obar1

## Move data between HDF-S and Spark - pyspark
# read some text file
sc.textFile("sqoop_import/departments").take(1)
# looping into a rdd
data=sc.textFile("sqoop_import/departments")
for i in data.collect():
    print(i)
# reading from local fs
sc.textFile("file:///tmp/departments.json").take(3)

obar1

mplayer -vo null -ao null -frames 0 -identify "file.mpg"

// LOOK FOR id_length
//8000

mpayer -vo jpeg -sstep 5 -endpos 8000 -nosound  "file.mpg"

obar1

 import org.apache.spark.{SparkConf,SparkContext}
 
 var appName:String ="TODO"
val conf = new SparkConf().setAppName(appName).setMaster("local[*]").set("spark.executor.memory","1g");
val sc =  SparkContext.getOrCreate(conf)
sc.setLogLevel("WARN")

obar1

/*
** get revenueeach daya
orders(id and date)
orders_items (order)id and subtotal)

then join and reduce on date
*/

var path = raw"C:\Users\mario\Desktop\CCA Spark and Hadoop Developer as per revised syllabus - YouTube\data-master\retail_db\orders"
val orders = sc.textFile(path)

obar1

#!/usr/bin/env bash
# Takes a YouTube URI to a playlist (fairly liberal, it's fine as long
# as the playlist id can be extracted), and prints a list of URIs in a
# YouTube playlist.
#
# Requires youtube-dl 2014.10.24, tested on youtube-dl
# 2014.11.02.1. Feature subject to change.
youtube-dl -j --flat-playlist "$1" | jq -r '.id' | sed 's_^_https://youtube.com/v/_'

obar1

Download Udemy course videos using youtube-dl

$ youtube-dl --list-extractors | grep udemy

Steps

1. Get link to the course to download. e.g. https://www.udemy.com/course-name/
2. Login into udemy website, save the cookie from chrome using Chrome (Cookie.txt)[1] export extension. Save it to file udemy-cookies.txt
3. Get the link of the video that you want to download. usually in format. Use the command provided below where you have to replace the {course_link} and {path_to_cookies_file} with respective paths.

# $ youtube-dl {course_link} --cookies {path_to_cookies_file}

obar1

cd /d %~dp0
xcopy /E /Y c:\Users\Mario_Amatucci\git\bigdata_qa  bigdata_qa /EXCLUDE:no.txt >> xc_bigdata_qa.log

obar1

"C:\Program Files\7-Zip\7z.exe" a -r  -x!*.7z -xr!.git gh_"%DATE:~7,2%.%DATE:~4,2%.%DATE:~-4% %TIME:~0,2%.%TIME:~3,2% Backup".7z
move C:\Users\Mario_Amatucci\Documents\GitHub\*.7z  "C:\Users\Mario_Amatucci\OneDrive - EPAM\_bkp"

obar1

#!/usr/bin/env bash
# using it 

echo ... $1
mkdir $1
cd $1
git clone [email protected]:obar1/learn.obar1.xyz.git 

obar1

hadoop_urls="`ip route get 8.8.8.8 | awk '{print $NF; exit}'`:\ {50070'\n',8800'\n'}"
eval echo -e "$hadoop_urls"