BeatriceMoissinac

# List hdfs file system
hadoop fs -ls

BeatriceMoissinac

import pandas as pd

df['bin'] = pd.cut(df['1'], [0, 50, 100,200], labels=['0-50', '50-100', '100-200'])

         0    1        file      bin
0  person1   24     age.csv     0-50
1  person2   17     age.csv     0-50
2  person3   98     age.csv   50-100
3  person4    6     age.csv     0-50
4  person2  166  Height.csv  100-200

BeatriceMoissinac

mongoexport --db car --collection groundtruth --query "{groundtruth:'movies'}" --out movies.json

BeatriceMoissinac

# UNZIP SESSION FILES
# Reference: https://aws.amazon.com/blogs/big-data/seven-tips-for-using-s3distcp-on-amazon-emr-to-move-data-efficiently-between-hdfs-and-amazon-s3/
# http://docs.aws.amazon.com/emr/latest/ReleaseGuide/UsingEMR_s3distcp.html
# Hadoop has to be installed on the cluster (add Name=Hadoop)
# As a step
aws emr add-steps --profile $KEY --cluster-id $CLUSTER --steps Type=CUSTOM_JAR,Name="S3DistCp",ActionOnFailure=CONTINUE,Jar="command-runner.jar",Args=[s3-dist-cp,--src,s3://osu-is/aruba/raw/Client_Session_1479414885.zip,--dest,hdfs:///output,--outputCodec,none]
# or as direct command in ssh consol					  					  
# s3-dist-cp --src s3://osu-is/aruba/raw --dest hdfs:///output --srcPattern .*\.zip --outputCodec=none

# aws s3 cp --profile $KEY src/main/shell/unzipper.sh s3://osu-is/jar

BeatriceMoissinac

// vim: syntax=markdown

References

• http://mywiki.wooledge.org/BashFAQ/035#getopts
• http://wiki.bash-hackers.org/howto/getopts_tutorial

BeatriceMoissinac

from datetime import datetime
import time

#-------------------------------------------------
# conversions to strings
#-------------------------------------------------
# datetime object to string
dt_obj = datetime(2008, 11, 10, 17, 53, 59)
date_str = dt_obj.strftime("%Y-%m-%d %H:%M:%S")
print date_str

BeatriceMoissinac

// vim: syntax=shell

$JAR=/usr/lib/spark/lib/spark-examples.jar
$KEY=MoissinB

# Create cluster with 1st step
aws emr create-cluster  --profile $KEY \
                        --name "Moissinb Cluster" \
                        --release-label emr-5.10.0 \
                        --applications Name=Spark \

BeatriceMoissinac

// vim: syntax=shell

# Install Homebrew
/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"

# Install Java
brew install java

# Install Scala
brew install scala