randyzwitch

#Cube every number from 1 to 100

#Python map function 
cubes = map(lambda(x): x*x*x, range(1,100))

#Python list comprehension
cubes= [x*x*x for x in range(1,100)]

#R sapply function
cubes <- sapply(seq(1,100), function(x) x*x*x)

randyzwitch

julia> airline_array = readdlm("/Users/randyzwitch/airline/1987.csv", ',');

julia> size(airline_array)
(1311827,29)

julia> typeof(airline_array)
Array{Any,2}

randyzwitch

julia> airline_array = readdlm("/Users/randyzwitch/airline/1987.csv", ',' , String);

julia> size(airline_array)
(1311827,29)

julia> typeof(airline_array)
Array{String,2}

randyzwitch

julia> airline_array = readdlm("/Users/randyzwitch/airline/1987.csv", ',' , String);

julia> size(airline_array)
(1311827,29)

julia> typeof(airline_array)
Array{String,2}

randyzwitch

julia> using DataFrames

julia> airline_df = readtable("/Users/randyzwitch/airline/1987.csv.gz");

julia> size(airline_df)
(1311826,29)

julia> typeof(airline_df)
DataFrame  (use methods(DataFrame) to see constructors)

randyzwitch

import collections 
import nltk

#Dictionary from Unix
internal_dict = open("/usr/share/dict/words")
#Stopwords corpus from NLTK
stopwords = nltk.corpus.stopwords.words('english')

#Build english_dictionary of prospect words
english_dictionary = []

randyzwitch

from mrjob.job import MRJob

class MRWordCounter(MRJob):    

  def mapper(self, english_dict, line):
  english_dict = ['aal', 'aalii', 'aam', 'aani'...'zythum', 'zyzomys', 'zyzzogeton']

  for word in english_dict:
            if word in line:
                yield word, 1

randyzwitch

python ~/Desktop/mapreduce.py -r emr s3://<s3bucket>/url_unload/0000_part_01 --output-dir=s3://<s3bucket>/url_output --num-ec2-instances=81

randyzwitch

julia> using ODBC

julia> ODBC.connect("MySQL")
Connection 1 to MySQL successful.

randyzwitch

#Amazon Redshift/Postgres connection string
Julia> red = advancedconnect("Driver={psqlODBC};ServerName=reporting.XXXXX.us-east-1.redshift.amazonaws.com;Username=XXXX;Password=XXXX;Database=XXXX;Port=XXXX");
Connection 1 to Driver={psqlODBC};ServerName=reporting.XXXXX.us-east-1.redshift.amazonaws.com;Username=XXXX;Password=XXXX;Database=XXXX;Port=XXXX successful.

#MySQL connection string
julia> my = advancedconnect("Driver={MySQL};user=root;server=localhost;database=airline;")
Connection 1 to Driver={MySQL};user=root;server=localhost;database=airline; successful.