tomz

#!/bin/bash
set -x -e

# AWS EMR bootstrap script 
# for installing open-source R (www.r-project.org) with RHadoop packages and RStudio on AWS EMR
#
# tested with AMI 4.0.0 (hadoop 2.6.0)
#
# [email protected]
# 24. September 2014

tomz

#!/bin/bash
set -x -e

# Usage:
# --no-rstudio - don't install rstudio-server
# --sparklyr - install RStudio's sparklyr package
# --sparkr - install SparkR package
# --shiny - install Shiny server
#
# --user - set user for rstudio, default "hadoop"

tomz

Loading SQL Server tables to Spark on EC2 and output to S3

0. Setup sqlserver on Azure

Follow the instructions here https://azure.microsoft.com/en-us/documentation/articles/sql-database-get-started/ to setup a SQL Server instance on Azure

1. Launch spark cluster using spark-ec2 script

Export AWS credentials

tomz

Dir["./spark/lib/*.jar"].each { |jar| require jar }
require "~/sqljdbc/sqljdbc42.jar"

import java.util.HashMap
import org.apache.spark.SparkConf
import org.apache.spark.api.java.JavaSparkContext
import org.apache.spark.sql.SQLContext

AWS_KEY = ENV["AWS_KEY"]
AWS_SECRET = ENV["AWS_SECRET"]

tomz

# prestogres jdbc
require "java"
java_import java.util.Properties
require "./postgresql-9.4-1203.jdbc41.jar"
Java::org.postgresql.Driver

db_url = "jdbc:postgresql://localhost:5439/hive"
props = Properties.new
props.set_property "protocolVersion", "2"
props.set_property "user","presto"

tomz

#!/bin/bash -ex

# Install Git
sudo yum -y install git

# Install Maven
wget -P /tmp http://apache.mirrors.spacedump.net/maven/maven-3/3.3.3/binaries/apache-maven-3.3.3-bin.tar.gz
sudo mkdir /opt/apache-maven
sudo tar -xvzf /tmp/apache-maven-3.3.3-bin.tar.gz -C /opt/apache-maven

tomz

# If you are using Spark 1.4, then launch SparkR with the command
#
# ./bin/sparkR --packages com.databricks:spark-csv_2.10:1.0.3
# as the `sparkPackages=` flag was only added in Spark 1.4.1.

# # This will work in Spark 1.4.1. 
sc <- sparkR.init(spark_link, sparkPackages = "com.databricks:spark-csv_2.10:1.0.3") 
sqlContext <- sparkRSQL.init(sc) 

flights <- read.df(sqlContext, "s3n://sparkr-data/nycflights13.csv","com.databricks.spark.csv", header="true")

tomz

Sys.setenv(SPARK_HOME="/Users/shivaram/spark-1.4.1")
.libPaths(c(file.path(Sys.getenv("SPARK_HOME"), "R", "lib"), .libPaths()))
library(SparkR)
sc <- sparkR.init(master="local")
sqlContext <- sparkRSQL.init(sc)

df <- createDataFrame(sqlContext, faithful)
# Select one column
head(select(df, df$eruptions))

tomz

sudo wget http://repos.fedorapeople.org/repos/dchen/apache-maven/epel-apache-maven.repo -O /etc/yum.repos.d/epel-apache-maven.repo
sudo sed -i s/\$releasever/6/g /etc/yum.repos.d/epel-apache-maven.repo
sudo yum install -y apache-maven
mvn --version

tomz

var casper = require('casper').create();
var url = 'http://localhost:3000/#!/';

casper.start(url, function() {
  var js = this.evaluate(function() {
    return document;
  });
  this.echo(js.all[0].outerHTML);
});
casper.run();