randerzander

set -eu

FILE=$1
HEADER_LINE_NUM=$2
DELIM=$3
TABLE_NAME=$4
LOCATION=$5

# This script assumes columns are all strings. Edit the DDL file after running the script and change column types at will.

randerzander

# Put a list of URLs in a file, inputs.txt
echo ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/daily/by_year/2009.csv.gz > input.txt
echo ftp://ftp.ncdc.noaa.gov/pub/data/ghcn/daily/by_year/2010.csv.gz >> input.txt

# Trick to force MapReduce to treat each line in inputs.txt as a single map task
mkdir input
cd input
# I used 1 line per file - tweak for smaller downloads & fewer map tasks
split -l 1 ../input.txt
hadoop fs -put input .

randerzander

set number
set autoindent
set smartindent
set expandtab
set shiftwidth=2
set tabstop=2

randerzander

# Git clone, build, and copy the necessary MongoDB jars to your worker node $HADOOP_HOME/lib directories
cd ~/
git clone https://github.com/mongodb/mongo-hadoop
cd mongo-hadoop
./gradlew jar 
sudo cp build/libs/* /usr/lib/hadoop/lib
sudo cp core/build/libs/* /usr/lib/hadoop/lib
sudo cp hive/build/libs/* /usr/lib/hadoop/lib

cd ~/

randerzander

wget http://dagobah.ftphosting.net/yum/smartfile.repo -O /etc/yum.repos.d/smartfile.repo
mv smartfile.repo /etc/yum.repos.d/
wget http://dl.fedoraproject.org/pub/epel/5/x86_64/epel-release-5-4.noarch.rpm
wget http://rpms.famillecollet.com/enterprise/remi-release-5.rpm
sudo rpm -Uvh remi-release-5*.rpm epel-release-5*.rpm
yum install -y libffi python26

randerzander

USER='admin'
PASS='admin'
CLUSTER='dev'
HOST=$(hostname -f):8080

function start(){
  curl -u $USER:$PASS -i -H 'X-Requested-By: ambari' -X PUT -d \
    '{"RequestInfo": {"context" :"Start '"$1"' via REST"}, "Body": {"ServiceInfo": {"state": "STARTED"}}}' \
    http://$HOST/api/v1/clusters/$CLUSTER/services/$1
}