jbenninghoff

#!/bin/bash

/opt/mapr/spark/spark-1.2.1/bin/spark-submit --master yarn-client \
  --class org.apache.spark.examples.terasort.TeraGen \
  --name 'TeraGen' \
  --conf 'mapreduce.terasort.num.partitions=5' \
  --executor-cores 30 \
  --executor-memory 7G \
  --num-executors 9 \
  terasort-project_2.10-1.0.jar 50G /user/$USER/spark-terasort

jbenninghoff

root@cent01 redhat 4C 02:43pm# sed -n '/^#/d;/@/d;s/\(^.*\): \(.*$\)/[\1]\n\2\n/p' /etc/clustershell/groups | sed '/\[.*\]/s/-/:/;/.* .*/s/ /\n/'
[all]
cent[01:05]

[zk]
cent[01:03]

[cldb]
cent01
cent02

jbenninghoff

hduser@master:~$ cat wordcount.pig
A = load '/user/jbenninghoff/somefile.txt';
B = foreach A generate flatten(TOKENIZE((chararray)$0)) as word;
C = filter B by word matches '\\w+';
D = group C by word;
E = foreach D generate COUNT(C), group;
store E into '/user/jbenninghoff/somefileWordcount';

jbenninghoff

// Use Gists to store code you would like to remember later on
console.log(window); // log the "window" object to the console