jorisbontje

;; Logical / unsigned shift right in Clojure (>>> operator in Java)
;;
;; Algorithm from http://www.sitepoint.com/forums/php-34/unsigned-right-bitwise-shift-449434.html

(defn logical-shift-right [n s] 
  (if (neg? n) 
    (bit-or (bit-shift-right (bit-and n 0x7fffffff) s)
            (bit-shift-right 0x40000000 (dec s)))
    (bit-shift-right n s)))

jorisbontje

ERROR in (test-parse-line-op) (FlowStepJob.java:173)
Uncaught exception, not in assertion.
expected: nil
  actual: cascading.flow.FlowException: step failed: (1/1) ...QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/sink/0ba51806-c2a2-41bf-96cf-06bfe61b86bc"]"], with job id: job_local_0001, please see cluster logs for failure messages
 at cascading.flow.FlowStepJob.blockOnJob (FlowStepJob.java:173)
    cascading.flow.FlowStepJob.start (FlowStepJob.java:138)
    cascading.flow.FlowStepJob.call (FlowStepJob.java:127)
    cascading.flow.FlowStepJob.call (FlowStepJob.java:39)
    java.util.concurrent.FutureTask$Sync.innerRun (FutureTask.java:303)
    java.util.concurrent.FutureTask.run (FutureTask.java:138)

jorisbontje

$ lein test
Testing cascalog-weather.test.weather
11/06/29 22:46:49 INFO hadoop.Hadoop18TapUtil: setting up task: 'attempt_002147483647_0000_m_000000_0' - file:/var/folders/YZ/YZO0QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/tap57/4abafe2e-a136-4415-b7bc-09fd51acc301/_temporary/_attempt_002147483647_0000_m_000000_0
11/06/29 22:46:49 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable
11/06/29 22:46:49 INFO hadoop.TapCollector: closing tap collector for: /var/folders/YZ/YZO0QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/tap57/4abafe2e-a136-4415-b7bc-09fd51acc301/part-00000
11/06/29 22:46:49 INFO hadoop.Hadoop18TapUtil: committing task: 'attempt_002147483647_0000_m_000000_0' - file:/var/folders/YZ/YZO0QDWpEp0jBsowT4Bo4U+++TI/-Tmp-/tap57/4abafe2e-a136-4415-b7bc-09fd51acc301/_temporary/_attempt_002147483647_0000_m_000000_0
11/06/29 22:46:49 INFO hadoop.Hadoop18TapUtil: saved output of task 'attempt_002147483647_0000_m_000000_0' to file:/var/folders/YZ/YZO0QDWpEp0jBsow

jorisbontje

java.lang.RuntimeException: java.lang.RuntimeException: java.lang.RuntimeException: java.lang.NullPointerException

LazySeq.java:47	clojure.lang.LazySeq.sval
LazySeq.java:56	clojure.lang.LazySeq.seq
Cons.java:39	clojure.lang.Cons.next
RT.java:1178	clojure.lang.RT.length
RT.java:1157	clojure.lang.RT.seqToArray
LazySeq.java:126	clojure.lang.LazySeq.toArray
RT.java:1135	clojure.lang.RT.toArray
core.clj:300	clojure.core/to-array

jorisbontje

@SuppressWarnings("serial")
@PlatformRunner.Platform({ LocalPlatform.class, HadoopPlatform.class})
public class SortTest extends PlatformTestCase {

	private static final inputFileSort = "src/test/data/sort.txt";

	public SortTest() {
		super(false);
	}

jorisbontje

drop table if exists raw_tweets;
drop table if exists tweets;
drop table if exists positive_hashtags_per_day;
drop table if exists count_positive_hashtags_per_day;
drop table if exists top5_positive_hashtags_per_day;

create table raw_tweets (json string);
load data local inpath 'sample.json' into table raw_tweets;

create table tweets as

jorisbontje

#!/bin/bash
USERNAME=admin
PASSWORD=admin
SCM_URL=http://localhost:7180
COOKIES_FILE=cookies.txt
EXPORT_FILE=export.txt

wget -q --post-data="j_username=${USERNAME}&j_password=${PASSWORD}" --save-cookies ${COOKIES_FILE} --keep-session-cookies -O /dev/null ${SCM_URL}/j_spring_security_check
wget -q -O ${EXPORT_FILE} --load-cookies ${COOKIES_FILE} ${SCM_URL}/cmf/exportCLI

jorisbontje

#!/bin/sh

# Return total number of Heroku dynos for an account
#
# Uses:
#     jutil <https://github.com/misterfifths/jutil.git>
#     underscore.js <http://underscorejs.org/>

API_KEY="<your Heroku API key>"
curl -s -H "Accept: application/json" -u :$API_KEY https://api.heroku.com/apps | jselect 'dynos' | jutil 'return _.reduce($, function(memo, num){ return memo + num; }, 0);'

jorisbontje

curl -O http://python-distribute.org/distribute_setup.py
sudo python distribute_setup.py

curl -O https://raw.github.com/pypa/pip/master/contrib/get-pip.py
sudo python get-pip.py

sudo pip install webhdfs

cp /usr/lib/python2.6/site-packages/webhdfs/example.py .

jorisbontje

0) Download avro-tools jar file from avro.apache.org

1) Extract Avro schema using avro-tools.jar

java -jar avro-tools*.jar getschema file.avro > file.avsc

2) Upload Avro schema to hdfs

hadoop fs -cp file.avsc /use/training/file.avsc