gistfile1.clj

;; here's a simpler query w/the same result - using a single file downloaded 
;; from the s3 source path below.
;; preliminary investigation suggests a dependency issue - the newer
;; version of Kryo required by an updated dependency maybe can't read
;; data created with an older version. The input files were created a few months ago
;; and we've since bumped version numbers with a few dependencies

(?- (hfs-textline "/tmp/test" :sinkmode :replace) (hfs-seqfile "/home/hadoop/part-00000"))


;; the original query
(let [sink (hfs-seqfile "s3n://formatemp/adjusted-screened" :sinkmode :replace)
      adjusted-path "s3n://pailbucket/updates/adjusted"
      t-res "16"
      adjusted-src (hfs-seqfile adjusted-path)
      adjusted-long-query (<- [?s-res ?mod-h ?mod-v ?sample ?line ?start-idx ?ndvi-ts ?precl-ts]
                              (adjusted-src ?s-res ?mod-h ?mod-v ?sample ?line ?start-idx ?ndvi-ts ?precl-ts)
                              
                              (<= 28 ?mod-h)
;;                              (ts->long ?start-idx ?ndvi-ts :> ?pd ?ndvi)
;;                              (ts->long ?start-idx ?precl-ts :> ?pd ?precl)
;;                              (u/nest-vals ?ndvi ?precl :> ?vals)
;;                              (date/period->datetime t-res ?pd :> ?date)
                              )]
  (?- sink adjusted-long-query))


cascading.tuple.TupleException: unable to read from input identifier: s3n://pailbucket/updates/adjusted/part-00000
  at cascading.tuple.TupleEntrySchemeIterator.hasNext(TupleEntrySchemeIterator.java:127)
	at cascading.flow.stream.SourceStage.map(SourceStage.java:76)
	at cascading.flow.stream.SourceStage.run(SourceStage.java:58)
	at cascading.flow.hadoop.FlowMapper.run(FlowMapper.java:124)
	at org.apache.hadoop.mapred.MapTask.runOldMapper(MapTask.java:441)
	at org.apache.hadoop.mapred.MapTask.run(MapTask.java:377)
	at org.apache.hadoop.mapred.Child$4.run(Child.java:255)
	at java.security.AccessController.doPrivileged(Native Method)
	at javax.security.auth.Subject.doAs(Subject.java:396)
	at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1059)
	at org.apache.hadoop.mapred.Child.main(Child.java:249)
Caused by: com.esotericsoftware.kryo.KryoException: Buffer underflow.
	at com.esotericsoftware.kryo.io.Input.require(Input.java:157)
	at com.esotericsoftware.kryo.io.Input.readAscii_slow(Input.java:546)
	at com.esotericsoftware.kryo.io.Input.readAscii(Input.java:524)
	at com.esotericsoftware.kryo.io.Input.readString(Input.java:402)
	at carbonite.serializer$clj_read.invoke(serializer.clj:35)
	at clojure.lang.Var.invoke(Var.java:415)
	at carbonite.ClojureReaderSerializer.read(Unknown Source)
	at com.esotericsoftware.kryo.Kryo.readObject(Kryo.java:612)
	at cascading.kryo.KryoDeserializer.deserialize(KryoDeserializer.java:42)
	at cascading.tuple.hadoop.TupleSerialization$SerializationElementReader.read(TupleSerialization.java:590)
	at cascading.tuple.hadoop.io.HadoopTupleInputStream.readType(HadoopTupleInputStream.java:105)
	at cascading.tuple.hadoop.io.HadoopTupleInputStream.getNextElement(HadoopTupleInputStream.java:52)
	at cascading.tuple.io.TupleInputStream.readTuple(TupleInputStream.java:78)
	at cascading.tuple.hadoop.io.TupleDeserializer.deserialize(TupleDeserializer.java:40)
	at cascading.tuple.hadoop.io.TupleDeserializer.deserialize(TupleDeserializer.java:28)
	at org.apache.hadoop.io.SequenceFile$Reader.deserializeValue(SequenceFile.java:1886)
	at org.apache.hadoop.io.SequenceFile$Reader.getCurrentValue(SequenceFile.java:1859)
	at org.apache.hadoop.mapred.SequenceFileRecordReader.getCurrentValue(SequenceFileRecordReader.java:103)
	at org.apache.hadoop.mapred.SequenceFileRecordReader.next(SequenceFileRecordReader.java:78)
	at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.moveToNext(MapTask.java:238)
	at org.apache.hadoop.mapred.MapTask$TrackedRecordReader.next(MapTask.java:218)
	at cascading.tap.hadoop.util.MeasuredRecordReader.next(MeasuredRecordReader.java:61)
	at cascading.scheme.hadoop.SequenceFile.source(SequenceFile.java:89)
	at cascading.tuple.TupleEntrySchemeIterator.getNext(TupleEntrySchemeIterator.java:140)
	at cascading.tuple.TupleEntrySchemeIterator.hasNext(TupleEntrySchemeIterator.java:120)
	... 10 more