debug MODIS time series in FORMA processing

check-data.clj

(use 'forma.hadoop.jobs.forma)
(in-ns 'forma.hadoop.jobs.forma)
(use 'forma.hadoop.pail)

(let [pail-path "s3n://pailbucket/all-master"
      out-path "s3n://pailbucket/output/run-2015-05-14/date-counts"
      s-res "500"
      t-res "16"
      pail-src (split-chunk-tap pail-path ["ndvi" (format "%s-%s" s-res t-res)])
      sink (hfs-textline out-path :sinkmode :replace)]
      (?<- sink [?date ?h ?v ?count]
          (pail-src _ ?tile-chunk)
          (thrift/unpack ?tile-chunk :> ?name ?tile-loc ?data ?t-res ?date _)
          (thrift/unpack ?tile-loc :> ?s-res ?h ?v ?id ?size)
          (= ?date "2015-03-22")
          (c/count ?count)))

query.clj

(use 'forma.hadoop.jobs.timeseries)
(in-ns 'forma.hadoop.jobs.timeseries)

(defn debug-tseries
  "Given a source of chunks, this subquery extracts the proper
  position information and outputs new datachunks containing
  timeseries."
  [missing-val tile-chunk-src]
  (let [mk-tseries (form-tseries missing-val)
        val-src (<- [?name ?t-res ?date ?s-res ?h ?v ?id ?size ?data]
                    (tile-chunk-src _ ?tile-chunk)
                    (thrift/unpack ?tile-chunk :> ?name ?tile-loc ?data ?t-res ?date _)
                    (thrift/unpack ?tile-loc :> ?s-res ?h ?v ?id ?size)
                     (:distinct false))
        ts-src (<- [?start-date ?end-date ?len]
                    (val-src _ ?t-res ?date _ _ _ _ _ ?val)
                    (mk-tseries ?t-res ?date ?val :> ?pixel-idx ?start ?end ?series)
                    (count ?series :> ?len)
                    (date/period->datetime ?t-res ?start :> ?start-date)
                    (date/period->datetime ?t-res ?end :> ?end-date))]
    (<- [?start-date ?end-date ?len ?count]
      (ts-src ?start-date ?end-date ?len)
      (c/count ?count))))


     
(defn modis-timeseries-debug
  "Convert unordered MODIS chunks from pail into pixel-level timeseries."
  [pail-path output-path s-res t-res dataset]
  (->> (pail/split-chunk-tap pail-path [dataset (format "%s-%s" s-res t-res)])
       (debug-tseries *missing-val*)
       (?- (hfs-textline output-path :sinkmode :replace))))
       
(modis-timeseries-debug "s3n://pailbucket/all-master" "s3n://pailbucket/output/run-2015-05-12/modis-timeseries-debug" "500" "16" "ndvi")

stacktrace.clj

Caused by: java.lang.AssertionError: Assert failed: (let [len (if (coll? vals) (count vals) (count (unpack vals)))] (= len (inc (- end start))))
	at forma.thrift$TimeSeries_STAR_.invoke(thrift.clj:252)