Investigating issue where optional fields aren't getting serialized. The optional param-break field is getting lost on serialization. Making it required solves the problem.

gistfile1.clj

(use 'forma.hadoop.jobs.runner-test)
(in-ns 'forma.hadoop.jobs.runner-test)

(def ts-dc
  "FormaValues wrapped in a TimeSeries object wrapped in a DataChunk"
  (let [[forma-vals] (series->forma-values nil [1. 2.] [2. 3.] [3. 4.] [4. 5.])
        loc (apply thrift/ModisPixelLocation* pix-loc)
        ts (thrift/TimeSeries* 827 forma-vals)
        dc (thrift/DataChunk* "trends" loc ts t-res)]
    dc))

(def fv-dc
  "FormaValue wrapped just in a DataChunk"
  (let [forma-val (thrift/FormaValue* (thrift/FireValue* 0 0 0 0) 1. 2. 3. 4.)
        loc (apply thrift/ModisPixelLocation* pix-loc)
        dc (thrift/DataChunk* "trends" loc forma-val t-res)]
    dc))

(defn ts-dc-to-formaval
  "Unpack DataChunk and enclosed TimeSeries, all the way to the first FormaValue"
  [data-chunk]
  (thrift/unpack (first (thrift/unpack (last (thrift/unpack (nth (thrift/unpack data-chunk) 2)))))))

(defn fv-dc-to-formaval
  "Unpack DataChunk and enclosed FormaValue"
  [data-chunk]
  (thrift/unpack (nth (thrift/unpack data-chunk) 2)))


(defn fv-checker
  (let [pail-path "/tmp/pail-fv"
      _ (p/to-pail pail-path [[fv-dc]])
      data (??- (p/split-chunk-tap pail-path ["trends" "500-16"]))]
  (fv-dc-to-formaval (last (ffirst data)))))
;=> [#<FireValue FireValue(temp330:0, conf50:0, bothPreds:0, count:0)> 1.0 2.0 3.0 0.0]
;; last element of the vector above should be 4.0, not 0.0

(defn fv-ts-checker
  (let [pail-path "/tmp/pail-ts"
      _ (p/to-pail pail-path [[ts-dc]])
      data (??- (p/split-chunk-tap pail-path ["trends" "500-16"]))]
  (ts-dc-to-formaval (last (ffirst data)))))
;=> [#<FireValue FireValue(temp330:0, conf50:0, bothPreds:0, count:0)> 1.0 2.0 3.0 0.0]
;; last element of the vector above should be 4.0, not 0.0