gistfile1.clj

(use 'forma.hadoop.jobs.scatter)

;; put stuff in pail
(let [pail-path "/tmp/newpail"
      vcf-src [(thrift/DataChunk* "vcf"
                                  (thrift/ModisPixelLocation* "500" 28 8 0 0)
                                  25
                                  "00")]
      gadm-src [(thrift/DataChunk* "gadm"
                                   (thrift/ModisPixelLocation* "500" 28 8 0 0)
                                   16435
                                   "00")]
           vcf-query (<- [?dc]
                         (vcf-src ?dc))
           gadm-query (<- [?dc]
                          (gadm-src ?dc))]
  ;; Data appear in pail no problem:
  (to-pail pail-path vcf-query) 
  (to-pail pail-path gadm-query))

;; helpers

(defn unpack-pixel-loc
  [loc]
  (let [p-loc (.getPixelLocation (:property (bean loc)))]
    [(.getResolution p-loc)
     (.getTileH p-loc)
     (.getTileV p-loc)
     (.getSample p-loc)
     (.getLine p-loc)]))

(defn unpack-long
  "Unpack long value from static source"
  [dv]
  (.getLongVal dv))

;; get values out of pail
;; data comes out when there's one source, but joining on sres h v s l produces nothing

;; these queries run just fine - we get the expected values back
(let [pail-path "/tmp/newpail"
      vcf-src (split-chunk-tap pail-path ["vcf"])]
  (??<- [?sres ?h ?v ?s ?l ?vcf]
      (vcf-src _ ?vcf-obj)
      (bean ?vcf-obj :> ?vcf-map)
      (get ?vcf-map :locationProperty :> ?vcf-loc)
      (unpack-pixel-loc ?vcf-loc :> ?sres ?h ?v ?s ?l)
      (get ?vcf-map :chunkValue :> ?vcf-dv)
      (unpack-long ?vcf-dv :> ?vcf)))

=> (["500" 28 8 0 0 25])

(let [pail-path "/tmp/newpail"
      gadm-src (split-chunk-tap pail-path ["gadm"])]
  (??<- [?sres ?h ?v ?s ?l ?gadm]
        (gadm-src _ ?gadm-obj)
        (bean ?gadm-obj :> ?gadm-map)
        (get ?gadm-map :locationProperty :> ?gadm-loc)
        (unpack-pixel-loc ?gadm-loc :> ?sres ?h ?v ?s ?l)
        (get ?gadm-map :chunkValue :> ?gadm-dv)
        (unpack-long ?gadm-dv :> ?gadm)))

=> (["500" 28 8 0 0 16435])

;; runs but no output:
(let [pail-path "/tmp/newpail"
      gadm-src (split-chunk-tap pail-path ["gadm"])
      vcf-src (split-chunk-tap pail-path ["vcf"])]
  (??<- [?sres ?h ?v ?s ?l ?gadm ?vcf]
        (gadm-src _ ?gadm-obj)
        (bean ?gadm-obj :> ?gadm-map)
        (get ?gadm-map :locationProperty :> ?gadm-loc)
        (unpack-pixel-loc ?gadm-loc :> ?sres ?h ?v ?s ?l)
        (get ?gadm-map :chunkValue :> ?gadm-dv)
        (unpack-long ?gadm-dv :> ?gadm)

        (vcf-src _ ?vcf-obj)
        (bean ?vcf-obj :> ?vcf-map)
        (get ?vcf-map :locationProperty :> ?vcf-loc)
        (unpack-pixel-loc ?vcf-loc :> ?sres ?h ?v ?s ?l)
        (get ?vcf-map :chunkValue :> ?vcf-dv)
        (unpack-long ?vcf-dv :> ?vcf)))

=> ()