cascalog.clj

(ns cascalog.helpers
  (:use cascalog.api)
  (:import [org.apache.hadoop.mapred JobConf]
           [cascalog Util]))

(defn lazy-generator
  "Returns a cascalog generator on the supplied lazy
  sequence. `lazy-generator` serializes each item in the lazy sequence
  into a sequencefile located at the supplied temporary directory, and
  returns a tap into its guts.

I recommend wrapping queries that use this tap with
`cascalog.io/with-fs-tmp`; for example,

    (with-fs-tmp [fs tmp-dir]
      (let [lazy-tap (pixel-generator tmp-path lazy-seq)]
      (?<- (stdout)
           [?field1 ?field2 ... etc]
           (lazy-tap ?field1 ?field2)
           ...)))"
  [tmp-path lazy-seq]
  (let [tap (hfs-seqfile tmp-path)]
    (with-open [collector (.openForWrite tap (JobConf.))]
      (doseq [item lazy-seq]
        (.add collector (Util/coerceToTuple item))))
    tap))