Lazy IO in Clojure

lazyio.clj

(ns Experiments.lazyio)

;; Self contained example - Lazy I/O etc
;; (C) Johan Astborg, 2014

(def filename "out.gz")

(defn lazy-gzip-read  [file]
  "lazy gzip file reader"
  (let [zip (java.util.zip.GZIPInputStream. (java.io.FileInputStream. file))
      reader (java.io.BufferedReader. (java.io.InputStreamReader. zip "UTF-8"))]
  (letfn [(helper [rdr]
                  (lazy-seq
                    (if-let [line (.readLine rdr)]
                      (cons line (helper rdr))
                      (do (.close rdr) nil))))]
         (helper reader))))

(defn eager-gzip-write [file l]
  "writes seq to gzip file"
  (let [zip (java.util.zip.GZIPOutputStream. (java.io.FileOutputStream. file))
      writer (java.io.BufferedWriter. (java.io.OutputStreamWriter. zip "UTF-8"))]
    (letfn [(helper [wtr data]
                    (doto wtr (.append data) (.newLine)))]
      (doall (map (partial helper writer) l)) (.close writer))))

;; Write sample data
(def sample-data (map str (take 10 (reductions + (cycle [(-> 1 Math/exp)])))))
(eager-gzip-write filename sample-data)

;; Read the same data and check
(def input-data (take 10 (lazy-gzip-read filename)))
(= input-data sample-data)

;;---------------------------------------------------------------------------------

(def csv "table.csv")

;; Lazy CSV reader with simple parser
(defn lazy-csv-read [file del skip]
  "lazy csv file reader, and parser"
  (let [parse (fn [s] (clojure.string/split s (re-pattern del)))
        csv (java.io.FileInputStream. file)
        reader (java.io.BufferedReader. (java.io.InputStreamReader. csv "UTF-8"))]
  (if (true? skip) (.readLine reader))
  (letfn [(helper [rdr]
              (lazy-seq
                (if-let [line (.readLine rdr)]
                  (cons (parse line) (helper rdr))
                  (do (.close rdr) nil))))]
     (helper reader))))

;; Read a CSV (Yahoo finance) and skip first row
(take 5 (lazy-csv-read csv "," true))

;;---------------------------------------------------------------------------------

(def url "http://www.google.com")

(defn lazy-http-reader [url]
  "lazy http reader, given url string"
  (let [conn (.openConnection (java.net.URL. url))
        stream (.getInputStream conn)
        reader (java.io.BufferedReader. (java.io.InputStreamReader. stream))]
  (.connect conn)
  (letfn [(helper [rdr]
             (lazy-seq
               (if-let [line (.readLine rdr)]
                 (cons line (helper rdr))
                 (do (.close rdr) nil))))]
   (helper reader))))

(take 1 (lazy-http-reader url))