Example of downloading an arbitrarily large table and saving to CSV

util.clj

(ns etl.util
  (:require [clojure.java.jdbc :as sql]
            [clojure.java.io :as io]
            [clojure.string :as str]
            [clojure.data.csv :as csv]))

(defn seq-counter
  "Calls callback after every n'th entry in sequence is evaluated.
   Optionally takes another callback to call once the seq is fully evaluated."
  ([sqn n callback]
   (map #(do (if (= (rem %1 n) 0) (callback %1)) %2) (iterate inc 1) sqn))
  ([sqn n callback finished-callback]
   (drop-last (lazy-cat (seq-counter sqn n callback)
                        (lazy-seq (cons (finished-callback) ()))))))

(defn write-header [outfile header]
  (with-open [w (io/writer outfile)]
    (csv/write-csv w header)))

(defn write-result-set [outfile batch-size row-fn rows]
  (let [progress-fn #(println (format "wrote %d rows" %))]
    (with-open [w (io/writer outfile :append true)]
      (doseq [row (seq-counter rows batch-size progress-fn)]
        (csv/write-csv w (row-fn row))))))

(defn table->csv [spec sql fetch-size outfile header row-fn]
  (write-header outfile header)
  (sql/with-db-connection [conn spec]
    (.setAutoCommit (:connection conn) false)
    (let [stmt (sql/prepare-statement (:connection conn) sql :fetch-size fetch-size)
          results (sql/query conn [stmt]
                             :identifiers (comp str/lower-case #(.replace % "_" "-"))
                             :result-set-fn (partial write-result-set outfile fetch-size row-fn))])))