Last active
March 17, 2018 10:55
-
-
Save gavlooth/a4bddc85443e159f28c4e6b501caa2af to your computer and use it in GitHub Desktop.
An improved file-seq reads a file to a lazy sequence of lines and closes the reader after
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
(ns chunked-line-seq | |
(:require [clojure.java.io :as io])) | |
(defn chunked-line-seq [file-name & {:keys [chunck-size] :or {chunck-size 1000}}] | |
(let [rd (io/reader (io/input-stream (io/file file-name))) | |
eof&? (atom false) | |
chunck-lines (fn [ ] | |
(seq (loop [lines (transient [] ) counter 0] | |
(if-let [line (.readLine rd)] | |
(if (< counter chunck-size) | |
(recur (conj! lines line) (inc counter)) | |
(persistent! lines)) | |
(do (swap! eof&? not) | |
(persistent! lines))))))] | |
(letfn [(chuncked-seq [] (if-not @eof&? | |
(concat (chunck-lines) | |
(lazy-seq (chuncked-seq))) | |
(.close ^java.io.BufferedReader rd)))] | |
(chuncked-seq)))) | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment