fbmnds

(require '[clojure.java.io :as io])

(def in-file "/path/to/datafile-4gb.tsv")
(def out-file "/path/to/datafile-16gb.tsv")

(defn multiply-string [s n]
  (cond
   (> n 1) (str s \newline (multiply-string s (dec n)))
   (= n 1) (str s \newline)
   :else nil))

fbmnds

;;
;; iota bails out main memory:
;;
user> (def test-file "/path/to/datafile-16gb.tsv")
#'user/test-file
user> (require '[iota]
         '[clojure.core.reducers :as r])
nil
user> (time (def test-vec  (iota/vec test-file)))

fbmnds

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;
;; !!!! WARNING !!!!
;;
;; This code is intented to be analysed in it´s flaws.
;; It does NOT claim to demonstrate good coding practice.
;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

(require '[clojure.java.io :as io])

fbmnds

(require '[iota]
         '[clojure.core.reducers :as r])
(def test-file "/path/to/test-file")

(def test-vec  (iota/vec test-file))


(defn f [m]
  (let [rem (:remaining m)]
    (cond (nil? rem) {:end true}

fbmnds

(defn lazy-read-csv
  "This one has lazily reads from the file and generates the CSV data.

  It will leak if the sequence is not fully consumed."
  ([csv-file]
   (let [in-file (io/text-reader csv-file)                    ;; FB: io/reader
         csv-seq (csv/read-csv in-file)
         lazy (fn lazy [wrapped]
                (lazy-seq
                  (if-let [s (seq wrapped)]

fbmnds

;; benchmark 1
;;
user> (time (count in-seq))
"Elapsed time: 2225 msecs"
812990
;;
;; memoizing
;;
user> (time (count in-seq))
"Elapsed time: 208 msecs"

fbmnds

(use '(incanter core stats charts))

;; number of processed lines:
;;
(def n '(10 100 1000 10000 20000 40000 60000 80000 100000))
;;
;; runtime per volume (# of lines) from measure:
;;
(def t-in-ms '(0.19248 1.718639 77.929438 5387.727517 25963.054403 137354.431354 303994.793612 551822.123044 737414.582849))
(def t-in-ms-1-5-1 '(1.70966 1.606149 56.427781 4738.202508 19550.828503 79468.576165 203903.7885 377308.347139 1013598.323447))

fbmnds

(require '[clojure.clr.data.csv :as csv]
         '[clojure.clr.io :as io])


(set! *print-length* 10)

(def in-file "/path/to/datafile/NYSE-2000-2001.tsv")
(def in-seq (line-seq (io/text-reader in-file)))   ;; FB: io/reader

;; ...

fbmnds

(require '[clojure.data.csv :as csv]
         '[clojure.java.io :as io])

;; avoid REPL timeouts by large data output (during testing)
;;
(set! *print-length* 10)

;; download from https://s3.amazonaws.com/hw-sandbox/tutorial1/NYSE-2000-2001.tsv.gz
;; 812,990 data records, 40 MB on disk
;;

fbmnds

;;
(load-file "./src/rserve_cli2/rconnection.clj")
(load-file "./src/rserve_cli2/rconnection_sexp.clj")

;;
(require ['rserve-cli2.rconnection :as 'rc])

;; clean up an eventually existing connection
(try (rc/close) (catch Exception e nil))