expimp.clj

(ns expimp.core
  (:use [korma.core]
        [korma.db]))

(defdb src (mysql
              {:db "xx"
               :user "xx" :password "xx"
               :host "myrds.us-east-1.rds.amazonaws.com"}))

(defdb target (postgres {:db "xx" :user "xx" :password "xx"}))


(defentity ceps_src
  (table :CEPS)
  (database src))

(defentity ceps_target
  (table :CEPS)
  (database target))

;takes every l rows from db. the them offseting it (((0 + o) + o) + ...)
(defn fetch-every [l o] (-> (select* ceps_src) (limit l) (order :cep :ASC) (offset o)))

(defn extract-every
  "q is a fn that represents a query. q is a fn that takes 2 args, one for limit 
  (l) and one for offset (o).
  Returns a lazy seq that stops when there is no more results for q."
  ([q l] (extract-every q l 0))
  ([q l o] 
    (when-let [ret (seq (select (q l o)))]
      (cons ret (lazy-seq (extract-every q l (+ o l)))))))

;use #(persist % true) to dry-run the insert
(defn persist [v & d]
  (time (if d
           (dry-run (insert ceps_target (values v)))
           (insert ceps_target (values v))))
(println "Last saved" (last v)))

;; call the following from the repl:
;;(map #(persist % true) (extract-every q 20))