elasticsearch-index-migration

core.clj

(ns core
  (:require
    [clojurewerkz.elastisch.rest.document :as doc]
    [clojurewerkz.elastisch.rest :as esr]
    [clojurewerkz.elastisch.query :as q]
    [clojurewerkz.elastisch.rest.response :as esrsp]))

(defn move-documents [conn <-index scroll-id]
  (let [;; scroll query - step 2
        scan-response (doc/scroll conn scroll-id :scroll "1m")
        scan-hits (esrsp/hits-from scan-response)
        ;; take the fresh scroll id
        scroll-id (:_scroll_id scan-response)]
    (when-not (empty? scan-hits)
      ;; Save hits to target index - step 3
      (doseq [hit scan-hits]
        (doc/create conn <-index (:_type hit) (:_source hit) :id (:_id hit)))
      ;; repeat - step 4; Good old-school recursion
      (move-documents conn <-index scroll-id))))


(defn -main [& args]
  (let [index-> (first args)  ; The index that provides data
        <-index (second args) ; The index that receives data
        _ (println "from" index-> "to" <-index) ; Tell me something ...
        ; Open connection to ES
        conn (esr/connect "http://localhost:9200")
        ;; Initialize scroll id - step 1
        response (doc/search conn index-> "test"
                             :query (q/match-all)
                             :search_type "scan"
                             :scroll "1m"
                             :size 10)
        scroll-id (:_scroll_id response)]
    ;; The hard work
    (move-documents conn <-index scroll-id)))

feeddata.sh

for i in $(seq 100); do
   curl -s -XPOST http://localhost:9200/threeshards/test/$i?pretty -d "{ \"when\" : \"$(date --rfc-3339=ns)\"}" > /dev/null
done;

project.clj

(defproject
  elasticsearch-index-migration "0.1.0-SNAPSHOT"

  :description "A Clojure based tool for migrating data between 2 indices."

  :url "https://gist.github.com/martinhynar"
  
  :license {:name "Eclipse Public License"
            :url  "http://www.eclipse.org/legal/epl-v10.html"}

  :dependencies [[org.clojure/clojure "1.6.0"]
                 [org.clojure/data.json "0.2.5"]
                 [clojurewerkz/elastisch "2.1.0"]
                 [clj-http "1.0.1" :exclusions [org.clojure/tools.reader]]
                 ]
  )

threeshards.sh

curl -XPUT 'http://localhost:9200/threeshards' -d '
{
    "settings" : {
        "index" : {
            "number_of_shards" : 3,
            "number_of_replicas" : 0
        }
    }
}'

twoshards.sh

curl -XPUT 'http://localhost:9200/twoshards' -d '
{
    "settings" : {
        "index" : {
            "number_of_shards" : 2,
            "number_of_replicas" : 0
        }
    }
}'