Differential delete from Solr to catch stale docs

gistfile1.clj

(ns solr-disj.core
  (:require [clojure-solr :as csolr]
            [clojure.set]
            [clojure.string])
  (:use [solr-disj.secret :only [solr-server]]
        [solr-disj.db]
        [icarus.core :only [-?>]]
        [korma.core]
        [korma.db]
        [clojure.data.xml]))

;; Lazy sequence of all values of the uid field in the Solr index.
(defn idx-uids
  ([j] (lazy-seq (mapcat vals (-?> solr-server "*:*" {:start j
                                                      :rows 4096
                                                      :fl "uid"
                                                      :sort "uid asc"}))))
  ([j k] (lazy-seq (mapcat vals (-?> solr-server "*:*" {:start j
                                                        :rows k
                                                        :fl "uid"
                                                        :sort "uid asc"})))))

(defn uid-diff [a b]
  (lazy-seq (clojure.set/difference (set a) (set b))))

(defn del-stale-docs [idx-count]
  "Delete documents from the index that have grown stale; that is, docs
indexed from rows in the database that no longer exist. Takes the count
of documents in the Solr index as its only argument."
  (let [uids (lazy-seq (mapcat vals (select jobs (fields :uid))))]
    (for [i (partition-all 4096 (range idx-count))]
      (let [diff (uid-diff (idx-uids (first i)) uids)]
        (if (not (empty? diff))
          (csolr/with-connection (csolr/connect solr-server)
            (csolr/delete-query!
             (str "uid:(" (clojure.string/join " OR " diff) ")"))
            (csolr/commit!)))))))

(del-stale-docs 599339)