heap-reducing.clj

;; use a heap to keep only the max N (here 10) items in memory at a time. imagine reducing over a 
;; quite large result set. don't want to realize it all, sort, and then take N. Just keep discarding
;; compared to our heap's min item.

(let [threshold 10
      cmp       (reify java.util.Comparator
                  (compare [_ x y]
                    ;; score is second so want to compare [score item]
                    (compare (vec (reverse x)) (vec (reverse y)))))
      xf        (comp (map (fn [{:keys [name]}] [name (count name)]))
                      (filter (fn [[name length]] (> length 5))))
      rf        (fn queue-accumulator
                  ([] (PriorityQueue. 30 cmp))
                  ([^PriorityQueue q]
                   (loop [acc []]
                     (if-let [x (.poll q)]
                       (recur (conj acc x))
                       acc)))
                  ([^PriorityQueue q item]
                   (if (>= (.size q) threshold)
                     (let [smallest (.peek q)]
                       (if (pos? (.compare cmp item smallest))
                         (doto q
                           (.poll)
                           (.offer item))
                         q))
                     (doto q
                       (.offer item)))))]
  (transduce xf rf (db/reducible-query {:select [:name]
                                        :from   [:metabase_field]})))