dupes.clj

(defn find-dupes-naieve [inp-seq]
  (reduce (fn [res item]
            (assoc res item (inc (get res item 0))))
          {}
          (map #(second (.split %1 "\t")) inp-seq)))


(defn find-dupes-with-bloom-filter [inp-seq expected-size fp-prob]
  (let [flt (bloom/make-optimal-filter expected-size fp-prob)]
    (reduce (fn [res item]
              (if-not (bloom/include? flt item)
                (do
                  (bloom/add! flt item)
                  res)
                (assoc res item (inc (get res item 1)))))
            {}
            (map #(second (.split %1 "\t")) inp-seq))))