distinct_bench.clj

(require '[criterium.core :as c])

(defn format-time [estimate]
  (let [mean (first estimate)
        [factor unit] (c/scale-time mean)]
    (c/format-value mean factor unit)))


(defmacro race [body1 body2]
 `(let [_#        (assert (= ~body1 ~body2))
        results1# (c/quick-benchmark ~body1 {})
        results2# (c/quick-benchmark ~body2 {})
        percent#  (->> (/ (first (:mean results2#))
                          (first (:mean results1#)))
                       (* 100)
                       (- 100)
                       (int))]
    
    (println ~(pr-str body1) "\t"
             (format-time (:mean results1#)) "=>" (format-time (:mean results2#))
             (str "(" (- percent#) "%)"))))


(defn transient-distinct
  "Returns a lazy sequence of the elements of coll with duplicates removed.
  Returns a stateful transducer when no collection is provided."
  {:added "1.0"
   :static true}
  ([]
   (fn [rf]
     (let [seen ^clojure.lang.ITransientSet (transient #{})]
       (fn
         ([] (rf))
         ([result] (rf result))
         ([result input]
          (if (.contains seen input)
            result
            (do (conj! seen input)
                (rf result input))))))))
  ([coll]
   (let [step (fn step [xs seen]
                (lazy-seq
                  ((fn [[f :as xs] ^clojure.lang.ITransientSet seen]
                     (when-let [s (seq xs)]
                       (if (.contains seen f)
                         (recur (rest s) seen)
                         (cons f (step (rest s) (conj! seen f))))))
                   xs seen)))]
     (step coll (transient #{})))))

(doseq [size [10 100 1000 10000]
          :let [coll (vec (repeatedly size #(rand-int 1000)))]]
    (println size "elements")
    (race (doall (distinct coll))
          (doall (transient-distinct coll)))
    
    (race (into [] (distinct) coll)
          (into [] (transient-distinct) coll)))

results

10 elements
(doall (distinct coll)) 	 5.773439 µs => 4.179092 µs (-27%)
(into [] (distinct) coll) 	 3.238236 µs => 1.943254 µs (-39%)

100 elements
(doall (distinct coll)) 	 67.725764 µs => 42.129993 µs (-37%)
(into [] (distinct) coll) 	 35.702741 µs => 16.495947 µs (-53%)

1000 elements
(doall (distinct coll)) 	 540.652739 µs => 399.053873 µs (-26%)
(into [] (distinct) coll) 	 301.423077 µs => 164.025500 µs (-45%)

10000 elements
(doall (distinct coll)) 	 3.439137 ms => 3.058872 ms (-11%)
(into [] (distinct) coll) 	 1.437390 ms => 848.277178 µs (-40%)