Simple Clojure memoize vs deref benchmark

memoize-vs-deref.core.clj

(ns memoize-vs-deref.core
  (:use criterium.core))

(def test-vec
  [{ :a   1 }
   { :f   6 }
   { :c   3 }
   { :d   4 }
   { :g   7 }
   { :h   8 }
   { :b   2 }
   { :i   9 }
   { :e   5 }
   { :w nil }
   { :j  10 }])

(defn test-vec-indexes
  []
  (remove #(nil? (first (vals %))) test-vec))

(defn test-vec-sorted
  []
  (sort-by #(first (vals %)) (test-vec-indexes)))

(defn test-vec-fn
  []
  (vec (map #(first (keys %)) (test-vec-sorted))))

(def test-vec-memo
  (memoize test-vec-fn))

(def test-vec-ref
  (ref (test-vec-fn)))

(def test-vec-atom
  (atom (test-vec-fn)))

(defn run!
  []
  (do
    (assert (= (test-vec-fn) (test-vec-memo) @test-vec-ref @test-vec-atom
               [:a :b :c :d :e :f :g :h :i :j]))
    (println "--")
    (println "test-vec-fn")
    (quick-bench (test-vec-fn))
    (println "--")
    (println "test-vec-memo")
    (quick-bench (test-vec-memo))
    (println "--")
    (println "test-vec-ref")
    (quick-bench @test-vec-ref)
    (println "--")
    (println "test-vec-atom")
    (quick-bench @test-vec-atom)))

project.clj

(defproject memoize-vs-deref "0.1.0-SNAPSHOT"
  :description "memoize vs deref performance"
  :license {:name "Eclipse Public License"
            :url "http://www.eclipse.org/legal/epl-v10.html"}
  :dependencies [[org.clojure/clojure "1.5.1"]
                 [criterium "0.4.1"]]
  :main memoize-vs-deref.core)

results.md

Results

Clojure 1.5.1

memoize-vs-deref.core=> (run!)
--
test-vec-fn
WARNING: Final GC required 3.21877537550772 % of runtime
WARNING: Final GC required 23.647212126881602 % of runtime
Evaluation count : 42636 in 6 samples of 7106 calls.
             Execution time mean : 15.303148 µs
    Execution time std-deviation : 685.379138 ns
   Execution time lower quantile : 14.560349 µs ( 2.5%)
   Execution time upper quantile : 16.047630 µs (97.5%)
                   Overhead used : 10.577454 ns
--
test-vec-memo
WARNING: Final GC required 32.76437686598905 % of runtime
Evaluation count : 8513190 in 6 samples of 1418865 calls.
             Execution time mean : 62.860000 ns
    Execution time std-deviation : 4.553484 ns
   Execution time lower quantile : 59.290363 ns ( 2.5%)
   Execution time upper quantile : 68.965085 ns (97.5%)
                   Overhead used : 10.577454 ns
--
test-vec-ref
WARNING: Final GC required 32.70806895663869 % of runtime
Evaluation count : 7643292 in 6 samples of 1273882 calls.
             Execution time mean : 61.320885 ns
    Execution time std-deviation : 7.609807 ns
   Execution time lower quantile : 53.959136 ns ( 2.5%)
   Execution time upper quantile : 70.726486 ns (97.5%)
                   Overhead used : 10.577454 ns
--
test-vec-atom
WARNING: Final GC required 46.98620379263601 % of runtime
Evaluation count : 21262062 in 6 samples of 3543677 calls.
             Execution time mean : 15.089389 ns
    Execution time std-deviation : 1.825023 ns
   Execution time lower quantile : 12.199170 ns ( 2.5%)
   Execution time upper quantile : 16.953413 ns (97.5%)
                   Overhead used : 10.577454 ns

Clojure 1.4.0

memoize-vs-deref.core=> (run!)
--
test-vec-fn
WARNING: Final GC required 2.823049126734798 % of runtime
WARNING: Final GC required 24.78557205458619 % of runtime
Evaluation count : 39738 in 6 samples of 6623 calls.
             Execution time mean : 14.888890 µs
    Execution time std-deviation : 180.895752 ns
   Execution time lower quantile : 14.714069 µs ( 2.5%)
   Execution time upper quantile : 15.113945 µs (97.5%)
                   Overhead used : 10.828664 ns
--
test-vec-memo
WARNING: Final GC required 28.52671789875035 % of runtime
Evaluation count : 8862114 in 6 samples of 1477019 calls.
             Execution time mean : 58.913499 ns
    Execution time std-deviation : 1.550494 ns
   Execution time lower quantile : 57.545541 ns ( 2.5%)
   Execution time upper quantile : 61.062252 ns (97.5%)
                   Overhead used : 10.828664 ns
--
test-vec-ref
WARNING: Final GC required 32.43716370139703 % of runtime
Evaluation count : 9604512 in 6 samples of 1600752 calls.
             Execution time mean : 53.306192 ns
    Execution time std-deviation : 2.022363 ns
   Execution time lower quantile : 51.685079 ns ( 2.5%)
   Execution time upper quantile : 56.675624 ns (97.5%)
                   Overhead used : 10.828664 ns

Found 1 outliers in 6 samples (16.6667 %)
  low-severe   1 (16.6667 %)
 Variance from outliers : 13.8889 % Variance is moderately inflated by outliers
--
test-vec-atom
WARNING: Final GC required 50.13639085875575 % of runtime
Evaluation count : 27334188 in 6 samples of 4555698 calls.
             Execution time mean : 10.486730 ns
    Execution time std-deviation : 0.524414 ns
   Execution time lower quantile : 10.080755 ns ( 2.5%)
   Execution time upper quantile : 11.362611 ns (97.5%)
                   Overhead used : 10.828664 ns

Found 1 outliers in 6 samples (16.6667 %)
  low-severe   1 (16.6667 %)
 Variance from outliers : 13.8889 % Variance is moderately inflated by outliers