hiphip benchmarks

bench.clj

(ns user
  (:require
   [criterium.core :as criterium]
   [hiphip.double :as d]))

(do
  (def ^:const sz 10000)
  (defmacro print-time
    "Better, but involves external lib."
    [expr]
    `(println '~expr (* 1000 (first (:mean (criterium/quick-benchmark ~expr {})))) "ms")))

#_
(do
  (def ^:const sz 10000000)
  (defmacro print-time
    "Hand-rolled benchmarks like this are extremely dangerous since you may not measure what you
  think; hotspot may even completely optimize away your code if it realizes the result can't
  matter."
    [expr]
    `(do (println '~expr)
         (dotimes [_# 3] (time ~expr)))))



(defn map-inc [xs]
  (mapv inc xs))

(defn map-inc-clojure [^doubles xs]
  (amap xs i ret (aset ret i (inc (aget xs i)))))

(defn map-inc-hiphip [xs]
  (d/amap [x xs] (inc x)))

(defn test-map-inc []
  (let [a (double-array (repeatedly 100 rand))]
    (apply =
           (for [f [map-inc map-inc-clojure map-inc-hiphip]]
             (apply + (f a))))))

(defn bench-map-inc []
  ;; clojure on unchunked seq
  (let [unchunked (doall (take sz (iterate inc 0.0)))]
    (print-time (doall (map inc unchunked))))
  ;; clojure mapv on chunked seq
  (let [chunked (vec (map double (range sz)))]
    (print-time (doall (mapv inc chunked))))
  ;; ops on arrays
  (let [a (double-array (repeatedly sz rand))]
    (print-time (map-inc a))
    (print-time (map-inc-clojure a))
    (print-time (map-inc-hiphip a))))



(defn dot-product [ws xs]
  (reduce + (map * ws xs)))

(defn dot-product-clojure [^doubles ws ^doubles xs]
  (areduce ws i out 0.0 (+ out (* (aget ws i) (aget xs i)))))

(defn dot-product-hiphip [ws xs]
  (d/asum [x xs w ws] (* x w)))

(defn test-dot-product []
  (let [a (double-array (repeatedly 100 rand))
        b (double-array (repeatedly 100 rand))]
    (apply =
           (for [f [dot-product dot-product-clojure dot-product-hiphip]]
             (f a b)))))

(defn bench-dot-product []
  ;; clojure on unchunked seqs
  (let [unchunked (doall (take sz (iterate inc 0.0)))]
    (print-time (dot-product unchunked unchunked)))
  ;; clojure on chunked seqs
  (let [chunked (vec (map double (range sz)))]
    (print-time (dot-product chunked chunked)))
  (let [a (double-array (repeatedly sz rand))]
    (print-time (dot-product a a))
    (print-time (dot-product-clojure a a))
    (print-time (dot-product-hiphip a a))))

;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;; Results:

;;; With criterium -- much more reliable and precise

;; user> (bench-map-inc)
;; (doall (map inc unchunked)) 1.2935320512820514 ms
;; (doall (mapv inc chunked)) 0.24182537788385045 ms
;; (map-inc a) 0.14261173708920188 ms
;; (map-inc-clojure a) 0.020098884976525823 ms
;; (map-inc-hiphip a) 0.010556760360739728 ms

;; user> (bench-dot-product)
;; (dot-product unchunked unchunked) 1.6849508196721312 ms
;; (dot-product chunked chunked) 1.2446234567901235 ms
;; (dot-product a a) 1.2617703252032524 ms
;; (dot-product-clojure a a) 0.011385344533999319 ms
;; (dot-product-hiphip a a) 0.008520291649470354 ms


;;; With `time` (-- removing the #_ above ) -- all the
;;; usual caveats of rolling your own benchmarks apply.

;; user> (bench-map-inc)
;; (doall (map inc unchunked))
;; "Elapsed time: 8700.094 msecs"
;; "Elapsed time: 6611.289 msecs"
;; "Elapsed time: 15440.647 msecs"
;; (doall (mapv inc chunked))
;; "Elapsed time: 1192.988 msecs"
;; "Elapsed time: 518.504 msecs"
;; "Elapsed time: 449.79 msecs"
;; (map-inc a)
;; "Elapsed time: 237.254 msecs"
;; "Elapsed time: 229.945 msecs"
;; "Elapsed time: 193.055 msecs"
;; (map-inc-clojure a)
;; "Elapsed time: 36.357 msecs"
;; "Elapsed time: 30.505 msecs"
;; "Elapsed time: 33.487 msecs"
;; (map-inc-hiphip a)
;; "Elapsed time: 34.218 msecs"
;; "Elapsed time: 26.645 msecs"
;; "Elapsed time: 24.491 msecs"

;; user> (bench-dot-product)
;; (dot-product unchunked unchunked)
;; "Elapsed time: 6029.915 msecs"
;; "Elapsed time: 6368.012 msecs"
;; "Elapsed time: 3586.885 msecs"
;; (dot-product chunked chunked)
;; "Elapsed time: 6445.413 msecs"
;; "Elapsed time: 8418.37 msecs"
;; "Elapsed time: 1853.243 msecs"
;; (dot-product a a)
;; "Elapsed time: 5912.253 msecs"
;; "Elapsed time: 7310.311 msecs"
;; "Elapsed time: 7055.002 msecs"
;; (dot-product-clojure a a)
;; "Elapsed time: 18.105 msecs"
;; "Elapsed time: 13.286 msecs"
;; "Elapsed time: 12.907 msecs"
;; (dot-product-hiphip a a)
;; "Elapsed time: 18.922 msecs"
;; "Elapsed time: 22.388 msecs"
;; "Elapsed time: 9.573 msecs"