Benchmarking realm vs status-go

bench.md

Disclaimer: the code is just there so that the curious reader can experiment himself and verify the numbers on his own setup, it is not 'good code' or anything, just the fastest way to get numbers as correct as possible and see if we'd have a perf issue with moving the db to status-go as opposed to keep it in realm-js.

Adding 4000 items in status-go db

(time (doall
       (dotimes [x 4000]
         (call {:method "wallet_addBrowser" :params [{"id" (str x) "name" "third", "timestamp" "0x0"
                                                      "dapp" false
                                                      "historyIndex" "0x3"
                                                      "history" ["hist1" "hist2"]}] :on-success #()}))))

"Elapsed time: 4065.000000 msecs"

Timing could be improved by working on the json-rpc/call function, however here we are doing something that is never done in the app, and if we were, we'd rather be doing one call that adds the 4000 items at once rather than 4000 calls adding one item

Benchmarking reads of the 4000 items in status-go

In status-im.ethereum.json-rpc namespace add the following code:

(do (def json-timer (atom nil))
    (def json-timer2 (atom nil))
    (def json-callback-sum (atom nil))
    (def results (atom nil))

    (defn call2
      [{:keys [method params on-success on-error]}]
      (when-let [method-options (json-rpc-api method)]
        (let [{:keys [id on-result subscription?]
               :or {on-result identity
                    id        1
                    params    []}} method-options
              on-error (or on-error
                           #(log/warn :json-rpc/error method :params params :error %))]
          (if (nil? method)
            (log/error :json-rpc/method-not-found method)
            (status/call-private-rpc
             (types/clj->json {:jsonrpc "2.0"
                               :id      id
                               :method  (if subscription?
                                          "eth_subscribeSignal"
                                          method)
                               :params  (if subscription?
                                          [method params]
                                          params)})
             (fn [response]
               (reset! json-timer2 (status-im.utils.datetime/timestamp))
               (if (string/blank? response)
                 (on-error {:message "Blank response"})
                 (let [{:keys [error result] :as response2} (types/json->clj response)]
                   (if error
                     (on-error error)
                     (if subscription?
                       (re-frame/dispatch
                        [:ethereum.callback/subscription-success
                         result on-success])
                       (on-success (on-result result))))))
               (swap! json-callback-sum + (- (status-im.utils.datetime/timestamp) @json-timer2))))))))

    (defn get-browsers-benchmark
      [n x]
      (call2 {:method "wallet_getBrowsers"
              :on-success
              #(if (= n x)
                 (let [avg-cb (/ @json-callback-sum x)
                       avg-tot (/ (- (status-im.utils.datetime/timestamp) @json-timer) x)
                       avg-async (- avg-tot avg-cb)]
                   (do (reset! results [avg-cb avg-tot avg-async])
                       (println :average-UI-thread avg-cb)
                       (println :average-async avg-async)
                       (println :average-total avg-tot)))
                 (do (println n "/" x)
                     (get-browsers-benchmark (inc n) x)))}))

    (defn benchmark-json []
      (reset! json-callback-sum 0)
      (reset! json-timer (status-im.utils.datetime/timestamp))
      (get-browsers-benchmark 0 30))

    (def transit-timer (atom nil))
    (def transit-timer2 (atom nil))
    (def transit-callback-sum (atom nil))

    (defn call3
      [{:keys [method params on-success on-error]}]
      (when-let [method-options (json-rpc-api method)]
        (let [{:keys [id on-result subscription?]
               :or {on-result identity
                    id        1
                    params    []}} method-options
              on-error (or on-error
                           #(log/warn :json-rpc/error method :params params :error %))]
          (if (nil? method)
            (log/error :json-rpc/method-not-found method)
            (status/call-private-rpc
             (types/clj->json {:jsonrpc "2.0"
                               :id      id
                               :method  (if subscription?
                                          "eth_subscribeSignal"
                                          method)
                               :params  (if subscription?
                                          [method params]
                                          params)})
             (fn [response]
               (reset! transit-timer2 (status-im.utils.datetime/timestamp))
               (if (string/blank? response)
                 (on-error {:message "Blank response"})
                 (let [{:keys [error result] :as response2} (deserialize response)]
                   (if error
                     (on-error error)
                     (if subscription?
                       (re-frame/dispatch
                        [:ethereum.callback/subscription-success
                         result on-success])
                       (on-success (on-result result))))))
               (swap! transit-callback-sum + (- (status-im.utils.datetime/timestamp) @transit-timer2))))))))

    (defn get-browsers-transit-benchmark
      [n x]
      (call3 {:method "wallet_getBrowsers"
              :on-success
              #(if (= n x)
                 (let [avg-cb (/ @transit-callback-sum x)
                       avg-tot (/ (- (status-im.utils.datetime/timestamp) @transit-timer) x)
                       avg-async (- avg-tot avg-cb)
                       [j1 j2 j3] @results]
                   (println :transit-encoding)
                   (println :average-UI-thread avg-cb " x" (/ j1 avg-cb))
                   (println :average-async avg-async " x" (/ j3 avg-async))
                   (println :average-total avg-tot " x" (/ j2 avg-tot)))
                 (do (println n "/" x)
                     (get-browsers-transit-benchmark (inc n) x)))}))

    (defn benchmark-transit []
      (reset! transit-callback-sum 0)
      (reset! transit-timer (status-im.utils.datetime/timestamp))
      (get-browsers-transit-benchmark 0 30)))

To start the benchmark do (benchmark-json) once it is done (benchmark-transit)

Results:

json

:average-UI-thread 847.2666666666667
:average-async 186.4000000000001
:average-total 1033.6666666666667

transit

:average-UI-thread 196.16666666666666  x 4.3191163976210705
:average-async 141.5666666666667  x 1.3166941370379095
:average-total 337.73333333333335  x 3.0606000789577577

So with json encoding status-go on average there is about 1000ms between the query and a response in a cljs datastructure usable by status-react

With transit this is about 3 times faster with about 350ms, but more importantly it is more than 4 times faster on the UI thread, with only around 200ms of blocking time.

Adding 4000 browser items in realm

(status-im.data-store.realm.core/write
 @status-im.data-store.realm.core/account-realm
 #(doseq [x (range 4000)]
    (status-im.data-store.realm.core/create
     @status-im.data-store.realm.core/account-realm
     :browser
     {:browser-id (str x)
      :name "third",
      :timestamp 0
      :dapp false
      :historyIndex 3
      :history ["hist1" "hist2"]}
     true)))

Note: at this point the emulator is begging me to end its miserable existence, but I refuse.

Out of pity, I remove the browsers from app-db to relieve it.

(swap! re-frame.db/app-db dissoc :browser/browsers)

Reading 4000 browser items from realm

In status-im.data-store.browser namespace run:

(do (def realm-timer (atom nil))
    (def realm-counter (atom 0))
    (def realm-total (atom 0))

    (defn benchmark-realm
      []
      (reset! realm-timer (status-im.utils.datetime/timestamp))
      (do (-> @core/account-realm
              (core/get-all :browser)
              (core/sorted :timestamp :desc)
              (core/all-clj :browser))
          1)
      (let [current (- (status-im.utils.datetime/timestamp) @realm-timer)]
        (swap! realm-total + current)
        (swap! realm-counter inc)
        (println "Current " current " Average " (/ @realm-total @realm-counter)))))

Becomes realm is synchronous testing it in the repl is a bit tedious. The (benchmark-realm) call will make one query and everytime you call it you will get the average of all your previous calls.

Using this method I got an average of 2500ms between the moment the query is done and the moment I get a usable cljs datastructure. All of it is a synchronous call blocking the UI thread.

Conclusion

In this benchmark, it took on average more than 7 times more time for realm to get 4000 browsers items than status-go using transit encoding and 2.5 times more time than status-go using json encoding. On top of that because part of the status-go queries is async and doesn't block the UI thread, we can also add that realm queries blocks the UI thread 10 times longer than status-go queries using transit encoding and 3 times longer than status-go using json encoding.

Before that benchmark we were worried about the risk of lesser performances when moving the data to status-go. It turns out that there is actually a performance gain, and that by trivially adding transit encoding to the equation, we get an order of magnitude performance improvement in terms of time spent blocking the UI thread.