preliminary benchmarks for json on MacRuby

bench.rb

require 'rubygems'
gem 'minitest', '>= 2.3.1'
require 'minitest/autorun'
require 'minitest/benchmark'
require 'json'
framework 'Foundation'

class Bench < MiniTest::Unit::TestCase
  def self.order
    :alpha
  end

  def self.bench_range
    bench_exp 1_000, 100_000
  end

  def json val
    assert_performance_linear do |n|
      n.times { val.to_json }
    end
  end

  # @todo varying lengths of strings, not important now because it does
  #       not stress the part of the code that is responsible for the
  #       slow down
  def bench_string
    json ['blah']
  end

  def bench_nsstring
    json [NSString.stringWithString('blah')]
  end

  def bench_number
    json [100]
  end

  def bench_hash
    # hash = { this:{is:{a:{test:{}}}}, life:{universe:42}, pie:'cake', hash:{with:1,few:2,keys:3} }
    json a: {}, b: {}
  end

  def bench_nsdict
    x = NSDictionary
    def NSDictionary.y *args
      dictionaryWithObjectsAndKeys *args.reverse, nil
    end
    # hash = x.y('this', x.y('a', x.y('test', x.y())), 'life', x.y('universe',42), 'pie', 'cake', 'hash', x.y('with',1,'few',2,'keys',3))
    json x.y('a',x.y, 'b',x.y)
  end

  def bench_array
    # array = [[[[[[]]]]], [[]], [[[[],[]],[[],[],[]],[]]], [[],[]], [[[],[]]]]
    json [[1],[2]]
  end

  def bench_nsarray
    x = NSArray
    def x.y *args
      arrayWithObjects *args, nil
    end
    # array = x.y x.y(x.y(x.y(x.y(x.y())))), x.y(x.y), x.y(x.y(x.y(x.y,x.y),x.y(x.y,x.y,x.y), x.y)), x.y(x.y,x.y), x.y(x.y(x.y,x.y))
    json x.y(x.y(1), x.y(2))
  end

  def bench_nil
    json [nil]
  end

  def bench_boolean
    json [true, false]
  end

  def bench_bignum
    json [1_000_000_000_000_000_000_000_000_000_000_000_000_000]
  end

  def bench_float
    json [2.71828183]
  end

  def bench_object
    json [Object.new]
  end

  def bench_smorgasbord
    json can: [{you: 'take', me: 'to', funky: 'town?'}, 1, 2, 3000, Object.new, true, -3.1415, nil]
  end
end

MacRuby 0.11 (nightly)

Bench            1000        10000       100000
bench_array      0.017672    0.141334    1.419953
bench_nsarray    0.004926    0.056099    0.510012
bench_bignum     0.021498    0.178389    1.814271
bench_boolean    0.005183    0.058042    0.541852
bench_float      0.012118    0.109843    1.036338
bench_hash       0.048230    0.432598    4.216660
bench_nsdict     0.060144    0.535491    5.368870
bench_nil        0.005834    0.059063    0.528223
bench_number     0.010391    0.060289    0.535806
bench_object     0.027736    0.245261    2.318201
bench_smorgasbord    0.124294    1.133121   11.417242
bench_string         0.017989    0.161462    1.626397
bench_nsstring       0.019732    0.190960    1.715292

Finished benchmarks in 37.066402s, 0.3507 tests/s, 0.3507 assertions/s.

Something funky is going on with arrays. I'm guessing there is an optimized empty array class for NSArray.

After updating the benchmark:

Bench   1000    10000   100000
bench_array      0.018272    0.141912    1.359039
bench_nsarray    0.016848    0.154996    1.483461
bench_bignum     0.019732    0.183136    1.765198
bench_boolean    0.004825    0.057215    0.533383
bench_float      0.012839    0.107901    1.038991
bench_hash       0.046788    0.418722    4.183835
bench_nsdict     0.057859    0.524156    5.280439
bench_nil        0.004999    0.068118    0.520402
bench_number     0.004732    0.062262    0.524425
bench_object     0.023123    0.234684    2.284864
bench_smorgasbord    0.119892    1.136674   11.710081
bench_string     0.019586    0.175391    1.694385
bench_nsstring   0.015845    0.176576    1.705815

Finished benchmarks in 38.140245s, 0.3408 tests/s, 0.3408 assertions/s.

MacRuby 0.10

Bench   1000    10000   100000
bench_array      0.006472    0.060398    0.575754
bench_nsarray    0.007006    0.059219    0.615877
bench_bignum     0.017705    0.152745    1.480051
bench_boolean    0.002566    0.028827    0.285693
bench_float      0.006105    0.062891    0.686732
bench_hash       0.013962    0.096075    1.002865
bench_nsdict     0.017937    0.149407    1.524524
bench_nil        0.002588    0.027587    0.273539
bench_number     0.003858    0.045224    0.418574
bench_object     0.009791    0.102272    1.060915
bench_smorgasbord    0.034375    0.312462    3.113500
bench_string     0.002773    0.030755    0.279621
bench_nsstring   0.003853    0.043759    0.374686

Finished benchmarks in 14.116058s, 0.9209 tests/s, 0.9209 assertions/s.

Well, that's a little depressing.

This is from applying appropriate patches to the upstream JSON repository:

Bench   1000    10000   100000
bench_array      0.009178    0.068889    0.576111
bench_nsarray    0.006231    0.071549    0.659718
bench_bignum     0.013563    0.143416    1.418103
bench_boolean    0.005108    0.065971    0.591222
bench_float      0.008823    0.121051    1.025400
bench_hash       0.032613    0.262340    2.698372
bench_nsdict     0.067821    0.338544    3.332269
bench_nil        0.005587    0.066914    0.592831
bench_number     0.005283    0.068908    0.547478
bench_object     0.042540    0.205396    2.025258
bench_smorgasbord    0.077420    0.754987    7.198482
bench_string     0.013055    0.124828    1.179611
bench_nsstring   0.013120    0.150757    1.456123

Finished benchmarks in 26.300107s, 0.4943 tests/s, 0.4943 assertions/s.