Benchmarking Node

babel-results.json

{
  "name": "ec2", 
  "succeeded": {
    "fortune": [], 
    "plaintext": [
      "hapi"
    ], 
    "db": [], 
    "update": [], 
    "json": [
      "hapi"
    ], 
    "query": []
  }, 
  "frameworks": [
    "hapi"
  ], 
  "verify": {
    "hapi": {
      "plaintext": "pass", 
      "json": "pass"
    }
  }, 
  "completed": {
    "hapi": "20150713155236"
  }, 
  "failed": {
    "fortune": [], 
    "plaintext": [], 
    "db": [], 
    "update": [], 
    "json": [], 
    "query": []
  }, 
  "rawData": {
    "fortune": {}, 
    "plaintext": {
      "hapi": [
        {
          "totalRequests": 22081, 
          "endTime": 1436802680, 
          "startTime": 1436802665
        }, 
        {
          "latencyAvg": "425.19ms", 
          "latencyMax": "14.57s", 
          "latencyStdev": "938.99ms", 
          "totalRequests": 90433, 
          "startTime": 1436802682, 
          "endTime": 1436802698
        }, 
        {
          "latencyAvg": "700.54ms", 
          "latencyMax": "10.36s", 
          "latencyStdev": "1.20s", 
          "totalRequests": 70249, 
          "startTime": 1436802700, 
          "endTime": 1436802715
        }, 
        {
          "totalRequests": 0, 
          "endTime": 1436802734, 
          "startTime": 1436802717
        }
      ]
    }, 
    "slocCounts": {
      "hapi": 13
    }, 
    "db": {}, 
    "update": {}, 
    "json": {
      "hapi": [
        {
          "latencyAvg": "40.65ms", 
          "latencyMax": "55.43ms", 
          "latencyStdev": "2.51ms", 
          "totalRequests": 2961, 
          "startTime": 1436802531, 
          "endTime": 1436802546
        }, 
        {
          "latencyAvg": "40.53ms", 
          "latencyMax": "58.47ms", 
          "latencyStdev": "2.20ms", 
          "totalRequests": 5903, 
          "startTime": 1436802548, 
          "endTime": 1436802563
        }, 
        {
          "latencyAvg": "41.71ms", 
          "latencyMax": "92.53ms", 
          "latencyStdev": "4.53ms", 
          "totalRequests": 11500, 
          "startTime": 1436802565, 
          "endTime": 1436802580
        }, 
        {
          "latencyAvg": "42.76ms", 
          "latencyMax": "131.89ms", 
          "latencyStdev": "6.32ms", 
          "totalRequests": 22420, 
          "startTime": 1436802582, 
          "endTime": 1436802597
        }, 
        {
          "latencyAvg": "50.82ms", 
          "latencyMax": "212.73ms", 
          "latencyStdev": "14.70ms", 
          "totalRequests": 37921, 
          "startTime": 1436802599, 
          "endTime": 1436802614
        }, 
        {
          "latencyAvg": "91.76ms", 
          "latencyMax": "1.43s", 
          "latencyStdev": "60.69ms", 
          "totalRequests": 42990, 
          "startTime": 1436802616, 
          "endTime": 1436802631
        }
      ]
    }, 
    "commitCounts": {
      "hapi": 32
    }, 
    "query": {}
  }, 
  "duration": 15, 
  "queryIntervals": [
    1, 
    5, 
    10, 
    15, 
    20
  ], 
  "concurrencyLevels": [
    8, 
    16, 
    32, 
    64, 
    128, 
    256
  ]
}

harmony-results.json

{
  "name": "ec2", 
  "succeeded": {
    "fortune": [], 
    "plaintext": [
      "hapi"
    ], 
    "db": [], 
    "update": [], 
    "json": [
      "hapi"
    ], 
    "query": []
  }, 
  "frameworks": [
    "hapi"
  ], 
  "verify": {
    "hapi": {
      "plaintext": "pass", 
      "json": "pass"
    }
  }, 
  "completed": {
    "hapi": "20150713154433"
  }, 
  "failed": {
    "fortune": [], 
    "plaintext": [], 
    "db": [], 
    "update": [], 
    "json": [], 
    "query": []
  }, 
  "rawData": {
    "fortune": {}, 
    "plaintext": {
      "hapi": [
        {
          "totalRequests": 10034, 
          "endTime": 1436802197, 
          "startTime": 1436802182
        }, 
        {
          "latencyAvg": "7.87s", 
          "latencyMax": "14.74s", 
          "latencyStdev": "7.11s", 
          "totalRequests": 70128, 
          "startTime": 1436802199, 
          "endTime": 1436802214
        }, 
        {
          "latencyAvg": "483.77ms", 
          "latencyMax": "9.56s", 
          "latencyStdev": "1.11s", 
          "totalRequests": 39584, 
          "startTime": 1436802217, 
          "endTime": 1436802232
        }, 
        {
          "totalRequests": 0, 
          "endTime": 1436802251, 
          "startTime": 1436802234
        }
      ]
    }, 
    "slocCounts": {
      "hapi": 13
    }, 
    "db": {}, 
    "update": {}, 
    "json": {
      "hapi": [
        {
          "latencyAvg": "40.11ms", 
          "latencyMax": "55.95ms", 
          "latencyStdev": "1.54ms", 
          "totalRequests": 2985, 
          "startTime": 1436802049, 
          "endTime": 1436802064
        }, 
        {
          "latencyAvg": "40.29ms", 
          "latencyMax": "53.25ms", 
          "latencyStdev": "1.69ms", 
          "totalRequests": 5936, 
          "startTime": 1436802066, 
          "endTime": 1436802081
        }, 
        {
          "latencyAvg": "41.44ms", 
          "latencyMax": "107.68ms", 
          "latencyStdev": "4.75ms", 
          "totalRequests": 11609, 
          "startTime": 1436802083, 
          "endTime": 1436802098
        }, 
        {
          "latencyAvg": "43.21ms", 
          "latencyMax": "119.70ms", 
          "latencyStdev": "6.51ms", 
          "totalRequests": 22240, 
          "startTime": 1436802100, 
          "endTime": 1436802115
        }, 
        {
          "latencyAvg": "54.13ms", 
          "latencyMax": "215.80ms", 
          "latencyStdev": "21.47ms", 
          "totalRequests": 36840, 
          "startTime": 1436802117, 
          "endTime": 1436802132
        }, 
        {
          "latencyAvg": "89.10ms", 
          "latencyMax": "1.85s", 
          "latencyStdev": "61.67ms", 
          "totalRequests": 44707, 
          "startTime": 1436802134, 
          "endTime": 1436802149
        }
      ]
    }, 
    "commitCounts": {
      "hapi": 32
    }, 
    "query": {}
  }, 
  "duration": 15, 
  "queryIntervals": [
    1, 
    5, 
    10, 
    15, 
    20
  ], 
  "concurrencyLevels": [
    8, 
    16, 
    32, 
    64, 
    128, 
    256
  ]
}

munger.rb

require 'json'

NODE_FILE = '/Users/nwest/Documents/node-benchmark/results.json'
HARMONY_FILE = '/Users/nwest/Documents/node-harmony-benchmark/results.json'
BABEL_FILE = '/Users/nwest/Documents/babel-node-benchmark/results.json'

def get_json_latency(result_set)
  latencies = result_set["rawData"]["json"]["hapi"].map { |run| run.fetch("latencyAvg") }
  latencies.map { |latency_string| latency_string.gsub("ms", "").to_f }.reduce(&:+) / latencies.length
end

def get_json_rps(result_set)
  rpses = result_set["rawData"]["json"]["hapi"].map { |run| run.fetch("totalRequests") }
  rpses.reduce(&:+) / rpses.length
end

def get_plaintext_latency(result_set)
  latencies = result_set["rawData"]["plaintext"]["hapi"]
              .select { |run| run.keys.include?("latencyAvg") }
              .map { |run| run.fetch("latencyAvg") }
  latencies.map { |latency_string| latency_string.gsub("ms", "").to_f }.reduce(&:+) / latencies.length
end

def get_plaintext_rps(result_set)
  rpses = result_set["rawData"]["plaintext"]["hapi"].map { |run| run.fetch("totalRequests") }
  rpses.reduce(&:+) / rpses.length
end

def result_set_to_results(name, result_set)
  {
    name: name,
    json_latency: get_json_latency(result_set),
    json_rps: get_json_rps(result_set),
    plaintext_latency: get_plaintext_latency(result_set),
    plaintext_rps: get_plaintext_rps(result_set)
  }
end

def results_from_file(name, file)
  result_set = JSON.parse(File.read(file))
  result_set_to_results(name, result_set)
end

def find_least_json_latency(results)
  winner = results.min { |a, b| a[:json_latency] <=> b[:json_latency] }
  winner[:name]
end

def find_highest_json_rps(results)
  winner = results.max { |a, b| a[:json_rps] <=> b[:json_rps] }
  winner[:name]
end

def find_least_plaintext_latency(results)
  winner = results.min { |a, b| a[:plaintext_latency] <=> b[:plaintext_latency] }
  winner[:name]
end

def find_highest_plaintext_rps(results)
  winner = results.max { |a, b| a[:palintext_rps] <=> b[:palintext_rps] }
  winner[:name]
end

def munge
  all_results = {node: NODE_FILE, harmony: HARMONY_FILE, babel: BABEL_FILE}
                .map { |(name, file)| results_from_file(name, file) }

  least_json_latency = find_least_json_latency(all_results)
  highest_json_rps = find_highest_json_rps(all_results)
  least_plaintext_latency = find_least_plaintext_latency(all_results)
  highest_plaintext_rps = find_highest_plaintext_rps(all_results)

  puts "Least json latency: #{least_json_latency}"
  puts "Highest json throughput: #{highest_json_rps}"
  puts "Least plaintext latency: #{least_plaintext_latency}"
  puts "Highest plaintext throughput: #{highest_plaintext_rps}"
end

munge

node-results.json

{
  "name": "ec2", 
  "succeeded": {
    "fortune": [], 
    "plaintext": [
      "hapi"
    ], 
    "db": [], 
    "update": [], 
    "json": [
      "hapi"
    ], 
    "query": []
  }, 
  "frameworks": [
    "hapi"
  ], 
  "verify": {
    "hapi": {
      "plaintext": "pass", 
      "json": "pass"
    }
  }, 
  "completed": {
    "hapi": "20150713153635"
  }, 
  "failed": {
    "fortune": [], 
    "plaintext": [], 
    "db": [], 
    "update": [], 
    "json": [], 
    "query": []
  }, 
  "rawData": {
    "fortune": {}, 
    "plaintext": {
      "hapi": [
        {
          "totalRequests": 30944, 
          "endTime": 1436801720, 
          "startTime": 1436801705
        }, 
        {
          "latencyAvg": "5.91s", 
          "latencyMax": "14.77s", 
          "latencyStdev": "7.02s", 
          "totalRequests": 80960, 
          "startTime": 1436801722, 
          "endTime": 1436801738
        }, 
        {
          "latencyAvg": "874.00ms", 
          "latencyMax": "13.97s", 
          "latencyStdev": "1.18s", 
          "totalRequests": 71200, 
          "startTime": 1436801740, 
          "endTime": 1436801755
        }, 
        {
          "startTime": 1436801757, 
          "read": 0, 
          "totalRequests": 10, 
          "write": 0, 
          "connect": 0, 
          "timeout": 1828, 
          "endTime": 1436801773
        }
      ]
    }, 
    "slocCounts": {
      "hapi": 13
    }, 
    "db": {}, 
    "update": {}, 
    "json": {
      "hapi": [
        {
          "latencyAvg": "40.28ms", 
          "latencyMax": "57.11ms", 
          "latencyStdev": "1.57ms", 
          "totalRequests": 2969, 
          "startTime": 1436801572, 
          "endTime": 1436801587
        }, 
        {
          "latencyAvg": "40.23ms", 
          "latencyMax": "57.21ms", 
          "latencyStdev": "1.56ms", 
          "totalRequests": 5927, 
          "startTime": 1436801590, 
          "endTime": 1436801605
        }, 
        {
          "latencyAvg": "41.22ms", 
          "latencyMax": "63.13ms", 
          "latencyStdev": "2.81ms", 
          "totalRequests": 11638, 
          "startTime": 1436801607, 
          "endTime": 1436801622
        }, 
        {
          "latencyAvg": "43.98ms", 
          "latencyMax": "174.35ms", 
          "latencyStdev": "8.71ms", 
          "totalRequests": 21960, 
          "startTime": 1436801624, 
          "endTime": 1436801639
        }, 
        {
          "latencyAvg": "50.23ms", 
          "latencyMax": "160.19ms", 
          "latencyStdev": "12.78ms", 
          "totalRequests": 38515, 
          "startTime": 1436801641, 
          "endTime": 1436801656
        }, 
        {
          "latencyAvg": "87.78ms", 
          "latencyMax": "1.37s", 
          "latencyStdev": "54.57ms", 
          "totalRequests": 43296, 
          "startTime": 1436801658, 
          "endTime": 1436801673
        }
      ]
    }, 
    "commitCounts": {
      "hapi": 32
    }, 
    "query": {}
  }, 
  "duration": 15, 
  "queryIntervals": [
    1, 
    5, 
    10, 
    15, 
    20
  ], 
  "concurrencyLevels": [
    8, 
    16, 
    32, 
    64, 
    128, 
    256
  ]
}

nwest at 3442344 in ~/code/FrameworkBenchmarks (master●●)
$ ruby munger.rb
Least json latency: node
Highest json throughput: harmony
Least plaintext latency: harmony
Highest plaintext throughput: node