crossfilter_performance.coffee

process = require('process')

_start = Date.now()
_title = ""
_mem = 0
tic = (title) ->
  _title = title
  _start = Date.now()
  console.log(_title, '..');
  _mem = process.memoryUsage().heapUsed
toc = () ->
  time = (Date.now() - _start) / 1000
  console.log('time:', time + ' sec')
  total = (process.memoryUsage().heapUsed) / 1000000
  up = (process.memoryUsage().heapUsed - _mem) / 1000000
  console.log('mem total:', total + ' mb, up:', up);
  console.log()

{crossfilter} = require('./src/crossfilter')

data = []

tips = [0,1,2]
types = [0..300]

tic("create data")
i = 0
while i < 5000000
  date = (new Date(Date.now() + Math.random() * 1000 * 60 * 60 * 24 * 365)).getTime()
  total = Math.random() * 300
  tip = tips[Math.floor(Math.random() * 3)]
  type = types[Math.floor(Math.random() * 300)]
  data.push {
    date
    total
    tip
    type
  }
  i += 1
toc()

tic("constructor")
cf = crossfilter(data);
toc()

tic("create dim")
allDim = cf.dimension (d) -> 1
dateDim = cf.dimension (d) -> d.date
totalDim = cf.dimension (d) -> d.total
typeDim = cf.dimension (d) -> d.type
tipDim = cf.dimension (d) -> d.tip
toc()

tic("create group")
dateGroup = dateDim.group (date) -> return (new Date(date)).getMonth()
dateReduced = dateGroup.reduceSum (d) -> return d.total
typeGroup = typeDim.group (type) -> return type
typeReduced = typeGroup.reduceSum (d) -> return d.total
totalGroup = totalDim.group (total) -> return Math.floor(total / 10)
totalReduced = totalGroup.reduceSum (d) -> return d.total
countGroup = allDim.group () -> return 1
countReduced = countGroup.reduceCount()
sumReduced = countGroup.reduceSum () -> return 1
toc()

tic("filter")
totalDim.filter([100, 200])
typeDim.filter('tab')
tipDim.filter(0)
toc()

tic("all count")
countReduced.top(1)
toc()

tic("all sum")
sumReduced.top(1)
toc()

tic("top3 by group")
typeReduced.top(3)
totalReduced.top(3)
dateReduced.top(3)
toc()

tic("update filter")
totalDim.filter([200, 300])
toc()

tic("top3 by group")
typeReduced.top(3)
totalReduced.top(3)
dateReduced.top(3)
toc()

> coffee -cp test.coffee | node --max-old-space-size=4048

create data ..
time: 1.731 sec
mem total: 430.75004 mb, up: 425.54612

constructor ..
time: 0.109 sec
mem total: 470.783952 mb, up: 40.025776

create dim ..
time: 17.996 sec
mem total: 768.262352 mb, up: 297.47712

create group ..
time: 2.828 sec
mem total: 973.6212 mb, up: 205.354632

filter ..
time: 0.753 sec
mem total: 1105.836848 mb, up: 132.214584

all count ..
time: 0.043 sec
mem total: 1105.87152 mb, up: 0.030448

all sum ..
time: 0 sec
mem total: 1105.875456 mb, up: 0.00152

top3 by group ..
time: 0.907 sec
mem total: 1119.252768 mb, up: 13.374856

update filter ..
time: 1.449 sec
mem total: 1285.779952 mb, up: 166.524704

top3 by group ..
time: 0 sec
mem total: 1286.045752 mb, up: 0.00116

構造上distributed/treeにすることも可能。処理が group x filter x reduce であればやはりこれはありな気がする。