node mem tests

gistfile1.js

/*

This generates 4Mio timeseries values and stores them in different data structures in memory, dependent on process.argv[2].

Test results on my laptop are included. Performance was not important here, only mem usage. The background: In mem node DB.

Results:

- 115mb : { 1 : 'd,12,1d,lo', ... }
- 150mb : { 1 : [12,34,56,78], ... }
- 160mb : { 1 : '12,34,56,78', ... }
- 240mb : { 1 : { 0 : 12, 1 : 34, 2 : 56, 3 : 78 } }
- 250mb : [ 12, 34, 56, 78, ... ]
- 350mb : [ [12, 34, 56, 78], ... ]
- 470mb : '12,34,56,78,...'

The takeaway (if any):

- if mem usage counts, fat arrays suck (because they preallocate space)
- small strings are small, but big strings are really big
- node is not a good solution for those kinds of problems!!

*/

var len = 4000000;

var db = {};
var data, lastId;

// 115 mb - Hash - String (Base32)
if (process.argv[2] == 'hashStrBase') {
  for (var i = 0; i < len; i++) {
    data = genData();
    db[data.id] = [
      data.tick.toString(32),
      data.ts.toString(32),
      data.last.toString(32),
      data.lastUnit.toString(32),
      data.lastCum.toString(32)
    ].join(',');
    lastId = data.id;
  }
}

// 150 mb - Hash - Array - Int
if (process.argv[2] == 'hashArr') {
  for (var i = 0; i < len; i++) {
    data = genData();
    db[data.id] = [data.tick, data.ts, data.last, data.lastUnit, data.lastCum];
    lastId = data.id;
  }
}

// 160mb - Hash - String
if (process.argv[2] == 'hashStr') {
  for (var i = 0; i < len; i++) {
    data = genData();
    db[data.id] = [
      data.tick, data.ts, data.last, data.lastUnit, data.lastCum
    ].join(',');
    lastId = data.id;
  }
}

// 240 mb - Hash - Map - Int
if (process.argv[2] == 'hashHash') {
  for (var i = 0; i < len; i++) {
    data = genData();
    db[data.id] = {
      0 : data.tick,
      1 : data.ts,
      2 : data.last,
      3 : data.lastUnit,
      4 : data.lastCum
    };
    lastId = data.id;
  }
}

// 250 mb - Array - Int
if (process.argv[2] == '1darray') {
  db = [];
  var data, lastid;
  for (var i = 0; i < len; i++) {
    data = genData();
    object.keys(data).foreach(function (key) {
      db.push(data[key])
    });
    lastid = db.length - 1;
  }
}

// 350 mb - Array - Array  Int
if (process.argv[2] == '2dArray') {
  db = [];
  var data, lastId;
  for (var i = 0; i < len; i++) {
    data = genData();
    db.push([
      data.id, data.tick, data.ts, data.last, data.lastUnit, data.lastCum
    ]);
    lastId = db.length - 1;
  }
}

// 470 mb - String
if (process.argv[2] == 'str') {
  db = '';
  var data, lastid;
  for (var i = 0; i < len; i++) {
    data = genData();
    db += [
      data.id, data.tick, data.ts, data.last, data.lastUnit, data.lastCum
    ].join(',') + '\n';
    lastid = db.length - 1;
  }
}

console.log('generated');
console.log(lastId, db[lastId]);

require('http').createServer().listen(2345);

function genData() {
  return {
    id : ofLen(6),
    tick : ofLen(6),
    ts : Date.now(),
    last : ofLen(6)/100,
    lastUnit : ofLen(7),
    lastCum : ofLen(9)
  }

  function ofLen(len) {
    var rand = Math.random();
    return parseInt(Math.floor(rand*Math.pow(10, len)),10);
  }
}