A proof of concept script for streaming a binary file of variable length integers through a nodejs context.

read_variable_length_integer_file.js

var fs, vi, sys, rs, bufferSize, remainder, startTime;

startTime = Date.now();

fs = require('fs');
vi = require('varint');
sys = require('sys');

rs = fs.createReadStream('/path/to.file');
total = 0;

rs.on('data', function(buffer) {
  var buffer, value, byteLen, offset;

  if (remainder) {
    buffer = Buffer.concat([remainder, buffer]);    
    remainder = null;
  } 

  offset = 0;

  while(offset < buffer.length){
    // decode the value and check how many bytes it required
    value = vi.decode(buffer, offset);
    offset += vi.decode.bytesRead;

    if (offset <= buffer.length){
      // keep the result and update position in the buffer
      total += value;
    } else {
      // result is incomplete, store remainder to be prepended to the next chunk
      remainder = new Buffer(buffer.length - offset);
      buffer.copy(remainder, 0, offset);
    }
  }
});

rs.on('end', function() {
  sys.puts((total));
  sys.puts(Date.now() - startTime);
});