nodejs_fileparser.md

NodeJS File parsing

Here's a skelleton for ripping files apart in NodeJS and processing each line.

var fs = require('fs');
var zlib = require('zlib');
var stream = require('stream');
var es = require('event-stream');

var return_object = {
  logs: []
};

// Create a readStream object...
fs.createReadStream(filename)
  // Do something when done processing the file.
  .on('end', function(){
    console.log(return_object);
  })
  // If it's a gzip, ungzip it on the fly.
  .pipe(zlib.createGunzip())
  // Split on \n
  .pipe(es.split())
  // Now, do something with each line.
  .pipe(es.map(function(data, callback){
      // in my case I have a line parse function elsewhere in the script.
      my.parse(data, function(error, object){
        if ( error ) {
          console.log(error);
        } else {
          // And I have a filter function as well to rate logs.
          my.filter(object, 70, function(error, log){
            if ( error ) {
              console.log(error);
              console.log(data);
            } else {
              return_object.logs.push(object);
            }
          });
        }
      });
  }));