access.log Parsing Node.js

access.log.parse.js

#!/usr/bin/env node

var fs =        require('fs'),
    geoip =     require('geoip-lite'),
    filename =  process.argv[2] || 'access.log',
    byHour =    {}; // storage hash

console.log('! loading ', filename);

fs.readFile(filename, function(error, data) {
  if(error) throw(error);

  data.toString().split(/\n/).forEach(function(line) {

    var parts =     line.split('"'),

        ipDate =    parts[0].split(' '),
        ipELB =     ipDate[0],

        datetime =  ipDate[3] ?
                      ipDate[3].replace(/^\[/, '').split(/[\/\:]/) :
                      null,
        day =       datetime ? datetime[0] : null,
        month =     datetime ? datetime[1] : null,
        year =      datetime ? datetime[2] : null,
        hour =      datetime ? datetime[3] : null,
        minute =    datetime ? datetime[4] : null,
        second =    datetime ? datetime[5] : null,
        key =       (year && month && day && hour) ?
                      year.concat(month.concat(day.concat(hour))) :
                      null,

        reqUrl =    parts[1],
        code =      parts[2] ? parts[2].trim().split(' ')[0] : null,
        refer =     parts[3],
        agent =     parts[5],
        ipOrig =    parts[7],

        ip =        ipOrig || ipELB,
        geo =       geoip.lookup(ip),
        record =    {
                      agent:  agent,
                      code:   code,
                      geo:    geo,
                      ip:     ip,
                      url:    reqUrl,
                      when:   {
                        day:    day,
                        month:  month,
                        year:   year,
                        hour:   hour,
                        minute: minute,
                        second: second
                      }
                    };

    if((! ip) || (! key)) return;
    if(agent.match(/HealthChecker/)) return;

    if(! byHour[key]) byHour[key] = {};
    if(! byHour[key][ip]) byHour[key][ip] = [];
    byHour[key][ip].push(record);
  });

  Object.keys(byHour).forEach(function(hour) {
    var total = 0;
    console.log(hour);

    Object.keys(byHour[hour]).forEach(function(origin) {
      var rows =  byHour[hour][origin],
          geo =   null;

      total += rows.length;

      console.log("\t" + rows.length + "\t" + origin);

      if(geo = geoip.lookup(origin)) {
        console.log("\t\tgeo =", geo.city, geo.region, geo.country);
      }

      rows.forEach(function(row) {
        console.log("\t\t\t", row.url);
      });
    });

    console.log("\t\thourTotal =", total);
  });

});