unique circulations/check-outs: oct 28 2015 - mar 18 2016 (w/ some gaps)

main.js

// script to gather unique circulations from WMS All_Checked_out reports, stored in './data' directory
var fs = require('fs')
var path = require('path')
var Level = require('level')
var split = require('split2')
var rimraf = require('rimraf')

var db = Level('./db')
var LevelUnique = require('level-unique-filter')

var uniq = LevelUnique(db)
var reports = fs.readdirSync('./data')

var out = {}, date, count, atHeader
;(function main (rep) {
  if (!rep) {
    return fs.writeFile('./summary.json', JSON.stringify(out), function (err) {
      if (err) throw err
      return rimraf('./db', function () {
        console.log('all set, check out summary.json')
      })
    })
  }

  date = (rep.match(/\d{8}/) || [])[0]
  count = 0
  atHeader = false

  console.log(`reading ${rep}`)
  fs.createReadStream(path.join(__dirname, 'data', rep))
    .pipe(split())
    .pipe(uniq.stream())
    .on('data', () => count++)
    .on('end', () => {
      console.log(`finished ${rep} with ${count} new items, moving on`)
      out[date] = count
      return main(reports.shift())
    })
})(reports.shift())

unique-circulations.json

{
  "20151028":105,
  "20151029":88,
  "20151030":85,
  "20151031":43,
  "20151101":18,
  "20151102":56,
  "20151103":87,
  "20151104":105,
  "20151105":95,
  "20151106":97,
  "20151107":58,
  "20151108":24,
  "20151109":54,
  "20151110":103,
  "20151111":70,
  "20151112":75,
  "20151113":76,
  "20151114":68,
  "20151115":33,
  "20151116":36,
  "20151117":81,
  "20151118":101,
  "20151119":108,
  "20151120":66,
  "20151121":48,
  "20151122":52,
  "20151123":91,
  "20151124":133,
  "20151125":77,
  "20151126":58,
  "20151127":14,
  "20151128":0,
  "20151129":0,
  "20151130":8,
  "20151201":102,
  "20151202":102,
  "20151203":107,
  "20151204":67,
  "20151205":62,
  "20151206":33,
  "20151207":35,
  "20151208":87,
  "20151209":126,
  "20151210":65,
  "20151211":42,
  "20151212":86,
  "20151213":41,
  "20151214":55,
  "20151215":101,
  "20151216":131,
  "20151217":71,
  "20151218":81,
  "20151219":96,
  "20151220":17,
  "20151221":5,
  "20151222":178,
  "20151223":75,
  "20151224":60,
  "20151225":0,
  "20151226":1,
  "20151227":0,
  "20151228":57,
  "20151229":1,
  "20151230":249,
  "20151231":38,
  "20160101":11,
  "20160102":0,
  "20160103":0,
  "20160104":0,
  "20160105":112,
  "20160106":83,
  "20160107":39,
  "20160108":75,
  "20160109":55,
  "20160110":6,
  "20160111":1,
  "20160112":53,
  "20160113":67,
  "20160114":1,
  "20160115":59,
  "20160116":52,
  "20160117":0,
  "20160118":1,
  "20160119":63,
  "20160120":101,
  "20160121":71,
  "20160122":60,
  "20160123":64,
  "20160124":18,
  "20160125":14,
  "20160126":26,
  "20160127":78,
  "20160128":79,
  "20160129":82,
  "20160130":88,
  "20160201":17,
  "20160202":59,
  "20160203":68,
  "20160204":84,
  "20160205":52,
  "20160206":58,
  "20160207":14,
  "20160208":13,
  "20160209":86,
  "20160210":59,
  "20160211":87,
  "20160212":78,
  "20160213":38,
  "20160214":31,
  "20160215":26,
  "20160216":64,
  "20160217":75,
  "20160218":68,
  "20160219":57,
  "20160220":82,
  "20160221":15,
  "20160222":51,
  "20160223":84,
  "20160224":34,
  "20160225":88,
  "20160226":54,
  "20160227":48,
  "20160228":18,
  "20160229":14,
  "20160301":90,
  "20160302":73,
  "20160305":262,
  "20160306":1,
  "20160308":47,
  "20160309":46,
  "20160310":30,
  "20160311":31,
  "20160312":28,
  "20160313":0,
  "20160314":17,
  "20160315":128,
  "20160316":54,
  "20160317":69,
  "20160318":55
}