Bulk insert of data into Google Cloud Datastore from JSON in CSV

importframes-ds.js

const fs = require('fs');
const readline = require('readline');
const Datastore = require('@google-cloud/datastore');
const parse = require('csv-parse');

const instream = fs.createReadStream(process.argv[2]);

const datastore = Datastore({
  projectId: 'cloudprojectid',
  keyFilename: 'serviceaccount.json'
});

const kind = 'frame';

const rl = readline.createInterface({
  input: instream,
  terminal: false
});

// Constructor
function Batcher() {
  this.flushAtCt = 500; // no more than 500 entities can be upserted in a single API call to Datastore
  this.items = []; // default value
}

Batcher.prototype.push = function(item) {
  if (this.items.length >= this.flushAtCt) {
    this.flush();
  } else {
    let skip = false;
    for (let i=0; i < this.items.length; i++) {
      //console.log(item.key,this.items[i].key);
        if (item.key.name === this.items[i].key.name) { // do not modify one key more than once
          skip = true;
    }
  }
   if (skip === false) {
    this.items.push(item);
  }
  }
};

Batcher.prototype.flush = function() {
  console.log('bulk insert of ' + this.items.length);
  datastore.upsert(this.items, (err, success) => {
          if (err) {
            console.log(err);
          } else {
            console.log("success!");
        }
  });
  this.items = [];
};

let batcher = new Batcher();

rl.on('line', (line) => {
  
  if (line === null) {return;}
  
  let success = true;
  
  let components = line.split(',');
  let path = components[0];
  let serial = components[1];
  let makeModel = components[2];
  
  components.shift(); components.shift(); components.shift();
  
  let theData;
  
  try {
    theData = JSON.parse(components.join());
  } catch (e) {
      success = false;
    if (e instanceof SyntaxError) {
      console.log("Bad JSON");
  }
  }

  if (success === true) {
    theData.insertDate = Date.now();
    theData.path = path;
    theData.serial = serial;
    theData.makeModel = makeModel;
  
    let entity = {
      key: datastore.key([kind, theData.serial+theData.IQimage.time]),
      data: {
        sequence: theData.IQimage.sequence,
        time: theData.IQimage.time,
        event: theData.IQimage.event,
        motionWindows: theData.IQimage.motionWindows,
        imgjdbg: theData.IQimage.imgjdbg,
        insertDate: theData.insertDate,
        path: theData.path,
        serial: theData.serial,
        makeModel: theData.makeModel
      },
    };
    batcher.push(entity);
  }
});

rl.on('close', () => {
  batcher.flush();
});

Is this up to date?