MongoDB update data for schema changes

Readme.md

/** *

• Run this script with the command: node schema-migration.js ./example-schema-update-rules.json
• Replace example-schema-update-rules.json with your own version of the file.
• There are three JSON operations for modifying data...
• @addFields will add new fields to the data with a value you can optionally specify
• @updateFields will map a field over to to a new key. Persisting the data value.
• @deleteFields will delete the fields you specify
• @TODO: make this script class based

*/

example-schema-update-rules.json

{
  "collections": [
    {
      "name": "vehicle_test",
      "modifiers": {
        "addFieldsNotes": "@addFields will add new fields to the data with a value you can optionally specify. Otherwise use null. Ex) {new.path: someValue}.",
        "addFields": {
          "created.newnewTestAdd": null
        },
        "updateFieldsNotes": "@updateFields will map a field over from it's old value to a new path. Persisting the data value. Ex) {New_Field: Old_Field}.",
        "updateFields": {
          "created.newnewName": "name"
        },
        "deleteFieldsNotes": "@deleteFields will delete the fields you specify. Ex) 'delete.from.path'.",
        "deleteFields": [
          "model_id"
        ]
      }
    },
    {
      "name": "vehicle_make",
      "modifiers": {
        "addFieldsNotes": "@addFields will add new fields to the data with a value you can optionally specify. Otherwise use null. Ex) {new.path: someValue}.",
        "addFields": {
          "created": null
        },
        "updateFieldsNotes": "@updateFields will map a field over from it's old value to a new path. Persisting the data value. Ex) {New_Field: Old_Field}.",
        "updateFields": {
          "newnewName": "state"
        },
        "deleteFieldsNotes": "@deleteFields will delete the fields you specify. Ex) 'delete.from.path'.",
        "deleteFields": [
          "type"
        ]
      }
    }
  ]
}

schema-migration.js

/**
 *
 * Run this script with the command: `node schema-migration.js ./example-schema-update-rules.json`
 * Replace `example-schema-update-rules.json` with your own version of the file.
 *
 * There are three JSON operations for modifying data...
 * @addFields will add new fields to the data with a value you can optionally specify
 * @updateFields will map a field over to to a new key. Persisting the data value.
 * @deleteFields will delete the fields you specify
 *
 * @TODO: make this script class based
 *
 */


'use strict';
// Parse command line arguments to determine input file
const cmdArgs = process.argv.slice(2);
const schemaUpdateRulesFile = cmdArgs[0] ? cmdArgs[0] : 'example-schema-update-rules';
const schemaUpdateRules = require(schemaUpdateRulesFile);

const mongoUrl = require('../server/datasources').db.url;
let MongoClient = require( 'mongodb' ).MongoClient;
let ObjectId = require('mongodb').ObjectID;
let _db;
let dbo;
let collectionIterator = 0;

/**
 * Open MongoDB connection
 * Save connect in global variables:
 * @_db
 * @dbo
 */
let mongoConnection = function() {
  MongoClient.connect(mongoUrl, function (err, db) {
    if (err) throw err;
    _db = db;
    dbo = db.db('vtax');
    initialize();
  });
}

/**
 * Initialize
 */
let initialize = function () {
  const numberCollections = schemaUpdateRules.collections.length;
  for (let i = 0; i < numberCollections; i++) {
    const collectionName = schemaUpdateRules.collections[i].name;
    const modifiers = schemaUpdateRules.collections[i].modifiers;
    getData(collectionName, modifiers, numberCollections);
  }
}

/**
 * Pull down data from MongoDB
 * @param collectionName
 * @param modifiers
 * @TODO: In future iterations, the find should iterate through documents for very large collections where local memory is a concern.
 */
let getData =  function(collectionName, modifiers, numberCollections) {
  dbo.collection(collectionName).find({}).toArray(function(err, data) {
    if (err) throw err;
    console.log('Updating ' + collectionName + '...');
    generateQuery(data, collectionName, modifiers, numberCollections);
  });
}

/**
 * Iterate through documents in a collection. Using our add, update,
 * and delete functions to create $set and $unset rules for mongo.
 * @param data
 * @param collectionName
 * @param modifiers
 * @param numberCollections
 */
let generateQuery = function (data, collectionName, modifiers, numberCollections) {
  let record;
  let updateQuery = {$set: {}, $unset: {}};
  let updatePromiseArray = [];
  // Loop through all records from collection
  for (let i = 0; i < data.length; i++) {
    record = JSON.parse(JSON.stringify(data[i])); // deep clone
    updateQuery = addFields(updateQuery, modifiers);
    updateQuery = updateFields(updateQuery, record, modifiers);
    updateQuery = deleteFields(updateQuery, record, modifiers);
    updatePromiseArray.push(update(collectionName, updateQuery, record._id));
  }
  Promise.all(updatePromiseArray).then(function(values) {
    collectionIterator++;
    if (numberCollections === collectionIterator) cleanExit()
  });
}

/**
 * Adds properties to be updated to the $set part of the query.
 * @param record
 * @param modifiers
 */
let addFields = function(updateQuery, modifiers) {
  for (let key in modifiers.addFields) {
    updateQuery.$set[key] = modifiers.addFields[key]
  }
  return updateQuery
}

/**
 * Adds properties to be updated to the $set part of the query.
 * Also, adds the properties to be deleted to the $unset part of the query.
 * @param record
 * @param modifiers
 */
let updateFields = function(updateQuery, record, modifiers) {
  if (modifiers.updateFields.length < 1) return
  for (let key in modifiers.updateFields) {
    const path = modifiers.updateFields[key];
    let addValue = readProp(record, path);
    if (!addValue) addValue = null;
    updateQuery.$set[key] = addValue;
    updateQuery.$unset[path] = '';
  }
  return updateQuery
}

/**
 * Adds properties to be deleted to the $unset part of the query
 * @param record
 * @param modifiers
 * @returns {*}
 */
let deleteFields = function(updateQuery, record, modifiers) {
  for (let i = 0; i < modifiers.deleteFields.length; i++) {
    updateQuery.$unset[modifiers.deleteFields[i]] = '';
  }
  return updateQuery
}

/**
 * Reads the value from the record object using the property using a string with dot notation
 * @param record
 * @param path
 */
let readProp = function(record, path) {
  try {
    let o = record
    path = path.replace(/\[(\w+)\]/g, '.$1')
    path = path.replace(/^\./, '')
    let a = path.split('.')
    while (a.length) {
      let n = a.shift()
      if (n in o) {
        o = o[n]
      } else {
        return
      }
    }
    return o
  } catch (err) {
    console.log('Could not read empty path: ' + path);
  }
}

/**
 * Utility Function for detecting empty objects
 * @param obj
 * @returns {boolean}
 */
function isEmpty(obj) {
  for(var key in obj) {
    if(obj.hasOwnProperty(key))
      return false;
  }
  return true;
}

/**
 * Update a document based on matching _id, and $set and unset rules.
 * @param collectionName
 * @param field
 */
let update = function(collectionName, updateQuery, _id) {
  return new Promise(function(resolve, reject) {
    if (!_id) {
      resolve();
      return;
    }
    let findQuery = {};
    try {
      findQuery['_id'] = ObjectId(_id);
    } catch (err) {
      findQuery['_id'] = _id;
    }
    if (isEmpty(updateQuery.$set)) delete updateQuery.$set;
    if (isEmpty(updateQuery.$unset)) delete updateQuery.$unset;
    dbo.collection(collectionName).updateOne(findQuery, updateQuery, {}, function (err, result) {
      if (err) console.log(err);
      resolve();
    });
  });
}

/**
 * Disconnect from database and exit script
 */
let cleanExit = function() {
  _db.close();
  console.log('Done! Exiting...');
  process.exit();
}

mongoConnection();