Skip to content

Instantly share code, notes, and snippets.

@dclowd9901
Last active December 21, 2015 17:49
Show Gist options
  • Save dclowd9901/6342967 to your computer and use it in GitHub Desktop.
Save dclowd9901/6342967 to your computer and use it in GitHub Desktop.
Node.js script to dedupe a JSON-compliant array
/**
* Node script to dedupe an array in a file.
*
* Prereqs: lodash (globally)
*
* Use:
* > node dedupe.js path/to/file.js [path/to/newfile.js]
*/
var fs = require('fs'),
_ = require('/usr/local/lib/node_modules/lodash'),
file = process.cwd() + '/' + process.argv[2],
fileNew = process.argv[3] || file;
console.log('Reading \'' + file + '\'');
fs.readFile(file, 'utf8', function (err, data) {
var usableData = JSON.parse(data),
deduped = _.uniq(usableData, true);
writeFile(deduped);
});
function writeFile(deduped) {
console.log('Writing file ' + fileNew);
fs.writeFile(fileNew, JSON.stringify(deduped), function (err) {
if (err) throw err;
console.log('Saved ' + fileNew);
});
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment