Created
February 12, 2016 17:40
-
-
Save nicholasblexrud/f9e033afb348afac2ac6 to your computer and use it in GitHub Desktop.
Bulk upload files using Node.js to Elasticsearch
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
// credit goes to this stack overflow post - http://stackoverflow.com/questions/20646836/is-there-any-way-to-import-a-json-filecontains-100-documents-in-elasticsearch | |
var elasticsearch = require('elasticsearch'), | |
fs = require('fs'), | |
pubs = JSON.parse(fs.readFileSync(__dirname + '/pubs.json')), // name of my first file to parse | |
forms = JSON.parse(fs.readFileSync(__dirname + '/forms.json')); // and the second set | |
var client = new elasticsearch.Client({ // default is fine for me, change as you see fit | |
host: 'localhost:9200', | |
log: 'trace' | |
}); | |
for (var i = 0; i < pubs.length; i++ ) { | |
client.create({ | |
index: "epubs", // name your index | |
type: "pub", // describe the data thats getting created | |
id: i, // increment ID every iteration - I already sorted mine but not a requirement | |
body: pubs[i] // *** THIS ASSUMES YOUR DATA FILE IS FORMATTED LIKE SO: [{prop: val, prop2: val2}, {prop:...}, {prop:...}] - I converted mine from a CSV so pubs[i] is the current object {prop:..., prop2:...} | |
}, function(error, response) { | |
if (error) { | |
console.error(error); | |
return; | |
} | |
else { | |
console.log(response); // I don't recommend this but I like having my console flooded with stuff. It looks cool. Like I'm compiling a kernel really fast. | |
} | |
}); | |
} | |
for (var a = 0; a < forms.length; a++ ) { // Same stuff here, just slight changes in type and variables | |
client.create({ | |
index: "epubs", | |
type: "form", | |
id: a, | |
body: forms[a] | |
}, function(error, response) { | |
if (error) { | |
console.error(error); | |
return; | |
} | |
else { | |
console.log(response); | |
} | |
}); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
been banging my head against elastic client for a week now and this just sorted it for me, brilliant!!