Skip to content

Instantly share code, notes, and snippets.

@emg110
Last active August 6, 2019 14:03
Show Gist options
  • Save emg110/2e034b3bf8ad19938339e442848bc54c to your computer and use it in GitHub Desktop.
Save emg110/2e034b3bf8ad19938339e442848bc54c to your computer and use it in GitHub Desktop.
const fs = require('fs');
const logger = require('./logger');//JUST PICKUP A LOGGER LIKE THIS LOGGER HERE: https://gist.github.com/emg110/1ab0584ff59b6f216304ec68b0977f7c
const { Client } = require('@elastic/elasticsearch');
const client = new Client({ node: 'http://YOUR_ES_SERVER:ES_HTTP/S_PORT' });
const inStream = fs.createReadStream('././data/YOUR_NEDB_FILE.db');
var lineReader = require('readline').createInterface({
input: inStream
});
async function writeToES(doc){
await client.index({
index: 'YOUR_ES_INDEX_NAME',
type: '_doc',
body: doc
}).then(()=>{
counter++;
logger.info(counter+" Records has been stored in ES")
}).catch(err=>{
logger.error(err)
})
}
var counter=0;
lineReader.on('line', function (line) {
//line = line.replace(/\"/g, '"');// JUST IN CASE! YOU KNOW.
//line = line.replace(/\//g, '');//JUST IN CASE! YOU KNOW.
let rec = JSON.parse(line)
//DO WHAT EVER TRANSFORM & VALIDATION YOU WANT WITH RECORD'S JSON BEFORE WRITING IT TO ES
writeToES(rec);
});
inStream.on('end', function() {
logger.info('ETL process has ended successfully....'+' & around '+counter+' Records have been transfered in total')
client.indices.refresh({ index: 'YOUR_ES_INDEX' })
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment