Created
December 30, 2014 03:56
-
-
Save toddgeist/9766463ea17810eb2401 to your computer and use it in GitHub Desktop.
creating a massive CSV with node streams
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
/** | |
* Created by toddgeist on 12/28/14. | |
*/ | |
var faker = require('faker'); | |
var stringify = require('csv-stringify'); | |
var uuid =require('uuid'); | |
var moment = require('moment'); | |
var fs = require('fs'); | |
var Readable = require('stream').Readable; | |
var start = moment(); | |
var birthDateStart = moment().subtract(100, 'years'); | |
var birthDateEnd = moment().subtract(18, 'years'); | |
var rows = 10000000; | |
var fileName = 'data.csv'; | |
var columns = [ | |
'uuid', | |
'firstName', | |
'lastName', | |
'email', | |
'mobilePhone', | |
'workPhone', | |
'address', | |
'city', | |
'state', | |
'postalCode', | |
'birthDate', | |
'company', | |
// 'slogan', | |
// 'notes', | |
'userName' | |
].join(',') + '\n'; | |
var person = function(){ | |
return [ | |
uuid.v4(), | |
faker.name.firstName(), | |
faker.name.lastName(), | |
faker.internet.email(), | |
faker.phone.phoneNumber(), | |
faker.phone.phoneNumber(), | |
faker.address.streetAddress(), | |
faker.address.city(), | |
faker.address.state(), | |
faker.address.zipCode(), | |
moment( faker.date.between(birthDateStart, birthDateEnd)).format('M/d/YYYY'), | |
faker.company.companyName(), | |
// faker.company.catchPhrase(), | |
// faker.lorem.paragraphs(), | |
faker.internet.userName() | |
].join(',') + '\n'; | |
} | |
rs = new Readable; | |
var writable = fs.createWriteStream(__dirname + '/data.csv'); | |
var i = 0; | |
rs._read = function(){ | |
i++; | |
if( i===1) { | |
rs.push(columns) | |
}else if(i > rows){ | |
rs.push(null) | |
}else{ | |
rs.push(person()) | |
} | |
}; | |
rs.pipe(writable); | |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment