Skip to content

Instantly share code, notes, and snippets.

@summer4096
Last active August 29, 2015 14:12
Show Gist options
  • Save summer4096/33c8a753560dbb40ae3e to your computer and use it in GitHub Desktop.
Save summer4096/33c8a753560dbb40ae3e to your computer and use it in GitHub Desktop.
var fs = require('fs');
var through2 = require('through2');
var csv = require('fast-csv');
var util = require('util');
var writeStream = fs.createWriteStream('out.csv');
var progress = {
total: 0,
done: 0
};
//the following reads thing.csv (a 50k line csv), parses it, counts the lines, stringifies it, and counts the lines again
var csvStream = fs.createReadStream('thing.csv')
.pipe(csv({headers: true, delimiter: '|'}))
.pipe(through2.obj(function(chunk, enc, cb){
progress.total++;
cb(null, chunk);
}))
.pipe(csv.createWriteStream())
.pipe(through2(function(chunk, enc, cb){
progress.done++;
console.log(progress.done, progress.total);
cb(null, chunk);
}));
var stockReadablePipe = require('stream').Readable.prototype.pipe;
var stockPipe = require('stream').prototype.pipe;
//This works
stockPipe.call(csvStream, writeStream);
//This doesn't work
//stockReadablePipe.call(csvStream, writeStream);
//This doesn't work (same as above)
//csvStream.pipe(writeStream);
//This works
/*csvStream.on('data', function(chunk, enc){
if (!writeStream.write(chunk, enc)) {
csvStream.pause();
}
});
writeStream.on('drain', function(){
csvStream.resume();
});
csvStream.on('end', function(){
writeStream.end();
});*/
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment