Skip to content

Instantly share code, notes, and snippets.

@myndzi
Last active November 3, 2016 06:06
Show Gist options
  • Save myndzi/5133064c4a37da7f607ab1d154a2b35a to your computer and use it in GitHub Desktop.
Save myndzi/5133064c4a37da7f607ab1d154a2b35a to your computer and use it in GitHub Desktop.
var split = require('binary-split-streams2'),
fs = require('fs');
function requestStream(url) {
return Kefir.stream(emitter => {
let req = bhttp.get(url, { stream: true })
.pipe(split()); // default is \n -- no regex support, use a different module if you need variable line terminators
req.on('data', chunk => emitter.emit(chunk));
req.on('error', err => {
emitter.error(err); emitter.end();
});
req.on('end', () => emitter.end());
})
}
var outFile = fs.createWriteStream('./outfile.log');
Kefir.sequentially(0, listOfUrls)
.flatMapConcurLimit(requestStream, 3) // 3 open at once max
.onValue(chunk => outFile.write(chunk))
.onError(err => console.error(err))
.onEnd(() => {
outFile.end();
console.log('done')
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment