Skip to content

Instantly share code, notes, and snippets.

@baptistemanson
Created November 6, 2015 18:44
Show Gist options
  • Save baptistemanson/82cf389b2f4a2b7b338c to your computer and use it in GitHub Desktop.
Save baptistemanson/82cf389b2f4a2b7b338c to your computer and use it in GitHub Desktop.
Parallel wget with node in a few lines
var fs = require('fs');
var queue = require('queue');
var wget = require('wget-improved');
var downloadUrl = function(url) {
return function(cb) {
try {
// you can also change the mapping url to file here by changing the function
var download = wget.download(url, () => ('./downloads/' + url.substring(url.lastIndexOf('/') + 1) ) , {});
download.on('end', function() {
() => console.log("done downloading")
});
} catch (e) {
}
};
};
var q = queue();
//change this one as needed.
q.concurrency = 5;
//here the urls can come from anything really.
var urls = ['http://example.com/image-1.jpg','http://example.com/image-2.jpg']
for (var i in urls) {
q.push(downloadUrl(urls[i]));
console.log("all urls pushed");
}
q.start();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment