Skip to content

Instantly share code, notes, and snippets.

@bsansouci
Last active February 3, 2019 12:15
Show Gist options
  • Save bsansouci/549879d9b2b69e087b4c to your computer and use it in GitHub Desktop.
Save bsansouci/549879d9b2b69e087b4c to your computer and use it in GitHub Desktop.
var request = require("request");
var async = require("async");
var fs = require("fs");
function searchFor(searchQuery, pageNum, callback) {
for(var i = 0; i < pageNum; i++) {
(function(i) {
request.get("https://www.google.fr/search?q=" + searchQuery + "&start=" + i * 10, function(err, res) {
if(err) throw err;
fs.writeFile(searchQuery + "-page" + i + ".html", res.body, done);
});
})(i);
}
var count = 0;
function done() {
count++;
if(count === pageNum) {
callback();
}
}
}
var pageNum = 3;
var arr = fs.readFileSync("filename.txt", 'utf8').split("\n").map(function(v) {return v.split(" ").join("+");});
async.mapLimit(arr, 5, function(v, cb) {
searchFor(v, pageNum, cb);
}, function(err) {
if(err) throw err;
console.log("Done");
});
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment