Created
April 9, 2016 00:16
-
-
Save atorralb/1d1d5e9333427fa6de0675a2a5c8f371 to your computer and use it in GitHub Desktop.
iterate through a pagination site with nightmarejs
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
var Nightmare = require('nightmare'); | |
var vo = require('vo'); | |
vo(run)(function(err, result) { | |
if (err) throw err; | |
}); | |
function* run() { | |
var nightmare = Nightmare(), | |
MAX_PAGE = 10, | |
currentPage = 0, | |
nextExists = true, | |
links = []; | |
yield nightmare | |
.goto('https://www.yahoo.com') | |
.type('.input-query', 'github nightmare') | |
.click('#search-submit') | |
.wait('body') | |
nextExists = yield nightmare.visible('.next'); | |
while (nextExists && currentPage < MAX_PAGE) { | |
links.push(yield nightmare | |
.evaluate(function() { | |
var links = document.querySelectorAll("ol.searchCenterMiddle a"); | |
return links[0].href; | |
})); | |
yield nightmare | |
.click('.next') | |
.wait('body') | |
currentPage++; | |
nextExists = yield nightmare.visible('.next'); | |
} | |
console.dir(links); | |
yield nightmare.end(); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
The examples really outdated and taken from a Github error a year ago. I made some adjustments and it works now:
var Nightmare = require('nightmare');
var vo = require('vo');
vo(run)(function(err, result) {
if (err) throw err;
});
function* run() {
var nightmare = Nightmare(),
MAX_PAGE = 10,
currentPage = 0,
nextExists = true,
links = [];
}