Created
April 9, 2016 00:16
-
-
Save atorralb/1d1d5e9333427fa6de0675a2a5c8f371 to your computer and use it in GitHub Desktop.
iterate through a pagination site with nightmarejs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| var Nightmare = require('nightmare'); | |
| var vo = require('vo'); | |
| vo(run)(function(err, result) { | |
| if (err) throw err; | |
| }); | |
| function* run() { | |
| var nightmare = Nightmare(), | |
| MAX_PAGE = 10, | |
| currentPage = 0, | |
| nextExists = true, | |
| links = []; | |
| yield nightmare | |
| .goto('https://www.yahoo.com') | |
| .type('.input-query', 'github nightmare') | |
| .click('#search-submit') | |
| .wait('body') | |
| nextExists = yield nightmare.visible('.next'); | |
| while (nextExists && currentPage < MAX_PAGE) { | |
| links.push(yield nightmare | |
| .evaluate(function() { | |
| var links = document.querySelectorAll("ol.searchCenterMiddle a"); | |
| return links[0].href; | |
| })); | |
| yield nightmare | |
| .click('.next') | |
| .wait('body') | |
| currentPage++; | |
| nextExists = yield nightmare.visible('.next'); | |
| } | |
| console.dir(links); | |
| yield nightmare.end(); | |
| } |
The examples really outdated and taken from a Github error a year ago. I made some adjustments and it works now:
var Nightmare = require('nightmare');
var vo = require('vo');
vo(run)(function(err, result) {
if (err) throw err;
});
function* run() {
var nightmare = Nightmare(),
MAX_PAGE = 10,
currentPage = 0,
nextExists = true,
links = [];
yield nightmare
.goto('https://www.yahoo.com')
.type('#uh-search-box', 'github nightmare')
.click('#uh-search-button')
.wait('ol.searchCenterMiddle')
nextExists = yield nightmare.visible('.next');
while (nextExists && currentPage < MAX_PAGE) {
links.push(yield nightmare
.evaluate(function() {
var links = document.querySelectorAll("ol.searchCenterMiddle a");
console.log(links[0].href);
return links[0].href;
}));
yield nightmare
.click('.next')
.wait('body')
currentPage++;
nextExists = yield nightmare.visible('.next');
}
console.dir(links);
yield nightmare.end();
}
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
This code does not work. On my Ubuntu 16.04 box with the latest stable version of node, I got a series of errors, starting with:
Note that I am not using the co library. I have explicitly required vo. But for some reason I am seeing co listed in the error.
Just for grins, I npm uninstalled co and vo from my packages. Then reinstalled them. After that, the code would execute, except that I got another error, which seems to reference the first if (err) throw err; as follows:
After removing the err statement, I run the code and it simply hangs.