Created
March 12, 2020 17:08
-
-
Save modster/a8566e53c0f141c8d5fd3eb1cd7d80e9 to your computer and use it in GitHub Desktop.
scroll page to end of document with nodejs puppeteer
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const puppeteer = require('puppeteer'); | |
(async () => { | |
const browser = await puppeteer.launch(); | |
const [page] = await browser.pages(); | |
await page.goto('https://www.reddit.com/r/GameDeals/', { waitUntil: 'networkidle0' }); | |
const links = await page.evaluate(async () => { | |
window.scrollBy(0, document.body.clientHeight); | |
await new Promise(resolve => setTimeout(resolve, 5000)); // wait for some time, you might need to figure out a good value for this yourself | |
return [...document.querySelectorAll('.scrollerItem div:nth-of-type(2) article div div:nth-of-type(3) a')] | |
.map((el) => el.href); | |
}); | |
console.log(links, links.length); | |
await browser.close(); | |
})(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment