-
-
Save iqbmo04/72f551a085fbbf02801a566e28b106ac to your computer and use it in GitHub Desktop.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
const Apify = require('apify'); | |
Apify.main(async () => { | |
// Get queue and enqueue first url. | |
const requestQueue = await Apify.openRequestQueue(); | |
const enqueueUrl = async url => requestQueue.addRequest(new Apify.Request({ url })); | |
await enqueueUrl('https://news.ycombinator.com/'); | |
const crawlerConfig = { | |
launchPuppeteerOptions: { | |
liveView: true | |
}, | |
requestQueue, | |
disableProxy: true, | |
// This page is executed for each request. | |
// If request failes then it's retried 3 times. | |
// Parameter page is Puppeteers page object with loaded page. | |
handlePageFunction: async ({ page, request }) => { | |
console.log(`Request ${request.url} succeeded!`); | |
// Extract all posts. | |
const data = await page.$$eval('.athing', (els) => { | |
return els.map(el => el.innerText); | |
}); | |
// Save data. | |
await Apify.pushData({ | |
url: request.url, | |
data, | |
}); | |
// Enqueue next page. | |
try { | |
const nextHref = await page.$eval('.morelink', el => el.href); | |
await enqueueUrl(nextHref); | |
} catch (err) { | |
console.log(`Url ${request.url} is the last page!`); | |
} | |
}, | |
// If request failed 4 times then this function is executed. | |
handleFailedRequestFunction: async ({ request }) => { | |
console.log(`Request ${request.url} failed 4 times`); | |
await Apify.pushData({ | |
url: request.url, | |
errors: request.errorMessages, | |
}) | |
}, | |
} | |
// Create crawler. | |
const crawler = new Apify.PuppeteerCrawler(crawlerConfig); | |
secondAct(); | |
// Run crawler. | |
await crawler.run(); | |
}); | |
async function secondAct() { | |
// Start browser. | |
const browser = await Apify.launchPuppeteer({liveView: true}); | |
// Load http://goldengatebridge75.org/news/webcam.html and get an iframe | |
// containing webcam stream. | |
const page = await browser.newPage(); | |
await page.goto('http://goldengatebridge75.org/news/webcam.html'); | |
// Get a screenshot of that image. | |
const imageBuffer = await page.screenshot(); | |
// Save it as an output. | |
await Apify.setValue('OUTPUT', imageBuffer, { contentType: 'image/jpeg' }); | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment