Puppeteer Crawler Example

index.js

var Walker = require('puppeteer-walker')
var fs = require('fs')

var walker = Walker()

fs.writeFile('output.csv', 'URL, Title, H2\r\n', 'utf8', function (err) {
  console.log(`Header written`)
})

walker.on('end', () => console.log('finished walking'))
walker.on('error', (err) => console.log('error', err))
walker.on('page', async (page) => {
  var url = await page.url()
  var title = await page.title()
  try {
    var h2 = await page.evaluate(() => document.querySelector('h2').innerText);
      fs.appendFile('output.csv', `${url}, ${title}, ${h2}\r\n`, 'utf8', function (err) {
        console.log(
          `url: ${url}, title: ${title}, h2: ${h2}`)
      })
  } catch(e) {
    fs.appendFile('output.csv', `${url}, ${title}, ERROR\r\n`, 'utf8', function (err) {
    console.log(
      `url: ${url}, title: ${title}, h2: ERROR`)
    })
  }
})

walker.walk('http://jsseo.expert/')