Recursive site scraper using Puppeteer

puppeteerScrape.js

import puppeteer from "puppeteer-core"

const unique = (arr) => [...new Set(arr)]

(async () => {
  const browser = await puppeteer.launch({
    headless: true,
    channel: "chrome",
    timeout: 60000,
  })

  const page = await browser.newPage()

  async function scrape(links, callback, visited = new Set()) {
    const [url, ...rest] = links

    if (!url) return [...visited.entries()]

    if (visited.has(url)) return scrape(rest, callback, visited)

    await page.goto(url, { waitUntil: "networkidle0" })

    const hrefs = await page.$$eval("a[href]", (anchors) =>
      anchors.map((el) => el.href)
    )

    const html = await page.content()

    callback(url, html)

    visited.add(url)

    return scrape(unique(rest.concat(hrefs)), callback, visited)
  }

  await scrape(["http://localhost:3000/"], (href, html) => {
    // ...
  })
})()