brandonb927 · February 22, 2022 00:51
diff --git a/puppeteer-vscode-scraper.js b/puppeteer-vscode-scraper.js
 #!/usr/bin/env node

 // $ npm install -d fs-extra ora open puppeteer

 const { spawn } = require('child_process')
 const readline = require('readline')
 const path = require('path')

 const fse = require('fs-extra')
 const open = require('open')
 const puppeteer = require('puppeteer')
 const ora = require('ora')

 let extensionList = []
 let extensionsDetailedList = []

 const OPEN_IN_BROWSER = process.env.OPEN_IN_BROWSER || false

 const htmlFilePath = path.resolve(__dirname, 'dist/index.html')
 const extensionsListPath = path.resolve(__dirname, 'dist/extensions.json')

 const writeExtensionsFile = extensions => {
  fse.outputFile(extensionsListPath, JSON.stringify(extensions))
 }

 const writeHtmlFile = extensions => {
  const content = `<!DOCTYPE html>
 <html lang="en">
  <head>
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <link rel="stylesheet" href="https://unpkg.com/tachyons/css/tachyons.min.css"/>
  </head>
  <body class="athelas mw7 ph3 center">
    <div>
      <h1 class="f-headline lh-solid">
        ${extensions.length} extensions installed!
      </h1>
      ${extensions.sort((a, b) => {
        const textA = a.title.toUpperCase()
        const textB = b.title.toUpperCase()
        return (textA < textB) ? -1 : (textA > textB) ? 1 : 0
      }).map(ext => {
        return `<div class="pv3 bb b--black-10 flex justify-between">
          <div class="flex items-center mw4 mr4 w-100">
            <img src="${ext.img || 'https://seeklogo.com/images/A/atom-logo-19BD90FF87-seeklogo.com.png'}" class="db" />
          </div>
          <div class="w-100">
            <h2 class="mv0 f2 lh-copy">
              <a href="${ext.url}" class="link underline black hover-orange">${ext.title}</a>
            </h2>
            <h3 class="mv0 lh-copy">${ext.name}</h3>
            <p class="lh-copy">${ext.desc}</p>
          </div>
        </div>`
      }).join('')}
      <br />
      <br />
    </div>
    <div class="pv4">
      <p class="lh-copy tc">Generated at ${new Date().toISOString()}</a>
      <p class="lh-copy tc">Made with <3 by <a href="https://brandonb.ca">Brandon Brown</a>
    </div>
  </body>
 </html>`

  fse.outputFile(htmlFilePath, content).then(() => {
    if (OPEN_IN_BROWSER) open(htmlFilePath)
  })
 }

 ;(async () => {
  const fileExists = await fse.pathExists(extensionsListPath)

  if (fileExists) {
    const spinner = ora('Reading cached extensions').start()
    const extensionsJson = await fse.readJson(extensionsListPath)

    writeHtmlFile(extensionsJson)

    setTimeout(() => {
      spinner.succeed('All done! Please refresh your browser to a view rebuilt page.')
    }, 1000)
  } else {
    const installedExtensions = spawn('code', [
      '--list-extensions'
    ])

    const rl = readline.createInterface({
      input: installedExtensions.stdout,
      terminal: false
    })

    rl.on('line', (name) => {
      extensionList.push({
        name,
        url: `https://marketplace.visualstudio.com/items?itemName=${name}`
      })
    })
    .on('close', async () => {
      const spinner = ora('Scraping extension info from the marketplace').start()

      const browser = await puppeteer.launch() // For debugging {headless: false}

      const scrape = async extension => {
        const page = await browser.newPage()
        await page.goto(extension.url, {
          waitUntil: 'networkidle0',
          timeout: 0
        })

        const results = await page.evaluate(({ url, name }) => {
          const title = document.querySelector('.item-header .ux-item-name').innerText
          const desc = document.querySelector('.item-header .ux-item-shortdesc').innerText
          const img = document.querySelector('.item-img .image-display').getAttribute('src')
          console.log(img)

          return {
            title,
            desc,
            img,
            name,
            url
          }
        }, extension)

        return results
      }

      for (let i = 0; i < extensionList.length; i++) {
        let extension = extensionList[i]
        try {
          let data = await scrape(extension)
          extensionsDetailedList.push(data)
          spinner.text = `Received extension: ${data.name}`
        } catch (e) {
          throw e
        }
      }

      await browser.close()

      writeExtensionsFile(extensionsDetailedList)
      writeHtmlFile(extensionsDetailedList)

      setTimeout(() => {
        spinner.succeed('All done!')
      }, 1000)
    })
  }
 })()
	#!/usr/bin/env node

	// $ npm install -d fs-extra ora open puppeteer

	const { spawn } = require('child_process')
	const readline = require('readline')
	const path = require('path')

	const fse = require('fs-extra')
	const open = require('open')
	const puppeteer = require('puppeteer')
	const ora = require('ora')

	let extensionList = []
	let extensionsDetailedList = []

	const OPEN_IN_BROWSER = process.env.OPEN_IN_BROWSER \|\| false

	const htmlFilePath = path.resolve(__dirname, 'dist/index.html')
	const extensionsListPath = path.resolve(__dirname, 'dist/extensions.json')

	const writeExtensionsFile = extensions => {
	fse.outputFile(extensionsListPath, JSON.stringify(extensions))
	}

	const writeHtmlFile = extensions => {
	const content = `<!DOCTYPE html>
	<html lang="en">
	<head>
	<meta name="viewport" content="width=device-width, initial-scale=1">
	<link rel="stylesheet" href="https://unpkg.com/tachyons/css/tachyons.min.css"/>
	</head>
	<body class="athelas mw7 ph3 center">
	<div>
	<h1 class="f-headline lh-solid">
	${extensions.length} extensions installed!
	</h1>
	${extensions.sort((a, b) => {
	const textA = a.title.toUpperCase()
	const textB = b.title.toUpperCase()
	return (textA < textB) ? -1 : (textA > textB) ? 1 : 0
	}).map(ext => {
	return `<div class="pv3 bb b--black-10 flex justify-between">
	<div class="flex items-center mw4 mr4 w-100">
	<img src="${ext.img \|\| 'https://seeklogo.com/images/A/atom-logo-19BD90FF87-seeklogo.com.png'}" class="db" />
	</div>
	<div class="w-100">
	<h2 class="mv0 f2 lh-copy">
	<a href="${ext.url}" class="link underline black hover-orange">${ext.title}</a>
	</h2>
	<h3 class="mv0 lh-copy">${ext.name}</h3>
	<p class="lh-copy">${ext.desc}</p>
	</div>
	</div>`
	}).join('')}
	<br />
	<br />
	</div>
	<div class="pv4">
	<p class="lh-copy tc">Generated at ${new Date().toISOString()}</a>
	<p class="lh-copy tc">Made with <3 by <a href="https://brandonb.ca">Brandon Brown</a>
	</div>
	</body>
	</html>`

	fse.outputFile(htmlFilePath, content).then(() => {
	if (OPEN_IN_BROWSER) open(htmlFilePath)
	})
	}

	;(async () => {
	const fileExists = await fse.pathExists(extensionsListPath)

	if (fileExists) {
	const spinner = ora('Reading cached extensions').start()
	const extensionsJson = await fse.readJson(extensionsListPath)

	writeHtmlFile(extensionsJson)

	setTimeout(() => {
	spinner.succeed('All done! Please refresh your browser to a view rebuilt page.')
	}, 1000)
	} else {
	const installedExtensions = spawn('code', [
	'--list-extensions'
	])

	const rl = readline.createInterface({
	input: installedExtensions.stdout,
	terminal: false
	})

	rl.on('line', (name) => {
	extensionList.push({
	name,
	url: `https://marketplace.visualstudio.com/items?itemName=${name}`
	})
	})
	.on('close', async () => {
	const spinner = ora('Scraping extension info from the marketplace').start()

	const browser = await puppeteer.launch() // For debugging {headless: false}

	const scrape = async extension => {
	const page = await browser.newPage()
	await page.goto(extension.url, {
	waitUntil: 'networkidle0',
	timeout: 0
	})

	const results = await page.evaluate(({ url, name }) => {
	const title = document.querySelector('.item-header .ux-item-name').innerText
	const desc = document.querySelector('.item-header .ux-item-shortdesc').innerText
	const img = document.querySelector('.item-img .image-display').getAttribute('src')
	console.log(img)

	return {
	title,
	desc,
	img,
	name,
	url
	}
	}, extension)

	return results
	}

	for (let i = 0; i < extensionList.length; i++) {
	let extension = extensionList[i]
	try {
	let data = await scrape(extension)
	extensionsDetailedList.push(data)
	spinner.text = `Received extension: ${data.name}`
	} catch (e) {
	throw e
	}
	}

	await browser.close()

	writeExtensionsFile(extensionsDetailedList)
	writeHtmlFile(extensionsDetailedList)

	setTimeout(() => {
	spinner.succeed('All done!')
	}, 1000)
	})
	}
	})()