Use Puppeteer to download a webpage after its been processed by javascript

fetch2.js

// save as index.js
// npm install https://github.com/GoogleChrome/puppeteer/
// node index.js URL
const puppeteer = require('puppeteer');

(async () => {
  const url = process.argv[2];
  const browser = await puppeteer.launch();
  // use tor
  //const browser = await puppeteer.launch({args:['--proxy-server=socks5://127.0.0.1:9050']});
  const page = await browser.newPage();
  page.on('request', (request) => {
    console.log(`Intercepting: ${request.method} ${request.url}`);
    request.continue();
  });
  await page.goto(url, {waitUntil: 'load'});

  //const title = await page.title();
  //console.log(title);
  await page.screenshot({path:'example.png'});
  const html = await page.content();
  console.log(html);

  browser.close();
})();

Thank you, you saved me ! :)

I thought this would download the webpage as an html file 😞😞

@Winstone-Were it outputs the webpage to the console. you can redirect the output or you can change console.log(html) to write to the file of your choice

thanks Schollz, devs help devs

…

On Wed, Aug 26, 2020 at 7:03 PM Zack ***@***.***> wrote: ***@***.**** commented on this gist. ------------------------------ @Winstone-Were <https://github.com/Winstone-Were> it outputs the webpage to the console. you can redirect the output or you can change console.log(html) to write to the file of your choice — You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub <https://gist.github.com/4dcd045a95196f567ba0abdd0ac70452#gistcomment-3431896>, or unsubscribe <https://github.com/notifications/unsubscribe-auth/AOBFKPBMTUW3X6KAKRCTLP3SCUW5VANCNFSM4KH4D3PQ> .

redirect

thanks fellow developer