simple node script to get headers and data from a webpage. Uses puppeteer.

puppeteer-json.js

'use strict'

const puppeteer = require('puppeteer');
const fs = require('fs');

var args = process.argv.slice(2);
if (!(Array.isArray(args) && args.length)) {
    console.log('puppeteer-json.js url [fileprefix]');
    process.exit(1);
}

const pageURL = args[0];
const prefixfile = args[1] ? args[1] : './output';
const pageTimeout = 250000;

(async() => {
    const browser = await puppeteer.launch({ headless: true });
    const page = await browser.newPage();
    let responses = [];
    await page.setCacheEnabled(false);
    await page.setRequestInterception(true);
    await page.setViewport({ width: 1920, height: 1080 });
    page.on('request', (request) => {
            request.continue();
            });
    page.on('response', (response) => {
            responses.push(response);
            });
    try {
        await page.goto(pageURL, {
                            timeout: pageTimeout,
                            waitUntil: 'networkidle0'
                        });
        await page.screenshot({
                    path: prefixfile + '.png',
                    type: 'png',
                    fullPage: true
            });
        await page.emulateMedia('screen');
        await page.pdf({
                    path: prefixfile + '.pdf',
                    format: 'A4'
            });
        await page.close();
    } catch (err) {
        console.log(`failed to open the page: ${pageURL} with the error: ${err}`);
    }
    await browser.close();

    let reportResponse = []
    try {
        for (let  r of responses) {
            const hasChain = r._request._redirectChain;
            let chain = [];
            for (let c of hasChain) {
                chain.push({url: c.url(), status: c.response().status()});
            }
            reportResponse.push({
                url: r._url,
                remoteAddress: r._remoteAddress,
                requestHeaders: r._request._headers,
                resourceType: r._resourceType,
                status: r._status,
                securityDetails: r._securityDetails,
                responseHeaders: r._headers,
                chain: chain
            });
        }
    } catch (err) {
        console.log(`failed with the error: ${err}`);
    }
    try {
        fs.writeFileSync(prefixfile + '.json', JSON.stringify(reportResponse, null, 2));
    } catch(err) {
        console.error('Data not written!', err);
    }
    console.log(`Output to files ${prefixfile}.{json,png,pdf} the url ${pageURL}`)
    process.exit()
})();