tableau-covid-scraping

index.js

const URL = 'https://public.tableau.com/views/PPV_15924847800480/ppv_db?%3Aembed=y&%3AshowVizHome=no&%3Adisplay_count=y&%3Adisplay_static_image=n&%3AbootstrapWhenNotified=true&%3Alanguage=en&:embed=y&:showVizHome=n&:apiID=host0';

const puppeteer = require('puppeteer');

// Below, largely cribbed from Thomas Dondorf at https://stackoverflow.com/questions/52969381/how-can-i-capture-all-network-requests-and-full-response-data-when-loading-a-pag

(async () => {
    const browser = await puppeteer.launch();
    const [page] = await browser.pages();

    let paused = false;
    let pausedRequests = [];

    const nextRequest = () => { // continue the next request or "unpause"
        if (pausedRequests.length === 0) {
            paused = false;
        } else {
            // continue first request in "queue"
            (pausedRequests.shift())(); // calls the request.continue function
        }
    };

    await page.setRequestInterception(true);

    page.on('request', request => {
        if (paused) {
            pausedRequests.push(() => request.continue());
        } else {
            paused = true; // pause, as we are processing a request now
            request.continue();
        }
    });

    page.on('requestfinished', async (request) => {
        const response = await request.response();

        let responseBody;
        if (request.url().includes('bootstrapSession')){
            responseBody = await response.buffer();
            responseBody = responseBody.toString();
            responseBody = responseBody.replace(/^\d+;{/g,'{');
            responseBody = responseBody.replace(/\d+;{/g,',{');
            responseBody = '[' + responseBody + ']';
            
            let jsonParsed = JSON.parse(  responseBody );

            let dataColumns = jsonParsed[1].secondaryInfo.presModelMap.dataDictionary.presModelHolder
                              .genDataDictionaryPresModel.dataSegments["0"].dataColumns;

            let dataArray = dataColumns[1].dataValues;

            let vizPaneColumns = jsonParsed[1].secondaryInfo.presModelMap.vizData.presModelHolder
                              .genPresModelMapPresModel.presModelMap.PPV.presModelHolder.genVizDataPresModel
                              .paneColumnsData.paneColumnsList[1].vizPaneColumns;

            console.log( JSON.stringify(dataArray) );

            //console.log( responseBody );
        }

        nextRequest(); // continue with next request
    });

    page.on('requestfailed', (request) => {
        // handle failed request
        nextRequest();
    });

    await page.goto(URL, { waitUntil: 'networkidle0' });
    await browser.close();
})();