twitter-analytics-data-scraper

dataMiner.js

// 1. Go to https://analytics.twitter.com/ 

// 2. Keep scrolling till the end until all the stats data is loaded

// 3. Right click on page click on last option "Inspect" a window should open select console from that

// 4. copy this entire function

function dataMiner(){  
        let masterArray = [];
        const totalSummary = document.getElementsByClassName("home-columns").length;
        for(let j=0; j< totalSummary; j++){
            let tweets = 0;
            let tweetviews = 0;
            let profileViews = 0;
            let mentions = 0;
            let followers = 0;
            let title = "";
            let item = document.getElementsByClassName("home-columns")[j];
            if(!!item.querySelector(".metric-tweets")){
                tweets = !!item.querySelector(".metric-tweets").textContent ? item.querySelector(".metric-tweets").textContent : null;
            }

            if(!!item.querySelector(".metric-tweetviews")){
                tweetviews = !!item.querySelector(".metric-tweetviews").textContent ? item.querySelector(".metric-tweetviews").textContent : null;
            }

            if(!!item.querySelector(".metric-profile-views")){
                profileViews = !!item.querySelector(".metric-profile-views").textContent ? item.querySelector(".metric-profile-views").textContent : null;
            }

            if(!!item.querySelector(".metric-mentions")){
                mentions = !!item.querySelector(".metric-mentions").textContent ? item.querySelector(".metric-mentions").textContent: null;
            }

            if(!!item.querySelector(".metric-followers")){
                followers = !!item.querySelector(".metric-followers").textContent ? item.querySelector(".metric-followers").textContent : null;
            }

            if(!!item.querySelectorAll('.home-group-header')){
                const totalLength = item.querySelectorAll('.home-group-header').length;
                title = item.querySelectorAll('.home-group-header')[totalLength - 1].innerHTML;
            }

            masterArray.push({tweets, tweetviews, profileViews, mentions, followers, title })
        } 
        // to view data in table format
        console.log(console.table(masterArray))
        
        // to view data as a JSON String
        console.log(JSON.stringify(masterArray))
    };
    
// 5. Hit Enter

// 6. type in console => dataMiner()

// 7. Hit Enter

// 8. You will see output in console as a JSON object, copy that entire object and paste it into https://json-csv.com/ 
// and convert it csv if you ever want it

// 9. Import this CSV to airtable if you want to save it further