linkedin_profile_post_scraper.js

// Updated extractLinkedInPosts function with continuous scrolling
async function extractLinkedInPosts() {
    const postsData = [];
    const seenTexts = new Set();
    let lastLength = 0;

    while (postsData.length < 20) {
        window.scrollTo(0, document.body.scrollHeight);
        await new Promise(resolve => setTimeout(resolve, 2000)); // Wait for more posts to load

        const postsList = document.getElementsByTagName('ul')[1].getElementsByTagName('li');
        if (postsList.length === lastLength) {
            break; // Break if no new posts are loaded
        }
        lastLength = postsList.length;

        for (let i = 0; i < postsList.length && postsData.length < 20; i++) {
            const post = postsList[i];
            const seeMoreButton = post.querySelector('button.feed-shared-inline-show-more-text__see-more-less-toggle');
            if (seeMoreButton) seeMoreButton.click();
            await new Promise(resolve => setTimeout(resolve, 1000)); // Wait for the text to expand

            const textContainer = post.querySelector('div.feed-shared-update-v2__description-wrapper > div');
            const text = textContainer ? textContainer.textContent.trim() : '';
            const linkAnchor = post.querySelector('a.app-aware-link');
            const link = linkAnchor ? linkAnchor.href : '';

            if (!seenTexts.has(text) && text) {
                seenTexts.add(text);
                postsData.push({ text, link });
            }
        }
    }
    return postsData;
}

// Function to download posts data as JSON
function downloadPostsData(postsData) {
    const dataStr = "data:text/json;charset=utf-8," + encodeURIComponent(JSON.stringify(postsData));
    const downloadAnchorNode = document.createElement('a');
    downloadAnchorNode.setAttribute("href", dataStr);
    downloadAnchorNode.setAttribute("download", "LinkedInPostsData.json");
    document.body.appendChild(downloadAnchorNode);
    downloadAnchorNode.click();
    downloadAnchorNode.remove();
}

// Main runner function
async function main() {
    const postsData = await extractLinkedInPosts();
    console.log(postsData); // Log the extracted data
    downloadPostsData(postsData); // Download the data
}

main();