Crawl linkedin pages with crawl4ai

linkedin.py

from crawl4ai import (
    CrawlerRunConfig,
    BrowserConfig,
    AsyncWebCrawler,
    DefaultMarkdownGenerator,
    BM25ContentFilter,
)
import asyncio
from pprint import pprint

browser_config = BrowserConfig(
    headless=False,
    user_data_dir="/Users/aravindkarnam/.crawl4ai/profiles/linkedin",
    use_managed_browser=True,
)


async def getProfile():
    async with AsyncWebCrawler(config=browser_config) as crawler:
        result = await crawler.arun(
            "https://www.linkedin.com/in/aravindkarnam/",
            config=CrawlerRunConfig(
                delay_before_return_html=3,
                scan_full_page=True,
                scroll_delay=1,
                markdown_generator=DefaultMarkdownGenerator(
                    content_filter=BM25ContentFilter(
                        user_query="who is aravind karnam? what are his professional qualifications and work experience?"
                    )
                ),
            ),
        )
        pprint(result.markdown.fit_markdown)


asyncio.run(getProfile())