Basic Wikipedia scrapping with pyppeteer

simple_scrapping_pyppeteer.py

# -*- coding: utf-8 -*-

import pprint
import asyncio
from pyppeteer import launch


async def get_browser():
    return await launch({"headless": False})


async def get_page(browser, url):
    page = await browser.newPage()
    await page.goto(url)
    return page


async def extract_data(page):
    # Select tr with a th and td descendant from table
    elements = await page.xpath(
        '//table[@class="infobox"]/tbody/tr[th and td]')
    # Extract data
    result = {}
    for element in elements:
        title, content = await page.evaluate(
            '''(element) =>
                [...element.children].map(child => child.textContent)''',
            element)
        result.update({title: content})
    return result


async def extract(name, url):
    browser = await get_browser()
    page = await get_page(browser, url)
    return {name: await extract_data(page)}


if __name__ == "__main__":
    languages = {
        "python": "https://es.wikipedia.org/wiki/Python",
        "Rust": "https://es.wikipedia.org/wiki/Rust_(lenguaje_de_programaci%C3%B3n)",
        "Java": "https://es.wikipedia.org/wiki/Java_(lenguaje_de_programaci%C3%B3n)",
        "Javascript": "https://es.wikipedia.org/wiki/JavaScript"
    }

    loop = asyncio.get_event_loop()
    tasks = [
        loop.create_task(extract(name, url)) for name, url in languages.items()
    ]

    done, pending = loop.run_until_complete(asyncio.wait(tasks))

    result = {}
    for task in done:
        result.update(task.result())

    pprint.pprint(result)