Web Scraping Wikipedia

python_requests_lxml.py

import requests
import lxml.html as html


def get_page(url):
    return requests.get(url, timeout =1.).text 
    

def parse_page(song_url):
    page_source = get_page(song_url)
    assert isinstance(page_source, str)
    root = html.fromstring(page_source)
    TARGET_XPATH = '//*[@id="mw-content-text"]/div/p[11]'
    target_html = root.xpath(TARGET_XPATH)
    assert len(target_html) == 1
    target_html = target_html[0] 
    output = [target_html.text]
    for c in target_html.getchildren():
        output.extend([c.text, c.tail])
    return list(filter(lambda x : x, output))

target_url = 'https://en.wikipedia.org/wiki/Periodic_table'
print(' '.join(parse_page(target_url)))