Download all searchable content from GOV.UK, using the sitemaps.

all-searchable-content.py

#! /usr/bin/env nix-shell
#! nix-shell -i python3 -p "python3.withPackages (p: [p.requests])"

import os
import requests
from xml.etree import ElementTree


def get_xml(url):
    """Stream some XML and return an iterator over the elements.
    """

    response = requests.get(url, stream=True)
    response.raise_for_status()
    response.raw.decode_content = True
    return ElementTree.iterparse(response.raw)


def url_to_base_path(url):
    """Turn a GOV.UK URL into a base path.
    """

    prefix = 'https://www.gov.uk/'
    if not url.startswith(prefix):
        raise Exception(url)
    return url[len(prefix):]


def fetch_from_content_api(base_path):
    """Fetch a content item from the content API and write it to disk.
    """

    url = f'https://www.gov.uk/api/content/{base_path}'
    filename = f'govuk/{base_path}.json'
    os.makedirs(os.path.dirname(filename), exist_ok=True)
    with requests.get(url, stream=True) as response:
        response.raise_for_status()
        with open(filename, 'wb') as f:
            for chunk in response.iter_content(chunk_size=128):
                f.write(chunk)


for event0, elem0 in get_xml('https://www.gov.uk/sitemap.xml'):
    if event0 == 'end' and elem0.tag == '{http://www.sitemaps.org/schemas/sitemap/0.9}loc':
        sitemap_url = elem0.text
        for event, elem in get_xml(sitemap_url):
            if event == 'end' and elem.tag == '{http://www.sitemaps.org/schemas/sitemap/0.9}loc':
                base_path = url_to_base_path(elem.text)
                fetch_from_content_api(base_path)