Compare different ways to get a html tree from an url with streaming

parse_url.py

"""Compare ways to return HTML tree streamed and parsed from a given URL."""

import contextlib
from typing import Literal, overload
import urllib.request
import xml.etree.ElementTree as etree

import certifi
import html5lib
import lxml.html
import requests
import urllib3


def parse(url: str, /) -> etree.ElementTree:
    # pure stdlib, but well-formed XHTML only
    with urllib.request.urlopen(url) as response:
        return etree.parse(response)


def parse(url: str, /) -> lxml.etree.ElementTree:
    if url.lower().startswith('https:'):
        raise NotImplementedError('no HTTPS support')
    return lxml.html.parse(url)


def parse(url: str, /) -> lxml.etree.ElementTree:
    with urllib.request.urlopen(url) as response:
        return lxml.html.parse(response)


@overload
def parse(url: str, /, *,
          treebuilder: Literal['etree'] = ...) -> etree.Element:
    """Return stdlib etree Element for treebuilder='etree'."""

@overload
def parse(url: str, /, *,
          treebuilder: Literal['lxml']) -> lxml.etree.ElementTree:
    """Return lxml ElementTree for treebuilder='lxml'."""

def parse(url: str, /, *,
          treebuilder: Literal['etree', 'lxml'] = 'etree'):
    with urllib.request.urlopen(url) as response:
        encoding = response.headers.get_content_charset()
        return html5lib.parse(response,
                              treebuilder=treebuilder,
                              transport_encoding=encoding,
                              namespaceHTMLElements=False)


def parse(url: str, /) -> lxml.etree.ElementTree:
    with urllib3.PoolManager(cert_reqs='CERT_REQUIRED', ca_certs=certifi.where()) as http:
        response = http.request('GET', url, preload_content=False)
        try:
            return lxml.html.parse(response)
        finally:
            response.release_conn()


def parse(url: str, /) -> lxml.etree.ElementTree:
    with contextlib.closing(requests.get(url, stream=True)) as response:
        response.raw.decode_content = True
        return lxml.html.parse(response.raw)


def pprint(doc: etree.Element | etree.ElementTree | lxml.etree.ElementTree, /,
           **kwargs) -> None:
    if isinstance(doc, (etree.ElementTree, lxml.etree.ElementTree)):
        doc = doc.getroot()
    print(etree.tostring(doc, encoding='unicode', **kwargs))