fetch latest image links for https://fedoraproject.org/wiki/Test_Day:2020-08-31_Btrfs_default

f33-latest-images.py

#!/usr/bin/python3

import asyncio
import pathlib
import textwrap

import bs4
import httpx
import rfc3986


compose_url = rfc3986.urlparse('https://kojipkgs.fedoraproject.org/compose/branched/latest-Fedora-33/compose/')
compose_path = pathlib.Path(compose_url.path)
results = {}
client = httpx.AsyncClient()


async def get_image_url(variant, arch, name=None, raw=False):
    # if name is not given, set it to the variant name
    if not name:
        name = variant

    # set the appropriate extension
    extension = '.raw.xz' if raw else '.iso'

    # construct the parent directory url
    image_dir_path = compose_path / variant / arch / ('images' if raw else 'iso')
    image_dir_url = compose_url.copy_with(path=str(image_dir_path))

    # http call
    print(f'GET {image_dir_url.unsplit()}')
    response = await client.get(image_dir_url.unsplit())
    response.raise_for_status()

    # find download url in the html
    soup = bs4.BeautifulSoup(response.text, features='lxml')
    tags = soup.find_all(
        href=lambda href: href and href.endswith(extension),
        string=lambda string: name in string
    )
    try:
        [tag] = tags
    except ValueError:
        raise SystemExit(f'more than one link contains "{name}", be more specific')

    print(f'found image: {tag["href"]}')
    image_path = image_dir_path / tag['href']
    image_url = image_dir_url.copy_with(path=str(image_path))
    results[f'{name}-{arch}{extension}'] = image_url.unsplit()


async def main():
    await asyncio.gather(
        get_image_url('Workstation', 'x86_64'),
        get_image_url('Workstation', 'ppc64le'),
        get_image_url('Workstation', 'aarch64'),
        get_image_url('Everything', 'x86_64'),
        get_image_url('Spins', 'x86_64', name='KDE'),
        get_image_url('Spins', 'x86_64', name='Xfce'),
        get_image_url('Spins', 'x86_64', name='SoaS'),
        get_image_url('Spins', 'x86_64', name='Cinnamon'),
        get_image_url('Spins', 'x86_64', name='LXDE'),
        get_image_url('Spins', 'x86_64', name='LXQt'),
        get_image_url('Labs', 'x86_64', name='Astronomy'),
        get_image_url('Labs', 'x86_64', name='Comp_Neuro'),
        get_image_url('Labs', 'x86_64', name='Games'),
        get_image_url('Labs', 'x86_64', name='Python-Classroom'),
        get_image_url('Labs', 'x86_64', name='Security'),
        get_image_url('Silverblue', 'x86_64'),
        get_image_url('Silverblue', 'ppc64le'),
        get_image_url('Workstation', 'aarch64', raw=True),
        get_image_url('Workstation', 'armhfp', raw=True),
        get_image_url('Spins', 'armhfp', name='KDE', raw=True),
        get_image_url('Spins', 'aarch64', name='Xfce', raw=True),
        get_image_url('Spins', 'armhfp', name='Xfce', raw=True),
        get_image_url('Spins', 'armhfp', name='LXQt', raw=True),
    )

    await client.aclose()

    print(textwrap.dedent(f"""
        Live ISOs:<br />
        &spades; [{results['Workstation-x86_64.iso']} Fedora-Workstation-Live-x86_64-33]<br />
        [{results['Workstation-ppc64le.iso']} Fedora-Workstation-Live-ppc64le-33]<br />
        &spades; [{results['Workstation-aarch64.iso']} Fedora-Workstation-Live-aarch64-33]<br />
        &spades; [{results['Everything-x86_64.iso']} Fedora-Everything-netinst-x86_64-33]<br />
        &spades; [{results['KDE-x86_64.iso']} Fedora-KDE-Live-x86_64-33]<br />
        [{results['Xfce-x86_64.iso']} Fedora-Xfce-Live-x86_64-33]<br />
        [{results['SoaS-x86_64.iso']} Fedora-SoaS-Live-x86_64-33]<br />
        [{results['Cinnamon-x86_64.iso']} Fedora-Cinnamon-Live-x86_64-33]<br />
        [{results['LXDE-x86_64.iso']} Fedora-LXDE-Live-x86_64-33]<br />
        [{results['LXQt-x86_64.iso']} Fedora-LXQt-Live-x86_64-33]<br />
        [{results['Astronomy-x86_64.iso']} Fedora-Astronomy_KDE-Live-x86_64-33]<br />
        [{results['Comp_Neuro-x86_64.iso']} Fedora-Comp_Neuro-Live-x86_64-33]<br />
        [{results['Games-x86_64.iso']} Fedora-Games-Live-x86_64-33]<br />
        [{results['Python-Classroom-x86_64.iso']} Fedora-Python-Classroom-Live-x86_64-33]<br />
        [{results['Security-x86_64.iso']} Fedora-Security-Live-x86_64-33]

        Silverblue ISOs:<br />
        [{results['Silverblue-x86_64.iso']} Fedora-Silverblue-ostree-x86_64-33]<br />
        [{results['Silverblue-ppc64le.iso']} Fedora-Silverblue-ostree-ppc64le-33]

        ARM Images:<br />
        &spades; [{results['Workstation-aarch64.raw.xz']} Fedora-Workstation-33.aarch64.raw.xz]<br />
        [{results['Workstation-armhfp.raw.xz']} Fedora-Workstation-armhfp-33.raw.xz]<br />
        [{results['KDE-armhfp.raw.xz']} Fedora-KDE-armhfp-33.raw.xz]<br />
        [{results['Xfce-aarch64.raw.xz']} Fedora-Xfce-33.aarch64.raw.xz]<br />
        [{results['Xfce-armhfp.raw.xz']} Fedora-Xfce-armhfp-33.raw.xz]<br />
        [{results['LXQt-armhfp.raw.xz']} Fedora-LXQt-armhfp-33.raw.xz]
    """))


asyncio.run(main())