Gets the latest Xonotic map URLs

xonmaps.py

#!/usr/bin/env python3
import os
import sys
from glob import glob
from lxml import html
from typing import List, NamedTuple, Optional
from urllib.parse import urljoin
from urllib.request import urlopen

BASE_URL = "https://beta.xonotic.org/autobuild-bsp/"
MAP_DIR = "~/mirror/maps"
XONOTIC_DIR = "~/.xonotic/data"

IGNORED_MAP_NAMES = {
    "_hudsetup",
    "dotc_test4",
    "fanatic_experimental_notforplay",
    "snafu-test",
    "snafu-test2",
    "test",
    "test_do_not_download",
    "tutorial",
    "tutorial_level1",
    "tutorial_level2",
}


class Map(NamedTuple):
    name: str
    pk3: str
    master: bool


def maps_load(data: str) -> List[Map]:
    maps: List[Map] = []
    last_name: Optional[str] = None
    for row in html.fromstring(data).xpath("//tr[@class]"):
        map_name = row.xpath('td[@class="mapname"]/text()')
        name = map_name[0] if len(map_name) > 0 else last_name
        last_name = name
        if name is None or name in IGNORED_MAP_NAMES:
            continue

        branch = row.xpath('td[@class="branches"]/code/text()')
        master = len(branch) > 0 and branch[0] == "master"
        pk3 = row.xpath('td[@class="fullpk3"]/a[@href]')[0].get("href")
        maps.append(Map(name, pk3, master))

    return maps


def maps_select(maps: List[Map]) -> List[Map]:
    selection = {}
    for map in reversed(maps):
        if map.name not in selection:
            selection[map.name] = map
            continue
        if not selection[map.name].master:
            selection[map.name] = map
            continue
        if map.master:
            selection[map.name] = map

    return list(selection.values())


def map_download(map_dir: str, map: Map) -> None:
    path = os.path.join(map_dir, map.pk3)
    if os.path.exists(path):
        return

    with urlopen(urljoin(BASE_URL, map.pk3)) as url, open(path, "wb") as fp:
        size = int(url.info().get("Content-Length"))
        if size < 1:
            return

        last_pct = 0
        while True:
            data = url.read(1024)
            if not data:
                break

            fp.write(data)
            dl_pct = int((fp.tell() // size) * 100)
            if last_pct == dl_pct:
                continue

            sys.stdout.write("\033[2K")
            sys.stdout.write("Downloading (%d%%): %s\r" % (dl_pct, map.pk3))
            sys.stdout.flush()
            last_pct = dl_pct

    print("Downloaded:", path)


def map_link(map_dir: str, xonotic_dir: str, map: Map) -> None:
    link = os.path.join(xonotic_dir, map.pk3)
    if not os.path.exists(link):
        path = os.path.join(map_dir, map.pk3)
        os.symlink(path, link)


def map_remove(map_dir: str, xonotic_dir: str, map: Map) -> None:
    glob_path = os.path.join(map_dir, map.name + "-full-*.pk3")
    for path in glob(glob_path):
        basename = os.path.basename(path)
        if basename == map.pk3:
            continue

        link = os.path.join(xonotic_dir, basename)
        if os.path.exists(link):
            os.unlink(link)

        os.unlink(path)


def main() -> None:
    with urlopen(BASE_URL) as url:
        data = url.read()
        maps = maps_select(maps_load(data))

    map_dir = os.path.expanduser(MAP_DIR)
    if not os.path.exists(map_dir):
        os.makedirs(map_dir)

    xonotic_dir = os.path.expanduser(XONOTIC_DIR)
    if not os.path.exists(xonotic_dir):
        os.makedirs(xonotic_dir)

    for map in maps:
        map_remove(map_dir, xonotic_dir, map)
        map_download(map_dir, map)
        map_link(map_dir, xonotic_dir, map)


if __name__ == "__main__":
    main()