Dump FreeIPA wiki pages as restructured text

importwiki.py

#!/usr/bin/env python3
"""Dump FreeIPA wiki pages as restructured text


Example:
    python3 importwiki.py V4/Kerberos_PKINIT V4/Healthcheck
"""
import argparse
import os
import subprocess
import sys
import urllib.request

# raw content URL on wiki
BASEURL = "https://www.freeipa.org/page/{}?action=raw"


def fetch_page(name):
    """Download raw MediaWiki content

    :param name: page name (after /page/)
    :return: page content as bytes
    """
    url = BASEURL.format(name)
    f = urllib.request.urlopen(url)
    # main title is not included in content
    content = b'= %s =\n\n' % name.encode('utf-8')
    content += f.read()
    return content


def convert_content(content):
    """Convert MediaWiki to RST

    :param content: media wiki content (bytes)
    :return: rst (bytes)
    """
    result = subprocess.run(
        ["pandoc", "-f", "mediawiki", "-t", "rst"],
        input=content,
        capture_output=True,
    )
    return result.stdout


def name2filename(name):
    name = name.lower()
    name = name.replace("/", "_")
    return name + ".rst"


parser = argparse.ArgumentParser()
parser.add_argument("pages", nargs="+")
parser.add_argument("--force", action="store_true")


def main():
    args = parser.parse_args()
    for name in args.pages:
        filename = name2filename(name)
        if os.path.isfile(filename) and not args.force:
            print(f"Skip {filename}", file=sys.stderr)
            continue
        content = fetch_page(name)
        rst = convert_content(content)
        with open(filename, "wb") as f:
            f.write(rst)
        print(f"Written {filename}", file=sys.stderr)


if __name__ == '__main__':
    main()