Parse buildout.cfg and recursively download all extends files that are online

buildout_extends_grabber.py

"""
Parse buildout.cfg and recursively download all extends files that are online

Usage::

  python buildout_extends_grabber.py [url or path to buildout file]

Needs Python 3.7 or higher.

Examples::

  python buildout_extends_grabber.py https://dist.plone.org/release/5.2.8/versions.cfg
  python buildout_extends_grabber.py versions.cfg
  python buildout_extends_grabber.py --plone=6.0.0a6
  python buildout_extends_grabber.py

When no file is specified, buildout.cfg in the current directory is used.
With --plone=6.0.0a6, we download files from dist.plone.org for this Plone version.

When run with this buildout.cfg::

  [buildout]
  extends = http://dist.plone.org/release/4.1/versions.cfg

the result is that these files are downloaded in the current directory:

- dist.plone.org_release_4.1_versions.cfg

- download.zope.org_Zope2_index_2.13.8_versions.cfg

- download.zope.org_zopetoolkit_index_1.0.3_zopeapp-versions.cfg

- download.zope.org_zopetoolkit_index_1.0.3_ztk-versions.cfg

We rename them afterwards.
"""

from configparser import ConfigParser
from configparser import NoOptionError
from configparser import NoSectionError
from pkg_resources import parse_version

import os.path
import re
import sys
import urllib.error
import urllib.parse
import urllib.request


# Default file to use:
FILENAME = "buildout.cfg"

RENAMES = [
    (re.compile("dist.plone.org_release_.*_versions.cfg"), "plone-versions.cfg"),
    # Plone 6.0.0a6+
    (
        re.compile("dist.plone.org_release_.*_versions-ecosystem.cfg"),
        "versions-ecosystem.cfg",
    ),
    (re.compile(".*_requirements.txt"), "requirements.txt"),
    (re.compile("dist.plone.org_release_.*_versions-extra.cfg"), "versions-extra.cfg"),
    # Zope 2:
    (re.compile("dist.plone.org_versions_zope-.*-versions.cfg"), "zope-versions.cfg"),
    (
        re.compile("dist.plone.org_versions_zopetoolkit-.*-ztk-versions.cfg"),
        "ztk-versions.cfg",
    ),
    (
        re.compile("dist.plone.org_versions_zopetoolkit-.*-zopeapp-versions.cfg"),
        "zopeapp-versions.cfg",
    ),
    # Zope 4:
    (
        re.compile("raw.githubusercontent.com_zopefoundation_Zope_.*_versions.cfg"),
        "zope-versions.cfg",
    ),
    (
        re.compile(
            "raw.githubusercontent.com_zopefoundation_Zope_.*_versions-prod.cfg"
        ),
        "zope-prod-versions.cfg",
    ),
    # Zope 4.4 / Plone 5.2.2:
    (
        re.compile("zopefoundation.github.io_Zope_releases_.*_versions.cfg"),
        "zope-versions.cfg",
    ),
    (
        re.compile("zopefoundation.github.io_Zope_releases_.*_versions-prod.cfg"),
        "zope-prod-versions.cfg",
    ),
]


def rename_file(filename, tab=""):
    for pattern, newname in RENAMES:
        if pattern.match(filename):
            os.rename(filename, newname)
            print(f"{tab} Renamed {filename} to {newname}")
            return newname
    return filename


def get_file_or_download(extend, tab=""):
    """Get local file or download it."""
    if os.path.isfile(extend):
        print(f"{tab} Local file found: {extend}")
        return extend
    print(f"{tab} Downloading {extend}")
    try:
        response = urllib.request.urlopen(extend)
    except Exception as exc:
        print(f"{tab} DOWNLOAD ERROR: {exc}")
        return None
    text = "# Downloaded from %s\n" % extend
    for line in response.readlines():
        line = line.decode("utf-8")
        text += line.rstrip() + "\n"
    # Ensure one empty line at the end.
    text = text.strip() + "\n"
    filename = ".".join(extend.split("://")[1:]).replace("/", "_")
    print(f"{tab} Writing file {filename}")
    with open(filename, "w") as myfile:
        myfile.write(text)
    return filename


def comment_out_non_versions(filename, tab=""):
    with open(filename) as response:
        # We only want the [versions] sections, and comment out anything above it.
        text = ""
        versions_section_found = False
        for line in response.readlines():
            if line.startswith("[versions]"):
                versions_section_found = True
            if versions_section_found:
                text += line
            else:
                line_cmp = line.strip()
                if not line_cmp or line_cmp.startswith("#"):
                    text += line
                else:
                    text += "# " + line
    print(f"{tab} Commenting out non-versions part of file {filename}")
    with open(filename, "w") as myfile:
        myfile.write(text)

# Gather list of 'extends', so we can print them in the right order.
extends_list = []


def parse_buildout_file(filename, orig_url, level=1):
    base_url = "/".join(orig_url.split("/")[:-1])
    tab = " " * level * 4
    print(f"{tab} Looking for extra extends in {filename}")
    config = ConfigParser()
    config.read(filename)
    try:
        extends = config.get("buildout", "extends").strip()
    except (NoSectionError, NoOptionError):
        extends_list.append(filename)
        return

    orig_filename = filename
    for extend in extends.splitlines():
        print()  # empty line
        if extend.startswith("http"):
            url = extend
        else:
            url = "/".join([base_url, extend])
        filename = get_file_or_download(url, tab)
        if not filename:
            continue
        filename = rename_file(filename, tab)
        parse_buildout_file(filename, url, level=level + 1)
        comment_out_non_versions(filename, tab)
    extends_list.append(orig_filename)


def handle_url(url):
    filename = get_file_or_download(url)
    filename = rename_file(filename)
    if filename.startswith("requirements") and filename.endswith(".txt"):
        return
    parse_buildout_file(filename, url)
    comment_out_non_versions(filename)


# We have special support for Plone.
plone = None
if len(sys.argv) == 2:
    url = sys.argv[1]
    # lazy version of parsing something like --plone=6.0.0a6
    if url.startswith("--plone="):
        plone = url.split("=")[-1]
        url = f"https://dist.plone.org/release/{plone}/versions.cfg"
        print(f"Getting files for Plone {plone}.")
else:
    url = FILENAME
handle_url(url)
if plone:
    handle_url(f"https://dist.plone.org/release/{plone}/requirements.txt")
    if parse_version(plone) >= parse_version("5.3"):
        # Get extra versions files:
        handle_url(f"https://dist.plone.org/release/{plone}/versions-ecosystem.cfg")
        handle_url(f"https://dist.plone.org/release/{plone}/versions-extra.cfg")

print("Usage:")
print("extends =")
for extend in extends_list:
    print(f"    {extend}")