Backporting of PRs and language changes

backport-languages.py

"""
Put this file in a master checkout under .github/.
It should be next to backport.py.
"""

import argparse
import glob
import subprocess
import shlex

def backport_language(language_file, blacklisted_ids, diff_to_stdout=False):
    result = subprocess.run(
        shlex.split(
            "git diff HEAD..upstream/master -- %s" % language_file
        ), check=True, stdout=subprocess.PIPE)

    input_lines = []
    chunk = []
    # We start with this set to True, to pick up any headers before the
    # patch really begins
    chunk_has_modification = True

    # Decode the result, skip the 4 line header
    for line in result.stdout.decode().split('\n'):
        if not line or line.startswith('@@') or line.startswith(('---', '+++')):
            # Only add the chunk if there was a modification to it.
            # 'git apply' cannot handle chunks with no modifications.
            if chunk_has_modification:
                input_lines.extend(chunk)
            chunk = []
            chunk_has_modification = False

            # Check for the start of a new chunk
            if line.startswith('@@'):
                chunk.append(line)
            else:
                input_lines.append(line)
            continue

        # Passthrough all the unmodified lines (they are just context)
        if not line.startswith(('-', '+')):
            chunk.append(line)
            continue

        id = line[1:].split(':')[0]
        if id not in blacklisted_ids:
            # Modification is not blacklisted; this is fine
            chunk_has_modification = True
            chunk.append(line)
            continue

        # A blacklisted id; skip the modification
        if line.startswith('+'):
            pass
        else:
            chunk.append(' ' + line[1:])

    # No chunks found, so nothing to do
    if len(input_lines) < 6:
        return

    total_input = "\n".join(input_lines)
    if diff_to_stdout:
        print(total_input)
        return

    result = subprocess.run(
        shlex.split(
            "git apply --recount"
        ), check=True, input=total_input.encode())


def create_blacklisted_ids():
    # First check what changed in english.txt. Every change is blacklisted and
    # translations in these lines will not be backported
    result = subprocess.run(
        shlex.split(
            "git diff HEAD..upstream/master -- src/lang/english.txt"
        ), check=True, stdout=subprocess.PIPE)

    blacklisted_ids = []

    # Walk the diff line by line
    for line in result.stdout.decode().split('\n'):
        # Ignore headers
        if line.startswith(('---', '+++')) or not line:
            continue

        # Find all the lines that are modified
        if line.startswith(('-', '+')):
            # Store that id in a blacklist
            id = line[1:].split(':')[0]
            blacklisted_ids.append(id)

    return blacklisted_ids

def parse_command_line():
    parser = argparse.ArgumentParser(description='Backport languages from master to release branch')
    parser.add_argument('languages', metavar='LANGUAGE', type=str, nargs='*', help='which languages to backport (empty for all)')
    parser.add_argument('--diff', action='store_true', help='only show the diff; do not apply')
    return parser.parse_args()

def main():
    args = parse_command_line()

    blacklisted_ids = create_blacklisted_ids()

    if args.languages:
        language_files = [ "src/lang/%s.txt" % language for language in args.languages ]
    else:
        language_files = glob.glob("src/lang/*.txt") + glob.glob("src/lang/unfinished/*.txt")

    for language_file in language_files:
         print("Backporting %s ..." % language_file[len("src/lang/"):])
         backport_language(language_file, blacklisted_ids, diff_to_stdout=args.diff)


if __name__ == "__main__":
    main()

backport.py

"""
Put this file in a master checkout under .github/.
It should be next to backport-languages.py.

This assumes your git "origin" points to your fork, and "upstream" to upstream.
This will force-push to a branch called "release-backport".

Execute with:

$ python3 .github/backport.py

And follow the instructions. After the PR is merged, run:

$ python3 .github/backport.py --mark-done <PR-NUMBER>
"""

import json
import os
import subprocess
import sys

# NOTE: Replace this with your own toekn
BEARER_TOKEN = "ghp_???"
# NOTE: Replace this with your own GitHub username
USERNAME = "TrueBrain"
# NOTE: Replace with the version branch to backport to
RELEASE = "13"

pr_query = """
query ($number: Int!) {
  repository(owner: "OpenTTD", name: "OpenTTD") {
    pullRequest(number: $number) {
      body
    }
  }
}
"""

pr_search_query = """
query ($search: String!) {
  search(query: $search, type: ISSUE, first: 100) {
    issueCount
    edges {
      node {
        ... on PullRequest {
          number
          title
          commits(first: 100) {
            totalCount
          }
          mergedAt
          mergeCommit {
            oid
          }
          labels(first: 10) {
            nodes {
              name
            }
          }
        }
      }
    }
  }
}
"""


def do_query(query, variables):
    query = query.replace("\n", "").replace('\\', '\\\\').replace('"', '\\"')
    variables = json.dumps(variables).replace('\\', '\\\\').replace('"', '\\"')
    res = subprocess.run(["curl", "-H", "Authorization: bearer " + BEARER_TOKEN, "-X", "POST", "-d", '{"query": "' + query + '", "variables": "' + variables + '"}', "https://api.github.com/graphql"], capture_output=True)
    if res.returncode != 0:
        return None
    return json.loads(res.stdout)


def do_remove_label(number):
    return subprocess.run(["curl", "-H", "Authorization: bearer " + BEARER_TOKEN, "-X", "DELETE", f"https://api.github.com/repos/OpenTTD/OpenTTD/issues/{number}/labels/backport%20requested"], capture_output=True)


def do_add_label(number):
    return subprocess.run(["curl", "-H", "Authorization: bearer " + BEARER_TOKEN, "-X", "POST", "-d", '{"labels": ["backported"]}', f"https://api.github.com/repos/OpenTTD/OpenTTD/issues/{number}/labels"], capture_output=True)


def do_command(command):
    return subprocess.run(command, capture_output=True)


def main():
    if len(sys.argv) > 1 and sys.argv[1] == "--mark-done":
        backport_pr = do_query(pr_query, {"number": int(sys.argv[2])})
        if backport_pr is None:
            print("ERROR: couldn't fetch backport PR")
            return

        for line in backport_pr["data"]["repository"]["pullRequest"]["body"].split("\n"):
            if line.startswith("<!-- Backported: "):
                prs = [int(pr) for pr in line.split(":")[1].split(" ")[1].split(",")]

        print("Update labels from backported PRs")
        for pr in prs:
            print(f"- #{pr} ..")
            res = do_remove_label(pr)
            if res.returncode != 0:
                print(f"ERROR: failed to remove label from {pr}")
            res = do_add_label(pr)
            if res.returncode != 0:
                print(f"ERROR: failed to add label to {pr}")

        print("All done")
        return

    dont_push = False
    if len(sys.argv) > 1 and sys.argv[1] == "--dont-push":
        dont_push = True

    resume = None
    resume_i = None
    if os.path.exists(".backport-resume"):
        with open(".backport-resume", "r") as fp:
            resume_str, _, resume_i_str = fp.read().partition(",")
            resume = int(resume_str)
            resume_i = int(resume_i_str)
        print(f"Resuming backporting from {resume}")

    all_prs = do_query(pr_search_query, {"search": "is:closed is:pr label:\"backport requested\" repo:OpenTTD/OpenTTD"})
    if all_prs is None:
        print("ERROR: couldn't fetch all Pull Requests marked for 'backport requested'")
        return

    if not resume:
        do_command(["git", "fetch", "upstream"])
        do_command(["git", "checkout", "upstream/release/" + RELEASE, "-B", "release-backport"])

    for pr in sorted(all_prs["data"]["search"]["edges"], key=lambda x: x["node"]["mergedAt"]):
        if resume:
            if resume != pr['node']['number']:
                continue
            resume = None
            print(f"Merging #{pr['node']['number']}: {pr['node']['title']} (resuming)")
        else:
            print(f"Merging #{pr['node']['number']}: {pr['node']['title']}")

        if next((node for node in pr['node']['labels']['nodes'] if node['name'] == 'backport squash'), None) is not None:
            print(" -> was squashed")
            pr["node"]["commits"]["totalCount"] = 1
            
        for i in range(pr["node"]["commits"]["totalCount"]):
            if resume_i is not None:
                if resume_i != i:
                    continue
                resume_i = None
                continue

            commit = pr["node"]["commits"]["totalCount"] - i - 1
            commit_str = f'{pr["node"]["mergeCommit"]["oid"]}' + "".join(["^"] * commit)

            print(f'  Commit #{i}: {commit_str} ...')

            res = do_command(["git", "cherry-pick", commit_str])
            if res.returncode != 0:
                with open(".backport-resume", "w") as fp:
                    fp.write(str(pr['node']['number']) + "," + str(i))
                print(res.stdout.decode())
                print("")
                print("Cherry-pick failed: please fix the issue manually and run script again.")
                return

    if os.path.exists(".backport-resume"):
        os.unlink(".backport-resume")

    print("")
    print("Done cherry-picking")
    print("Backporting language changes")
    res = do_command(["python3", ".github/backport-languages.py"])
    if res.returncode != 0:
        print("ERROR: backporting language changes failed")
        return
    do_command(["git", "add", "src/lang/*.txt"])
    do_command(["git", "commit", "-m", "Update: Backport language changes"])
    print("Done backporting language changes")
    print("")

    print("Your commit message:")
    print("")

    marker = []

    print("## Description")
    print(f"Backport of all closed Pull Requests labeled as 'backport requested' into `release/{RELEASE}`.")
    for pr in sorted(all_prs["data"]["search"]["edges"], key=lambda x: x["node"]["mergedAt"]):
        print(f"- https://github.com/OpenTTD/OpenTTD/pull/{pr['node']['number']}")
        marker.append(str(pr['node']['number']))
    print("- All language changes")
    print(f"<!-- Backported: {','.join(marker)} -->")

    print("")

    if dont_push:
        print("Done with backport; you can now push this branch to remote:")
        print(" git push -f --set-upstream origin release-backport")
        print("After that, go to this URL:")
    else:
        res = do_command(["git", "push", "-f", "--set-upstream", "origin", "release-backport"])
        if res.returncode != 0:
            print("ERROR: failed to push to remote")
        else:
            print("Pushed to remote")
            print("You can create the PR here:")

    print("  https://github.com/OpenTTD/OpenTTD/compare/release/" + RELEASE + "..." + USERNAME + ":release-backport?expand=1&title=Backport%20master%20into%20release%2f" + RELEASE)


if __name__ == "__main__":
    main()