SourceForge to GitHub migration script for Issues

rest.py

#!/usr/bin/python3

# This script uses the undocumented new issues API of GitHub.
# Read more about it here: https://gist.github.com/jonmagic/5282384165e0f86ef105

import datetime
import pprint
import pytz
import re
import requests
import time

project = "p/pgfplots"
ghproject = "pgf-tikz-bot/pgfplots"
token = "..."  # Generate TOKEN on https://github.com/settings/token

url = "https://api.github.com/repos/{}/import/issues".format(ghproject)
auth = {
    "Authorization": "token {token}".format(token=token),
    "Accept": "application/vnd.github.golden-comet-preview+json",
}


def isotimestamp(string):
    try:
        ts = datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S.%f")
    except:
        ts = datetime.datetime.strptime(string, "%Y-%m-%d %H:%M:%S")
    return ts.isoformat() + "Z"  # Just make it UTC (no idea whether it actually is)


for tracker, trackerlabel in [
    ("bugs", []),
    ("feature-requests", ["Feature Request"]),
    ("support-requests", ["Support Request"]),
    ("patches", ["Patch"]),
]:
    # Get number of bugs
    req = requests.get(
        "https://sourceforge.net/rest/{project}/{tracker}".format(
            project=project, tracker=tracker
        )
    )
    if req.status_code == 401:
        print("Failed to obtain bugs")
        exit(1)
    info = req.json()
    count = info["count"]

    for bug in range(1, count + 1):
        req = requests.get(
            "https://sourceforge.net/rest/{project}/{tracker}/{bug}".format(
                project=project, tracker=tracker, bug=bug
            )
        )
        if req.status_code == 401:
            print("Bug", bug, "doesn't exist")
            continue
        info = req.json()
        ticket = info["ticket"]

        comments = []
        for n, comment in enumerate(ticket["discussion_thread"]["posts"]):
            body = (
                "*Migrated from SourceForge*\n"
                + "https://sourceforge.net/{project}/{tracker}/{bug}/#{slug}\n"
                + "Author: {author}\n"
                + "Timestamp: {timestamp}\n"
            ).format(project=project, tracker=tracker, bug=bug, **comment)
            for attachment in comment["attachments"]:
                body += "\n" + attachment["url"]
            body += "\n"
            body += re.sub("~~~~*", "```", comment["text"])

            if body == "":
                body = "*empty*"
            comments.append(
                {"body": body, "created_at": isotimestamp(comment["timestamp"])}
            )

        body = (
            "*Migrated from SourceForge*\n"
            + "https://sourceforge.net/{project}/{tracker}/{bug}/\n"
            + "Author: {reported_by}\n"
            + "Timestamp: {created_date}\n"
        ).format(project=project, tracker=tracker, bug=bug, **ticket)
        for artifact in ticket["related_artifacts"]:
            body += "\n" + "Related: https://sourceforge.net" + artifact
        for attachment in ticket["attachments"]:
            body += "\n" + attachment["url"]
        body += "\n"
        body += re.sub("~~~~*", "```", ticket["description"])

        status = [] if ticket["status"] in ["open", "closed"] else [ticket["status"]]
        labels = ["bug" if l == "Bug" else l for l in ticket["labels"]]
        issue = {
            "issue": {
                "title": ticket["summary"],
                "labels": status + trackerlabel + labels,
                "closed": ticket["status"].startswith("closed")
                or ticket["status"] == "wont-fix",
                "created_at": isotimestamp(ticket["created_date"]),
                "body": body,
            },
            "comments": comments,
        }

        print("Creating #{bug} in {tracker}".format(bug=bug, tracker=tracker), end="")
        while True:
            try:
                req = requests.post(url=url, headers=auth, json=issue)
                info = req.json()
                issueurl = info["url"]
            except ConnectionResetError:
                print("Timeout. Trying again in 1 second.")
                time.sleep(1)  # Don't retry immediately
                continue
            else:
                break

        # Check status of new issue. Print error and exit or wait until it was
        # posted.
        req = requests.get(issueurl, headers=auth)
        info = req.json()
        while info["status"] != "imported":
            if info["status"] == "failed":
                print(
                    "\nError creating #{bug} in {tracker}".format(
                        bug=bug, tracker=tracker
                    )
                )
                pprint.pprint(info["errors"])
                exit()
            elif info["status"] == "pending":
                print(".", end="")
                time.sleep(0.1)
                req = requests.get(issueurl, headers=auth)
                info = req.json()
        print(
            "\nSuccessfully created #{bug} in {tracker}\n    at {issue_url}".format(
                bug=bug, tracker=tracker, issue_url=info["issue_url"]
            )
        )