Migrate from Pivotal Tracker to GitHub issues

pivotal_to_github.py

import re
import sys
import csv
import json
import urllib2

from datetime import datetime

from collections import defaultdict as dd

DATE_FORMAT = "%m-%d-%Y"
NOTE_FORMAT = re.compile(r"(.*) \((.*) - (.*)\)", re.DOTALL)
P_NUMBER_FORMAT = re.compile(r"\[P#:(\d+)\]")
MONTHS = {'Jan': 1,
          'Feb': 2, 
          'Mar': 3, 
          'Apr': 4, 
          'May': 5, 
          'Jun': 6, 
          'Jul': 7, 
          'Aug': 8, 
          'Sep': 9, 
          'Oct': 10, 
          'Nov': 11, 
          'Dec': 12}

def todatetime(s):
    # Using %b to change the month is harder than expected
    # so I'm doing this the manual way.
    fields = s.split()
    month = MONTHS[fields[0]]
    day = int(fields[1].strip(', '))
    year = int(fields[2])
    return datetime(year, month, day)

def today():
    # can't compare date and datetime so I'm gonna simulate them
    return datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)

def buildauthormap(f):
    d = {}
    for line in open(f, 'rb'):
        if not line.strip(): continue
        
        aliases, credentials = line.strip().split(":")
        for alias in aliases.split(","):
            alias = alias.strip()
            if alias in d:
                raise Exception("%s alias appears in different accounts" % (alias,))
            username, password = credentials.strip().split(",")
            d[alias] = dict(username=username.strip(), password=password.strip())
    d[''] = dict(username="", password="")  # set a default for non assigned tickets
    return d

def togithubissue(d, authormap):
    body = "Pivotal Issue: %s\n\n%s" % (d['URL'][0], d['Description'][0])
    return {'title': "[P#:%s] %s" % (d['Id'], d['Story'][0]),
            'body': body,
            'assignee': authormap[d['Owned By'][0]]['username'],
            'labels': d['Labels']}

def togithubcomment(d):
    body = "%s\n\n%s" % (d['date'].strftime(DATE_FORMAT), d['Note'])
    return {"body": body}

def main(o):
    """
    I run the show.

    * Build a dictionary of PT issues from the csv file.
    * Removes issues that are finished
    * Creates an issue map with the issues already in the github project
    * Creates issues or comments depending on what you picked through arguments

    It doesn't keep track of which comments were created so that one should run
    fine immediately.

    Problem: You either get the passwords of all your users or you'll have to
    agree with them to run the file one by one first on all the issues and then
    on all the comments one by one, or you'll need to change this script to use
    OAuth2. In case you don't have OAuth2 you'll also have issues in comment
    ordering. This is why we leave a date in the comment body.
    """
    heads = None
    issues = {}
    authormap = buildauthormap(o.authormap)
    for i, line in enumerate(csv.reader(open(o.pivotal_file, 'rb'))):
        if not i:
            heads = line
            continue

        d = dd(lambda : [])
        for head, field in zip(heads, line):
            if head == "Note":
                if not field:
                    continue
                body, author, created_date = NOTE_FORMAT.match(field).groups()
                field = {'Note': body,
                         'author': author,
                         'date': todatetime(created_date)}

            d[head].append(field)

        issues[d['Id'][0]] = d

    # Take out closed issues, you can't really created closed issues in github and
    # pivotal tracker doesn't really change the status of tickets that were already
    # done in an iteration except that they have iteration end set.
    issues = dict((id, issue)
                    for id, issue in issues.iteritems()
                      if (not issue['Iteration End'][0] or
                          todatetime(issue['Iteration End'][0]) > today()))

    # Issues are created to have the pivotal tracker id in the title so that we can map
    # created ones to already to be created ones and avoid re-creating them.
    issuesmap = get_issues_map(o.base_url, authormap, o.github_username)
    
    if o.do_comments:
        for i, (issue_id, issue) in enumerate(issues.iteritems()):
            for note in issue['Note']:
                github_comment = togithubcomment(note)
                if not note['author']:
                    raise Exception("Unknown author of comment: %s" % (note,))
                credentials = authormap[note['author']]
                if credentials['password'] == "password":
                    print "Skipping comment to issue %s from user %s" % (
                        issue_id, note['author'])
                    continue

                github_issue_id = issuesmap[issue_id]
                sendcomment(o.base_url, github_comment, credentials, github_issue_id)
    else:
        for i, (issue_id, issue) in enumerate(issues.iteritems()):
            if issue_id in issuesmap:
                continue

            github_issue = togithubissue(issue, authormap)
            if not issue['Requested By'][0]:
                raise Exception("Unknown author of issue: %s" % (github_issue,))
            credentials = authormap[issue['Requested By'][0]]
            if credentials['password'] == "password":
                print "Skipping issue %s from user %s" % (
                    issue_id, issue['Requested By'][0])
                continue
            sendissue(o.base_url, github_issue, credentials)
        
def sendissue(base_url, github_issue, credentials):
    data = json.dumps(github_issue)

    try:
        indata = get_page("POST", base_url, credentials, data)
        print "Created issue", json.loads(indata)[number]

    except Exception, e:
        print e
        return None

def sendcomment(base_url, github_comment, credentials, github_issue_id):
    data = json.dumps(github_comment)

    try:
        indata = get_page("POST", "%s/%s/comments" % (
                base_url, github_issue_id), credentials, data)

    except Exception, e:
        print e
        return None
    
def get_page(method, url, credentials, data=None, and_response=False):
    import httplib
    import base64
    base64string = base64.encodestring('%(username)s:%(password)s' % credentials)[:-1]
    headers = {"Authorization": "Basic %s" % base64string}
    conn = httplib.HTTPSConnection("api.github.com")
    conn.request(method, url, data, headers)
    response = conn.getresponse()
    data = response.read()
    conn.close()
    if response.status != 200:
        raise Exception("Couldn't fetch %s with status %s, response %s" % (
                url, response.status, data))
    if and_response:
        return response, data
    return data

def get_issues_map(base_url, authormap, github_username):
    # find credentials for this query
    for credentials in authormap.values():
        if credentials['username'] == github_username:
            break
    else:
        raise Exception("%s is not in the authormap, can't determine github password" % (
                github_username))

    issues = []
    current_page = 1
    while True:
        response, data = get_page("GET", base_url + "?page=%s" % (current_page,),
                                  credentials, and_response=True)
        issues.extend(json.loads(data))
        
        link_header = response.getheader("Link", None)

        # It appears that github puts the 'last' link unless you're in the last page
        if '; link="last"' not in link_header:
            break

        current_page += 1    

    d = {}
    for issue in issues:
        old_ticket = P_NUMBER_FORMAT.match(issue['title'])
        if not old_ticket:
            continue

        d[old_ticket.groups()[0]] = issue['number']
    return d
        
if __name__ == "__main__":
    from argparse import ArgumentParser

    parser = ArgumentParser(prog="pivotal.py")

    aa = parser.add_argument

    aa("--authormap", action="store", dest="authormap", required=True,
       help=("Map of authors with corresponding username/password for GitHub "
             "Formatted one per line as: 'PT_user_label:gh_username,gh_password' "
             "gh_password should be 'password' if you want to skip the user"))

    aa("--pivotal-file", action="store", dest="pivotal_file",
       help="csv of exported pivotal data", required=True)
    
    aa("--base-github-url", action="store", dest="base_url", required=True,
       help="Base url for the github repo, such as /repos/:user/:repo/issues")

    aa("--do-comments-instead", action="store_true", dest="do_comments",
       help="import comments from pivotal to github")

    aa("--github-username", action="store", dest="github_username", required=True,
       help="Your GitHub username to fetch issues list for a project")
    
    args = parser.parse_args()

    print main(args)

@shyam-habarakada

We've also moved from Pivotal to Github Issues.
Mainly this is to mesh better with our engineering workflow - but it's also cheaper. :)

We now use Zenhub.io to create a Kanban style board for tracking our backlog, current sprint, and items ready for review. Zenhub also allows you to upload ANY file type

SO far, it's ben doable.

Thanks for sharing the script. We moved to GitHub Issues quite a long time ago and it's definitely doable.
I don't want to advertise anything here but tools like our http://overv.io can greatly help to organize the work in similar manner to how PT does it (if you guys want to give it a go we're about to start the second round of private beta testing this week).