JIRA to Github Issues Migration Script

migrate-jira-to-github-issues.groovy

@Grab(group='com.github.groovy-wslite', module='groovy-wslite', version='1.1.0')
@Grab(group='joda-time', module='joda-time', version='2.7')
import wslite.rest.*
import org.joda.time.*
import org.joda.time.format.*
import groovy.xml.*
import groovy.json.*
import static java.lang.System.*
import groovy.transform.*

def xml = new XmlSlurper()

// The path of the JIRA XML export
entities = xml.parse(new File("data/entities.xml"))
// You should set your Github API token to the GH_TOKEN environment variable
githubToken = getenv('GH_TOKEN')
// configure these variables to modify JIRA source and Github target project
projectToMigrate = 'GRAILS'
repoSlug= 'grails/grails-core'
// if your milestone names use a prefix modify it here
milestonePrefix = "grails-"
jiraDateFormat ='yyyy-MM-dd HH:mm:ss.S'
dateFormatter = ISODateTimeFormat.dateTime()

// Whether to migrate only closed/resolved issues or to also migrate open issues
onlyClosed = true

// Configure how JIRA usernames map to Github usernames
jiraToGibhubAuthorMappings = [
    graemerocher: 'graemerocher',
    pledbrook:'pledbrook',
    brownj:'jeffbrown',
    burtbeckwith:'burtbeckwith',
    wangjammer7:'marcpalmer',
    wangjammer5:'marcpalmer',
    ldaley:'alkemist',
    lhotari:'lhotari',
    fletcherr:'robfletcher',
    pred:'smaldini'
]

def projects = entities.Project.collect {
    new Project(key: it.@originalkey, id: it.@id)
}

urlFragment = "https://api.github.com/repos/$repoSlug"
Project project = projects.find { it.key == projectToMigrate }

if(!githubToken) {
    println "No GH_TOKEN environment variable set"

    exit 1
}

hasHitRateLimit = { response ->
    response.headers['X-RateLimit-Remaining'] && response.headers['X-RateLimit-Remaining'].toInteger() == 0
}

waitOnRateLimit = { response ->
    long sleepTime = response.headers['X-RateLimit-Reset'].toLong() * 1000
    long currentTime = currentTimeMillis()
    while(currentTime < sleepTime) {
        println "Rate Limit Reached! Sleeping until ${new Date(sleepTime)}. Please wait...."
        sleep( sleepTime - currentTime )
        currentTime = currentTimeMillis()
    }

    println "Resuming..."
}

if(project) {
    def projectId = project.id
    def versions = entities.Version.findAll {
        [email protected]() == project.id
    }.collect {
        new Version([email protected](),
                    project,
                    [email protected](),
                    [email protected](),
                    Boolean.valueOf([email protected]()),
                    [email protected]() )
    }.collectEntries {
        [(it.id): it]
    }

    def statuses = entities.Status.collectEntries { status ->
        def name = [email protected]()
        [ ([email protected]()) :
            new Status(name: [email protected]())
        ]
    }

    def components = entities.Component.findAll {
        [email protected]().startsWith("Grails-")
    }.collectEntries { component ->
        [ ([email protected]()): [email protected]() ]
    }

    def priorities = entities.Priority.collectEntries { priority ->
        [ ([email protected]()): [email protected]() ]
    }

    def resolutions = entities.Resolution.collectEntries { resolution ->
        [ ([email protected]()): [email protected]() ]
    }

    def issueTypes = entities.IssueType.collectEntries { issueType ->
        [ ([email protected]()): [email protected]() ]
    }



    println "Statuses: ${statuses.values()*.name}"
    println "Priorities: ${priorities.values()}"
    println "Resolutions: ${resolutions.values()}"
    println "Issue Types: ${issueTypes.values()}"

    // First read existing Milestone data
    def milestones = [:]

    def milestoneData = new RESTClient("$urlFragment/milestones?state=all")
                                .get(headers:[Authorization: "token $githubToken"])
                                .json

    int page = 1
    while(milestoneData) {


        for(m in milestoneData) {
            milestones[m.title] = m.number
        }
        page++
        milestoneData = new RESTClient("$urlFragment/milestones?state=all&page=$page")
                                    .get(headers:[Authorization: "token $githubToken"])
                                    .json

    }

    for(version in versions.values()) {
        def milestoneTitle = "${milestonePrefix}${version.name}".toString()
        def existingNumber = milestones[milestoneTitle]

        // if the milestone already exists just populate it
        if(existingNumber) {
            version.milestoneId = existingNumber
        }
        else {
            // otherwise create a new milestone for the version
            println "Creating Milestone: $version"
            def client = new RESTClient("$urlFragment/milestones")
            try {
                def response = client.post(headers:[Authorization: "token $githubToken"]) {
                    json    title: milestoneTitle,
                            description: version.description,
                            state: version.released ? 'closed' : 'open',
                            due_on: dateFormatter.print( new DateTime(version.releaseDate ?: new Date()) )
                }

                version.milestoneId = response.json.number.toInteger()
                if(response.statusCode == 200 || response.statusCode == 201) {
                    println "Milestone Created $version"
                }
                else {
                    println "Error occurred: ${response.statusCode}"
                    println response.json.toString()
                }
            }
            catch(RESTClientException e) {
                println "Error occurred Creating Milestone: ${e.response.statusCode}"
                println e.response.contentAsString
                if ( hasHitRateLimit(response) ) {

                    waitOnRateLimit(response)

                    try {
                        def response = client.post(headers:[Authorization: "token $githubToken"]) {
                            json    title: milestoneTitle,
                                    description: version.description,
                                    state: version.released ? 'closed' : 'open',
                                    due_on: dateFormatter.print( new DateTime(version.releaseDate ?: new Date()) )
                        }

                        version.milestoneId = response.json.number.toInteger()
                    }
                    catch(RESTClientException e2) {
                        // no further attempts
                        println "Error occurred Creating Milestone: ${e2.response.statusCode}"
                        println e2.response.contentAsString

                    }
                }

            }
        }
    }


    def nodeAssociations = entities.NodeAssociation
    def issues = entities.Issue.findAll {
        [email protected]() == project.id
    }.collect {
        // to obtain fix version and milestone version
        // <NodeAssociation sourceNodeId="31735" sourceNodeEntity="Issue" sinkNodeId="10995" sinkNodeEntity="Version" associationType="IssueFixVersion"/>

        def dateCreated
        if( it.@created ) {
            try {
                dateCreated = new Date().parse(jiraDateFormat, [email protected]())
            } catch(e) {
                // ignore
            }

        }

        // create base issue data
        def issue = new Issue(
                id: [email protected](),
                jiraKey: [email protected](),
                reporter: [email protected](),
                assignee: [email protected](),
                project: project,
                summary: [email protected](),
                environment: [email protected](),
                description: it.description.text(),
                priority: priorities[[email protected]()],
                type: issueTypes[[email protected]()],
                status: statuses[[email protected]()],
                resolution: resolutions[[email protected]()],
                created: dateCreated
        )

        def votes = [email protected]()
        if(votes) {
            issue.popular = votes.toInteger() > 9
        }

        def versionId = nodeAssociations.find {
            [email protected]() == issue.id && [email protected]() == 'Issue' && [email protected]() == "IssueVersion"
        }?.@sinkNodeId?.text()
        def fixVersionId = nodeAssociations.find {
            [email protected]() == issue.id && [email protected]() == 'Issue' && [email protected]() == "IssueFixVersion"
        }?.@sinkNodeId?.text()


        issue.version = versions[versionId]
        issue.fixVersion = versions[fixVersionId]

        // parse issue comments
        issue.comments = entities.Action.findAll {
            ([email protected]() == issue.id)  && (it.@type == "comment")
        }.collect {
            def commentCreated

            try {
                commentCreated = new Date().parse(jiraDateFormat, [email protected]())
            } catch(e) {
                commentCreated = new Date()
            }


            new Comment(id: [email protected](),
                        author: [email protected](),
                        body:  [email protected]() ?: it.body.text(),
                        created: commentCreated)
        }.sort {
            it.created
        }

        println "Created Issue Object for Issue: ${issue.jiraKey}"

        if( onlyClosed && !issue.status.closed && !issue.popular) {
            // we're only migrating historically closed issues and issues with significant votes
            return issue
        }
        println "Publishing Issue: ${issue.jiraKey}"
        try {
            def searchClient = new RESTClient("https://api.github.com/search/issues?q=repo:${repoSlug}+${issue.jiraKey}")

            def searchResults = searchClient.get(headers:[Authorization: "token $githubToken"]).json
            def issueExists = 0 < searchResults.total_count ?: 0

            if(issueExists) {
                if( searchResults.items[0].title.contains(issue.jiraKey) ) {
                    println "Issue ${issue.jiraKey} already exists, skipping..."

                    return issue
                }
            }

        }
        catch(RESTClientException e) {
            // probably hit the rate limit
            println "Error occurred searching for existing issue: ${e.response.statusCode}"
            println e.response.contentAsString

            if ( hasHitRateLimit(e.response) ) {
                waitOnRateLimit(e.response)
            }
        }

        def client = new RESTClient("$urlFragment/import/issues")

        def labels = []
        def comments = []
        def assignee = jiraToGibhubAuthorMappings[issue.assignee]


        if(issue.resolution) {
            labels << issue.resolution
        }
        if(issue.type) {
            labels << issue.type
        }
        if(issue.priority) {
            labels << issue.priority
        }

        if(issue.comments) {
            for(comment in issue.comments) {
                if(comment.body.trim()) {
                    comments << [
                        created_at:  dateFormatter.print( new DateTime( comment.created ) ),
                        body: """$comment.author said:
    $comment.body"""
                    ]

                }
            }
        }

        def issueJson = [
            title: "${issue.jiraKey}: ${issue.summary}",
            body: """
Original Reporter: ${issue.reporter}
Environment: ${issue.environment ?: 'Not Specified'}
Version: ${issue.version?.name ?: 'Not Specified'}
Migrated From: http://jira.grails.org/browse/${issue.jiraKey}

${issue.description}""",
            created_at: dateFormatter.print( new DateTime( issue.created ) ),
            closed: issue.resolution ? true : false,
            labels: labels
        ]

        if(assignee) {
            issueJson.assignee = assignee
        }
        if(issue.fixVersion) {
            issueJson.milestone = issue.fixVersion.milestoneId
        }

        try {
            def response = client.post(headers:[Authorization: "token $githubToken",
                                                Accept: "application/vnd.github.golden-comet-preview+json"]) {
                json(
                    issue: issueJson,
                    comments:comments
                )
            }

            println "Issue Created. API Limit: ${response.headers['X-RateLimit-Remaining']}"
        }
        catch(RESTClientException e) {
            println "Error occurred: ${e.response.statusCode}"
            println e.response.contentAsString

            if ( hasHitRateLimit(e.response) ) {

                waitOnRateLimit(e.response)

                try {
                    client.post(headers:[Authorization: "token $githubToken",
                                                        Accept: "application/vnd.github.golden-comet-preview+json"]) {
                        json(
                            issue: issueJson,
                            comments:comments
                        )
                    }
                    println "Issue Created."
                }
                catch(RESTClientException e2 ) {
                    println "Error occurred: ${e2.response.statusCode}"
                    println e2.response.contentAsString

                }
            }
        }
        return issue
    }



    println "Issue Migration Complete."
}
else {
    println "Project not found"
    exit 1
}




// Model Classes
@ToString
class Project {
    String key
    String id
}

@ToString
class Version {

    String id
    Project project
    String name
    String description
    boolean released
    Date releaseDate
    int milestoneId

    Version(String id, Project project, String name, String description, boolean released = false, String releaseDate = null) {
        this.id = id
        this.project = project
        this.name = name
        this.description = description
        this.released = released
        if(releaseDate) {
            this.releaseDate = new Date().parse('yyyy-MM-dd HH:mm:ss.S', releaseDate)
        }
    }

}

@ToString
class Issue {
    String id
    String jiraKey
    String reporter
    String assignee
    Project project
    String summary
    String environment
    String description
    String priority
    String type
    Status status
    String resolution
    Date created
    Version version
    Version fixVersion
    boolean popular

    Collection<String> components = []
    Collection<Comment> comments = []
}

@ToString
class Comment {
    String id
    String author
    Date created
    String body
}

@ToString
class Status {
    String name

    boolean isClosed() {
        name == "Closed" || name == "Resolved"
    }
}