My Jenkins pipeline

Jenkinsfile.cd

#!groovy

// Load the dsbjenkins library for some Jenkins pipeline utilities
@Library('[email protected]') _

// Start the pipeline
pipeline {

    // Run this pipeline in the dmsbuildsys docker agent
    agent { label 'DMS-2019.2-DMSBUILDSYS' }

    // Jenkins Job Options
    options {
        // makes Jenkins "Console Output" colorized
        ansiColor('xterm')

        // Adds timestamps to the "Console Output"
        timestamps()

        // Configures the discarding of old builds
        buildDiscarder(logRotator(artifactDaysToKeepStr: '', artifactNumToKeepStr: '', daysToKeepStr: '365', numToKeepStr: ''))

        // This option skips the automatic 'checkout scm' on each agent.
        // Skipping it allows us to do a custom checkout.
        skipDefaultCheckout true

        // Do NOT allow concurrent builds, since it will break versioning
        disableConcurrentBuilds()
    }

    // Configure job environment
    environment {
        // timestamp for the build
        BUILD_TIMESTAMP = getTimestamp()

        // this path is baked into the docker images
        JAVA_HOME = "/usr/local/Java/jdk11.0.1-64"

        RELEASE_ARCHITECTURE = "x86_64"

        // artifactory server and repositories
        ARTIFACTORY_SERVER_ID = 'bytesalad'
        DEV_RPM_REPO = 'dms-rpm-dev'
        TST_RPM_REPO = 'dms-rpm-test'
        OPS_RPM_REPO = 'dms-rpm-ops'
        DEV_PKG_REPO = 'dms-software-releases-dev'
        TST_PKG_REPO = 'dms-software-releases-test'
        OPS_PKG_REPO = 'dms-software-releases-ops'

        // JIRA config
        JIRA_SITE = 'STScI JIRA'
        JIRA_FAIL_ON_ERROR = false
        JIRA_ISSUE_PATTERN = "\\b(DADS|DADSCOMMON)-\\d+"
        JIRA_RELEASE_NOTES_BASE_CLAUSE = '\
project in (DADS, DADSCOMMON) and (issuetype not in (subTaskIssueTypes()) \
and status not in (Open, "In Progress", "In Review", Rejected, "On Hold"))'
    }

    // Job stages
    stages {
        stage('Initialization') {
            steps {
                // talk about it
                notifySlack "STARTED", "#dsb_rss"

                // clean out the workspace
                cleanWs()

                script {
                    // checkout the revision we're building and get the git info
                    GIT_SCM = checkout(scm)

                    // store some properties for this product, to be ammended later
                    // store the git properties separate since we have multiple,
                    // somewhat related products to publish
                    GIT_PROPS = [
                            'git.branch': GIT_SCM.GIT_BRANCH,
                            'git.commit': GIT_SCM.GIT_COMMIT,
                            'git.abbrevcommit': GIT_SCM.GIT_COMMIT[0..7],
                            'git.url': GIT_SCM.GIT_URL
                    ]
                    PROPS = [:]
                    GIT_PROPS.each { k, v -> PROPS[k] = v }

                    // configure an artifactory server for use later
                    ARTIFACTORY_SERVER = Artifactory.server "${env.ARTIFACTORY_SERVER_ID}"

                    def gradleProps = readProperties file: 'gradle.properties'
                    PROPS['mission'] = "${gradleProps['systemProp.dads.mission']}"
                    PROPS['rpm.metadata.name'] = "DADS-${PROPS['mission']}"
                    PROPS['rpm.metadata.version'] = [
                            "${gradleProps['systemProp.dads.version.major']}",
                            "${gradleProps['systemProp.dads.version.minor']}",
                            "${gradleProps['systemProp.dads.version.patch']}"].join('.')

                    // NOTE: We track SID and HST under DADS-<version>, NOT DADS-<mission>-<version>!!!
                    //       Hence, the hard-coded 'DADS' below.
                    PROPS['jira.fixversion'] = "DADS-${PROPS['rpm.metadata.version']}"
                }
            }
        }

        // if this is a release branch, verify that it's name matches the Jira fix version
        stage('Verify Release Branch') {
            when {
                branch 'releases/*'
                expression { !(PROPS['git.branch'] ==~ /releases\/${PROPS['jira.fixversion']}$/ ) }
            }
            steps {
                error "Release branch name, ${PROPS['git.branch']}, is not consistent with version in gradle.properties: ${PROPS['jira.fixversion']}"
            }
        }

        stage('Transition Jira Issues') {
            when { anyOf { branch 'master'; branch 'releases/*' } }
            steps {
                script {
                    transitionJiraIssuesAndAddFixVersion(
                            jiraTransitions: ['Done', 'Resolve Issue', 'Change to Ready To Test', 'RTT'],
                            jiraIssuePattern: "${env.JIRA_ISSUE_PATTERN}",
                            fixVersion: "${PROPS['jira.fixversion']}")
                }
            }
        }

        stage('Versioning') {
            when { anyOf { branch 'master'; branch 'releases/*' } }
            steps {
                script {
                    PROPS['rpm.metadata.release'] = getNextReleaseForVersion(
                            artifactProperties: PROPS.subMap(['rpm.metadata.name', 'rpm.metadata.version']),
                            artifactoryRepository: "${env.DEV_RPM_REPO}",
                            releasePropertyName: 'rpm.metadata.release',
                            releaseGitHashSplitStr: '~',
                            gitRevision: PROPS['git.abbrevcommit'],
                            artifactoryServerId: "${env.ARTIFACTORY_SERVER_ID}")

                    PROPS['package.buildid'] = [
                            PROPS['rpm.metadata.version'],
                            PROPS['rpm.metadata.release']].join('-')

                    PROPS['package.fullname'] = [
                            'DADS',
                            PROPS['rpm.metadata.version'],
                            PROPS['rpm.metadata.release']].join('-')

                    // populate the gradle properties so we don't have to think about it
                    sh("sed -i -e 's/systemProp\\.dads\\.release\\s*=.\\+/systemProp.dads.release=${PROPS['rpm.metadata.release']}/' gradle.properties")

                    // get all the Jira issues that we care about
                    BUILD_ISSUES = jiraJqlSearch jql: "(${JIRA_RELEASE_NOTES_BASE_CLAUSE}) and fixVersion in (\"${PROPS['jira.fixversion']}\") order by key asc"

                    // generate Jira release notes for inclusion in RPM
                    def context = [
                            "PRODUCT_VERSION": PROPS['rpm.metadata.version'],
                            "PRODUCT_RELEASE": PROPS['rpm.metadata.release'],
                            "BUILD_TIMESTAMP": BUILD_TIMESTAMP,
                            "BUILD_ISSUES": BUILD_ISSUES
                    ]
                    def templateText = readFile file: 'cm/rpmReleaseNotesTemplate.groovy'
                    def filledTemplate = populateGroovyTemplate(templateText, context)
                    writeFile file: 'pkg/release_notes', text: filledTemplate.toString()
                }
            }
        }

        stage('Build & Test') {
            steps {
                // -parallel breaks the maven dependency checking
                // not starting with a fresh copy?
                sh "./gradlew --stacktrace --no-daemon clean buildRPM generateJavaDocs dependencyCheckAnalyze"
                // this runs the code coverage report, robot is broken
                sh "./cm/run_tests_in_docker"
                sh "mv dist dist-${PROPS['mission']}"
            }
            post {
                always {
                    // publish unit test results
                    junit healthScaleFactor: 100, keepLongStdio: true, testResults: 'build/*/test-results/unitTest/*.xml'

                    // publish jacoco report (with graphs and stuff)
                    jacoco classPattern: 'build/**/classes', exclusionPattern: '**/*Tests.class', \
                            execPattern: 'build/**/*.exec', inclusionPattern: '**/*.class', sourcePattern: 'src'

                    // publish fancy html from jacoco and gradle
                    publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: false, reportDir: 'build/reports',
                            reportFiles: 'unitTests/index.html,codeCoverageReport/html/index.html', reportName: 'Fancy Report',
                            reportTitles: 'Unit Test Results,Code Coverage'])

                    // publish dbDoc from liquibase
                    publishHTML([allowMissing: false, alwaysLinkToLastBuild: false, keepAll: false, reportDir: 'cm/dbDoc',
                            reportFiles: 'fsb/docs/index.html,archive/docs/index.html', reportName: 'DB Documentation',
                            reportTitles: 'FSB Database Documentation,AC Database Documentation'])

                    // publish the javadocs
                    step([$class: 'JavadocArchiver', javadocDir: 'javadoc', keepAll: false])

                }
            }
        }

        // run sonarqube analysis
        stage('SonarQube') {
            environment {
                // sonar-scanner tool
                SONAR_SCANNER_HOME = "${tool 'SonarQube Scanner 4.1.0'}"
            }
            steps {
                // run bandit + sonarqube
                withSonarQubeEnv('DMD SonarQube') {
                    // needs to be put into a script to check branch name
                    script {
                        def sonarScannerOpts = [
                            '-X',
                            '-Dsonar.projectKey=DADS-AEI',
                            '-Dsonar.sources=src',
                            '-Dsonar.java.binaries=build/**/classes/java/main',
                            '-Dsonar.login=6b908108fbc5508d26740f1c351dd1f317fbc90a',
// this should work but it doesn't?
//                            '-Dsonar.java.libraries=dist/lib/*.jar',
                            '-Dsonar.coverage.jacoco.xmlReportPaths=build/jacoco/jacocoMerge.xml',
                            '-Dsonar.dependencyCheck.reportPath=build/**/reports/dependency-check-report.xml',
                            '-Dsonar.dependencyCheck.htmlReportPath=build/**/reports/dependency-check-report.html'
                        ]

                        // parse the branch to see if this is a Gerrit change and run sonar scanner accordingly
                        if (GIT_SCM.GIT_BRANCH ==~ /[0-9]{2}\/[0-9]+\/[0-9]+/) {
                            sonarScannerOpts += [
                                "-Dsonar.pullrequest.branch=${GIT_SCM.GIT_BRANCH}",
                                "-Dsonar.pullrequest.key=${GIT_SCM.GIT_BRANCH}"
                            ]
                        } else {
                            sonarScannerOpts += [
                                "-Dsonar.branch.name=${GIT_SCM.GIT_BRANCH}"
                            ]
                        }
                        sh "${SONAR_SCANNER_HOME}/bin/sonar-scanner ${sonarScannerOpts.join(' ')}"

                    }
                }
                // FAIL the Jenkins build if sonarqube analysis fails the quality gate (set on sonarqube webserver)
                timeout(time: 2, unit: 'MINUTES') {
                    waitForQualityGate abortPipeline: true
                }
            }
        }

        stage('Build SID') {
            when { anyOf { branch 'master'; branch 'releases/*' } }
            steps {
                // build sid RPMs
                sh "./gradlew -Ddads.mission=SID --stacktrace --parallel clean buildRPM && mv dist dist-SID"
            }
        }

        stage('Artifactory Deploy') {
            when { anyOf { branch 'master'; branch 'releases/*' } }
            steps {
                // push artifacts to artifactory
                script {
                    // create build info for publishing
                    BUILD_INFO = Artifactory.newBuildInfo()
                    BUILD_INFO.env.capture = true
                    BUILD_INFO.number = "${PROPS['package.buildid']}"

                    // publish the RPMs separately by mission
                    ['HST', 'SID'].each { mission ->
                        PROPS['mission'] = mission
                        PROPS['rpm.metadata.name'] = "DADS-${PROPS['mission']}"
                        publishFilesToArtifactory(
                                fileGlobRegex: "dist-${mission}/rpm/RPMS/${env.RELEASE_ARCHITECTURE}/DADS-${mission}-.*rpm",
                                targetRepoPath: "${env.DEV_RPM_REPO}/el/7/${env.RELEASE_ARCHITECTURE}/Packages/",
                                properties: PROPS,
                                buildInfo: BUILD_INFO,
                                publishBuildInfo: false,
                                artifactoryServerId: "${env.ARTIFACTORY_SERVER_ID}")
                    }

                    // publish the OneGUI and BuildInfo
                    publishFilesToArtifactory(
                            fileGlobRegex: "dist-HST/dadsgui/DADS-.*-OneGUI.*",
                            targetRepoPath: "${env.DEV_PKG_REPO}/dms/multimission/DADS/OneGUI/${PROPS['rpm.metadata.version']}/",
                            properties: GIT_PROPS,
                            buildInfo: BUILD_INFO,
                            publishBuildInfo: true,
                            artifactoryServerId: "${env.ARTIFACTORY_SERVER_ID}")
                }
            }
        }

        stage('HST-CI Deploy') {
            // ONLY deploy to HST-CI for master branch
            when { branch 'master' }
            steps {
                script {
                    // query artifactory for the new artifact to get the properties to pass to
                    // the DMS Deploy job below - ONLY HST FOR NOW
                    // NOTE: we have to wait for the **ARTIFACTORY CALCULATED PROPERTIES**
                    //       that are calculated by the yum repository indexer, otherwise
                    //       we end up with half-baked properties
                    PROPS['mission'] = "HST"
                    PROPS['rpm.metadata.name'] = "DADS-${PROPS['mission']}"
                    def searchProps = PROPS.subMap([
                            'rpm.metadata.name',
                            'rpm.metadata.version',
                            'rpm.metadata.release']) + ['rpm.metadata.arch': "${env.RELEASE_ARCHITECTURE}"]
                    ARTIFACT_PROPS = getArtifactPropertiesFromArtifactory(
                            artifactProperties: searchProps,
                            artifactoryRepository: "${env.DEV_RPM_REPO}",
                            artifactoryServerId: "${env.ARTIFACTORY_SERVER_ID}")
                }

                // Trigger downstream job to deploy to CI-string, wait for results, and fail if it fails
                // NOTE: THIS ONLY APPLIES TO HST CURRENTLY
                build job: '../HST DMS Deploy', parameters: [string(name: 'DEPLOY_ENVIRONMENT', value: 'HST-CI'),
                        string(name: 'DEPLOYMENTS', value: 'DADS'),
                        string(name: 'DADS_VERSION', value: ARTIFACT_PROPS)],
                        propagate: true, wait: true
            }
        }

        stage('Artifact Promotion') {
            // WARNING: Promotion to test for releases/* branches are "blind", meaning
            //          they could not have been tested on the HST-CI cluster
            when { anyOf { branch 'master'; branch 'releases/*' } }
            steps {
                // push artifacts to artifactory
                script {
                    // define a new build info to workaround artifactory bug:
                    //   - https://www.jfrog.com/jira/browse/RTFACT-18781
                    BUILD_INFO = Artifactory.newBuildInfo()
                    BUILD_INFO.env.capture = true
                    BUILD_INFO.number = "${PROPS['package.buildid']}"

                    // publish the RPMs separately by mission
                    ['HST', 'SID'].each { mission ->
                        PROPS['mission'] = mission
                        PROPS['rpm.metadata.name'] = "DADS-${PROPS['mission']}"
                        publishFilesToArtifactory(
                                fileGlobRegex: "dist-${mission}/rpm/RPMS/${env.RELEASE_ARCHITECTURE}/DADS-${mission}-.*rpm",
                                targetRepoPath: "${env.TST_RPM_REPO}/el/7/${env.RELEASE_ARCHITECTURE}/Packages/",
                                properties: PROPS,
                                buildInfo: BUILD_INFO,
                                publishBuildInfo: false,
                                artifactoryServerId: "${env.ARTIFACTORY_SERVER_ID}")
                    }

                    // publish the OneGUI and BuildInfo
                    publishFilesToArtifactory(
                            fileGlobRegex: "dist-HST/dadsgui/DADS-.*-OneGUI.*",
                            targetRepoPath: "${env.TST_PKG_REPO}/dms/multimission/DADS/OneGUI/${PROPS['rpm.metadata.version']}/",
                            properties: GIT_PROPS,
                            buildInfo: BUILD_INFO,
                            publishBuildInfo: true,
                            artifactoryServerId: "${env.ARTIFACTORY_SERVER_ID}")

                    // If we get here, then we have a build that is available for deployment
                    // to Test. So we should update the display name of the jenkins job to
                    // reflect the version.
                    currentBuild.displayName = "${PROPS['package.buildid']}"

                    // we should also keep the build forever
                    keepBuildForever
                }
                // Deployment to Test/Ops is triggered manually using the DMS Deploy Jenkins job.
                // It determines the available artifacts to deploy from artifactory directly (active choices plugin)
            }
        }

        stage('Release Promotion') {
            // Promote builds on releases/* branches to Ops repository automatically
            // WARNING: Promotion to ops for releases/* branches are "blind", meaning
            //          they could not have been tested on the JWST-CI-string
            when { branch 'releases/*' }
            steps {
                // push artifacts to artifactory
                script {
                    // define a new build info to workaround artifactory bug:
                    //   - https://www.jfrog.com/jira/browse/RTFACT-18781
                    BUILD_INFO = Artifactory.newBuildInfo()
                    BUILD_INFO.env.capture = true
                    BUILD_INFO.number = "${PROPS['package.buildid']}"

                    // publish the RPMs separately by mission
                    ['HST', 'SID'].each { mission ->
                        PROPS['mission'] = mission
                        PROPS['rpm.metadata.name'] = "DADS-${PROPS['mission']}"
                        publishFilesToArtifactory(
                                fileGlobRegex: "dist-${mission}/rpm/RPMS/${env.RELEASE_ARCHITECTURE}/DADS-${mission}-.*rpm",
                                targetRepoPath: "${env.OPS_RPM_REPO}/el/7/${env.RELEASE_ARCHITECTURE}/Packages/",
                                properties: PROPS,
                                buildInfo: BUILD_INFO,
                                publishBuildInfo: false,
                                artifactoryServerId: "${env.ARTIFACTORY_SERVER_ID}")
                    }

                    // publish the OneGUI and BuildInfo
                    publishFilesToArtifactory(
                            fileGlobRegex: "dist-HST/dadsgui/DADS-.*-OneGUI.*",
                            targetRepoPath: "${env.OPS_PKG_REPO}/dms/multimission/DADS/OneGUI/${PROPS['rpm.metadata.version']}/",
                            properties: GIT_PROPS,
                            buildInfo: BUILD_INFO,
                            publishBuildInfo: true,
                            artifactoryServerId: "${env.ARTIFACTORY_SERVER_ID}")
                }
            }
        }

        stage('Create Confluence Release Notes') {
            when { anyOf { branch 'master'; branch 'releases/*' } }
            steps {
                script {
                    // we are replacing the python template filler with this groovy one
                    // which gives us more leeway in defining the template
                    // we still have to use the stupid filenames that createConfluencePagesForBuild
                    // expects, so we fudge it.
                    def context = [
                        "PRODUCT_NAME": 'DADS',
                        "PRODUCT_VERSION": PROPS['rpm.metadata.version'],
                        "PRODUCT_RELEASE": PROPS['rpm.metadata.release'],
                        "BUILD_TIMESTAMP": "${env.BUILD_TIMESTAMP}",
                        "BUILD_URL": "${env.BUILD_URL}",
                        "BUILD_INFO": BUILD_INFO,
                        "ARTIFACTORY_SERVER": ARTIFACTORY_SERVER,
                        "BUILD_ISSUES": BUILD_ISSUES,
                        "ISSUE_BASE_URL": jiraGetServerInfo().data.baseUrl + '/browse'
                    ]
                    def templateText = readFile file: 'cm/confluenceReleaseNotesTemplate.groovy'
                    def filledTemplate = populateGroovyTemplate(templateText, context)
                    writeFile file: 'cm/DADS_confluence_build_tag_template.xml', text: filledTemplate.toString()

                    // Create Confluence pages for this release
                    def utils = new edu.stsci.dsb.jenkins.utilities()
                    utils.createConfluencePagesForBuild('DADS', PROPS['jira.fixversion'],
                            PROPS['package.fullname'], 'NA', "cm", [])
                }
            }
        }
    }

    // post-op
    post {
        success {
            notifySlack "SUCCESS", "#dsb_rss"
            gerritReview labels: [Verified: 1], message: "SUCCESS: ${env.BUILD_URL}"
        }
        failure {
            notifySlack "FAILURE", "#dsb_rss"
            gerritReview labels: [Verified: -1], message: "FAILURE: ${env.BUILD_URL}"
            emailext body: "FAILURE: Job ${env.JOB_NAME} build ${env.BUILD_NUMBER}\nMore info at: ${env.BUILD_URL}\n\n\${BUILD_LOG}",
                recipientProviders: [developers(), requestor(), culprits()],
                subject: "FAILURE: ${env.JOB_NAME}"
        }
        unstable {
            notifySlack "UNSTABLE", "#dsb_rss"
            gerritReview labels: [Verified: -1], message: "UNSTABLE: ${env.BUILD_URL}"
            emailext body: "UNSTABLE: Job ${env.JOB_NAME} build ${env.BUILD_NUMBER}\nMore info at: ${env.BUILD_URL}\n\n\${BUILD_LOG}",
                recipientProviders: [developers(), requestor(), culprits()],
                subject: "UNSTABLE: ${env.JOB_NAME}"
        }
        fixed {
            emailext body: "FIXED: Job ${env.JOB_NAME} build ${env.BUILD_NUMBER}\nMore info at: ${env.BUILD_URL}\n",
                recipientProviders: [developers(), requestor(), culprits()],
                subject: "FIXED: ${env.JOB_NAME}"
        }
        always {
            // activate chuck
            chuckNorris()
        }
    }
}