Maven Build only Changed

MavenBOC.groovy

import hudson.FilePath
import hudson.model.Hudson

import java.nio.file.Paths
import java.rmi.UnexpectedException

return this;

/**
 * Dope thing to enhance analysis of inter-module dependencies in Maven and getting info about what was changed
 * since last commit in order to enable Build Only Changed functionality
 */
class MavenBOC implements Serializable {

    private static final long serialVersionUID = 1437352647376250L

    String rootPomFile


    String slave

    String workspace

    /**
     * Key - name of module
     * Value - list of other modules or artifacts
     *         (denoted by colon, e.g. openshift is module, :openshift_scrips - artifact)
     */
    Map<String, List<String>> moduleInterDependencyMap

    /**
     * Key - name of module
     * Value - list of root-level directory names
     *
     * This kludge is needed for Sterling projects, where sources live in Foundation directory.
     * Such directories can be configured for modules in ci.boc.nonModuleUsed property as comma-separated list
     * Only root-level directories supported by this class automation
     */
    Map<String, List<String>> nonModuleTriggerList

    /**
     * List of modules, which have ci.boc.skipCI property set to true. They will be excluded from CI-triggered builds
     */
    Set<String> skipCIList

    /**
     * List of modules, which have ci.boc.skipPR property set to true. They will be excluded from PR-triggered builds
     */
    Set<String> skipPRList

    /**
     * Handy list of profiles. Depedening on profile, inter-dependendy map can be adjusted
     */
    List<String> activeProfileList

    MavenBOC(String rootPomFile, List<String> activeProfileList, String slave, String workspace) {
        this.rootPomFile = rootPomFile
        this.slave = slave
        this.workspace = workspace
        this.activeProfileList = activeProfileList
        moduleInterDependencyMap = [:]
        nonModuleTriggerList = [:]
        skipCIList = []
        skipPRList = []
    }

    /**
     * This method should be executed in the first place in order to run BoC.
     * Method flow:
     *  - Get all modules represented in rootPomFile
     *  - For each of module, identify its and parent groupIDs and versions
     *  - For each dependency for in module, analyse, whether there are any
     *    other modules as dependencies. Module dependency will be taken IF:
     *      - groupID is among ${project.groupId}, ${project.parent.groupId} or corresponding values
     *      - version is among ${project.version}, ${project.parent.version} or corresponding values
     *      - so if groupID is among aforementioned values, but version is RELEASE one, module WON'T BE ADDED
     *  - Parse ci.boc.* values from <properties>
     *  - Perform the same for all active profiles
     *  - Set the resulting map to moduleInterDependencyMap
     */
    @NonCPS
    void initializeDependencyMap() {
        def xmlSlurper = new XmlSlurper()
        def pomData = xmlSlurper.parseText(fastReadFile(Paths.get(workspace, rootPomFile) as String))
        def allModules = ['.'] + pomData.modules?.module?.collect { it as String }


        Map<String, String> modulesWithCustomArtifactIdMap = [:]

        allModules.each { module ->
            def moduleData = xmlSlurper.parseText(fastReadFile(Paths.get(workspace, module as String, 'pom.xml').toString()))
            def projectGroupId = moduleData.groupId
            def parentGroupId = moduleData.parent.groupId
            def projectVersion = moduleData.version
            def parentVersion = moduleData.parent.version

            def allowedGID = ['${project.groupId}', projectGroupId, parentGroupId, '${project.parent.groupId}']
            def allowedVersion = ['${project.version}', projectVersion, parentVersion, '${project.parent.version}']

            def condition = { d -> d.groupId in allowedGID && (d.version == null || d.version in allowedVersion) }

            parseCIValues(moduleData, module)

            List<String> dependencyListForModule = []

            def moduleAid = (moduleData.artifactId as String)
            if (moduleAid != module) {
                def name = (moduleAid in allModules) ? moduleAid : ":${moduleAid}"
                dependencyListForModule << name
                modulesWithCustomArtifactIdMap.put(module, name)
            }

            moduleData.dependencies?.dependency?.findAll(condition)?.each { d ->
                def name = (d.artifactId as String)
                dependencyListForModule << ((name in allModules) ? name : ":${name}")
            }


            moduleData.profiles?.profile?.findAll { p -> (p.id as String) in activeProfileList }?.each { p ->

                parseCIValues(p, module)

                p.dependencies?.dependency?.findAll(condition)?.each { d ->
                    def name = (d.artifactId as String)
                    dependencyListForModule << ((name in allModules) ? name : ":${name}")
                }
            }

            moduleInterDependencyMap.put(module as String, dependencyListForModule)
        }

        moduleInterDependencyMap.each { _, v ->
            modulesWithCustomArtifactIdMap.each { customK, customV ->
                if (v.contains(customV)) {
                    v.add(customK)
                }
            }
        }
    }

    /**
     * Recursive search for dependencies of changed modules for example.
     * For each module, method will analyse moduleInterDependencyMap and add its dependencies,
     * then for each of added dependencies operation will repeat, until all the dependencies are added:
     * size of resulting list equals to all mentioned modules and their dependencies. Check will be executed
     * and find any incomplete modules. If none of them added on new iteration, we break it.
     *
     * Let's visualise it:
     * moduleInterDependencyMap = [A: ['B', 'C'], B: [':x', 'D', 'J'], C: [], D:['I'], J:[]]
     * changedModulesList = ['J']
     *
     * 1. ['J']
     * 2. ['J', 'B', ':x', 'D', 'A'] // add B as it's key in moduleInterDependencyMap and add whole list
     * 3. ['J', 'B', ':x', 'D', 'A', 'C', 'I'] // add C as A is key and A is in result and moduleNameList
     *                                         // add ['I'] as D is key and B is in result and moduleNameList
     *
     *
     * Also, there is a 10^3 limit on dependencies as protection from endless loop.
     *
     * @param changedModulesList set of strings of what was changed or wanted to be built
     * @return complete set of strings of modules which are subject to build basing on recursive analysis
     */
    @NonCPS
    Set<String> generateFullDependencyList(Set<String> changedModulesList) {
        Set<String> result = []
        Set<String> moduleNameList = moduleInterDependencyMap.keySet() // understand which modules we have

        result.addAll(changedModulesList) // add all SCM-changed modules
        changedModulesList.each { result.addAll(moduleInterDependencyMap.get(it)) } // add all siblings for SCM-changes
        def i = Math.pow(10, 3)
        while (i != 0) {
            result.findAll { it in moduleNameList } // for all modules which are not artifacts
                    .each {
                // if the module is the key or contained in list of siblings
                // add both key and all the siblings to rebuild
                moduleInterDependencyMap.findAll { k, v -> k == it || v.contains(it) }.each { k, v ->
                    result.add(k)
                    result.addAll(v)
                }
            }

            // break check
            def incomplete = result.findAll { r ->
                r in moduleNameList // for all modules which are not artifacts
            }.findAll { r ->
                def ll = moduleInterDependencyMap.get(r).toSet() // get all siblings
                def check1 = result.intersect(ll).toSet() != ll  // if all siblings already in resulting list

                // inverse: all the modules dependent on this one are in list
                def usedByList = moduleInterDependencyMap.findAll { _, v -> v.contains(r) }.collect { k, _ -> k }
                def check2 = true
                usedByList.each { z ->
                    def l2 = moduleInterDependencyMap.get(z).toSet()
                    check2 = check2 && (result.intersect(l2).toSet() == l2)
                }
                check1 || !check2
            }

            if (incomplete.size()) { // if there are modules which don't match criteria, we should add them and retry
                result.addAll(incomplete)
            } else {
                break // otherwise break it
            }
            i--
        }
        if (!i) {
            throw new UnexpectedException("Unable to build dependency list")
        }

        return result
    }

    /**
     * Method to understand which modules were changed basing on list of files changed in Git
     * Flow:
     *  - If amongst files we find rootPomFile, we rebuild whole thing. No exceptions
     *  - Otherwise, we firstly put modules  (checking that they are modules by moduleInterDependencyMap.keySet())
     *    which denoted by same-name directories
     *  - Finally, we analyse if any root level directories mentioned in any modules ci.boc.nonModuleUsed property
     *    If it's true, add corresponding module
     * @param scmChangedFiles list of files changed by Git
     * @return  list of modules to build (no inter-module dependency analysis yet)
     */
    @NonCPS
    Set<String> getChangedModulesListBySCMFiles(Set<String> scmChangedFiles) {
        Set<String> result = []
        // if root pom was changed, rebuild everything
        if (scmChangedFiles.contains(rootPomFile)) {
            return moduleInterDependencyMap.keySet().toList()
        }
        // for each file change of which was traced in SCM
        scmChangedFiles.each { directory ->
            // if this directory is a module, add as is
            if (directory in moduleInterDependencyMap.keySet()) {
                result << directory
            } else {
                // otherwise add all modules that are requiring this directory (ci.boc.nonModuleUsed)
                result.addAll(nonModuleTriggerList.findAll { _, v -> v.contains(directory)}.collect { k, _ -> k})
            }
        }
        return result
    }

    @NonCPS
    Set<String> excludeModulesForCI(Set<String> currentSelection) {
        return excludeModulesWithRelated(currentSelection, skipCIList)
    }

    @NonCPS
    Set<String> excludeModulesForPR(Set<String> currentSelection) {
        return excludeModulesWithRelated(currentSelection, skipPRList)
    }

    /**
     * Flow:
     *  - Get modules from directories
     *  - Generate module inter-dependency list
     *  - Exclude what is not needed for PR
     * @param scmChangedModules root-level directories was changed in PR
     * @return complete list of modules to build
     */
    @NonCPS
    Set<String> anaylseModulesForPR(Set<String> scmChangedModules) {
        return excludeModulesForPR(generateFullDependencyList(getChangedModulesListBySCMFiles(scmChangedModules)))
    }

    /**
     * Understand what was changed since last commit, using rules:
     * - If there is "Merge:" line in commit metadata, compare between those two commits:
     * https://stackoverflow.com/questions/5072693/how-to-git-show-a-merge-commit-with-combined-diff-output-even-when-every-chang/7335580#7335580
     * - If there is not "Merge:", compare with latest commit (HEAD~)
     * - We do not support rebase properly here
     *
     * @param runtime Pipeline runtime to run shell scripts for Git
     * @return list of root-level directories and/or pom.xml which where changed
     */
    Set<String> analyseSCMChange(runtime) {
        String sha = runtime.sh(returnStdout: true, script: "git rev-parse HEAD").trim()
        runtime.sh("git show -s ${sha} > .tmp; cat .tmp | grep Merge: | cut -d ' ' -f2 > ${workspace}/hisTo")
        runtime.sh("git show -s ${sha} > .tmp; cat .tmp | grep Merge: | cut -d ' ' -f3 > ${workspace}/hisFrom")
        String historyTo = fastReadFile("${workspace}/hisTo").trim()
        String historyFrom = fastReadFile("${workspace}/hisFrom").trim()

        String revisionParameter
        if (historyTo && historyFrom) {
            // merge case
            revisionParameter = "${historyFrom}..${historyTo}"
        } else {
            // squash & merge case
            revisionParameter = "HEAD~"
        }

        return runtime.sh (returnStdout: true, script: "git diff --name-only ${revisionParameter} | cut -d / -f1").tokenize('\n').toSet()
    }

    /**
     * Entry-point for manual/CI builds
     * @param runtime            Pipeline runtime to print out things
     * @param originalModuleList list of selected modules by user (disregarded in case isBoC = true)
     * @param isExpert           expert mode = no scm analysis no inter-module analysis
     * @param isBoC              build-only-changed = scm analysis + inter-module analysis
     * @param isCI               exclude some modules from CI or not
     * @return list of modules to build
     */
    Set<String> runForRegularBuild(Object runtime,
                           List<String> originalModuleList,
                           Boolean isExpert,
                           Boolean isBoC,
                           Boolean isCI) {
        runtime.println "modules = ${originalModuleList}; expert = ${isExpert}; boc = ${isBoC}; isCI = ${isCI}"
        def fullChangedModules = []
        if (isExpert) {
            runtime.println "Expert mode selected, leaving only selected modules: ${originalModuleList}"
            fullChangedModules = originalModuleList
        } else {

            initializeDependencyMap()
            runtime.println "${toString()}"

            def originalChangedModules
            if (isBoC) {
                def gitDiffResult = analyseSCMChange(runtime)
                runtime.println "Root files changed: ${gitDiffResult}"
                originalChangedModules = getChangedModulesListBySCMFiles(gitDiffResult)
                runtime.println "MODULES changed BoC: ${originalChangedModules}"
            } else {
                originalChangedModules = originalModuleList
                runtime.println "MODULES selected: ${originalChangedModules}"
            }

            fullChangedModules = generateFullDependencyList(originalChangedModules.toSet())
            runtime.println "MODULES (including sibling dependencies): ${fullChangedModules}"

            if (isCI) {
                fullChangedModules = excludeModulesForCI(fullChangedModules)
            }
        }

        // this condition is needed for cases of single-module projects
        if (fullChangedModules.size() == 0) {
            fullChangedModules.add(".")
        }


        return fullChangedModules
    }

    @NonCPS
    private String fastReadFile(String path) {
        return (new FilePath(Hudson.instance.getNode(slave).getChannel(), path).readToString() as String)
    }

    /**
     * This method allows to setup required properties
     *
     * nonModuleUsed - list of directories in core, which are required by modules but are not modules
     * skipCI - forcibly disable module for CI
     * skipPR - forciby disable modules for PR
     *
     * @param leaf
     * @param module
     */
    @NonCPS
    private void parseCIValues(Object leaf, String module) {
        def nonModuleUsedDirectories = leaf.properties.'ci.boc.nonModuleUsed' as String
        if (nonModuleUsedDirectories.size()) {
            nonModuleTriggerList.put(module as String, nonModuleUsedDirectories.tokenize(','))
        }
        if (leaf.properties.'ci.boc.skipCI' == true) {
            skipCIList << (module as String)
        }
        if (leaf.properties.'ci.boc.skipPR' == true) {
            skipPRList << (module as String)
        }
    }

    // TODO: likely contains cases when it exclude what actually needed, now protected by -am/-amd options in Maven
    @NonCPS
    private Set<String> excludeModulesWithRelated(Set setToTrim, Set keysToTrim) {
        def completeExclusionList = []
        moduleInterDependencyMap.findAll { k, _ -> k in keysToTrim }.each { k, v ->
            completeExclusionList << k
            completeExclusionList.addAll(v)
        }
        return  (setToTrim - setToTrim.intersect(completeExclusionList as Set) - '.')
    }

    @Override
    String toString() {
        return "MavenBOC{" +
                "rootPomFile='" + rootPomFile + '\'' +
                ", slave='" + slave + '\'' +
                ", workspace='" + workspace + '\'' +
                ", moduleInterDependencyMap=" + moduleInterDependencyMap +
                ", nonModuleTriggerList=" + nonModuleTriggerList +
                ", skipCIList=" + skipCIList +
                ", skipPRList=" + skipPRList +
                ", activeProfileList=" + activeProfileList +
                '}';
    }
}

MavenBOC createInstance(String mvnPomFile, List<String> activeProfileList, String slave, String workspace) {
    return new MavenBOC(mvnPomFile, activeProfileList, slave, workspace)
}