Collect Build Scan publication monthly metrics

collect-monthly-kpi.sh

#!/bin/bash

# Parse CLI arguments
if [[ $# -lt 3 ]]; then
    echo "Usage: $0 <develocity_url> <access_key> <yyyy-mm>"
    exit 1
fi

# Init parameters
readonly develocityUrl=$1
readonly apiKey=$2
readonly yearMonth=$3

# Init global vars
readonly maxBuildsPerBatch=1000
readonly recoveryFile="$0.${yearMonth}.recovery"
readonly outputFile="$0.${yearMonth}.csv"
buildScans=""
batchCount=1
buildCount=1

#############
# functions #
#############

function getBuildScansByFilter() {
    local queryFilter=$1
    
    buildScans=$(curl --silent "${develocityUrl}/api/builds?${queryFilter}&models=gradle-attributes&models=maven-attributes&models=bazel-attributes&models=npm-attributes&models=python-attributes&maxBuilds=${maxBuildsPerBatch}" --header "authorization: Bearer ${apiKey}" | jq -r '.[].models | to_entries | .[].value | select(has("problem") | not) | [ .model.rootProjectName // .model.topLevelProjectName // "Unknown project" , .model.id, .model.buildStartTime, .model.environment.username // .model.user , (.model.values[] | select(.name == "Department") | .value) // "Unknown department", (.model.tags[] | select(.=="CI")) // "Local" ] | @csv')
    if [ "$?" != "0" ]; then
        echo "Failed to retrieve build scans"
        exit 1
    fi
}

function getBuildScansByMonth() {
    local firstDayOfMonth=$(date -d "${yearMonth}-1" +"%Y-%m-%d")
    local lastDayOfMonth=$(date -d"$(date --date="${yearMonth}-1 +1month" +"%Y-%m-01") -1sec" +%F)
    local queryFilter="fromInstant=0&query=buildStartTime:%5B${firstDayOfMonth}%20to%20${lastDayOfMonth}%5D"

    echo "buildStartTime:[${firstDayOfMonth} to ${lastDayOfMonth}]"
    getBuildScansByFilter "${queryFilter}"
}

function getBuildScansFromBuildId() {
    local buildId=$1

    getBuildScansByFilter "fromBuild=${buildId}"
}

########
# main #
########

# check if recovery data are present
recoveryBuildScanId=$(cat "${recoveryFile}" 2>/dev/null)

if [[ -z "${recoveryBuildScanId}" ]]
then
    # Clean output files
    rm -f "${outputFile}"

    # Get first batch of builds
    getBuildScansByMonth "${yearMonth}"
else
    echo "Recovering data from build scan id ${recoveryBuildScanId} (you can trigger a fresh run by deleting ${recoveryFile})"

    # Get first batch of builds
    getBuildScansFromBuildId "${recoveryBuildScanId}"
fi

# Iterate over build scan list
while [ -n "${buildScans}" ]
do 
    echo "Processing batch ${batchCount}"
    currentBuildScanId=""

    while IFS="\n" read buildScan; do
        while IFS="," read buildScanProject buildScanId buildScanDate buildScanUser buildScanDepartment buildScanEnvironment; do
            # Remove quotes from buildScanId
            currentBuildScanId=$(echo "${buildScanId}" | tr -d \")

            echo "Processing build ${buildCount}: (published $(date -d @$(echo ${buildScanDate} | head -c 10))"
            #echo "[${buildScanDepartment} / ${buildScanProject} / ${buildScanUser} / ${currentBuildScanId} / ${buildScanEnvironment}]"

            # save data in output file
            echo "${buildScanDepartment},${buildScanProject},${buildScanUser},${currentBuildScanId},${buildScanEnvironment},${buildScanDate}" >> "${outputFile}"

            # save build scan id for recovery mode
            echo "${currentBuildScanId}" > "${recoveryFile}"

            buildCount=$((buildCount+1))

        done < <(echo "${buildScan}")
    done < <(echo "${buildScans}")

    # Get next batch of builds
    getBuildScansFromBuildId "${currentBuildScanId}"

    batchCount=$((batchCount+1))
done 

# Remove recovery data
rm -f "${recoveryFile}"

echo "Process completed successfully, see results in ${outputFile}"

• Tested on MacOs and Ubuntu
• Requires curl and jq

Usage:

./collect-monthly-kpi.sh <develocity_url> <access_key> <yyyy-mm>