Collect git repository URLs having non readcted usernames

collect-git-repository-urls.sh

#!/bin/bash

# Parse CLI arguments
if [[ $# -lt 3 ]]; then
    echo "Usage: $0 <develocity_url> <access_key> <yyyy-mm>"
    exit 1
fi

# Init parameters
readonly develocityUrl=$1
readonly apiKey=$2
readonly daysAgo=$3

# Init global vars
readonly maxBuildsPerBatch=1000
readonly recoveryFile="$0.recovery"
readonly outputFile="$0.csv"
fromInstantInMs=0
buildScans=""
buildCount=1

#############
# functions #
#############

function getFromInstantByDaysAgo() {
    local currentInstant=$(date +%s)
    fromInstantInMs=$((${currentInstant}*1000 - ${daysAgo}*60*60*24*1000))
}

function getBuildScansByFilter() {
    local queryFilter=$1

    buildScans=$(curl "${develocityUrl}/api/builds?${queryFilter}&models=gradle-attributes&models=maven-attributes&maxBuilds=${maxBuildsPerBatch}" --header "authorization: Bearer ${apiKey}" | jq -r '.[].models | to_entries | .[].value | select(has("problem") | not) | {id: .model.id, project: (.model.rootProjectName // .model.topLevelProjectName // "Unknown project"), buildStartTime: .model.buildStartTime, user: (.model.environment.username // .model.user), repository: ((.model.values[] | select(.name == "Git repository") | .value) // "Unknown repository")} | [.id, .buildStartTime, .project, .user, .repository] | @csv')
    if [ "$?" != "0" ]; then
        echo "Failed to retrieve build scans"
        exit 1
    fi
}

function getBuildScansFromInstant() {
    local instant=$1
    echo "Fetching builds fromInstant=${instant} => $(date -d @$(echo ${fromInstantInMs} | head -c 10))"
    getBuildScansByFilter "fromInstant=${instant}"
}

function getBuildScansFromBuildId() {
    local buildId=$1
    echo "Fetching builds fromBuild=${buildId}"
    getBuildScansByFilter "fromBuild=${buildId}"
}

########
# main #
########

# check if recovery data are present
recoveryBuildScanId=$(cat "${recoveryFile}" 2>/dev/null)

if [[ -z "${recoveryBuildScanId}" ]]
then
    # Clean output files
    rm -f ${outputFile}
    rm -f ${workFile}

    # Get current timestamp
    getFromInstantByDaysAgo

    # Get first batch of builds
    getBuildScansFromInstant ${fromInstantInMs}
else
    echo "Recovering data from build scan id ${recoveryBuildScanId} (you can trigger a fresh run by deleting ${recoveryFile})"

    # Get first batch of builds
    getBuildScansFromBuildId ${recoveryBuildScanId}
fi

# Iterate over build scan list
while [ ! -z "${buildScans}" ]
do 
    buildScanId=""

    IFS=$'\n'
    for buildScan in ${buildScans}; do
        # Remove "
        buildScan=$(echo "${buildScan}" | sed -e 's/"//g')

        # Parse fields
        buildScanId=$(echo "${buildScan}" | cut -d, -f 1)
        buildScanDate=$(echo "${buildScan}" | cut -d, -f 2)
        project=$(echo "${buildScan}" | cut -d, -f 3)
        user=$(echo "${buildScan}" | cut -d, -f 4)
        repository=$(echo "${buildScan}" | cut -d, -f 5)

        echo "Build ${buildCount}: Checking ${buildScanId} published at $(date -d @$(echo ${buildScanDate} | head -c 10))"
        
        # Filter repositories having a username@password with username non redacted username
        pattern="^https.*[^*]@.*$"
        if [[ $repository =~ $pattern ]] 
        then
            echo "Found ${buildScanId} with ${repository} building ${project}"
        fi

        # save build scan id for recovery mode
        echo "${buildScanId}" > ${recoveryFile}

        # append to file if not already present
        grep -q "${repository}" $0.out.tmp.txt 2>/dev/null || echo ${repository} >> $0.out.tmp.txt
        buildCount=$((buildCount+1))
    done

    getBuildScansFromBuildId ${buildScanId}
done 

# Remove recovery data
rm -f ${recoveryFile}

# Sort results
sort ${workFile} > ${outputFile}

# Display results
echo "List of repositories found:"
echo ""
cat ${outputFile}