gitlab multi project pipeline alternative

trigger-gitlab-job.sh

#!/bin/bash

# downstream, a tool to trigger gitlab CI pipelines
#
# DESCRIPTION
#
# downstream offers a way to trigger a pipeline on a specific branch of one or multiple projects
# Project are identified by their numerical IDs.
# Project IDs can be found under Settings->General->General project settings
# When a branch is not found, it is created from the master and then the pipeline is triggered
#
# USAGE
#
# ./downstream.sh $GITLAB_API_URL $API_TOKEN $TARGET_BRANCH <LIST OF PROJECT IDS>
#
# (ex : ./downstream.sh https://git.si.mbs/api/v4/projects/ s2CWCfd8x5Djasx1fUAx move_to_gitlab_ci 1232 1236 1235)
#
# DEPENDENCIES
#
# - curl, transfer a URL
# - jg, command-line JSON processor
#
# BUGS
#
# Presumably. Report them or discuss them at:
#  http://10.33.2.89:3000/channel/ceo
#

function stdout {
    echo "$(date "+%H:%m:%s") $@"
}

WATCH_PERIOD=5 # in seconds
NUM_OF_CYCLES=150 # script will timeout after WATCH_PERIOD x NUM_OF_CYCLES seconds

## MAIN PARAMETERS
GITLAB_API_URL=$1
API_TOKEN=$2
TARGET_BRANCH=$3

## INTERNAL VARS
declare -a PROJECT_IDS
declare -a PIPELINE_IDS
declare -a PATHS_WITH_NAMESPACE
declare -a URLS

num_of_projects=$(($#-3))

stdout "Prepare to downstream $num_of_projects jobs..."

## 1 - trigger pipelines
i=0
while [ $i -lt $num_of_projects ];
do
    # retrieve project ID, shifting the 4 first elements of command line
    project_id=$((i+4))
    PROJECT_IDS[${i}]=${!project_id}

    # create a branch from master if the branch does not exist yet (this request has not effect if the branch already exists)
    # This strategy allows to :
    # - group pipelines with the same branch name
    # - link parent<->children pipelines
    stdout "create a branch on sub-project if it does not exits..."
    curl -X POST -sk --header "PRIVATE-TOKEN: $API_TOKEN" --silent "${GITLAB_API_URL}${PROJECT_IDS[$i]}/repository/branches?branch=$TARGET_BRANCH&ref=master"

    # If a pipeline already exists on this branch, we use it
    ref=$(curl -sk --header "PRIVATE-TOKEN: $API_TOKEN" ${GITLAB_API_URL}${PROJECT_IDS[$i]}/pipelines | jq  " .[] | select(.status | contains(\"running\")) | select(.ref | contains(\"$TARGET_BRANCH\")) | .id" | head -n 1)

    if [[ -z $ref ]]; then
        ## trigger pipeline on branch if branch exists
        HTTP_RESPONSE=$(curl -sk --header "PRIVATE-TOKEN: $API_TOKEN" --silent --write-out "HTTPSTATUS:%{http_code}" -X POST "${GITLAB_API_URL}${PROJECT_IDS[$i]}/pipeline?ref=$TARGET_BRANCH")
        STATUS_CODE=$(echo $HTTP_RESPONSE | tr -d '\n' | sed -e 's/.*HTTPSTATUS://')
        if [ $STATUS_CODE != 201 ]; then
                stdout "Cannot trigger pipeline, got status code $STATUS_CODE with an HTTP POST on ${GITLAB_API_URL}${PROJECT_IDS[$i]}/pipeline?ref=$TARGET_BRANCH"
                exit 2;
        fi
        ref=$(echo $HTTP_RESPONSE | sed -e 's/HTTPSTATUS\:.*//g' | jq ".id")
    else
        stdout "pipeline#${ref} already exists for project ${PROJECT_IDS[$i]}"
    fi

    PIPELINE_IDS[${i}]=$ref
    PATHS_WITH_NAMESPACE[${i}]=$(curl -sk --header "PRIVATE-TOKEN: $API_TOKEN" "${GITLAB_API_URL}${PROJECT_IDS[$i]}" | jq -r ".path_with_namespace")
    URLS[${i}]="https://${GITLAB_URL}/${PATHS_WITH_NAMESPACE[$i]}/pipelines/${PIPELINE_IDS[$i]}"

    stdout "--> ${URLS[$i]} started for project ${PATHS_WITH_NAMESPACE[$i]}"

    ((i++))
done

# 2- watch pipelines statuses
for (( c=1; c<=$NUM_OF_CYCLES; c++ ))
do
    i=0
    num_of_success=0
    while [ $i -lt $num_of_projects ];
    do
        stdout ${URLS[$i]}

        PIPELINE_STATUS=$(curl -sk --header "PRIVATE-TOKEN: $API_TOKEN" "${GITLAB_API_URL}${PROJECT_IDS[$i]}/pipelines/${PIPELINE_IDS[$i]}" | jq -r ".status")
        if [ $PIPELINE_STATUS = "success" ]; then
            stdout "... is in success"
            ((num_of_success++))
        elif [ $PIPELINE_STATUS = "failed" ]; then
            stdout "... failed"
            exit 2;
        elif [ $PIPELINE_STATUS = "canceled" ]; then
            stdout "... was canceled but others sub-jobs may persist"
            exit 2;
        else
            stdout "... is in state $PIPELINE_STATUS"
        fi

        ((i++))
    done

    # check if all subjobs succeded
    if [ $num_of_success -eq $num_of_projects ]; then
        stdout "Everything succeeded !!! Great job !"
        exit 0;
    fi

    stdout "-- wait $WATCH_PERIOD seconds and check again"
    sleep $WATCH_PERIOD

done

stdout "pipeline has timed out, exit"
exit 2