track-gitlab-runner.sh

#!/usr/bin/env bash

# integrate gitlab (shell) runner with jenkins scheduler
# uses flock filesystem locking for coordination

# the target is to ensure the gitlab runner obtains an jenkins executor so the machine where both runner and jenkins are running on is not
# overloaded while the resources can still be fully utilized.

# Christoph Schulz <[email protected]>
# Aug 2020

JENKINS_HOST=localhost
JENKINS_BASE_URL=http://$JENKINS_HOST:8080/jenkins
JENKINS_JOB_PATH=job/track-gitlab-runner
JENKINS_TRIG=1

# set these here or via a "config" file next to this script
# JENKINS_USER=
# JENKINS_TOKEN=

# jenkins ALL=(gitlab-runner) NOPASSWD: kill
JENKINS_SUDO="sudo -u gitlab-runner"

action=$1
lock_name=$2

lock_prefix=/tmp/runner-$lock_name

# used by jenkins to block until the runner is done; locks the executor
# locked and released by runner, waited on by jenkins
executor_lock=$lock_prefix-executor

# used by runner to block execution until the jenkins job is scheduled;
# gate for starting runner execution
# locked and waited on by runner, released by jenkins
# better name maybe: runner_start_gate
scheduler_gate=$lock_prefix-runner

config=$(dirname $0)/config
[ -f $config ] && source $config

# function to be executed asynchronously that locks and releases the jenkins executor
# because the script needs to return so the runner execution can start, a FD for opening the gate for that needs to be passed.
lock_jenkins_executor(){

    # the fd to the gate to start runner execution
    runner_gate_fd=$1

    # initialize and close jenkins executor lock
    exec {executor_lock_fd}>$executor_lock
    flock $executor_lock_fd

    # trigger job in jenkins, i.e. request an executor

    if [ -n "$JENKINS_TRIG" ]; then
        curl -s -X POST --netrc-file <(cat <<<"machine $JENKINS_HOST login $JENKINS_USER password $JENKINS_TOKEN") \
            "$JENKINS_BASE_URL/$JENKINS_JOB_PATH/buildWithParameters" -d delay=0sec -d LOCK=$lock_name \
            -d CI_PROJECT_PATH=$CI_PROJECT_PATH -d CI_PIPELINE_ID=$CI_PIPELINE_ID -d CI_PIPELINE_URL=$CI_PIPELINE_URL \
            -d CI_JOB_STAGE=$CI_JOB_STAGE -d CI_JOB_ID=$CI_JOB_ID -d CI_JOB_NAME=$CI_JOB_NAME -d CI_JOB_URL=$CI_JOB_URL
            
            # $CI_PROJECT_PATH: $CI_JOB_STAGE @ $CI_JOB_NAME (#$BUILD_NUMBER)
            # <a href="$CI_PIPELINE_URL">Pipeline #$CI_PIPELINE_ID</a> / <a href="$CI_JOB_URL">Job #$CI_JOB_ID</a>
            
            # [[ $LOCK =~ ^[a-zA-Z0-9_-]+$ ]]
            # ./track.sh jenkins $LOCK

    fi

    # block to wait until jenkins has scheduled the job / assigned us an executor
    {
        # close gate and provoke a deadlock in the background.
        # write pid file so jenkins can kill the deadlock to open the gate for us
        flock $gateFD; flock $scheduler_gate true &
        echo $! >$scheduler_gate.pid

        echo "Waiting for Jenkins to schedule our job..."
        wait %+
        echo "Scheduled!"

    } {gateFD}>$scheduler_gate

    # okay, executor is obtained, the runner now can start executing.

    # provoke deadlock in the background
    # the runner's post script will resolve when it's done executing
    flock $executor_lock true & echo $! >$executor_lock.pid

    # release lock and close FD for runner exection
    # this will make the pre script return
    flock -u $runner_gate_fd
    exec {runner_gate_fd}<&-

    # wait for the post script to open gate
    # so we can in turn open the gate to release the executor in jenkins
    wait
    echo "Runner finished."

    # cleanup locks
    rm $lock_prefix*
    flock -u $executor_lock_fd
    exec {executor_lock_fd}>&-

}

case $action in
    start|lock)

        if [ -z "$JENKINS_TOKEN" ]
        then
            echo >&2 "JENKINS_TOKEN required!"
            exit 1
        fi

        echo $PPID > $executor_lock.ppid

        # internal gate to determine when to exit the script so the runner execution can start
        runner_gate=$(mktemp)

        {
            flock $runner_gate_fd

            # asynchronously obtain executor and lock
            # will unlock our gate when we can start executing
            lock_jenkins_executor $runner_gate_fd &

            # wait for executor to be obtained
            flock $runner_gate true

            echo "Executor obtained, starting runner execution."

            # remove process relationship between the executor job and our own process
            # otherwise bash will still wait for all jobs to finish
            # this will make the runner's pre script return and start the main execution
            disown

            # cleanup
            rm $runner_gate*

        } {runner_gate_fd}>$runner_gate
        ;;
    finish|stop|unlock)
        kill $(<$executor_lock.pid)
        ;;
    check)
        if flock -n $executor_lock true
        then echo unlocked
        else echo locked
        fi
        ;;
    jenkins)

        # open the gate to start runner execution
        eval $JENKINS_SUDO kill $(<$scheduler_gate.pid)

        runner_proc=/proc/$(<$executor_lock.ppid)

        while [ -d $runner_proc ]
        do
            # wait for executor release gate
            flock -w 60 $executor_lock true
        done

        if [ ! -d $runner_proc ]
        then
            eval $JENKINS_SUDO kill $(<$executor_lock.pid)
            echo "Runner died without releasing lock"
            exit 64
        fi

        ;;
esac