Shutdown GitHub self-hosted Actions Runners gracefully

README.md

Shutdown GitHub self-hosted Actions Runners gracefully

Sometimes you may want to shut down GitHub runners to perform maintenance.

This may cause race conditions where a job is scheduled while a runner is shutting down.

Assuming you're selecting your runners via a custom label (i.e. runs-on: custom-label), this script gracefully prepares the runners for shutdown by removing custom labels (which prevents jobs new from being scheduled).

It then waits for the runners to be idle.

The script expects the following variables to be defined, either in the environment or in an .env file next to the script.

• GITHUB_ORG
• GITHUB_TOKEN
• RUNNER_LABELS

shutdown_runners.sh

#!/bin/bash
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")"

API_URL="https://api.github.com"

# Load environment variables from .env file
if [[ -f .env ]]; then
    source .env
fi

if [[ -z "$GITHUB_ORG" ]] || [[ -z "$GITHUB_TOKEN" ]] || [[ -z "$RUNNER_LABELS" ]]; then
    echo "GITHUB_ORG, GITHUB_TOKEN, and RUNNER_LABELS must be set in the .env file"
    exit 1
fi
IFS=',' read -r -a LABELS <<<"$RUNNER_LABELS"
echo "Shutting down runners with labels: ${LABELS[*]}"

function gh_curl() {
    curl -sSL \
        -H "Accept: application/vnd.github+json" \
        -H "Authorization: Bearer $GITHUB_TOKEN" \
        -H "X-GitHub-Api-Version: 2022-11-28" \
        "$@"
}

function get_runners() {
    local runners
    runners=$(gh_curl "$API_URL/orgs/$GITHUB_ORG/actions/runners")
    for label in "${LABELS[@]}"; do
        if [[ -z "$label" ]]; then
            continue
        fi
        echo "$runners" |
            jq -r ".runners[] | select(.labels[].name == \"$label\") | (.id | tostring) + \"=\" + .name"
    done | sort -u
}

function remove_labels() {
    local runner_id="$1"
    for label in "${LABELS[@]}"; do
        gh_curl -X DELETE "$API_URL/orgs/$GITHUB_ORG/actions/runners/$runner_id/labels/$label"
    done
}

function is_runner_idle() {
    local runner_id="$1"
    local busy
    busy="$(gh_curl "$API_URL/orgs/$GITHUB_ORG/actions/runners/$runner_id" | jq -r .busy)"
    [[ "$busy" == "false" ]]
}

function to_assoc() {
    local -n assoc_ref="$1"
    local -a arr
    mapfile -t arr
    for line in "${arr[@]}"; do
        IFS='=' read -r key value <<<"$line"
        # shellcheck disable=SC2034
        assoc_ref["$key"]="$value"
    done
}

# Loop through all runners and remove labels if they exist
declare -A runners
to_assoc runners < <(get_runners)

echo "Preventing new jobs from being assigned to runners..."

for runner_id in "${!runners[@]}"; do
    name="${runners[$runner_id]}"
    echo "Removing labels from runner $name ($runner_id)..."
    remove_labels "$runner_id"
done

echo "Waiting for runners to become idle..."

for runner_id in "${!runners[@]}"; do
    name="${runners[$runner_id]}"
    while ! is_runner_idle "$runner_id"; do
        echo "Runner $name ($runner_id) is not idle, waiting..."
        sleep 5
    done
    echo "Runner $name ($runner_id) is idle"
done

echo "All runners are idle, you can now safely shut them down."