recover_zk.sh

#!/bin/bash
SCRIPT_VERSION=v1.0.0

: "${DATE_FORMAT:="%Y-%m-%d %H:%I:%S,%3N"}"

echo "[$(date +"$DATE_FORMAT")][INFO] VERSION: $SCRIPT_VERSION"

# This script performs taks according to a provided command. The supported command values are:
# - generate-certs - Will create temporary ZK stunnel certs if the existing ca-file is found to be expired
# - validate-zk - Will verify ZK quorum after certs have been re-generated and distributed to all directors
# - restore-backups - Will restore backed up ZK stunnel cert files
# - delete-backups - Will delete backed up ZK stunnel cert files

# Exit codes:
#   1 An unexpected error has occurred
#   2 Indicates ZK certs are not expired and do not need to be forcefully re-generated
#   5 Indicates new ZK certs have been generated and must be externally copied to other ZK nodes as described. 
#     Then this script should be run again with COMMAND=validate-zk before proceeding with director cert rotation.
#   9 Indicates the ZK quorum has been validated with the current certs. We are ready to run the director cert rotation.


# 10 minutes of ZK validation retries by default
: "${MAX_RETRIES:=60}"
HOST_STORAGE_PATH=$(docker exec frc-runners-runner printenv HOST_STORAGE_PATH)
RUNNER_ID=$(docker exec frc-runners-runner printenv RUNNER_ID)
MANAGED_PATH=$HOST_STORAGE_PATH/$RUNNER_ID/services/zookeeper/managed
cd $MANAGED_PATH


function generate-certs() {
   
    if [ ! -f "stunnel.conf" ]; then
        echo "stunnel.conf not found"
        exit 1
    fi

    # Check if any ZK cert file is expired
    echo "Verifying if any zookeeper cert is expired"
    declare -i expired_certs=0
    for FILE in ca-file.pem clients-ca-file.pem cert.pem clients-cert.pem #perform cert renewal for all zk certificates.
    do
        if openssl x509 -checkend 1 -noout -in $FILE > /dev/null 2>&1; then
        echo "$FILE is not expired."
        else
        echo "Verified $FILE cert has expired."
        expired_certs=$expired_certs+1
        fi      
    done
    
    if [ ${expired_certs} -eq 0 ]; then
    echo "No cert file is expired. Nothing to do"
    exit 2
    fi

    # Backup existing certs
    echo "Backing up ZK CA and cert files"
    cp ca-file.pem ca-file.pem.bak
    cp cert.pem cert.pem.bak
    cp clients-ca-file.pem clients-ca-file.pem.bak
    cp clients-cert.pem clients-cert.pem.bak

    # Generate new certs
    echo "Generating temporary ZK certs"
    openssl req -x509 -newkey rsa:4096 -keyout key.pem -out ca-file.pem -days 365 -nodes -subj "/CN=elastic ce zk servers root"
    cat ca-file.pem key.pem > cert.pem
    openssl req -x509 -newkey rsa:4096 -keyout key.pem -out clients-ca-file.pem -days 365 -nodes -subj "/CN=elastic ce zk clients root"
    cat clients-ca-file.pem key.pem > clients-cert.pem

    # Check for other Director hosts
    OTHER_DIRECTORS=`cat stunnel.conf | grep "connect=" | grep -wv "localhost" | cut -d '=' -f2 | cut -d ':' -f1 | uniq | sort`
    if [ -z "$OTHER_DIRECTORS" ]; then
        echo "No other director hosts detected"
        validate-zk
    else
        print_director_copy_message
    fi
}

function validate-zk() {
    echo "Restarting ZK stunnel"
    docker exec frc-zookeeper-servers-zookeeper sv restart stunnel

    echo "Validating ZK quorum. This step requires that certificates have already been generated and copied to all directors."

    # Validate local ZK connectivity
    local_zk_port=$((`cat $HOST_STORAGE_PATH/$RUNNER_ID/services/zookeeper/data/myid` + 2181))
    verify-zk-quorum "localhost" $local_zk_port

    # Cross-reference replicated.cfg.dynamic with stunnel.conf to validate remote ZK connectivity
    zkports=`cat replicated.cfg.dynamic | cut -d '=' -f2 | cut -d ':' -f2,5`
    for zkp in $zkports; do
        stunnel_port=$((`echo $zkp | cut -d ':' -f1` + 10000))
        local_port=`echo $zkp | cut -d ':' -f2`
        host=`cat stunnel.conf | grep "connect=" | grep $stunnel_port | cut -d '=' -f2 | cut -d ':' -f1`
        if [ "$host" ]; then
            verify-zk-quorum $host $local_port
        fi
    done
    exit 9
}

function verify-zk-quorum() {
    if [[ "$1" = "" || "$2" == "" ]]; then
        echo "Missing required arguments: [host, port]"
        return
    fi
    declare -i attempts=0
    
    until [ ${attempts} -ge ${MAX_RETRIES} ]
    do
        echo mntr | nc $1 $2 | grep -q "not currently serving requests" 
        if [[ $? -eq 0 ]]; then
            echo "$1 is not part of a quorum. Will check again in 10 seconds."
            attempts=$attempts+1
            sleep 10
        else
            echo "Verified quorum on $1"
            return
        fi
    done
    echo "Failed to verify $1 quorum connectivity after 10 minutes. Exiting."
    exit 1
}

function print_director_copy_message() {
    printf "\nPlease copy the following CA and cert files:\n\n"
    echo "$MANAGED_PATH/ca-file.pem"
    echo "$MANAGED_PATH/cert.pem"
    echo "$MANAGED_PATH/clients-ca-file.pem"
    echo "$MANAGED_PATH/clients-cert.pem"
    printf "\ninto the corresponding directories on the following director hosts:\n\n"
    for d in $OTHER_DIRECTORS; do
        echo $d
    done
    printf "\nOnce that is complete, run this script again on all directors concurrently with the command: validate-zk.\n"
    exit 5
}

function restore-backups() {
    cd $HOST_STORAGE_PATH/$RUNNER_ID/services/zookeeper/managed
    echo "Restoring ZK CA and cert files from backups"
    cp ca-file.pem.bak ca-file.pem
    cp cert.pem.bak cert.pem
    cp clients-ca-file.pem.bak clients-ca-file.pem
    cp clients-cert.pem.bak clients-cert.pem
    docker exec frc-zookeeper-servers-zookeeper sv restart stunnel
}

function delete-backups() {
    cd $HOST_STORAGE_PATH/$RUNNER_ID/services/zookeeper/managed
    echo "Deleting ZK CA and cert files from backups"
    rm ca-file.pem.bak cert.pem.bak clients-ca-file.pem.bak clients-cert.pem.bak 
}

if [ "$1" == "generate-certs" ]; then 
    generate-certs
elif [ "$1" == "validate-zk" ]; then
    validate-zk
elif [ "$1" == "restore-backups" ]; then
    restore-backups
elif [ "$1" == "delete-backups" ]; then
    delete-backups
else
    echo "Please specify one of the following commands: [generate-certs, validate-zk, restore-backups, delete-backups]"
    exit 1
fi