Demonstrates how to do rolling restarts... log, ready, pid, port files need to be cleaned up in the shutdown part.

rolling_restart.sh

#!/usr/bin/env bash
#This script is intended to be run from the root of the sauce connect package.
SCBIN="bin/sc"
LOGPREFIX="./tmp"
##Uncomment the lines below and set your credentials
#UNAME=""
#AKEY=""

function start_sauce_connect() {
    tunnelId=$1

    if [ -z "$2" ]; then
        port="4445"
    else
        port=$2
    fi

    echo "tunnel id: ${tunnelId}"
    echo "se port: ${port}"

    args="--user ${UNAME} \
          --api-key ${AKEY} \
    	  --max-logsize 524288000 \
          --se-port ${port} \
          --logfile ${LOGPREFIX}/sc_log_${tunnelId}_${port}.log \
          --pidfile ${LOGPREFIX}/sc_${tunnelId}_${port}.pid \
          --readyfile ${LOGPREFIX}/sc_${tunnelId}_${port}.rdy \
          --no-remove-colliding-tunnels \
          --wait-tunnel-shutdown"

    if ! [ -z "$tunnelId" ]; then
        args="${args} --tunnel-identifier ${tunnelId}"
    fi

    echo $SCBIN $args
    $SCBIN $args &
    echo $port > $LOGPREFIX/sc_${tunnelId}_${port}.port
}

function is_tunnel_ready() {
    tunnelId=$1

    if [ -z "$2" ]; then
        port="4445"
    else
        port=$2
    fi
    ready_file="${LOGPREFIX}/sc_${tunnelId}_${port}.rdy"
    ctr=0
    while ! [[ -e "$ready_file" ]] && [[ $ctr -lt 10 ]];
    do
        echo "Waiting ${ctr}"
        echo "se port: ${port}"
        echo "tunnel id: ${tunnelId}"
        sleep 3
        let ctr=ctr+1
    done

    if [[ ctr -gt 9 ]]; then
        echo "Failed:"
        echo "se port: ${port}"
        echo "tunnel id: ${tunnelId}"
    else
        echo "Ready:"
        echo "se port: ${port}"
        echo "tunnel id: ${tunnelId}"
    fi
}

function check_credentials() {
    if ! [ -z "$UNAME" ]; then
        # echo $UNAME
        export SAUCE_USERNAME=$UNAME
        echo "Username supplied!"
    else
        echo "No username supplied"
        exit 1
    fi

    if ! [ -z "$AKEY" ]; then
        # echo $akey
        export SAUCE_ACCESS_KEY=$AKEY
        echo "Access key supplied!"
    else
        echo "No access key supplied"
        exit 1
    fi
}

# usage start port end port
function start_all_tunnels() {
    for p in `seq $1 $2`;
    do
      start_sauce_connect "" $p
      sleep 1
    done

    for p in `seq $1 $2`;
    do
      is_tunnel_ready "" $p
      sleep 1
    done
}

function get_active_pids(){
	#cat $LOGPREFIX/sc*.pid
	active_pids=$(cat $LOGPREFIX/sc*.pid)
}

function get_active_ports(){
	#cat $LOGPREFIX/sc*.port
	active_ports=$(cat $LOGPREFIX/sc*.port)
}

function get_active_pid_count(){
	local pids=$1
	echo $pids
	pid_count=0
	for pid in $pids
	do
		if ps -p $pid > /dev/null
		then
   			echo "$pid is running"
   			((pid_count++))
		fi
	done
}

function shutdown_tunnels() {
	local pids=$1
	if ! [[ -z $pids ]]; then
		echo $pids | xargs kill
	fi
	# you may need to adjust the max count and the sleep delay
	ctr=0
	pid_count=1
	while  ((proc_cnt > 0)) && ((ctr < 100));
	do
		echo "Waiting for tunnel shutdown ${ctr}"
		let ctr=ctr+1
		sleep 3
		get_active_pid_count $pids
		echo $proc_cnt
	done
}

function generate_port_range(){
	read -r -a ports <<< $1
	l=${#ports[@]}
	((l--))
	echo $ports
	start=$((${ports[@]: -1}+1))
	end=$((start+l))
	new_range="$start $end"
}

#ulimit -n needs to be set to something >8192 or more
check_credentials
start_all_tunnels 4445 4450
get_active_pids
get_active_ports
echo $active_ports
get_active_pid_count "$active_pids"
echo $pid_count
echo "We know what's running now!"
#launch replacement set
generate_port_range "$active_ports"
echo "Start replacement set!"
sleep 20
start_all_tunnels "$new_range"
sleep 20
shutdown_tunnels "$active_pids"