A helper script to workaround hanging tiflash instance

check_tiflash_alive.sh

#!/bin/bash

# set -x

DOC="A tiflash alive detect helper script.\
If tiflash is hanging, it receives the packets but doesn't respond. \
Dropping the packets will slow down the queries caused by the tidb \
alive detecting mechanism. \
This script checks the status port to detect whether TiFlash is hanging. \
Add a rule to iptables to REJECT the TCP packets once tiflash is hanging, \
And remove the REJECT rule once tiflash becomes normal"

function check_tiflash_alive
{
    IP="172.16.6.50"
    STATUS_PORT="20292"
    SERVICE_PORT="3930"

    curl "http://${IP}:${STATUS_PORT}/status" --max-time 2 --silent
    curl_exit_code=$?
    if [[ "$curl_exit_code" == 0 ]]; then
        # tiflash running normally, remove the iptables if exist
        iptables --list | grep "REJECT.*dpt:${SERVICE_PORT}"
        grep_exit_code=$?
        if [[ "$grep_exit_code" == 0 ]]; then
            iptables -D INPUT -p tcp --dport "${SERVICE_PORT}" -j REJECT
            echo "Drop iptables REJECT rule @ $(date +"%Y-%m-%d %H:%M:%S")"
        fi
    elif [[ "$curl_exit_code" == 28 ]]; then
        # status show timeout, disable the MPP port explicitly
        iptables --list | grep "REJECT.*dpt:${SERVICE_PORT}" -q
        grep_exit_code=$?
        if [[ "$grep_exit_code" == 1 ]]; then
            iptables -A INPUT -p tcp --dport "${SERVICE_PORT}" -j REJECT
            echo "Add iptables DROP rule @ $(date +"%Y-%m-%d %H:%M:%S")"
        else
            echo "REJECT rule has been added"
        fi
    fi
}

echo "TiFlash alive detecting helper script"
echo "[Hit CTRL+C to stop]"
for (( ; ; )); do
    check_tiflash_alive
    sleep 2
done