Storj Bash Script for Docker Healthcheck + Report via Whatsapp (callmebot api)

storj_healthcheck.sh

#!/bin/bash

# Configuration
CALLMEBOT_API="https://api.callmebot.com/whatsapp.php"
PHONE_NUMBER="4917632323232"
API_KEY="12345678"
DOCKER_CONTAINERS=("storagenode1" "storagenode2")
STORAGE_PATHS=("/mnt/user/Storj/" "/mnt/user/Storj2/")
IDENTITY_PATHS=("/mnt/user/appdata/storj/Identity/storagenode/" "/mnt/user/appdata/storj/Identity/storagenode2/")
NODE_API_URLS=("http://192.168.0.2:14002/api/sno/" "http://192.168.0.2:14003/api/sno/")
SATELLITES_API_URLS=("http://192.168.0.2:14002/api/sno/satellites" "http://192.168.0.2:14003/api/sno/satellites")
LAST_HEALTH_CHECK_FILE="/tmp/storj_last_health_check"

# Function to send WhatsApp notification
send_notification() {
    local message="$1"
    local max_length=800
    echo "Message length: ${#message} characters"
    echo -e "Message content:\n$message"

    if [ "${#message}" -le "$max_length" ]; then
        curl -s --get --data-urlencode "phone=$PHONE_NUMBER" --data-urlencode "text=$message" --data-urlencode "apikey=$API_KEY" "$CALLMEBOT_API"
    else
        echo "Message exceeds $max_length characters. Unable to send."
    fi
}

# Check if a given folder is available
check_folder_available() {
    local folder="$1"
    if [ -d "$folder" ]; then
        return 0 # Folder exists
    else
        return 1 # Folder does not exist
    fi
}

# Fetch main node API data
fetch_node_api_data() {
    local api_url="$1"
    local container="$2"
    local data=""

    api_response=$(curl -s --max-time 60 "$api_url" || echo "error")
    if [ "$api_response" == "error" ]; then
        data+="  ⚠️ API for $container is not reachable.\n"
    else
        local quic_status=$(echo "$api_response" | jq -r '.quicStatus // "N/A"')
        local last_quic_ping_raw=$(echo "$api_response" | jq -r '.lastQuicPingedAt // "N/A"')
        local started_at_raw=$(echo "$api_response" | jq -r '.startedAt // "N/A"')
        local disk_space_used_bytes=$(echo "$api_response" | jq -r '.diskSpace.used // 0')
        local disk_space_available_bytes=$(echo "$api_response" | jq -r '.diskSpace.available // 0')

        local now=$(date +%s)
        local last_quic_ping_seconds=$(date -d "$last_quic_ping_raw" +%s 2>/dev/null || echo "0")
        if (( last_quic_ping_seconds > 0 )); then
            local quic_ping_diff=$(( (now - last_quic_ping_seconds) / 60 ))
            local quic_ping_human="$quic_ping_diff minutes ago"
        else
            local quic_ping_human="N/A"
        fi
        local started_at=$(date -d "$started_at_raw" '+%Y-%m-%d %H:%M:%S' 2>/dev/null || echo "N/A")
        local disk_space_used=$(awk "BEGIN {printf \"%.2f\", $disk_space_used_bytes / 1000000000000}")TB
        local disk_space_available=$(awk "BEGIN {printf \"%.2f\", $disk_space_available_bytes / 1000000000000}")TB

        data+="  🔹 QUIC: $quic_status (Last Ping: $quic_ping_human)\n"
        data+="  🔹 Node Started: $started_at\n"
        data+="  🔹 Disk: $disk_space_used / $disk_space_available\n"
    fi

    echo "$data"
}

# Fetch satellites data
fetch_satellites_data() {
    local satellites_api_url="$1"
    local container="$2"
    local satellites_info=""

    satellites_response=$(curl -s --max-time 60 "$satellites_api_url" || echo "error")
    if [ "$satellites_response" == "error" ]; then
        satellites_info+="  ⚠️ Satellites API for $container is not reachable.\n"
    else
        satellites_info="\n  🔹 Scores %:\n"
        while IFS= read -r audit; do
            local satellite_name=$(echo "$audit" | jq -r '.satelliteName' | cut -d'.' -f1)
            local audit_score=$(echo "$audit" | jq -r '.auditScore | . * 100 | floor / 1')
            local suspension_score=$(echo "$audit" | jq -r '.suspensionScore | . * 100 | floor / 1')
            local online_score=$(echo "$audit" | jq -r '.onlineScore | . * 100 | floor / 1')
            satellites_info+="    - $satellite_name: Audit($audit_score), Susp($suspension_score), Online($online_score)\n"
        done <<< "$(echo "$satellites_response" | jq -c '.audits[]')"

        if [ -z "$(echo "$satellites_response" | jq -c '.audits[]')" ]; then
            satellites_info+="    No satellite data available.\n"
        fi
    fi

    echo "$satellites_info"
}

# Generate health report for a single container
generate_container_health_report() {
    local container="$1"
    local api_url="$2"
    local satellites_api_url="$3"
    local status="Running"

    if ! docker ps --format '{{.Names}}' | grep -q "$container"; then
        status="Stopped"
    fi
    
    local status_icon=""
    if [ "$status" == "Running" ]; then
        status_icon="🟢"
    else
        status_icon="🔴"
    fi

    local message="- $status_icon️ Node: $container\n"
    message+="  🔹 *Status: $status*\n"

    if [ "$status" == "Running" ]; then
        message+="$(fetch_node_api_data "$api_url" "$container")"
        message+="$(fetch_satellites_data "$satellites_api_url" "$container")\n"
    fi

    echo "$message"
}

# Check and restart Docker containers
check_and_restart_containers() {
    for i in "${!DOCKER_CONTAINERS[@]}"; do
        local container="${DOCKER_CONTAINERS[$i]}"
        local storage_path="${STORAGE_PATHS[$i]}"
        local identity_path="${IDENTITY_PATHS[$i]}"

        # Check if the container is running
        if ! docker ps --format '{{.Names}}' | grep -q "$container"; then
            send_notification "⚠️ Docker container $container is down. Checking storage folder before restarting."

            # Check if the storage folder is available
            if check_folder_available "$storage_path" && check_folder_available "$identity_path"; then
                send_notification "✅ Storage folders are available. Attempting to restart $container."

                # Attempt to restart the container
                docker start "$container"

                # Verify if the container restarted successfully
                if docker ps --format '{{.Names}}' | grep -q "$container"; then
                    send_notification "🎉 $container restarted successfully."
                else
                    send_notification "❌ Failed to restart $container. Manual intervention needed."
                fi
            else
                send_notification "❌ Storage folders are not available. Cannot restart $container. Ensure the storage is mounted."
            fi

            # Trigger a one-time health report for the container
            local report_message=$(generate_container_health_report "$container" "${NODE_API_URLS[$i]}" "${SATELLITES_API_URLS[$i]}")
            send_notification "$report_message"
        fi
    done
}

# Send periodic health report
send_periodic_report() {
    local now=$(date +%s)
    local current_hour=$(date +%H)
    local last_check=0
    local six_hours=$((6 * 60 * 60))
    local report_timestamp=$(date '+%Y-%m-%d %H:%M:%S')

    # Skip sending reports outside specific hours (9 AM to 9 PM)
    if ! [[ $current_hour =~ ^(9|10|11|12|13|14|15|16|17|18|19|20|21)$ ]]; then
        echo "Skipping report: Outside of reporting hours (9 AM - 9 PM)"
        return
    fi

    if [ -f "$LAST_HEALTH_CHECK_FILE" ]; then
        last_check=$(cat "$LAST_HEALTH_CHECK_FILE")
    fi

    if ((now - last_check >= six_hours)); then
        echo "$now" > "$LAST_HEALTH_CHECK_FILE"

        # Initialize a single message
        local consolidated_message="📅 Report from $report_timestamp\n\n"

        # Collect reports for all containers
        for i in "${!DOCKER_CONTAINERS[@]}"; do
            consolidated_message+=$(generate_container_health_report "${DOCKER_CONTAINERS[$i]}" "${NODE_API_URLS[$i]}" "${SATELLITES_API_URLS[$i]}")
        done

        # Send the consolidated message
        send_notification "$consolidated_message"
    fi
}

# Main execution
check_and_restart_containers
send_periodic_report

Related: https://forum.storj.io/t/i-did-a-bash-healthcheck-script-that-restart-my-docker-and-report-my-node-status-to-my-whatsapp-details-here/29113