Run a command repeatedly until it fails

until-it-fails

#! /bin/bash

set -e -o pipefail

usage() {
    local exit_code=${1:-1}
    echo "Usage: until-it-fails [-v] [-i <iterations>] <command> [args...]"
    echo "       until-it-fails -h | --help"
    echo ""
    echo "Repeatedly runs a command until it fails, logging the final output."
    echo ""
    echo "Options:"
    echo "  -v               Verbose mode: print output of each iteration."
    echo "  -i <iterations>  Stop after the requested number of successful iterations."
    echo "  -h, --help       Show this help message and exit."
    exit "$exit_code"
}

# Check for arguments
if [ $# -eq 0 ]; then
    usage 1
fi

verbose=false
target_iterations=0

while [[ $# -gt 0 ]]; do
    case "$1" in
        -v)
            verbose=true
            shift
            ;;
        -i)
            if [[ -n "$2" && "$2" =~ ^[0-9]+$ ]]; then
                target_iterations=$2
                shift 2
            else
                echo "Error: -i requires a numeric argument." >&2
                usage 1
            fi
            ;;
        -h|--help)
            usage 0
            ;;
        *)
            # Stop parsing options if we hit something that doesn't look like an option
            break
            ;;
    esac
done

if [ $# -eq 0 ]; then
    echo "Error: No command specified." >&2
    usage 1
fi

# Create unique log file in /tmp
logfile=$(mktemp /tmp/until-it-fails.XXXXXX.log)
readonly logfile

# Function to print log file location
print_logfile() {
    # Ensure a newline if we were in dot mode and didn't finish a line
    if [ -n "$mode" ] && [ "$mode" = "dot" ] && [ "$(( iterations % 50 ))" -ne 0 ]; then
        echo ""
    fi
    echo "Log file: $logfile"
}

# Trap to ensure log file location is always printed on exit
trap print_logfile EXIT

# Trap to handle interrupts with proper exit code
trap 'exit 130' INT TERM

iterations=0
start=$SECONDS
mode=""

while true; do
    iterations=$((iterations + 1))
    istart=$SECONDS
    # Clear the log file before this iteration
    : > "$logfile"

    # Run the command
    set +e
    if [ "$verbose" = true ]; then
        "$@" 2>&1 | tee "$logfile"
        exit_code=${PIPESTATUS[0]}
    else
        "$@" &> "$logfile"
        exit_code=$?
    fi
    set -e

    now=$SECONDS
    ielapsed=$((now - istart))
    total_elapsed=$((now - start))
    avg=$((total_elapsed / iterations))

    if [ -z "$mode" ]; then
        if [ "$ielapsed" -lt 15 ]; then
            mode="dot"
        else
            mode="table"
        fi
    fi

    if [ "$exit_code" -ne 0 ]; then
        if [ "$verbose" = true ]; then
            echo "iteration: $iterations, elapsed: ${ielapsed}s, average: ${avg}s, total: ${total_elapsed}s"
        fi
        break
    fi

    if [ "$verbose" = true ]; then
        echo "iteration: $iterations, elapsed: ${ielapsed}s, average: ${avg}s, total: ${total_elapsed}s"
    elif [ "$mode" = "dot" ]; then
        printf "."
        if [ "$((iterations % 50))" -eq 0 ]; then
            printf "\n"
        elif [ "$((iterations % 10))" -eq 0 ]; then
            printf " "
        fi
    else
        if [ "$(( (iterations - 1) % 20 ))" -eq 0 ]; then
            printf "\n%-10s %-10s %-10s %-10s\n" "Iteration" "Time" "Rate" "Total"
            printf "%-10s %-10s %-10s %-10s\n" "----------" "----------" "----------" "----------"
        fi
        printf "%-10d %-10s %-10s %-10s\n" "$iterations" "${ielapsed}s" "${avg}s" "${total_elapsed}s"
    fi

    if [ "$target_iterations" -gt 0 ] && [ "$iterations" -ge "$target_iterations" ]; then
        if [ "$mode" = "dot" ] && [ "$(( iterations % 50 ))" -ne 0 ]; then
            echo ""
        fi
        echo "Successfully completed $iterations iterations. Total time: ${total_elapsed}s"
        exit 0
    fi
done

if [ "$mode" = "dot" ] && [ "$(( iterations % 50 ))" -ne 0 ]; then
    echo ""
fi
echo "XXXXXXXXXX Failed after: $iterations iterations  Total time: ${total_elapsed}s XXXXXXXXXX"