failure_frequency.sh

#!/bin/bash
N_RUNS=30

function drive_test {
    test_meta=$1
    test_target=$2
    n_failures=0

    echo "********************************************************************************************************************************************"
    echo "$test_meta EXECUTION"
    echo "********************************************************************************************************************************************"
    for ((run_i=1;run_i<=N_RUNS;run_i++)); do
        echo "///////////////////////////////////////////////////////////////////////////////////////"
        echo "RUN $run_i"
        pytest --durations=0 $test_target

        if [[ "$?" != "0" ]]; then
            let n_failures++
        fi
    done

    return $n_failures
}


drive_test TEST_SUITE_X proj_x/test_foo.py
n_x_failures=$?
drive_test TEST_SUITE_Y proj_y/test_foo.py
n_y_failures=$?


echo -e "\n\n\nSUMMARY"
echo "    n_x_failures = $n_x_failures / $N_RUNS"
echo "    n_y_failures = $n_y_failures / $N_RUNS"