run-mix-failed-tests.sh

#!/opt/homebrew/bin/bash
#
# A script that checks for previously failed tests, caches them to a file, and reruns only
# those tests again. Slightly more useful than `mix test --failed` alone because you can
# share/store your set of test failures however you want.
# 
# TODO: Someday turn this into an ExUnit formatter: https://hexdocs.pm/ex_unit/ExUnit.Formatter.html
# - https://github.com/crertel/exunit_json_formatter/blob/master/lib/exunit_json_formatter.ex
# - https://github.com/findmypast-oss/exunit_json_formatter/blob/master/lib/exunit_json_formatter.ex

# DEBUGGING:
# set -v
# set -x

# Configure these as needed:
input_file="scripts/mix-failed-tests.txt"
max_tags_to_check=5
tag_to_apply="recently_failed"

function run() {
  if [[ ! -f "$input_file" ]]; then
    echo "File '$input_file' does not exist."
    run_all_tests
  else 
    echo "File '$input_file' exists."
    maybe_run_all_tests
  fi
}

function maybe_run_all_tests() {
  tests=$(get_tests_from_file)
  if [[ -z "$tests" ]]; then
    echo "File exists but no tests inside, deleting file."
    rm "$input_file"

    run_all_tests
  else
    run_failed_tests
  fi
}

function run_all_tests() {
  echo "Running all tests to check for failures."
  # Run all tests, then rerun any failed ones once and output the results to a file 'cache'
  # Modified version of: https://angelika.me/2024/01/08/do-not-run-mix-test-failed/
  mix test --warnings-as-errors || if [[ $? = 2 ]]; then clear; mix test --warnings-as-errors --trace --failed 2>&1 | filter_excluded_and_skipped | tee "$input_file"; else false; fi 
  tag_failed_tests
}

function run_failed_tests() {
  tests=$(get_tests_from_file)
  tests_count=$(echo "$tests" | wc -l)
  echo "Running $tests_count previously-failed tests."
  # shellcheck disable=SC2086
  mix test --warnings-as-errors --trace $tests 2>&1 | filter_excluded_and_skipped
  echo "Ran $tests_count previously-failed tests."
  tag_failed_tests
}

function tag_failed_tests() {
  tests=$(get_tests_from_file)

  # Create associative arrays to track line offsets for each file
  declare -A file_offsets

  for test in $tests; do
    filename=$(echo "$test" | cut -d':' -f1)
    test_start_line=$(echo "$test" | cut -d':' -f2)

    # Initialize offsets for this file if not exist
    : "${file_offsets[$filename]:=0}"
    
    # Adjust line number based on previous insertions
    adjusted_start_line=$((test_start_line + ${file_offsets[$filename]}))
    lines_before_test_start=$((adjusted_start_line - max_tags_to_check))
    [ $lines_before_test_start -lt 1 ] && lines_before_test_start=1

    # Check if tag exists in the lines before the test
    range_to_check=$(gsed --quiet "${lines_before_test_start},${adjusted_start_line}p" "$filename")
    echo "$range_to_check" | grep --quiet "@tag :$tag_to_apply"
    tag_exists_in_range=$?
    if [[ $tag_exists_in_range -ne 0 ]]; then
      # DEBUGGING:
      # cat -n "$filename" | gsed --quiet "${lines_before_test_start},${adjusted_start_line}p"

      # Insert the tag before the test
      gsed --in-place "${adjusted_start_line}i @tag :$tag_to_apply" "$filename"
      echo "Tagged $filename:$adjusted_start_line: :$tag_to_apply"
    else
      echo "Skipping $filename:$adjusted_start_line: :$tag_to_apply (already tagged)"
    fi

    # Increment the offset (and lines to check) for this file
    file_offsets[$filename]=$((file_offsets[$filename] + 1))
  done

  mix format
  
  echo "Done tagging recently-failed tests!"
  echo -e "Run the following command to only run these tests:"
  echo -e "\tmix test --warnings-as-errors --only $tag_to_apply"
}

function get_tests_from_file() {
  grep --extended-regexp "test/.+\.exs:[0-9]+$" "$input_file" | sort -t: -k1,1 -k2n | uniq
}

function filter_excluded_and_skipped() {
  grep --invert-match --regexp="(excluded)" --regexp="(skipped)"
}

run

run-mix-slowest-tests.sh

#!/opt/homebrew/bin/bash
#
# A script that checks for previous slow tests, caches them to a file,
# and reruns only those tests again. Extends `mix test --slowest` to
# allow you to store/share your set of slow tests however you want.
#
# On macOS, requires `gnu-sed` from Homebrew: https://stackoverflow.com/a/4247319
#
# TODO: Someday turn this into an ExUnit formatter: https://hexdocs.pm/ex_unit/ExUnit.Formatter.html
# - https://github.com/crertel/exunit_json_formatter/blob/master/lib/exunit_json_formatter.ex
# - https://github.com/findmypast-oss/exunit_json_formatter/blob/master/lib/exunit_json_formatter.ex

# DEBUGGING:
# set -v
# set -x

# Configure these as needed:
input_file="scripts/mix-slowest-tests.txt"
number_of_tests=30
max_tags_to_check=5
tag_to_apply="slowest_$number_of_tests"

function run() {
  if [[ ! -f "$input_file" ]]; then
    echo "File '$input_file' does not exist."
    run_slowest_tests
  else 
    echo "File '$input_file' exists. Running previously slow tests."
    maybe_run_slowest_tests
    echo "To rerun all tests, delete the following file and rerun this script: $input_file"
  fi
}

function maybe_run_slowest_tests() {
  tests=$(get_tests_from_file)
  if [[ -z "$tests" ]]; then
    echo "File exists but no tests inside, deleting file."
    rm "$input_file"

    run_slowest_tests
  else
    run_cached_tests
  fi
}

function run_slowest_tests() {
  echo "Running all tests to determine slowest $number_of_tests."
  mix test --warnings-as-errors --slowest $number_of_tests 2>&1 | filter_excluded_and_skipped | tee "$input_file"
  tag_slowest_tests
}

function run_cached_tests() {
  echo "Running cached slow tests."
  tests=$(get_tests_from_file)
  # shellcheck disable=SC2086
  mix test --warnings-as-errors --slowest $number_of_tests $tests 2>&1 | filter_excluded_and_skipped
  tag_slowest_tests
}

function tag_slowest_tests() {
  tests=$(get_tests_from_file)

  # Create associative arrays to track line offsets for each file
  declare -A file_offsets

  for test in $tests; do
    filename=$(echo "$test" | cut -d':' -f1)
    test_start_line=$(echo "$test" | cut -d':' -f2)

    # Initialize offsets for this file if not exist
    : "${file_offsets[$filename]:=0}"

    # Adjust line number based on previous insertions
    adjusted_start_line=$((test_start_line + ${file_offsets[$filename]}))
    lines_before_test_start=$((adjusted_start_line - max_tags_to_check))
    [ $lines_before_test_start -lt 1 ] && lines_before_test_start=1

    # Check if tag exists in the lines before the test
    range_to_check=$(gsed --quiet "${lines_before_test_start},${adjusted_start_line}p" "$filename")
    echo "$range_to_check" | grep --quiet "@tag :$tag_to_apply"
    tag_exists_in_range=$?
    if [[ $tag_exists_in_range -ne 0 ]]; then
      # DEBUGGING
      # cat -n "$filename" | gsed --quiet "${lines_before_test_start},${adjusted_start_line}p"

      # Insert the tag before the test
      gsed --in-place "${adjusted_start_line}i @tag :$tag_to_apply" "$filename"
      echo "Tagged $filename:$adjusted_start_line: :$tag_to_apply"
    else
      echo "Skipping $filename:$adjusted_start_line: :$tag_to_apply (already tagged)"
    fi

    # Increment the offset (and lines to check) for this file
    file_offsets[$filename]=$((file_offsets[$filename] + 1))
  done

  mix format

  echo "Done tagging slowest tests!"
  echo -e "Run the following command to only run these slow tests:"
  echo -e "\tmix test --warnings-as-errors --include $tag_to_apply --slowest $number_of_tests | grep --invert-match --regexp=\"(excluded)\" --regexp=\"(skipped)\""
}

function get_tests_from_file() {
  # shellcheck disable=SC2016
  gsed -n '/Finished in/,$p' "$input_file" | awk -F'[][]' '/\[.*:[0-9]+\]/ {print $2}' | sort -t: -k1,1 -k2,2n | uniq
}

function filter_excluded_and_skipped() {
  grep --invert-match --regexp="(excluded)" --regexp="(skipped)"
}

run