POSIX test runner

test-runner.sh

#!/bin/sh

# test.sh
# ----------------------
# scaffold example test:
# sh  --init

# 
# quickstart:
#
# 1. $ sh test.sh init  # scaffolds sample `foo.test` file
# 2. $ sh test.sh run   # runs all tests within `foo.test`
# 
# commands:
# 
# init           scaffold a sample <PATTERN>
# test           run the test-runner tests
# run            execute the run function
#
# options:
#
# -h, --help       print this help
# -v, --version    print version
#


set -e  # exit on err
set -u  # no undefined
set -f  # no globbing

# config
# ------------------------

APP_SEMV="1.0.0"
APP_NAME=$(basename "$0") || exit 1
APP_DESC="shellscript test runner"

PRETTY=0           # pretty output, default off
TIMEOUT=5          # per-test timeout, seconds
PATTERN=".test"    # test files glob pattern
TEMPDIR=".tmp"     # temporary directory

# private config  
# ------------------------

# Exit codes
PASS=0; FAIL=99; INVD=2; TFAIL=99; TOUT=124

# Get own directory
_get_own_dir() {
: "1:"
_d=$1
while [ -h "$_d" ]; do
  _ls=$(ls -ld "$_d") || exit 1
  _link=$(expr "$_ls" : '.*-> \(.*\)$')
  if expr "$_link" : '/.*' > /dev/null; then
    _d="$_link"
  else
    _d=$(dirname "$_d")/"$_link"
  fi
done
_pwd=$(cd "$(dirname "$_d")" && pwd) || exit 1
printf '%s' "$_pwd"
unset _d _ls _link _pwd
}
OWN_DIR=$(_get_own_dir "$0")

# Environment protection
PATH=/usr/local/bin:/usr/bin:/bin
LC_ALL=C; LANG=C; umask 022

# default reporter ---------------- 
# test reporters must implement:
# - reporter_error
# - reporter_debug
# - reporter_suite_start
#   - reporter_test_fail
#   - reporter_test_pass
# - reporter_suite_end

reporter_error() {
printf '# error: %s' "1:-}"
}

reporter_debug() {
printf '# %s' "1:-}"
}

# suite loggers

reporter_suite_start() { 
printf 'TAP version 14'; 
printf "1..%d" "$_test_plan"
}

# test loggers

reporter_test_fail() {
: "1:"
printf 'not ok %s' "1:-}"
}

reporter_test_pass() {
: "1:"
printf 'ok %s' "1:-}"
}

reporter_suite_end() {
: "1:"

[ "$PRETTY" -eq 1 ] || return 0

printf ""



printf ""
}

# Utilities 
# --------------------------------------------

help() {
# config --------------------
_BGN="_BGN:-# \`\`\`[[:space:]]*console}" 
_END="_END:-# \`\`\`}"

# extract helptext from fenced code block
_block=$(sed -n "/^_BGN}/,/^_END}/p" "$0" | \
  sed -e "/^_BGN}/d" -e "/^_END}/d" -e "s/^# *//")

[ -z "$_block" ] && {
  printf '%s' "error: cannot find block." >&2
  return 1
}

# print example command
printf ''
[ -n "APP_DESC-}" ] && printf ' > %s' "$APP_DESC"
printf '  %s <command>' "APP_NAME:-$(basename "$0")}"
# print extracted helptext
printf '  %s' "$_block" | sed "s/^/  /; s/$/  /"

printf ''
}

clean() {
rm -rf "OWN_DIR:/TEMPDIR:"
} 

# Test functions ------------------------------------------------

execute_test() {
: "1:" "_test_dir:"    
_name=$1
_expect_status=$2
_expect_output=$3
_failed=0
_tmpfile="_test_dir}/output.$$.tmp"
_status=0
_timed_out=no

# Execute test with output capture
"$_name" > "$_tmpfile" 2>&1 & _pid=$!

# Wait with timeout
_timeout=$TEST_TIMEOUT
while [ "$_timeout" -gt 0 ]; do
  if ! kill -0 "$_pid" 2>/dev/null; then
    wait "$_pid" && _status=$? || _status=$?
    break
  fi
  sleep 1
  _timeout=$(expr $_timeout - 1)
done

# Handle timeout
if [ "$_timeout" -eq 0 ]; then
  kill -TERM "$_pid" 2>/dev/null
  wait "$_pid" 2>/dev/null || true
  _status=$TOUT
  _timed_out=yes
fi

# Get output
[ -f "$_tmpfile" ] && _output=$(cat "$_tmpfile") || _output=""
rm -f "$_tmpfile"

# Check expectations
[ "$_status" = "$_expect_status" ] || _failed=1
if [ -n "$_expect_output" ] && [ "$_output" != "$_expect_output" ]
then
  _failed=1
fi

# Export results
test_output=$_output
test_status=$_status
test_timeout=$_timed_out

# Clean up
unset _name _expect_status _expect_output _pid \
      _tmpfile _output _status _timeout

return "$_failed"
}

# Own Unit Tests  ----------------------------------------------

self_test() {    
: "OWN_DIR:" "TEMPDIR:" "TIMEOUT:"

_test_timeout=2
_test_plan=6

_test_count=1
_test_dir="OWN_DIR}/TEMPDIR}"
_orig_pwd=$(pwd) || exit 1
_exit_code=0

trap 'cd "_orig_pwd:" || exit 1; clean' EXIT INT TERM
clean

# Simulate what a user would do:

# ... creates a test directory
mkdir -p "_test_dir:" || {
  reporter_error "Failed to create test directory"
  return "$FAIL"
}

# ... has some functions 
printf '%s' \
  'simulate_pass() { printf "foo"; return 0; }' \
  'simulate_fail() { printf "bar"; return 99; }' \
  'simulate_hang() { printf "baz"; sleep 2; }' \
  > "_test_dir:/functions.sh" || return "$FAIL"

# ... has some tests for the functions
printf '%s' \
  '#!/bin/sh' \
  '. ./functions.sh' \
  'test_zero() { simulate_pass; }' \
  'test_fail() { simulate_fail; }' \
  'test_hang() { simulate_hang; }' \
  'test_zero_output() { simulate_pass; }' \
  'test_fail_output() { simulate_fail; }' \
  'test_fail_both() { simulate_fail; }' \
  > "_test_dir:/tests.sh" || return "$FAIL"

cd "_test_dir:" || {
  reporter_error "Failed to change directory"
  return "$FAIL"
}

. "./tests.sh" || {
  reporter_error "Failed to source tests" 
  return "$FAIL"
}

[ -n "TEST_TIMEOUT:+x}" ] || TEST_TIMEOUT=$TIMEOUT
_orig_timeout=$TEST_TIMEOUT
TEST_TIMEOUT=$_test_timeout

reporter_suite_start "$_test_plan"

for _test in test_zero test_fail test_hang \
            test_zero_output test_fail_output \
            test_fail_both; do
  case "$_test" in
    test_zero|test_zero_output)
      _exp_status=0
      _exp_output="foo"
      ;;
    test_hang)
      _exp_status=$TOUT
      _exp_output="baz"
      ;;
    *)
      _exp_status=99
      _exp_output="bar"
      ;;
  esac

  if ! execute_test "$_test" "$_exp_status" "$_exp_output"; then
    reporter_test_fail "$_test_count - $_test"
    _exit_code=99
  else
    reporter_test_pass "$_test_count - $_test"
  fi

  reporter_debug ""
  reporter_debug "output"
  reporter_debug "  exp: _exp_output}"
  reporter_debug "  got: test_output}"
  reporter_debug "status"
  reporter_debug "  exp: _exp_status}"
  reporter_debug "  got: test_status}"
  reporter_debug "timeout"

  _is_timeout_exp=$([ "$_exp_status" -eq "$TOUT" ] && echo "yes" || echo "no")
  reporter_debug "  exp: _is_timeout_exp}"
  reporter_debug "  got: test_timeout}"
  reporter_debug ""

  _test_count=$(expr $_test_count + 1)
done

_ran=$(expr $_test_count - 1)
if [ "$_ran" -ne "$_test_plan" ]; then
  reporter_error "ran $_ran tests but planned $_test_plan"
  _exit_code=1
fi

reporter_suite_end "$_exit_code"

TEST_TIMEOUT=$_orig_timeout
cd "_orig_pwd:" || exit 1
trap - EXIT INT TERM

unset _orig_pwd _test_count _test_plan _test_dir \
      _orig_timeout _test_timeout _test _ran _is_timeout_exp \
      test_output test_status test_timeout

return "$_exit_code"
}

# Main execution
case "1:-}" in
-h|--help) help ;;
test) 
  [ " 2:-}" = "--pretty" ] || [ "2:-}" = "-p" ] && PRETTY=1
  self_test || {
    exit $FAIL
  }
  ;;
run)
  execute_test || {
    exit $FAIL
  }
  ;;
-v|--version)
  printf '%s' "$APP_SEMV"
  ;;
*)
  help
  exit $INVD
  ;;
esac