Shell test prototype

README.md

Description

This is a prototype for doing shell tests. To run:

$ ./test/suite.sh

To add another test suite, make a script in the test directory that ends with _test.sh and ensure it is executable. Then follow the pattern in example_test.sh:

#!/bin/sh
. ${0%/$TEST_CASE}/helper.sh

test_echo_echos_a_string_to_stdout () {
# The assert_equal method will check that the echo command exits
# with status 0, and that the stdout is 'hello world'.
#
# There are similar assertions to just check the exit status
# or just check stdout.

  assert_equal 0 "$(
  echo 'hello world'
)" $LINENO <<stdout
hello world
stdout

# Write as many assertions as you please in a given method.
}

run_test_case

You can write as many test methods as you like... just follow the same conventions as in ruby's Test::Unit (ie start the method with test_).

Commentary

I think shell-based testing has a lot of potential. One nice feature is that there are no special requirements for the server -- you don't have to install ruby, or anything else. This prototype should only require POSIX compatibility. I've only tested it with bash, but I think it should work with zsh, tcsh, or whaterver-sh.

Another nice feature is that it allows you to use the same commands you use already. There is a great correspondence between the checks you'd manually run and what goes in here.

Obviously it's a bit confusing if you're unfamiliar with the shell, but there are ways around that... like learning the shell, or potentially something like linecook to which I intend to add support for this kind of thing.

example_test.sh

#!/bin/sh
. "$TS_UNIT_TEST"

#
# assert_equal test
#

# set | grep "TS_"
# TS_BIN_DIR=/Users/bahuvrihi/Documents/Gems/1128147/bin
# TS_TEST=test_assert_equal_pass
# TS_TEST_CASE=example_test
# TS_TEST_CASE_DIR=/var/folders/hJ/hJgPGtAvG6WpSaJZENLmGU+++TU/-Tmp-/example_test
# TS_TEST_DIR=/var/folders/hJ/hJgPGtAvG6WpSaJZENLmGU+++TU/-Tmp-/example_test/test_assert_equal_pass
# TS_TEST_FILE=test/example_test.sh
# TS_TMP_DIR=/var/folders/hJ/hJgPGtAvG6WpSaJZENLmGU+++TU/-Tmp-
# TS_UNIT_TEST=/Users/bahuvrihi/Documents/Gems/1128147/bin/ts-unit-test
# TS_USER_DIR=/Users/bahuvrihi/Documents/Gems/1128147

test_assert_equal_pass () {
  assert_equal 0 "$(
echo 'hello world'
echo 'hello world'
echo 'hello world'
)" $LINENO <<stdout
hello world
hello world
hello world
stdout
}

test_assert_equal_fail_by_status () {
  assert_equal 1 "$(
echo 'hello world'
echo 'hello world'
echo 'hello world'
)" $LINENO <<stdout
hello world
hello world
hello world
stdout
}

test_assert_equal_fail_by_output () {
  assert_equal 0 "$(
echo 'hello world'
echo 'ellow world'
echo 'hello world'
)" $LINENO <<stdout
hello world
hello world
hello world
stdout
}

#
# assert_status_equal test
#

test_assert_status_equal_pass () {
  true
  assert_status_equal 0 $? $LINENO
}

#
# assert_output test
#

test_assert_output_equal_pass () {
  assert_output_equal "$(
echo 'hello world'
echo 'hello world'
echo 'hello world'
)" $LINENO <<stdout
hello world
hello world
hello world
stdout
}

#
# assert test
#

test_assert_pass () {
  assert true
  assert [ 1 -eq 1 ]
}

test_assert_fail () {
  assert false
}

#
# ts test
#

test_ts_can_capture_stdout_and_stderr () {
exec >> "$TS_TEST_LOG" 2>&1
echo 'hello world'
echo 'goodnight moon' >&2
exit 1
}

test_ts_provides_message_on_exit () {
  exit 1
}

run_test

fail_test.sh

#!/bin/sh
. "$TS_UNIT_TEST"

test_fail () {
false
}

test_exit_fail () {
exit 8
}

test_assert_fail () {
assert false
}

test_assert_str_fail () {
assert_str "hello world" "hello w0rld" $LINENO
}

test_assert_file_fail () {
mkdir -p "$TS_TEST_DIR"
file="$TS_TEST_DIR/file.txt"
echo 'hello world' > "$file"

assert_file "$file" $LINENO <<doc
hello w0rld
doc
}

test_run_status_fail () {
run false
assert_status 0 $status $LINENO
}

test_run_output_fail () {
run echo 'hello world'
assert_output "$output" $LINENO <<doc
hello w0rld
doc
}

test_capture_output_fail () {
capture
echo 'hello world'
assert_file "$output" $LINENO <<doc
hello w0rld
doc
}

run_test

pass_test.sh

#!/bin/sh
. "$TS_UNIT_TEST"

test_pass () {
true
}

test_exit_pass () {
exit
}

test_assert_pass () {
assert true $LINENO
}

test_assert_str_pass () {
assert_str "hello world" "hello world" $LINENO
}

test_assert_file_pass () {
mkdir -p "$TS_TEST_DIR"
file="$TS_TEST_DIR/file.txt"
echo 'hello world' > "$file"

assert_file "$file" $LINENO <<doc
hello world
doc
}

test_run_status_pass () {
run true
assert_status 0 $status $LINENO
}

test_run_output_pass () {
run echo 'hello world'
assert_output "$output" $LINENO <<doc
hello world
doc
}

test_capture_output_pass () {
capture
echo 'hello world'
assert_file "$output" $LINENO <<doc
hello world
doc
}

run_test

teardown_exit_test.sh

#!/bin/sh
. "$TS_UNIT_TEST"

teardown () {
  exit 2
}

test_pass_on_teardown_fail () {
true
}

test_fail_on_teardown_fail () {
false
}

run_test

ts

#!/bin/sh
############################################################################
tmpdir=${TMPDIR:-tmp}
xtrace=false

usage="usage: %s [-h] [-t TMPDIR] [-x] [TEST_FILES...]\n"
option="       %s   %s\n"
while getopts "ht:x" opt
do
  case $opt in
  h  )  printf "$usage" "$0"
        printf "$option" "-h" "prints this help"
        printf "$option" "-t" "test temp dir"
        printf "$option" "-x" "xtrace this script"
        exit 0 ;;
  t  )  tmpdir=$OPTARG ;;
  x  )  xtrace=true ;;
  \? )  printf "$usage" "$0"
        exit 2 ;;
  esac
done
shift $(($OPTIND - 1))

mkdir -p "$tmpdir"
tmpdir="$(cd "$tmpdir"; pwd)"

export TS_USER_DIR="$(pwd)"
export TS_BIN_DIR="$(cd "$(dirname "$0")"; pwd)"
export TS_TMP_DIR="${tmpdir%/}"
export TS_UNIT_TEST="$TS_BIN_DIR/ts-unit-test"
export TS_TEST_FILE
export TS_TEST_CASE
export TS_TEST_CASE_DIR
export TS_TEST_LINENO
export TS_TEST_NAME
export TS_TEST_DIR
export TS_TEST_LOG
export TS_LOG="$TS_TMP_DIR/log"

npass=0
nfail=0
touch "$TS_LOG"

if [ "$xtrace" = "true" ]
then set -x
fi
############################################################################

#
#  Run the test cases
#

printf "Started\n"

start_time=$SECONDS
for TS_TEST_FILE in "$@"
do
  if [ -x "$TS_TEST_FILE" ] && [ -f "$TS_TEST_FILE" ]
  then
    TS_TEST_CASE="$(basename "${TS_TEST_FILE%\.*}")"
    TS_TEST_CASE_DIR="$TS_TMP_DIR/$TS_TEST_CASE"
    mkdir -p "$TS_TEST_CASE_DIR"

    for test in $(
      grep -onE "^ *${NAME:-test_\w+} +\(\)" "$TS_TEST_FILE" |
      tr -d " ()"
    )
    do
      TS_TEST_LINENO=$(echo "$test" | cut -d ':' -f 1)
      TS_TEST_NAME=$(echo "$test" | cut -d ':' -f 2)
      TS_TEST_DIR="$TS_TEST_CASE_DIR/$TS_TEST_NAME"
      TS_TEST_LOG="$TS_TEST_DIR.log"

      rm -rf "$TS_TEST_DIR"
      printf "[$TS_TEST_FILE:$TS_TEST_LINENO] $TS_TEST_NAME\n" > "$TS_TEST_LOG"

      if "$TS_TEST_FILE" "$TS_TEST_NAME"
      then
        printf "."
        npass=$(( npass + 1 ))
      else
        printf "exit: $?\n" >> "$TS_TEST_LOG"
        cat "$TS_TEST_LOG" >> "$TS_LOG"
        printf "\n" >> "$TS_LOG"
        printf "F"
        nfail=$(( nfail + 1 ))
      fi

      if [ "${KEEP_OUTPUTS:-false}" != "true" ]
      then rm -rf "$TS_TEST_DIR" "$TS_TEST_DIR".*
      fi
    done

    rmdir "$TS_TEST_CASE_DIR" 2>/dev/null
  elif ! [ -f "$TS_TEST_FILE" ]
  then
    printf "-"
    printf "[$TS_TEST_FILE] not a file\n\n" >> "$TS_LOG"
  else
    printf "-"
    printf "[$TS_TEST_FILE] not executable\n\n" >> "$TS_LOG"
  fi
done
end_time=$SECONDS

printf "\nFinished in $(($end_time - $start_time))s\n\n"
cat "$TS_LOG"
printf "$npass pass, $nfail fail\n"

rm "$TS_LOG"
rmdir "$TS_TMP_DIR" 2>/dev/null

ts-unit-test

#
#  Defines assertions and other functions needed
#  to run the tests.
#

output=
status=

setup () {
  true
}

teardown () {
  true
}

flunk () {
  printf "[$TS_TEST_FILE:${lineno:-$TS_TEST_LINENO}] $TS_TEST_NAME\n$1\n" 1>"$TS_TEST_LOG"
  exit 1
}

assert_status () {
  expected=$1; actual=$2; lineno=$3

  if [ $actual -ne $expected ]
  then flunk "expected exit status $expected but was $actual"
  fi
}

assert_output () {
  expected=$(cat); actual=$1; lineno=$2

  if [ "$actual" != "$expected" ]
  then
    echo -e "$expected" > "$TS_TEST_DIR.expect"
    echo -e "$actual"   > "$TS_TEST_DIR.actual"

    flunk "unequal stdout:\n$(diff "$TS_TEST_DIR.expect" "$TS_TEST_DIR.actual")"

    rm "$TS_TEST_DIR.expect" "$TS_TEST_DIR.actual"
    exit 1
  fi
}

assert () {
  assert_status 0 $1 $2
}

assert_file () {
  if [ -f "$1" ]
  then assert_output "$(cat "$1")" $2
  else flunk "no such file: $1"
  fi
}

assert_str () {
  [ "$1" = "$2" ]
  assert $? $3
}

capture () {
  mkdir -p "$TS_TEST_DIR"
  exec >> "$TS_TEST_DIR/output" 2>&1
  output="$TS_TEST_DIR/output"
}

run () {
  output="$($@)"
  status=$?
}

run_test () {
  output=
  status=

  setup
  "$TS_TEST_NAME"
  trap "exit $?" exit
  teardown
}