TAP-14 compliant zsh test runner; 1 file

ok.zsh

#!/usr/bin/env zsh
#
# ok.zsh -- tiny TAP 14 test runner for zsh
#
# Author:  https://github.com/nicholaswmin
# License: MIT
#
# Discover and run *.test.zsh files. Each file runs in its own subshell
# with the assertion API preloaded, so test files call the assertors
# directly -- no `source` line.
#
# Usage:
#   ok.zsh <dir>        run every *.test.zsh under <dir>, recursively
#   ok.zsh -h, --help   print this help and exit
#
# A test file:
#   plan 3
#   is     "hello" "hello"        "literal equality"
#   like   "hello world" "*world*" "anchored glob"
#   unlike "abc" "x*"             "negated glob"
#   finish
#
# assertions:
#   plan N                       declare N assertions; emits 1..N
#   is     actual expected desc  literal string equality
#   like   actual pattern  desc  anchored zsh glob match
#   unlike actual pattern  desc  inverse of like
#   runs   N cmd... desc         run cmd; assert exit N; sets REPLY=output
#   runs_like N pat cmd... desc  run cmd; assert exit N and glob match
#   ok     desc                  low-level pass
#   not_ok desc                  low-level fail
#   diag   msg                   TAP comment
#   register name cond diag      define a custom assertor
#   finish                       end the file; sets its exit status
#
# Custom assertors:
#   - `register name cond diag` creates a function named `name`
#   - on call, $1..$N are predicate args; last positional is the desc
#   - cond is a test condition; diag is the failure message
#   - single-quote cond and diag so $1, $2 expand at call time
#
#   register file_exists '[[ -f $1 ]]'    'expected file at: $1'
#   register matches     '[[ $1 == $2 ]]' 'expected $2, got $1'
#   file_exists /etc/hostname "hostname is a file"
#   matches "a" "a" "literal equality"
#
# Exit codes:
#   0   every test file passed
#   1   one or more files failed, or no test files were found
#   2   usage error (bad option, missing/extra operand, bad directory)
#
# Behaviour:
#   - discovery is recursive: <dir>/**/*.test.zsh, sorted, plain files only
#   - each file runs in its own subshell, so `finish`'s exit cannot leak
#     into the runner and per-file counters start from zero
#   - each file emits an independent TAP 14 stream on stdout (version
#     line + plan + assertions); progress and summary go to stderr
#   - a file that never calls `finish` fails (its plan stays unverified)
#   - color is emitted only when stderr is a terminal, from the ANSI palette
#   - assertion semantics: plan under/overflow fails the run; "#", CR, LF
#     in descriptions are stripped (with a warning); "like" is anchored --
#     use "*foo*" for substrings

emulate -L zsh
setopt err_return pipe_fail no_unset extended_glob warn_create_global

typeset -gi __ok_num=0
typeset -gi __ok_plan=0
typeset -gi __ok_failed=0

# --- assertion API ---------------------------------------------------------
# Preloaded into every test file's subshell. Each uses `emulate -L zsh` so a
# test file's setopt cannot leak in.

plan() {
  emulate -L zsh
  __ok_plan=$1
  print -r -- "1..$__ok_plan"
}

diag() {
  emulate -L zsh
  local line
  for line in ${(f)${*//$'\r'/}}; do
    print -r -- "# $line"
  done
}
_ok_sanitise() {
  emulate -L zsh
  local raw=$1 clean
  clean=${${${raw//$'\r'/}//$'\n'/ }//\#/}
  if [[ $clean != $raw ]]; then
    print -ru2 -- "ok: warning: stripped unsafe chars from description: $raw"
  fi
  REPLY=$clean
}

ok() {
  emulate -L zsh
  local REPLY
  _ok_sanitise $1
  (( ++__ok_num ))
  print -r -- "ok $__ok_num - $REPLY"
}

not_ok() {
  emulate -L zsh
  local REPLY
  _ok_sanitise $1
  (( ++__ok_num ))
  (( ++__ok_failed ))
  print -r -- "not ok $__ok_num - $REPLY"
}

is() {
  emulate -L zsh
  local actual=$1 expected=$2 desc=$3
  if [[ $actual == $expected ]]; then
    ok $desc
    return 0
  fi
  not_ok $desc
  diag "expected: $expected"
  diag "got:      $actual"
  return 1
}

like() {
  emulate -L zsh
  local actual=$1 pattern=$2 desc=$3
  if [[ $actual == ${~pattern} ]]; then
    ok $desc
    return 0
  fi
  not_ok $desc
  diag "expected pattern: $pattern"
  diag "got:             $actual"
  return 1
}

unlike() {
  emulate -L zsh
  local actual=$1 pattern=$2 desc=$3
  if [[ $actual != ${~pattern} ]]; then
    ok $desc
    return 0
  fi
  not_ok $desc
  diag "unexpected pattern: $pattern"
  diag "got:               $actual"
  return 1
}

runs() {
  emulate -L zsh
  local -i expected=$1; shift
  local desc=${@[-1]}
  local -a cmd=( ${@[1,-2]} )
  local out
  local -i actual=0
  out=$( $cmd 2>&1 ) || actual=$?
  REPLY=$out
  if (( actual == expected )); then
    ok $desc
    return 0
  fi
  not_ok $desc
  diag "expected exit: $expected"
  diag "got:           $actual"
  diag "output:        $out"
  return 1
}

runs_like() {
  emulate -L zsh
  local -i expected=$1
  local pattern=$2
  shift 2
  local desc=${@[-1]}
  local -a cmd=( ${@[1,-2]} )
  local out
  local -i actual=0
  out=$( $cmd 2>&1 ) || actual=$?
  REPLY=$out
  if (( actual == expected )) && [[ $out == ${~pattern} ]]; then
    ok $desc
    return 0
  fi
  not_ok $desc
  (( actual != expected )) && {
    diag "expected exit: $expected"
    diag "got:           $actual"
  }
  [[ $out != ${~pattern} ]] && {
    diag "expected pattern: $pattern"
    diag "got:              $out"
  }
  return 1
}

register() {
  emulate -L zsh
  local name=$1 cond=$2 diag_msg=${3-}
  [[ -z $name || -z $cond ]] && {
    print -ru2 -- "ok: register: name and condition required"
    return 2
  }
  case $name in
    plan|is|like|unlike|runs|runs_like|ok|not_ok|diag|finish|register|_ok_sanitise)
      print -ru2 -- "ok: register: cannot shadow built-in: $name"
      return 2
      ;;
  esac
  functions[$name]="
    emulate -L zsh
    local desc=\${@[-1]}
    if $cond; then ok \$desc; return 0; fi
    not_ok \$desc
    [[ -n \"$diag_msg\" ]] && diag \"$diag_msg\"
    return 1
  "
}
finish() {
  emulate -L zsh
  (( __ok_num != __ok_plan )) &&
    diag "planned $__ok_plan but ran $__ok_num"
  exit $(( __ok_failed > 0 || __ok_num != __ok_plan ))
}

# --- runner ----------------------------------------------------------------

typeset -g _C_RED='' _C_GREEN='' _C_MUTE='' _C_RESET=''
if [[ -t 2 && -z ${NO_COLOR-} ]]; then
  _C_RED=${(%):-%F{red}}
  _C_GREEN=${(%):-%F{green}}
  _C_MUTE=${(%):-%F{8}}
  _C_RESET=${(%):-%f}
fi

# _say <color-var-name> <msg...>  -- write a colored line to stderr
_say() {
  emulate -L zsh
  local color=$1; shift
  print -ru2 -- "${(P)color}$*${_C_RESET}"
}

_usage() {
  emulate -L zsh
  print -r -- 'ok.zsh -- tiny zsh test runner

usage:
  ok.zsh <dir>        run every *.test.zsh under <dir>, recursively
  ok.zsh -h, --help   print this help and exit

A test file calls the assertion API directly -- no source line:

  plan 2
  is   "$(id -un)" root   "running as root"
  like "$PWD" "/*"        "absolute cwd"
  finish

assertions:
  plan N                       declare N assertions; emits 1..N
  is     actual expected desc  literal string equality
  like   actual pattern  desc  anchored zsh glob match
  unlike actual pattern  desc  inverse of like
  runs   N cmd... desc         run cmd; assert exit N; sets REPLY=output
  runs_like N pat cmd... desc  run cmd; assert exit N and glob match
  ok     desc                  low-level pass
  not_ok desc                  low-level fail
  diag   msg                   TAP comment
  register name cond diag      define a custom assertor
  finish                       end the file; sets its exit status

exit status:
  0  every test file passed
  1  a file failed, or no test files were found
  2  usage error'
}

# usage error: message, blank line, help -- all to stderr; exit 2
_die() {
  emulate -L zsh
  _say _C_RED "ok: $*"
  print -ru2 --
  _usage >&2
  exit 2
}

# Run one test file. The subshell inherits the API, contains `finish`'s exit,
# and starts from the zeroed global counters. `finish` is what validates the
# plan, so reaching past `source` on a clean run means it was never called.
_run_file() {
  emulate -L zsh
  print -r -- 'TAP version 14'
  source $1 &&
    _say _C_RED "ok: $1: finished without calling finish"
  exit 1
}

main() {
  emulate -L zsh
  local -a opt_help
  zparseopts -D -E -- h=opt_help -help=opt_help

  (( $#opt_help )) && { _usage; return 0 }

  local arg
  for arg in $argv; do
    [[ $arg == -* ]] && _die "unknown option: $arg"
  done
  (( $#argv == 0 )) && _die "missing directory operand"
  (( $#argv > 1 ))  && _die "too many arguments"

  local dir=$argv[1]
  [[ -e $dir ]] || _die "no such file or directory: $dir"
  [[ -d $dir ]] || _die "not a directory: $dir"

  local -a files=( $dir/**/*.test.zsh(.N) )
  (( $#files )) || {
    _say _C_RED "ok: no *.test.zsh files under: $dir"
    return 1
  }

  local plural=s
  (( $#files == 1 )) && plural=''
  _say _C_MUTE "running $#files test file$plural under $dir ..."

  integer passed=0 failed=0
  local f
  for f in $files; do
    if ( _run_file $f ); then
      (( ++passed ))
      _say _C_MUTE "ok   $f"
    else
      (( ++failed ))
      _say _C_RED "FAIL $f"
    fi
  done

  local prefix=all\ 
  (( $#files == 1 )) && prefix=''
  (( failed == 0 )) && {
    _say _C_GREEN "${prefix}$#files file$plural passed"
    return 0
  }
  _say _C_RED "$failed of $#files file$plural failed"
  return 1
}

main $@ || exit $?