Diff the stdout, stderr and return code of a program with known stdout, stderr, and return code

progtest.sh

#!/bin/bash

# CS 343 - A#Q# Test Script
# by m9chang
# based on progtest.sh by m9chang
# based on progdiff.sh by jlfwong (available at https://gist.github.com/3829517)

retval1=0 retval2=0 retcode=0

#    Method: run
#    Purpose: Execute a command with the given arguments and input, and records its output, stderr, and return code.
#    Returns: The return code from the execution of the command.
#    Errors: The executable is not found. Script is terminated.
#    Globals:
#        sandbox - temporary directory to store files for this execution
run () {
    local srcpath="$1" trialnum="$2" retval=0
    shift 2

    if [ ! -x "$srcpath" ]; then
        echo "No executable found at $(pwd)/$srcpath"
        exit 1
    fi

    local curdir="$(pwd)" execpath="$sandbox/exec"
    cp "$srcpath" "$execpath"
    cd $sandbox

    if $hasstdin; then
        ./exec < "$sandbox/stdin.txt" > "$sandbox/stdout.$trialnum.txt" 2> "$sandbox/stderr.$trialnum.txt" $@
        retval=$?
    else
        ./exec > "$sandbox/stdout.$trialnum.txt" 2> "$sandbox/stderr.$trialnum.txt" $@
        retval=$?
    fi

    cd $curdir
    return $retval
}

#    Method: tc_run
#    Purpose: Convenience wrapper around run
#    Returns, Errors, Globals: See run
tc_run() {
  prog="$1"
  shift 1
  run $prog 2 $@
  retval2=$?
}

#    Method: tc_def
#    Purpose: Execute a test case and report (differences in) its output, stderr, and return code.
#    Returns: void
#    Errors: The executable is not found. Script is terminated. If recording mode is disabled, and there is a
#        difference in the output, stderr, or return code between the test case and the results from tc_run, set a flag
#        to cause the script to return an error after executing all tests.
#    Globals:
#        sandbox - temporary directory to store files for this execution
#        hasstdin - determines if data needs to be passed in to stdin while executing the program
#        recording - determine if we should record the output, stderr, and return code and display it to the user (in
#            lieu of comparing it to the expect output)
#        retval1 - expected return value
#        retval2 - return value from run
tc_def() {
  sandbox="$(mktemp -d)" hasstdin=true recording=false tcname="$1"
  shift 1

  tc_init_$tcname
  tc_run_$tcname

  echo -e "\e[00;32m# Test Case: $tcname\e[00m"

  if $recording; then
    echo -e "  hasstdin=$hasstdin"
    echo -e "  retval1=$retval2"

    if $hasstdin; then
      echo -e "  cat > \"\$sandbox/stdin.txt\" <<TC_STDIN_${tcname^^}"
      cat "$sandbox/stdin.txt"
      echo -e "TC_STDIN_${tcname^^}"
    fi;

    echo -e "  cat > \"\$sandbox/stdout.1.txt\" <<TC_STDOUT_${tcname^^}"
    cat "$sandbox/stdout.2.txt"
    echo -e "TC_STDOUT_${tcname^^}"

    echo -e "  cat > \"\$sandbox/stderr.1.txt\" <<TC_STDERR_${tcname^^}"
    cat "$sandbox/stderr.2.txt"
    echo -e "TC_STDERR_${tcname^^}"
  else
    if [ "$retval1" -ne "$retval2" ]; then
        retcode=1
        echo -e "\e[00;31mERROR: Return code differs.\e[00m"
        echo -e "\texpected $retval1, got $retval2"
        echo
    fi
    
    diff -y "$sandbox/stdout.1.txt" "$sandbox/stdout.2.txt" > "$sandbox/stdout.diff.txt"
    if [ "$?" -ne "0" ]; then
        retcode=1
        echo -e "\e[00;31mERROR: Standard output differs.\e[00m"
        cat "$sandbox/stdout.diff.txt"
    fi
    
    diff -y "$sandbox/stderr.1.txt" "$sandbox/stderr.2.txt" > "$sandbox/stderr.diff.txt"
    if [ "$?" -ne "0" ]; then
        retcode=1
        echo -e "\e[00;31mERROR: Standard error differs.\e[00m"
        cat "$sandbox/stderr.diff.txt"
    fi
  fi
  rm -r $sandbox #clean up
}

#    Test Case: retval (Return Value)
#    Purpose: Check a return value
#    Returns, Errors, Globals: See tc_def
tc_init_retval() {
  hasstdin=false
  retval1=1
  cat > "$sandbox/stdout.1.txt" <<TC_STDOUT_RETVAL
TC_STDOUT_RETVAL
  cat > "$sandbox/stderr.1.txt" <<TC_STDERR_RETVAL
TC_STDERR_RETVAL
}
tc_run_retval() { tc_run $(which false); }
tc_def retval

#    Test Case: retvalfail (Return Value Fail)
#    Purpose: Check a failing return value
#    Returns, Errors, Globals: See tc_def
tc_init_retvalfail() { tc_init_retval; } # Same as retval, above
tc_run_retvalfail() { tc_run $(which true); }
tc_def retvalfail

#    Test Case: rec (Recording Mode)
#    Purpose: Record some output.
#    Returns, Errors, Globals: See tc_def
tc_init_rec() {
  recording=true
  #hasstdin=false
  retval1=0
  cat > "$sandbox/stdin.txt" <<TC_STDIN_REC
A
TC_STDIN_REC
  cat > "$sandbox/stdout.1.txt" <<TC_STDOUT_REC
A
TC_STDOUT_REC
  cat > "$sandbox/stderr.1.txt" <<TC_STDERR_REC
TC_STDERR_REC
}
tc_run_rec() { tc_run $(which cat); }
tc_def rec 

# End of Test Cases
if [ "$retcode" -eq "0" ]; then echo 'All tests passed.'; fi;
exit $retcode