progtest.sh

#!/bin/bash


VERSION='3.0.0'


BOLD='\033[1m'
RED='\033[0;31m'
GREEN='\033[0;32m'
ORANGE='\033[0;33m'
DARKGRAY='\033[0;90m'
NC='\033[0m'


error () {
    echo -e "${RED}$1${NC}" >&2
}


usage () {
    echo
    echo -e "${BOLD}USAGE:${NC}"
    echo -e " -> ${BOLD}$0 [SOURCE_FILE ...]${NC}"
    echo -e "    - where ${BOLD}SOURCE_FILE${NC} is an absolute or relative path to"
    echo -e "      the .c file you would like to test, not a compiled binary."
    echo -e "      You can test several files at once, just pass them in as"
    echo -e "      separate arguments."
    echo
    echo -e " -> ${BOLD}$0 [--help | -h]${NC}"
    echo -e "    - print more information about the usage of the script."
    echo
    echo -e " -> ${BOLD}$0 [--version | -v]${NC}"
    echo -e "    - print more detailed information about the version of the script and its dependencies."
    if [[ $# -ne 1 && ! "$1" == "--no-link" ]]
    then
        echo
        echo -e "${BOLD} See ${ORANGE}$0 --help${NC}${BOLD} for more info."
    fi
}


print_header () {
    echo -e "${BOLD}PA1 ProgTest validation script${NC}"
}


print_bugs () {
    echo
    echo -e "${BOLD}BUGS:${NC}"
    echo -e "I'm pretty much a noob with shell scripting, so if this thing doesn't work the way it should then please let me know in person,"
    echo -e "on Discord (${ORANGE}jakcharvat#3310${NC}), or if ur really into that sort of stuff through email (${ORANGE}mailto:[email protected]${NC})"
    echo -e "${GREEN}${BOLD}Thanks :)${NC}"
}


print_plug () {
    echo
    echo -e "${BOLD}SMALL PLUG:${NC}"
    echo -e "Written by ${ORANGE}${BOLD}Jakub Charvat${NC}. Pls check out my GitHub (${BOLD}https://github.com/jakcharvat${NC}). TY ${RED}${BOLD}<3${NC}"
}


print_help () {
    print_header
    echo "Version ${VERSION}"
    echo
    echo -e "${BOLD}OVERVIEW:${NC}"
    echo "A script to simplify testing ProgTest challenge solutions. The script only needs the"
    echo "path to the file that should be tested, and handles all compilation and testing itself."
    echo
    echo -e "${BOLD}REQUIREMENTS:${NC}"
    echo "For simplicity, this script makes the following assumption about your file structure:"
    echo -e "  The test data for the solution you would like to test lives in a directory called ${ORANGE}\"sample\"${NC}"
    echo "  that's located in the same directory as the solution. So if your solution file is called"
    echo -e "  ${ORANGE}solution.c${NC}, then the file tree would look like so:"
    echo
    echo "      projectDir/"
    echo "        |- solution.c"
    echo "        |- sample/"
    echo "             |- CZE/"
    echo "                  |- 0000_in.txt"
    echo "                  |- 0000_out.txt"
    echo "                  |- 0001_in.txt"
    echo "                  |- 0001_out.txt"
    echo "                         ..."
    echo
    echo "  This structure can be located anywhere in your filesystem, and when calling the script"
    echo -e "  you must point at the ${ORANGE}solution.c${NC} file as the only argument."
    usage --no-link
    print_bugs
    print_plug
}


print_version () {
    print_header
    echo -e "Version ${ORANGE}${VERSION}${NC}"
    echo
    echo -e "${BOLD}DEPENDENCIES:${NC}"
    echo -e " -> ${ORANGE}${BOLD}bash${NC}"
    echo -e " -> ${ORANGE}${BOLD}g++${NC}"
    echo -e "     - On Darwin (macOS) the script uses g++-11, available from"
    echo -e "       ${BOLD}https://formulae.brew.sh/formula/gcc#default${NC}."
    echo -e "       This is because by default the g++ command links to Xcode's"
    echo -e "       clang C compiler, whereas the brew version is actual GCC."
    echo -e " -> a bunch of other ${DARKGRAY}${BOLD}(hopefully)${NC} built-in commands:"
    echo -e "     - ${ORANGE}echo${NC}"
    echo -e "     - ${ORANGE}basename${NC}"
    echo -e "     - ${ORANGE}dirname${NC}"
    echo -e "     - ${ORANGE}grep${NC}"
    echo -e "     - ${ORANGE}sed${NC}"
    echo -e "     - ${ORANGE}diff${NC}"
    echo -e "     - ${ORANGE}timeout${NC}"
    echo -e "     - ${ORANGE}trap${NC}"
    print_bugs
    print_plug
}


builddir () {
    DIR='./.builddir'
    if [[ ! "$1" == "--no-create" ]]
    then
        if [[ ! -d "$DIR" ]]
        then
            mkdir -p "$DIR"
        fi
    fi
    echo "$DIR"
}


cleanup_exit () {
    if [[ -d "$( builddir --no-create )" ]]
    then
        rm -r "$( builddir --no-create )"
    fi
    exit $1
}


sigint_exit () {
    error
    error
    error "SIGINT: Cleaning up and exiting..."
    cleanup_exit 130
}


trap sigint_exit INT


if [[ $# -lt 1 ]]
then
    error "ERROR: You must provide at least one argument - either"
    error "       an input file or a script flag."
    usage >&2
    cleanup_exit 2
fi


PRINT_HELP=0
PRINT_VERSION=0
SOURCE_FILES=""

while [[ $# -gt 0 ]]
do
    ARG="$1"

    if [[ "$ARG" == "--help" || "$ARG" == "-h" ]]
    then
        PRINT_HELP=1
    elif [[ "$ARG" == "--version" || "$ARG" == "-v" ]]
    then
        PRINT_VERSION=1
    else
        if [[ ! -z "$SOURCE_FILES" ]]
        then
            SOURCE_FILES="$SOURCE_FILES "
        fi
        SOURCE_FILES="$SOURCE_FILES$ARG"
    fi

    if [[ ! -z "$SOURCE_FILES" && $PRINT_HELP != 0 ]]
    then
        error "ERROR: Combining help flag with sources."
        usage >&2
        cleanup_exit 2
    fi

    if [[ ! -z "$SOURCE_FILES" && $PRINT_VERSION != 0 ]]
    then
        error "ERROR: Combining version flag with sources."
        usage >&2
        cleanup_exit 2
    fi

    if [[ $PRINT_HELP != 0 && $PRINT_VERSION != 0 ]]
    then
        error "ERROR: Combining version flag with help flag."
        usage >&2
        cleanup_exit 2
    fi

    shift
done


if [[ $PRINT_HELP == 1 ]]
then
    print_help
    cleanup_exit 0
fi


if [[ $PRINT_VERSION == 1 ]]
then
    print_version
    cleanup_exit 0
fi



# ========================= BUILD PHASE =========================
compile() {
    GCC="g++"
    if [[ "$( uname -s )" == "Darwin" ]]; then GCC="g++-11"; fi
    "$GCC" -Wall -pedantic "$1" -o "$( builddir )/$2"
}


copy_tests () {
    SAMPLE_DIR="$( dirname "$1" )/sample"

    if [[ -d "$( builddir --no-create )/tests/$2" ]]
    then
        rm -r "$( builddir --no-create )/tests/$2"
    fi

    mkdir -p "$( builddir )/tests/$2"
    ASSERT_MODE=0
    
    if [[ -d "$SAMPLE_DIR/CZE" ]]
    then
        cp "$SAMPLE_DIR/CZE"/* "$( builddir )/tests/$2/"
    elif [[ -d "$SAMPLE_DIR" ]]
    then
        TEST_DIRS=$( find "$SAMPLE_DIR" -name 'CZE' )
        for DIR in $TEST_DIRS
        do
            TEST_NAME="$( basename "$( dirname "$DIR" )" )"
            mkdir -p "$( builddir )/tests/$2/$TEST_NAME"
            cp "$DIR"/* "$( builddir )/tests/$2/$TEST_NAME"
        done
    else
        ASSERT_MODE=1
    fi
}


get_build_file_base() {
    DIR="$( dirname "${1##*./}" )"
    BASE="$( tr '/' '-' <<<"${DIR#/}" )"
    if [[ -z "$( echo "$BASE" | tr -d '.' | tr -d '/' )" ]]
    then
        BASE="$( basename "${1%.*}" )"
    fi
    echo "$BASE"
}


get_build_file() {
    echo "$1.out"
}


build_source_file() {
    SOURCE_FILE="$1"
    echo -ne "${BOLD}Building file ${ORANGE}${SOURCE_FILE}${NC}${BOLD}..."
    BUILD_FILE_BASE=$( get_build_file_base "$SOURCE_FILE" )
    BUILD_FILE=$( get_build_file "$BUILD_FILE_BASE" )
    copy_tests "$SOURCE_FILE" "$BUILD_FILE_BASE"
    ERROR="$( compile "$SOURCE_FILE" "$BUILD_FILE" 2>&1 )"
    GCC_EXIT="$?"
    if [[ "$GCC_EXIT" -ne 0 ]]
    then
        echo
        echo -e "$ERROR"
        echo -e "${RED}ERROR Compiling ${SOURCE_FILE}. There should be more info above."
        cleanup_exit "$GCC_EXIT"
    fi
    echo -ne "${GREEN}${BOLD} BUILT ${BUILD_FILE}${NC}"
    if [[ "$ASSERT_MODE" == 1 ]]; then echo -ne "${ORANGE} -- Assert mode${NC}"; fi
    echo
    if [[ ! -z "$ERROR" ]]; then echo -e "${DARKGRAY}${ERROR}${NC}"; fi
    echo
}


if [[ -d "$( builddir --no-create )" ]]; then rm -r "$( builddir --no-create )"; fi

for SOURCE_FILE in $SOURCE_FILES
do
     if [[ ! -e "$SOURCE_FILE" ]]
     then
         if [[ -z "$SOURCE_FILE" ]]; then SOURCE_FILE="NO SOURCE"; fi
         echo -e "${RED}ERROR Cannot find a source file in the specified path: ${BOLD}../$1${RED} (${BOLD}${SOURCE_FILE}${RED}).${NC}" >&2
         cleanup_exit 2
     fi
     build_source_file "$SOURCE_FILE"
done



# ========================= TEST PHASE =========================
ERROR_COUNT=0
TIMEOUT_COUNT=0
for BINARY in "$( builddir )"/*.out
do
    echo
    echo
    BINARY_NAME="$( basename "$BINARY" )"
    echo -ne "${BOLD}Testing ${ORANGE}${BINARY_NAME}${NC}"

    # the %.* drops the last component of the filename - the extension
    BASENAME="${BINARY_NAME%.*}"

    if [[ -z "$BASENAME" ]]; then BASENAME="$( basename "$BINARY" )"; fi
    TEST_DIR="$( dirname "$BINARY" )/tests/$BASENAME"

    if [[ ! -d "$TEST_DIR" || ! "$( ls -A "$TEST_DIR" )" ]]
    then
        echo -e "${ORANGE} -- Assert mode${NC}"
        TEST_TIMEOUT=10
        { timeout --foreground "$TEST_TIMEOUT" "$BINARY"; } > "$( builddir )/out.txt" 2>&1

        if [[ $? == 124 ]]
        then
            echo -e "${ORANGE}TIMEOUT:${NC} ${BINARY}"
            TIMEOUT_COUNT=$(( TIMEOUT_COUNT + 1 ))
        else
            if [[ -e "$( builddir )/out.txt" && ! -s "$( builddir )/out.txt" ]]
            then
                echo -e "${GREEN}OK:${NC}      ${BINARY}"
            else
                echo -e "${RED}FAIL:${NC}    ${BINARY}"
                ERROR_COUNT=$(( ERROR_COUNT + 1 ))
                echo -e "${RED}Unexpected output:${NC}"
                cat "$( builddir )/out.txt"
                echo
            fi
        fi

        continue
    fi

    echo
    MULTIPLE_DIRS=1
    REF_DIRS=$( ls -p "$TEST_DIR" | grep '^.*/$' | sed 's /$  ' )
    if [[ -z "$REF_DIRS" ]]
    then
        REF_DIRS="$TEST_DIR"
        MULTIPLE_DIRS=0
    fi

    TEST_IDX=0
    for REF_DIR in $REF_DIRS
    do
        if [[ $MULTIPLE_DIRS == 1 ]]
        then
            echo
            echo -e "-> Test set ${ORANGE}${REF_DIR}${NC}"
        fi
        if [[ ! -d "$REF_DIR" ]]; then REF_DIR="${TEST_DIR}/${REF_DIR}"; fi
        for INPUT_FILE in "$REF_DIR"/????_in.txt
        do
            REF_FILE=$( sed 's-\(.*\)_in\.txt-\1_out.txt-' <<<"$INPUT_FILE" )
            if [[ ! -e "$REF_FILE" ]]
            then
                echo "ERROR: The reference file ($REF_FILE) doesn't exist."
                cleanup_exit 2
            fi

            INPUT_FILE_FMT="${INPUT_FILE#*/${BASENAME}/}"

            TEST_TIMEOUT=3
            # Give a longer timeout for first iterations of tests to allow caches to load
            # up and shit. Basically first tests timed out on my machine so I tried
            # doing this and it helped, therefore I'm keeping it here ¯\_(ツ)_/¯
            if [[ $TEST_IDX == 0 ]]; then TEST_TIMEOUT=10; fi
            timeout --foreground "$TEST_TIMEOUT" "$BINARY" < "$INPUT_FILE" > "$( builddir )/out.txt"

            if [[ $? == 124 ]]
            then
                echo -e "${ORANGE}TIMEOUT:${NC} ${INPUT_FILE_FMT}"
                TIMEOUT_COUNT=$(( TIMEOUT_COUNT + 1 ))
            else
                if ! diff "$REF_FILE" "$( builddir )/out.txt" > "$( builddir )/temp.txt"
                then
                    echo -e "${RED}FAIL:${NC}    ${INPUT_FILE_FMT}"
                    ERROR_COUNT=$(( ERROR_COUNT + 1 ))
                    cat "$( builddir )/temp.txt"
                else
                    echo -e "${GREEN}OK:${NC}      ${INPUT_FILE_FMT}"
                fi
            fi
        done
        (( TEST_IDX++ ))
    done
done

echo

if [[ $ERROR_COUNT == 0 && $TIMEOUT_COUNT == 0 ]]
then
    echo -e "${GREEN}${BOLD}SUCCESS: All tests passed :)${NC}"
    cleanup_exit 0
fi


if [[ $ERROR_COUNT -ne 0 ]]
then
    echo -e "${RED}${BOLD}FAIL: There were $ERROR_COUNT errors :(${NC}"
fi

if [[ $TIMEOUT_COUNT -ne 0 ]]
then
    echo -e "${ORANGE}${BOLD}FAIL: There were $TIMEOUT_COUNT timeouts :/${NC}"
fi

cleanup_exit 5

CHANGELOG

3.0.0

2.11.2021

• Add assert mode testing

2.1.1

25.10.2021

• Remove leading dash from filenames of files built from absolute paths.

---

2.1.0

25.10.2021

• Fix an issue where the build phase wouldn't properly build source files where their relative paths didn't contain any directories (source files in the current working directory) as their filenames would be ..out
• Fix an issue where the build phase wouldn't properly build source files where their relative paths contained parent directories (..) as they would be built outside of the build directory