Benchmarking compilation times v0.3

benchmark.sh

#!/bin/bash

set -e

if [ "$#" -ne 3 ]; then
    echo "Usage: $0 <start_commit> <end_commit> <project_dir>"
    exit 1
fi

start_commit="$1"
end_commit="$2"
project_dir="$3"


# Create CSV and error files
SCRIPT_DIR=$(dirname "$(realpath "$0")")
CSV_FILE="$SCRIPT_DIR/results.csv"
ERROR_FILE="$SCRIPT_DIR/errors.log"

# Change to project directory
cd "$project_dir"

echo "commit,date,build_time,description" > "$CSV_FILE"
echo "commit,error" > "$ERROR_FILE"

# Loop over commits
# for commit in $(git log --reverse "$start_commit".."$end_commit" --pretty=format:"%H"); do
for commit in $(git tag --sort version:refname | grep -A999 "$start_commit" | grep -B999 "$end_commit"); do
    # Checkout commit and print info
    git checkout "$commit"
    description=$(git log --format=%s -n 1)
    date=$(git log --format=%ad -n 1 --date=short)
    echo "Commit: $commit"
    echo "Description: $description"
    echo "Date: $date"


    # clean the repo and build directories entirely as qtkeychain's translation files keep getting messed up
    rm -rf build/
    git clean -fdX
    git clean -fdx


    # Configure cmake and compile project three times
    if cmake -Bbuild -H. -Wno-dev -DCMAKE_PREFIX_PATH=/media/vadi/SSDer/Programs/Qt/5.14.2/gcc_64/ > /dev/null 2>&1; then
        echo "configured cmake"
        # fix up translation files that can go missing
        cd 3rdparty/qtkeychain
        git checkout -- translations/qtkeychain_de.ts translations/qtkeychain_fr.ts translations/qtkeychain_ro.ts translations/qtkeychain_ru.ts translations/qtkeychain_zh.ts
        cd ..
        cd ..
        echo "restored files"

        for i in {1..3}; do
            output=$(mktemp)
            echo "now compiling..."
            if (time cmake --build build -- -j$(nproc)) > $output 2>&1; then
                # Parse build time from output
                build_time=$(grep "^real" "$output" | awk '{print $2}')
                # Print commit info and build time
                echo "\"$commit\",\"$date\",\"$build_time\",\"$description\""
                echo "\"$commit\",\"$date\",\"$build_time\",\"$description\"" >> "$CSV_FILE"
            else
                # Log error and commit hash
                echo "Build error for commit $commit"
                cat $output
                echo "\"$commit\",\"Build error\"" >> "$ERROR_FILE"
            fi
            rm -f "$output"
        done
    else
        # Log cmake configuration error and commit hash
        echo "CMake configuration error for commit $commit"
        echo "\"$commit\",\"CMake configuration error\"" >> "$ERROR_FILE"
        cmake -Bbuild -H. -Wno-dev -DCMAKE_PREFIX_PATH=/media/vadi/SSDer/Programs/Qt/5.14.2/gcc_64/ 2>&1 >> "$ERROR_FILE"
    fi
done