A reproducible experiment runner script that captures Git state, metadata, outputs, and execution details. 

run.sh

#!/usr/bin/env bash

# Experiment Runner
# ---------------
# This script runs experiments in a reproducible way by:
# - Recording git repository state and metadata
# - Creating a unique experiment directory for each run
# - Capturing command output and execution details
# - Generating a comprehensive summary report
# 
# Ideal for research and experiments where reproducibility is important.

set -euo pipefail

function usage() {
  SHOWING_HELP=true
  echo "Usage: $0 [--force] [--dir <directory>] [--no-pushd] [--cleanup-on-fail] [--dev] <command> [arguments]"
  echo
  echo "Description:"
  echo "  This script runs commands in a reproducible way by recording the git repository"
  echo "  state and capturing all command output. Each run creates a unique experiment"
  echo "  directory with logs and a summary markdown file containing execution details."
  echo
  echo "Options:"
  echo "  --force         Allow experiments to run with uncommitted changes."
  echo "  --dir <dir>     Base directory for experiment output [default: experiments]."
  echo "  --no-pushd      Execute command in current directory (don't cd to experiment dir)."
  echo "  --cleanup-on-fail  Remove experiment directory if command fails (for development)."
  echo "  --dev           Development mode: enables both --force and --cleanup-on-fail."
  echo "  -h, --help      Display this help message."
  echo
  echo "Examples:"
  echo "  $0 ./my_script.sh arg1 arg2        # Run a script with arguments"
  echo "  $0 --force julia train.jl          # Force run with uncommitted changes"
  echo "  $0 --dir results python test.py    # Use 'results' as experiment base directory"
  echo "  $0 --no-pushd ./relative_path.sh   # Run without changing directory"
  echo "  $0 --dev julia train.jl            # Development mode with auto cleanup on failure"
  exit 1
}

# --- GLOBAL VARIABLES for traps ---
FORCE_DIRTY=false
EXP_BASE_DIR="experiments"
NO_PUSHD=false        # New variable to control directory change
CLEANUP_ON_FAIL=false # Whether to remove the experiment directory on failure
EXIT_STATUS=0               # Will hold the command's exit code
EXP_DIR=""                  # Will be set later
INTERRUPTED=false           # Will be set to true if user interrupts
SHOWING_HELP=false          # Will be set to true if help is requested
DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
START_TIME_SECONDS=$(date +%s)  # Start time in seconds for duration calculation

# --- HELPER FUNCTIONS ---

# Called on normal script exit (including failures and interrupts).
# Logs the final status to Summary.md.
function finalize() {
  local code=$?
  
  # Exit quietly if we're just showing help
  if $SHOWING_HELP; then
    exit $code
  fi
  
  local finish_time
  finish_time=$(date '+%Y-%m-%dT%H:%M:%S')
  
  # Calculate execution time
  local end_time_seconds=$(date +%s)
  local duration_seconds=$((end_time_seconds - START_TIME_SECONDS))
  
  # Format duration in a human-readable way
  local hours=$((duration_seconds / 3600))
  local minutes=$(( (duration_seconds % 3600) / 60 ))
  local seconds=$((duration_seconds % 60))
  local duration_formatted
  
  if [[ $hours -gt 0 ]]; then
    duration_formatted="${hours}h ${minutes}m ${seconds}s"
  elif [[ $minutes -gt 0 ]]; then
    duration_formatted="${minutes}m ${seconds}s"
  else
    duration_formatted="${seconds}s"
  fi

  # If we are not yet inside a valid EXP_DIR, just exit
  if [[ -z "$EXP_DIR" || ! -d "$EXP_DIR" ]]; then
    echo "Experiment directory not found"
    exit $code
  fi

  # Report the exit status
  if $INTERRUPTED; then
    echo -e "\033[0;33mCommand interrupted with exit status: $code (execution time: $duration_formatted)\033[0m"
  elif [[ $code -eq 0 ]]; then
    echo -e "\033[0;32mCommand completed with exit status: $code (execution time: $duration_formatted)\033[0m"
  else
    echo -e "\033[0;31mCommand failed with exit status: $code (execution time: $duration_formatted)\033[0m"
    
    # Clean up if requested and command failed
    if $CLEANUP_ON_FAIL; then
      echo -e "\033[0;33mCleaning up experiment directory due to failure...\033[0m"
      cd / # Change to a safe directory before removal
      rm -rf "$EXP_DIR"
      echo -e "\033[0;33mExperiment directory removed.\033[0m"
      exit $code # Exit now since we've removed the directory
    fi
  fi

  {
    echo -e "\n## Execution Results"
    echo "- **Execution finished:** $finish_time"
    echo "- **Execution time:** $duration_formatted"
    echo "- **Exit status:** $code"
    if $INTERRUPTED; then
      echo "- **Terminated by user**"
    fi
  } >> "$EXP_DIR/Summary.md"

  # Exit with the script’s real code
  exit "$code"
}

# Called if the user sends SIGINT (Ctrl+C) or SIGTERM
function handle_int() {
  INTERRUPTED=true
  # Set exit code for user interruption as 130 by convention
  exit 130
}

# --- SET TRAPS ---
trap finalize EXIT     # Always run 'finalize' at the end
trap handle_int INT    # Handle Ctrl+C
trap handle_int TERM   # Handle kill/termination

# --- PARSE OPTIONS ---
while [[ $# -gt 0 ]]; do
  case "$1" in
    --force)
      FORCE_DIRTY=true
      shift
      ;;
    --dir)
      EXP_BASE_DIR="$2"
      shift 2
      ;;
    --no-pushd)
      NO_PUSHD=true
      shift
      ;;
    --cleanup-on-fail)
      CLEANUP_ON_FAIL=true
      shift
      ;;
    --dev)
      FORCE_DIRTY=true
      CLEANUP_ON_FAIL=true
      shift
      ;;
    -h|--help)
      usage
      ;;
    *)
      # Once we hit a non-option, assume it's the command and stop parsing
      break
      ;;
  esac
done

if [[ $# -eq 0 ]]; then
  usage
fi

COMMAND=("$@")

# --- CHECK FOR GIT COMMAND ---
if ! command -v git &> /dev/null; then
  echo -e "\033[0;31mError: Git command not found.\033[0m"
  echo "Please install Git or ensure it's in your PATH."
  exit 1
fi

# --- CHECK GIT REPO VALIDITY ---
if ! git rev-parse --is-inside-work-tree &> /dev/null; then
  echo "Not a valid Git repository. Exiting."
  exit 1
fi

# --- CHECK GIT STATE (unless forced) ---
if ! $FORCE_DIRTY; then
  # Ignore untracked files, but check for uncommitted changes
  if [[ -n $(git status --porcelain --untracked-files=no) ]]; then
    echo -e "\033[0;31mError: Uncommitted changes detected.\033[0m"
    echo "Use --force to override. Exiting."
    exit 1
  fi
fi

# --- GATHER METADATA ---
GIT_HASH=$(git rev-parse --short HEAD)
FULL_GIT_HASH=$(git rev-parse HEAD)
BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)

# --- CREATE EXPERIMENT DIRECTORY ---
# Directory naming format: experiments/YYYY-MM-DDTHH:MM:SS_branch-name_git-hash
# This ensures unique, sortable directories that can be traced back to specific commits
EXP_DIR="${EXP_BASE_DIR}/${DATETIME}_${BRANCH_NAME}_${GIT_HASH}"
mkdir -p "$EXP_DIR"
EXP_DIR="$(realpath "$EXP_DIR")"

# --- WRITE INITIAL SUMMARY ---
{
  echo "# Experiment Summary"
  echo "## Metadata"
  echo "- **Execution datetime:** $DATETIME"
  echo "- **Branch:** \`$BRANCH_NAME\`"
  echo "- **Commit hash:** \`$FULL_GIT_HASH\`"
  echo "- **Command:** \`${COMMAND[*]}\`"
  echo "- **Hostname:** \`$(hostname)\`"
  echo "- **Working directory:** \`$EXP_DIR\`"

  echo -e "\n## Latest Commit Details"
  echo '```diff'
  git show
  echo '```'

  echo -e "\n## Git Status"
  echo '```'
  git status
  echo '```'

  echo -e "\n## Uncommitted Changes (Diff)"
  echo '```diff'
  git diff
  echo '```'

  echo -e "\n## Environment Info"
  echo '```'
  uname -a
  echo '```'
} > "$EXP_DIR/Summary.md"

# --- RUN THE COMMAND ---
echo -e "\033[0;32mExperiment directory: $EXP_DIR\033[0m"
echo -e "\033[0;32mRunning command: ${COMMAND[*]}\033[0m"

# We need to capture the real exit status of the command
if $NO_PUSHD; then
  # Run in current directory
  # Use bash 'set -o pipefail' temporarily to ensure pipe failures are propagated
  set -o pipefail
  "${COMMAND[@]}" > >(tee "$EXP_DIR/stdout.log") 2> >(tee "$EXP_DIR/stderr.log" >&2)
  EXIT_STATUS=$?
  set +o pipefail  # Reset pipefail to previous state
else
  # Run in experiment directory
  pushd "$EXP_DIR" > /dev/null
  # Use bash 'set -o pipefail' temporarily to ensure pipe failures are propagated
  set -o pipefail
  "${COMMAND[@]}" > >(tee stdout.log) 2> >(tee stderr.log >&2)
  EXIT_STATUS=$?
  set +o pipefail  # Reset pipefail to previous state
  popd > /dev/null
fi

# Explicitly exit with the command's code, so 'finalize' trap can log it.
exit "$EXIT_STATUS"