A script to repeatedly run the `delta` test case reducer since it gets stuck.

readme.md

A script that can repeatedly run the delta test case reducer, since in my experience it gets stuck in local minima. The example command is currently geared toward WebAssembly WAT reducing, which is my main use case.

deltaforever

#!/usr/bin/env python3

import argparse
import re
import subprocess
import sys
import time
from pathlib import Path

TEMPLATE_CONTENT = """#!/bin/bash

# Run the script and capture stderr and stdout
wasm-tools parse $1 -o $1.wasm
output=$(js --no-threads --no-ion -e '
    const b = read(scriptArgs[0], "binary");
    const m = new WebAssembly.Module(b);
    const { f } = new WebAssembly.Instance(m).exports;
    f();
' -- $1.wasm 2>&1)
exit_code=$?

echo $output

# Check for non-zero exit code and specific assertion message
if [[ $exit_code -ne 0 && "$output" == *"YOUR ERROR MESSAGE HERE"* ]]; then
    echo "Triggered expected error"
    exit 0
else
    echo "Did not trigger expected error"
    exit 1
fi
"""

def get_latest_tmp_dir():
    tmp_dirs = [d for d in Path('.').glob('tmp[0-9]*') if d.is_dir()]
    if not tmp_dirs:
        raise RuntimeError("No tmp directory found after running delta.")
    return max(tmp_dirs, key=lambda d: int(d.name[3:]))

def get_latest_valid_file_from_log(tmp_dir):
    log_path = tmp_dir / "log"
    if not log_path.is_file():
        raise RuntimeError(f"Log file not found in {tmp_dir}")

    last_valid_file = None
    pattern = re.compile(r'^(\d+\.c), lines: \d+')

    with open(log_path, 'r') as f:
        for line in f:
            match = pattern.match(line.strip())
            if match:
                last_valid_file = match.group(1)

    if last_valid_file is None:
        raise RuntimeError(f"No valid reduced files found in log {log_path}")

    candidate = tmp_dir / last_valid_file
    if not candidate.is_file():
        raise RuntimeError(f"File {candidate} from log does not exist")

    return candidate

def run_delta(test_script, input_file):
    print(f"Running delta on {input_file}")
    result = subprocess.run(['delta', f'-test={test_script}', str(input_file)])
    if result.returncode != 0:
        print(f"Delta failed on {input_file}. Check your test script.")
        sys.exit(result.returncode)

def reduce_loop(test_script, initial_file):
    current_file = Path(initial_file)

    try:
        while True:
            run_delta(test_script, current_file)

            tmp_dir = get_latest_tmp_dir()
            latest_file = get_latest_valid_file_from_log(tmp_dir)
            current_file = latest_file.resolve()
            print(f"Restarting with: {current_file}")

    except KeyboardInterrupt:
        print("\nInterrupted by user.")
        final_tmp = get_latest_tmp_dir()
        final_file = get_latest_valid_file_from_log(final_tmp)
        print(f"✅ Most reduced file: {final_file.resolve()}")
        sys.exit(0)

def write_template_script():
    path = Path("test.sh")
    if path.exists():
        print("⚠️ test.sh already exists. Aborting.")
        sys.exit(1)

    with open(path, "w") as f:
        f.write(TEMPLATE_CONTENT)

    path.chmod(path.stat().st_mode | 0o111)  # make executable
    print("✅ Created test.sh in the current directory.")

def main():
    parser = argparse.ArgumentParser(description="Automate delta test case reduction.")
    subparsers = parser.add_subparsers(dest="command")

    # Reduce command (default)
    reduce_parser = subparsers.add_parser("reduce", help="Run delta reduction loop. The default command.")
    reduce_parser.add_argument('--test', required=True, help='Path to the test script used by delta')
    reduce_parser.add_argument('initial_file', help='Initial .wat file to begin reduction')
    reduce_parser.set_defaults(func=lambda args: reduce_loop(args.test, args.initial_file))

    # Template command
    template_parser = subparsers.add_parser("template", help="Write a template test.sh script to current directory")
    template_parser.set_defaults(func=lambda args: write_template_script())

    if len(sys.argv) == 1 or (len(sys.argv) > 1 and sys.argv[1] in ('-h', '--help')):
        parser.print_help()
        sys.exit(0)

    # Inject "reduce" as default if no subcommand is provided
    known_commands = {"reduce", "template"}
    if sys.argv[1] not in known_commands:
        args = parser.parse_args(['reduce'] + sys.argv[1:])
    else:
        args = parser.parse_args()

    args.func(args)

if __name__ == "__main__":
    main()