Script for reproducing errors in flaky tests in Microsoft/vstest. i.e. Tests that fail randomly with low probability.

repro.py

# Usage
# Python repro.py <VSTEST folder> <ProjectPattern> <TestFilter> 
#
# Uncomment the two lines for test output
# Log paths are hardcoded at D:\logs
#
# Logs will be cleared after every successful run
# (What's the point of having the logs when it didn't repro?)

import subprocess
import os
import sys
import signal
import shutil
import psutil

def kill_proc_tree(pid, including_parent=True):    
    parent = psutil.Process(pid)
    children = parent.children(recursive=True)
    for child in children:
        child.kill()
    gone, still_alive = psutil.wait_procs(children, timeout=5)
    if including_parent:
        parent.kill()
        parent.wait(5)

def clearlogs(folder):
    for the_file in os.listdir(folder):
        file_path = os.path.join(folder, the_file)
        try:
            if os.path.isfile(file_path):
                os.unlink(file_path)
            #elif os.path.isdir(file_path): shutil.rmtree(file_path)
        except Exception as e:
            print(e)


pattern = ""
filter = ""
if len(sys.argv) > 1:
    vstest_dir = sys.argv[1]
else:
    print("vstest directory?")
    exit(1)
if len(sys.argv) > 2:
    pattern = "-p " + sys.argv[2]
if len(sys.argv) > 3:
    filter = "-filter " + sys.argv[3]

buildcmd = os.path.join(vstest_dir, "build.cmd")
testcmd = os.path.join(vstest_dir, "test.cmd")

os.chdir(vstest_dir)

def run_command(command):
    status = 1
    print(command.split(" "))
    process = subprocess.Popen(command.strip().split(" "), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    while True:
        output = process.stdout.readline().decode("utf-8")

        if output == '' and process.poll() is not None:
            break

        if "... .. . Failed tests:" in output.strip():
            print("\nNOT ALL TESTS WERE SUCCESSFULL.\n")
            status = 0

        if output.strip() == 'Test Run Successful.':
            print("\nALL TESTS SUCCESSFUL. KILLING PROCESS")
            try:
                kill_proc_tree(process.pid)
            except:
                pass
            return status

        # if output:
            # print(output.strip())
    return status

print("Building...\n")
run_command(buildcmd + " " + pattern)
print("\nTesting...")

l = []

for i in range(0,10):
    i = 0
    while True:
        i += 1
        print("\n" + str(l) + " Iteration " + str(i) + "...\n\n")
        status = run_command(testcmd + " " + pattern + " " + filter)
        if status == 0:
            break
        else:
            clearlogs("D:\\logs")
        
    l.append(i)

print(l)