A simple method to manage experiments

experiment_manager.py

"""
A simple experiment manager.  You can add experiments as functions with the @register decorator.  Then, call the experiments you want to run
using the -r flag followed by the function names of the experiments or run all the experiments with the -a flag. 

Experiment runs are stored in an .experiments/ folder created on the first time you run an experiment.  Each run creates a folder inside
.experiments/ with the name of the function and the datetime it was run.  Every experiment should accept a logging_path argument.  
This is the path to the folder created for the specific experiment run.  You can use this to store experiment specific files.   
"""
import argparse
import datetime
import os

experiments = {}
parser = argparse.ArgumentParser(description='Run experiments.')
experiment_folder = './experiments'

STRIP = -7

def register(experiment):
	experiments[experiment.__name__] = experiment 
  
##
# Begin Experiments:
# Each experiment function call should accept a logging path argument.
# This is the path of the directory created for each new run of this experiment 
# and can be used to store files specific to the experiment.
###
@register
def test_exp_1(logging_path):
  print ("Experiment 1 here")
  
@register
def test_exp_2(logging_path):
  print ("Experiment 2 here")
###

def setup(exp):
	time_of_creation = str(datetime.datetime.now()).replace(" ", ".")[:STRIP]
	exp_path = os.path.join(experiment_folder,f"{exp}_{time_of_creation}")

	if not os.path.exists(exp_path):
		os.makedirs(exp_path)
	return exp_path

def main():
	parser.add_argument('-s', action='store_true', default=False,
                   help='show the current experiments avaliable to run')
	parser.add_argument('-r', metavar='--run', nargs='+',
				   help='run these experiments')
	parser.add_argument('-a', action='store_true', default=False,
				   help='run all the experiments avaliable')

	args = vars(parser.parse_args())

	if args['s'] or not args['r']:
		print ("Avaliable Experiments:")
		print ("----------------------")
		for key in experiments:
			print (key)
		print ("----------------------")	
		print ("Run experiments with -r flag.")
		exit()

	if args['a']:
		for exp in experiments:
			print (f"Beginning experiment {exp}")
			print ("-----------------------------")
			logging_path = setup(exp)
			experiments[exp](logging_path)
			print (f"Saving in {logging_path}")
			print ("-----------------------------")
		exit()

	for exp in args['r']:
		if exp in experiments:
			print (f"Beginning experiment {exp}")
			print ("-----------------------------")
			logging_path = setup(exp)
			experiments[exp](logging_path)
			print (f"Saving in {logging_path}")
			print ("-----------------------------")
		else:
			print(f"Experiment {exp} not registered -- skipping!.")


if __name__ == "__main__":
	main()