Python function decorator for "infallible" functions: a decorated function will be retried as many times as needed until no exceptions occur. Includes fancy logging.

restart_on_exception.py

def restart_on_crash(log_exprs=[]):
	"""
	A function decorator that re-runs the wrapped function in case it raises an exception.
	This is repeated until the function succeeds.
	
	`log_exprs` is a list of strings, each string being an expression whose value at the time
	of exception is displayed. Example:
	>>> @restart_on_crash(log_exprs=['b', 'a+b'])
	>>> def divider(a):
	>>> 	import random; random.seed(time.time())
	>>> 	for t in range(5):
	>>> 		print a, 'divided by', b, 'is', a/b
	>>> 	print 'done'
	
	The error report is also written to a (hardcoded) file.
	"""
	def decorator(func):
		REPORT_FILE = os.path.abspath('./_crash_report.txt')
		def wrapped_func(*args, **kwargs):
			alles_gut = False
			while not alles_gut: 
				try:
					func(*args, **kwargs)
					alles_gut = True
				except:
					print '%s() was restarted at %s because of the following error:' % (func.func_name, datetime.datetime.now().isoformat())
					traceback.print_exc()

					try:
						# find the most nested invocation of `func` in the traceback
						func_frame = None
						tb = sys.exc_info()[2]
						while True:
							if tb.tb_frame.f_code == func.func_code:
								func_frame = tb.tb_frame
							if not tb.tb_next: break
							tb = tb.tb_next
						# evaluate the expression-to-be-logged in the scope of func
						with open(REPORT_FILE, 'w') as f:
							f.write('Crash in function %s at %s\n\n' % (func.func_name, datetime.datetime.now().isoformat()))
							traceback.print_exc(file=f)
							f.write('\n\nLogged variables/expressions:\n')
							for log_expr in log_exprs:
								try: log_val = repr(eval(log_expr, globals(), func_frame.f_locals))
								except: log_val = '(error while evaluating expression; %r)' % sys.exc_info()[1]
								f.write('>>> %s: %s\n' % (log_expr, log_val))
						print 'More info can be found in %r' % REPORT_FILE
					except:
						print 'Additionally, an error was encountered trying to write the crash report to %r:' % REPORT_FILE
						traceback.print_exc()
		return wrapped_func
	return decorator