Watchers. When something goes wrong, and only restart helps.

watcher.py

#!/usr/bin/env python
import argparse
import shlex
import subprocess
import time


def get_arguments():
    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--period', type=int, default=30)
    parser.add_argument('checker')
    parser.add_argument('action')
    return parser.parse_args()


def check(arguments):
    ret = subprocess.call(shlex.split(arguments.checker), stdout=None, stderr=None)
    if ret != 0:
        ret = subprocess.call(shlex.split(arguments.action))


def main():
    arguments = get_arguments()
    while True:
        check(arguments)
        time.sleep(arguments.period)

if __name__ == '__main__':
    main()

watchers.ini

[program:watcher]
command = watcher.py 'curl --fail --silent http://service' 'sudo supervisorctl restart service_name'
autorestart = true
stdout_logfile = /var/log/supervisor/watcher.log
stderr_logfile = NONE
redirect_stderr = true