An implementation of Scheduler that catches jobs that fail. For use with https://github.com/dbader/schedule

safe_schedule.py

import logging
from traceback import format_exc
import datetime

from schedule import Scheduler


logger = logging.getLogger('schedule')


class SafeScheduler(Scheduler):
    """
    An implementation of Scheduler that catches jobs that fail, logs their
    exception tracebacks as errors, optionally reschedules the jobs for their
    next run time, and keeps going.

    Use this to run jobs that may or may not crash without worrying about
    whether other jobs will run or if they'll crash the entire script.
    """

    def __init__(self, reschedule_on_failure=True):
        """
        If reschedule_on_failure is True, jobs will be rescheduled for their
        next run as if they had completed successfully. If False, they'll run
        on the next run_pending() tick.
        """
        self.reschedule_on_failure = reschedule_on_failure
        super().__init__()

    def _run_job(self, job):
        try:
            super()._run_job(job)
        except Exception:
            logger.error(format_exc())
            job.last_run = datetime.datetime.now()
            job._schedule_next_run()

test_safe_schedule.py

#!/usr/bin/env python3

import time
from safe_schedule import SafeScheduler


def good_task_1():
    print('Good Task 1')


def good_task_2():
    print('Good Task 2')


def good_task_3():
    print('Good Task 3')


def bad_task_1():
    print('Bad Task 1')
    print(1/0)


def bad_task_2():
    print('Bad Task 2')
    raise Exception('Something went wrong!')


scheduler = SafeScheduler()
scheduler.every(3).seconds.do(good_task_1)
scheduler.every(5).seconds.do(bad_task_1)
scheduler.every(7).seconds.do(good_task_2)
scheduler.every(8).seconds.do(bad_task_2)
scheduler.every(12).seconds.do(good_task_3)

while True:
    scheduler.run_pending()
    time.sleep(1)

You don't need to use the traceback module, logging has the exception method to deal with exceptions, which automatically appends the traceback to the log.

Example:
logger.exception("Here's the error: ")

what if I just want to restart the whole script instead of scheduling the next run?

I used your code to run something that needs to talk to a reader. I tested by purposely disconnecting to the internet to cause some issues.

How do I resolve this? Perhaps restart the script after say 60 seconds and then keep doing the restart every 60 seconds?

Hello, @simkimsia, @abelsonlive, @El3k0n, @mplewis

I have been using SafeScheduler for a while with success...But now I need to use run_continuously(). So excuse me if my question sounds too naive but how do I implement run_continously() into SafeScheduler?

Thanks!

Hey @mplewis
im new to python and im wonder about how reschedule_on_failure=True work?
seems has no effect to anything
pls help me understand of this

Thanks <3

Hi @h-2-0,

If you well fixed the schedule/init.py as here, you can just implement it as following, using super() and the inherited method from Scheduler <Scheduler>:

class SafeScheduler(Scheduler):
    """
    An implementation of Scheduler that catches jobs that fail, logs their
    exception tracebacks as errors, optionally reschedules the jobs for their
    next run time, and keeps going.
    Use this to run jobs that may or may not crash without worrying about
    whether other jobs will run or if they'll crash the entire script.
    """

    def __init__(self, reschedule_on_failure=True):
        """
        If reschedule_on_failure is True, jobs will be rescheduled for their
        next run as if they had completed successfully. If False, they'll run
        on the next run_pending() tick.
        """
        self.reschedule_on_failure = reschedule_on_failure
        super().__init__()

    def _run_job(self, job):
        try:
            super()._run_job(job)
        except Exception:
            logger.error(format_exc())
            job.last_run = datetime.datetime.now()
            job._schedule_next_run()
            
    def run_continuously(self, interval=1):
        try:
            super().run_continuously(interval)
        except Exception:
            logger.error(format_exc())
            job.last_run = datetime.datetime.now()
            job._schedule_next_run()

Hey @mplewis
im new to python and im wonder about how reschedule_on_failure=True work?
seems has no effect to anything
pls help me understand of this

Thanks <3

You're right, I don't think reschedule_on_failure does anything. I checked the parent class (version 0.6.0) and it doesn't reference that variable at all.

Hi,
I modified your extension in a way that the job can be rescheduled in minutes o seconds after a failure, or can be even canceled on failure.

Thanks for the work.

Bye

class SafeScheduler(Scheduler):
    """
    An implementation of Scheduler that catches jobs that fail, logs their
    exception tracebacks as errors, optionally reschedules the jobs for their
    next run time, and keeps going.

    Use this to run jobs that may or may not crash without worrying about
    whether other jobs will run or if they'll crash the entire script.
    """

    def __init__(self, reschedule_on_failure=True, minutes_after_failure=0, seconds_after_failure=0):
        """
        If reschedule_on_failure is True, jobs will be rescheduled for their
        next run as if they had completed successfully. If False, they'll run
        on the next run_pending() tick.
        """
        self.reschedule_on_failure = reschedule_on_failure
        self.minutes_after_failure = minutes_after_failure
        self.seconds_after_failure = seconds_after_failure
        super().__init__()

    def _run_job(self, job):
        try:
            super()._run_job(job)
        except Exception:
            logger.error(format_exc())
            if(self.reschedule_on_failure):
                if(self.minutes_after_failure!=0 or self.seconds_after_failure!=0):
                    logger.warn("Rescheduled in %s minutes and %s seconds." % (self.minutes_after_failure, self.seconds_after_failure))
                    job.last_run = None
                    job.next_run = datetime.now() + timedelta(minutes=self.minutes_after_failure, seconds=self.seconds_after_failure)
                else:
                    logger.warn("Rescheduled.")
                    job.last_run = datetime.now()
                    job._schedule_next_run()
            else:
                logger.warn("Job canceled.")
                self.cancel_job(job)

Hi @fimasini , @mplewis

I got the following error " AttributeError: module 'datetime' has no attribute 'now' ", so I changed datetime.now() to datetime.datetime.now().

@fimasini Thanks for the contributed
@mplewis Thanks for awesome initiative

class SafeScheduler(Scheduler):
    """
    An implementation of Scheduler that catches jobs that fail, logs their
    exception tracebacks as errors, optionally reschedules the jobs for their
    next run time, and keeps going.

    Use this to run jobs that may or may not crash without worrying about
    whether other jobs will run or if they'll crash the entire script.
    """

    def __init__(self, reschedule_on_failure=True, minutes_after_failure=0, seconds_after_failure=0):
        """
        If reschedule_on_failure is True, jobs will be rescheduled for their
        next run as if they had completed successfully. If False, they'll run
        on the next run_pending() tick.
        """
        self.reschedule_on_failure = reschedule_on_failure
        self.minutes_after_failure = minutes_after_failure
        self.seconds_after_failure = seconds_after_failure
        super().__init__()

    def _run_job(self, job):
        try:
            super()._run_job(job)
        except Exception:
            logger.error(format_exc())
            if(self.reschedule_on_failure):
                if(self.minutes_after_failure!=0 or self.seconds_after_failure!=0):
                    logger.warn("Rescheduled in %s minutes and %s seconds." % (self.minutes_after_failure, self.seconds_after_failure))
                    job.last_run = None
                    job.next_run = datetime.datetime.now() + timedelta(minutes=self.minutes_after_failure, seconds=self.seconds_after_failure)
                else:
                    logger.warn("Rescheduled.")
                    job.last_run = datetime.datetime.now()
                    job._schedule_next_run()
            else:
                logger.warn("Job canceled.")
                self.cancel_job(job)

Hello! I have noticed that if my job keeps failing too many times, the entire process gets killed because I get queue.Full error. Any way to clear that up?

from datetime import timedelta, datetime

import schedule


class RetryableJob(schedule.Job):
    """
    A job that can be retried if failed.
    """

    def __init__(self, retry_after: timedelta = timedelta(minutes=30), cancel_after_consecutive_failures: int = 3,
                 *args, **kwargs):
        self.retry_after = retry_after
        self.cancel_after_consecutive_failures = cancel_after_consecutive_failures
        self._failure_count = 0
        super().__init__(*args, **kwargs)

    def mark_failed(self) -> None:
        self._failure_count += 1

    @property
    def should_retry(self) -> bool:
        return self._failure_count < self.cancel_after_consecutive_failures

    @property
    def prepare_next_run_time(self) -> datetime:
        return datetime.now() + self.retry_after


class SafeScheduler(schedule.Scheduler):
    """
    An implementation of Scheduler that catches jobs that fail, logs their
    exception tracebacks as errors, optionally reschedules the jobs for their
    next run time, and keeps going.

    Use this to run jobs that may or may not crash without worrying about
    whether other jobs will run or if they'll crash the entire script.
    """

    def __init__(self):
        super().__init__()

    def _run_job(self, job):
        try:
            super()._run_job(job)
            return
        except Exception as e:
            logging.error(e)
            if not isinstance(job, RetryableJob):
                logger.warn("Failed job cancelled")
                self.cancel_job(job)
                return

            job: RetryableJob = job  # type: ignore
            job.mark_failed()

            if job.should_retry:
                next_run_time = job.prepare_next_run_time
                logger.warn(f"Rescheduled failed retryable job for {next_run_time.isoformat()}")
                job.last_run = None
                job.next_run = next_run_time
                job._schedule_next_run()
                return
            else:
                logger.warn("Retryable job failed too many times, cancelled.")
                self.cancel_job(job)
                return

    def every_safe(self,
                   interval: int = 1,
                   /,
                   retry_after: timedelta = timedelta(minutes=30),
                   cancel_after_consecutive_failures: int = 3) -> RetryableJob:
        """
        Schedule a new periodic retryable job

        :param interval: A quantity of a certain time unit
        :param retry_after: An optional timedelta to use as the retry wait time
        :param cancel_after_consecutive_failures: An optional count of retry attempts before cancelling the job
        :return: An unconfigured :class:`RetryableJob <RetryableJob>`
        """

        return RetryableJob(
            retry_after=retry_after,
            cancel_after_consecutive_failures=cancel_after_consecutive_failures,
            interval=interval,
            scheduler=self
        )


scheduler = SafeScheduler()

def test_task():
    raise Exception("Retrigger the task")


scheduler.every_safe(3, retry_after=timedelta(minutes=31), cancel_after_consecutive_failures=3).seconds.do(test_task)

while True:
    scheduler.run_pending()
    time.sleep(1)