Part of the blog post - Timeouts and Retries which will walk through how to combine timeouts and retries in Celery - http://adikrishnan.in/2018/09/05/timeouts-and-retries/

run.py

from tasks import timeout_and_chords

timeout_and_chords.delay()

tasks.py

import time
import shlex
import random
import datetime
import subprocess
from celery import Celery, chord
from celery.exceptions import SoftTimeLimitExceeded, MaxRetriesExceededError

conf = {
    "broker_url": "redis://dev-qa-pliny01.kendall.corp.akamai.com:6379/7",
    "result_backend": "redis://dev-qa-pliny01.kendall.corp.akamai.com:6379/7",
    "result_expires": "600",
    "task_soft_time_limit": "30",
}

celery_app = Celery()
celery_app.config_from_object(conf)

@celery_app.task
def add(x, y):
    return x + y


@celery_app.task
def tsum(numbers):
    print(sum(numbers))


@celery_app.task(bind=True, default_retry_delay=30, retry_kwargs={'max_retries': 5})
def timeout_and_chords(self):
    try:
        print("Gonna start a long task!")
        if self.request.retries == 0:
            print("Raising an Error")
            raise KeyError("Testing Key Errors")
        if self.request.retries == 1:
            print("Running chords")
            chord(add.s(i, i) for i in range(10))(tsum.s())
            raise KeyError("Intentional raise for another retry")
        if self.request.retries == 2:
            print("Running chords")
            chord(add.s(i, i) for i in range(10000000000))(tsum.s())
    except SoftTimeLimitExceeded:
        print("Clean up now!")
    except KeyError:
        print("Retrying now")
        raise self.retry()