Created
October 31, 2018 12:24
-
-
Save gtors/f2983b6bc3b877555eb495bbbfab97fc to your computer and use it in GitHub Desktop.
Celery + Environ
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Celery (4.2) | |
| # http://docs.celeryproject.org/en/latest/userguide/configuration.html | |
| from datetime import timedelta | |
| from libs.env import env | |
| PREFIX = 'PRJ' | |
| # General settings | |
| CELERY_ACCEPT_CONTENT = env.list( | |
| f'{PREFIX}_CELERY_ACCEPT_CONTENT', default=['json']) | |
| # Time and date settings | |
| CELERY_ENABLE_UTC = env.bool( | |
| f'{PREFIX}_CELERY_ENABLE_UTC', default=True) | |
| CELERY_TIMEZONE = env( | |
| f'{PREFIX}_CELERY_TIMEZONE', default='UTC') | |
| # Task settings | |
| CELERY_TASK_ANNOTATIONS = env.json( | |
| f'{PREFIX}_CELERY_TASK_ANNOTATIONS', default=None) | |
| CELERY_TASK_COMPRESSION = env.str( | |
| f'{PREFIX}_CELERY_TASK_COMPRESSION', default=None) | |
| CELERY_TASK_PROTOCOL = env.int( | |
| f'{PREFIX}_CELERY_TASK_PROTOCOL', default=2) | |
| CELERY_TASK_SERIALIZER = env.str( | |
| f'{PREFIX}_CELERY_TASK_SERIALIZER', default='json') | |
| CELERY_TASK_PUBLISH_RETRY = env.bool( | |
| f'{PREFIX}_CELERY_TASK_PUBLISH_RETRY', default=True) | |
| CELERY_TASK_PUBLISH_RETRY_POLICY = env.json( | |
| f'{PREFIX}_CELERY_TASK_PUBLISH_RETRY_POLICY', default={ | |
| 'max_retries': 3, | |
| 'interval_start': 0, | |
| 'interval_step': 0.2, | |
| 'interval_max': 0.2, | |
| }) | |
| # Task execution settings | |
| CELERY_TASK_ALWAYS_EAGER = env.bool( | |
| f'{PREFIX}_CELERY_TASK_ALWAYS_EAGER', default=False) | |
| CELERY_TASK_EAGER_PROPAGATES = env.bool( | |
| f'{PREFIX}_CELERY_TASK_EAGER_PROPAGATES', default=False) | |
| CELERY_TASK_REMOTE_TRACEBACKS = env.bool( | |
| f'{PREFIX}_CELERY_TASK_REMOTE_TRACEBACKS', default=False) | |
| CELERY_TASK_IGNORE_RESULT = env.bool( | |
| f'{PREFIX}_CELERY_TASK_IGNORE_RESULT', default=False) | |
| CELERY_TASK_STORE_ERRORS_EVEN_IF_IGNORED = env.bool( | |
| f'{PREFIX}_CELERY_TASK_STORE_ERRORS_EVEN_IF_IGNORED', default=False) | |
| CELERY_TASK_TRACK_STARTED = env.bool( | |
| f'{PREFIX}_CELERY_TASK_TRACK_STARTED', default=False) | |
| CELERY_TASK_SOFT_TIME_LIMIT = env.int( | |
| f'{PREFIX}_CELERY_TASK_SOFT_TIME_LIMIT', default=None) | |
| CELERY_TASK_TIME_LIMIT = env.int( | |
| f'{PREFIX}_CELERY_TASK_TIME_LIMIT', default=None) | |
| CELERY_TASK_ACKS_LATE = env.bool( | |
| f'{PREFIX}_CELERY_TASK_ACKS_LATE', default=False) | |
| CELERY_TASK_DEFAULT_RATE_LIMIT = env.int( | |
| f'{PREFIX}_CELERY_TASK_DEFAULT_RATE_LIMIT', default=None) | |
| # Task result backend settings | |
| CELERY_RESULT_BACKEND = env.str( | |
| f'{PREFIX}_CELERY_RESULT_BACKEND', default=None) | |
| CELERY_RESULT_TRANSPORT_OPTIONS = env.json( | |
| f'{PREFIX}_CELERY_RESULT_TRANSPORT_OPTIONS', default={}) | |
| CELERY_RESULT_SERIALIZER = env.str( | |
| f'{PREFIX}_CELERY_RESULT_SERIALIZER', default='json') | |
| CELERY_RESULT_COMPRESSION = env.str( | |
| f'{PREFIX}_CELERY_RESULT_COMPRESSION', default=None) | |
| CELERY_RESULT_EXPIRES = env.int( | |
| f'{PREFIX}_CELERY_RESULT_EXPIRES', default=timedelta(days=1).total_seconds()) | |
| CELERY_RESULT_CACHE_MAX = env.int( | |
| f'{PREFIX}_CELERY_RESULT_CACHE_MAX', default=None) | |
| # Database backend settings | |
| CELERY_DATABASE_ENGINE_OPTIONS = env.json( | |
| f'{PREFIX}_CELERY_DATABASE_ENGINE_OPTIONS', default={}) | |
| CELERY_DATABASE_SHORT_LIVED_SESSIONS = env.bool( | |
| f'{PREFIX}_CELERY_DATABASE_SHORT_LIVED_SESSIONS', default=False) | |
| CELERY_DATABASE_DB_NAMES = env.json( | |
| f'{PREFIX}_CELERY_DATABASE_DB_NAMES', default={}) | |
| # RPC backend settings | |
| CELERY_RESULT_PERSISTENT = env.bool( | |
| f'{PREFIX}_CELERY_RESULT_PERSISTENT', default=False) | |
| # Cache backend settings | |
| CELERY_CACHE_BACKEND_OPTIONS = env.json( | |
| f'{PREFIX}_CELERY_CACHE_BACKEND_OPTIONS', default={}) | |
| # Redis backend settings | |
| CELERY_REDIS_BACKEND_USE_SSL = env.json( | |
| f'{PREFIX}_CELERY_REDIS_BACKEND_USE_SSL', default=None) | |
| CELERY_REDIS_MAX_CONNECTIONS = env.int( | |
| f'{PREFIX}_CELERY_REDIS_MAX_CONNECTIONS', default=None) | |
| CELERY_REDIS_SOCKET_CONNECT_TIMEOUT = env.int( | |
| f'{PREFIX}_CELERY_REDIS_SOCKET_CONNECT_TIMEOUT', default=None) | |
| CELERY_REDIS_SOCKET_TIMEOUT = env.int( | |
| f'{PREFIX}_CELERY_REDIS_SOCKET_TIMEOUT', default=120) | |
| # Cassandra backend settings | |
| CELERY_CASSANDRA_SERVERS = env.list( | |
| f'{PREFIX}_CELERY_CASSANDRA_SERVERS', default=[]) | |
| CELERY_CASSANDRA_PORT = env.int( | |
| f'{PREFIX}_CELERY_CASSANDRA_PORT', default=9042) | |
| CELERY_CASSANDRA_KEYSPACE = env.str( | |
| f'{PREFIX}_CELERY_CASSANDRA_KEYSPACE', default=None) | |
| CELERY_CASSANDRA_TABLE = env.str( | |
| f'{PREFIX}_CELERY_CASSANDRA_TABLE', default=None) | |
| CELERY_CASSANDRA_READ_CONSISTENCY = env.str( | |
| f'{PREFIX}_CELERY_CASSANDRA_READ_CONSISTENCY', default=None) | |
| CELERY_CASSANDRA_WRITE_CONSISTENCY = env.str( | |
| f'{PREFIX}_CELERY_CASSANDRA_WRITE_CONSISTENCY', default=None) | |
| CELERY_CASSANDRA_ENTRY_TTL = env.int( | |
| f'{PREFIX}_CELERY_CASSANDRA_ENTRY_TTL', default=None) | |
| CELERY_CASSANDRA_AUTH_PROVIDER = env.str( | |
| f'{PREFIX}_CELERY_CASSANDRA_AUTH_PROVIDER', default=None) | |
| CELERY_CASSANDRA_AUTH_KWARGS = env.json( | |
| f'{PREFIX}_CELERY_CASSANDRA_AUTH_KWARGS', default={}) | |
| CELERY_CASSANDRA_OPTIONS = env.json( | |
| f'{PREFIX}_CELERY_CASSANDRA_OPTIONS', default={}) | |
| # Elasticsearch backend settings | |
| CELEREY_ELASTICSEARCH_RETRY_ON_TIMEOUT = env.bool( | |
| f'{PREFIX}_CELEREY_ELASTICSEARCH_RETRY_ON_TIMEOUT', default=False) | |
| CELEREY_ELASTICSEARCH_MAX_RETRIES = env.int( | |
| f'{PREFIX}_CELEREY_ELASTICSEARCH_MAX_RETRIES', default=3) | |
| CELEREY_ELASTICSEARCH_TIMEOUT = env.int( | |
| f'{PREFIX}_CELEREY_ELASTICSEARCH_TIMEOUT', default=10) | |
| # Riak backend settings | |
| CELERY_RIAK_BACKEND_SETTINGS = env.json( | |
| f'{PREFIX}_CELERY_RIAK_BACKEND_SETTINGS', default={}) | |
| # Couchbase backend settings | |
| CELERY_COUCHBASE_BACKEND_SETTINGS = env.json( | |
| f'{PREFIX}_CELERY_COUCHBASE_BACKEND_SETTINGS', default={}) | |
| # Message Routing | |
| CELERY_TASK_QUEUES = env.json( | |
| f'{PREFIX}_CELERY_TASK_QUEUES', default=None) | |
| CELERY_TASK_ROUTES = env.json( | |
| f'{PREFIX}_CELERY_TASK_ROUTES', default=None) | |
| CELERY_TASK_QUEUE_HA_POLICY = env.list( | |
| f'{PREFIX}_CELERY_TASK_QUEUE_HA_POLICY', default=None) | |
| CELERY_TASK_QUEUE_MAX_PRIORITY = env.int( | |
| f'{PREFIX}_CELERY_TASK_QUEUE_MAX_PRIORITY', default=None) | |
| CELERY_WORKER_DIRECT = env.bool( | |
| f'{PREFIX}_CELERY_WORKER_DIRECT', default=False) | |
| CELERY_TASK_CREATE_MISSING_QUEUES = env.bool( | |
| f'{PREFIX}_CELERY_TASK_CREATE_MISSING_QUEUES', default=True) | |
| CELERY_TASK_DEFAULT_QUEUE = env.str( | |
| f'{PREFIX}_CELERY_TASK_DEFAULT_QUEUE', default="celery") | |
| CELERY_TASK_DEFAULT_EXCHANGE = env.str( | |
| f'{PREFIX}_CELERY_TASK_DEFAULT_EXCHANGE', default="celery") | |
| CELERY_TASK_DEFAULT_EXCHANGE_TYPE = env.str( | |
| f'{PREFIX}_CELERY_TASK_DEFAULT_EXCHANGE_TYPE', default="direct") | |
| CELERY_TASK_DEFAULT_ROUTING_KEY = env.str( | |
| f'{PREFIX}_CELERY_TASK_DEFAULT_ROUTING_KEY', default="celery") | |
| CELERY_TASK_DEFAULT_DELIVERY_MODE = env.str( | |
| f'{PREFIX}_CELERY_TASK_DEFAULT_DELIVERY_MODE', default="persistent") | |
| # Broker Settings | |
| CELERY_BROKER_URL = env.str( | |
| f'{PREFIX}_CELERY_BROKER_URL', default="amqp://") | |
| CELERY_BROKER_READ_URL = env.str( | |
| f'{PREFIX}_CELERY_BROKER_READ_URL', default=CELERY_BROKER_URL) | |
| CELERY_BROKER_WRITE_URL = env.str( | |
| f'{PREFIX}_CELERY_BROKER_WRITE_URL', default=CELERY_BROKER_URL) | |
| CELERY_BROKER_FAILOVER_STRATEGY = env.str( | |
| f'{PREFIX}_CELERY_BROKER_FAILOVER_STRATEGY', default="round-robin") | |
| CELERY_BROKER_HEARTBEAT = env.int( | |
| f'{PREFIX}_CELERY_BROKER_HEARTBEAT', default=120) | |
| CELERY_BROKER_HEARTBEAT_CHECKRATE = env.int( | |
| f'{PREFIX}_CELERY_BROKER_HEARTBEAT_CHECKRATE', default=2) | |
| CELERY_BROKER_USE_SSL = env.json( | |
| f'{PREFIX}_CELERY_BROKER_USE_SSL', default=None) | |
| CELERY_BROKER_POOL_LIMIT = env.int( | |
| f'{PREFIX}_CELERY_BROKER_POOL_LIMIT', default=10) | |
| CELERY_BROKER_CONNECTION_TIMEOUT = env.int( | |
| f'{PREFIX}_CELERY_BROKER_CONNECTION_TIMEOUT', default=4) | |
| CELERY_BROKER_CONNECTION_RETRY = env.bool( | |
| f'{PREFIX}_CELERY_BROKER_CONNECTION_RETRY', default=True) | |
| CELERY_BROKER_CONNECTION_MAX_RETRIES = env.int( | |
| f'{PREFIX}_CELERY_BROKER_CONNECTION_MAX_RETRIES', default=100) | |
| CELERY_BROKER_LOGIN_METHOD = env.str( | |
| f'{PREFIX}_CELERY_BROKER_LOGIN_METHOD', default="AMQPLAIN") | |
| CELERY_BROKER_TRANSPORT_OPTIONS = env.json( | |
| f'{PREFIX}_CELERY_BROKER_TRANSPORT_OPTIONS', default={}) | |
| # Worker | |
| CELERY_IMPORTS = env.list( | |
| f'{PREFIX}_CELERY_IMPORTS', default=[]) | |
| CELERY_INCLUDE = env.list( | |
| f'{PREFIX}_CELERY_INCLUDE', default=[]) | |
| CELERY_WORKER_CONCURRENCY = env.int( | |
| f'{PREFIX}_CELERY_WORKER_CONCURRENCY', default=None) | |
| CELERY_WORKER_PREFETCH_MULTIPLIER = env.int( | |
| f'{PREFIX}_CELERY_WORKER_PREFETCH_MULTIPLIER', default=4) | |
| CELERY_WORKER_LOST_WAIT = env.int( | |
| f'{PREFIX}_CELERY_WORKER_LOST_WAIT', default=10) | |
| CELERY_WORKER_MAX_TASKS_PER_CHILD = env.int( | |
| f'{PREFIX}_CELERY_WORKER_MAX_TASKS_PER_CHILD', default=None) | |
| CELERY_WORKER_MAX_MEMORY_PER_CHILD = env.int( | |
| f'{PREFIX}_CELERY_WORKER_MAX_MEMORY_PER_CHILD', default=None) | |
| CELERY_WORKER_DISABLE_RATE_LIMITS = env.bool( | |
| f'{PREFIX}_CELERY_WORKER_DISABLE_RATE_LIMITS', default=False) | |
| CELERY_WORKER_STATE_DB = env.str( | |
| f'{PREFIX}_CELERY_WORKER_STATE_DB', default=None) | |
| CELERY_WORKER_TIMER_PRECISION = env.int( | |
| f'{PREFIX}_CELERY_WORKER_TIMER_PRECISION', default=1) | |
| CELERY_WORKER_ENABLE_REMOTE_CONTROL = env.bool( | |
| f'{PREFIX}_CELERY_WORKER_ENABLE_REMOTE_CONTROL', default=True) | |
| # Events | |
| CELERY_WORKER_SEND_TASK_EVENTS = env.bool( | |
| f'{PREFIX}_CELERY_WORKER_SEND_TASK_EVENTS', default=False) | |
| CELERY_TASK_SEND_SENT_EVENT = env.bool( | |
| f'{PREFIX}_CELERY_TASK_SEND_SENT_EVENT', default=False) | |
| CELERY_EVENT_QUEUE_TTL = env.int( | |
| f'{PREFIX}_CELERY_EVENT_QUEUE_TTL', default=5) | |
| CELERY_EVENT_QUEUE_EXPIRES = env.int( | |
| f'{PREFIX}_CELERY_EVENT_QUEUE_EXPIRES', default=60) | |
| CELERY_EVENT_QUEUE_PREFIX = env.str( | |
| f'{PREFIX}_CELERY_EVENT_QUEUE_PREFIX', default="celeryev") | |
| CELERY_EVENT_SERIALIZER = env.str( | |
| f'{PREFIX}_CELERY_EVENT_SERIALIZER', default="json") | |
| # Remote Control Commands | |
| CELERY_CONTROL_QUEUE_TTL = env.int( | |
| f'{PREFIX}_CELERY_CONTROL_QUEUE_TTL', default=300) | |
| CELERY_CONTROL_QUEUE_EXPIRES = env.int( | |
| f'{PREFIX}_CELERY_CONTROL_QUEUE_EXPIRES', default=10) | |
| # Logging | |
| CELERY_WORKER_HIJACK_ROOT_LOGGER = env.bool( | |
| f'{PREFIX}_CELERY_WORKER_HIJACK_ROOT_LOGGER', default=True) | |
| CELERY_WORKER_LOG_COLOR = env.bool( | |
| f'{PREFIX}_CELERY_WORKER_LOG_COLOR', default=None) | |
| CELERY_WORKER_LOG_FORMAT = env.str( | |
| f'{PREFIX}_CELERY_WORKER_LOG_FORMAT', | |
| default="[%(asctime)s: %(levelname)s/%(processName)s] %(message)s") | |
| CELERY_WORKER_TASK_LOG_FORMAT = env.str( | |
| f'{PREFIX}_CELERY_WORKER_TASK_LOG_FORMAT', | |
| default=( | |
| "[%(asctime)s: %(levelname)s/%(processName)s] " | |
| "[%(task_name)s(%(task_id)s)] %(message)s" | |
| )) | |
| CELERY_WORKER_REDIRECT_STDOUTS = env.bool( | |
| f'{PREFIX}_CELERY_WORKER_REDIRECT_STDOUTS', default=True) | |
| CELERY_WORKER_REDIRECT_STDOUTS_LEVEL = env.str( | |
| f'{PREFIX}_CELERY_WORKER_REDIRECT_STDOUTS_LEVEL', default="WARNING") | |
| # Security | |
| CELERY_SECURITY_KEY = env.str( | |
| f'{PREFIX}_CELERY_SECURITY_KEY', default=None) | |
| CELERY_SECURITY_CERTIFICATE = env.str( | |
| f'{PREFIX}_CELERY_SECURITY_CERTIFICATE', default=None) | |
| CELERY_SECURITY_CERT_STORE = env.str( | |
| f'{PREFIX}_CELERY_SECURITY_CERT_STORE', default=None) | |
| # Custom Component Classes | |
| CELERY_WORKER_POOL = env.str( | |
| f'{PREFIX}_CELERY_WORKER_POOL', default="prefork") | |
| CELERY_WORKER_POOL_RESTARTS = env.bool( | |
| f'{PREFIX}_CELERY_WORKER_POOL_RESTARTS', default=False) | |
| CELERY_WORKER_AUTOSCALER = env.str( | |
| f'{PREFIX}_CELERY_WORKER_AUTOSCALER', | |
| default="celery.worker.autoscale:Autoscaler") | |
| CELERY_WORKER_CONSUMER = env.str( | |
| f'{PREFIX}_CELERY_WORKER_CONSUMER', | |
| default="celery.worker.consumer:Consumer") | |
| CELERY_WORKER_TIMER = env.str( | |
| f'{PREFIX}_CELERY_WORKER_TIMER', | |
| default="kombu.asynchronous.hub.timer:Timer") | |
| # Beat Settings (celery beat) | |
| CELERY_BEAT_SCHEDULE = env.json( | |
| f'{PREFIX}_CELERY_BEAT_SCHEDULE', default={}) | |
| CELERY_BEAT_SCHEDULER = env.str( | |
| f'{PREFIX}_CELERY_BEAT_SCHEDULER', | |
| default="celery.beat:PersistentScheduler") | |
| CELERY_BEAT_SCHEDULE_FILENAME = env.str( | |
| f'{PREFIX}_CELERY_BEAT_SCHEDULE_FILENAME', | |
| default="celerybeat-schedule") | |
| CELERY_BEAT_SYNC_EVERY = env.int( | |
| f'{PREFIX}_CELERY_BEAT_SYNC_EVERY', default=0) | |
| CELERY_BEAT_MAX_LOOP_INTERVAL = env.int( | |
| f'{PREFIX}_CELERY_BEAT_MAX_LOOP_INTERVAL', default=0) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment