async_job_queues.yml

back_ends:
  delayed_job:
    base_pool_workers: 0
    base_pool_queues: nil
    queue_pools:
      - workers: <%= Integer(ENV['IOU_COPY_POOL_WORKERS'] || 1) %>
        disabled: <%= ENV['IOU_COPY_POOL_DISABLE'] == 'true' %>
        dj_opts:
          queues:
            - 'iou_copy'
      - workers: <%= Integer(ENV['AWS_STATES_POOL_WORKERS'] || 1) %>
        disabled: <%= ENV['AWS_STATES_POOL_DISABLE'] == 'true' %>
        dj_opts:
          queues:
            - 'aws_states'

dj_worker_pools.rb

require 'yaml'
require 'erb'
require 'active_support'
require 'active_support/core_ext/hash/indifferent_access'

path = File.expand_path('../async_job_queues.yml.erb', __FILE__)
DJ_SETTINGS = YAML.safe_load(
  ERB.new(File.new(path).read).result
).with_indifferent_access[:back_ends][:delayed_job]

workers       DJ_SETTINGS[:base_pool_workers] || 0
queues        DJ_SETTINGS[:base_pool_queues]  || nil
min_priority  DJ_SETTINGS[:base_pool_min_priority] if DJ_SETTINGS[:base_pool_min_priority]
max_priority  DJ_SETTINGS[:base_pool_max_priority] if DJ_SETTINGS[:base_pool_max_priority]
sleep_delay   DJ_SETTINGS[:base_pool_sleep_delay]  if DJ_SETTINGS[:base_pool_sleep_delay]
read_ahead    DJ_SETTINGS[:base_pool_read_ahead]   if DJ_SETTINGS[:base_pool_read_ahead]

# pooled_queues expects an array of pooled queues:
# [
#   { workers: 2, dj_opts: {queues:['a_named_queue']} }
# ]
#
# The above example is a "minimum" required config for a pooled queue
# the dj_opts: hash may contain any values that the Delayed::Worker
# accepts.  Refer to https://github.com/collectiveidea/delayed_job
pooled_queues(
  DJ_SETTINGS[:queue_pools]
  .reject { |pool| pool[:disabled] }
  .map { |pool| pool.except(:disabled) }
)

preload_app

# This runs in the master process after it preloads the app
after_preload_app do
  puts "Master #{Process.pid} preloaded app"

  # Don't hang on to database connections from the master after we've
  # completed initialization
  ActiveRecord::Base.connection_pool.disconnect!
end

# This runs in the worker processes after it has been forked
on_worker_boot do |worker_info|
  puts "Worker #{worker_info.name} : #{Process.pid} started"

  # Reconnect to the database
  ActiveRecord::Base.establish_connection
end

# This runs in the master process after a worker starts
after_worker_boot do |worker_info|
  puts "Master #{Process.pid} booted worker #{worker_info.name} with " \
        "process id #{worker_info.process_id}"
end

# This runs in the master process after a worker shuts down
after_worker_shutdown do |worker_info|
  puts "Master #{Process.pid} detected dead worker #{worker_info.name} " \
        "with process id #{worker_info.process_id}"
end