Created
November 20, 2015 04:14
-
-
Save winston/bf1758a3d5c57e5cfb3f to your computer and use it in GitHub Desktop.
Heroku + Rails + Sidekiq 4.0 Config
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
web: bundle exec puma -C config/puma.rb -p $PORT | |
worker: bundle exec sidekiq -e $RAILS_ENV -q default -q mailers | |
# By default, sidekiq only operates on the `default` queue | |
# So if you do `Mailer.new(...).deliver_later`, those are queued to the `mailers` queue | |
# Hence you need to explicitly tell Sidekiq to operate on `mailers` queue too | |
# You can also put this into sidekiq.yml |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# config/initializers/sidekiq.rb | |
require 'sidekiq_calculations' | |
if defined?(Sidekiq) | |
Sidekiq::Worker::ClassMethods.class_eval do | |
def perform_async_with_retry(*args) | |
perform_async_count = 0 | |
begin | |
perform_async_without_retry(*args) | |
rescue Redis::CannotConnectError | |
if (perform_async_count+=1) <= 3 | |
sleep 1 | |
retry | |
else | |
raise | |
end | |
end | |
end | |
alias_method :perform_async_without_retry, :perform_async | |
alias_method :perform_async, :perform_async_with_retry | |
end | |
# NOTE: The configuration hash must have symbolized keys. | |
Sidekiq.configure_client do |config| | |
sidekiq_calculations = SidekiqCalculations.new | |
sidekiq_calculations.raise_error_for_env! | |
config.redis = { | |
url: ENV['REDISCLOUD_URL'], | |
size: sidekiq_calculations.client_redis_size | |
} | |
end | |
# NOTE: The configuration hash must have symbolized keys. | |
Sidekiq.configure_server do |config| | |
sidekiq_calculations = SidekiqCalculations.new | |
sidekiq_calculations.raise_error_for_env! | |
config.options[:concurrency] = sidekiq_calculations.server_concurrency_size | |
config.redis = { | |
url: ENV['REDISCLOUD_URL'] | |
} | |
end | |
end |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# lib/sidekiq_calculations | |
class SidekiqCalculations | |
DEFAULT_CLIENT_REDIS_SIZE = 2 | |
DEFAULT_SERVER_CONCURRENCY = 25 | |
def raise_error_for_env! | |
return if !Rails.env.production? | |
max_redis_connection | |
web_dynos | |
worker_dynos | |
rescue KeyError, TypeError # Integer(nil) raises TypeError | |
raise <<-ERROR | |
Sidekiq Server Configuration failed. | |
!!!======> Please add ENV: | |
- MAX_REDIS_CONNECTION | |
- NUMBER_OF_WEB_DYNOS | |
- NUMBER_OF_WORKER_DYNOS | |
ERROR | |
end | |
def client_redis_size | |
return DEFAULT_CLIENT_REDIS_SIZE if !Rails.env.production? | |
puma_workers * (puma_threads/2) * web_dynos | |
end | |
def server_concurrency_size | |
return DEFAULT_SERVER_CONCURRENCY if !Rails.env.production? | |
(max_redis_connection - client_redis_size - sidekiq_reserved) / paranoid_divisor | |
end | |
private | |
def web_dynos | |
Integer(ENV.fetch('NUMBER_OF_WEB_DYNOS')) | |
end | |
def worker_dynos | |
Integer(ENV.fetch('NUMBER_OF_WORKER_DYNOS')) | |
end | |
def max_redis_connection | |
Integer(ENV.fetch('MAX_REDIS_CONNECTION')) | |
end | |
# Copied from `config/puma.rb`. | |
# If default changes, then this needs to be updated | |
def puma_workers | |
Integer(ENV.fetch("WEB_CONCURRENCY", 2)) | |
end | |
# Copied from `config/puma.rb`. | |
# If default changes, then this needs to be updated | |
def puma_threads | |
Integer(ENV.fetch("WEB_MAX_THREADS", 5)) | |
end | |
def sidekiq_reserved | |
2 | |
end | |
# This is added to bring down the value of Concurrency | |
# so that there's leeway to grow | |
def paranoid_divisor | |
2 | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Do I need to set all of the ENV variables that are defined here?