resque_extra_workers.god

rails_env   = ENV['RAILS_ENV']  || "production"
rails_root  = ENV['RAILS_ROOT'] || "/mnt/data-store/html/gogobot/current"

WORKER_TIMEOUT = 60 * 10 # 10 minutes

# Stale workers
Thread.new do
  loop do
    begin
      `ps -e -o pid,command | grep [r]esque`.split("\n").each do |line|
        parts   = line.split(' ')
        next if parts[-2] != "at"
        started = parts[-1].to_i
        elapsed = Time.now - Time.at(started)

        if elapsed >= WORKER_TIMEOUT
          ::Process.kill('USR1', parts[0].to_i)
        end
      end
    rescue
      # don't die because of stupid exceptions
      nil
    end
    sleep 30
  end
end

queue_name = "social_graph_facebook"
num_workers = 10

num_workers.times do |num|
  God.watch do |w|
    w.dir      = "#{rails_root}"
    w.name     = "resque-#{num}-#{queue_name}"
    w.group    = "resque"
    w.interval = 2.minutes
    w.env      = {"QUEUE"=>"#{queue_name}", "RAILS_ENV"=>rails_env, "PIDFILE" => "#{rails_root}/tmp/resque_#{queue_name}_#{w}.pid"}
    w.pid_file = "#{rails_root}/tmp/resque_#{queue_name}_#{w}.pid"
    w.start    = "cd #{rails_root}/ && bundle exec rake environment resque:work QUEUE=#{queue_name} RAILS_ENV=#{rails_env}"
    w.log      = "#{rails_root}/log/resque_god.log"

    w.uid = 'root'
    w.gid = 'root'

    # restart if memory gets too high
    w.transition(:up, :restart) do |on|
      on.condition(:memory_usage) do |c|
        c.above = 350.megabytes
        c.times = 2
      end
    end

    # determine the state on startup
    w.transition(:init, { true => :up, false => :start }) do |on|
      on.condition(:process_running) do |c|
        c.running = true
      end
    end

    # determine when process has finished starting
    w.transition([:start, :restart], :up) do |on|
      on.condition(:process_running) do |c|
        c.running = true
        c.interval = 5.seconds
      end

      # failsafe
      on.condition(:tries) do |c|
        c.times = 5
        c.transition = :start
        c.interval = 5.seconds
      end
    end

    # start if process is not running
    w.transition(:up, :start) do |on|
      on.condition(:process_running) do |c|
        c.running = false
      end
    end
  end
end