Monitor delayed_job processes.

delayed_job.rake

namespace :jobs do

  #bundle exec rake delayed_job:monit RAILS_ENV=demo
  desc "Monitor delayed_job processes."
  task :monit => :environment do
    @server_name = `uname -n`.split("\n").first
    @application_name = Rails.application.class.parent_name
    delayed_job_found = Hash.new

    def do_alert(msg)
      NotificationMailer.background_job_alert(msg,
        {:subject => "ALERT from  #{@application_name} #{Rails.env} #{@server_name.upcase} server delayed_job monit !!!"}).deliver
    end

    MAX_JOB_COUNT = 20
    def delayed_job_seems_dead?
      DelayedJob.all.count > MAX_JOB_COUNT
    end

    puts "#{Time.now} START: delayed_job process monit for #{@application_name} #{Rails.env} environment on #{@server_name.upcase} server."

    delayed_job_pids = `cat #{Rails.root.join('tmp','pids','delayed_job*.pid')} 2>/dev/null`.split("\n")
    delayed_job_pids.each do |delayed_job_pid|
      delayed_job_pid = delayed_job_pid.to_i
      processes = `ps aux | grep delayed_job`
      processes = processes.split("\n")
      processes.each do |proc|
        user, pid, cpu, mem, virt, res, tty, stat, start, time, *other = proc.strip.split(/\s+/)
        if pid.to_i == delayed_job_pid
          #If process found then OK
          delayed_job_found = delayed_job_found.merge({delayed_job_pid=>true})
          puts "   #{Time.now} Delayed job process with pid #{delayed_job_pid} is active."
          break
        end
      end
      puts "   #{Time.now} Delayed job process with pid #{delayed_job_pid} must be active." unless delayed_job_found[delayed_job_pid]
    end

    #If no active delayed_job instance found
    if delayed_job_pids.present? and !delayed_job_found.present?
      satus = `cd #{Rails.root.to_s}; RAILS_ENV=#{Rails.env} script/delayed_job -p "#{@application_name}(#{Rails.env})" restart`

      msg = "Delayed_job worker not found for #{@application_name} #{Rails.env} environment on #{@server_name.upcase} server."<<
        "\n Monit tried to start this process: #{satus}"
      puts "   #{Time.now} #{msg}"
      do_alert(msg)

    #send notification if there are no active delayed jobs and the environment is production
    elsif delayed_job_pids.blank? and %w[ development production qa ].include?(Rails.env)
      msg = "There are no active delayed workers for #{@application_name} #{Rails.env} environment on #{@server_name.upcase} server."<<
        "\n Please execute 'cap #{Rails.env} deploy:delayed_worker' to start one."
      puts "   #{Time.now} #{msg}"
      do_alert(msg)
    elsif delayed_job_seems_dead?
      msg = "Delayed job process seems to be dead. There are more than #{MAX_JOB_COUNT} delayed jobs pending for #{@application_name}"<<
        " #{Rails.env} environment on #{@server_name.upcase} server."
      puts "   #{Time.now} #{msg}"
      do_alert(msg)
    end

    puts "#{Time.now} END: delayed_job process monit for #{@application_name} #{Rails.env} environment on #{@server_name.upcase} server."
  end
end