Heroku dyno killer

heroku_killer.rb

require 'pty'
require 'active_support'
require 'active_support/core_ext/time'
require 'active_support/core_ext/numeric'

TIMEOUT_TO_KILL_DYNO = 3.minute.from_now
TIMEOUT_TO_ANY_KILL  = 30.seconds
MAX_PROBLEMS         = 3

PROBLEMS = {
  r14: "Error R14",
  h12: "code=H12",
  h13: "code=H13",
  rack_timeout: "state=timed_out"
}

# GREP = "grep --line-buffered '\\(#{ PROBLEMS.values.join('\|') }\\)'"
GREP = 'grep --line-buffered "\(Error R14\|code=H12\|code=H13\|state=timed_out\)"'


# To test:
# CMD  = "tail -f fixtures/dev.log | #{GREP}"
# $ echo `cat fixtures/r14.log` > fixtures/dev.log
# AND comment the ps:stop

CMD  = "heroku logs -t -p router -a enjoei | #{GREP}"

dynos = {}

PTY.spawn(CMD) do |stdout, stdin, pid|
  stdout.each do |line|
    if line =~ /dyno=web\.(\d+)/ || line =~ /heroku web\.(\d+)/ || line =~ /app web\.(\d+)/
      dyno = $1.to_i

      if dyno > 0
        dynos[dyno] ||= {}
        dynos[dyno][:kill_count] ||= 0
        dynos[dyno][:kill_count] += 1

        # If the log print some error about the dyno and the dyno was already
        # killed less then TIMEOUT_TO_KILL_DYNO then we skip this line
        if dynos[dyno][:next_kill] && Time.current < dynos[dyno][:next_kill]
          dynos[dyno][:kill_count] = 0
          next
        end

        # If the log printed more than MAX_PROBLEMS problems then the dyno will
        # be killed and the problems count reseted.
        # If the log printed a R14 error (memory problem) we will stop the dyno
        # right away.
        if dynos[dyno][:kill_count] > MAX_PROBLEMS || line.include?(PROBLEMS[:r14])
          system("heroku ps:stop web.#{dyno} -a enjoei")

          puts "Stopping web.#{dyno} at #{Time.now.to_s}"

          dynos[dyno][:next_kill] = TIMEOUT_TO_KILL_DYNO
          dynos[dyno][:kill_count] = 0

          if TIMEOUT_TO_ANY_KILL.present?
            sleep TIMEOUT_TO_ANY_KILL
          end
        end
      end
    end
  end
end