Remove orphaned passenger processes.

passenger_process.rb

require 'chronic'

class PassengerProcess
  attr_accessor :pid, :uptime
  
  def initialize(passenger_status_line)
    values = passenger_status_line.match(/PID:\s(\d*).*Uptime:\s*(.*)$/)
    @pid = values[1].to_i
    @uptime = values[2]
  end
  
  def uptime_in_seconds
    uptime_values = uptime.split(/\s/).map { |u| u.to_i }
    seconds = uptime_values[-1] || 0
    minutes = uptime_values[-2] || 0
    hours = uptime_values[-3] || 0
    seconds + (minutes*60) + (hours*3600)
  end


  def self.passenger_status
    @passenger_status ||= `passenger-status | grep PID`
  end
  
  def self.active
    passengers = []
    passenger_status.each_line do |line|
      passengers << PassengerProcess.new(line)
    end
    passengers
  end

  def self.stale
    stale_passengers = []
    potentially_stale_processes = active.select { |p| p.uptime_in_seconds > 600 }
    potentially_stale_processes.each do |process|
      process_last_log_entry = last_log_entry(process.pid)
      etime = (Time.now - parse_time_from_log_entry(process_last_log_entry)).to_i
      if  etime > 600
        puts "Stale process last log entry: #{process_last_log_entry}" if debug?
        stale_passengers << process
      end
    end
    stale_passengers
  end
  
  def self.last_log_entry(pid)
    pwd = `pwd`.chomp
    `grep 'rails\\[#{pid}\\]' #{pwd}/log/production.log | tail -n 1`.chomp
  end
  
  def self.last_log_entry_time(pid)
    log_entry = last_log_entry(pid)
    log_entry_time = parse_time_from_log_entry(log_entry)
    (Time.now - log_entry_time).to_i
  end
  
  def self.parse_time_from_log_entry(entry)
    Chronic.parse(entry.match(/.*\s.*\s\d{1,2}:\d{1,2}:\d{1,2}/)[0])    
  end

  def self.active_passenger_pids
    active.map{ |p| p.pid }
  end
  
  def self.passenger_memory_stats
    # 17630  287.0 MB  64.5 MB  Rails: /var/www/apps/gldl/current
    # 17761  285.9 MB  64.9 MB  Rails: /var/www/apps/gldl/current
    # 18242  293.1 MB  71.4 MB  Rails: /var/www/apps/gldl/current
    # 18255  285.9 MB  60.6 MB  Rails: /var/www/apps/gldl/current
    @passenger_memory_stats ||= `passenger-memory-stats | grep 'Rails:.*\/current'`
  end

  def self.all_passenger_pids
    passengers = []
    passenger_memory_stats.each_line do |line|
      matcher = line.match(/\s?(\d*)\s/)
      passengers << matcher[1] if matcher
    end
    passengers
  end

  def self.inactive_passenger_pids
    inactive_passenger_pids = []
    (all_passenger_pids - active_passenger_pids).each do |pid|
      raw_etime = `ps -p #{pid} --no-headers -o etime`.chomp
      etime = PsEtime.new(raw_etime)
      inactive_passenger_pids << pid unless (etime.age_in_seconds < (MINIMUM_ETIME || 600))
    end
    inactive_passenger_pids
  end

  def self.kill_inactive_passengers
    inactive_passenger_pids.each do |pid|
      kill(pid)
    end
  end
  
  def self.kill_stale_passengers
    stale.each do |p|
      kill(p.pid)
    end
  end
  
  def self.kill(pid)
    signal = ARGV.include?('--hard') ? 9 : 15
    command = "kill -#{signal} #{pid}"
    puts command
    `#{command}` unless noop?
  end
  
  def self.inactive_passengers_last_log_entry
    inactive_passenger_pids.each do |pid|
      puts last_log_entry(pid)[0, 150]
    end
  end  

  private
  
  def self.noop?
    @noop ||= ARGV.include?('--noop')
  end

  def self.debug?
    @noop ||= ARGV.include?('--debug')
  end
end

passenger_reaper

#!/usr/bin/env ruby
require 'rubygems'
require File.join(File.dirname(__FILE__), 'ps_etime')
require File.join(File.dirname(__FILE__), 'passenger_process')

MINIMUM_ETIME = 600 # 10 minutes

if %W(active inactive debug status).include?(ARGV.first)
  case ARGV.first
  when 'active'
    PassengerProcess.kill_stale_passengers
  when 'inactive'
    PassengerProcess.kill_inactive_passengers
  when 'debug'
    PassengerProcess.inactive_passengers_last_log_entry
  when 'status'
    puts "Total of passengers: #{PassengerProcess.all_passenger_pids.count}"
    puts "Stale passengers: #{PassengerProcess.stale.count}"
  end
else
  help =<<EOF
Error: please use the following syntax:
  
    passenger_reaper <command> <options>

Commands:

status    displays the number of total passenger workers and the number of active workers
active    kills stale workers that passengers has in the pool
inactive  kills workers that passenger no longer controls
debug     shows the last log entry from each inactive worker

Options:

--noop    don't actually kill processes but show which ones would have been killed
--hard    send the KILL signal

EOF
  puts help
  exit 1
end

passenger_reaper_spec.rb

require 'ps_etime'
require 'passenger_process'

require 'chronic'

describe PassengerProcess do
  before do
    @passenger_status =<<EOF
PID: 14548   Sessions: 0    Processed: 42      Uptime: 40s
PID: 14521   Sessions: 0    Processed: 29      Uptime: 1h 20m 43s
PID: 14525   Sessions: 0    Processed: 32      Uptime: 20m 5s
PID: 14007   Sessions: 0    Processed: 36      Uptime: 1m 47s
PID: 14830   Sessions: 1    Processed: 1       Uptime: 5s
EOF
    PassengerProcess.stub!(:passenger_status).and_return(@passenger_status)
  end

  it "should return active passenger processes" do
    all_passenger_processes = PassengerProcess.active
    all_passenger_processes.count.should eql(5)
    passenger = all_passenger_processes.first
    passenger.pid.should eql(14548)
    passenger.uptime.should eql('40s')
  end
  
  it "should parse the uptime into seconds" do
    all_passenger_processes = PassengerProcess.active
    all_passenger_processes[0].uptime_in_seconds.should eql(40)
    all_passenger_processes[4].uptime_in_seconds.should eql(5)
    all_passenger_processes[3].uptime_in_seconds.should eql(107)
    all_passenger_processes[1].uptime_in_seconds.should eql(4843)
  end
  
  it "should return the stale passengers" do
    stub_time = Chronic.parse('Sep 26 17:40:34')
    Time.stub!(:now).and_return(stub_time)
    PassengerProcess.stub!(:last_log_entry).with(14521).and_return('Sep 26 16:40:34 web1 rails[14521]: Rendering promotions/index')
    PassengerProcess.stub!(:last_log_entry).with(14525).and_return('Sep 26 17:39:34 web1 rails[14525]: Rendering promotions/index')
    stale_passengers = PassengerProcess.stale
    stale_passengers.count.should eql(1)
    stale_passengers[0].pid.should eql(14521)
  end
end

ps_etime.rb

class PsEtime
  attr_accessor :seconds, :minutes, :hours, :days
  attr_reader :etime

  def initialize(raw_etime)
    @etime = raw_etime
    split_etime = @etime.split(/\-|\:/).map {|a| a.to_i}
    @seconds = split_etime[-1] || 0
    @minutes = split_etime[-2] || 0 
    @hours = split_etime[-3] || 0
    @days = split_etime[-4] || 0
  end
  
  def age_in_seconds
    @seconds + (@minutes*60) + (@hours*3600) + (@days*86400)
  end
end