Stashboard <-> Pingdom Integration

stashboard-pingdom.rb

#!/usr/bin/env ruby
require 'time'
require 'logger'

require 'rubygems'
require 'pingdom-client'
require 'active_support/core_ext/numeric/time' # time extensions, e.g., 5.days
require 'stashboard'

TIME_FORMAT = "%Y-%m-%d %H:%M:%S %Z" # YYYY-MM-DD HH:MM:SS ZZZ
logger = Logger.new(STDOUT)
logger.level = Logger::WARN

pingdom_auth = {
  :username => '[email protected]',
  :password => 'xxxxxxx',
  :key => '1234567890123456789012345678901234567890'
}

stashboard_auth = {
  :url          => 'https://yourstashboard.appspot.com',
  :oauth_token  => 'AaBbCcDdEeFfgG0102030405060708',
  :oauth_secret => 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'
}

# Stashboard service id => Regex matching pingdom check name(s)
services = {
  'foo'    => /foo/i,
  'bar'    => /bar/i,
  'quux'   => /moopie/i,
}

stashboard = Stashboard::Stashboard.new(
  stashboard_auth[:url],
  stashboard_auth[:oauth_token],
  stashboard_auth[:oauth_secret]
)

pingdom = Pingdom::Client.new pingdom_auth.merge(:logger => logger)

stashboard.service_ids.each do |service|
  puts "WARNING:  Missing mapping for stashboard service '#{service}'" unless services[service]
end

up_services = services
warning_services = {}

# Synchronize recent pingdom outages over to stashboard and determine which services
# are currently up.
pingdom.checks.each do |check|
  service = services.keys.find do |service|
    regex = services[service]
    check.name =~ regex
  end

  unless service
    next
  end

  puts "Check: #{check.name} => #{check.status} [#{service}]"

  yesterday = Time.now - 1.day
  recent_outages = check.summary.outages.select do |outage|
    outage.timefrom > yesterday || outage.timeto > yesterday
  end

  recent_events = stashboard.events(service, "start" => yesterday.strftime("%Y-%m-%d"))

  recent_outages.each do |outage|
    msg = "Service #{check.name} unavailable: " +
          "#{outage.timefrom.strftime(TIME_FORMAT)} - #{outage.timeto.strftime(TIME_FORMAT)}"

    unless recent_events.any? { |event| event["message"] == msg }
      puts "New outage: " + msg
      stashboard.create_event(service, "down", msg)
    end

  end

  unless recent_outages.empty?
    up_services.delete(service)
    warning_services[service] = true
  end

  # if any pingdom check fails for a given service, consider the service down.
  if check.status == "down"
    up_services.delete(service)
  end
end

# Mark any up/warning services as such in stashboard
#
# Note: Custom stashboard status messages *will not* be changed unless
#       they match the regex /Service .* unavailable/i or there is a new outage
#       reported by Pingdom.  This is intentional to allow overriding "up" automated
#       updates.
puts "Up services: #{up_services.inspect}"
up_services.each_key do |service|
  current = stashboard.current_event(service)
  if current["message"] =~ /(Service .* unavailable)|(Service operational but has experienced outage)/i
    puts "Service now operating normally: #{service}"
    stashboard.create_event(service, "up", "Service operating normally.")
  end
end

puts "Warning services: #{warning_services.inspect}"
warning_services.each_key do |service|
  current = stashboard.current_event(service)
  if current["message"] =~ /Service .* unavailable/i
    puts "Service experienced outages in past 24 hours: #{service}"
    stashboard.create_event(service, "warning", "Service operational but has experienced outage(s) in past 24 hours.")
  end
end