Sensu freshness_threshold checker

sensu-freshness_threshold-checker.rb

#!/usr/bin/env ruby

# A small daemon-like script that looks for checks with "freshness_threshold" attribute and applies it.
# Meant to be used with a process supervisor like runit, so it just fails if exception occurs.

require 'eventmachine'
require 'json'
require 'net/http'
require 'sensu/transport'
require 'sensu/settings'
require 'logger'

INITIAL_STATUS = 0 # Initial status for checks that haven't reported anything, yet

Thread.new { EM.run }
while not EM.reactor_running?; end

logger = Logger.new(STDERR)
settings = Sensu::Settings.load(:config_file => "/etc/sensu/config.json")
transport_name = settings[:transport][:name] || "rabbitmq"
transport = Sensu::Transport.connect(transport_name, settings[transport_name])

while true
  Net::HTTP.start('127.0.0.1', 4567) do |http|
    checks = Hash[JSON.load(http.get("/checks").body).select{|check| check['freshness_threshold']}.map{|check| [check['name'], check]}]
    subscriptions = checks.map{|name, check| check['subscribers']}.flatten.uniq
    clients = JSON.load(http.get("/clients").body).select{|client| (client['subscriptions'] & subscriptions).size > 0}

    clients.each do |client|
      subscribed_checks = checks.select{|name, check| (check['subscribers'] & client['subscriptions']).size > 0}
      history = Hash[JSON.load(http.get("/clients/#{client['name']}/history").body).map{|hist| [hist['check'], hist]}]

      found_checks = history.select{|check_name, history| subscribed_checks.keys.include? check_name}
      notfound_checks = subscribed_checks.select{|name, check| not history.keys.include? name}

      notfound_checks.each do |name, check|
        payload = {
          :client => client['name'],
          :check => {
            :name => name,
            :output => "No submitted status",
            :status => INITIAL_STATUS,
            :issued => Time.now.to_i,
            :executed => Time.now.to_i
          }
        }
        logger.info("Publishing first status check to #{client['name']} on #{name}")
        transport.publish(:direct, "results", JSON.dump(payload))
      end

      found_checks.select{|check_name, history| Time.now.to_i - history['last_execution'] > checks[check_name]['freshness_threshold']}.each do |check_name, history|
        stale = Time.now.to_i - history['last_execution']
        payload = {
          :client => client['name'],
          :check => {
            :name => check_name,
            :output => "Stale check, freshness_threshold: #{checks[check_name]['freshness_threshold']}",
            :status => 2,
            :issued => Time.now.to_i,
            :executed => Time.now.to_i
          }
        }
        logger.info("Publishing stale check error for #{client['name']} on #{check_name}: #{Time.now.to_i - history['last_execution']}s stale")
        transport.publish(:direct, "results", JSON.dump(payload))
      end
    end
  end

  sleep 15
end