sensu hipchat extension

hipchat.rb

#/usr/bin/env ruby

# Sends events to hipchat for wonderful chatty notifications
#
# This extension requires the hipchat gem
#
# The reason I wrote this instead of using the normal hipchat handler, is that with Flapjack
# all events are handled unless you do crazy filtering stuff. Also with a large number of events
# and checks the sensu server can get overloaded with forking stuff. So anyway, hipchat extension :)
#
# Here is an example of what the Sensu configuration for hipchat should
# look like. It's fairly configurable:
#
# {
#   "hipchat": {
#     "apiversion": "{hipchat_apiversion}",
#     "apikey": "{hipchat_apikey}",
#     "room": "{hipchat_room}",
#     "from": "{hipchat_from}",
#     "keepalive": {
#       "room": "{hipchat_keepalive_room}",
#       "occurrences": {hipchat_keepalive_occurrences}
#     }
#   }
# }
#
# The first four variables should be fairly self-explanatory.
# The 'keepalive' block is for keepalive check settings, in case you want to have keepalive alerts
# going to a different room, repeating at different intervals. This could probably be done better.
#
# Checks can also define a hipchat room, and other options.
# Options that can be passed in event data are as follows:
#   "hipchat_room"   => Room to send hipchat events for this check to
#   "hipchat_from"   => Name to send message from (defaults to Sensu)
#   "hipchat_notify" => Turn on/off the hipchat 'notify' option (defines if the room goes red on new message)
#    - Defaults to 'true' for OK, Critical, Warning and 'false' for Unknown
#   "playbook"       => URL or HTML for playbook for that check
#
# Copyright 2014 Steve Berryman and contributors.
#
# Released under the same terms as Sensu (the MIT license); see LICENSE for details.


require 'rubygems' if RUBY_VERSION < '1.9.0'
require 'hipchat'
require 'timeout'
require 'net/http'
require 'slack-notifier'

module Sensu::Extension

  class Hipchat < Handler

    # The post_init hook is called after the main event loop has started
    # At this time EventMachine is available for interaction.
    def post_init
    end

    # Must at a minimum define type and name. These are used by
    # Sensu during extension initialization.
    def definition
      {
        type: 'extension',  # Always.
        name: 'hipchat',   # Usually just class name lower case.
        mutator: 'ruby_hash'
      }
    end

    # Simple accessor for the extension's name. You could put a different
    # value here that is slightly more descriptive, or you could simply
    # refer to the definition name.
    def name
      definition[:name]
    end

    # A simple, brief description of the extension's function.
    def description
      'Hipchat extension. Because otherwise the sensu server will forking die.'
    end

    # Sends an event to the specified hipchat room etc
    def send_hipchat(room, from, message, color, notify)

      # I've found that the v2 api basically doesn't work YMMV
      apiversion = @settings["hipchat"]["apiversion"] || 'v1'

      apikey = @settings["hipchat"]["apikey"]

      hipchatmsg = HipChat::Client.new(apikey, :api_version => apiversion)

      begin
        timeout(3) do
          hipchatmsg[room].send(from, "#{message}.", :color => color, :notify => notify)
          return "Sent hipchat message"
        end
      rescue Timeout::Error
        return "Timed out while attempting to message #{room} [#{message}]"
      rescue HipChat::UnknownResponseCode
        return "Hipchat returned an unknown response code (rate limited?)"
      end
    end

    # Sends an event to the specified slack room etc
    def send_slack(message)
      slack_token = @settings["hipchat"]["slack_token"]
      slack_org = @settings["hipchat"]["slack_team"]

      slack_client = Slack::Notifier.new(slack_org, slack_token)

      begin
        timeout(3) do
          slack_client.ping("Sensu Alert", message)
          return "Sent slack message"
        end
      rescue Timeout::Error
        return "Timed out while attempting to message"
      rescue
        return "Exception occurred sending slack message"
      end
    end

    # Log something and return false.
    def bail(msg, event)
      @logger.info("Hipchat handler: " + msg + ': ' + event[:client][:name] + '/' + event[:check][:name])
      false
    end

    # Lifted from the sensu-plugin gem, makes an api request to sensu
    def api_request(method, path, &blk)
      http = Net::HTTP.new(@settings['api']['host'], @settings['api']['port'])
      req = net_http_req_class(method).new(path)
      if @settings['api']['user'] && @settings['api']['password']
        req.basic_auth(@settings['api']['user'], @settings['api']['password'])
      end
      yield(req) if block_given?
      http.request(req)
    end

    # also lifted from the sensu-plugin gem. In fact, most of the rest was.
    def net_http_req_class(method)
      case method.to_s.upcase
      when 'GET'
        Net::HTTP::Get
      when 'POST'
        Net::HTTP::Post
      when 'DELETE'
        Net::HTTP::Delete
      when 'PUT'
        Net::HTTP::Put
      end
    end

    def stash_exists?(path)
      api_request(:GET, '/stash' + path).code == '200'
    end

    def event_exists?(client, check)
      api_request(:GET, '/event/' + client + '/' + check).code == '200'
    end

    # Has this check been disabled from handlers?
    def filter_disabled(event)
      if event[:check].has_key?(:handler)
        if event[:check][:handler] == false
          bail 'alert disabled', event
        end
      end
      return true
    end

    # Don't spam hipchat too much!
    def filter_repeated(event)
      defaults = {
        'occurrences' => 1,
        'interval'    => 60,
        'refresh'     => 1800
      }

      occurrences = event[:check][:occurrences] || defaults['occurrences']
      interval    = event[:check][:interval]    || defaults['interval']
      refresh     = event[:check][:refresh]     || defaults['refresh']

      if event[:occurrences] < occurrences
        return bail 'not enough occurrences', event
      end

      if event[:occurrences] > occurrences && event[:action] == :create
        number = refresh.fdiv(interval).to_i
        unless number == 0 || event[:occurrences] % number == 0
          return bail 'only handling every ' + number.to_s + ' occurrences', event
        end
      end

      return true
    end

    # Has the event been silenced through the API?
    def filter_silenced(event)
      stashes = [
        ['client', '/silence/' + event[:client][:name]],
        ['check', '/silence/' + event[:client][:name] + '/' + event[:check][:name]],
        ['check', '/silence/all/' + event[:check][:name]]
      ]
      stashes.each do |(scope, path)|
        begin
          timeout(2) do
            if stash_exists?(path)
              return bail scope + ' alerts silenced', event
            end
          end
        rescue Timeout::Error
          @logger.warn('timed out while attempting to query the sensu api for a stash')
        end
      end
      true
    end

    # Does this event have dependencies?
    def filter_dependencies(event)
      if event[:check].has_key?(:dependencies)
        if event[:check][:dependencies].is_a?(Array)
          event[:check][:dependencies].each do |dependency|
            begin
              timeout(2) do
                check, client = dependency.split('/').reverse
                if event_exists?(client || event[:client][:name], check)
                  return bail 'check dependency event exists', event
                end
              end
            rescue Timeout::Error
              @logger.warn( 'timed out while attempting to query the sensu api for an event')
            end
          end
        end
      end
      return true
    end

    # Run all the filters in some order. Only run the handler if they all return true
    def filters(event_data)
      return false unless filter_repeated(event_data)

      return false unless filter_silenced(event_data)

      return false unless filter_dependencies(event_data)

      return false unless filter_disabled(event_data)
      @logger.info("#{event_data[:client][:name]}/#{event_data[:check][:name]} not being filtered!")
      return true
    end


    def color_to_hex(color)
      if color == "green"
        return "#09B524"
      elsif color == "red"
        return "#E31717"
      elsif color == "yellow"
        return "#FFFF00"
      else
        return "#FFFFFF"
      end
    end

    def build_link(type, url, text)
      if type == :slack
        return "<#{url}|#{text}>"
      else
        return "<a href='#{url}'>#{text}</a>"
      end
    end

    def build_playbook(check, type)
      # If the playbook attribute exists and is a URL, "[<a href='url'>playbook</a>]" will be output.
      # To control the link name, set the playbook value to the HTML output you would like.
      if check[:playbook]
        begin
          uri = URI.parse(check[:playbook])
          if %w( http https ).include?(uri.scheme)
            return "[#{build_link(type, check[:playbook], "Playbook")}]"
          else
            return "Playbook:  #{check[:playbook]}"
          end
        rescue
          return "Playbook:  #{check[:playbook]}"
        end
      else
        return nil
      end

    end

    def build_hipchat_message(event, state_msg, status_msg)
      check = event[:check]
      client_name = check[:source] || event[:client][:name]
      check_name = check[:name]
      output = check[:notification] || check[:output]
      playbook = build_playbook(check, :hipchat)

      return "#{status_msg} #{client_name}/#{check_name} - #{state_msg}: #{output} #{playbook}"
    end

    def build_slack_message(event, room, from, color, state_msg, status_msg)
      check = event[:check]
      client_name = check[:source] || event[:client][:name]
      check_name = check[:name]
      output = check[:notification] || check[:output]
      duration = check[:duration] || Time.at(check[:occurrences] * check[:interval]).utc.strftime("%Hh %Mm %Ss") || "Unknown"
      playbook = build_playbook(check, :slack)
      snoopy_link = build_link(:slack, "https://internal-tool-thing", "Snoopy")
      dash_link = build_link(:slack, "http://sensu-dashboard.wherever.com:3000/#/client/Sensu/#{client_name}", "Sensu")

      if room == "compose"
        room = "#general"
      else
        room = "#" + room.downcase.gsub(/\s+/, "-")
      end

      fields = [
        {
          "title" => "Output",
          "value" => output,
          "short" => false
        }
      ]
      playbook = build_playbook(event[:check], :slack)
      if !playbook.nil?
        fields << {
          "title" => "Playbook",
          "value" => playbook,
          "short" => false
        }
      end
      return {
        "channel" => room,
        "username" => from,
        "icon_url" => "https://files.slack.com/files-pri/T024FNVQB-F02G53U5L/sensu_small.png",
        "attachments" => [
          {
            "mrkdwn_in" => ["text", "title", "fallback"],
            "fallback" => "Host: #{client_name}, Check: #{check_name}\nOutput: #{output}\nAlerting for: #{duration} | [ #{snoopy_link} ] [ #{dash_link} ]",
            "color" => color_to_hex(color),
            "text" => "*Host*: #{client_name}, *Check*: #{check_name}\n*Alerting for:* #{duration} | [ #{snoopy_link} ] [ #{dash_link} ]",
            "fields" => fields
           }
         ]
       }
    end
    def clarify_state(state, check)
      if state == 0
        state_msg = 'OK'
        color = 'green'
        notify = check[:hipchat_notify] || true
      elsif state == 1
        state_msg = 'WARNING'
        color = 'yellow'
        notify = check[:hipchat_notify] || true
      elsif state == 2
        state_msg = 'CRITICAL'
        color = 'red'
        notify = check[:hipchat_notify] || true
      else
        state_msg = 'UNKNOWN'
        color = 'gray'
        notify = check[:hipchat_notify] || false
      end
      return state_msg, color, notify
    end

    # run() is passed a copy of the event_data hash
    def run(event_data)
      event = event_data
      # Is this event a resolution?
      resolved = true if event[:action].eql?("resolve")

      # Is this event a keepalive?
      # Adding extra config on every client is annoying. Just make some extra settings for it.
      keepalive = @settings["hipchat"]["keepalive"] || {}
      if event[:check][:name] == "keepalive"
        event[:check][:occurrences] = keepalive["occurrences"] || 6
        event[:check][:hipchat_room] = keepalive["room"] || @settings["hipchat"]["room"]
        event[:check][:hipchat_from] = keepalive["from"] || @settings["hipchat"]["from"] || 'Sensu'
      end

      # If this event is resolved, or in one of the 'bad' states, and it passes all the filters,
      # send the message to hipchat
      if (resolved or [1,2,3].include?(event[:check][:status])) && filters(event)
        check = event[:check]

        room = check[:hipchat_room] || @settings["hipchat"]["room"]
        from = check[:hipchat_from] || @settings["hipchat"]["from"] || 'Sensu'
        state = check[:status]
        state_msg, color, notify = clarify_state(state, check)
        status_msg = resolved ? "[RESOLVED]" : "[PROBLEM]"

        hipchat_msg = build_hipchat_message(event, state_msg, status_msg)
        slack_msg = build_slack_message(event, room, from, color, state_msg, status_msg)

        operation = proc {
          send_hipchat(room, from, hipchat_msg, color, notify)
          send_slack(slack_msg)
        }
        callback = proc {|result|
          yield "Hipchat message: #{result}", 0
        }
        EM::defer(operation, callback)
      else
        yield "", 0
      end
    end

    # Called when Sensu begins to shutdown.
    def stop
      true
    end

  end
end