Created
March 3, 2016 10:19
-
-
Save hanynowsky/9c700af0913d96ee4c99 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Sends events to Flapjack for notification routing. See http://flapjack.io/ | |
# | |
# This extension requires Flapjack >= 0.8.7 and Sensu >= 0.13.1 | |
# | |
# In order for Flapjack to keep its entities up to date, it is necssary to set | |
# metric to "true" for each check that is using the flapjack handler extension. | |
# | |
# Here is an example of what the Sensu configuration for flapjack should | |
# look like, assuming your Flapjack's redis service is running on the | |
# same machine as the Sensu server: | |
# | |
# { | |
# "flapjack": { | |
# "host": "localhost", | |
# "port": 6380, | |
# "db": "0" | |
# } | |
# } | |
# | |
# Copyright 2014 Jive Software and contributors. | |
# | |
# Released under the same terms as Sensu (the MIT license); see LICENSE for details. | |
require 'net/http' | |
require 'uri' | |
require 'json' | |
require 'sensu-plugin/utils' | |
require 'mixlib/cli' | |
require 'sensu/redis' | |
module Sensu | |
module Extension | |
class Flapjack < Handler | |
def name | |
'flapjack' | |
end | |
def description | |
'sends sensu events to the flapjack redis queue' | |
end | |
def options | |
return @options if @options | |
@options = { | |
host: '127.0.0.1', | |
port: 6379, | |
channel: 'events', | |
db: 0 | |
} | |
if @settings[:flapjack].is_a?(Hash) | |
@options.merge!(@settings[:flapjack]) | |
end | |
@options | |
end | |
def definition | |
{ | |
type: 'extension', | |
name: name, | |
mutator: 'ruby_hash' | |
} | |
end | |
def post_init | |
@redis = Sensu::Redis.connect(options) | |
@redis.on_error do |_error| | |
@logger.warn('Flapjack Redis instance not available on ' + options[:host]) | |
end | |
end | |
### CUSTOM | |
# Log something and return false. | |
def bail(msg, event) | |
@logger.info("Flapjack handler: #{msg} : #{event[:client][:name]}/#{event[:check][:name]}") | |
false | |
end | |
# Lifted from the sensu-plugin gem, makes an api request to sensu | |
def api_request(method, path, &blk) | |
http = Net::HTTP.new(@settings['api']['host'], @settings['api']['port']) | |
req = net_http_req_class(method).new(path) | |
if @settings['api']['user'] && @settings['api']['password'] | |
req.basic_auth(@settings['api']['user'], @settings['api']['password']) | |
end | |
yield(req) if block_given? | |
http.request(req) | |
end | |
# Lifted from sensu-plugin for REST | |
def net_http_req_class(method) | |
case method.to_s.upcase | |
when 'GET' | |
Net::HTTP::Get | |
when 'POST' | |
Net::HTTP::Post | |
when 'DELETE' | |
Net::HTTP::Delete | |
when 'PUT' | |
Net::HTTP::Put | |
end | |
end | |
# Check stashes | |
def stash_exists?(path) | |
api_request(:GET, '/stash' + path).code == '200' | |
end | |
# Check whether a given client's check event exists | |
def event_exists?(client, check) | |
api_request(:GET, '/event/' + client + '/' + check).code == '200' | |
end | |
# Has this check been disabled from handlers? | |
def filter_disabled(event) | |
if event[:check].key?(:alert) | |
if event[:check][:alert] == false | |
bail 'alert disabled', event | |
end | |
end | |
true | |
end | |
# Don't spam flapjack too much! | |
@not_occured = "_no?_" | |
@occured = "_o?_" | |
def filter_repeated(event) | |
defaults = { | |
'occurrences' => 1, | |
'interval' => 60, | |
'refresh' => 1800 | |
} | |
occurrences = event[:check][:occurrences] || defaults['occurrences'] | |
interval = event[:check][:interval] || defaults['interval'] | |
refresh = event[:check][:refresh] || defaults['refresh'] | |
if event[:occurrences].to_i < occurrences.to_i | |
@not_occured = "Not enough occurrences" | |
return bail 'not enough occurrences', event | |
end | |
if event[:occurrences].to_i > occurrences.to_i && event[:action] == :create | |
number = refresh.fdiv(interval).to_i | |
occudiff = "#{event[:occurrences]} VS #{occurrences}" | |
@occured = "Event occurrences > Check occurrences #{occudiff}" | |
modulo = (event[:occurrences] - occurrences) % number | |
unless number == 0 || modulo == 0 | |
@occured = 'only handling every ' + number.to_s + ' occurrences' + " : #{occudiff}" + " MODULO: #{modulo}" | |
return bail 'only handling every ' + number.to_s + ' occurrences', event | |
else | |
@occured = "Event occurrences > Check occurrences MODULO HANDLE ON: #{occudiff} MODULO: #{modulo}" | |
end | |
else | |
@occured = "Event occurrences < Check occurrences OR action not CREATE #{event[:action]} - #{occudiff} - MODULO: #{modulo}" | |
end | |
true | |
end | |
# Has the event been silenced through the API? | |
def filter_silenced(event) | |
stashes = [ | |
['client', '/silence/' + event[:client][:name]], | |
['check', '/silence/' + event[:client][:name] + '/' + event[:check][:name]], | |
['check', '/silence/all/' + event[:check][:name]] | |
] | |
stashes.each do |(scope, path)| | |
begin | |
timeout(2) do | |
if stash_exists?(path) | |
return bail scope + ' alerts silenced', event | |
end | |
end | |
rescue Timeout::Error | |
@logger.warn('timed out while attempting to query the sensu api for a stash') | |
end | |
end | |
true | |
end | |
# Does this event have dependencies? | |
def filter_dependencies(event) | |
if event[:check].has_key?(:dependencies) | |
if event[:check][:dependencies].is_a?(Array) | |
event[:check][:dependencies].each do |dependency| | |
begin | |
timeout(2) do | |
check, client = dependency.split('/').reverse | |
if event_exists?(client || event[:client][:name], check) | |
return bail 'check dependency event exists', event | |
end | |
end | |
rescue Timeout::Error | |
@logger.warn('timed out while attempting to query the sensu api for an event') | |
end | |
end | |
end | |
end | |
true | |
end | |
# Run all the filters in some order. Only run the handler if they all return true | |
def filters(event_data) | |
return false unless filter_repeated(event_data) | |
return false unless filter_silenced(event_data) | |
return false unless filter_dependencies(event_data) | |
return false unless filter_disabled(event_data) | |
@logger.info("#{event_data[:client][:name]}/#{event_data[:check][:name]} not being filtered!") | |
true | |
end | |
def run(event) | |
client = event[:client] | |
check = event[:check] | |
tags = [] | |
tags.concat(client[:tags]) if client[:tags].is_a?(Array) | |
tags.concat(check[:tags]) if check[:tags].is_a?(Array) | |
tags << client[:environment] unless client[:environment].nil? | |
# #YELLOW | |
unless check[:subscribers].nil? || check[:subscribers].empty? # rubocop:disable UnlessElse | |
tags.concat(client[:subscriptions] - (client[:subscriptions] - check[:subscribers])) | |
else | |
tags.concat(client[:subscriptions]) | |
end | |
details = ['Address:' + client[:address]] | |
details << 'Tags:' + tags.join(',') | |
details << "Raw Output: #{check[:output]}" if check[:notification] | |
flapjack_event = { | |
entity: client[:name], | |
check: check[:name], | |
type: 'service', | |
state: Sensu::SEVERITIES[check[:status]] || 'unknown', | |
summary: check[:notification] || check[:output], | |
details: details.join(' '), | |
time: check[:executed], | |
tags: tags | |
} | |
# CUSTOM TEST | |
dependencies = check[:dependencies] | |
has_dep = false | |
if dependencies.is_a?(Array) | |
if dependencies.size > 0 | |
has_dep = true | |
end | |
end | |
apihost = @settings['api']['host'] || 'NO_HOST_DEFINED' | |
# CUSTOM TEST END | |
resolved = event[:action].eql?(:resolve) | |
fired = [1, 2, 3].include?(event[:check][:status]) | |
if (filter_repeated(event) == false or filter_silenced(event) == false or filter_dependencies(event) == false or filter_disabled(event) == false ) && !resolved | |
yield "FLAPJACK Skipping Flapjack event delivery. FILTERS: #{filter_repeated(event)} #{filter_silenced(event)} #{filter_dependencies(event)} #{filter_disabled(event)} - STATUS: #{event[:check][:status]} Occurrence: #{event[:occurrences]} vs #{event[:check][:occurrences]} - #{client[:name]} #{check[:name]} - ACTION: #{event[:action]} - RESOLUTION: #{resolved} - OCC #{@occured} vs NOT_OCC #{@not_occured}", 0 | |
else | |
@redis.lpush(options[:channel], MultiJson.dump(flapjack_event)) | |
yield "FLAPJACK filters: #{filter_repeated(event)} #{filter_silenced(event)} #{filter_dependencies(event)} #{filter_disabled(event)} #{client[:name]} : #{check[:name]} . Occurrence: #{event[:occurrences]} vs #{event[:check][:occurrences]} : STATUS: #{event[:check][:status]} - ACTION: #{event[:action]} - RESOLUTIOn: #{resolved} - OCC #{@occured} vs NOT_OCC #{@not_occured} : sent an event to the flapjack redis queue", 0 | |
end | |
end | |
# Called when Sensu begins to shutdown. | |
def stop | |
true | |
end | |
end | |
end | |
end |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment