Wikimedia labs snmptrap thing setup

gistfile1.txt

Set check_external_commands=1 in /etc/icinga/icinga.cfg

apt-get install snmpd snmptt

cat > /etc/init.d/snmptrapd <<'EOF'
#! /bin/sh -e
### BEGIN INIT INFO
# Provides:           snmptrapd
# Required-Start:     $network $remote_fs $syslog
# Required-Stop:      $network $remote_fs $syslog
# Default-Start:      2 3 4 5
# Default-Stop:       0 1 6
# Short-Description:  SNMP agents
# Description:        NET SNMP (Simple Network Management Protocol) Agents
### END INIT INFO
#
# Author:    Jochen Friedrich <[email protected]>
#
# Modified by:  Leslie Carr <[email protected]>
#
# Splits apart snmpd and snmptrapd
#
 
set -e
 
. /lib/lsb/init-functions
 
export PATH=/sbin:/usr/sbin:/bin:/usr/bin
 
test -x /usr/sbin/snmptrapd || exit 0
 
# Defaults
export MIBDIRS=/usr/share/mibs/site:/usr/share/snmp/mibs:/usr/share/mibs/iana:/usr/share/mibs/ietf:/usr/share/mibs/netsnmp
TRAPDRUN=yes
TRAPDOPTS='-On -Lsd -p /var/run/snmptrapd.pid'
 
# Reads config file (will override defaults above)
[ -r /etc/default/snmptrapd ] && . /etc/default/snmptrapd
 
# Cd to / before starting any daemons.
cd /
 
# Create compatibility link to old AgentX socket location
if [ "$SNMPDCOMPAT" = "yes" ] && [ "$1" != status ]; then
  ln -sf /var/agentx/master /var/run/agentx
fi
 
case "$1" in
  start)
    log_daemon_msg "Starting snmptrapd:"
    if [ "$TRAPDRUN" = "yes" -a -f /etc/snmp/snmptrapd.conf ]; then
   start-stop-daemon --quiet --start --oknodo --exec /usr/sbin/snmptrapd \
       -- $TRAPDOPTS
   log_progress_msg " snmptrapd"
    fi
    ;;
  stop)
    log_daemon_msg "Stopping snmptrapd:"
    start-stop-daemon --quiet --stop --oknodo --exec /usr/sbin/snmptrapd
    log_progress_msg " snmptrapd"
    ;;
  restart|reload|forcereload)
    log_daemon_msg "Restarting snmptrapd:"
    start-stop-daemon --quiet --stop --oknodo --exec /usr/sbin/snmptrapd
    # Allow the daemons time to exit completely.
    sleep 2
    if [ "$TRAPDRUN" = "yes" -a -f /etc/snmp/snmptrapd.conf ]; then
   sleep 1
   start-stop-daemon --quiet --start --exec /usr/sbin/snmptrapd -- $TRAPDOPTS
   log_progress_msg " snmptrapd"
    fi
    ;;
  status)
    status=0
    if [ "$TRAPDRUN" = "yes" -a -f /etc/snmp/snmptrapd.conf ]; then
       status_of_proc /usr/sbin/snmptrapd snmptrapd || status=$?
    fi
    exit $status
    ;;
  *)
    echo "Usage: /etc/init.d/snmptrapd {start|stop|restart|status}"
    exit 1
esac
 
exit 0
EOF
chmod +x /etc/init.d/snmptrapd

cat > /etc/snmp/snmptrapd.conf <<'EOF'
traphandle default /usr/sbin/snmptthandler
disableAuthorization yes
donotlogtraps no
EOF

cat > /etc/snmp/snmptt.conf <<'EOF'
EVENT enterpriseSpecific .1.3.6.1.4.1.33298.0.1004 "Status Events" Normal
FORMAT Exec of puppet event from $aA.
EXEC /usr/lib/nagios/plugins/puppet_run_callback --ip=$aA
SDESC
snmp trap that's sent whenever puppet runs on a wikimedia host.
EDESC
EOF

cat > /usr/lib/nagios/plugins/puppet_run_callback <<'EOF'
#!/usr/bin/env python
'''
Nagios puppet notifier for wmflabs.

Author: Damian Zaremba <[email protected]>

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.
'''
# Import modules we need
import sys
import os
import time
import ldap
import logging, logging.handlers
from optparse import OptionParser

# How much to spam
logging_level = logging.INFO

# LDAP details
ldap_config_file = "/etc/ldap.conf"
ldap_base_dn = "dc=wikimedia,dc=org"
ldap_filter = '(objectClass=dcobject)'
ldap_attrs = ['puppetVar', 'puppetClass', 'dc', 'aRecord', 'associatedDomain']

# Setup logging, everyone likes logging
formatter = logging.Formatter("%(asctime)s [%(levelname)s] %(message)s")
stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.setFormatter(formatter)
syslog_handler = logging.handlers.SysLogHandler(address = '/dev/log')
logger = logging.getLogger(__name__)
logger.setLevel(logging_level)
logger.addHandler(stdout_handler)
logger.addHandler(syslog_handler)


def get_ldap_config():
    '''
    Simple function to load the ldap config into a dict
    '''
    ldap_config = {}
    with open(ldap_config_file, 'r') as fh:
        for line in fh.readlines():
            line_parts = line.split(' ', 1)

            if len(line_parts) == 2:
                ldap_config[line_parts[0].strip()] = line_parts[1].strip()

    return ldap_config


def ldap_connect():
    '''
    Simple function to connect to ldap
    '''
    ldap_config = get_ldap_config()
    if 'uri' not in ldap_config:
        logger.error('Could get URI from ldap config')
        return False

    if 'binddn' not in ldap_config or 'bindpw' not in ldap_config:
        logger.error('Could get bind details from ldap config')
        return False

    ldap_connection = ldap.initialize(ldap_config['uri'])
    ldap_connection.start_tls_s()

    try:
        ldap_connection.simple_bind_s(ldap_config['binddn'],
                                      ldap_config['bindpw'])
    except ldap.LDAPError:
        logger.error('Could not bind to LDAP')
    else:
        logger.debug('Connected to ldap')
        return ldap_connection


def ldap_disconnect(ldap_connection):
    '''
    Simple function to disconnect from ldap
    '''
    try:
        ldap_connection.unbind_s()
    except ldap.LDAPError:
        logger.error('Could not cleanly disconnect from LDAP')
    else:
        logger.debug('Disconnected from ldap')


def get_puppet_vars(instance):
    '''
    Function to determine what puppet vars an instance has
    '''
    logger.debug('Processing puppet vars for %s' % instance['dc'][0])
    vars = {}
    if 'puppetVar' in instance.keys():
        for var in instance['puppetVar']:
            (k, v) = var.split('=', 1)
            vars[k] = v

    return vars


def get_hosts():
    '''
    Simple function to get minimal host attributes
    '''
    hosts = {}

    logger.debug('Searching ldap for hosts')
    results = ldap_connection.search_s(ldap_base_dn, ldap.SCOPE_SUBTREE,
                                       ldap_filter, ldap_attrs)
    if not results:
        logger.error('Could not get the list of hosts from ldap')

    for (dn, instance) in results:
        logger.debug('Processing info for %s' % dn)

        # Get puppet vars
        puppet_vars = get_puppet_vars(instance)

        # Get the dc - don't rely on this for anything other than
        # being unique (used for file names etc)
        dc = instance['dc'][0]

        # We only care about instances
        if 'instancename' not in puppet_vars:
            logger.debug('Skipping %s, not an instance' % dn)
            continue

        # If an instance doesn't have an ip it's probably building
        ips = []
        for ip in instance['aRecord']:
            if len(ip.strip()) > 0:
                ips.append(ip)

        if len(ips) == 0:
            logger.debug('Skipping %s, no ips' % dn)
            continue

        # Sort out the host info
        hosts[dc] = {
            'fqdn': instance['dc'][0],
            'ips': ips,
        }

        # Fix the fqdn if required
        for domain in instance['associatedDomain']:
            if domain.startswith(puppet_vars['instancename']):
                hosts[dc]['fqdn'] = domain
    return hosts


def notify_icinga(hostname):
    service = 'Puppet freshness'
    date = time.strftime('%a %b %d %H:%M:%S %Y')

    message = "[%d] PROCESS_SERVICE_CHECK_RESULT;%s;%s;0;puppet ran at %s\n" % (int(time.time()),
                                                                                hostname,
                                                                                service,
                                                                                date)
    logger.info('Submitting %s' % message)
    try:
        fh = open('/var/lib/icinga/rw/icinga.cmd', 'w')
        fh.write(message)
        fh.close()
    except Exception, e:
        logger.error('Could not submit message: %s' % str(e))

if __name__ == "__main__":
    parser = OptionParser()
    parser.add_option('-d', '--debug', action='store_true', dest='debug')
    parser.add_option('--ip', dest='ip_address')

    (options, args) = parser.parse_args()
    if options.debug:
        logger.setLevel(logging.DEBUG)
        debug_mode = True

    if not options.ip_address:
        logger.error('No IP specified')
        sys.exit(1)

    # Connect
    ldap_connection = ldap_connect()
    if ldap_connection:
        # Get all hosts (can't do a search on aRecord :()
        hosts = get_hosts()
        for host in hosts:
            logger.debug('Looking for %s in %s' % (options.ip_address, ','.join(hosts[host]['ips'])))
            if options.ip_address in hosts[host]['ips']:
                # Write the status message
                notify_icinga(hosts[host]['fqdn'])
                break

        # Cleanup
        ldap_disconnect(ldap_connection)
        sys.exit(0)

    sys.exit(2)
EOF
chmod +x /usr/lib/nagios/plugins/puppet_run_callback 

update-rc.d snmptt enable
update-rc.d snmptrapd defaults
update-rc.d snmptrapd enable
service snmptt restart
service snmptrapd restart