Last active
August 29, 2015 14:00
-
-
Save mattyjones/f796faf539d97b8afd7f to your computer and use it in GitHub Desktop.
Parse a logfile in realtime for an email storm
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
#! /usr/bin/env python | |
''' | |
email_throttle.py | |
Matt Jones [email protected] | |
Created 04.23.14 | |
Last Update 05.05.14 | |
Notes: | |
Usage: ./email_throttle | |
ToDo: | |
''' | |
import time | |
import subprocess | |
import sys | |
import os | |
sys.path.append('/usr/local/icingadata/store/icinga_service_checks/modules/python26/psutil-2.1.0-py2.6-linux-x86_64.egg') | |
import psutil | |
log_file = '/var/log/icinga/icinga.log' | |
email_threshold = 25 | |
f = open(log_file, 'r') | |
def py_tail(): | |
''' | |
py_tail | |
takes: nothing | |
returns: the current line | |
function: get the newest line in a file | |
This will function the same as the unix command 'tail -f' | |
''' | |
# get the current position of the cursor as set in the main loop | |
where = f.tell() | |
line = '' | |
tail = f.readline() | |
# this will help prevent partial lines | |
if not tail: | |
time.sleep(0.1) | |
f.seek(where) | |
else: | |
line += tail | |
return line | |
def check_email(end_time): | |
''' | |
check_email | |
takes: end_time | |
returns: bool | |
function: see if the current line is a notification command | |
This will cheeck the current line to see if it is a | |
notification command, and return 'True' if so. | |
''' | |
email_counter = 0 | |
while time.time() <= end_time: | |
if 'NOTIFICATION' in py_tail(): | |
email_counter += 1 | |
if email_counter >= email_threshold: | |
# email monitoring and the goc | |
sleep(10) | |
result = subprocess.Popen(['service', 'sendmail', 'stop'], shell=True, stdout=subprocess.PIPE) | |
if check_maillog == True: | |
# stop the service | |
else: | |
result = subprocess.Popen(['service', 'ido2db', 'stop'], shell=True, stdout=subprocess.PIPE) | |
result = subprocess.Popen(['service', 'icinga', 'stop'], shell=True, stdout=subprocess.PIPE) | |
if check_maillog == True: | |
# stop the service | |
else: | |
result = subprocess.Popen(['shutdown', '-h', 'now'], shell=True, stdout=subprocess.PIPE) | |
else: | |
# return to the start of the while loop | |
continue | |
return True | |
def check_service(): | |
''' | |
check_service | |
takes: nothing | |
returns: a bool | |
function: check to make sure the Icinga service is running | |
This will check to make sure the Icinga service is currently running, | |
if so it will return 'True' | |
''' | |
service_pid = 00 | |
# Get the pid | |
for x in psutil.process_iter(): | |
if x.name() == 'icinga': | |
service_pid = x.pid | |
else: | |
pass | |
# Check to make sure the service is present | |
if service_pid == 00: | |
print "the service is not running" | |
return False | |
else: | |
return True | |
def check_maillog(): | |
''' | |
check_maillog | |
takes: nothing | |
returns: a bool | |
function: check to make sure all outgoing email has stopped | |
This will check to make sure all outgoing email has stopped, | |
if so it will return 'True' | |
''' | |
mail_log = '/var/log/maillog' | |
f = open('mail_log', r) | |
results = os.stat(mail_log) | |
size_1 = st_results[6] | |
sleep(10) | |
results = os.stat(mail_log) | |
size_2 = st_results[6] | |
if size_2 != size_1: | |
return False | |
else: | |
return True | |
def main(): | |
try: | |
try: | |
if os.fork() > 0: | |
sys.exit(0) | |
except OSError, e: | |
sys.stderr.write('fork #1 failed" (%d) %s\n' % (e.errno, e.strerror)) | |
sys.exit(1) | |
os.setsid() | |
os.chdir('/home/majones') | |
os.umask(0) | |
try: | |
pid = os.fork() | |
if pid > 0: | |
fpid = open('pidfile', 'wb') | |
fpid.write(str(pid)) | |
fpid.close() | |
sys.exit(0) | |
except OSError, e: | |
sys.stderr.write('fork #2 failed" (%d) %s\n' % (e.errno, e.strerror)) | |
sys.exit(1) | |
si = open('/dev/null', 'r') | |
so = open('out_log', 'a+', 0) | |
se = open('err_log', 'a+', 0) | |
os.dup2(si.fileno(), sys.stdin.fileno()) | |
os.dup2(so.fileno(), sys.stdout.fileno()) | |
os.dup2(se.fileno(), sys.stderr.fileno()) | |
except Exception, e: | |
sys.stderr.write(str(e)) | |
# number of times to check if Icinga is running before stopping the service | |
check_count = 0 | |
check_count_threshold = 3 | |
while check_count < check _count_threshold: | |
check_count += 1 | |
while check_service() == True: | |
start_time = time.time() | |
end_time = start_time + 60 | |
#Find the size of the file and move to the end | |
st_results = os.stat(log_file) | |
st_size = st_results[6] | |
f.seek(st_size) | |
check_email(end_time) | |
continue | |
# sleep for 1 minute then recheck the service | |
sleep(60) | |
continue | |
# make the service stop and exit cleanly | |
if __name__ == "__main__": | |
main() |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment