log_reader.py

import os
import re
import socket
import ssl
import sys
import time
from collections import namedtuple

import ConfigParser
import httplib

try:
    import simplejson as json
except ImportError:
    import json

"""
[Install]
WantedBy=multi-user.target

[Unit]
Description=Log reader
After=network.target

[Service]
User=root
#Group={{ as_user }}
WorkingDirectory=/home/user/log_reader
ExecStart=python /home/user/log_reader/log_reader.py
ExecReload=/bin/kill -s HUP $MAINPID
ExecStop=/bin/kill -s TERM $MAINPID
PrivateTmp=true

[Install]
WantedBy=multi-user.target
"""


class FailedToSend(Exception):
    """An item could not be sent"""


class Http:
    def __init__(self, host, url, path, port=80):
        self.host = host
        self.url = url
        self.port = int(port)
        self.path = path
        self.headers = {
            'Host': host,
            'Content-Type': 'application/json; charset=UTF-8',
        }
        self.conn = None
        self._connect()

    def _connect(self):
        try:
            if self.port in [443, 8443]:
                self.conn = httplib.HTTPSConnection(self.url, port=self.port, context=ssl._create_unverified_context())
            else:
                self.conn = httplib.HTTPConnection(self.url, port=self.port)
        except socket.error:
            pass

    def _block_until_succeeded(self, body):
        while True:
            try:
                self._connect()
                self.send_post(body)
                break
            except FailedToSend:
                time.sleep(1)  # TODO add throttling/tar pitting

    def send_post(self, body):
        try:
            self.conn.request(
                method='POST',
                url=self.path,
                headers=self.headers,
                body=json.dumps(body),
            )

            res = self.conn.getresponse()
            if res.status not in [200, 202]:
                raise FailedToSend
            # read must be called on the response,
            # or httplib could throw ResponseNotReady under some circumstances
            res.read()
        except (httplib.HTTPException, socket.error, AttributeError) as e:
            sys.stderr.write(str(e) + str(type(e)) + '\n')
            raise FailedToSend

    def send_request(self, body):
        try:
            self.send_post(body)
        except FailedToSend:
            self._block_until_succeeded(body)

    # noinspection PyUnusedLocal,PyUnusedLocal
    def __exit__(self, *args, **kwargs):
        self.conn.close()


def prompt_config():
    print("In order to save to /etc/ root is needed. once to create ")
    filename = raw_input("input file to to monitor: ")
    while not os.path.isfile(filename):
        print("non-existing file try again")
        filename = raw_input("input file name and path: ")

    regexes = {}
    add_regex = True
    while add_regex:
        regex = raw_input("input regex: ")
        reason = raw_input("input name for regex number {}: ".format(len(regexes) + 1))
        regexes[reason] = regex
        add_regex = raw_input("add another y/n?: ").lower() in ['yes', 'y']

    start_from_top = False
    # TODO set better default
    url = "172.31.141.206"
    host = "edra-cno.aorta.net"
    port = 443
    path = '/logs'

    config = ConfigParser.ConfigParser()
    config.add_section('general')
    config.set('general', 'url', url)
    config.set('general', 'host', host)
    config.set('general', 'path', path)
    config.set('general', 'port', port)
    config.set('general', 'filename', filename)
    config.set('general', 'start_from_top', start_from_top)

    config.add_section('regexes')
    for reason, regex in regexes.iteritems():
        config.set('regexes', reason, regex)

    with open('/etc/log_reader.ini', 'wb') as configfile:
        config.write(configfile)


def config_section_to_dict(config, section):
    dic = dict(config._sections[section])
    del dic['__name__']
    return dic


def get_or_create_config():
    ini_file = '/etc/log_reader.ini'
    config = ConfigParser.ConfigParser()

    if not os.path.isfile(ini_file):
        sys.exit("Configuration file not found, create one with '-config' or '-c' flag")

    _ = config.read(ini_file)

    named_config = namedtuple("Config", ["url", "host", "port", "path", "hostname", "start_from_top", "filename"])
    general = config_section_to_dict(config, 'general')
    general['hostname'] = socket.gethostname()
    return named_config(**general), config_section_to_dict(config, 'regexes')


def create_message(line, reason):
    return {
        "msg": line,
        "hostname": config.hostname,
        "time": time.time(),
        "reason": reason,
    }


def check_regexes(regexes):
    errors = {}
    for reason, regex in regexes.iteritems():
        try:
            re.compile(regex)
        except re.error:
            errors[reason] = {regex}
    return errors


class Bucket:
    def __init__(self, base=0.1, diff=0.1, max_amount=300, min_amount=0.1, counter=10):
        self.base = base
        self.current = base
        self.diff_up = 1 + diff
        self.diff_down = 1 - diff
        self.max_amount = max_amount
        self.min_amount = min_amount
        self.counter = 0
        self.max_counter = counter

    def up_counter(self):
        self.counter += 1
        if self.counter >= self.max_counter:
            self.counter = 0
        return self.counter == 0

    def reset(self):
        self.current = self.base

    def up(self):
        val = self.current * self.diff_up
        self.current = max(min(val, self.max_amount), self.min_amount)
        return self.current

    def down(self):
        val = self.current * self.diff_down
        self.current = max(min(val, self.max_amount), self.min_amount)
        return self.current


class FileRotated(Exception):
    """File was rotated"""


if __name__ == '__main__':
    if any(flag in sys.argv for flag in ['-c', '-config']):
        prompt_config()
        sys.stdout.write('config has been written under /etc/log_reader.ini\n')
        sys.exit(0)

    config, regexes = get_or_create_config()
    conn = Http(config.host, config.url, config.path, config.port)
    inactive_bucket = Bucket()

    error_msg = check_regexes(regexes)
    if error_msg:
        sys.stderr.write(str(error_msg) + '\n')
        sys.exit(1)

    sys.stdout.write('log reader started\n')
    sys.stdout.write(str(config) + '\n')

    while True:
        with open(config.filename) as f:
            f_id = os.fstat(f.fileno()).st_ino
            if config.start_from_top != "True":
                f.seek(0, os.SEEK_END)

            while True:
                line = f.readline()
                if not line:
                    time.sleep(inactive_bucket.up())
                    if inactive_bucket.up_counter() and os.stat(config.filename).st_ino != f_id:
                        break  # File has been rotated, close this file, and reopen the new one
                continue

            inactive_bucket.down()
            for reason, regex in regexes.iteritems():
                if bool(re.search(regex, line)):
                    # print(bool(re.search(regex, line)), regex, line)
                    conn.send_request(create_message(line, reason))