wynimator.py

#!/usr/bin/env python2.7

import re
from elasticsearch import Elasticsearch
from elasticsearch_dsl import Search
from elasticsearch_dsl.query import Q

class RequestList(object):
    def __init__(self, host, method_name):
        #self.max_distance =
        self.method_name = method_name
        self.request_re = re.compile(r"^Request messageId (?P<requestId>(\d+)): (?P<rest>(.*))", re.U | re.I)
        self.response_re = re.compile(r"^Response for messageId (?P<requestId>(\d+)): (?P<rest>(.*))", re.U | re.I)

        level_query = Q("match", level=7)
        app_query = Q("match", application="Dialog Big")
        logger_query = Q("match", logger="im.dlg.server.session.RpcHandler")

        self.base_query = Q("bool", must=[app_query, logger_query, level_query])
        self.client = Elasticsearch(host)

        self.requests = {}

    def get_search(self, index):
        return Search(using=self.client, index=index)\
                .query(self.base_query)\
                .sort('@timestamp')\
                .source(['@timestamp', 'message', 'authId'])

    def get_message_type_with_params(self, msg):
        req = self.request_re.match(msg)
        if req is not None:
            return (True, req.groupdict())
        res = self.response_re.match(msg)
        if res is not None:
            return (False, res.groupdict())

    def insert(self, authId, requestId, is_start, ts):
        print 'TS = ', ts, type(ts)
        # self.requests.setdefault(authId, {})
        # self.requests[authId].setdefault(requestId, [])
        # # lookup nearest by ts counterpart for this item
        # items = [x for x in self.requests[authId][requestId] if abs(x['ts'] - ts) < self.max_distance]
        # for k, item in enumerate(items):
        #     if item.get('start') is None and is_start:
        #         item[]

    def store_hit(self, hit):
        result = self.get_message_type_with_params(hit.message)
        if result is None:
            return
        start, data = result
        if self.method_name not in data['rest']:
            return
        self.insert(i.authId, data['requestId'], start, hit['@timestamp'])

    def scan_index(self, index):
        for hit in self.get_search(index).scan():
            self.store_hit(hit)

    def scan(self):
        for index in sorted(self.client.indices.get('dlg-servers-*')):
            self.scan_index(index)

if __name__ == '__main__':
    RequestList('localhost:9200', 'GetDifference').scan()