druid_metrics_parser.py

#!/usr/bin/env python3

import collections
import json

def parse(filename):
    res = []
    with open(filename, "r") as fd:
        lines = fd.readlines()
        for line in lines:
            line = line.strip()
            if "metrics" in line and line[-1] == "}":
                begin = line.find("{")
                json_str = line[begin:]
                record = json.loads(json_str)
                res.append(record)
    return res

def distinct_types(dump):
    types = []
    for record in dump:
        metric_type = record.get("metric", "UNKNOWN")
        if metric_type not in types:
            types.append(metric_type)
    return types

def group_by_metric(dump, metric_type):
    res = []
    for record in dump:
       metric_t = record.get("metric", "UNKNOWN")
       if metric_t == metric_type:
           time = record.get("timestamp", "")
           value = record.get("value", 0)
           data_source = record.get("dataSource", "")
           duration = record.get("duration", "")
           res.append({
               "timestamp": time,
               "value": value,
               "dataSource": data_source,
               "duration": duration
           })
    return res

def pretty_print(metric_records):
    for record in metric_records:
        print("%-40s %10s %15s %s" % (record["dataSource"], record["value"], record["duration"], record["timestamp"]))

def group_by_data_source(metric_records):
    res = collections.Counter()
    for record in metric_records:
        res[record["dataSource"]] += record["value"]
    return res

if __name__ == "__main__":
    filename = "broker-service.log"
    raw = parse(filename)
    metrics = group_by_metric(raw, "query/time")
    pretty_print(metrics)