Write stats to redis

write_stats.py

#!/bin/python

import redis
import sys
import datetime as DT

class Stat:
    @staticmethod
    def store_new_stat(r, stat):
        p = r.pipeline()

        hash_key = Stat.key_for_stat(stat)

        p.hmset(hash_key, stat)
        p.sadd("idx:hash:stat", hash_key)
        p.sadd("idx:hash:stat:{destination}".format(**stat), hash_key)
        p.sadd("idx:hash:stat:{destination}:{port}".format(**stat), hash_key)

        p.execute()

        return hash_key

    @staticmethod
    def key_for_stat(stat):
        return \
            "hash:stat:{destination}:{port}:{timestamp}" \
            .format(**stat)


def read_lines(f):
    while True:
        line = f.readline()
        if line:
            yield line
        else:
            return


r = redis.Redis(charset='utf-8', decode_responses=True)
known_keys = r.smembers('idx:hash:stat') or set()

last = DT.datetime.now()
records_inserted = 0

for line in read_lines(sys.stdin):
    destination, port = line.strip().split(" ")
    now = DT.datetime.now()
    timestamp = DT.datetime(
        now.year, now.month, now.day, now.hour, 0, 0, 0
    ).strftime("%s")

    stat = {
        'destination': destination,
        'port': port,
        'timestamp': timestamp,
        'count': 1
    }

    key = Stat.key_for_stat(stat)

    if not key in known_keys:
        Stat.store_new_stat(r, stat)
        known_keys.add(key)
    else:
        r.hincrby(key, 'count')

    records_inserted += 1

    time_diff = (now - last).total_seconds()
    if time_diff > 10:
        print("{} records/s"
            .format(records_inserted / time_diff), file=sys.stderr
        )

        last = now
        records_inserted = 0