parse_log.py

import re, json, itertools

path = 'log.txt'
text = open(path).readlines()
data = [ json.loads(re.findall(r".+?\d{3}\s-\s(.*)$", x.strip())[0]) for x in  text]

rules = list(set([r.strip() for r in itertools.chain(*[x['rule'].split(',') for x in data])]))

rules_data = {r:[] for r in rules}
for rule in rules:
    for line in data:
        if rule in line['rule']:
            rules_data[rule].append(line['hypotheses'])
rules_worlds = {k: list(itertools.chain(*v))  for k, v in rules_data.items()}

counts = {}
for rule, items in rules_worlds.items():
    counts[rule] = {i:items.count(i) for i in set(items)}