Log file parsing and sorting demonstration

problem-2.py

import re

log_list = []
hosts = set()
hosts_status = set()
test_str = ''

filepath = 'problem-2.txt'  
with open(filepath) as fp:  
    test_str = fp.read().strip()

# test_str = ("p.p1 {margin: 0.0px 0.0px 0.0px 0.0px; font: 10.0px Monaco; color: #f4f4f4; background-color: #000000; background-color: rgba(0, 0, 0, 0.85)} span.s1 {font-variant-ligatures: no-common-ligatures}\n 2017-02-09T02:37:44 [ERROR] Consumer iwjef99 could not be contacted 2017-02-09T02:37:46 [INFO] Message received from Producer w89fj93 2017-02-09T02:37:51 [ERROR] Consumer 7sjeuf returned 504 2017-02-09T02:37:53 [INFO] Message received from Producer a99jef9 2017-02-09T02:37:59 [INFO] Message sent to Consumer a99jef9 2017-02-09T02:38:02 [INFO] Message received from Producer a99jef9 2017-02-09T02:38:08 [INFO] Message received from Producer 7sjeuf 2017-02-09T02:38:10 [INFO] Message sent to Consumer 98w9jfi9 2017-02-09T02:38:10 [INFO] Message sent to Consumer a9i5f 2017-02-09T02:38:12 [INFO] Message received from Producer 98w9jfi9 2017-02-09T02:38:15 [INFO] Message received from Producer w89fj93 2017-02-09T02:38:17 [INFO] Message sent to Consumer i9wjf99 2017-02-09T02:38:21 [INFO] Message sent to Consumer 7sjeuf 2017-02-09T02:38:27 [INFO] Message received from Producer iwjef99 2017-02-09T02:38:29 [INFO] Message received from Producer i9wjf99 2017-02-09T02:38:31 [INFO] Message received from Producer i9wjf99 2017-02-09T02:38:35 [INFO] Message received from Producer 98w9jfi9 2017-02-09T02:38:40 [INFO] Message received from Producer 7sjeuf 2017-02-09T02:38:40 [INFO] Message sent to Consumer w89fj93 2017-02-09T02:38:44 [INFO] Message received from Producer i9wjf99 2017-02-09T02:38:49 [INFO] Message received from Producer 98w9jfi9 2017-02-09T02:38:55 [ERROR] Consumer a99jef9 disconnected unexpectedly 2017-02-09T02:38:55 [ERROR] Consumer i9wjf99 returned 503 2017-02-09T02:38:58 [INFO] Message received from Producer w89fj93 2017-02-09T02:39:02 [INFO] Message received from Producer iwjef99 2017-02-09T02:39:04 [INFO] Message sent to Consumer a99jef9 2017-02-09T02:39:07 [INFO] Message sent to Consumer 98w9jfi9 2017-02-09T02:39:11 [INFO] Message received from Producer 63di9 2017-02-09T02:39:15 [INFO] Message sent to Consumer 8efkwi 2017-02-09T02:39:20 [INFO] Message sent to Consumer a99jef9")

time_stamp_regex = r'\d{1,4}-\d{1,2}-\d{1,2}T\d{1,2}:\d{1,2}:\d{1,2}'
regex = r"\d{1,4}-\d{1,2}-\d{1,2}T\d{1,2}:\d{1,2}:\d{1,2} \[.*?\] (?:\w*) ((?:\w+)\s+)+"
last_word_regex = r'(\w+$)'
hosts_regex = r'((?:Consumer|Producer)) ((?:\w*))'

matches = re.finditer(regex, test_str, re.IGNORECASE | re.MULTILINE)
last_word_matches = re.finditer(last_word_regex, test_str, re.IGNORECASE | re.MULTILINE)
hosts_matches = re.finditer(hosts_regex, test_str, re.IGNORECASE | re.MULTILINE)

last_word = ''

for cnt,match in enumerate(hosts_matches):
    host = match.group()
    hosts_status.add(host)
    hosts.add(host.split()[1])

for cnt,match in enumerate(last_word_matches):
    last_word = match.group()

for cnt,match in enumerate(matches):
    log_list.append(match.group())
log_list[-1] += last_word

final = dict()

for host in hosts:
    final[host] = dict()
    for hs in hosts_status:
        if host in hs:
            temp = list()
            for x in log_list:
                if hs in x:
                    temp.append(re.split(time_stamp_regex,x)[1].strip())
            final[host][hs] = temp
grand = 0
host_number = 0
for k in final.keys():
    print('')
    print("Host {}".format(k),end='\n')
    print("==========================")
    print('')
    total = 0
    for l in final[k].keys():
        temp = final[k][l]
        for x in set(temp):
            times = final[k][l].count(x)
            total += times
            print("{} - {}".format(times,x))
    print('')
    print("{}% Error/Info".format(total))
    print('')
    grand += total
    host_number += 1
print("Total: {}% Error/Info - {} Messages sent/received".format((host_number/grand)*100,grand))
print('')