Parsing build logs

parse_colcon_times.py

#!/usr/bin/env python3

import sys
import re
import statistics
from collections import defaultdict

def time_to_milliseconds(time_str):
    seconds = 0
    if 'min' in time_str:
        min_sec = time_str.split('min')
        seconds += int(min_sec[0].strip()) * 60
        if len(min_sec) > 1:
            seconds += float(min_sec[1].strip()[:-1])
    elif 's' in time_str:
        seconds += float(time_str.strip()[:-1])
    return seconds * 1000

def milliseconds_to_human_readable(milliseconds):
    minutes, milliseconds = divmod(milliseconds, 60000)
    seconds = milliseconds / 1000
    if minutes:
        return f'{int(minutes)}min {seconds:.2f}s'
    return f'{seconds:.2f}s'

data = defaultdict(list)

for line in sys.stdin:
    name_match = re.search('<<< (.*?) \[', line)
    time_match = re.search('\[(.*?)\]$', line)
    if name_match and time_match:
        name = name_match.group(1)
        time = time_match.group(1)
        milliseconds = time_to_milliseconds(time)
        data[name].append((milliseconds, time))

data_for_sorting = []
times_for_computation = []

for name, times in data.items():
    times.sort(key=lambda x: x[0])
    max_time_milliseconds, max_time_human = times[-1]
    times_for_computation.append(max_time_milliseconds)
    data_for_sorting.append((name, times))

data_for_sorting.sort(key=lambda x: x[1][-1][0])

for name, times in data_for_sorting:
    print(f'Package: {name}')
    for time_milliseconds, time_human in times:
        print(f'Time: {time_milliseconds} ms ({time_human})')

times_for_computation.sort()

total_time = sum(times_for_computation)
average_time = total_time / len(times_for_computation)
median_time = statistics.median(times_for_computation)

print('\nSummary:')
print(f'Total time: {total_time} ms ({milliseconds_to_human_readable(total_time)})')
print(f'Average time: {average_time} ms ({milliseconds_to_human_readable(average_time)})')
print(f'Median time: {median_time} ms ({milliseconds_to_human_readable(median_time)})')

parse_step_times.py

#!/usr/bin/env python3

import sys
import re

def convert_ms(ms):
    minutes, ms = divmod(ms, 60000)
    seconds, ms = divmod(ms, 1000)
    return f"{int(minutes)}m {int(seconds)}s {int(ms)}ms"

def extract_lines(output):
    pattern = r'\[\s*.+?\s*\d+/\d+\]'
    done_pattern = r'DONE'

    results = []
    temp_line = None
    lines = output.split('\n')
    for line in lines:
        if re.search(pattern, line):
            temp_line = line
        elif temp_line and 'DONE' in line:
            time_in_seconds = float(re.search(r'DONE (\d+\.\d+)s', line).group(1))
            results.append((temp_line, time_in_seconds * 1000))
            temp_line = None

    results.sort(key=lambda x: x[1] )

    return results

output = sys.stdin.read()

relevant_lines = extract_lines(output)
for line, time in relevant_lines:
    print(f"{line} took {convert_ms(time)}")