Experimenting with MOT data

aggregate_data.py

#!/usr/bin/env python
import sys
import json
from collections import defaultdict

if __name__ == '__main__':

    data = defaultdict(
            lambda: defaultdict(
                lambda: defaultdict(
                    lambda: defaultdict(int))))

    for line in sys.stdin:
        row = line.split('|')
        try:
            year_of_test = int(row[2][:4])
            year_first_used = int(row[13][:4])
        except ValueError:
            continue
        result = row[5]
        make = row[8]
        model = row[9]

        age = year_of_test - year_first_used

        data[make][model][age][result] += 1

    json.dump(data, sys.stdout, indent=2)

get_data.sh

#!/bin/bash
# Use zcat so we can work directly on the gzipped data without unzipping it first
zcat test_result_* | ./aggregate_data.py > breakdown.json