parse_events.py

# Look for problematic ordering of events in Glean events pings

import codecs
import gzip
import json
import sys

results = json.load(open(sys.argv[-1], "r"))

total = 0
first_and_last_identical = 0
first_and_last_same_timestamp = 0
sent_due_to_many_events = 0
out_of_order = 0
for entry in results:
    payload = gzip.decompress(codecs.decode(entry["f0_"].encode('ascii'), 'base64')).decode('utf8')
    if not payload:
        continue

    payload = json.loads(payload)
    total += 1

    events = payload['events']
    timestamps = [ev['timestamp'] for ev in events]

    if len(events) >= 499:
        sent_due_to_many_events += 1

    if timestamps[0] == 0 and timestamps[-1] == 0:
        first_and_last_same_timestamp += 1
        if events[0] == events[-1]:
            first_and_last_identical += 1
    elif len(timestamps) >= 5 and timestamps[0] == 0 and timestamps[-2] == 0:
        if events[0:2] == events[-2:]:
            first_and_last_identical += 1
    elif timestamps != sorted(timestamps):
        out_of_order += 1

print(
    f"total: {total}\n"
    f"first and last same timestamp: {first_and_last_same_timestamp}\n"
    f"first and last identical: {first_and_last_identical}\n"
    f"sent due to many events: {sent_due_to_many_events}\n"
    f"out of order: {out_of_order}\n"
)