Python script to update fields.csv and methods.csv, merging both the versions from 9.01 alpha and MCPTest. NOTE, this is a hack and makes way too many assumptions for anyone to run without reading it through and figuring out what to do from there. NOTE, at time of writing, one of the methods from MCPTest has a name conflict with one of the metho…

update_mcptest

import csv
import os.path
import re
from collections import defaultdict
from contextlib import closing
from datetime import timedelta
from operator import itemgetter
from time import time as now
from urllib.request import urlopen

csv_field_names = ["searge","name","side","desc"]

re_lookup = { "fields": re.compile("field_(\d+)_\w+"),
              "methods": re.compile("func_(\d+)_\w+") }

def get_id(searge, base_name):
    expression = re_lookup[base_name]
    m = re.search(expression, searge)
    return int(m.group(1))

def update_from_mcptest(base_name, age_to_update = timedelta(hours=1)):
    file_name = base_name + "_mcptest.csv"
    if os.path.exists(file_name):
        last_modification_time = os.path.getmtime(file_name)
        current_time = now()
        age = timedelta(seconds = current_time - last_modification_time)
        if age < age_to_update:
            print("Not updating from mcptest, since we just did it {age} minutes ago.".format(age = age.seconds / 60))
            return

    url = "http://mcpold.ocean-labs.de/files/mcptest/" + base_name + ".csv"
    with open(file_name, "wb") as outfile, \
         closing(urlopen(url)) as response:
        outfile.write(response.read())

def merge_csvs(base_name):
    update_from_mcptest(base_name)

    # we store one mapping per side (client / server / future_thing)
    results = defaultdict(lambda: defaultdict(dict))
    for csvfile in (base_name + "_901alpha.csv", base_name + "_mcptest.csv"):
        with open(csvfile, encoding="utf-8", newline="") as infile:
            reader = csv.DictReader(infile)
            for row in reader:
                id = get_id(row["searge"], base_name)
                side = row["side"]
                results[side][id] = row

    with open(base_name + ".csv", "w", encoding="utf-8", newline="") as outfile:
        writer = csv.DictWriter(outfile, csv_field_names)
        writer.writeheader()

        # right now, results is {"0": {srg_number: {"searge": "field_123_a", "side": "0", ...}, ...}, "1": {...}}
        # we don't store the two dictionaries separately for any other reason than to make sure we keep both client and server.
        # so let's flatten them into just {srg_number: {"searge": "field_123_a", ...}, ...}, merging both "side": "0" and "side": "1"
        flattened_items = ((key, val) for s in results.values() for (key, val) in s.items())

        # now that we have the flattened list, let's order that by srg_number
        # so both client and server entries are together.
        for _, row in sorted(flattened_items, key=itemgetter(0)):
            writer.writerow(row)

if __name__ == "__main__":
    merge_csvs("fields")
    merge_csvs("methods")