Generate master list of fields for Super Search

generate-master-list.py

import copy
import json


PARAM_TO_FIELD_MAPPING = {
    # Processed crash keys.
    'build_id': 'build',
    'date': 'date_processed',
    'platform': 'os_name',
    'platform_version': 'os_version',
    'plugin_name': 'PluginName',
    'plugin_filename': 'PluginFilename',
    'plugin_version': 'PluginVersion',
    'winsock_lsp': 'Winsock_LSP',
    # Raw crash keys.
    'accessibility': 'Accessibility',
    'adapter_device_id': 'AdapterDeviceID',
    'adapter_vendor_id': 'AdapterVendorID',
    'android_board': 'Android_Board',
    'android_brand': 'Android_Brand',
    'android_cpu_abi': 'Android_CPU_ABI',
    'android_cpu_abi2': 'Android_CPU_ABI2',
    'android_device': 'Android_Device',
    'android_display': 'Android_Display',
    'android_fingerprint': 'Android_Fingerprint',
    'android_hardware': 'Android_Hardware',
    'android_manufacturer': 'Android_Manufacturer',
    'android_model': 'Android_Model',
    'android_version': 'Android_Version',
    'async_shutdown_timeout': 'AsyncShutdownTimeout',
    'available_page_file': 'AvailablePageFile',
    'available_physical_memory': 'AvailablePhysicalMemory',
    'available_virtual_memory': 'AvailableVirtualMemory',
    'b2g_os_version': 'B2G_OS_Version',
    'bios_manufacturer': 'BIOS_Manufacturer',
    'cpu_usage_flash_process1': 'CpuUsageFlashProcess1',
    'cpu_usage_flash_process2': 'CpuUsageFlashProcess2',
    'dom_ipc_enabled': 'DOMIPCEnabled',
    'em_check_compatibility': 'EMCheckCompatibility',
    'frame_poison_base': 'FramePoisonBase',
    'frame_poison_size': 'FramePoisonSize',
    'is_garbage_collecting': 'IsGarbageCollecting',
    'min_arm_version': 'Min_ARM_Version',
    'number_of_processors': 'NumberOfProcessors',
    'oom_allocation_size': 'OOMAllocationSize',
    'plugin_cpu_usage': 'PluginCpuUsage',
    'plugin_hang': 'PluginHang',
    'plugin_hang_ui_duration': 'PluginHangUIDuration',
    'startup_time': 'StartupTime',
    'system_memory_use_percentage': 'SystemMemoryUsePercentage',
    'theme': 'Theme',
    'throttleable': 'Throttleable',
    'total_virtual_memory': 'TotalVirtualMemory',
    'vendor': 'Vendor',
}


FIELD_TO_PARAM_MAPPING = dict(
    (PARAM_TO_FIELD_MAPPING[x], x) for x in PARAM_TO_FIELD_MAPPING
)


FIELDS_WITH_FULL_VERSION = (
    'processed_crash.cpu_info',
    'processed_crash.os_name',
    'processed_crash.product',
    'processed_crash.reason',
    'processed_crash.signature',
    'processed_crash.user_comments',
    'processed_crash.PluginFilename',
    'processed_crash.PluginName',
    'processed_crash.PluginVersion',
    'raw_crash.Android_Model',
)


FORM_FIELD_TYPES = {
    "address": "StringField",
    "app_notes": "MultipleValueField",
    "build_id": "IntegerField",
    "cpu_info": "StringField",
    "cpu_name": "MultipleValueField",
    "date": "DateTimeField",
    "distributor": "MultipleValueField",
    "distributor_version": "MultipleValueField",
    "dump": "StringField",
    "flash_version": "MultipleValueField",
    "hang_type": "MultipleValueField",
    "install_age": "IntegerField",
    "java_stack_trace": "MultipleValueField",
    "last_crash": "IntegerField",
    "platform": "MultipleValueField",
    "platform_version": "StringField",
    "plugin_name": "MultipleValueField",
    "plugin_filename": "MultipleValueField",
    "plugin_version": "MultipleValueField",
    "process_type": "MultipleValueField",
    "processor_notes": "MultipleValueField",
    "product": "MultipleValueField",
    "productid": "MultipleValueField",
    "reason": "StringField",
    "release_channel": "MultipleValueField",
    "signature": "StringField",
    "topmost_filenames": "MultipleValueField",
    "uptime": "IntegerField",
    "user_comments": "StringField",
    "version": "MultipleValueField",
    "winsock_lsp": "MultipleValueField",
    "accessibility": "BooleanField",
    "additional_minidumps": "MultipleValueField",
    "adapter_device_id": "MultipleValueField",
    "adapter_vendor_id": "MultipleValueField",
    "android_board": "MultipleValueField",
    "android_brand": "MultipleValueField",
    "android_cpu_abi": "MultipleValueField",
    "android_cpu_abi2": "MultipleValueField",
    "android_device": "MultipleValueField",
    "android_display": "MultipleValueField",
    "android_fingerprint": "MultipleValueField",
    "android_hardware": "MultipleValueField",
    "android_manufacturer": "MultipleValueField",
    "android_model": "StringField",
    "android_version": "MultipleValueField",
    "async_shutdown_timeout": "MultipleValueField",
    "available_page_file": "IntegerField",
    "available_physical_memory": "IntegerField",
    "available_virtual_memory": "IntegerField",
    "b2g_os_version": "MultipleValueField",
    "bios_manufacturer": "MultipleValueField",
    "cpu_usage_flash_process1": "IntegerField",
    "cpu_usage_flash_process2": "IntegerField",
    "dom_ipc_enabled": "BooleanField",
    "em_check_compatibility": "BooleanField",
    "frame_poison_base": "MultipleValueField",
    "frame_poison_size": "IntegerField",
    "is_garbage_collecting": "BooleanField",
    "min_arm_version": "MultipleValueField",
    "number_of_processors": "IntegerField",
    "oom_allocation_size": "IntegerField",
    "plugin_cpu_usage": "IntegerField",
    "plugin_hang": "BooleanField",
    "plugin_hang_ui_duration": "IntegerField",
    "startup_time": "IntegerField",
    "system_memory_use_percentage": "IntegerField",
    "throttleable": "BooleanField",
    "throttle_rate": "IntegerField",
    "theme": "MultipleValueField",
    "total_virtual_memory": "IntegerField",
    "useragent_locale": "MultipleValueField",
    "vendor": "MultipleValueField",
}


BASE_STRUCT = {
    "name": "xxx",
    "in_database_name": "xxx",
    "storage_mapping": None,
    "query_type": "enum",
    "data_validation_type": "enum",
    "namespace": "processed_crash",
    "has_full_version": False,
    "form_field_type": "MultipleValueField",
    "form_field_choices": None,
    "permissions_needed": [],
    "default_value": None,
    "is_exposed": True,
    "is_returned": True,
    "is_mandatory": False,
}


def populate_fields(all_fields, fields_list, namespace, mapping):
    for f in fields_list:
        field_name = FIELD_TO_PARAM_MAPPING.get(f, f)

        field_data = copy.deepcopy(BASE_STRUCT)
        field_data['name'] = field_name
        field_data['in_database_name'] = f
        field_data['namespace'] = namespace
        field_data['has_full_version'] = '%s.%s' % (namespace, f) in FIELDS_WITH_FULL_VERSION
        field_data['is_exposed'] = field_name in FORM_FIELD_TYPES.keys()
        field_data['is_returned'] = field_name in FORM_FIELD_TYPES.keys()

        form_field_type = FORM_FIELD_TYPES.get(field_name, None)

        field_data['form_field_type'] = form_field_type

        if form_field_type:
            if form_field_type == 'DateField':
                field_data['query_type'] = 'date'
                field_data['data_validation_type'] = 'date'
            elif form_field_type == 'DateTimeField':
                field_data['query_type'] = 'date'
                field_data['data_validation_type'] = 'datetime'
            elif form_field_type == 'IntegerField':
                field_data['query_type'] = 'number'
                field_data['data_validation_type'] = 'int'
            elif form_field_type == 'BooleanField':
                field_data['query_type'] = 'bool'
                field_data['data_validation_type'] = 'bool'
            elif form_field_type == 'StringField':
                field_data['query_type'] = 'string'
                field_data['data_validation_type'] = 'str'
            else:
                field_data['query_type'] = 'enum'
                field_data['data_validation_type'] = 'enum'

        field_mapping = mapping.get(field_data['in_database_name'])
        if field_mapping:
            field_data['storage_mapping'] = field_mapping

        all_fields[field_name] = field_data


def main():
    all_fields = {}

    settings_json = open(
        './socorro/external/elasticsearch/mappings/socorro_index_settings.json'
    ).read()
    mapping = json.loads(settings_json % 'index')
    mapping = mapping['mappings']['index']['properties']

    populate_fields(
        all_fields,
        mapping['processed_crash']['properties'].keys(),
        'processed_crash',
        mapping['processed_crash']['properties']
    )
    populate_fields(
        all_fields,
        mapping['raw_crash']['properties'].keys(),
        'raw_crash',
        mapping['raw_crash']['properties']
    )

    print json.dumps(all_fields, indent=4, sort_keys=True)


if __name__ == "__main__":
    main()