Choosing an Analytics Schema Scales

big_query_dsl.py

__author__ = 'ekampf'

class FieldTypes(object):
    integer = 'INTEGER'
    float = 'FLOAT'
    string = 'STRING'
    record = 'RECORD'
    ts = 'TIMESTAMP'

class FieldMode(object):
    nullable = 'NULLABLE'
    required = 'REQUIRED'
    repeated = 'REPEATED'

def Field(name, column_type, description=None, mode=None, fields=None):
    field = dict(name=name, type=column_type)
    if description:
        field['description'] = description
    if mode:
        field['mode'] = mode
    if fields:
        field['fields'] = fields

    return field

def StringField(name, mode=None, description=None):
    return Field(name, FieldTypes.string, mode=mode, description=description)

def FloatField(name, mode=None, description=None):
    return Field(name, FieldTypes.float, mode=mode, description=description)

def IntField(name, mode=None, description=None):
    return Field(name, FieldTypes.integer, mode=mode, description=description)

def TSField(name, mode=None, description=None):
    return Field(name, FieldTypes.ts, mode=mode, description=description)

def RecordField(name, fields, mode=None, description=None):
    return Field(name, FieldTypes.record, fields=fields, description=description, mode=mode)

schema.py

import big_query_dsl

OBJECT_SCHEMA = [
    StringField('key', description="The object's key/id", mode=FieldMode.required),
    StringField('type', description="The object's type", mode=FieldMode.required),
    StringField('display', description="The object's display name.", mode=FieldMode.nullable)
]

ANALYTICS_SCHEMA = [
    TSField('timestamp'),
    RecordField('subject', OBJECT_SCHEMA, description="This is the entity which is carrying out the action. Ex: '*Eran* wrote a letter'"),
    StringField('verb', description="Describes the action being done. Ex:'UserX *wrote* a letter'"),

    RecordField('direct_object', OBJECT_SCHEMA, description="The noun. The entity on which action is being done. Ex: 'Eran wrote *a letter*"),
    RecordField('indirect_object', OBJECT_SCHEMA, description="The entity indirectly affected by the action. Ex: 'Eran wrote a letter *to Lior*'"),
    RecordField('prepositional_object', OBJECT_SCHEMA, description="An object introduced by a preposition (in, for, of etc), but not the direct or indirect object. Ex: 'Eran put a letter *in an envelope*'"),

    StringField('context', description="JSON providing extra event-specific data"),

    # meta about event collection
    StringField('tracker_version', description="Version string of the software sending events from client."),
    StringField('collection_version', description="Version string of server-side receiving frontend."),
]