Test doppelganger on different-sized marginals

accuracy.py

import csv
import os
os.chdir('/Users/gregmacfarlane/tf/doppelganger/examples')

import pandas as pd
import logging
log = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)


from doppelganger import (
    allocation,
    inputs,
    Configuration,
    HouseholdAllocator,
    PumsData,
    SegmentedData,
    BayesianNetworkModel,
    Population,
    Preprocessor,
    Marginals
)

STATE = '06'
PUMA = '00106'
output_dir = ".test"

log.info("Loading configuration and data")
configuration = Configuration.from_file('sample_data/config.json')
preprocessor = Preprocessor.from_config(configuration.preprocessing_config)

household_fields = tuple(set(
    field.name for field in allocation.DEFAULT_HOUSEHOLD_FIELDS).union(
        set(configuration.household_fields)
))


households_data = PumsData.from_csv('sample_data/households_00106_dirty.csv').clean(
    household_fields, preprocessor, puma=PUMA
)

persons_fields = tuple(set(
    field.name for field in allocation.DEFAULT_PERSON_FIELDS).union(
        set(configuration.person_fields)
))
persons_data = PumsData.from_csv('sample_data/persons_00106_dirty.csv').clean(
    persons_fields, preprocessor, puma=PUMA
)

log.info("Loading model")


person_segmentation = lambda x: x[inputs.AGE.name]

person_training_data = SegmentedData.from_data(
    persons_data,
    list(configuration.person_fields),
    inputs.PERSON_WEIGHT.name,
    person_segmentation
)
person_model = BayesianNetworkModel.train(
    person_training_data,
    configuration.person_structure,
    configuration.person_fields
)

household_segmenter = lambda x: x[inputs.NUM_PEOPLE.name]

household_training_data = SegmentedData.from_data(
    households_data,
    list(configuration.household_fields),
    inputs.HOUSEHOLD_WEIGHT.name,
    household_segmenter,
)
household_model = BayesianNetworkModel.train(
    household_training_data,
    configuration.household_structure,
    configuration.household_fields
)



def generate_population(marginals_file):
    controls = Marginals.from_csv(marginals_file)
    allocator = HouseholdAllocator.from_cleaned_data(controls, households_data, persons_data)

    population = Population.generate(
        allocator, person_model, household_model
    )

    log.info("%s \t \t %d \t %d \t %d", marginals_file,
             households_data.data['household_weight'].sum(),
             controls.data['num_people_count'].sum(),
             population.generated_households['household_id'].count())


log.info("File \t \t PUMS \t Controls \t Generated")
generate_population('sample_data/marginals_00106.csv')
generate_population('sample_data/marginals_00106_modified.csv')

marginals_00106_modified.csv