model drift using sm models brief example

model_drift.py

import pandas as pd
import statsmodels.api as sm


def test_rolling_mode():
    """
    The goal of this application level monitoring is to react in light of changes
    to re-model fits and movement in the modelling process. In the linear model
    scenario, confidence intervals around the coefficients can be provided which
    can be used as a measure for model drift over time.

    We will use stats model to demonstrate this. Note that GD methods don't
    have confidence intervals as it doesn't really make sense...
    """
    model_data = pd.DataFrame({"x": [1, 2, 3, 4, 5], "y": [1, 2, 3, 4, 4.5]})
    model_data2 = pd.DataFrame({"x": [1, 2, 3, 4, 5], "y": [1.5, 2, 3.5, 4.7, 5]})
    model_data3 = pd.DataFrame({"x": [1, 2, 3, 4, 5], "y": [1.7, 2, 5.5, 7.9, 5]})

    model_data = sm.add_constant(model_data)
    lm = sm.GLM(
        model_data["y"], model_data[["x", "const"]], family=sm.families.Gaussian()
    )
    lm_results = lm.fit()
    lm_df = pd.read_html(
        lm_results.summary().tables[1].as_html(), header=0, index_col=0
    )[0]
    lm_df.columns = ["coef", "std", "z", "Pz", "lower", "upper"]

    # compare with model that shouldn't change too much...
    model_data2 = sm.add_constant(model_data2)
    lm2 = sm.GLM(
        model_data2["y"], model_data2[["x", "const"]], family=sm.families.Gaussian()
    )
    lm2_results = lm2.fit()
    lm2_df = pd.read_html(
        lm2_results.summary().tables[1].as_html(), header=0, index_col=0
    )[0]
    lm2_df.columns = ["coef", "std", "z", "Pz", "lower", "upper"]

    assert (
        lm2_df.loc["x"]["coef"] >= lm_df.loc["x"]["lower"]
        and lm2_df.loc["x"]["coef"] <= lm_df.loc["x"]["upper"]
    )
    assert (
        lm2_df.loc["const"]["coef"] >= lm_df.loc["const"]["lower"]
        and lm2_df.loc["x"]["coef"] <= lm_df.loc["x"]["upper"]
    )


if __name__ == "__main__":
    test_rolling_mode()