erykml

name: Data validation with deepchecks

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

permissions: write-all

erykml

from deepchecks.vision.simple_classification_data import load_dataset
from deepchecks.vision.suites import train_test_validation
from config import PROCESSED_IMAGES_DIR

train_ds = load_dataset(PROCESSED_IMAGES_DIR, train=True, object_type="VisionData", image_extension="jpg")

test_ds = load_dataset(PROCESSED_IMAGES_DIR, train=False, object_type="VisionData", image_extension="jpg")

suite = train_test_validation()

erykml

from config import PROCESSED_IMAGES_DIR, MODELS_DIR
import os
import tensorflow.keras
import mlflow
from dagshub import dagshub_logger

mlflow.set_tracking_uri("https://dagshub.com/eryk.lewinson/mario_vs_wario_v2.mlflow")
os.environ['MLFLOW_TRACKING_USERNAME'] = USER_NAME
os.environ['MLFLOW_TRACKING_PASSWORD'] = PASSWORD

erykml

stages:
  get_videos:
    cmd: python src/get_videos.py
    deps:
    - src/config.py
    outs:
    - data/videos
  extract_frames:
    cmd: python src/extract_frames.py
    deps:

erykml

dvc run -n get_videos -d src/config.py -o data/videos python src/get_videos.py
git add data/.gitignore dvc.lock dvc.yaml
git commit -m "added get_videos step"
git push
dvc push -r origin

dvc run -n extract_frames -d src/config.py -d src/utils.py -d data/videos -o data/raw  python src/extract_frames.py
git add dvc.yaml dvc.lock data/.gitignore
git commit -m "added extract_frames step"
git push

erykml

dvc remote add origin https://dagshub.com/eryk.lewinson/mario_vs_wario_v2.dvc
dvc remote modify origin --local auth basic
dvc remote modify origin --local user XXXX
dvc remote modify origin --local password XXXX

erykml

.
├── README.md
├── .github
│   └── workflows
│       └── data_validation.yml
├── data
│   ├── processed
│   │   ├── test
│   │   │   ├── mario
│   │   │   ── wario

erykml

from statsmodels.tsa.seasonal import DecomposeResult 

manual_decomposition = DecomposeResult(
    seasonal=seasonal_df["seasonality"],
    trend=seasonal_df["trend"],
    resid=seasonal_df["resid"],
    observed=seasonal_df["n_passengers"],
)

def add_second_decomp_plot(fig, res, legend):

erykml

(
    seasonal_df
    .loc[:, ["n_passengers", "trend", "seasonality", "resid"]]
    .plot(subplots=True, title="Seasonal decomposition - additive")
);

erykml

# create the DF
seasonal_df = y.to_frame()

# calculate the trend component
seasonal_df["trend"] = seasonal_df["n_passengers"].rolling(window=13, center=True).mean()

# detrend the series
seasonal_df["detrended"] = seasonal_df["n_passengers"] - seasonal_df["trend"]

# calculate the seasonal component