Skip to content

Instantly share code, notes, and snippets.

View ywolff's full-sized avatar

Yannick Wolff ywolff

View GitHub Profile
import typer
from src.utils.mlflow_run_decorator import mlflow_run
@mlflow_run
def preprocess_data(input_folder, output_folder):
...
from functools import wraps
import mlflow
from src.constants import PROJECT_EXPERIMENT_NAME
def mlflow_run(wrapped_function):
@wraps(wrapped_function)
def wrapper(*args, **kwargs):
@ywolff
ywolff / Makefile
Last active February 17, 2022 11:27
run_pipeline:
export MLFLOW_RUN_ID=`python src/utils/start_pipeline.py $(RUN_NAME)`; \
dvc repro
import mlflow
import typer
from src.constants import PROJECT_ROOT_PATH, PROJECT_EXPERIMENT_NAME
def start_pipeline(run_name):
mlflow.set_experiment(PROJECT_EXPERIMENT_NAME)
with mlflow.start_run(run_name=run_name):
print(mlflow.active_run().info.run_id)
stages:
preprocess_data:
foreach:
- train
- val
- test
do:
cmd: python scripts/preprocess_data.py --input-folder datasets/raw/${item} --output-folder datasets/preprocessed/${item}
deps:
- scripts/preprocess_data.py
from pathlib import Path
import tensorflow as tf
from build_big_model import build_big_model
MODEL_NAME = 'big_model'
MODEL_VERSION = '1'
model = build_big_model()
tf.saved_model.save(model, str(Path('models') / MODEL_NAME / MODEL_VERSION))
import threading
import time
from queue import Empty, Queue
import numpy as np
from flask import Flask, request as flask_request
from build_big_model import build_big_model
BATCH_SIZE = 20
import numpy as np
from locust import HttpLocust, TaskSet, task, between
INPUT_WIDTH = 10
class PredictTaskSet(TaskSet):
@task(1)
def predict(self):
payload = {
import numpy as np
from flask import Flask, request
from build_big_model import build_big_model
model = build_big_model()
app = Flask(__name__)
from tensorflow.keras.layers import Dense
from tensorflow.keras import Model, Input
def build_big_model(
# small input and output in order to avoid latency due to data transfer through the internet during our tests
input_width=10,
output_width=10,
hidden_layers_width=10**4,
nb_hidden_layers=200,