This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import typer | |
from src.utils.mlflow_run_decorator import mlflow_run | |
@mlflow_run | |
def preprocess_data(input_folder, output_folder): | |
... | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from functools import wraps | |
import mlflow | |
from src.constants import PROJECT_EXPERIMENT_NAME | |
def mlflow_run(wrapped_function): | |
@wraps(wrapped_function) | |
def wrapper(*args, **kwargs): |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
run_pipeline: | |
export MLFLOW_RUN_ID=`python src/utils/start_pipeline.py $(RUN_NAME)`; \ | |
dvc repro |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import mlflow | |
import typer | |
from src.constants import PROJECT_ROOT_PATH, PROJECT_EXPERIMENT_NAME | |
def start_pipeline(run_name): | |
mlflow.set_experiment(PROJECT_EXPERIMENT_NAME) | |
with mlflow.start_run(run_name=run_name): | |
print(mlflow.active_run().info.run_id) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
stages: | |
preprocess_data: | |
foreach: | |
- train | |
- val | |
- test | |
do: | |
cmd: python scripts/preprocess_data.py --input-folder datasets/raw/${item} --output-folder datasets/preprocessed/${item} | |
deps: | |
- scripts/preprocess_data.py |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from pathlib import Path | |
import tensorflow as tf | |
from build_big_model import build_big_model | |
MODEL_NAME = 'big_model' | |
MODEL_VERSION = '1' | |
model = build_big_model() | |
tf.saved_model.save(model, str(Path('models') / MODEL_NAME / MODEL_VERSION)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import threading | |
import time | |
from queue import Empty, Queue | |
import numpy as np | |
from flask import Flask, request as flask_request | |
from build_big_model import build_big_model | |
BATCH_SIZE = 20 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import numpy as np | |
from locust import HttpLocust, TaskSet, task, between | |
INPUT_WIDTH = 10 | |
class PredictTaskSet(TaskSet): | |
@task(1) | |
def predict(self): | |
payload = { |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
import numpy as np | |
from flask import Flask, request | |
from build_big_model import build_big_model | |
model = build_big_model() | |
app = Flask(__name__) | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from tensorflow.keras.layers import Dense | |
from tensorflow.keras import Model, Input | |
def build_big_model( | |
# small input and output in order to avoid latency due to data transfer through the internet during our tests | |
input_width=10, | |
output_width=10, | |
hidden_layers_width=10**4, | |
nb_hidden_layers=200, |
NewerOlder