Skip to content

Instantly share code, notes, and snippets.

@Eligijus112
Eligijus112 / train_iterator_nyc.py
Last active October 8, 2022 16:52
Trains a deep learning model using iterators in Python
# Data wrangling
import pandas as pd
# Deep learning
import tensorflow as tf
import keras
# Import feature engineering functions
from utils import create_date_vars, distance_calculation, custom_transform
@Eligijus112
Eligijus112 / DataGeneratorNYC.py
Last active October 8, 2022 19:08
Data generator for NYC data
import pandas as pd
import numpy as np
import keras
# Defining the class for the batches creation
class DataGenerator(keras.utils.Sequence):
def __init__(
self,
csv_generator: pd.io.parsers.readers.TextFileReader,
n_batches: int,
@Eligijus112
Eligijus112 / train_whole_data.py
Last active October 8, 2022 15:11
Whole modeling pipeline
# Importing the feature engineering pipeline
from utils import ft_engineering_pipeline
# Data wrangling
import pandas as pd
# Memory tracking
from memory_profiler import profile
# Command line arguments
@Eligijus112
Eligijus112 / create_model.py
Created October 7, 2022 05:08
Loading a model in memory
# Deep learning
import tensorflow as tf
import keras
# Memory tracking
from memory_profiler import profile
@profile
def create_model(
input_size: int,
@Eligijus112
Eligijus112 / dummy_vars.py
Created October 6, 2022 19:48
Function to create dummy variables for the NYC dataset
import pandas as pd
from sklearn.preprocessing import OneHotEncoder
import numpy as np
# Defining the function for dummy creation
def create_dummy(df: pd.DataFrame, dummy_var_list: list) -> Tuple:
"""
Creates dummy variables for the variables in dummy_var_list
Returns a tuple of the following
@Eligijus112
Eligijus112 / get_distance.py
Created October 6, 2022 19:41
Function to calclulation the distance between two points on a map used for NYC dataset
import numpy as np
import pandas as pd
# Defining the function for distance calculation
def distance_calculation(df: pd.DataFrame) -> pd.DataFrame:
"""
Calculates the distance between two points on the earth's surface.
The distance is in meters
"""
@Eligijus112
Eligijus112 / nyc_date_conversion.py
Last active October 6, 2022 19:57
Date conversion and feature engineering for NYC cab data
import pandas as pd
from datetime import datetime
import numpy as np
import re
# To datetime conversion
def to_datetime(x: str) -> datetime:
"""
Converts a string to a datetime object
An example of the string is 2010-02-02 17:24:55
@Eligijus112
Eligijus112 / analysis.ipynb
Created September 25, 2022 11:09
Regularization in Python
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
@Eligijus112
Eligijus112 / elastic_net_to_keras.py
Created September 23, 2022 05:10
From sklearn to tensorflow regularization
def elastic_net_to_keras(alpha: float, l1_ratio: float):
"""
Converts ElasticNet parameters from sklearn to Keras regularizers.
Arguments
---------
alpha: float
The regularization strength of the model.
l1_ratio: float
The l1 regularization ratio of the model.
@Eligijus112
Eligijus112 / NMSE.py
Created September 23, 2022 04:57
Custom MSE loss for tensorflow
# Importing the package
import tensorflow as tf
class NMSE(tf.keras.losses.Loss):
def __init__(self):
super().__init__()
def call(self, y_true, y_pred):
# Calculating the mse;