Skip to content

Instantly share code, notes, and snippets.

View vikramsoni2's full-sized avatar

Vikram vikramsoni2

View GitHub Profile
@vikramsoni2
vikramsoni2 / parallel_bars_progress_apply.py
Created April 9, 2019 23:58
tqdm progress_apply on multiprocessing, showing parallel progress bars.
import numpy as np
import pandas as pd
from multiprocessing import cpu_count, Pool, current_process
from functools import partial
from tqdm import tqdm
def split_df_by_group(_df, entity, chunks):
df_split=[]
entities = _df[entity].unique()
for i in range(chunks): df_split.append(_df[_df[entity].isin(entities[i::chunks])].copy())
@vikramsoni2
vikramsoni2 / remove_highly_correlated_feats.py
Created April 16, 2019 14:57
remove highly linearly correlated feature
corr_matrix = all_data.corr().abs()
upper = corr_matrix.where(np.triu(np.ones(corr_matrix.shape), k=1).astype(np.bool))
to_drop = [c for c in upper.columns if any(upper[c] > 0.95)]
del upper
drop_column = all_data.columns[to_drop]
print('drop columns:', drop_column)
from scipy import optimize
def calc_shifted_ewm(series, alpha, adjust=True):
return series.shift().ewm(alpha=alpha, adjust=adjust).mean()
def find_best_signal(series, adjust=False, eps=10e-5):
def f(alpha):
@vikramsoni2
vikramsoni2 / tensorflow_dynamic_gpu_allocation.py
Last active July 23, 2020 09:23
Tensorflow dynamic gpu resource allocation
# actual code you can copy-paste into your Keras code to have Tensorflow dynamically allocate the GPU memory:
import tensorflow as tf
from keras.backend.tensorflow_backend import set_session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True # dynamically grow the memory used on the GPU
config.log_device_placement = True # to log device placement (on which device the operation ran)
# (nothing gets printed in Jupyter, only if you run it standalone)
sess = tf.Session(config=config)
set_session(sess) # set this TensorFlow session as the default session for Keras
@vikramsoni2
vikramsoni2 / fast_confusion_matrix.py
Last active September 5, 2022 19:30
fast confusion matrix
from bokeh.models import HoverTool
from bokeh.layouts import gridplot, column
from bokeh.plotting import figure, output_notebook, show, ColumnDataSource
from sklearn.metrics import precision_recall_curve
def fast_confusion_matrix(true_labels, predicted_labels, decimals=3, plot=True, only_best=False, min_recall=0):
precision, recall, thresholds = precision_recall_curve(true_labels, predicted_labels)
@vikramsoni2
vikramsoni2 / bokeh_residual_plot.py
Last active March 2, 2022 15:25
regression residual plot bokeh
def residual_plot(y_true, y_pred, **kwargs):
line_end = np.max([df['O_target'].max(), df['prediction'].max()])
p = figure(plot_width=900, plot_height=500, title="Residual Plot",
x_axis_label='Actual', y_axis_label='Predicted', **kwargs )
data = { 'Actual': y_true, 'Predicted': y_pred }
p.circle(x='Actual', y='Predicted', source=data, fill_color='red', alpha=0.8, line_width=0.2, line_color='black')
@vikramsoni2
vikramsoni2 / super_describe.py
Created June 26, 2020 07:57
describe pandas dataframe
PERCENTILES = [.005, .01, .025, .05, .10, .20, .25, .50, .75, .80, .90, .95, .975, .99]
def super_describe(_d: pd.DataFrame, percentiles: Iterable[float]=PERCENTILES, missing_only: bool=False) -> pd.DataFrame:
"""
Include counts of missing and of unique values.
"""
if isinstance(_d, pd.Series):
_d = _d.to_frame()
_dd = pd.concat((_d.isnull().sum().rename('missing'), _d.nunique(axis=0).rename('unique'), _d.describe(percentiles=percentiles).T, ), axis=1, sort=False)
return _dd.loc[_dd.missing > 0] if missing_only else _dd
@vikramsoni2
vikramsoni2 / course.md
Last active July 12, 2020 13:25
Chepter 1

Fundamentals of computer

What is computer?

A computer is an electronic device that manipulates information, or data. It has the ability to store, retrieve, and process data. It can process information in very fast speed. You can use a computer to type documents, send email play games, and browse the Web. You can also use it to edit or create spreadsheets, presentations, and even videos


Hardware

@vikramsoni2
vikramsoni2 / seaborn_heatmap.py
Last active March 2, 2022 15:23
seaborn heatmap with correction where boxes are cut in half
correlation = df.corr(method='pearson')
correlation = np.abs(correlation)*100
f, ax = plt.subplots(figsize=(30, 30))
cmap = sns.cubehelix_palette(light=1, as_cmap=True)
ax = sns.heatmap(correlation, cbar=False, annot=True, cmap=cmap, square=True, fmt='.0f',
annot_kws={'size': 10})
plt.title('Corrlation Between Features')
@vikramsoni2
vikramsoni2 / templates.json
Last active January 12, 2021 03:14
portainer templates
{
"version": "2",
"templates": [
{
"type": 1,
"title": "Airsonic",
"name": "airsonic",
"description": "Airsonic is a free, web-based media streamer, providing ubiqutious access to your music. Use it to share your music with friends, or to listen to your own music while at work. You can stream to multiple players simultaneously, for instance to one player in your kitchen and another in your living room.",
"logo": "https://raw.githubusercontent.com/SelfhostedPro/selfhosted_templates/master/Images/airsonic-logo.png",
"image": "linuxserver/airsonic:latest",