Dmitry Nikitko Puzer

Chief data scientist

Puzer / perspective_transformation.py

Created August 17, 2017 15:15

Perspective transformation for imgaug

	seq = iaa.Sequential([
	Perspective(Normal(0, 0.03), Normal(0, 0.15)), # perspective transformation
	])

Puzer / limit_gpu_mem.py

Last active February 12, 2020 12:59

Limit GPU memory

	# https://www.tensorflow.org/programmers_guide/using_gpu#allowing_gpu_memory_growth

	import tensorflow as tf
	from keras.backend.tensorflow_backend import set_session

	config = tf.ConfigProto()
	config.gpu_options.allow_growth = True
	config.gpu_options.per_process_gpu_memory_fraction = 0.5
	sess = tf.Session(config=config)

Puzer / gist:dc31ed6eff81a13aaf4d361a0ce711cf

Created March 13, 2019 14:29

Global permissions for a dir for a group

setfacl -dm g:shared_group:rwx some_dir

Puzer / another_script.py

Last active March 25, 2019 12:39

Parse arguments with defaults

	from run_something import main as run_something

	run_something({'input':'a.txt',
	'output':'b.txt'})
	# Output: default 5


	run_something({'input':'a.txt',
	'default':10,
	'output':'b.txt'})

Puzer / cv2_video.py

Last active April 10, 2019 10:43

Tensorflow ffmpeg cuda dali

	import cv2
	from tqdm import tqdm
	import numpy as np
	import tensorflow as tf

	VIDEO_PATH = 'ch03_20181228181500.avi'
	MODEL = 'ssd_resnet50_v1_fpn_shared_box_predictor_640x640_coco14_sync_2018_07_03/frozen_inference_graph.pb'
	OUTPUT_TENSORS = ['num_detections:0', 'detection_boxes:0', 'detection_scores:0', 'detection_classes:0']
	BATCH_SIZE = 15

Puzer / bpe.py

Last active October 9, 2019 17:14

	bpemb_ru = BPEmb(lang='ru', dim=50)


	def extract_text(json_data):
	ru_text = list(filter(lambda x: any(1040 <= ord(y) <= 1103 for y in x), json_data.split('"')))
	return ' '.join(ru_text)


	def embed_text(text):
	ids = bpemb_ru.encode_ids(text)

Puzer / install.sh

Last active November 21, 2019 11:52

	#!/bin/bash
	wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O $HOME/miniconda.sh
	bash $HOME/miniconda.sh -b -p $HOME/miniconda
	rm miniconda.sh
	$HOME/miniconda/condabin/conda install jupyter scipy pandas numpy scikit-learn -y
	$HOME/miniconda/condabin/conda init
	$HOME/miniconda/bin/jupyter-notebook --generate-config
	echo "c.NotebookApp.allow_remote_access = True" >> $HOME/.jupyter/jupyter_notebook_config.py
	echo "c.NotebookApp.open_browser = False" >> $HOME/.jupyter/jupyter_notebook_config.py
	echo "c.NotebookApp.password_required = False" >> $HOME/.jupyter/jupyter_notebook_config.py

Puzer / pytorch_compile.sh

Created December 6, 2019 11:41

Compile PyTorch from sources

	# Assuming that conda enviroment installed and activated
	sudo apt install gcc build-essential
	conda install numpy ninja pyyaml mkl mkl-include setuptools cmake cffi
	git clone --recursive https://github.com/pytorch/pytorch
	export USE_CUDA=0
	export USE_DISTRIBUTED=0
	export CMAKE_PREFIX_PATH=${CONDA_PREFIX:-"$(dirname $(which conda))/../"}
	python setup.py install

Puzer / tensorflow_affine_to_projective.py

Created December 25, 2019 13:00

	import tensorflow as tf
	from tensorflow.contrib.image import matrices_to_flat_transforms, transform

	sess = tf.Session()


	# affine_M = [[ 0.14, 0. , -18.4 ],
	# [ -0. , 0.14, -16.28],
	# [0, 0, 1]]

Puzer / optimize_and_predict_daal4py.py

Created April 23, 2021 20:27

LGBM Inference Optimization

	import daal4py as d4p
	from lightgbm import LGBMClassifier

	model = LGBMClassifier()
	model.fit(X_data, y_data)
	y_pred_orig = model.predict_proba(x_data_dense)[:,1]

	daal_model = d4p.get_gbt_model_from_lightgbm(model.booster_)

	predictions_container = d4p.gbt_classification_prediction(nClasses=2, resultsToEvaluate='computeClassProbabilities', fptype='float')