negedng’s gists

negedng / default_covariance.py

Last active March 6, 2024 09:26

NumPy and Scikit-learn has different default value for DDOF.

	import numpy as np
	from sklearn import linear_model

	# The data
	X = np.array([10,11,15,20,30,50,60,61,70])
	Y = np.array([3,4,3,5,10,10,12,11,13])

	# Calculating mean
	mean_X = np.mean(X)
	mean_Y = np.mean(Y)

negedng / subtitles2vocab.py

Last active May 13, 2021 20:20

Get vocab for a movie

	# Parameters
	language = 'es' # Tested: 'en' or 'es'
	text_file_path = 'movie_subtitle.xml'
	known_words_path = 'known_word_list.txt' # one word per line
	# More about universal part-of-speech: https://universaldependencies.org/u/pos/
	skip_upos = ['PUNCT', 'PRON', 'DET', 'ADP', 'SYM', 'X']
	most_common = 30

	# Loading dependencies
	import re

negedng / BERT_HUB_model_v3.py

Created November 15, 2020 11:50

Updated v3 BERT model from https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/3

	import tensorflow_text as text # Registers the ops.
	import tensorflow as tf
	import tensorflow_hub as hub

	# text_input = ["This is a sample sentence."]
	text_input = tf.keras.layers.Input(shape=(), dtype=tf.string)
	preprocessor = hub.KerasLayer(
	"https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/1")
	encoder_inputs = preprocessor(text_input) # dict with keys: 'input_mask', 'input_type_ids', 'input_word_ids'
	encoder = hub.KerasLayer(

negedng / names_BERT.csv

Created November 11, 2020 16:54

Black-white female-male name representations in BERT

negedng / huggingface_transformer_example.py

Last active November 11, 2020 14:25

Example from https://github.com/huggingface/transformers

	from transformers import AutoTokenizer, AutoModel

	tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
	model = AutoModel.from_pretrained("bert-base-cased")

	inputs = tokenizer("Hello world!", return_tensors="pt")
	outputs = model(**inputs)

negedng / nlp_datasets_hfds_training.py

Created October 18, 2020 22:04

	ds2_train = (
	ds2_train
	.shuffle(100000)
	.batch(32)
	.prefetch(tf.data.experimental.AUTOTUNE)
	)
	ds2_val = (
	ds2_val
	.batch(32)
	.prefetch(tf.data.experimental.AUTOTUNE)

negedng / nlp_datasets_hfds_format_data.py

Created October 18, 2020 21:56

see tutorial: https://huggingface.co/docs/datasets/quicktour.html

	# Encoding the data to integer token ids
	def encode(examples):
	tokens = [enc.ids for enc in tokenizer.encode_batch(examples['text'])]
	return {'tokens': tokens}
	ds2_train = ds2_train.map(encode, batched=True)
	ds2_train = ds2_train.map(lambda examples: {'labels': examples['label']}, batched=True)
	ds2_val = ds2_val.map(encode, batched=True)
	ds2_val = ds2_val.map(lambda examples: {'labels': examples['label']}, batched=True)

	# Format to TensorFlow Dataset

negedng / nlp_datasets_hfds_tokenizer_freq.py

Created October 18, 2020 21:49

Tokenizer from the frequency list

	# Formating vocab dictionary from the most common words
	vocab_dict = {k:i+4 for i,k in enumerate([l for l,m in vocabulary_counter.most_common(20000-4)])}
	# Adding the special characters
	vocab_dict["[PAD]"]=0
	vocab_dict["[UNK]"]=1
	vocab_dict["[CLS]"]=2
	vocab_dict["[SEP]"]=3
	vocab_dict["[MASK]"]=4

	tokenizer_2 = BertWordPieceTokenizer(vocab_dict)

negedng / nlp_datasets_hfds_tokenizer_training.py

Created October 18, 2020 21:36

	with open("imdb_train_plain_lines.txt",'w') as f:
	for examples in ds2_train:
	f.write(examples['text'])
	f.write('\n')

	from tokenizers import Tokenizer, models, pre_tokenizers, decoders, trainers, processors, BertWordPieceTokenizer

	# Initialize a tokenizer
	tokenizer = BertWordPieceTokenizer()

negedng / nlp_datasets_hfds_data_import.py

Created October 18, 2020 21:17

	ds2 = datasets.load_dataset("imdb")

	ds2_train = ds2['train']
	ds2_val = ds2['test']

Gergely Nemeth negedng