Skip to content

Instantly share code, notes, and snippets.

View Guitaricet's full-sized avatar

Vlad Lialin Guitaricet

85 followers · 49 following
View GitHub Profile
@Guitaricet
Guitaricet / check_wandb_login.py
Created April 19, 2024 21:23
Wandb check if logged in
import netrc
import os
def is_wandb_logged_in():
netrc_path = os.path.expanduser("~/.netrc")
if not os.path.exists(netrc_path):
return False
auth = netrc.netrc(netrc_path).authenticators("api.wandb.ai")
return bool(auth)
@Guitaricet
Guitaricet / better_repr.py
Last active December 8, 2022 06:01
Read pytorch module descriptions more easily
from torch.nn import ModuleList
def _addindent(s_, numSpaces):
s = s_.split('\n')
# don't do anything for single-line stuff
if len(s) == 1:
return s_
first = s.pop(0)
s = [(numSpaces * ' ') + line for line in s]
@Guitaricet
Guitaricet / named_shape.py
Created December 8, 2022 02:39
add NamedShape property to torch.Tensor
import torch
class NamedShape:
"""A convenience class to make beautifully named shapes."""
def __init__(self, tensor) -> None:
self.names = tensor.names
self.shape = tensor.shape
def __repr__(self) -> str:
@Guitaricet
Guitaricet / generate_multigpu.py
Last active September 1, 2022 18:18
Short example of distributed generation. Pure PyTorch Distributed.
# torchrun --nproc_per_node 2 generate_multigpu.py
# tested on torch==1.12.1 and transformers==4.21
import os
import json
import torch
import torch.distributed as dist
import transformers
import datasets
@Guitaricet
Guitaricet / logger.py
Created June 16, 2022 15:41
Good default logger
import os
import sys
import logging
logging.basicConfig(
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
level=logging.INFO,
stream=sys.stdout,
)
@Guitaricet
Guitaricet / learning_subspaces.ipynb
Created April 5, 2021 19:33
A quick and dirty reimplementation of the training algorithm from Learning Neural Network Subspaces, Wortsman et al., https://arxiv.org/abs/2102.10472
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
@Guitaricet
Guitaricet / lite_transformer.py
Last active April 30, 2020 19:46
Lite transformer encoder layer
class LiteTransformerLayer(nn.Module):
def __init__(self, dim=496, heads=4, kernel_size=4):
super().__init__()
assert dim % 2 == 0
self.attention = Attention(dim // 2, heads=heads)
self.cnn = LightweightConv(dim // 2, kernel=kernel_size) # or Dynamic conv
self.fc = nn.Sequential(
nn.Linear(dim, dim),
@Guitaricet
Guitaricet / transformer.py
Last active April 30, 2020 19:46
Simple transformer encoder layer
class TransformerLayer(nn.Module):
def __init__(self, dim=496, heads=4, ffn_dim=1984):
super().__init__()
self.attention = Attention(dim, heads=heads)
self.fc = nn.Sequential(
nn.Linear(dim, ffn_dim),
nn.ReLU(), # chose your favorite nonlinearity here
nn.Linear(ffn_dim, dim),
)
@Guitaricet
Guitaricet / att.py
Last active October 20, 2021 13:42
Very simple self attention implementation
# Just dot product self-attention
class SelfAttention(nn.Module):
def __init__(self, dim=7):
super().__init__()
self.K = nn.Linear(dim, dim)
self.Q = nn.Linear(dim, dim)
self.V = nn.Linear(dim, dim)
self.scale = dim ** 0.5
def forward(self, x):
@Guitaricet
Guitaricet / ipynb_output_filter.py
Created April 9, 2020 17:39
A copy of https://github.com/toobaz/ipynb_output_filter in one gist
#!/usr/bin/env python3
import sys
try:
runarg_idx = sys.argv.index('--rundir')
rundir = sys.argv[runarg_idx+1]
import os
os.chdir(os.path.expanduser(rundir))
except ValueError: