Skip to content

Instantly share code, notes, and snippets.

@allanj
allanj / iob1toiob2_funct.py
Last active March 29, 2021 14:37
Convert the tags from IOB1 to IOB2 tagging scheme
"""
IOB1: O I I B I
IOB2: O B I B I
"""
from typing import List
def iob2(tags: List[str]):
"""
Check that tags have a valid IOB format.
@BIGBALLON
BIGBALLON / extract_ILSVRC.sh
Created May 13, 2018 20:09
script for ImageNet data extract.
#!/bin/bash
#
# script to extract ImageNet dataset
# ILSVRC2012_img_train.tar (about 138 GB)
# ILSVRC2012_img_val.tar (about 6.3 GB)
# make sure ILSVRC2012_img_train.tar & ILSVRC2012_img_val.tar in your current directory
#
# https://github.com/facebook/fb.resnet.torch/blob/master/INSTALL.md
#
# train/
@jihunchoi
jihunchoi / masked_cross_entropy.py
Last active January 22, 2024 19:20
PyTorch workaround for masking cross entropy loss
def _sequence_mask(sequence_length, max_len=None):
if max_len is None:
max_len = sequence_length.data.max()
batch_size = sequence_length.size(0)
seq_range = torch.range(0, max_len - 1).long()
seq_range_expand = seq_range.unsqueeze(0).expand(batch_size, max_len)
seq_range_expand = Variable(seq_range_expand)
if sequence_length.is_cuda:
seq_range_expand = seq_range_expand.cuda()
seq_length_expand = (sequence_length.unsqueeze(1)