kklemon

import torch
import torch.distributed as dist
import torch.multiprocessing as mp

from torch.utils.data import IterableDataset, DataLoader


class DistributedIterableDataset(IterableDataset):
    """
    Example implementation of an IterableDataset that handles both multiprocessing (num_workers > 0) 

simonthompson99

Running

python -m pudb <python_file> - opens up the script in pudb window, not executed anything yet. For fire modules better to make separate script that executes the individual function of interest, difficult to debug the original main file. Need pudb in virtualenv.

Or:

from pudb import set_trace
...
set_trace()

sgraaf

#!/usr/bin/env python
# -*- coding: utf-8 -*-
from argparse import ArgumentParser

import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel as DDP
from torch.utils.data import DataLoader, Dataset
from torch.utils.data.distributed import DistributedSampler
from transformers import BertForMaskedLM

sujitpal

aditya-malte

GongXinyuu

import torch
import torch.nn.functional as F


def gumbel_softmax(logits, tau=1, hard=False, eps=1e-10, dim=-1):
    # type: (Tensor, float, bool, float, int) -> Tensor
    r"""
    Samples from the `Gumbel-Softmax distribution`_ and optionally discretizes.
    You can use this function to replace "F.gumbel_softmax".
    

yzh119

from __future__ import print_function
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable

def sample_gumbel(shape, eps=1e-20):
    U = torch.rand(shape).cuda()
    return -Variable(torch.log(-torch.log(U + eps) + eps))

alvations

# -*- coding: utf-8 -*-
# Natural Language Toolkit: GLEU Score
#
# Copyright (C) 2001-2017 NLTK Project
# Authors:
# Contributors:
# URL: <http://nltk.org/>
# For license information, see LICENSE.TXT

""" GLEU score implementation. """

ericjang

def sample_gumbel(shape, eps=1e-20): 
  """Sample from Gumbel(0, 1)"""
  U = tf.random_uniform(shape,minval=0,maxval=1)
  return -tf.log(-tf.log(U + eps) + eps)

def gumbel_softmax_sample(logits, temperature): 
  """ Draw a sample from the Gumbel-Softmax distribution"""
  y = logits + sample_gumbel(tf.shape(logits))
  return tf.nn.softmax( y / temperature)

chikamichi

# Optional but convenient: use C-b v to paste the tmux buffer.
bind v paste-buffer

# Do not exit from copy-mode when selecting text.
# @see https://github.com/tmux/tmux/issues/337
# Note: the setting might be renamed MouseDragEndX.
# Depending on whether you activated tmux or vi keybindings (I'm using vi mode):
#bind -temaics-copy MouseDragEnd1Pane copy-selection -x
bind -tvi-copy MouseDragEnd1Pane copy-selection -x