stas00

from transformers import CONFIG_MAPPING
from huggingface_hub import HfApi
api = HfApi()
keys = list(CONFIG_MAPPING.keys())
downloads = {}
for key in keys:
    models = api.list_models(filter=key)
    total_downloads = sum(model.downloads if hasattr(model, "downloads") else 0 for model in models)
    downloads[key] = total_downloads
ordered = sorted(downloads.items(), reverse=True, key=lambda t: t[1])

stas00

stas00

# torch/autograd/profiler.py
def build_table(
        events,
        sort_by=None,
        header=None,
        row_limit=100,
        max_src_column_width=75,
        with_flops=False,
        profile_memory=False,
        top_level_events_only=False):

stas00

# conftest.py
# to run:
# TRACE_START_STOP=1 pytest tests/test_trainer.py
import pytest
import os
trace = os.environ.get('TRACE_START_STOP', "")
@pytest.hookimpl(tryfirst=True, hookwrapper=True)
def pytest_runtest_makereport(item, call):
    outcome = yield
    res = outcome.get_result()

stas00

# Samyam: I have three thoughts here:
# 1) would dropping off large activations push the network towards producing smaller activations? I don't the answer but it feels unlikely as the network is not getting penalized in anyway for producing large activations,
# 2) dropout is meant to be used as a regularization but by dropping out only large values, it's introducing a bias. It may have unexpected impact on convergence,
# 3) if 1 does not happen then during time of inference where there is no dropout, we have the inf again


def dropout_abs_max_values(x, p=0.2):
    """ Like Dropout but instead of random sampling, this one zeroth the p fraction of the biggest absolute values """
    topk = int(p * x.shape[-1])
    indices = torch.topk(x.abs(), topk, dim=-1, largest=True)[1]

stas00

import argparse

def compose_actions(*actions):
    """Compose many argparse actions into one callable action.

    Args:
        *actions: The actions to compose.

    Returns:
        argparse.Action: Composed action.

stas00

# python -m torch.distributed.launch --nproc_per_node=2 all_reduce_bench.py

import torch
import torch.distributed as dist
import time
import argparse
import os
import fcntl

TRIALS = 5

stas00

# same as the other script, but this time each thread allocates on a different device
# still reports correctly
import threading
import time
import torch

def print_mem_usage(prefix):
    n_gpus = torch.cuda.device_count()
    for id in range(n_gpus):
        with torch.cuda.device(id):

stas00

# this goes into transformers/testing_utils.py

_pytest_num_workers = 1

def set_pytest_num_workers(n):
    """
    This is helper method that sets how many pytest workers are used (if under pytest-xdist's -n option)
    """
    _pytest_num_workers = n

stas00