LiutongZhou

"""MinHeap and MaxHeap (Optimized Implementation)"""
from abc import ABC, abstractmethod
from collections import Counter, UserList
from functools import singledispatchmethod
from heapq import (
    _heapify_max,
    _heappop_max,
    _heapreplace_max,
    _siftdown,
    _siftdown_max,

LiutongZhou

"""BitMask"""

class BitMask:
    __slots__ = ("size", "mask")
    
    def __init__(self, size: int = 16):
        """Create a bit mask to store (size) 0/1 status"""
        self.size = size
        self.mask = 1 << size
        

LiutongZhou

Large-Scale Distributed Data and Model Parallel Training

Data Streaming

FastFile Mode

sagemaker.inputs.TrainingInput(S3_INPUT_FOLDER, input_mode='FastFile') 

LiutongZhou

How to document a project

How to update the docs and publish to the Home Page?

Prerequisites

One-time installation of dependencies

python3 -m pip install -U jupyter-book ghp-import

LiutongZhou

Docker Tips

Move default docker storage to another location

nano /etc/docker/daemon.json

## add this config
{
"data-root": "/newlocation"

LiutongZhou

"""Data Strutures that extend OrderedDict"""
from collections import Counter, OrderedDict
from typing import Any, Hashable, Optional, Tuple, List

from hypothesis import given, strategies as st

__all__ = ["OrderedDefaultDict", "MinMaxCounter"]


class OrderedDefaultDict(OrderedDict):

LiutongZhou

Memory Efficient Training

Optimizers

Short Name	Full Name	URL
CAME	Confidence-guided Adaptive Memory Efficient Optimization (ACL23 outstanding paper award)	https://github.com/huawei-noah/Pretrained-Language-Model/blob/master/CAME/came.py
LOMO	LOw-Memory Optimization	https://github.com/OpenLMLab/LOMO/blob/main/src/lomo.py

Low Rank Decomposition

LORA (Low Rank Adaption): https://github.com/microsoft/LoRA or https://github.com/huggingface/peft/blob/main/src/peft/tuners/lora.py

LiutongZhou

"""Universal Decorator

Universal decorators can decorate functions, classes, bound methods
(class method / instance method) referenced outside of class definition
and descriptors (class methods, static methods) defined inside class definition.
"""

from __future__ import annotations

import inspect

LiutongZhou

"""Distributed Data Parallel Inference for Hugging Face Transformers."""

from typing import Union

import torch
from accelerate import Accelerator
from accelerate.utils import gather_object
from tqdm import tqdm
from transformers import (
    PreTrainedModel,

LiutongZhou

"""OpenAI OSS sdpa and moe implementations that are suitable for both training and inference."""

from typing import Final

import torch
import torch.nn.functional as F
from einops import einsum, rearrange, repeat
from torch import Tensor, nn

__all__ = ["sdpa", "MOEBlock"]