Lyken17

from typing import Any
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch import autograd


def scientific_precision(number):
    suffix = ["KB", "MB", "GB", "TB", "PB"]
    for idx_, s in enumerate(suffix):

Lyken17

import torch
from torch import nn

# net = nn.Linear(500, 500)
# input = torch.randn(64, 500)

net = nn.Conv2d(3, 3, kernel_size=3, padding=1)
input = torch.randn(1, 3, 32, 32)

# only calculate input grad, prints ('_saved_mat2', torch.Size([500, 500]))

Lyken17

# e408b27

MODEL=facebook/opt-6.7b
torchrun --nproc_per_node=8 --master_port=24567 train.py \
    --model_name_or_path $MODEL \
    --data_path ./alpaca_data.json \
    --bf16 True \
    --output_dir ./output/$MODEL \
    --num_train_epochs 3 \
    --per_device_train_batch_size 2 \

Lyken17

batch_size, seq_length, embed_dim = x.size()
# B, T, D
qkv = self.qkv_proj(x) # B, T, 3xE
# head_dim = embed_dim // num_heads

# Separate Q, K, V from linear output
qkv = qkv.reshape(batch_size, seq_length, self.num_heads, 3 * self.head_dim) # B, T, H, 3xHD
qkv = qkv.permute(0, 2, 1, 3)  # B, H, T, 3xHD
q, k, v = qkv.chunk(3, dim=-1) # B, H, T, HD 

Lyken17

import numpy as np

from collections import Counter

import tvm
from tvm import relay
from tvm.relay import ExprFunctor, ExprMutator, ExprVisitor
from tvm.relay.expr_functor import ExprMutator, Call
from tvm.relay.dataflow_pattern import wildcard, is_op, is_constant, is_expr, rewrite, DFPatternCallback

Lyken17

import numpy as np

from collections import Counter

import tvm
from tvm import relay
# from tvm.relay import ExprFunctor, ExprMutator, ExprVisitor
from tvm.relay.expr_functor import ExprMutator, Call

Lyken17

import numpy as np

from collections import Counter

import tvm
from tvm import relay
from tvm.relay import ExprFunctor, ExprMutator, ExprVisitor
from tvm.relay.expr_functor import ExprMutator, Call

Lyken17

import os, sys
import os.path as osp
import math

import numpy as np
import torch
import torch.nn as nn
from torchvision import transforms, datasets

from ofa.model_zoo import ofa_net

Lyken17

import torch
import torch.nn as nn
import torchvision

from torchvision import models


batch = 1 
dim = 3
res = 224

Lyken17

# dense update
# forward 
input: 1, 48, 8, 8
weight: 240, 48, 1, 1
output: 1, 240, 8, 8

# input 
# (n, c, h, w) => (1, n * c, h, w)
input_1 = 1, 48, 8, 8