Skip to content

Instantly share code, notes, and snippets.

@HDCharles

HDCharles/repro.py

Created July 29, 2026 20:40
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save HDCharles/6c65dca0953d4ec7a0d835d26126ff66 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save HDCharles/6c65dca0953d4ec7a0d835d26126ff66 to your computer and use it in GitHub Desktop.
Download ZIP
repro for humming issues
Raw
repro.py
#!/usr/bin/env python3
"""Repro: humming GEMM produces NaN with quantized input activations.
Run: python tests/repro_int8_nan.py
Sweeps all combinations of weight bit width (3-8), input quantization
(int8, int4, bf16), batch size, and shape. int4 inputs require weight
nbits < 4 and SM >= 80.
"""
import json
import torch
from humming.layer import HummingMethod
from humming.schema import BaseWeightSchema, BaseInputSchema, HummingInputSchema
def make_layer(N, K, nbits, input_bits=None):
"""Build a humming layer with auto-tuned config, matching vllm's path.
input_bits: 8 for int8 input, 4 for int4 input, None for bf16 (no quant).
"""
quant_config = {
"quant_method": "humming",
"dtype": f"int{nbits}",
"group_size": 0,
}
weight_schema = BaseWeightSchema.from_config(quant_config)
if input_bits is not None:
input_schema = BaseInputSchema.from_config({
"num_bits": input_bits, "type": "int", "strategy": "channel",
"symmetric": True, "dynamic": True, "group_size": 0,
"quant_method": "compressed-tensors", "format": "int-quantized",
})
else:
input_schema = HummingInputSchema()
layer = torch.nn.Module()
packed_k = K * nbits // 32
weight = torch.randint(0, 2**nbits, (N, packed_k),
dtype=torch.int32, device="cuda")
weight_scale = torch.randn(N, 1, dtype=torch.bfloat16, device="cuda") * 0.01
layer.weight = torch.nn.Parameter(weight, requires_grad=False)
layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
ws, tensors = weight_schema.convert_humming(
tensors=dict(layer.named_parameters()),
shape_n_stacks=[N], shape_k_stacks=[K],
param_dtype=torch.bfloat16,
)
isc, _ = input_schema.convert_humming(
tensors={}, shape_n_stacks=[N], shape_k_stacks=[K],
param_dtype=torch.bfloat16,
)
layer.weight_schema = ws
for pn in list(dict(layer.named_parameters()).keys()):
delattr(layer, pn)
for pn, t in tensors.items():
if isinstance(t, torch.nn.Parameter):
t = t.data
setattr(layer, pn, torch.nn.Parameter(t, requires_grad=False))
HummingMethod.prepare_layer_meta(
layer=layer, shape_n=N, shape_k=K,
weight_schema=ws, input_schema=isc,
pad_n_to_multiple=256, pad_k_to_multiple=128,
has_bias=False, torch_dtype=torch.bfloat16,
)
HummingMethod.transform_humming_layer(layer)
layer.register_buffer(
"locks", torch.zeros(1024, dtype=torch.int32, device="cuda"),
)
layer.compute_config = json.dumps({
"use_batch_invariant": False,
"use_f16_accum": False,
"gemm_type": "dense",
})
return layer
BATCH_SIZES = [256, 1024, 4096, 16384]
SHAPES = [
(19456, 2560), # Qwen3-4B gate_up_proj
(2560, 9728), # Qwen3-4B down_proj
(2560, 2560), # Qwen3-4B o_proj
]
NBITS = [3, 4, 5, 6, 7, 8]
INPUT_MODES = [8, 4, None] # int8, int4, bf16
sm = torch.cuda.get_device_capability()
sm_version = sm[0] * 10 + sm[1]
device_name = torch.cuda.get_device_name()
print(f"Device: {device_name} (SM{sm_version})")
print()
header = f"{'nbits':>5s} {'input':>5s} {'M':>5s} {'N':>5s} {'K':>5s} Result"
print(header)
print("=" * len(header) + "=" * 40)
fail_count = 0
pass_count = 0
skip_count = 0
for nbits in NBITS:
for input_bits in INPUT_MODES:
if input_bits == 4 and nbits >= 4:
continue
if input_bits == 4 and sm_version != 80:
continue
input_tag = f"int{input_bits}" if input_bits else "bf16"
for N, K in SHAPES:
for M in BATCH_SIZES:
try:
layer = make_layer(N, K, nbits, input_bits=input_bits)
except Exception as e:
print(f"{nbits:5d} {input_tag:>5s} {M:5d} {N:5d} {K:5d} "
f"SKIP: {e}")
skip_count += 1
continue
x = torch.randn(M, K, dtype=torch.bfloat16, device="cuda")
x = x.clamp(-2, 2)
try:
out = HummingMethod.forward_layer(
layer=layer, inputs=x,
compute_config=layer.compute_config,
)
except RuntimeError as e:
msg = str(e).split('\n')[0][:60]
print(f"{nbits:5d} {input_tag:>5s} {M:5d} {N:5d} {K:5d} "
f"ERR: {msg}")
skip_count += 1
del layer
torch.cuda.empty_cache()
continue
nan_count = torch.isnan(out).sum().item()
nan_rows = torch.isnan(out).any(dim=-1).sum().item()
if nan_count > 0:
print(f"{nbits:5d} {input_tag:>5s} {M:5d} {N:5d} {K:5d} "
f"FAIL: {nan_count} NaN, {nan_rows}/{M} rows")
fail_count += 1
else:
print(f"{nbits:5d} {input_tag:>5s} {M:5d} {N:5d} {K:5d} "
f"PASS")
pass_count += 1
del layer, out
torch.cuda.empty_cache()
print(f"\nTotal: {pass_count} passed, {fail_count} failed, {skip_count} skipped")
@HDCharles

HDCharles commented Jul 29, 2026

Copy link
Copy Markdown
Author

Command: run python /home/HDCharles/repos/humming/tests/repro_int8_nan.py
Reserved 1 GPU(s): [2] for command execution
Device: NVIDIA H100 80GB HBM3 (SM90)

nbits input M N K Result

3  int8   256 19456  2560  PASS
3  int8  1024 19456  2560  PASS
3  int8  4096 19456  2560  PASS
3  int8 16384 19456  2560  FAIL: 176 NaN, 106/16384 rows
3  int8   256  2560  9728  PASS
3  int8  1024  2560  9728  PASS
3  int8  4096  2560  9728  PASS
3  int8 16384  2560  9728  PASS
3  int8   256  2560  2560  PASS
3  int8  1024  2560  2560  PASS
3  int8  4096  2560  2560  PASS
3  int8 16384  2560  2560  PASS
3  bf16   256 19456  2560  PASS
3  bf16  1024 19456  2560  PASS
3  bf16  4096 19456  2560  PASS
3  bf16 16384 19456  2560  PASS
3  bf16   256  2560  9728  PASS
3  bf16  1024  2560  9728  PASS
3  bf16  4096  2560  9728  PASS
3  bf16 16384  2560  9728  PASS
3  bf16   256  2560  2560  PASS
3  bf16  1024  2560  2560  PASS
3  bf16  4096  2560  2560  PASS
3  bf16 16384  2560  2560  PASS
4  int8   256 19456  2560  PASS
4  int8  1024 19456  2560  PASS
4  int8  4096 19456  2560  PASS
4  int8 16384 19456  2560  FAIL: 259 NaN, 136/16384 rows
4  int8   256  2560  9728  PASS
4  int8  1024  2560  9728  PASS
4  int8  4096  2560  9728  PASS
4  int8 16384  2560  9728  PASS
4  int8   256  2560  2560  PASS
4  int8  1024  2560  2560  PASS
4  int8  4096  2560  2560  PASS
4  int8 16384  2560  2560  PASS
4  bf16   256 19456  2560  PASS
4  bf16  1024 19456  2560  PASS
4  bf16  4096 19456  2560  PASS
4  bf16 16384 19456  2560  PASS
4  bf16   256  2560  9728  PASS
4  bf16  1024  2560  9728  PASS
4  bf16  4096  2560  9728  PASS
4  bf16 16384  2560  9728  PASS
4  bf16   256  2560  2560  PASS
4  bf16  1024  2560  2560  PASS
4  bf16  4096  2560  2560  PASS
4  bf16 16384  2560  2560  PASS
5  int8   256 19456  2560  PASS
5  int8  1024 19456  2560  PASS
5  int8  4096 19456  2560  FAIL: 434 NaN, 238/4096 rows
5  int8 16384 19456  2560  FAIL: 2001 NaN, 994/16384 rows
5  int8   256  2560  9728  PASS
5  int8  1024  2560  9728  PASS
5  int8  4096  2560  9728  PASS
5  int8 16384  2560  9728  FAIL: 9 NaN, 8/16384 rows
5  int8   256  2560  2560  PASS
5  int8  1024  2560  2560  PASS
5  int8  4096  2560  2560  PASS
5  int8 16384  2560  2560  PASS
5  bf16   256 19456  2560  PASS
5  bf16  1024 19456  2560  PASS
5  bf16  4096 19456  2560  PASS
5  bf16 16384 19456  2560  PASS
5  bf16   256  2560  9728  PASS
5  bf16  1024  2560  9728  PASS
5  bf16  4096  2560  9728  PASS
5  bf16 16384  2560  9728  PASS
5  bf16   256  2560  2560  PASS
5  bf16  1024  2560  2560  PASS
5  bf16  4096  2560  2560  PASS
5  bf16 16384  2560  2560  PASS
6  int8   256 19456  2560  PASS
6  int8  1024 19456  2560  PASS
6  int8  4096 19456  2560  FAIL: 44 NaN, 27/4096 rows
6  int8 16384 19456  2560  FAIL: 1002 NaN, 630/16384 rows
6  int8   256  2560  9728  PASS
6  int8  1024  2560  9728  PASS
6  int8  4096  2560  9728  PASS
6  int8 16384  2560  9728  PASS
6  int8   256  2560  2560  PASS
6  int8  1024  2560  2560  PASS
6  int8  4096  2560  2560  PASS
6  int8 16384  2560  2560  PASS
6  bf16   256 19456  2560  PASS
6  bf16  1024 19456  2560  PASS
6  bf16  4096 19456  2560  PASS
6  bf16 16384 19456  2560  PASS
6  bf16   256  2560  9728  PASS
6  bf16  1024  2560  9728  PASS
6  bf16  4096  2560  9728  PASS
6  bf16 16384  2560  9728  PASS
6  bf16   256  2560  2560  PASS
6  bf16  1024  2560  2560  PASS
6  bf16  4096  2560  2560  PASS
6  bf16 16384  2560  2560  PASS
7  int8   256 19456  2560  PASS
7  int8  1024 19456  2560  PASS
7  int8  4096 19456  2560  FAIL: 665 NaN, 387/4096 rows
7  int8 16384 19456  2560  FAIL: 1222 NaN, 672/16384 rows
7  int8   256  2560  9728  PASS
7  int8  1024  2560  9728  PASS
7  int8  4096  2560  9728  PASS
7  int8 16384  2560  9728  FAIL: 280 NaN, 178/16384 rows
7  int8   256  2560  2560  PASS
7  int8  1024  2560  2560  PASS
7  int8  4096  2560  2560  PASS
7  int8 16384  2560  2560  FAIL: 24 NaN, 17/16384 rows
7  bf16   256 19456  2560  PASS
7  bf16  1024 19456  2560  PASS
7  bf16  4096 19456  2560  PASS
7  bf16 16384 19456  2560  PASS
7  bf16   256  2560  9728  PASS
7  bf16  1024  2560  9728  PASS
7  bf16  4096  2560  9728  PASS
7  bf16 16384  2560  9728  PASS
7  bf16   256  2560  2560  PASS
7  bf16  1024  2560  2560  PASS
7  bf16  4096  2560  2560  PASS
7  bf16 16384  2560  2560  PASS
8  int8   256 19456  2560  PASS
8  int8  1024 19456  2560  FAIL: 10 NaN, 7/1024 rows
8  int8  4096 19456  2560  FAIL: 89 NaN, 62/4096 rows
8  int8 16384 19456  2560  FAIL: 918 NaN, 591/16384 rows
8  int8   256  2560  9728  PASS
8  int8  1024  2560  9728  PASS
8  int8  4096  2560  9728  PASS
8  int8 16384  2560  9728  FAIL: 323 NaN, 205/16384 rows
8  int8   256  2560  2560  PASS
8  int8  1024  2560  2560  PASS
8  int8  4096  2560  2560  PASS
8  int8 16384  2560  2560  FAIL: 383 NaN, 245/16384 rows
8  bf16   256 19456  2560  PASS
8  bf16  1024 19456  2560  PASS
8  bf16  4096 19456  2560  PASS
8  bf16 16384 19456  2560  PASS
8  bf16   256  2560  9728  PASS
8  bf16  1024  2560  9728  PASS
8  bf16  4096  2560  9728  PASS
8  bf16 16384  2560  9728  PASS
8  bf16   256  2560  2560  PASS
8  bf16  1024  2560  2560  PASS
8  bf16  4096  2560  2560  PASS
8  bf16 16384  2560  2560  PASS

Total: 128 passed, 16 failed, 0 skipped

@HDCharles

HDCharles commented Jul 29, 2026

Copy link
Copy Markdown
Author

Command: run python /home/HDCharles/repos/llm-compressor/repro.py
Reserved 1 GPU(s): [5] for command execution
Device: NVIDIA A100-SXM4-80GB (SM80)

nbits input M N K Result

3  int8   256 19456  2560  PASS
3  int8  1024 19456  2560  PASS
3  int8  4096 19456  2560  PASS
3  int8 16384 19456  2560  PASS
3  int8   256  2560  9728  PASS
3  int8  1024  2560  9728  PASS
3  int8  4096  2560  9728  PASS
3  int8 16384  2560  9728  PASS
3  int8   256  2560  2560  PASS
3  int8  1024  2560  2560  PASS
3  int8  4096  2560  2560  PASS
3  int8 16384  2560  2560  PASS
3  int4   256 19456  2560  PASS
3  int4  1024 19456  2560  PASS
3  int4  4096 19456  2560  PASS
3  int4 16384 19456  2560  PASS
3  int4   256  2560  9728  PASS
3  int4  1024  2560  9728  PASS
3  int4  4096  2560  9728  PASS
3  int4 16384  2560  9728  PASS
3  int4   256  2560  2560  PASS
3  int4  1024  2560  2560  PASS
3  int4  4096  2560  2560  PASS
3  int4 16384  2560  2560  PASS
3  bf16   256 19456  2560  PASS
3  bf16  1024 19456  2560  PASS
3  bf16  4096 19456  2560  PASS
3  bf16 16384 19456  2560  PASS
3  bf16   256  2560  9728  PASS
3  bf16  1024  2560  9728  PASS
3  bf16  4096  2560  9728  PASS
3  bf16 16384  2560  9728  PASS
3  bf16   256  2560  2560  PASS
3  bf16  1024  2560  2560  PASS
3  bf16  4096  2560  2560  PASS
3  bf16 16384  2560  2560  PASS
4  int8   256 19456  2560  PASS
4  int8  1024 19456  2560  PASS
4  int8  4096 19456  2560  PASS
4  int8 16384 19456  2560  PASS
4  int8   256  2560  9728  PASS
4  int8  1024  2560  9728  PASS
4  int8  4096  2560  9728  PASS
4  int8 16384  2560  9728  PASS
4  int8   256  2560  2560  PASS
4  int8  1024  2560  2560  PASS
4  int8  4096  2560  2560  PASS
4  int8 16384  2560  2560  PASS
4  bf16   256 19456  2560  PASS
4  bf16  1024 19456  2560  PASS
4  bf16  4096 19456  2560  PASS
4  bf16 16384 19456  2560  PASS
4  bf16   256  2560  9728  PASS
4  bf16  1024  2560  9728  PASS
4  bf16  4096  2560  9728  PASS
4  bf16 16384  2560  9728  PASS
4  bf16   256  2560  2560  PASS
4  bf16  1024  2560  2560  PASS
4  bf16  4096  2560  2560  PASS
4  bf16 16384  2560  2560  PASS
5  int8   256 19456  2560  PASS
5  int8  1024 19456  2560  PASS
5  int8  4096 19456  2560  PASS
5  int8 16384 19456  2560  PASS
5  int8   256  2560  9728  PASS
5  int8  1024  2560  9728  PASS
5  int8  4096  2560  9728  PASS
5  int8 16384  2560  9728  PASS
5  int8   256  2560  2560  PASS
5  int8  1024  2560  2560  PASS
5  int8  4096  2560  2560  PASS
5  int8 16384  2560  2560  PASS
5  bf16   256 19456  2560  PASS
5  bf16  1024 19456  2560  PASS
5  bf16  4096 19456  2560  PASS
5  bf16 16384 19456  2560  PASS
5  bf16   256  2560  9728  PASS
5  bf16  1024  2560  9728  PASS
5  bf16  4096  2560  9728  PASS
5  bf16 16384  2560  9728  PASS
5  bf16   256  2560  2560  PASS
5  bf16  1024  2560  2560  PASS
5  bf16  4096  2560  2560  PASS
5  bf16 16384  2560  2560  PASS
6  int8   256 19456  2560  PASS
6  int8  1024 19456  2560  PASS
6  int8  4096 19456  2560  PASS
6  int8 16384 19456  2560  PASS
6  int8   256  2560  9728  PASS
6  int8  1024  2560  9728  PASS
6  int8  4096  2560  9728  PASS
6  int8 16384  2560  9728  PASS
6  int8   256  2560  2560  PASS
6  int8  1024  2560  2560  PASS
6  int8  4096  2560  2560  PASS
6  int8 16384  2560  2560  PASS
6  bf16   256 19456  2560  PASS
6  bf16  1024 19456  2560  PASS
6  bf16  4096 19456  2560  PASS
6  bf16 16384 19456  2560  PASS
6  bf16   256  2560  9728  PASS
6  bf16  1024  2560  9728  PASS
6  bf16  4096  2560  9728  PASS
6  bf16 16384  2560  9728  PASS
6  bf16   256  2560  2560  PASS
6  bf16  1024  2560  2560  PASS
6  bf16  4096  2560  2560  PASS
6  bf16 16384  2560  2560  PASS
7  int8   256 19456  2560  PASS
7  int8  1024 19456  2560  PASS
7  int8  4096 19456  2560  PASS
7  int8 16384 19456  2560  PASS
7  int8   256  2560  9728  PASS
7  int8  1024  2560  9728  PASS
7  int8  4096  2560  9728  PASS
7  int8 16384  2560  9728  PASS
7  int8   256  2560  2560  PASS
7  int8  1024  2560  2560  PASS
7  int8  4096  2560  2560  PASS
7  int8 16384  2560  2560  PASS
7  bf16   256 19456  2560  PASS
7  bf16  1024 19456  2560  PASS
7  bf16  4096 19456  2560  PASS
7  bf16 16384 19456  2560  PASS
7  bf16   256  2560  9728  PASS
7  bf16  1024  2560  9728  PASS
7  bf16  4096  2560  9728  PASS
7  bf16 16384  2560  9728  PASS
7  bf16   256  2560  2560  PASS
7  bf16  1024  2560  2560  PASS
7  bf16  4096  2560  2560  PASS
7  bf16 16384  2560  2560  PASS
8  int8   256 19456  2560  PASS
8  int8  1024 19456  2560  PASS
8  int8  4096 19456  2560  PASS
8  int8 16384 19456  2560  PASS
8  int8   256  2560  9728  PASS
8  int8  1024  2560  9728  PASS
8  int8  4096  2560  9728  PASS
8  int8 16384  2560  9728  PASS
8  int8   256  2560  2560  PASS
8  int8  1024  2560  2560  PASS
8  int8  4096  2560  2560  PASS
8  int8 16384  2560  2560  PASS
8  bf16   256 19456  2560  PASS
8  bf16  1024 19456  2560  PASS
8  bf16  4096 19456  2560  PASS
8  bf16 16384 19456  2560  PASS
8  bf16   256  2560  9728  PASS
8  bf16  1024  2560  9728  PASS
8  bf16  4096  2560  9728  PASS
8  bf16 16384  2560  9728  PASS
8  bf16   256  2560  2560  PASS
8  bf16  1024  2560  2560  PASS
8  bf16  4096  2560  2560  PASS
8  bf16 16384  2560  2560  PASS

Total: 156 passed, 0 failed, 0 skipped

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment