vikhyat

Non-Neural Drafting Baseline for Qwen Math Traces

This gist contains a small reproducibility bundle for measuring a deterministic draft program on qwen35 math reasoning traces.

The measurement is an offline speculative-decoding analogue: at each response position, a causal program sees only the prompt plus already accepted response prefix, drafts 16 tokens, and counts how many leading tokens match the held-out trace continuation.

vikhyat

from typing import List, Dict, Tuple

import torch
import torch.nn as nn

from einops import rearrange
from transformers import pipeline, Owlv2Processor, Owlv2ForObjectDetection
from transformers.image_transforms import center_to_corners_format
from flash_attn.modules.mha import FlashSelfAttention

vikhyat

import torch
import torch.nn as nn
import torch.optim as optim
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.animation import FuncAnimation

act, filename = nn.GELU, 'gelu_training.gif'

# Step 1: Generate Data

vikhyat

# Copyright (c) Microsoft Corporation.
# Licensed under the MIT license.
#
# Copyright (c) 2022, Tri Dao, trid@cs.stanford.edu.
# Licensed under the BSD 3-Clause License.

from __future__ import annotations

import math
from dataclasses import dataclass, field

vikhyat

tok_embeddings.weight torch.Size([32000, 4096])
norm.weight torch.Size([4096])
output.weight torch.Size([32000, 4096])
layers.0.attention_norm.weight torch.Size([4096])
layers.0.attention.wq.weight torch.Size([4096, 4096])
layers.0.attention.wk.weight torch.Size([1024, 4096])
layers.0.attention.wv.weight torch.Size([1024, 4096])
layers.0.attention.wo.weight torch.Size([4096, 4096])
layers.0.feed_forward.gate.weight torch.Size([8, 4096])
layers.0.ffn_norm.weight torch.Size([4096])

vikhyat

test

vikhyat

Here's how it's going to look like from the end user perspective:

1 hour before my PRO membership ends, if I am on a recurring plan 
Hummingbird will try to charge my credit card. If it succeeds 
everything is good. If it fails my subscription will run out and 
Hummingbird will try to charge my credit card again tomorrow, 3 
days from now and 5 days from now. I will receive an email from 
Hummingbird if the charge is successful, and a different email 
every time charging my card fails.

vikhyat

function readAnime(results) {
  return _.map(results.anime, function(anime) {
    anime.linked = {};
    _.each(Object.keys(anime.links), function(rel) {
      var relVal;
      if (Array.isArray(anime.links[rel])) {
        relVal = _.find(results.linked[rel], function(candidate) {
          // return whether candidate.id is in anime.links[rel]
        });
      } else {

vikhyat

int fib(n) {
  return n < 2 ? n : fib(n-1) + fib(n-2);
}

int main() {
  int t, n;
  scanf("%d\n", &t);
  while (t--) {
    scanf("%d\n", &n);
    printf("%f\n", fib(n+2) / pow(2, n));

vikhyat

#define classify(x) (((x) >= 0.5) ? '+' : '-')

float sigmoid(float x) {
  return 1 / (1 + exp((double) -x));
}

main() {
  int d, q, i, a, t;
  scanf("%d %d\n", &d, &q);
  int beta[d+1];