cloneofsimo

Simplifying the Contrastive Flow Matching Objective

We show that CFM (Contrastive flow matching) objective is fundamentally indifferent from FM because it's simply affine-transformation of the target velocity, which could be learned post-hoc.

Assumptions:

• $\alpha_t = 1 - t, \sigma_t = t \Rightarrow v = -x_i + \epsilon_i, \tilde v = -\tilde x + \tilde\epsilon$
• $(\tilde x,\tilde\epsilon)$ is drawn i.i.d. from the dataset, independent of $(x_i,\epsilon_i)$
• Sampled noise has zero mean: $\mathbb{E}[\tilde\epsilon] = 0$
• Let $\mu_x := \mathbb{E}[\tilde x]$ (empirical average $\bar x$ in practice)

cloneofsimo

import os
from typing import List

import torch
import torch.distributed.distributed_c10d as c10d
import torch.distributed as dist

from torch._C._distributed_c10d import _SymmetricMemory  # type: ignore
from torch.distributed._symmetric_memory import (        # type: ignore
    rendezvous as _sm_rendezvous,

cloneofsimo

# credit: https://github.com/Nordth/humanize-ai-lib/blob/main/src/humanize-string.ts
import re

_HIDDEN_CHARS = re.compile(
    r"[\u00AD\u180E\u200B-\u200F\u202A-\u202E\u2060\u2066-\u2069\uFEFF]"
)
_TRAILING_WS  = re.compile(r"[ \t\x0B\f]+$", re.MULTILINE)
_NBSP         = re.compile(r"\u00A0")
_DASHES       = re.compile(r"[—–]+")         # em- & en-dashes → ASCII hyphen
_DQUOTES      = re.compile(r"[“”«»„]")       # curly / guillemets → "

cloneofsimo

Credit: How to write ML papers by Neel Nanda

You are chatbot that gives constructive analysis of the following work. Specifically, you care about the following criteria:

## Core Narrative Quality
- **Clear Claims**: Contains 1-3 specific, concrete claims that fit within a cohesive theme
- **Strong Motivation**: Clearly explains why readers should care ("so what?")

cloneofsimo

#!/usr/bin/env python

import torch
import time
import random
import numpy as np
import multiprocessing
from multiprocessing import Process, Manager, Event

import plotly.express as px

cloneofsimo

import torch


class TwoDimRotary(torch.nn.Module):
    def __init__(self, dim, base=100, h = 128, w = 128):
        super().__init__()
        self.inv_freq = 1.0 / (base ** (torch.arange(0, dim, 2).float() / (dim)))
        self.h = h
        self.w = w
        

cloneofsimo

https://x.com/jxbz/status/1857145985480438073

import torch

def polar_factor_newton_schulz(M, max_iter=50):

M_t = M / M.norm(p='fro')
for _ in range(max_iter):
M_t = 1.5 * M_t - 0.5 * M_t @ M_t.T @ M_t
return M_t

cloneofsimo

# batch_eigendecomp.py
import torch
from torch.utils.cpp_extension import load_inline
import argparse
import os
import shutil

def clear_cuda_cache():
    cache_path = os.path.expanduser('~/.cache/torch_extensions')
    if os.path.exists(cache_path):

cloneofsimo

#!/bin/bash

# Install required packages if not present
check_and_install_dependencies() {
    local packages=("inotify-tools" "texlive" "texlive-latex-extra" "biber")
    
    echo "Checking and installing dependencies..."
    for package in "${packages[@]}"; do
        if ! dpkg -l | grep -q "^ii  $package "; then
            sudo apt-get install -y "$package"

cloneofsimo

import torch
import torch.nn as nn
import torch.nn.functional as F
from torchvision import datasets, transforms
import numpy as np
import math


def compute_activation_std(model, dataset, device='cpu', batch_size=32, num_workers=0, layer_names=None):
    activations = {}