RohanAwhad

Tensor Parallelism Knowledge Compilation

Core Understanding

Tensor Parallelism (TP) shards individual tensors across devices, following the Megatron-LM pattern:

• Column-wise: Input projections (q/k/v, gate/up)
• Row-wise: Output projections (o_proj, down_proj)
• Sequence Parallel: Shards activations on sequence dimension for memory savings
• Loss Parallel: Keep logits sharded on vocab dimension for efficient cross-entropy

RohanAwhad

# /// script
# dependencies = [
#   "numpy",
# ]
# ///

# multi head latent attention

import numpy as np

RohanAwhad

#!/Users/rohan/miniconda3/bin/python

import asyncio
import dataclasses
import json
import openai
import os
import requests
import sys
import time

RohanAwhad

import json
import openai
from typing import Any
from jupyter_client.manager import KernelManager

kernel_manager: KernelManager = None
def execute_code_in_notebook(code: str) -> list[Any]:
  if not code: return []
  global kernel_manager
  if kernel_manager is None:

RohanAwhad

import base64
import cv2
import os
import openai
import whisper
import pyaudio
import wave
import uuid
import threading
from queue import Queue

RohanAwhad

from transformers import AutoTokenizer, AutoModel
import torch


# MODEL CKPT is downloaded from: "jinaai/jina-embeddings-v2-base-en"  # has context len of 8192
MODEL_CKPT = "/Users/rohan/3_Resources/ai_models/jina-embeddings-v2-base-en"

def recursive_splitter(text: str, separators: list[str], chunk_size: int) -> list[str]:
  if len(separators) == 0:
    words = text.strip().split(' ')

RohanAwhad

import dataclasses
import openai
import os
import re
import yaml

from pydantic import BaseModel, ValidationError
from typing import Optional

RohanAwhad

--[[

=====================================================================
==================== READ THIS BEFORE CONTINUING ====================
=====================================================================
========                                    .-----.          ========
========         .----------------------.   | === |          ========
========         |.-""""""""""""""""""-.|   |-----|          ========
========         ||                    ||   | === |          ========
========         ||   KICKSTART.NVIM   ||   |-----|          ========

RohanAwhad

# Inspiration from Eric Hartford github2file: https://github.com/cognitivecomputations/github2file

"""
A utility script to download and process files from a GitHub repository based on language-specific criteria.

Features:
- Download files from a specific branch or tag of a GitHub repository.
- Filter files based on their extension to include only those relevant to the specified programming language.
- Exclude files located in certain directories or those that match a set of predefined non-useful criteria.
- Check and exclude test files based on content indicators specific to each supported language.

RohanAwhad

function sleep(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

function clickShowTranscriptButton() {
  // Selecting the button based on its class and aria-label
  var buttons = document.querySelectorAll(
    "button.yt-spec-button-shape-next.yt-spec-button-shape-next--outline.yt-spec-button-shape-next--call-to-action.yt-spec-button-shape-next--size-m"
  );