Skip to content

Instantly share code, notes, and snippets.

@myazdani

myazdani/color-ablation.ipynb

Created June 22, 2026 20:49
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save myazdani/2315a7493c500c2e78c632b553846e86 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save myazdani/2315a7493c500c2e78c632b553846e86 to your computer and use it in GitHub Desktop.
Download ZIP
CoLoR-ablation.ipynb
Display the source blob
Display the rendered blob
Raw
color-ablation.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"machine_shape": "hm",
"gpuType": "A100",
"authorship_tag": "ABX9TyNIiSrWq9xu3UUiaYSJc7JL",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/myazdani/2315a7493c500c2e78c632b553846e86/color-ablation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "markdown",
"source": [
"# 0. Runtime, Drive, and Token"
],
"metadata": {
"id": "BNKA9Es5ySaK"
}
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "x9LZ8VAfIhpd",
"outputId": "2fab628c-0f73-4f14-b254-0f25bbc60194"
},
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Mon Jun 22 18:59:46 2026 \n",
"+-----------------------------------------------------------------------------------------+\n",
"| NVIDIA-SMI 580.82.07 Driver Version: 580.82.07 CUDA Version: 13.0 |\n",
"+-----------------------------------------+------------------------+----------------------+\n",
"| GPU Name Persistence-M | Bus-Id Disp.A | Volatile Uncorr. ECC |\n",
"| Fan Temp Perf Pwr:Usage/Cap | Memory-Usage | GPU-Util Compute M. |\n",
"| | | MIG M. |\n",
"|=========================================+========================+======================|\n",
"| 0 NVIDIA A100-SXM4-80GB Off | 00000000:00:05.0 Off | 0 |\n",
"| N/A 33C P0 53W / 400W | 0MiB / 81920MiB | 0% Default |\n",
"| | | Disabled |\n",
"+-----------------------------------------+------------------------+----------------------+\n",
"\n",
"+-----------------------------------------------------------------------------------------+\n",
"| Processes: |\n",
"| GPU GI CI PID Type Process name GPU Memory |\n",
"| ID ID Usage |\n",
"|=========================================================================================|\n",
"| No running processes found |\n",
"+-----------------------------------------------------------------------------------------+\n"
]
}
],
"source": [
"!nvidia-smi"
]
},
{
"cell_type": "code",
"source": [
"from google.colab import drive\n",
"drive.mount('/content/drive')"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "n_ce4sG1IsZo",
"outputId": "9c58cbb3-dfc8-48e1-9749-8647d6ac9be7"
},
"execution_count": 2,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Mounted at /content/drive\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"import os\n",
"from google.colab import userdata\n",
"os.environ[\"HF_TOKEN\"] = userdata.get(\"HF_TOKEN\")"
],
"metadata": {
"id": "KbrxQhQ3NnYD"
},
"execution_count": 3,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# 1. Clone Repos and Install"
],
"metadata": {
"id": "wjgBaLiqbnk5"
}
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"import os\n",
"\n",
"PROJECT = \"/content/CoLoR-ablation\"\n",
"OLMO = \"/content/color-filter-olmo\"\n",
"OLMO_SHA = \"3e0424c8cc6c53aaad70d3a3dea3fd683658cdd4\"\n",
"\n",
"# This repo. If you uploaded the repo folder manually and it already exists,\n",
"# this clone step is skipped.\n",
"if not os.path.isdir(PROJECT):\n",
" !git clone https://github.com/myazdani/CoLoR-ablation.git {PROJECT}\n",
"print(\"repo cloned\")\n",
"# Paper fork, pinned.\n",
"if not os.path.isdir(OLMO):\n",
" !git clone https://github.com/davidbrandfonbrener/color-filter-olmo.git {OLMO}\n",
"!git -C {OLMO} checkout {OLMO_SHA}\n",
"print(\"paper cloned\")\n",
"%cd {PROJECT}\n",
"!pip install -q --no-deps -r requirements-colab.txt\n",
"print(\"reqs installed\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "VGVH3lTqNv19",
"outputId": "78b9d73c-65a0-4bc7-973d-66bf475d4759"
},
"execution_count": 4,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Cloning into '/content/CoLoR-ablation'...\n",
"remote: Enumerating objects: 109, done.\u001b[K\n",
"remote: Counting objects: 100% (109/109), done.\u001b[K\n",
"remote: Compressing objects: 100% (62/62), done.\u001b[K\n",
"remote: Total 109 (delta 43), reused 94 (delta 28), pack-reused 0 (from 0)\u001b[K\n",
"Receiving objects: 100% (109/109), 50.31 KiB | 5.03 MiB/s, done.\n",
"Resolving deltas: 100% (43/43), done.\n",
"repo cloned\n",
"Cloning into '/content/color-filter-olmo'...\n",
"remote: Enumerating objects: 16650, done.\u001b[K\n",
"remote: Counting objects: 100% (2809/2809), done.\u001b[K\n",
"remote: Compressing objects: 100% (2510/2510), done.\u001b[K\n",
"remote: Total 16650 (delta 319), reused 299 (delta 299), pack-reused 13841 (from 1)\u001b[K\n",
"Receiving objects: 100% (16650/16650), 81.20 MiB | 28.87 MiB/s, done.\n",
"Resolving deltas: 100% (11419/11419), done.\n",
"fatal: reference is not a tree: 3e0424c8cc6c53aaad70d3a3dea3fd683658cdd4\n",
"paper cloned\n",
"/content/CoLoR-ablation\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m79.5/79.5 kB\u001b[0m \u001b[31m5.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m140.0/140.0 kB\u001b[0m \u001b[31m12.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m15.3/15.3 MB\u001b[0m \u001b[31m128.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m90.1/90.1 kB\u001b[0m \u001b[31m10.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
"\u001b[?25hreqs installed\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"import boto3, botocore, rich, cached_path, omegaconf\n",
"print(\"paper-fork import deps OK\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "6pKMVT8CfH4o",
"outputId": "a72a7e7e-bd7e-45f1-ef1b-115ca078eb55"
},
"execution_count": 5,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"paper-fork import deps OK\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"import sys\n",
"if OLMO not in sys.path:\n",
" sys.path.insert(0, OLMO)"
],
"metadata": {
"id": "mlitHRloWksn"
},
"execution_count": 6,
"outputs": []
},
{
"cell_type": "markdown",
"source": [
"# 2. Drive Layout"
],
"metadata": {
"id": "-B4-g0Hyb7Zw"
}
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"DRIVE = \"/content/drive/MyDrive/color-filter-ablation\"\n",
"for sub in [\"assets/raw\", \"assets/hf\", \"data\", \"results\",\n",
" \"reports/layer-ablated-color-filter/figures\"]:\n",
" os.makedirs(f\"{DRIVE}/{sub}\", exist_ok=True)\n",
"print(DRIVE)"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "RRJuuqHTOOy1",
"outputId": "a3112d08-1216-4814-8cca-ad519810eefa"
},
"execution_count": 7,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"/content/drive/MyDrive/color-filter-ablation\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# 3. Get the Converted Checkpoints onto Drive"
],
"metadata": {
"id": "7YBysEFZb_k2"
}
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"RAW = f\"{DRIVE}/assets/raw\"\n",
"for filename in [\n",
" \"models/prior/config.yaml\",\n",
" \"models/prior/model.pt\",\n",
" \"models/conditional_books/config.yaml\",\n",
" \"models/conditional_books/model.pt\",\n",
"]:\n",
" !hf download hlzhang109/CoLoR-filter {filename} --local-dir \"{RAW}\""
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "IcsCICDTPllV",
"outputId": "a1b0e411-b04e-4523-d88b-d4419da7e6f1"
},
"execution_count": 8,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"\u001b[90mHint: A new version of huggingface_hub (1.20.1) is available! You are using version 1.19.0.\n",
"To update, run: hf update\u001b[0m\n",
"Ignored metadata for 'models/prior/config.yaml' (outdated). Will re-compute hash.\n",
"config.yaml: 100% 20.6k/20.6k [00:00<00:00, 14.6MB/s]\n",
"\u001b[32m✓ Downloaded\u001b[0m\n",
" path: /content/drive/MyDrive/color-filter-ablation/assets/raw/models/prior/config.yaml\n",
"\u001b[32m✓ Downloaded\u001b[0m\n",
" path: /content/drive/MyDrive/color-filter-ablation/assets/raw/models/prior/model.pt\n",
"Ignored metadata for 'models/conditional_books/config.yaml' (outdated). Will re-compute hash.\n",
"config.yaml: 100% 11.8k/11.8k [00:00<00:00, 33.5MB/s]\n",
"\u001b[32m✓ Downloaded\u001b[0m\n",
" path: /content/drive/MyDrive/color-filter-ablation/assets/raw/models/conditional_books/config.yaml\n",
"\u001b[32m✓ Downloaded\u001b[0m\n",
" path: /content/drive/MyDrive/color-filter-ablation/assets/raw/models/conditional_books/model.pt\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"HF_ASSETS = f\"{DRIVE}/assets/hf\"\n",
"!mkdir -p \"{HF_ASSETS}/books_marg_hf\" \"{HF_ASSETS}/books_cond_hf\"\n",
"!cp -R \"{RAW}/models/prior/.\" \"{HF_ASSETS}/books_marg_hf/\"\n",
"!cp -R \"{RAW}/models/conditional_books/.\" \"{HF_ASSETS}/books_cond_hf/\"\n",
"\n",
"!PYTHONPATH=\"{OLMO}\" python scripts/05_convert_olmo_to_hf.py --checkpoint-dir \"{HF_ASSETS}/books_marg_hf\"\n",
"!PYTHONPATH=\"{OLMO}\" python scripts/05_convert_olmo_to_hf.py --checkpoint-dir \"{HF_ASSETS}/books_cond_hf\""
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "DC2wmG_YW4ys",
"outputId": "eeb0fbbd-d2f9-478e-98dc-44a1b30fdd6b"
},
"execution_count": 9,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Converted /content/drive/MyDrive/color-filter-ablation/assets/hf/books_marg_hf\n",
"Converted /content/drive/MyDrive/color-filter-ablation/assets/hf/books_cond_hf\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"from pathlib import Path\n",
"import yaml\n",
"\n",
"cfg_path = Path(\"configs/default.yaml\")\n",
"cfg = yaml.safe_load(cfg_path.read_text())\n",
"\n",
"cfg[\"target\"][\"cond_checkpoint\"] = f\"{DRIVE}/assets/hf/books_cond_hf\"\n",
"cfg[\"target\"][\"marg_checkpoint\"] = f\"{DRIVE}/assets/hf/books_marg_hf\"\n",
"cfg[\"target\"][\"pool_tokens\"] = f\"{DRIVE}/data/books_pool.npy\"\n",
"cfg[\"target\"][\"pool_meta\"] = f\"{DRIVE}/data/books_pool_meta.parquet\"\n",
"\n",
"cfg[\"paths\"][\"paper_code\"] = OLMO\n",
"cfg[\"paths\"][\"results_dir\"] = f\"{DRIVE}/results\"\n",
"cfg[\"paths\"][\"figures_dir\"] = f\"{DRIVE}/reports/layer-ablated-color-filter/figures\"\n",
"cfg[\"paths\"][\"metrics_csv\"] = f\"{DRIVE}/results/metrics.csv\"\n",
"\n",
"cfg[\"pool\"][\"enriched\"][\"source\"] = \"davidbrandfonbrener/color-filtered-c4\"\n",
"cfg[\"pool\"][\"enriched\"][\"name\"] = \"color-filtered-c4-books\"\n",
"\n",
"cfg_path.write_text(yaml.safe_dump(cfg, sort_keys=False))\n",
"print(cfg_path.read_text())"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "CDFhZy53P6Ow",
"outputId": "0d7f7e69-4faf-4429-90c0-b768c12611a9"
},
"execution_count": 10,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"seed: 17\n",
"target:\n",
" name: books\n",
" cond_checkpoint: /content/drive/MyDrive/color-filter-ablation/assets/hf/books_cond_hf\n",
" marg_checkpoint: /content/drive/MyDrive/color-filter-ablation/assets/hf/books_marg_hf\n",
" pool_tokens: /content/drive/MyDrive/color-filter-ablation/data/books_pool.npy\n",
" pool_meta: /content/drive/MyDrive/color-filter-ablation/data/books_pool_meta.parquet\n",
"paths:\n",
" paper_code: /content/color-filter-olmo\n",
" results_dir: /content/drive/MyDrive/color-filter-ablation/results\n",
" figures_dir: /content/drive/MyDrive/color-filter-ablation/reports/layer-ablated-color-filter/figures\n",
" metrics_csv: /content/drive/MyDrive/color-filter-ablation/results/metrics.csv\n",
"pool:\n",
" sequence_length: 512\n",
" tokenizer: allenai/eleuther-ai-gpt-neox-20b-pii-special\n",
" c4:\n",
" source: allenai/c4\n",
" name: en\n",
" split: train\n",
" n_sequences: 100000\n",
" seed: 17\n",
" shuffle_buffer: 10000\n",
" enriched:\n",
" source: davidbrandfonbrener/color-filtered-c4\n",
" name: color-filtered-c4-books\n",
" split: train\n",
" n: 5000\n",
" seed: 17\n",
" shuffle_buffer: 10000\n",
"scoring:\n",
" batch_size: 64\n",
" dtype: bf16\n",
" device: auto\n",
" shard_size: 5000\n",
" total_layers: 12\n",
"variants:\n",
"- id: full\n",
"- id: top1\n",
"- id: top2\n",
"- id: top4\n",
"- id: top6\n",
"- id: mid2\n",
"- id: mid4\n",
"- id: bot2\n",
"- id: bot4\n",
"- id: skip2\n",
"metrics:\n",
" selection_rates:\n",
" - 0.015625\n",
" - 0.03125\n",
" - 0.0625\n",
" - 0.125\n",
" - 0.25\n",
" bootstrap_reps: 1000\n",
" bootstrap_seed: 17\n",
" tail_rate_for_local_spearman: 0.125\n",
" main_plot_selection_rate: 0.0625\n",
"\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"from pathlib import Path\n",
"import yaml\n",
"\n",
"cfg = yaml.safe_load(Path(\"configs/default.yaml\").read_text())\n",
"for key in [\"marg_checkpoint\", \"cond_checkpoint\"]:\n",
" p = Path(cfg[\"target\"][key])\n",
" print(key, p, \"exists=\", p.is_dir())\n",
" if not p.is_dir():\n",
" raise FileNotFoundError(\n",
" f\"{key} does not exist: {p}. Drive may not be mounted, or the \"\n",
" \"converted checkpoint folders are not under \"\n",
" \"MyDrive/color-filter-ablation/assets/hf/.\"\n",
" )"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Wjd_TMILQeS0",
"outputId": "0ca89aa7-f273-4208-bc79-f54be658886c"
},
"execution_count": 11,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"marg_checkpoint /content/drive/MyDrive/color-filter-ablation/assets/hf/books_marg_hf exists= True\n",
"cond_checkpoint /content/drive/MyDrive/color-filter-ablation/assets/hf/books_cond_hf exists= True\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"required = {\"config.json\", \"pytorch_model.bin\", \"tokenizer.json\",\n",
" \"tokenizer_config.json\", \"special_tokens_map.json\"}\n",
"\n",
"for key in [\"marg_checkpoint\", \"cond_checkpoint\"]:\n",
" p = Path(cfg[\"target\"][key])\n",
" files = set(x.name for x in p.iterdir())\n",
" missing = sorted(required - files)\n",
" print(key, p)\n",
" print(\"present:\", sorted(files))\n",
" if missing:\n",
" raise FileNotFoundError(\n",
" f\"{key} is not a converted HF checkpoint. Missing: {missing}. \"\n",
" \"Run the fallback conversion in Step 3 or upload the local assets/hf folders.\"\n",
" )"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "WR0Lz6p2QmTt",
"outputId": "6846a7fd-a1e2-49c8-fc5f-7aae84b6a7ba"
},
"execution_count": 12,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"marg_checkpoint /content/drive/MyDrive/color-filter-ablation/assets/hf/books_marg_hf\n",
"present: ['config.json', 'config.yaml', 'model.pt', 'pytorch_model.bin', 'special_tokens_map.json', 'tokenizer.json', 'tokenizer_config.json']\n",
"cond_checkpoint /content/drive/MyDrive/color-filter-ablation/assets/hf/books_cond_hf\n",
"present: ['config.json', 'config.yaml', 'model.pt', 'pytorch_model.bin', 'special_tokens_map.json', 'tokenizer.json', 'tokenizer_config.json']\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"enriched = cfg[\"pool\"][\"enriched\"]\n",
"print(enriched)\n",
"if enriched.get(\"source\") != \"davidbrandfonbrener/color-filtered-c4\":\n",
" raise ValueError(\"pool.enriched.source must be davidbrandfonbrener/color-filtered-c4\")\n",
"if enriched.get(\"name\") != \"color-filtered-c4-books\":\n",
" raise ValueError(\"pool.enriched.name must be color-filtered-c4-books\")"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "NGiBG_SxtbXM",
"outputId": "94f4a802-b635-44c9-c859-f164c68648f7"
},
"execution_count": 13,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"{'source': 'davidbrandfonbrener/color-filtered-c4', 'name': 'color-filtered-c4-books', 'split': 'train', 'n': 5000, 'seed': 17, 'shuffle_buffer': 10000}\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# 4. GPU Load Check (cheap gate before building the pool)"
],
"metadata": {
"id": "PQad5ywqZzoM"
}
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"!python scripts/06_local_validation.py --config configs/default.yaml --device cuda"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "eMLnf6qWZcPr",
"outputId": "6129a683-5db9-4613-9fa8-1c6e241041fb"
},
"execution_count": 14,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Tokenizer: OLMoTokenizerFast, eos=50279, pad=1\n",
"Packing 200 Gutenberg-ish sequences...\n",
"Packing 200 C4 sequences...\n",
"README.md: 100% 41.1k/41.1k [00:00<00:00, 63.4MB/s]\n",
"Resolving data files: 100% 1024/1024 [00:00<00:00, 341330.95it/s]\n",
"Resolving data files: 100% 1024/1024 [00:00<00:00, 22100.84it/s]\n",
"Packed pool shape=(400, 512)\n",
"Loading converted models; scoring device=cuda, dtype=bf16...\n",
"Loading weights: 100% 99/99 [00:00<00:00, 309.03it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 311.49it/s]\n",
"Block path: model.transformer.blocks; blocks=12\n",
"Module tree preview:\n",
"<root>: OLMoForCausalLM\n",
"model: OLMo\n",
"model.transformer: ModuleDict\n",
"model.transformer.wte: Embedding\n",
"model.transformer.emb_drop: Dropout\n",
"model.transformer.ln_f: LayerNorm\n",
"model.transformer.blocks: ModuleList len=12\n",
"model.transformer.blocks.0: OLMoSequentialBlock layer_id=0\n",
"model.transformer.blocks.0.dropout: Dropout\n",
"model.transformer.blocks.0.k_norm: LayerNorm\n",
"model.transformer.blocks.0.q_norm: LayerNorm\n",
"model.transformer.blocks.0.act: GELU\n",
"model.transformer.blocks.0.attn_out: Linear\n",
"model.transformer.blocks.0.ff_out: Linear\n",
"model.transformer.blocks.0.rotary_emb: RotaryEmbedding\n",
"model.transformer.blocks.0.attn_norm: LayerNorm\n",
"model.transformer.blocks.0.ff_norm: LayerNorm\n",
"model.transformer.blocks.0.att_proj: Linear\n",
"model.transformer.blocks.0.ff_proj: Linear\n",
"model.transformer.blocks.1: OLMoSequentialBlock layer_id=1\n",
"model.transformer.blocks.1.dropout: Dropout\n",
"model.transformer.blocks.1.k_norm: LayerNorm\n",
"model.transformer.blocks.1.q_norm: LayerNorm\n",
"model.transformer.blocks.1.act: GELU\n",
"model.transformer.blocks.1.attn_out: Linear\n",
"model.transformer.blocks.1.ff_out: Linear\n",
"model.transformer.blocks.1.rotary_emb: RotaryEmbedding\n",
"model.transformer.blocks.1.attn_norm: LayerNorm\n",
"model.transformer.blocks.1.ff_norm: LayerNorm\n",
"model.transformer.blocks.1.att_proj: Linear\n",
"model.transformer.blocks.1.ff_proj: Linear\n",
"model.transformer.blocks.2: OLMoSequentialBlock layer_id=2\n",
"model.transformer.blocks.2.dropout: Dropout\n",
"model.transformer.blocks.2.k_norm: LayerNorm\n",
"model.transformer.blocks.2.q_norm: LayerNorm\n",
"model.transformer.blocks.2.act: GELU\n",
"model.transformer.blocks.2.attn_out: Linear\n",
"model.transformer.blocks.2.ff_out: Linear\n",
"model.transformer.blocks.2.rotary_emb: RotaryEmbedding\n",
"model.transformer.blocks.2.attn_norm: LayerNorm\n",
"model.transformer.blocks.2.ff_norm: LayerNorm\n",
"model.transformer.blocks.2.att_proj: Linear\n",
"model.transformer.blocks.2.ff_proj: Linear\n",
"model.transformer.blocks.3: OLMoSequentialBlock layer_id=3\n",
"model.transformer.blocks.3.dropout: Dropout\n",
"model.transformer.blocks.3.k_norm: LayerNorm\n",
"model.transformer.blocks.3.q_norm: LayerNorm\n",
"model.transformer.blocks.3.act: GELU\n",
"model.transformer.blocks.3.attn_out: Linear\n",
"model.transformer.blocks.3.ff_out: Linear\n",
"model.transformer.blocks.3.rotary_emb: RotaryEmbedding\n",
"model.transformer.blocks.3.attn_norm: LayerNorm\n",
"model.transformer.blocks.3.ff_norm: LayerNorm\n",
"model.transformer.blocks.3.att_proj: Linear\n",
"model.transformer.blocks.3.ff_proj: Linear\n",
"model.transformer.blocks.4: OLMoSequentialBlock layer_id=4\n",
"model.transformer.blocks.4.dropout: Dropout\n",
"model.transformer.blocks.4.k_norm: LayerNorm\n",
"model.transformer.blocks.4.q_norm: LayerNorm\n",
"model.transformer.blocks.4.act: GELU\n",
"model.transformer.blocks.4.attn_out: Linear\n",
"model.transformer.blocks.4.ff_out: Linear\n",
"model.transformer.blocks.4.rotary_emb: RotaryEmbedding\n",
"model.transformer.blocks.4.attn_norm: LayerNorm\n",
"model.transformer.blocks.4.ff_norm: LayerNorm\n",
"model.transformer.blocks.4.att_proj: Linear\n",
"model.transformer.blocks.4.ff_proj: Linear\n",
"model.transformer.blocks.5: OLMoSequentialBlock layer_id=5\n",
"model.transformer.blocks.5.dropout: Dropout\n",
"model.transformer.blocks.5.k_norm: LayerNorm\n",
"model.transformer.blocks.5.q_norm: LayerNorm\n",
"model.transformer.blocks.5.act: GELU\n",
"model.transformer.blocks.5.attn_out: Linear\n",
"model.transformer.blocks.5.ff_out: Linear\n",
"model.transformer.blocks.5.rotary_emb: RotaryEmbedding\n",
"model.transformer.blocks.5.attn_norm: LayerNorm\n",
"model.transformer.blocks.5.ff_norm: LayerNorm\n",
"model.transformer.blocks.5.att_proj: Linear\n",
"model.transformer.blocks.5.ff_proj: Linear\n",
"model.transformer.blocks.6: OLMoSequentialBlock layer_id=6\n",
"Scoring full models on cuda...\n",
"c4_color_mean: 0.661101\n",
"c4_color_std: 0.162787\n",
"c4_n: 200\n",
"c4_nll_cond_mean: 4.171017\n",
"c4_nll_marg_mean: 3.509916\n",
"color_mean_gap_gutenberg_minus_c4: -1.963580\n",
"gutenberg_color_mean: -1.302479\n",
"gutenberg_color_std: 0.150317\n",
"gutenberg_n: 200\n",
"gutenberg_nll_cond_mean: 3.277694\n",
"gutenberg_nll_marg_mean: 4.580173\n",
"Applying paired ablation top4 and scoring 8 real sequences...\n",
"Wrote /content/CoLoR-ablation/reports/layer-ablated-color-filter/local_validation.md\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# 5. Build the Frozen Pool"
],
"metadata": {
"id": "KSiu2x-6Zxx0"
}
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"!python scripts/01_build_pool.py --config configs/default.yaml"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "ax0ClMLUZhU2",
"outputId": "8c2dafd9-b3e9-4ad8-ee9c-4b35895073fc"
},
"execution_count": 15,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"tokenizer_config.json: 100% 264/264 [00:00<00:00, 890kB/s]\n",
"tokenizer.json: 100% 2.11M/2.11M [00:00<00:00, 108MB/s]\n",
"special_tokens_map.json: 100% 134/134 [00:00<00:00, 668kB/s]\n",
"README.md: 100% 41.1k/41.1k [00:00<00:00, 82.5MB/s]\n",
"Resolving data files: 100% 1024/1024 [00:00<00:00, 76809.69it/s]\n",
"Resolving data files: 100% 1024/1024 [00:00<00:00, 22542.68it/s]\n",
"README.md: 100% 2.95k/2.95k [00:00<00:00, 7.30MB/s]\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/data/books_pool.npy with shape=(105000, 512)\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/data/books_pool_meta.parquet with n_enriched=5000\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"import pandas as pd\n",
"meta = pd.read_parquet(f\"{DRIVE}/data/books_pool_meta.parquet\")\n",
"print(meta[\"enriched\"].value_counts(dropna=False))\n",
"print(meta.head())"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "OzJGwadrZus9",
"outputId": "a007ca7f-a9bf-48a2-cfc0-7c648cc0dffe"
},
"execution_count": 16,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"enriched\n",
"False 100000\n",
"True 5000\n",
"Name: count, dtype: int64\n",
" seq_idx source enriched source_sequence_idx seed\n",
"0 0 allenai/c4 False 0 17\n",
"1 1 allenai/c4 False 1 17\n",
"2 2 allenai/c4 False 2 17\n",
"3 3 allenai/c4 False 3 17\n",
"4 4 allenai/c4 False 4 17\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# 6. Ground Truth + Noise Floor (defines the baseline)"
],
"metadata": {
"id": "CX_RqgnRvcMv"
}
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"!python scripts/02_score.py --config configs/default.yaml --variant full\n",
"!python scripts/02_score.py --config configs/default.yaml --variant full --score-id full_rescore\n",
"!python scripts/03_metrics.py --config configs/default.yaml"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "1oDlMFJVtveG",
"outputId": "ed2f7e82-6064-457e-dab8-ae911e128608"
},
"execution_count": 17,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Loading weights: 100% 99/99 [00:00<00:00, 287.55it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 287.87it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_full.parquet\n",
"{\n",
" \"elapsed_seconds\": 485.5985937590001,\n",
" \"tokens_per_second\": 221417.4451529849,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"Loading weights: 100% 99/99 [00:00<00:00, 290.04it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 292.52it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_full_rescore.parquet\n",
"{\n",
" \"elapsed_seconds\": 486.007337558,\n",
" \"tokens_per_second\": 221231.22778402205,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/metrics.csv\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# 7. Ablation Scoring Grid"
],
"metadata": {
"id": "zs9V-v3g01Dh"
}
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"for variant in [\"top1\", \"top2\", \"top4\", \"top6\", \"mid2\", \"mid4\", \"bot2\", \"bot4\", \"skip2\"]:\n",
" print(f\"=== {variant} ===\")\n",
" !python scripts/02_score.py --config configs/default.yaml --variant {variant}"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "qUgyhQTrvhtH",
"outputId": "9293f3e0-4025-4cf8-931c-6cec45223bca"
},
"execution_count": 19,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"=== top1 ===\n",
"Loading weights: 100% 99/99 [00:00<00:00, 292.99it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 280.41it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top1_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_top1.parquet\n",
"{\n",
" \"elapsed_seconds\": 457.60804622300066,\n",
" \"tokens_per_second\": 234960.90352310712,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"=== top2 ===\n",
"Loading weights: 100% 99/99 [00:00<00:00, 301.24it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 285.70it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top2_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_top2.parquet\n",
"{\n",
" \"elapsed_seconds\": 429.50918993999994,\n",
" \"tokens_per_second\": 250332.24554524655,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"=== top4 ===\n",
"Loading weights: 100% 99/99 [00:00<00:00, 293.50it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 289.79it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top4_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_top4.parquet\n",
"{\n",
" \"elapsed_seconds\": 373.3852946579991,\n",
" \"tokens_per_second\": 287959.92112780595,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"=== top6 ===\n",
"Loading weights: 100% 99/99 [00:00<00:00, 297.02it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 282.47it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_top6_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_top6.parquet\n",
"{\n",
" \"elapsed_seconds\": 316.91414802800045,\n",
" \"tokens_per_second\": 339271.69446060905,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"=== mid2 ===\n",
"Loading weights: 100% 99/99 [00:00<00:00, 300.06it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 270.83it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid2_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_mid2.parquet\n",
"{\n",
" \"elapsed_seconds\": 429.4130174130023,\n",
" \"tokens_per_second\": 250388.31064729704,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"=== mid4 ===\n",
"Loading weights: 100% 99/99 [00:00<00:00, 287.05it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 283.85it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_mid4_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_mid4.parquet\n",
"{\n",
" \"elapsed_seconds\": 373.02652139100246,\n",
" \"tokens_per_second\": 288236.87816904223,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"=== bot2 ===\n",
"Loading weights: 100% 99/99 [00:00<00:00, 306.71it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 296.19it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot2_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_bot2.parquet\n",
"{\n",
" \"elapsed_seconds\": 429.29979027800164,\n",
" \"tokens_per_second\": 250454.35016488892,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"=== bot4 ===\n",
"Loading weights: 100% 99/99 [00:00<00:00, 290.26it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 290.29it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_bot4_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_bot4.parquet\n",
"{\n",
" \"elapsed_seconds\": 372.5894944189995,\n",
" \"tokens_per_second\": 288574.96416441427,\n",
" \"tokens_scored\": 107520000\n",
"}\n",
"=== skip2 ===\n",
"Loading weights: 100% 99/99 [00:00<00:00, 287.65it/s]\n",
"[transformers] `use_return_dict` is deprecated! Use `return_dict` instead!\n",
"Loading weights: 100% 99/99 [00:00<00:00, 282.18it/s]\n",
"Scoring shard 0:5000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00000000_00005000.parquet\n",
"Scoring shard 5000:10000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00005000_00010000.parquet\n",
"Scoring shard 10000:15000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00010000_00015000.parquet\n",
"Scoring shard 15000:20000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00015000_00020000.parquet\n",
"Scoring shard 20000:25000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00020000_00025000.parquet\n",
"Scoring shard 25000:30000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00025000_00030000.parquet\n",
"Scoring shard 30000:35000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00030000_00035000.parquet\n",
"Scoring shard 35000:40000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00035000_00040000.parquet\n",
"Scoring shard 40000:45000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00040000_00045000.parquet\n",
"Scoring shard 45000:50000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00045000_00050000.parquet\n",
"Scoring shard 50000:55000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00050000_00055000.parquet\n",
"Scoring shard 55000:60000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00055000_00060000.parquet\n",
"Scoring shard 60000:65000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00060000_00065000.parquet\n",
"Scoring shard 65000:70000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00065000_00070000.parquet\n",
"Scoring shard 70000:75000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00070000_00075000.parquet\n",
"Scoring shard 75000:80000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00075000_00080000.parquet\n",
"Scoring shard 80000:85000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00080000_00085000.parquet\n",
"Scoring shard 85000:90000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00085000_00090000.parquet\n",
"Scoring shard 90000:95000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00090000_00095000.parquet\n",
"Scoring shard 95000:100000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00095000_00100000.parquet\n",
"Scoring shard 100000:105000 -> /content/drive/MyDrive/color-filter-ablation/results/scores_skip2_shards/part_00100000_00105000.parquet\n",
"Wrote /content/drive/MyDrive/color-filter-ablation/results/scores_skip2.parquet\n",
"{\n",
" \"elapsed_seconds\": 316.5678678459981,\n",
" \"tokens_per_second\": 339642.8093968957,\n",
" \"tokens_scored\": 107520000\n",
"}\n"
]
}
]
},
{
"cell_type": "markdown",
"source": [
"# 8. Metrics and Plots"
],
"metadata": {
"id": "3HZbo0E4DwAw"
}
},
{
"cell_type": "code",
"source": [
"# PYTHON CELL\n",
"!python scripts/03_metrics.py --config configs/default.yaml\n",
"!python scripts/04_plots.py --config configs/default.yaml --scatter-variant top4"
],
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/"
},
"id": "Y65SkQ2Y1DQ5",
"outputId": "1952ff37-2bc8-439a-f908-83f5461671ea"
},
"execution_count": 20,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Wrote /content/drive/MyDrive/color-filter-ablation/results/metrics.csv\n",
"Wrote figures under /content/drive/MyDrive/color-filter-ablation/reports/layer-ablated-color-filter/figures\n"
]
}
]
},
{
"cell_type": "code",
"source": [],
"metadata": {
"id": "y921veSSDyRL"
},
"execution_count": null,
"outputs": []
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment