test-impact-of-lit-gpt-qa-training-set-tokenization-fix.ipynb

test-impact-of-lit-gpt-qa-training-set-tokenization-fix.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNel57SEx9WSKL8tRt9xke9",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/ochafik/e83e51aa48d717eaa75e8635cafa62d7/test-impact-of-lit-gpt-qa-training-set-tokenization-fix.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Download the Alpaca Dataset\n",
        "!wget -O alpaca_data.json https://raw.githubusercontent.com/tatsu-lab/stanford_alpaca/main/alpaca_data.json\n",
        "!wget -O alpaca_data_cleaned.json https://huggingface.co/datasets/magicgh/alpaca-cleaned/resolve/main/alpaca_data_cleaned.json?download=true\n",
        "\n",
        "# Download some tokenizers.\n",
        "!wget -O mistral-tokenizer.json https://huggingface.co/mistralai/Mistral-7B-v0.1/raw/main/tokenizer.json\n",
        "!wget -O falcon-40b-tokenizer.json https://huggingface.co/tiiuae/falcon-40b/raw/main/tokenizer.json\n",
        "!wget -O llama-2-tokenizer.json https://huggingface.co/hf-internal-testing/llama-tokenizer/raw/main/tokenizer.json\n",
        "# !wget -O llama-2-tokenizer.json https://huggingface.co/meta-llama/Llama-2-7b-chat-hf/raw/main/tokenizer.json"
      ],
      "metadata": {
        "id": "urwcddMwCnV6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5KiKxlIkCf8r",
        "outputId": "8e3fccc9-c7c2-4da2-b422-426d3d5eb30a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total = 52002 data entries\n",
            "# llama-2-tokenizer.json: 23 tokens starting w/ :, matches 1753 data entries (3.3710241913772547%)\n",
            "  <:->: 656 occurrences\n",
            "  <:\">: 611 occurrences\n",
            "  <:[>: 179 occurrences\n",
            "  <:#>: 122 occurrences\n",
            "  <:(>: 46 occurrences\n",
            "  <://>: 42 occurrences\n",
            "  <:{>: 35 occurrences\n",
            "  <:$>: 30 occurrences\n",
            "  <:\\>: 8 occurrences\n",
            "  <:`>: 7 occurrences\n",
            "  <:/>: 5 occurrences\n",
            "  <:@>: 4 occurrences\n",
            "  <:%>: 3 occurrences\n",
            "  <:=>: 2 occurrences\n",
            "  <:'>: 2 occurrences\n",
            "  <::>: 1 occurrences\n",
            "\n",
            "# mistral-tokenizer.json: 23 tokens starting w/ :, matches 1369 data entries (2.6325910541902235%)\n",
            "  <:->: 656 occurrences\n",
            "  <:\">: 611 occurrences\n",
            "  <://>: 42 occurrences\n",
            "  <:{>: 35 occurrences\n",
            "  <:\\>: 8 occurrences\n",
            "  <:`>: 7 occurrences\n",
            "  <:%>: 3 occurrences\n",
            "  <:'>: 2 occurrences\n",
            "  <:*>: 2 occurrences\n",
            "  <:_>: 2 occurrences\n",
            "  <::>: 1 occurrences\n",
            "\n",
            "# falcon-40b-tokenizer.json: 33 tokens starting w/ :, matches 2011 data entries (3.867158955424791%)\n",
            "  <:->: 656 occurrences\n",
            "  <:\">: 611 occurrences\n",
            "  <:<>: 244 occurrences\n",
            "  <:[>: 179 occurrences\n",
            "  <:#>: 122 occurrences\n",
            "  <:(>: 46 occurrences\n",
            "  <://>: 42 occurrences\n",
            "  <:{>: 35 occurrences\n",
            "  <:$>: 30 occurrences\n",
            "  <:.>: 12 occurrences\n",
            "  <:\\>: 8 occurrences\n",
            "  <:`>: 7 occurrences\n",
            "  <:/>: 5 occurrences\n",
            "  <:+>: 4 occurrences\n",
            "  <:%>: 3 occurrences\n",
            "  <:=>: 2 occurrences\n",
            "  <:'>: 2 occurrences\n",
            "  <:*>: 2 occurrences\n",
            "  <::>: 1 occurrences\n",
            "\n"
          ]
        }
      ],
      "source": [
        "# Context: https://github.com/Lightning-AI/lit-gpt/compare/main...ochafik:lit-gpt:tmp_ochafik_fix_qa_tokenization?expand=1\n",
        "\n",
        "import json\n",
        "import re\n",
        "\n",
        "offending_question_suffix = ':'\n",
        "\n",
        "# with open(\"alpaca_data_cleaned.json\", 'r+t') as f:\n",
        "with open(\"alpaca_data.json\", 'r+t') as f:\n",
        "  data = json.load(f)\n",
        "print(f'Total = {len(data)} data entries')\n",
        "\n",
        "for tokenizer_file in [\"llama-2-tokenizer.json\", \"mistral-tokenizer.json\", \"falcon-40b-tokenizer.json\"]:\n",
        "  with open(tokenizer_file) as f:\n",
        "    vocab = list(json.load(f)['model']['vocab'].keys())\n",
        "\n",
        "  # Sort by longest token first (skip token <:> itself)\n",
        "  matching_tokens = list(sorted((k for k in vocab if k.startswith(offending_question_suffix) and len(k) > len(offending_question_suffix)), key=lambda t: -len(t)))\n",
        "\n",
        "  token_matches = {t: 0 for t in matching_tokens}\n",
        "  pattern = re.compile('^(' + '|'.join(re.escape(t) for t in matching_tokens) + ').*')\n",
        "  # print(pattern)\n",
        "\n",
        "  matching = 0\n",
        "  for d in data:\n",
        "    output = d[\"output\"]\n",
        "    combined_str = offending_question_suffix + output\n",
        "    if pattern.match(combined_str): # or any(output.startswith(t[2: ]) for t in tokens):\n",
        "      for t in matching_tokens:\n",
        "        if combined_str.startswith(t):\n",
        "          token_matches[t] += 1\n",
        "          break\n",
        "      # if matching < 100:\n",
        "      #   s = output[:15].split('\\n')[0]\n",
        "      #   print(f\"  Example maching output: {s}\")\n",
        "      matching += 1\n",
        "\n",
        "  print(f'# {tokenizer_file}: {len(matching_tokens)} tokens starting w/ {offending_question_suffix}, matches {matching} data entries ({100 * matching / len(data)}%)')\n",
        "  for t, c in sorted(token_matches.items(), key=lambda tc: -tc[1]):\n",
        "    if c > 0:\n",
        "      print(f'  <{t}>: {c} occurrences')\n",
        "  print('')\n"
      ]
    }
  ]
}