RAG_on_hsbc_uk_num_of_tokens.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "QZ_E1Ma1jKYT",
        "-6LGAl2AGj4j",
        "Ooplt3-YJJRb",
        "gUMZZ3XKSb1A"
      ],
      "gpuType": "T4"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Ensure num of input tokens does not exceed limit.\n",
        "\n",
        "How to calculate num of tokens: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb"
      ],
      "metadata": {
        "id": "CvJWKtYlWnJT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "encoding = tiktoken.get_encoding(\"p50k_base\")\n",
        "num_tokens_template = len(encoding.encode(prompt_template))\n",
        "num_tokens = len(encoding.encode(llm_full_query))\n",
        "print('num_tokens', num_tokens)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CPFSAkFwePtg",
        "outputId": "69aa06ec-bd15-4495-f180-d7c2a6aa54ea"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "num_tokens 8673\n"
          ]
        }
      ]
    }
  ]
}