Skip to content

Instantly share code, notes, and snippets.

@jussker

jussker/OpenAI Token Counting.ipynb

Forked from CGamesPlay/OpenAI Token Counting.ipynb
Last active December 15, 2023 16:58
Show Gist options
  • Save jussker/91e578cccb770ed3bf8673d44539851e to your computer and use it in GitHub Desktop.
Save jussker/91e578cccb770ed3bf8673d44539851e to your computer and use it in GitHub Desktop.
Download ZIP
Display the source blob
Display the rendered blob
Raw
OpenAI Token Counting.ipynb
{
"cells": [
{
"cell_type": "code",
"execution_count": 68,
"id": "92f64b19-bc5b-4b40-a752-0f7364f525b7",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"import textwrap\n",
"\n",
"import openai\n",
"import tiktoken\n",
"\n",
"encoder = tiktoken.encoding_for_model(\"gpt-3.5-turbo\")\n",
"token_length = lambda x: len(encoder.encode(x))"
]
},
{
"cell_type": "code",
"execution_count": 50,
"id": "9d4b26b6-5635-4382-90db-4e449e10cc2a",
"metadata": {
"tags": []
},
"outputs": [],
"source": [
"# Defining some sample tools to use.\n",
"\n",
"simple_tool = dict(\n",
" name=\"get_location\",\n",
" description=\"Get the user's location\",\n",
" parameters={\"type\": \"object\", \"properties\": {}},\n",
")\n",
"\n",
"create_model_tool = dict(\n",
" name=\"create_model\",\n",
" description=\"Create a new conversational agent\",\n",
" parameters={\n",
" \"title\": \"GptParams\",\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"temperature\": {\n",
" \"title\": \"Temperature\",\n",
" \"default\": 1,\n",
" \"minimum\": 0,\n",
" \"maximum\": 2,\n",
" \"type\": \"number\",\n",
" },\n",
" \"max_tokens\": {\n",
" \"title\": \"Max Tokens\",\n",
" \"description\": \"The maximum response length of the model\",\n",
" \"default\": 512,\n",
" \"type\": \"integer\",\n",
" },\n",
" \"reserve_tokens\": {\n",
" \"title\": \"Reserve Tokens\",\n",
" \"description\": \"Number of tokens reserved for the model's output\",\n",
" \"default\": 512,\n",
" \"type\": \"integer\",\n",
" },\n",
" },\n",
" \"additionalProperties\": False,\n",
" },\n",
")\n",
"\n",
"send_message_tool = dict(\n",
" name=\"send_message\",\n",
" description=\"Send a new message\",\n",
" parameters={\n",
" \"title\": \"ConversationCreate\",\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"params\": {\"$ref\": \"#/definitions/ConversationParams\"},\n",
" \"messages\": {\n",
" \"title\": \"Messages\",\n",
" \"type\": \"array\",\n",
" \"items\": {\"$ref\": \"#/definitions/MessageCreate\"},\n",
" },\n",
" },\n",
" \"required\": [\"params\", \"messages\"],\n",
" \"definitions\": {\n",
" \"ConversationParams\": {\n",
" \"title\": \"ConversationParams\",\n",
" \"description\": \"Parameters to use for the conversation. Extra fields are permitted and\\npassed to the model directly.\",\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"model\": {\n",
" \"title\": \"Model\",\n",
" \"description\": \"Completion model to use for the conversation\",\n",
" \"pattern\": \"^.+:.+$\",\n",
" \"example\": \"openai:Gpt35Model\",\n",
" \"type\": \"string\",\n",
" },\n",
" \"model_params\": {\n",
" \"title\": \"Model Params\",\n",
" \"type\": \"object\",\n",
" \"properties\": {},\n",
" \"additionalProperties\": True,\n",
" },\n",
" \"features\": {\n",
" \"title\": \"Features\",\n",
" \"description\": \"Set of enabled features for this conversation and the parameters for them.\",\n",
" \"example\": {\"test:dummy_feature\": {\"enable_weather\": True}},\n",
" \"type\": \"object\",\n",
" },\n",
" },\n",
" \"required\": [\"model\", \"model_params\", \"features\"],\n",
" },\n",
" \"MessageRole\": {\n",
" \"title\": \"MessageRole\",\n",
" \"description\": \"An enumeration.\",\n",
" \"enum\": [\"user\", \"assistant\", \"system\", \"tool_call\", \"tool_result\"],\n",
" \"type\": \"string\",\n",
" },\n",
" \"MessageCreate\": {\n",
" \"title\": \"MessageCreate\",\n",
" \"type\": \"object\",\n",
" \"properties\": {\n",
" \"role\": {\"$ref\": \"#/definitions/MessageRole\"},\n",
" \"name\": {\n",
" \"title\": \"Name\",\n",
" \"description\": \"\\n Sender of the message. For tool_call and tool_result, this is the\\n name of the tool being referenced. Otherwise, it is optional.\\n \",\n",
" \"example\": \"user\",\n",
" \"type\": \"string\",\n",
" },\n",
" \"content\": {\n",
" \"title\": \"Content\",\n",
" \"description\": \"\\n Arbitrary data. For regular messages (not tool calls/results), this\\n must include a 'text' field containing the message text.\\n \",\n",
" \"example\": {\"text\": \"Why is the sky blue?\"},\n",
" \"additionalProperties\": True,\n",
" \"type\": \"object\",\n",
" },\n",
" },\n",
" \"required\": [\"role\", \"content\"],\n",
" },\n",
" },\n",
" },\n",
")"
]
},
{
"cell_type": "code",
"execution_count": 58,
"id": "45ff05fa-dd35-45f7-a695-c373fb909d65",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"The following text document has been provided for context.\n",
"\n",
"# Tools\n",
"\n",
"## functions\n",
"\n",
"namespace functions {\n",
"\n",
"// Get the user's location\n",
"type get_location = () => any;\n",
"\n",
"// Create a new conversational agent\n",
"type create_model = (_: {\n",
"temperature?: number, // default: 1.0\n",
"// The maximum response length of the model\n",
"max_tokens?: number, // default: 512\n",
"// Number of tokens reserved for the model's output\n",
"reserve_tokens?: number, // default: 512\n",
"}) => any;\n",
"\n",
"// Send a new message\n",
"type send_message = (_: {\n",
"// Parameters to use for the conversation. Extra fields are permitted and\n",
"// passed to the model directly.\n",
"params: {\n",
" model: string,\n",
" model_params: object,\n",
"},\n",
"messages: {\n",
" role: \"user\" | \"assistant\" | \"system\" | \"tool_call\" | \"tool_result\",\n",
" name: string,\n",
"}[],\n",
"}) => any;\n",
"\n",
"} // namespace functions\n"
]
}
],
"source": [
"def dump_encoding(functions):\n",
" response = openai.ChatCompletion.create(\n",
" model=\"gpt-3.5-turbo\",\n",
" temperature=0,\n",
" messages=[\n",
" {\n",
" \"role\": \"system\",\n",
" \"content\": \"The following text document has been provided for context.\",\n",
" },\n",
" {\n",
" \"role\": \"system\",\n",
" \"content\": \"End of document. Please repeat, verbatim, the text document, to verify understanding.\",\n",
" },\n",
" ],\n",
" functions=functions,\n",
" )\n",
" print(response.choices[0].message.content)\n",
"\n",
"\n",
"dump_encoding([simple_tool, create_model_tool, send_message_tool])"
]
},
{
"cell_type": "markdown",
"id": "568492f8-eed7-42d0-bd0b-cea0c3edd24e",
"metadata": {
"tags": []
},
"source": [
"## Observations from the model output\n",
"\n",
"OpenAI is injecting the function descriptions as the second message in the thread, presumably as a system message. It renders the JSON schema to a test format, which is possible to emulate locally. Some notable things:\n",
"\n",
"- Examples, titles, and most validations are not exposed to the model, but default values and required fields are.\n",
"- OpenAI will de-indent the descriptions that it does include.\n",
"- Nested objects are permitted, but the description fields will be omitted from the model (see the start_conversation output).\n",
"- Object types with unspecified keys are handled inconsistently (see ConversationParams model_params, features, and MessageCreate content).\n",
"- Optional fields are handled inconsistently (see MessageCreate name)."
]
},
{
"cell_type": "code",
"execution_count": 72,
"id": "448d1830-007a-4d9e-a91d-6a04903a5654",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"12\n"
]
}
],
"source": [
"FUNCTION_OVERHEAD = -1 + len(\n",
" tiktoken.encoding_for_model(\"gpt-3.5-turbo\").encode(\n",
" \"\"\"# Tools\n",
"\n",
"## functions\n",
"\n",
"namespace functions {\n",
"\n",
"} // namespace functions\"\"\"\n",
" )\n",
")\n",
"print(FUNCTION_OVERHEAD)\n",
"# original content: Function overhead is 16: 3 for the system message plus this template.\n",
"# Up to December 16, 2023, after multiple tests, the FUNCTION_OVERHEAD should be 12."
]
},
{
"cell_type": "code",
"execution_count": 76,
"id": "df4fd16e-2911-4cdb-8170-03873076a5d5",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"// Get the user's location\n",
"type get_location = () => any;\n",
"\n",
"\n",
"// Create a new conversational agent\n",
"type create_model = (_: {\n",
"temperature?: number, // default: 1.0\n",
"// The maximum response length of the model\n",
"max_tokens?: number, // default: 512\n",
"// Number of tokens reserved for the model's output\n",
"reserve_tokens?: number, // default: 512\n",
"}) => any;\n",
"\n",
"\n",
"// Send a new message\n",
"type send_message = (_: {\n",
"// Parameters to use for the conversation. Extra fields are permitted and\n",
"// passed to the model directly.\n",
"params: {\n",
" model: string,\n",
" model_params: object,\n",
"},\n",
"messages: {\n",
" role: \"user\" | \"assistant\" | \"system\" | \"tool_call\" | \"tool_result\",\n",
" name?: string,\n",
" content: object,\n",
"}[],\n",
"}) => any;\n",
"\n",
"\n"
]
}
],
"source": [
"def format_tool(tool):\n",
" def resolve_ref(schema):\n",
" if schema.get(\"$ref\") is not None:\n",
" ref = schema[\"$ref\"][14:]\n",
" schema = json_schema[\"definitions\"][ref]\n",
" return schema\n",
"\n",
" def format_schema(schema, indent):\n",
" schema = resolve_ref(schema)\n",
" if \"enum\" in schema:\n",
" return format_enum(schema, indent)\n",
" elif schema[\"type\"] == \"object\":\n",
" return format_object(schema, indent)\n",
" elif schema[\"type\"] == \"integer\":\n",
" return \"number\"\n",
" elif schema[\"type\"] in [\"string\", \"number\"]:\n",
" return schema[\"type\"]\n",
" elif schema[\"type\"] == \"array\":\n",
" return format_schema(schema[\"items\"], indent) + \"[]\"\n",
" else:\n",
" raise ValueError(\"unknown schema type \" + schema[\"type\"])\n",
"\n",
" def format_enum(schema, indent):\n",
" # `json.dumps(o, ensure_ascii=False)` to support Japanese and Chinese.\n",
" return \" | \".join(json.dumps(o, ensure_ascii=False) for o in schema[\"enum\"])\n",
"\n",
" def format_object(schema, indent):\n",
" result = \"{\\n\"\n",
" if \"properties\" not in schema or len(schema[\"properties\"]) == 0:\n",
" if schema.get(\"additionalProperties\", False):\n",
" return \"object\"\n",
" return None\n",
" for key, value in schema[\"properties\"].items():\n",
" value = resolve_ref(value)\n",
" value_rendered = format_schema(value, indent + 1)\n",
" if value_rendered is None:\n",
" continue\n",
" if \"description\" in value and indent == 0:\n",
" for line in textwrap.dedent(value[\"description\"]).strip().split(\"\\n\"):\n",
" result += f\"{' '*indent}// {line}\\n\"\n",
" optional = \"\" if key in schema.get(\"required\", {}) else \"?\"\n",
" comment = (\n",
" \"\"\n",
" if value.get(\"default\") is None\n",
" else f\" // default: {format_default(value)}\"\n",
" )\n",
" result += f\"{' '*indent}{key}{optional}: {value_rendered},{comment}\\n\"\n",
" result += (\" \" * (indent - 1)) + \"}\"\n",
" return result\n",
"\n",
" def format_default(schema):\n",
" v = schema[\"default\"]\n",
" if schema[\"type\"] == \"number\":\n",
" return f\"{v:.1f}\" if float(v).is_integer() else str(v)\n",
" else:\n",
" return str(v)\n",
"\n",
" json_schema = tool[\"parameters\"]\n",
" result = f\"// {tool['description']}\\ntype {tool['name']} = (\"\n",
" formatted = format_object(json_schema, 0)\n",
" if formatted is not None:\n",
" result += \"_: \" + formatted\n",
" result += \") => any;\\n\\n\"\n",
" return result\n",
"\n",
"\n",
"print(format_tool(simple_tool))\n",
"print(format_tool(create_model_tool))\n",
"print(format_tool(send_message_tool))"
]
},
{
"cell_type": "code",
"execution_count": 65,
"id": "a38dd30b-7f98-49a9-ba4d-0f32fe820857",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"14\n"
]
}
],
"source": [
"base_message = {\"role\": \"user\", \"content\": \"What is the meaning of life?\"}\n",
"response = openai.ChatCompletion.create(\n",
" model=\"gpt-3.5-turbo\", max_tokens=1, messages=[base_message]\n",
")\n",
"base_usage = response.usage.prompt_tokens\n",
"print(base_usage)"
]
},
{
"cell_type": "code",
"execution_count": 73,
"id": "3ab9413d-4659-441d-b43f-8f75ea3dcea5",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"45 45\n"
]
}
],
"source": [
"response = openai.ChatCompletion.create(\n",
" model=\"gpt-3.5-turbo\",\n",
" max_tokens=1,\n",
" messages=[base_message],\n",
" functions=[simple_tool],\n",
")\n",
"actual = response.usage.prompt_tokens\n",
"expected = base_usage + FUNCTION_OVERHEAD + token_length(format_tool(simple_tool))\n",
"print(actual, expected)\n",
"assert actual == expected"
]
},
{
"cell_type": "code",
"execution_count": 79,
"id": "495626ae-a772-467f-9d69-0033fce557f2",
"metadata": {
"tags": []
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"118 118\n"
]
}
],
"source": [
"functions = [simple_tool, create_model_tool]\n",
"response = openai.ChatCompletion.create(\n",
" model=\"gpt-3.5-turbo\",\n",
" max_tokens=1,\n",
" messages=[base_message],\n",
" functions=functions,\n",
")\n",
"actual = response.usage.prompt_tokens\n",
"expected = base_usage + FUNCTION_OVERHEAD + sum(token_length(format_tool(f)) for f in functions)\n",
"print(actual, expected)\n",
"assert actual == expected"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.4"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment