Cameron Pfiffer slide at NousCon

notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9b3145fe-2b5f-4a94-884d-1ad6e5d4dba2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:18.104655Z",
     "iopub.status.busy": "2024-09-19T12:36:18.104166Z",
     "iopub.status.idle": "2024-09-19T12:36:18.121825Z",
     "shell.execute_reply": "2024-09-19T12:36:18.120522Z",
     "shell.execute_reply.started": "2024-09-19T12:36:18.104604Z"
    }
   },
   "outputs": [],
   "source": [
    "from enum import Enum\n",
    "\n",
    "class GameSettingType(str, Enum):\n",
    "    cyberpunk = \"cyberpunk\"\n",
    "    solarpunk = \"solarpunk\"\n",
    "    fantasy = \"fantasy\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7d7dbe56-a093-4e75-b537-7d7d0be30edf",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:18.123510Z",
     "iopub.status.busy": "2024-09-19T12:36:18.122941Z",
     "iopub.status.idle": "2024-09-19T12:36:18.248657Z",
     "shell.execute_reply": "2024-09-19T12:36:18.247543Z",
     "shell.execute_reply.started": "2024-09-19T12:36:18.123476Z"
    }
   },
   "outputs": [],
   "source": [
    "from pydantic import BaseModel\n",
    "\n",
    "\n",
    "class GameSetting(BaseModel):\n",
    "    setting: GameSettingType\n",
    "    description: str\n",
    "    id: int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bc6b56e7-eb05-487c-ba39-06f005849225",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:18.249643Z",
     "iopub.status.busy": "2024-09-19T12:36:18.249480Z",
     "iopub.status.idle": "2024-09-19T12:36:18.258524Z",
     "shell.execute_reply": "2024-09-19T12:36:18.257838Z",
     "shell.execute_reply.started": "2024-09-19T12:36:18.249628Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'$defs': {'GameSettingType': {'enum': ['cyberpunk', 'solarpunk', 'fantasy'],\n",
       "   'title': 'GameSettingType',\n",
       "   'type': 'string'}},\n",
       " 'properties': {'setting': {'$ref': '#/$defs/GameSettingType'},\n",
       "  'description': {'title': 'Description', 'type': 'string'},\n",
       "  'id': {'title': 'Id', 'type': 'integer'}},\n",
       " 'required': ['setting', 'description', 'id'],\n",
       " 'title': 'GameSetting',\n",
       " 'type': 'object'}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "\n",
    "json_schema = GameSetting.model_json_schema()\n",
    "json_schema"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "63ecce82-faf3-439e-a7e5-dd334c5a561d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:18.259289Z",
     "iopub.status.busy": "2024-09-19T12:36:18.259139Z",
     "iopub.status.idle": "2024-09-19T12:36:21.975451Z",
     "shell.execute_reply": "2024-09-19T12:36:21.974724Z",
     "shell.execute_reply.started": "2024-09-19T12:36:18.259275Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\\\{[ ]?\"setting\"[ ]?:[ ]?(\"cyberpunk\"|\"solarpunk\"|\"fantasy\")[ ]?,[ ]?\"description\"[ ]?:[ ]?\"([^\"\\\\\\\\\\\\x00-\\\\x1F\\\\x7F-\\\\x9F]|\\\\\\\\[\"\\\\\\\\])*\"[ ]?,[ ]?\"id\"[ ]?:[ ]?(-)?(0|[1-9][0-9]*)[ ]?\\\\}'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from outlines.integrations.utils import convert_json_schema_to_str\n",
    "from outlines.fsm.json_schema import build_regex_from_schema\n",
    "\n",
    "schema_str = convert_json_schema_to_str(json_schema=json_schema)\n",
    "regex_str = build_regex_from_schema(schema_str, whitespace_pattern=\"[ ]?\")\n",
    "regex_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "0d964971-601a-447f-a7ff-ec15131eb225",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:21.976548Z",
     "iopub.status.busy": "2024-09-19T12:36:21.976266Z",
     "iopub.status.idle": "2024-09-19T12:36:25.300222Z",
     "shell.execute_reply": "2024-09-19T12:36:25.299030Z",
     "shell.execute_reply.started": "2024-09-19T12:36:21.976531Z"
    }
   },
   "outputs": [],
   "source": [
    "import llama_cpp\n",
    "from llama_cpp import Llama\n",
    "from outlines import generate, models\n",
    "\n",
    "llm = Llama(\n",
    "    \"/big_storage/llms/models/Hermes-3-Llama-3.1-8B.Q6_K.gguf\",\n",
    "    tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained(\n",
    "        \"NousResearch/Hermes-3-Llama-3.1-8B\"\n",
    "    ),\n",
    "    n_gpu_layers=-1,\n",
    "    flash_attn=True,\n",
    "    n_ctx=8192,\n",
    "    verbose=False,\n",
    ")\n",
    "model = models.LlamaCpp(llm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "fa647578-95a0-4973-8566-108660a4cf20",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:25.301973Z",
     "iopub.status.busy": "2024-09-19T12:36:25.301800Z",
     "iopub.status.idle": "2024-09-19T12:36:25.305607Z",
     "shell.execute_reply": "2024-09-19T12:36:25.304949Z",
     "shell.execute_reply.started": "2024-09-19T12:36:25.301958Z"
    }
   },
   "outputs": [],
   "source": [
    "def generate_hermes_prompt(user_prompt):\n",
    "    return (\n",
    "        \"<|im_start|>system\\n\"\n",
    "        \"You are a world class AI model who answers questions in JSON with correct Pydantic schema. \"\n",
    "        f\"Here's the json schema you must adhere to:\\n<schema>\\n{schema_str}\\n</schema><|im_end|>\\n\"\n",
    "        \"<|im_start|>user\\n\"\n",
    "        + user_prompt\n",
    "        + \"<|im_end|>\"\n",
    "        + \"\\n<|im_start|>assistant\\n\"\n",
    "        \"<schema>\"\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e2db2793-6c82-4036-9e4d-277713f9af43",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:25.306318Z",
     "iopub.status.busy": "2024-09-19T12:36:25.306176Z",
     "iopub.status.idle": "2024-09-19T12:36:25.331308Z",
     "shell.execute_reply": "2024-09-19T12:36:25.330219Z",
     "shell.execute_reply.started": "2024-09-19T12:36:25.306304Z"
    }
   },
   "outputs": [],
   "source": [
    "user_prompt = \"Pick a game setting and describe it in JSON format\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6ac9b98c-c600-457a-a174-7bb6d097a919",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:25.333188Z",
     "iopub.status.busy": "2024-09-19T12:36:25.332740Z",
     "iopub.status.idle": "2024-09-19T12:36:27.628974Z",
     "shell.execute_reply": "2024-09-19T12:36:27.628393Z",
     "shell.execute_reply.started": "2024-09-19T12:36:25.333144Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/asilva/2024-PyData-Paris/.venv/lib/python3.10/site-packages/llama_cpp/llama.py:1138: RuntimeWarning: Detected duplicate leading \"<|begin_of_text|>\" in prompt, this will likely reduce response quality, consider removing it...\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "generator = generate.json(model, GameSetting)\n",
    "prompt = generate_hermes_prompt(user_prompt)\n",
    "response = generator(prompt, max_tokens=1024, temperature=0, seed=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "641edabb-ba71-4c2f-9d76-b2ae335f05de",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:27.629913Z",
     "iopub.status.busy": "2024-09-19T12:36:27.629704Z",
     "iopub.status.idle": "2024-09-19T12:36:27.633864Z",
     "shell.execute_reply": "2024-09-19T12:36:27.633423Z",
     "shell.execute_reply.started": "2024-09-19T12:36:27.629895Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GameSetting(setting=<GameSettingType.cyberpunk: 'cyberpunk'>, description='A dystopian future where technology has advanced exponentially, leading to a world of advanced AI, cybernetic enhancements, and a stark divide between the wealthy and the poor. The setting is often characterized by dark, neon-lit streets, high-tech gadgets, and a gritty, action-packed atmosphere.', id=1)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "67a5f721-2717-4469-976f-ac47cc398d83",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:27.634615Z",
     "iopub.status.busy": "2024-09-19T12:36:27.634446Z",
     "iopub.status.idle": "2024-09-19T12:36:27.661725Z",
     "shell.execute_reply": "2024-09-19T12:36:27.660501Z",
     "shell.execute_reply.started": "2024-09-19T12:36:27.634599Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'A dystopian future where technology has advanced exponentially, leading to a world of advanced AI, cybernetic enhancements, and a stark divide between the wealthy and the poor. The setting is often characterized by dark, neon-lit streets, high-tech gadgets, and a gritty, action-packed atmosphere.'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c4437fbe-266c-4fda-81fa-c20701b1111f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:27.663660Z",
     "iopub.status.busy": "2024-09-19T12:36:27.663151Z",
     "iopub.status.idle": "2024-09-19T12:36:27.680389Z",
     "shell.execute_reply": "2024-09-19T12:36:27.679001Z",
     "shell.execute_reply.started": "2024-09-19T12:36:27.663609Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b934e2b5-25a5-45fd-8b08-75374bc9fe2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-09-19T12:36:27.682948Z",
     "iopub.status.busy": "2024-09-19T12:36:27.682216Z",
     "iopub.status.idle": "2024-09-19T12:36:27.698028Z",
     "shell.execute_reply": "2024-09-19T12:36:27.696609Z",
     "shell.execute_reply.started": "2024-09-19T12:36:27.682869Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<GameSettingType.cyberpunk: 'cyberpunk'>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "response.setting"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}