Skip to content

Instantly share code, notes, and snippets.

@richard087

richard087/phi3-strictjson-test.ipynb

Last active May 1, 2024 05:50
Show Gist options
  • Save richard087/8e5d5ec4442da4dd4bdeec2a69c72caf to your computer and use it in GitHub Desktop.
Save richard087/8e5d5ec4442da4dd4bdeec2a69c72caf to your computer and use it in GitHub Desktop.
Download ZIP
phi3-strictjson-test.ipynb
Display the source blob
Display the rendered blob
Raw
phi3-strictjson-test.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"provenance": [],
"gpuType": "T4",
"authorship_tag": "ABX9TyPfEpD86V36GMQICU+HPxOc",
"include_colab_link": true
},
"kernelspec": {
"name": "python3",
"display_name": "Python 3"
},
"language_info": {
"name": "python"
},
"accelerator": "GPU",
"widgets": {
"application/vnd.jupyter.widget-state+json": {
"ca4e1f29f3de404999df2e2c5d5f8b06": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HBoxModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HBoxModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HBoxView",
"box_style": "",
"children": [
"IPY_MODEL_953fe0f55bc84b6dbd24a54d6d9dac8c",
"IPY_MODEL_feac8d2604fa4bc28b06217652951451",
"IPY_MODEL_161119c9a40d454ba18fec5c0bac8e8b"
],
"layout": "IPY_MODEL_540cbc7bf8154788b8fce224ca98a960"
}
},
"953fe0f55bc84b6dbd24a54d6d9dac8c": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_93d031e6944244e3b37488b783de3b76",
"placeholder": "​",
"style": "IPY_MODEL_ee15f16d7e334ab78ad6b726856d725e",
"value": "Loading checkpoint shards: 100%"
}
},
"feac8d2604fa4bc28b06217652951451": {
"model_module": "@jupyter-widgets/controls",
"model_name": "FloatProgressModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "FloatProgressModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "ProgressView",
"bar_style": "success",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_b8f0ed5a84514de3bac13cfd8ccac15e",
"max": 2,
"min": 0,
"orientation": "horizontal",
"style": "IPY_MODEL_6a780c1615164b86bb12253c7ace3d57",
"value": 2
}
},
"161119c9a40d454ba18fec5c0bac8e8b": {
"model_module": "@jupyter-widgets/controls",
"model_name": "HTMLModel",
"model_module_version": "1.5.0",
"state": {
"_dom_classes": [],
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "HTMLModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/controls",
"_view_module_version": "1.5.0",
"_view_name": "HTMLView",
"description": "",
"description_tooltip": null,
"layout": "IPY_MODEL_a3d564541c3f4d42b4329d2e31a62567",
"placeholder": "​",
"style": "IPY_MODEL_05fbb529bc3e4d1c87aeabe87c41a8ad",
"value": " 2/2 [00:29<00:00, 14.22s/it]"
}
},
"540cbc7bf8154788b8fce224ca98a960": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"93d031e6944244e3b37488b783de3b76": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"ee15f16d7e334ab78ad6b726856d725e": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
},
"b8f0ed5a84514de3bac13cfd8ccac15e": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"6a780c1615164b86bb12253c7ace3d57": {
"model_module": "@jupyter-widgets/controls",
"model_name": "ProgressStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "ProgressStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"bar_color": null,
"description_width": ""
}
},
"a3d564541c3f4d42b4329d2e31a62567": {
"model_module": "@jupyter-widgets/base",
"model_name": "LayoutModel",
"model_module_version": "1.2.0",
"state": {
"_model_module": "@jupyter-widgets/base",
"_model_module_version": "1.2.0",
"_model_name": "LayoutModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "LayoutView",
"align_content": null,
"align_items": null,
"align_self": null,
"border": null,
"bottom": null,
"display": null,
"flex": null,
"flex_flow": null,
"grid_area": null,
"grid_auto_columns": null,
"grid_auto_flow": null,
"grid_auto_rows": null,
"grid_column": null,
"grid_gap": null,
"grid_row": null,
"grid_template_areas": null,
"grid_template_columns": null,
"grid_template_rows": null,
"height": null,
"justify_content": null,
"justify_items": null,
"left": null,
"margin": null,
"max_height": null,
"max_width": null,
"min_height": null,
"min_width": null,
"object_fit": null,
"object_position": null,
"order": null,
"overflow": null,
"overflow_x": null,
"overflow_y": null,
"padding": null,
"right": null,
"top": null,
"visibility": null,
"width": null
}
},
"05fbb529bc3e4d1c87aeabe87c41a8ad": {
"model_module": "@jupyter-widgets/controls",
"model_name": "DescriptionStyleModel",
"model_module_version": "1.5.0",
"state": {
"_model_module": "@jupyter-widgets/controls",
"_model_module_version": "1.5.0",
"_model_name": "DescriptionStyleModel",
"_view_count": null,
"_view_module": "@jupyter-widgets/base",
"_view_module_version": "1.2.0",
"_view_name": "StyleView",
"description_width": ""
}
}
}
}
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/richard087/8e5d5ec4442da4dd4bdeec2a69c72caf/phi3-strictjson-test.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"source": [
"res = !pip show accelerate 2>&1 | grep \"Package(s) not found: \"\n",
"if res.spstr != \"\":\n",
" !pip install accelerate\n",
" print('Stopping RUNTIME! Please run again.')\n",
" import os\n",
" os.kill(os.getpid(), 9)\n",
"else:\n",
" print(\"Already installed accelerate\")"
],
"metadata": {
"id": "cpgfR5snaqw2",
"outputId": "2426e674-f6ad-4ae8-d2f0-1949c9441965",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"execution_count": 1,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Already installed accelerate\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"!pip install transformers torch huggingface_hub[hf_transfer] taskgen-ai"
],
"metadata": {
"id": "x1Rti5taaK4b",
"outputId": "0b6270d5-5c77-4e7c-bd28-ef499ce154c0",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"execution_count": 2,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Requirement already satisfied: transformers in /usr/local/lib/python3.10/dist-packages (4.40.1)\n",
"Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (2.2.1+cu121)\n",
"Requirement already satisfied: huggingface_hub[hf_transfer] in /usr/local/lib/python3.10/dist-packages (0.20.3)\n",
"Requirement already satisfied: taskgen-ai in /usr/local/lib/python3.10/dist-packages (1.3.2)\n",
"Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.13.4)\n",
"Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.25.2)\n",
"Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (24.0)\n",
"Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
"Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.12.25)\n",
"Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
"Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.19.1)\n",
"Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.4.3)\n",
"Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.2)\n",
"Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch) (4.11.0)\n",
"Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch) (1.12)\n",
"Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch) (3.3)\n",
"Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch) (3.1.3)\n",
"Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch) (2023.6.0)\n",
"Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
"Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
"Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
"Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch) (8.9.2.26)\n",
"Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.3.1)\n",
"Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch) (11.0.2.54)\n",
"Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch) (10.3.2.106)\n",
"Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch) (11.4.5.107)\n",
"Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.0.106)\n",
"Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch) (2.19.3)\n",
"Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch) (12.1.105)\n",
"Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch) (2.2.0)\n",
"Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch) (12.4.127)\n",
"\u001b[33mWARNING: huggingface-hub 0.20.3 does not provide the extra 'hf_transfer'\u001b[0m\u001b[33m\n",
"\u001b[0mRequirement already satisfied: openai>=1.3.6 in /usr/local/lib/python3.10/dist-packages (from taskgen-ai) (1.25.0)\n",
"Requirement already satisfied: dill>=0.3.7 in /usr/local/lib/python3.10/dist-packages (from taskgen-ai) (0.3.8)\n",
"Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.3.6->taskgen-ai) (3.7.1)\n",
"Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>=1.3.6->taskgen-ai) (1.7.0)\n",
"Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.3.6->taskgen-ai) (0.27.0)\n",
"Requirement already satisfied: pydantic<3,>=1.9.0 in /usr/local/lib/python3.10/dist-packages (from openai>=1.3.6->taskgen-ai) (2.7.1)\n",
"Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>=1.3.6->taskgen-ai) (1.3.1)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch) (2.1.5)\n",
"Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.3.2)\n",
"Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.7)\n",
"Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2.0.7)\n",
"Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2024.2.2)\n",
"Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch) (1.3.0)\n",
"Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>=1.3.6->taskgen-ai) (1.2.1)\n",
"Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>=1.3.6->taskgen-ai) (1.0.5)\n",
"Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.3.6->taskgen-ai) (0.14.0)\n",
"Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.3.6->taskgen-ai) (0.6.0)\n",
"Requirement already satisfied: pydantic-core==2.18.2 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1.9.0->openai>=1.3.6->taskgen-ai) (2.18.2)\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"!pip install packaging ninja wheel setuptools # build tools to support flash-attn\n",
"!pip install flash-attn --no-build-isolation"
],
"metadata": {
"id": "0zAlW2BEbtIu",
"outputId": "d04a53d1-b554-4de4-f2ad-2eb285d3f843",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"execution_count": 3,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (24.0)\n",
"Requirement already satisfied: ninja in /usr/local/lib/python3.10/dist-packages (1.11.1.1)\n",
"Requirement already satisfied: wheel in /usr/local/lib/python3.10/dist-packages (0.43.0)\n",
"Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (67.7.2)\n",
"Requirement already satisfied: flash-attn in /usr/local/lib/python3.10/dist-packages (2.5.8)\n",
"Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from flash-attn) (2.2.1+cu121)\n",
"Requirement already satisfied: einops in /usr/local/lib/python3.10/dist-packages (from flash-attn) (0.8.0)\n",
"Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from flash-attn) (24.0)\n",
"Requirement already satisfied: ninja in /usr/local/lib/python3.10/dist-packages (from flash-attn) (1.11.1.1)\n",
"Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (3.13.4)\n",
"Requirement already satisfied: typing-extensions>=4.8.0 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (4.11.0)\n",
"Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (1.12)\n",
"Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (3.3)\n",
"Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (3.1.3)\n",
"Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (2023.6.0)\n",
"Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.105)\n",
"Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.105)\n",
"Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.105)\n",
"Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (8.9.2.26)\n",
"Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.3.1)\n",
"Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (11.0.2.54)\n",
"Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (10.3.2.106)\n",
"Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (11.4.5.107)\n",
"Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.0.106)\n",
"Requirement already satisfied: nvidia-nccl-cu12==2.19.3 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (2.19.3)\n",
"Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (12.1.105)\n",
"Requirement already satisfied: triton==2.2.0 in /usr/local/lib/python3.10/dist-packages (from torch->flash-attn) (2.2.0)\n",
"Requirement already satisfied: nvidia-nvjitlink-cu12 in /usr/local/lib/python3.10/dist-packages (from nvidia-cusolver-cu12==11.4.5.107->torch->flash-attn) (12.4.127)\n",
"Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch->flash-attn) (2.1.5)\n",
"Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->flash-attn) (1.3.0)\n"
]
}
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {
"colab": {
"base_uri": "https://localhost:8080/",
"height": 66,
"referenced_widgets": [
"ca4e1f29f3de404999df2e2c5d5f8b06",
"953fe0f55bc84b6dbd24a54d6d9dac8c",
"feac8d2604fa4bc28b06217652951451",
"161119c9a40d454ba18fec5c0bac8e8b",
"540cbc7bf8154788b8fce224ca98a960",
"93d031e6944244e3b37488b783de3b76",
"ee15f16d7e334ab78ad6b726856d725e",
"b8f0ed5a84514de3bac13cfd8ccac15e",
"6a780c1615164b86bb12253c7ace3d57",
"a3d564541c3f4d42b4329d2e31a62567",
"05fbb529bc3e4d1c87aeabe87c41a8ad"
]
},
"id": "dj-5OJjlaCkP",
"outputId": "e20eb3af-7fd0-46b3-d6c6-8b95ceab11c8"
},
"outputs": [
{
"output_type": "display_data",
"data": {
"text/plain": [
"Loading checkpoint shards: 0%| | 0/2 [00:00<?, ?it/s]"
],
"application/vnd.jupyter.widget-view+json": {
"version_major": 2,
"version_minor": 0,
"model_id": "ca4e1f29f3de404999df2e2c5d5f8b06"
}
},
"metadata": {}
},
{
"output_type": "stream",
"name": "stderr",
"text": [
"Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
]
}
],
"source": [
"from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline\n",
"from taskgen import strict_json\n",
"\n",
"model_name = \"microsoft/Phi-3-mini-4k-instruct\"\n",
"#model_name = \"microsoft/Phi-3-mini-128k-instruct\" #same result but takes longer\n",
"\n",
"model = AutoModelForCausalLM.from_pretrained(\n",
" model_name,\n",
" device_map=\"cuda\",\n",
" torch_dtype=\"auto\",\n",
" trust_remote_code=True,\n",
")\n",
"tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
"\n",
"pipe = pipeline(\n",
" \"text-generation\",\n",
" model=model,\n",
" tokenizer=tokenizer,\n",
")\n",
"\n",
"generation_args = {\n",
" \"max_new_tokens\": 200,\n",
" \"return_full_text\": False,\n",
" #\"temperature\": 0.0,\n",
" #\"do_sample\": False,\n",
"}"
]
},
{
"cell_type": "code",
"source": [
"def test():\n",
" messages = [\n",
" {\"role\": \"user\", \"content\": \"Can you provide ways to eat combinations of bananas and dragonfruits?\"},\n",
" {\"role\": \"assistant\", \"content\": \"Sure! Here are some ways to eat bananas and dragonfruits together: 1. Banana and dragonfruit smoothie: Blend bananas and dragonfruits together with some milk and honey. 2. Banana and dragonfruit salad: Mix sliced bananas and dragonfruits together with some lemon juice and honey.\"},\n",
" {\"role\": \"user\", \"content\": \"What about solving an 2x + 3 = 7 equation?\"},\n",
" ]\n",
" output = pipe(messages, **generation_args)\n",
" res = output[0]['generated_text']\n",
" return res\n",
"\n",
"print(test())"
],
"metadata": {
"id": "LsgfjyH-sZVe",
"outputId": "94813fde-2845-4af0-e720-53956ebdb66d",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"execution_count": 8,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"To solve the equation 2x + 3 = 7, follow these steps:\n",
"\n",
"1. Subtract 3 from both sides of the equation: 2x + 3 - 3 = 7 - 3\n",
"2. Simplify: 2x = 4\n",
"3. Divide both sides by 2: 2x/2 = 4/2\n",
"4. Simplify: x = 2\n",
"\n",
"So, the solution to the equation 2x + 3 = 7 is x = 2.\n"
]
}
]
},
{
"cell_type": "code",
"source": [
"def llm(system_prompt: str, user_prompt: str):\n",
" messages = [{\"role\": \"system\", \"content\": system_prompt}, {\"role\": \"user\", \"content\": user_prompt}]\n",
" output = pipe(messages, **generation_args)\n",
" res = output[0]['generated_text']\n",
" return res\n",
"\n",
"res = strict_json(system_prompt = 'You are a classifier',\n",
" user_prompt = 'It is a beautiful and sunny day',\n",
" output_format = {'Sentiment': 'Type of Sentiment',\n",
" 'Adjectives': 'Array of adjectives',\n",
" 'Words': 'Number of words'},\n",
" debug=True,\n",
" llm = llm) # set this to your own LLM\n",
"print(res)"
],
"metadata": {
"id": "1PjO-xEGsZZE",
"outputId": "645cfea1-9b38-40cb-9188-6e7fe93750e1",
"colab": {
"base_uri": "https://localhost:8080/"
}
},
"execution_count": 9,
"outputs": [
{
"output_type": "stream",
"name": "stdout",
"text": [
"An exception occurred: string index out of range\n",
"Current invalid json format: \n",
"An exception occurred: string index out of range\n",
"Current invalid json format: \n",
"An exception occurred: string index out of range\n",
"Current invalid json format: \n",
"{}\n"
]
}
]
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment