ollama_colab_ngrok.ipynb

ollama_colab_ngrok.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/flaviodelgrosso/028777f03591be9903de5b2d27725dd9/ollama_colab_ngrok.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Download and install ollama to the system\n",
        "!curl -fsSL https://ollama.com/install.sh | sed 's#https://ollama.com/download#https://github.com/jmorganca/ollama/releases/download/v0.1.27#' | sh\n",
        "\n",
        "!pip install aiohttp pyngrok\n",
        "\n",
        "import threading\n",
        "import time\n",
        "import os\n",
        "import asyncio\n",
        "import queue\n",
        "\n",
        "from pyngrok import ngrok\n",
        "from threading import Thread\n",
        "\n",
        "# Get your ngrok token from your ngrok account:\n",
        "# https://dashboard.ngrok.com/get-started/your-authtoken\n",
        "token=\"YOUR_NGROK_TOKEN\"\n",
        "ngrok.set_auth_token(token)\n",
        "\n",
        "# set up a stoppable thread (not mandatory, but cleaner if you want to stop this later\n",
        "class StoppableThread(threading.Thread):\n",
        "    def __init__(self, *args, **kwargs):\n",
        "        super(StoppableThread, self).__init__(*args, **kwargs)\n",
        "        self._stop_event = threading.Event()\n",
        "\n",
        "    def stop(self):\n",
        "        self._stop_event.set()\n",
        "\n",
        "    def is_stopped(self):\n",
        "        return self._stop_event.is_set()\n",
        "\n",
        "def start_ngrok(q, stop_event):\n",
        "    try:\n",
        "        # Start an HTTP tunnel on the specified port\n",
        "        public_url = ngrok.connect(11434)\n",
        "        # Put the public URL in the queue\n",
        "        q.put(public_url)\n",
        "        # Keep the thread alive until stop event is set\n",
        "        while not stop_event.is_set():\n",
        "            time.sleep(1)  # Adjust sleep time as needed\n",
        "    except Exception as e:\n",
        "        print(f\"Error in start_ngrok: {e}\")\n",
        "\n",
        "\n",
        "# Create a queue to share data between threads\n",
        "url_queue = queue.Queue()\n",
        "\n",
        "# Start ngrok in a separate thread\n",
        "ngrok_thread = StoppableThread(target=start_ngrok, args=(url_queue, StoppableThread.is_stopped))\n",
        "ngrok_thread.start()\n",
        "\n",
        "# Wait for the ngrok tunnel to be established\n",
        "while True:\n",
        "    try:\n",
        "        public_url = url_queue.get()\n",
        "        if public_url:\n",
        "            break\n",
        "        print(\"Waiting for ngrok URL...\")\n",
        "        time.sleep(1)\n",
        "    except Exception as e:\n",
        "        print(f\"Error in retrieving ngrok URL: {e}\")\n",
        "\n",
        "print(\"Ngrok tunnel established at:\", public_url)\n",
        "\n",
        "async def run_process(cmd):\n",
        "    print('>>> starting', *cmd)\n",
        "    process = await asyncio.create_subprocess_exec(\n",
        "        *cmd,\n",
        "        stdout=asyncio.subprocess.PIPE,\n",
        "        stderr=asyncio.subprocess.PIPE\n",
        "    )\n",
        "\n",
        "    # define an async pipe function\n",
        "    async def pipe(lines):\n",
        "        async for line in lines:\n",
        "            print(line.decode().strip())\n",
        "\n",
        "        await asyncio.gather(\n",
        "            pipe(process.stdout),\n",
        "            pipe(process.stderr),\n",
        "        )\n",
        "\n",
        "    # call it\n",
        "    await asyncio.gather(pipe(process.stdout), pipe(process.stderr))\n",
        "\n",
        "async def start_ollama_serve():\n",
        "    await run_process(['ollama', 'serve'])\n",
        "\n",
        "def run_async_in_thread(loop, coro):\n",
        "    asyncio.set_event_loop(loop)\n",
        "    loop.run_until_complete(coro)\n",
        "    loop.close()\n",
        "\n",
        "# Create a new event loop that will run in a new thread\n",
        "new_loop = asyncio.new_event_loop()\n",
        "\n",
        "# Start ollama serve in a separate thread so the cell won't block execution\n",
        "thread = threading.Thread(target=run_async_in_thread, args=(new_loop, start_ollama_serve()))\n",
        "thread.start()"
      ],
      "metadata": {
        "id": "WiIWYU4_UMhf"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "L4",
      "provenance": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}