image validation experiment

experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "from urllib3.exceptions import NotOpenSSLWarning\n",
    "warnings.filterwarnings(\"ignore\", category=NotOpenSSLWarning)\n",
    "\n",
    "import requests\n",
    "from io import BytesIO\n",
    "\n",
    "import os\n",
    "import torch\n",
    "import openai\n",
    "from PIL import Image\n",
    "from transformers import CLIPProcessor, CLIPModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      ".venv/lib/python3.9/site-packages/urllib3/__init__.py:35: NotOpenSSLWarning: urllib3 v2 only supports OpenSSL 1.1.1+, currently the 'ssl' module is compiled with 'LibreSSL 2.8.3'. See: https://github.com/urllib3/urllib3/issues/3020\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Step 1: Load CLIP model and processor\n",
    "model = CLIPModel.from_pretrained(\"openai/clip-vit-base-patch32\")\n",
    "processor = CLIPProcessor.from_pretrained(\"openai/clip-vit-base-patch32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "truth_set_directory = \"truth_set\"\n",
    "\n",
    "truth_images = [\n",
    "    os.path.join(truth_set_directory, f)\n",
    "    for f in os.listdir(truth_set_directory)\n",
    "    if os.path.isfile(os.path.join(truth_set_directory, f))\n",
    "]\n",
    "\n",
    "\n",
    "truth_embeddings = []\n",
    "\n",
    "for image_path in truth_images:\n",
    "    image = Image.open(image_path).convert(\"RGB\")\n",
    "    image = image.resize((1024, 1024)) #3024 X 4032 original\n",
    "    inputs = processor(images=image, return_tensors=\"pt\", padding=True)\n",
    "    with torch.no_grad():\n",
    "        embedding = model.get_image_features(**inputs).squeeze()\n",
    "    truth_embeddings.append(embedding)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average the truth set embeddings\n",
    "truth_set_avg_embedding = torch.stack(truth_embeddings).mean(dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated image URLs:\n",
      "1: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-CjxWzUIl3LmmHIUXjJzHYTKi.png?st=2024-12-15T16%3A30%3A49Z&se=2024-12-15T18%3A30%3A49Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-15T10%3A26%3A25Z&ske=2024-12-16T10%3A26%3A25Z&sks=b&skv=2024-08-04&sig=gkHPbYtbyaDCl32j%2BDNo5ixN6x/DXJQ2D5hzvbb8yuI%3D\n",
      "2: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-iHjkDPaAfgElniB9XJx1njF9.png?st=2024-12-15T16%3A31%3A01Z&se=2024-12-15T18%3A31%3A01Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-15T05%3A20%3A56Z&ske=2024-12-16T05%3A20%3A56Z&sks=b&skv=2024-08-04&sig=7%2BwM9UlY5RGtu5AHOcadDGYUQCZ8tVKmBWFyA%2BXqpGY%3D\n",
      "3: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-GZ1Y8sBS9e7VIIbAZd5Jm1MF.png?st=2024-12-15T16%3A31%3A12Z&se=2024-12-15T18%3A31%3A12Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-15T03%3A50%3A52Z&ske=2024-12-16T03%3A50%3A52Z&sks=b&skv=2024-08-04&sig=9YcvayhlejFHqP1F2lQ2DkpsnQTNz409L2%2B/upLVYKs%3D\n",
      "4: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-v8wr48W37DnQIbL4K8bvZm3K.png?st=2024-12-15T16%3A31%3A27Z&se=2024-12-15T18%3A31%3A27Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-15T03%3A24%3A12Z&ske=2024-12-16T03%3A24%3A12Z&sks=b&skv=2024-08-04&sig=dkHy7eNaYbh7Fhk6sYojBcQRuI4qzstEFGsqbuc%2Bvpo%3D\n",
      "5: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-NOtroSPzqpFEb3K8fz0ZSHey.png?st=2024-12-15T16%3A31%3A49Z&se=2024-12-15T18%3A31%3A49Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-15T03%3A56%3A17Z&ske=2024-12-16T03%3A56%3A17Z&sks=b&skv=2024-08-04&sig=V3FBtVY4kdphgKITj/ZsqGfpJ%2B3qMGBGx4whz9y/4Vw%3D\n",
      "6: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-Zm0jEohrT6w9YB2nrdskkEHJ.png?st=2024-12-15T16%3A31%3A59Z&se=2024-12-15T18%3A31%3A59Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-15T15%3A41%3A27Z&ske=2024-12-16T15%3A41%3A27Z&sks=b&skv=2024-08-04&sig=o2T4OgKG1ALKPdD1VFgdegJqW%2Bojt55fomo%2BMuAAEs0%3D\n",
      "7: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-yO566auNqRJYgocMYNC54Ll9.png?st=2024-12-15T16%3A32%3A12Z&se=2024-12-15T18%3A32%3A12Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-14T20%3A15%3A16Z&ske=2024-12-15T20%3A15%3A16Z&sks=b&skv=2024-08-04&sig=6oX2mezMg9sQ4onOz9Rx1dbsubAIraLtTxlqXd4dvO8%3D\n",
      "8: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-CfJ5FDtmTB6rK5iJMFhTMSPj.png?st=2024-12-15T16%3A32%3A25Z&se=2024-12-15T18%3A32%3A25Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-15T08%3A46%3A54Z&ske=2024-12-16T08%3A46%3A54Z&sks=b&skv=2024-08-04&sig=Ku03sNdrXJ/%2Bc5Y7ockcAUxCNQB8Mhw1cwLEevgQWRc%3D\n",
      "9: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-X96HGWKBsIPbROeVcHrvdhh9.png?st=2024-12-15T16%3A32%3A38Z&se=2024-12-15T18%3A32%3A38Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-15T05%3A46%3A03Z&ske=2024-12-16T05%3A46%3A03Z&sks=b&skv=2024-08-04&sig=0PBJ4uRbZQJXTMegqTc%2B%2B5nVey1VQIIlTHrLtLE9/4Y%3D\n",
      "10: https://oaidalleapiprodscus.blob.core.windows.net/private/org-jXMZj5kjVPIA0aF9jkikgCi6/user-hn35HYMc07e601A7nEXk4LTH/img-YHMaTBRmHpKKAjUpjEBc7Z6n.png?st=2024-12-15T16%3A32%3A52Z&se=2024-12-15T18%3A32%3A52Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=d505667d-d6c1-4a0a-bac7-5c84a87759f8&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2024-12-15T03%3A15%3A43Z&ske=2024-12-16T03%3A15%3A43Z&sks=b&skv=2024-08-04&sig=IwIaPz033sacEyRdZ05/NSss2kMkFCgVEd3vfhxkIWg%3D\n"
     ]
    }
   ],
   "source": [
    "#create test images\n",
    "open_ai_key = \"XXX\"\n",
    "os.environ[\"OPENAI_API_KEY\"] = open_ai_key\n",
    "\n",
    "prompt = (\n",
    "    \"A diverse set of household plumbing pipes with cracks, rust , and other imperfections \"\n",
    "    \"suitable for testing a machine learning model. The images should feature a variety of backgrounds, textures, \"\n",
    "    \"and compositions, with some objects appearing plastic, others metal, others cast-ion, and some PVC.\"\n",
    ")\n",
    "\n",
    "n_images = 10\n",
    "\n",
    "test_image_urls = []\n",
    "for _ in range(n_images):\n",
    "    response = openai.images.generate(prompt=prompt, n=1, size=\"1024x1024\")\n",
    "    image_url = response.data[0].url \n",
    "    test_image_urls.append(image_url)\n",
    "\n",
    "print(\"Generated image URLs:\")\n",
    "for idx, url in enumerate(test_image_urls, 1):\n",
    "    print(f\"{idx}: {url}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_query_embedding(query_image_url):\n",
    "    response = requests.get(query_image_url)\n",
    "    response.raise_for_status()\n",
    "    query_image = Image.open(BytesIO(response.content)).convert(\"RGB\")\n",
    "    query_inputs = processor(images=query_image, return_tensors=\"pt\", padding=True)\n",
    "    with torch.no_grad():\n",
    "        query_embedding = model.get_image_features(**query_inputs).squeeze()\n",
    "    return query_embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_similarity(query_embedding):\n",
    "    cosine_sim = torch.nn.functional.cosine_similarity(query_embedding, truth_set_avg_embedding, dim=0)\n",
    "    threshold = 0.7\n",
    "    is_similar = cosine_sim > threshold\n",
    "    return is_similar"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query image is similar to the truth set: True\n",
      "Query image is similar to the truth set: True\n",
      "Query image is similar to the truth set: False\n",
      "Query image is similar to the truth set: True\n",
      "Query image is similar to the truth set: True\n",
      "Query image is similar to the truth set: True\n",
      "Query image is similar to the truth set: False\n",
      "Query image is similar to the truth set: True\n",
      "Query image is similar to the truth set: True\n",
      "Query image is similar to the truth set: False\n"
     ]
    }
   ],
   "source": [
    "for test_image_url in test_image_urls:\n",
    "    query_embedding = get_query_embedding(test_image_url)\n",
    "    is_similar = get_similarity(query_embedding)\n",
    "    print(f\"Query image is similar to the truth set: {is_similar.item()}\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}