MultiDiffusion_Panorama_withQoLUpdatesByNorod78.ipynb

multidiffusion_panorama_withqolupdatesbynorod78.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/Norod/abe291445bc5f70be2629a3414a2e005/multidiffusion_panorama_withqolupdatesbynorod78.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HWT5Q9c_qJ6v"
      },
      "source": [
        "Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "IIbLF70Rp-Dt"
      },
      "outputs": [],
      "source": [
        "!pip install diffusers\n",
        "!pip install transformers\n",
        "!pip install accelerate\n",
        "!pip install exif"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cXXNWiTAqWNS"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from diffusers import StableDiffusionPanoramaPipeline, DDIMScheduler\n",
        "\n",
        "#model_ckpt = \"stabilityai/stable-diffusion-2-1-base\"\n",
        "model_ckpt = \"Norod78/sd2-simpsons-blip\"\n",
        "#model_ckpt = \"Norod78/sd15-arthur-blip-captions\"\n",
        "#model_ckpt = \"Norod78/SD15-VinageStyle\"\n",
        "#model_ckpt = \"runwayml/stable-diffusion-v1-5\"\n",
        "scheduler = DDIMScheduler.from_pretrained(model_ckpt, subfolder=\"scheduler\")\n",
        "pipe = StableDiffusionPanoramaPipeline.from_pretrained(\n",
        "     model_ckpt, scheduler=scheduler, torch_dtype=torch.float16\n",
        ")\n",
        "\n",
        "pipe = pipe.to(\"cuda\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8CZuMAP5sh0l"
      },
      "source": [
        "Create a Panorama!"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KXkdF7HOs77b"
      },
      "source": [
        "For more details, please visit their [project page](https://multidiffusion.github.io/) and their [paper](https://arxiv.org/abs/2302.08113)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rjwy0QZHsflu"
      },
      "outputs": [],
      "source": [
        "H = 512\n",
        "W = 2048\n",
        "STEPS = 50\n",
        "SEED = 42\n",
        "\n",
        "torch.manual_seed(SEED)\n",
        "\n",
        "#prompt = \"a photo of the dolomites\"\n",
        "prompt = \"a photo of the surface of an alien planet in The Simpsons\"\n",
        "image = pipe(prompt, height=H, width=W, num_inference_steps=STEPS).images[0]\n",
        "image"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b741-dbksxYB"
      },
      "source": [
        "Save the result"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2O10IhwBqdwS"
      },
      "outputs": [],
      "source": [
        "import exif\n",
        "import cv2\n",
        "import numpy as np\n",
        "\n",
        "file_name = f'{prompt.replace(\" \", \"_\")}_{H}x{W}_{STEPS}_{SEED}_{model_ckpt.split(\"/\")[-1]}.jpg'\n",
        "\n",
        "npimg = np.array(image)[...,::-1]\n",
        "print(npimg.shape)\n",
        "#npimg.transpose(0, 1, 2, 3)\n",
        "\n",
        "# encode to the appropriate format\n",
        "status, image_jpg_coded = cv2.imencode('.jpg', npimg)\n",
        "print('successful jpg encoding: %s' % status)\n",
        "\n",
        "# to a byte string\n",
        "image_jpg_coded_bytes = image_jpg_coded.tobytes()\n",
        "\n",
        "# using the exif format to add information\n",
        "exif_jpg = exif.Image(image_jpg_coded_bytes)\n",
        "\n",
        "# providing some information\n",
        "user_comment = f'{prompt} {H}x{W} {STEPS} steps {model_ckpt.split(\"/\")[-1]}'\n",
        "software = 'MultiDiffusion Panorama'\n",
        "\n",
        "# adding information to exif files:\n",
        "exif_jpg[\"software\"]  = software\n",
        "exif_jpg[\"user_comment\"] = user_comment\n",
        "exif_jpg.set(\"ProjectionType\", \"equirectangular\")\n",
        "exif_jpg.set(\"CroppedAreaImageWidthPixels\", W)\n",
        "exif_jpg.set(\"CroppedAreaImageHeightPixels\", H)\n",
        "exif_jpg.set(\"FullPanoWidthPixels\", W)\n",
        "exif_jpg.set(\"FullPanoHeightPixels\", H)\n",
        "exif_jpg.set(\"CroppedAreaLeftPixels\", 0)\n",
        "exif_jpg.set(\"CroppedAreaTopPixels\", 0)\n",
        "exif_jpg.set(\"UsePanoramaViewer\", \"True\")\n",
        "exif_jpg.set(\"PoseHeadingDegrees\", 0)\n",
        "\n",
        "# save image\n",
        "with open(file_name, 'wb') as new_image_file:\n",
        "    new_image_file.write(exif_jpg.get_file())\n"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "private_outputs": true,
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.13"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}