controlnetdiffusersdevelopment.ipynb

controlnetdiffusersdevelopment.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "private_outputs": true,
      "provenance": [],
      "machine_shape": "hm",
      "authorship_tag": "ABX9TyNrZLCPqNK71uDuzCmDmvBJ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/takuma104/7517f0b50a1840ff6e86340a31f7872d/controlnetdiffusersdevelopment.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Install Diffusers in development version and dependencies.\n",
        "and import libs"
      ],
      "metadata": {
        "id": "hnugbDmHBid-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "SHy4vGktxP6F"
      },
      "outputs": [],
      "source": [
        "! pip install \"git+https://github.com/takuma104/diffusers.git@controlnet\" # Diffusers in development version \n",
        "! pip install transformers accelerate safetensors xformers opencv-python"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from diffusers import StableDiffusionControlNetPipeline, EulerAncestralDiscreteScheduler\n",
        "from diffusers.utils import load_image\n",
        "import torch\n",
        "import cv2\n",
        "import numpy as np\n",
        "from PIL import Image\n"
      ],
      "metadata": {
        "id": "VTxt5Dmg9E0t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Common "
      ],
      "metadata": {
        "id": "8HjYAvVCL-Ph"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "euler_scheduler = EulerAncestralDiscreteScheduler.from_config(\"takuma104/control_sd15_canny\", subfolder=\"scheduler\")"
      ],
      "metadata": {
        "id": "MjcNfWrD_-ss"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Canny Edge model"
      ],
      "metadata": {
        "id": "w6hYYszZBqzf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pipe_canny = StableDiffusionControlNetPipeline.from_pretrained(\"takuma104/control_sd15_canny\", torch_dtype=torch.float16).to(\"cuda\")\n",
        "pipe_canny.scheduler = euler_scheduler\n",
        "pipe_canny.enable_xformers_memory_efficient_attention()\n"
      ],
      "metadata": {
        "id": "PTqP_XCN9HV0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Control by preprocessed image"
      ],
      "metadata": {
        "id": "KmqYshFNMKfx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "canny_edged_image = load_image(\"https://huggingface.co/takuma104/controlnet_dev/resolve/main/vermeer_canny_edged.png\")\n",
        "canny_edged_image"
      ],
      "metadata": {
        "id": "vfMEVnkhxxfZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "generator = torch.Generator(device=\"cpu\").manual_seed(3)\n",
        "image = pipe_canny(prompt=\"best quality, extremely detailed\", \n",
        "                   negative_prompt=\"monochrome, lowres, bad anatomy, worst quality, low quality\",\n",
        "                   controlnet_hint=canny_edged_image,\n",
        "                   num_inference_steps=30, \n",
        "                   generator=generator).images[0]\n",
        "image"
      ],
      "metadata": {
        "id": "joNSeXop_KjN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### Control by generated image"
      ],
      "metadata": {
        "id": "v7QTIQdgMV_y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "original_image = load_image(\"https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_imgvar/input_image_vermeer.png\")\n",
        "original_image"
      ],
      "metadata": {
        "id": "9O0hCaQGMVus"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "control = cv2.Canny(np.array(original_image), threshold1=100, threshold2=200)\n",
        "#control = cv2.cvtColor(control, cv2.COLOR_GRAY2RGB)\n",
        "Image.fromarray(control)"
      ],
      "metadata": {
        "id": "2TuyVDb2MVsK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "control.shape"
      ],
      "metadata": {
        "id": "-YEeqGqcQRNe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "generator = torch.Generator(device=\"cpu\").manual_seed(3)\n",
        "image = pipe_canny(prompt=\"best quality, extremely detailed\", \n",
        "                   negative_prompt=\"monochrome, lowres, bad anatomy, worst quality, low quality\",\n",
        "                   controlnet_hint=control,\n",
        "                   num_inference_steps=30, \n",
        "                   generator=generator).images[0]\n",
        "image"
      ],
      "metadata": {
        "id": "3-BogkxdP74V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## OpenPose model"
      ],
      "metadata": {
        "id": "ON-MLTZCBcf9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pose_image = load_image('https://huggingface.co/takuma104/controlnet_dev/resolve/main/pose.png')\n",
        "pose_image"
      ],
      "metadata": {
        "id": "WkrgAzpSyfHj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pipe_op = StableDiffusionControlNetPipeline.from_pretrained(\"takuma104/control_sd15_openpose\", torch_dtype=torch.float16).to(\"cuda\")\n",
        "pipe_op.scheduler = euler_scheduler\n",
        "pipe_op.enable_xformers_memory_efficient_attention()\n"
      ],
      "metadata": {
        "id": "uxNpbFh79BXw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "generator = torch.Generator(device=\"cpu\").manual_seed(0)\n",
        "image = pipe_op(prompt=\"best quality, extremely detailed, football, a boy\", \n",
        "                negative_prompt=\"lowres, bad anatomy, worst quality, low quality\",\n",
        "                controlnet_hint=pose_image, \n",
        "                generator=generator,\n",
        "                num_inference_steps=30).images[0]\n",
        "image"
      ],
      "metadata": {
        "id": "B5nYufyO9dzJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ZZ6fyLAB_Iw3"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}