LipSync.ipynb

lipsync.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyMsnUd6pM0LWrH5ETtHbyRA",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/hathibelagal-dev/fe393508fd1f3dd9d2c3066564325a7b/lipsync.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/bytedance/LatentSync.git"
      ],
      "metadata": {
        "id": "aercf7KTTSoF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%cd LatentSync\n",
        "!pip install -r requirements.txt"
      ],
      "metadata": {
        "id": "QX2X7H7jWPgj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -U accelerate==0.32.0"
      ],
      "metadata": {
        "id": "j7e1tPGYEKb4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!mkdir -p checkpoints"
      ],
      "metadata": {
        "id": "IC6b_b0KTnUy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!huggingface-cli download ByteDance/LatentSync-1.5 whisper/tiny.pt --local-dir checkpoints\n",
        "!huggingface-cli download ByteDance/LatentSync-1.5 latentsync_unet.pt --local-dir checkpoints"
      ],
      "metadata": {
        "id": "i2HIcr4-TYm2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls .."
      ],
      "metadata": {
        "id": "Qrb3BDr0YQom"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!python -m scripts.inference \\\n",
        "    --unet_config_path \"configs/unet/stage2.yaml\" \\\n",
        "    --inference_ckpt_path \"checkpoints/latentsync_unet.pt\" \\\n",
        "    --inference_steps 18 \\\n",
        "    --guidance_scale 2.0 \\\n",
        "    --video_path \"../i2--animation_driver.mp4\" \\\n",
        "    --audio_path \"../speech.wav\" \\\n",
        "    --video_out_path \"../out.mp4\""
      ],
      "metadata": {
        "id": "pgHUsbAVVHws"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}