openpose+video2bvh.ipynb

openpose-video2bvh.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "openpose+video2bvh.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [],
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/gist/eyaler/886ecdff6429ef9cd585d5dc8fdd51ec/openpose-video2bvh.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FOdkDhb6ga6N"
      },
      "source": [
        "# openpose setup - taken from https://colab.research.google.com/github/Kasidit0052/MatchingOpenpose/blob/main/OpenPose.ipynb\n",
        "# note this can take 10 minutes +\n",
        "\n",
        "%cd /content\n",
        "from os.path import exists, join, basename, splitext\n",
        "git_repo_url = 'https://github.com/CMU-Perceptual-Computing-Lab/openpose.git'\n",
        "project_name = splitext(basename(git_repo_url))[0]\n",
        "if not exists(project_name):\n",
        "  # see: https://github.com/CMU-Perceptual-Computing-Lab/openpose/issues/949\n",
        "  # install new CMake becaue of CUDA10\n",
        "  !wget -q https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.tar.gz\n",
        "  !tar xfz cmake-3.13.0-Linux-x86_64.tar.gz --strip-components=1 -C /usr/local\n",
        "  # clone openpose\n",
        "  !git clone -q --depth 1 --branch v1.6.0 $git_repo_url\n",
        "  !sed -i 's/execute_process(COMMAND git checkout master WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}\\/3rdparty\\/caffe)/execute_process(COMMAND git checkout f019d0dfe86f49d1140961f8c7dec22130c83154 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}\\/3rdparty\\/caffe)/g' openpose/CMakeLists.txt\n",
        "  # install system dependencies\n",
        "  !apt-get -qq install -y libatlas-base-dev libprotobuf-dev libleveldb-dev libsnappy-dev libhdf5-serial-dev protobuf-compiler libgflags-dev libgoogle-glog-dev liblmdb-dev opencl-headers ocl-icd-opencl-dev libviennacl-dev\n",
        "  # build openpose\n",
        "  !cd openpose && rm -rf build || true && mkdir build && cd build && cmake -DBUILD_PYTHON=ON .. && make -j`nproc`\n",
        "\n",
        "!apt install imagemagick\n",
        "\n",
        "!git clone -q --depth 1 https://github.com/KevinLTT/video2bvh"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZIhQA8Lf5qpH"
      },
      "source": [
        "# video2bvh demo from https://colab.research.google.com/github/KevinLTT/video2bvh/blob/master/demo.ipynb \n",
        "\n",
        "# Set Python Openpose Directory for python api (Important)\n",
        "import sys\n",
        "pyopenpose_dir = '/content/openpose/build/python'\n",
        "if pyopenpose_dir not in sys.path:\n",
        "    sys.path.append(pyopenpose_dir)\n",
        "from openpose import pyopenpose as op\n",
        "\n",
        "%cd /content/video2bvh\n",
        "from pose_estimator_2d.estimator_2d import Estimator2D\n",
        "\n",
        "class OpenPoseEstimator(Estimator2D):\n",
        "\n",
        "    def __init__(self, model_folder):\n",
        "        \"\"\"\n",
        "        OpenPose 2D pose estimator. See [https://github.com/\n",
        "        CMU-Perceptual-Computing-Lab/openpose/tree/ master/examples/\n",
        "        tutorial_api_python] for help.\n",
        "        Args: \n",
        "        \"\"\"\n",
        "        super().__init__()\n",
        "        params = {'model_folder': model_folder, 'render_pose': 0}\n",
        "        self.opWrapper = op.WrapperPython()\n",
        "        self.opWrapper.configure(params)\n",
        "        self.opWrapper.start()\n",
        "\n",
        "    def estimate(self, img_list, bbox_list=None):\n",
        "        \"\"\"See base class.\"\"\"\n",
        "        keypoints_list = []\n",
        "        for i, img in enumerate(img_list):\n",
        "            if bbox_list:\n",
        "                x, y, w, h = bbox_list[i]\n",
        "                img = img[y:y+h, x:x+w]\n",
        "            datum = op.Datum()\n",
        "            datum.cvInputData = img\n",
        "            self.opWrapper.emplaceAndPop([datum])\n",
        "            keypoints = datum.poseKeypoints\n",
        "            if bbox_list:\n",
        "                # TODO: restore coordinate\n",
        "                pass\n",
        "            keypoints_list.append(datum.poseKeypoints)\n",
        "\n",
        "        return keypoints_list\n",
        "\n",
        "from pose_estimator_3d import estimator_3d\n",
        "from utils import smooth, vis, camera\n",
        "from bvh_skeleton import openpose_skeleton, h36m_skeleton, cmu_skeleton\n",
        "\n",
        "import cv2\n",
        "import importlib\n",
        "import numpy as np\n",
        "import os\n",
        "from pathlib import Path\n",
        "from IPython.display import HTML"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K89boUgdHm49"
      },
      "source": [
        "## Initialize 2d pose estimator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jn_kJtgBHm4_"
      },
      "source": [
        "# more 2d pose estimators like HRNet, PoseResNet, CPN, etc., will be added later\n",
        "e2d = OpenPoseEstimator(model_folder='/content/openpose/models/') # set model_folder to /path/to/openpose/models"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PAPD741WHm4_"
      },
      "source": [
        "## Estimate 2D pose from video"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "89uKXH4LHm5A"
      },
      "source": [
        "video_file = Path('miscs/cxk.mp4') # video file to process\n",
        "output_dir = Path(f'miscs/{video_file.stem}_cache')\n",
        "if not output_dir.exists():\n",
        "    os.makedirs(output_dir)\n",
        "    \n",
        "cap = cv2.VideoCapture(str(video_file))\n",
        "keypoints_list = []\n",
        "img_width, img_height = None, None\n",
        "while True:\n",
        "    ret, frame = cap.read()\n",
        "    if not ret:\n",
        "        break\n",
        "    img_height = frame.shape[0]\n",
        "    img_width = frame.shape[1]\n",
        "    \n",
        "    # returned shape will be (num_of_human, 25, 3)\n",
        "    # last dimension includes (x, y, confidence)\n",
        "    keypoints = e2d.estimate(img_list=[frame])[0]\n",
        "    if not isinstance(keypoints, np.ndarray) or len(keypoints.shape) != 3:\n",
        "        # failed to detect human\n",
        "        keypoints_list.append(None)\n",
        "    else:\n",
        "        # we assume that the image only contains 1 person\n",
        "        # multi-person video needs some extra processes like grouping\n",
        "        # maybe we will implemented it in the future\n",
        "        keypoints_list.append(keypoints[0])\n",
        "cap.release()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I1AJIHVJHm5A"
      },
      "source": [
        "## Process 2D pose"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pHIaoawpHm5B"
      },
      "source": [
        "# filter out failed result\n",
        "keypoints_list = smooth.filter_missing_value(\n",
        "    keypoints_list=keypoints_list,\n",
        "    method='ignore' # interpolation method will be implemented later\n",
        ")\n",
        "\n",
        "# smooth process will be implemented later\n",
        "\n",
        "# save 2d pose result\n",
        "pose2d = np.stack(keypoints_list)[:, :, :2]\n",
        "pose2d_file = Path(output_dir / '2d_pose.npy')\n",
        "np.save(pose2d_file, pose2d)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GJr82cNlHm5B"
      },
      "source": [
        "## Visualize 2D pose"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kd9e1P_zHm5B"
      },
      "source": [
        "cap = cv2.VideoCapture(str(video_file))\n",
        "vis_result_dir = output_dir / '2d_pose_vis' # path to save the visualized images\n",
        "if not vis_result_dir.exists():\n",
        "    os.makedirs(vis_result_dir)\n",
        "    \n",
        "op_skel = openpose_skeleton.OpenPoseSkeleton()\n",
        "\n",
        "for i, keypoints in enumerate(keypoints_list):\n",
        "    ret, frame = cap.read()\n",
        "    if not ret:\n",
        "        break\n",
        "    \n",
        "    # keypoint whose detect confidence under kp_thresh will not be visualized\n",
        "    vis.vis_2d_keypoints(\n",
        "        keypoints=keypoints,\n",
        "        img=frame,\n",
        "        skeleton=op_skel,\n",
        "        kp_thresh=0.4,\n",
        "        output_file=vis_result_dir / f'{i:04d}.png'\n",
        "    )\n",
        "cap.release()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g1s87IqtHm5C"
      },
      "source": [
        "## Initialize 3D pose estimator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5H_WGXoiHm5C"
      },
      "source": [
        "os.makedirs('models/openpose_video_pose_243f', exist_ok=True)\n",
        "\n",
        "if not os.path.exists('models/openpose_video_pose_243f/video_pose.yaml'):\n",
        "  !gdown https://drive.google.com/uc?id=1Ieqj1pyQSO6qU7gB1WRbWmd5CQu1bb5l -O models/openpose_video_pose_243f/video_pose.yaml\n",
        "\n",
        "if not os.path.exists('models/openpose_video_pose_243f/best_58.58.pth'):\n",
        "  !gdown https://drive.google.com/uc?id=1lfTWNqnqIvsf2h959Ole7t8-j86fO1xU -O models/openpose_video_pose_243f/best_58.58.pth\n",
        "\n",
        "importlib.reload(estimator_3d)\n",
        "e3d = estimator_3d.Estimator3D(\n",
        "    config_file='models/openpose_video_pose_243f/video_pose.yaml',\n",
        "    checkpoint_file='models/openpose_video_pose_243f/best_58.58.pth'\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tudBreUzHm5D"
      },
      "source": [
        "## Estimate 3D pose from 2D pose"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PsOavR3SHm5D"
      },
      "source": [
        "pose2d = np.load(pose2d_file)\n",
        "pose3d = e3d.estimate(pose2d, image_width=img_width, image_height=img_height)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Oq64m1nPHm5E"
      },
      "source": [
        "## Convert 3D pose from camera coordinates to world coordinates"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lQxuqMz8Hm5E"
      },
      "source": [
        "subject = 'S1'\n",
        "cam_id = '55011271'\n",
        "cam_params = camera.load_camera_params('cameras.h5')[subject][cam_id]\n",
        "R = cam_params['R']\n",
        "T = 0\n",
        "azimuth = cam_params['azimuth']\n",
        "\n",
        "pose3d_world = camera.camera2world(pose=pose3d, R=R, T=T)\n",
        "pose3d_world[:, :, 2] -= np.min(pose3d_world[:, :, 2]) # rebase the height\n",
        "\n",
        "pose3d_file = output_dir / '3d_pose.npy'\n",
        "np.save(pose3d_file, pose3d_world)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Eft5jUVtHm5E"
      },
      "source": [
        "## Visualize 3D pose"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YVfBy2pdHm5E"
      },
      "source": [
        "h36m_skel = h36m_skeleton.H36mSkeleton()\n",
        "gif_file = output_dir / '3d_pose_300.gif' # output format can be .gif or .mp4 \n",
        "\n",
        "ani = vis.vis_3d_keypoints_sequence(\n",
        "    keypoints_sequence=pose3d_world[0:300],\n",
        "    skeleton=h36m_skel,\n",
        "    azimuth=azimuth,\n",
        "    fps=60,\n",
        "    output_file=gif_file\n",
        ")\n",
        "HTML(ani.to_jshtml())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6bxjAjy9Hm5F"
      },
      "source": [
        "## Convert 3D pose to BVH"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XDqtM4SSHm5F"
      },
      "source": [
        "bvh_file = output_dir / f'{video_file.stem}.bvh'\n",
        "cmu_skel = cmu_skeleton.CMUSkeleton()\n",
        "channels, header = cmu_skel.poses2bvh(pose3d_world, output_file=bvh_file)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ltQzN-wTHm5G"
      },
      "source": [
        "output = 'miscs/h36m_cxk.bvh'\n",
        "h36m_skel = h36m_skeleton.H36mSkeleton()\n",
        "_ = h36m_skel.poses2bvh(pose3d_world, output_file=output)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}