Skip to content

Instantly share code, notes, and snippets.

@eyaler

eyaler/openpose-video2bvh.ipynb

Created October 27, 2021 16:06
Show Gist options
  • Save eyaler/886ecdff6429ef9cd585d5dc8fdd51ec to your computer and use it in GitHub Desktop.
Save eyaler/886ecdff6429ef9cd585d5dc8fdd51ec to your computer and use it in GitHub Desktop.
Download ZIP
openpose+video2bvh.ipynb
Display the source blob
Display the rendered blob
Raw
openpose-video2bvh.ipynb
{
"nbformat": 4,
"nbformat_minor": 0,
"metadata": {
"colab": {
"name": "openpose+video2bvh.ipynb",
"private_outputs": true,
"provenance": [],
"collapsed_sections": [],
"machine_shape": "hm",
"include_colab_link": true
},
"kernelspec": {
"display_name": "Python 3",
"name": "python3"
},
"accelerator": "GPU"
},
"cells": [
{
"cell_type": "markdown",
"metadata": {
"id": "view-in-github",
"colab_type": "text"
},
"source": [
"<a href=\"https://colab.research.google.com/gist/eyaler/886ecdff6429ef9cd585d5dc8fdd51ec/openpose-video2bvh.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
]
},
{
"cell_type": "code",
"metadata": {
"id": "FOdkDhb6ga6N"
},
"source": [
"# openpose setup - taken from https://colab.research.google.com/github/Kasidit0052/MatchingOpenpose/blob/main/OpenPose.ipynb\n",
"# note this can take 10 minutes +\n",
"\n",
"%cd /content\n",
"from os.path import exists, join, basename, splitext\n",
"git_repo_url = 'https://github.com/CMU-Perceptual-Computing-Lab/openpose.git'\n",
"project_name = splitext(basename(git_repo_url))[0]\n",
"if not exists(project_name):\n",
" # see: https://github.com/CMU-Perceptual-Computing-Lab/openpose/issues/949\n",
" # install new CMake becaue of CUDA10\n",
" !wget -q https://cmake.org/files/v3.13/cmake-3.13.0-Linux-x86_64.tar.gz\n",
" !tar xfz cmake-3.13.0-Linux-x86_64.tar.gz --strip-components=1 -C /usr/local\n",
" # clone openpose\n",
" !git clone -q --depth 1 --branch v1.6.0 $git_repo_url\n",
" !sed -i 's/execute_process(COMMAND git checkout master WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}\\/3rdparty\\/caffe)/execute_process(COMMAND git checkout f019d0dfe86f49d1140961f8c7dec22130c83154 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}\\/3rdparty\\/caffe)/g' openpose/CMakeLists.txt\n",
" # install system dependencies\n",
" !apt-get -qq install -y libatlas-base-dev libprotobuf-dev libleveldb-dev libsnappy-dev libhdf5-serial-dev protobuf-compiler libgflags-dev libgoogle-glog-dev liblmdb-dev opencl-headers ocl-icd-opencl-dev libviennacl-dev\n",
" # build openpose\n",
" !cd openpose && rm -rf build || true && mkdir build && cd build && cmake -DBUILD_PYTHON=ON .. && make -j`nproc`\n",
"\n",
"!apt install imagemagick\n",
"\n",
"!git clone -q --depth 1 https://github.com/KevinLTT/video2bvh"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "ZIhQA8Lf5qpH"
},
"source": [
"# video2bvh demo from https://colab.research.google.com/github/KevinLTT/video2bvh/blob/master/demo.ipynb \n",
"\n",
"# Set Python Openpose Directory for python api (Important)\n",
"import sys\n",
"pyopenpose_dir = '/content/openpose/build/python'\n",
"if pyopenpose_dir not in sys.path:\n",
" sys.path.append(pyopenpose_dir)\n",
"from openpose import pyopenpose as op\n",
"\n",
"%cd /content/video2bvh\n",
"from pose_estimator_2d.estimator_2d import Estimator2D\n",
"\n",
"class OpenPoseEstimator(Estimator2D):\n",
"\n",
" def __init__(self, model_folder):\n",
" \"\"\"\n",
" OpenPose 2D pose estimator. See [https://github.com/\n",
" CMU-Perceptual-Computing-Lab/openpose/tree/ master/examples/\n",
" tutorial_api_python] for help.\n",
" Args: \n",
" \"\"\"\n",
" super().__init__()\n",
" params = {'model_folder': model_folder, 'render_pose': 0}\n",
" self.opWrapper = op.WrapperPython()\n",
" self.opWrapper.configure(params)\n",
" self.opWrapper.start()\n",
"\n",
" def estimate(self, img_list, bbox_list=None):\n",
" \"\"\"See base class.\"\"\"\n",
" keypoints_list = []\n",
" for i, img in enumerate(img_list):\n",
" if bbox_list:\n",
" x, y, w, h = bbox_list[i]\n",
" img = img[y:y+h, x:x+w]\n",
" datum = op.Datum()\n",
" datum.cvInputData = img\n",
" self.opWrapper.emplaceAndPop([datum])\n",
" keypoints = datum.poseKeypoints\n",
" if bbox_list:\n",
" # TODO: restore coordinate\n",
" pass\n",
" keypoints_list.append(datum.poseKeypoints)\n",
"\n",
" return keypoints_list\n",
"\n",
"from pose_estimator_3d import estimator_3d\n",
"from utils import smooth, vis, camera\n",
"from bvh_skeleton import openpose_skeleton, h36m_skeleton, cmu_skeleton\n",
"\n",
"import cv2\n",
"import importlib\n",
"import numpy as np\n",
"import os\n",
"from pathlib import Path\n",
"from IPython.display import HTML"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "K89boUgdHm49"
},
"source": [
"## Initialize 2d pose estimator"
]
},
{
"cell_type": "code",
"metadata": {
"id": "Jn_kJtgBHm4_"
},
"source": [
"# more 2d pose estimators like HRNet, PoseResNet, CPN, etc., will be added later\n",
"e2d = OpenPoseEstimator(model_folder='/content/openpose/models/') # set model_folder to /path/to/openpose/models"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "PAPD741WHm4_"
},
"source": [
"## Estimate 2D pose from video"
]
},
{
"cell_type": "code",
"metadata": {
"id": "89uKXH4LHm5A"
},
"source": [
"video_file = Path('miscs/cxk.mp4') # video file to process\n",
"output_dir = Path(f'miscs/{video_file.stem}_cache')\n",
"if not output_dir.exists():\n",
" os.makedirs(output_dir)\n",
" \n",
"cap = cv2.VideoCapture(str(video_file))\n",
"keypoints_list = []\n",
"img_width, img_height = None, None\n",
"while True:\n",
" ret, frame = cap.read()\n",
" if not ret:\n",
" break\n",
" img_height = frame.shape[0]\n",
" img_width = frame.shape[1]\n",
" \n",
" # returned shape will be (num_of_human, 25, 3)\n",
" # last dimension includes (x, y, confidence)\n",
" keypoints = e2d.estimate(img_list=[frame])[0]\n",
" if not isinstance(keypoints, np.ndarray) or len(keypoints.shape) != 3:\n",
" # failed to detect human\n",
" keypoints_list.append(None)\n",
" else:\n",
" # we assume that the image only contains 1 person\n",
" # multi-person video needs some extra processes like grouping\n",
" # maybe we will implemented it in the future\n",
" keypoints_list.append(keypoints[0])\n",
"cap.release()"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "I1AJIHVJHm5A"
},
"source": [
"## Process 2D pose"
]
},
{
"cell_type": "code",
"metadata": {
"id": "pHIaoawpHm5B"
},
"source": [
"# filter out failed result\n",
"keypoints_list = smooth.filter_missing_value(\n",
" keypoints_list=keypoints_list,\n",
" method='ignore' # interpolation method will be implemented later\n",
")\n",
"\n",
"# smooth process will be implemented later\n",
"\n",
"# save 2d pose result\n",
"pose2d = np.stack(keypoints_list)[:, :, :2]\n",
"pose2d_file = Path(output_dir / '2d_pose.npy')\n",
"np.save(pose2d_file, pose2d)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "GJr82cNlHm5B"
},
"source": [
"## Visualize 2D pose"
]
},
{
"cell_type": "code",
"metadata": {
"id": "kd9e1P_zHm5B"
},
"source": [
"cap = cv2.VideoCapture(str(video_file))\n",
"vis_result_dir = output_dir / '2d_pose_vis' # path to save the visualized images\n",
"if not vis_result_dir.exists():\n",
" os.makedirs(vis_result_dir)\n",
" \n",
"op_skel = openpose_skeleton.OpenPoseSkeleton()\n",
"\n",
"for i, keypoints in enumerate(keypoints_list):\n",
" ret, frame = cap.read()\n",
" if not ret:\n",
" break\n",
" \n",
" # keypoint whose detect confidence under kp_thresh will not be visualized\n",
" vis.vis_2d_keypoints(\n",
" keypoints=keypoints,\n",
" img=frame,\n",
" skeleton=op_skel,\n",
" kp_thresh=0.4,\n",
" output_file=vis_result_dir / f'{i:04d}.png'\n",
" )\n",
"cap.release()"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "g1s87IqtHm5C"
},
"source": [
"## Initialize 3D pose estimator"
]
},
{
"cell_type": "code",
"metadata": {
"id": "5H_WGXoiHm5C"
},
"source": [
"os.makedirs('models/openpose_video_pose_243f', exist_ok=True)\n",
"\n",
"if not os.path.exists('models/openpose_video_pose_243f/video_pose.yaml'):\n",
" !gdown https://drive.google.com/uc?id=1Ieqj1pyQSO6qU7gB1WRbWmd5CQu1bb5l -O models/openpose_video_pose_243f/video_pose.yaml\n",
"\n",
"if not os.path.exists('models/openpose_video_pose_243f/best_58.58.pth'):\n",
" !gdown https://drive.google.com/uc?id=1lfTWNqnqIvsf2h959Ole7t8-j86fO1xU -O models/openpose_video_pose_243f/best_58.58.pth\n",
"\n",
"importlib.reload(estimator_3d)\n",
"e3d = estimator_3d.Estimator3D(\n",
" config_file='models/openpose_video_pose_243f/video_pose.yaml',\n",
" checkpoint_file='models/openpose_video_pose_243f/best_58.58.pth'\n",
")"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "tudBreUzHm5D"
},
"source": [
"## Estimate 3D pose from 2D pose"
]
},
{
"cell_type": "code",
"metadata": {
"id": "PsOavR3SHm5D"
},
"source": [
"pose2d = np.load(pose2d_file)\n",
"pose3d = e3d.estimate(pose2d, image_width=img_width, image_height=img_height)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "Oq64m1nPHm5E"
},
"source": [
"## Convert 3D pose from camera coordinates to world coordinates"
]
},
{
"cell_type": "code",
"metadata": {
"id": "lQxuqMz8Hm5E"
},
"source": [
"subject = 'S1'\n",
"cam_id = '55011271'\n",
"cam_params = camera.load_camera_params('cameras.h5')[subject][cam_id]\n",
"R = cam_params['R']\n",
"T = 0\n",
"azimuth = cam_params['azimuth']\n",
"\n",
"pose3d_world = camera.camera2world(pose=pose3d, R=R, T=T)\n",
"pose3d_world[:, :, 2] -= np.min(pose3d_world[:, :, 2]) # rebase the height\n",
"\n",
"pose3d_file = output_dir / '3d_pose.npy'\n",
"np.save(pose3d_file, pose3d_world)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "Eft5jUVtHm5E"
},
"source": [
"## Visualize 3D pose"
]
},
{
"cell_type": "code",
"metadata": {
"id": "YVfBy2pdHm5E"
},
"source": [
"h36m_skel = h36m_skeleton.H36mSkeleton()\n",
"gif_file = output_dir / '3d_pose_300.gif' # output format can be .gif or .mp4 \n",
"\n",
"ani = vis.vis_3d_keypoints_sequence(\n",
" keypoints_sequence=pose3d_world[0:300],\n",
" skeleton=h36m_skel,\n",
" azimuth=azimuth,\n",
" fps=60,\n",
" output_file=gif_file\n",
")\n",
"HTML(ani.to_jshtml())"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "markdown",
"metadata": {
"id": "6bxjAjy9Hm5F"
},
"source": [
"## Convert 3D pose to BVH"
]
},
{
"cell_type": "code",
"metadata": {
"id": "XDqtM4SSHm5F"
},
"source": [
"bvh_file = output_dir / f'{video_file.stem}.bvh'\n",
"cmu_skel = cmu_skeleton.CMUSkeleton()\n",
"channels, header = cmu_skel.poses2bvh(pose3d_world, output_file=bvh_file)"
],
"execution_count": null,
"outputs": []
},
{
"cell_type": "code",
"metadata": {
"id": "ltQzN-wTHm5G"
},
"source": [
"output = 'miscs/h36m_cxk.bvh'\n",
"h36m_skel = h36m_skeleton.H36mSkeleton()\n",
"_ = h36m_skel.poses2bvh(pose3d_world, output_file=output)"
],
"execution_count": null,
"outputs": []
}
]
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment