▶︎ What's this?: RM motion capture demo using pixiv three-vrm
▶︎ Reference: https://github.com/pixiv/three-vrm
[Gist] https://gist.github.com/Hirosaji/00d3c9ceba65ef7ddf0f2890968132dc
Built with blockbuilder.org
# three-vrm - VRM motion capture by PoseNet | |
license: mit |
▶︎ What's this?: RM motion capture demo using pixiv three-vrm
▶︎ Reference: https://github.com/pixiv/three-vrm
[Gist] https://gist.github.com/Hirosaji/00d3c9ceba65ef7ddf0f2890968132dc
Built with blockbuilder.org
<!DOCTYPE html> | |
<html> | |
<head> | |
<meta charset="utf-8" /> | |
<title>three-vrm example</title> | |
<meta | |
name="viewport" | |
content="width=device-width, initial-scale=1.0, minimum-scale=1.0, maximum-scale=1.0, user-scalable=no" | |
/> | |
<style> | |
body { | |
margin: 0; | |
} | |
canvas { | |
display: block; | |
} | |
#motion { | |
position: absolute; | |
bottom: 0; | |
right: 0; | |
color: white; | |
} | |
#webacamCanvas { transform: rotateY(180deg); } | |
</style> | |
</head> | |
<body> | |
<div id="vrm"></div> | |
<div id="motion"> | |
<div id="loading-indicator">PoseNet model is loading.</div> | |
<canvas id="webacamCanvas" width="480" height="320"></canvas> | |
<video id="video" width="480" height="320" style="display:none;" autoplay playsinline>Video stream not | |
available.</video> | |
</div> | |
<script src="https://unpkg.com/three@0.106.2/build/three.js"></script> | |
<script src="https://unpkg.com/three@0.106.2/examples/js/loaders/GLTFLoader.js"></script> | |
<script src="https://unpkg.com/three@0.106.2/examples/js/controls/OrbitControls.js"></script> | |
<script src="./three-vrm.min.js"></script> | |
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs"></script> | |
<script src="https://cdn.jsdelivr.net/npm/@tensorflow-models/posenet"></script> | |
<script> | |
//=================================== | |
// motion capture process | |
let poseStore = {}; | |
const webacamCanvas = document.getElementById("webacamCanvas"); | |
const webcamCtx = webacamCanvas.getContext("2d"); | |
const video = document.getElementById('video'); | |
// display camera movie Canvas detected parts | |
function detectAndDraw(net) { | |
webcamCtx.drawImage(video, 0, 0, 480, 320); | |
net.estimateSinglePose(video, { | |
flipHorizontal: false | |
}) | |
.then(function(pose) { | |
drawKeypoints(pose); | |
}); | |
} | |
// draw detected parts by PoseNet | |
function drawKeypoints(pose) { | |
pose.keypoints.forEach(keypoint => { | |
if (keypoint.score > 0.4) { | |
poseStore[keypoint.part] = { | |
x: 480/2 - keypoint.position.x, | |
y: 320/2 - keypoint.position.y | |
}; | |
webcamCtx.beginPath(); | |
webcamCtx.fillStyle = "rgb(255, 255, 0)"; // 黄色 | |
webcamCtx.arc( | |
keypoint.position.x, | |
keypoint.position.y, | |
5, | |
(10 * Math.PI) / 180, | |
(80 * Math.PI) / 180, | |
true | |
); | |
webcamCtx.fill(); | |
webcamCtx.fillText( | |
keypoint.part, | |
keypoint.position.x, | |
keypoint.position.y + 10 | |
); | |
} | |
}); | |
} | |
// get camera movie | |
navigator.mediaDevices.getUserMedia({ audio: false, video: true }) | |
.then(function (mediaStream) { | |
// set video tag srcObject | |
video.srcObject = mediaStream; | |
video.onloadedmetadata = function (e) { | |
video.play(); | |
}; | |
return posenet.load(); | |
}) | |
.then(function (net) { | |
var loadingIndicator = document.getElementById("loading-indicator"); | |
loadingIndicator.style.display = 'none'; | |
setInterval(function () { detectAndDraw(net); }, 100); | |
}); | |
//=================================== | |
// renderer | |
const renderer = new THREE.WebGLRenderer(); | |
renderer.setSize( window.innerWidth, window.innerHeight ); | |
renderer.setPixelRatio( window.devicePixelRatio ); | |
document.getElementById('vrm').appendChild( renderer.domElement ); | |
// camera | |
const camera = new THREE.PerspectiveCamera( 30.0, window.innerWidth / window.innerHeight, 0.1, 20.0 ); | |
camera.position.set( 0.0, 1.0, 5.0 ); | |
// camera controls | |
const controls = new THREE.OrbitControls( camera, renderer.domElement ); | |
controls.screenSpacePanning = true; | |
controls.target.set( 0.0, 1.0, 0.0 ); | |
controls.update(); | |
// scene | |
const scene = new THREE.Scene(); | |
// light | |
const light = new THREE.DirectionalLight( 0xffffff ); | |
light.position.set( 1.0, 1.0, 1.0 ).normalize(); | |
scene.add( light ); | |
// gltf and vrm | |
let currentVrm = undefined; | |
const loader = new THREE.GLTFLoader(); | |
loader.crossOrigin = 'anonymous'; | |
loader.load( | |
"./three-vrm-girl.vrm", | |
( gltf ) => { | |
console.log(gltf) | |
THREE.VRM.from( gltf ).then( ( vrm ) => { | |
scene.add( vrm.scene ); | |
currentVrm = vrm; | |
vrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.Hips ).rotation.y = Math.PI; | |
console.log( vrm ); | |
} ); | |
}, | |
( progress ) => console.log( 'Loading model...', 100.0 * ( progress.loaded / progress.total ), '%' ), | |
( error ) => console.error( error ) | |
); | |
// helpers | |
const gridHelper = new THREE.GridHelper( 10, 10 ); | |
scene.add( gridHelper ); | |
const axesHelper = new THREE.AxesHelper( 5 ); | |
scene.add( axesHelper ); | |
// animate | |
const clock = new THREE.Clock(); | |
let angleStore = {}; | |
// X axis | |
function getAngleFromX(pos2, pos1) { | |
return Math.atan2(pos2.y - pos1.y, pos2.x - pos1.x); | |
} | |
function animate() { | |
requestAnimationFrame( animate ); | |
const deltaTime = clock.getDelta(); | |
if ( currentVrm ) { | |
if (poseStore) { | |
if (poseStore.leftShoulder && poseStore.rightShoulder) { | |
// spine & shoulder | |
let angle = getAngleFromX(poseStore.rightShoulder, poseStore.leftShoulder); | |
if (angle !== null) { | |
angle = angle * -1; | |
angleStore.Spine = angle; | |
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.Spine ).rotation.z = angle; | |
} | |
} | |
if (poseStore.leftEye && poseStore.rightEye) { | |
// neck $ eyes | |
let angle = getAngleFromX(poseStore.rightEye, poseStore.leftEye); | |
if (angle !== null) { | |
angle = angle * -1; | |
angleStore.Neck = angle; | |
angle = angle - (angleStore.Spine || 0); | |
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.Neck ).rotation.z = angle; | |
} | |
} | |
if (poseStore.leftShoulder && poseStore.leftElbow) { | |
// arms | |
let angle = getAngleFromX(poseStore.leftElbow, poseStore.leftShoulder); | |
if (angle !== null) { | |
angle = Math.PI - angle; | |
angleStore.RightUpperArm = angle; | |
angle = angle - (angleStore.Spine || 0); | |
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.RightUpperArm ).rotation.z = angle; | |
} | |
} | |
if (poseStore.leftWrist && poseStore.leftElbow) { | |
// arms | |
let angle = getAngleFromX(poseStore.leftWrist, poseStore.leftElbow); | |
if (angle !== null) { | |
angle = Math.PI - angle; | |
angleStore.RightLowerArm = angle; | |
angle = angle - (angleStore.RightUpperArm || 0); | |
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.RightLowerArm ).rotation.z = angle; | |
} | |
} | |
if (poseStore.rightShoulder && poseStore.rightElbow) { | |
// arms | |
let angle = getAngleFromX(poseStore.rightElbow, poseStore.rightShoulder); | |
if (angle !== null) { | |
angle = angle * -1; | |
angleStore.LeftUpperArm = angle; | |
angle = angle - (angleStore.Spine || 0); | |
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.LeftUpperArm ).rotation.z = angle; | |
} | |
} | |
if (poseStore.rightWrist && poseStore.rightElbow) { | |
// arms | |
let angle = getAngleFromX(poseStore.rightWrist, poseStore.rightElbow); | |
if (angle !== null) { | |
angle = angle * -1; | |
angleStore.LeftLowerArm = angle; | |
angle = angle - (angleStore.LeftUpperArm || 0); | |
currentVrm.humanoid.getBoneNode( THREE.VRMSchema.HumanoidBoneName.LeftLowerArm ).rotation.z = angle; | |
} | |
} | |
} | |
// update vrm | |
currentVrm.update( deltaTime ); | |
} | |
renderer.render( scene, camera ); | |
} | |
animate(); | |
</script> | |
</body> | |
</html> |