🖐️
[Next.js] Mediapipe + カメラで手のジェスチャー認識スケッチパッド
Demo Link: https://cygra.github.io/hand-gesture-whiteboard/
参考:https://ai.google.dev/edge/mediapipe/solutions/vision/gesture_recognizer
環境を構築し、Mediapipeをインストールする
npx create-next-app@latest
npm i @mediapipe/tasks-vision
ビデオを取得する
const prepareVideoStream = async () => {
const stream = await navigator.mediaDevices.getUserMedia({
audio: false,
video: true,
});
if (videoRef.current) {
videoRef.current.srcObject = stream;
videoRef.current.addEventListener("loadeddata", () => {
process();
});
}
};
Midiapipeを初期化する
const vision = await FilesetResolver.forVisionTasks(
"https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
);
const gestureRecognizer = await GestureRecognizer.createFromOptions(
vision,
{
baseOptions: {
modelAssetPath:
"https://storage.googleapis.com/mediapipe-tasks/gesture_recognizer/gesture_recognizer.task",
delegate: "GPU",
},
numHands: 1,
runningMode: "VIDEO",
}
);
大体
パフォーマンス向上のために requestAnimationFrame
を使用する
const process = async () => {
const vision = await FilesetResolver.forVisionTasks(...);
const gestureRecognizer = await GestureRecognizer.createFromOptions(...);
const renderLoop = () => {
const result = gestureRecognizer.recognizeForVideo(video, startTimeMs);
drawLandmarks(landmarks);
drawLine(strokeCanvasCtx, x, y);
requestAnimationFrame(() => {
renderLoop();
});
};
renderLoop();
};
手のジェスチャーを取得する
let lastWebcamTime = -1;
...
lastWebcamTime = video.currentTime;
const startTimeMs = performance.now();
const result = gestureRecognizer.recognizeForVideo(video, startTimeMs);
👌🏻 ジェスチャーを判定する
if (result.landmarks) {
const width = landmarkCanvas.width;
const height = landmarkCanvas.height;
if (result.landmarks.length === 0) {
landmarkCanvasCtx.clearRect(0, 0, width, height);
} else {
result.landmarks.forEach((landmarks) => {
const thumbTip = landmarks[THUMB_TIP_INDEX];
const indexFingerTip = landmarks[INDEX_FINGER_TIP_INDEX];
const dx = (thumbTip.x - indexFingerTip.x) * width;
const dy = (thumbTip.y - indexFingerTip.y) * height;
const connected = dx < 50 && dy < 50;
if (connected) {
const x = (1 - indexFingerTip.x) * width;
const y = indexFingerTip.y * height;
drawLine(strokeCanvasCtx, x, y);
} else {
prevX = prevY = 0;
}
drawLandmarks(
landmarks,
landmarkCanvasCtx,
width,
height,
connected
);
});
}
}
描き
const SMOOTHING_FACTOR = 0.3;
const drawLine = (ctx: CanvasRenderingContext2D, x: number, y: number) => {
if (!prevX || !prevY) {
prevX = x;
prevY = y;
}
const smoothedX = prevX + SMOOTHING_FACTOR * (x - prevX);
const smoothedY = prevY + SMOOTHING_FACTOR * (y - prevY);
ctx.lineWidth = 5;
ctx.moveTo(prevX, prevY);
ctx.lineTo(smoothedX, smoothedY);
ctx.strokeStyle = "white";
ctx.stroke();
ctx.save();
prevX = smoothedX;
prevY = smoothedY;
};
ジェスチャのレンダリング
const drawLandmarks = (
landmarks: NormalizedLandmark[],
ctx: CanvasRenderingContext2D,
width: number,
height: number,
connected: boolean
) => {
const drawingUtils = new DrawingUtils(ctx);
ctx.clearRect(0, 0, width, height);
drawingUtils.drawConnectors(landmarks, GestureRecognizer.HAND_CONNECTIONS, {
color: "#00FF00",
lineWidth: connected ? 5 : 2,
});
drawingUtils.drawLandmarks(landmarks, {
color: "#FF0000",
lineWidth: 1,
});
};
Discussion