🖐️

[Next.js] Mediapipe + カメラで手のジェスチャー認識スケッチパッド

2025/01/21に公開

Demo Link: https://cygra.github.io/hand-gesture-whiteboard/

https://github.com/Cygra/hand-gesture-whiteboard

参考:https://ai.google.dev/edge/mediapipe/solutions/vision/gesture_recognizer

環境を構築し、Mediapipeをインストールする

https://www.npmjs.com/package/@mediapipe/tasks-vision

npx create-next-app@latest

npm i @mediapipe/tasks-vision

ビデオを取得する

  const prepareVideoStream = async () => {
    const stream = await navigator.mediaDevices.getUserMedia({
      audio: false,
      video: true,
    });

    if (videoRef.current) {
      videoRef.current.srcObject = stream;
      videoRef.current.addEventListener("loadeddata", () => {
        process();
      });
    }
  };

Midiapipeを初期化する

    const vision = await FilesetResolver.forVisionTasks(
      "https://cdn.jsdelivr.net/npm/@mediapipe/tasks-vision@latest/wasm"
    );

    const gestureRecognizer = await GestureRecognizer.createFromOptions(
      vision,
      {
        baseOptions: {
          modelAssetPath:
            "https://storage.googleapis.com/mediapipe-tasks/gesture_recognizer/gesture_recognizer.task",
          delegate: "GPU",
        },
        numHands: 1,
        runningMode: "VIDEO",
      }
    );

大体

パフォーマンス向上のために requestAnimationFrame を使用する

  const process = async () => {
    const vision = await FilesetResolver.forVisionTasks(...);
    const gestureRecognizer = await GestureRecognizer.createFromOptions(...);

    const renderLoop = () => {
      const result = gestureRecognizer.recognizeForVideo(video, startTimeMs);
      
      drawLandmarks(landmarks);
      drawLine(strokeCanvasCtx, x, y);
      
      requestAnimationFrame(() => {
        renderLoop();
      });
    };

    renderLoop();
  };

手のジェスチャーを取得する

      let lastWebcamTime = -1;
      
      ...
      
      lastWebcamTime = video.currentTime;
      const startTimeMs = performance.now();
      const result = gestureRecognizer.recognizeForVideo(video, startTimeMs);

👌🏻 ジェスチャーを判定する

      if (result.landmarks) {
        const width = landmarkCanvas.width;
        const height = landmarkCanvas.height;
        if (result.landmarks.length === 0) {
          landmarkCanvasCtx.clearRect(0, 0, width, height);
        } else {
          result.landmarks.forEach((landmarks) => {
            const thumbTip = landmarks[THUMB_TIP_INDEX];
            const indexFingerTip = landmarks[INDEX_FINGER_TIP_INDEX];

            const dx = (thumbTip.x - indexFingerTip.x) * width;
            const dy = (thumbTip.y - indexFingerTip.y) * height;

            const connected = dx < 50 && dy < 50;
            if (connected) {
              const x = (1 - indexFingerTip.x) * width;
              const y = indexFingerTip.y * height;
              drawLine(strokeCanvasCtx, x, y);
            } else {
              prevX = prevY = 0;
            }
            drawLandmarks(
              landmarks,
              landmarkCanvasCtx,
              width,
              height,
              connected
            );
          });
        }
      }

描き

const SMOOTHING_FACTOR = 0.3;

  const drawLine = (ctx: CanvasRenderingContext2D, x: number, y: number) => {
    if (!prevX || !prevY) {
      prevX = x;
      prevY = y;
    }

    const smoothedX = prevX + SMOOTHING_FACTOR * (x - prevX);
    const smoothedY = prevY + SMOOTHING_FACTOR * (y - prevY);
    ctx.lineWidth = 5;
    ctx.moveTo(prevX, prevY);
    ctx.lineTo(smoothedX, smoothedY);
    ctx.strokeStyle = "white";
    ctx.stroke();
    ctx.save();

    prevX = smoothedX;
    prevY = smoothedY;
  };

ジェスチャのレンダリング

  const drawLandmarks = (
    landmarks: NormalizedLandmark[],
    ctx: CanvasRenderingContext2D,
    width: number,
    height: number,
    connected: boolean
  ) => {
    const drawingUtils = new DrawingUtils(ctx);
    ctx.clearRect(0, 0, width, height);
    drawingUtils.drawConnectors(landmarks, GestureRecognizer.HAND_CONNECTIONS, {
      color: "#00FF00",
      lineWidth: connected ? 5 : 2,
    });
    drawingUtils.drawLandmarks(landmarks, {
      color: "#FF0000",
      lineWidth: 1,
    });
  };

Discussion