Mediapipe Hand Tracking in Unity in Sync with Webcam

HandLandmarkRunnerSync.cs

using System.Collections;
using Mediapipe.Tasks.Vision.HandLandmarker;
using UnityEngine;
using UnityEngine.Rendering;

using Stopwatch = System.Diagnostics.Stopwatch;


public class HandLandmarkerRunnerSync : MonoBehaviour
{
  private WebCamTexture _webCamTexture;
  public UnityEngine.UI.RawImage _rawImage;

  public TextAsset _handLandmarkerBytes;
  HandLandmarker _handLandmarker;

  private GameObject[] _spheres = new GameObject[20];

  private float _fpsAccumulator = 0.0f;
  private int _frameCount = 0;
  private float _fpsUpdateInterval = 1.0f; // Update FPS every 1 second
  private float _fps = 0.0f;
  private Stopwatch _stopwatch = new Stopwatch();

  public void Start()
  {
    _webCamTexture = new WebCamTexture();
    _webCamTexture.Play();

    var options = new HandLandmarkerOptions(
      baseOptions: new Mediapipe.Tasks.Core.BaseOptions(
        Mediapipe.Tasks.Core.BaseOptions.Delegate.CPU,
        modelAssetBuffer: _handLandmarkerBytes.bytes
      ),
      // runningMode: Mediapipe.Tasks.Vision.Core.RunningMode.IMAGE
      runningMode: Mediapipe.Tasks.Vision.Core.RunningMode.VIDEO
    );

    _handLandmarker = HandLandmarker.CreateFromOptions(options, null);

    for (int i = 0; i < 20; i++)
    {
      _spheres[i] = GameObject.CreatePrimitive(PrimitiveType.Sphere);
      _spheres[i].transform.position = new Vector3(0, 0, 0);
      _spheres[i].transform.localScale = new Vector3(10f, 10f, 10f);
      // _spheres[i].transform.parent = _rawImage.transform;
    }
  }

  public void Update()
  {
    // time how long detection takes
    _stopwatch.Start();
    if (_webCamTexture == null || !_webCamTexture.isPlaying)
    {
      return;
    }

    var mpInputTexture = new Texture2D(_webCamTexture.width, _webCamTexture.height, TextureFormat.RGBA32, false);
    mpInputTexture.SetPixels32(_webCamTexture.GetPixels32());
    mpInputTexture.Apply();
    using var image = new Mediapipe.Image(mpInputTexture);

    // var result = _handLandmarker.Detect(image);
    var result = _handLandmarker.DetectForVideo(image, (int)(Time.time*1000));

    _rawImage.texture = mpInputTexture;

    if (result.handLandmarks == null)
    {
      // hide all spheres
      foreach (var sphere in _spheres)
      {
        sphere.SetActive(false);
      }
      return;
    }

    foreach (var handLandmarkList in result.handLandmarks)
    {
      for (int i = 0; i < 20; i++)
      {  
        Vector3 position = new Vector3((handLandmarkList.landmarks[i].x- 0.5f)*1280, (handLandmarkList.landmarks[i].y - 0.5f)*720, 0);
        _spheres[i].SetActive(true);
        _spheres[i].transform.localPosition = position;
      }
    }
    _stopwatch.Stop();
    Debug.Log("Detection took " + _stopwatch.ElapsedMilliseconds + " ms");
    _stopwatch.Reset();

  }
}