บั้นเด้า รถแห่ ชุมแพ with P5.js, ML5.js, Lyria 2 and MoveNet

bundao-chumpae-p5-ml5.js

// อภิมหาบั้นเด้า รถแห่ อำเภอเมืองชุมแพ The Game
// Technologies: P5.js, ML5.js, MoveNet, TensorFlow.js, Patchies.app, DeepMind Lyria 2 

let video;
let bodyPose;
let poses = [];
let connections;

// Variables to store current hip and shoulder positions for drawing ellipses
let lHip = [0, 0];
let rHip = [0, 0];
let lShould = [0, 0];
let rShould = [0, 0];

// Global variables to store previous hip and shoulder positions
// Initialized to null to indicate no previous pose has been detected yet.
let prevLHipX = null, prevLHipY = null;
let prevRHipX = null, prevRHipY = null;
let prevLShouldX = null, prevLShouldY = null;
let prevRShouldX = null, prevRShouldY = null;

// Global variables for the movement scores
// These scores accumulate movement over time but also decay,
// providing a measure of recent activity.
let shoulderMoveScore = 0;
let hipMoveScore = 0;

// Decay factor for the scores. A higher value (closer to 1) means the score
// persists longer, a lower value makes it more reactive to immediate movement.
const scoreDecay = 0.85;

// Minimum confidence threshold for a keypoint to be considered valid
const confidenceThreshold = 0.1;

let isModelLoaded = false
let isZaap = false

let zaapFrames = 0
let notZaapFrames = 0

/**
 * Preloads the BodyPose model.
 * This function is called once before setup().
 */
function preload() {
  bodyPose = ml5.bodyPose("MoveNet", {
    modelUrl: undefined
  }, () => {
    console.log('MoveNet Loaded')

    // Start detecting poses in the webcam video.
    // The 'gotPoses' function will be called when new pose data is available.
    bodyPose.detectStart(video, gotPoses);

    // Get the skeleton connection information to draw lines between keypoints
    connections = bodyPose.getSkeleton();

    isModelLoaded = true
  });
}

/**
 * Sets up the canvas, video capture, and starts pose detection.
 * This function is called once when the sketch starts.
 */
function setup() {
  // Create a canvas to draw on
  createCanvas(640, 480);

  // Create a video capture object from the webcam
  video = createCapture(VIDEO);
  video.size(width, height); // Set video size to match canvas
  video.hide(); // Hide the HTML video element, only draw to canvas
}

/**
 * Main drawing loop. This function is called repeatedly (default 60 times per second).
 */
function draw() {
  // Draw the webcam video onto the canvas
  image(video, 0, 0, width, height);

  // Apply decay to the scores each frame.
  // This makes the scores decrease over time if no new movement is detected,
  // preventing them from growing infinitely and reflecting recent activity.
  shoulderMoveScore *= scoreDecay;
  hipMoveScore *= scoreDecay;

  // Check if a pose is detected (poses array is not empty)
  if (poses[0]) {
    const pose = poses[0]; // Get the first detected pose

    // --- Update current keypoint positions and draw ellipses ---
    // Only update and draw if the keypoint's confidence is above the threshold
    if (pose.left_hip.confidence > confidenceThreshold) {
      lHip = [pose.left_hip.x, pose.left_hip.y];
      fill(0, 255, 0); // Green for left hip
      ellipse(lHip[0], lHip[1], 40, 40);
    }

    if (pose.right_hip.confidence > confidenceThreshold) {
      rHip = [pose.right_hip.x, pose.right_hip.y];
      fill(0, 0, 255); // Blue for right hip
      ellipse(rHip[0], rHip[1], 40, 40);
    }

    if (pose.left_shoulder.confidence > confidenceThreshold) {
      lShould = [pose.left_shoulder.x, pose.left_shoulder.y];
      fill(50, 0, 200); // Purple for left shoulder
      ellipse(lShould[0], lShould[1], 40, 40);
    }

    if (pose.right_shoulder.confidence > confidenceThreshold) {
      rShould = [pose.right_shoulder.x, pose.right_shoulder.y];
      fill(200, 0, 50); // Red for right shoulder
      ellipse(rShould[0], rShould[1], 40, 40);
    }

    // --- Calculate Movement Scores ---
    let currentFrameShoulderMove = 0;
    let currentFrameHipMove = 0;

    // Calculate movement for left hip if previous position is available and current confidence is high
    if (prevLHipX !== null && pose.left_hip.confidence > confidenceThreshold) {
      currentFrameHipMove += dist(lHip[0], lHip[1], prevLHipX, prevLHipY);
    }
    // Calculate movement for right hip
    if (prevRHipX !== null && pose.right_hip.confidence > confidenceThreshold) {
      currentFrameHipMove += dist(rHip[0], rHip[1], prevRHipX, prevRHipY);
    }
    // Calculate movement for left shoulder
    if (prevLShouldX !== null && pose.left_shoulder.confidence > confidenceThreshold) {
      currentFrameShoulderMove += dist(lShould[0], lShould[1], prevLShouldX, prevLShouldY);
    }
    // Calculate movement for right shoulder
    if (prevRShouldX !== null && pose.right_shoulder.confidence > confidenceThreshold) {
      currentFrameShoulderMove += dist(rShould[0], rShould[1], prevRShouldX, prevRShouldY);
    }

    // Add the calculated movement for the current frame to the overall scores
    shoulderMoveScore += currentFrameShoulderMove;
    hipMoveScore += currentFrameHipMove;

    // --- Update Previous Positions for the next frame ---
    // Only update previous positions if the current keypoint confidence is high enough.
    // This prevents storing unreliable data as "previous".
    if (pose.left_hip.confidence > confidenceThreshold) {
      prevLHipX = lHip[0];
      prevLHipY = lHip[1];
    }
    if (pose.right_hip.confidence > confidenceThreshold) {
      prevRHipX = rHip[0];
      prevRHipY = rHip[1];
    }
    if (pose.left_shoulder.confidence > confidenceThreshold) {
      prevLShouldX = lShould[0];
      prevLShouldY = lShould[1];
    }
    if (pose.right_shoulder.confidence > confidenceThreshold) {
      prevRShouldX = rShould[0];
      prevRShouldY = rShould[1];
    }

    // --- Display Scores and Keypoint Coordinates ---
    fill(255, 0, 0); // White text
    textSize(24);
    // Use floor() to display integer scores for better readability
    text(`SHOULDER: ${floor(shoulderMoveScore)}`, 10, 20);
    text(`ASS: ${floor(hipMoveScore)}`, 10, 40);
    text(`ZAAP: ${isZaap}, ZF: ${zaapFrames}, NZF: ${notZaapFrames}`, 10, 60);

    // if (hipMoveScore > 0 && shoulderMoveScore > 0) {
    //   send({ hipMoveScore, shoulderMoveScore })
    // }
  } else {
    // If no pose is detected in the current frame, reset previous positions to null.
    // This prevents a large score jump if a person moves out of frame and then back in.
    prevLHipX = null; prevLHipY = null;
    prevRHipX = null; prevRHipY = null;
    prevLShouldX = null; prevLShouldY = null;
    prevRShouldX = null; prevRShouldY = null;
  }

  // --- Draw the skeleton connections ---
  for (let i = 0; i < poses.length; i++) {
    let pose = poses[i];
    for (let j = 0; j < connections.length; j++) {
      let pointAIndex = connections[j][0];
      let pointBIndex = connections[j][1];
      let pointA = pose.keypoints[pointAIndex];
      let pointB = pose.keypoints[pointBIndex];
      // Only draw a line if both points are confident enough
      if (pointA.confidence > confidenceThreshold && pointB.confidence > confidenceThreshold) {
        stroke(255, 0, 0); // Red lines
        strokeWeight(2);
        line(pointA.x, pointA.y, pointB.x, pointB.y);
      }
    }
  }

  // --- Draw all the tracked landmark points (circles) ---
  for (let i = 0; i < poses.length; i++) {
    let pose = poses[i];
    for (let j = 0; j < pose.keypoints.length; j++) {
      let keypoint = pose.keypoints[j];
      // Only draw a circle if the keypoint's confidence is bigger than the threshold
      if (keypoint.confidence > confidenceThreshold) {
        fill(0, 255, 0); // Green circles
        noStroke();
        circle(keypoint.x, keypoint.y, 10);
      }
    }
  }

  superWow()
}

function superWow() {
  if (hipMoveScore > 200 && shoulderMoveScore > 200) {
    zaapFrames += 1
    notZaapFrames = 0

    if (!isZaap) {
      send({
        type: 'setPrompts',
        prompts: {
          'thai cha cha cha': 0.8,
          'ass dance': 0.4,
        }
      })

      filter(POSTERIZE)
      send({ type: 'play' })
    }

    if (zaapFrames > 30) {
      isZaap = true
    }
  } else {
    notZaapFrames += 1

    // u not zaap too long!!
    if (notZaapFrames > 20) {
      zaapFrames = 0
      isZaap = false
      send({ type: 'pause' })
      filter(GRAY)
    }
  }
}

/**
 * Callback function for when bodyPose outputs data.
 * This function is called asynchronously whenever the model detects poses.
 * @param {Array} results - An array of detected poses.
 */
function gotPoses(results) {
  // Save the output to the poses variable
  poses = results;
}