human_kodi_control.js

/**
 * Mostly from Human demo for NodeJS https://github.com/vladmandic/human/blob/main/demo/nodejs/node-webcam.js

package.json :
{
  "dependencies": {
    "@tensorflow/tfjs-node": "^3.8.0",
    "@vladmandic/human": "^2.1.4",
    "@vladmandic/pilogger": "^0.2.18",
    "kodi-controller": "^1.2.0",
    "node-webcam": "^0.8.0"
  }
}

*/

const KodiController = require("kodi-controller");
kodi = new KodiController("192.168.0.125", 8080);

let initial = true; // remember if this is the first run to print additional details
const log = require('@vladmandic/pilogger');
// eslint-disable-next-line node/no-missing-require
const nodeWebCam = require('node-webcam');
// for NodeJS, `tfjs-node` or `tfjs-node-gpu` should be loaded before using Human
// eslint-disable-next-line no-unused-vars, @typescript-eslint/no-unused-vars
const tf = require('@tensorflow/tfjs-node'); // or const tf = require('@tensorflow/tfjs-node-gpu');
// load specific version of Human library that matches TensorFlow mode
const Human = require('@vladmandic/human').default; // points to @vladmandic/human/dist/human.node.js


// options for node-webcam
const tempFile = '/run/user/1000/webcam-snap'; // node-webcam requires writting snapshot to a file, recommended to use tmpfs to avoid excessive disk writes
const optionsCamera = {
  callbackReturn: 'buffer', // this means whatever `fswebcam` writes to disk, no additional processing so it's fastest
  saveShots: false, // don't save processed frame to disk, note that temp file is still created by fswebcam thus recommendation for tmpfs
};
const camera = nodeWebCam.create(optionsCamera);

const refreshRate = 1 // be cautious that fswebcam is also quite slow
// options for human
const optionsHuman = {
  backend: 'tensorflow',
  modelBasePath: 'file://models/',
  face:{enabled:true,
	  iris:{enabled:false},
	  /*
	  mesh:{enabled:false},
	  */
	  emotion:{enabled:false},
	  description:{enabled:false},
  },
  body:{enabled:false}, // 0.5 fps
  hand:{enabled:false}, // 0.2 fps
};
const human = new Human(optionsHuman);

function buffer2tensor(buffer) {
  return human.tf.tidy(() => {
    if (!buffer) return null;
    const decode = human.tf.node.decodeImage(buffer, 3);
    let expand;
    if (decode.shape[2] === 4) { // input is in rgba format, need to convert to rgb
      const channels = human.tf.split(decode, 4, 2); // tf.split(tensor, 4, 2); // split rgba to channels
      const rgb = human.tf.stack([channels[0], channels[1], channels[2]], 2); // stack channels back to rgb and ignore alpha
      expand = human.tf.reshape(rgb, [1, decode.shape[0], decode.shape[1], 3]); // move extra dim from the end of tensor and use it as batch number instead
    } else {
      expand = human.tf.expandDims(decode, 0); // inpur ia rgb so use as-is
    }
    const cast = human.tf.cast(expand, 'float32');
    return cast;
  });
}

async function detect() {
  // trigger next frame every 5 sec
  // triggered here before actual capture and detection since we assume it will complete in less than 5sec
  // so it's as close as possible to real 5sec and not 5sec + detection time
  // if there is a chance of race scenario where detection takes longer than loop trigger, then trigger should be at the end of the function instead
  setTimeout(() => detect(), refreshRate * 1000);

  camera.capture(tempFile, (err, data) => { // gets the (default) jpeg data from from webcam
    if (err) {
      log.error('error capturing webcam:', err);
    } else {
      const tensor = buffer2tensor(data); // create tensor from image buffer
      if (initial) log.data('input tensor:', tensor.shape);
      // eslint-disable-next-line promise/no-promise-in-callback
      human.detect(tensor).then((result) => {
        if (result && result.gesture && result.gesture.length > 0) {	
		console.log(result.gesture) 
		// assume Kodi has a player running.
		// kodi.getActivePlayerID( (e,p) => console.log(p) ) // usually returns 1 if player running, even if on pause
		if ( result.gesture.filter( g => g.gesture.indexOf('down') > 0 ).length > 0 )
			//kodi.getVolume((err, volume) =>{ console.log(`The Volume is currently ${volume}.`); });
			kodi.playPause() // only works if already playing
		if ( result.gesture.filter( g => g.gesture.indexOf('left') > 0 ).length > 0 )
			kodi.volumeDown()
		if ( result.gesture.filter( g => g.gesture.indexOf('right') > 0 ).length > 0 )
			kodi.volumeUp()
	}
      });
    }
    initial = false;
  });
  // alternatively to triggering every 5sec sec, simply trigger next frame as fast as possible
  // setImmediate(() => process());
}

async function main() {
  camera.list((list) => {
    log.data('detected camera:', list);
  });
  await human.load();
  detect();
}

log.header();
main();