detect.js

const tf = require('@tensorflow/tfjs-node-gpu');
const cv = require("@u4/opencv4nodejs");
const posenet = require("@tensorflow-models/posenet")

let camera = new cv.VideoCapture(0);
camera.set(cv.CAP_PROP_FRAME_WIDTH, 640);
camera.set(cv.CAP_PROP_FRAME_HEIGHT, 480);

async function loadPosenet() {
    net = await posenet.load({
        architecture: 'ResNet50',
        outputStride: 32,
        inputResolution: { width: 640, height: 480 },
        quantBytes: 2
      });
    detectPerson()
}
loadPosenet()

const blue = new cv.Vec(255, 0, 0);
const green = new cv.Vec(0, 255, 0);
const red = new cv.Vec(0, 0, 255);

function detectPerson() {

    camera.readAsync(async (err, mat) => {

        if (err) { console.log('Capture error:', err); return; }
        if (mat.empty) { console.log('Empty frame captured'); return; };

        // convert to RGB
        const plot = new cv.Mat(300, 600, cv.CV_8UC3, [255, 255, 255]);
        const buffer = new Uint8Array(mat.getData().buffer);
        const tFrame = tf.tensor3d(buffer, [480, 640, 3]);

        // // net is your poseNet instance
        const pose = await net.estimateSinglePose(tFrame)

        // console.log(pose)
        // // rightShoulder x - leftShoulder x
        const getHistAxis = channel => ([
            {
                channel,
                bins: 256,
                ranges: [0, 256]
            }
        ]);

        ls = pose.keypoints[6].position
        rh = pose.keypoints[11].position
        
        const width = rh.x - ls.x
        const height = rh.y - ls.y
        console.log(`${width} - ${height}`)
        
        // check region is meaningful
        // TODO this check could be improved
        if (ls.x > 0 && ls.y > 0 && width > 0 && height > 0) {
            try {
                mat.drawCircle(new cv.Point2(rh.x, rh.y), 3, green)
                mat.drawCircle(new cv.Point2(ls.x, ls.y), 3, red)

                //TODO sometimes getRegion complains, not sure why
                const torso = mat.getRegion(new cv.Rect(ls.x, ls.y, width, height));
                cv.imshow('torso', torso);
                cv.waitKey(1);
                const bHist = cv.calcHist(torso, getHistAxis(0));
                const gHist = cv.calcHist(torso, getHistAxis(1));
                const rHist = cv.calcHist(torso, getHistAxis(2));

                // plot channel histograms
                
                cv.plot1DHist(bHist, plot, blue, 2);
                cv.plot1DHist(gHist, plot, green, 2);
                cv.plot1DHist(rHist, plot, red, 2);
                cv.imshow('rgb histogram', plot);
                cv.waitKey(1);
            } catch (error) {
                console.log(error)
            }

        }

        cv.imshow('rgb image', mat);
        
        cv.waitKey(1);
        detectPerson()

    });
}