Skip to content

Instantly share code, notes, and snippets.

@Narottam04

Narottam04/index.html

Created November 26, 2022 13:25
Show Gist options
  • Save Narottam04/9d3383270101c69a6fd6389f589163a8 to your computer and use it in GitHub Desktop.
Save Narottam04/9d3383270101c69a6fd6389f589163a8 to your computer and use it in GitHub Desktop.
Download ZIP
Mediapipe hands is not working on iOS. Seems to work on all other platforms like windows, android
Raw
index.html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<!-- css -->
<link rel="stylesheet" href="/css/styles.css" ></link>
<!-- mediapipe and tensorflow library -->
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/camera_utils/camera_utils.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/control_utils/control_utils.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/drawing_utils/drawing_utils.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@mediapipe/hands/hands.js" crossorigin="anonymous"></script>
<script src="https://cdn.jsdelivr.net/npm/@tensorflow/tfjs/dist/tf.min.js"></script>
<script defer src="https://unpkg.com/[email protected]/dist/cdn.min.js"></script>
<!-- <link rel="stylesheet" href="style.css"> -->
<script type="module" src="/js/numbersQuiz.js"></script>
<title>Detect Alphabet</title>
</head>
<body >
<div class="fixed top-0 left-0 right-0 bottom-0 w-full h-full">
<video class="input_video fixed h-full w-full object-cover" ></video>
</div>
</div>
<canvas class="output_canvas fixed w-full h-full object-cover z-10"></canvas>
<div class="absolute left-0 right-0 py-8 content-center text-center z-50">
<h1 class="text-2xl md:text-4xl font-bold text-white font-white ">Show your hand to webcam to see the magic!</h1>
<h1 id="predection" class="py-2 text-[4rem] md:text-[6rem] font-extrabold text-white font-white "></h1>
<!-- <img src="/assets/asl_alphabets/Ahand.svg" alt="predection" > -->
</div>
</body>
<!-- <script type="module" src="../../main.js"></script> -->
</html>
Raw
numbersQuiz.js
// import cv from "@techstark/opencv-js";
// import Alpine from 'alpinejs'
const videoElement = document.getElementsByClassName('input_video')[0];
const canvasElement = document.getElementsByClassName('output_canvas')[0];
const canvasCtx = canvasElement.getContext('2d');
// predection h1
const showPredection = document.getElementById('predection')
window.Alpine = Alpine
// Alpine.data('dropdown', () => ({
// detectedValue:'Show your hands to see the magic!',
// }))
// Alpine.start()
// document.addEventListener('alpine:init', () => {
// Alpine.data('employee', () => ({
// name: 'John Doe',
// email: '[email protected]',
// age: '25',
// }));
// });
function calc_landmark_list(imgWidth,imgHeight,landmarks) {
const landmarkArray = landmarks.map((landmark) => {
const landmark_X = Math.round(Math.min((landmark.x * imgWidth), imgWidth - 1))
const landmark_Y = Math.round(Math.min((landmark.y * imgHeight), imgHeight - 1))
return [landmark_X,landmark_Y]
})
return landmarkArray
}
function preProcessLandmark(landmarks) {
let baseX = 0
let baseY = 0
const preProcessLandmark = landmarks.map((landmark,index) => {
if(index == 0){
baseX = landmark[0]
baseY = landmark[1]
}
return [(landmark[0] - baseX),(landmark[1] - baseY)]
})
// Convert to a one-dimensional list
const arr1d = [].concat(...preProcessLandmark);
// absolute value array
const absArray = arr1d.map(value => Math.abs(value))
// max value
const maxValue = Math.max(...absArray)
// normalization
const normalizedArray = arr1d.map((value) => value/maxValue)
return normalizedArray
}
function onResults(results) {
if(videoElement.readyState === 4){
// Get Video Properties
const videoWidth = videoElement.videoWidth
const videoHeight = videoElement.videoHeight
// set video width
videoElement.width = videoWidth
videoElement.height = videoHeight
// Set canvas width and height
canvasElement.width = videoWidth
canvasElement.height = videoHeight
// canvasElement.width = videoWidth ? videoWidth: window.innerWidth
// canvasElement.height = videoHeight ? videoHeight : window.innerHeight
// canvasElement.width = window.innerWidth
// canvasElement.height = window.innerHeight
canvasCtx.save();
canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
canvasCtx.drawImage(
results.image, 0, 0, canvasElement.width, canvasElement.height);
if (results.multiHandLandmarks) {
for (const landmarks of results.multiHandLandmarks) {
drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS,
{color: '#F0F0F0', lineWidth: 1});
// drawLandmarks(canvasCtx, landmarks, {color: '#FF0000', lineWidth: 1});
drawLandmarks(canvasCtx, landmarks, {color: '#F0F0F0', lineWidth: 2,
radius: (data) => {
return lerp(data.from.z, -0.15, .1, 10, 1);
}
});
}
// console.log(results.multiHandLandmarks[0])
}
if (results.multiHandLandmarks.length > 0){
// console.log("hello world")
const landmarks = calc_landmark_list(videoWidth,videoHeight,results.multiHandLandmarks[0])
const processedLandmarks = preProcessLandmark(landmarks)
const data = tf.tensor2d([processedLandmarks])
const predict = customModel.predict(data).dataSync()
const gesture = Math.max(...predict)
const gestureIndex = predict.indexOf(gesture)
showPredection.innerHTML = `${gestureIndex}`
// console.log(gestureIndex)
}
canvasCtx.restore();
}
}
const hands = new Hands({locateFile: (file) => {
return `https://cdn.jsdelivr.net/npm/@mediapipe/hands/${file}`;
}});
let customModel = null
hands.setOptions({
maxNumHands: 1,
modelComplexity: 1,
minDetectionConfidence: 0.5,
minTrackingConfidence: 0.5,
selfieMode: true,
});
hands.onResults(onResults);
async function loadModel(){
// https://raw.githubusercontent.com/Narottam04/SignLanguage/master/frontend/model/asl_numbers_1-9/model.json
// https://raw.githubusercontent.com/Narottam04/SignLanguage/master/frontend/model/asl_numbers_1-10/model.json
customModel = await tf.loadLayersModel("https://raw.githubusercontent.com/Narottam04/SignLanguage/master/frontend/model/asl_numbers_1-10/model.json")
}
loadModel()
const camera = new Camera(videoElement, {
onFrame: async () => {
await hands.send({image: videoElement});
},
});
camera.start();
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment