Skip to content

Instantly share code, notes, and snippets.

@PhoenixIllusion

PhoenixIllusion/index.html

Last active October 3, 2025 15:05
Show Gist options
  • Save PhoenixIllusion/47bb9ea7467997abe3690f62cd15e7f0 to your computer and use it in GitHub Desktop.
Save PhoenixIllusion/47bb9ea7467997abe3690f62cd15e7f0 to your computer and use it in GitHub Desktop.
Download ZIP
Harpagia: Zen Video Tracking tool
Raw
index.html
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Harpagia: Zen Video Tracking</title>
<style>
body > .container {
position: relative;
}
.debug-overlay {
position: absolute;
top: calc(1px * var(--grid-top));
left: calc(1px * var(--grid-left));
width: calc(1px * calc(var(--grid-right) - var(--grid-left)));
height: calc(1px * calc(var(--grid-bottom) - var(--grid-top)));
.grid {
position: absolute;
top: calc(1px * var(--grid-y-inset, 0));
left: calc(1px * var(--grid-x-inset, 0));
right: 0;
bottom: 0;
display: grid;
grid-template-columns: repeat(var(--grid-x-count, 1), calc(1px * var(--grid-x-width, 0)));
grid-template-rows: repeat(var(--grid-y-count, 1), calc(1px * var(--grid-y-width, 0)));
row-gap: calc(1px * var(--grid-y-gap, 0));
column-gap: calc(1px * var(--grid-x-gap, 0));
.tile {
background: rgba(255,0,255, 0.5);
opacity: 0.7;
}
}
}
</style>
</head>
<body>
<div id="app"></div>
<button id="startCapture">Start Window Capture</button>
<button id="setupGrid">Setup Grid</button>
<div class="container">
<video id="videoFeed" muted></video>
<div class="debug-overlay" style="display: none">
<canvas id="captureCanvas"></canvas>
<div class="grid"></div>
</div>
</div>
<script type="module" src="./src/main.ts"></script>
</body>
</html>
Raw
main.ts
const startButton = document.getElementById('startCapture') as HTMLButtonElement;
const video = document.getElementById('videoFeed') as HTMLVideoElement;
const canvas = document.getElementById('captureCanvas') as HTMLCanvasElement;
const setupGrid = document.getElementById('setupGrid') as HTMLCanvasElement;
let ctx = canvas.getContext('2d', { willReadFrequently: true});
interface Rect { x: number, y: number, width: number, height: number};
let readRegion: Rect | null = null;
let tileData: {x: TileDim, y: TileDim} = { x: { inset: -1, dim: -1, gap: -1}, y: { inset: -1, dim: -1, gap: -1} }
let tiles: HTMLDivElement[] = [];
const dbg_overlay = document.querySelector('.debug-overlay') as HTMLDivElement;
const debugGrid = dbg_overlay?.querySelector('.grid') as HTMLDivElement;
function luma(r: number, g: number, b: number): number {
return (r * 2126 + g * 7152 + b * 722) >> 13; // scaled to 0–255
}
/*
function chromaDistance(c1: number, c2: number): number {
const r1 = (c1 >> 16) & 0xFF, g1 = (c1 >> 8) & 0xFF, b1 = c1 & 0xFF;
const r2 = (c2 >> 16) & 0xFF, g2 = (c2 >> 8) & 0xFF, b2 = c2 & 0xFF;
const drg = (r1 - g1) - (r2 - g2);
const dbg = (b1 - g1) - (b2 - g2);
return drg * drg + dbg * dbg; // No sqrt
}
*/
function colorDistance(c1: number, c2: number): number {
const r1 = (c1 >> 16) & 0xFF, g1 = (c1 >> 8) & 0xFF, b1 = c1 & 0xFF;
const r2 = (c2 >> 16) & 0xFF, g2 = (c2 >> 8) & 0xFF, b2 = c2 & 0xFF;
const dluma = luma(r1, g1, b1) - luma(r2, g2, b2);
const drg = (r1 - g1) - (r2 - g2);
const dbg = (b1 - g1) - (b2 - g2);
return dluma * dluma + drg * drg + dbg * dbg;
}
const BG_COLOR = 0x5e4733;
const TILE_COLOR = 0x555555;
const TRAIN_COLOR = 0xdc9735;
const GREEN_u32 = 0x5cac24;
const CYAN_u32_mask = 0x84a000;
class VideoFrameReader {
buffer = new ArrayBuffer(0);
uint32 = new Uint32Array(this.buffer);
width = 0;
height = 0;
copyData(video: HTMLVideoElement, rect: Rect) {
const frame = new VideoFrame(video);
if(rect.width != this.width || rect.height != this.height) {
this.width = rect.width;
this.height = rect.height;
this.buffer = new ArrayBuffer(this.width * this.height * 4);
this.uint32 = new Uint32Array(this.buffer);
}
frame.copyTo(this.buffer, { rect, format: 'RGBX'});
frame.close();
}
getPixel(x: number, y: number) { return this.uint32[x + y * this.width] & 0xffffff; }
testPixel(x: number, y: number, color: number) {
return colorDistance(this.getPixel(x,y),(color & 0xffffff)) < 50;
}
countPixel(x: number, y: number, dx: number, dy: number, count: number, color: number) {
let c = 0;
for(let i=0; i< count; i++) {
if(this.testPixel(x + i * dx, y + i * dy, color)) c++;
}
return c;
}
}
const frameReader = new VideoFrameReader();
function onFrameRequest(_dt: number) {
if (readRegion) {
frameReader.copyData(video, readRegion);
analyzeFeatures(frameReader);
}
requestAnimationFrame(onFrameRequest)
}
startButton.onclick = async () => {
try {
const stream = await navigator.mediaDevices.getDisplayMedia({
video: { displaySurface: 'window' },
audio: false
});
video.srcObject = stream;
// Wait for video to load
await video.play();
video.requestVideoFrameCallback(() => {requestAnimationFrame(onFrameRequest)})
} catch (err) {
console.error('Error capturing window:', err);
}
};
interface TileDim { inset: number, dim: number, gap: number}
function setCssVars(data: Record<string,any>) {
Object.entries(data).forEach(([k,v]) => {
dbg_overlay.attributeStyleMap.set('--'+k, ''+v)
})
}
const gridFindReader = new VideoFrameReader();
setupGrid.onclick = function () {
gridFindReader.copyData(video, { x: 0, y: 0, width: video.videoWidth, height: video.videoHeight});
let x = 0, y = gridFindReader.height/2;
let hasSeen = false;
for(x = 10; x < gridFindReader.width/2; x++) {
if(hasSeen)
if(!gridFindReader.testPixel(x,y, BG_COLOR))
break;
if(gridFindReader.testPixel(x,y, BG_COLOR))
hasSeen = true;
}
if( x == gridFindReader.width/2)
return;
const left = x;
let top = -1;
let bottom = -1;
const y_offset = Math.floor(gridFindReader.height/2);
for(let y = 0; y < y_offset && (top < 0 || bottom < 0); y++) {
if(top < 0) {
if(gridFindReader.countPixel(left, y_offset - y, 5, 0, 4, BG_COLOR) == 4)
top = y_offset - y;
}
if(bottom < 0) {
if(gridFindReader.countPixel(left, y_offset + y, 5, 0, 4, BG_COLOR) == 4)
bottom = y_offset + y;
}
}
if( top < 0 || bottom < 0)
return;
setCssVars({
'grid-left': left, 'grid-right': (gridFindReader.width-left),
'grid-top': top, 'grid-bottom': bottom
})
hasSeen = false;
tileData = { x: { inset: -1, dim: -1, gap: -1}, y: { inset: -1, dim: -1, gap: -1} }
let prevX = false;
let prevY = false;
function setTileDim(tile: TileDim, prev: boolean, cur: boolean, i: number) {
if(cur && !prev) {
if(tile.inset < 0) tile.inset = i;
else
if(tile.gap < 0) tile.gap = i - tile.dim - tile.inset;
}
if(!cur && prev) {
if(tile.dim < 0) tile.dim = i - tile.inset;
}
}
for(let i=0; i < 300; i++) {
const x = gridFindReader.testPixel(left+ i, top + 30, TILE_COLOR) && gridFindReader.testPixel(left+ i + 1, top + 30, TILE_COLOR);
const y = gridFindReader.testPixel(left+ 30, top + i, TILE_COLOR) && gridFindReader.testPixel(left+ 30, top + i + 1, TILE_COLOR);
setTileDim(tileData.x, prevX, x, i);
setTileDim(tileData.y, prevY, y, i);
if(tileData.x.gap >=0 && tileData.y.gap >= 0)
break;
prevX = x;
prevY = y;
}
const gridWidth = gridFindReader.width - 2 * left;
const colCount = Math.round((gridWidth-2*tileData.x.inset+tileData.x.gap)/(tileData.x.dim+tileData.x.gap))
const gridHeight = bottom-top;
const rowCount = Math.round((gridHeight-2*tileData.y.inset+tileData.y.gap)/(tileData.y.dim+tileData.y.gap));
setCssVars({
'grid-x-count': colCount, 'grid-x-inset': tileData.x.inset, 'grid-x-width': tileData.x.dim, 'grid-x-gap': tileData.x.gap,
'grid-y-count': rowCount, 'grid-y-inset': tileData.y.inset, 'grid-y-width': tileData.y.dim, 'grid-y-gap': tileData.y.gap,
})
debugGrid?.querySelectorAll('.tile').forEach( x => x.remove())
tiles = [];
let tileTop = tileData.y.inset;
for(let y = 0; y < rowCount; y++) {
let tileLeft = tileData.x.inset;
for(let x = 0; x < colCount; x++){
const tile = document.createElement('div');
tile.className = 'tile';
const meta: Rect = {
x: tileLeft, y: tileTop, width: tileData.x.dim, height: tileData.y.dim
};
(tile as any).meta = meta;
tiles.push(tile);
debugGrid?.append(tile);
tileLeft += tileData.x.gap + tileData.x.dim
}
tileTop += tileData.y.gap + tileData.y.dim
}
dbg_overlay && (dbg_overlay.style.display = 'block');
canvas.width = gridFindReader.width - 2 * left;
canvas.height = bottom - top;
ctx = canvas.getContext('2d', { willReadFrequently: true});
readRegion = { y: top, x: left, width: canvas.width, height: canvas.height}
}
function isTrain(reader: VideoFrameReader): boolean {
const offsetX = 50
const offsetY = Math.round(reader.height/2) - 10;
return reader.countPixel(offsetX, offsetY, 5, 0, 20, TRAIN_COLOR) > 12;
}
function testTile(data: Uint32Array, width: number, loc: Rect) {
let max = -1;
const histogram: Record<number,number> = {}
for(let y=0;y<10;y++)
for(let x=0;x<10;x++) {
const c = data[(3*x+loc.x+5) + (3*y + loc.y+5) * width];
if(max < 0) max = c;
const count = histogram[c] = (histogram[c]||0)+1;
if(count > histogram[max]) max = c;
}
return max;
}
let gameState = {
active: false,
phase: -1,
sequence: [] as any[],
user_step: 0,
last_seen: -1
}
function analyzeFeatures(reader: VideoFrameReader) {
const menuVisible = isTrain(reader);
dbg_overlay.classList.toggle('is-train', menuVisible);
if(!menuVisible) {
const NOW = performance.now();
if(!gameState.active) {
gameState.active = true;
gameState.last_seen = NOW;
gameState.phase = 0;
}
let activeTile: HTMLElement | undefined = undefined;
if(gameState.phase == 1) {
activeTile = gameState.sequence[gameState.sequence.length-1-gameState.user_step]
}
tiles.forEach((tile) => {
const tileActive = testTile(reader.uint32, reader.width, (tile as any).meta);
const active = ((tileActive & 0xffff00) == CYAN_u32_mask);
const is_green = ((tileActive & 0xfcfcfc) == GREEN_u32);
tile.style.backgroundColor = active ? 'red' : 'transparent';
if(is_green && tile == activeTile) {
gameState.user_step ++;
}
if(active) {
if(gameState.sequence[0] != tile) gameState.sequence.unshift(tile);
gameState.last_seen = NOW;
if(gameState.phase == 1) {
gameState.phase = 0;
gameState.user_step = 0;
gameState.sequence = []
console.log('Entering Pattern Display Mode')
}
}
})
if(NOW - gameState.last_seen > 500 && gameState.phase == 0) {
gameState.phase = 1;
console.log('Entering User Enter Mode')
}
if(gameState.phase == 1) {
activeTile = gameState.sequence[gameState.sequence.length-1-gameState.user_step]
activeTile && (activeTile.style.backgroundColor = 'magenta');
const nextTile = gameState.sequence[gameState.sequence.length-2-gameState.user_step]
ctx?.clearRect(0,0, ctx.canvas.width, ctx.canvas.height);
if(activeTile && nextTile) {
const curRect: Rect = (activeTile as any).meta;
const nextRect: Rect = (nextTile as any).meta;
if(ctx) {
ctx.strokeStyle = gameState.user_step == gameState.sequence.length - 2 ? 'yellow': 'magenta';
ctx.beginPath();
ctx.moveTo(curRect.x + curRect.width/2, curRect.y + curRect.height/2);
ctx.lineTo(nextRect.x + nextRect.width/2, nextRect.y + nextRect.height/2);
ctx.stroke();
}
}
}
} else {
if(gameState.active) {
console.log(gameState);
console.log('clearing game state')
gameState = {
active: false,
phase: -1,
sequence: [] as any[],
user_step: 0,
last_seen: -1
}
}
}
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment