spooky_mode.py

import time
import math
import random
import numpy as np
import pyvirtualcam
import cv2

# Toggle to visually debug sprite sheets on the output stream
_DEBUG_GESTURES = False  # gesture diagnostics
_UNMIRROR_DEBUG_TEXT = True  # if True, flip debug text region back for readability
_MIRROR_OUTPUT = True  # overall mirror for user-friendly webcam feel

# Use MediaPipe Face Mesh & Hands
try:
    import mediapipe as mp
except ImportError as e:
    raise ImportError(
        "This cell now uses MediaPipe. Install with: pip install mediapipe"
    ) from e

mp_face_mesh = mp.solutions.face_mesh
mp_hands = mp.solutions.hands

# Global handles so we can close on shutdown
_face_mesh = None
_hands = None

# Jump scare
_jump_scare_img = None
_jump_scare_end_time = 0

# Bats & Skeletons
_bat_sprite_sheet = None
_bats = []
_skeleton_sprite_sheet = None
_skeletons = []

# Effect toggle state
_eyes_enabled = False             # googly eyes start OFF
_halloween_enabled = False        # bats + skeletons start OFF per user request
_eye_wide_start = None            # timestamp when wide eyes first detected
_hands_raised_start = None        # timestamp when both raised open hands detected
_last_eyes_toggle_time = 0.0
_last_halloween_toggle_time = 0.0

# Gesture timing constants
EYES_TOGGLE_WIDE_SECONDS = 2.0     # hold wide eyes duration
HANDS_TOGGLE_HOLD_SECONDS = 1.0
TOGGLE_COOLDOWN_SECONDS = 2.0

# Eye & hand threshold constants
WIDE_EYE_THRESHOLD = 0.36          # openness ratio to consider "wide"
WRIST_RAISED_THRESHOLD = 0.50      # wrist.y < threshold => raised
REQUIRED_FINGERS_OPEN = 3          # minimum fingertips above wrist
FINGER_DELTA = 0.015               # vertical margin above wrist

# Sprite constants
BAT_FRAMES = 1
SKELETON_FRAMES = 1
_bat_frame_count = None
_skeleton_frame_count = None

def _ensure_alpha(img: np.ndarray | None) -> np.ndarray | None:
    if img is None:
        return None
    if img.ndim == 2:
        bgr = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR)
        alpha = np.full((bgr.shape[0], bgr.shape[1], 1), 255, dtype=np.uint8)
        return np.concatenate([bgr, alpha], axis=2)
    if img.shape[2] == 3:
        alpha = np.full((img.shape[0], img.shape[1], 1), 255, dtype=np.uint8)
        return np.concatenate([img, alpha], axis=2)
    return img

def _extract_frame_from_sheet(sheet: np.ndarray, frame_idx: int, expected_frames: int) -> np.ndarray:
    sheet = _ensure_alpha(sheet)
    if sheet is None:
        return None
    if expected_frames <= 1:
        return sheet
    h, w = sheet.shape[:2]
    if w % expected_frames == 0:
        frame_w = w // expected_frames
        x0 = frame_idx * frame_w
        x1 = min(w, x0 + frame_w)
        return sheet[:, x0:x1]
    return sheet

def _detect_frame_count(sheet: np.ndarray | None, hint: int) -> int:
    sheet = _ensure_alpha(sheet)
    if sheet is None:
        return 1
    return 1

class Bat:
    def __init__(self, w, h):
        self.w = w
        self.h = h
        self.angle = random.uniform(0, 2 * math.pi)
        self.radius = random.uniform(w / 4, w / 2.5)
        self.orbit_speed = random.uniform(0.02, 0.05) * random.choice([-1, 1])
        self.x = w / 2 + self.radius * math.cos(self.angle)
        self.y = h / 2 + self.radius * math.sin(self.angle)
        self.frame = 0
        self.last_update = time.time()

    def update(self, t, head_center):
        self.angle += self.orbit_speed
        if self.angle > 2 * math.pi:
            self.angle -= 2 * math.pi
        elif self.angle < 0:
            self.angle += 2 * math.pi
        self.x = head_center[0] + self.radius * math.cos(self.angle)
        self.y = head_center[1] + self.radius * math.sin(self.angle)
        if t - self.last_update > 0.15:
            frames = globals().get('_bat_frame_count') or BAT_FRAMES
            self.frame = (self.frame + 1) % max(1, frames)
            self.last_update = t

    def draw(self, frame):
        if _bat_sprite_sheet is None:
            return
        sprite = _extract_frame_from_sheet(_bat_sprite_sheet, self.frame, globals().get('_bat_frame_count') or BAT_FRAMES)
        if sprite is None or sprite.size == 0:
            return
        orig_h, orig_w = sprite.shape[:2]
        scale = 0.8
        bat_w = max(24, int(orig_w * scale))
        bat_h = max(18, int(orig_h * scale))
        sprite = cv2.resize(sprite, (bat_w, bat_h), interpolation=cv2.INTER_AREA)
        y1, y2 = int(self.y - bat_h / 2), int(self.y + bat_h / 2)
        x1, x2 = int(self.x - bat_w / 2), int(self.x + bat_w / 2)
        if y1 < self.h and x1 < self.w and y2 > 0 and x2 > 0:
            y1_clamp = max(0, y1)
            y2_clamp = min(self.h, y2)
            x1_clamp = max(0, x1)
            x2_clamp = min(self.w, x2)
            sprite_y1 = max(0, -y1)
            sprite_y2 = sprite_y1 + (y2_clamp - y1_clamp)
            sprite_x1 = max(0, -x1)
            sprite_x2 = sprite_x1 + (x2_clamp - x1_clamp)
            region = sprite[sprite_y1:sprite_y2, sprite_x1:sprite_x2]
            if region.size == 0:
                return
            region = _ensure_alpha(region)
            alpha_s = region[:, :, 3] / 255.0
            alpha_l = 1.0 - alpha_s
            for c in range(0, 3):
                frame[y1_clamp:y2_clamp, x1_clamp:x2_clamp, c] = (
                    alpha_s * region[:, :, c] + alpha_l * frame[y1_clamp:y2_clamp, x1_clamp:x2_clamp, c]
                ).astype(np.uint8)

class Skeleton:
    def __init__(self, w, h, sheet: np.ndarray):
        self.w = w
        self.h = h
        self.sheet = _ensure_alpha(sheet)
        self.last_update = time.time()
        self.flip = False
        self.flip_interval = 0.5
        orig_h, orig_w = self.sheet.shape[:2]
        target_h = min(320, max(240, int(h * 0.28)))
        self.scale = target_h / orig_h
        self.draw_w = int(orig_w * self.scale)
        self.draw_h = int(orig_h * self.scale)
        self.x = 0
        self.y = self.h - self.draw_h - 20

    def update(self, t):
        if t - self.last_update > self.flip_interval:
            self.flip = not self.flip
            self.last_update = t

    def draw(self, frame):
        if self.sheet is None:
            return
        sprite = cv2.resize(self.sheet, (self.draw_w, self.draw_h), interpolation=cv2.INTER_AREA)
        if self.flip:
            sprite = cv2.flip(sprite, 1)
        bob = int(6 * math.sin(time.time() * 2.2))
        y1, y2 = int(self.y + bob), int(self.y + bob + self.draw_h)
        x1, x2 = int(self.x), int(self.x + self.draw_w)
        if y1 < self.h and x1 < self.w and y2 > 0 and x2 > 0:
            y1_clamp = max(0, y1)
            y2_clamp = min(self.h, y2)
            x1_clamp = max(0, x1)
            x2_clamp = min(self.w, x2)
            sprite_y1 = max(0, -y1)
            sprite_y2 = sprite_y1 + (y2_clamp - y1_clamp)
            sprite_x1 = max(0, -x1)
            sprite_x2 = sprite_x1 + (x2_clamp - x1_clamp)
            region = sprite[sprite_y1:sprite_y2, sprite_x1:sprite_x2]
            if region.size == 0:
                return
            region = _ensure_alpha(region)
            alpha_s = region[:, :, 3] / 255.0
            alpha_l = 1.0 - alpha_s
            for c in range(0, 3):
                frame[y1_clamp:y2_clamp, x1_clamp:x2_clamp, c] = (
                    alpha_s * region[:, :, c] + alpha_l * frame[y1_clamp:y2_clamp, x1_clamp:x2_clamp, c]
                ).astype(np.uint8)

def _clamp(val: float, lo: float, hi: float) -> float:
    return max(lo, min(hi, val))

def _draw_googly_eye(img: np.ndarray, center: tuple[int, int], eye_radius: int, pupil_offset: tuple[float, float]) -> None:
    x, y = int(center[0]), int(center[1])
    r = int(max(4, eye_radius))
    cv2.circle(img, (x, y), r, (255, 255, 255), -1, lineType=cv2.LINE_AA)
    cv2.circle(img, (x, y), r, (0, 0, 0), 2, lineType=cv2.LINE_AA)
    pupil_r = max(3, int(r * 0.35))
    max_offset = r - pupil_r - 2
    dx, dy = pupil_offset
    mag = math.hypot(dx, dy)
    if mag > 1e-6:
        scale = _clamp(max_offset / mag, 0.0, 1.0)
        dx *= scale
        dy *= scale
    px = int(round(x + dx))
    py = int(round(y + dy))
    cv2.circle(img, (px, py), pupil_r, (30, 30, 30), -1, lineType=cv2.LINE_AA)
    cv2.circle(img, (px - pupil_r // 3, py - pupil_r // 3), max(1, pupil_r // 4), (230, 230, 230), -1, lineType=cv2.LINE_AA)

def _eyes_wide(face_landmarks, w, h):
    def lm_xy(idx: int):
        lm = face_landmarks.landmark[idx]
        return lm.x * w, lm.y * h
    r_outer = lm_xy(33)
    r_inner = lm_xy(133)
    l_outer = lm_xy(263)
    l_inner = lm_xy(362)
    right_width = math.dist(r_outer, r_inner)
    left_width = math.dist(l_outer, l_inner)
    r_top = lm_xy(159)
    r_bottom = lm_xy(145)
    l_top = lm_xy(386)
    l_bottom = lm_xy(374)
    right_open = math.dist(r_top, r_bottom) / max(1e-6, right_width)
    left_open = math.dist(l_top, l_bottom) / max(1e-6, left_width)
    wide = right_open > WIDE_EYE_THRESHOLD and left_open > WIDE_EYE_THRESHOLD
    return wide, right_open, left_open

def _evaluate_hands(hand_landmarks_list):
    diagnostics = []
    if not hand_landmarks_list:
        return diagnostics
    for hand in hand_landmarks_list:
        wrist = hand.landmark[0]
        tips = [hand.landmark[i] for i in (4,8,12,16,20)]
        open_count = sum(1 for tip in tips if tip.y < wrist.y - FINGER_DELTA)
        wrist_y = wrist.y
        raised = wrist_y < WRIST_RAISED_THRESHOLD
        hand_ok = (open_count >= REQUIRED_FINGERS_OPEN) and raised
        diagnostics.append({
            'open_count': open_count,
            'wrist_y': wrist_y,
            'raised': raised,
            'ok': hand_ok
        })
    return diagnostics

def _both_hands_open_and_raised(hand_landmarks_list):
    diags = _evaluate_hands(hand_landmarks_list)
    if len(diags) < 2:
        return False, diags
    ok_hands = sum(1 for d in diags if d['ok'])
    return ok_hands >= 2, diags

def edit_frame(frame: np.ndarray, t: float) -> np.ndarray:
    global _jump_scare_end_time, _eyes_enabled, _halloween_enabled, _eye_wide_start, _hands_raised_start, _last_eyes_toggle_time, _last_halloween_toggle_time
    out = cv2.flip(frame, 1) if _MIRROR_OUTPUT else frame.copy()
    if _face_mesh is None or _hands is None:
        return out

    if _jump_scare_end_time > t:
        h_js, w_js = out.shape[:2]
        return cv2.resize(_jump_scare_img, (w_js, h_js))

    h, w = out.shape[:2]
    rgb = cv2.cvtColor(out, cv2.COLOR_BGR2RGB)
    rgb.flags.writeable = False
    face_results = _face_mesh.process(rgb)
    hand_results = _hands.process(rgb)
    rgb.flags.writeable = True

    right_open = left_open = None

    if face_results.multi_face_landmarks:
        face_landmarks = face_results.multi_face_landmarks[0]
        eyes_wide_now, right_open, left_open = _eyes_wide(face_landmarks, w, h)
        if eyes_wide_now:
            if _eye_wide_start is None:
                _eye_wide_start = t
            else:
                hold_time = t - _eye_wide_start
                if (hold_time >= EYES_TOGGLE_WIDE_SECONDS and
                    (t - _last_eyes_toggle_time) >= TOGGLE_COOLDOWN_SECONDS):
                    _eyes_enabled = not _eyes_enabled
                    _last_eyes_toggle_time = t
                    if _DEBUG_GESTURES:
                        print(f"[Gesture] Eyes toggled via WIDE -> {_eyes_enabled} at t={t:.2f}")
                    _eye_wide_start = None
        else:
            _eye_wide_start = None

        def lm_xy(idx: int):
            lm = face_landmarks.landmark[idx]
            return int(lm.x * w), int(lm.y * h)

        if _eyes_enabled:
            r_outer = np.array(lm_xy(33), dtype=np.float32)
            r_inner = np.array(lm_xy(133), dtype=np.float32)
            r_iris = np.array(lm_xy(468), dtype=np.float32)
            l_outer = np.array(lm_xy(263), dtype=np.float32)
            l_inner = np.array(lm_xy(362), dtype=np.float32)
            l_iris = np.array(lm_xy(473), dtype=np.float32)
            r_center = (r_outer + r_inner) * 0.5
            l_center = (l_outer + l_inner) * 0.5
            r_radius = max(6.0, 1.2 * (np.linalg.norm(r_outer - r_inner) * 0.5))
            l_radius = max(6.0, 1.2 * (np.linalg.norm(l_outer - l_inner) * 0.5))
            r_offset = (r_iris - r_center)
            l_offset = (l_iris - l_center)
            wobble = 0.3 * math.sin(6.0 * t)
            r_offset *= (1.0 + wobble)
            l_offset *= (1.0 + wobble)
            _draw_googly_eye(out, (int(r_center[0]), int(r_center[1])), int(r_radius), (float(r_offset[0]), float(r_offset[1])))
            _draw_googly_eye(out, (int(l_center[0]), int(l_center[1])), int(l_radius), (float(l_offset[0]), float(l_offset[1])))

        upper_lip = lm_xy(13)
        lower_lip = lm_xy(14)
        mouth_opening = np.linalg.norm(np.array(upper_lip) - np.array(lower_lip))
        if mouth_opening > 50:
            _jump_scare_end_time = t + 1

    hands_list = hand_results.multi_hand_landmarks if hand_results.multi_hand_landmarks else []
    hands_condition, hands_diags = _both_hands_open_and_raised(hands_list)
    if hands_condition:
        if _hands_raised_start is None:
            _hands_raised_start = t
        else:
            hold_time = t - _hands_raised_start
            if (hold_time >= HANDS_TOGGLE_HOLD_SECONDS and
                (t - _last_halloween_toggle_time) >= TOGGLE_COOLDOWN_SECONDS):
                _halloween_enabled = not _halloween_enabled
                _last_halloween_toggle_time = t
                if _DEBUG_GESTURES:
                    print(f"[Gesture] Halloween toggled -> {_halloween_enabled} at t={t:.2f}")
                _hands_raised_start = None
    else:
        _hands_raised_start = None

    if _halloween_enabled:
        head_center = (w // 2, h // 2)
        if face_results.multi_face_landmarks:
            lm = face_results.multi_face_landmarks[0].landmark[1]
            head_center = (int(lm.x * w), int(lm.y * h))
        for bat in _bats:
            bat.update(t, head_center)
            bat.draw(out)
        for skeleton in _skeletons:
            skeleton.update(t)
            skeleton.draw(out)

    status_lines = [
        f"Eyes: {'ON' if _eyes_enabled else 'OFF'}",
        f"Spooky: {'ON' if _halloween_enabled else 'OFF'}"
    ]
    y_text = 18
    for line in status_lines:
        cv2.putText(out, line, (10, y_text), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0), 2, lineType=cv2.LINE_AA)
        y_text += 24

    if _DEBUG_GESTURES:
        dbg_lines = []
        if right_open is not None and left_open is not None:
            wide_hold = 0.0
            if _eye_wide_start is not None:
                wide_hold = t - _eye_wide_start
            dbg_lines.append(f"EyesWide R:{right_open:.2f} L:{left_open:.2f} thr>{WIDE_EYE_THRESHOLD:.2f}")
            dbg_lines.append(f"WideHold {wide_hold:.2f}/{EYES_TOGGLE_WIDE_SECONDS:.1f}s")
        if hands_diags:
            for i, d in enumerate(hands_diags):
                dbg_lines.append(f"Hand{i+1} open:{d['open_count']} wristY:{d['wrist_y']:.2f} raised:{d['raised']} ok:{d['ok']}")
            hand_hold = 0.0
            if _hands_raised_start is not None:
                hand_hold = t - _hands_raised_start
            dbg_lines.append(f"HandsHold {hand_hold:.2f}/{HANDS_TOGGLE_HOLD_SECONDS:.1f}s cond:{hands_condition}")
        dbg_lines.append(f"CooldownEyes {(t - _last_eyes_toggle_time):.1f}s")
        dbg_lines.append(f"CooldownHall {(t - _last_halloween_toggle_time):.1f}s")
        for l in dbg_lines:
            cv2.putText(out, l, (10, y_text), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (255,200,0), 1, lineType=cv2.LINE_AA)
            y_text += 18
        if _UNMIRROR_DEBUG_TEXT and _MIRROR_OUTPUT:
            region_w = min(w, 600)
            region_h = y_text + 4
            region = out[0:region_h, 0:region_w]
            region = cv2.flip(region, 1)
            out[0:region_h, 0:region_w] = region

    return out

camera_index = 0
cap = cv2.VideoCapture(camera_index)
if not cap.isOpened():
    raise RuntimeError(f"Failed to open camera index {camera_index}")
ok, frame = cap.read()
if not ok or frame is None:
    cap.release()
    raise RuntimeError("Failed to read initial frame from the camera")
h, w = frame.shape[:2]
fps = cap.get(cv2.CAP_PROP_FPS)
if fps is None or fps <= 0 or (isinstance(fps, float) and np.isnan(fps)):
    fps = 30.0
print(f"Input camera opened: index={camera_index}, size={w}x{h}, fps≈{fps:.1f}")
try:
    _jump_scare_img = cv2.imread('/Users/kennyr/workspace/notebooks/data/scary_face.jpeg')
    _bat_sprite_sheet = cv2.imread('/Users/kennyr/workspace/notebooks/data/bat_sprite.png', cv2.IMREAD_UNCHANGED)
    _skeleton_sprite_sheet = cv2.imread('/Users/kennyr/workspace/notebooks/data/skeleton_dance.png', cv2.IMREAD_UNCHANGED)
    print('bat sprite:', None if _bat_sprite_sheet is None else _bat_sprite_sheet.shape)
    print('skeleton sprite:', None if _skeleton_sprite_sheet is None else _skeleton_sprite_sheet.shape)
    _bat_sprite_sheet = _ensure_alpha(_bat_sprite_sheet)
    _skeleton_sprite_sheet = _ensure_alpha(_skeleton_sprite_sheet)
    _bat_frame_count = _detect_frame_count(_bat_sprite_sheet, BAT_FRAMES)
    _skeleton_frame_count = _detect_frame_count(_skeleton_sprite_sheet, SKELETON_FRAMES)
    print('detected bat frames:', _bat_frame_count)
    print('detected skel frames:', _skeleton_frame_count)
    for _ in range(5):
        _bats.append(Bat(w, h))
    desired_skeletons = 2
    for _ in range(desired_skeletons):
        _skeletons.append(Skeleton(w, h, _skeleton_sprite_sheet))
    if _skeletons:
        total = len(_skeletons)
        for i, sk in enumerate(_skeletons):
            center_x = (i + 1) / (total + 1) * w
            sk.x = int(center_x - sk.draw_w / 2)
    global _face_mesh, _hands
    _face_mesh = mp_face_mesh.FaceMesh(
        static_image_mode=False,
        refine_landmarks=True,
        max_num_faces=5,
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5,
    )
    _hands = mp_hands.Hands(
        static_image_mode=False,
        max_num_hands=2,
        model_complexity=0,
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5,
    )
    with pyvirtualcam.Camera(width=w, height=h, fps=int(round(fps))) as cam:
        print(f"Using virtual camera: {cam.device}")
        print("Streaming frames… Interrupt the kernel (stop button) to end.")
        t0 = time.time()
        while True:
            ok, frame = cap.read()
            if not ok or frame is None:
                continue
            t = time.time() - t0
            frame = edit_frame(frame, t)
            frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
            cam.send(frame_rgb)
            cam.sleep_until_next_frame()
except KeyboardInterrupt:
    print("Stopping stream.")
finally:
    if _face_mesh is not None:
        _face_mesh.close()
        _face_mesh = None
    if _hands is not None:
        _hands.close()
        _hands = None
    cap.release()
    print("Camera released and resources cleaned up.")