Files
flowforum/src/point.py
T

224 lines
7.5 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env python3
"""Point at a BCC145 — MediaPipe hand aiming + Art-Net.
Camera is centred on the row of 4 (#3–#6) and faces the performer
(toward +Y / audience). Point with your index finger; that can lights.
"""
from __future__ import annotations
import time
import urllib.request
from pathlib import Path
import cv2
import mediapipe as mp
import numpy as np
from can_lights import CanLights
from stage_layout import load_cans, pick_can
BaseOptions = mp.tasks.BaseOptions
HandLandmarker = mp.tasks.vision.HandLandmarker
HandLandmarkerOptions = mp.tasks.vision.HandLandmarkerOptions
VisionRunningMode = mp.tasks.vision.RunningMode
mp_drawing = mp.tasks.vision.drawing_utils
mp_drawing_styles = mp.tasks.vision.drawing_styles
mp_hands = mp.tasks.vision.HandLandmarksConnections
WRIST = 0
INDEX_MCP = 5
INDEX_TIP = 8
MIDDLE_TIP = 12
RING_TIP = 16
PINKY_TIP = 20
ROOT = Path(__file__).resolve().parent.parent
HAND_MODEL_PATH = ROOT / "models" / "hand_landmarker.task"
HAND_MODEL_URL = (
"https://storage.googleapis.com/mediapipe-models/"
"hand_landmarker/hand_landmarker/float16/1/hand_landmarker.task"
)
WINDOW_NAME = "flowforum-point"
CAMERA_WIDTH = 1920
CAMERA_HEIGHT = 1080
HOLD_NEED = 4
def ensure_model(path: Path, url: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
if path.exists():
return
print(f"Downloading {path.name}...")
urllib.request.urlretrieve(url, path)
def is_pointing(hand) -> bool:
"""Index extended; middle/ring/pinky curled back toward wrist."""
wrist = hand[WRIST]
index_tip = hand[INDEX_TIP]
index_mcp = hand[INDEX_MCP]
def dist(a, b) -> float:
return float(np.hypot(a.x - b.x, a.y - b.y))
if dist(index_tip, index_mcp) < 0.06:
return False
index_reach = dist(index_tip, wrist)
for tip_i in (MIDDLE_TIP, RING_TIP, PINKY_TIP):
if dist(hand[tip_i], wrist) > index_reach * 0.85:
return False
return True
def enter_fullscreen() -> None:
cv2.namedWindow(WINDOW_NAME, cv2.WINDOW_NORMAL)
cv2.imshow(WINDOW_NAME, np.zeros((2, 2, 3), dtype=np.uint8))
cv2.waitKey(1)
cv2.setWindowProperty(WINDOW_NAME, cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN)
cv2.waitKey(1)
def draw_stage_overlay(rgb: np.ndarray, cans, active_index: int | None) -> None:
h, w = rgb.shape[:2]
for can in cans:
x = int((1.0 - can.u) * w)
y = int(can.v * h)
on = can.index == active_index
color = (0, 255, 128) if on else (80, 80, 80)
cv2.circle(rgb, (x, y), 22 if on else 14, color, -1)
cv2.putText(
rgb,
can.label,
(x - 12, y + 5),
cv2.FONT_HERSHEY_SIMPLEX,
0.5,
(0, 0, 0) if on else (200, 200, 200),
1,
cv2.LINE_AA,
)
def main() -> None:
ensure_model(HAND_MODEL_PATH, HAND_MODEL_URL)
cans = load_cans()
lights = CanLights(colour="White")
enter_fullscreen()
cap = cv2.VideoCapture(0)
cap.set(cv2.CAP_PROP_FRAME_WIDTH, CAMERA_WIDTH)
cap.set(cv2.CAP_PROP_FRAME_HEIGHT, CAMERA_HEIGHT)
hand_options = HandLandmarkerOptions(
base_options=BaseOptions(model_asset_path=str(HAND_MODEL_PATH)),
running_mode=VisionRunningMode.VIDEO,
num_hands=2,
min_hand_detection_confidence=0.5,
min_tracking_confidence=0.5,
)
active: int | None = None
hold_index: int | None = None
hold_frames = 0
try:
with HandLandmarker.create_from_options(hand_options) as hand_landmarker:
start = time.monotonic()
last_timestamp_ms = -1
while cap.isOpened():
success, image = cap.read()
if not success:
print("Ignoring empty camera frame.")
continue
rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)
timestamp_ms = int((time.monotonic() - start) * 1000)
if timestamp_ms <= last_timestamp_ms:
timestamp_ms = last_timestamp_ms + 1
last_timestamp_ms = timestamp_ms
hand_results = hand_landmarker.detect_for_video(mp_image, timestamp_ms)
aimed: int | None = None
if hand_results.hand_landmarks:
for hand_landmarks in hand_results.hand_landmarks:
mp_drawing.draw_landmarks(
rgb,
hand_landmarks,
mp_hands.HAND_CONNECTIONS,
mp_drawing_styles.get_default_hand_landmarks_style(),
mp_drawing_styles.get_default_hand_connections_style(),
)
if not is_pointing(hand_landmarks):
continue
mcp = hand_landmarks[INDEX_MCP]
tip = hand_landmarks[INDEX_TIP]
can = pick_can((mcp.x, mcp.y), (tip.x, tip.y), cans)
if can is None:
continue
aimed = can.index
h, w = rgb.shape[:2]
p0 = (int(mcp.x * w), int(mcp.y * h))
p1 = (int(tip.x * w), int(tip.y * h))
extend = np.array([tip.x - mcp.x, tip.y - mcp.y], dtype=float)
n = float(np.linalg.norm(extend))
if n > 1e-3:
extend /= n
p2 = (
int((tip.x + extend[0] * 0.45) * w),
int((tip.y + extend[1] * 0.45) * h),
)
cv2.line(rgb, p0, p2, (0, 255, 255), 3)
cv2.circle(rgb, p1, 12, (255, 80, 0), -1)
break
if aimed is not None:
if aimed == hold_index:
hold_frames += 1
else:
hold_index = aimed
hold_frames = 1
if hold_frames >= HOLD_NEED and aimed != active:
lights.light_only(aimed)
active = aimed
print(f"ON {cans[aimed].label}", flush=True)
else:
hold_index = None
hold_frames = 0
draw_stage_overlay(rgb, cans, active)
label = cans[active].label if active is not None else "—"
cv2.putText(
rgb,
f"point → {label} [q] quit [b] blackout",
(40, 60),
cv2.FONT_HERSHEY_SIMPLEX,
1.0,
(240, 240, 240),
2,
cv2.LINE_AA,
)
display = cv2.flip(cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR), 1)
cv2.imshow(WINDOW_NAME, display)
cv2.setWindowProperty(
WINDOW_NAME, cv2.WND_PROP_FULLSCREEN, cv2.WINDOW_FULLSCREEN
)
key = cv2.waitKey(5) & 0xFF
if key == ord("q"):
break
if key == ord("b"):
lights.blackout()
active = None
print("blackout", flush=True)
finally:
lights.close()
cap.release()
cv2.destroyAllWindows()
if __name__ == "__main__":
main()