239 lines
9.9 KiB
Python
239 lines
9.9 KiB
Python
"""Camera hand-gesture pointer control. Installed to /opt/gesture-control/.
|
|
|
|
Open hand moves the pointer, closed fist clicks. Started by /usr/local/bin/gesture-control
|
|
from the sway config; see that wrapper for why it is a session process.
|
|
|
|
PRIVACY, and the reason the structure of this file looks the way it does: enabling this
|
|
means a camera continuously captures and analyses video of the room. main() therefore
|
|
reads gesture-config.json and returns before OpenCV or MediaPipe are imported at all —
|
|
the "enabled": false default does not mean the camera is opened and its frames dropped,
|
|
it means the video stack is never loaded and /dev/video* is never opened. Keep the
|
|
imports where they are.
|
|
|
|
Why this is not a module inside thinclient_agent/, despite reusing two of its modules:
|
|
that package is installed to /opt/thinclient-agent and runs on the SYSTEM interpreter
|
|
against apt's python3-paho-mqtt. MediaPipe is PyPI-only and lives in its own venv
|
|
(1100-gesture-control.hook.chroot, same PEP 668 reasoning as the voice satellite). A
|
|
module sitting in a package whose interpreter cannot import its own dependencies would
|
|
be a trap. The two modules it does import — input_control and runtime_state — are
|
|
stdlib-only, so they load fine from inside the venv via PYTHONPATH.
|
|
|
|
Why velocity control rather than trackpad-style frame-to-frame deltas: at the inference
|
|
rates this hardware can sustain (~10-15 fps, see README) a delta model is both jittery
|
|
and runs out of frame — you would have to lift and re-place your hand like a mouse. Hand
|
|
offset from the centre of the frame driving a pointer SPEED instead is self-recentering,
|
|
has an obvious rest state (hand in the middle = pointer stopped), and degrades into
|
|
"slightly slower pointer" rather than "wrong pointer" when frames are dropped.
|
|
|
|
The pointer anchor is the middle-finger knuckle (landmark 9), not a fingertip: it barely
|
|
moves as the fingers curl, so the open-hand-to-fist transition does not drag the pointer
|
|
off whatever you were about to click.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import math
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
from thinclient_agent.input_control import InputControl
|
|
from thinclient_agent.runtime_state import ensure_runtime_copy, load_json
|
|
|
|
log = logging.getLogger("gesture-control")
|
|
|
|
CONFIG_FILENAME = "gesture-config.json"
|
|
|
|
WRIST = 0
|
|
PALM_ANCHOR = 9
|
|
FINGER_TIPS = (8, 12, 16, 20)
|
|
FINGER_PIPS = (6, 10, 14, 18)
|
|
|
|
# Curl is measured per finger as "is the tip nearer the wrist than its middle joint",
|
|
# which is scale- and rotation-invariant and so needs no calibration for how far away
|
|
# the person is standing. The gap between the two thresholds is deliberate: a hand
|
|
# somewhere between the two states is neither, and does nothing.
|
|
FIST_MIN_CURLED = 4
|
|
OPEN_MAX_CURLED = 1
|
|
|
|
|
|
def _distance(a, b) -> float:
|
|
return math.hypot(a.x - b.x, a.y - b.y)
|
|
|
|
|
|
def _curled_fingers(landmarks) -> int:
|
|
wrist = landmarks[WRIST]
|
|
return sum(
|
|
1
|
|
for tip, pip in zip(FINGER_TIPS, FINGER_PIPS)
|
|
if _distance(landmarks[tip], wrist) < _distance(landmarks[pip], wrist)
|
|
)
|
|
|
|
|
|
class GesturePointer:
|
|
def __init__(self, config: dict, input_control: InputControl):
|
|
self._input = input_control
|
|
self._dead_zone = float(config.get("dead_zone") or 0.08)
|
|
self._speed = float(config.get("pointer_speed") or 900)
|
|
self._mirror = bool(config.get("mirror", True))
|
|
self._fist_hold = float(config.get("fist_hold_seconds") or 0.4)
|
|
self._cooldown = float(config.get("click_cooldown_seconds") or 1.0)
|
|
self._fist_since: float | None = None
|
|
self._click_armed = True
|
|
# -inf, not 0.0: these are time.monotonic() values, which are uptime-relative, so
|
|
# 0.0 would silently swallow the first click of a session started soon after boot.
|
|
self._last_click = float("-inf")
|
|
|
|
def _axis_delta(self, normalised: float, elapsed: float) -> float:
|
|
offset = normalised - 0.5
|
|
magnitude = abs(offset) - self._dead_zone
|
|
if magnitude <= 0:
|
|
return 0.0
|
|
# Rescaled so the speed ramps from zero at the edge of the dead zone up to the
|
|
# full configured speed at the frame edge, rather than jumping to a fraction of
|
|
# it the moment the dead zone is crossed.
|
|
travel = max(0.5 - self._dead_zone, 1e-6)
|
|
return math.copysign(magnitude / travel, offset) * self._speed * elapsed
|
|
|
|
def handle(self, landmarks, now: float, elapsed: float) -> None:
|
|
if landmarks is None:
|
|
self._fist_since = None
|
|
self._click_armed = True
|
|
return
|
|
|
|
curled = _curled_fingers(landmarks)
|
|
|
|
if curled >= FIST_MIN_CURLED:
|
|
# No movement while the fist is closed: a click that drifts the pointer
|
|
# between the press and whatever the user was aiming at is worse than a
|
|
# click that does not fire.
|
|
if self._fist_since is None:
|
|
self._fist_since = now
|
|
elif (
|
|
self._click_armed
|
|
and now - self._fist_since >= self._fist_hold
|
|
and now - self._last_click >= self._cooldown
|
|
):
|
|
log.info("fist held, clicking")
|
|
self._input.click("LEFT")
|
|
self._last_click = now
|
|
self._click_armed = False
|
|
return
|
|
|
|
self._fist_since = None
|
|
if curled <= OPEN_MAX_CURLED:
|
|
# Re-arming only on a clearly open hand, not merely on "not a fist", is what
|
|
# makes a held fist one click instead of a repeat.
|
|
self._click_armed = True
|
|
anchor = landmarks[PALM_ANCHOR]
|
|
x = 1.0 - anchor.x if self._mirror else anchor.x
|
|
self._input.move_relative(
|
|
self._axis_delta(x, elapsed), self._axis_delta(anchor.y, elapsed)
|
|
)
|
|
|
|
|
|
def run(config: dict) -> int:
|
|
# Imported here rather than at module scope so that the disabled default in main()
|
|
# never loads the video stack. See the module docstring.
|
|
import cv2
|
|
import mediapipe as mp
|
|
from mediapipe.tasks import python as mp_python
|
|
from mediapipe.tasks.python import vision as mp_vision
|
|
|
|
model_path = str(config.get("model_path") or "/opt/gesture-control/hand_landmarker.task")
|
|
device = str(config.get("camera_device") or "/dev/video0")
|
|
|
|
capture = cv2.VideoCapture(device, cv2.CAP_V4L2)
|
|
if not capture.isOpened():
|
|
log.error("could not open %s — is a camera plugged in and is this user in the "
|
|
"'video' group?", device)
|
|
return 1
|
|
|
|
capture.set(cv2.CAP_PROP_FRAME_WIDTH, int(config.get("frame_width") or 640))
|
|
capture.set(cv2.CAP_PROP_FRAME_HEIGHT, int(config.get("frame_height") or 480))
|
|
capture.set(cv2.CAP_PROP_FPS, int(config.get("capture_fps") or 30))
|
|
capture.set(cv2.CAP_PROP_BUFFERSIZE, 1)
|
|
|
|
inference_fps = float(config.get("inference_fps") or 12)
|
|
inference_period = 1.0 / inference_fps if inference_fps > 0 else 0.0
|
|
|
|
landmarker = mp_vision.HandLandmarker.create_from_options(
|
|
mp_vision.HandLandmarkerOptions(
|
|
base_options=mp_python.BaseOptions(model_asset_path=model_path),
|
|
# VIDEO rather than LIVE_STREAM: LIVE_STREAM hands results back on a callback
|
|
# thread, which buys nothing here because the loop below is already the only
|
|
# consumer and is deliberately rate-limited.
|
|
running_mode=mp_vision.RunningMode.VIDEO,
|
|
num_hands=1,
|
|
min_hand_detection_confidence=0.6,
|
|
min_hand_presence_confidence=0.6,
|
|
min_tracking_confidence=0.5,
|
|
)
|
|
)
|
|
|
|
# Plain os.environ, not SwayControl.session_env(): unlike thinclient-agent, which is
|
|
# a system service outside the session and has to reconstruct SWAYSOCK/XDG_RUNTIME_DIR
|
|
# by hand, this process is started by sway itself and already has them.
|
|
pointer = GesturePointer(config, InputControl(os.environ.copy))
|
|
log.info("gesture control running on %s at ~%.0f inference fps", device, inference_fps)
|
|
|
|
# Seeded with the current time rather than 0, so the first analysed frame gets a
|
|
# sane elapsed and cannot start the session by flinging the pointer a clamped step.
|
|
last_inference = time.monotonic()
|
|
try:
|
|
while True:
|
|
# Every frame is read even though most are discarded: leaving them queued in
|
|
# V4L2 would mean the frame that does get analysed is progressively older
|
|
# than the hand actually in front of the camera.
|
|
ok, frame = capture.read()
|
|
if not ok:
|
|
log.warning("camera read failed; stopping")
|
|
return 1
|
|
|
|
now = time.monotonic()
|
|
if now - last_inference < inference_period:
|
|
continue
|
|
elapsed = min(now - last_inference, 0.5)
|
|
last_inference = now
|
|
|
|
image = mp.Image(
|
|
image_format=mp.ImageFormat.SRGB,
|
|
data=cv2.cvtColor(frame, cv2.COLOR_BGR2RGB),
|
|
)
|
|
result = landmarker.detect_for_video(image, int(now * 1000))
|
|
hands = getattr(result, "hand_landmarks", None) or []
|
|
pointer.handle(hands[0] if hands else None, now, elapsed)
|
|
except KeyboardInterrupt:
|
|
return 0
|
|
finally:
|
|
capture.release()
|
|
landmarker.close()
|
|
|
|
|
|
def main() -> int:
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
|
|
stream=sys.stdout,
|
|
)
|
|
|
|
config = load_json(ensure_runtime_copy(CONFIG_FILENAME))
|
|
if config.get("enabled") is not True:
|
|
log.info(
|
|
"gesture control is disabled (the default) — the camera will not be opened. "
|
|
"Set \"enabled\": true in /var/lib/thinclient-agent/%s to turn it on.",
|
|
CONFIG_FILENAME,
|
|
)
|
|
return 0
|
|
|
|
log.warning(
|
|
"gesture control is ENABLED — a camera is about to start continuously capturing "
|
|
"and analysing video of this room for as long as this session is up."
|
|
)
|
|
return run(config)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|