diff --git a/CoreSystemConfig.json.template b/CoreSystemConfig.json.template index e0dd45d..f45ace3 100644 --- a/CoreSystemConfig.json.template +++ b/CoreSystemConfig.json.template @@ -66,6 +66,8 @@ "portainer": true, "mealie": false, "gallery_smb": false, + "photos_web": false, + "workshop": false, "music_assistant": false, "backups": false } @@ -109,6 +111,9 @@ "transit": 8099, "otp": 8100, "music_assistant": 8101, + "workshop": 8102, + "workshop_web": 8103, + "photos_web": 2283, "ollama": 11434, "proxy_http": 80, "proxy_https": 443 @@ -122,22 +127,77 @@ "mqtt_username": "", "mqtt_password": "", "ha_token": "", - "opnsense_api_key": "", - "opnsense_api_secret": "", + "_opnsense_keys": "One entry per firewall in the opnsense list above, keyed by its `name`. A single-firewall household writes {\"main\": {...}} and is done. Keyed rather than positional because a list that has to line up with another list by index is a bug waiting for the day somebody reorders one of them.", + "opnsense_keys": { + "main": { "api_key": "", "api_secret": "" } + }, + "_checkmk": "A CheckMK automation user with the read-only Guest role, and its AUTOMATION SECRET (Setup -> Users -> the user -> Automation secret) \u2014 not its login password.", + "checkmk_username": "", + "checkmk_secret": "", + "_freeipa_bind_password": "The password for freeipa.bind_dn. Prefer a dedicated service account with READ-ONLY access to the household subtree \u2014 the mirror never writes to the directory, so a bind that can is a grant with no matching use.", + "freeipa_bind_password": "", + "_workshop_token": "Required when container_host.enable.workshop is true. openssl rand -hex 32", + "workshop_token": "", + "_gitea": "Optional, and only for the workshop assistant's own repositories. Gitea -> Settings -> Applications -> Generate New Token, scope write:repository. STRONGLY PREFER a dedicated 'workshop-bot' user scoped to one organisation over a token on your own account: this token can create repositories, and the blast radius of a leaked env file should be one org of generated repos rather than everything you own. The assistant commits and pushes but never rewrites history, and repo deletion is not implemented at all \u2014 see workshop/README.md.", + "gitea_url": "", + "gitea_token": "", + "gitea_owner": "", "ssh_authorized_key": "", "kiosk_password": "", "admin_password_hash": "" }, - "opnsense": { - "_comment": "The household's existing OPNsense firewall. Used by digest-engine's network digest, which pulls a Suricata intrusion-detection summary from it (GET /api/ids/service/status and POST /api/ids/service/query_alerts — two read endpoints, nothing else, ever). A build writes this block plus the secrets above into the container host's IDSconf.json; the long-form documentation for every field lives in digest-engine/IDSconf.json.example. Leave base_url empty to skip that file entirely. TWO THINGS THIS DOES NOT DO: it does not enable Suricata (do that at Services -> Intrusion Detection on the firewall, then download a ruleset), and it does not enable the ingest (set ENABLE_OPNSENSE_IDS_INGEST=true in digest-engine.env). SCOPE THE API KEY: give it its own OPNsense user with only the 'Services: Intrusion Detection' privilege — that privilege still covers api/ids/* including start/stop, because OPNsense ACLs are page-level, so the read-only guarantee comes from digest-engine calling exactly two endpoints and not from the firewall enforcing it.", + "opnsense": [ + { + "_comment": "EVERY OPNsense firewall in the household, as a list. It was a single object until more than one firewall became plausible; a list is the shape that does not need changing again, and a one-firewall household just has one entry. `name` distinguishes them everywhere downstream — in the digest's network section, in the workshop health table, and in the alert text itself — so 'the IDS is quiet' can never silently mean 'one of the two is quiet'. Used by digest-engine's network digest (GET /api/ids/service/status and POST /api/ids/service/query_alerts \u2014 two read endpoints, nothing else, ever) and by workshop's health poller. A build writes this block plus the secrets into IDSconf.json; long-form field docs live in digest-engine/IDSconf.json.example. An empty list skips that file entirely. TWO THINGS THIS DOES NOT DO: it does not enable Suricata (Services -> Intrusion Detection on the firewall, then download a ruleset), and it does not enable the ingest (ENABLE_OPNSENSE_IDS_INGEST=true in digest-engine.env). SCOPE EACH API KEY: its own OPNsense user with only the 'Services: Intrusion Detection' privilege \u2014 which still covers api/ids/* including start/stop, because OPNsense ACLs are page-level, so the read-only guarantee comes from this project calling exactly two endpoints and not from the firewall enforcing it.", + "name": "main", + "base_url": "", + "verify_tls": true, + "interfaces": [], + "max_alerts_scanned": 5000, + "top_signatures": 8, + "top_hosts": 5, + "packet_capture_reference": "" + } + ], + + "checkmk": { + "_comment": "An existing CheckMK server, polled read-only for host/service state. Feeds two places: digest-engine's network section (so a failing disk shows up in the quarter-daily digest) and workshop's own infra_status table (so the workshop display can overlay system health on the gallery). Nothing here ever acknowledges, downtimes or reschedules anything \u2014 the API user only needs to read. site is the CheckMK site name, which is the path segment in every URL: http:////check_mk/api/1.0/...", + "_credentials": "CheckMK: Setup -> Users -> add a user, Roles = 'Guest' (read-only), then its 'Automation secret' \u2014 NOT its login password. Put the username and secret in secrets.checkmk_username / secrets.checkmk_secret. A Guest-role automation user cannot change anything, which is the actual boundary here rather than a promise about which endpoints get called.", "base_url": "", + "site": "cmk", "verify_tls": true, - "interfaces": [], - "max_alerts_scanned": 5000, - "top_signatures": 8, - "top_hosts": 5, - "packet_capture_reference": "" + "_only_problems": "true keeps the payload to hosts and services that are not OK, which is what both consumers want; false pulls everything and is mostly useful once, to see what the site knows about.", + "only_problems": true, + "max_rows": 200 + }, + + "identity_provider": { + "_comment": "An EXTERNAL identity provider (Keycloak) in front of the household's web surfaces. NOTHING IN THIS REPO IMPLEMENTS SSO YET \u2014 this block exists so the decision and its values are recorded in one place before the work happens, and so the proxy config can be generated from it when it does. Leave issuer_url empty to skip it entirely, which is the current default and the tested path.", + "_pairs_with_freeipa": "KEYCLOAK AND FREEIPA ARE ONE DECISION, NEVER TWO. Keycloak federates FreeIPA as its user store; it is not a place people are created. Configuring Keycloak without FreeIPA would mean a second, parallel set of household accounts \u2014 which is the specific outcome having a directory exists to prevent \u2014 so the validator refuses either one alone. Group membership comes from FreeIPA and rides through Keycloak as a claim; roles are never assigned in Keycloak itself, or the directory stops being the answer to 'who is in this household'. WHAT IT WOULD PROTECT: the proxy-fronted web UIs (Home Assistant, the photo frontend, workshop, identity's admin panel). WHAT IT MUST NEVER PROTECT: the kiosk-to-service APIs \u2014 pantry-vision, identity's /presence, workshop's own API \u2014 which are bearer-token gated because a wall panel cannot complete an interactive login. Putting an OIDC redirect in front of those turns every kiosk into a dead screen.", + "issuer_url": "", + "_realm": "Keycloak realm name; part of the issuer URL too, kept separately because the proxy config and any client library both want it on its own.", + "realm": "smarthome", + "client_id": "smarthome-proxy", + "_protected_hosts": "Which proxied hostnames would sit behind SSO. Advisory until the work is done \u2014 see the _comment above.", + "protected_hosts": [] + }, + + "freeipa": { + "_comment": "An existing FreeIPA domain, mirrored INTO identity as a source of people and group memberships. NOT IMPLEMENTED YET \u2014 this block records the decision and its values ahead of the work, like identity_provider above. Leave server empty to skip it, which is the current default and the only tested path.", + "_direction": "ONE WAY, FreeIPA -> identity, and it must stay that way. identity holds household facts FreeIPA has no opinion about (BLE identifiers, chore reminder style, digest preferences, a colour) and writing any of that back would make a directory serving real logins into a store of smart-home preferences. A mirrored person is matched on uid and their name/group memberships are refreshed; everything else identity knows about them is left alone.", + "_groups": "Group membership maps to roles here. `chore_exempt_group` members are dropped from the chore rotation exactly as the per-person flag does today \u2014 which means a FreeIPA group can grant an exemption but must never remove one somebody set by hand, or a directory sync would silently re-enrol a guest. `household_group` is who gets mirrored at all: without it every service account in the directory becomes a household member.", + "_naming": "EVERY group this project reads is named USR_HA_, matching the field name after the _group suffix is dropped: household_group -> USR_HA_household, chore_exempt_group -> USR_HA_chore_exempt, admin_group -> USR_HA_admins. The prefix is what makes a directory shared with other systems auditable \u2014 'which groups does the smart home read?' is answerable with one filter instead of by reading this file. The validator warns on anything that does not follow it rather than erroring, since an existing directory may already have its own convention and renaming groups in FreeIPA is not a thing a config file should force.", + "server": "", + "domain": "", + "base_dn": "", + "bind_dn": "", + "verify_tls": true, + "household_group": "USR_HA_household", + "chore_exempt_group": "USR_HA_chore_exempt", + "admin_group": "USR_HA_admins", + "_sync_interval_minutes": "How often the mirror runs, once it exists. Directory changes are not urgent \u2014 somebody joining the household is a thing you also tell the door panel about.", + "sync_interval_minutes": 60 }, "proxy": { @@ -159,6 +219,7 @@ "_comment": "type must be one of: thin-client, touch-panel, door-panel, kitchen-display. hostname must be unique and a valid DNS label — it is what the HA device shows up as.", "type": "door-panel", "hostname": "door-panel", + "room": "hallway", "friendly_name": "Door panel", "kiosk_username": "kiosk", "voice_satellite": true, @@ -167,6 +228,7 @@ { "type": "kitchen-display", "hostname": "kitchen-display", + "room": "kitchen", "friendly_name": "Kitchen fridge display", "kiosk_username": "kiosk", "voice_satellite": false, @@ -175,6 +237,7 @@ { "type": "thin-client", "hostname": "thin-client-living", + "room": "living_room", "friendly_name": "Living room thin client", "kiosk_username": "kiosk", "voice_satellite": false, @@ -185,6 +248,7 @@ { "type": "touch-panel", "hostname": "touch-panel-kitchen", + "room": "kitchen", "friendly_name": "Kitchen touch panel", "kiosk_username": "kiosk", "voice_satellite": false, @@ -196,11 +260,13 @@ { "_comment": "Headless Spotify Connect appliances for rooms with no thin client. arch picks the toolchain — and they are genuinely different toolchains producing different artifacts, not one image for both: 'amd64' is a mini PC + USB DAC built with live-build (an .iso), 'arm64' is a Raspberry Pi + HiFiBerry Amp2 built with rpi-image-gen (an .img). build-all.sh builds every entry here, so listing both architectures gets you both. hostname doubles as the Spotify Connect device name and must be unique across kiosks too — they're all devices on one network.", "hostname": "audio-endpoint-livingroom", + "room": "living_room", "friendly_name": "Living room", "arch": "amd64" }, { "hostname": "audio-endpoint-kitchen", + "room": "kitchen", "friendly_name": "Kitchen", "arch": "arm64" } diff --git a/README.md b/README.md index 53208ca..e501de0 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,9 @@ hosts/ browser, switched via an always-on touch dock or by HA/ the local LLM over MQTT kitchen-display/ Single-purpose Sway kiosk for the fridge/pantry: one - Chromium window showing pantry-vision's Scan/Inventory/ - Recipes frontend, camera capture via the browser itself + Chromium window showing pantry-vision's unload/consume/ + expired/edit frontend, camera capture via the browser + itself door-panel/ Single-purpose Sway kiosk by the door/wardrobe: identity's weather+clothing/who's-home/groceries-low dashboard by default, registration on demand — this @@ -60,7 +61,10 @@ admin-canvas/ On-demand sys-admin-llm display surface for the thin static serving) pantry-vision/ Kitchen-display backend: a photo held up to the camera -> an Ollama vision-model proposal -> human-confirmed - write into Grocy stock; also proxies Grocy's inventory + write into Grocy stock. All four stock movements are + camera-driven (unload / consume / bin what expired / + correct by hand), counted in individual units and folded + brand-free; also proxies Grocy's inventory (soonest-expiring first) and recipes to the kiosk frontend (write API + frontend/ static serving) trash-calendar/ Reads Kennelbach's personal trash-collection ICS feed, @@ -94,12 +98,24 @@ chores/ Presence/calendar-driven household chore nudging + - [ ] ESP32-S3-Touch-LCD-1.85C-V2 voice satellite + status display (`firmware/esp32-s3-touch-lcd-1.85c/`) — ESPHome config written and passes `esphome config`, not yet flashed to real hardware; `media_player`/`weather` entity IDs still need to be chosen, see `docs/project-plan.md` §4 - [ ] Headless audio endpoint (`hosts/audio-endpoint/`) — per-room independent Spotify Connect appliance for rooms without a thin client, arm64 (Raspberry Pi + HiFiBerry Amp2, rpi-image-gen) and amd64 (mini PC + USB DAC/amp, live-build) build pipelines written, **neither built/flashed/booted on real hardware** — rpi-image-gen's exact config schema in particular is unverified, see `hosts/audio-endpoint/README.md` - [ ] Sway touch panel (`hosts/touch-panel/`) — touch-driven Sway image: full Spotify GUI (Flathub), a dedicated Home Assistant Chromium kiosk window, a general web browser, an always-on touch dock for app switching, an on-screen keyboard (toggled manually, no auto-show), and `touchpanel-agent` (HA MQTT control, same LLM-mediated-through-HA security model as the thin client) — built, **no touch-panel hardware chosen and nothing booted on real metal**, see `hosts/touch-panel/README.md` -- [ ] Kitchen/fridge display + `pantry-vision` (`hosts/kitchen-display/`, `pantry-vision/`) — hold a grocery item up to the camera, an Ollama vision model proposes what it is and roughly how long it keeps, a human confirms (never auto-committed) before it's written into Grocy stock; the display then shows inventory sorted by soonest-to-expire, groceries running low, and Grocy's recipes — built and wired into `setup-container-host.sh` (`ENABLE_PANTRY_VISION`, off by default), **nothing run against a real camera, vision model, or Grocy instance** — the Grocy API call shapes in particular are written from documentation only, see `pantry-vision/README.md` and `hosts/kitchen-display/README.md` +- [ ] Kitchen/fridge display + `pantry-vision` (`hosts/kitchen-display/`, `pantry-vision/`) — hold a grocery item up to the camera, an Ollama vision model proposes what it is and roughly how long it keeps, a human confirms (never auto-committed) before it's written into Grocy stock; the display then shows inventory sorted by soonest-to-expire, groceries running low, and Grocy's recipes. All four stock movements now run off the camera — **unload** (a scan loop, one confirm per item, with pack size and where to put it away), **consume** (asks which brand and how many), **list expired** (cleared by scanning what you're binning, booked out as spoiled), and **edit inventory** (the deliberately camera-free correction screen) — with two invariants: stock counts *individual units* (a twelve-pack of eggs is twelve) and folds *brand-free* via Grocy product groups (12 of brand X + 10 of brand Y = 22 eggs, expandable per brand). Built and wired into `setup-container-host.sh` (`ENABLE_PANTRY_VISION`, off by default), **nothing run against a real camera, vision model, or Grocy instance** — the Grocy API call shapes in particular are written from documentation only, see `pantry-vision/README.md` and `hosts/kitchen-display/README.md` +- [x] Where-is-it-actually, for multiple fridges — `docs/fridge-item-location.md`: separates "which appliance" (a software-only change: Grocy locations + a transfer action) from "which shelf" (a *hint* at best) and "exact position" (occlusion makes it unbuildable), and rules out interior cameras on power/condensation/−18 °C grounds — which is exactly the compartment the question starts from. **Built**: appliances as Grocy locations, `POST /transfer` with a "Move to…" picker on the edit screen, and the door-sensor→camera→hint path (`pantry-vision/doorway.py`, `POST /doorway-event` — an HA automation on a Zigbee contact sensor, answered 202 with the camera burst running off-thread; what it recognises is stored in its own SQLite file with a timestamp and a confidence and is **never** written to stock). **Not bought, not tested**: no door sensor, no doorway camera, and the assumption the camera half rests on — that a local vision model can identify an item in a moving hand at ~1.5 m — has never been checked (open decision #40). An appliance can be configured with a sensor and no camera, which is still the recommended way to start +- [x] Per-person **colour** and a settable **profile picture** in `identity` — eight colours assigned automatically at registration, avoiding any colour already worn by somebody with the same initial (an Anna and an Amir are two identical "A"s on a wall panel, and the colour is what makes that readable), then least-used overall; editable in the admin panel, backfilled oldest-first for existing people so nobody's colour reshuffles on restart. `color` + `initial` now ride on `/people`, `/presence` and every `/floorplan/presence` occupant, so no consumer derives an initial or invents a palette. New `POST /people//photo` sets a picture without a walk to the door panel — the registration capture was the only source before, which left a device-less household member unable to have a face at all. **The palette values sit on the 2-bits-per-channel lattice a colour Pebble renders natively**, so the colour on a watch is the colour in the panel, see below +- [x] **Rooms are one vocabulary, and devices declare theirs** — `docs/rooms-and-endpoints.md`: the room id is an HA `area_id` everywhere (`CoreSystemConfig.json` → `config-export.py` → the ISO builder → the agent → `suggested_area` in MQTT discovery), which is the same string `identity`'s floorplan rooms join on. Every kiosk and audio endpoint now carries `room`, `tools/validate-config.py` rejects anything that isn't already an area_id (helpfully slugifying it is how you end up with two rooms) and warns rather than errors when it's missing. **The honest limit: `suggested_area` is only honoured at first discovery** — move a device and you move it in HA once, by hand +- [x] **`workshop/`** — built: project notebook (`workshop.db`), a **never-pruned knowledge store** (`workshop-knowledge.db`: workflow instructions by activity, keyword facts, project learnings, and the household's **one** hardware inventory), `GET /context` to hand an assistant everything that applies at once, a **web inventory editor**, **Gitea repos with append-only history** (commit/push/branch yes; force-push/rebase/reset/filter-repo never, enforced by branch protection server-side, with a print-only scrub-request for the token-in-history case), a **health poller** for CheckMK + every OPNsense firewall, a **cameras** tab over go2rtc, and a **fleet-scripts admin surface**: one monitoring-agent script per kind of machine (Debian x86, arm64 Pi, the Docker host, the GPU host, plus slots for the ESP32s and network appliances that *can't* run a script and whose CheckMK-server-side config goes there instead), fetched by each endpoint's `fleet-bootstrap` timer. It's remote code execution by design, so the constraints are the design: **upload is a draft, publishing is a separate click**, the service never executes anything, scripts live in SQLite rather than on the writable share (a share credential must not be a whole-fleet root-execution credential), every version is kept, and the endpoint verifies the checksum, runs a version once, and **reports back pass or fail** — a script that was served is not a script that succeeded. Off by default (`ENABLE_WORKSHOP`), **nothing run against a real deployment** — the CheckMK API shape and Gitea's branch-protection payload are both written from documentation and have version-sensitive field names +- [x] **Multi-firewall + CheckMK, and a config that says so** — `opnsense` is a **list** now (each with its own name and key pair, keyed by name rather than by index), `checkmk` is a first-class block, and both feed the digest (`ingest/infra_health.py`, which reads the *poller* rather than polling, so the digest can say "critical since Tuesday" instead of quoting a six-hour-old snapshot) **and** the workshop's `infra_status` table. Three states — `ok`/`problem`/**`unreachable`** — because "I could not ask" is not "nothing is wrong". Validator rejects duplicate firewall names and half-configured credentials for both +- [x] **Photo web frontend (Immich)** alongside the SMB share — `ENABLE_PHOTOS_WEB`, search/albums/faces over the **same** directory, mounted **read-only**: two writers to one photo tree with different ideas of the layout is how a collection gets quietly reorganised. The share stays right for bulk copy and for keeping photos openable with no software at all +- [x] **Keycloak + FreeIPA config blocks** — `identity_provider` and `freeipa`, validated and exported, **declaration-only**: nothing implements SSO or the directory mirror yet, and the validator says so out loud rather than letting a filled-in block imply it works. **They are one decision, not two** — Keycloak federates FreeIPA rather than being its own user store, so the validator errors on either configured alone; a Keycloak without the directory behind it is a second parallel set of household accounts, which is the thing a directory exists to prevent. Groups follow **`USR_HA_`** (`USR_HA_household`, `USR_HA_chore_exempt`, `USR_HA_admins`), warned-not-errored so an existing directory's convention isn't overridden by a config file. Two rules recorded before the work: the mirror is **one-way** (a directory serving real logins must not become a store of chore-reminder preferences), and a group may **grant** a chore exemption but never **remove** one somebody set by hand +- [ ] Workshop/office assistant — `docs/workshop-assistant.md`: **read the label, don't recognise the object** (a closed T480 and T490 are the same black rectangle; the identity is in the service tag and the PCB silkscreen, so the pipeline is OCR/barcode-first with the VLM only locating the label). Specs must be **quoted from a fetched document with its URL, never generated** — a hallucinated pinout destroys hardware — which also makes this the first component here that deliberately reaches the open internet (outbound-only, allowlisted, cached). Per-room scoping = the room selects the toolset, riding the `room` plumbing above. An SMB **workspace share** for everything it produces (reusing the existing Samba container — a second one would collide on 445, which the gallery already holds — with its own volume, own account, and read-write where the gallery is read-only). Display widgets collapse to **one `svg` window kind plus server-side renderers** (Graphviz for code-flow/data-structure, netlistsvg for schematics, KiCad export for board plans), because a megabyte of JS on a kiosk buys a picture the server could render once — **IEC/EU notation is a symbol-library decision**, so "can I supply my own symbols?" disqualifies a tool before output quality does. Purple/magenta holo theme is one CSS variable override over the existing `glow.css`, with the drawings left untinted on purpose. Analysis only, nothing built +- [x] **Now-playing visualiser + 3D floorplan** — built, in `render/` (shared, vendored, dependency-free, config from `?query=params`). `media-visualiser/`: circular spectrum, album-art palette, LRC lyrics — **two-tier by necessity**, since most endpoints have no local audio (a kitchen panel showing what the living room plays cannot analyse anything), so it is real FFT where audio is local and a tempo-driven ring elsewhere, **which says on screen that it is not an analysis**. Palette rejects near-greys/near-blacks before ranking and lifts each colour until it clears the background — the step whose absence makes art-coloured visualisers invisible on dark covers. Plain lyrics are shown but never auto-scrolled at a guessed rate. `floorplan-3d/`: the same `/floorplan/presence` payload as the Pebble app, extruded — **canvas 2D, not three.js**, a change from the plan made while building it: the scene is prisms on a plane, which an isometric projection with painter's sorting draws in ~200 lines, predictably on weak panels, with the frontend still at zero dependencies. Lit/dark rooms plus the dashed third state for rooms HA never reports on, photos-or-initials in the person's colour ring, and a visible shelf for people who are home but unlocatable. Geometry unit-tested headlessly; **never opened in a real browser** +- [ ] ~~Now-playing visualiser + 3D floorplan (design)~~ — `docs/endpoint-surfaces.md`: a circular CAVA-style ring behind every playing screen, coloured from the album art, lyrics under the cover when they exist; and the Pebble app's presence view in 3D on any endpoint. Two findings shape both: **most endpoints have no local audio** (a kitchen panel showing what the living room plays cannot analyse anything), so the visualiser is explicitly two-tier — real FFT where audio is local, tempo-driven "mood light" elsewhere, and the synthetic tier must never claim to be the real one; and the 3D plan is an **extrusion of the existing 2D polygons**, not a hand-authored model, so it can't go stale when a room is redrawn. Three.js is ~1MB and a deliberate break with the dependency-free SDK rule — make it once, explicitly. Analysis only, nothing built +- [ ] Pebble presence **watchapp** feasibility — `docs/pebble-presence-watchface.md`: the floorplan drawn as quantised polygons (~200–250 bytes for a whole floor, against a ~2 KB AppMessage budget), occupants as initial-on-colour rather than photos (a face gets ~20×20px in 64 colours — four skin-toned blobs, and three of them exceed the message budget the entire floor fits in). Targets the **Pebble Round 2** that's actually owned: the plan is inscribed in the circle (`w = D·a/√(a²+1)`, so ~170×170 for a squarish plan at a 240px usable diameter — comparable to a Time 2 after chrome), the projection change is one line of phone-side JS, and the only real loss is corner furniture like a status footer. Occupancy reads as **dark room / lit room** in lightness rather than hue, since colour is already spoken for by *who* — with a third dashed state for rooms HA never reports on, because drawing "no data" as "empty" is a quiet lie. **Button-cycling through rooms forces a watchapp, not a watchface** — Pebble gives watchfaces no button events and restricts touch to apps — and the watchapp is the decision: Up/Down cycles rooms into a plain-text list of who is in each (plus a final "somewhere in the house" entry for people who are home but unlocatable, who are exactly who you picked the watch up to find). The cost is that it's a menu entry rather than your default screen, and its data only lives while it's open; a watchface variant later is a second `main()`, not a second project. Analysis only, nothing built. The finding that actually decides it is upstream: room-level presence has never been measured in this house, and the admin panel's floorplan tab with **Live** ticked tests it today for free +- [ ] "Put the groceries away" chore — `chores`' new `groceries_out_of_place` watch point: `litter`'s twin (same "whoever the camera just saw" attribution, same exemption-proof, same unassignable) with a **30-minute** neglect fuse instead of four hours, because the failure is spoiled food rather than untidiness. Its prompt names what to *ignore* (fruit bowls, bread bins, coffee, cookware) as well as what to look for — the general question gets a YES for any normal kitchen, and a false chore every two hours is how a household learns to mute the channel (open decision #41). Knows food is out, never what the food is; nothing here touches Grocy. Untested against a real camera or model, like every other watch point - [ ] `identity` + door panel (`identity/`, `hosts/door-panel/`) — the person <-> BLE-identifier registry: "register me as ``" by voice or touchscreen, multi-phone support (multiple identifiers per person), anti-spoofing (only allowlisted IRK-resolved/fixed-tag entities are ever accepted as candidates, never a raw MAC), device-less people (a "no device" flag plus a hand-operated Home/Away toggle — the concrete case: a grandmother without a smartphone), and an anonymous "Guest" path. Backs `hosts/door-panel/`'s weather+clothing/who's-home/groceries-running-low dashboard and `hosts/kitchen-display/`'s "Show registration" screen — built and wired into `setup-container-host.sh` (`ENABLE_IDENTITY`, off by default), **nothing run against a real HA instance, real Private BLE Device entities, or a real voice pipeline** — `TRUSTED_ENTITY_PREFIXES` above all needs checking against Developer Tools -> States, see `identity/README.md` and `hosts/door-panel/README.md` - [ ] `identity` also corroborates presence from Frigate face recognition (Phase 20, Tapo pan/tilt cameras) — an OR-ed-in second signal only, **never** a registration signal; and owns the per-person chore-system settings (`chore_exempt`, `chore_reminder_style`, plus chore assignment) consumed by `chores/`, see `identity/README.md` - [ ] `identity`'s admin panel (`identity/frontend/admin.html`, Phase 6b) — managing people/guests: edit every field, **nicknames** (an input alias only — `/resolve` accepts them, but the assistant always speaks the real `speak_name`), **visit history** sampled from `/presence` plus a derived "who was home with whom" view, **"select all that last visited before ``"** pruning (the filter selects, a human confirms the exact list, the filter is never re-run at delete time), **per-device rights** for self-entry via a smart lock (`identity` only ever *answers* `GET /device-access` — HA asks and HA acts, deny is the default), chore assignment, a **floorplan editor** (draw levels and rooms as polygons, map each to an HA area, and watch occupied rooms light up — resolves the long-deferred open decision #22), and **opt-in arrival push notifications** ("tell me when someone gets home", via the self-hosted ntfy this stack already runs — `identity` itself never touches the WAN; ntfy stays LAN-only and remote delivery rides a WireGuard split tunnel — see `docs/network-integration.md` §2.2 for why a DMZ/port-forward was weighed and rejected). Deliberately **not** a kiosk page and not linked from any wall panel. Covered by API-level tests; **never opened in a real browser**, and `DEPARTURE_GRACE_SECONDS` is an untuned guess — see `identity/README.md` - [ ] `trash-calendar` + `transit` (Phase 19, Kennelbach AT trash pickup + Vorarlberg public transit) — built and wired into `setup-container-host.sh` (`ENABLE_TRASH_CALENDAR`/`ENABLE_TRANSIT`/`ENABLE_TRIP_PLANNING`, all off by default), **nothing run against a live ICS feed, a live GTFS feed, or a real OpenTripPlanner instance** — trip planning also needs a manually-built OTP graph this repo does not build for you, see `trash-calendar/README.md` and `transit/README.md`'s "Route planning scope" -- [ ] `chores` (Phase 20) — presence/calendar-driven household chore nudging: "I don't care who does it, as long as it gets done" — prefers whoever's been assigned a chore in `identity`'s admin panel but falls through to whoever's actually home rather than waiting (`CHORE_ASSIGNMENT_STRICT` flips that), redirects to someone else if a chore goes neglected, keeps a passive fairness tally that never feeds back into who gets nudged, and camera-checks trash bins/dishes/litter via Frigate + an Ollama vision model. **Litter remains the exception to everything** — it ignores both chore-exemption and assignment, because cleaning up what you left out was never a task anyone could be assigned. Built and wired into `setup-container-host.sh` (`ENABLE_CHORES`, off by default, every-2-hours systemd timer), **no Tapo camera hardware chosen and nothing run against real hardware**, see `chores/README.md` +- [ ] `chores` (Phase 20) — presence/calendar-driven household chore nudging: "I don't care who does it, as long as it gets done" — prefers whoever's been assigned a chore in `identity`'s admin panel but falls through to whoever's actually home rather than waiting (`CHORE_ASSIGNMENT_STRICT` flips that), redirects to someone else if a chore goes neglected, keeps a passive fairness tally that never feeds back into who gets nudged, and camera-checks trash bins/dishes/litter/groceries-left-out via Frigate + an Ollama vision model. **Litter and groceries-left-out remain the exception to everything** — both ignore chore-exemption and assignment, because cleaning up what you left out was never a task anyone could be assigned; groceries additionally get a 30-minute fuse instead of four hours, since that failure spoils food rather than merely looking untidy. Built and wired into `setup-container-host.sh` (`ENABLE_CHORES`, off by default, every-2-hours systemd timer), **no Tapo camera hardware chosen and nothing run against real hardware**, see `chores/README.md` - [ ] Music Assistant (optional, additive multi-room audio) — wired into `setup-container-host.sh` (`ENABLE_MUSIC_ASSISTANT`, off by default), **its default port is an unverified guess that collides with `PANTRY_VISION_PORT`** if both are enabled together, see `docs/project-plan.md` open decision #31 - [ ] `docs/network-integration.md` (OPNsense VLAN segmentation, the WireGuard split tunnel that carries arrival notifications, and why nothing here — ntfy included — gets port-forwarded to the WAN) — written, not run against a real OPNsense instance - [ ] `tools/` + `CoreSystemConfig.json` — every build and setup script in one place, reading one config. The container host and LLM host build as a **twinned pair**: you set two last octets and the container host's `OLLAMA_HOST` is *derived* from the LLM host's, so the two ISOs cannot be built disagreeing about where the other one is; every kiosk's service URLs derive from the container host's address the same way. `build-all.sh` builds the set, `validate-config.py` refuses a build on duplicate ports (the `music_assistant`/`pantry_vision` 8095 clash, open decision #31), placeholder or padded tokens, duplicate hostnames, or a kiosk pointed at a disabled service. All secrets are burnt into the images so installs are unattended — **which makes every ISO a credential**; the filled-in config and `iso-out/` are gitignored. **No ISO has ever been built with this** (`lb build` needs live-build, root and a long fetch) — what is tested is config validation/derivation and every generated artifact, with `lb` stubbed. See `tools/README.md` diff --git a/chores/README.md b/chores/README.md index 3aab0f5..2b00b50 100644 --- a/chores/README.md +++ b/chores/README.md @@ -21,18 +21,43 @@ module docstring for the full reasoning on each: Frigate snapshot per configured watch point (optionally moving a PTZ camera to a preset first), asks an Ollama vision model a one-word question ("is this bin FULL/PARTIAL/EMPTY", "is this counter DIRTY/CLEAN", "is there litter left out - here, YES/NO"), opens a chore on "needs attention," auto-closes one on "clear." + here, YES/NO", "are there groceries standing out that belong in a fridge/freezer/ + cupboard, YES/NO"), opens a chore on "needs attention," auto-closes one on "clear." 3. **Nudging** — ASAP, not on a fixed schedule: the first run after a chore opens nudges whoever `identity` reports home right now — preferring anyone **assigned** that chore type, minus anyone `chore_exempt`, both below. If the chore is still open `NEGLECT_THRESHOLD_HOURS` after the last nudge (and the household calendar isn't showing a busy window), the nudge goes to **someone different from who was last asked** — "the next person that walks - by" — rather than re-nagging the same person. `litter` chores are special-cased - to prefer whoever the camera most recently recognized nearby (a best-effort - "who left this" guess), since the point there is telling the actual person, not - just whoever's around — and `litter` also ignores `chore_exempt` entirely, see - below. Each nudge's wording is a plain template unless the target has a - `chore_reminder_style` set, see below. + by" — rather than re-nagging the same person. `litter` and + `groceries_out_of_place` chores are special-cased to prefer whoever the camera + most recently recognized nearby (a best-effort "who left this" guess), since the + point there is telling the actual person, not just whoever's around — and both + also ignore `chore_exempt` entirely, see below. Each nudge's wording is a plain + template unless the target has a `chore_reminder_style` set, see below. + +## Putting the groceries away is litter with a shorter fuse + +`groceries_out_of_place` is built as `litter`'s twin on purpose — same "whoever the +camera just saw" attribution, same exemption-proofing, same unassignability, because +it is the same situation: somebody carried something in and put it down. Its one +deliberate difference is **`GROCERIES_NEGLECT_THRESHOLD_HOURS`, 30 minutes instead of +the usual four**, and that is the whole reason it isn't simply another watch point on +the `litter` prompt: litter left for an afternoon is untidiness, and a tub of ice +cream left for an afternoon is a bin bag. It is the only chore type in `check.py` +treated as more urgent than the others, because it is the only one where being late +costs something other than tidiness. + +Two things worth knowing before pointing a camera at this: + +- **The prompt names what to ignore, not just what to look for** — fruit bowls, bread + bins, coffee, oil, spices, appliances, cookware. Ask a vision model the general + question ("is anything out of place?") and it will say YES to a kitchen that is + simply a kitchen, every two hours, until somebody mutes the topic. +- **It is a chore, not an inventory update.** It notices that food is standing out; it + does not know *what* the food is and never books anything into Grocy. That is + `pantry-vision`'s job, on a display someone is actually standing at. Two systems + looking at the same counter for two different reasons is the intended shape, not a + duplication to collapse. ## Assignment is a preference, not a lock @@ -52,20 +77,22 @@ legitimate and this file can't pick for you, so it's one env var rather than a hard-coded opinion. Strict mode still never stalls a chore that was assigned to *nobody* — that falls through regardless. -**Litter can't be assigned** (`_ASSIGNMENTS_DONT_APPLY` in `check.py`), for the same -reason it ignores exemptions: it goes to whoever left the mess, and cleaning up after -yourself was never a task anyone could be handed. +**Litter and groceries-left-out can't be assigned** (`_ASSIGNMENTS_DONT_APPLY` in +`check.py`), for the same reason they ignore exemptions: they go to whoever left the +mess, and cleaning up after yourself was never a task anyone could be handed. -## Chore-exempt people — everyone except litter +## Chore-exempt people — everyone except litter and groceries Set in the same place (`identity`'s admin panel, or `POST /people//chore-settings` directly). A `chore_exempt` person is dropped from the nudge rotation entirely — the "cousin visits often but doesn't owe -me chores" case. **Litter is the deliberate exception** (`_EXEMPTIONS_DONT_APPLY` -in `check.py`): an exempt person still gets told to put trash they left out into -the bin, because that isn't "doing a chore," it's cleaning up after yourself. If -everyone currently home is chore_exempt for a non-litter chore, that run just logs -and skips — the chore stays open until someone eligible is around. +me chores" case. **Litter and groceries-left-out are the deliberate exceptions** +(`_EXEMPTIONS_DONT_APPLY` in `check.py`): an exempt person still gets told to put +trash they left out into the bin and to put the milk away, because neither is "doing +a chore," both are cleaning up after yourself — and a guest who helped unpack the +shopping is exactly as able to finish the job as anyone else. If everyone currently +home is chore_exempt for one of the other chore types, that run just logs and skips — +the chore stays open until someone eligible is around. ## Reminder tone is per-person and LLM-phrased, but never LLM-decided @@ -163,7 +190,11 @@ when left unconfigured. 2. Whether the vision model's one-word FULL/PARTIAL/EMPTY/DIRTY/CLEAN/YES/NO answers are actually reliable for a real bin/sink/hallway from a real camera angle — completely unmeasured, same caveat as pantry-vision's own vision-model - accuracy note. + accuracy note. **`groceries_out_of_place` is the one to check first**: its whole + design rests on a model reliably telling a carton of milk from a fruit bowl, and + its failure mode is not a missed chore but a false one every two hours, which is + how a household learns to ignore the notification channel. Point it at a *clean* + counter for a day before trusting a YES. 3. `_likely_culprit()`'s reliance on `identity`'s `face_seen_recently` field assumes Frigate face-recognition presence corroboration is actually wired up and working (`identity/README.md`'s own "Camera face recognition" section is diff --git a/chores/check.py b/chores/check.py index bc298fa..ca1f230 100644 --- a/chores/check.py +++ b/chores/check.py @@ -18,8 +18,9 @@ in case something else is running" jitter) and does three things, in order: camera hardware has been chosen yet, see docs/project-plan.md §1.18): for each configured watch point (a Frigate camera + optional PTZ preset), grabs a snapshot via Frigate's own API and asks an Ollama vision model whether it shows - a full bin / dirty dishes. A "needs attention" result opens a chore if one isn't - already open; a "clear" result auto-closes one if it was. + a full bin, dirty dishes, litter left out, or **groceries still standing out that + belong in a fridge/freezer/cupboard**. A "needs attention" result opens a chore if + one isn't already open; a "clear" result auto-closes one if it was. 3. **Nudging**: for every open chore, ASAP, not "wait for a schedule" — the first run after a chore opens nudges whoever `identity` reports home right now (using `identity`'s own room field to prefer someone actually near the relevant spot, @@ -35,10 +36,20 @@ in case something else is running" jitter) and does three things, in order: `identity`'s `chore_exempt` flag (set via `POST /people//chore-settings`, see identity/README.md) takes a person out of the nudge rotation entirely — a frequent -guest who isn't a household member doesn't owe chores. **`litter` is the one -exception** (`_EXEMPTIONS_DONT_APPLY` below): everyone, exempt or not, still gets -told to put trash they left out into the bin — that isn't "doing a chore," it's -cleaning up after yourself. +guest who isn't a household member doesn't owe chores. **`litter` and +`groceries_out_of_place` are the exceptions** (`_EXEMPTIONS_DONT_APPLY` below): +everyone, exempt or not, still gets told to put trash they left out into the bin and +to put the milk away — neither is "doing a chore," both are cleaning up after +yourself, and a guest who unpacked the shopping is exactly as able to finish the job +as anyone else. + +`groceries_out_of_place` is deliberately built as litter's twin — same +culprit-attribution, same exemption-proofing, same unassignability — with **one +difference that matters: a much shorter fuse** (`_NEGLECT_HOURS_OVERRIDE`, 30 minutes +against the usual four hours). Litter left for an afternoon is untidiness; a tub of +ice cream left for an afternoon is a bin bag. It is the only place in this file where +one chore type is treated as more urgent than another, and the reason is that the cost +of being late differs in kind, not degree. ## Assignment is a preference, not a lock @@ -54,9 +65,10 @@ waiting. Set `CHORE_ASSIGNMENT_STRICT=true` if you'd rather it wait for the assi that's the honest opposite reading of the same feature, and which one a household wants isn't something this file can decide for it. -**`litter` ignores assignment entirely** (`_ASSIGNMENTS_DONT_APPLY`), for the same -reason it ignores exemptions: it goes to whoever left the mess, and "cleaning up after -yourself" was never a task anyone could be assigned in the first place. +**`litter` and `groceries_out_of_place` ignore assignment entirely** +(`_ASSIGNMENTS_DONT_APPLY`), for the same reason they ignore exemptions: they go to +whoever left the mess, and "cleaning up after yourself" was never a task anyone could +be assigned in the first place. `identity`'s `chore_reminder_style` free-text field (same endpoint) is passed to an LLM that **phrases** the ntfy message in that person's preferred tone ("be @@ -139,26 +151,65 @@ _CHORE_PROMPTS = { "that does not belong there (not properly disposed of in a bin)? Answer with exactly " "one word: YES or NO." ), + # Groceries left standing out — the same shape as litter (somebody put something + # down and walked away) with a much shorter fuse, because the failure here is not + # untidiness, it is food going off. The prompt names the perishable cases + # explicitly rather than asking the general "is anything out of place" question, + # which a vision model will happily answer YES to for a fruit bowl, a bread bin, + # or a kettle — and a chore that fires every two hours about the fruit bowl is one + # the household will mute within a week. + "groceries_out_of_place": ( + "Look at this photo of a kitchen counter, table or worktop. Are there GROCERIES sitting " + "out that belong in a fridge, freezer or cupboard — for example milk, yoghurt, cheese, " + "meat, fish, eggs, opened jars, frozen food, or a shopping bag that has not been put " + "away? Ignore things that normally live on a worktop: fruit bowls, bread bins, coffee, " + "spices, oil, salt, appliances, cookware and dishes. Answer with exactly one word: " + "YES or NO." + ), } # Which watch points get "who was just seen here" culprit-attribution treatment # (see _likely_culprit()) instead of the general "whoever's around" nudge — litter # is specifically about telling whoever left it, not just whoever's nearby now. -_ATTRIBUTE_TO_RECENT_VIEWER = {"litter"} +# Groceries left out is the same situation: somebody carried it there. +_ATTRIBUTE_TO_RECENT_VIEWER = {"litter", "groceries_out_of_place"} # Chore types where identity's chore_exempt flag does NOT apply — everyone still # gets told to clean up litter they left out, exempt household member or not (see # module docstring). Currently the same set as _ATTRIBUTE_TO_RECENT_VIEWER, but # they mean different things — one is about attribution, this is about eligibility # — so they're kept as separate names rather than reusing one for both purposes. -_EXEMPTIONS_DONT_APPLY = {"litter"} +_EXEMPTIONS_DONT_APPLY = {"litter", "groceries_out_of_place"} # Chore types that can't be assigned to anyone — see the module docstring's -# "Assignment is a preference, not a lock". Third set with the same one member as the +# "Assignment is a preference, not a lock". Third set with the same members as the # two above, and kept separate for the third distinct reason: attribution, then # eligibility, now assignability. If they ever diverge (a chore that's assignable but # exempt-proof, say) collapsing them now would be the thing that made that painful. -_ASSIGNMENTS_DONT_APPLY = {"litter"} +_ASSIGNMENTS_DONT_APPLY = {"litter", "groceries_out_of_place"} + +# How the one-word camera answer is read, per chore type. Extracted from an inline +# expression once there were four of them: adding a chore type should be a line in a +# table, not a new clause in a boolean nobody can read. +_ATTENTION_ANSWERS = { + # "PARTIAL" contains neither FULL nor a separate word, but a model answering + # "PARTIALLY FULL" must not be read as full — hence the exclusion, kept from the + # original inline check. + "bin_full": lambda a: "FULL" in a and "PARTIAL" not in a, + "dishes": lambda a: "DIRTY" in a, + "litter": lambda a: "YES" in a, + "groceries_out_of_place": lambda a: "YES" in a, +} + +# Per-type override for how long a chore may sit before it is redirected to somebody +# else. A dropped crisp packet can wait the default four hours; a tub of ice cream on +# the counter cannot, and neither can the shopping nobody unpacked. This is the one +# place in this file where a chore type is treated as more urgent than another, and it +# is here because the cost of being late is different in kind — spoiled food, not an +# untidy room. +_NEGLECT_HOURS_OVERRIDE = { + "groceries_out_of_place": float(os.environ.get("GROCERIES_NEGLECT_THRESHOLD_HOURS", "0.5")), +} # Whether an assigned person who ISN'T home blocks the chore from falling through to # whoever is. Default false — "as long as it gets done" is the house rule; true makes @@ -261,7 +312,8 @@ def check_trash_day(conn) -> None: # --- 2. Camera checks -------------------------------------------------------------- def _watchpoints() -> list[tuple[str, str, str | None]]: """CAMERA_WATCHPOINTS format: "type:camera[:preset],type:camera[:preset],...". - type must be a key in _CHORE_PROMPTS other than "trash" (bin_full, dishes). + type must be a key in _CHORE_PROMPTS other than "trash" — i.e. bin_full, dishes, + litter, or groceries_out_of_place. """ raw = os.environ.get("CAMERA_WATCHPOINTS", "").strip() if not raw: @@ -269,7 +321,10 @@ def _watchpoints() -> list[tuple[str, str, str | None]]: points = [] for entry in raw.split(","): parts = [p.strip() for p in entry.split(":")] - if len(parts) < 2 or parts[0] not in _CHORE_PROMPTS or parts[0] == "trash": + # _ATTENTION_ANSWERS as well as _CHORE_PROMPTS: a chore type with a prompt but + # no way to read its answer is a KeyError in check_cameras(), i.e. a crash on + # the timer rather than here. Adding a type means adding both. + if len(parts) < 2 or parts[0] not in _CHORE_PROMPTS or parts[0] not in _ATTENTION_ANSWERS: LOG.warning("chores: ignoring malformed CAMERA_WATCHPOINTS entry %r", entry) continue points.append((parts[0], parts[1], parts[2] if len(parts) > 2 else None)) @@ -334,9 +389,7 @@ def check_cameras(conn) -> None: if answer is None: continue - needs_attention = ("FULL" in answer and "PARTIAL" not in answer) or "DIRTY" in answer or ( - chore_type == "litter" and "YES" in answer - ) + needs_attention = _ATTENTION_ANSWERS[chore_type](answer) existing = _open_chore(conn, chore_type) if needs_attention and existing is None: @@ -458,7 +511,8 @@ def nudge_open_chores(conn) -> None: if last_nudged_at is not None: elapsed_hours = (datetime.now(timezone.utc) - datetime.fromisoformat(last_nudged_at.replace("Z", "+00:00"))).total_seconds() / 3600 - if elapsed_hours < NEGLECT_THRESHOLD_HOURS: + threshold = _NEGLECT_HOURS_OVERRIDE.get(chore["type"], NEGLECT_THRESHOLD_HOURS) + if elapsed_hours < threshold: continue # not neglected yet — leave whoever was last nudged alone for now # chore_exempt people are out of the rotation entirely, EXCEPT litter — see @@ -525,12 +579,54 @@ def nudge_open_chores(conn) -> None: ) +# The culprit-framed wording, per type. Both are "you probably left this, please deal +# with it" — but "put it in the bin" is exactly wrong for a tub of ice cream, and a +# reminder that tells you to do the wrong thing is one you stop reading. +_CULPRIT_MESSAGES = { + "litter": "{name}, looks like something was left out — could you put it in the bin?", + "groceries_out_of_place": ( + "{name}, there are groceries still standing out — could you put them away before they spoil?" + ), +} + +# What the LLM phrasing pass is told the situation is, when there is a model +# configured. Same split, same reason. +_CULPRIT_SITUATIONS = { + "litter": ( + "Something was left out and may belong to them specifically — ask them to put it away, " + "don't accuse them outright. " + ), + "groceries_out_of_place": ( + "Groceries have been left standing out instead of being put in the fridge, freezer or " + "cupboard, and they may be the person who left them — ask them to put the food away " + "before it spoils, don't accuse them outright. " + ), +} + + +# Chore types are snake_case identifiers; people are not. Everything user-facing goes +# through here — a notification that says "could you take care of: +# groceries_out_of_place" is a notification that reads like a stack trace. +_CHORE_LABELS = { + "trash": "putting the bins out", + "bin_full": "emptying the bin", + "dishes": "the dishes", + "litter": "clearing up what was left out", + "groceries_out_of_place": "putting the groceries away", +} + + +def _label(chore_type: str) -> str: + return _CHORE_LABELS.get(chore_type, chore_type.replace("_", " ")) + + def _default_message(name: str, chore_type: str, redirected: bool, is_culprit: bool) -> str: if is_culprit: - return f"{name}, looks like something was left out — could you put it in the bin?" + template = _CULPRIT_MESSAGES.get(chore_type, "{name}, looks like something was left out — could you deal with it?") + return template.format(name=name) if redirected: - return f"{name}, this one's still open — could you take care of: {chore_type}?" - return f"{name}, could you take care of: {chore_type}?" + return f"{name}, this one's still open — could you take care of: {_label(chore_type)}?" + return f"{name}, could you take care of: {_label(chore_type)}?" def _compose_message(name: str, chore_type: str, redirected: bool, is_culprit: bool, reminder_style: str | None) -> str: @@ -545,10 +641,13 @@ def _compose_message(name: str, chore_type: str, redirected: bool, is_culprit: b prompt = ( f"Write ONE short household chore reminder (max 2 sentences) addressed to {name}. " - f"The chore is: {chore_type}. " + f"The chore is: {_label(chore_type)}. " f"Follow {name}'s own stated preference for how they like to be reminded: \"{reminder_style}\". " - + ("Something was left out and may belong to them specifically — ask them to put it away, don't accuse them outright. " - if is_culprit else "") + + (_CULPRIT_SITUATIONS.get( + chore_type, + "Something was left out and may belong to them specifically — ask them to put it away, " + "don't accuse them outright. ", + ) if is_culprit else "") + ("They were already asked about this once before and it's still not done. " if redirected else "") + "Reply with ONLY the message text itself — no preamble, no quotation marks." ) diff --git a/chores/chores.env.example b/chores/chores.env.example index 668a3f3..e7ef2cb 100644 --- a/chores/chores.env.example +++ b/chores/chores.env.example @@ -48,15 +48,28 @@ CALDAV_QUIET_KEYWORDS=busy,meeting,call,movie,sleep # sources. # # CAMERA_WATCHPOINTS format: "type:frigate_camera_name[:ptz_preset_name],...". -# type must be "bin_full", "dishes", or "litter" ("has someone left trash out -# somewhere it doesn't belong" — gets told to whoever the camera most recently -# recognized nearby, not just whoever's home in general, see README.md). preset -# is optional — omit it if the camera doesn't need to move. Example: -# CAMERA_WATCHPOINTS=bin_full:driveway_cam:trash_preset,dishes:kitchen_cam,litter:hallway_cam +# type must be one of: +# bin_full — is the bin full +# dishes — is the sink/counter dirty +# litter — has someone left trash out somewhere it doesn't belong +# groceries_out_of_place — is there food still standing out that belongs in a +# fridge/freezer/cupboard +# The last two are told to whoever the camera most recently recognized nearby, not +# just whoever's home in general, and neither can be assigned or exempted away — +# see README.md. groceries_out_of_place also has a much shorter neglect fuse than +# everything else (below), because the failure mode is spoiled food, not untidiness. +# preset is optional — omit it if the camera doesn't need to move. Example: +# CAMERA_WATCHPOINTS=bin_full:driveway_cam:trash_preset,dishes:kitchen_cam,litter:hallway_cam,groceries_out_of_place:kitchen_cam # --------------------------------------------------------------------------- FRIGATE_URL= CAMERA_WATCHPOINTS= +# How long groceries may stand out before the nudge is redirected to somebody else, +# overriding NEGLECT_THRESHOLD_HOURS for that one chore type. 30 minutes by default: +# the default four hours is a sensible wait for a crisp packet and far too long for a +# tub of ice cream. +GROCERIES_NEGLECT_THRESHOLD_HOURS=0.5 + # Same LLM host as digest-engine/pantry-vision. OLLAMA_VISION_MODEL must be a # vision-capable model (see pantry-vision/README.md's identical caveat — plain text # models cannot see images at all). diff --git a/digest-engine/README.md b/digest-engine/README.md index ba112bc..c3a8f9d 100644 --- a/digest-engine/README.md +++ b/digest-engine/README.md @@ -224,11 +224,15 @@ trusting a scheduled run, verify by hand: half is verified — `ingest/rci_social.py` was run live against the committed `feeds/rci-social.json` on 2026-08-06 and returned real YouTube and podcast entries with durations — but no local model has yet been asked to produce the - four question windows, per-marker summaries and `sources` arrays in one JSON + five question windows, per-marker summaries and `sources` arrays in one JSON document. Check on the first real run that a 14B model actually fills `sources` rather than dropping the field, that it does not put episodes in the analysis, and that the compact pass still fits the HA card now that the - section has more to say. + section has more to say. The roll-call (`political-struggles`) is the window + to read most sceptically on that first run: confirm every line traces to an + entry that is actually in the context rather than to the model's own memory + of a famous strike, and that a long-running dispute stays on the list across + consecutive runs instead of being dropped as stale. 14. **The Telegram channel entry in `feeds/rci-social.json`.** The channel name comes from marxist.com's own footer but was never fetched — reading it needs the Telethon session, which only exists on the real deployment. Also confirm @@ -264,9 +268,9 @@ trusting a scheduled run, verify by hand: a Times of Israel claim about Palestinians never reaches the digest in the section's own voice. -## The political section — sources, ownership, and the four questions +## The political section — sources, ownership, and the five questions -The political digest answers four questions, in this order, one window each +The political digest answers five questions, in this order, one window each (`synth/prompts/political.md` is written around them): 1. What is relevant for the communist and class struggle **globally** right now. @@ -278,6 +282,17 @@ The political digest answers four questions, in this order, one window each sources are the `theory` feeds, the organisation's own social output, and the reports that arrive in the user's own **mail** — the one input here no feed can supply. +5. What class struggles are **currently going on around the world** — the + `political-struggles` roll-call. This one is an inventory, not a curation: + one line per live strike, occupation or mass movement, kept on the list while + it runs even in a quarter that carried no fresh news of it, which is what the + archive's `times_seen`/`first_seen_at` annotations are for. It is deliberately + the one window where a story being old is not a reason to drop it, and it + survives the compact pass (shortened, and saying it is shortened) because a + roll-call folded into a summary line stops being a roll-call. Everything on it + still has to come from this run's context — the prompt says in as many words + that the model's own knowledge of the world is not a source, since an + inventory question is exactly the shape of prompt that invites one. Plus a **watch-later** window: new videos and podcast episodes from the organisation's channels, kept out of the analysis entirely because they are diff --git a/digest-engine/ingest/infra_health.py b/digest-engine/ingest/infra_health.py new file mode 100644 index 0000000..fc93d3b --- /dev/null +++ b/digest-engine/ingest/infra_health.py @@ -0,0 +1,96 @@ +"""Infrastructure health for the network digest — CheckMK and every firewall. + +READS THE POLLER, DOES NOT POLL. `workshop/health.py` already asks CheckMK and each +OPNsense box every few minutes and keeps a month of samples; this module fetches that +service's `GET /health` and reshapes it for the network section. + +That indirection is the whole design and it is worth one paragraph. The digest runs +four times a day. "Is the NAS disk failing right now" is not a question with a +six-hour answer, so polling from here would have produced a snapshot taken at 06:00 +and quoted at 12:00. Reading the poller instead gives the digest something a snapshot +cannot have: **how long the state has held**. "Critical since Tuesday" is a different +sentence from "critical", and it is the one that tells you whether to get up. + +It also means one set of credentials in one place. digest-engine never learns the +CheckMK secret or any firewall's API key for this — those live in workshop's env file, +and this module needs only workshop's own bearer token. + +WHY THIS IS NOT IN opnsense_ids.py +----------------------------------- +That module reads Suricata's *alert log*, with paging and a time window, from one +firewall. This one reads *service state* from many. They answer different questions +("what fired" vs "is it running and is anything broken"), and the alert query is +deliberately left where it already works rather than reimplemented here. + +DEGRADES TO NOTHING, LOUDLY +---------------------------- +An unreachable workshop service returns `None` and the run continues — same rule as +every other ingestion module here. But when the poller *is* reachable and reports a +target as `unreachable`, that is passed through as a finding rather than dropped: a +firewall nobody can reach is exactly the thing a network digest exists to mention. +""" + +from __future__ import annotations + +import json +import logging +import os +import urllib.error +import urllib.request + +LOG = logging.getLogger(__name__) + +WORKSHOP_URL = os.environ.get("WORKSHOP_URL", "").rstrip("/") +WORKSHOP_TOKEN = os.environ.get("WORKSHOP_TOKEN", "") +TIMEOUT = float(os.environ.get("WORKSHOP_TIMEOUT", "10")) + + +def fetch() -> list[dict]: + """Health entries for the digest context, or [] when unavailable. + + Every entry carries `target` — the firewall's name or the CheckMK site — because a + household with two firewalls must never be handed "the IDS is running" as if there + were one of them. The prompt is told to name the target in anything it says. + """ + if not (WORKSHOP_URL and WORKSHOP_TOKEN): + LOG.info("infra_health: WORKSHOP_URL/WORKSHOP_TOKEN unset, skipping") + return [] + + try: + req = urllib.request.Request(f"{WORKSHOP_URL}/health") + req.add_header("Authorization", f"Bearer {WORKSHOP_TOKEN}") + with urllib.request.urlopen(req, timeout=TIMEOUT) as resp: + payload = json.loads(resp.read() or b"{}") + except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, ValueError): + LOG.warning("infra_health: could not reach the workshop health poller", exc_info=True) + return [] + + if not payload.get("configured"): + # Nothing is being watched. Distinct from "everything is fine", and the prompt + # must never render the second when the truth is the first. + return [] + + entries = [] + for row in payload.get("results", []) or []: + problems = row.get("detail") or [] + entries.append({ + "category": "infra_health", + "source": row.get("source"), + "target": row.get("target"), + "state": row.get("state"), + "title": f"{row.get('target')}: {row.get('state')}", + "summary": row.get("summary", ""), + "problem_count": row.get("problem_count", 0), + # When this state started. None means it has held for the whole retention + # window, which the prompt should read as "long-standing", not "just now". + "since": row.get("since"), + "checked_at": row.get("checked_at"), + # Capped: a site with 200 failing services is a real state, and putting all + # 200 in an LLM context is not how you say so. + "problems": problems[:12], + "problems_truncated": max(0, len(problems) - 12), + }) + + if entries: + LOG.info("infra_health: %d target(s), overall %s", len(entries), payload.get("overall")) + return entries diff --git a/digest-engine/run.py b/digest-engine/run.py index c2a2a28..aeb4343 100644 --- a/digest-engine/run.py +++ b/digest-engine/run.py @@ -61,6 +61,7 @@ from ingest import ( financial, flight_traffic, grocy, + infra_health, naval_traffic, news_rss, opnsense_ids, @@ -91,6 +92,9 @@ SOURCES = ( ("ENABLE_FLIGHT_TRAFFIC_INGEST", "flight_traffic", flight_traffic), ("ENABLE_NAVAL_TRAFFIC_INGEST", "naval_traffic", naval_traffic), ("ENABLE_OPNSENSE_IDS_INGEST", "opnsense_ids", opnsense_ids), + # Reads workshop's health poller rather than polling CheckMK/the firewalls itself — + # see ingest/infra_health.py for why the digest must not be the thing that polls. + ("ENABLE_INFRA_HEALTH_INGEST", "infra_health", infra_health), ("ENABLE_CALDAV_INGEST", "calendar", caldav_ingest), ("ENABLE_GROCY_INGEST", "grocy", grocy), ) @@ -109,7 +113,7 @@ SECTION_SOURCES = { "political": ("news", "rci_social", "financial", "email", "flight_traffic", "naval_traffic", "calendar"), "household": ("calendar", "grocy"), - "network": ("opnsense_ids",), + "network": ("opnsense_ids", "infra_health"), } DEFAULT_SCHEDULE = "00,06,12,18" @@ -323,6 +327,9 @@ def build_section_contexts(collected, lookback_hours, is_evening_run, previous_r contexts["network"] = { "lookback_hours": lookback_hours, "network_security": collected.get("opnsense_ids", []), + # Host/service state from CheckMK and every firewall's IDS status, each row + # naming its own target. See ingest/infra_health.py and synth/prompts/network.md. + "infra_health": collected.get("infra_health", []), } # A section nobody asked for is dropped here, before its context is ever built into diff --git a/digest-engine/synth/prompts/network.md b/digest-engine/synth/prompts/network.md index fe59728..c40f753 100644 --- a/digest-engine/synth/prompts/network.md +++ b/digest-engine/synth/prompts/network.md @@ -31,6 +31,31 @@ section to pad. no network data was collected this run. Do not infer that the network was quiet, and do not invent an alert, a device, or a signature. +## Machine health: CheckMK, and every firewall by name + +The context may carry entries tagged `"category": "infra_health"` — one per monitored +target, from the always-on poller rather than from this run (see +`ingest/infra_health.py`). Each has a `target`, a `state`, a `summary`, a `since`, and +up to twelve `problems`. + +- **Always name the target.** There may be more than one firewall, and `main` being + healthy says nothing about `dmz`. "The IDS is running" is a sentence you must never + write when the context has two firewalls in it; write "Suricata is running on main; + dmz has not answered since 14:20." +- **`state: "unreachable"` is a finding, not an absence.** It means the poller asked + and got nothing — a machine that is off, a credential that expired, a cable. Report + it as prominently as a real failure, because it is one, and never as "no problems". +- **`since` is what makes this worth reading.** A critical service that went critical + four minutes ago and one that has been critical since Tuesday call for different + reactions. Say which, using the timestamp, whenever `since` is present. When it is + null the state has held for the whole retention window — say "long-standing", never + "just started". +- **`problems_truncated` is a count of what you were not shown.** If it is non-zero, + say so plainly ("12 of 47 shown"). Never summarise 47 failures from 12 of them. +- If there are no `infra_health` entries at all, say nothing about machine health. + Nothing being monitored and everything being fine look identical from here, and only + one of them is good news. + ## History: one alert is noise, the same alert every night is a fact The context may carry a `history` block from the digest's own archive of past diff --git a/digest-engine/synth/prompts/political.md b/digest-engine/synth/prompts/political.md index 29b3e83..dea0d8d 100644 --- a/digest-engine/synth/prompts/political.md +++ b/digest-engine/synth/prompts/political.md @@ -15,9 +15,9 @@ Vorarlberg. Write for someone who already holds this politics and needs to be oriented in the world this week — not for someone who needs to be convinced of it. -## The four questions +## The five questions -Everything below serves four questions. They are the structure of the section, +Everything below serves five questions. They are the structure of the section, not a checklist to append: decide what goes in the digest by asking which question an item answers, and drop it if it answers none. @@ -46,10 +46,59 @@ question an item answers, and drop it if it answers none. sections arrive there, and they are the one source in this context that no feed can supply. Treat a comrade's report as a report, not as an anonymous claim: say which section or comrade it came from where the mail says so. +5. **What class struggles are currently going on around the world?** Strikes, + occupations, factory takeovers, mass movements and general strikes that are + *live right now* — an inventory, not a curation. This is the one question that + does not ask what is most relevant or what moved this run: a dispute that has + been running for six weeks with nothing new to report still belongs on the + list, because the reader needs to know what is going on, not only what + happened since breakfast. See "The roll-call" below for how to build it. Each question gets its own window (see "Output"). A question with nothing worth saying this run gets one honest line, never filler. +Questions 1 and 5 look at the same material and are not the same job. Question 1 +asks what a communist needs to understand — the featured, quoted, analysed items, +few of them. Question 5 asks what is happening — the full standing list, one line +each, no analysis. An item can be in both: featured in `political-global` with +its quote and its theory, and named again in the roll-call as one of the +struggles currently running. That is not duplication, it is the difference +between the briefing and the map. + +## The roll-call + +Question 5's window is a list of the class struggles this run's context shows to +be under way. It has its own rules, because a standing inventory fails in +different ways from a curated section. + +- **Everything on it still comes from this run's context.** "No speculation" + below is not relaxed here: a struggle goes on the list because entries in this + run's material describe it, never because you know of it from elsewhere or + remember it from a previous digest. Your own knowledge of the world is not a + source. +- **`times_seen` and `first_seen_at` are what keep it standing.** An entry that + has appeared in six consecutive runs is exactly the long strike this question + exists to hold on to — it stays on the roll-call at full standing, and the + "a story featured for three runs needs a reason to be featured again" rule in + "History" governs questions 1–4, not this one. +- **One line each**: what, where, who is out or occupying, and since when if the + context says. "Rail workers, France — national strike over pensions, running + since 4 Aug (first seen 5 Aug, in every run since)." No analysis, no impact + paragraph; that belongs to question 1 if the item earns it. +- **Say when a struggle's state is unknown rather than assuming it continues.** + If the last thing the context said about a strike was a fortnight ago and + nothing since, the line says that: "no entry in this run's material since 27 + Jul — outcome unknown here." A roll-call that quietly implies everything on it + is still live is worse than a short one. +- **Ended is a finding.** A strike settled, defeated, sold out or won this run + stays on the list once, marked with how it ended and by whose account, then + drops off. Defeats are not omitted to keep the list encouraging. +- Sources attach the same way as everywhere else — see "Curating and quoting". + A line with no entry behind it does not go on the list. +- Cap it at roughly a dozen entries. If more clear the bar, keep the largest and + those a reader could act on or be asked about, and say plainly that the list + is cut ("12 of ~20 disputes in this run's material"). + ## What this section is, and is not This is a curation-and-correlation exercise, not a running commentary on @@ -149,7 +198,7 @@ the reader might choose to watch or listen to later. says** — never guess at the content of a video from its title alone. - Order newest first, cap it at about six entries, and omit the window entirely when there are no new episodes this run. Never carry an episode over into one - of the four question windows or onto the globe. + of the five question windows or onto the globe. - A `"theory"` article whose title is marked as a podcast (marxist.com prefixes these with `[Podcast]`) belongs in this window too, not in the analysis. @@ -328,7 +377,9 @@ context — a specific entry, figure, or quote. This is not a style preference: ## Curating and quoting For each entry that passes the relevance filter above — i.e. that answers one of -the four questions — and inside the window belonging to that question: +the five questions — and inside the window belonging to that question (question +5's window is the exception: it is one line per struggle with its sources, and +the quoting and analysis rules below do not apply to it, see "The roll-call"): - Feature it explicitly with a short excerpt or quotation taken verbatim from the entry's own text (its title/description/body field, not your @@ -523,6 +574,14 @@ Rules: Omit the window entirely if the run genuinely has none. - `political-rci` — question 4, the International and comrades' reports. Omit if there is nothing; never manufacture organisational news. + - `political-struggles`, titled something like **"Struggles under way"** — + question 5, the roll-call. `kind: "list"`, one line per live struggle, per + "The roll-call" above. Most of the amber `star` and red `hammer-sickle` + markers on the globe should correspond to a line here; keep the two saying + the same thing. Omit the window only when this run's material describes no + live struggle anywhere — say so in one line in `political-global` when that + happens, because it is a surprising claim about the world rather than a quiet + run. - `political-agenda` — the upcoming meetings and their agenda points, per "The agendas" above. Omit when no agenda arrived. - `political-todo`, titled **"Political todos"** — the tasks those agendas @@ -533,11 +592,14 @@ Rules: You may add further windows beyond these when an item needs its own space (a long piece of analysis, the financial indicators as their own `kind: "list"` with each line stating the move *and* what it means for working people). Do not - drop or rename the four question windows to make room. -- At `detail_level: "compact"` keep the globe and fold the questions into a + drop or rename the five question windows to make room. +- At `detail_level: "compact"` keep the globe and fold questions 1–4 into a single `kind: "list"` window — one or two lines each for questions 1 and 2, one line each for 3 and 4 if they have anything. Sources still attach; they cost no - space when folded. **Keep `political-todo` as its own window even here**: it is + space when folded. **Keep `political-struggles` as its own window even here**, + shortened to about six lines and to the largest and nearest struggles, saying + it is cut: a roll-call folded into a summary line stops being a roll-call. + **Keep `political-todo` as its own window even here** too: it is the one part of this section a person acts on rather than reads, and it is the first thing they will look for on a phone. - `narration` is spoken aloud by a TTS voice, so no markdown, no URLs, no emoji. diff --git a/docs/components.md b/docs/components.md index 5b387dc..2a80e43 100644 --- a/docs/components.md +++ b/docs/components.md @@ -44,6 +44,45 @@ Components 1xSound System 1xRuView Presence Node (ESP32-S3 CSI board) +#Workshop / Office + +The room the workshop assistant runs in (`workshop/`, docs/workshop-assistant.md). +Structurally the kitchen display again — a Sway kiosk, a screen, a camera — with two +differences that matter for buying: the screen wants to be BIG (schematics, board +plans and a camera grid at the same time), and the camera is a MACRO problem, not a +wide-angle one. + +1xTiny PC (have — "near infinite Tiny Pcs") +1xLarge monitor or TV, wall-mounted above the bench +1xUSB inspection/macro camera -> reading a service tag or PCB silkscreen at 10-20cm +1xVoice Reciever - Round screen thingy +1xRuView Presence Node (ESP32-S3 CSI board) + +#Network cameras + +Distinct from every USB camera in this project, which are all fixed-purpose at angles +useless for anything else (an item held to the kitchen lens; an appliance door). These +are the ones on the network, ingested by the Frigate/go2rtc this stack already runs, +and they are what the workshop display can actually show. + +2-4xPoE or WiFi camera (RTSP/ONVIF) -> count depends on which rooms; see #Open items + +#Appliance monitoring (fridges + freezers) + +The household runs more than one cold appliance — the main fridge's freezer +compartment is too small, so there is at least a second freezer elsewhere. That is +what makes "which one is it in?" a real question; see +[`fridge-item-location.md`](fridge-item-location.md) for why the answer is *door +sensors plus an outward-facing camera*, and why nothing goes inside the appliance. + +Per appliance (assume 3 to start — kitchen fridge, kitchen freezer compartment, +second freezer): +1xZigbee door contact sensor (mounted on the OUTSIDE face of the door, never inside) +1xDoorway camera (looking at the door opening from outside, ~1.8-2m up) + +The appliances themselves are already owned and are not bought by this project. The +room the second freezer lives in is not decided yet — see #Open items. + ##Lighting ~16x RGB smart bulb (exact count/room split TBD — see #Open items). Zigbee, not @@ -62,13 +101,17 @@ ecosystem. See #Need for the specific pick and why. 2x ThinClient (tiny) 1x Miniscreen System (Lenovo all-in-one, built-in touchscreen — already owned) 1x Beamer -4x Voice Reciever - round screen thingy (Living Room, Loggia, Linus Room, Kitchen) +5x Voice Reciever - round screen thingy (Living Room, Loggia, Linus Room, Kitchen, Workshop) 1x HA Voice PE 5x Sound System (all confirmed: Living Room, Loggia, Linus Room, Amirs Room, Kitchen) 1x Tiny PC with Mini Touchscreen (Kitchen) 1x Camera (Kitchen) 2x Spare webcam (destination TBD) -5x RuView Presence Node (ESP32-S3 CSI board — one per room: Living Room, Loggia, Linus Room, Amirs Room, Kitchen) +6x RuView Presence Node (ESP32-S3 CSI board — one per room: Living Room, Loggia, Linus Room, Amirs Room, Kitchen, Workshop) +3x Zigbee door contact sensor (one per cold appliance door — count depends on the final appliance list) +2x Doorway camera (network/RTSP, one per appliance that isn't already covered by a camera pointing the right way) +1x Tiny PC + 1x large monitor/TV + 1x USB inspection/macro camera (Workshop/Office) +2-4x Network camera (RTSP/ONVIF — rooms TBD, see #Open items) #Lighting ~16x RGB smart bulb (Zigbee) — count/room split not finalized @@ -78,12 +121,30 @@ Living Room: 1x ThinClient, 1x Voice Reciever, 1x Sound System (have), 1x RuVie Loggia: 1x Miniscreen System (have), 1x Sound System, 1x Voice Reciever, 1x RuView node Linus Room: 1x Voice Reciever, 1x Sound System, 1x RuView node, +1 unresolved (???) Amirs Room: 1x Beamer, 1x ThinClient, 1x HA Voice PE, 1x Sound System, 1x RuView node -Kitchen: 1x Tiny PC w/ Touchscreen, 1x Camera, 1x Voice Reciever, 1x Sound System, 1x RuView node +Kitchen: 1x Tiny PC w/ Touchscreen, 1x Camera, 1x Voice Reciever, 1x Sound System, 1x RuView node, + 2x door contact sensor (fridge + freezer compartment), 1x doorway camera +Workshop: 1x Tiny PC (have), 1x large monitor/TV, 1x inspection camera, 1x Voice Reciever, 1x RuView node +Wherever the second freezer is: 1x door contact sensor, 1x doorway camera #Open items - Linus Room's "???" line is still undecided. - Bulb count/room split is a guess ("like 16") — not yet mapped to specific rooms or fixture counts per room. +- **Which room the second freezer is in**, and therefore whether its doorway camera + needs its own PoE/power run or can share the kitchen's. Also whether the kitchen + fridge and its freezer compartment are one door or two — that decides whether the + contact-sensor count is 2 or 3. +- **How many network cameras, and in which rooms.** Priced at 3 as a placeholder. The + workshop display can show any of them; which ones are worth watching is a decision + nobody has made, and unlike the bulbs this one has a privacy dimension — a camera in + a room is a camera in a room, the same caveat `hosts/thin-client`'s README already + spends a section on for gesture control. +- **Whether the workshop monitor needs to be new at all.** It is the only display here + with no touch requirement; almost any HDMI panel or old TV does the job. +- **Whether the kitchen's existing C920 can double as the kitchen fridge's doorway + camera.** It is already there and already on a host, so it is free if the display + happens to face the fridge — and useless if it doesn't. Nobody has measured the + angle; do that before buying a second camera for that room. #Have 1xSound System (earmarked for Living Room) @@ -180,6 +241,74 @@ the identity registration flow, not just kitchen-display/door-panel). pantry-vision's food-photo use case that specifically wants the C920's autofocus, so there's no reason to pay for that here. +#Appliance door sensors — need ~3 +One per cold-appliance door, feeding the Zigbee2MQTT mesh this project already runs +(§1.3) — the same reasoning as the bulbs: one local mesh, not a manufacturer cloud. +These are what trigger the doorway cameras below (`pantry-vision`'s +`POST /doorway-event`, see `pantry-vision/README.md`), and they are worth buying +**even if no camera is ever installed**: on their own they already answer "which +appliance was opened, when" and "has the freezer been standing open for four +minutes", which prevents more spoilage per euro than any amount of item-locating. +- [Aqara Door and Window Sensor (Zigbee)](https://www.amazon.com/Aqara-Window-Sensor-Wireless-Security/dp/B07D37FKGX) + (~€10–15 each) — plain reed-switch contact sensor, long-standing Zigbee2MQTT + support, coin-cell powered. +- **Mount the sensor body on the OUTSIDE face of the door and the magnet on the + outside of the frame** (or vice versa). Not inside the compartment: a coin cell at + −18 °C loses a large part of its usable capacity, and every door opening condenses + humid room air onto whatever is in there. The reed switch does not care which side + of the door it is on, so there is no reason to pay the cold penalty. +- Battery life in this application is **unverified** — a fridge door that opens forty + times a day is a much higher duty cycle than the window these are sold for. Budget + for spare CR1632/CR2032s and check one after a month. + +#Doorway cameras (appliances) — need ~2 +Outward-facing, above the door on the hinge side, ~1.8–2 m, angled so the field of +view is the door aperture and the ~40 cm in front of it. **Never inside the +appliance** — `docs/fridge-item-location.md` has the full argument (no interior +power, condensation on every opening, and −18 °C being outside every consumer camera +module's rating, which rules out exactly the compartment the question came from). +- [TP-Link Tapo C120 (2K, RTSP/ONVIF)](https://www.amazon.com/TP-Link-Tapo-Security-Detection-C120/dp/B0CL5RRZ2M) + (~€30–40 each) — needs to expose a plain **RTSP** stream so Frigate can ingest it, + the same way `chores`' watch points already work. **Unverified**: whether this + specific model serves RTSP without go2rtc as a bridge — the same open question + Phase 20 already carries for its own camera pick (project-plan open decision #23), + and worth resolving once for both rather than twice. +- One of the two **spare webcams** already on this list is a legitimate substitute + for whichever appliance sits next to an existing host — a USB webcam needs a + machine, and the kitchen has one; the second freezer's room probably doesn't. +- Buy **one** first, not both. `docs/fridge-item-location.md` recommends running door + sensors alone for a month before deciding whether item-level localisation is a + thing the household actually wanted, and the test that decides whether the camera + works at all — can a local vision model identify an item in a moving hand at + doorway distance? — can be run today with the kitchen's existing webcam. + +#Workshop display — need 1 screen + 1 macro camera +The Tiny PC is covered by "Have". These two are not: +- Large monitor or TV (32-43", 1080p is enough; 4K only if you want two schematics + side by side). **Any HDMI display works** — this is the one screen in the project + with no touch requirement, because the workshop interaction is voice and keyboard + with dirty hands, not fingers on glass. Reuse anything you have before buying. +- [USB digital microscope / inspection camera, 1080p with adjustable stand](https://www.amazon.com/Microscope-Compatible-Magnification-Adjustable-Compatible/dp/B08BC7GDVL) + (~€30-60) — **not a webcam.** The job is reading 2mm text on a service tag or a PCB + silkscreen, so autofocus at 10-20cm and enough resolution to resolve small print + matter far more than field of view. The C920's autofocus does not go near enough. + This is the single most important pick for whether the OCR-first identification in + `docs/workshop-assistant.md` works at all, and it is cheap enough to test the + premise before committing to the rest. + +#Network cameras — need 2-4 +For the workshop display's camera view, and to give Frigate something to watch beyond +the peephole cam. Must expose **RTSP** — that is the ingest path Frigate and go2rtc +use, and it is the one thing worth checking on the listing before buying. +- [TP-Link Tapo C120 (2K, RTSP/ONVIF)](https://www.amazon.com/TP-Link-Tapo-Security-Detection-C120/dp/B0CL5RRZ2M) + (~€30-40 each) — same model already listed for the appliance doorways, deliberately: + one camera model across the house means one set of quirks to learn, one stream + configuration to get right, and spares that fit anywhere. **Unverified**: whether + this model serves RTSP without go2rtc as a bridge (project-plan open decision #23). +- Buy **one first** and get it into Frigate before ordering the rest. Every camera + after the first is a repeat of a solved problem; the first one is where you find out + whether the model was the right pick. + #Fixed BLE tags — need ~4 to start For household members who don't (or shouldn't have to) carry a phone for presence to work — a grandmother without a smartphone is the concrete case, but this is also just @@ -255,8 +384,8 @@ local pricing/VAT/shipping. | TMY 1080P mini projector | 1 | €40–65 | €40–65 | | HDMI cable | 1 | €6–10 | €6–10 | | Ceiling/shelf mount *(optional)* | 1 | €15–25 | €15–25 | -| Waveshare ESP32-S3-Touch-LCD-1.85C board | 4 | €35–45 | €140–180 | -| USB-C 5V/3A wall adapter | 4 | €7–12 | €28–48 | +| Waveshare ESP32-S3-Touch-LCD-1.85C board | 5 | €35–45 | €175–225 | +| USB-C 5V/3A wall adapter | 5 | €7–12 | €35–60 | | Fosi Audio V3 amp | 4 | €130 | €520 | | Micca MB42X G2 speakers, pair | 4 | €100 | €400 | | 16AWG speaker wire, 100ft spool *(one-time)* | 1 | €15–22 | €15–22 | @@ -265,10 +394,24 @@ local pricing/VAT/shipping. | Logitech C270 spare webcam | 2 | €25–35 | €50–70 | | innr RB 285 C Zigbee RGB bulb | ~16 (TBD) | €15–20 | €240–320 | | iTag-Tiny fixed BLE tag | 4 | €3–8 | €12–32 | -| Espressif ESP32-S3-DevKitC-1-N8R2 (RuView) | 5 | €8–12 | €40–60 | +| Espressif ESP32-S3-DevKitC-1-N8R2 (RuView) | 6 | €8–12 | €48–72 | +| Aqara Zigbee door contact sensor (appliance doors) | ~3 | €10–15 | €30–45 | +| Tapo C120 doorway camera *(buy 1 first — see that section)* | ~2 | €30–40 | €60–80 | +| Tapo C120 network camera (rooms/workshop view) | ~3 | €30–40 | €90–120 | +| Workshop monitor/TV *(reuse one first if you have it)* | 1 | €120–250 | €120–250 | +| USB inspection/macro camera (workshop) | 1 | €30–60 | €30–60 | -**Subtotal (excludes the optional beamer mount): ~€1,831–2,182** -**Subtotal, including the optional beamer mount: ~€1,846–2,207** +**Subtotal (excludes the optional beamer mount): ~€2,244–2,846** +**Subtotal, including the optional beamer mount: ~€2,259–2,871** + +The workshop line items are the ones to sequence rather than buy at once: the macro +camera is cheap and decides whether the OCR-first identification works at all, the +monitor can be anything you already own, and the network cameras are one-then-more. + +The two appliance-monitoring lines are the only ones on this list bought in a +deliberate order rather than all at once: the door sensors are worth having on their +own merits, the cameras are not worth having until the sensors have proved the +question is real. Buying one camera instead of two costs €30–40 to find out. All 5 Sound Systems (Living Room, Loggia, Linus Room, Amirs Room, Kitchen) are confirmed — see `#Open items` above — so all 4 needed amp+speaker kits are in the diff --git a/docs/endpoint-surfaces.md b/docs/endpoint-surfaces.md new file mode 100644 index 0000000..cc43502 --- /dev/null +++ b/docs/endpoint-surfaces.md @@ -0,0 +1,173 @@ +# Two shared endpoint surfaces: the now-playing visualiser and the 3D floorplan + +Design notes for two things that must look the same on every screen in the house — a +thin client on a TV, the Loggia all-in-one, the kitchen panel, the door panel — and +which therefore belong in a **shared, vendored SDK** rather than being written four +times. + +> **Status: both are built** — `render/media-visualiser/` and `render/floorplan-3d/`. +> One thing changed in the building, and it is called out in section 2: the 3D view is +> **canvas 2D, not three.js**. The scene turned out to be prisms on a plane, which an +> isometric projection draws in ~200 lines without the megabyte, without a build step, +> and with predictable performance on the small panels. Everything else below was +> implemented as specified. + +--- + +## 1. The now-playing visualiser + +Wanted: a circular CAVA-style spectrum behind every playing-status screen, coloured +from the album art, with lyrics scrolling under the cover when they exist. + +### The audio problem, which is the whole problem + +CAVA reads an audio stream. The endpoints do not all have one: + +| Endpoint | Is the audio local? | Can it get a spectrum? | +|---|---|---| +| Thin client running mpv/spotifyd | **yes** | Yes — real FFT of the actual output | +| Audio endpoint (Spotify Connect box) | yes, but it is headless | Yes, but there is no screen to draw on | +| Kitchen / door panel showing *what the living room is playing* | **no** | **No. There is no audio here at all** | + +So a design that requires real audio analysis works on one endpoint and silently +degrades to a dead circle on the others — the worst outcome, because the screen looks +broken rather than looking deliberate. + +**Two-tier, declared up front:** + +- **Reactive tier** — where audio is local, CAVA (or a WebAudio `AnalyserNode` when the + player is in the browser) drives the bars. `cava` has a `raw` output mode writing + plain numbers to a FIFO, which a small agent can publish over MQTT; that is the least + fragile path on a machine already running an agent. +- **Synthetic tier** — everywhere else, the ring is animated from **track position and + tempo**, not from silence. It breathes with the beat rather than pretending to + analyse it. Nobody watching a kitchen panel from across the room can tell, and it + never looks broken. + +Do not let the synthetic tier claim to be the reactive one anywhere in the UI. It is a +mood light, and the moment somebody believes it is a spectrum they will report it as a +bug every time it does not match a bass drop. + +### Colours from the album art + +Pull 3–5 dominant colours from the cover, client-side, at load: + +1. Draw the cover into a small offscreen canvas (64×64 is plenty — this is a palette, + not a photograph). +2. Bucket pixels in RGB space, take the top buckets by population. +3. **Reject near-greys and near-blacks** before ranking. Album art is full of them, and + a palette extracted naively from a dark cover gives you four indistinguishable dark + greys and a visualiser that looks switched off. +4. **Force a minimum contrast against the background.** Lift the chosen colours in + lightness until they clear the panel behind them; a visualiser you cannot see is + the same as no visualiser, and this is the step that gets skipped. + +No library needed — that is about forty lines of canvas work, and it keeps the +"vendored, dependency-free" property the canvas SDKs already have. + +### Lyrics + +- **Source**: whatever the player exposes. Music Assistant and MPD both surface + synced-lyrics fields when the provider has them; `.lrc` sidecar files are the other + common case for local libraries. +- **Synced (LRC) vs plain**: with timestamps, highlight the current line and scroll it + to centre. Without them, do not fake it — scroll slowly, or just show the text. A + plain-lyrics block auto-scrolled at a guessed rate is wrong within ten seconds and + stays wrong. +- **Absent is the normal case.** Most tracks in most libraries have no lyrics. The + layout must be designed for "no lyrics" as the default state, with lyrics as the + addition — not a gap where they would go. +- Never fetch lyrics from the internet at render time. If lyrics are worth having, + they are worth caching where the track is. + +### Where it lives + +A new `render/media-visualiser/` in the shared SDK style: one JS file, one CSS file, +vendored into each host that needs it, exactly as `digest-canvas-sdk` and +`canvas-sdk` already are. Inputs are a normalised now-playing object +(`{title, artist, album, art_url, position_ms, duration_ms, tempo?, lyrics?}`) and an +optional spectrum feed. Every endpoint already receives now-playing over MQTT; nothing +new has to be plumbed for the synthetic tier. + +--- + +## 2. The 3D floorplan + +Wanted: the Pebble app's presence view, in 3D, on any endpoint — rooms, and who is in +them, with profile pictures. + +### It is the same data, and that is the point + +`GET /floorplan/presence` already returns everything: room polygons normalised 0–1, +each room's occupants, each occupant's `color`, `initial` and `has_photo`, plus +`unplaced` for people who are home but not locatable. The Pebble app and this render +the same JSON at different fidelities — which is the reason to build the second one at +all. If they diverge, one of them is lying. + +### Extrusion, not modelling + +Do not author a 3D model. Take the existing 2D polygons and **extrude them to a wall +height**, which is one `THREE.ExtrudeGeometry` per room and needs no new data beyond a +single `wall_height_m` per level. A hand-authored model would be prettier, immediately +stale the first time a room is redrawn, and unmaintainable by the person who drew the +plan in a 2D editor. + +Camera: fixed isometric by default, drag to orbit, no free-fly. An orbit camera on a +wall panel is something people knock out of alignment and cannot get back. + +### The occupant markers + +This is where 3D earns itself, because the Pebble's constraint — 18×18 px, no room for +a face — is gone: + +- A **billboarded disc** above each occupied room, always facing the camera (never a + flat sprite lying on the floor, which is unreadable at a glance). +- **Profile picture inside the disc**, ringed in the person's colour, falling back to + the initial on the colour when `has_photo` is false. The ring matters even with a + photo: it is what ties this marker to the same person's marker on the watch and in + the admin panel. +- **Occupied rooms lit, empty rooms dark**, the same rule the watchapp uses, and for + the same reason: state should read before detail. In 3D that is an emissive floor + material rather than a fill colour, and the third state — *drawn but never reported + by HA* — stays visually distinct, because rendering "no data" as "empty" is a quiet + lie on any screen. +- **`unplaced` people get a shelf**, not a hidden list: a strip along the bottom + showing everyone who is home but unlocatable. They are the people you are most often + looking for. + +### The honest constraint — and how it was resolved + +The plan was to vendor three.js and call it a deliberate break with the +dependency-free rule. Building it made the cheaper answer obvious and it was taken +instead: **canvas 2D with an isometric projection and painter's-algorithm sorting.** + +The scene is prisms standing on a plane — no lighting model worth the name, no +textures, no physics, no camera motion beyond an orbit. That is ~200 lines of canvas +2D, it redraws in well under a millisecond on the small panels (where a WebGL context +on integrated graphics is a much less predictable proposition), and the frontend stays +at zero dependencies with nothing to keep patched. + +This is not a compromise version of the three.js plan; it is the smaller correct tool +for this specific scene. If the view ever grows real lighting or an imported model, +three.js becomes right and `floorplan3d.js` becomes the fallback for the weak panels. + +And the same caveat as the watchapp, which no amount of rendering fixes: **this is only +as right as room-level presence is**, which has never been measured in this house. A +beautiful 3D house with everybody sitting in the `unplaced` shelf is a beautiful 3D +house that tells you nothing. Test that first — the admin panel's floorplan tab with +**Live** ticked answers it for free. + +--- + +## Build order + +1. **The visualiser's synthetic tier**, on one endpoint. It needs no new data and no + audio plumbing, and it is what tells you whether the effect is worth the reactive + tier's complexity. +2. **Album-art palette extraction**, which is where the look actually comes from. +3. **Lyrics**, synced-only at first. +4. **The reactive tier**, on the thin clients that have local audio. +5. **The 3D floorplan** — built. It turned out not to be a dependency addition at all + (see above), so the sequencing argument that put it last no longer applies. The + *other* reason still does, in full: **it is only as right as room-level presence, + which has never been measured here.** Test that before believing the picture. diff --git a/docs/fridge-item-location.md b/docs/fridge-item-location.md new file mode 100644 index 0000000..e910ddb --- /dev/null +++ b/docs/fridge-item-location.md @@ -0,0 +1,223 @@ +# Knowing *where* a thing is: multiple fridges, shelves, and what it would cost + +A feasibility note that has since grown a build. `pantry-vision` (Phase 17) knows +**what** the household has and **when** it expires. This asks what it would take to +also know **where** — which of several fridges, and how far inside one. + +The recommendation at the bottom is deliberately smaller than the question, and the +reason is in the middle. + +> **Status, 2026-08-10.** Steps 1 and 4 of the recommendation are now written: +> appliances as Grocy locations with a `POST /transfer` action and a "Move to…" +> control on the edit screen, and the door-sensor→camera→hint path +> (`pantry-vision/doorway.py`, `POST /doorway-event`). **Step 2 is hardware nobody has +> bought and step 3 is a month nobody has spent** — which means the camera half is +> written against an assumption that has never been tested: that a local vision model +> can identify an item in a moving hand at doorway distance. An appliance can be +> configured with a door sensor and no camera, and that remains the recommended way to +> start. Nothing about writing the code changes the order in which it is worth +> switching on. + +## First, decide which question is actually being asked + +These are three different problems with three different price tags, and they get +conflated constantly: + +| Level | The question | Honest difficulty | +| --- | --- | --- | +| **L1 — which appliance** | "Is the mustard in the kitchen fridge or the loggia one?" | Solved today, no new hardware, one extra tap | +| **L2 — which zone** | "Door shelf, middle shelf, or the crisper drawer?" | Achievable at *hint* quality (60–80%), real hardware, real work | +| **L3 — exact position** | "Second from the left, behind the milk" | Not worth it. See "Occlusion" below | + +The value is overwhelmingly at L1, and it is the level this household actually has a +problem at — the whole premise of the question is *multiple fridges*, because the +freezer compartment of the main one is too small. Somebody standing in the kitchen +wondering whether to walk to the loggia is asking an L1 question. L2 saves ten seconds +of looking; L3 saves none, because by the time you have the door open you can see. + +**Do not build downward from L3.** Every camera-in-the-fridge design starts there, +because that is the impressive demo, and every one of them founders on the same rock. + +## L1: available now, and mostly a data-model decision + +`pantry-vision` already resolves the confirm screen's fridge/freezer/cupboard answer +to a **Grocy location**, created by name (`PLACEMENT_LOCATION_NAMES` in `server.py`, +`PANTRY_LOCATION_*` in the env file), and passes `location_id` on the stock add. Grocy +has modelled multiple locations natively since long before this project existed. + +So L1 is not a vision problem at all. It is three small changes, and two and a half of +them are now written: + +1. **The placement answer becomes the real appliances** — "Fridge (kitchen)", "Freezer + (loggia)", "Cupboard" — named in `PANTRY_LOCATION_*` and resolved to Grocy + locations. *Done, with a caveat*: the confirm screen still offers the four + categories (fridge / freezer / cupboard / counter) and each maps to one location + name, so a household with two fridges has to pick which one that word means and + move the other by hand. Reading the appliance list straight off Grocy's locations + is the obvious next step and has not been taken. +2. **`/inventory` carries the location through**, so every list says where. *Done* — + `location_id`/`location` on every row, `locations` on the response. +3. **Moving something between fridges needs a "moved" action**, or the data rots + within a fortnight. *Done* — `POST /transfer` onto Grocy's own transfer endpoint, + with a picker on the edit screen. + +Point 3 is the whole ballgame, and it is the thing every camera proposal below is +really trying to buy its way out of: **the expensive part is not learning where things +are, it is noticing when they move.** + +## L2/L3: what the camera approaches actually run into + +### Occlusion is the rock + +A single camera sees the front row. Groceries are stored in rows. The mustard behind +the milk is invisible, and no model — local, cloud, or otherwise — recovers it from a +picture that does not contain it. Multi-camera stereo does not fix this either; it is +not a depth problem, it is a line-of-sight problem. + +This is why L3 is off the table and why L2 is a *hint*, not an assertion. Any design +that needs the camera to enumerate a shelf's contents is buying an answer that is +silently incomplete, which for an inventory is the worst failure available: "we have +no mustard" from a system that simply could not see it produces a second jar of +mustard. + +### The interior-camera problems, in the order they will bite + +- **Power.** Fridges have no interior outlet. The three options are a battery (poor: + cold cuts usable lithium capacity substantially, and this is a duty-cycled camera, + not a doorbell), a flat ribbon cable through the door gasket (works — this is how + retrofit fridge cams do it — but it is a modification to the seal of an appliance + that may be rented), or drilling a grommet (irreversible). +- **The freezer is out of scope for interior cameras, full stop.** Off-the-shelf + camera modules — ESP32-CAM, Pi camera, USB webcams — are specified to roughly 0 °C + at best. −18 °C is outside every consumer part's rating: lubricants stiffen, + electrolytics lose capacitance, and condensation cycling on every door open + eventually gets inside the enclosure. Industrial cold-chain cameras exist and cost + more than the freezer. **Given that the second appliance exists precisely because + the built-in freezer is too small, this rules interior cameras out of the exact + compartment the question started from.** +- **Condensation.** Every door opening puts warm humid air on a cold lens. It fogs, + and it fogs for minutes — longer than the interaction. Mitigations (sealed housing + with desiccant, conformal coating, a lens heater) all cost power, which brings back + problem one. +- **Light.** The interior lamp is on only while the door is open, so the only usable + frames are the ones during an opening — at which point the camera might as well be + outside. + +Every one of those bullets is a *class* of problem, not a part-selection detail. Taken +together they say: **do not put the camera inside.** + +### The approach that survives all four: watch the doorway, not the shelf + +Point a camera at the **plane of the door opening**, from outside, and trigger on a +door sensor. Then you are not parsing a packed shelf — you are watching one item, held +in one hand, unoccluded, at a known moment, against a known background, with the +kitchen's own lighting. That is the *same* recognition problem `/identify` already +solves, which means it reuses the prompt, the model, and the "propose, never +auto-commit" guardrail instead of needing a new perception stack. + +What it buys: **transactions, not state.** "A jar of X went into the loggia fridge at +18:42." Transactions are what keep L1 from rotting (the problem above), and a +sequence of them gives L2 for free if the camera can also see *which shelf the hand +went to* — a much weaker inference than reading the shelf, and one that degrades +gracefully into "somewhere in this fridge". + +What it costs in honesty: it misses. Two items at once, an arm across the lens, a foot +closing the door. So its output is a **hint with a timestamp** — "last seen going into +the loggia fridge, Tuesday 18:42" — never an assertion the UI treats as fact. That +framing is what makes a 70%-accurate system useful instead of infuriating, and it is +the same distinction `digest-engine` already draws between what a source *says* and +what is *so*. + +This is what `pantry-vision/doorway.py` implements. An HA automation on the contact +sensor POSTs `/doorway-event`, the service answers 202 and takes a short burst off the +request thread (three frames, stopping at the first that recognises anything), and the +result lands in **its own SQLite file** — not in Grocy, because Grocy owns stock and +has nowhere to put "a camera thinks it saw something like this go past that door, and +might be wrong". Hints expire after 30 days, because "last seen a month ago" tells +nobody anything they didn't know. The edit screen shows the sighting next to the +location Grocy records, and acting on it is a tap on *Move*. + +### Where the cameras go, concretely + +- **One camera per appliance**, mounted above the door on the hinge side, ~1.8–2 m, + angled down so its field of view is the door aperture and the ~40 cm in front of it. + Not inside the door swing. `docs/components.md` already lists **2× spare webcam + (destination TBD)** — this is a plausible destination for one of them, which makes + the experiment nearly free. +- **A contact sensor on every fridge and freezer door.** Zigbee, on the coordinator + this project already has. This is the single highest value-per-euro item in this + whole note and it is worth doing *even if no camera is ever installed*: it is what + triggers the capture, it is what tells you which appliance a transaction belongs to, + and on its own it already supports "the loggia freezer has been open for four + minutes" — which prevents more spoilage per euro than any amount of localisation. +- **No new inter-VLAN paths.** Both devices sit where the existing camera/IoT devices + sit; the capture is pulled by the container host, same direction as everything else. + +### Approaches ruled out, and why + +- **RFID/NFC tags per item.** The only technology that genuinely answers L3. It also + requires tagging every single item by hand, which is a *worse* manual step than the + one tap L1 costs — and a fridge is a metal box full of water, which detunes UHF tags + badly. Rules itself out twice. +- **Load cells under each shelf.** Excellent at "how much is left", blind to identity, + and a per-shelf retrofit on appliances that may be rented. +- **BLE/UWB.** Nothing to attach a transmitter to. Not applicable to passive goods. + +## The compute question nobody asks until it's too late + +Reading a shelf is not the same task as reading one held-up jar: small objects, +partial labels, angles, occlusion. A 7–13B local VLM will be markedly worse at it than +at the current task, and the honest architecture is detect-then-crop — a small object +detector proposing boxes, the VLM reading each crop — which is a Frigate-class, +always-on GPU workload. + +There is one RTX 3060 in this design, and contention on it is already an open +question (project plan open decision #4, `MAX_LOADED_MODELS=1`). Adding a +continuous detector to a GPU that also has to answer Assist within voice latency and +run four digests a day is a scheduling problem, not a spare-capacity problem. **The +doorway-camera approach avoids this too**: it is event-driven, a handful of inferences +per day, on the model that is already loaded. + +## Recommendation + +In order, stopping wherever the household stops caring: + +1. **Make L1 real in software** — appliances as Grocy locations, location shown in + every list, and a transfer action so moves get recorded. No hardware. This is the + only step that is unambiguously worth doing, and it must come first regardless: + without it there is nowhere to *put* a camera's answer. **Written.** +2. **Contact sensors on every fridge/freezer door.** Cheap, Zigbee, immediately useful + on their own, and the prerequisite for everything below. See + `docs/components.md`'s "Appliance door sensors" — ~€10–15 each, mounted on the + *outside* face of the door, because a coin cell at −18 °C is a false economy. + **Not bought.** +3. **Live with 1+2 for a month.** The real finding will be whether anyone ever wanted + L2, or whether "which fridge" plus "the door's been open" was the whole need. This + step is not padding — it is the only way to avoid building L2 for a question nobody + turned out to be asking. **Not done, and writing step 4 early does not skip it.** +4. **Only then**, a doorway camera on one appliance, producing timestamped *hints*, + reusing `/identify` unchanged. One appliance, not all of them, until it earns the + second. **Written** (`pantry-vision/doorway.py`), **untested against a real camera, + and configurable per appliance so it can stay off.** +5. **Never** an interior camera in the freezer, and probably not in the fridge either. + +A related chore was added alongside this: `chores`' **`groceries_out_of_place`** watch +point, which notices food standing out on a counter and nudges whoever the camera last +saw — litter's twin, with a 30-minute fuse instead of four hours, because the failure +there is spoilage rather than untidiness. It is a *different* system looking at a +*different* surface for a *different* reason, and deliberately so: it knows food is +out, not what the food is, and it never touches Grocy. See `chores/README.md`. + +## What in here is unverified + +Everything with a number in it. Specifically: consumer camera modules' actual +low-temperature behaviour (the 0 °C figure is a datasheet-class generalisation, not a +part this project has picked or tested); how badly a lens really fogs on a door +opening in this kitchen's humidity; whether a local VLM can identify an item held in a +moving hand at doorway distance at all — that last one is the assumption step 4 lives +or dies on, and it is testable today with the kitchen display's existing camera and +about twenty minutes, long before anything is bought. + +Related open decisions in `docs/project-plan.md`: #18 (no vision model picked, +benchmarked, or measured for latency) and #4 (GPU contention). Both apply here +unchanged, and neither gets easier with a second camera pointed at anything. diff --git a/docs/pebble-presence-watchface.md b/docs/pebble-presence-watchface.md new file mode 100644 index 0000000..cde8381 --- /dev/null +++ b/docs/pebble-presence-watchface.md @@ -0,0 +1,280 @@ +# A Pebble watchapp that shows who is in which room + +Feasibility note. The question: take the floorplan drawn in `identity`'s admin panel, +put it on a Pebble, and mark each room with who is standing in it. + +**Verdict: the watch is the easy part.** Drawing the plan is a few hundred bytes and +some `gpath` calls, and the data already exists at `GET /floorplan/presence`. It is a +**watchapp**, not a watchface — see "Zooming into a room" for why the buttons decide +that. + +The hard part is upstream and unchanged by any of this: **room-level presence +itself.** Build it if you want it, but do not expect it to be more right than the +presence data feeding it, which today is "which HA area a trusted BLE entity reports" and has +never been measured for room-level accuracy in this house. + +## The hardware: a Pebble Round 2 + +**That is the target — it is the watch that exists.** 260×260, 64-colour e-paper, +touch. For reference, the rest of the line: + +| Model | Screen | Colour | Notes | +|---|---|---|---| +| **Pebble Round 2** | 260×260 | 64-colour e-paper, touch | **The target.** The most pixels in the line, and circular — see the next section, which is entirely about that | +| Pebble Time 2 | 200×228 | 64-colour e-paper, touch | Rectangular, so a plan fits without the geometry below. Fewer pixels, more usable ones | +| Pebble 2 Duo | 144×168 | Black and white | The per-person colour collapses to a grey; two people sharing an initial stop being distinguishable | + +Classic Pebbles (144×168) run the same code, cramped. + +## Fitting a rectangular plan on a round screen + +A floorplan is a rectangle and the screen is a circle, so the plan gets inscribed in +the circle and the corners are simply not available. The arithmetic, for a plan of +aspect ratio `a = w/h` on a usable diameter `D`: + +``` +w = D·a / √(a²+1) +h = D / √(a²+1) +``` + +With `D = 240` (260 less a ~10 px margin, because a circular screen's outermost pixels +are where you least want a room boundary): + +| Plan aspect | Usable box | +|---|---| +| 1:1 | 170 × 170 | +| 4:3 | 192 × 144 | +| 16:9 | 209 × 118 | + +So a squarish plan gets **170×170** — comparable to what a Time 2 gives after chrome, +and more than the 144×168 classics ever had. The Round is not a downgrade for this; +it just has to be told the truth about its shape. + +Three consequences to design to: + +- **Compute the box in PebbleKit JS, not on the watch.** The projection step already + planned for (normalised polygons → pixels) simply takes the inscribed box instead of + the full screen. No watch-side change at all. +- **No corner furniture.** The "3 of 4 home placed" footer has nowhere to live on a + circle — a line of text at the bottom gets clipped by the bezel radius. Put it at the + vertical centre-bottom, short, or move it into the room-detail screen where there is + a full-width line to use. +- **Round screens have less usable area than their diameter suggests**, and a ten-room + plan at 170×170 gives each room roughly 40×35 px. That is still enough for a 16 px + occupant dot with an initial in it — which is the number that decided against + profile pictures above, and it does not change here. + +## The data path + +``` +identity ──HTTP──> PebbleKit JS ──AppMessage/BT──> watchapp (C) +(container host) (runs on the Android ~2 KB budget gpath + text + /floorplan/presence phone, inside the rendering + Pebble app) +``` + +Three consequences, all of which shape the design: + +- **The watch has no network of its own.** Everything goes through PebbleKit JS, which + runs on the phone, inside the Pebble app, and — this is the important part — + **only while the app is open.** It cannot poll in the background — which bounds the + battery cost, and also means every launch begins with a fetch. +- **The phone has to be able to reach `identity`.** On the home network that is direct. + Off it, that is the WireGuard split tunnel this project already uses for exactly this + class of problem — nothing here justifies putting `identity` on the WAN. If the + tunnel is down it shows its last state with an age on it, not a blank plan. +- **Android only**, which this household is anyway, so no iOS-side caveats apply. + +## The payload budget, and why it is not a problem + +Pebble's guaranteed AppMessage buffers are small — 124 bytes in / 636 out at the +documented minimum, ~2 KB each way in practice for a JS-backed app. That sounds +alarming until you count what a floorplan actually is: + +- Room polygons are already stored **normalised 0.0–1.0** (`floorplan_rooms.points`). + Quantise each coordinate to one byte and a vertex costs 2 bytes. +- A ten-room plan at eight vertices a room: `10 × (1 + 8×2) = 170 bytes`. +- An occupant marker is 3 bytes: room index, colour index, initial. + +So a whole floor with everybody on it lands around **200–250 bytes** — comfortably +inside one message, with room to spare for a level name and a timestamp. Send it as a +single byte-array tuple rather than one tuple per room; the dictionary overhead is +what would actually cost you. + +**Do the projection on the phone, not the watch.** PebbleKit JS picks the level, scales +the normalised polygons to the watch's pixel box — the inscribed box, on the Round — +drops rooms too small to draw, and sends integers. The watch does `gpath_create` → `gpath_draw_filled` → outline → text. +No floating point, no layout logic, no second copy of the floorplan model on a device +with 64 KB to its name. + +## Empty rooms dark, occupied rooms lit + +The plan should read as a *state* at a glance, not as a drawing you have to search. So +occupancy is carried by the room fill itself, before you look at any marker: + +| | Fill | Outline | Contents | +|---|---|---|---| +| **Empty** | near-black, barely above the background | dim | nothing, or the room's initial letter in the dim outline colour | +| **Occupied** | light — a pale warm grey/white | bright | the occupant dots, in their own colours | + +Three details that decide whether this works: + +- **Carry it in lightness, not hue.** The occupant dots are already using colour to + mean *who*; if the room fill also used colour to mean *occupied*, the two would + compete on the one channel that matters most on a 64-colour panel. Dark-vs-light is + the strongest signal e-paper has and it costs nothing. +- **A lit room must be lighter than any occupant dot is dark**, or the dot vanishes into + its own room. With `PERSON_COLORS` being mid-to-bright, a near-white fill and a dark + dot outline keeps every one of the eight readable — this is the same reason the admin + panel draws initials in near-black on the person's colour. +- **"Unknown" is not "empty".** A room the plan has drawn but whose area HA never + reports is neither occupied nor confirmed-empty, and rendering it as empty is a + quiet lie. Give it the empty fill with a **dashed or dotted outline** — a third state + that costs one drawing call and is the difference between "nobody is in the study" + and "nothing can see the study". + +E-paper is reflective, so "lit" here means a lighter fill, not a backlight. The effect +is exactly the one you want in a dark hallway at 2am: the rooms with people in them are +the bright shapes. + +## Profile pictures: no, and the arithmetic says so + +On the Round 2, a ten-room plan inscribed at 170×170 gives each room about 40×35 px. A +face inside one, with the room outline still visible, gets about **18×18 px, in 64 +colours, on e-paper**. That is not a picture of a person; it is four skin-toned blobs. +And each one costs ~330 bytes to ship, so three of them exceed the entire message +budget that currently carries the whole floor. The rectangular Time 2 is no better — +45×35 px rooms, a 20×20 face — so this is a conclusion about the class of device, not +about the shape of this one. + +**Initial plus colour is the right answer, and it is why the colour exists.** A filled +16 px circle in the person's own colour with their initial in `GOTHIC_14_BOLD` reads +at arm's length, costs 3 bytes, and degrades honestly: on a black-and-white Pebble the +colour becomes a grey and the letter still works. + +`identity` now serves both fields ready-made — `color` and `initial` on `/people`, +`/presence`, and each occupant in `/floorplan/presence` — so no consumer has to derive +an initial from a name or invent a palette. **The eight palette colours are chosen on +the 2-bits-per-channel lattice (`00/55/AA/FF`) that a colour Pebble renders natively**, +precisely so the colour on the watch is the colour in the admin panel and not a +dithered approximation of it. + +## Zooming into a room: why this is an app and not a face + +The wanted behaviour — press a button, cycle to the next room, see a plain list of who +is in it — is easy to draw and cheap in bytes. **The catch is that it forces a +watchapp instead of a watchface**, and that is a real product decision, not a detail: + +- **Watchfaces do not receive button events.** The buttons belong to the system there + (Select opens the app menu, Up/Down are the system shortcuts), and Pebble's own docs + say **touch is deliberately restricted to watchapps too** — "easier to allow it later + than to take it away once apps depend on it." +- A **watchapp** gets buttons and touch, and can do exactly the requested cycling. What + it does not get is being your default screen: you launch it from the menu, look, and + leave. + +So pick which the thing actually is: + +| | Watchface | Watchapp | +|---|---|---| +| Shows without launching | **yes** — this is the whole glance-at-wrist value | no, it's a menu entry | +| Button cycling through rooms | no | **yes** | +| Shake to cycle | yes (`accel_tap_service_subscribe`, the long-standing shake-to-reveal trick — worth confirming on your firmware) | yes | +| Practical shape | the plan, occupant dots, nothing else | the plan **plus** the per-room drill-down | + +**Decided: build the watchapp.** Button cycling is the point, so the drill-down wins +over being the default screen. What that costs, stated plainly so it isn't a surprise +later: you launch it from the app menu rather than seeing it by raising your wrist, and +its JS — and therefore its data — only lives while it is open, so every launch starts +with one fetch and a moment of "loading". Design for that: draw the last-known state +immediately with its age on it, then repaint when the fetch lands, rather than showing +a spinner on a screen that already has something true to say. + +One package is either a face or an app — no binary is both — but the rendering, the +AppMessage handler and the JS are all shared, so **a watchface variant later is a +second `main()` and a build target, not a second project.** Worth keeping that seam +clean while writing it, in case the glance turns out to be what you actually reach for. + +### What the drill-down shows + +Rooms in the order they are drawn, wrapping at both ends, with Back leaving. (On the +Round 2, confirm which of buttons and touch you actually want to drive this — it has +both, and a circular screen makes a swipe more natural than a button press for +"next". The click config provider and a touch handler are the same twenty lines either +way.) + +``` + Kitchen <- room name, Up/Down cycles + ───────────── + ● Amir <- the person's colour, then their full name + ● Anna + <- "Nobody here" when empty, never a blank screen + 3 of 4 home placed <- the honest footer, see below +``` + +Full names cost bytes the overview does not need — ten people at ~12 bytes is ~120, +still nothing against the ~2 KB budget, so send them with the plan rather than making a +second request per room. The colour dot stays even though there is room for the name: +it is what ties this screen back to the marker on the plan. + +**Give the empty and the unknown cases real text.** "Nobody here" is a finding. +"3 of 4 home placed" is the truth that the overview can only gesture at — and the +`unplaced` list from `/floorplan/presence` deserves its own entry at the end of the +cycle ("Somewhere in the house: Bibi"), because a person the system cannot locate is +exactly who you were looking for when you picked up the watch. + +Where a photo *does* belong: nowhere here either. Once you have room to print +"Amir" you have already solved the problem the picture was for. + +## Update cadence + +The JS only lives while the face is displayed, so "polling" means "while you are +looking at it": + +- Fetch once on load. +- Refresh on a `tick_timer` every 2–5 minutes while visible. +- Refresh on tap/shake, for the "who's home *right now*" glance that is the actual use + case. + +Do not refresh every second, and do not attempt a background service to keep it warm: +Bluetooth wakeups are the battery cost on both devices, and a presence display that is +four minutes stale is not wrong in any way that matters. + +## What is already in place, and what is missing + +Already there: + +- `GET /floorplan/presence` — the drawn plan joined to who is in each room, with + `unplaced` for people who are home but not locatable and `unmapped_areas` for areas + HA reports that nothing on the plan claims. Both matter on a small screen: "3 home, + 1 not locatable" is honest, and quietly dropping two people is not. +- `color` and `initial` on every person, in every presence payload. +- Bearer-token auth on the whole API — the token would have to live in the app's + Clay settings, which is a real consideration: it is stored on the phone in the Pebble + app's config, and it is a token that reads the household's presence history. + +Missing, in the order it would need doing: + +1. **A compact serialisation.** Doing the quantisation in PebbleKit JS keeps the server + general and is the right first move. If polygon vertex counts turn out to be large + (a hand-drawn plan can easily have 20-vertex rooms), a `?format=compact` on + `/floorplan/presence` that simplifies and quantises server-side is the fallback. +2. **The watchapp itself** — `gpath` rendering for the overview, a click config + provider, the room-detail window, an AppMessage handler, and a Clay settings page + for the API URL and token. This is a weekend, not a project. +3. **A level picker**, because the plan is per-level and a watch shows one at a time. + Another button, or fold the levels into the same Up/Down cycle after the last room. + +## The thing that decides whether this is worth building + +**Room-level presence.** `identity` resolves a person's room from whatever +`AREA_ATTRIBUTE` holds on their trusted BLE entity — which is as good as the BLE +proxy layout and HA's area assignment, and this project has never measured it. If in +practice everyone resolves to "home, room unknown", the app is a picture of a +floorplan with everybody sitting in the `unplaced` list at the bottom, and no amount +of watch-side work fixes that. + +That is testable today, without buying anything: open the admin panel's floorplan tab +with **Live** ticked, walk between two rooms, and see whether the marker moves. If it +does, the app is a weekend. If it doesn't, the work is in the BLE proxies, and +the watch is a distraction from it. diff --git a/docs/project-plan.md b/docs/project-plan.md index 7401e5b..5c083cb 100644 --- a/docs/project-plan.md +++ b/docs/project-plan.md @@ -113,10 +113,13 @@ from Phase 1, same as every other MQTT-connected host in this plan.)* |---|---|---| | Mini PC or SBC with a touchscreen, mounted near the fridge/pantry | €150–300 | Prefers a native Wayland `wl_touch` device, same as §1.14 — unlike that host, `hosts/kitchen-display/` does not (yet) ship the `type:pointer`/udev-override fallback described there; a misclassified touchscreen here is still an open problem | | USB webcam, pointed at wherever items get held up for scanning | €15–40 | Either built into the panel or on a short cable/gooseneck so its angle can be set independently of the screen. Any UVC webcam works — this is the same "no depth camera, no accelerator needed" bar as the thin client's gesture-control camera (§1.11), just used for a photo instead of continuous tracking | +| Zigbee door contact sensor, one per cold appliance door | €10–15 | ~3 to start (fridge, freezer compartment, second freezer). Mounted on the **outside** face of the door — a coin cell at −18 °C is a false economy and every opening condenses room air onto whatever is inside. Worth buying on their own merits ("has the freezer been open four minutes") before any camera exists; see `docs/fridge-item-location.md` | +| Doorway camera, one per appliance (optional, later) | €30–40 | Outward-facing, above the door on the hinge side, ~1.8–2 m, ingested by Frigate like every other camera in this project. **Never inside the appliance.** Buy one, not two — and only after the door sensors have been lived with, per `docs/fridge-item-location.md`'s recommended order | *(No new container-host hardware — `pantry-vision` is a container on the existing Phase 1 host, calling the existing Phase 3/9 LLM host for vision inference and the -already-running `grocy` container for storage.)* +already-running `grocy` container for storage. It does gain one small bind mount, +`/data`, for the doorway-hint database — see Phase 17 item 4a.)* ### 1.16 Door/wardrobe panel hardware (Phase 18) | Item | Est. Price (EUR) | Notes | @@ -215,7 +218,7 @@ real hardware" callouts for everything downstream of this.)* | Grocery vision recognition | **Ollama** (a vision-capable model, e.g. `llava`/`qwen2.5vl` — TBD, not yet pulled or benchmarked) | Identifies a grocery item from one photo and estimates its shelf life; the proposal is always human-reviewed before anything is written (see `pantry-vision/README.md`) | | Grocery inventory backend | **pantry-vision** (custom Python, stdlib `http.server`) | `POST /identify` (photo → proposal), `POST /confirm` (human-reviewed proposal → Grocy stock write), `GET /inventory`/`GET /recipes` (proxy Grocy, reshaped). Bearer-token gated, **and — unlike admin-canvas — LAN-published**, since the kitchen display's kiosk browser calls it directly rather than through Home Assistant | | Grocery inventory storage | **Grocy** (already running, Phase 1/7) | The single source of truth for stock/best-before-dates/recipes; `pantry-vision` is a client of Grocy's own REST API, not a replacement for it | -| Kitchen-display static serving | **pantry-web** (nginx:alpine) | Serves `pantry-vision/frontend/`'s Scan/Inventory/Recipes single-page app read-only to the kitchen display — same role `digest-web`/`admin-web` play for their own hosts | +| Kitchen-display static serving | **pantry-web** (nginx:alpine) | Serves `pantry-vision/frontend/`'s single-page app (unload / consume / expired / edit, plus Inventory and Recipes) read-only to the kitchen display — same role `digest-web`/`admin-web` play for their own hosts | | Identity registry backend | **identity** (custom Python, stdlib `http.server` + `sqlite3`) | Person <-> BLE-identifier registry: registration (voice or touchscreen), presence resolution, visit history, per-device rights, a weather proxy. **Published**, unlike admin-canvas — `hosts/kitchen-display/`'s and `hosts/door-panel/`'s kiosk browsers call it directly, bearer-token gated | | Identity static serving | **identity-web** (nginx:alpine) | Serves `identity/frontend/`'s `register.html`/`dashboard.html`/`admin.html` read-only — same role as `pantry-web`/`digest-web`/`admin-web`. The two kiosk pages are what the wall panels load; `admin.html` is deliberately not linked from either (see Phase 6b) | | Door-panel OS build | **live-build** (custom config, `hosts/door-panel/live-build/`) | Reuses the thin client's build tool/convention, structurally `hosts/kitchen-display/`'s twin — see Phase 18 | @@ -627,7 +630,34 @@ service) plus `hosts/kitchen-display/` (a third, simpler kiosk image). whole phase is designed around, the same "propose, never auto-commit" rule this project already applies to identity-merge confirmation (the *Identity store* row in §2). A wrong camera guess costs one tap to fix on the kiosk screen, not a wrong - fact silently written into the household's inventory. + fact silently written into the household's inventory. The book-out direction obeys + it too, and it costs a tap there as well: recognition never consumes anything by + itself, an ambiguous brand is asked about instead of picked, and a throw-away + states its amount first. +2a. **Stock only moves in four ways, so the display has four screens** — *Unload + groceries* (the camera runs in a loop, one confirm per item, straight on to the + next), *Consume article* (hold up what you are eating; multi-unit rows ask how + many, multi-brand kinds ask which), *List expired foods* (cleared by scanning what + you are binning, booked out as **spoiled** so Grocy keeps waste apart from eating), + and *Edit inventory* (`−`/`+` and a freeform amount — the one screen that + deliberately does not use the camera, because it is where you go when the camera + got something wrong). Corrections use Grocy's inventory-correction endpoint, not + consume/add, so the stock journal doesn't fill up with corrections dressed as + meals. +2b. **Two invariants the whole feature rests on.** (i) *Stock is counted in individual + units, never packages* — a twelve-pack of eggs is twelve, because that is the + question people ask and it makes consuming three of them arithmetic instead of + fractions of a pack; the model's `units_per_package` is a multiplier applied once + on the confirm screen, where it is editable and where the result is stated before + anything is written. (ii) *The fold key is the brand-free product kind* — twelve + eggs of brand X plus ten of brand Y are twenty-two eggs, recorded as a Grocy + **product group** so the grouping lives in Grocy's own data model rather than in a + second classification scheme here, with the per-brand rows kept underneath because + "take ten off brand Y" has to stay possible on the edit screen. The folding is + deliberately dumb (casefold, strip the brand suffix); the clever version is a + synonym problem nobody wants adjudicated by a kitchen display at 19:00, and its + failure mode — two lines that should be one — is at least visible, which a wrongly + merged line is not. 3. **A real, LAN-published network listener — the one deliberate exception to this project's usual "the LLM only ever reaches a device through HA→MQTT" shape.** `admin-canvas` (Phase 13) has no published port because only Home Assistant calls @@ -636,18 +666,51 @@ service) plus `hosts/kitchen-display/` (a third, simpler kiosk image). with no HA round-trip in the loop for that specific call. The bearer token — not network placement — is the actual boundary here; see `pantry-vision/README.md`'s "A real network listener, unlike admin-canvas" - section. Controlling *which screen is showing* (Scan/Inventory/Recipes) is a - separate, narrower path that does stay HA/MQTT-mediated: `kitchen-display-agent`, - identical in shape to every other host's agent in this project. + section. Controlling *which screen is showing* is a separate, narrower path that + does stay HA/MQTT-mediated: `kitchen-display-agent`, identical in shape to every + other host's agent in this project. Its "Show scan" button still publishes the + fragment `scan`, which `frontend/app.js` treats as an alias for the unload screen — + an old name living on costs nothing next to an HA button that silently stops + working. 4. **Grocy is the system of record, not a new inventory store** — `pantry-vision` is a client of Grocy's own REST API (already running unconditionally since Phase 1/7) - for both reads (`/inventory`, `/recipes`) and the one write (`/confirm` → Grocy - stock). Nothing here duplicates or replaces Grocy's own data model. + for the reads and for every stock write (`/confirm`, `/consume`, `/adjust`, + `/transfer`). Nothing here duplicates or replaces Grocy's own data model. The one + thing `pantry-vision` keeps in a file of its own is the doorway hints below, and + that is precisely because they are *not* inventory. +4a. **Multiple cold appliances, answered twice over — and only one of the answers is + authoritative.** Grocy **locations** are the record: the confirm screen's placement + resolves to one, `/inventory` reports it, and `POST /transfer` moves an amount + between appliances. That transfer action is the whole point — without it a location + decays into "where it was when it was bought", which is worse than no answer. + Alongside it, **doorway hints** (`pantry-vision/doorway.py`): a Zigbee contact + sensor on each appliance door drives an HA automation that POSTs `/doorway-event`, + which answers 202 immediately and pulls a short camera burst off the request + thread. Whatever the vision model recognises is written to its own SQLite file with + a timestamp and a confidence, **never to stock** — a camera at a door cannot tell + in from out, misses two-items-at-once, and sees nothing behind an arm, so it + produces "last seen at Freezer (loggia), 20 min ago" for a person to evaluate. + `docs/fridge-item-location.md` is the full argument, including why nothing goes + *inside* an appliance (no interior power, condensation on every opening, −18 °C + outside every consumer camera module's rating) and why the door sensors are worth + buying even if no camera ever is. **An appliance configured with a sensor and no + camera is a supported, recommended starting point.** 5. **The vision-identification prompt asks for one photo, not a live video stream**, and degrades to a low-confidence placeholder proposal (never an error page) if the model call fails or its response isn't parseable JSON — same "degrade, don't blank" rule as the digest/admin canvas renderers and `llm_client.py`'s own - `_fallback_document`. + `_fallback_document`. The unload screen's "camera searching for products" is still + built out of single photos: `frontend/app.js` samples a 32×24 greyscale thumbnail + every 700 ms and spends an `/identify` call only when the frame has **settled** and + **changed** since the last identification. Both gates are latency economics — see + open decision #18 — and a degraded answer stops the loop rather than + re-photographing the counter at a model that is down. +5a. **The model is asked to read a printed best-before date off the packaging**, and + that is the one place the "propose, never auto-commit" rule is deliberately looser + than a pure estimate would be. The screen always labels which of the two it is + showing ("read off the packaging — check it" vs "estimated from the category"), and + the server discards any date more than a year past or ten years out, because a + misread label is this feature's most likely failure and that is its shape. 6. **The kitchen display has no native camera-capture app at all** — Chromium's own `getUserMedia()`, called from `pantry-vision/frontend/app.js`, talks to the webcam directly inside the kiosk page and hands a captured frame straight to @@ -655,11 +718,12 @@ service) plus `hosts/kitchen-display/` (a third, simpler kiosk image). auto-accepts the permission prompt that would otherwise sit unanswered on an unattended screen. 7. **`hosts/kitchen-display/` is one workspace, one app — no touch dock, no - multi-app switching** the way `hosts/touch-panel` has: Scan/Inventory/Recipes is - in-page tab navigation inside `pantry-vision/frontend/`'s single-page app, since - there's only one thing this device does. `kitchen-display-agent`'s MQTT surface - is correspondingly narrow: three "Show X" buttons and nothing else — no - media_player, no capture/audio/remote-desktop entities. + multi-app switching** the way `hosts/touch-panel` has: Home/Inventory/Recipes is + in-page tab navigation inside `pantry-vision/frontend/`'s single-page app, with the + four stock-movement screens opening from Home, since there's only one thing this + device does. `kitchen-display-agent`'s MQTT surface is correspondingly narrow: + three "Show X" buttons and nothing else — no media_player, no + capture/audio/remote-desktop entities. 8. **The Grocy API integration is written from documentation, not verified against a live instance** — the exact `GET /api/stock` response shape (whether product names arrive nested by default), the minimum required fields for @@ -758,13 +822,28 @@ entirely container-host services plus additional Frigate camera sources. "Camera face recognition" section) specifically so that Tapo cameras plug into the exact same anti-spoofing-respecting presence pipeline as everything else, rather than this phase inventing its own. -3. **Trash-bin fullness / dishes / litter checks are `chores/`'s job**: every ~2 - hours (systemd timer, `RandomizedDelaySec=1800` — the "+/-30 min in case - something else is running" jitter, systemd's own built-in feature, not custom +3. **Trash-bin fullness / dishes / litter / groceries-left-out checks are `chores/`'s + job**: every ~2 hours (systemd timer, `RandomizedDelaySec=1800` — the "+/-30 min in + case something else is running" jitter, systemd's own built-in feature, not custom code), `chores/check.py` grabs a Frigate snapshot per configured watch point (optionally moving a PTZ camera to a preset first) and asks an Ollama vision model a one-word question. A "needs attention" result opens a chore; a "clear" result auto-closes one. +3a. **`groceries_out_of_place` is `litter`'s twin, with a shorter fuse.** Same + culprit-attribution (whoever the camera last recognised nearby), same + exemption-proofing, same unassignability — because it is the same situation, + somebody carried something in and put it down. The one deliberate difference is + `GROCERIES_NEGLECT_THRESHOLD_HOURS` (30 minutes against the usual four), and it is + the only place in `chores/` where one chore type is treated as more urgent than + another: litter left for an afternoon is untidiness, a tub of ice cream left for an + afternoon is a bin bag. Its prompt names what to **ignore** (fruit bowls, bread + bins, coffee, spices, cookware) as well as what to look for, because the general + question — "is anything out of place?" — is one a vision model answers YES to for a + kitchen that is simply a kitchen, every two hours, until the household mutes the + topic. It knows food is out; it never knows *what* the food is and never writes to + Grocy. That is `pantry-vision`'s job, at a display someone is standing at — two + systems looking at the same counter for two different reasons, which is the + intended shape rather than a duplication to collapse. 4. **Household chore distribution — presence/calendar-driven nudging, not LLM assignment.** The governing principle, stated in `chores/check.py`'s own module docstring: *"I don't care who does it, as long as it gets done."* This is a @@ -901,3 +980,6 @@ These need a decision before their respective implementation steps can be built 37. ~~iOS cannot receive `identity`'s arrival notifications without WAN egress through a third party~~ — **household decision made** (Phase 6b): **no Apple devices are used here**, which removes the only forcing function for exposing ntfy at all. ntfy therefore **stays LAN-only** — no DMZ, no port forward, no NAT-reflection hairpin; `identity → ntfy` is a container-to-container call that never reaches OPNsense, and remote delivery rides a **WireGuard split tunnel** routing only the smart-home VLAN. The full comparison (including why the hairpin variant, which avoids an inter-VLAN rule, was rejected for making an internal path depend on the ISP) is recorded in `docs/network-integration.md` §2.2 so it doesn't get re-litigated. **Still genuinely open**: nothing has been delivered to a real phone yet, and the split tunnel's `AllowedIPs` is the fiddly part — routing all of `192.168.0.0/16` would collide with typical café/hotel LANs and break the phone's connectivity there. Revisit §2.2 only if an Apple device ever joins the household. 38. **Both of `chores`' shipped container URLs pointed at `127.0.0.1` and could never have worked** (found while wiring Phase 6b's arrival notifications) — inside the `chores` container that address is the chores container itself, not the host. `NTFY_URL=http://127.0.0.1:8090` meant every nudge failed to send, and `IDENTITY_URL=http://127.0.0.1:8097` meant `_presence()` never reached identity at all, so **`chores` would have done nothing whatsoever** — and silently, since both failure paths log and continue by design (a deliberate never-block-on-a-dependency choice that here hid a total outage). Corrected in the template to `http://ntfy` and `http://identity:8097` (compose-bridge DNS; ntfy on its internal port 80, not the 8090 published to the LAN). **An already-deployed `/opt/smart-home/chores/chores.env` still has the old values** — templates are copied once at setup and never re-synced, so existing installs need this edited by hand. The broader lesson worth acting on: several services fail soft on an unreachable dependency, which means a misconfigured address produces silence rather than an error — nothing in this repo currently distinguishes "nothing to do" from "never reached the thing that would have told me". 39. **Nothing enforces that voice/TTS consumers actually read `speak_name` rather than `nickname`** (new, Phase 6b) — the field exists, is documented, and `chores/` uses it, but a future HA intent script that reaches for the friendlier-looking `nickname` would break the "the assistant uses real names" rule silently. Worth checking whenever a new consumer of `/presence` or `/resolve` is written. +40. **Whether a vision model can identify a grocery item in a moving hand at doorway distance** (new, Phase 17) — the assumption the entire `/doorway-event` path rests on, and a materially harder task than the kitchen display's held-still-against-a-plain-background one. It is also the cheapest open decision on this list to close: point the kitchen's existing webcam at somebody walking past with a carton and run `/identify` on the frames. Until then the door sensors are the part worth deploying, and `PANTRY_DOOR_APPLIANCES` entries can be configured without a camera — see `docs/fridge-item-location.md`, whose recommended order (sensors, then a month of living with them, *then* one camera) the code deliberately does not shortcut. +41. **Whether `groceries_out_of_place` can tell a carton of milk from a fruit bowl** (new, Phase 20) — its prompt lists what to ignore precisely because the general question gets a YES for any normal kitchen, but that is a mitigation written blind. The failure mode is not a missed chore, it is a false one every two hours, which is how a household learns to ignore the notification channel entirely. Point it at a *clean* counter for a day before trusting a YES, and note this is the first chore type whose false positives cost more than its false negatives. +42. **Contact-sensor battery life on an appliance door is unverified** (new, Phase 17) — a fridge door opens far more often than the window these sensors are sold for, and the sensor sits in a cold, humid draught even when mounted outside the compartment (which `docs/components.md` says to do, for the separate reason that a coin cell at −18 °C is a false economy). Check one after a month before buying more. diff --git a/docs/rooms-and-endpoints.md b/docs/rooms-and-endpoints.md new file mode 100644 index 0000000..7582445 --- /dev/null +++ b/docs/rooms-and-endpoints.md @@ -0,0 +1,78 @@ +# Rooms, and what is in them + +One vocabulary for rooms across the whole project, and one answer to "what hardware is +in this room". Written because three separate features now need it — the floorplan +presence view, the Pebble app, and anything room-scoped like the workshop assistant — +and each of them was about to invent its own. + +## The room id is an HA `area_id`, everywhere + +`living_room`, `kitchen`, `workshop`. Lowercase, digits, underscores; Home Assistant +slugifies area names into exactly that shape. + +That one string is the join key for every part of this project that cares where +something is: + +| Where it appears | As what | +|---|---| +| Home Assistant | the device's **area** | +| `CoreSystemConfig.json` | `room` on each kiosk and audio endpoint | +| Each host's agent config | `_ROOM`, baked in at build time | +| MQTT discovery | `suggested_area` on the device object | +| `identity` | `floorplan_rooms.ha_area_id` — the drawn polygon's tie to reality | +| `identity`'s `/presence` | the `room` field, read off `AREA_ATTRIBUTE` of a trusted entity | + +Nothing translates between these. A room named `Living Room` in one file and +`living_room` in another is two rooms as far as every join above is concerned, which is +why `tools/validate-config.py` rejects anything that is not already an area_id rather +than helpfully slugifying it — helpfully slugifying it is how you end up with two. + +## How a device learns which room it is in + +``` +CoreSystemConfig.json tools/config-export.py the ISO builder + kiosks[].room ────────────> CORE_KIOSK_ROOM ───────────> _ROOM in the + agent's config file + │ + HA files the device in that area <───────────┘ + (suggested_area, on first discovery) +``` + +Declared once, in the same file that already knows every other fact about that device. +Nobody drags devices into areas in the HA UI, and nobody types a room name twice. + +**The one real limitation: `suggested_area` is a suggestion, and it is only honoured +when HA first discovers the device.** Move a panel to another room, rebuild its image, +and HA keeps it in the old area — the suggestion is not reapplied. Moving a device +means moving it in HA too, once. This is a property of HA's discovery, not something +this project can paper over, and it is why the field is named *suggested*. + +An empty `room` is allowed and is a **warning, not an error**: a household that hasn't +settled its room names must still be able to build an image. What it loses is automatic +area assignment, which is a nuisance to fix by hand — not a broken device. + +## What is not covered by this + +- **Zigbee devices, cameras, and anything not built by `tools/`.** Their room lives in + HA only, set when they are paired or added. That is fine — HA is the registry; this + file is about the devices this repo builds images for, which otherwise had no way to + say where they were. +- **Where a *person* is.** That is `identity`'s `/presence`, and it is a different + problem with a different reliability story — see + `docs/pebble-presence-watchface.md`'s closing section. +- **Where an *item* is.** `pantry-vision` locations for food + (`docs/fridge-item-location.md`); nothing tracks tools or parts yet + (`docs/workshop-assistant.md`). + +## Adding a room + +1. Create the area in Home Assistant (or let a device's `suggested_area` create it). +2. Use its area_id as `room` on every kiosk/audio endpoint that lives there. +3. Draw it on the floorplan in `identity`'s admin panel and set its `ha_area_id` to + the same string. The room editor offers a pick-list from + `GET /floorplan/areas` — every area HA is currently reporting — so this step is + choosing from a list rather than retyping an id. + +Step 3 is optional and independent: presence works without a drawn plan, and a drawn +plan is useful before presence is wired up. They only need each other for the views +that show people *on* the plan. diff --git a/docs/workshop-assistant.md b/docs/workshop-assistant.md new file mode 100644 index 0000000..f2a14e1 --- /dev/null +++ b/docs/workshop-assistant.md @@ -0,0 +1,341 @@ +# A workshop/office assistant: what's worth building + +Feasibility note. The ask: identify laptops and computers by camera, identify +components (mainboards especially), pull up disassembly guides, research part specs +automatically, and hold a conversation about planning a project — all scoped per room. + +**The one finding that reorganises the whole thing: stop trying to recognise the +object, and read the label on it.** Everything below follows from that. + +## Why "identify it by shape" is the wrong problem + +Ask a local 7–13B vision model what laptop it is looking at and you will get a +confident, wrong answer, because: + +- **Laptops are visually identical by design.** A closed ThinkPad T480 and a T490 are + the same black rectangle. So are half of Dell's Latitude line. There is no visual + feature to learn, and a model that claims otherwise is pattern-matching on the + wallpaper. +- **Mainboards are worse, not better.** Board layout varies more than a laptop shell + does, but the discriminating detail — where the fourth M.2 slot is, which VRM + configuration — is exactly what a photo at workbench distance and a general-purpose + VLM cannot resolve. +- **Getting it wrong is expensive here**, unlike in the pantry. A wrong yoghurt costs a + tap. A disassembly guide for the wrong laptop revision costs a broken clip or a + ribbon cable, and a wrong pinout costs the board. + +Meanwhile, every one of these objects **carries its own identity in printed text**: + +| Object | Where its identity actually is | +|---|---| +| Laptop | Service tag / serial sticker on the base — and on Dell/Lenovo/HP that tag resolves to the *exact factory configuration* | +| Mainboard | Model silkscreened on the PCB (`PRIME B450M-A`), usually near the RAM slots or the PCIe bracket | +| GPU / PSU / drives | Model and part number on the label | +| ICs | Top-marking, when it hasn't been sanded | + +So the pipeline is **OCR-first**: photo → detect text regions → read them → match +against a parts database. The VLM's job is the small, tractable one it is good at — +"what kind of thing is this, and where on it is the label" — and the identification +comes from characters, not from vibes. This also degrades honestly: an unreadable +label produces "I can't read it, hold it closer" rather than a plausible wrong model. + +**What stays hard, permanently:** an unmarked part. A capacitor, a sanded IC, an +anonymous barrel connector. Nothing here will identify those, and the assistant should +say so rather than guess. + +## The four capabilities, ranked by whether they're worth it + +### 1. Conversational project planning — build this first + +The cheapest and the most useful, and the only one with no perception problem in it. +It is a text LLM with a notebook: what am I building, what have I got, what's the next +step, what did I decide last Tuesday and why. The value is entirely in the *notebook* +being persistent and per-project, not in the model being clever. + +This is also the piece that makes the others worth having: an identified mainboard is +only useful if there is somewhere to put it ("this is for the NAS build"). + +Shape: a small service in the pattern this repo already uses four times over +(`identity`, `chores`, `pantry-vision`) — SQLite, an HTTP API, bearer token, a static +frontend on a display. Projects, notes, parts, photos, decisions. + +### 2. Part-spec research — build it, with one hard rule + +**Every spec must be quoted from a fetched document, with its URL, and never generated +from the model's memory.** This is the same rule `digest-engine`'s political prompt +enforces on claims about the world and `pantry-vision` enforces on stock writes, and +here it is a safety property rather than a quality one: a hallucinated TDP wastes an +afternoon, a hallucinated pinout destroys hardware. + +Practically: fetch the vendor page or datasheet PDF, cache it locally, extract, and +show the extract next to its source. If nothing was fetched, the answer is "I couldn't +find a datasheet", not a paragraph of plausible numbers. + +**This is the first component in this project that deliberately reaches the open +internet for content**, which is worth stating plainly given the local-first doctrine +everything else follows. It is outbound-only, it does not require anything to be +exposed, and it should run against an explicit allowlist of sources (vendor domains, +iFixit, datasheet archives) with everything it fetches cached on disk — so the same +board is looked up once, not once per question. + +### 3. Disassembly guides — mostly a link, and that's fine + +iFixit is the corpus, and it has a public API. The honest scope is: resolve the +identified model to a guide, show the steps and images on the workshop display, cache +what you fetch. Do not rewrite the guide through an LLM — the steps are the thing, and +an LLM paraphrase of "disconnect the battery before removing the board" is strictly +worse than the sentence itself. + +Vendor service manuals (Lenovo's HMM PDFs, Dell's service manuals) cover what iFixit +doesn't, and are the reason the cache is worth having: they are large PDFs you want +locally, once. + +### 4. Camera identification — build it last, and OCR-first + +By the time the three above exist, this is a convenience: it saves typing a model +number. Useful convenience — hands are usually dirty or full — but it is the piece +with the perception risk, and it should not be the first thing attempted. + +The interaction that actually works: hold the label to the camera, get the text back +with a confidence, **confirm before anything acts on it** — the identical +propose-never-auto-commit rule `pantry-vision` runs on. Scanning a barcode or QR code +where one exists (most service tags have one) is strictly better than OCR and should +be tried first. + +## Per-room scoping, which the plumbing now supports + +"Make this a per-room system" splits into two questions, and the boring one is now +answered: + +**Which devices are in which room** — done. Every kiosk and audio endpoint declares a +`room` (an HA area_id) in `CoreSystemConfig.json`; it is baked into the agent and +published as `suggested_area`, so HA files each device in the right area by itself. See +`docs/rooms-and-endpoints.md`. The workshop is a room like any other: give it a display +and it declares `room: workshop`. + +**Which assistant is active in which room** — the design question, and the answer that +fits this project is: *the room selects the toolset, not the personality.* A voice +request in the workshop can reach the parts database and the project notebook; the same +words in the kitchen reach `pantry-vision` and the recipes. That is a routing table +from area_id to capability set, sitting in front of HA Assist, and it is small. + +Two reasons to do it that way rather than one assistant with everything: + +- **Ambiguity collapses.** "What have I got?" means stock in the kitchen and parts in + the workshop, and no amount of prompt engineering makes one agent reliably guess + which room the words came from — but `identity`'s `/speaker` already answers "who + spoke in this area", and the area is right there in the request. +- **The dangerous tools stay where they belong.** Nothing in the kitchen should be able + to open a disassembly guide, and nothing in the workshop needs to book out groceries. + +## The workspace, and why it is an SMB share + +The assistant needs somewhere to put things: fetched datasheets, extracted specs, +generated diagrams, photos of the board, project notes, a scratch directory per +project. Making that a **plain directory on the container host, exported over SMB**, is +right for a reason worth stating: it means every artefact is a file you can open from a +laptop with no API, no export step, and no dependence on this project still existing in +two years. The assistant's output should outlive the assistant. + +``` +/opt/smart-home/workshop/ + projects// + notes/ markdown the assistant and you both write + datasheets/ everything fetched, cached, named by part + diagrams/ generated SVGs (see the next section) + photos/ what the camera captured + scratch/ the assistant's working directory +``` + +**This project already runs Samba** — `gallery-smb`, from `setup-container-host.sh`. +Two things follow, and the second is a real trap: + +- **Reuse the container, add a share.** Samba serves many shares on one port; a second + Samba container would fight the first for **445, which is already claimed** + (`ports.gallery_smb`). That is the same class of collision already recorded as open + decision #31 for Music Assistant, and it is avoidable here by construction: one + `smb` container, a `gallery` share and a `workshop` share, separate accounts. +- **The gallery share is `read only = yes`. This one cannot be**, and that is the + entire security difference between them. A writable share is fine; a writable share + that an LLM writes into deserves three limits, none of which are exotic: + - **Its own volume**, not a subdirectory of anything else. The blast radius of a bad + path is then "the workshop workspace", which is recoverable. + - **Its own account**, not the gallery's. Guest access off, same as the gallery. + - **No execution.** Nothing in the workspace is ever run by anything on the host — + it is a place for documents, and the moment it becomes a place for scripts it is a + different security question than this note answers. + +Keep the assistant's *state* — projects, parts, decisions — in SQLite as with every +other service here, and the *artefacts* on the share. The database is the index; the +share is the filing cabinet. Mixing them (blobs in SQLite, or state in files) gets you +the worst of both: a database you cannot browse and files nothing can query. + +## Technical display widgets, and the one decision that makes them cheap + +The thin-client canvas SDK today has `stat`, `chart`, `image`, `video`, and plain +markdown-ish text. The wanted additions — schematics, board plans, code-flow and +data-structure diagrams, breadboard layouts, 3D — look like six new widgets. They are +mostly one: + +> **Render to SVG on the container host; show it in the canvas.** One new `svg` window +> kind, plus renderers server-side. + +That keeps the SDK's dependency-free, no-build-step property (the thing that makes it +maintainable), puts heavy tooling on the machine that already has heavy tooling, and +means a diagram is a *file in the workspace share* as well as something on a screen. +Client-side rendering libraries would put a megabyte of JavaScript on a kiosk to +produce a picture the server could have produced once and cached. + +| Widget | How | Honest difficulty | +|---|---|---| +| **Circuit diagrams** | `netlistsvg` (or KiCad's own SVG export) from a netlist the assistant produces | Medium — see the notation note below, which is the whole job | +| **Board plans** | KiCad `.kicad_pcb`/Gerber → SVG, server-side, pre-rendered | Medium, and only for boards you have files for. There is no path from a *photo* of a board to a layout drawing | +| **Code-flow diagrams** | Graphviz `dot` → SVG | **Easy.** Graphviz is small, deterministic, and an LLM writes `dot` reliably | +| **Data-structure diagrams** | Graphviz (records/HTML-like labels) → SVG | Easy, same pipeline | +| **Assembly guides** | The guide's own step images in sequence — see the iFixit section | Easy in 2D | +| **Breadboard diagrams** | Fritzing-style, and Fritzing's value *is* its part library, which is not callable | **Hard, and least worth it** — a photo of the actual breadboard is usually better and always more honest | +| **3D models** | glTF + a vendored three.js on the kiosk | The only one that genuinely needs client-side code — see below | + +### EU/IEC notation is a symbol-library decision, not a rendering one + +Rectangular resistors, not zigzags. This matters more than it sounds: **most schematic +tooling defaults to ANSI/US symbols**, and a renderer that cannot be given a symbol set +will quietly produce American schematics forever. + +So the selection criterion for the schematic renderer is *"can I supply my own symbol +library?"* before anything about output quality. `netlistsvg` takes custom skins, which +is the concrete lever — an IEC skin, written once, vendored in the repo like every +other asset here. KiCad can be configured the same way. Anything that hardcodes its +symbols is disqualified regardless of how good its output looks, because the first +zigzag resistor is the last time anyone trusts the diagram. + +The same applies downstream: state the convention in the prompt that generates the +netlist, and check it on the way out. A diagram in the wrong notation is not a style +complaint — it is a diagram that reads wrong to the person holding the soldering iron. + +### The holographic look is a variable override, not a new stylesheet + +The canvas SDK is already a holo aesthetic — glow text, glowing panel edges, a pulse +animation, a dark blue base — and, importantly, it is already parameterised. Every +colour in `glow.css` comes from custom properties on `:root` +(`--admin-accent`, `--admin-glow-color`, `--admin-edge`, `--admin-panel`, `--admin-bg`). + +So "make the workshop look holographic in purple/magenta" is **one small override file +loaded after the SDK**, not a fork of it: + +```css +/* workshop-theme.css — loaded after glow.css. Overrides only. */ +:root { + --admin-bg: #0a0510; /* near-black, violet-biased */ + --admin-panel: rgba(28, 12, 44, 0.82); + --admin-edge: rgba(200, 120, 255, 0.32); + --admin-accent: #c084fc; /* the purple everything glows in */ + --admin-text: #eadcff; + --admin-muted: #a98fc4; + --admin-glow-color: #ff3ec8; /* magenta glow against a purple accent */ +} +``` + +Two magenta-on-purple choices worth making deliberately: the **accent** (borders, +labels, chrome) reads better as the lighter purple, and the **glow** as the magenta — +glow bleeds and saturates, so the more aggressive colour belongs to the effect rather +than to the thing being read. Keep `--admin-good`/`--admin-warn` as they are; green +and amber mean something, and recolouring them into the theme would cost that meaning. + +**Do not tint the technical drawings.** This is the one rule that keeps the look from +becoming a liability: a schematic, a board plan or a pinout table needs contrast and +neutral colour far more than it needs to match the frame — trace colours mean things, +and a magenta glow over a resistor value is how you misread it. So: + +- Holo treatment on the **chrome**: window frames, titles, edges, the pulse, the + background. +- The **drawing surface stays neutral** — a light panel for schematics (they are drawn + for paper and read best that way), or high-contrast line art on near-black. The + frame around it can glow all it likes. +- Anything conveying state by colour — a red LED in a diagram, an error, a + live measurement — keeps its own colour. The theme owns the furniture, not the data. + +The same override trick works if a future room wants its own palette; the SDK never +needed to know about themes for this to be possible, which is why it costs one file. + +### 3D: worth doing, but not first, and not with a live viewer at first + +3D is the one item that cannot be an SVG. Two paths: + +- **Pre-rendered orbit frames** — render N views server-side, ship them as an image + sequence, scrub with a drag. Cheap, works on every surface including the ones with + no GPU, and covers "let me see what this connector looks like from behind", which is + most of the actual need. +- **A real viewer** — vendored three.js plus glTF. It is roughly a megabyte of + JavaScript on a kiosk that currently ships none, which is a real break with the + SDK's "vendored, dependency-free" rule and should be a deliberate decision rather + than a drift into one. + +Start with the frames. Move to a viewer if and only if manipulating the model turns +out to be what people do, rather than glancing at it. + +**3D assembly guides** — exploded views with per-step visibility — are the most +speculative item in this entire note. They need a model *with assembly structure*, +which almost nothing you download has, and authoring one per project is more work than +the repair. Park it. + +## Hardware + +The workshop needs what the kitchen already has: a screen you can touch with one +knuckle, a camera, a microphone. `hosts/kitchen-display/` is exactly this shape +already — a Sway kiosk, a webcam, an agent, one workspace — so a workshop display is +that image with a different frontend, not a new class of device. + +The camera wants to be better than the kitchen's, though, and differently: reading a +service tag or a PCB silkscreen is a **macro** problem. Autofocus at 10–20 cm and +enough resolution to resolve 2 mm text matter far more than field of view. A cheap +USB microscope/inspection camera is a better fit than a webcam, and is worth pricing +before assuming the C920 answer carries over. + +## The compute problem, again + +One RTX 3060, already carrying Assist, four digests a day, `pantry-vision` and +`chores`. Adding OCR is fine (it is small, and CPU OCR is viable). Adding a second +large model that has to be resident is not — and "the workshop assistant felt slow +because someone asked for a digest" is exactly the contention already flagged as open +decision #4. Design for one model swapped predictably, not two resident. + +## Recommended order + +1. **The project notebook + conversational planning, and the workspace share.** No + camera, no internet, no new hardware. Useful on day one, and it is where everything + else lands — including every artefact the later steps produce. +2. **Spec research with mandatory citation**, against an allowlist, cached on disk. +3. **The `svg` window kind + a Graphviz renderer.** One kind and one small tool buys + code-flow and data-structure diagrams immediately, and is the pipeline the schematic + and board-plan renderers then plug into. +4. **Guide lookup**, by typed model number. +5. **Schematics with an IEC symbol skin** — the notation work is the work. +6. **Camera identification**, barcode first and OCR second, proposing rather than + acting. +7. **3D**, as pre-rendered orbit frames. A live viewer only if manipulation turns out + to be what people actually do. + +The purple/magenta holo theme is not in that order because it is not a step: it is one +override file and can land whenever. + +Steps 1–3 need no new hardware at all — a browser on any existing thin client reaches +them. That matters: it means the whole idea can be proved before buying a workshop +display, and if the conversation-and-notebook half turns out to be the only part +anyone uses, that is a complete and useful outcome rather than a failure. + +## What is unverified here + +Everything past step 1. Specifically: whether iFixit's API terms permit caching what +this would cache; whether a local VLM can read a PCB silkscreen at all under workshop +lighting (testable today with any webcam and a spare board — do this before buying an +inspection camera); whether service-tag lookups work without a vendor account; and +whether the OCR-first pipeline holds up on the labels that are scratched, curved, or +under a warranty sticker, which in a workshop is most of them. + +On the display side specifically: whether `netlistsvg`'s skin mechanism really can +express a full IEC symbol set (the notation requirement rests entirely on that, and it +is worth a two-hour spike before committing to the tool); whether an LLM produces +netlists that are *correct* rather than merely well-formed, which is a different and +much harder property; and how big a vendored three.js actually is against the kiosk's +budget before deciding 3D is affordable. diff --git a/hosts/door-panel/agent/door_panel_agent/main.py b/hosts/door-panel/agent/door_panel_agent/main.py index b8a5df0..04f768b 100644 --- a/hosts/door-panel/agent/door_panel_agent/main.py +++ b/hosts/door-panel/agent/door_panel_agent/main.py @@ -71,6 +71,9 @@ def main() -> int: hostname = socket.gethostname() node_id = "".join(c if c.isalnum() else "_" for c in hostname).strip("_") or "doorpanel" friendly_name = config.get("DOOR_PANEL_NAME") or f"Door panel ({hostname})" + # The HA area this device sits in, published as suggested_area — see + # docs/rooms-and-endpoints.md. Blank is fine and means "no suggestion". + room = config.get("DOOR_PANEL_ROOM", "") broker_host = config.get("MQTT_BROKER_HOST", "") broker_port = int(config.get("MQTT_BROKER_PORT") or 1883) @@ -79,7 +82,7 @@ def main() -> int: if config.get("MQTT_USERNAME"): client.username_pw_set(config["MQTT_USERNAME"], config.get("MQTT_PASSWORD") or None) - discovery = Discovery(client, node_id, friendly_name) + discovery = Discovery(client, node_id, friendly_name, room) def on_show(screen: str) -> None: if screen == "register": diff --git a/hosts/door-panel/agent/door_panel_agent/mqtt_discovery.py b/hosts/door-panel/agent/door_panel_agent/mqtt_discovery.py index 2deeb64..4724179 100644 --- a/hosts/door-panel/agent/door_panel_agent/mqtt_discovery.py +++ b/hosts/door-panel/agent/door_panel_agent/mqtt_discovery.py @@ -25,10 +25,11 @@ DISCOVERY_PREFIX = "homeassistant" class Discovery: - def __init__(self, client, node_id: str, friendly_name: str): + def __init__(self, client, node_id: str, friendly_name: str, room: str = ""): self.client = client self.node_id = node_id self.friendly_name = friendly_name + self.room = (room or "").strip() self.base = f"doorpanel/{node_id}" self.availability_topic = f"{self.base}/availability" self._handlers: dict[str, Callable[[str], None]] = {} @@ -40,6 +41,13 @@ class Discovery: "model": "Sway door panel", "sw_version": __version__, } + # Which room this physically sits in, as an HA area_id. `suggested_area` + # is honoured by HA only when the device is FIRST discovered — moving a + # device later means moving it in HA too, this cannot un-file it. Omitted + # entirely when unset, because an empty suggested_area is not the same + # request as no suggestion. See docs/rooms-and-endpoints.md. + if self.room: + self.device["suggested_area"] = self.room def _publish_config(self, component: str, object_id: str, payload: dict) -> None: payload = { diff --git a/hosts/kitchen-display/README.md b/hosts/kitchen-display/README.md index 13fe94d..d5efc2d 100644 --- a/hosts/kitchen-display/README.md +++ b/hosts/kitchen-display/README.md @@ -9,8 +9,9 @@ inventory (soonest-to-expire first) and Grocy's recipes, all touch-driven. that's [`../../pantry-vision/`](../../pantry-vision/README.md), a container-host service this device's kiosk browser calls directly. This host is deliberately thin: a Sway session, one Chromium kiosk window, and `kitchen-display-agent`, which only -ever controls *which screen is showing* (Scan / Inventory / Recipes, plus -Registration — see below) over MQTT — the same "the agent controls the surface, a +ever controls *which screen is showing* (the unload-groceries scanner, Inventory or +Recipes, plus Registration — see below) over MQTT — the same "the agent controls the +surface, a separate write API owns the content" split `hosts/thin-client`'s admin canvas already established. @@ -88,7 +89,11 @@ would otherwise sit unanswered on a screen nobody is there to click "Allow" on. `kitchen-display-agent` publishes four buttons on connect: **Show scan**, **Show inventory**, **Show recipes**, **Show registration** — the first three kill and -relaunch the kiosk Chromium window at `PANTRY_WEB_URL/index.html?...#`; the +relaunch the kiosk Chromium window at `PANTRY_WEB_URL/index.html?...#`. ("Show +scan" still publishes the fragment `scan`, which the frontend treats as an alias for +its unload-groceries screen; the four stock-movement screens are reached by tapping, +not over MQTT, since they are all things a person is standing at the display doing.) +The fourth relaunches it at `IDENTITY_WEB_URL/register.html?...` instead (a different backend, `identity`, behind the same "Show X" shape — see `kitchen_display_agent/main.py`'s `on_show()`). That's the entire MQTT surface; there diff --git a/hosts/kitchen-display/agent/kitchen_display_agent/main.py b/hosts/kitchen-display/agent/kitchen_display_agent/main.py index 1223c7e..b5b524b 100644 --- a/hosts/kitchen-display/agent/kitchen_display_agent/main.py +++ b/hosts/kitchen-display/agent/kitchen_display_agent/main.py @@ -72,6 +72,9 @@ def main() -> int: hostname = socket.gethostname() node_id = "".join(c if c.isalnum() else "_" for c in hostname).strip("_") or "kitchendisplay" friendly_name = config.get("KITCHEN_DISPLAY_NAME") or f"Kitchen display ({hostname})" + # The HA area this device sits in, published as suggested_area — see + # docs/rooms-and-endpoints.md. Blank is fine and means "no suggestion". + room = config.get("KITCHEN_DISPLAY_ROOM", "") broker_host = config.get("MQTT_BROKER_HOST", "") broker_port = int(config.get("MQTT_BROKER_PORT") or 1883) @@ -80,7 +83,7 @@ def main() -> int: if config.get("MQTT_USERNAME"): client.username_pw_set(config["MQTT_USERNAME"], config.get("MQTT_PASSWORD") or None) - discovery = Discovery(client, node_id, friendly_name) + discovery = Discovery(client, node_id, friendly_name, room) def on_show(fragment: str) -> None: # "register" is a different backend (identity's register.html) behind the diff --git a/hosts/kitchen-display/agent/kitchen_display_agent/mqtt_discovery.py b/hosts/kitchen-display/agent/kitchen_display_agent/mqtt_discovery.py index ec5e48d..ab0e265 100644 --- a/hosts/kitchen-display/agent/kitchen_display_agent/mqtt_discovery.py +++ b/hosts/kitchen-display/agent/kitchen_display_agent/mqtt_discovery.py @@ -28,10 +28,11 @@ DISCOVERY_PREFIX = "homeassistant" class Discovery: - def __init__(self, client, node_id: str, friendly_name: str): + def __init__(self, client, node_id: str, friendly_name: str, room: str = ""): self.client = client self.node_id = node_id self.friendly_name = friendly_name + self.room = (room or "").strip() self.base = f"kitchendisplay/{node_id}" self.availability_topic = f"{self.base}/availability" self._handlers: dict[str, Callable[[str], None]] = {} @@ -43,6 +44,13 @@ class Discovery: "model": "Sway kitchen display", "sw_version": __version__, } + # Which room this physically sits in, as an HA area_id. `suggested_area` + # is honoured by HA only when the device is FIRST discovered — moving a + # device later means moving it in HA too, this cannot un-file it. Omitted + # entirely when unset, because an empty suggested_area is not the same + # request as no suggestion. See docs/rooms-and-endpoints.md. + if self.room: + self.device["suggested_area"] = self.room def _publish_config(self, component: str, object_id: str, payload: dict) -> None: payload = { diff --git a/hosts/thin-client/README.md b/hosts/thin-client/README.md index 60b859c..90c019c 100644 --- a/hosts/thin-client/README.md +++ b/hosts/thin-client/README.md @@ -294,6 +294,44 @@ vendor/product IDs) and passes it to mpv as `--external-file=alsa://hw:X,0` if found. Video-only playback (not a crash) if nothing matches. **Unverified against real hardware** — see the checklist below. +## A wireless USB remote works out of the box + +The kind with TV buttons on the front and a small keyboard on the back. To Linux that +dongle is two ordinary HID keyboards — a normal one and a "consumer control" one — so +there is nothing to configure per device: the front buttons arrive as `XF86*` keysyms +and the back keyboard arrives as keys. `configs/sway/config`'s remote-control section +binds the full standardised set: + +| Buttons | What they do | +|---|---| +| ▶ ⏸ ⏹ ⏭ ⏮ | `playerctl -p mpv,spotifyd` — play/pause, pause, stop, next, previous | +| ⏪ ⏩ | seek 10s back / 30s forward. Skip, not scan: remotes get *pressed*, not held, and the podcast convention is already in people's fingers | +| Volume, mute, mic mute | the **sink**, not the player — the rocker should move the room's volume whatever is making the noise | +| Channel ± | next/previous workspace. A media station's "channels" are its workspaces, which is the closest honest analogy | +| Home / Back | the media workspace / `back_and_forth` | +| Power, Sleep | **the display, not the machine** — see below | + +**Two deliberate choices worth knowing before you remap anything.** + +*Plain arrows and Return are not bound, on purpose.* A remote's D-pad and OK send +exactly those, unmodified, and Chromium, mpv and every kiosk page need them — binding +them at the compositor would break scrolling a web page with the remote, which is most +of what the remote is for. Window focus stays on `$mod`+arrows. + +*The power button turns the screen off, not the computer.* On a TV that button turns +the picture off; on a thin client that autologins into a kiosk, `poweroff` takes the +room's screen away until somebody walks over to press a physical button. So it runs +`display-toggle`, which is a **toggle** rather than two bindings because with the +outputs dark there is no other way back — Sway is still running and still receiving +keys, so the next press wakes it. A stuck button therefore cannot leave the screen +dark either. + +**Remotes vary more than their marketing does.** Run `wev` (or +`sudo libinput debug-events`) from the maintenance shell (`$mod+Shift+Ctrl+m`), press +every button, and add any that comes back with an unbound keysym. A button that reports +*no keysym at all* is one the kernel has no mapping for — that is a udev hwdb entry, +not a Sway binding, and is worth knowing before blaming the config. + ## Home Assistant entities `thinclient-agent` publishes MQTT-discovery configs on connect. Under the MQTT @@ -452,6 +490,14 @@ default on most sets, and the reason a TV that sleeps fine refuses to wake. ## Manual verification still outstanding +0. **The remote.** No remote has been plugged into anything — the keysym list above is + the standardised set, not one read off a specific device. Expect one or two buttons + on any given remote to report something unbound (or nothing at all); `wev` from the + maintenance shell is the two-minute check, and `display-toggle`'s grep for + `'"dpms": true'` in `swaymsg -t get_outputs` is worth confirming against the Sway + version this image actually ships, since that field's spelling is the one thing + that would make the power button silently do nothing. + None of this has been run on hardware. In rough order: 1. The ISO builds at all (`lb build` is network-heavy and can fail on mirror hiccups). diff --git a/hosts/thin-client/agent/thinclient_agent/main.py b/hosts/thin-client/agent/thinclient_agent/main.py index 6d4087d..ef54015 100644 --- a/hosts/thin-client/agent/thinclient_agent/main.py +++ b/hosts/thin-client/agent/thinclient_agent/main.py @@ -152,6 +152,9 @@ def main() -> int: hostname = socket.gethostname() node_id = "".join(c if c.isalnum() else "_" for c in hostname).strip("_") or "thinclient" friendly_name = config.get("THINCLIENT_NAME") or f"Thin client ({hostname})" + # The HA area this device sits in, published as suggested_area — see + # docs/rooms-and-endpoints.md. Blank is fine and means "no suggestion". + room = config.get("THINCLIENT_ROOM", "") broker_host = config.get("MQTT_BROKER_HOST", "") broker_port = int(config.get("MQTT_BROKER_PORT") or 1883) @@ -183,7 +186,7 @@ def main() -> int: if config.get("MQTT_USERNAME"): client.username_pw_set(config["MQTT_USERNAME"], config.get("MQTT_PASSWORD") or None) - discovery = Discovery(client, node_id, friendly_name) + discovery = Discovery(client, node_id, friendly_name, room) mpris = MprisBridge(discovery.publish_media_state, sway.session_env) def on_detail_level(payload: str) -> None: diff --git a/hosts/thin-client/agent/thinclient_agent/mqtt_discovery.py b/hosts/thin-client/agent/thinclient_agent/mqtt_discovery.py index 30fa288..ba2cc50 100644 --- a/hosts/thin-client/agent/thinclient_agent/mqtt_discovery.py +++ b/hosts/thin-client/agent/thinclient_agent/mqtt_discovery.py @@ -29,10 +29,11 @@ DISCOVERY_PREFIX = "homeassistant" class Discovery: - def __init__(self, client, node_id: str, friendly_name: str): + def __init__(self, client, node_id: str, friendly_name: str, room: str = ""): self.client = client self.node_id = node_id self.friendly_name = friendly_name + self.room = (room or "").strip() self.base = f"thinclient/{node_id}" self.availability_topic = f"{self.base}/availability" self.media_state_topic = f"{self.base}/media/state" @@ -51,6 +52,13 @@ class Discovery: "model": "Sway thin client", "sw_version": __version__, } + # Which room this physically sits in, as an HA area_id. `suggested_area` + # is honoured by HA only when the device is FIRST discovered — moving a + # device later means moving it in HA too, this cannot un-file it. Omitted + # entirely when unset, because an empty suggested_area is not the same + # request as no suggestion. See docs/rooms-and-endpoints.md. + if self.room: + self.device["suggested_area"] = self.room # --- plumbing ----------------------------------------------------------- def _publish_config(self, component: str, object_id: str, payload: dict) -> None: diff --git a/hosts/thin-client/configs/sway/config b/hosts/thin-client/configs/sway/config index 42b0f85..83e5e19 100644 --- a/hosts/thin-client/configs/sway/config +++ b/hosts/thin-client/configs/sway/config @@ -133,12 +133,64 @@ bindsym $mod+Right focus right bindsym $mod+Up focus up bindsym $mod+Down focus down +# --------------------------------------------------------------------------- +# Remote control — the full standardised media-key set. +# +# THE DEVICE THIS IS FOR: a wireless USB remote with TV controls on the front and a +# small keyboard on the back. To Linux that is just two HID keyboards on one dongle +# (a normal keyboard plus a "consumer control" device), so there is nothing to +# configure per-device — the front buttons arrive as the XF86* keysyms below and the +# back keyboard arrives as ordinary keys. Any remote in that class works; the bindings +# are what make it useful. +# +# WHAT IS DELIBERATELY *NOT* BOUND: plain Up/Down/Left/Right and Return. A remote's +# D-pad and OK button send exactly those, unmodified — and Chromium, mpv and every +# kiosk page need them. Stealing them at the compositor would break scrolling a web +# page with the remote, which is most of what the remote is for. Window focus stays on +# $mod+arrows above; the bare arrows belong to whatever is on screen. +# +# CHECKING WHAT YOUR REMOTE ACTUALLY SENDS: remotes vary more than their marketing +# does. `wev` (or `sudo libinput debug-events`) prints the keysym for each button — +# run it once from the maintenance shell and add any button that comes back unbound. +# A button that reports no keysym at all is one the kernel has no mapping for, which +# is a udev hwdb entry, not a Sway binding. +# --------------------------------------------------------------------------- + +# Transport. `playerctl -p mpv,spotifyd` matches the two players this image runs; +# the order is the priority when both are alive. bindsym XF86AudioPlay exec playerctl -p mpv,spotifyd play-pause +bindsym XF86AudioPause exec playerctl -p mpv,spotifyd pause +bindsym XF86AudioStop exec playerctl -p mpv,spotifyd stop bindsym XF86AudioNext exec playerctl -p mpv,spotifyd next bindsym XF86AudioPrev exec playerctl -p mpv,spotifyd previous +# Skip rather than scan: a remote's ⏪/⏩ are pressed repeatedly, not held, and +# playerctl has no scan mode. 30s forward / 10s back is the podcast convention and is +# the one people already have in their fingers. +bindsym XF86AudioForward exec playerctl -p mpv,spotifyd position 30+ +bindsym XF86AudioRewind exec playerctl -p mpv,spotifyd position 10- + +# Volume. Sinks, not players — the remote's volume rocker should move the room's +# volume regardless of what is making the noise. bindsym XF86AudioRaiseVolume exec wpctl set-volume @DEFAULT_AUDIO_SINK@ 5%+ bindsym XF86AudioLowerVolume exec wpctl set-volume @DEFAULT_AUDIO_SINK@ 5%- bindsym XF86AudioMute exec wpctl set-mute @DEFAULT_AUDIO_SINK@ toggle +bindsym XF86AudioMicMute exec wpctl set-mute @DEFAULT_AUDIO_SOURCE@ toggle + +# Navigation. A media station's "channels" are its workspaces, which is the closest +# honest analogy and means the channel rocker does something useful instead of nothing. +bindsym XF86ChannelUp workspace next +bindsym XF86ChannelDown workspace prev +bindsym XF86HomePage workspace $ws_media +bindsym XF86Back workspace back_and_forth +bindsym XF86Forward workspace back_and_forth + +# Power. **The display, not the machine.** A remote's power button on a TV turns the +# picture off, and that is what people expect it to do here — while `poweroff` on a +# thin client that autologins is a button that takes the room's screen away until +# somebody walks over to it. See the display-toggle script for the wake side. +bindsym XF86PowerOff exec display-toggle +bindsym XF86Sleep exec display-toggle +bindsym XF86ScreenSaver exec display-toggle # Deliberately no exit binding: `swaymsg exit` would drop to a black VT, and greetd # would just autologin straight back in. Use SSH or the local terminal to administer. diff --git a/hosts/thin-client/configs/sway/display-toggle b/hosts/thin-client/configs/sway/display-toggle new file mode 100644 index 0000000..656439c --- /dev/null +++ b/hosts/thin-client/configs/sway/display-toggle @@ -0,0 +1,26 @@ +#!/bin/sh +# Turn every output off, or back on. Installed to /usr/local/bin/display-toggle and +# bound to the remote's power/sleep buttons in the Sway config. +# +# WHY THE POWER BUTTON DOES NOT POWER ANYTHING OFF: on a TV, that button turns the +# picture off. On a thin client that autologins into a kiosk, `poweroff` takes the +# room's screen away until somebody walks over and presses a physical button — which +# is a worse outcome than any it prevents. So the remote's power button does the thing +# the person pressing it actually meant. +# +# THE WAKE SIDE IS THE HARD HALF. With outputs powered off, Sway is still running and +# still receiving keys, so pressing power again lands here and turns them back on. +# That is why this is a toggle rather than two bindings: there is no other way back. +# It also means a *stuck* remote button cannot leave the screen dark — the next press +# fixes it. +set -eu + +# `swaymsg -t get_outputs` reports each output's dpms state. If ANY output is still on, +# the intent of a press is "turn it off"; only when everything is already dark does a +# press mean "wake up". That ordering matters on a multi-output machine, where asking +# per-output would leave the remote toggling one screen at a time. +if swaymsg -t get_outputs | grep -q '"dpms": true'; then + swaymsg 'output * dpms off' +else + swaymsg 'output * dpms on' +fi diff --git a/hosts/thin-client/live-build/config/package-lists/thin-client.list.chroot b/hosts/thin-client/live-build/config/package-lists/thin-client.list.chroot index c3056f7..3fdcf45 100644 --- a/hosts/thin-client/live-build/config/package-lists/thin-client.list.chroot +++ b/hosts/thin-client/live-build/config/package-lists/thin-client.list.chroot @@ -22,6 +22,9 @@ mpv mpv-mpris libmpv2 playerctl +# wev prints the keysym for each key press — the two-minute way to find out what a +# given wireless remote's buttons actually send. See README's remote-control section. +wev # --- Audio --- pipewire diff --git a/hosts/touch-panel/agent/touchpanel_agent/main.py b/hosts/touch-panel/agent/touchpanel_agent/main.py index 55456e9..85f6c29 100644 --- a/hosts/touch-panel/agent/touchpanel_agent/main.py +++ b/hosts/touch-panel/agent/touchpanel_agent/main.py @@ -117,6 +117,9 @@ def main() -> int: hostname = socket.gethostname() node_id = "".join(c if c.isalnum() else "_" for c in hostname).strip("_") or "touchpanel" friendly_name = config.get("TOUCHPANEL_NAME") or f"Touch panel ({hostname})" + # The HA area this device sits in, published as suggested_area — see + # docs/rooms-and-endpoints.md. Blank is fine and means "no suggestion". + room = config.get("TOUCHPANEL_ROOM", "") broker_host = config.get("MQTT_BROKER_HOST", "") broker_port = int(config.get("MQTT_BROKER_PORT") or 1883) @@ -128,7 +131,7 @@ def main() -> int: if config.get("MQTT_USERNAME"): client.username_pw_set(config["MQTT_USERNAME"], config.get("MQTT_PASSWORD") or None) - discovery = Discovery(client, node_id, friendly_name) + discovery = Discovery(client, node_id, friendly_name, room) mpris = MprisBridge(discovery.publish_media_state, sway.session_env) def on_launch(key: str) -> None: diff --git a/hosts/touch-panel/agent/touchpanel_agent/mqtt_discovery.py b/hosts/touch-panel/agent/touchpanel_agent/mqtt_discovery.py index 0bfac9c..af19d3d 100644 --- a/hosts/touch-panel/agent/touchpanel_agent/mqtt_discovery.py +++ b/hosts/touch-panel/agent/touchpanel_agent/mqtt_discovery.py @@ -30,10 +30,11 @@ DISCOVERY_PREFIX = "homeassistant" class Discovery: - def __init__(self, client, node_id: str, friendly_name: str): + def __init__(self, client, node_id: str, friendly_name: str, room: str = ""): self.client = client self.node_id = node_id self.friendly_name = friendly_name + self.room = (room or "").strip() self.base = f"touchpanel/{node_id}" self.availability_topic = f"{self.base}/availability" self.media_state_topic = f"{self.base}/media/state" @@ -46,6 +47,13 @@ class Discovery: "model": "Sway touch panel", "sw_version": __version__, } + # Which room this physically sits in, as an HA area_id. `suggested_area` + # is honoured by HA only when the device is FIRST discovered — moving a + # device later means moving it in HA too, this cannot un-file it. Omitted + # entirely when unset, because an empty suggested_area is not the same + # request as no suggestion. See docs/rooms-and-endpoints.md. + if self.room: + self.device["suggested_area"] = self.room # --- plumbing ----------------------------------------------------------- def _publish_config(self, component: str, object_id: str, payload: dict) -> None: diff --git a/identity/README.md b/identity/README.md index 5ec8286..d361a48 100644 --- a/identity/README.md +++ b/identity/README.md @@ -72,6 +72,57 @@ actually decides who's registering. **If you want camera-based identity later, Frigate's own face-recognition + enrollment (0.16+) is the piece to wire in, not a new pipeline here.** +## How a person looks on a screen too small for their name + +Every person carries two derived fields, served on `/people`, `/presence` and each +occupant in `/floorplan/presence`: + +- **`initial`** — the first letter of the **real name**, never the nickname. Same rule + as `speak_name`, same reason: what a machine shows for somebody should follow who + they are, not what the household happens to call them this year. +- **`color`** — one of eight, assigned automatically at registration and editable in + the admin panel. + +**The colour exists because initials collide.** A household with an Anna and an Amir +gets two identical "A"s on a wall panel or a watch face, and the colour is what makes +that readable. So the assignment rule is: first avoid any colour already worn by +somebody with the same initial — that is worth spending the whole palette on — then +take the least-used colour overall. Past eight people sharing one letter it repeats, +and you pick by hand. + +**Why those eight values.** Every channel is `00`/`55`/`AA`/`FF`, which is exactly the +2-bits-per-channel space a colour Pebble renders natively. Anything else is snapped or +dithered by the watch, and a colour that shifts between the admin panel and the watch +defeats the point of having one. They also vary in lightness rather than only in hue, +so they stay distinguishable when a black-and-white screen reduces them to greys, and +they survive the common colour-vision deficiencies better than a rainbow would. A hex +outside the palette is accepted if you set one deliberately; it just won't be exact on +a watch. See `docs/pebble-presence-watchface.md`. + +Existing people are backfilled with a colour at startup, oldest first, so the +assignment is stable across restarts and never reshuffles a colour somebody has +already learned. + +## The photo, and where it comes from + +Two ways a person gets a profile picture: + +- **Registration** — the door panel's capture becomes the picture, "most recent + registration wins". Good for the picture registration took; useless for a device-less + household member registered by hand, who could never have one at all. +- **`POST /people//photo`** — upload one from the admin panel. Written into the + same photo directory with the same filename shape, so anything that serves or backs + up a registration capture handles this identically. Deliberately **not** recorded as + a registration event: nobody registered. + +Removing a picture (`clear_photo`) blanks the reference but leaves the file on disk — +it may also be a `registration_events` audit artifact, and "stop showing this photo" is +a different request from "destroy the record that it was taken". The same reasoning +applies to replacing one. + +A person with no picture is not a gap in a UI: their initial on their colour is the +fallback everywhere, and on the small screens it is the *preferred* rendering anyway. + ## People without a device Two paths, distinct on purpose because they solve different problems: @@ -559,9 +610,11 @@ not network placement. | `POST /register` | `{"name", "device_id", "photo_id"?, "entity_id"?, "no_device"?}` -> registers, or returns a reason it couldn't (see above) | | `POST /register/guest` | `{"device_id", "photo_id"?}` -> registers "Guest N", no name needed | | `GET /people` | admin/audit list of every person: identifiers, device grants, chore assignments, `nickname`/`speak_name`, `last_visit_at`, `visit_count`, `currently_home_since` | -| `POST /people/` | edit any editable field — `{"name"?, "nickname"?, "note"?, "chore_exempt"?, "chore_reminder_style"?, "notify_on_arrival"?, "announce_arrivals"?, "notify_topic"?, "digest_sections"?, "clear_photo"?}`. Omitted keys are left alone | +| `POST /people/` | edit any editable field — `{"name"?, "nickname"?, "note"?, "color"?, "chore_exempt"?, "chore_reminder_style"?, "notify_on_arrival"?, "announce_arrivals"?, "notify_topic"?, "digest_sections"?, "clear_photo"?}`. Omitted keys are left alone; an empty `color` re-derives one rather than blanking it | | `POST /people//test-notification` | push a test message to this person's ntfy topic, to prove it works | -| `GET /people//photo` | the person's profile picture (raw JPEG) — their most recent registration photo | +| `GET /people//photo` | the person's profile picture (raw JPEG) | +| `POST /people//photo` | raw image bytes -> set the profile picture directly, without a walk to the door panel | +| `GET /person-colors` | the eight-colour palette the admin panel offers (see below) | | `POST /people//identifiers` | `{"entity_id"}` — attach an identifier by hand (a fixed BLE tag not in range yet). Still enforces `TRUSTED_ENTITY_PREFIXES` | | `DELETE /people//identifiers/` | revoke a mistaken or compromised identifier | | `DELETE /people/` | remove a person entirely (their identifiers, grants and visits go with them) | @@ -589,7 +642,7 @@ not network placement. | `GET /floorplan/areas` | area values HA is currently reporting — the editor's pick list | | `POST /presence/manual` | `{"person_id", "home"}` — hand-operated Home/Away for anyone with no identifiers | | `POST /people//chore-settings` | `{"chore_exempt"?, "chore_reminder_style"?}` — see below; either field omitted/`null` leaves it unchanged | -| `GET /presence` | `{"people": [{"id", "name", "nickname", "speak_name", "home", "room", "has_device", "has_photo", "chore_exempt", "chore_reminder_style", "chore_assignments"}], "generated_at"}` — `home` is `true`/`false`/`null` (unknown), `room` is best-effort floor-plan groundwork (see below) | +| `GET /presence` | `{"people": [{"id", "name", "nickname", "speak_name", "home", "room", "has_device", "has_photo", "color", "initial", "chore_exempt", "chore_reminder_style", "chore_assignments"}], "generated_at"}` — `home` is `true`/`false`/`null` (unknown), `room` is best-effort floor-plan groundwork (see below) | | `GET /weather` | proxies `smarthome/weather/current`, same JSON shape (`temperature`/`condition`/`location`) `hosts/thin-client`'s weather overlay already uses | **Every person gets a profile picture, automatically** — whichever registration photo diff --git a/identity/frontend/admin.html b/identity/frontend/admin.html index 40adc24..78c78d5 100644 --- a/identity/frontend/admin.html +++ b/identity/frontend/admin.html @@ -203,6 +203,28 @@ +
+ How they look on small screens +

+ A wall panel or a watch face has no room for a name — it shows this colour and + the first letter of the real name. Two people whose names start the same letter + get different colours automatically; change one here if you'd rather pick. +

+
Colour +
+
+
Profile picture +
+ + + +
+ + Otherwise this is whatever the door panel captured at their last registration. + +
+
+
Chores