dump a malformed digest gets. Still never a blank page.
+ if (!visible.length) {
+ global.DigestWindow.open({
+ title: 'Nothing in this digest',
+ content:
+ 'No digest section is switched on for this screen. Sections are chosen ' +
+ 'per person in the identity admin panel.',
+ container: container
+ });
+ return;
+ }
+
+ visible.forEach(function (doc) {
try {
renderSection(container, doc);
} catch (err) {
diff --git a/digest-engine/render/digest-canvas-sdk/window-chrome.js b/digest-engine/render/digest-canvas-sdk/window-chrome.js
index 1e02e05..e5268ef 100644
--- a/digest-engine/render/digest-canvas-sdk/window-chrome.js
+++ b/digest-engine/render/digest-canvas-sdk/window-chrome.js
@@ -71,7 +71,83 @@
return body;
}
+ // Only http(s) becomes a clickable link. The content of every window here is
+ // model output, and `javascript:` in an href would be script execution handed to
+ // whatever the LLM emitted — the one place this renderer's degrade-don't-throw
+ // habit is not enough. Anything else is shown as plain text instead.
+ function safeHref(url) {
+ var text = String(url || '').trim();
+ return /^https?:\/\//i.test(text) ? text : null;
+ }
+
+ // The fold-out sources block. Collapsed by default because the digest is read
+ // across a room: the claim is what you see, the receipts are one tap away, and
+ // an open citation list would push the next window off the screen.
+ function renderSources(sources, label) {
+ var usable = (sources || []).filter(function (source) {
+ return source && (source.title || source.outlet || source.url || source.quote);
+ });
+ if (!usable.length) { return null; }
+
+ var details = document.createElement('details');
+ details.className = 'digest-sources';
+
+ var summary = document.createElement('summary');
+ summary.textContent = (label || 'Sources') + ' (' + usable.length + ')';
+ details.appendChild(summary);
+
+ var list = document.createElement('ul');
+ list.className = 'digest-source-list';
+
+ usable.forEach(function (source) {
+ var li = document.createElement('li');
+
+ var head = document.createElement('div');
+ head.className = 'digest-source-head';
+ var href = safeHref(source.url);
+ var titleText = source.title || source.url || source.outlet;
+ if (href) {
+ var link = document.createElement('a');
+ link.href = href;
+ link.textContent = titleText;
+ link.rel = 'noopener noreferrer';
+ head.appendChild(link);
+ } else {
+ head.appendChild(document.createTextNode(titleText));
+ }
+ li.appendChild(head);
+
+ // Outlet and ownership sit with the citation rather than in the prose: who
+ // published a claim is part of reading it, and it belongs next to the claim.
+ var attribution = [source.outlet, source.owner, source.bias]
+ .filter(function (part) { return part; })
+ .join(' · ');
+ if (attribution) {
+ var meta = document.createElement('div');
+ meta.className = 'digest-source-meta';
+ meta.textContent = attribution;
+ li.appendChild(meta);
+ }
+
+ if (source.quote) {
+ var quote = document.createElement('blockquote');
+ quote.className = 'digest-source-quote';
+ quote.textContent = source.quote;
+ li.appendChild(quote);
+ }
+
+ list.appendChild(li);
+ });
+
+ details.appendChild(list);
+ return details;
+ }
+
var DigestWindow = {
+ // Exposed so render.js can attach the same block under a globe marker's brief,
+ // where there is no window of its own to hang it on.
+ sources: renderSources,
+
open: function (options) {
options = options || {};
@@ -98,7 +174,11 @@
bar.appendChild(title);
el.appendChild(bar);
- el.appendChild(renderContent(options.content));
+
+ var body = renderContent(options.content);
+ var sources = renderSources(options.sources);
+ if (sources) { body.appendChild(sources); }
+ el.appendChild(body);
if (typeof options.x === 'number' && typeof options.y === 'number') {
el.classList.add('digest-window-positioned');
diff --git a/digest-engine/render/templates/compact.html b/digest-engine/render/templates/compact.html
index e2f0b1c..a014fca 100644
--- a/digest-engine/render/templates/compact.html
+++ b/digest-engine/render/templates/compact.html
@@ -54,8 +54,15 @@
var canvas = document.getElementById('canvas');
+ // Optional ?person= — put it in the Lovelace card's URL to get
+ // that person's own digest sections (chosen in identity's admin panel). Unlike
+ // full.html this defaults to showing everything that was generated, personal
+ // section included: this card is embedded in somebody's own HA dashboard, which is
+ // already a per-account surface, not a kiosk in a hallway that anyone walks past.
+ var person = new URLSearchParams(location.search).get('person') || '';
+
function load() {
- DigestRender.load(canvas, DIGEST_URL, { detailLevel: 'compact' });
+ DigestRender.load(canvas, DIGEST_URL, { detailLevel: 'compact', person: person });
}
load();
diff --git a/digest-engine/render/templates/full.html b/digest-engine/render/templates/full.html
index c81c6c0..ec87f0f 100644
--- a/digest-engine/render/templates/full.html
+++ b/digest-engine/render/templates/full.html
@@ -61,8 +61,20 @@
var canvas = document.getElementById('canvas');
+ // ?person=, set by thinclient_agent/digest_canvas.py from a
+ // request Home Assistant had ALREADY resolved — this page never works out who is
+ // standing in front of it. With it, only that person's own digest sections are
+ // drawn (they pick them in identity's admin panel); without it, the personal
+ // section is left out rather than shown to whoever happens to walk past, which is
+ // the plan's Phase 11.8 rule: never guess whose personal section this is.
+ var person = new URLSearchParams(location.search).get('person') || '';
+
function load() {
- DigestRender.load(canvas, DIGEST_URL, { detailLevel: 'full' });
+ DigestRender.load(canvas, DIGEST_URL, {
+ detailLevel: 'full',
+ person: person,
+ requirePersonForPersonal: true
+ });
}
load();
diff --git a/digest-engine/requirements.txt b/digest-engine/requirements.txt
index a71b911..c6f0201 100644
--- a/digest-engine/requirements.txt
+++ b/digest-engine/requirements.txt
@@ -7,3 +7,4 @@ websocket-client>=1.7
caldav>=2.0
icalendar>=5.0
paho-mqtt>=1.6
+pypdf>=4.0
diff --git a/digest-engine/run.py b/digest-engine/run.py
index 2afd2e6..c2a2a28 100644
--- a/digest-engine/run.py
+++ b/digest-engine/run.py
@@ -10,10 +10,19 @@ Every ingestion source is independently toggleable and independently fallible. A
source that is disabled, misconfigured, or simply broken (a stale Telegram
session, a signal-cli container that is down) logs a warning and contributes an
empty list — it must never take down the rest of the run, because a digest with
-two of three sections is worth far more than no digest at all.
+most of its sections is worth far more than no digest at all.
Everything here is read-only. See docs/project-plan.md Phase 12 step 8.
+WHICH DIGESTS GET GENERATED
+---------------------------
+Not necessarily all four. Each person in `identity` has a per-person set of the
+sections they want (network, household, personal/social, political/news), and
+this run generates the union of what the household asked for — a section nobody
+wants costs no LLM call and never reaches output/. See preferences.py, including
+why every failure in that lookup means "generate everything" rather than
+"generate nothing".
+
Before anything is written to output/, every generated document passes through
synth/counter_run.py — a second LLM call that checks the document against the
same context it was generated from and drops anything that doesn't trace back
@@ -55,11 +64,16 @@ from ingest import (
naval_traffic,
news_rss,
opnsense_ids,
+ rci_social,
signal_ingest,
telegram_ingest,
whatsapp_ingest,
)
from synth import counter_run, llm_client
+import agenda
+import archive
+import notify
+import preferences
import viewed_tracker
LOG = logging.getLogger("digest")
@@ -72,6 +86,7 @@ SOURCES = (
("ENABLE_DISCORD_INGEST", "discord", discord_ingest),
("ENABLE_WHATSAPP_INGEST", "whatsapp", whatsapp_ingest),
("ENABLE_NEWS_INGEST", "news", news_rss),
+ ("ENABLE_RCI_SOCIAL_INGEST", "rci_social", rci_social),
("ENABLE_FINANCIAL_INGEST", "financial", financial),
("ENABLE_FLIGHT_TRAFFIC_INGEST", "flight_traffic", flight_traffic),
("ENABLE_NAVAL_TRAFFIC_INGEST", "naval_traffic", naval_traffic),
@@ -80,8 +95,22 @@ SOURCES = (
("ENABLE_GROCY_INGEST", "grocy", grocy),
)
-PERSONAL_SOURCES = ("email", "signal", "telegram", "discord", "whatsapp")
-POLITICAL_SOURCES = ("news", "financial", "email", "flight_traffic", "naval_traffic")
+# Which ingested sources feed which section. Two jobs: it is what the personal
+# context's `messages` map is built from, and it is how collect() knows that a source
+# whose only consumers are switched off for this run needs no fetching at all — a
+# digest section nobody asked for should cost neither an LLM call nor the WAN round
+# trips its sources would have made. `email` appears twice on purpose: mail is both
+# personal correspondence and, occasionally, politically relevant evidence.
+SECTION_SOURCES = {
+ "personal": ("email", "signal", "telegram", "discord", "whatsapp"),
+ # `calendar` is here for the agendas, not for the diary: the political section
+ # owns the branch agenda's contents and the to-do list that comes out of it (see
+ # agenda.py), and it cannot match an agenda to a meeting it has never fetched.
+ "political": ("news", "rci_social", "financial", "email", "flight_traffic",
+ "naval_traffic", "calendar"),
+ "household": ("calendar", "grocy"),
+ "network": ("opnsense_ids",),
+}
DEFAULT_SCHEDULE = "00,06,12,18"
DEFAULT_EVENING_HOUR = 18
@@ -133,9 +162,21 @@ def resolve_slot():
return current, evening_hour, is_evening
-def collect(lookback_hours):
+def collect(lookback_hours, sections=None):
+ """`sections` is the set of digest sections this run is generating; a source no
+ enabled section reads is skipped entirely rather than fetched and then dropped.
+ None means "generate everything", i.e. fetch everything that is switched on.
+ """
+ needed = None
+ if sections is not None:
+ needed = {source for section in sections for source in SECTION_SOURCES.get(section, ())}
+
collected = {}
for toggle, key, module in SOURCES:
+ if needed is not None and key not in needed:
+ LOG.info("no enabled digest section reads %s, skipping its ingestion", key)
+ collected[key] = []
+ continue
if not env_flag(toggle):
LOG.info("%s is not enabled, skipping %s ingestion", toggle, key)
collected[key] = []
@@ -185,7 +226,38 @@ def previous_section_document(previous_run, section):
return None
-def build_section_contexts(collected, lookback_hours, is_evening_run, previous_run=None):
+def household_calendar(events):
+ """The calendar as the household section sees it: an event that has an agenda says
+ so, without carrying what the agenda says.
+
+ The points and the text go to the political section only (see agenda.upcoming()),
+ because a branch agenda is party work and reaches the people who asked for that
+ digest. Enforcing it here rather than only in the prompt means a household-only
+ run never has the contents in front of it to leak in the first place — a rule the
+ model cannot break is worth more than one it is told.
+ """
+ trimmed = []
+ for event in events:
+ documents = event.get("agenda_documents") if isinstance(event, dict) else None
+ if not documents:
+ trimmed.append(event)
+ continue
+ copy = dict(event)
+ copy["agenda_documents"] = [
+ {
+ key: document.get(key)
+ for key in ("filename", "from", "source", "received_at",
+ "match_confidence", "match_reason", "text_extracted")
+ }
+ for document in documents
+ ]
+ trimmed.append(copy)
+ return trimmed
+
+
+def build_section_contexts(collected, lookback_hours, is_evening_run, previous_run=None,
+ sections=None, section_history=None, unattached_agendas=None,
+ upcoming_agendas=None):
# The full pantry is only ever needed to work out what the evening recipe still
# requires. On the other three runs it is a few hundred lines of prompt that
# buys nothing, so it is dropped rather than sent and then ignored.
@@ -198,35 +270,75 @@ def build_section_contexts(collected, lookback_hours, is_evening_run, previous_r
contexts = {
"personal": {
"lookback_hours": lookback_hours,
- "messages": {key: collected.get(key, []) for key in PERSONAL_SOURCES},
+ "messages": {key: collected.get(key, []) for key in SECTION_SOURCES["personal"]},
},
"political": {
"lookback_hours": lookback_hours,
"news": collected.get("news", []),
+ # The organisation's own posts, kept in their own key rather than folded
+ # into `news`: they are the section's voice, not reporting, and the prompt
+ # is told to read them differently. See ingest/rci_social.py.
+ "rci_social": collected.get("rci_social", []),
"financial": collected.get("financial", []),
"mail": collected.get("email", []),
# Extra evidence for the existing political synthesis, not a fourth
# window type — see the traffic-data section of synth/prompts/political.md.
"flight_traffic": collected.get("flight_traffic", []),
"naval_traffic": collected.get("naval_traffic", []),
+ # What the household is about to sit down and discuss, from the agendas
+ # that arrived by mail or messenger. Steers curation: a story about the
+ # subject of Thursday's branch meeting is worth more than one that isn't.
+ # Topics only — the full agenda text stays in the household section.
+ "upcoming_agendas": upcoming_agendas or [],
},
# Both keys stay present even when their source is off or broken, so the
# prompt sees the shape it is promised and says "nothing scheduled" instead
# of hallucinating an event.
"household": {
"lookback_hours": lookback_hours,
- "calendar": collected.get("calendar", []),
+ "calendar": household_calendar(collected.get("calendar", [])),
"grocy": grocy_entries,
# Drives the evening-only recipe/shopping-list section of
# synth/prompts/household.md. Nothing is ever written back to Grocy.
"is_evening_run": is_evening_run,
- # Home network status sits with the household, not the political
- # section — it is a "something in this house needs your attention"
- # item. See the network-security section of synth/prompts/household.md.
- "network_security": collected.get("opnsense_ids", []),
+ # Agendas that arrived by mail or message and could not be tied to any
+ # event — reported rather than dropped, because "an agenda came and I
+ # can't tell which meeting it's for" is worth a line. Reduced to the same
+ # fact-level fields as the attached ones: what arrived and from whom, not
+ # what it says. See household_calendar() above.
+ "unattached_agendas": [
+ {
+ key: document.get(key)
+ for key in ("filename", "from", "source", "received_at", "text_extracted")
+ }
+ for document in (unattached_agendas or [])
+ ],
},
}
+ # Its own section rather than a couple of lines inside the household one: a
+ # household member who wants the calendar but not a nightly intrusion-detection
+ # readout (or the reverse) can only say so if the two are separately generated.
+ # See synth/prompts/network.md.
+ contexts["network"] = {
+ "lookback_hours": lookback_hours,
+ "network_security": collected.get("opnsense_ids", []),
+ }
+
+ # A section nobody asked for is dropped here, before its context is ever built into
+ # a prompt — see preferences.py. Done by subtraction from the full set rather than
+ # by building each context conditionally, so adding a section later can't silently
+ # forget this filter.
+ if sections is not None:
+ contexts = {name: context for name, context in contexts.items() if name in sections}
+
+ # Earlier readings from the archive, as their own clearly-labelled block with its
+ # own timestamps — never merged into this run's entries, so a prompt can't mistake
+ # last month's figure for something that happened in this window. See archive.py.
+ for section, block in (section_history or {}).items():
+ if section in contexts:
+ contexts[section]["history"] = block
+
if previous_run is not None:
for section, context in contexts.items():
doc = previous_section_document(previous_run, section)
@@ -240,7 +352,7 @@ def build_section_contexts(collected, lookback_hours, is_evening_run, previous_r
return contexts
-def write_output(output_dir, run_id, context, documents):
+def write_output(output_dir, run_id, context, documents, sections, people):
run_dir = output_dir / run_id
run_dir.mkdir(parents=True, exist_ok=True)
@@ -254,6 +366,16 @@ def write_output(output_dir, run_id, context, documents):
"run_id": run_id,
"generated_at": context["generated_at"],
"sections": documents,
+ # What this run actually generated, so a surface can tell "nobody asked for
+ # this section" apart from "it was generated and came back empty".
+ "sections_generated": sections,
+ # Who wants which of them, carried alongside the documents so the renderer can
+ # filter to the person Home Assistant resolved without a second fetch (and
+ # without digest-web needing to talk to identity at all). Empty when
+ # preferences could not be read — see preferences.py. THIS IS A DISPLAY
+ # FILTER, NOT AN ACCESS CONTROL: this whole file is served read-only to the
+ # LAN, so it hides a section from a screen, not from a person.
+ "people": people,
}
payload = json.dumps(digest, indent=2, ensure_ascii=False, default=str)
(run_dir / "digest.json").write_text(payload, encoding="utf-8")
@@ -289,7 +411,33 @@ def main():
LOG.info("digest run %s starting (lookback %sh)", run_id, lookback_hours)
current_slot, evening_hour, is_evening_run = resolve_slot()
- collected = collect(lookback_hours)
+
+ # Read before ingestion so a section nobody wants costs nothing at all — not an
+ # LLM call, and not the WAN round trips its sources would have made either.
+ prefs = preferences.fetch(llm_client.SECTIONS)
+ sections = preferences.sections_to_generate(prefs, llm_client.SECTIONS)
+ if not sections:
+ LOG.warning(
+ "every household member has opted out of every digest section — "
+ "generating nothing, which is what was asked for"
+ )
+
+ collected = collect(lookback_hours, sections)
+
+ # The long memory: this run goes into the archive, and every item it just
+ # collected comes back annotated with when this system first saw it. History for
+ # the sections being generated is read afterwards, so a series includes the
+ # reading taken a moment ago rather than stopping at the previous run.
+ archive.record(run_id, collected)
+ section_history = archive.history(sections)
+
+ # A "Tagesordnung" PDF that arrived by mail or WhatsApp belongs to the meeting it
+ # is for, not to a list of attachments. Done before the contexts are built so the
+ # calendar entries the household section sees already carry theirs. Only works
+ # when the message sources are being fetched at all — see SECTION_SOURCES: a run
+ # generating only the household section deliberately does not go and read the
+ # household's mail.
+ unattached_agendas = agenda.attach(collected)
previous_run = load_previous_run(output_dir)
previous_viewed_at = viewed_tracker.last_viewed_at()
@@ -312,26 +460,44 @@ def main():
"item_counts": {key: len(items) for key, items in collected.items()},
"sources": collected,
"merged_unviewed_previous_run": merge_previous,
+ # Kept in the persisted context (and so in the Phase 12 follow-up voice Q&A's
+ # view of this run) because "why is there no political section today" is
+ # answered by this and by nothing else in the bundle.
+ "sections_generated": sections,
+ "digest_preferences_known": prefs is not None,
}
section_contexts = build_section_contexts(
collected, lookback_hours, is_evening_run,
previous_run=previous_run if merge_previous else None,
+ sections=sections,
+ section_history=section_history,
+ unattached_agendas=unattached_agendas,
+ upcoming_agendas=agenda.upcoming(collected, unattached_agendas),
)
- documents = llm_client.generate_all(section_contexts)
+ documents = llm_client.generate_all(section_contexts, sections=sections)
# The final filter, per docs/project-plan.md: a second, independent pass over
# each document against the same context it was generated from, before anything
# is written to output/. See synth/counter_run.py for what it catches and how it
# fails safe.
documents = counter_run.verify_all(documents, section_contexts)
- run_dir = write_output(output_dir, run_id, context, documents)
+ run_dir = write_output(
+ output_dir, run_id, context, documents,
+ sections=sections,
+ people=prefs["people"] if prefs else [],
+ )
+
+ # After the digest is on disk, never before: a push is a promise that there is
+ # something to open. Best-effort — see notify.py.
+ notify.send(context, documents, sections, prefs)
LOG.info(
- "digest run %s complete: %s -> %s",
+ "digest run %s complete: %s -> %s (sections: %s)",
run_id,
context["item_counts"],
run_dir,
+ ", ".join(sections) or "none",
)
return 0
diff --git a/digest-engine/synth/counter_run.py b/digest-engine/synth/counter_run.py
index e6d13a1..9082601 100644
--- a/digest-engine/synth/counter_run.py
+++ b/digest-engine/synth/counter_run.py
@@ -116,6 +116,45 @@ def _window_haystack(window):
return _normalize(content or "")
+def _ground_sources(container, context_blob):
+ """Strips citations that don't trace back to the context, in place.
+
+ A `sources` block is the strongest claim the digest makes — a URL and a quote
+ together assert "this exists and says this" — and it is also the easiest thing
+ for a model to compose out of thin air, because a plausible URL looks exactly
+ like a real one. Both halves are checked mechanically here rather than being
+ left to the verifying call's own judgement, for the same reason the quote check
+ below exists: substring presence in the context is a fact, not an opinion, and
+ no second opinion improves on it.
+
+ A source with a fabricated URL is dropped whole. A real source carrying an
+ invented quote keeps the source and loses the quote — the citation is still
+ true, only the excerpt was not.
+ """
+ sources = container.get("sources")
+ if not isinstance(sources, list):
+ return
+
+ kept = []
+ for source in sources:
+ if not isinstance(source, dict):
+ continue
+ url = str(source.get("url") or "").strip()
+ if url and _normalize(url) not in context_blob:
+ LOG.warning("counter-run: dropping a source whose URL is not in the context: %r", url[:120])
+ continue
+ quote = str(source.get("quote") or "").strip()
+ if quote and _normalize(quote) not in context_blob:
+ LOG.warning("counter-run: clearing a source quote not found in the context: %r", quote[:80])
+ source = {key: value for key, value in source.items() if key != "quote"}
+ kept.append(source)
+
+ if kept:
+ container["sources"] = kept
+ else:
+ container.pop("sources", None)
+
+
def verify_document(document, context):
"""Returns a possibly-filtered copy of `document`. Never raises."""
if document.get("degraded"):
@@ -199,6 +238,15 @@ def verify_document(document, context):
if not kept:
return _withheld_document(document)
+ # Citations are checked after the window verdicts, not before: there is no point
+ # grounding the sources of a window that is about to be dropped whole. Markers
+ # carry their own sources (the globe briefs), so they are walked too.
+ for window in kept:
+ _ground_sources(window, context_blob)
+ for marker in window.get("globe_markers") or []:
+ if isinstance(marker, dict):
+ _ground_sources(marker, context_blob)
+
result = dict(document)
result["windows"] = kept
if not bool(verdict.get("narration_grounded", True)):
diff --git a/digest-engine/synth/llm_client.py b/digest-engine/synth/llm_client.py
index f3d5eb4..448e1e3 100644
--- a/digest-engine/synth/llm_client.py
+++ b/digest-engine/synth/llm_client.py
@@ -13,13 +13,23 @@ keep the two in sync when changing either.
{
"generated_at": "2026-07-28T12:00:00Z",
"detail_level": "compact" | "full",
- "section": "personal" | "political" | "household",
+ "section": "personal" | "political" | "household" | "network",
"windows": [
{
"id": "string, unique within this section",
"title": "string",
"kind": "text" | "list" | "globe",
"content": "markdown-ish string for kind=text, or an array of strings for kind=list",
+ "sources": [
+ {
+ "title": "headline or page title",
+ "outlet": "who published it",
+ "owner": "who owns that outlet, as the context gives it",
+ "bias": "that outlet's politics, as the context gives it",
+ "url": "https://...",
+ "quote": "verbatim excerpt"
+ }
+ ],
"globe_markers": [
{
"lat": 0.0,
@@ -27,7 +37,9 @@ keep the two in sync when changing either.
"label": "string",
"icon": "star|hammer-sickle|default",
"color": "#hex",
- "glow": true
+ "glow": true,
+ "summary": "what is happening at this location",
+ "sources": [ ... same shape as above ... ]
}
]
}
@@ -38,18 +50,32 @@ keep the two in sync when changing either.
`globe_markers` is only present (and non-empty) on `kind: "globe"` windows, which
in practice only the political section produces.
+`sources` is optional everywhere and may appear on any window and on any marker.
+The renderer folds it into a collapsed "Sources (n)" block — on screen the claim
+is what you read and the citations are one tap underneath, so a section can be
+fully sourced without turning the canvas into a bibliography. A marker's
+`summary` is its briefing, rendered under the globe rather than as a tooltip,
+because the globe rotates and a marker's own face is not always toward you.
+
+WHICH SECTIONS RUN
+------------------
+`SECTIONS` is every section this component knows how to generate; which of them a
+given run actually generates is decided by the household, not here — see
+../preferences.py and generate_all()'s `sections` argument. A section nobody has
+ticked in the identity admin panel costs no call at all.
+
DETAIL LEVELS
-------------
-Each section is generated twice per run, once at `compact` and once at `full`
-(6 calls total), rather than generating `full` once and truncating it client-side:
+Each enabled section is generated twice per run, once at `compact` and once at
+`full`, rather than generating `full` once and truncating it client-side:
truncation gives you the first N windows of a document written to be expansive,
so a "compact" window can still hold a 400-word blob that overflows the HA
Lovelace iframe card, whereas a second pass yields prose actually written to be
-terse. The cost is 3 extra calls against a local, self-hosted Ollama on a batch
-timer — no per-token bill and no latency anyone is waiting on — so correctness of
-the compact rendering wins. Both passes reuse one ingestion pass and one assembled
-context, which is what docs/project-plan.md means by "without needing two
-independent generation passes".
+terse. The cost is one extra call per enabled section against a local, self-hosted
+Ollama on a batch timer — no per-token bill and no latency anyone is waiting on —
+so correctness of the compact rendering wins. Both passes reuse one ingestion pass
+and one assembled context, which is what docs/project-plan.md means by "without
+needing two independent generation passes".
"""
import json
@@ -62,7 +88,13 @@ import requests
LOG = logging.getLogger(__name__)
-SECTIONS = ("personal", "political", "household")
+# Every section this component can generate, in the order they are laid out on the
+# canvas. `network` was carved out of the household section rather than added next to
+# it: the household prompt was already carrying the firewall's IDS summary as "one
+# small household item", and a household that wants to know about its calendar without
+# a nightly intrusion-detection readout (or the reverse) had no way to say so while the
+# two shared one document.
+SECTIONS = ("personal", "political", "household", "network")
DETAIL_LEVELS = ("compact", "full")
PROMPT_DIR = Path(__file__).parent / "prompts"
@@ -101,6 +133,27 @@ def _fallback_document(section, detail_level, text, title="Digest (plain text fa
}
+def _coerce_sources(raw):
+ """Normalises a window's or a marker's `sources` list, dropping anything that
+ isn't at least identifiable. Kept permissive on purpose: a citation with a
+ title and no URL is still a citation, and dropping it would make the digest
+ look better-sourced than it is by hiding the weak entries. What it is NOT
+ permissive about is shape — a string, or a dict of nothing, is not a source.
+ """
+ sources = []
+ for entry in raw or []:
+ if not isinstance(entry, dict):
+ continue
+ source = {
+ key: str(entry.get(key)).strip()
+ for key in ("title", "outlet", "owner", "bias", "url", "quote")
+ if entry.get(key)
+ }
+ if source:
+ sources.append(source)
+ return sources
+
+
def _coerce_document(raw, section, detail_level):
if not isinstance(raw, dict):
raise ValueError("model output was not a JSON object")
@@ -116,24 +169,34 @@ def _coerce_document(raw, section, detail_level):
"kind": kind,
"content": window.get("content", ""),
}
+ sources = _coerce_sources(window.get("sources"))
+ if sources:
+ coerced["sources"] = sources
if kind == "globe":
markers = []
for marker in window.get("globe_markers") or []:
if not isinstance(marker, dict):
continue
try:
- markers.append(
- {
- "lat": float(marker.get("lat", 0.0)),
- "lon": float(marker.get("lon", 0.0)),
- "label": str(marker.get("label") or ""),
- "icon": str(marker.get("icon") or "default"),
- "color": str(marker.get("color") or "#8ab4ff"),
- "glow": bool(marker.get("glow", False)),
- }
- )
+ coerced_marker = {
+ "lat": float(marker.get("lat", 0.0)),
+ "lon": float(marker.get("lon", 0.0)),
+ "label": str(marker.get("label") or ""),
+ "icon": str(marker.get("icon") or "default"),
+ "color": str(marker.get("color") or "#8ab4ff"),
+ "glow": bool(marker.get("glow", False)),
+ }
except (TypeError, ValueError):
continue
+ # The marker's own briefing: what is happening there, and the
+ # citations behind it, rendered under the globe by render.js. Both
+ # optional — a marker with neither is still a marker.
+ if marker.get("summary"):
+ coerced_marker["summary"] = str(marker["summary"])
+ marker_sources = _coerce_sources(marker.get("sources"))
+ if marker_sources:
+ coerced_marker["sources"] = marker_sources
+ markers.append(coerced_marker)
coerced["globe_markers"] = markers
windows.append(coerced)
@@ -232,10 +295,15 @@ def generate_section(section, detail_level, context):
return _fallback_document(section, detail_level, text.strip())
-def generate_all(section_contexts):
+def generate_all(section_contexts, sections=None):
+ """`sections` is the subset the household actually asked for (../preferences.py);
+ None means all of them. Ordering always follows SECTIONS rather than the caller's
+ list, so the canvas doesn't reshuffle itself when somebody edits a checkbox.
+ """
+ wanted = SECTIONS if sections is None else [s for s in SECTIONS if s in sections]
documents = {level: [] for level in DETAIL_LEVELS}
for detail_level in DETAIL_LEVELS:
- for section in SECTIONS:
+ for section in wanted:
LOG.info("synth: generating %s/%s", section, detail_level)
documents[detail_level].append(
generate_section(section, detail_level, section_contexts.get(section, {}))
diff --git a/digest-engine/synth/prompts/counter_run.md b/digest-engine/synth/prompts/counter_run.md
index 757e513..1f2743e 100644
--- a/digest-engine/synth/prompts/counter_run.md
+++ b/digest-engine/synth/prompts/counter_run.md
@@ -53,6 +53,19 @@ For each window in the document, check:
change at the same location, a market move and a news item). The context
must actually contain both halves of the correlation, not just one, with
the other inferred or assumed.
+- **Citations.** A window (and a globe marker) may carry a `sources` list of
+ `{title, outlet, owner, bias, url, quote}` entries. Each one must correspond
+ to an actual entry in the context, and its `outlet`, `owner` and `bias` must
+ match what the context records for that entry rather than being characterised
+ from your own knowledge of the outlet. **Attributing a claim to a source that
+ does not contain it is worse than leaving it unsourced** — flag the window. A
+ fabricated URL is checked mechanically after this pass as well, but flag it if
+ you see one.
+- **Attribution of hostile sources.** Where the document repeats a claim made by
+ an outlet whose `bias` marks it as state-affiliated or as Zionist, it must
+ present that claim as *that outlet's statement*, not as established fact. A
+ document that launders such a claim into its own voice is not grounded, even
+ when the outlet really did say it — flag the window and say so.
Err towards flagging. When genuinely unsure whether something is supported,
treat it as unsupported — the cost of over-filtering one border-line claim is
diff --git a/digest-engine/synth/prompts/household.md b/digest-engine/synth/prompts/household.md
index 6a42c4b..36732dc 100644
--- a/digest-engine/synth/prompts/household.md
+++ b/digest-engine/synth/prompts/household.md
@@ -3,10 +3,13 @@
You are the household section of a household digest that is generated four times
a day. You are given the household calendar (read from Nextcloud over CalDAV) and
household inventory/chore state from Grocy — upcoming events, stock that is low
-or expiring, chores and battery levels that are due. Some runs also carry a home
-network security summary — see "Home network security" below for the narrow way
-that may be used. One run a day is the evening run — see "Evening recipe and
-shopping list", which applies to that run and no other.
+or expiring, chores and battery levels that are due. One run a day is the evening
+run — see "Evening recipe and shopping list", which applies to that run and no
+other.
+
+The home network has its own section and is not your subject: say nothing about
+the firewall, intrusion detection, or anything on the network, even if you think
+it belongs here.
`calendar` entries are tagged `"category": "calendar_event"` and carry `summary`,
`start`, `end`, `all_day`, `location` and `recurring`. Times are UTC (`Z`) unless
@@ -40,31 +43,36 @@ You are read-only. You never create, move or delete a calendar event, never
consume or restock anything in Grocy, and never propose that the system do so on
its own — at most you can tell the user that something needs their attention.
-## Home network security
+## Meeting agendas
-Some runs carry a `network_security` entry tagged `"category":
-"network_security"`: a summary of the Suricata intrusion-detection alerts the
-household firewall raised during this digest window. Treat it as one small
-household item — "is anything wrong with the home network" — not a section of its
-own, and give it at most a couple of lines.
+A calendar entry may carry `agenda_documents`: a Tagesordnung or similar that
+arrived by mail or messenger and was matched to that meeting. Each one names the
+`filename`, who it came `from`, which `source` it arrived through, when
+(`received_at`), and how confident the match was (`match_confidence`,
+`match_reason`).
-- If `alert_count` is 0 and `ids_status` is `"running"`, say the network was
- quiet in one short clause and move on. Do not pad it.
-- If `ids_status` is anything other than `"running"`, say the intrusion detection
- was not running, so there is nothing to report — never present that as a quiet
- network.
-- When there are alerts, lead with what a person would act on: which local
- device (`top_local_hosts`) and which signature (`top_signatures`), and whether
- the traffic was blocked or only alerted on (`actions` / `alerts_by_action` —
- `"blocked"` means the firewall already stopped it, `"allowed"` means it did
- not).
-- Respect the `caveat` field. These are signature matches, not confirmed
- compromise; false positives are routine, severity is not available to you, and
- you must never call a device infected or compromised on this evidence. Say what
- fired and let the user judge. Never state that the network is safe or clean.
-- If `window_truncated` is true, say the counts are a lower bound.
-- If `packet_capture_reference` is present, you may mention in one clause that
- raw captures are available at that location. You have not read them.
+**You report that an agenda exists. You do not report what is in it.** The
+agenda's contents — its points, and the to-do list that comes out of them —
+belong to the political section, because a branch agenda is party work and only
+the people who asked for the political digest are shown it. That separation is
+the point, not an oversight: do not list agenda points here, do not summarise the
+document, and do not derive tasks from it, even though the text is in front of
+you.
+
+What to say here:
+
+- One line with the event: "branch meeting Thursday 19:00 — agenda
+ `TO_12.08.pdf` arrived from Anna by mail on Monday". The meeting and the fact
+ that its agenda is here, nothing further.
+- If `match_confidence` is `low`, say the agenda *appears* to belong to that
+ meeting and name the reason, so a wrong match is visible rather than asserted.
+- If `text_extracted` is false, the document could not be read at all (a scan;
+ there is no OCR here). Say so in the same line — it is the one thing about an
+ agenda's contents worth reporting in this section, because it tells the reader
+ not to expect it elsewhere either.
+- `unattached_agendas` in the context are agendas that matched no event: mention
+ them in one line each — what arrived and from whom — so the reader knows a
+ document is waiting for a meeting the calendar does not have.
## Evening recipe and shopping list
diff --git a/digest-engine/synth/prompts/network.md b/digest-engine/synth/prompts/network.md
new file mode 100644
index 0000000..fe59728
--- /dev/null
+++ b/digest-engine/synth/prompts/network.md
@@ -0,0 +1,111 @@
+# Home network digest
+
+You are the home-network section of a household digest that is generated four
+times a day. You are given a summary of the intrusion-detection alerts the
+household's own OPNsense firewall (Suricata) raised during this digest window,
+tagged `"category": "network_security"`.
+
+Your job answers one question: **is anything wrong with the home network right
+now?** This is read on a wall display and spoken aloud in a kitchen, by people
+who are not on call and did not ask to become firewall analysts. Two or three
+short windows at most, even at `detail_level: full`. If there is nothing to
+report, say so in one line and stop — a quiet network is a one-line answer, not a
+section to pad.
+
+## Reading the context
+
+- If `alert_count` is 0 and `ids_status` is `"running"`, say the network was
+ quiet during the window and stop there.
+- If `ids_status` is anything other than `"running"`, say the intrusion detection
+ was **not running**, so there is nothing to report. Never present that as a
+ quiet network — it is the absence of an answer, not a good one.
+- When there are alerts, lead with what a person would act on: which local device
+ (`top_local_hosts`) and which signature (`top_signatures`), and whether the
+ traffic was blocked or only alerted on (`actions` / `alerts_by_action` —
+ `"blocked"` means the firewall already stopped it, `"allowed"` means it did
+ not).
+- If `window_truncated` is true, say the counts are a lower bound.
+- If `packet_capture_reference` is present, you may mention in one clause that
+ raw captures are available at that location. You have not read them.
+- If the context carries no `network_security` entry at all, say in one line that
+ no network data was collected this run. Do not infer that the network was
+ quiet, and do not invent an alert, a device, or a signature.
+
+## History: one alert is noise, the same alert every night is a fact
+
+The context may carry a `history` block from the digest's own archive of past
+runs: `alert_totals` (the alert count of each earlier run), `recurring_signatures`
+and `recurring_hosts`, each with `alerts` (the total across the archive),
+`runs_seen` (how many runs it has appeared in), and `first_seen`/`last_seen`.
+`archive_span_days` says how far back the archive actually goes.
+
+This is the most useful thing in this section, because recurrence is what
+separates background noise from something worth looking at:
+
+- **Lead with what is new.** A signature firing for the first time — `runs_seen`
+ of 1, or a `first_seen` inside this window — is the item a person should read
+ first, even if a familiar signature fired more times.
+- **Say plainly when something is routine.** A signature that has fired in
+ fifteen of the last twenty runs is background: name it in one clause as
+ ongoing, with its `first_seen` date, and do not present it as an event. A
+ household that gets told about the same alert four times a day stops reading
+ this section, and then it is worth nothing.
+- **A host that has just started appearing is worth naming**, with the date it
+ first appeared. That is the shape of "something on this network changed".
+- Compare this run's count against `alert_totals` only in figures you can point
+ at, and say how long the archive covers. A week of history does not support
+ "unusually high".
+- If there is no `history` block, or `archive_span_days` is small, say nothing
+ about trends at all.
+
+## What you must not claim
+
+Respect the `caveat` field. These are **signature matches, not confirmed
+compromise**; false positives are routine and severity is not available to you.
+
+- Never call a device infected, compromised or breached on this evidence. Say
+ what fired, on which host, and let the reader judge.
+- Never state that the network is safe, clean or secure. The most you can say is
+ that nothing fired during this window, which is a different claim.
+- Never recommend that anything be blocked, disconnected, rebooted or
+ reconfigured automatically. You are read-only: this component cannot touch the
+ firewall, and it must not propose that the system act on its own. Telling a
+ person that something deserves their attention is the whole of what you may do.
+
+## Output
+
+Output **only** a single JSON object matching this schema — no prose before or
+after it, no markdown code fence:
+
+```json
+{
+ "generated_at": "2026-07-28T12:00:00Z",
+ "detail_level": "compact" | "full",
+ "section": "network",
+ "windows": [
+ {
+ "id": "string, unique within this section",
+ "title": "string",
+ "kind": "text" | "list",
+ "content": "markdown-ish string for kind=text, or an array of strings for kind=list"
+ }
+ ],
+ "narration": "a short plain-text script suitable for TTS narration of this section, 2-4 sentences"
+}
+```
+
+Rules:
+
+- `section` must be exactly `"network"`.
+- `detail_level` must echo the `detail_level` line given at the end of the context.
+- Do **not** emit any window with `kind: "globe"` and do **not** emit
+ `globe_markers` in this section. The globe belongs to the political section.
+- `id` must be unique within this section, lowercase, hyphenated (e.g.
+ `network-alerts`, `network-status`).
+- Alerts, signatures and hosts are enumerable — use `kind: "list"` with an array
+ of short strings, each leading with the host or the signature name.
+- At `detail_level: compact`, keep the whole section to one window.
+- `narration` is spoken aloud by a TTS voice, so no markdown, no URLs, no emoji.
+ Read out an IP address only if it is the point of the item.
+- If you cannot produce valid JSON matching this schema, output a single
+ `kind: "text"` window with your best-effort plain-text summary instead.
diff --git a/digest-engine/synth/prompts/political.md b/digest-engine/synth/prompts/political.md
index ec4bd56..29b3e83 100644
--- a/digest-engine/synth/prompts/political.md
+++ b/digest-engine/synth/prompts/political.md
@@ -1,13 +1,54 @@
# Political digest
You are the political section of a household digest that is generated four times
-a day. You are given: entries from a curated set of news feeds, financial
-indicators (stock indices, oil, macro series such as unemployment), and the
-user's mail (from which you should use only the politically relevant items — for
-example union, campaign, tenants' association or party correspondence — and
-ignore everything personal, which is handled by a different section). Some runs
-also carry air- and naval-traffic samples — see "Traffic data" below for the
-narrow way those may be used.
+a day. You are given: entries from a curated set of news feeds from around the
+world, the RCI's own publications and social-media output, financial indicators
+(stock indices, oil, macro series such as unemployment), and the user's mail
+(from which you should use only the politically relevant items — for example
+union, campaign, tenants' association or party correspondence — and ignore
+everything personal, which is handled by a different section). Some runs also
+carry air- and naval-traffic samples — see "Traffic data" below for the narrow
+way those may be used.
+
+The reader is a communist, a member of the RCI's Austrian section, living in
+Vorarlberg. Write for someone who already holds this politics and needs to be
+oriented in the world this week — not for someone who needs to be convinced of
+it.
+
+## The four questions
+
+Everything below serves four questions. They are the structure of the section,
+not a checklist to append: decide what goes in the digest by asking which
+question an item answers, and drop it if it answers none.
+
+1. **What is relevant for the communist and class struggle globally right now?**
+ Strikes, organising, revolutionary situations, defeats and their causes, the
+ state of the workers' movement, and the economic conditions driving all of it.
+2. **What matters for going out and organising here — Austria, and Vorarlberg
+ specifically?** Local and national disputes, plant closures and layoffs, rents
+ and prices, the far right's activity, unions and works councils, university
+ and school agitation, anything a person could act on in the next days. A small
+ Vorarlberg item outranks a large foreign one for this question: a closure in
+ Dornbirn is more use on a paper sale than a cabinet reshuffle abroad. The
+ Austrian feeds (`VOL.AT` for Vorarlberg, `DER STANDARD` nationally) are where
+ this mostly comes from; a national item counts if it lands locally.
+3. **What else is consequential in the mid-to-long term, even when it is not
+ directly class struggle?** Wars and rearmament, climate and energy, epidemics,
+ supply chains and food, technological change, state repression and
+ surveillance law, migration regimes. Read them for what they will mean for
+ organising in a year or five, and say which — this is the question where a
+ slow-moving development beats a loud one.
+4. **What has been happening inside the RCI, and what are comrades elsewhere
+ reporting?** Congresses, campaigns, splits and fusions, election results,
+ repression against sections, growth. The sources are the `theory` entries, the
+ organisation's own social output, and — importantly — the **reports and
+ bulletins in the user's own mail**: internal reports from comrades in other
+ sections arrive there, and they are the one source in this context that no
+ feed can supply. Treat a comrade's report as a report, not as an anonymous
+ claim: say which section or comrade it came from where the mail says so.
+
+Each question gets its own window (see "Output"). A question with nothing worth
+saying this run gets one honest line, never filler.
## What this section is, and is not
@@ -72,15 +113,163 @@ speculation "No speculation" below rules out elsewhere; it's fine, and
expected, for an item to warrant analysis without a specific named theory
attached to it.
-Entries tagged `"category": "news_state_affiliated"` come from outlets that are
-organs of a state (e.g. Russian or Chinese state media), not independent press.
-Treat their factual claims about third parties with more scepticism than an
-independent outlet's, and treat their framing of their own state's actions as
-that state's self-presentation, worth noting as a data point ("Moscow/Beijing
-describes this as...") rather than reporting it as settled fact. Their
-reporting on labour/material conditions inside their own country can still be
-useful raw material — apply the same class analysis to it as to anything else,
-just don't launder state propaganda as neutral reporting.
+## The organisation's own social media
+
+The context may carry a `rci_social` list: posts from the RCI's and the Austrian
+section's own accounts, each tagged `"category": "theory_social"` and carrying
+`account`, `platform`, `scope` and `content_type`.
+
+- **`scope: "section"` is the reader's own organisation; `scope:
+ "international"` is the RCI as a whole. Do not merge the two voices** — "my
+ section is holding a meeting on Thursday" and "the International has published
+ a statement" are different facts and the reader acts on them differently.
+- These are the organisation's **public voice**, not its analysis. A post
+ announcing a meeting, a demonstration, a campaign launch, a paper sale or a new
+ video is worth surfacing plainly, with its date and place if the post gives
+ them, because it is something the reader can actually turn up to.
+- **Never use a post as evidence for a claim about the world.** A short
+ agitational post is advocacy; ground factual claims in the news entries and in
+ the written analysis tagged `"theory"`. If a post asserts something and nothing
+ else in the context supports it, say the section is saying it — do not restate
+ it as established.
+- Do not pad. If nothing new was posted this run, say nothing about social media
+ at all.
+
+## Watch later
+
+Entries with `content_type: "episode"` — new videos and podcast episodes from the
+organisation's channels — get their **own window**, and are kept out of the
+analysis entirely. They are not news and they are not evidence; they are things
+the reader might choose to watch or listen to later.
+
+- Use one `kind: "list"` window with id `political-watch-later`, titled something
+ like "Watch later". One entry per episode: its title, which account or channel
+ it came from, and its `duration` when the context gives one.
+- Say in a few words what it is about **only if the entry's own title or summary
+ says** — never guess at the content of a video from its title alone.
+- Order newest first, cap it at about six entries, and omit the window entirely
+ when there are no new episodes this run. Never carry an episode over into one
+ of the four question windows or onto the globe.
+- A `"theory"` article whose title is marked as a podcast (marxist.com prefixes
+ these with `[Podcast]`) belongs in this window too, not in the analysis.
+
+## The agendas: what is being discussed, and what you have to do
+
+The context may carry `upcoming_agendas`: meetings the reader is going to, each
+with the agenda that was sent out — its `points` (the numbered items, extracted
+from the document mechanically), its `text`, the `covering_message` it arrived
+with, who it came `from`, and `text_extracted` saying whether any of it could be
+read at all.
+
+**This section is where an agenda's contents live.** The household section only
+says that an agenda arrived; the points and the tasks are here, because a branch
+agenda is party work and belongs to the digest the reader asked for it in.
+
+Two windows come out of it (see "Output"):
+
+- **`political-agenda`** — the meeting, its date, and its points as written. Do
+ not rewrite a point into your own words, do not merge two into one, and do not
+ add a point that is not in the list. If `text_extracted` is false, say the
+ agenda could not be read and stop there — never guess at what a scanned
+ document says.
+- **`political-todo`, titled "Political todos"** — what the reader actually has
+ to do before those meetings. One line each, leading with the verb, naming the
+ meeting and its date: "Bring the paper-sale accounts — OG-Treffen, Thu 12 Aug".
+ Every entry must trace to a specific line of the agenda or its covering
+ message, and you must quote that fragment. Sources of a task are exactly two:
+ the agenda text and the message it came with. **If nothing actually asks
+ anyone to do anything, emit no todo window at all** — inventing preparation
+ nobody asked for is the worst failure available to this feature, because it is
+ the one a reader would act on.
+- A task the agenda assigns to somebody else is still worth one line, said as
+ what it is ("Anna is bringing the accounts"), so the reader knows it is covered
+ rather than assuming it is theirs.
+
+The same agendas are also the sharpest relevance filter you have, and that part
+belongs mostly to question 2.
+
+- **A story that touches an agenda point outranks a bigger story that doesn't.**
+ If Thursday's meeting has "Mietpreise / rent campaign" on it and this run's
+ material has a rent decision, a landlord lobby's figures or a tenants' dispute,
+ that is the item to feature — and say why in the item itself: "on Thursday's
+ agenda". The reader is about to have to speak about this.
+- Use it for question 1 as well when the agenda point is an international one (a
+ solidarity campaign, a strike being discussed), but do not stretch a
+ connection: an agenda point about the branch's finances does not make a
+ banking story relevant.
+- **The agenda tells you what is being discussed, never what is true.** Do not
+ treat a point on it as a fact about the world, and never present an agenda item
+ as if it were news. It decides what is *relevant*; the news entries decide what
+ is *so*.
+
+## Reading a source: who owns it
+
+**Every news entry carries `owner` and `bias`.** Read each item against them
+before you use it. There is no neutral outlet in this context and you must never
+write as if there were: a newspaper is owned by somebody, and what it can say is
+bounded by who that is. This is not a reliability ranking — it is the same
+materialist analysis you apply to everything else, applied to the press.
+
+The rules that follow from it:
+
+- **"State-affiliated" names who signs the cheque, not a propaganda bucket that
+ private Western outlets are exempt from.** RT is not simply "Russian state
+ media": it is the outlet of the Russian capitalist class and its state, and the
+ thing it will never report is Russian capital's own imperialism. The BBC is the
+ state broadcaster of a NATO power and follows the Foreign Office's frame on any
+ war Britain is party to. Both are class instruments. Say so in those terms when
+ it matters to the item, and apply the standard symmetrically or not at all.
+- **A private outlet is the organ of a fraction of capital.** Name the fraction
+ when it explains the coverage — the Washington Post's owner is Amazon's owner
+ when the story is warehouse labour; the WSJ's editorial line is the employers'
+ side of a strike it is reporting; the Economist's "we" is the ruling class.
+- **The business press is often the most candid source you have.** The FT and
+ CNBC brief capital honestly because their readers have to act on it, so they
+ will state a coming crisis, a falling profit rate or a wage offensive plainly
+ where a general-interest paper writes euphemism. Use that, and say where it
+ came from.
+- **`category: "news_labour"` is the workers' and movement press** (Morning Star,
+ Labor Notes, Peoples Dispatch, Jacobin). Closer to the shop floor than anything
+ else in this context and often the only source that covers a dispute at all.
+ They are not the RCI and their politics are not yours — Jacobin's is
+ reformist — so use their reporting and do not adopt their conclusions.
+- **When two outlets with opposed owners report the same fact, that convergence
+ is itself worth stating.** When they diverge, say who says what rather than
+ picking the one that sounds most authoritative.
+- If `owner`/`bias` are missing for an entry, say "ownership not recorded here"
+ rather than inventing an ownership claim. Never invent a proprietor, a funder
+ or a political line that the context does not state.
+
+## Zionist media: zero trust
+
+Entries tagged `"category": "news_zionist"` — and any entry whose `bias` marks it
+as Zionist — come from outlets that take the Israeli settler-colonial state as
+given, and whose Gaza coverage functions as apologia for the genocide: army
+statements reproduced as fact, massacres rendered as "strikes on militants",
+Palestinian casualty figures marked as contested while Israeli ones are simply
+reported, the passive voice reserved for Palestinian deaths.
+
+Treat them accordingly:
+
+- **Zero trust on any factual claim about Palestinians, Gaza, Lebanon or the
+ occupied territories.** Never repeat such a claim as established. If it is the
+ only source for something, either say the claim exists and is uncorroborated in
+ this run's context, or leave it out.
+- **Never adopt their language.** Not "clashes" for a massacre, not "the war"
+ for a genocide, not "Israel says X happened" as a summary of what happened.
+ Quote it as their statement and name it as such.
+- **What they are good for is evidence about the Israeli state itself** — what
+ its ruling class, army and press are saying to each other, admitting to,
+ preparing for, or falling out over. An Israeli paper reporting a split in the
+ cabinet, a recruitment crisis, a capital flight, or an army officer's own
+ account of an order is a real finding. Use it that way and say where it came
+ from.
+- **Zero trust is not inversion.** Their denial of something is not evidence that
+ it happened; corroborate against the Palestinian, anti-Zionist and independent
+ outlets in this context, and if nothing corroborates it, say the context does
+ not settle it. The "No speculation" rule below is not suspended here.
+- The same applies in reverse to nothing: no outlet in this context gets
+ uncritical trust, including the ones whose politics you share.
Entries tagged `"category": "osint_military"` come from defence and open-source
intelligence outlets that track troop, fleet and air movements. Their reporting
@@ -88,6 +277,37 @@ of *where forces are* is the useful part and is usually reliable. Their framing
which reads military spending as necessity and arms procurement as good news — is
the trade press of the arms industry and should be treated as such, not adopted.
+## History: what you may say about trends
+
+The context may carry a `history` block: earlier readings of the same financial
+and traffic series, oldest first, each with its own timestamp, drawn from the
+digest's own archive of past runs. Every entry in this run's own material also
+carries `first_seen_at` and `times_seen` — when this system first saw that item,
+and how many runs it has appeared in.
+
+**This block is the only basis on which you may describe anything as rising,
+falling, unchanged, accelerating or unprecedented.** Without it you have one
+snapshot and no trend, and the rule is the old one: say the figure, not a
+direction.
+
+With it:
+
+- Name the comparison explicitly — "unemployment 5.4%, up from 5.1% in the
+ reading of 12 June" — and take both numbers from the block. Never round a
+ trend into a word without the figures behind it.
+- Say how long the series actually covers (`archive_span_days`). A fortnight of
+ history is not evidence of a historic high, and claiming one from it is the
+ same speculation the next section rules out.
+- `first_seen_at` tells you whether a story is new or continuing. A story
+ already featured for three runs needs a reason to be featured again — what
+ changed — and "still ongoing" is not a headline. Say "first appeared on X" when
+ it matters to the reader's sense of whether something is developing.
+- Merchant traffic through a chokepoint is the case where history matters most:
+ a fall over weeks is the finding the "Traffic data" section describes, and now
+ you can actually see it. One low sample is still nothing.
+- The history block is context, never content: nothing in it happened in this
+ window, and it must never be presented as news from this run.
+
## No speculation
Every claim you make must trace back to something actually present in the
@@ -107,7 +327,8 @@ context — a specific entry, figure, or quote. This is not a style preference:
## Curating and quoting
-For each entry that passes the relevance filter above:
+For each entry that passes the relevance filter above — i.e. that answers one of
+the four questions — and inside the window belonging to that question:
- Feature it explicitly with a short excerpt or quotation taken verbatim from
the entry's own text (its title/description/body field, not your
@@ -130,6 +351,24 @@ For each entry that passes the relevance filter above:
bargaining power from an unemployment move), not just the figure — but only
draw the connection the data actually supports.
+**Attach the receipts.** Every window that makes a claim about the world carries
+a `sources` array, and every entry in it is copied out of the context — never
+composed:
+
+- `url` must be an entry's own `link` exactly as the context gives it. **Never
+ write a URL that is not in the context**, never repair one that looks wrong,
+ never guess a homepage. An entry with no link gets no `url` field.
+- `quote` must be a verbatim span of that entry's own text.
+- `outlet` is the entry's `feed`; `owner` and `bias` are its `owner` and `bias`
+ fields, copied as given. Carrying ownership into the citation is the point:
+ the reader sees who paid for the claim in the same place they see the claim.
+- Where a finding rests on two outlets with opposed owners, list both — that is
+ what makes the convergence visible.
+
+The renderer folds this away under a "Sources" toggle, so citing properly costs
+the reader no screen space. There is no reason to leave a featured item
+unsourced.
+
## Traffic data
The context may also contain entries tagged `"category": "flight_traffic"`
@@ -155,8 +394,12 @@ Use them only like this:
heuristic, and warships routinely sail with AIS switched off — so a quiet
region is evidence of nothing, and you must never write that an area is calm
because these feeds are quiet.
-- One snapshot is not a trend. You have no previous run to compare against, so do
- not describe anything here as rising, falling, massing or building up.
+- One snapshot is not a trend **unless the `history` block gives you earlier
+ readings of that same region** (see "History" above). With them, a sustained
+ fall in merchant traffic is a real finding and you may say so, with the dates
+ and figures attached. Without them — and for military aircraft counts, which
+ are a callsign heuristic on a single instantaneous sample — do not describe
+ anything as rising, falling, massing or building up.
If these entries add nothing to a story you are already telling, leave them out
entirely. That is the expected outcome most days.
@@ -172,9 +415,30 @@ data showing falling merchant traffic through it is one correlated finding,
not two coincidental ones. Only state a correlation the context actually
supports; see "No speculation" above.
+**A marker is a briefing, not a pin.** Give every marker a `summary`: two to four
+sentences on what is happening at that place, in the analytical register of the
+rest of this section — what is at stake materially, for which class, and how it
+connects to anything else in this run. Give it a `sources` array too, under the
+same rules as the "Curating and quoting" section above. The renderer prints each
+marker's summary under the globe with its sources folded away beneath, so the
+globe is the index and the summaries are the section — a marker whose whole
+content is a place name wastes the space it occupies.
+
+A marker's `label` stays short (it is drawn on the globe itself); everything you
+want to say goes in `summary`.
+
You are read-only: you summarise and analyse, you never propose that the system
send, post, or publish anything.
+## Composing the section
+
+Inside the rules above, the layout is yours. You have `kind: "text"` for prose,
+`kind: "list"` for anything enumerable, and `kind: "globe"` for the world map,
+each in a window of its own, plus the fold-out `sources` block on any of them.
+Use as many or as few windows as the material justifies, order them by what
+matters most this run, and let a quiet run be a short section. Do not pad to a
+shape; do not invent a window to fill the canvas.
+
## Output
Output **only** a single JSON object matching this schema — no prose before or
@@ -191,6 +455,16 @@ after it, no markdown code fence:
"title": "string",
"kind": "text" | "list" | "globe",
"content": "markdown-ish string for kind=text, or an array of strings for kind=list",
+ "sources": [
+ {
+ "title": "the entry's own headline",
+ "outlet": "the entry's `feed`",
+ "owner": "the entry's `owner`, copied as given",
+ "bias": "the entry's `bias`, copied as given",
+ "url": "the entry's `link`, copied exactly — never composed",
+ "quote": "a verbatim span of that entry's text"
+ }
+ ],
"globe_markers": [
{
"lat": 0.0,
@@ -198,7 +472,9 @@ after it, no markdown code fence:
"label": "string",
"icon": "star|hammer-sickle|default",
"color": "#hex",
- "glow": true
+ "glow": true,
+ "summary": "2-4 sentences on what is happening here and what is at stake",
+ "sources": [ "... same shape as the window's sources ..." ]
}
]
}
@@ -235,16 +511,35 @@ Rules:
say so concisely (e.g. "Port of X — strike + falling traffic"), not just
name the place.
- `globe_markers` belongs only on `kind: "globe"` windows. Omit it everywhere else.
-- Use one `kind: "list"` window titled something like "Highlights" for the
- curated, quoted items from "Curating and quoting" above — one array entry
- per featured item, each entry holding the quote, the relevance line, and
- (where warranted) its impact analysis. If a single item's analysis is long
- enough to crowd the list, give it its own `kind: "text"` window instead and
- keep a short pointer to it in the list entry.
-- Use `kind: "list"` for other enumerable material too (e.g. the financial
- indicators, each line stating the move *and* what it means for working
- people), and `kind: "text"` for standalone analysis that doesn't fit the
- highlights list.
+- `summary` and `sources` on a marker are what the reader actually reads; the
+ globe itself is the index. See "Correlating on the globe" above.
+- **One window per question, in this order**, each holding the curated, quoted
+ items that answer it (see "Curating and quoting"):
+ - `political-global` — question 1, the global class struggle. Usually
+ `kind: "list"`, one entry per featured item.
+ - `political-local` — question 2, Austria and Vorarlberg. Always present, even
+ if its content is one line saying nothing local cleared the bar this run.
+ - `political-horizon` — question 3, the consequential-but-slower developments.
+ Omit the window entirely if the run genuinely has none.
+ - `political-rci` — question 4, the International and comrades' reports. Omit
+ if there is nothing; never manufacture organisational news.
+ - `political-agenda` — the upcoming meetings and their agenda points, per "The
+ agendas" above. Omit when no agenda arrived.
+ - `political-todo`, titled **"Political todos"** — the tasks those agendas
+ actually ask for, one line each with the quote they came from. Omit entirely
+ when nothing is asked.
+ - `political-watch-later` — new episodes, per "Watch later" above. Omit when
+ there are none.
+ You may add further windows beyond these when an item needs its own space (a
+ long piece of analysis, the financial indicators as their own `kind: "list"`
+ with each line stating the move *and* what it means for working people). Do not
+ drop or rename the four question windows to make room.
+- At `detail_level: "compact"` keep the globe and fold the questions into a
+ single `kind: "list"` window — one or two lines each for questions 1 and 2, one
+ line each for 3 and 4 if they have anything. Sources still attach; they cost no
+ space when folded. **Keep `political-todo` as its own window even here**: it is
+ the one part of this section a person acts on rather than reads, and it is the
+ first thing they will look for on a phone.
- `narration` is spoken aloud by a TTS voice, so no markdown, no URLs, no emoji.
- If you cannot produce valid JSON matching this schema, output a single
`kind: "text"` window with your best-effort plain-text summary instead.
diff --git a/digest-engine/whatsapp-bridge/index.js b/digest-engine/whatsapp-bridge/index.js
index b1fcecd..7de8e10 100644
--- a/digest-engine/whatsapp-bridge/index.js
+++ b/digest-engine/whatsapp-bridge/index.js
@@ -17,14 +17,21 @@
const fs = require('fs');
const path = require('path');
+const crypto = require('crypto');
const qrcode = require('qrcode-terminal');
const { Client, LocalAuth } = require('whatsapp-web.js');
const DATA_DIR = process.env.WHATSAPP_DATA_DIR || '/data';
const MESSAGES_PATH = path.join(DATA_DIR, 'messages.jsonl');
const AUTH_PATH = path.join(DATA_DIR, '.wwebjs_auth');
+// Document attachments land here. digest-engine mounts the parent of this bridge's
+// /data, so what is /data/documents here is /data/whatsapp-bridge/documents there —
+// see ingest/whatsapp_ingest.py, which resolves exactly that.
+const DOCUMENTS_DIR = path.join(DATA_DIR, 'documents');
+const MAX_DOCUMENT_BYTES = Number(process.env.WHATSAPP_MAX_DOCUMENT_BYTES || 10 * 1024 * 1024);
fs.mkdirSync(DATA_DIR, { recursive: true });
+fs.mkdirSync(DOCUMENTS_DIR, { recursive: true });
const client = new Client({
authStrategy: new LocalAuth({ dataPath: AUTH_PATH }),
@@ -90,8 +97,46 @@ client.on('message', async (message) => {
is_group: chat ? Boolean(chat.isGroup) : false,
timestamp: message.timestamp,
type: message.type,
- body: message.body || ''
+ body: message.body || '',
+ has_media: Boolean(message.hasMedia),
+ // whatsapp-web.js exposes the document's own filename on the raw payload. This
+ // is not part of its documented API and may simply be undefined on some
+ // message types or library versions — in which case the digest still gets the
+ // caption and the type, and agenda matching falls back to those.
+ filename: (message._data && message._data.filename) || null,
+ mimetype: (message._data && message._data.mimetype) || null,
+ document_file: null
};
+
+ // DOCUMENTS ONLY, and only documents: a meeting agenda arrives as a PDF, and
+ // digest-engine reads its text to pull out the agenda points. Photos, video and
+ // audio are never downloaded — they are the bulk of what a group chat carries,
+ // this container has no use for them, and every download is one more request
+ // through a session that is already the highest-risk part of this project.
+ if (message.hasMedia && message.type === 'document') {
+ try {
+ const media = await message.downloadMedia();
+ const size = media && media.data ? Buffer.byteLength(media.data, 'base64') : 0;
+ if (!media || !media.data) {
+ console.error('A document had no downloadable data; recording it by name only.');
+ } else if (size > MAX_DOCUMENT_BYTES) {
+ console.error(`Document is ${size} bytes, over the ${MAX_DOCUMENT_BYTES} cap; recording it by name only.`);
+ } else {
+ // Sanitised basename plus a hash, so a hostile or merely awkward filename
+ // ("../../etc/passwd", or the fourth "Tagesordnung.pdf" this month) can
+ // neither escape this directory nor overwrite an earlier file.
+ const raw = media.filename || record.filename || `${record.id || Date.now()}.bin`;
+ const safe = path.basename(raw).replace(/[^A-Za-z0-9._-]+/g, '_').slice(0, 80) || 'document.bin';
+ const stamp = crypto.createHash('sha256').update(String(record.id || raw)).digest('hex').slice(0, 8);
+ const target = path.join(DOCUMENTS_DIR, `${stamp}-${safe}`);
+ fs.writeFileSync(target, Buffer.from(media.data, 'base64'));
+ record.document_file = path.basename(target);
+ }
+ } catch (err) {
+ // A failed download costs the document's text, not the message.
+ console.error(`Could not save a document: ${err}`);
+ }
+ }
// appendFileSync opens/appends/closes per message, so whatsapp_ingest.py can
// rename the file out from under us mid-run without losing a partial write.
fs.appendFileSync(MESSAGES_PATH, JSON.stringify(record) + '\n', 'utf8');
diff --git a/docs/project-plan.md b/docs/project-plan.md
index e5860b6..7401e5b 100644
--- a/docs/project-plan.md
+++ b/docs/project-plan.md
@@ -184,7 +184,7 @@ real hardware" callouts for everything downstream of this.)*
| Thin-client browser | **Firefox (kiosk)** | General browsing + the rendering surface for the LLM-generated digest canvas |
| Thin-client voice | **wyoming-satellite** + **openWakeWord** | Local wake-word spotting, streams to the existing Phase 3 Wyoming faster-whisper/Piper Assist pipeline — no new STT/TTS infrastructure |
| Digest scheduling | **systemd timer** | 4x/day cadence, same `OnCalendar` pattern as the existing restic backup timer |
-| Digest engine | **digest-engine** (custom Python) | Ingests mail/messages/news/financial data, calls the Phase 3 Ollama host, renders the personal/political/household digest sections |
+| Digest engine | **digest-engine** (custom Python) | Ingests mail/messages/news/financial data, calls the Phase 3 Ollama host, renders the personal/political/household/network digest sections, generating only the ones the household ticked in `identity` |
| Digest static serving | **digest-web** (nginx:alpine/Caddy) | Serves the rendered digest artifact read-only to both the thin client and an HA iframe card |
| Digest ingestion — Signal | **signal-cli** | Linked-device (JSON-RPC) read access to Signal messages |
| Digest ingestion — Telegram | **Telethon** | MTProto client logged in as the real account — the Bot API can't read personal DMs |
@@ -478,7 +478,7 @@ recurrence/TLS traps. Nextcloud itself is pre-existing; nothing in this repo dep
8. **Voice interactivity**:
- Only the rooms with a chosen thin client *and* an attached mic run `wyoming-satellite` (openWakeWord), streaming to the existing Phase 3 Wyoming faster-whisper/Piper pipeline — no new STT/TTS infrastructure.
- `thinclient-agent` accepts an Assist-resolved "play my digest" intent: switches the dedicated Firefox workspace into the "full/thorough" canvas view (vs. the HA dashboard's "compact" view) and narrates via the existing Piper TTS output.
- - **Room/person routing**: reuse Phase 2's presence system (`person.*`/area entities). If exactly one recognized person is in the room where the wake word fired, play that person's personal digest section. If more than one is present, Assist asks **"whose digest?"** and disambiguates by spoken name before playing the personal section — never guesses. Political/household sections always play regardless of presence ambiguity.
+ - **Room/person routing — voice-activated, automatically recognized, never asked**: the digest canvas is shown only when somebody asks for it (a spoken "play my digest", or the HA button). **Nothing displays it because a person walked past a screen**, and nothing here polls presence to decide to show something. When the wake word fires, HA calls `identity`'s `GET /speaker?area=`, which resolves who is asking from the two signals it already fuses: an IRK-resolved BLE identifier in that area, and a Frigate face sighting inside `FACE_PRESENCE_WINDOW_SECONDS`. One person in the room is them; several, and the most recent camera sighting decides. That answer becomes the `person` parameter the thin client passes to the canvas, which then shows exactly that person's chosen sections (Phase 12 step 5). **An unresolved answer means show less, not ask**: the canvas renders every section except the personal one, which is the same rule as before — automating the recognition is only acceptable because the unresolved case still fails closed. This replaces the earlier "Assist asks *whose digest?*" disambiguation: the household wanted recognition, not an interrogation. There is no speaker identification in this stack and `/speaker` does not pretend otherwise — it identifies who is in the room, not whose voice it was.
9. Network placement: plain trusted LAN for now (no VLAN precedent exists for a general client device class yet — only the unbuilt camera-VLAN concept). Revisitable later as a Phase-10-style expansion item, not a blocker now.
10. **Validate the image boots to a working kiosk session (Sway, local mpv/Spotify playback) with Mosquitto/HA/container-host powered off** — must not hang waiting on the network, same "reactive path never depends on a remote service" philosophy applied to the thin client's own boot path.
@@ -494,10 +494,14 @@ recurrence/TLS traps. Nextcloud itself is pre-existing; nothing in this repo dep
- **WhatsApp** — no officially-sanctioned API option exists. Rather than a protocol-reimplementation library (Baileys), run a small **`whatsapp-bridge`** sidecar (Node.js, `digest-engine/whatsapp-bridge/`): a real Chromium logged into the actual web.whatsapp.com client via **whatsapp-web.js** (Puppeteer), inside its own container running **Xvfb** so Chromium executes **headful** (not `headless: true`) — WhatsApp's automation detection specifically fingerprints headless Chrome, so a virtual-display "real browser" session is meaningfully lower-risk than either Baileys or true-headless whatsapp-web.js, though not zero-risk (it's still automated use of a personal account). One-time interactive QR-code login persists a session directory (mounted volume) so subsequent runs don't need re-scanning. The bridge exposes incoming messages over a local-only channel (e.g. a Unix socket or a small internal HTTP endpoint on the compose network, never published to the LAN) that `digest-engine/ingest/whatsapp_ingest.py` reads each run. Still gate behind `ENABLE_WHATSAPP_INGEST="false"`, off by default, with a warning in script output + `digest-engine/README.md`; recommend a secondary/non-critical number if enabled. Build this one last.
- **News** — `feedparser` over a curated OPML list (`digest-engine/feeds/curated-feeds.opml`), seeded with the confirmed `https://www.marxist.com/feed/rss` plus a mainstream-outlet list (exact outlets: see open decisions).
- **Financial** — FRED API (macro/unemployment, e.g. `UNRATE`) + Stooq keyless CSV (stocks/oil/commodities, preferred over Alpha Vantage's tight free-tier cap).
-5. LLM synthesis: assemble the run's ingested content into context, call the existing Phase 3 Ollama host with three separate prompt templates (`digest-engine/synth/prompts/{personal,political,household}.md`):
+5. LLM synthesis: assemble the run's ingested content into context, call the existing Phase 3 Ollama host with one prompt template per section (`digest-engine/synth/prompts/{personal,political,household,network}.md`):
- **Personal** — from personal-flagged mail/messages.
- **Political** — Marxist/working-class analytical framing (marxist.com feed as theoretical basis) synthesizing mainstream news + financial indicators + politically-flagged mail, laid out on the "holo globe" with colored/glowing markers (e.g. revolutionary-situation markers in red with a hammer-and-sickle/star motif).
- **Household/calendar** — from the existing Nextcloud CalDAV integration (Phase 8) and Grocy state (Phase 7).
+ - **Network** — the OPNsense/Suricata intrusion-detection summary, split out of the household section (implementation note, later than the original plan) so the two can be wanted separately.
+ - **The archive** (implementation note): `digest-engine/archive.py` keeps every ingested item and every measured number in SQLite across runs, so a run can say "up from 5.1% in June", "this IDS signature has fired every night this week", or "this story first appeared on Monday". Items are deduplicated on a fingerprint so `first_seen_at` means something; numbers are stored one row per measurement so a trend is a query. History enters each prompt as its own timestamped block, and it is what lifts the "one snapshot is not a trend" prohibition — but only with the figures and dates attached. It also persists mail and messages for its retention window, which `DIGEST_ARCHIVE_EXCLUDE_SOURCES` exists to bound.
+ - **Meeting agendas** (implementation note): `digest-engine/agenda.py` matches a Tagesordnung PDF arriving by mail or WhatsApp to the calendar event it belongs to (date in the filename/subject/heading, then wording in common, each labelled with its confidence), reads it with pypdf, and extracts its numbered points mechanically. The household section lists the points and derives a todo window from what the document actually asks for; the political section gets the points as a relevance filter — a story touching Thursday's agenda outranks a bigger one that doesn't. A scanned agenda extracts nothing and is named but never characterised; there is no OCR here.
+ - **Per-person section toggles** (implementation note): each person picks their own sections in `identity`'s admin panel (`people.digest_sections`, `GET /digest-preferences`). A run generates the **union** of what the household asked for — a section nobody wants costs neither an LLM call nor its sources' ingestion — and each surface filters to the person Home Assistant resolved. That last half is a display filter, not an access control: `digest-web` serves the whole artifact read-only to the LAN. An unreachable `identity` means "generate everything", never "generate nothing".
- A **detail-level** parameter (`compact` for the HA iframe, `full` for the thin-client fullscreen view) makes the thin-client rendering genuinely more thorough without needing two independent generation passes.
6. Rendering: vendor the offline **digest-canvas SDK** under `digest-engine/render/digest-canvas-sdk/` (globe + `addMarker()`, window/panel chrome, glow/holo CSS utility, no CDN dependency). Each run's LLM job is to call into this SDK with structured content, not hand-roll projection math. Use a custom inline SVG or Unicode ☭ (U+262D, explicit font-fallback + CSS glow) for hammer-and-sickle iconography since Nerd Fonts has no such glyph.
7. **Live follow-up voice Q&A**: persist each run's actually-used ingested-context bundle (not full raw content) as `digest-engine/output//context.json`. Expose a small HA tool (`digest_followup_query`) so a spoken follow-up ("tell me more about the unemployment numbers") feeds the cached context + question back into Ollama for a grounded, low-latency answer — no fresh ingestion pass. The answer can push a new small window/card onto the already-open thin-client canvas via a websocket, keeping the "flexible windows" idea alive live, not just at generation time.
diff --git a/hosts/thin-client/README.md b/hosts/thin-client/README.md
index aa7c68f..60b859c 100644
--- a/hosts/thin-client/README.md
+++ b/hosts/thin-client/README.md
@@ -311,6 +311,9 @@ integration you should get one device per thin client with:
plugged in (dynamic, re-scanned periodically — see "Capture-card /
receiver-box viewing" above); switches to `5:capture` and shows the picked one
full-screen via mpv.
+- **Display** (switch) — the TV's own power, over HDMI-CEC with a Sway DPMS
+ fallback. Meant to be driven by room presence; see "Turning the TV off when the
+ room is empty" below.
- **Workspace** (select) — `1:web` / `2:digest` / `3:media` / `4:admin` / `5:capture`
- **Launch Firefox**, **Launch web browser**, **Launch Steam Link** (buttons)
- **Playback state** (sensor, with track metadata as attributes), **Volume** (number),
@@ -376,6 +379,77 @@ never the payload — that reaches `capture-view` as an argv element. An unknown
or since-unplugged selection resolves to "no source," never to acting on
whatever string HA sent.
+## Turning the TV off when the room is empty
+
+A wall-mounted TV showing a canvas to an empty room is the largest power draw
+this machine is attached to — 60–150 W of lit panel against the thin client's own
+handful of watts. The **Display** switch turns it off on demand, and the intended
+driver is room presence.
+
+**How the agent does it** (`display_power.py`): HDMI-CEC first, over the same
+cable that carries the picture — `cec-ctl --to 0 --standby` to sleep the panel,
+`--image-view-on` plus `--active-source` to wake it and claim the input back. No
+network path to the TV, no pairing, no account, and it keeps working with the LAN
+down. Then, always, `swaymsg output power off`, which stops the compositor
+driving pixels — that is the fallback for a set whose CEC is broken or switched
+off, and belt-and-braces on one where it works.
+
+`cec-ctl` comes from `v4l-utils`, already in the image's package list. Most TVs
+ship CEC **disabled**; enable it once in the TV's settings, where it will be called
+HDMI-CEC, Bravia Sync, Anynet+, SimpLink, Viera Link or similar. `CEC_DEVICE`,
+`DISPLAY_OUTPUTS` and `DISPLAY_USE_CEC` in the agent's config cover the machine
+with two adapters, several screens, or a panel whose CEC you want left alone.
+
+**"Off" means standby, honestly.** A TV in CEC standby still draws roughly half a
+watt — that is what lets it hear the wake. This turns 60–150 W into ~0.5 W; it is
+not a smart plug and does not claim to be.
+
+**The presence decision stays in Home Assistant**, where presence already lives —
+this agent only does what it is told, same as every other entity here. A worked
+example, unverified against a running HA like every other HA snippet in this repo:
+
+```yaml
+# automations.yaml (excerpt). Replace the entity ids with your own.
+- alias: "Living room TV on when the room is occupied"
+ trigger:
+ - platform: state
+ entity_id: binary_sensor.living_room_occupancy
+ to: "on"
+ action:
+ - service: switch.turn_on
+ target:
+ entity_id: switch.thinclient_living_room_display
+
+- alias: "Living room TV off when the room empties"
+ trigger:
+ - platform: state
+ entity_id: binary_sensor.living_room_occupancy
+ to: "off"
+ # Long enough that walking to the kitchen for a glass of water does not
+ # cycle the panel. A TV that flickers off behind you is worse than one
+ # left on, and CEC wake takes a second or two.
+ for: "00:05:00"
+ condition:
+ # Don't black out a film. media_player state comes from the agent's own
+ # playback sensor — see "Home Assistant entities" above.
+ - condition: not
+ conditions:
+ - condition: state
+ entity_id: sensor.thinclient_living_room_playback_state
+ state: "playing"
+ action:
+ - service: switch.turn_off
+ target:
+ entity_id: switch.thinclient_living_room_display
+```
+
+**An Android TV with no thin client attached** is a Home Assistant question
+rather than one for this repo: pair it with the **Android TV Remote** integration
+and swap the `switch.turn_on`/`turn_off` calls above for
+`media_player.turn_on`/`turn_off` on that entity. Waking one over the network
+needs the TV's own "network standby"/"wake on cast" setting enabled — off by
+default on most sets, and the reason a TV that sleeps fine refuses to wake.
+
## Manual verification still outstanding
None of this has been run on hardware. In rough order:
@@ -509,3 +583,12 @@ None of this has been run on hardware. In rough order:
back up from being powered off, and whether the overlay's text is legible
against a bright/high-contrast real photo rather than the dark backgrounds
assumed while choosing the text-shadow-only styling in `eww.scss`.
+22. **Display power over CEC has never been run against a real TV.** The command
+ shapes are from `cec-ctl`'s own documentation, not from a session with a
+ panel on the other end. Three things to check on the first set it meets:
+ that CEC standby actually darkens it rather than just blanking the picture
+ (compare the mains draw, not the screen); that waking it comes back to *this*
+ input rather than to whatever it was on before; and that the Sway DPMS half
+ does not leave a "no signal" banner glowing on a set that ignored the CEC
+ standby. `DISPLAY_USE_CEC=false` is the escape hatch if a TV reacts badly to
+ being addressed at all.
diff --git a/hosts/thin-client/agent/thinclient_agent/display_power.py b/hosts/thin-client/agent/thinclient_agent/display_power.py
new file mode 100644
index 0000000..c535871
--- /dev/null
+++ b/hosts/thin-client/agent/thinclient_agent/display_power.py
@@ -0,0 +1,136 @@
+"""Turning the attached TV on and off, so an empty room does not power a panel.
+
+A wall-mounted Android TV driven by one of these thin clients draws 60-150 W while
+it shows a canvas nobody is in the room to look at. This module is what Home
+Assistant calls when presence says the room is occupied or empty — the decision
+lives in HA (an area's occupancy, the same presence system everything else here
+uses), and the doing lives here.
+
+TWO MECHANISMS, IN THIS ORDER
+-----------------------------
+1. **HDMI-CEC** (`cec-ctl`, from v4l-utils). The thin client is the HDMI *source*,
+ so it can put the display into standby and wake it again over the HDMI cable
+ itself. That is the one that actually saves the panel's power, and it needs no
+ network path to the TV, no pairing, no credentials, and no account — it keeps
+ working with the LAN down, which is this project's whole posture. Android TV
+ and Google TV sets implement CEC as "HDMI-CEC", "Bravia Sync", "Anynet+",
+ "Simplink" and a dozen other brand names for the same standard; it usually has
+ to be enabled in the TV's settings once.
+2. **Sway DPMS** (`swaymsg output power on|off`) as the fallback, and as a
+ belt-and-braces companion: it stops the compositor driving pixels and drops the
+ HDMI signal, which most panels treat as "go to sleep" on their own. It always
+ works because it needs nothing but the compositor already running here — but on
+ its own it may leave a TV showing a "no signal" banner rather than sleeping,
+ which is why CEC is tried first.
+
+Both are attempted on every call unless CEC is switched off, because they fail in
+different ways and neither reports reliably.
+
+WHAT "OFF" HONESTLY MEANS
+-------------------------
+Standby, not disconnected. A TV in CEC standby still draws roughly half a watt to
+keep listening on the HDMI line — that is what makes waking it possible at all.
+This turns 60-150 W of lit panel into ~0.5 W of standby; it is not a smart plug
+and does not pretend to be. If a set is one of the ones that ignores CEC standby
+entirely, you will see it immediately (the panel stays lit) — that is what the
+verification note in hosts/thin-client/README.md is for.
+
+SECURITY POSTURE, UNCHANGED
+---------------------------
+This is another enumerated MQTT command, exactly like the workspace switch and the
+canvas buttons: HA -> MQTT -> a fixed action here. A payload never becomes an argv
+element — `set_power()` takes a boolean, and the device names come from local
+configuration, never from the message. See mqtt_discovery.py's module docstring.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+import shutil
+import subprocess
+
+log = logging.getLogger(__name__)
+
+CEC_TIMEOUT_SECONDS = 10
+
+
+class DisplayPower:
+ def __init__(self, sway, cec_device: str | None = None, outputs: str = "*",
+ use_cec: bool = True):
+ self.sway = sway
+ # The CEC adapter, e.g. /dev/cec0. Most systems have exactly one and cec-ctl
+ # finds it on its own; this is for the machine that has two.
+ self.cec_device = cec_device or os.environ.get("CEC_DEVICE", "")
+ # Which Sway outputs to power down. "*" is every output, which is right for a
+ # thin client driving one TV; name an output (e.g. "HDMI-A-1") on a machine
+ # where only one of several screens is the TV.
+ self.outputs = outputs or "*"
+ self.use_cec = use_cec
+ self.state = True
+
+ # --- CEC ----------------------------------------------------------------
+ def _cec(self, *args: str) -> bool:
+ binary = shutil.which("cec-ctl")
+ if not binary:
+ log.info("cec-ctl is not installed; falling back to DPMS only")
+ return False
+
+ command = [binary]
+ if self.cec_device:
+ command += ["-d", self.cec_device]
+ command += list(args)
+
+ try:
+ result = subprocess.run(
+ command, capture_output=True, text=True, timeout=CEC_TIMEOUT_SECONDS
+ )
+ except (OSError, subprocess.SubprocessError) as exc:
+ log.warning("cec-ctl %s failed: %s", " ".join(args), exc)
+ return False
+
+ if result.returncode != 0:
+ log.warning("cec-ctl %s: %s", " ".join(args), (result.stderr or "").strip())
+ return False
+ return True
+
+ # --- the one public action ---------------------------------------------
+ def set_power(self, on: bool) -> bool:
+ """Turn the display on or off. Returns the state it believes it left it in.
+
+ Deliberately not idempotent-by-early-return: HA asking for "on" when this
+ object already thinks it is on must still send the wake, because the TV may
+ have been turned off with its own remote and nothing here would know. The
+ state field is for reporting, never for skipping work.
+ """
+ log.info("display: turning the panel %s", "on" if on else "off")
+
+ if self.use_cec:
+ # --to 0 addresses the TV specifically (logical address 0) rather than
+ # broadcasting, so a soundbar or receiver on the same bus is left alone.
+ if on:
+ self._cec("--to", "0", "--image-view-on")
+ # Ask to become the active source too: waking a TV that then shows a
+ # different input is the same as not waking it.
+ self._cec("--to", "0", "--active-source", "phys-addr=0.0.0.0")
+ else:
+ self._cec("--to", "0", "--standby")
+
+ # Always also drive the compositor: on a set that ignores CEC this is what
+ # stops it displaying, and on one that honours CEC it stops the thin client
+ # rendering to a panel nobody is looking at.
+ self.sway.swaymsg("output", self.outputs, "power", "on" if on else "off")
+
+ self.state = on
+ return self.state
+
+ def handle_command(self, payload: str) -> bool:
+ """MQTT payload -> action. Anything that isn't a known ON/OFF word is ignored
+ rather than guessed at, per the enumerated-command rule."""
+ value = (payload or "").strip().upper()
+ if value in ("ON", "TRUE", "1"):
+ return self.set_power(True)
+ if value in ("OFF", "FALSE", "0"):
+ return self.set_power(False)
+ log.warning("display: ignoring unknown power payload %r", payload)
+ return self.state
diff --git a/hosts/thin-client/agent/thinclient_agent/main.py b/hosts/thin-client/agent/thinclient_agent/main.py
index 1400cff..6d4087d 100644
--- a/hosts/thin-client/agent/thinclient_agent/main.py
+++ b/hosts/thin-client/agent/thinclient_agent/main.py
@@ -18,6 +18,7 @@ from .admin_canvas import AdminCanvas
from .audio_control import AudioControl
from .capture_control import CaptureControl, find_audio_card
from .digest_canvas import DETAIL_LEVELS, DigestCanvas
+from .display_power import DisplayPower
from .input_control import InputControl
from .mpris_bridge import MprisBridge
from .mqtt_discovery import Discovery
@@ -158,6 +159,15 @@ def main() -> int:
sway = SwayControl()
canvas = DigestCanvas(sway, config.get("DIGEST_WEB_URL", ""))
admin_canvas = AdminCanvas(sway, config.get("ADMIN_WEB_URL", ""))
+ # The TV's own power. DISPLAY_OUTPUTS names which Sway output(s) are the TV
+ # ("*" on a machine driving one screen); DISPLAY_USE_CEC=false drops to DPMS
+ # only, for a panel whose CEC is broken or deliberately disabled.
+ display = DisplayPower(
+ sway,
+ cec_device=config.get("CEC_DEVICE", ""),
+ outputs=config.get("DISPLAY_OUTPUTS", "*"),
+ use_cec=str(config.get("DISPLAY_USE_CEC", "true")).strip().lower() == "true",
+ )
apps = build_apps(config)
audio = AudioControl(sway.session_env)
capture = CaptureControl()
@@ -216,6 +226,13 @@ def main() -> int:
admin_canvas.show()
discovery.publish_workspace(WS_ADMIN)
+ def on_display_power(payload: str) -> None:
+ # Driven by an HA automation on room occupancy (see hosts/thin-client's
+ # README): a TV showing a canvas to an empty room is the single largest
+ # power draw this machine is attached to. The decision stays in HA, where
+ # presence already lives; this only does what it is told.
+ discovery.publish_display_power(display.handle_command(payload))
+
def on_audio_output(payload: str) -> None:
discovery.publish_audio_output(audio.select(payload))
@@ -264,6 +281,7 @@ def main() -> int:
discovery.register_media_player(mpris.handle_command, mpris.set_volume)
discovery.register_digest(on_show_digest, on_detail_level, DETAIL_LEVELS, canvas.detail_level)
discovery.register_admin_canvas(on_show_admin_canvas)
+ discovery.register_display_power(on_display_power, display.state)
discovery.register_app_launchers(apps, on_launch)
discovery.register_workspace_select(WORKSPACES, on_workspace, WS_DIGEST)
# audio.apply_preferred() already ran once at startup (before MQTT was even
diff --git a/hosts/thin-client/agent/thinclient_agent/mqtt_discovery.py b/hosts/thin-client/agent/thinclient_agent/mqtt_discovery.py
index 62961b1..30fa288 100644
--- a/hosts/thin-client/agent/thinclient_agent/mqtt_discovery.py
+++ b/hosts/thin-client/agent/thinclient_agent/mqtt_discovery.py
@@ -41,6 +41,7 @@ class Discovery:
self.capture_source_state_topic = f"{self.base}/capture/source/state"
self.remote_target_state_topic = f"{self.base}/remote/target/state"
self.input_text_state_topic = f"{self.base}/input/text/state"
+ self.display_power_state_topic = f"{self.base}/display/power/state"
self._handlers: dict[str, Callable[[str], None]] = {}
self.device = {
@@ -211,6 +212,35 @@ class Discovery:
},
)
+ def register_display_power(self, on_set, current: bool) -> None:
+ """The TV's own power, as a switch HA can drive from room presence.
+
+ A switch rather than a button, because the interesting automation is "this
+ area became unoccupied" -> off, "somebody walked in" -> on, and that needs a
+ state HA can read back as well as set. The state is what this agent last
+ did, not what the panel reports: CEC gives no reliable read-back, and a
+ state that lies about the TV having been turned off by its own remote is
+ better than one that blocks the next wake — see display_power.set_power().
+ """
+ self._publish_config(
+ "switch",
+ "display_power",
+ {
+ "name": "Display",
+ "command_topic": self._command_topic("display/power/set", on_set),
+ "state_topic": self.display_power_state_topic,
+ "payload_on": "ON",
+ "payload_off": "OFF",
+ "icon": "mdi:television",
+ },
+ )
+ self.publish_display_power(current)
+
+ def publish_display_power(self, on: bool) -> None:
+ self.client.publish(
+ self.display_power_state_topic, "ON" if on else "OFF", qos=1, retain=True
+ )
+
def register_app_launchers(self, apps, on_launch) -> None:
for key, app in apps.items():
self._publish_config(
diff --git a/identity/README.md b/identity/README.md
index 9e2e695..5ec8286 100644
--- a/identity/README.md
+++ b/identity/README.md
@@ -166,7 +166,8 @@ http://:8098/admin.html?api=http://:8097&token=
```
Four tabs: **People** (tap anyone to edit every field, their devices, their door
-rights and their chores), **Prune**, **History**, and **Access log**.
+rights, their chores and which digests get generated for them), **Prune**,
+**History**, and **Access log**.
> The token is in the URL, exactly like the two kiosk pages — that's the existing
> pattern here, not a new decision, and it's why this service treats the token as the
@@ -393,6 +394,69 @@ where "how do I relate to this specific household member" facts belong, not dupl
into `chores/`'s own database. Assignments get their own table only because they're
many-per-person, not because they belong anywhere else.
+## "Who just spoke?" — automatic recognition for the voice path
+
+`GET /speaker?area=` answers who is asking, so a spoken "play my digest"
+shows *that person's* digest without anybody typing or spelling a name, and
+without the assistant interrogating the room.
+
+It resolves from the two presence signals this service already fuses:
+
+1. **One person in that area** — that's them.
+2. **Several** — the one a camera recognised most recently, if any did inside
+ `FACE_PRESENCE_WINDOW_SECONDS`. A face seen thirty seconds ago is the best
+ evidence available that a particular person is the one standing there talking.
+3. **Nobody in the area but exactly one person home** — them.
+4. **Otherwise `person` is null**, with the candidates named.
+
+**An unresolved answer means show less, not ask.** The caller's fallback is a
+digest with no personal section — never a prompt, never a guess. Automating the
+recognition is only defensible *because* the ambiguous case still fails closed;
+that is the same rule the registry applies to registration, applied to display.
+
+**Two things it is not.** It is not speaker identification — nothing here listens
+to a voice; it works out who is *in the room*, so two people in a kitchen where
+one was just recognised by a camera resolve to that one even if the other spoke.
+And it is not a display trigger: nothing in this project shows a digest because
+somebody walked past a screen. The canvas opens when it is asked for, and this
+endpoint only answers *by whom*.
+
+## Digest settings — owned here, used by `digest-engine/`
+
+Which of `digest-engine`'s four digests get generated for a person — **network**,
+**household**, **social** (its `personal` section: mail and messages) and **political /
+news** — is a per-person setting on this registry, editable in the admin panel's person
+editor. Same reasoning as the chore fields above: it is a standing fact about a
+household member, and `identity` is already this project's source of truth for those,
+so it lives as a column on `people` rather than in a second database over in
+`digest-engine`.
+
+`digest-engine` reads `GET /digest-preferences` once at the start of each of its four
+daily runs — the same shape-for-the-consumer pattern as `GET /chore-assignments` — and:
+
+- generates the **union** of what the household asked for. A section nobody has ticked
+ costs no LLM call, and `digest-engine` also skips ingesting the sources only that
+ section reads (turn the political one off for everybody and its news, financial and
+ flight/naval fetches stop happening at all).
+- filters each surface to the person Home Assistant resolved, using the per-person sets
+ this endpoint hands over.
+
+**The second half is a display filter, not an access control**, and it should not be
+described to anyone as privacy: `digest-web` serves the rendered digest read-only to
+anything on the LAN, so an unticked section is off somebody's screen and out of their
+narration, not out of their reach. The part that genuinely does not exist anywhere is
+the part that was never generated.
+
+Two deliberate defaults:
+
+- **Never set means all four.** A household that never opens this panel keeps exactly
+ the digest it had before the setting existed. An explicitly empty set is different and
+ is honoured as written — "generate nothing for me" is a real answer, and the column
+ stores `''` rather than `NULL` to keep the two distinguishable.
+- **This service being unreachable means all four too.** `digest-engine` treats any
+ failed lookup as "no preferences known" and generates everything, because one
+ container being down must never silently cost the household its whole digest.
+
## Camera face recognition — a second presence signal, never a registration one
If Tapo pan/tilt cameras are wired into Frigate as additional camera sources
@@ -495,7 +559,7 @@ not network placement.
| `POST /register` | `{"name", "device_id", "photo_id"?, "entity_id"?, "no_device"?}` -> registers, or returns a reason it couldn't (see above) |
| `POST /register/guest` | `{"device_id", "photo_id"?}` -> registers "Guest N", no name needed |
| `GET /people` | admin/audit list of every person: identifiers, device grants, chore assignments, `nickname`/`speak_name`, `last_visit_at`, `visit_count`, `currently_home_since` |
-| `POST /people/` | edit any editable field — `{"name"?, "nickname"?, "note"?, "chore_exempt"?, "chore_reminder_style"?, "notify_on_arrival"?, "announce_arrivals"?, "notify_topic"?, "clear_photo"?}`. Omitted keys are left alone |
+| `POST /people/` | edit any editable field — `{"name"?, "nickname"?, "note"?, "chore_exempt"?, "chore_reminder_style"?, "notify_on_arrival"?, "announce_arrivals"?, "notify_topic"?, "digest_sections"?, "clear_photo"?}`. Omitted keys are left alone |
| `POST /people//test-notification` | push a test message to this person's ntfy topic, to prove it works |
| `GET /people//photo` | the person's profile picture (raw JPEG) — their most recent registration photo |
| `POST /people//identifiers` | `{"entity_id"}` — attach an identifier by hand (a fixed BLE tag not in range yet). Still enforces `TRUSTED_ENTITY_PREFIXES` |
@@ -512,6 +576,9 @@ not network placement.
| `GET /device-access/events?limit=` | the audit log of every access check, allowed and denied |
| `GET`/`POST /people//chore-assignments` | read/replace this person's assigned chore types (`{"chore_types": [...]}`) |
| `GET /chore-assignments` | the same facts keyed by chore type — the shape `chores/` reads |
+| `GET`/`POST /people//digest-settings` | read/replace which digests are generated for this person (`{"digest_sections": ["network", "household", "personal", "political"]}`) |
+| `GET /digest-preferences` | every person's set plus `wanted`, the union — the shape `digest-engine/` reads |
+| `GET /speaker?area=` | who just spoke in that area — `{"person", "reason", "candidates"}`, `person` null when it can't tell (see above) |
| `GET /floorplan` | every level and its drawn rooms (polygons in normalised 0–1 coordinates) |
| `POST /floorplan/levels` | create or rename a level — `{"id"?, "name", "sort_order"?}` |
| `DELETE /floorplan/levels/` | remove a level and its rooms |
@@ -599,3 +666,11 @@ no way to send an `Authorization` header.
*outside* the `condition: template` guard would open the door regardless of what
this service answered. `identity` cannot enforce that from its side — it only ever
answers the question.
+15. **The digest-section list in `frontend/admin.js` is kept in step with `server.py`'s
+ `DIGEST_SECTIONS` by hand**, like `CHORE_TYPES` above it. Unlike `CHORE_TYPES` the
+ server does validate these, so drift shows up as a visible refusal rather than a
+ silent bad write — but the labels ("Social", "Political / news") are the panel's
+ own words and match nothing on the server, so renaming a section in
+ `digest-engine` needs all three places checked. `digest-engine`'s end of it (which
+ sections a run actually generates, and what a stopped `identity` does to a run) is
+ on that component's own verification list.
diff --git a/identity/frontend/admin.html b/identity/frontend/admin.html
index 0aaf8f5..40adc24 100644
--- a/identity/frontend/admin.html
+++ b/identity/frontend/admin.html
@@ -34,7 +34,7 @@
People
- Tap a person to edit every field, their devices, door rights and chores.
+ Tap a person to edit every field, their devices, door rights, chores and digests.
Loading…
@@ -223,6 +223,20 @@
+
+