400 lines
17 KiB
Python
400 lines
17 KiB
Python
"""Meeting agendas — attaching a "Tagesordnung" to the meeting it belongs to.
|
|
|
|
An agenda arrives as a PDF in mail or WhatsApp, days before the meeting it is
|
|
for. The meeting is in the calendar. Nothing connected the two, so the digest
|
|
would report "a message from Anna with an attachment" in one section and "branch
|
|
meeting, Thursday 19:00" in another, and leave the reader to notice.
|
|
|
|
This module makes that one fact: a calendar event that has an agenda says so,
|
|
naming the file, who sent it and where it arrived.
|
|
|
|
READING THE DOCUMENT
|
|
--------------------
|
|
The text is extracted (pypdf for PDFs, straight decode for plain text) and put in
|
|
the context, so the digest can list what is actually on the agenda, pull out what
|
|
the reader has been asked to prepare, and — in the political section — let this
|
|
week's agenda points steer which news is worth featuring. That last one is the
|
|
real payoff: "my branch is discussing rent controls on Thursday" is exactly the
|
|
filter that makes a news digest useful rather than merely comprehensive.
|
|
|
|
Extraction is best-effort and honest about failing. A scanned agenda is a page of
|
|
images and yields nothing; there is no OCR here and adding one would be a
|
|
different project. When nothing could be read, the document still attaches to its
|
|
meeting with `text_extracted: false`, and the prompts are told that a document
|
|
without text is one they may name but must not characterise.
|
|
|
|
The bytes never leave the household: the file is spooled to the digest's own
|
|
`/data` volume by the ingest modules, read here, and the extracted text goes only
|
|
into the local Ollama prompt like everything else.
|
|
|
|
HOW A MATCH IS MADE, IN ORDER
|
|
-----------------------------
|
|
1. **A date in the filename or subject** ("TO_12.08.pdf", "Tagesordnung 12.8.",
|
|
"agenda 2026-08-12") against an event starting that day. This is the strong
|
|
signal and it is tried first.
|
|
2. **Words in common** between the agenda's own text and an event's summary
|
|
("OG-Treffen" in both), among events in the lookahead window.
|
|
3. Both, which is the confident case and is labelled as such.
|
|
|
|
Anything that matches nothing is returned as an unattached agenda rather than
|
|
dropped — "an agenda arrived and I could not tell which meeting it is for" is
|
|
useful to a reader, and silently swallowing it would be the one outcome that
|
|
makes this feature worse than not having it.
|
|
|
|
WHY "TO" IS NOT MATCHED LIKE THE OTHER WORDS
|
|
--------------------------------------------
|
|
`TO` is the abbreviation everybody actually uses, and it is also the most common
|
|
two-letter word in English mail. It is therefore matched only as a standalone
|
|
uppercase token, and only in a filename or subject — never in body text. The
|
|
long words (Tagesordnung, Traktanden, agenda) are matched case-insensitively
|
|
anywhere, because they cannot collide with anything.
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
# Sources whose messages can carry an agenda. Telegram and Signal are here because a
|
|
# branch that uses them will send agendas there too; they contribute nothing unless
|
|
# their ingestion is enabled and actually carries attachment metadata.
|
|
MESSAGE_SOURCES = ("email", "whatsapp", "telegram", "signal", "discord")
|
|
|
|
DEFAULT_KEYWORDS = "tagesordnung,traktanden,traktandenliste,agenda,einladung"
|
|
|
|
# Standalone uppercase abbreviations, matched only in filenames and subjects.
|
|
ABBREVIATION_RE = re.compile(r"(?:^|[\s_\-\[(])((?:TO|TOP)\d*)(?:[\s_\-.\])]|$)")
|
|
|
|
DOCUMENT_TYPES = (".pdf", ".doc", ".docx", ".odt", ".rtf")
|
|
|
|
# 12.08.2026 / 12.8. / 2026-08-12 / 12-08-2026
|
|
DATE_PATTERNS = (
|
|
re.compile(r"(?P<year>20\d{2})[-_.](?P<month>\d{1,2})[-_.](?P<day>\d{1,2})"),
|
|
re.compile(r"(?P<day>\d{1,2})[-_.](?P<month>\d{1,2})[-_.](?P<year>20\d{2})"),
|
|
re.compile(r"(?P<day>\d{1,2})[-_.](?P<month>\d{1,2})\.?(?![\d.])"),
|
|
)
|
|
|
|
# Words too common to prove two strings are about the same meeting.
|
|
STOPWORDS = {
|
|
"der", "die", "das", "und", "für", "fur", "mit", "von", "zum", "zur", "des", "dem",
|
|
"the", "and", "for", "with", "our", "next", "meeting", "termin", "treffen",
|
|
"tagesordnung", "agenda", "einladung", "traktanden", "traktandenliste", "pdf",
|
|
}
|
|
|
|
MIN_TOKEN_LENGTH = 4
|
|
|
|
# How much of a document reaches the prompt. An agenda is a page or two; anything
|
|
# past this is a bundle of minutes and attachments that would crowd out the run's
|
|
# actual material.
|
|
MAX_TEXT_CHARS = 6000
|
|
MAX_PDF_PAGES = 12
|
|
|
|
# How much of a document counts as its heading for matching purposes — the block at
|
|
# the top where an agenda names its meeting and its date.
|
|
HEADING_CHARS = 300
|
|
|
|
# How much of it goes to the political section, which has to fit a world's worth of
|
|
# news around it. Enough for a page of agenda plus its preamble.
|
|
BRIEF_TEXT_CHARS = 3000
|
|
|
|
# Lines that look like agenda points: "1. Bericht", "TOP 3 — Kasse", "- Anträge".
|
|
POINT_RE = re.compile(r"^\s*(?:TOP\s*)?(?:\d{1,2}[.)]|[-•*])\s+(?P<text>\S.{2,160})$", re.IGNORECASE)
|
|
|
|
|
|
def _extract_text(path):
|
|
"""The document's text, or "" if it cannot be read. Never raises.
|
|
|
|
pypdf is imported here rather than at module scope so a missing or broken
|
|
optional dependency costs the agenda text and nothing else — the same rule
|
|
every ingestion module follows for its own client library.
|
|
"""
|
|
if not path:
|
|
return ""
|
|
file_path = Path(path)
|
|
if not file_path.is_file():
|
|
LOG.info("agenda: %s is not readable, keeping the document by name only", path)
|
|
return ""
|
|
|
|
suffix = file_path.suffix.lower()
|
|
try:
|
|
if suffix in (".txt", ".md"):
|
|
return file_path.read_text(encoding="utf-8", errors="replace")[:MAX_TEXT_CHARS]
|
|
if suffix != ".pdf":
|
|
# .docx/.odt are zip containers and .doc is a binary format; parsing them
|
|
# would mean another dependency for a case nobody in this household has
|
|
# hit yet. Named, not read — which the prompts handle.
|
|
return ""
|
|
|
|
from pypdf import PdfReader
|
|
|
|
reader = PdfReader(str(file_path))
|
|
pages = []
|
|
for page in reader.pages[:MAX_PDF_PAGES]:
|
|
pages.append(page.extract_text() or "")
|
|
return "\n".join(pages)[:MAX_TEXT_CHARS]
|
|
except Exception:
|
|
LOG.warning("agenda: could not extract text from %s", path, exc_info=True)
|
|
return ""
|
|
|
|
|
|
def _agenda_points(text):
|
|
"""The numbered/bulleted lines of an agenda, in order.
|
|
|
|
Extracted here rather than left to the model because these are the one part of
|
|
the document that has a reliable shape, and a list pulled out mechanically is a
|
|
list that cannot be invented. The model still gets the full text; this is what
|
|
it can quote from without having to find structure in a wall of PDF extraction.
|
|
"""
|
|
points = []
|
|
for line in (text or "").splitlines():
|
|
match = POINT_RE.match(line.strip())
|
|
if not match:
|
|
continue
|
|
point = " ".join(match.group("text").split())
|
|
if point and point not in points:
|
|
points.append(point)
|
|
return points[:30]
|
|
|
|
|
|
def _keywords():
|
|
raw = os.environ.get("AGENDA_KEYWORDS", DEFAULT_KEYWORDS)
|
|
return [word.strip().lower() for word in raw.split(",") if word.strip()]
|
|
|
|
|
|
def _looks_like_document(name, content_type):
|
|
name = (name or "").lower()
|
|
content_type = (content_type or "").lower()
|
|
if any(name.endswith(suffix) for suffix in DOCUMENT_TYPES):
|
|
return True
|
|
return "pdf" in content_type or "document" in content_type or "msword" in content_type
|
|
|
|
|
|
def _agenda_label(text, keywords):
|
|
"""Why this looked like an agenda, or None. Returned as a string because it ends
|
|
up in the digest: "matched on 'Tagesordnung'" is checkable by a human, "true" is
|
|
not."""
|
|
if not text:
|
|
return None
|
|
lowered = text.lower()
|
|
for word in keywords:
|
|
if word in lowered:
|
|
return word
|
|
match = ABBREVIATION_RE.search(text)
|
|
return match.group(1) if match else None
|
|
|
|
|
|
def _parse_date(text, reference_year):
|
|
"""First date-looking thing in `text`, as (year, month, day). A bare "12.8." takes
|
|
the year from the run itself — an agenda written without a year is always for the
|
|
near future, never for two years ago."""
|
|
if not text:
|
|
return None
|
|
for pattern in DATE_PATTERNS:
|
|
match = pattern.search(text)
|
|
if not match:
|
|
continue
|
|
groups = match.groupdict()
|
|
try:
|
|
day = int(groups["day"])
|
|
month = int(groups["month"])
|
|
year = int(groups.get("year") or reference_year)
|
|
except (TypeError, ValueError):
|
|
continue
|
|
if 1 <= month <= 12 and 1 <= day <= 31:
|
|
return year, month, day
|
|
return None
|
|
|
|
|
|
def _tokens(text):
|
|
words = re.split(r"[^0-9A-Za-zÄÖÜäöüß]+", (text or "").lower())
|
|
return {
|
|
word for word in words
|
|
if len(word) >= MIN_TOKEN_LENGTH and word not in STOPWORDS and not word.isdigit()
|
|
}
|
|
|
|
|
|
def _event_date(event):
|
|
raw = str(event.get("start") or "")
|
|
if not raw:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
# All-day events arrive as a bare date.
|
|
try:
|
|
parsed = datetime.strptime(raw[:10], "%Y-%m-%d")
|
|
except ValueError:
|
|
return None
|
|
return parsed.year, parsed.month, parsed.day
|
|
|
|
|
|
def _find_agendas(collected, keywords):
|
|
"""Every agenda-looking document in this run's messages, with the evidence."""
|
|
found = []
|
|
for source in MESSAGE_SOURCES:
|
|
for message in collected.get(source) or []:
|
|
if not isinstance(message, dict):
|
|
continue
|
|
attachments = [a for a in (message.get("attachments") or []) if isinstance(a, dict)]
|
|
documents = [
|
|
a for a in attachments
|
|
if _looks_like_document(a.get("filename"), a.get("content_type"))
|
|
]
|
|
if not documents:
|
|
continue
|
|
|
|
# The subject/caption counts as a label too: plenty of people send
|
|
# "Tagesordnung für Donnerstag" with the file named scan_0001.pdf.
|
|
heading = " ".join(str(message.get(key) or "") for key in ("subject", "body", "chat"))
|
|
for document in documents:
|
|
filename = str(document.get("filename") or "")
|
|
label = _agenda_label(filename, keywords)
|
|
matched_in = "filename"
|
|
if not label:
|
|
# The abbreviation is only trusted in a subject line, never in body
|
|
# prose — see the module docstring.
|
|
label = _agenda_label(str(message.get("subject") or ""), keywords)
|
|
matched_in = "subject"
|
|
if not label:
|
|
lowered = heading.lower()
|
|
label = next((word for word in keywords if word in lowered), None)
|
|
matched_in = "message text"
|
|
if not label:
|
|
continue
|
|
|
|
text = _extract_text(document.get("path"))
|
|
found.append({
|
|
"filename": filename or "(unnamed attachment)",
|
|
"content_type": document.get("content_type"),
|
|
"source": source,
|
|
"from": message.get("from"),
|
|
"chat": message.get("chat"),
|
|
"subject": message.get("subject"),
|
|
# The covering note. Kept because a good half of what a branch
|
|
# actually asks people to do is written here rather than in the
|
|
# numbered agenda — "bringt bitte eure Beiträge mit".
|
|
"body": str(message.get("body") or "")[:400],
|
|
"received_at": message.get("timestamp"),
|
|
"matched_keyword": label,
|
|
"matched_in": matched_in,
|
|
# False for a scanned agenda, an unsupported format, or a file
|
|
# the ingest could not spool. The prompts treat that case as
|
|
# "name it, do not characterise it".
|
|
"text_extracted": bool(text.strip()),
|
|
"points": _agenda_points(text),
|
|
"text": text.strip(),
|
|
})
|
|
return found
|
|
|
|
|
|
def attach(collected, reference_year=None):
|
|
"""Attaches agendas to calendar events in place and returns the unmatched ones.
|
|
|
|
Mutating the calendar entries is deliberate: the household section already
|
|
renders those events, so an event that carries its own `agenda_documents` needs
|
|
no second list to be cross-referenced against.
|
|
"""
|
|
events = [event for event in (collected.get("calendar") or []) if isinstance(event, dict)]
|
|
agendas = _find_agendas(collected, _keywords())
|
|
if not agendas:
|
|
return []
|
|
if not events:
|
|
LOG.info("agenda: %d agenda-looking document(s) but no calendar events to match", len(agendas))
|
|
return agendas
|
|
|
|
reference_year = reference_year or datetime.now(timezone.utc).year
|
|
unmatched = []
|
|
|
|
for agenda in agendas:
|
|
# The document's own heading — its first few lines, where an agenda puts the
|
|
# name of the meeting and its date. Only the heading: matching against the
|
|
# whole text would find a word in common with almost any event in the
|
|
# calendar and turn a confident match into a coincidence.
|
|
heading = (agenda.get("text") or "")[:HEADING_CHARS]
|
|
date = (
|
|
_parse_date(agenda["filename"], reference_year)
|
|
or _parse_date(agenda.get("subject"), reference_year)
|
|
or _parse_date(heading, reference_year)
|
|
)
|
|
agenda_tokens = _tokens(f"{agenda['filename']} {agenda.get('subject') or ''} {heading}")
|
|
|
|
best = None
|
|
for event in events:
|
|
same_day = date is not None and _event_date(event) == date
|
|
shared = agenda_tokens & _tokens(event.get("summary"))
|
|
if same_day and shared:
|
|
confidence, why = "high", f"same date and shared wording ({', '.join(sorted(shared))})"
|
|
elif same_day:
|
|
confidence, why = "medium", "the date in the file name matches this event"
|
|
elif shared:
|
|
confidence, why = "low", f"shared wording ({', '.join(sorted(shared))})"
|
|
else:
|
|
continue
|
|
rank = {"high": 3, "medium": 2, "low": 1}[confidence]
|
|
if best is None or rank > best[0]:
|
|
best = (rank, event, confidence, why)
|
|
|
|
if best is None:
|
|
unmatched.append(agenda)
|
|
continue
|
|
|
|
_, event, confidence, why = best
|
|
attached = dict(agenda)
|
|
attached["match_confidence"] = confidence
|
|
attached["match_reason"] = why
|
|
event.setdefault("agenda_documents", []).append(attached)
|
|
LOG.info(
|
|
"agenda: attached %r to %r (%s: %s)",
|
|
agenda["filename"], event.get("summary"), confidence, why,
|
|
)
|
|
|
|
if unmatched:
|
|
LOG.info("agenda: %d agenda(s) matched no event", len(unmatched))
|
|
return unmatched
|
|
|
|
|
|
def _brief(document, meeting, starts_at):
|
|
return {
|
|
"meeting": meeting,
|
|
"starts_at": starts_at,
|
|
"agenda_file": document.get("filename"),
|
|
"from": document.get("from"),
|
|
"received_at": document.get("received_at"),
|
|
# What was sent alongside the file. Half of what a branch actually asks
|
|
# people to do arrives in the covering message ("bringt eure Beiträge mit"),
|
|
# not in the numbered agenda itself.
|
|
"covering_message": " ".join(
|
|
part for part in (document.get("subject"), document.get("body")) if part
|
|
)[:400] or None,
|
|
"points": document.get("points") or [],
|
|
"text": (document.get("text") or "")[:BRIEF_TEXT_CHARS],
|
|
"text_extracted": document.get("text_extracted", False),
|
|
}
|
|
|
|
|
|
def upcoming(collected, unattached=None):
|
|
"""The agendas, for the political section.
|
|
|
|
This carries the document's actual text, not just its topics, because the
|
|
political section is where the agenda's content lives: it lists the points and
|
|
derives the reader's own to-do list from them. That placement is deliberate —
|
|
a branch agenda is party work, so it reaches only the people who asked for the
|
|
political digest, and a household member who did not is told a meeting has an
|
|
agenda without being shown what is on it.
|
|
|
|
The text is capped harder than the household section's copy: this prompt has a
|
|
world's worth of news to fit alongside it.
|
|
"""
|
|
briefs = []
|
|
for event in collected.get("calendar") or []:
|
|
if not isinstance(event, dict):
|
|
continue
|
|
for document in event.get("agenda_documents") or []:
|
|
briefs.append(_brief(document, event.get("summary"), event.get("start")))
|
|
for document in unattached or []:
|
|
briefs.append(_brief(document, None, None))
|
|
return briefs
|