"""WhatsApp ingestion — the Python-side consumer of the whatsapp-bridge sidecar. The bridge (digest-engine/whatsapp-bridge/) runs a real, headful Chromium logged into web.whatsapp.com and appends one JSON object per received message to a shared file on a Docker volume. This module drains that file each run. A file on a shared volume beats an HTTP call to the bridge here: a digest run fires on a systemd timer with no regard for whether the bridge container happens to be restarting, re-authenticating, or mid-Chromium-crash. Messages the bridge already wrote are still on disk and still get read; there is no liveness or retry/backoff coordination problem to solve. Draining is a rename-then-read, not a truncate-in-place: the bridge appends with an open-write-close per message, so renaming the file out from under it is atomic from the reader's side and the bridge simply recreates the original path on its next append. A read-then-truncate would silently drop anything written in between. This is opt-in and off by default — see the ban-risk warning in README.md. """ import json import logging import os from datetime import datetime, timedelta, timezone LOG = logging.getLogger(__name__) DEFAULT_MESSAGES_PATH = "/data/whatsapp-bridge/messages.jsonl" MAX_BODY_CHARS = 2000 def fetch(lookback_hours): messages_path = os.environ.get("WHATSAPP_MESSAGES_PATH", DEFAULT_MESSAGES_PATH) drained_path = messages_path + ".draining" if not os.path.exists(messages_path): LOG.warning( "whatsapp: no message file at %s (is whatsapp-bridge running and logged in?), skipping", messages_path, ) return [] try: os.replace(messages_path, drained_path) except OSError: LOG.warning("whatsapp: could not claim %s for reading, skipping", messages_path, exc_info=True) return [] since = datetime.now(timezone.utc) - timedelta(hours=lookback_hours) messages = [] try: with open(drained_path, "r", encoding="utf-8") as handle: for line in handle: line = line.strip() if not line: continue try: record = json.loads(line) timestamp = record.get("timestamp") sent_at = ( datetime.fromtimestamp(timestamp, tz=timezone.utc) if timestamp else None ) if sent_at and sent_at < since: continue messages.append( { "source": "whatsapp", "from": record.get("from_name") or record.get("from"), "chat": record.get("chat"), "timestamp": sent_at.isoformat() if sent_at else None, "body": (record.get("body") or "")[:MAX_BODY_CHARS], } ) except Exception: LOG.warning("whatsapp: could not parse a bridge line, skipping", exc_info=True) except Exception: LOG.warning("whatsapp: could not read %s, returning nothing", drained_path, exc_info=True) return [] finally: try: os.remove(drained_path) except OSError: LOG.warning("whatsapp: could not remove %s", drained_path, exc_info=True) LOG.info("whatsapp: %d message(s) in the last %sh", len(messages), lookback_hours) return messages