112 lines
4.9 KiB
Python
112 lines
4.9 KiB
Python
"""WhatsApp ingestion — the Python-side consumer of the whatsapp-bridge sidecar.
|
|
|
|
The bridge (digest-engine/whatsapp-bridge/) runs a real, headful Chromium logged
|
|
into web.whatsapp.com and appends one JSON object per received message to a
|
|
shared file on a Docker volume. This module drains that file each run.
|
|
|
|
A file on a shared volume beats an HTTP call to the bridge here: a digest run
|
|
fires on a systemd timer with no regard for whether the bridge container happens
|
|
to be restarting, re-authenticating, or mid-Chromium-crash. Messages the bridge
|
|
already wrote are still on disk and still get read; there is no liveness or
|
|
retry/backoff coordination problem to solve.
|
|
|
|
Draining is a rename-then-read, not a truncate-in-place: the bridge appends with
|
|
an open-write-close per message, so renaming the file out from under it is atomic
|
|
from the reader's side and the bridge simply recreates the original path on its
|
|
next append. A read-then-truncate would silently drop anything written in between.
|
|
|
|
This is opt-in and off by default — see the ban-risk warning in README.md.
|
|
"""
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
DEFAULT_MESSAGES_PATH = "/data/whatsapp-bridge/messages.jsonl"
|
|
MAX_BODY_CHARS = 2000
|
|
|
|
|
|
def fetch(lookback_hours):
|
|
messages_path = os.environ.get("WHATSAPP_MESSAGES_PATH", DEFAULT_MESSAGES_PATH)
|
|
drained_path = messages_path + ".draining"
|
|
|
|
if not os.path.exists(messages_path):
|
|
LOG.warning(
|
|
"whatsapp: no message file at %s (is whatsapp-bridge running and logged in?), skipping",
|
|
messages_path,
|
|
)
|
|
return []
|
|
|
|
try:
|
|
os.replace(messages_path, drained_path)
|
|
except OSError:
|
|
LOG.warning("whatsapp: could not claim %s for reading, skipping", messages_path, exc_info=True)
|
|
return []
|
|
|
|
# The bridge writes documents next to its message file, in its own /data. Derived
|
|
# from the message path rather than configured separately, so the two can never
|
|
# be pointed at different places by half-updating the environment.
|
|
documents_dir = Path(messages_path).parent / "documents"
|
|
|
|
since = datetime.now(timezone.utc) - timedelta(hours=lookback_hours)
|
|
messages = []
|
|
|
|
try:
|
|
with open(drained_path, "r", encoding="utf-8") as handle:
|
|
for line in handle:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
record = json.loads(line)
|
|
timestamp = record.get("timestamp")
|
|
sent_at = (
|
|
datetime.fromtimestamp(timestamp, tz=timezone.utc) if timestamp else None
|
|
)
|
|
if sent_at and sent_at < since:
|
|
continue
|
|
messages.append(
|
|
{
|
|
"source": "whatsapp",
|
|
"from": record.get("from_name") or record.get("from"),
|
|
"chat": record.get("chat"),
|
|
"timestamp": sent_at.isoformat() if sent_at else None,
|
|
"body": (record.get("body") or "")[:MAX_BODY_CHARS],
|
|
# Shaped like email's `attachments` so agenda.py can treat
|
|
# both the same way. `path` is filled only for documents
|
|
# the bridge actually saved; photos and video are recorded
|
|
# by type and never downloaded. Older bridge lines predate
|
|
# these fields and simply carry no attachment.
|
|
"attachments": (
|
|
[{
|
|
"filename": record.get("filename") or "",
|
|
"content_type": record.get("mimetype") or record.get("type") or "",
|
|
"path": (
|
|
str(documents_dir / record["document_file"])
|
|
if record.get("document_file")
|
|
else None
|
|
),
|
|
}]
|
|
if record.get("has_media")
|
|
else []
|
|
),
|
|
}
|
|
)
|
|
except Exception:
|
|
LOG.warning("whatsapp: could not parse a bridge line, skipping", exc_info=True)
|
|
except Exception:
|
|
LOG.warning("whatsapp: could not read %s, returning nothing", drained_path, exc_info=True)
|
|
return []
|
|
finally:
|
|
try:
|
|
os.remove(drained_path)
|
|
except OSError:
|
|
LOG.warning("whatsapp: could not remove %s", drained_path, exc_info=True)
|
|
|
|
LOG.info("whatsapp: %d message(s) in the last %sh", len(messages), lookback_hours)
|
|
return messages
|