"""IMAP mail ingestion. Auth is a plain password via EMAIL_PASSWORD, not OAuth2/XOAUTH2: for Gmail that means an App Password (enable 2FA, then generate one at myaccount.google.com -> Security -> App passwords); other providers have an equivalent app-specific password. OAuth2 would need a browser consent round-trip plus refresh-token storage, which is far too heavy for a single-user cron-style job — App Password is the documented, supported path for exactly this case. """ import email import email.utils import logging import os from datetime import datetime, timedelta, timezone from email.header import decode_header, make_header LOG = logging.getLogger(__name__) MAX_BODY_CHARS = 2000 def _decode(value): if not value: return "" try: return str(make_header(decode_header(value))) except Exception: return str(value) def _body_text(message): if message.is_multipart(): for part in message.walk(): if part.get_content_type() == "text/plain" and "attachment" not in str( part.get("Content-Disposition", "") ): payload = part.get_payload(decode=True) if payload: return payload.decode(part.get_content_charset() or "utf-8", "replace") return "" payload = message.get_payload(decode=True) if not payload: return "" return payload.decode(message.get_content_charset() or "utf-8", "replace") def fetch(lookback_hours): host = os.environ.get("EMAIL_IMAP_HOST", "").strip() user = os.environ.get("EMAIL_USERNAME", "").strip() password = os.environ.get("EMAIL_PASSWORD", "") folder = os.environ.get("EMAIL_FOLDER", "INBOX") port = int(os.environ.get("EMAIL_IMAP_PORT", "993")) lookback_hours = float(os.environ.get("EMAIL_LOOKBACK_HOURS", lookback_hours)) if not (host and user and password): LOG.warning("email: EMAIL_IMAP_HOST/EMAIL_USERNAME/EMAIL_PASSWORD not all set, skipping") return [] since = datetime.now(timezone.utc) - timedelta(hours=lookback_hours) messages = [] try: # Imported here, not at module scope, so a missing/broken optional # dependency degrades this one source instead of the whole run. from imapclient import IMAPClient with IMAPClient(host, port=port, ssl=True) as client: client.login(user, password) # readonly=True is the whole point: it stops the server from setting # \Seen on anything we FETCH. digest-engine never mutates the mailbox. client.select_folder(folder, readonly=True) uids = client.search(["SINCE", since.date()]) if not uids: return [] for uid, data in client.fetch(uids, ["RFC822"]).items(): raw = data.get(b"RFC822") if not raw: continue try: parsed = email.message_from_bytes(raw) sent_at = email.utils.parsedate_to_datetime(parsed.get("Date")) if sent_at is not None and sent_at.tzinfo is None: sent_at = sent_at.replace(tzinfo=timezone.utc) if sent_at is not None and sent_at < since: continue messages.append( { "source": "email", "uid": int(uid), "from": _decode(parsed.get("From")), "to": _decode(parsed.get("To")), "subject": _decode(parsed.get("Subject")), "timestamp": sent_at.isoformat() if sent_at else None, "body": _body_text(parsed).strip()[:MAX_BODY_CHARS], } ) except Exception: LOG.warning("email: could not parse message uid=%s, skipping", uid, exc_info=True) except Exception: LOG.warning("email: ingestion failed, returning nothing", exc_info=True) return [] LOG.info("email: %d message(s) in the last %sh", len(messages), lookback_hours) return messages