107 lines
4.1 KiB
Python
107 lines
4.1 KiB
Python
"""IMAP mail ingestion.
|
|
|
|
Auth is a plain password via EMAIL_PASSWORD, not OAuth2/XOAUTH2: for Gmail that
|
|
means an App Password (enable 2FA, then generate one at myaccount.google.com ->
|
|
Security -> App passwords); other providers have an equivalent app-specific
|
|
password. OAuth2 would need a browser consent round-trip plus refresh-token
|
|
storage, which is far too heavy for a single-user cron-style job — App Password
|
|
is the documented, supported path for exactly this case.
|
|
"""
|
|
|
|
import email
|
|
import email.utils
|
|
import logging
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
from email.header import decode_header, make_header
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
MAX_BODY_CHARS = 2000
|
|
|
|
|
|
def _decode(value):
|
|
if not value:
|
|
return ""
|
|
try:
|
|
return str(make_header(decode_header(value)))
|
|
except Exception:
|
|
return str(value)
|
|
|
|
|
|
def _body_text(message):
|
|
if message.is_multipart():
|
|
for part in message.walk():
|
|
if part.get_content_type() == "text/plain" and "attachment" not in str(
|
|
part.get("Content-Disposition", "")
|
|
):
|
|
payload = part.get_payload(decode=True)
|
|
if payload:
|
|
return payload.decode(part.get_content_charset() or "utf-8", "replace")
|
|
return ""
|
|
payload = message.get_payload(decode=True)
|
|
if not payload:
|
|
return ""
|
|
return payload.decode(message.get_content_charset() or "utf-8", "replace")
|
|
|
|
|
|
def fetch(lookback_hours):
|
|
host = os.environ.get("EMAIL_IMAP_HOST", "").strip()
|
|
user = os.environ.get("EMAIL_USERNAME", "").strip()
|
|
password = os.environ.get("EMAIL_PASSWORD", "")
|
|
folder = os.environ.get("EMAIL_FOLDER", "INBOX")
|
|
port = int(os.environ.get("EMAIL_IMAP_PORT", "993"))
|
|
lookback_hours = float(os.environ.get("EMAIL_LOOKBACK_HOURS", lookback_hours))
|
|
|
|
if not (host and user and password):
|
|
LOG.warning("email: EMAIL_IMAP_HOST/EMAIL_USERNAME/EMAIL_PASSWORD not all set, skipping")
|
|
return []
|
|
|
|
since = datetime.now(timezone.utc) - timedelta(hours=lookback_hours)
|
|
messages = []
|
|
|
|
try:
|
|
# Imported here, not at module scope, so a missing/broken optional
|
|
# dependency degrades this one source instead of the whole run.
|
|
from imapclient import IMAPClient
|
|
|
|
with IMAPClient(host, port=port, ssl=True) as client:
|
|
client.login(user, password)
|
|
# readonly=True is the whole point: it stops the server from setting
|
|
# \Seen on anything we FETCH. digest-engine never mutates the mailbox.
|
|
client.select_folder(folder, readonly=True)
|
|
uids = client.search(["SINCE", since.date()])
|
|
if not uids:
|
|
return []
|
|
|
|
for uid, data in client.fetch(uids, ["RFC822"]).items():
|
|
raw = data.get(b"RFC822")
|
|
if not raw:
|
|
continue
|
|
try:
|
|
parsed = email.message_from_bytes(raw)
|
|
sent_at = email.utils.parsedate_to_datetime(parsed.get("Date"))
|
|
if sent_at is not None and sent_at.tzinfo is None:
|
|
sent_at = sent_at.replace(tzinfo=timezone.utc)
|
|
if sent_at is not None and sent_at < since:
|
|
continue
|
|
messages.append(
|
|
{
|
|
"source": "email",
|
|
"uid": int(uid),
|
|
"from": _decode(parsed.get("From")),
|
|
"to": _decode(parsed.get("To")),
|
|
"subject": _decode(parsed.get("Subject")),
|
|
"timestamp": sent_at.isoformat() if sent_at else None,
|
|
"body": _body_text(parsed).strip()[:MAX_BODY_CHARS],
|
|
}
|
|
)
|
|
except Exception:
|
|
LOG.warning("email: could not parse message uid=%s, skipping", uid, exc_info=True)
|
|
except Exception:
|
|
LOG.warning("email: ingestion failed, returning nothing", exc_info=True)
|
|
return []
|
|
|
|
LOG.info("email: %d message(s) in the last %sh", len(messages), lookback_hours)
|
|
return messages
|