"""News ingestion — feedparser over the curated OPML feed list.

The feed list is user-editable at digest-engine/feeds/curated-feeds.opml; each
`<outline>` with an `xmlUrl` attribute is fetched. marxist.com is tagged in the
OPML with `category="theory"` and that category is carried through onto every
entry, because the political prompt treats it as the analytical basis rather than
as one more headline source.

Three OPML attributes are passed through onto every entry: `category`, `owner`
and `bias`. The last two are what let the political prompt read a story against
who paid for it instead of treating "the news" as a single undifferentiated
input — see the source-criticism section of synth/prompts/political.md. They are
free text and are never parsed here; this module carries them, it does not
interpret them.
"""

import calendar
import html
import logging
import os
import re
import xml.etree.ElementTree as ElementTree
from datetime import datetime, timedelta, timezone

LOG = logging.getLogger(__name__)

DEFAULT_OPML_PATH = "/app/feeds/curated-feeds.opml"
MAX_SUMMARY_CHARS = 1200

_TAG_RE = re.compile(r"<[^>]+>")


def _plain_text(markup):
    return html.unescape(_TAG_RE.sub(" ", markup or "")).strip()


def _read_opml(path):
    feeds = []
    tree = ElementTree.parse(path)
    for outline in tree.iter("outline"):
        url = outline.get("xmlUrl")
        if not url:
            continue
        feeds.append(
            {
                "url": url,
                "title": outline.get("title") or outline.get("text") or url,
                "category": outline.get("category") or "news",
                # Who owns the outlet and where it sits politically, both free text,
                # both carried onto every entry. The political prompt is told to read
                # each item against them rather than against a "reliable/unreliable"
                # ranking: no outlet in this file is neutral, and which fraction of
                # capital (or which capitalist state) pays for one is a fact about the
                # reporting, not a footnote. Missing attributes just arrive as null —
                # the prompt handles that as "ownership not recorded here".
                "owner": outline.get("owner"),
                "bias": outline.get("bias"),
            }
        )
    return feeds


def _entry_time(entry):
    parsed = entry.get("published_parsed") or entry.get("updated_parsed")
    if not parsed:
        return None
    return datetime.fromtimestamp(calendar.timegm(parsed), tz=timezone.utc)


def fetch(lookback_hours):
    opml_path = os.environ.get("NEWS_OPML_PATH", DEFAULT_OPML_PATH)
    max_per_feed = int(os.environ.get("NEWS_MAX_ENTRIES_PER_FEED", "15"))

    # Imported here, not at module scope, so a missing/broken optional dependency
    # degrades this one source instead of the whole run.
    try:
        import feedparser
    except ImportError:
        LOG.warning("news: feedparser is not installed, skipping", exc_info=True)
        return []

    try:
        feeds = _read_opml(opml_path)
    except Exception:
        LOG.warning("news: could not read OPML at %s, returning nothing", opml_path, exc_info=True)
        return []

    if not feeds:
        LOG.warning("news: no feeds with an xmlUrl in %s, skipping", opml_path)
        return []

    since = datetime.now(timezone.utc) - timedelta(hours=lookback_hours)
    entries = []

    for feed in feeds:
        try:
            parsed = feedparser.parse(feed["url"])
            if parsed.get("bozo") and not parsed.get("entries"):
                LOG.warning("news: feed %s did not parse, skipping", feed["url"])
                continue
            kept = 0
            for entry in parsed.entries:
                if kept >= max_per_feed:
                    break
                published = _entry_time(entry)
                if published and published < since:
                    continue
                entries.append(
                    {
                        "source": "news",
                        "feed": feed["title"],
                        "category": feed["category"],
                        "owner": feed["owner"],
                        "bias": feed["bias"],
                        "title": entry.get("title", "").strip(),
                        "link": entry.get("link", ""),
                        "timestamp": published.isoformat() if published else None,
                        "summary": _plain_text(entry.get("summary"))[:MAX_SUMMARY_CHARS],
                    }
                )
                kept += 1
        except Exception:
            LOG.warning("news: feed %s failed, skipping", feed["url"], exc_info=True)

    LOG.info("news: %d entr(ies) from %d feed(s) in the last %sh", len(entries), len(feeds), lookback_hours)
    return entries