127 lines
4.7 KiB
Python
127 lines
4.7 KiB
Python
"""News ingestion — feedparser over the curated OPML feed list.
|
|
|
|
The feed list is user-editable at digest-engine/feeds/curated-feeds.opml; each
|
|
`<outline>` with an `xmlUrl` attribute is fetched. marxist.com is tagged in the
|
|
OPML with `category="theory"` and that category is carried through onto every
|
|
entry, because the political prompt treats it as the analytical basis rather than
|
|
as one more headline source.
|
|
|
|
Three OPML attributes are passed through onto every entry: `category`, `owner`
|
|
and `bias`. The last two are what let the political prompt read a story against
|
|
who paid for it instead of treating "the news" as a single undifferentiated
|
|
input — see the source-criticism section of synth/prompts/political.md. They are
|
|
free text and are never parsed here; this module carries them, it does not
|
|
interpret them.
|
|
"""
|
|
|
|
import calendar
|
|
import html
|
|
import logging
|
|
import os
|
|
import re
|
|
import xml.etree.ElementTree as ElementTree
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
DEFAULT_OPML_PATH = "/app/feeds/curated-feeds.opml"
|
|
MAX_SUMMARY_CHARS = 1200
|
|
|
|
_TAG_RE = re.compile(r"<[^>]+>")
|
|
|
|
|
|
def _plain_text(markup):
|
|
return html.unescape(_TAG_RE.sub(" ", markup or "")).strip()
|
|
|
|
|
|
def _read_opml(path):
|
|
feeds = []
|
|
tree = ElementTree.parse(path)
|
|
for outline in tree.iter("outline"):
|
|
url = outline.get("xmlUrl")
|
|
if not url:
|
|
continue
|
|
feeds.append(
|
|
{
|
|
"url": url,
|
|
"title": outline.get("title") or outline.get("text") or url,
|
|
"category": outline.get("category") or "news",
|
|
# Who owns the outlet and where it sits politically, both free text,
|
|
# both carried onto every entry. The political prompt is told to read
|
|
# each item against them rather than against a "reliable/unreliable"
|
|
# ranking: no outlet in this file is neutral, and which fraction of
|
|
# capital (or which capitalist state) pays for one is a fact about the
|
|
# reporting, not a footnote. Missing attributes just arrive as null —
|
|
# the prompt handles that as "ownership not recorded here".
|
|
"owner": outline.get("owner"),
|
|
"bias": outline.get("bias"),
|
|
}
|
|
)
|
|
return feeds
|
|
|
|
|
|
def _entry_time(entry):
|
|
parsed = entry.get("published_parsed") or entry.get("updated_parsed")
|
|
if not parsed:
|
|
return None
|
|
return datetime.fromtimestamp(calendar.timegm(parsed), tz=timezone.utc)
|
|
|
|
|
|
def fetch(lookback_hours):
|
|
opml_path = os.environ.get("NEWS_OPML_PATH", DEFAULT_OPML_PATH)
|
|
max_per_feed = int(os.environ.get("NEWS_MAX_ENTRIES_PER_FEED", "15"))
|
|
|
|
# Imported here, not at module scope, so a missing/broken optional dependency
|
|
# degrades this one source instead of the whole run.
|
|
try:
|
|
import feedparser
|
|
except ImportError:
|
|
LOG.warning("news: feedparser is not installed, skipping", exc_info=True)
|
|
return []
|
|
|
|
try:
|
|
feeds = _read_opml(opml_path)
|
|
except Exception:
|
|
LOG.warning("news: could not read OPML at %s, returning nothing", opml_path, exc_info=True)
|
|
return []
|
|
|
|
if not feeds:
|
|
LOG.warning("news: no feeds with an xmlUrl in %s, skipping", opml_path)
|
|
return []
|
|
|
|
since = datetime.now(timezone.utc) - timedelta(hours=lookback_hours)
|
|
entries = []
|
|
|
|
for feed in feeds:
|
|
try:
|
|
parsed = feedparser.parse(feed["url"])
|
|
if parsed.get("bozo") and not parsed.get("entries"):
|
|
LOG.warning("news: feed %s did not parse, skipping", feed["url"])
|
|
continue
|
|
kept = 0
|
|
for entry in parsed.entries:
|
|
if kept >= max_per_feed:
|
|
break
|
|
published = _entry_time(entry)
|
|
if published and published < since:
|
|
continue
|
|
entries.append(
|
|
{
|
|
"source": "news",
|
|
"feed": feed["title"],
|
|
"category": feed["category"],
|
|
"owner": feed["owner"],
|
|
"bias": feed["bias"],
|
|
"title": entry.get("title", "").strip(),
|
|
"link": entry.get("link", ""),
|
|
"timestamp": published.isoformat() if published else None,
|
|
"summary": _plain_text(entry.get("summary"))[:MAX_SUMMARY_CHARS],
|
|
}
|
|
)
|
|
kept += 1
|
|
except Exception:
|
|
LOG.warning("news: feed %s failed, skipping", feed["url"], exc_info=True)
|
|
|
|
LOG.info("news: %d entr(ies) from %d feed(s) in the last %sh", len(entries), len(feeds), lookback_hours)
|
|
return entries
|