286 lines
12 KiB
Python
286 lines
12 KiB
Python
"""RCI social-media ingestion — the organisation's own public output.
|
|
|
|
Feeds the **political** section alongside the RCI/Der Funke RSS already in
|
|
feeds/curated-feeds.opml, tagged `"category": "theory_social"` so
|
|
synth/prompts/political.md can tell the section's agitational and organisational
|
|
posts ("come to this meeting", "watch this explainer") apart from its written
|
|
analysis, which stays tagged `"theory"`. Sources are listed in the committed,
|
|
user-editable feeds/rci-social.json.
|
|
|
|
WHAT IS REACHABLE WITHOUT SCRAPING, AND WHAT IS NOT
|
|
---------------------------------------------------
|
|
Only platforms with a public, keyless, first-party read path are implemented.
|
|
That is a hard line, not a to-do list:
|
|
|
|
- **YouTube** — `https://www.youtube.com/feeds/videos.xml?channel_id=UC...`
|
|
is YouTube's own Atom feed, no key and no quota. Verified live against both
|
|
configured channels 2026-08-06.
|
|
- **Any RSS/Atom feed** — a Mastodon account's `.rss`, a podcast feed, a
|
|
section's own site. Same `feedparser` this project already vendors for news.
|
|
- **Telegram** — read through the Telethon session `telegram_ingest.py`
|
|
already logs in with. A public channel resolves by username and its history
|
|
reads **without joining it**: no `JoinChannelRequest`, no
|
|
`send_read_acknowledge()`, nothing written anywhere. If that session does
|
|
not exist, these entries are skipped and the RSS ones still run.
|
|
|
|
Deliberately NOT built, because none of them has a read path that is both
|
|
keyless and within the platform's terms — and this component's whole premise is
|
|
that it never scrapes and never logs in as a person to a platform that forbids
|
|
it (the one exception, WhatsApp, is opt-in and carries its own ban warning in
|
|
README.md):
|
|
|
|
- **Instagram** (`@rkp_austria`, `@revcomintern`) — the Basic Display API was
|
|
retired in December 2024, and the Graph API only reads accounts you own,
|
|
through a reviewed Meta app. Reading somebody else's public account means
|
|
scraping, which is both against Instagram's terms and the fastest way to get
|
|
an account or an IP blocked. This is the biggest real gap in this module:
|
|
Instagram is where the section posts most.
|
|
- **Facebook** (`/derfunke.at`) — page RSS was killed in 2018; Page Public
|
|
Content Access needs Meta app review for a business use case this is not.
|
|
- **WhatsApp channel** — no API of any kind. The existing `whatsapp-bridge`
|
|
sidecar is a linked *personal* device and its library's channel support is
|
|
experimental; wiring it up would extend that sidecar's ban risk to a feature
|
|
that can be had by reading the website instead.
|
|
- **X/Twitter** (`@revcomintern`) — the free API tier is write-oriented and
|
|
reads essentially nothing; Nitter instances are gone.
|
|
|
|
If you want those, the honest answer is to follow the accounts on your phone.
|
|
Do not "solve" it here by pointing this module at a scraping proxy: that moves
|
|
the terms-of-service problem onto a third party without removing it.
|
|
"""
|
|
|
|
import html
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
DEFAULT_CONF_PATH = "/app/feeds/rci-social.json"
|
|
|
|
# YouTube's own feed endpoint. Documented and keyless, but it takes the opaque
|
|
# channel id (`UC...`) only — an `@handle` is not accepted, so the config file
|
|
# holds ids and says where to find them.
|
|
YOUTUBE_FEED_URL = "https://www.youtube.com/feeds/videos.xml?channel_id={channel_id}"
|
|
|
|
MAX_BODY_CHARS = 1200
|
|
DEFAULT_MAX_PER_SOURCE = 10
|
|
|
|
# What a source publishes, unless its config entry says otherwise. A YouTube channel
|
|
# is episodes by definition; a plain feed could be either (a podcast is episodes, a
|
|
# Mastodon account is posts), so it defaults to the less presumptuous of the two.
|
|
DEFAULT_CONTENT_TYPES = {"youtube": "episode", "rss": "post", "telegram": "post"}
|
|
|
|
_TAG_RE = re.compile(r"<[^>]+>")
|
|
|
|
|
|
# Deliberately duplicated from news_rss.py rather than shared: every module in
|
|
# this package stands on its own so that one source's breakage can never reach
|
|
# another's, and a shared helpers module would be a single import for all of them
|
|
# to fail on. Same trade counter_run.py makes against llm_client.py.
|
|
def _plain_text(markup):
|
|
return html.unescape(_TAG_RE.sub(" ", markup or "")).strip()
|
|
|
|
|
|
def _entry_time(entry):
|
|
parsed = entry.get("published_parsed") or entry.get("updated_parsed")
|
|
if not parsed:
|
|
return None
|
|
import calendar
|
|
|
|
return datetime.fromtimestamp(calendar.timegm(parsed), tz=timezone.utc)
|
|
|
|
|
|
def _load_sources(path):
|
|
try:
|
|
with open(path, encoding="utf-8") as handle:
|
|
config = json.load(handle)
|
|
except FileNotFoundError:
|
|
LOG.warning("rci_social: %s not found, skipping", path)
|
|
return []
|
|
except Exception:
|
|
LOG.warning("rci_social: %s could not be parsed, skipping", path, exc_info=True)
|
|
return []
|
|
|
|
sources = []
|
|
for entry in config.get("sources") or []:
|
|
if not isinstance(entry, dict) or entry.get("enabled") is False:
|
|
continue
|
|
platform = str(entry.get("platform") or "").strip().lower()
|
|
if platform not in ("rss", "youtube", "telegram"):
|
|
LOG.warning("rci_social: unknown platform %r, skipping that entry", platform)
|
|
continue
|
|
sources.append(
|
|
{
|
|
"platform": platform,
|
|
"name": str(entry.get("name") or "").strip() or platform,
|
|
# "episode" (something to watch or listen to later) vs "post"
|
|
# (something said now). The political prompt keeps episodes out of
|
|
# the analysis and puts them in their own watch-later window, so a
|
|
# 40-minute video never competes with a strike for space.
|
|
"content_type": str(entry.get("content_type") or DEFAULT_CONTENT_TYPES.get(platform, "post")).strip().lower(),
|
|
# `scope` is passed straight through to the prompt: "section" is the
|
|
# user's own national section, "international" the RCI as a whole.
|
|
# The two are not the same voice and the digest should not merge them.
|
|
"scope": str(entry.get("scope") or "section").strip().lower(),
|
|
"url": str(entry.get("url") or "").strip(),
|
|
"channel_id": str(entry.get("channel_id") or "").strip(),
|
|
"channel": str(entry.get("channel") or "").strip().lstrip("@"),
|
|
}
|
|
)
|
|
return sources
|
|
|
|
|
|
def _item(source, title, body, url, timestamp):
|
|
return {
|
|
"source": "rci_social",
|
|
# Not "theory": these are posts, not the written analysis the political
|
|
# prompt reasons from. See that prompt's "The organisation's own social
|
|
# media" section for the difference it is told to keep.
|
|
"category": "theory_social",
|
|
"platform": source["platform"],
|
|
"account": source["name"],
|
|
"scope": source["scope"],
|
|
"content_type": source["content_type"],
|
|
"title": title,
|
|
"url": url,
|
|
"timestamp": timestamp,
|
|
"body": (body or "")[:MAX_BODY_CHARS],
|
|
}
|
|
|
|
|
|
def _fetch_feed(source, since, max_per_source):
|
|
import feedparser
|
|
|
|
url = source["url"]
|
|
if source["platform"] == "youtube":
|
|
if not source["channel_id"]:
|
|
LOG.warning("rci_social: %s has no channel_id, skipping", source["name"])
|
|
return []
|
|
url = YOUTUBE_FEED_URL.format(channel_id=source["channel_id"])
|
|
if not url:
|
|
LOG.warning("rci_social: %s has no url, skipping", source["name"])
|
|
return []
|
|
|
|
parsed = feedparser.parse(url)
|
|
items = []
|
|
for entry in parsed.entries[:max_per_source]:
|
|
published = _entry_time(entry)
|
|
if published is None or published < since:
|
|
continue
|
|
item = _item(
|
|
source,
|
|
_plain_text(entry.get("title")),
|
|
_plain_text(entry.get("summary") or entry.get("description")),
|
|
entry.get("link") or url,
|
|
published.isoformat(),
|
|
)
|
|
# Podcast feeds carry <itunes:duration>; YouTube's Atom feed does not. Passed
|
|
# through when it exists because "is this 8 minutes or 90" is most of what
|
|
# decides whether something makes it onto a watch-later list.
|
|
duration = entry.get("itunes_duration")
|
|
if duration:
|
|
item["duration"] = str(duration)
|
|
items.append(item)
|
|
return items
|
|
|
|
|
|
async def _fetch_telegram_async(sources, since, api_id, api_hash, session_path, max_per_source):
|
|
from telethon import TelegramClient
|
|
|
|
items = []
|
|
client = TelegramClient(session_path, api_id, api_hash)
|
|
await client.connect()
|
|
try:
|
|
if not await client.is_user_authorized():
|
|
LOG.warning(
|
|
"rci_social: telegram session at %s is not authorized — run "
|
|
"`python ingest/telegram_login.py` once; skipping the telegram sources",
|
|
session_path,
|
|
)
|
|
return []
|
|
|
|
for source in sources:
|
|
try:
|
|
# Resolving a public channel by username is a read. Nothing here
|
|
# joins it, subscribes to it, or marks anything read — same
|
|
# invariant as telegram_ingest.py, which never calls
|
|
# send_read_acknowledge() either.
|
|
entity = await client.get_entity(source["channel"])
|
|
async for message in client.iter_messages(entity, limit=max_per_source):
|
|
if message.date is None or message.date < since:
|
|
break
|
|
if not message.message:
|
|
continue
|
|
items.append(
|
|
_item(
|
|
source,
|
|
"",
|
|
message.message,
|
|
f"https://t.me/{source['channel']}/{message.id}",
|
|
message.date.isoformat(),
|
|
)
|
|
)
|
|
except Exception:
|
|
LOG.warning(
|
|
"rci_social: could not read telegram channel %r, skipping",
|
|
source["channel"],
|
|
exc_info=True,
|
|
)
|
|
finally:
|
|
await client.disconnect()
|
|
|
|
return items
|
|
|
|
|
|
def _fetch_telegram(sources, since, max_per_source):
|
|
api_id = os.environ.get("TELEGRAM_API_ID", "").strip()
|
|
api_hash = os.environ.get("TELEGRAM_API_HASH", "").strip()
|
|
session_path = os.environ.get("TELEGRAM_SESSION_PATH", "/data/telegram.session")
|
|
|
|
# The same credentials and session file telegram_ingest.py uses. A household
|
|
# that never set Telegram up simply doesn't get these sources; it is not an
|
|
# error and must not cost the RSS ones.
|
|
if not (api_id and api_hash and os.path.exists(session_path)):
|
|
LOG.info("rci_social: no usable telegram session, skipping the telegram sources")
|
|
return []
|
|
|
|
import asyncio
|
|
|
|
return asyncio.run(
|
|
_fetch_telegram_async(sources, since, int(api_id), api_hash, session_path, max_per_source)
|
|
)
|
|
|
|
|
|
def fetch(lookback_hours):
|
|
sources = _load_sources(os.environ.get("RCI_SOCIAL_CONF_PATH", DEFAULT_CONF_PATH))
|
|
if not sources:
|
|
return []
|
|
|
|
max_per_source = int(os.environ.get("RCI_SOCIAL_MAX_PER_SOURCE", DEFAULT_MAX_PER_SOURCE))
|
|
since = datetime.now(timezone.utc) - timedelta(hours=float(lookback_hours))
|
|
|
|
items = []
|
|
for source in sources:
|
|
if source["platform"] == "telegram":
|
|
continue
|
|
try:
|
|
items.extend(_fetch_feed(source, since, max_per_source))
|
|
except Exception:
|
|
# One dead account never costs the others, exactly like a dead feed in
|
|
# news_rss.py — a section that renamed a channel shouldn't silence the rest.
|
|
LOG.warning("rci_social: %s failed, skipping it", source["name"], exc_info=True)
|
|
|
|
telegram_sources = [source for source in sources if source["platform"] == "telegram"]
|
|
if telegram_sources:
|
|
try:
|
|
items.extend(_fetch_telegram(telegram_sources, since, max_per_source))
|
|
except Exception:
|
|
LOG.warning("rci_social: telegram ingestion failed, skipping it", exc_info=True)
|
|
|
|
items.sort(key=lambda item: item["timestamp"], reverse=True)
|
|
LOG.info("rci_social: %d post(s) in the last %sh", len(items), lookback_hours)
|
|
return items
|