85 lines
2.4 KiB
Python
85 lines
2.4 KiB
Python
import datetime as dt
|
|
import logging
|
|
|
|
import feedparser
|
|
import yaml
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from .config import settings
|
|
from .geocode import cluster_key_for, locate
|
|
from .models import Article
|
|
|
|
log = logging.getLogger("newsatlas.ingest")
|
|
|
|
|
|
def load_sources() -> list[dict]:
|
|
with open(settings.sources_file, encoding="utf-8") as f:
|
|
return yaml.safe_load(f)["sources"]
|
|
|
|
|
|
def _parse_published(entry) -> dt.datetime:
|
|
for key in ("published_parsed", "updated_parsed"):
|
|
value = getattr(entry, key, None)
|
|
if value:
|
|
return dt.datetime(*value[:6])
|
|
return dt.datetime.utcnow()
|
|
|
|
|
|
def fetch_source(session: Session, source: dict) -> int:
|
|
"""Fetch one RSS feed, geocode + store new entries. Returns count added."""
|
|
added = 0
|
|
try:
|
|
parsed = feedparser.parse(source["url"])
|
|
except Exception:
|
|
log.exception("Failed to fetch %s", source["name"])
|
|
return 0
|
|
|
|
if getattr(parsed, "bozo", 0) and not parsed.entries:
|
|
log.warning("Feed %s returned no usable entries (bozo=%s)", source["name"], parsed.bozo_exception)
|
|
return 0
|
|
|
|
seen_urls: set[str] = set()
|
|
for entry in parsed.entries:
|
|
url = getattr(entry, "link", None)
|
|
if not url or url in seen_urls:
|
|
continue
|
|
seen_urls.add(url)
|
|
|
|
exists = session.execute(select(Article.id).where(Article.url == url)).first()
|
|
if exists:
|
|
continue
|
|
|
|
title = getattr(entry, "title", "").strip()
|
|
summary = getattr(entry, "summary", "").strip()
|
|
place = locate(f"{title} {summary}")
|
|
|
|
article = Article(
|
|
source=source["name"],
|
|
source_bias=source.get("bias", ""),
|
|
title=title,
|
|
url=url,
|
|
summary=summary,
|
|
published_at=_parse_published(entry),
|
|
fetched_at=dt.datetime.utcnow(),
|
|
)
|
|
if place:
|
|
article.location_name = place.name
|
|
article.country = place.country
|
|
article.lat = place.lat
|
|
article.lon = place.lon
|
|
article.cluster_key = cluster_key_for(place.lat, place.lon)
|
|
|
|
session.add(article)
|
|
added += 1
|
|
|
|
session.commit()
|
|
return added
|
|
|
|
|
|
def fetch_all(session: Session) -> int:
|
|
total = 0
|
|
for source in load_sources():
|
|
total += fetch_source(session, source)
|
|
return total
|