68 lines
2.3 KiB
Python
68 lines
2.3 KiB
Python
import datetime as dt
|
|
import re
|
|
from collections import Counter, defaultdict
|
|
|
|
from sqlalchemy import select
|
|
from sqlalchemy.orm import Session
|
|
|
|
from .config import settings
|
|
from .models import Article
|
|
|
|
_STOPWORDS = {
|
|
"the", "a", "an", "in", "on", "of", "to", "for", "and", "or", "is", "as",
|
|
"at", "by", "with", "from", "after", "over", "amid", "amid", "into",
|
|
"says", "say", "will", "has", "have", "had", "its", "it", "his", "her",
|
|
"new", "up", "out", "how", "why", "what", "who", "be", "are", "was",
|
|
"were", "this", "that", "than", "not", "no", "us", "u.s.",
|
|
}
|
|
_WORD_RE = re.compile(r"[A-Za-z][A-Za-z'-]{2,}")
|
|
|
|
|
|
def _topic_label(titles: list[str]) -> str:
|
|
words = Counter()
|
|
for title in titles:
|
|
for word in _WORD_RE.findall(title.lower()):
|
|
if word not in _STOPWORDS:
|
|
words[word] += 1
|
|
top = [w for w, _ in words.most_common(3)]
|
|
return ", ".join(top) if top else ""
|
|
|
|
|
|
def build_clusters(session: Session) -> list[dict]:
|
|
cutoff = dt.datetime.utcnow() - dt.timedelta(hours=settings.article_window_hours)
|
|
rows = session.execute(
|
|
select(Article)
|
|
.where(Article.cluster_key.is_not(None))
|
|
.where(Article.published_at >= cutoff)
|
|
.order_by(Article.published_at.desc())
|
|
).scalars().all()
|
|
|
|
groups: dict[str, list[Article]] = defaultdict(list)
|
|
for article in rows:
|
|
groups[article.cluster_key].append(article)
|
|
|
|
clusters = []
|
|
for key, articles in groups.items():
|
|
lat = sum(a.lat for a in articles) / len(articles)
|
|
lon = sum(a.lon for a in articles) / len(articles)
|
|
sources = sorted({a.source for a in articles})
|
|
clusters.append(
|
|
{
|
|
"cluster_key": key,
|
|
"lat": lat,
|
|
"lon": lon,
|
|
"location_name": articles[0].location_name,
|
|
"country": articles[0].country,
|
|
"article_count": len(articles),
|
|
"source_count": len(sources),
|
|
"sources": sources,
|
|
"topic": _topic_label([a.title for a in articles]),
|
|
"latest_published_at": max(a.published_at for a in articles).isoformat(),
|
|
"headline": articles[0].title,
|
|
"article_ids": [a.id for a in articles],
|
|
}
|
|
)
|
|
|
|
clusters.sort(key=lambda c: c["article_count"], reverse=True)
|
|
return clusters
|