57 lines
2.7 KiB
Python
57 lines
2.7 KiB
Python
"""Plain word-frequency text analysis — no ML/LLM involved.
|
|
|
|
Shared by clustering.py (per-cluster topic labels) and markets.py (heuristic
|
|
"likely factors" behind a detected market spike): tokenize, drop stopwords,
|
|
count. That's the whole technique — it surfaces words that recur across a
|
|
set of headlines, nothing more. Treat the result as a hint worth reading the
|
|
linked articles over, not an explanation.
|
|
"""
|
|
|
|
import re
|
|
from collections import Counter
|
|
|
|
_WORD_RE = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ][A-Za-zÀ-ÖØ-öø-ÿ'-]{2,}")
|
|
|
|
# English + German (taz, Der Standard contribute German-language headlines)
|
|
# function words. Deliberately does NOT include short, meaningful tokens
|
|
# common in market/political news (us, uk, eu, fed, opec, ...) — those are
|
|
# exactly the signal this is trying to surface, not noise to remove.
|
|
_STOPWORDS = {
|
|
# English
|
|
"the", "a", "an", "in", "on", "of", "to", "for", "and", "or", "is", "as",
|
|
"at", "by", "with", "from", "after", "over", "amid", "into", "says",
|
|
"say", "said", "will", "has", "have", "had", "its", "it", "his", "her",
|
|
"new", "up", "out", "how", "why", "what", "who", "be", "are", "was",
|
|
"were", "this", "that", "than", "not", "no", "been", "but", "also",
|
|
"more", "their", "they", "you", "your", "we", "our", "can", "could",
|
|
"would", "should", "about", "against", "all", "any", "some", "such",
|
|
"one", "two", "first", "if", "so", "just", "most", "other", "which",
|
|
"when", "where", "while", "during", "before", "between", "under",
|
|
"again", "then", "there", "here", "still", "now", "only", "own", "same",
|
|
"too", "very", "did", "does", "do", "being", "them", "he", "she", "i",
|
|
"my", "me", "him", "off", "down", "because", "each", "few", "further",
|
|
"once", "both", "don",
|
|
# German
|
|
"der", "die", "das", "und", "zu", "den", "von", "mit", "auf", "für",
|
|
"ist", "im", "dem", "des", "ein", "eine", "einer", "eines", "auch",
|
|
"nach", "bei", "aus", "wie", "was", "wird", "werden", "sich", "sie",
|
|
"er", "es", "noch", "nur", "schon", "über", "um", "als", "aber", "oder",
|
|
"wenn", "wir", "ihr", "man", "sind", "hat", "haben", "kann", "können",
|
|
"wurde", "wurden", "einem", "einen", "diese", "dieser", "dieses", "vor",
|
|
"durch", "zum", "zur", "doch",
|
|
}
|
|
|
|
|
|
def extract_keywords(texts: list[str], top_n: int = 8) -> list[tuple[str, int]]:
|
|
"""Word-frequency count across `texts`, stopwords dropped, most common first."""
|
|
counts: Counter[str] = Counter()
|
|
for text in texts:
|
|
for word in _WORD_RE.findall((text or "").lower()):
|
|
if word not in _STOPWORDS:
|
|
counts[word] += 1
|
|
return counts.most_common(top_n)
|
|
|
|
|
|
def topic_label(titles: list[str], top_n: int = 3) -> str:
|
|
return ", ".join(w for w, _ in extract_keywords(titles, top_n))
|