diff --git a/README.md b/README.md index 17349de..0cc1b19 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,14 @@ composition diagram when Wikipedia has one. - **Top-left "i" button** — opens the **Economic Incident History** view: every detected market spike, full clock-hour window, with the complete list of articles published in it (not just the top few) — a comprehensive log, - distinct from the quick preview in the right rail. + distinct from the quick preview in the right rail. Spikes across + instruments within an hour of each other are merged into one incident + (e.g. WTI and Brent crude spiking together shows as one entry, not two — + a timestamp heuristic, not evidence the moves actually share a cause). + Each incident also shows its **likely factors**: the words that recur + most across every headline published in that window, stopwords filtered + out — plain word-frequency counting, no AI/LLM involved (`textutil.py`). + Read it as "worth checking these articles," not an explanation. - **Bottom drawer** — collapsible conflict/military-event log (ACLED-backed; see below). - **Flights toggle** — live global air traffic (OpenSky Network) as airplane @@ -160,6 +167,10 @@ into the red dots. alone qualifies a story as a candidate. Click a ticker item for a quick preview, or the "i" button (top-left) for the full log. Heuristic correlation, not a verified causal link. +- `GET /api/markets/incidents?symbol=&hours=` — the same spikes merged + across instruments within `INCIDENT_MERGE_HOURS` (1h, `markets.py`) of + each other, each with a `top_keywords` word-frequency list. Backs the + Economic Incident History view. - `GET /api/conflict-events?hours=168` - `GET /api/flights` — latest global OpenSky snapshot, with `is_military` per aircraft (see the heuristic caveat above) @@ -173,8 +184,14 @@ into the red dots. - Add gazetteer locations: `backend/app/data/gazetteer.csv`. - Add/fix parliament page mappings: `backend/app/data/parliaments.yaml`. - Add/fix military ICAO24/callsign ranges: `backend/app/data/military_ranges.yaml`. +- Extend stopwords for keyword extraction (cluster topics, incident "likely + factors"): `backend/app/textutil.py`. - Change poll intervals / clustering window: `.env`. +New model fields need a matching entry in `_COLUMN_MIGRATIONS` in +`backend/app/db.py` — `create_all()` only creates missing tables, not +columns on tables that already exist in a deployed `data/newsatlas.db`. + ## Deploying as a Proxmox LXC See `deploy/lxc/README.md` for a `pct`-based build script that provisions a diff --git a/backend/app/clustering.py b/backend/app/clustering.py index 48b2d89..087ac81 100644 --- a/backend/app/clustering.py +++ b/backend/app/clustering.py @@ -1,31 +1,12 @@ import datetime as dt -import re -from collections import Counter, defaultdict +from collections import defaultdict from sqlalchemy import select from sqlalchemy.orm import Session from .config import settings from .models import Article - -_STOPWORDS = { - "the", "a", "an", "in", "on", "of", "to", "for", "and", "or", "is", "as", - "at", "by", "with", "from", "after", "over", "amid", "amid", "into", - "says", "say", "will", "has", "have", "had", "its", "it", "his", "her", - "new", "up", "out", "how", "why", "what", "who", "be", "are", "was", - "were", "this", "that", "than", "not", "no", "us", "u.s.", -} -_WORD_RE = re.compile(r"[A-Za-z][A-Za-z'-]{2,}") - - -def _topic_label(titles: list[str]) -> str: - words = Counter() - for title in titles: - for word in _WORD_RE.findall(title.lower()): - if word not in _STOPWORDS: - words[word] += 1 - top = [w for w, _ in words.most_common(3)] - return ", ".join(top) if top else "" +from .textutil import topic_label as _topic_label def build_clusters(session: Session) -> list[dict]: diff --git a/backend/app/db.py b/backend/app/db.py index aa3817f..a22f22f 100644 --- a/backend/app/db.py +++ b/backend/app/db.py @@ -1,6 +1,6 @@ from pathlib import Path -from sqlalchemy import create_engine +from sqlalchemy import create_engine, text from sqlalchemy.orm import DeclarativeBase, sessionmaker from .config import settings @@ -18,10 +18,30 @@ class Base(DeclarativeBase): pass +# Columns added to existing tables after their first release. create_all() +# only creates missing *tables*, so an upgrade on an existing SQLite file +# needs these added by hand — a lightweight alternative to a full migration +# framework, appropriate for this project's scale. Add a new (table, column, +# ddl-type-and-default) tuple here whenever a model gains a field. +_COLUMN_MIGRATIONS = [ + ("market_spikes", "top_keywords_json", "TEXT DEFAULT '[]'"), +] + + +def _run_column_migrations() -> None: + with engine.connect() as conn: + for table, column, ddl in _COLUMN_MIGRATIONS: + existing = {row[1] for row in conn.execute(text(f"PRAGMA table_info({table})"))} + if column not in existing: + conn.execute(text(f"ALTER TABLE {table} ADD COLUMN {column} {ddl}")) + conn.commit() + + def init_db() -> None: from . import models # noqa: F401 (registers tables on Base.metadata) Base.metadata.create_all(bind=engine) + _run_column_migrations() def get_session(): diff --git a/backend/app/main.py b/backend/app/main.py index 7cc471f..036986d 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -15,7 +15,7 @@ from .config import settings from .db import get_session, init_db, SessionLocal from .flights import get_latest_flights from .ingest import fetch_all -from .markets import poll_markets, INSTRUMENTS +from .markets import poll_markets, merge_spikes_into_incidents, INSTRUMENTS from .conflict import enabled as conflict_enabled, poll_conflict_events from .models import Article, ConflictEvent, MarketPrice, MarketSpike from .scheduler import start_scheduler @@ -212,6 +212,48 @@ def api_markets_spikes(symbol: str | None = None, hours: int = Query(168, le=24 "window_end": s.window_end.isoformat(), "detected_at": s.detected_at.isoformat(), "candidate_articles": articles, + "top_keywords": json.loads(s.top_keywords_json or "[]"), + } + ) + return out + finally: + session.close() + + +@app.get("/api/markets/incidents") +def api_markets_incidents(symbol: str | None = None, hours: int = Query(720, le=24 * 365)): + """Spikes merged across instruments when they land within + MERGE_INCIDENT_HOURS of each other (see markets.merge_spikes_into_incidents) + — the comprehensive view behind the Economic Incident History UI. + `symbol`, if given, filters to incidents that include that instrument, + but still returns every instrument in the merged incident, not just it.""" + since = dt.datetime.utcnow() - dt.timedelta(hours=hours) + session = next(get_session()) + try: + spikes = session.execute(select(MarketSpike).where(MarketSpike.detected_at >= since)).scalars().all() + incidents = merge_spikes_into_incidents(spikes) + if symbol: + incidents = [i for i in incidents if any(instr["symbol"] == symbol for instr in i["instruments"])] + + out = [] + for inc in incidents: + article_ids = inc["article_ids"] + articles = [] + if article_ids: + rows = session.execute(select(Article).where(Article.id.in_(article_ids))).scalars().all() + by_id = {a.id: a for a in rows} + articles = [_article_dict(by_id[i]) for i in article_ids if i in by_id] + out.append( + { + "instruments": [ + {**instr, "detected_at": instr["detected_at"].isoformat()} for instr in inc["instruments"] + ], + "window_start": inc["window_start"].isoformat(), + "window_end": inc["window_end"].isoformat(), + "detected_at": inc["detected_at"].isoformat(), + "latest_detected_at": inc["latest_detected_at"].isoformat(), + "candidate_articles": articles, + "top_keywords": inc["top_keywords"], } ) return out diff --git a/backend/app/markets.py b/backend/app/markets.py index 2c40cc1..dc58f99 100644 --- a/backend/app/markets.py +++ b/backend/app/markets.py @@ -1,6 +1,7 @@ import datetime as dt import json import logging +from collections import Counter import httpx from sqlalchemy import select @@ -8,6 +9,14 @@ from sqlalchemy.orm import Session from .config import settings from .models import Article, MarketPrice, MarketSpike +from .textutil import extract_keywords + +# Two spikes within this many hours of each other are treated as one +# "incident" — e.g. WTI and Brent crude almost always move together, and a +# real shock often shows up across several indices within the same hour. +# Chain-merged: A+B within range and B+C within range merges all three even +# if A and C individually aren't, same declutter pattern used for the globe. +INCIDENT_MERGE_HOURS = 1.0 log = logging.getLogger("newsatlas.markets") @@ -100,7 +109,9 @@ def _score_article(article: Article, keywords: list[str], country: str | None) - return score -def _find_candidate_articles(session: Session, symbol: str, window_start: dt.datetime, window_end: dt.datetime, limit: int = 60) -> list[int]: +def _find_candidate_articles( + session: Session, symbol: str, window_start: dt.datetime, window_end: dt.datetime, limit: int = 60 +) -> tuple[list[int], list[tuple[str, int]]]: hints = _SPIKE_HINTS.get(symbol, {"keywords": [], "country": None}) rows = session.execute( select(Article) @@ -115,7 +126,14 @@ def _find_candidate_articles(session: Session, symbol: str, window_start: dt.dat # first), it just no longer excludes zero-score articles. scored = [(_score_article(a, hints["keywords"], hints["country"]), a) for a in rows] scored.sort(key=lambda pair: (pair[0], pair[1].published_at), reverse=True) - return [a.id for _, a in scored[:limit]] + + # "Most likely reasons," heuristically: just the words that recur across + # every headline in the window (stopwords dropped) — no AI/LLM involved, + # see textutil.py. Computed over the whole window, not just the + # `limit`-truncated list below, for a more robust signal. + keywords = extract_keywords([a.title for _, a in scored], top_n=8) + + return [a.id for _, a in scored[:limit]], keywords def _floor_hour(t: dt.datetime) -> dt.datetime: @@ -140,7 +158,7 @@ def _detect_and_record_spike( # "14:00-16:00"), covering at least the full hour the spike landed in. window_start = _floor_hour(prev.recorded_at - dt.timedelta(hours=settings.market_spike_lookback_hours)) window_end = _ceil_hour(recorded_at) - article_ids = _find_candidate_articles(session, symbol, window_start, window_end) + article_ids, keywords = _find_candidate_articles(session, symbol, window_start, window_end) session.add( MarketSpike( @@ -153,6 +171,7 @@ def _detect_and_record_spike( window_end=window_end, detected_at=recorded_at, article_ids_json=json.dumps(article_ids), + top_keywords_json=json.dumps(keywords), ) ) log.info("Spike detected: %s %.2f%% (%d candidate articles)", symbol, pct_change, len(article_ids)) @@ -194,3 +213,63 @@ def poll_markets(session: Session) -> int: session.commit() return added + + +def merge_spikes_into_incidents(spikes: list[MarketSpike]) -> list[dict]: + """Chain-merge spikes within INCIDENT_MERGE_HOURS of each other into + "incidents" spanning possibly multiple instruments, on the assumption + that near-simultaneous moves are more likely to share a cause than + coincidence. Purely a timestamp heuristic — no correlation of the + actual price movements is attempted.""" + ordered = sorted(spikes, key=lambda s: s.detected_at) + groups: list[list[MarketSpike]] = [] + threshold = dt.timedelta(hours=INCIDENT_MERGE_HOURS) + + for spike in ordered: + target = next( + (g for g in groups if any(abs(spike.detected_at - member.detected_at) <= threshold for member in g)), + None, + ) + if target is not None: + target.append(spike) + else: + groups.append([spike]) + + incidents = [] + for group in groups: + group.sort(key=lambda s: s.detected_at) + article_ids: list[int] = [] + seen_ids: set[int] = set() + keyword_counts: Counter[str] = Counter() + for s in group: + for aid in json.loads(s.article_ids_json or "[]"): + if aid not in seen_ids: + seen_ids.add(aid) + article_ids.append(aid) + for word, count in json.loads(s.top_keywords_json or "[]"): + keyword_counts[word] += count + + incidents.append( + { + "instruments": [ + { + "symbol": s.symbol, + "label": s.label, + "from_price": s.from_price, + "to_price": s.to_price, + "pct_change": s.pct_change, + "detected_at": s.detected_at, + } + for s in group + ], + "window_start": min(s.window_start for s in group), + "window_end": max(s.window_end for s in group), + "detected_at": group[0].detected_at, + "latest_detected_at": group[-1].detected_at, + "article_ids": article_ids[:60], + "top_keywords": keyword_counts.most_common(8), + } + ) + + incidents.sort(key=lambda i: i["latest_detected_at"], reverse=True) + return incidents diff --git a/backend/app/models.py b/backend/app/models.py index fdeb972..e6540eb 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -58,6 +58,10 @@ class MarketSpike(Base): # JSON-encoded list of Article.id, ranked most-to-least likely relevant. article_ids_json: Mapped[str] = mapped_column(Text, default="[]") + # JSON-encoded [[word, count], ...] — plain word-frequency across every + # article in the window's titles, stopwords dropped. A heuristic hint at + # "likely factors," not an explanation; see textutil.py. + top_keywords_json: Mapped[str] = mapped_column(Text, default="[]") class ConflictEvent(Base): diff --git a/backend/app/sources.yaml b/backend/app/sources.yaml index f407ef6..8879eca 100644 --- a/backend/app/sources.yaml +++ b/backend/app/sources.yaml @@ -61,3 +61,7 @@ sources: - name: Der Standard (International) url: https://www.derstandard.at/rss/international bias: austrian-mainstream + + - name: IRNA (Islamic Republic News Agency) + url: https://en.irna.ir/rss + bias: iranian-state-run diff --git a/backend/app/textutil.py b/backend/app/textutil.py new file mode 100644 index 0000000..bf26ebb --- /dev/null +++ b/backend/app/textutil.py @@ -0,0 +1,56 @@ +"""Plain word-frequency text analysis — no ML/LLM involved. + +Shared by clustering.py (per-cluster topic labels) and markets.py (heuristic +"likely factors" behind a detected market spike): tokenize, drop stopwords, +count. That's the whole technique — it surfaces words that recur across a +set of headlines, nothing more. Treat the result as a hint worth reading the +linked articles over, not an explanation. +""" + +import re +from collections import Counter + +_WORD_RE = re.compile(r"[A-Za-zÀ-ÖØ-öø-ÿ][A-Za-zÀ-ÖØ-öø-ÿ'-]{2,}") + +# English + German (taz, Der Standard contribute German-language headlines) +# function words. Deliberately does NOT include short, meaningful tokens +# common in market/political news (us, uk, eu, fed, opec, ...) — those are +# exactly the signal this is trying to surface, not noise to remove. +_STOPWORDS = { + # English + "the", "a", "an", "in", "on", "of", "to", "for", "and", "or", "is", "as", + "at", "by", "with", "from", "after", "over", "amid", "into", "says", + "say", "said", "will", "has", "have", "had", "its", "it", "his", "her", + "new", "up", "out", "how", "why", "what", "who", "be", "are", "was", + "were", "this", "that", "than", "not", "no", "been", "but", "also", + "more", "their", "they", "you", "your", "we", "our", "can", "could", + "would", "should", "about", "against", "all", "any", "some", "such", + "one", "two", "first", "if", "so", "just", "most", "other", "which", + "when", "where", "while", "during", "before", "between", "under", + "again", "then", "there", "here", "still", "now", "only", "own", "same", + "too", "very", "did", "does", "do", "being", "them", "he", "she", "i", + "my", "me", "him", "off", "down", "because", "each", "few", "further", + "once", "both", "don", + # German + "der", "die", "das", "und", "zu", "den", "von", "mit", "auf", "für", + "ist", "im", "dem", "des", "ein", "eine", "einer", "eines", "auch", + "nach", "bei", "aus", "wie", "was", "wird", "werden", "sich", "sie", + "er", "es", "noch", "nur", "schon", "über", "um", "als", "aber", "oder", + "wenn", "wir", "ihr", "man", "sind", "hat", "haben", "kann", "können", + "wurde", "wurden", "einem", "einen", "diese", "dieser", "dieses", "vor", + "durch", "zum", "zur", "doch", +} + + +def extract_keywords(texts: list[str], top_n: int = 8) -> list[tuple[str, int]]: + """Word-frequency count across `texts`, stopwords dropped, most common first.""" + counts: Counter[str] = Counter() + for text in texts: + for word in _WORD_RE.findall((text or "").lower()): + if word not in _STOPWORDS: + counts[word] += 1 + return counts.most_common(top_n) + + +def topic_label(titles: list[str], top_n: int = 3) -> str: + return ", ".join(w for w, _ in extract_keywords(titles, top_n)) diff --git a/frontend/css/style.css b/frontend/css/style.css index 08cd17d..39bf9c3 100644 --- a/frontend/css/style.css +++ b/frontend/css/style.css @@ -273,10 +273,22 @@ html, body { } .econ-incident { padding: 14px 0; border-bottom: 1px solid #241533; } -.econ-incident h3 { margin: 0 0 4px; font-size: 14px; color: #f2dede; } +.econ-incident h3 { margin: 0 0 4px; font-size: 14px; color: #f2dede; display: flex; flex-wrap: wrap; gap: 8px 14px; font-weight: 600; } +.instrument-chip { display: inline-flex; gap: 6px; align-items: baseline; } .econ-incident .window { font-size: 11px; color: var(--c-text-muted); margin-bottom: 8px; } .econ-incident .articles { margin-top: 8px; } +.keyword-tags { margin: 6px 0 10px; font-size: 11px; display: flex; flex-wrap: wrap; align-items: center; gap: 5px; } +.keyword-tag { + display: inline-block; + background: var(--c-tag-bg); + border: 1px solid var(--c-panel-border); + color: var(--c-dark-soft); + border-radius: 10px; + padding: 2px 8px; + font-size: 11px; +} + .panel-section { margin-bottom: 20px; } .panel-section h2 { font-size: 13px; diff --git a/frontend/js/app.js b/frontend/js/app.js index fe1f28c..17849fb 100644 --- a/frontend/js/app.js +++ b/frontend/js/app.js @@ -5,6 +5,12 @@ const API = "/api"; // than a separate alert layer, per the flights-overlay color spec. const GLOBE_TINT_HEX = 0x6a3fd9; const MILITARY_FLIGHT_HEX = 0xff33ff; +// News icons used to gradient violet->pink by corroborating-source count; +// flattened to a single bright magenta so they stay legible against the +// (also violet) globe tint — size still scales with source/article count. +const NEWS_ICON_HEX = 0xff33ff; +// Must match INCIDENT_MERGE_HOURS in backend/app/markets.py — display label only. +const INCIDENT_MERGE_HOURS_LABEL = "1h"; let config = { weather_enabled: false, conflict_enabled: false }; let clusters = []; // raw, server-side ~11km-grid clusters @@ -85,12 +91,6 @@ function resizeGlobe() { window.addEventListener("resize", resizeGlobe); setTimeout(resizeGlobe, 0); -function colorForCluster(d) { - // violet (few corroborating sources) -> hot pink (many) — theme accent gradient - const n = Math.min(d.source_count, 6); - const palette = ["#5018DD", "#7018C4", "#9018AB", "#B01092", "#D00879", "#E40046"]; - return palette[n - 1] || palette[0]; -} // ---- globe icon sprites (news clusters + conflict events) -------------- // @@ -164,7 +164,7 @@ function buildGlobeObject(d) { if (d.kind === "conflict") { return makeIconSprite(helmetIconTexture(), MILITARY_FLIGHT_HEX, HELMET_ICON_DEG * sceneUnitsPerDeg); } - return makeIconSprite(newspaperIconTexture(), colorForCluster(d), pointRadiusFor(d) * sceneUnitsPerDeg * 2.4); + return makeIconSprite(newspaperIconTexture(), NEWS_ICON_HEX, pointRadiusFor(d) * sceneUnitsPerDeg * 2.4); } function objectLabelFor(d) { @@ -846,6 +846,16 @@ function mountSparkline(container, history) { }); } +// "Likely factors": the words that recur most across every headline +// published in a spike's window (backend textutil.py — stopword-filtered +// word frequency, no AI/LLM). A hint worth reading the linked articles +// over, not an explanation, so it's labeled as such wherever it's shown. +function keywordTagsHtml(keywords) { + if (!keywords || !keywords.length) return ""; + const tags = keywords.map(([word, count]) => `${escapeHtml(word)} · ${count}`).join(""); + return `
Likely factors (common words in this window): ${tags}
`; +} + async function toggleMarketDetail(symbol, idx) { const el = document.getElementById(`market-detail-${idx}`); const wasHidden = el.classList.contains("hidden"); @@ -871,6 +881,7 @@ async function toggleMarketDetail(symbol, idx) {

${arrow} ${s.pct_change.toFixed(2)}% at ${new Date(s.detected_at).toLocaleString()} (${s.from_price.toLocaleString()} → ${s.to_price.toLocaleString()})

+ ${keywordTagsHtml(s.top_keywords)} ${articlesHtml} `; }) @@ -912,22 +923,23 @@ async function loadEconHistory() { listEl.innerHTML = `

Loading…

`; try { const url = symbol - ? `${API}/markets/spikes?symbol=${encodeURIComponent(symbol)}&hours=8760` - : `${API}/markets/spikes?hours=8760`; - const spikes = await (await fetch(url)).json(); - if (!spikes.length) { + ? `${API}/markets/incidents?symbol=${encodeURIComponent(symbol)}&hours=8760` + : `${API}/markets/incidents?hours=8760`; + const incidents = await (await fetch(url)).json(); + if (!incidents.length) { listEl.innerHTML = `

No moves ≥ the spike threshold recorded yet.

`; return; } - listEl.innerHTML = spikes.map(econIncidentHtml).join(""); + listEl.innerHTML = incidents.map(econIncidentHtml).join(""); } catch (e) { listEl.innerHTML = `

Could not load incident history.

`; } } -function econIncidentHtml(s) { - const dir = s.pct_change > 0 ? "up" : "down"; - const arrow = s.pct_change > 0 ? "▲" : "▼"; +// An "incident" is one or more spikes merged because they landed within an +// hour of each other (backend: markets.merge_spikes_into_incidents) — e.g. +// WTI and Brent crude spiking together shows as one incident, not two. +function econIncidentHtml(inc) { const fmtHour = (iso) => new Date(iso).toLocaleString(undefined, { month: "short", @@ -935,17 +947,25 @@ function econIncidentHtml(s) { hour: "2-digit", minute: "2-digit", }); - const articlesHtml = s.candidate_articles.length - ? s.candidate_articles.map(articleItemHtml).join("") + const instrumentsHtml = inc.instruments + .map((i) => { + const dir = i.pct_change > 0 ? "up" : "down"; + const arrow = i.pct_change > 0 ? "▲" : "▼"; + return `${escapeHtml(i.label)} ${arrow} ${i.pct_change.toFixed(2)}%`; + }) + .join(""); + const articlesHtml = inc.candidate_articles.length + ? inc.candidate_articles.map(articleItemHtml).join("") : `

No stories found published in that window.

`; return `
-

${escapeHtml(s.label)} ${arrow} ${s.pct_change.toFixed(2)}%

+

${instrumentsHtml}

- ${fmtHour(s.window_start)} – ${fmtHour(s.window_end)} - · ${s.from_price.toLocaleString()} → ${s.to_price.toLocaleString()} - · ${s.candidate_articles.length} stor${s.candidate_articles.length === 1 ? "y" : "ies"} in window + ${fmtHour(inc.window_start)} – ${fmtHour(inc.window_end)} + · ${inc.candidate_articles.length} stor${inc.candidate_articles.length === 1 ? "y" : "ies"} in window + ${inc.instruments.length > 1 ? ` · ${inc.instruments.length} instruments moved within ${INCIDENT_MERGE_HOURS_LABEL} of each other` : ""}
+ ${keywordTagsHtml(inc.top_keywords)}
${articlesHtml}
`;