From c3a266f6d92313e804312ad98e39d178ff47b076 Mon Sep 17 00:00:00 2001 From: The_miro Date: Mon, 20 Jul 2026 15:44:53 +0200 Subject: [PATCH] Backfill a week of market history on first startup; UI polish Backend: backfill_market_history() fetches 7 days of hourly price bars per instrument on first container start (skipped once an instrument has more than a day of real history, so restarts don't re-run it) and runs the same spike-detection logic retroactively, so the Economic Incident History view is already populated instead of empty for a week. Runs as a genuine one-time APScheduler job, separate from the recurring polls. Also: FROZEN/ERROR overlay badges on market ticker items (MOEX's known-stale Yahoo feed, and any instrument whose last poll actually failed), each labeled with the instrument name; a big magenta date/time heading per economic incident; and the incident article list is now a collapsed fold-out (some incidents merge to 60 stories) instead of always fully expanded. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Um48tTvZDrEgDeweFyhPYC --- README.md | 12 +++++ backend/app/main.py | 6 ++- backend/app/markets.py | 102 +++++++++++++++++++++++++++++++++++++++ backend/app/scheduler.py | 26 ++++++++-- frontend/css/style.css | 51 ++++++++++++++++++-- frontend/js/app.js | 61 ++++++++++++++++++----- 6 files changed, 236 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index a7276d5..fcc90ad 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,18 @@ On first boot the globe will be empty for a few seconds until the first RSS poll completes — hit "Refresh now" if you don't want to wait for the 10-minute interval. +The Economic Incident History, on the other hand, doesn't start empty: on +first startup the backend backfills a week of hourly price history per +instrument (`backfill_market_history()` in `markets.py`) and runs spike +detection retroactively across it, so there's already a populated week of +incidents rather than a week-long wait for live polling to accumulate +enough data. It's a one-time thing — skipped on later restarts once an +instrument has more than a day of real history. Backfilled incidents will +generally have few or no candidate articles, since RSS feeds only carry +recent items and there's no way to backfill a week of news to match a week +of backfilled prices — the price/incident data itself is real, the article +correlation just can't reach that far back. + ## Optional API keys (`.env`) Everything works with `.env` left blank except that the weather and conflict diff --git a/backend/app/main.py b/backend/app/main.py index 325d145..b7f92f3 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -15,7 +15,7 @@ from .config import settings from .db import get_session, init_db, SessionLocal from .flights import get_latest_flights from .ingest import fetch_all -from .markets import poll_markets, merge_spikes_into_incidents, INSTRUMENTS +from .markets import poll_markets, merge_spikes_into_incidents, has_fetch_error, FROZEN_SYMBOLS, INSTRUMENTS from .conflict import enabled as conflict_enabled, poll_conflict_events from .models import Article, ConflictEvent, MarketPrice, MarketSpike from .scheduler import start_scheduler @@ -160,6 +160,10 @@ def api_markets_latest(): "currency": row.currency, "change_pct": row.change_pct, "recorded_at": row.recorded_at.isoformat(), + "frozen": symbol in FROZEN_SYMBOLS, + # FROZEN already explains a stale price for that one + # documented instrument — don't also flag it ERROR. + "error": symbol not in FROZEN_SYMBOLS and has_fetch_error(symbol), } ) return out diff --git a/backend/app/markets.py b/backend/app/markets.py index f830211..23ec1c0 100644 --- a/backend/app/markets.py +++ b/backend/app/markets.py @@ -21,6 +21,11 @@ INCIDENT_MERGE_HOURS = 1.0 log = logging.getLogger("newsatlas.markets") +# Instruments whose feed is known stale/non-live (see the comment on +# IMOEX.ME below) — surfaced via /api/markets/latest so the UI can mark +# them clearly rather than presenting old data as a live price. +FROZEN_SYMBOLS = {"IMOEX.ME"} + # A spread of major indices as rough proxies for national/regional economies, # plus crude oil benchmarks. INSTRUMENTS = [ @@ -101,6 +106,86 @@ def _fetch_one(client: httpx.Client, symbol: str) -> tuple[float, float | None] return float(price), change_pct +BACKFILL_DAYS = 7 + + +def _fetch_history_bars(client: httpx.Client, symbol: str) -> list[tuple[dt.datetime, float]]: + resp = client.get( + CHART_URL.format(symbol=symbol), + params={"range": f"{BACKFILL_DAYS}d", "interval": "60m"}, + headers=_HEADERS, + timeout=20, + ) + resp.raise_for_status() + result = resp.json()["chart"]["result"][0] + timestamps = result["timestamp"] + closes = result["indicators"]["quote"][0]["close"] + return [ + (dt.datetime.utcfromtimestamp(ts), float(close)) + for ts, close in zip(timestamps, closes) + if close is not None + ] + + +def backfill_market_history(session: Session) -> int: + """Runs once per fresh deployment (skipped for any symbol that already + has price history — including from a prior backfill), so the app + doesn't start with an empty Economic Incident History and a week-long + wait for real polling to accumulate enough data to compare against. + + Candidate articles for anything detected here will likely be sparse or + empty: RSS feeds only carry recent items, so there's no way to backfill + a week of news to match against a week of backfilled prices. That's an + inherent limitation, not a bug — the price/incident data itself is real. + """ + added = 0 + with httpx.Client() as client: + for symbol, label, category, currency in INSTRUMENTS: + # Oldest-row check rather than "any row exists": the regular + # poll job also runs an immediate first pass at startup and may + # win the race, inserting a single "just now" row before this + # runs. That single fresh row shouldn't count as "already have + # history" and skip the backfill — only a row that's genuinely + # old (from a real prior deployment) should. + oldest = session.execute( + select(MarketPrice.recorded_at) + .where(MarketPrice.symbol == symbol) + .order_by(MarketPrice.recorded_at.asc()) + .limit(1) + ).scalar_one_or_none() + if oldest is not None and oldest <= dt.datetime.utcnow() - dt.timedelta(days=1): + continue + + try: + bars = _fetch_history_bars(client, symbol) + except Exception: + log.exception("Backfill failed for %s", symbol) + continue + + prev_row: MarketPrice | None = None + for recorded_at, price in bars: + row = MarketPrice( + symbol=symbol, + label=label, + category=category, + price=price, + currency=currency, + change_pct=None, + recorded_at=recorded_at, + ) + session.add(row) + session.flush() # so _recent_volatility below can see it as history for later bars + + if prev_row is not None: + _detect_and_record_spike(session, symbol, label, prev_row, price, recorded_at) + prev_row = row + added += 1 + + session.commit() + log.info("Backfilled %d historical price points across %d days", added, BACKFILL_DAYS) + return added + + def _score_article(article: Article, keywords: list[str], country: str | None) -> int: text = f"{article.title} {article.summary}".lower() score = sum(1 for kw in keywords if kw in text) @@ -218,6 +303,19 @@ def _detect_and_record_spike( ) + +# Whether the most recent poll attempt for a symbol succeeded — surfaced via +# /api/markets/latest so a genuinely broken feed (as opposed to a merely +# stale-by-design one, see FROZEN_SYMBOLS) is visible in the UI rather than +# silently showing an old price forever. In-memory only, reset each poll; +# not persisted since it's a "right now" health signal, not history. +_fetch_errors: dict[str, bool] = {} + + +def has_fetch_error(symbol: str) -> bool: + return _fetch_errors.get(symbol, False) + + def poll_markets(session: Session) -> int: added = 0 with httpx.Client() as client: @@ -225,8 +323,10 @@ def poll_markets(session: Session) -> int: try: result = _fetch_one(client, symbol) if result is None: + _fetch_errors[symbol] = True continue price, change_pct = result + recorded_at = dt.datetime.utcnow() prev = session.execute( @@ -248,9 +348,11 @@ def poll_markets(session: Session) -> int: ) ) _detect_and_record_spike(session, symbol, label, prev, price, recorded_at) + _fetch_errors[symbol] = False added += 1 except Exception: log.exception("Failed to fetch %s", symbol) + _fetch_errors[symbol] = True session.commit() return added diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py index fafde33..1db21c5 100644 --- a/backend/app/scheduler.py +++ b/backend/app/scheduler.py @@ -1,13 +1,15 @@ +import datetime as dt import logging from apscheduler.schedulers.background import BackgroundScheduler +from apscheduler.triggers.date import DateTrigger from .conflict import poll_conflict_events from .config import settings from .db import SessionLocal from .flights import poll_flights from .ingest import fetch_all -from .markets import poll_markets +from .markets import backfill_market_history, poll_markets log = logging.getLogger("newsatlas.scheduler") @@ -44,6 +46,15 @@ def _run_flights_job() -> None: log.info("Flight poll complete: %d aircraft", count) +def _run_market_backfill_job() -> None: + session = SessionLocal() + try: + added = backfill_market_history(session) + log.info("Market history backfill complete: %d historical points", added) + finally: + session.close() + + def start_scheduler() -> BackgroundScheduler: scheduler = BackgroundScheduler(timezone="UTC") scheduler.add_job(_run_rss_job, "interval", minutes=settings.rss_poll_minutes, next_run_time=None) @@ -52,12 +63,17 @@ def start_scheduler() -> BackgroundScheduler: scheduler.add_job(_run_flights_job, "interval", seconds=settings.flights_poll_seconds, next_run_time=None) scheduler.start() - # Kick off an immediate first run of each job in the background so the - # globe isn't empty while waiting for the first interval to elapse. - import datetime as dt - + # Kick off an immediate first run of each recurring job in the + # background so the globe isn't empty while waiting for the first + # interval to elapse. now = dt.datetime.utcnow() for job in scheduler.get_jobs(): job.modify(next_run_time=now) + # One-time (non-recurring) job: backfills a week of market history so + # the Economic Incident History view isn't empty on a fresh deployment. + # Separate from the loop above — it must run exactly once, not on + # market_poll_minutes' schedule. + scheduler.add_job(_run_market_backfill_job, trigger=DateTrigger(run_date=now), id="market_backfill") + return scheduler diff --git a/frontend/css/style.css b/frontend/css/style.css index bb454e7..b6ac169 100644 --- a/frontend/css/style.css +++ b/frontend/css/style.css @@ -192,6 +192,44 @@ html, body { cursor: pointer; } .market-item:hover { background: var(--c-panel-hover); } + +.market-item.frozen, .market-item.error { position: relative; } +.market-item.frozen > .market-row, .market-item.error > .market-row, +.market-item.frozen > .spark-container, .market-item.error > .spark-container { opacity: 0.35; } +.frozen-overlay, .error-overlay { + position: absolute; + inset: 0; + z-index: 3; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + gap: 2px; + background: rgba(11, 20, 36, 0.55); + border-radius: 6px; + font-size: 13px; + font-weight: 700; + letter-spacing: 0.08em; + cursor: help; + text-align: center; +} +.frozen-overlay .overlay-name, .error-overlay .overlay-name { + font-size: 10px; + font-weight: 500; + letter-spacing: normal; + text-shadow: none; + opacity: 0.85; +} +.frozen-overlay { + border: 1px solid #4fd1e6; + color: #9fe8f5; + text-shadow: 0 0 6px rgba(79, 209, 230, 0.6); +} +.error-overlay { + border: 1px solid var(--c-red); + color: #ffb3b3; + text-shadow: 0 0 6px rgba(245, 5, 5, 0.6); +} .market-row { display: flex; justify-content: space-between; align-items: baseline; font-size: 13px; } .market-row .label { color: var(--c-text); } .market-row .price { font-weight: 600; } @@ -273,12 +311,19 @@ html, body { } .econ-incident { padding: 14px 0; border-bottom: 1px solid #241533; } +.incident-datetime { + font-size: 21px; + font-weight: 700; + color: var(--c-magenta); + text-shadow: 0 0 10px rgba(255, 51, 255, 0.35); + margin-bottom: 6px; +} .econ-incident h3 { margin: 0 0 4px; font-size: 14px; color: #f2dede; display: flex; flex-wrap: wrap; gap: 8px 14px; font-weight: 600; } .instrument-chip { display: inline-flex; gap: 6px; align-items: baseline; } .econ-incident .window { font-size: 11px; color: var(--c-text-muted); margin-bottom: 8px; } .econ-incident .articles { margin-top: 8px; } -.overlay-toggle-btn { +.overlay-toggle-btn, .articles-toggle-btn { background: var(--c-tag-bg); border: 1px solid var(--c-panel-border); color: var(--c-dark-soft); @@ -286,9 +331,9 @@ html, body { padding: 4px 10px; font-size: 11px; cursor: pointer; - margin: 4px 0 6px; + margin: 4px 6px 6px 0; } -.overlay-toggle-btn:hover { background: #3d2260; color: #fff; } +.overlay-toggle-btn:hover, .articles-toggle-btn:hover { background: #3d2260; color: #fff; } .overlay-charts { margin: 6px 0 12px; } .overlay-charts h4 { font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em; color: var(--c-text-muted); margin: 12px 0 4px; font-weight: 600; } diff --git a/frontend/js/app.js b/frontend/js/app.js index 191e83f..26485e0 100644 --- a/frontend/js/app.js +++ b/frontend/js/app.js @@ -731,7 +731,20 @@ async function loadMarkets() { const dir = r.change_pct > 0 ? "up" : r.change_pct < 0 ? "down" : ""; const arrow = r.change_pct > 0 ? "▲" : r.change_pct < 0 ? "▼" : ""; return ` -
+
+ ${ + r.frozen + ? `
+
❄ FROZEN
+
${escapeHtml(r.label)}
+
` + : r.error + ? `
+
⚠ ERROR
+
${escapeHtml(r.label)}
+
` + : "" + }
${escapeHtml(r.label)} ${r.price.toLocaleString(undefined, { maximumFractionDigits: 2 })} ${r.currency} @@ -943,18 +956,30 @@ document.getElementById("econHistorySymbolFilter").onchange = () => loadEconHist // duplicate listeners: expand/collapse a card's comparison charts, fetching // them lazily on first expand only. document.getElementById("econHistoryList").addEventListener("click", (ev) => { - const btn = ev.target.closest(".overlay-toggle-btn"); - if (!btn) return; - const idx = btn.dataset.idx; - const wrap = document.getElementById(`overlay-charts-${idx}`); - if (wrap.dataset.loaded) { + const overlayBtn = ev.target.closest(".overlay-toggle-btn"); + if (overlayBtn) { + const idx = overlayBtn.dataset.idx; + const wrap = document.getElementById(`overlay-charts-${idx}`); + if (wrap.dataset.loaded) { + wrap.classList.toggle("hidden"); + overlayBtn.textContent = wrap.classList.contains("hidden") ? "📈 Show comparison charts" : "📉 Hide comparison charts"; + } else { + overlayBtn.textContent = "Loading charts…"; + loadIncidentOverlayCharts(currentIncidents[idx], idx).then(() => { + overlayBtn.textContent = "📉 Hide comparison charts"; + }); + } + return; + } + + const articlesBtn = ev.target.closest(".articles-toggle-btn"); + if (articlesBtn) { + const idx = articlesBtn.dataset.idx; + const wrap = document.getElementById(`articles-${idx}`); + const count = currentIncidents[idx].candidate_articles.length; + const noun = `stor${count === 1 ? "y" : "ies"}`; wrap.classList.toggle("hidden"); - btn.textContent = wrap.classList.contains("hidden") ? "📈 Show comparison charts" : "📉 Hide comparison charts"; - } else { - btn.textContent = "Loading charts…"; - loadIncidentOverlayCharts(currentIncidents[idx], idx).then(() => { - btn.textContent = "📉 Hide comparison charts"; - }); + articlesBtn.textContent = wrap.classList.contains("hidden") ? `📄 Show ${count} ${noun}` : `📄 Hide ${count} ${noun}`; } }); @@ -1012,8 +1037,17 @@ function econIncidentHtml(inc, idx) { const articlesHtml = inc.candidate_articles.length ? inc.candidate_articles.map(articleItemHtml).join("") : `

No stories found published in that window.

`; + const incidentDateTime = new Date(inc.detected_at).toLocaleString(undefined, { + weekday: "short", + year: "numeric", + month: "short", + day: "numeric", + hour: "2-digit", + minute: "2-digit", + }); return `
+
${incidentDateTime}

${instrumentsHtml}

${fmtHour(inc.window_start)} – ${fmtHour(inc.window_end)} @@ -1028,7 +1062,8 @@ function econIncidentHtml(inc, idx) {

This incident's instruments only

-
${articlesHtml}
+ +
`; }