Backfill a week of market history on first startup; UI polish

Backend: backfill_market_history() fetches 7 days of hourly price bars per
instrument on first container start (skipped once an instrument has more
than a day of real history, so restarts don't re-run it) and runs the same
spike-detection logic retroactively, so the Economic Incident History view
is already populated instead of empty for a week. Runs as a genuine
one-time APScheduler job, separate from the recurring polls.

Also: FROZEN/ERROR overlay badges on market ticker items (MOEX's known-stale
Yahoo feed, and any instrument whose last poll actually failed), each
labeled with the instrument name; a big magenta date/time heading per
economic incident; and the incident article list is now a collapsed
fold-out (some incidents merge to 60 stories) instead of always fully
expanded.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Um48tTvZDrEgDeweFyhPYC

README.md

@@ -96,6 +96,18 @@ On first boot the globe will be empty for a few seconds until the first RSS
 poll completes — hit "Refresh now" if you don't want to wait for the
 10-minute interval.
 
+The Economic Incident History, on the other hand, doesn't start empty: on
+first startup the backend backfills a week of hourly price history per
+instrument (`backfill_market_history()` in `markets.py`) and runs spike
+detection retroactively across it, so there's already a populated week of
+incidents rather than a week-long wait for live polling to accumulate
+enough data. It's a one-time thing — skipped on later restarts once an
+instrument has more than a day of real history. Backfilled incidents will
+generally have few or no candidate articles, since RSS feeds only carry
+recent items and there's no way to backfill a week of news to match a week
+of backfilled prices — the price/incident data itself is real, the article
+correlation just can't reach that far back.
+
 ## Optional API keys (`.env`)
 
 Everything works with `.env` left blank except that the weather and conflict

backend/app/main.py

@@ -15,7 +15,7 @@ from .config import settings
 from .db import get_session, init_db, SessionLocal
 from .flights import get_latest_flights
 from .ingest import fetch_all
-from .markets import poll_markets, merge_spikes_into_incidents, INSTRUMENTS
+from .markets import poll_markets, merge_spikes_into_incidents, has_fetch_error, FROZEN_SYMBOLS, INSTRUMENTS
 from .conflict import enabled as conflict_enabled, poll_conflict_events
 from .models import Article, ConflictEvent, MarketPrice, MarketSpike
 from .scheduler import start_scheduler
@@ -160,6 +160,10 @@ def api_markets_latest():
                         "currency": row.currency,
                         "change_pct": row.change_pct,
                         "recorded_at": row.recorded_at.isoformat(),
+                        "frozen": symbol in FROZEN_SYMBOLS,
+                        # FROZEN already explains a stale price for that one
+                        # documented instrument — don't also flag it ERROR.
+                        "error": symbol not in FROZEN_SYMBOLS and has_fetch_error(symbol),
                     }
                 )
         return out

backend/app/markets.py

@@ -21,6 +21,11 @@ INCIDENT_MERGE_HOURS = 1.0
 
 log = logging.getLogger("newsatlas.markets")
 
+# Instruments whose feed is known stale/non-live (see the comment on
+# IMOEX.ME below) — surfaced via /api/markets/latest so the UI can mark
+# them clearly rather than presenting old data as a live price.
+FROZEN_SYMBOLS = {"IMOEX.ME"}
+
 # A spread of major indices as rough proxies for national/regional economies,
 # plus crude oil benchmarks.
 INSTRUMENTS = [
@@ -101,6 +106,86 @@ def _fetch_one(client: httpx.Client, symbol: str) -> tuple[float, float | None]
     return float(price), change_pct
 
 
+BACKFILL_DAYS = 7
+
+
+def _fetch_history_bars(client: httpx.Client, symbol: str) -> list[tuple[dt.datetime, float]]:
+    resp = client.get(
+        CHART_URL.format(symbol=symbol),
+        params={"range": f"{BACKFILL_DAYS}d", "interval": "60m"},
+        headers=_HEADERS,
+        timeout=20,
+    )
+    resp.raise_for_status()
+    result = resp.json()["chart"]["result"][0]
+    timestamps = result["timestamp"]
+    closes = result["indicators"]["quote"][0]["close"]
+    return [
+        (dt.datetime.utcfromtimestamp(ts), float(close))
+        for ts, close in zip(timestamps, closes)
+        if close is not None
+    ]
+
+
+def backfill_market_history(session: Session) -> int:
+    """Runs once per fresh deployment (skipped for any symbol that already
+    has price history — including from a prior backfill), so the app
+    doesn't start with an empty Economic Incident History and a week-long
+    wait for real polling to accumulate enough data to compare against.
+
+    Candidate articles for anything detected here will likely be sparse or
+    empty: RSS feeds only carry recent items, so there's no way to backfill
+    a week of news to match against a week of backfilled prices. That's an
+    inherent limitation, not a bug — the price/incident data itself is real.
+    """
+    added = 0
+    with httpx.Client() as client:
+        for symbol, label, category, currency in INSTRUMENTS:
+            # Oldest-row check rather than "any row exists": the regular
+            # poll job also runs an immediate first pass at startup and may
+            # win the race, inserting a single "just now" row before this
+            # runs. That single fresh row shouldn't count as "already have
+            # history" and skip the backfill — only a row that's genuinely
+            # old (from a real prior deployment) should.
+            oldest = session.execute(
+                select(MarketPrice.recorded_at)
+                .where(MarketPrice.symbol == symbol)
+                .order_by(MarketPrice.recorded_at.asc())
+                .limit(1)
+            ).scalar_one_or_none()
+            if oldest is not None and oldest <= dt.datetime.utcnow() - dt.timedelta(days=1):
+                continue
+
+            try:
+                bars = _fetch_history_bars(client, symbol)
+            except Exception:
+                log.exception("Backfill failed for %s", symbol)
+                continue
+
+            prev_row: MarketPrice | None = None
+            for recorded_at, price in bars:
+                row = MarketPrice(
+                    symbol=symbol,
+                    label=label,
+                    category=category,
+                    price=price,
+                    currency=currency,
+                    change_pct=None,
+                    recorded_at=recorded_at,
+                )
+                session.add(row)
+                session.flush()  # so _recent_volatility below can see it as history for later bars
+
+                if prev_row is not None:
+                    _detect_and_record_spike(session, symbol, label, prev_row, price, recorded_at)
+                prev_row = row
+                added += 1
+
+    session.commit()
+    log.info("Backfilled %d historical price points across %d days", added, BACKFILL_DAYS)
+    return added
+
+
 def _score_article(article: Article, keywords: list[str], country: str | None) -> int:
     text = f"{article.title} {article.summary}".lower()
     score = sum(1 for kw in keywords if kw in text)
@@ -218,6 +303,19 @@ def _detect_and_record_spike(
     )
 
 
+
+# Whether the most recent poll attempt for a symbol succeeded — surfaced via
+# /api/markets/latest so a genuinely broken feed (as opposed to a merely
+# stale-by-design one, see FROZEN_SYMBOLS) is visible in the UI rather than
+# silently showing an old price forever. In-memory only, reset each poll;
+# not persisted since it's a "right now" health signal, not history.
+_fetch_errors: dict[str, bool] = {}
+
+
+def has_fetch_error(symbol: str) -> bool:
+    return _fetch_errors.get(symbol, False)
+
+
 def poll_markets(session: Session) -> int:
     added = 0
     with httpx.Client() as client:
@@ -225,8 +323,10 @@ def poll_markets(session: Session) -> int:
             try:
                 result = _fetch_one(client, symbol)
                 if result is None:
+                    _fetch_errors[symbol] = True
                     continue
                 price, change_pct = result
+
                 recorded_at = dt.datetime.utcnow()
 
                 prev = session.execute(
@@ -248,9 +348,11 @@ def poll_markets(session: Session) -> int:
                     )
                 )
                 _detect_and_record_spike(session, symbol, label, prev, price, recorded_at)
+                _fetch_errors[symbol] = False
                 added += 1
             except Exception:
                 log.exception("Failed to fetch %s", symbol)
+                _fetch_errors[symbol] = True
 
     session.commit()
     return added

backend/app/scheduler.py

@@ -1,13 +1,15 @@
+import datetime as dt
 import logging
 
 from apscheduler.schedulers.background import BackgroundScheduler
+from apscheduler.triggers.date import DateTrigger
 
 from .conflict import poll_conflict_events
 from .config import settings
 from .db import SessionLocal
 from .flights import poll_flights
 from .ingest import fetch_all
-from .markets import poll_markets
+from .markets import backfill_market_history, poll_markets
 
 log = logging.getLogger("newsatlas.scheduler")
 
@@ -44,6 +46,15 @@ def _run_flights_job() -> None:
     log.info("Flight poll complete: %d aircraft", count)
 
 
+def _run_market_backfill_job() -> None:
+    session = SessionLocal()
+    try:
+        added = backfill_market_history(session)
+        log.info("Market history backfill complete: %d historical points", added)
+    finally:
+        session.close()
+
+
 def start_scheduler() -> BackgroundScheduler:
     scheduler = BackgroundScheduler(timezone="UTC")
     scheduler.add_job(_run_rss_job, "interval", minutes=settings.rss_poll_minutes, next_run_time=None)
@@ -52,12 +63,17 @@ def start_scheduler() -> BackgroundScheduler:
     scheduler.add_job(_run_flights_job, "interval", seconds=settings.flights_poll_seconds, next_run_time=None)
     scheduler.start()
 
-    # Kick off an immediate first run of each job in the background so the
+    # Kick off an immediate first run of each recurring job in the
-    # globe isn't empty while waiting for the first interval to elapse.
+    # background so the globe isn't empty while waiting for the first
-    import datetime as dt
+    # interval to elapse.
-
     now = dt.datetime.utcnow()
     for job in scheduler.get_jobs():
         job.modify(next_run_time=now)
 
+    # One-time (non-recurring) job: backfills a week of market history so
+    # the Economic Incident History view isn't empty on a fresh deployment.
+    # Separate from the loop above — it must run exactly once, not on
+    # market_poll_minutes' schedule.
+    scheduler.add_job(_run_market_backfill_job, trigger=DateTrigger(run_date=now), id="market_backfill")
+
     return scheduler

frontend/css/style.css

@@ -192,6 +192,44 @@ html, body {
   cursor: pointer;
 }
 .market-item:hover { background: var(--c-panel-hover); }
+
+.market-item.frozen, .market-item.error { position: relative; }
+.market-item.frozen > .market-row, .market-item.error > .market-row,
+.market-item.frozen > .spark-container, .market-item.error > .spark-container { opacity: 0.35; }
+.frozen-overlay, .error-overlay {
+  position: absolute;
+  inset: 0;
+  z-index: 3;
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  justify-content: center;
+  gap: 2px;
+  background: rgba(11, 20, 36, 0.55);
+  border-radius: 6px;
+  font-size: 13px;
+  font-weight: 700;
+  letter-spacing: 0.08em;
+  cursor: help;
+  text-align: center;
+}
+.frozen-overlay .overlay-name, .error-overlay .overlay-name {
+  font-size: 10px;
+  font-weight: 500;
+  letter-spacing: normal;
+  text-shadow: none;
+  opacity: 0.85;
+}
+.frozen-overlay {
+  border: 1px solid #4fd1e6;
+  color: #9fe8f5;
+  text-shadow: 0 0 6px rgba(79, 209, 230, 0.6);
+}
+.error-overlay {
+  border: 1px solid var(--c-red);
+  color: #ffb3b3;
+  text-shadow: 0 0 6px rgba(245, 5, 5, 0.6);
+}
 .market-row { display: flex; justify-content: space-between; align-items: baseline; font-size: 13px; }
 .market-row .label { color: var(--c-text); }
 .market-row .price { font-weight: 600; }
@@ -273,12 +311,19 @@ html, body {
 }
 
 .econ-incident { padding: 14px 0; border-bottom: 1px solid #241533; }
+.incident-datetime {
+  font-size: 21px;
+  font-weight: 700;
+  color: var(--c-magenta);
+  text-shadow: 0 0 10px rgba(255, 51, 255, 0.35);
+  margin-bottom: 6px;
+}
 .econ-incident h3 { margin: 0 0 4px; font-size: 14px; color: #f2dede; display: flex; flex-wrap: wrap; gap: 8px 14px; font-weight: 600; }
 .instrument-chip { display: inline-flex; gap: 6px; align-items: baseline; }
 .econ-incident .window { font-size: 11px; color: var(--c-text-muted); margin-bottom: 8px; }
 .econ-incident .articles { margin-top: 8px; }
 
-.overlay-toggle-btn {
+.overlay-toggle-btn, .articles-toggle-btn {
   background: var(--c-tag-bg);
   border: 1px solid var(--c-panel-border);
   color: var(--c-dark-soft);
@@ -286,9 +331,9 @@ html, body {
   padding: 4px 10px;
   font-size: 11px;
   cursor: pointer;
-  margin: 4px 0 6px;
+  margin: 4px 6px 6px 0;
 }
-.overlay-toggle-btn:hover { background: #3d2260; color: #fff; }
+.overlay-toggle-btn:hover, .articles-toggle-btn:hover { background: #3d2260; color: #fff; }
 
 .overlay-charts { margin: 6px 0 12px; }
 .overlay-charts h4 { font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em; color: var(--c-text-muted); margin: 12px 0 4px; font-weight: 600; }

frontend/js/app.js

@@ -731,7 +731,20 @@ async function loadMarkets() {
         const dir = r.change_pct > 0 ? "up" : r.change_pct < 0 ? "down" : "";
         const arrow = r.change_pct > 0 ? "▲" : r.change_pct < 0 ? "▼" : "";
         return `
-        <div class="market-item" data-symbol="${r.symbol}" data-idx="${i}">
+        <div class="market-item${r.frozen ? " frozen" : r.error ? " error" : ""}" data-symbol="${r.symbol}" data-idx="${i}">
+          ${
+            r.frozen
+              ? `<div class="frozen-overlay" title="Yahoo's feed for this instrument appears frozen since mid-2022 — not a live price, shown for continuity only">
+                   <div>❄ FROZEN</div>
+                   <div class="overlay-name">${escapeHtml(r.label)}</div>
+                 </div>`
+              : r.error
+                ? `<div class="error-overlay" title="The last poll for this instrument failed — showing the most recent successful price, which may be stale">
+                     <div>⚠ ERROR</div>
+                     <div class="overlay-name">${escapeHtml(r.label)}</div>
+                   </div>`
+                : ""
+          }
           <div class="market-row">
             <span class="label">${escapeHtml(r.label)}</span>
             <span class="price">${r.price.toLocaleString(undefined, { maximumFractionDigits: 2 })} ${r.currency}</span>
@@ -943,19 +956,31 @@ document.getElementById("econHistorySymbolFilter").onchange = () => loadEconHist
 // duplicate listeners: expand/collapse a card's comparison charts, fetching
 // them lazily on first expand only.
 document.getElementById("econHistoryList").addEventListener("click", (ev) => {
-  const btn = ev.target.closest(".overlay-toggle-btn");
+  const overlayBtn = ev.target.closest(".overlay-toggle-btn");
-  if (!btn) return;
+  if (overlayBtn) {
-  const idx = btn.dataset.idx;
+    const idx = overlayBtn.dataset.idx;
     const wrap = document.getElementById(`overlay-charts-${idx}`);
     if (wrap.dataset.loaded) {
       wrap.classList.toggle("hidden");
-    btn.textContent = wrap.classList.contains("hidden") ? "📈 Show comparison charts" : "📉 Hide comparison charts";
+      overlayBtn.textContent = wrap.classList.contains("hidden") ? "📈 Show comparison charts" : "📉 Hide comparison charts";
     } else {
-    btn.textContent = "Loading charts…";
+      overlayBtn.textContent = "Loading charts…";
       loadIncidentOverlayCharts(currentIncidents[idx], idx).then(() => {
-      btn.textContent = "📉 Hide comparison charts";
+        overlayBtn.textContent = "📉 Hide comparison charts";
       });
     }
+    return;
+  }
+
+  const articlesBtn = ev.target.closest(".articles-toggle-btn");
+  if (articlesBtn) {
+    const idx = articlesBtn.dataset.idx;
+    const wrap = document.getElementById(`articles-${idx}`);
+    const count = currentIncidents[idx].candidate_articles.length;
+    const noun = `stor${count === 1 ? "y" : "ies"}`;
+    wrap.classList.toggle("hidden");
+    articlesBtn.textContent = wrap.classList.contains("hidden") ? `📄 Show ${count} ${noun}` : `📄 Hide ${count} ${noun}`;
+  }
 });
 
 async function loadEconHistory() {
@@ -1012,8 +1037,17 @@ function econIncidentHtml(inc, idx) {
   const articlesHtml = inc.candidate_articles.length
     ? inc.candidate_articles.map(articleItemHtml).join("")
     : `<p class="subtle">No stories found published in that window.</p>`;
+  const incidentDateTime = new Date(inc.detected_at).toLocaleString(undefined, {
+    weekday: "short",
+    year: "numeric",
+    month: "short",
+    day: "numeric",
+    hour: "2-digit",
+    minute: "2-digit",
+  });
   return `
     <div class="econ-incident">
+      <div class="incident-datetime">${incidentDateTime}</div>
       <h3>${instrumentsHtml}</h3>
       <div class="window">
         ${fmtHour(inc.window_start)} – ${fmtHour(inc.window_end)}
@@ -1028,7 +1062,8 @@ function econIncidentHtml(inc, idx) {
         <h4>This incident's instruments only</h4>
         <div class="overlay-chart-container" id="overlay-inc-${idx}"></div>
       </div>
-      <div class="articles">${articlesHtml}</div>
+      <button class="articles-toggle-btn" data-idx="${idx}">📄 Show ${inc.candidate_articles.length} stor${inc.candidate_articles.length === 1 ? "y" : "ies"}</button>
+      <div class="articles hidden" id="articles-${idx}">${articlesHtml}</div>
     </div>
   `;
 }