Backfill a week of market history on first startup; UI polish
Backend: backfill_market_history() fetches 7 days of hourly price bars per instrument on first container start (skipped once an instrument has more than a day of real history, so restarts don't re-run it) and runs the same spike-detection logic retroactively, so the Economic Incident History view is already populated instead of empty for a week. Runs as a genuine one-time APScheduler job, separate from the recurring polls. Also: FROZEN/ERROR overlay badges on market ticker items (MOEX's known-stale Yahoo feed, and any instrument whose last poll actually failed), each labeled with the instrument name; a big magenta date/time heading per economic incident; and the incident article list is now a collapsed fold-out (some incidents merge to 60 stories) instead of always fully expanded. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Um48tTvZDrEgDeweFyhPYCmain
parent
54022ad66f
commit
c3a266f6d9
12
README.md
12
README.md
|
|
@ -96,6 +96,18 @@ On first boot the globe will be empty for a few seconds until the first RSS
|
|||
poll completes — hit "Refresh now" if you don't want to wait for the
|
||||
10-minute interval.
|
||||
|
||||
The Economic Incident History, on the other hand, doesn't start empty: on
|
||||
first startup the backend backfills a week of hourly price history per
|
||||
instrument (`backfill_market_history()` in `markets.py`) and runs spike
|
||||
detection retroactively across it, so there's already a populated week of
|
||||
incidents rather than a week-long wait for live polling to accumulate
|
||||
enough data. It's a one-time thing — skipped on later restarts once an
|
||||
instrument has more than a day of real history. Backfilled incidents will
|
||||
generally have few or no candidate articles, since RSS feeds only carry
|
||||
recent items and there's no way to backfill a week of news to match a week
|
||||
of backfilled prices — the price/incident data itself is real, the article
|
||||
correlation just can't reach that far back.
|
||||
|
||||
## Optional API keys (`.env`)
|
||||
|
||||
Everything works with `.env` left blank except that the weather and conflict
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from .config import settings
|
|||
from .db import get_session, init_db, SessionLocal
|
||||
from .flights import get_latest_flights
|
||||
from .ingest import fetch_all
|
||||
from .markets import poll_markets, merge_spikes_into_incidents, INSTRUMENTS
|
||||
from .markets import poll_markets, merge_spikes_into_incidents, has_fetch_error, FROZEN_SYMBOLS, INSTRUMENTS
|
||||
from .conflict import enabled as conflict_enabled, poll_conflict_events
|
||||
from .models import Article, ConflictEvent, MarketPrice, MarketSpike
|
||||
from .scheduler import start_scheduler
|
||||
|
|
@ -160,6 +160,10 @@ def api_markets_latest():
|
|||
"currency": row.currency,
|
||||
"change_pct": row.change_pct,
|
||||
"recorded_at": row.recorded_at.isoformat(),
|
||||
"frozen": symbol in FROZEN_SYMBOLS,
|
||||
# FROZEN already explains a stale price for that one
|
||||
# documented instrument — don't also flag it ERROR.
|
||||
"error": symbol not in FROZEN_SYMBOLS and has_fetch_error(symbol),
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
|
|
|||
|
|
@ -21,6 +21,11 @@ INCIDENT_MERGE_HOURS = 1.0
|
|||
|
||||
log = logging.getLogger("newsatlas.markets")
|
||||
|
||||
# Instruments whose feed is known stale/non-live (see the comment on
|
||||
# IMOEX.ME below) — surfaced via /api/markets/latest so the UI can mark
|
||||
# them clearly rather than presenting old data as a live price.
|
||||
FROZEN_SYMBOLS = {"IMOEX.ME"}
|
||||
|
||||
# A spread of major indices as rough proxies for national/regional economies,
|
||||
# plus crude oil benchmarks.
|
||||
INSTRUMENTS = [
|
||||
|
|
@ -101,6 +106,86 @@ def _fetch_one(client: httpx.Client, symbol: str) -> tuple[float, float | None]
|
|||
return float(price), change_pct
|
||||
|
||||
|
||||
BACKFILL_DAYS = 7
|
||||
|
||||
|
||||
def _fetch_history_bars(client: httpx.Client, symbol: str) -> list[tuple[dt.datetime, float]]:
|
||||
resp = client.get(
|
||||
CHART_URL.format(symbol=symbol),
|
||||
params={"range": f"{BACKFILL_DAYS}d", "interval": "60m"},
|
||||
headers=_HEADERS,
|
||||
timeout=20,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
result = resp.json()["chart"]["result"][0]
|
||||
timestamps = result["timestamp"]
|
||||
closes = result["indicators"]["quote"][0]["close"]
|
||||
return [
|
||||
(dt.datetime.utcfromtimestamp(ts), float(close))
|
||||
for ts, close in zip(timestamps, closes)
|
||||
if close is not None
|
||||
]
|
||||
|
||||
|
||||
def backfill_market_history(session: Session) -> int:
|
||||
"""Runs once per fresh deployment (skipped for any symbol that already
|
||||
has price history — including from a prior backfill), so the app
|
||||
doesn't start with an empty Economic Incident History and a week-long
|
||||
wait for real polling to accumulate enough data to compare against.
|
||||
|
||||
Candidate articles for anything detected here will likely be sparse or
|
||||
empty: RSS feeds only carry recent items, so there's no way to backfill
|
||||
a week of news to match against a week of backfilled prices. That's an
|
||||
inherent limitation, not a bug — the price/incident data itself is real.
|
||||
"""
|
||||
added = 0
|
||||
with httpx.Client() as client:
|
||||
for symbol, label, category, currency in INSTRUMENTS:
|
||||
# Oldest-row check rather than "any row exists": the regular
|
||||
# poll job also runs an immediate first pass at startup and may
|
||||
# win the race, inserting a single "just now" row before this
|
||||
# runs. That single fresh row shouldn't count as "already have
|
||||
# history" and skip the backfill — only a row that's genuinely
|
||||
# old (from a real prior deployment) should.
|
||||
oldest = session.execute(
|
||||
select(MarketPrice.recorded_at)
|
||||
.where(MarketPrice.symbol == symbol)
|
||||
.order_by(MarketPrice.recorded_at.asc())
|
||||
.limit(1)
|
||||
).scalar_one_or_none()
|
||||
if oldest is not None and oldest <= dt.datetime.utcnow() - dt.timedelta(days=1):
|
||||
continue
|
||||
|
||||
try:
|
||||
bars = _fetch_history_bars(client, symbol)
|
||||
except Exception:
|
||||
log.exception("Backfill failed for %s", symbol)
|
||||
continue
|
||||
|
||||
prev_row: MarketPrice | None = None
|
||||
for recorded_at, price in bars:
|
||||
row = MarketPrice(
|
||||
symbol=symbol,
|
||||
label=label,
|
||||
category=category,
|
||||
price=price,
|
||||
currency=currency,
|
||||
change_pct=None,
|
||||
recorded_at=recorded_at,
|
||||
)
|
||||
session.add(row)
|
||||
session.flush() # so _recent_volatility below can see it as history for later bars
|
||||
|
||||
if prev_row is not None:
|
||||
_detect_and_record_spike(session, symbol, label, prev_row, price, recorded_at)
|
||||
prev_row = row
|
||||
added += 1
|
||||
|
||||
session.commit()
|
||||
log.info("Backfilled %d historical price points across %d days", added, BACKFILL_DAYS)
|
||||
return added
|
||||
|
||||
|
||||
def _score_article(article: Article, keywords: list[str], country: str | None) -> int:
|
||||
text = f"{article.title} {article.summary}".lower()
|
||||
score = sum(1 for kw in keywords if kw in text)
|
||||
|
|
@ -218,6 +303,19 @@ def _detect_and_record_spike(
|
|||
)
|
||||
|
||||
|
||||
|
||||
# Whether the most recent poll attempt for a symbol succeeded — surfaced via
|
||||
# /api/markets/latest so a genuinely broken feed (as opposed to a merely
|
||||
# stale-by-design one, see FROZEN_SYMBOLS) is visible in the UI rather than
|
||||
# silently showing an old price forever. In-memory only, reset each poll;
|
||||
# not persisted since it's a "right now" health signal, not history.
|
||||
_fetch_errors: dict[str, bool] = {}
|
||||
|
||||
|
||||
def has_fetch_error(symbol: str) -> bool:
|
||||
return _fetch_errors.get(symbol, False)
|
||||
|
||||
|
||||
def poll_markets(session: Session) -> int:
|
||||
added = 0
|
||||
with httpx.Client() as client:
|
||||
|
|
@ -225,8 +323,10 @@ def poll_markets(session: Session) -> int:
|
|||
try:
|
||||
result = _fetch_one(client, symbol)
|
||||
if result is None:
|
||||
_fetch_errors[symbol] = True
|
||||
continue
|
||||
price, change_pct = result
|
||||
|
||||
recorded_at = dt.datetime.utcnow()
|
||||
|
||||
prev = session.execute(
|
||||
|
|
@ -248,9 +348,11 @@ def poll_markets(session: Session) -> int:
|
|||
)
|
||||
)
|
||||
_detect_and_record_spike(session, symbol, label, prev, price, recorded_at)
|
||||
_fetch_errors[symbol] = False
|
||||
added += 1
|
||||
except Exception:
|
||||
log.exception("Failed to fetch %s", symbol)
|
||||
_fetch_errors[symbol] = True
|
||||
|
||||
session.commit()
|
||||
return added
|
||||
|
|
|
|||
|
|
@ -1,13 +1,15 @@
|
|||
import datetime as dt
|
||||
import logging
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
from apscheduler.triggers.date import DateTrigger
|
||||
|
||||
from .conflict import poll_conflict_events
|
||||
from .config import settings
|
||||
from .db import SessionLocal
|
||||
from .flights import poll_flights
|
||||
from .ingest import fetch_all
|
||||
from .markets import poll_markets
|
||||
from .markets import backfill_market_history, poll_markets
|
||||
|
||||
log = logging.getLogger("newsatlas.scheduler")
|
||||
|
||||
|
|
@ -44,6 +46,15 @@ def _run_flights_job() -> None:
|
|||
log.info("Flight poll complete: %d aircraft", count)
|
||||
|
||||
|
||||
def _run_market_backfill_job() -> None:
|
||||
session = SessionLocal()
|
||||
try:
|
||||
added = backfill_market_history(session)
|
||||
log.info("Market history backfill complete: %d historical points", added)
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def start_scheduler() -> BackgroundScheduler:
|
||||
scheduler = BackgroundScheduler(timezone="UTC")
|
||||
scheduler.add_job(_run_rss_job, "interval", minutes=settings.rss_poll_minutes, next_run_time=None)
|
||||
|
|
@ -52,12 +63,17 @@ def start_scheduler() -> BackgroundScheduler:
|
|||
scheduler.add_job(_run_flights_job, "interval", seconds=settings.flights_poll_seconds, next_run_time=None)
|
||||
scheduler.start()
|
||||
|
||||
# Kick off an immediate first run of each job in the background so the
|
||||
# globe isn't empty while waiting for the first interval to elapse.
|
||||
import datetime as dt
|
||||
|
||||
# Kick off an immediate first run of each recurring job in the
|
||||
# background so the globe isn't empty while waiting for the first
|
||||
# interval to elapse.
|
||||
now = dt.datetime.utcnow()
|
||||
for job in scheduler.get_jobs():
|
||||
job.modify(next_run_time=now)
|
||||
|
||||
# One-time (non-recurring) job: backfills a week of market history so
|
||||
# the Economic Incident History view isn't empty on a fresh deployment.
|
||||
# Separate from the loop above — it must run exactly once, not on
|
||||
# market_poll_minutes' schedule.
|
||||
scheduler.add_job(_run_market_backfill_job, trigger=DateTrigger(run_date=now), id="market_backfill")
|
||||
|
||||
return scheduler
|
||||
|
|
|
|||
|
|
@ -192,6 +192,44 @@ html, body {
|
|||
cursor: pointer;
|
||||
}
|
||||
.market-item:hover { background: var(--c-panel-hover); }
|
||||
|
||||
.market-item.frozen, .market-item.error { position: relative; }
|
||||
.market-item.frozen > .market-row, .market-item.error > .market-row,
|
||||
.market-item.frozen > .spark-container, .market-item.error > .spark-container { opacity: 0.35; }
|
||||
.frozen-overlay, .error-overlay {
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
z-index: 3;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 2px;
|
||||
background: rgba(11, 20, 36, 0.55);
|
||||
border-radius: 6px;
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
letter-spacing: 0.08em;
|
||||
cursor: help;
|
||||
text-align: center;
|
||||
}
|
||||
.frozen-overlay .overlay-name, .error-overlay .overlay-name {
|
||||
font-size: 10px;
|
||||
font-weight: 500;
|
||||
letter-spacing: normal;
|
||||
text-shadow: none;
|
||||
opacity: 0.85;
|
||||
}
|
||||
.frozen-overlay {
|
||||
border: 1px solid #4fd1e6;
|
||||
color: #9fe8f5;
|
||||
text-shadow: 0 0 6px rgba(79, 209, 230, 0.6);
|
||||
}
|
||||
.error-overlay {
|
||||
border: 1px solid var(--c-red);
|
||||
color: #ffb3b3;
|
||||
text-shadow: 0 0 6px rgba(245, 5, 5, 0.6);
|
||||
}
|
||||
.market-row { display: flex; justify-content: space-between; align-items: baseline; font-size: 13px; }
|
||||
.market-row .label { color: var(--c-text); }
|
||||
.market-row .price { font-weight: 600; }
|
||||
|
|
@ -273,12 +311,19 @@ html, body {
|
|||
}
|
||||
|
||||
.econ-incident { padding: 14px 0; border-bottom: 1px solid #241533; }
|
||||
.incident-datetime {
|
||||
font-size: 21px;
|
||||
font-weight: 700;
|
||||
color: var(--c-magenta);
|
||||
text-shadow: 0 0 10px rgba(255, 51, 255, 0.35);
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.econ-incident h3 { margin: 0 0 4px; font-size: 14px; color: #f2dede; display: flex; flex-wrap: wrap; gap: 8px 14px; font-weight: 600; }
|
||||
.instrument-chip { display: inline-flex; gap: 6px; align-items: baseline; }
|
||||
.econ-incident .window { font-size: 11px; color: var(--c-text-muted); margin-bottom: 8px; }
|
||||
.econ-incident .articles { margin-top: 8px; }
|
||||
|
||||
.overlay-toggle-btn {
|
||||
.overlay-toggle-btn, .articles-toggle-btn {
|
||||
background: var(--c-tag-bg);
|
||||
border: 1px solid var(--c-panel-border);
|
||||
color: var(--c-dark-soft);
|
||||
|
|
@ -286,9 +331,9 @@ html, body {
|
|||
padding: 4px 10px;
|
||||
font-size: 11px;
|
||||
cursor: pointer;
|
||||
margin: 4px 0 6px;
|
||||
margin: 4px 6px 6px 0;
|
||||
}
|
||||
.overlay-toggle-btn:hover { background: #3d2260; color: #fff; }
|
||||
.overlay-toggle-btn:hover, .articles-toggle-btn:hover { background: #3d2260; color: #fff; }
|
||||
|
||||
.overlay-charts { margin: 6px 0 12px; }
|
||||
.overlay-charts h4 { font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em; color: var(--c-text-muted); margin: 12px 0 4px; font-weight: 600; }
|
||||
|
|
|
|||
|
|
@ -731,7 +731,20 @@ async function loadMarkets() {
|
|||
const dir = r.change_pct > 0 ? "up" : r.change_pct < 0 ? "down" : "";
|
||||
const arrow = r.change_pct > 0 ? "▲" : r.change_pct < 0 ? "▼" : "";
|
||||
return `
|
||||
<div class="market-item" data-symbol="${r.symbol}" data-idx="${i}">
|
||||
<div class="market-item${r.frozen ? " frozen" : r.error ? " error" : ""}" data-symbol="${r.symbol}" data-idx="${i}">
|
||||
${
|
||||
r.frozen
|
||||
? `<div class="frozen-overlay" title="Yahoo's feed for this instrument appears frozen since mid-2022 — not a live price, shown for continuity only">
|
||||
<div>❄ FROZEN</div>
|
||||
<div class="overlay-name">${escapeHtml(r.label)}</div>
|
||||
</div>`
|
||||
: r.error
|
||||
? `<div class="error-overlay" title="The last poll for this instrument failed — showing the most recent successful price, which may be stale">
|
||||
<div>⚠ ERROR</div>
|
||||
<div class="overlay-name">${escapeHtml(r.label)}</div>
|
||||
</div>`
|
||||
: ""
|
||||
}
|
||||
<div class="market-row">
|
||||
<span class="label">${escapeHtml(r.label)}</span>
|
||||
<span class="price">${r.price.toLocaleString(undefined, { maximumFractionDigits: 2 })} ${r.currency}</span>
|
||||
|
|
@ -943,19 +956,31 @@ document.getElementById("econHistorySymbolFilter").onchange = () => loadEconHist
|
|||
// duplicate listeners: expand/collapse a card's comparison charts, fetching
|
||||
// them lazily on first expand only.
|
||||
document.getElementById("econHistoryList").addEventListener("click", (ev) => {
|
||||
const btn = ev.target.closest(".overlay-toggle-btn");
|
||||
if (!btn) return;
|
||||
const idx = btn.dataset.idx;
|
||||
const overlayBtn = ev.target.closest(".overlay-toggle-btn");
|
||||
if (overlayBtn) {
|
||||
const idx = overlayBtn.dataset.idx;
|
||||
const wrap = document.getElementById(`overlay-charts-${idx}`);
|
||||
if (wrap.dataset.loaded) {
|
||||
wrap.classList.toggle("hidden");
|
||||
btn.textContent = wrap.classList.contains("hidden") ? "📈 Show comparison charts" : "📉 Hide comparison charts";
|
||||
overlayBtn.textContent = wrap.classList.contains("hidden") ? "📈 Show comparison charts" : "📉 Hide comparison charts";
|
||||
} else {
|
||||
btn.textContent = "Loading charts…";
|
||||
overlayBtn.textContent = "Loading charts…";
|
||||
loadIncidentOverlayCharts(currentIncidents[idx], idx).then(() => {
|
||||
btn.textContent = "📉 Hide comparison charts";
|
||||
overlayBtn.textContent = "📉 Hide comparison charts";
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const articlesBtn = ev.target.closest(".articles-toggle-btn");
|
||||
if (articlesBtn) {
|
||||
const idx = articlesBtn.dataset.idx;
|
||||
const wrap = document.getElementById(`articles-${idx}`);
|
||||
const count = currentIncidents[idx].candidate_articles.length;
|
||||
const noun = `stor${count === 1 ? "y" : "ies"}`;
|
||||
wrap.classList.toggle("hidden");
|
||||
articlesBtn.textContent = wrap.classList.contains("hidden") ? `📄 Show ${count} ${noun}` : `📄 Hide ${count} ${noun}`;
|
||||
}
|
||||
});
|
||||
|
||||
async function loadEconHistory() {
|
||||
|
|
@ -1012,8 +1037,17 @@ function econIncidentHtml(inc, idx) {
|
|||
const articlesHtml = inc.candidate_articles.length
|
||||
? inc.candidate_articles.map(articleItemHtml).join("")
|
||||
: `<p class="subtle">No stories found published in that window.</p>`;
|
||||
const incidentDateTime = new Date(inc.detected_at).toLocaleString(undefined, {
|
||||
weekday: "short",
|
||||
year: "numeric",
|
||||
month: "short",
|
||||
day: "numeric",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
});
|
||||
return `
|
||||
<div class="econ-incident">
|
||||
<div class="incident-datetime">${incidentDateTime}</div>
|
||||
<h3>${instrumentsHtml}</h3>
|
||||
<div class="window">
|
||||
${fmtHour(inc.window_start)} – ${fmtHour(inc.window_end)}
|
||||
|
|
@ -1028,7 +1062,8 @@ function econIncidentHtml(inc, idx) {
|
|||
<h4>This incident's instruments only</h4>
|
||||
<div class="overlay-chart-container" id="overlay-inc-${idx}"></div>
|
||||
</div>
|
||||
<div class="articles">${articlesHtml}</div>
|
||||
<button class="articles-toggle-btn" data-idx="${idx}">📄 Show ${inc.candidate_articles.length} stor${inc.candidate_articles.length === 1 ? "y" : "ies"}</button>
|
||||
<div class="articles hidden" id="articles-${idx}">${articlesHtml}</div>
|
||||
</div>
|
||||
`;
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue