143 lines
4.9 KiB
Python
143 lines
4.9 KiB
Python
"""Financial ingestion — FRED (macro series) + Stooq (market/commodity prices).
|
|
|
|
Stooq is used instead of Alpha Vantage because it needs no key at all and has no
|
|
25-request/day cap, which a 4x/day digest would otherwise burn through
|
|
(docs/project-plan.md Phase 12 step 4).
|
|
|
|
Adding series/symbols is env-only, no code change:
|
|
|
|
FRED_SERIES — comma-separated FRED series IDs, default "UNRATE" (US
|
|
unemployment rate). Browse/search IDs at
|
|
https://fred.stlouisfed.org — e.g. "UNRATE,CPIAUCSL,FEDFUNDS"
|
|
for unemployment + CPI + the federal funds rate.
|
|
STOOQ_SYMBOLS — comma-separated Stooq symbols, default "^spx,^dax,cl.f".
|
|
Stooq notation: "^spx"/"^dax"/"^ndq" for indices, bare tickers
|
|
like "aapl.us" for US equities, and futures like "cl.f" (WTI
|
|
crude), "gc.f" (gold), "ng.f" (natural gas).
|
|
"""
|
|
|
|
import csv
|
|
import io
|
|
import logging
|
|
import os
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import requests
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
FRED_URL = "https://api.stlouisfed.org/fred/series/observations"
|
|
STOOQ_URL = "https://stooq.com/q/d/l/"
|
|
|
|
HTTP_TIMEOUT = 30
|
|
STOOQ_HISTORY_DAYS = 45
|
|
FRED_HISTORY_DAYS = 400
|
|
|
|
|
|
def _csv_list(name, default):
|
|
raw = os.environ.get(name, default)
|
|
return [item.strip() for item in raw.split(",") if item.strip()]
|
|
|
|
|
|
def _fetch_fred(series_ids, api_key):
|
|
observations = []
|
|
start = (datetime.now(timezone.utc) - timedelta(days=FRED_HISTORY_DAYS)).date()
|
|
|
|
for series_id in series_ids:
|
|
try:
|
|
response = requests.get(
|
|
FRED_URL,
|
|
params={
|
|
"series_id": series_id,
|
|
"api_key": api_key,
|
|
"file_type": "json",
|
|
"observation_start": start.isoformat(),
|
|
"sort_order": "desc",
|
|
"limit": 6,
|
|
},
|
|
timeout=HTTP_TIMEOUT,
|
|
)
|
|
response.raise_for_status()
|
|
points = [
|
|
{"date": item["date"], "value": item["value"]}
|
|
for item in response.json().get("observations", [])
|
|
if item.get("value") not in (None, ".")
|
|
]
|
|
if not points:
|
|
LOG.warning("financial: FRED series %s returned no usable observations", series_id)
|
|
continue
|
|
observations.append(
|
|
{
|
|
"source": "fred",
|
|
"series_id": series_id,
|
|
"latest_date": points[0]["date"],
|
|
"latest_value": points[0]["value"],
|
|
"previous_value": points[1]["value"] if len(points) > 1 else None,
|
|
"recent": points,
|
|
}
|
|
)
|
|
except Exception:
|
|
LOG.warning("financial: FRED series %s failed, skipping", series_id, exc_info=True)
|
|
|
|
return observations
|
|
|
|
|
|
def _fetch_stooq(symbols):
|
|
quotes = []
|
|
end = datetime.now(timezone.utc).date()
|
|
start = end - timedelta(days=STOOQ_HISTORY_DAYS)
|
|
|
|
for symbol in symbols:
|
|
try:
|
|
response = requests.get(
|
|
STOOQ_URL,
|
|
params={
|
|
"s": symbol,
|
|
"i": "d",
|
|
"d1": start.strftime("%Y%m%d"),
|
|
"d2": end.strftime("%Y%m%d"),
|
|
},
|
|
timeout=HTTP_TIMEOUT,
|
|
)
|
|
response.raise_for_status()
|
|
rows = [row for row in csv.DictReader(io.StringIO(response.text)) if row.get("Close")]
|
|
if not rows:
|
|
LOG.warning("financial: Stooq symbol %s returned no rows", symbol)
|
|
continue
|
|
latest = rows[-1]
|
|
previous = rows[-2] if len(rows) > 1 else None
|
|
first = rows[0]
|
|
quotes.append(
|
|
{
|
|
"source": "stooq",
|
|
"symbol": symbol,
|
|
"date": latest["Date"],
|
|
"close": float(latest["Close"]),
|
|
"previous_close": float(previous["Close"]) if previous else None,
|
|
"close_days_ago": float(first["Close"]),
|
|
"window_days": STOOQ_HISTORY_DAYS,
|
|
}
|
|
)
|
|
except Exception:
|
|
LOG.warning("financial: Stooq symbol %s failed, skipping", symbol, exc_info=True)
|
|
|
|
return quotes
|
|
|
|
|
|
def fetch(lookback_hours):
|
|
del lookback_hours # macro series and daily bars move slower than the digest cadence
|
|
|
|
results = []
|
|
|
|
fred_key = os.environ.get("FRED_API_KEY", "").strip()
|
|
fred_series = _csv_list("FRED_SERIES", "UNRATE")
|
|
if fred_key:
|
|
results.extend(_fetch_fred(fred_series, fred_key))
|
|
else:
|
|
LOG.warning("financial: FRED_API_KEY not set, skipping macro series")
|
|
|
|
results.extend(_fetch_stooq(_csv_list("STOOQ_SYMBOLS", "^spx,^dax,cl.f")))
|
|
|
|
LOG.info("financial: %d indicator(s)", len(results))
|
|
return results
|