From d52cb8b12e043ac16495827b399e76c11e7e15a7 Mon Sep 17 00:00:00 2001 From: The_miro Date: Mon, 20 Jul 2026 13:49:00 +0200 Subject: [PATCH] Initial commit: NewsAtlas news globe nginx-fronted stack that pulls RSS from an ideologically mixed set of outlets, geocodes stories onto a 3D globe with zoom-adaptive clustering, tracks market/oil prices with spike-to-article correlation, and overlays weather, conflict events, Wikipedia lookups, and parliament diagrams. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01Um48tTvZDrEgDeweFyhPYC --- .env.example | 36 ++ .gitignore | 4 + README.md | 138 +++++ backend/Dockerfile | 17 + backend/app/__init__.py | 0 backend/app/clustering.py | 67 +++ backend/app/config.py | 35 ++ backend/app/conflict.py | 90 ++++ backend/app/data/gazetteer.csv | 160 ++++++ backend/app/data/parliaments.yaml | 107 ++++ backend/app/db.py | 32 ++ backend/app/geocode.py | 68 +++ backend/app/ingest.py | 84 ++++ backend/app/main.py | 305 +++++++++++ backend/app/markets.py | 181 +++++++ backend/app/models.py | 82 +++ backend/app/scheduler.py | 56 +++ backend/app/sources.yaml | 63 +++ backend/app/wikipedia.py | 115 +++++ backend/requirements.txt | 10 + docker-compose.yml | 26 + frontend/css/style.css | 356 +++++++++++++ frontend/index.html | 61 +++ frontend/js/app.js | 808 ++++++++++++++++++++++++++++++ nginx/nginx.conf | 53 ++ 25 files changed, 2954 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 README.md create mode 100644 backend/Dockerfile create mode 100644 backend/app/__init__.py create mode 100644 backend/app/clustering.py create mode 100644 backend/app/config.py create mode 100644 backend/app/conflict.py create mode 100644 backend/app/data/gazetteer.csv create mode 100644 backend/app/data/parliaments.yaml create mode 100644 backend/app/db.py create mode 100644 backend/app/geocode.py create mode 100644 backend/app/ingest.py create mode 100644 backend/app/main.py create mode 100644 backend/app/markets.py create mode 100644 backend/app/models.py create mode 100644 backend/app/scheduler.py create mode 100644 backend/app/sources.yaml create mode 100644 backend/app/wikipedia.py create mode 100644 backend/requirements.txt create mode 100644 docker-compose.yml create mode 100644 frontend/css/style.css create mode 100644 frontend/index.html create mode 100644 frontend/js/app.js create mode 100644 nginx/nginx.conf diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..19af685 --- /dev/null +++ b/.env.example @@ -0,0 +1,36 @@ +# Copy this file to .env and fill in what you have. Everything has a safe +# default / degrades gracefully if left blank. + +# Port nginx will listen on (http://localhost:HTTP_PORT) +HTTP_PORT=8080 + +# --- Weather overlay (OpenWeatherMap tile layer) --- +# Free key: https://home.openweathermap.org/users/sign_up +# Without a key the weather overlay toggle stays disabled in the UI. +OWM_API_KEY= + +# --- Conflict / "military movement" overlay --- +# Backed by ACLED (Armed Conflict Location & Event Data), the closest thing +# to an open, structured, public feed of reported military/conflict events. +# Free academic/non-commercial access: https://acleddata.com/register/ +# Without these the overlay stays empty (endpoint returns [] and the UI hides it). +ACLED_API_KEY= +ACLED_EMAIL= + +# --- Polling intervals (minutes) --- +RSS_POLL_MINUTES=10 +MARKET_POLL_MINUTES=15 +CONFLICT_POLL_MINUTES=60 + +# How long an article stays "live" on the globe before aging out of clusters +ARTICLE_WINDOW_HOURS=72 + +# Minimum |% change| between two consecutive polls of an index/oil price +# before it's flagged as a "spike" (see /api/markets/spikes) +MARKET_SPIKE_THRESHOLD_PCT=1.5 +# How many hours before a spike's previous poll to search for candidate +# articles that might explain it +MARKET_SPIKE_LOOKBACK_HOURS=6 + +# SQLite DB location inside the backend container (mapped to ./data on host) +DATABASE_PATH=/data/newsatlas.db diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..848de11 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.env +/data/ +__pycache__/ +*.pyc diff --git a/README.md b/README.md new file mode 100644 index 0000000..ee9e2ec --- /dev/null +++ b/README.md @@ -0,0 +1,138 @@ +# NewsAtlas + +A self-hosted, nginx-fronted "news globe": pulls RSS from a deliberately +ideologically-mixed set of outlets, localizes each story onto a 3D globe, +merges same-place/same-topic stories into a single expandable point, tracks +major stock indices and oil prices over time, and overlays weather and +(where available) conflict-event data. Any place name or article text can be +looked up on Wikipedia, and country panels show that country's parliament +composition diagram when Wikipedia has one. + +## Layout + +- **Left rail** β€” latest news, newest first, auto-refreshing. A πŸ“ button on + any geocoded story flies the globe to it. +- **Right rail** β€” index/oil ticker. Click an instrument to expand its + recent-spikes-with-candidate-articles view (see below). +- **Center** β€” the globe. Clicking a point opens a modal with its stories, + a Wikipedia summary of the place, and (if available) its country's + parliament composition diagram. Text selected anywhere can be searched on + Wikipedia via the popup that appears. +- **Bottom drawer** β€” collapsible conflict/military-event log (ACLED-backed; + see below). + +Globe points use **zoom-adaptive clustering**: the backend groups articles +onto a fixed ~11km grid, then the frontend progressively folds nearby grid +cells into a single point as you zoom out (`regroupForAltitude` in +`frontend/js/app.js`), so a continent-level view doesn't leave dozens of +separate dots. The fold radius grows with camera altitude, capped at +subcontinent scale β€” tune it via `mergeRadiusDeg()` if you want tighter or +looser regional grouping. + +Color theme is the user's own "CyberQueer" palette (near-black base, hot +pink `#E40046` + electric violet `#5018DD` accents), including a tinted +globe material β€” see `frontend/css/style.css` `:root` variables to swap it. + +National borders render as a bright violet stroke over a transparent fill +(Natural Earth admin-0 polygons, loaded from `three-globe`'s own npm +package via unpkg β€” no key or backend endpoint needed). + +Each market instrument in the right rail shows a 7-day sparkline +(line + area, hover for a crosshair/tooltip) built from +`/api/markets/history`. Article listings (news feed, cluster modal, spike +candidate articles) show a small favicon next to each headline, resolved +from the article's own URL β€” decorative only, not a claim of any +publisher's official branding. + +## Stack + +- **backend/** β€” FastAPI + SQLite + APScheduler. Polls RSS feeds, geocodes + articles against a curated gazetteer, clusters them, polls Yahoo Finance + for markets/oil, optionally polls ACLED for conflict events, and proxies + Wikipedia + OpenWeatherMap so API keys never reach the browser. +- **frontend/** β€” static HTML/CSS/JS using [globe.gl](https://globe.gl) + (three.js) for the 3D globe. No build step. +- **nginx/** β€” reverse proxy (`/api/*` β†’ backend) + static file server, with + a short-lived cache in front of the API so many browser tabs don't hammer + SQLite. + +## Running it + +```bash +cp .env.example .env # fill in optional API keys, see below +docker compose up --build +``` + +Then open http://localhost:8080 (or whatever `HTTP_PORT` you set). + +On first boot the globe will be empty for a few seconds until the first RSS +poll completes β€” hit "Refresh now" if you don't want to wait for the +10-minute interval. + +## Optional API keys (`.env`) + +Everything works with `.env` left blank except that the weather and conflict +overlays disable themselves. Nothing else requires a key. + +| Var | What it unlocks | Get one | +|---|---|---| +| `OWM_API_KEY` | Weather overlay (clouds/precipitation/temp/wind/pressure) | free, [openweathermap.org](https://home.openweathermap.org/users/sign_up) | +| `ACLED_API_KEY` + `ACLED_EMAIL` | Conflict/"military movement" event overlay | free for registered (incl. non-commercial/research) use, [acleddata.com](https://acleddata.com/register/) | + +**Why ACLED for "military movements":** there is no single free, public, +real-time feed of military movements. ACLED is the closest widely-used open +dataset of sourced, georeferenced conflict/political-violence events and is +what the conflict overlay is built on. Treat it as "reported conflict +events," not live troop tracking. + +## What's a heuristic, not ground truth + +- **Geocoding** (`backend/app/geocode.py`, `data/gazetteer.csv`) is keyword + matching against a curated ~180-place gazetteer, not full NLP/NER. It will + miss unlisted places and can mismatch generic names. Add rows to the CSV + to extend coverage. +- **Clustering** groups articles whose matched location falls in the same + ~11km grid cell within the last `ARTICLE_WINDOW_HOURS` (default 72h). The + "topic" label shown per cluster is just the most frequent significant + words across its headlines, not topic modeling. +- **Weather overlay** stitches a handful of OpenWeatherMap Web-Mercator + tiles into one texture and maps it onto the globe's equirectangular UVs. + It's visually indicative (clouds/precip patterns are recognizable) but not + pixel-accurate, especially near the poles. +- **Parliament diagrams** (`backend/app/wikipedia.py`, + `data/parliaments.yaml`) reuse the infobox image of each country's + legislature article, which is usually β€” not always β€” the semicircle/ + hemicycle composition chart. Countries without a curated mapping are + resolved by a live Wikipedia search at request time, so accuracy varies; + fix a bad match by adding an override to `parliaments.yaml`. +- **MOEX (Russia) index** data from Yahoo Finance appears frozen at mid-2022 + values β€” Western data providers largely cut off live Russian market feeds + after sanctions. It's shown for continuity, not as a live price. +- **RSS feed URLs** (`backend/app/sources.yaml`) belong to third parties and + can change without notice β€” if a source goes quiet, check its site for a + current feed URL and update the file (no rebuild needed, it's a read-only + mount). + +## API surface (consumed by the frontend, but usable standalone) + +- `GET /api/clusters` β€” merged globe points +- `GET /api/clusters/{cluster_key}/articles` β€” articles behind one point +- `GET /api/articles?q=&limit=` β€” raw article search +- `POST /api/refresh` β€” force an immediate RSS poll +- `GET /api/markets/latest`, `GET /api/markets/history?symbol=^GSPC` +- `GET /api/markets/spikes?symbol=&hours=` β€” sudden index/oil moves + (`MARKET_SPIKE_THRESHOLD_PCT`) paired with relevance-ranked articles + published in the surrounding window (`MARKET_SPIKE_LOOKBACK_HOURS`) β€” click + a ticker item in the UI for this. It's a heuristic correlation (keyword + + country/location matching), not a verified causal link. +- `GET /api/conflict-events?hours=168` +- `GET /api/weather/tiles/{layer}/{z}/{x}/{y}.png` β€” OWM tile proxy +- `GET /api/wikipedia/summary?title=`, `GET /api/wikipedia/search?q=`, + `GET /api/wikipedia/parliament?country=` + +## Extending + +- Add/edit RSS sources: `backend/app/sources.yaml`. +- Add gazetteer locations: `backend/app/data/gazetteer.csv`. +- Add/fix parliament page mappings: `backend/app/data/parliaments.yaml`. +- Change poll intervals / clustering window: `.env`. diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..d941be4 --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.12-slim + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app ./app + +ENV PYTHONUNBUFFERED=1 +EXPOSE 8000 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/clustering.py b/backend/app/clustering.py new file mode 100644 index 0000000..48b2d89 --- /dev/null +++ b/backend/app/clustering.py @@ -0,0 +1,67 @@ +import datetime as dt +import re +from collections import Counter, defaultdict + +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .config import settings +from .models import Article + +_STOPWORDS = { + "the", "a", "an", "in", "on", "of", "to", "for", "and", "or", "is", "as", + "at", "by", "with", "from", "after", "over", "amid", "amid", "into", + "says", "say", "will", "has", "have", "had", "its", "it", "his", "her", + "new", "up", "out", "how", "why", "what", "who", "be", "are", "was", + "were", "this", "that", "than", "not", "no", "us", "u.s.", +} +_WORD_RE = re.compile(r"[A-Za-z][A-Za-z'-]{2,}") + + +def _topic_label(titles: list[str]) -> str: + words = Counter() + for title in titles: + for word in _WORD_RE.findall(title.lower()): + if word not in _STOPWORDS: + words[word] += 1 + top = [w for w, _ in words.most_common(3)] + return ", ".join(top) if top else "" + + +def build_clusters(session: Session) -> list[dict]: + cutoff = dt.datetime.utcnow() - dt.timedelta(hours=settings.article_window_hours) + rows = session.execute( + select(Article) + .where(Article.cluster_key.is_not(None)) + .where(Article.published_at >= cutoff) + .order_by(Article.published_at.desc()) + ).scalars().all() + + groups: dict[str, list[Article]] = defaultdict(list) + for article in rows: + groups[article.cluster_key].append(article) + + clusters = [] + for key, articles in groups.items(): + lat = sum(a.lat for a in articles) / len(articles) + lon = sum(a.lon for a in articles) / len(articles) + sources = sorted({a.source for a in articles}) + clusters.append( + { + "cluster_key": key, + "lat": lat, + "lon": lon, + "location_name": articles[0].location_name, + "country": articles[0].country, + "article_count": len(articles), + "source_count": len(sources), + "sources": sources, + "topic": _topic_label([a.title for a in articles]), + "latest_published_at": max(a.published_at for a in articles).isoformat(), + "headline": articles[0].title, + "article_ids": [a.id for a in articles], + } + ) + + clusters.sort(key=lambda c: c["article_count"], reverse=True) + return clusters diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000..e66da13 --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,35 @@ +from pathlib import Path + +from pydantic_settings import BaseSettings + +APP_DIR = Path(__file__).resolve().parent + + +class Settings(BaseSettings): + database_path: str = "/data/newsatlas.db" + + owm_api_key: str = "" + acled_api_key: str = "" + acled_email: str = "" + + rss_poll_minutes: int = 10 + market_poll_minutes: int = 15 + # Minimum |% change| between two consecutive polls of the same + # instrument before we flag it as a "spike" and go looking for articles + # that might explain it. + market_spike_threshold_pct: float = 1.5 + # How far back (and forward, capped at "now") from the previous poll to + # search for candidate-cause articles around a detected spike. + market_spike_lookback_hours: int = 6 + conflict_poll_minutes: int = 60 + + article_window_hours: int = 72 + + sources_file: str = str(APP_DIR / "sources.yaml") + gazetteer_file: str = str(APP_DIR / "data" / "gazetteer.csv") + + class Config: + env_file = ".env" + + +settings = Settings() diff --git a/backend/app/conflict.py b/backend/app/conflict.py new file mode 100644 index 0000000..62b5f8a --- /dev/null +++ b/backend/app/conflict.py @@ -0,0 +1,90 @@ +"""Conflict / military-movement overlay, backed by ACLED. + +There is no single authoritative, free, real-time public feed of "military +movements" β€” ACLED (Armed Conflict Location & Event Data, acleddata.com) is +the closest widely-used open dataset of georeferenced, sourced conflict and +political-violence events, and offers free access for registered users. If +ACLED_API_KEY / ACLED_EMAIL are not configured, this module is a no-op and +the overlay simply stays empty in the UI rather than erroring. +""" + +import datetime as dt +import logging + +import httpx +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .config import settings +from .models import ConflictEvent + +log = logging.getLogger("newsatlas.conflict") + +ACLED_URL = "https://api.acleddata.com/acled/read" + + +def enabled() -> bool: + return bool(settings.acled_api_key and settings.acled_email) + + +def poll_conflict_events(session: Session) -> int: + if not enabled(): + log.info("ACLED credentials not configured; skipping conflict-event poll") + return 0 + + since = (dt.datetime.utcnow() - dt.timedelta(days=7)).strftime("%Y-%m-%d") + params = { + "key": settings.acled_api_key, + "email": settings.acled_email, + "event_date": since, + "event_date_where": ">=", + "limit": 500, + } + + try: + resp = httpx.get(ACLED_URL, params=params, timeout=30) + resp.raise_for_status() + payload = resp.json() + except Exception: + log.exception("Failed to fetch ACLED data") + return 0 + + added = 0 + for row in payload.get("data", []): + external_id = str(row.get("event_id_cnty") or row.get("data_id")) + exists = session.execute( + select(ConflictEvent.id).where( + ConflictEvent.source == "acled", + ConflictEvent.external_id == external_id, + ) + ).first() + if exists: + continue + + try: + lat = float(row["latitude"]) + lon = float(row["longitude"]) + event_date = dt.datetime.strptime(row["event_date"], "%Y-%m-%d") + except (KeyError, ValueError, TypeError): + continue + + session.add( + ConflictEvent( + source="acled", + external_id=external_id, + event_type=row.get("event_type", ""), + actor1=row.get("actor1", ""), + actor2=row.get("actor2", ""), + fatalities=int(row["fatalities"]) if str(row.get("fatalities", "")).isdigit() else None, + notes=row.get("notes", ""), + location_name=row.get("location", ""), + country=row.get("country", ""), + lat=lat, + lon=lon, + event_date=event_date, + ) + ) + added += 1 + + session.commit() + return added diff --git a/backend/app/data/gazetteer.csv b/backend/app/data/gazetteer.csv new file mode 100644 index 0000000..566b87b --- /dev/null +++ b/backend/app/data/gazetteer.csv @@ -0,0 +1,160 @@ +name,lat,lon,type,country +Washington,38.9072,-77.0369,capital,United States +New York,40.7128,-74.0060,city,United States +Los Angeles,34.0522,-118.2437,city,United States +Chicago,41.8781,-87.6298,city,United States +London,51.5074,-0.1278,capital,United Kingdom +Paris,48.8566,2.3522,capital,France +Berlin,52.5200,13.4050,capital,Germany +Moscow,55.7558,37.6173,capital,Russia +Beijing,39.9042,116.4074,capital,China +Shanghai,31.2304,121.4737,city,China +Tokyo,35.6762,139.6503,capital,Japan +Seoul,37.5665,126.9780,capital,South Korea +Pyongyang,39.0392,125.7625,capital,North Korea +New Delhi,28.6139,77.2090,capital,India +Mumbai,19.0760,72.8777,city,India +Islamabad,33.6844,73.0479,capital,Pakistan +Kabul,34.5553,69.2075,capital,Afghanistan +Tehran,35.6892,51.3890,capital,Iran +Baghdad,33.3152,44.3661,capital,Iraq +Damascus,33.5138,36.2765,capital,Syria +Beirut,33.8938,35.5018,capital,Lebanon +Jerusalem,31.7683,35.2137,capital,Israel +Tel Aviv,32.0853,34.7818,city,Israel +Gaza,31.5017,34.4668,region,Palestine +Gaza Strip,31.5017,34.4668,region,Palestine +Rafah,31.2985,34.2429,city,Palestine +Ramallah,31.9038,35.2034,city,Palestine +West Bank,31.9466,35.3027,region,Palestine +Amman,31.9454,35.9284,capital,Jordan +Riyadh,24.7136,46.6753,capital,Saudi Arabia +Doha,25.2854,51.5310,capital,Qatar +Abu Dhabi,24.4539,54.3773,capital,United Arab Emirates +Dubai,25.2048,55.2708,city,United Arab Emirates +Kuwait City,29.3759,47.9774,capital,Kuwait +Sanaa,15.3694,44.1910,capital,Yemen +Cairo,30.0444,31.2357,capital,Egypt +Tripoli,32.8872,13.1913,capital,Libya +Tunis,36.8065,10.1815,capital,Tunisia +Algiers,36.7538,3.0588,capital,Algeria +Rabat,34.0209,-6.8416,capital,Morocco +Khartoum,15.5007,32.5599,capital,Sudan +Addis Ababa,9.0300,38.7400,capital,Ethiopia +Mogadishu,2.0469,45.3182,capital,Somalia +Nairobi,-1.2921,36.8219,capital,Kenya +Kyiv,50.4501,30.5234,capital,Ukraine +Kharkiv,49.9935,36.2304,city,Ukraine +Donbas,48.0159,37.8028,region,Ukraine +Donetsk,48.0159,37.8028,city,Ukraine +Mariupol,47.0971,37.5434,city,Ukraine +Crimea,45.3453,34.4697,region,Ukraine +Odesa,46.4825,30.7233,city,Ukraine +Minsk,53.9006,27.5590,capital,Belarus +Warsaw,52.2297,21.0122,capital,Poland +Vilnius,54.6872,25.2797,capital,Lithuania +Riga,56.9496,24.1052,capital,Latvia +Tallinn,59.4370,24.7536,capital,Estonia +Helsinki,60.1699,24.9384,capital,Finland +Stockholm,59.3293,18.0686,capital,Sweden +Oslo,59.9139,10.7522,capital,Norway +Copenhagen,55.6761,12.5683,capital,Denmark +Reykjavik,64.1466,-21.9426,capital,Iceland +Dublin,53.3498,-6.2603,capital,Ireland +Madrid,40.4168,-3.7038,capital,Spain +Lisbon,38.7223,-9.1393,capital,Portugal +Rome,41.9028,12.4964,capital,Italy +Vienna,48.2082,16.3738,capital,Austria +Bern,46.9480,7.4474,capital,Switzerland +Brussels,50.8503,4.3517,capital,Belgium +Amsterdam,52.3676,4.9041,capital,Netherlands +The Hague,52.0705,4.3007,city,Netherlands +Athens,37.9838,23.7275,capital,Greece +Ankara,39.9334,32.8597,capital,Turkey +Istanbul,41.0082,28.9784,city,Turkey +Budapest,47.4979,19.0402,capital,Hungary +Prague,50.0755,14.4378,capital,Czech Republic +Bucharest,44.4268,26.1025,capital,Romania +Sofia,42.6977,23.3219,capital,Bulgaria +Belgrade,44.7866,20.4489,capital,Serbia +Zagreb,45.8150,15.9819,capital,Croatia +Sarajevo,43.8563,18.4131,capital,Bosnia and Herzegovina +Skopje,41.9981,21.4254,capital,North Macedonia +Tirana,41.3275,19.8187,capital,Albania +Pristina,42.6629,21.1655,capital,Kosovo +Yerevan,40.1792,44.4991,capital,Armenia +Baku,40.4093,49.8671,capital,Azerbaijan +Nagorno-Karabakh,39.8000,46.7500,region,Azerbaijan +Tbilisi,41.7151,44.8271,capital,Georgia +Lagos,6.5244,3.3792,city,Nigeria +Abuja,9.0765,7.3986,capital,Nigeria +Accra,5.6037,-0.1870,capital,Ghana +Johannesburg,-26.2041,28.0473,city,South Africa +Pretoria,-25.7479,28.2293,capital,South Africa +Cape Town,-33.9249,18.4241,city,South Africa +Kinshasa,-4.4419,15.2663,capital,Democratic Republic of the Congo +Dakar,14.7167,-17.4677,capital,Senegal +Bamako,12.6392,-8.0029,capital,Mali +Niamey,13.5137,2.1098,capital,Niger +N'Djamena,12.1348,15.0557,capital,Chad +Ottawa,45.4215,-75.6972,capital,Canada +Mexico City,19.4326,-99.1332,capital,Mexico +Havana,23.1136,-82.3666,capital,Cuba +Caracas,10.4806,-66.9036,capital,Venezuela +Bogota,4.7110,-74.0721,capital,Colombia +Lima,-12.0464,-77.0428,capital,Peru +Brasilia,-15.8267,-47.9218,capital,Brazil +Sao Paulo,-23.5505,-46.6333,city,Brazil +Rio de Janeiro,-22.9068,-43.1729,city,Brazil +Buenos Aires,-34.6037,-58.3816,capital,Argentina +Santiago,-33.4489,-70.6693,capital,Chile +Taipei,25.0330,121.5654,capital,Taiwan +Taiwan Strait,24.5000,119.5000,region,Taiwan +Hong Kong,22.3193,114.1694,city,China +Manila,14.5995,120.9842,capital,Philippines +Jakarta,-6.2088,106.8456,capital,Indonesia +Bangkok,13.7563,100.5018,capital,Thailand +Hanoi,21.0278,105.8342,capital,Vietnam +Kuala Lumpur,3.1390,101.6869,capital,Malaysia +Singapore,1.3521,103.8198,city,Singapore +Canberra,-35.2809,149.1300,capital,Australia +Sydney,-33.8688,151.2093,city,Australia +Wellington,-41.2865,174.7762,capital,New Zealand +Dhaka,23.8103,90.4125,capital,Bangladesh +Yangon,16.8409,96.1735,city,Myanmar +Naypyidaw,19.7633,96.0785,capital,Myanmar +Colombo,6.9271,79.8612,capital,Sri Lanka +Kashmir,34.0837,74.7973,region,India/Pakistan +Srinagar,34.0837,74.7973,city,India +South China Sea,12.0000,114.0000,region,International Waters +Red Sea,20.0000,38.0000,region,International Waters +Strait of Hormuz,26.5000,56.2500,region,International Waters +United States,39.8283,-98.5795,country,United States +Russia,61.5240,105.3188,country,Russia +China,35.8617,104.1954,country,China +India,20.5937,78.9629,country,India +Ukraine,48.3794,31.1656,country,Ukraine +Israel,31.0461,34.8516,country,Israel +Palestine,31.9522,35.2332,country,Palestine +Iran,32.4279,53.6880,country,Iran +North Korea,40.3399,127.5101,country,North Korea +Taiwan,23.6978,120.9605,country,Taiwan +France,46.2276,2.2137,country,France +Germany,51.1657,10.4515,country,Germany +United Kingdom,55.3781,-3.4360,country,United Kingdom +Brazil,-14.2350,-51.9253,country,Brazil +Nigeria,9.0820,8.6753,country,Nigeria +South Africa,-30.5595,22.9375,country,South Africa +Australia,-25.2744,133.7751,country,Australia +Canada,56.1304,-106.3468,country,Canada +Mexico,23.6345,-102.5528,country,Mexico +Japan,36.2048,138.2529,country,Japan +South Korea,35.9078,127.7669,country,South Korea +Saudi Arabia,23.8859,45.0792,country,Saudi Arabia +Pakistan,30.3753,69.3451,country,Pakistan +Afghanistan,33.9391,67.7100,country,Afghanistan +Syria,34.8021,38.9968,country,Syria +Yemen,15.5527,48.5164,country,Yemen +Sudan,12.8628,30.2176,country,Sudan +Ethiopia,9.1450,40.4897,country,Ethiopia +Venezuela,6.4238,-66.5897,country,Venezuela diff --git a/backend/app/data/parliaments.yaml b/backend/app/data/parliaments.yaml new file mode 100644 index 0000000..f672c50 --- /dev/null +++ b/backend/app/data/parliaments.yaml @@ -0,0 +1,107 @@ +# Best-effort mapping of country -> English Wikipedia article for its +# (lower/only) national legislature. Most legislature articles carry a +# composition/hemicycle diagram as their infobox image, so we reuse the +# page-summary thumbnail as a "parliament diagram". +# +# Countries not listed here are resolved dynamically at request time via +# Wikipedia opensearch (" parliament"), which is best-effort and +# occasionally wrong β€” add an override here if you spot a bad match. +# Leave a country out entirely (or set to null) if it currently has no +# functioning/elected legislature. + +United States: United States House of Representatives +United Kingdom: House of Commons +France: National Assembly (France) +Germany: Bundestag +Russia: State Duma +China: National People's Congress +Japan: House of Representatives (Japan) +South Korea: National Assembly (South Korea) +North Korea: Supreme People's Assembly +India: Lok Sabha +Pakistan: National Assembly of Pakistan +Afghanistan: null +Iran: Islamic Consultative Assembly +Iraq: Council of Representatives of Iraq +Syria: People's Assembly of Syria +Lebanon: Parliament of Lebanon +Israel: Knesset +Palestine: Palestinian Legislative Council +Jordan: House of Representatives (Jordan) +Saudi Arabia: Shura Council (Saudi Arabia) +Qatar: Shura Council (Qatar) +United Arab Emirates: Federal National Council +Kuwait: National Assembly (Kuwait) +Yemen: House of Representatives (Yemen) +Egypt: House of Representatives (Egypt) +Libya: House of Representatives (Libya) +Tunisia: Assembly of the Representatives of the People +Algeria: People's National Assembly +Morocco: Assembly of Representatives of Morocco +Sudan: null +Ethiopia: House of Peoples' Representatives +Somalia: Federal Parliament of Somalia +Kenya: National Assembly (Kenya) +Ukraine: Verkhovna Rada +Belarus: House of Representatives (Belarus) +Poland: Sejm +Lithuania: Seimas +Latvia: Saeima +Estonia: Riigikogu +Finland: Parliament of Finland +Sweden: Riksdag +Norway: Storting +Denmark: Folketing +Iceland: Althing +Ireland: Dail Eireann +Spain: Congress of Deputies +Portugal: Assembly of the Republic (Portugal) +Italy: Chamber of Deputies (Italy) +Austria: National Council (Austria) +Switzerland: National Council (Switzerland) +Belgium: Chamber of Representatives (Belgium) +Netherlands: House of Representatives (Netherlands) +Greece: Hellenic Parliament +Turkey: Grand National Assembly of Turkey +Hungary: National Assembly (Hungary) +Czech Republic: Chamber of Deputies (Czech Republic) +Romania: Chamber of Deputies (Romania) +Bulgaria: National Assembly (Bulgaria) +Serbia: National Assembly (Serbia) +Croatia: Croatian Parliament +Bosnia and Herzegovina: House of Representatives (Bosnia and Herzegovina) +North Macedonia: Assembly of North Macedonia +Albania: Parliament of Albania +Kosovo: Assembly of Kosovo +Armenia: National Assembly (Armenia) +Azerbaijan: National Assembly (Azerbaijan) +Georgia: Parliament of Georgia +Nigeria: House of Representatives (Nigeria) +Ghana: Parliament of Ghana +South Africa: National Assembly of South Africa +Democratic Republic of the Congo: National Assembly (Democratic Republic of the Congo) +Senegal: National Assembly (Senegal) +Mali: null +Niger: null +Chad: National Transitional Council (Chad) +Canada: House of Commons of Canada +Mexico: Chamber of Deputies (Mexico) +Cuba: National Assembly of People's Power +Venezuela: National Assembly (Venezuela) +Colombia: Chamber of Representatives (Colombia) +Peru: Congress of the Republic of Peru +Brazil: Chamber of Deputies (Brazil) +Argentina: Chamber of Deputies of Argentina +Chile: Chamber of Deputies of Chile +Taiwan: Legislative Yuan +Philippines: House of Representatives of the Philippines +Indonesia: People's Representative Council +Thailand: House of Representatives (Thailand) +Vietnam: National Assembly (Vietnam) +Malaysia: Dewan Rakyat +Singapore: Parliament of Singapore +Australia: House of Representatives (Australia) +New Zealand: New Zealand Parliament +Bangladesh: Jatiya Sangsad +Myanmar: null +Sri Lanka: Parliament of Sri Lanka diff --git a/backend/app/db.py b/backend/app/db.py new file mode 100644 index 0000000..aa3817f --- /dev/null +++ b/backend/app/db.py @@ -0,0 +1,32 @@ +from pathlib import Path + +from sqlalchemy import create_engine +from sqlalchemy.orm import DeclarativeBase, sessionmaker + +from .config import settings + +Path(settings.database_path).parent.mkdir(parents=True, exist_ok=True) + +engine = create_engine( + f"sqlite:///{settings.database_path}", + connect_args={"check_same_thread": False}, +) +SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False) + + +class Base(DeclarativeBase): + pass + + +def init_db() -> None: + from . import models # noqa: F401 (registers tables on Base.metadata) + + Base.metadata.create_all(bind=engine) + + +def get_session(): + session = SessionLocal() + try: + yield session + finally: + session.close() diff --git a/backend/app/geocode.py b/backend/app/geocode.py new file mode 100644 index 0000000..1ab1bb3 --- /dev/null +++ b/backend/app/geocode.py @@ -0,0 +1,68 @@ +"""Lightweight, dependency-free geocoding. + +We deliberately avoid calling an external geocoding API per-article (rate +limits, latency, privacy) and avoid a heavyweight NER model. Instead we match +article text against a curated gazetteer of cities/regions/countries +(app/data/gazetteer.csv), preferring the most specific (longest) name found. +This is a heuristic: it will miss unlisted places and can occasionally match +the wrong entity for very generic names. Extend data/gazetteer.csv to +improve coverage. +""" + +import csv +import re +from dataclasses import dataclass +from functools import lru_cache + +from .config import settings + + +@dataclass(frozen=True) +class Place: + name: str + lat: float + lon: float + type: str + country: str + + +def _load_gazetteer() -> list[Place]: + places: list[Place] = [] + with open(settings.gazetteer_file, newline="", encoding="utf-8") as f: + for row in csv.DictReader(f): + places.append( + Place( + name=row["name"], + lat=float(row["lat"]), + lon=float(row["lon"]), + type=row["type"], + country=row["country"], + ) + ) + # Longest name first so "Gaza Strip" wins over "Gaza", "New York" over "York", etc. + places.sort(key=lambda p: len(p.name), reverse=True) + return places + + +@lru_cache(maxsize=1) +def _gazetteer() -> list[tuple[re.Pattern, Place]]: + compiled = [] + for place in _load_gazetteer(): + pattern = re.compile(r"\b" + re.escape(place.name) + r"\b", re.IGNORECASE) + compiled.append((pattern, place)) + return compiled + + +def locate(text: str) -> Place | None: + """Return the most specific gazetteer place mentioned in `text`, if any.""" + if not text: + return None + for pattern, place in _gazetteer(): + if pattern.search(text): + return place + return None + + +def cluster_key_for(lat: float, lon: float) -> str: + """Grid-snap coordinates (~11km cells) so nearby stories share a bucket.""" + return f"{round(lat, 1)}:{round(lon, 1)}" diff --git a/backend/app/ingest.py b/backend/app/ingest.py new file mode 100644 index 0000000..1a3c082 --- /dev/null +++ b/backend/app/ingest.py @@ -0,0 +1,84 @@ +import datetime as dt +import logging + +import feedparser +import yaml +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .config import settings +from .geocode import cluster_key_for, locate +from .models import Article + +log = logging.getLogger("newsatlas.ingest") + + +def load_sources() -> list[dict]: + with open(settings.sources_file, encoding="utf-8") as f: + return yaml.safe_load(f)["sources"] + + +def _parse_published(entry) -> dt.datetime: + for key in ("published_parsed", "updated_parsed"): + value = getattr(entry, key, None) + if value: + return dt.datetime(*value[:6]) + return dt.datetime.utcnow() + + +def fetch_source(session: Session, source: dict) -> int: + """Fetch one RSS feed, geocode + store new entries. Returns count added.""" + added = 0 + try: + parsed = feedparser.parse(source["url"]) + except Exception: + log.exception("Failed to fetch %s", source["name"]) + return 0 + + if getattr(parsed, "bozo", 0) and not parsed.entries: + log.warning("Feed %s returned no usable entries (bozo=%s)", source["name"], parsed.bozo_exception) + return 0 + + seen_urls: set[str] = set() + for entry in parsed.entries: + url = getattr(entry, "link", None) + if not url or url in seen_urls: + continue + seen_urls.add(url) + + exists = session.execute(select(Article.id).where(Article.url == url)).first() + if exists: + continue + + title = getattr(entry, "title", "").strip() + summary = getattr(entry, "summary", "").strip() + place = locate(f"{title} {summary}") + + article = Article( + source=source["name"], + source_bias=source.get("bias", ""), + title=title, + url=url, + summary=summary, + published_at=_parse_published(entry), + fetched_at=dt.datetime.utcnow(), + ) + if place: + article.location_name = place.name + article.country = place.country + article.lat = place.lat + article.lon = place.lon + article.cluster_key = cluster_key_for(place.lat, place.lon) + + session.add(article) + added += 1 + + session.commit() + return added + + +def fetch_all(session: Session) -> int: + total = 0 + for source in load_sources(): + total += fetch_source(session, source) + return total diff --git a/backend/app/main.py b/backend/app/main.py new file mode 100644 index 0000000..802c279 --- /dev/null +++ b/backend/app/main.py @@ -0,0 +1,305 @@ +import datetime as dt +import json +import logging +from contextlib import asynccontextmanager + +import httpx +from fastapi import FastAPI, HTTPException, Query +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import Response +from sqlalchemy import select + +from . import wikipedia +from .clustering import build_clusters +from .config import settings +from .db import get_session, init_db, SessionLocal +from .ingest import fetch_all +from .markets import poll_markets, INSTRUMENTS +from .conflict import enabled as conflict_enabled, poll_conflict_events +from .models import Article, ConflictEvent, MarketPrice, MarketSpike +from .scheduler import start_scheduler + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger("newsatlas") + +_scheduler = None + + +@asynccontextmanager +async def lifespan(app: FastAPI): + global _scheduler + init_db() + _scheduler = start_scheduler() + yield + if _scheduler: + _scheduler.shutdown(wait=False) + + +app = FastAPI(title="NewsAtlas API", lifespan=lifespan) +app.add_middleware( + CORSMiddleware, + allow_origins=["*"], + allow_methods=["*"], + allow_headers=["*"], +) + + +@app.get("/api/health") +def health(): + return {"status": "ok", "time": dt.datetime.utcnow().isoformat()} + + +@app.get("/api/config") +def public_config(): + """Feature flags the frontend needs to decide what UI to show.""" + return { + "weather_enabled": bool(settings.owm_api_key), + "conflict_enabled": conflict_enabled(), + "article_window_hours": settings.article_window_hours, + } + + +# ---- Articles / clusters ----------------------------------------------- + +@app.get("/api/clusters") +def api_clusters(): + session = next(get_session()) + try: + return build_clusters(session) + finally: + session.close() + + +@app.get("/api/clusters/{cluster_key}/articles") +def api_cluster_articles(cluster_key: str): + session = next(get_session()) + try: + rows = session.execute( + select(Article) + .where(Article.cluster_key == cluster_key) + .order_by(Article.published_at.desc()) + ).scalars().all() + return [_article_dict(a) for a in rows] + finally: + session.close() + + +@app.get("/api/articles") +def api_articles(limit: int = Query(100, le=500), q: str | None = None): + session = next(get_session()) + try: + stmt = select(Article).order_by(Article.published_at.desc()).limit(limit) + if q: + like = f"%{q}%" + stmt = select(Article).where(Article.title.ilike(like)).order_by( + Article.published_at.desc() + ).limit(limit) + rows = session.execute(stmt).scalars().all() + return [_article_dict(a) for a in rows] + finally: + session.close() + + +@app.post("/api/refresh") +def api_refresh(): + """Manually trigger an RSS poll (in addition to the scheduled interval).""" + session = SessionLocal() + try: + added = fetch_all(session) + return {"added": added} + finally: + session.close() + + +def _article_dict(a: Article) -> dict: + return { + "id": a.id, + "source": a.source, + "source_bias": a.source_bias, + "title": a.title, + "url": a.url, + "summary": a.summary, + "published_at": a.published_at.isoformat() if a.published_at else None, + "location_name": a.location_name, + "country": a.country, + "lat": a.lat, + "lon": a.lon, + } + + +# ---- Markets -------------------------------------------------------------- + +@app.get("/api/markets/latest") +def api_markets_latest(): + session = next(get_session()) + try: + out = [] + for symbol, label, category, _currency in INSTRUMENTS: + row = session.execute( + select(MarketPrice) + .where(MarketPrice.symbol == symbol) + .order_by(MarketPrice.recorded_at.desc()) + .limit(1) + ).scalar_one_or_none() + if row: + out.append( + { + "symbol": row.symbol, + "label": row.label, + "category": row.category, + "price": row.price, + "currency": row.currency, + "change_pct": row.change_pct, + "recorded_at": row.recorded_at.isoformat(), + } + ) + return out + finally: + session.close() + + +@app.get("/api/markets/history") +def api_markets_history(symbol: str, hours: int = Query(168, le=24 * 30)): + since = dt.datetime.utcnow() - dt.timedelta(hours=hours) + session = next(get_session()) + try: + rows = session.execute( + select(MarketPrice) + .where(MarketPrice.symbol == symbol, MarketPrice.recorded_at >= since) + .order_by(MarketPrice.recorded_at.asc()) + ).scalars().all() + return [{"price": r.price, "recorded_at": r.recorded_at.isoformat()} for r in rows] + finally: + session.close() + + +@app.get("/api/markets/spikes") +def api_markets_spikes(symbol: str | None = None, hours: int = Query(168, le=24 * 30)): + since = dt.datetime.utcnow() - dt.timedelta(hours=hours) + session = next(get_session()) + try: + stmt = select(MarketSpike).where(MarketSpike.detected_at >= since).order_by(MarketSpike.detected_at.desc()) + if symbol: + stmt = stmt.where(MarketSpike.symbol == symbol) + spikes = session.execute(stmt).scalars().all() + + out = [] + for s in spikes: + article_ids = json.loads(s.article_ids_json or "[]") + articles = [] + if article_ids: + rows = session.execute(select(Article).where(Article.id.in_(article_ids))).scalars().all() + by_id = {a.id: a for a in rows} + articles = [_article_dict(by_id[i]) for i in article_ids if i in by_id] + out.append( + { + "id": s.id, + "symbol": s.symbol, + "label": s.label, + "from_price": s.from_price, + "to_price": s.to_price, + "pct_change": s.pct_change, + "window_start": s.window_start.isoformat(), + "window_end": s.window_end.isoformat(), + "detected_at": s.detected_at.isoformat(), + "candidate_articles": articles, + } + ) + return out + finally: + session.close() + + +@app.post("/api/markets/refresh") +def api_markets_refresh(): + session = SessionLocal() + try: + added = poll_markets(session) + return {"added": added} + finally: + session.close() + + +# ---- Conflict / military-movement overlay --------------------------------- + +@app.get("/api/conflict-events") +def api_conflict_events(hours: int = Query(168, le=24 * 30)): + if not conflict_enabled(): + return [] + since = dt.datetime.utcnow() - dt.timedelta(hours=hours) + session = next(get_session()) + try: + rows = session.execute( + select(ConflictEvent).where(ConflictEvent.event_date >= since) + ).scalars().all() + return [ + { + "id": r.id, + "event_type": r.event_type, + "actor1": r.actor1, + "actor2": r.actor2, + "fatalities": r.fatalities, + "notes": r.notes, + "location_name": r.location_name, + "country": r.country, + "lat": r.lat, + "lon": r.lon, + "event_date": r.event_date.isoformat(), + } + for r in rows + ] + finally: + session.close() + + +@app.post("/api/conflict-events/refresh") +def api_conflict_refresh(): + session = SessionLocal() + try: + added = poll_conflict_events(session) + return {"added": added} + finally: + session.close() + + +# ---- Weather tile proxy (keeps the OWM key server-side) ------------------- + +_OWM_LAYERS = {"clouds", "precipitation", "pressure", "wind", "temp"} + + +@app.get("/api/weather/tiles/{layer}/{z}/{x}/{y}.png") +async def weather_tile(layer: str, z: int, x: int, y: int): + if not settings.owm_api_key: + raise HTTPException(503, "OWM_API_KEY not configured") + if layer not in _OWM_LAYERS: + raise HTTPException(404, "unknown layer") + + url = f"https://tile.openweathermap.org/map/{layer}_new/{z}/{x}/{y}.png" + async with httpx.AsyncClient(timeout=10) as client: + resp = await client.get(url, params={"appid": settings.owm_api_key}) + if resp.status_code != 200: + raise HTTPException(resp.status_code, "upstream weather tile error") + return Response(content=resp.content, media_type="image/png", headers={"Cache-Control": "public, max-age=600"}) + + +# ---- Wikipedia integration -------------------------------------------- + +@app.get("/api/wikipedia/summary") +async def api_wikipedia_summary(title: str): + result = await wikipedia.get_summary(title) + if not result: + raise HTTPException(404, "no Wikipedia summary found") + return result + + +@app.get("/api/wikipedia/search") +async def api_wikipedia_search(q: str, limit: int = Query(5, le=20)): + return await wikipedia.search(q, limit=limit) + + +@app.get("/api/wikipedia/parliament") +async def api_wikipedia_parliament(country: str): + result = await wikipedia.get_parliament(country) + if not result: + raise HTTPException(404, "no parliament diagram found for this country") + return result diff --git a/backend/app/markets.py b/backend/app/markets.py new file mode 100644 index 0000000..ffc5efe --- /dev/null +++ b/backend/app/markets.py @@ -0,0 +1,181 @@ +import datetime as dt +import json +import logging + +import httpx +from sqlalchemy import select +from sqlalchemy.orm import Session + +from .config import settings +from .models import Article, MarketPrice, MarketSpike + +log = logging.getLogger("newsatlas.markets") + +# A spread of major indices as rough proxies for national/regional economies, +# plus crude oil benchmarks. +INSTRUMENTS = [ + ("^GSPC", "S&P 500 (US)", "index", "USD"), + ("^DJI", "Dow Jones Industrial Average (US)", "index", "USD"), + ("^IXIC", "Nasdaq Composite (US)", "index", "USD"), + ("^FTSE", "FTSE 100 (UK)", "index", "GBP"), + ("^GDAXI", "DAX (Germany)", "index", "EUR"), + ("^FCHI", "CAC 40 (France)", "index", "EUR"), + ("^N225", "Nikkei 225 (Japan)", "index", "JPY"), + ("^HSI", "Hang Seng (Hong Kong/China)", "index", "HKD"), + ("000001.SS", "Shanghai Composite (China)", "index", "CNY"), + ("^BSESN", "BSE Sensex (India)", "index", "INR"), + # Yahoo's feed for this appears frozen since mid-2022 (Western data + # providers largely cut off live Russian market data after sanctions); + # treat it as historical, not live. + ("IMOEX.ME", "MOEX Russia Index", "index", "RUB"), + ("^BVSP", "Bovespa (Brazil)", "index", "BRL"), + ("CL=F", "WTI Crude Oil", "commodity", "USD"), + ("BZ=F", "Brent Crude Oil", "commodity", "USD"), +] + +# Yahoo's unauthenticated chart endpoint β€” no crumb/cookie dance required, +# unlike the quote/quoteSummary endpoints that the yfinance library wraps +# (which have proven flaky here: empty bodies once its session's crumb goes +# stale). This is the same data source, just called directly. +CHART_URL = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}" +_HEADERS = {"User-Agent": "Mozilla/5.0 (NewsAtlas market poller)"} + +# Rough relevance hints used only to *rank* candidate-cause articles for a +# detected spike β€” not a claim of causation, just "these are more likely to +# be about this instrument than the average headline in the window". +_SPIKE_HINTS: dict[str, dict] = { + "CL=F": {"keywords": ["oil", "crude", "opec", "barrel", "pipeline", "refinery", "hormuz"], "country": None}, + "BZ=F": {"keywords": ["oil", "crude", "opec", "barrel", "pipeline", "refinery", "hormuz"], "country": None}, + "^GSPC": {"keywords": ["fed", "federal reserve", "tariff", "inflation", "rate"], "country": "United States"}, + "^DJI": {"keywords": ["fed", "federal reserve", "tariff", "inflation", "rate"], "country": "United States"}, + "^IXIC": {"keywords": ["fed", "federal reserve", "tariff", "tech", "inflation", "rate"], "country": "United States"}, + "^FTSE": {"keywords": ["boe", "bank of england", "tariff", "inflation", "rate"], "country": "United Kingdom"}, + "^GDAXI": {"keywords": ["ecb", "european central bank", "tariff", "inflation", "energy"], "country": "Germany"}, + "^FCHI": {"keywords": ["ecb", "european central bank", "tariff", "inflation", "energy"], "country": "France"}, + "^N225": {"keywords": ["boj", "bank of japan", "yen", "tariff", "inflation"], "country": "Japan"}, + "^HSI": {"keywords": ["china", "yuan", "tariff", "property", "beijing"], "country": "China"}, + "000001.SS": {"keywords": ["china", "yuan", "tariff", "property", "beijing", "pboc"], "country": "China"}, + "^BSESN": {"keywords": ["rbi", "rupee", "tariff", "inflation"], "country": "India"}, + "IMOEX.ME": {"keywords": ["sanctions", "ruble", "central bank", "war"], "country": "Russia"}, + "^BVSP": {"keywords": ["real", "central bank", "tariff", "inflation", "election"], "country": "Brazil"}, +} +_GENERIC_SHOCK_KEYWORDS = [ + "war", "attack", "strike", "sanctions", "tariff", "default", "crisis", + "embargo", "shutdown", "election", "earthquake", "coup", "ceasefire", +] + + +def _fetch_one(client: httpx.Client, symbol: str) -> tuple[float, float | None] | None: + resp = client.get( + CHART_URL.format(symbol=symbol), + params={"range": "5d", "interval": "1d"}, + headers=_HEADERS, + timeout=15, + ) + resp.raise_for_status() + result = resp.json()["chart"]["result"][0] + meta = result["meta"] + price = meta.get("regularMarketPrice") + if price is None: + return None + + prev_close = meta.get("previousClose") or meta.get("chartPreviousClose") + if prev_close is None: + closes = [c for c in result["indicators"]["quote"][0]["close"] if c is not None] + if len(closes) >= 2: + prev_close = closes[-2] + + change_pct = None + if prev_close: + change_pct = round((price - prev_close) / prev_close * 100, 3) + return float(price), change_pct + + +def _score_article(article: Article, keywords: list[str], country: str | None) -> int: + text = f"{article.title} {article.summary}".lower() + score = sum(1 for kw in keywords if kw in text) + score += sum(1 for kw in _GENERIC_SHOCK_KEYWORDS if kw in text) + if country and (article.country == country or country.lower() in text): + score += 3 + return score + + +def _find_candidate_articles(session: Session, symbol: str, window_start: dt.datetime, window_end: dt.datetime, limit: int = 8) -> list[int]: + hints = _SPIKE_HINTS.get(symbol, {"keywords": [], "country": None}) + rows = session.execute( + select(Article) + .where(Article.published_at >= window_start, Article.published_at <= window_end) + .order_by(Article.published_at.desc()) + .limit(300) # cap the scan; this is a heuristic ranker, not a full-text search index + ).scalars().all() + + scored = [(_score_article(a, hints["keywords"], hints["country"]), a) for a in rows] + scored = [(s, a) for s, a in scored if s > 0] + scored.sort(key=lambda pair: (pair[0], pair[1].published_at), reverse=True) + return [a.id for _, a in scored[:limit]] + + +def _detect_and_record_spike( + session: Session, symbol: str, label: str, prev: MarketPrice | None, price: float, recorded_at: dt.datetime +) -> None: + if prev is None or not prev.price: + return + pct_change = (price - prev.price) / prev.price * 100 + if abs(pct_change) < settings.market_spike_threshold_pct: + return + + window_start = prev.recorded_at - dt.timedelta(hours=settings.market_spike_lookback_hours) + article_ids = _find_candidate_articles(session, symbol, window_start, recorded_at) + + session.add( + MarketSpike( + symbol=symbol, + label=label, + from_price=prev.price, + to_price=price, + pct_change=round(pct_change, 3), + window_start=window_start, + window_end=recorded_at, + detected_at=recorded_at, + article_ids_json=json.dumps(article_ids), + ) + ) + log.info("Spike detected: %s %.2f%% (%d candidate articles)", symbol, pct_change, len(article_ids)) + + +def poll_markets(session: Session) -> int: + added = 0 + with httpx.Client() as client: + for symbol, label, category, currency in INSTRUMENTS: + try: + result = _fetch_one(client, symbol) + if result is None: + continue + price, change_pct = result + recorded_at = dt.datetime.utcnow() + + prev = session.execute( + select(MarketPrice) + .where(MarketPrice.symbol == symbol) + .order_by(MarketPrice.recorded_at.desc()) + .limit(1) + ).scalar_one_or_none() + + session.add( + MarketPrice( + symbol=symbol, + label=label, + category=category, + price=price, + currency=currency, + change_pct=change_pct, + recorded_at=recorded_at, + ) + ) + _detect_and_record_spike(session, symbol, label, prev, price, recorded_at) + added += 1 + except Exception: + log.exception("Failed to fetch %s", symbol) + + session.commit() + return added diff --git a/backend/app/models.py b/backend/app/models.py new file mode 100644 index 0000000..fdeb972 --- /dev/null +++ b/backend/app/models.py @@ -0,0 +1,82 @@ +import datetime as dt + +from sqlalchemy import DateTime, Float, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from .db import Base + + +class Article(Base): + __tablename__ = "articles" + __table_args__ = (UniqueConstraint("url", name="uq_article_url"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source: Mapped[str] = mapped_column(String(128), index=True) + source_bias: Mapped[str] = mapped_column(String(64), default="") + title: Mapped[str] = mapped_column(Text) + url: Mapped[str] = mapped_column(Text, index=True) + summary: Mapped[str] = mapped_column(Text, default="") + + published_at: Mapped[dt.datetime] = mapped_column(DateTime, index=True) + fetched_at: Mapped[dt.datetime] = mapped_column(DateTime, default=dt.datetime.utcnow) + + location_name: Mapped[str | None] = mapped_column(String(128), nullable=True, index=True) + country: Mapped[str | None] = mapped_column(String(128), nullable=True) + lat: Mapped[float | None] = mapped_column(Float, nullable=True) + lon: Mapped[float | None] = mapped_column(Float, nullable=True) + + # Rounded lat/lon grid key articles are clustered on, e.g. "31.5:34.5" + cluster_key: Mapped[str | None] = mapped_column(String(64), nullable=True, index=True) + + +class MarketPrice(Base): + __tablename__ = "market_prices" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + symbol: Mapped[str] = mapped_column(String(32), index=True) + label: Mapped[str] = mapped_column(String(128)) + category: Mapped[str] = mapped_column(String(32)) # "index" | "commodity" + price: Mapped[float] = mapped_column(Float) + currency: Mapped[str] = mapped_column(String(8), default="USD") + change_pct: Mapped[float | None] = mapped_column(Float, nullable=True) + recorded_at: Mapped[dt.datetime] = mapped_column(DateTime, index=True, default=dt.datetime.utcnow) + + +class MarketSpike(Base): + __tablename__ = "market_spikes" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + symbol: Mapped[str] = mapped_column(String(32), index=True) + label: Mapped[str] = mapped_column(String(128)) + from_price: Mapped[float] = mapped_column(Float) + to_price: Mapped[float] = mapped_column(Float) + pct_change: Mapped[float] = mapped_column(Float) + + window_start: Mapped[dt.datetime] = mapped_column(DateTime) + window_end: Mapped[dt.datetime] = mapped_column(DateTime) + detected_at: Mapped[dt.datetime] = mapped_column(DateTime, index=True, default=dt.datetime.utcnow) + + # JSON-encoded list of Article.id, ranked most-to-least likely relevant. + article_ids_json: Mapped[str] = mapped_column(Text, default="[]") + + +class ConflictEvent(Base): + __tablename__ = "conflict_events" + __table_args__ = (UniqueConstraint("source", "external_id", name="uq_conflict_event"),) + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + source: Mapped[str] = mapped_column(String(32), default="acled") + external_id: Mapped[str] = mapped_column(String(64)) + + event_type: Mapped[str] = mapped_column(String(128)) + actor1: Mapped[str] = mapped_column(String(256), default="") + actor2: Mapped[str] = mapped_column(String(256), default="") + fatalities: Mapped[int | None] = mapped_column(Integer, nullable=True) + notes: Mapped[str] = mapped_column(Text, default="") + + location_name: Mapped[str] = mapped_column(String(128), default="") + country: Mapped[str] = mapped_column(String(128), default="") + lat: Mapped[float] = mapped_column(Float) + lon: Mapped[float] = mapped_column(Float) + + event_date: Mapped[dt.datetime] = mapped_column(DateTime, index=True) diff --git a/backend/app/scheduler.py b/backend/app/scheduler.py new file mode 100644 index 0000000..dcf853a --- /dev/null +++ b/backend/app/scheduler.py @@ -0,0 +1,56 @@ +import logging + +from apscheduler.schedulers.background import BackgroundScheduler + +from .conflict import poll_conflict_events +from .config import settings +from .db import SessionLocal +from .ingest import fetch_all +from .markets import poll_markets + +log = logging.getLogger("newsatlas.scheduler") + + +def _run_rss_job() -> None: + session = SessionLocal() + try: + added = fetch_all(session) + log.info("RSS poll complete: %d new articles", added) + finally: + session.close() + + +def _run_market_job() -> None: + session = SessionLocal() + try: + added = poll_markets(session) + log.info("Market poll complete: %d instruments recorded", added) + finally: + session.close() + + +def _run_conflict_job() -> None: + session = SessionLocal() + try: + added = poll_conflict_events(session) + log.info("Conflict-event poll complete: %d new events", added) + finally: + session.close() + + +def start_scheduler() -> BackgroundScheduler: + scheduler = BackgroundScheduler(timezone="UTC") + scheduler.add_job(_run_rss_job, "interval", minutes=settings.rss_poll_minutes, next_run_time=None) + scheduler.add_job(_run_market_job, "interval", minutes=settings.market_poll_minutes, next_run_time=None) + scheduler.add_job(_run_conflict_job, "interval", minutes=settings.conflict_poll_minutes, next_run_time=None) + scheduler.start() + + # Kick off an immediate first run of each job in the background so the + # globe isn't empty while waiting for the first interval to elapse. + import datetime as dt + + now = dt.datetime.utcnow() + for job in scheduler.get_jobs(): + job.modify(next_run_time=now) + + return scheduler diff --git a/backend/app/sources.yaml b/backend/app/sources.yaml new file mode 100644 index 0000000..f407ef6 --- /dev/null +++ b/backend/app/sources.yaml @@ -0,0 +1,63 @@ +# RSS sources for NewsAtlas. +# +# Publishers change feed URLs without notice β€” if a source stops showing +# new articles, check its /rss or /feed page and update the url below. +# `bias` is just a free-text label shown in the UI so merged clusters can +# show "reported by: BBC, RT, Al Jazeera" etc. It is not used for filtering. + +sources: + - name: Al Jazeera + url: https://www.aljazeera.com/xml/rss/all.xml + bias: qatari-state-funded + + - name: BBC News (World) + url: http://feeds.bbci.co.uk/news/world/rss.xml + bias: uk-public-broadcaster + + - name: New York Times (World) + url: https://rss.nytimes.com/services/xml/rss/nyt/World.xml + bias: us-mainstream + + - name: Washington Post (World) + url: https://feeds.washingtonpost.com/rss/world + bias: us-mainstream + + - name: In Defence of Marxism (marxist.com) + url: https://www.marxist.com/feed/rss + bias: marxist + + - name: Middle East Eye + url: https://www.middleeasteye.net/rss + bias: pro-palestinian-leaning + + - name: taz + url: https://taz.de/!p4608;rss/ + bias: german-left-leaning + + - name: RT (Russia Today) + url: https://www.rt.com/rss/ + bias: russian-state-funded + + - name: Xinhua (English) + url: http://www.xinhuanet.com/english/rss/worldrss.xml + bias: chinese-state-run + + - name: China Daily (World) + url: http://www.chinadaily.com.cn/rss/world_rss.xml + bias: chinese-state-run + + - name: Global Times + url: https://www.globaltimes.cn/rss/outbrain.xml + bias: chinese-state-run + + - name: CNN (World) + url: http://rss.cnn.com/rss/cnn_world.rss + bias: us-mainstream + + - name: Jacobin + url: https://jacobin.com/feed + bias: socialist + + - name: Der Standard (International) + url: https://www.derstandard.at/rss/international + bias: austrian-mainstream diff --git a/backend/app/wikipedia.py b/backend/app/wikipedia.py new file mode 100644 index 0000000..526fb8c --- /dev/null +++ b/backend/app/wikipedia.py @@ -0,0 +1,115 @@ +"""Wikipedia integration: location summaries, free-text search (for the +"search via Wikipedia" text-selection feature), and best-effort national +parliament composition diagrams. + +Everything here is a thin, cached proxy around Wikipedia's public REST/action +APIs so the frontend never talks to wikipedia.org directly (avoids CORS and +lets us cache/rate-limit centrally). +""" + +import time +from functools import lru_cache + +import httpx +import yaml + +from .config import APP_DIR + +REST_SUMMARY = "https://en.wikipedia.org/api/rest_v1/page/summary/{title}" +OPENSEARCH = "https://en.wikipedia.org/w/api.php" +HEADERS = {"User-Agent": "NewsAtlas/1.0 (self-hosted news globe; contact: admin@localhost)"} + +_PARLIAMENTS_FILE = APP_DIR / "data" / "parliaments.yaml" +_CACHE_TTL = 6 * 3600 # seconds +_summary_cache: dict[str, tuple[float, dict | None]] = {} +_parliament_resolve_cache: dict[str, tuple[float, str | None]] = {} + + +@lru_cache(maxsize=1) +def _parliament_map() -> dict[str, str | None]: + with open(_PARLIAMENTS_FILE, encoding="utf-8") as f: + return yaml.safe_load(f) or {} + + +def _cache_get(cache: dict, key: str): + entry = cache.get(key) + if entry and time.time() - entry[0] < _CACHE_TTL: + return entry[1] + return "MISS" + + +async def get_summary(title: str) -> dict | None: + cached = _cache_get(_summary_cache, title) + if cached != "MISS": + return cached + + result = None + async with httpx.AsyncClient(headers=HEADERS, timeout=10) as client: + resp = await client.get(REST_SUMMARY.format(title=title.replace(" ", "_"))) + if resp.status_code == 200: + data = resp.json() + if data.get("type") != "disambiguation": + result = { + "title": data.get("title"), + "extract": data.get("extract"), + "thumbnail": (data.get("thumbnail") or {}).get("source"), + "original_image": (data.get("originalimage") or {}).get("source"), + "page_url": (data.get("content_urls", {}).get("desktop") or {}).get("page"), + } + + _summary_cache[title] = (time.time(), result) + return result + + +async def search(query: str, limit: int = 5) -> list[dict]: + if not query or not query.strip(): + return [] + params = { + "action": "opensearch", + "search": query.strip(), + "limit": str(limit), + "namespace": "0", + "format": "json", + } + async with httpx.AsyncClient(headers=HEADERS, timeout=10) as client: + resp = await client.get(OPENSEARCH, params=params) + resp.raise_for_status() + _, titles, snippets, urls = resp.json() + return [ + {"title": t, "snippet": s, "page_url": u} + for t, s, u in zip(titles, snippets, urls) + ] + + +async def _resolve_parliament_title(country: str) -> str | None: + mapping = _parliament_map() + if country in mapping: + return mapping[country] # may be explicit None -> "no legislature" + + cached = _cache_get(_parliament_resolve_cache, country) + if cached != "MISS": + return cached + + results = await search(f"{country} parliament", limit=1) + resolved = results[0]["title"] if results else None + _parliament_resolve_cache[country] = (time.time(), resolved) + return resolved + + +async def get_parliament(country: str) -> dict | None: + title = await _resolve_parliament_title(country) + if not title: + return None + summary = await get_summary(title) + if not summary: + return None + diagram = summary.get("original_image") or summary.get("thumbnail") + if not diagram: + return None + return { + "country": country, + "legislature": summary["title"], + "diagram_url": diagram, + "extract": summary.get("extract"), + "page_url": summary.get("page_url"), + } diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 0000000..e0a4946 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,10 @@ +fastapi==0.115.0 +uvicorn[standard]==0.30.6 +feedparser==6.0.11 +apscheduler==3.10.4 +sqlalchemy==2.0.35 +httpx==0.27.2 +python-dateutil==2.9.0.post0 +pydantic==2.9.2 +pydantic-settings==2.5.2 +pyyaml==6.0.2 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..431900a --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +services: + backend: + build: ./backend + container_name: newsatlas-backend + restart: unless-stopped + env_file: + - .env + volumes: + - ./data:/data + - ./backend/app/sources.yaml:/app/app/sources.yaml:ro + - ./backend/app/data/gazetteer.csv:/app/app/data/gazetteer.csv:ro + - ./backend/app/data/parliaments.yaml:/app/app/data/parliaments.yaml:ro + expose: + - "8000" + + nginx: + image: nginx:alpine + container_name: newsatlas-nginx + restart: unless-stopped + depends_on: + - backend + ports: + - "${HTTP_PORT:-8080}:80" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + - ./frontend:/usr/share/nginx/html:ro diff --git a/frontend/css/style.css b/frontend/css/style.css new file mode 100644 index 0000000..c4617e6 --- /dev/null +++ b/frontend/css/style.css @@ -0,0 +1,356 @@ +* { box-sizing: border-box; } + +/* + * CyberQueer theme β€” sourced from the user's ~/Dotfiles/colors.conf: + * COLOR_TEXT=D6ABAB COLOR_BG=1A1A1A COLOR_HIGHLIGHT=E40046 + * COLOR_DARK=5018DD COLOR_RED=F50505 + * Everything else here is a derived tint/shade of those five anchors. + */ +:root { + --rail-width: 320px; + --topbar-height: 52px; + --drawer-handle-height: 38px; + + --c-bg: #1a1a1a; + --c-bg-deep: #0f0d16; /* darker-than-bg base for the page behind panels */ + --c-text: #d6abab; + --c-text-muted: #a68a9c; + --c-highlight: #e40046; /* hot pink β€” primary accent */ + --c-highlight-soft: #ff4d82; /* lightened highlight for small text/links on dark bg */ + --c-dark: #5018dd; /* electric violet β€” secondary accent */ + --c-dark-soft: #9d7cff; /* lightened violet for small text/links on dark bg */ + --c-red: #f50505; /* danger / down */ + --c-green: #34d399; /* gain β€” kept distinct from theme for finance semantics */ + + --c-panel: rgba(24, 15, 33, 0.97); + --c-panel-border: #3a2159; + --c-panel-hover: #241533; + --c-tag-bg: #2a1a45; +} + +html, body { + margin: 0; + height: 100%; + background: var(--c-bg-deep); + color: var(--c-text); + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; + overflow: hidden; +} + +#globeViz { + position: absolute; + top: var(--topbar-height); + left: var(--rail-width); + right: var(--rail-width); + bottom: 0; +} + +#topbar { + position: absolute; + top: 0; + left: 0; + right: 0; + height: var(--topbar-height); + display: flex; + align-items: center; + justify-content: space-between; + padding: 0 18px; + background: #120a1e; + border-bottom: 1px solid var(--c-panel-border); + z-index: 30; +} + +#topbar h1 { + font-size: 18px; + letter-spacing: 0.08em; + margin: 0; + font-weight: 600; + background: linear-gradient(90deg, var(--c-dark-soft), var(--c-highlight-soft)); + -webkit-background-clip: text; + background-clip: text; + color: transparent; +} + +#layerToggles { + display: flex; + align-items: center; + gap: 14px; + font-size: 13px; + color: var(--c-text); +} + +#layerToggles label { + display: flex; + align-items: center; + gap: 4px; + cursor: pointer; + white-space: nowrap; +} +#layerToggles input[type="checkbox"] { accent-color: var(--c-highlight); } + +#layerToggles select { + background: #1c1030; + color: var(--c-text); + border: 1px solid var(--c-panel-border); + border-radius: 4px; + font-size: 12px; + padding: 2px 4px; +} + +#refreshBtn { + background: var(--c-tag-bg); + color: var(--c-text); + border: 1px solid var(--c-panel-border); + border-radius: 6px; + padding: 4px 10px; + cursor: pointer; + font-size: 12px; +} +#refreshBtn:hover { background: #3d2260; border-color: var(--c-dark); } + +#lastUpdated { opacity: 0.6; font-size: 11px; } + +/* ---- docked rails (left = news, right = markets) ---- */ + +.rail { + position: absolute; + top: var(--topbar-height); + bottom: 0; + width: var(--rail-width); + background: var(--c-panel); + z-index: 15; + overflow-y: auto; + padding: 14px 14px 20px; +} +#leftPanel { left: 0; border-right: 1px solid var(--c-panel-border); } +#rightPanel { right: 0; border-left: 1px solid var(--c-panel-border); } + +.rail h2 { + font-size: 12px; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--c-dark-soft); + margin: 4px 0 10px; + position: sticky; + top: 0; + background: inherit; +} + +.scroll-list { display: flex; flex-direction: column; gap: 2px; } + +/* news feed items */ +.news-item { + padding: 9px 0; + border-bottom: 1px solid #241533; + cursor: default; +} +.news-item a { + color: var(--c-text); + text-decoration: none; + font-size: 13.5px; + line-height: 1.35; +} +.news-item a:hover { color: var(--c-highlight-soft); text-decoration: underline; } + +.favicon { + width: 13px; + height: 13px; + border-radius: 3px; + vertical-align: -2px; + margin-right: 5px; + background: #241533; /* placeholder box while loading / if the icon is transparent */ +} +.news-item .article-meta { font-size: 11px; color: var(--c-text-muted); margin-top: 3px; display: flex; align-items: center; gap: 6px; flex-wrap: wrap; } +.locate-btn { + background: var(--c-tag-bg); + border: 1px solid var(--c-panel-border); + color: #c9a8d8; + border-radius: 3px; + font-size: 10px; + padding: 0 4px; + cursor: pointer; +} +.locate-btn:hover { background: #3d2260; color: #fff; } + +.bias-tag { + display: inline-block; + font-size: 10px; + background: var(--c-tag-bg); + border: 1px solid var(--c-panel-border); + border-radius: 3px; + padding: 1px 5px; + color: #c9a8d8; +} + +/* markets rail items */ +.market-item { + padding: 9px 4px; + border-bottom: 1px solid #241533; + cursor: pointer; +} +.market-item:hover { background: var(--c-panel-hover); } +.market-row { display: flex; justify-content: space-between; align-items: baseline; font-size: 13px; } +.market-row .label { color: var(--c-text); } +.market-row .price { font-weight: 600; } +.market-detail { margin-top: 8px; font-size: 12px; } +.market-detail.hidden { display: none; } + +.spark-container { position: relative; margin-top: 6px; } +.spark { display: block; width: 100%; height: 44px; overflow: visible; } +.spark-tooltip { + position: absolute; + top: -4px; + background: #1c1030; + border: 1px solid var(--c-panel-border); + border-radius: 4px; + padding: 2px 6px; + font-size: 10px; + color: var(--c-text); + white-space: nowrap; + pointer-events: none; + z-index: 2; +} + +.up { color: var(--c-green); } +.down { color: var(--c-red); } + +/* ---- cluster modal ---- */ + +.modal { position: absolute; inset: 0; z-index: 40; display: flex; align-items: center; justify-content: center; } +.modal.hidden { display: none; } +#modalBackdrop { position: absolute; inset: 0; background: rgba(10, 0, 20, 0.65); } +.modal-card { + position: relative; + width: 420px; + max-width: 88vw; + max-height: 82vh; + overflow-y: auto; + background: #150a24; + border: 1px solid var(--c-panel-border); + border-radius: 10px; + padding: 20px 18px; + box-shadow: 0 12px 40px rgba(0,0,0,0.6), 0 0 60px rgba(80, 24, 221, 0.15); +} +#closeModal { + position: absolute; + top: 8px; + right: 10px; + background: none; + border: none; + color: var(--c-text-muted); + font-size: 22px; + cursor: pointer; +} +#closeModal:hover { color: var(--c-highlight-soft); } + +.panel-section { margin-bottom: 20px; } +.panel-section h2 { + font-size: 13px; + text-transform: uppercase; + letter-spacing: 0.06em; + color: var(--c-dark-soft); + margin: 0 0 8px; + border-bottom: 1px solid var(--c-panel-border); + padding-bottom: 4px; +} + +.headline { font-size: 17px; font-weight: 600; margin: 4px 0 4px; line-height: 1.3; color: #f2dede; } +.subtle { color: var(--c-text-muted); font-size: 12px; } + +.article-item { padding: 10px 0; border-bottom: 1px solid #241533; } +.article-item a { color: var(--c-text); text-decoration: none; font-size: 14px; line-height: 1.35; } +.article-item a:hover { color: var(--c-highlight-soft); text-decoration: underline; } +.article-meta { font-size: 11px; color: var(--c-text-muted); margin-top: 3px; } + +.wiki-box { display: flex; gap: 10px; } +.wiki-box img { width: 84px; height: 84px; object-fit: cover; border-radius: 6px; flex-shrink: 0; } +.wiki-box p { font-size: 13px; line-height: 1.4; margin: 0 0 6px; color: #c9b2ba; } +.wiki-box a { color: var(--c-dark-soft); font-size: 12px; } + +.parliament-diagram { max-width: 100%; border-radius: 6px; background: #fff; padding: 6px; } + +/* ---- wikipedia text-selection popup ---- */ + +#wikiSelectPopup { + position: absolute; + z-index: 50; + background: #1c1030; + border: 1px solid var(--c-panel-border); + border-radius: 6px; + padding: 6px; + box-shadow: 0 4px 16px rgba(0,0,0,0.5); + font-size: 12px; + max-width: 280px; +} +#wikiSelectPopup.hidden { display: none; } +#wikiSelectPopup button.trigger { + background: var(--c-tag-bg); + border: 1px solid var(--c-panel-border); + color: var(--c-text); + border-radius: 4px; + padding: 4px 8px; + cursor: pointer; + font-size: 12px; +} +#wikiSelectPopup .wiki-result { + display: block; + padding: 6px 4px; + border-top: 1px solid var(--c-panel-border); + color: #d8c3cc; + text-decoration: none; +} +#wikiSelectPopup .wiki-result:hover { background: #2a1a45; } +#wikiSelectPopup .wiki-result b { color: var(--c-highlight-soft); display: block; } + +/* ---- bottom drawer: military / conflict event log ---- */ + +#militaryDrawer { + position: absolute; + left: var(--rail-width); + right: var(--rail-width); + bottom: 0; + z-index: 25; + background: var(--c-panel); + border-top: 1px solid var(--c-panel-border); + max-height: 46vh; + display: flex; + flex-direction: column; + transition: max-height 0.2s ease; +} +#militaryDrawer.collapsed { max-height: var(--drawer-handle-height); } + +#drawerHandle { + height: var(--drawer-handle-height); + flex-shrink: 0; + width: 100%; + background: none; + border: none; + color: var(--c-text); + font-size: 12px; + letter-spacing: 0.04em; + cursor: pointer; + display: flex; + align-items: center; + gap: 8px; + justify-content: center; +} +#drawerHandle:hover { background: var(--c-panel-hover); } +#drawerArrow { display: inline-block; transition: transform 0.2s ease; font-size: 10px; color: var(--c-highlight); } +#militaryDrawer.collapsed #drawerArrow { transform: rotate(180deg); } + +.drawer-body { overflow-y: auto; padding: 4px 18px 16px; } +#militaryDrawer.collapsed .drawer-body { display: none; } + +.conflict-item { + display: flex; + justify-content: space-between; + gap: 10px; + padding: 7px 0; + border-bottom: 1px solid #241533; + font-size: 12.5px; +} +.conflict-item .desc { color: var(--c-text); } +.conflict-item .meta { color: var(--c-text-muted); font-size: 11px; white-space: nowrap; } +.conflict-item .fatalities { color: var(--c-red); } + +.hidden { display: none !important; } diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..ddf8df7 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,61 @@ + + + + + + NewsAtlas + + + +
+ +
+

NewsAtlas

+
+ + + + + +
+
+ + + + + + + + + +
+ +
+
+ + + + + + diff --git a/frontend/js/app.js b/frontend/js/app.js new file mode 100644 index 0000000..bc5d3ba --- /dev/null +++ b/frontend/js/app.js @@ -0,0 +1,808 @@ +const API = "/api"; + +let config = { weather_enabled: false, conflict_enabled: false }; +let clusters = []; // raw, server-side ~11km-grid clusters +let displayedClusters = []; // zoom-adaptive regrouping of `clusters`, currently on screen +let conflictEvents = []; +let clickRings = []; +let weatherMeshes = {}; // layer -> THREE.Mesh (built lazily, cached) +let activeWeatherLayer = null; +let currentAltitude = 2.2; + +// ---------------------------------------------------------------- globe -- + +const globe = Globe()(document.getElementById("globeViz")) + .globeImageUrl("https://unpkg.com/three-globe/example/img/earth-blue-marble.jpg") + .bumpImageUrl("https://unpkg.com/three-globe/example/img/earth-topology.png") + .backgroundColor("#0f0d16") + .pointLat("lat") + .pointLng("lon") + .pointAltitude(0.012) + .pointRadius(pointRadiusFor) + .pointColor(colorForCluster) + .pointLabel( + (d) => + `
+ ${escapeHtml(d.headline)}
+ ${d.article_count} stor${d.article_count === 1 ? "y" : "ies"} Β· ${d.source_count} source(s) + ${d.location_name ? `
${escapeHtml(d.location_name)}` : ""} +
` + ) + .onPointClick(onClusterClick) + .onZoom(({ altitude }) => { + currentAltitude = altitude; + scheduleRegroup(); + }) + .ringLat("lat") + .ringLng("lon") + .ringColor((d) => (t) => + d.kind === "conflict" ? `rgba(245,5,5,${1 - t})` : `rgba(228,0,70,${1 - t})` + ) + .ringMaxRadius((d) => d.maxR) + .ringPropagationSpeed(3) + .ringRepeatPeriod((d) => d.repeatPeriod); + +globe.pointOfView({ lat: 20, lng: 20, altitude: 2.2 }, 0); + +// Tint the globe surface (blue-marble texture) into the theme's violet +// range by multiplying it against the material's diffuse color, rather +// than a CSS filter on the whole canvas β€” that would also hue-shift the +// point/ring accent colors set below, which need to stay true to the +// palette. Must wait for onGlobeReady: three-globe (re)builds the mesh's +// material once the texture has loaded, which would otherwise orphan a +// tint applied immediately after construction. +globe.onGlobeReady(() => { + const globeMaterial = globe.globeMaterial(); + globeMaterial.color = new THREE.Color(0x6a3fd9); + globeMaterial.emissive = new THREE.Color(0x1a0e33); + globeMaterial.emissiveIntensity = 0.25; +}); + +// National border overlay: transparent fill (so it doesn't obscure the globe +// texture or news points) with a bright themed stroke. Natural Earth's +// admin-0 country polygons, served as a static asset from three-globe's own +// npm package via unpkg. +globe + .polygonAltitude(0.001) + .polygonCapColor(() => "rgba(0,0,0,0)") + .polygonSideColor(() => "rgba(0,0,0,0)") + .polygonStrokeColor(() => "#c9a6ff") + .polygonLabel((d) => escapeHtml(d.properties.ADMIN || d.properties.NAME || "")); + +fetch("https://unpkg.com/three-globe@2.32.0/example/country-polygons/ne_110m_admin_0_countries.geojson") + .then((res) => res.json()) + .then((geo) => globe.polygonsData(geo.features)) + .catch((e) => console.warn("Could not load country borders", e)); + +function resizeGlobe() { + const el = document.getElementById("globeViz"); + globe.width(el.clientWidth).height(el.clientHeight); +} +window.addEventListener("resize", resizeGlobe); +setTimeout(resizeGlobe, 0); + +function colorForCluster(d) { + // violet (few corroborating sources) -> hot pink (many) β€” theme accent gradient + const n = Math.min(d.source_count, 6); + const palette = ["#5018DD", "#7018C4", "#9018AB", "#B01092", "#D00879", "#E40046"]; + return palette[n - 1] || palette[0]; +} + +// pointRadius is in degrees of arc on the globe surface (three-globe's own +// unit for this accessor β€” not pixels, not a fraction of globe radius), so +// it's directly comparable to lat/lon distances for the decluttering pass +// below. A fixed degree-size reads as bigger on screen the closer the +// camera gets, so we also shrink it toward zoomed-in altitudes β€” that both +// matches "smaller when zoomed in" and leaves more breathing room for +// decluttering dense regions like the Levant. +function pointRadiusFor(d, altitude = currentAltitude) { + const zoomScale = Math.max(0.4, Math.min(1.2, altitude / 2.2)); + return zoomScale * Math.min(2.0, 0.22 + Math.log2(d.article_count + 1) * 0.28); +} + +// ---------------------------------------------- zoom-adaptive regrouping -- +// +// The server clusters articles onto a fixed ~11km grid (see backend +// clustering.py). That's the right resolution when zoomed in on a city, but +// when zoomed out to see a continent it leaves dozens of separate dots that +// visually belong together. Rather than re-query the server per zoom level, +// we do a second, cheap client-side merge pass: nearby base-clusters get +// folded into a single displayed point, with the fold radius growing with +// camera altitude. Clicking a merged point fetches articles from every +// constituent grid cell. + +let regroupScheduled = false; +function scheduleRegroup() { + if (regroupScheduled) return; + regroupScheduled = true; + requestAnimationFrame(() => { + regroupScheduled = false; + displayedClusters = regroupForAltitude(clusters, currentAltitude); + globe.pointsData(displayedClusters); + }); +} + +function mergeRadiusDeg(altitude) { + // altitude ~0.4 (close, city-level) -> ~0deg (no extra merging beyond the server grid) + // altitude ~2.2 (default view) -> ~13deg (nearby-country scale, e.g. Levant stays + // together but Ukraine/Iran/China stay apart) + // altitude ~4+ (fully zoomed out) -> capped at 32deg (subcontinent scale) + return Math.max(0, Math.min(32, (altitude - 0.4) * 7)); +} + +function approxDegDistance(a, b) { + const avgLatRad = (((a.lat + b.lat) / 2) * Math.PI) / 180; + const dx = (a.lon - b.lon) * Math.cos(avgLatRad); + const dy = a.lat - b.lat; + return Math.sqrt(dx * dx + dy * dy); +} + +function regroupForAltitude(baseClusters, altitude) { + const radius = mergeRadiusDeg(altitude); + let result; + if (radius <= 0.05) { + result = baseClusters.map((c) => ({ ...c, cluster_keys: [c.cluster_key] })); + } else { + const bySize = [...baseClusters].sort((a, b) => b.article_count - a.article_count); + const visited = new Set(); + const groups = []; + + for (const seed of bySize) { + if (visited.has(seed.cluster_key)) continue; + const group = [seed]; + visited.add(seed.cluster_key); + for (const other of bySize) { + if (visited.has(other.cluster_key)) continue; + if (approxDegDistance(seed, other) <= radius) { + group.push(other); + visited.add(other.cluster_key); + } + } + groups.push(group); + } + + result = groups.map(mergeGroup); + } + + // Grouping alone doesn't stop dense regions (e.g. the Levant, where Gaza/ + // Israel/West Bank/Jerusalem/Palestine all sit within ~1deg of each other) + // from rendering as several large dots stacked on top of each other. Push + // still-overlapping points apart so every one stays individually visible + // and clickable, rather than the biggest one blotting out its neighbors. + return declutter(result, altitude); +} + +function declutter(points, altitude, iterations = 8) { + // Work on plain {lat, lon} + a back-reference so we don't mutate the + // input objects mid-pass (order of processing would otherwise bias the + // result). + const pts = points.map((p) => ({ lat: p.lat, lon: p.lon, r: pointRadiusFor(p, altitude) })); + + for (let iter = 0; iter < iterations; iter++) { + let moved = false; + for (let i = 0; i < pts.length; i++) { + for (let j = i + 1; j < pts.length; j++) { + const a = pts[i]; + const b = pts[j]; + const minDist = (a.r + b.r) * 0.85; // slight overlap tolerance reads as "touching," not gapless + const avgLatRad = (((a.lat + b.lat) / 2) * Math.PI) / 180; + const cosLat = Math.max(0.05, Math.cos(avgLatRad)); + let dx = (b.lon - a.lon) * cosLat; + let dy = b.lat - a.lat; + let dist = Math.sqrt(dx * dx + dy * dy); + if (dist < 1e-4) { + const angle = Math.random() * Math.PI * 2; + dx = Math.cos(angle) * 0.01; + dy = Math.sin(angle) * 0.01; + dist = 0.01; + } + if (dist < minDist) { + moved = true; + const push = (minDist - dist) / 2; + const ux = dx / dist; + const uy = dy / dist; + a.lon -= (ux * push) / cosLat; + a.lat -= uy * push; + b.lon += (ux * push) / cosLat; + b.lat += uy * push; + } + } + } + if (!moved) break; + } + + return points.map((p, i) => ({ + ...p, + lat: Math.max(-85, Math.min(85, pts[i].lat)), + lon: pts[i].lon, + })); +} + +function mergeGroup(group) { + const totalArticles = group.reduce((sum, g) => sum + g.article_count, 0); + const lat = group.reduce((sum, g) => sum + g.lat * g.article_count, 0) / totalArticles; + const lon = group.reduce((sum, g) => sum + g.lon * g.article_count, 0) / totalArticles; + const sources = new Set(group.flatMap((g) => g.sources)); + const countries = new Set(group.map((g) => g.country).filter(Boolean)); + const representative = group.reduce((best, g) => (g.article_count > best.article_count ? g : best), group[0]); + + let locationName; + if (group.length === 1) locationName = group[0].location_name; + else if (countries.size === 1) locationName = `${[...countries][0]} (${group.length} locations)`; + else locationName = `${group.length} locations across ${countries.size} countries`; + + return { + cluster_key: group.map((g) => g.cluster_key).join("|"), + cluster_keys: group.map((g) => g.cluster_key), + lat, + lon, + location_name: locationName, + country: countries.size === 1 ? [...countries][0] : null, + article_count: totalArticles, + source_count: sources.size, + sources: [...sources], + topic: representative.topic, + latest_published_at: group.reduce((max, g) => (g.latest_published_at > max ? g.latest_published_at : max), group[0].latest_published_at), + headline: representative.headline, + }; +} + +function escapeHtml(s) { + return (s || "").replace(/[&<>"']/g, (c) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }[c])); +} + +function flyTo(lat, lon) { + globe.pointOfView({ lat, lng: lon, altitude: 1.4 }, 1200); + const ring = { lat, lon, kind: "click", maxR: 6, repeatPeriod: 4000 }; + clickRings.push(ring); + syncRings(); + setTimeout(() => { + clickRings = clickRings.filter((r) => r !== ring); + syncRings(); + }, 1600); +} + +function syncRings() { + const conflictRingData = document.getElementById("toggleConflict").checked + ? conflictEvents.map((e) => ({ lat: e.lat, lon: e.lon, kind: "conflict", maxR: 4, repeatPeriod: 2200 })) + : []; + globe.ringsData([...conflictRingData, ...clickRings]); +} + +// ------------------------------------------------------------ clusters -- + +async function loadClusters() { + try { + const res = await fetch(`${API}/clusters`); + clusters = await res.json(); + scheduleRegroup(); + document.getElementById("lastUpdated").textContent = "updated " + new Date().toLocaleTimeString(); + } catch (e) { + console.error("Failed to load clusters", e); + } +} + +function onClusterClick(cluster) { + flyTo(cluster.lat, cluster.lon); + openClusterModal(cluster); +} + +// -------------------------------------------------------------- modal -- + +const clusterModal = document.getElementById("clusterModal"); +const modalContent = document.getElementById("modalContent"); + +document.getElementById("closeModal").onclick = () => clusterModal.classList.add("hidden"); +document.getElementById("modalBackdrop").onclick = () => clusterModal.classList.add("hidden"); + +async function openClusterModal(cluster) { + clusterModal.classList.remove("hidden"); + modalContent.innerHTML = ` +
${escapeHtml(cluster.headline)}
+
${cluster.article_count} stories Β· ${cluster.source_count} source(s) + ${cluster.location_name ? " Β· " + escapeHtml(cluster.location_name) : ""} +
+

About this place

Loading…

+
+

Stories

Loading…
+ `; + + loadWikiSummary(cluster.location_name || cluster.country); + if (cluster.country) loadParliament(cluster.country); + loadClusterArticles(cluster.cluster_keys || [cluster.cluster_key]); +} + +async function loadClusterArticles(clusterKeys) { + const el = document.getElementById("articleList"); + try { + const results = await Promise.all( + clusterKeys.map((key) => fetch(`${API}/clusters/${encodeURIComponent(key)}/articles`).then((r) => r.json())) + ); + const seen = new Set(); + const articles = results + .flat() + .filter((a) => (seen.has(a.id) ? false : (seen.add(a.id), true))) + .sort((a, b) => new Date(b.published_at) - new Date(a.published_at)); + el.innerHTML = articles.map(articleItemHtml).join(""); + } catch (e) { + el.textContent = "Could not load stories."; + } +} + +// Decorative publisher favicon, derived from the article's own URL so it +// works for any source without hand-curating logo assets (and without +// hotlinking an outlet's actual trademarked logo β€” favicons are the +// standard low-risk "site icon" browsers/RSS readers already show). +function faviconFor(url) { + try { + const host = new URL(url).hostname; + return `https://www.google.com/s2/favicons?sz=32&domain=${encodeURIComponent(host)}`; + } catch (e) { + return ""; + } +} + +function faviconImgHtml(a) { + const src = faviconFor(a.url); + if (!src) return ""; + return ``; +} + +function articleItemHtml(a) { + return ` +
+ ${faviconImgHtml(a)}${escapeHtml(a.title)} +
+ ${escapeHtml(a.source)}${escapeHtml(a.source_bias || "")} + Β· ${new Date(a.published_at).toLocaleString()} +
+
`; +} + +async function loadWikiSummary(title) { + const el = document.getElementById("wikiSection"); + if (!title) { + el.innerHTML = ""; + return; + } + try { + const res = await fetch(`${API}/wikipedia/summary?title=${encodeURIComponent(title)}`); + if (!res.ok) throw new Error("not found"); + const w = await res.json(); + el.innerHTML = ` +

About ${escapeHtml(w.title)}

+
+ ${w.thumbnail ? `` : ""} +
+

${escapeHtml((w.extract || "").slice(0, 260))}${(w.extract || "").length > 260 ? "…" : ""}

+ Read on Wikipedia β†’ +
+
`; + } catch (e) { + el.innerHTML = ""; + } +} + +async function loadParliament(country) { + const el = document.getElementById("parliamentSection"); + try { + const res = await fetch(`${API}/wikipedia/parliament?country=${encodeURIComponent(country)}`); + if (!res.ok) throw new Error("not found"); + const p = await res.json(); + el.innerHTML = ` +

${escapeHtml(p.legislature)}

+ Composition of ${escapeHtml(p.legislature)} +

+ Full composition on Wikipedia β†’ +

`; + } catch (e) { + el.innerHTML = ""; + } +} + +// ----------------------------------------------------------- news rail -- + +async function loadNewsFeed() { + const el = document.getElementById("newsFeed"); + try { + const res = await fetch(`${API}/articles?limit=60`); + const articles = await res.json(); + el.innerHTML = articles + .map( + (a) => ` +
+ ${faviconImgHtml(a)}${escapeHtml(a.title)} +
+ ${escapeHtml(a.source)} + ${escapeHtml(a.source_bias || "")} + ${new Date(a.published_at).toLocaleString()} + ${ + a.lat != null + ? `` + : "" + } +
+
` + ) + .join(""); + el.querySelectorAll(".locate-btn").forEach((btn) => { + btn.onclick = () => flyTo(parseFloat(btn.dataset.lat), parseFloat(btn.dataset.lon)); + }); + } catch (e) { + console.error("Failed to load news feed", e); + } +} + +// -------------------------------------------------- wikipedia text-select + +const wikiPopup = document.getElementById("wikiSelectPopup"); + +document.addEventListener("mouseup", (ev) => { + if (wikiPopup.contains(ev.target)) return; + const sel = window.getSelection(); + const text = sel ? sel.toString().trim() : ""; + if (text.length < 3 || text.length > 120) { + wikiPopup.classList.add("hidden"); + return; + } + const range = sel.getRangeAt(0); + const rect = range.getBoundingClientRect(); + wikiPopup.style.left = Math.min(rect.left + window.scrollX, window.innerWidth - 300) + "px"; + wikiPopup.style.top = rect.bottom + window.scrollY + 6 + "px"; + wikiPopup.innerHTML = ``; + wikiPopup.classList.remove("hidden"); + wikiPopup.querySelector(".trigger").onclick = () => searchWikipedia(text); +}); + +async function searchWikipedia(query) { + wikiPopup.innerHTML = `
Searching…
`; + try { + const res = await fetch(`${API}/wikipedia/search?q=${encodeURIComponent(query)}`); + const results = await res.json(); + if (!results.length) { + wikiPopup.innerHTML = `
No Wikipedia results.
`; + return; + } + wikiPopup.innerHTML = results + .map( + (r) => + ` + ${escapeHtml(r.title)}${escapeHtml(r.snippet || "")} + ` + ) + .join(""); + } catch (e) { + wikiPopup.innerHTML = `
Search failed.
`; + } +} + +document.addEventListener("mousedown", (ev) => { + if (!wikiPopup.contains(ev.target)) wikiPopup.classList.add("hidden"); +}); + +// -------------------------------------------------------------- weather -- + +const TILE_ZOOM = 2; // 4x4 tiles => manageable fetch count for a demo overlay + +async function buildWeatherMesh(layer) { + const n = 2 ** TILE_ZOOM; + const canvas = document.createElement("canvas"); + canvas.width = n * 256; + canvas.height = n * 256; + const ctx = canvas.getContext("2d"); + + const loads = []; + for (let x = 0; x < n; x++) { + for (let y = 0; y < n; y++) { + loads.push( + loadImage(`${API}/weather/tiles/${layer}/${TILE_ZOOM}/${x}/${y}.png`).then((img) => { + ctx.drawImage(img, x * 256, y * 256, 256, 256); + }) + ); + } + } + await Promise.all(loads); + + const texture = new THREE.CanvasTexture(canvas); + const radius = globe.getGlobeRadius() * 1.012; + const geometry = new THREE.SphereGeometry(radius, 64, 64); + const material = new THREE.MeshBasicMaterial({ + map: texture, + transparent: true, + opacity: 0.55, + depthWrite: false, + }); + const mesh = new THREE.Mesh(geometry, material); + mesh.rotation.y = Math.PI / 2; + return mesh; +} + +function loadImage(src) { + return new Promise((resolve, reject) => { + const img = new Image(); + img.crossOrigin = "anonymous"; + img.onload = () => resolve(img); + img.onerror = reject; + img.src = src; + }); +} + +async function setWeatherLayer(layer) { + Object.values(weatherMeshes).forEach((m) => (m.visible = false)); + if (!layer) return; + if (!weatherMeshes[layer]) { + weatherMeshes[layer] = await buildWeatherMesh(layer); + globe.scene().add(weatherMeshes[layer]); + } + weatherMeshes[layer].visible = true; +} + +// --------------------------------------------- conflict / military drawer + +const militaryDrawer = document.getElementById("militaryDrawer"); +document.getElementById("drawerHandle").onclick = () => militaryDrawer.classList.toggle("collapsed"); + +async function loadConflictEvents() { + const logEl = document.getElementById("militaryLog"); + const countEl = document.getElementById("militaryCount"); + if (!config.conflict_enabled) { + logEl.innerHTML = `

Conflict-event overlay not configured β€” set ACLED_API_KEY / ACLED_EMAIL in .env to enable (see README).

`; + countEl.textContent = ""; + return; + } + try { + const res = await fetch(`${API}/conflict-events`); + conflictEvents = await res.json(); + conflictEvents.sort((a, b) => new Date(b.event_date) - new Date(a.event_date)); + countEl.textContent = `(${conflictEvents.length})`; + logEl.innerHTML = conflictEvents.length + ? conflictEvents + .map( + (e) => ` +
+
+ ${escapeHtml(e.event_type)} β€” ${escapeHtml(e.actor1)}${e.actor2 ? " vs " + escapeHtml(e.actor2) : ""} +
${escapeHtml(e.location_name)}, ${escapeHtml(e.country)} + ${e.fatalities ? ` Β· ${e.fatalities} fatalities` : ""} +
+
${new Date(e.event_date).toLocaleDateString()}
+
` + ) + .join("") + : `

No events recorded in the current window.

`; + syncRings(); + } catch (e) { + console.error("Failed to load conflict events", e); + } +} + +// ------------------------------------------------------------ markets -- + +async function loadMarkets() { + const el = document.getElementById("marketsList"); + try { + const res = await fetch(`${API}/markets/latest`); + const rows = await res.json(); + el.innerHTML = rows + .map((r, i) => { + const dir = r.change_pct > 0 ? "up" : r.change_pct < 0 ? "down" : ""; + const arrow = r.change_pct > 0 ? "β–²" : r.change_pct < 0 ? "β–Ό" : ""; + return ` +
+
+ ${escapeHtml(r.label)} + ${r.price.toLocaleString(undefined, { maximumFractionDigits: 2 })} ${r.currency} +
+
+ ${new Date(r.recorded_at).toLocaleTimeString()} + ${arrow} ${r.change_pct != null ? r.change_pct.toFixed(2) + "%" : ""} +
+
+ +
`; + }) + .join(""); + el.querySelectorAll(".market-item").forEach((item) => { + item.onclick = () => toggleMarketDetail(item.dataset.symbol, item.dataset.idx); + }); + rows.forEach((r, i) => loadSparkline(r.symbol, i)); + } catch (e) { + console.error("Failed to load markets", e); + } +} + +// 7-day price history as a small inline line+area chart, with a hover +// crosshair/tooltip (per dataviz interaction guidance: line charts ship +// hover by default). Single series per instrument, so no legend β€” the +// stroke reuses the theme's existing up/down status colors rather than +// introducing a new categorical hue. +async function loadSparkline(symbol, idx) { + const container = document.getElementById(`spark-${idx}`); + if (!container) return; + try { + const res = await fetch(`${API}/markets/history?symbol=${encodeURIComponent(symbol)}&hours=168`); + const history = await res.json(); + mountSparkline(container, history); + } catch (e) { + console.error("Failed to load sparkline for", symbol, e); + } +} + +function mountSparkline(container, history) { + container.innerHTML = ""; + if (!history || history.length < 2) { + container.innerHTML = `

not enough history yet

`; + return; + } + + const width = 280; + const height = 44; + const pad = 3; + const prices = history.map((h) => h.price); + const min = Math.min(...prices); + const max = Math.max(...prices); + const span = max - min || Math.abs(prices[0]) * 0.001 || 1; + const up = prices[prices.length - 1] >= prices[0]; + const color = up ? "var(--c-green)" : "var(--c-red)"; + const pts = history.map((h, i) => ({ + x: pad + (i / (history.length - 1)) * (width - 2 * pad), + y: height - pad - ((h.price - min) / span) * (height - 2 * pad), + price: h.price, + t: h.recorded_at, + })); + + const svgNS = "http://www.w3.org/2000/svg"; + const svg = document.createElementNS(svgNS, "svg"); + svg.setAttribute("viewBox", `0 0 ${width} ${height}`); + svg.setAttribute("preserveAspectRatio", "none"); + svg.classList.add("spark"); + + const linePath = pts.map((p, i) => `${i === 0 ? "M" : "L"}${p.x.toFixed(1)},${p.y.toFixed(1)}`).join(" "); + const areaPath = `${linePath} L${pts[pts.length - 1].x.toFixed(1)},${height - pad} L${pts[0].x.toFixed(1)},${height - pad} Z`; + + const area = document.createElementNS(svgNS, "path"); + area.setAttribute("d", areaPath); + area.setAttribute("fill", color); + area.setAttribute("fill-opacity", "0.14"); + area.setAttribute("stroke", "none"); + svg.appendChild(area); + + const line = document.createElementNS(svgNS, "path"); + line.setAttribute("d", linePath); + line.setAttribute("fill", "none"); + line.setAttribute("stroke", color); + line.setAttribute("stroke-width", "2"); + line.setAttribute("stroke-linecap", "round"); + line.setAttribute("stroke-linejoin", "round"); + svg.appendChild(line); + + const crosshair = document.createElementNS(svgNS, "line"); + crosshair.setAttribute("y1", "0"); + crosshair.setAttribute("y2", String(height)); + crosshair.setAttribute("stroke", "var(--c-text-muted)"); + crosshair.setAttribute("stroke-width", "1"); + crosshair.setAttribute("stroke-dasharray", "2,2"); + crosshair.style.display = "none"; + svg.appendChild(crosshair); + + const dot = document.createElementNS(svgNS, "circle"); + dot.setAttribute("r", "2.6"); + dot.setAttribute("fill", color); + dot.style.display = "none"; + svg.appendChild(dot); + + container.appendChild(svg); + const tooltip = document.createElement("div"); + tooltip.className = "spark-tooltip hidden"; + container.appendChild(tooltip); + + svg.addEventListener("mousemove", (ev) => { + const rect = svg.getBoundingClientRect(); + const relX = ((ev.clientX - rect.left) / rect.width) * width; + let nearest = 0; + let bestDist = Infinity; + pts.forEach((p, i) => { + const d = Math.abs(p.x - relX); + if (d < bestDist) { + bestDist = d; + nearest = i; + } + }); + const p = pts[nearest]; + crosshair.setAttribute("x1", p.x.toFixed(1)); + crosshair.setAttribute("x2", p.x.toFixed(1)); + crosshair.style.display = ""; + dot.setAttribute("cx", p.x.toFixed(1)); + dot.setAttribute("cy", p.y.toFixed(1)); + dot.style.display = ""; + tooltip.textContent = `${p.price.toLocaleString(undefined, { maximumFractionDigits: 2 })} Β· ${new Date(p.t).toLocaleString()}`; + tooltip.classList.remove("hidden"); + const leftPct = p.x / width; + tooltip.style.left = `${Math.min(70, Math.max(0, leftPct * 100 - 30))}%`; + }); + svg.addEventListener("mouseleave", () => { + crosshair.style.display = "none"; + dot.style.display = "none"; + tooltip.classList.add("hidden"); + }); +} + +async function toggleMarketDetail(symbol, idx) { + const el = document.getElementById(`market-detail-${idx}`); + const wasHidden = el.classList.contains("hidden"); + document.querySelectorAll(".market-detail").forEach((d) => d.classList.add("hidden")); + if (!wasHidden) return; + el.classList.remove("hidden"); + el.innerHTML = `

Loading recent moves…

`; + try { + const res = await fetch(`${API}/markets/spikes?symbol=${encodeURIComponent(symbol)}&hours=720`); + const spikes = await res.json(); + if (!spikes.length) { + el.innerHTML = `

No moves β‰₯ the spike threshold recorded yet.

`; + return; + } + el.innerHTML = spikes + .map((s) => { + const dir = s.pct_change > 0 ? "up" : "down"; + const arrow = s.pct_change > 0 ? "β–²" : "β–Ό"; + const articlesHtml = s.candidate_articles.length + ? s.candidate_articles.slice(0, 5).map(articleItemHtml).join("") + : `

No strongly-matching stories found in that window.

`; + return ` +

${arrow} ${s.pct_change.toFixed(2)}% + at ${new Date(s.detected_at).toLocaleString()} + (${s.from_price.toLocaleString()} β†’ ${s.to_price.toLocaleString()})

+ ${articlesHtml} + `; + }) + .join("
"); + } catch (e) { + el.innerHTML = `

Could not load spike data.

`; + } +} + +// --------------------------------------------------------------- wiring -- + +document.getElementById("refreshBtn").onclick = async () => { + await fetch(`${API}/refresh`, { method: "POST" }); + setTimeout(loadClusters, 1500); + setTimeout(loadNewsFeed, 1500); +}; + +document.getElementById("toggleWeather").onchange = (ev) => { + activeWeatherLayer = ev.target.checked ? document.getElementById("weatherLayer").value : null; + setWeatherLayer(activeWeatherLayer); +}; +document.getElementById("weatherLayer").onchange = (ev) => { + if (document.getElementById("toggleWeather").checked) setWeatherLayer(ev.target.value); +}; + +document.getElementById("toggleConflict").onchange = () => { + if (!conflictEvents.length) loadConflictEvents(); + else syncRings(); +}; + +async function init() { + try { + config = await (await fetch(`${API}/config`)).json(); + } catch (e) { + console.warn("Could not load config; assuming weather/conflict disabled", e); + } + document.getElementById("toggleWeather").disabled = !config.weather_enabled; + document.getElementById("toggleConflict").disabled = !config.conflict_enabled; + + await loadClusters(); + await loadNewsFeed(); + await loadMarkets(); + await loadConflictEvents(); + resizeGlobe(); + + setInterval(loadClusters, 60_000); + setInterval(loadNewsFeed, 60_000); + setInterval(loadMarkets, 5 * 60_000); + setInterval(loadConflictEvents, 10 * 60_000); +} + +init(); diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..6ee9de9 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,53 @@ +worker_processes auto; + +events { + worker_connections 1024; +} + +http { + include mime.types; + default_type application/octet-stream; + sendfile on; + keepalive_timeout 65; + + gzip on; + gzip_types text/css application/javascript application/json image/svg+xml; + + # Simple in-memory cache for upstream JSON responses so many globe + # clients don't hammer the backend/SQLite on every poll. + proxy_cache_path /tmp/newsatlas_cache levels=1:2 keys_zone=api_cache:10m max_size=100m inactive=5m; + + upstream backend { + server backend:8000; + } + + server { + listen 80; + server_name _; + + root /usr/share/nginx/html; + index index.html; + + location / { + try_files $uri $uri/ /index.html; + } + + location /api/ { + proxy_pass http://backend/api/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + proxy_cache api_cache; + proxy_cache_valid 200 30s; + proxy_cache_use_stale error timeout updating; + add_header X-Cache-Status $upstream_cache_status; + } + + location /healthz { + access_log off; + return 200 "ok\n"; + } + } +}