"""Fleet scripts — one uploaded script per platform, fetched by the endpoints. WHAT THIS IS FOR ---------------- Getting every machine in the house into CheckMK monitoring means running an agent installer on each of them, and the installer is not the same file on a Debian x86 thin client, an arm64 Raspberry Pi audio endpoint and the GPU host. So: one slot per platform, uploaded through the admin UI, fetched by each endpoint over HTTP. Every machine in the house gets a slot, including the two kinds that cannot run a script at all — see PLATFORMS. The slots are a FIXED SET, not a free-form list of files. A fixed set means every endpoint knows which slot is its own without being told, the UI can say plainly which platforms are covered and which are still empty, and nobody has to invent a naming convention that then has to be kept in two places. THIS SERVICE NEVER EXECUTES ANYTHING ------------------------------------- It stores text and it serves text. Execution happens on the endpoint, by that endpoint's own systemd unit, under that endpoint's own root — which is exactly where the decision to run it belongs. Nothing here reaches out to a machine, and there is no "deploy now" button, because a button that runs a script on eleven machines at once is a button that breaks eleven machines at once. UPLOAD IS NOT DEPLOY -------------------- An upload creates a new **draft** version. Endpoints only ever fetch the **published** one, and publishing is a second, deliberate action. This is the same propose-then- confirm rule pantry-vision applies to stock writes and identity applies to merges, and it matters more here than in either: the thing being confirmed will run as root on every machine in the house. WHY THE SCRIPTS LIVE IN SQLITE AND NOT ON THE SMB SHARE -------------------------------------------------------- The workspace share is writable by anyone with the SMB password. A file that will be executed as root on every endpoint must not be writable by that path — it would mean a share credential is silently a whole-fleet code-execution credential, with no version history and no record of who changed what. In the database each version is immutable, checksummed, and attributed. This is the one place where "artefacts go on the share" is the wrong rule. WHAT AN ENDPOINT SEES, AND WHAT IT REPORTS ------------------------------------------- `GET /fleet/script/` returns the published body plus its sha256 and version. The endpoint runs it and POSTs back what it ran. That report is the only way to answer "is this machine actually monitored yet", and it is deliberately reported by the endpoint rather than assumed by this service: a script that was served is not a script that succeeded. """ from __future__ import annotations import hashlib import logging import os import sqlite3 from datetime import datetime, timezone from pathlib import Path LOG = logging.getLogger("workshop.fleet") DB_PATH = Path(os.environ.get("WORKSHOP_KNOWLEDGE_DB_PATH", "/data/workshop-knowledge.db")) # One slot per KIND OF MACHINE in the house — every machine gets monitored, so every # machine needs a slot. Fixed set on purpose (see the module docstring): each endpoint # knows which slot is its own without being told, and the UI can show which platforms # are still uncovered, which is the thing somebody setting this up needs to see. # # The split is by "would the installer actually differ", not by role. A thin client and # a touch panel are both Debian x86 running the same agent, so they share a slot; the # container host does not, because a Docker host wants the agent plus container checks # and its own plugin set. Splitting a slot later is easy; merging two that people have # already written different scripts into is not. PLATFORMS = { "debian-x86": { "label": "Debian / x86_64 kiosk endpoints", "covers": "thin clients, touch panel, kitchen display, door panel, workshop display", "runs_on_device": True, }, "raspbian-arm": { "label": "Raspberry Pi OS / arm64 endpoints", "covers": "the arm64 audio endpoints, and any future Pi", "runs_on_device": True, }, "container-host": { "label": "Docker / container host", "covers": "the Phase 1 machine running the whole compose stack — the agent plus " "container and disk checks, which is a different install from a kiosk's", "runs_on_device": True, }, "llm-host": { "label": "LLM / GPU host", "covers": "the Ollama machine — a different package set, and the one host where " "GPU temperature and VRAM are worth checking at all", "runs_on_device": True, }, "esphome": { "label": "ESP32 devices (voice satellites, BLE proxies, RuView nodes)", "covers": "microcontrollers. THEY CANNOT RUN A SHELL SCRIPT — this slot holds the " "CheckMK-side piece instead (an active check or a special agent that " "polls their ESPHome/MQTT state from the monitoring server). The slot " "exists so 'have we got the ESPs monitored?' has a visible answer " "rather than being quietly out of scope", # Nothing fetches this slot: no endpoint of this kind runs an agent. "runs_on_device": False, }, "network-appliance": { "label": "Firewalls, switches and network cameras", "covers": "OPNsense and anything else with no place to install an agent. Same " "shape as the ESP slot: the script belongs on the CheckMK server " "(SNMP or a special agent), not on the device", # Nothing fetches this slot: no endpoint of this kind runs an agent. "runs_on_device": False, }, } MAX_SCRIPT_BYTES = int(os.environ.get("WORKSHOP_MAX_SCRIPT_KB", "256")) * 1024 def _now() -> str: return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") def init_db() -> None: with sqlite3.connect(DB_PATH, timeout=10) as conn: conn.executescript( """ -- Every version ever uploaded, kept. Rolling back is picking an older row, -- which is only possible if nothing deletes them — the same append-only -- reasoning git_ops.py applies to history, for the same reason: what runs -- as root on the fleet must always be recoverable. CREATE TABLE IF NOT EXISTS fleet_scripts ( id INTEGER PRIMARY KEY, platform TEXT NOT NULL, version INTEGER NOT NULL, body TEXT NOT NULL, -- Shown in the UI and returned to the endpoint, which records what it -- ran. This is how "what is actually on that machine" stops being a -- guess. sha256 TEXT NOT NULL, note TEXT, -- Upload is not deploy: a new row starts unpublished and endpoints -- never see it until somebody publishes it deliberately. published INTEGER NOT NULL DEFAULT 0, uploaded_at TEXT NOT NULL, published_at TEXT ); CREATE UNIQUE INDEX IF NOT EXISTS fleet_scripts_version ON fleet_scripts (platform, version); -- What each endpoint says it ran. REPORTED, never assumed: a script that -- was served is not a script that succeeded, and the difference is the -- whole point of having this table. CREATE TABLE IF NOT EXISTS fleet_reports ( id INTEGER PRIMARY KEY, hostname TEXT NOT NULL, platform TEXT NOT NULL, version INTEGER, sha256 TEXT, ok INTEGER NOT NULL DEFAULT 0, detail TEXT, reported_at TEXT NOT NULL ); CREATE INDEX IF NOT EXISTS fleet_reports_host ON fleet_reports (hostname, reported_at); """ ) def _sha256(body: str) -> str: return hashlib.sha256(body.encode("utf-8")).hexdigest() def upload(platform: str, body: str, note: str = "") -> dict: """Store a new draft version. Never published by this call.""" if platform not in PLATFORMS: return {"ok": False, "reason": "bad_platform", "message": f"Unknown platform. Known: {', '.join(sorted(PLATFORMS))}."} body = str(body or "") if not body.strip(): return {"ok": False, "reason": "bad_field", "message": "The script is empty."} if len(body.encode("utf-8")) > MAX_SCRIPT_BYTES: return {"ok": False, "reason": "too_large", "message": f"Scripts are capped at {MAX_SCRIPT_BYTES // 1024} KB. Something " "bigger than that is a package, not a bootstrap script."} digest = _sha256(body) with sqlite3.connect(DB_PATH, timeout=10) as conn: conn.row_factory = sqlite3.Row existing = conn.execute( "SELECT id, version, published FROM fleet_scripts WHERE platform = ? AND sha256 = ? " "ORDER BY version DESC LIMIT 1", (platform, digest) ).fetchone() if existing: # Byte-identical to a version already stored. Storing it again would create # two versions nobody can tell apart, and the honest answer to "did this # change?" is no. return {"ok": True, "unchanged": True, "version": existing["version"], "sha256": digest, "published": bool(existing["published"]), "message": f"Identical to version {existing['version']} — nothing stored."} row = conn.execute( "SELECT MAX(version) AS v FROM fleet_scripts WHERE platform = ?", (platform,) ).fetchone() version = int((row["v"] or 0)) + 1 conn.execute( "INSERT INTO fleet_scripts (platform, version, body, sha256, note, published, " "uploaded_at) VALUES (?, ?, ?, ?, ?, 0, ?)", (platform, version, body, digest, str(note or "").strip() or None, _now()), ) LOG.info("workshop: stored %s script v%d (%s), unpublished", platform, version, digest[:12]) return {"ok": True, "version": version, "sha256": digest, "published": False, "message": "Stored as a draft. Endpoints will not fetch it until you publish it."} def publish(platform: str, version: int) -> dict: """Make one version the one endpoints fetch. Exactly one per platform is live.""" if platform not in PLATFORMS: return {"ok": False, "reason": "bad_platform", "message": "Unknown platform."} with sqlite3.connect(DB_PATH, timeout=10) as conn: conn.row_factory = sqlite3.Row row = conn.execute( "SELECT id FROM fleet_scripts WHERE platform = ? AND version = ?", (platform, version) ).fetchone() if row is None: return {"ok": False, "reason": "no_such_version", "message": "No such version."} # Unpublish the rest first: two published versions of one platform would make # "what do the endpoints run" unanswerable, which is the question this exists # to answer. conn.execute("UPDATE fleet_scripts SET published = 0 WHERE platform = ?", (platform,)) conn.execute( "UPDATE fleet_scripts SET published = 1, published_at = ? WHERE id = ?", (_now(), row["id"]), ) LOG.info("workshop: published %s script v%d", platform, version) return {"ok": True, "platform": platform, "version": version} def published(platform: str) -> dict | None: """What an endpoint of this platform should run, or None.""" if platform not in PLATFORMS: return None with sqlite3.connect(DB_PATH, timeout=10) as conn: conn.row_factory = sqlite3.Row row = conn.execute( "SELECT version, body, sha256, note, published_at FROM fleet_scripts " "WHERE platform = ? AND published = 1", (platform,) ).fetchone() return {k: row[k] for k in row.keys()} if row else None def overview() -> dict: """Every slot, whether it is filled, and which endpoints have reported. Empty slots are listed rather than omitted — "no script for arm64 yet" is the thing somebody setting this up needs to see, and a UI that only lists what exists cannot show it. """ slots = [] with sqlite3.connect(DB_PATH, timeout=10) as conn: conn.row_factory = sqlite3.Row for platform, meta in PLATFORMS.items(): versions = [ {k: r[k] for k in r.keys()} for r in conn.execute( "SELECT version, sha256, note, published, uploaded_at, published_at " "FROM fleet_scripts WHERE platform = ? ORDER BY version DESC LIMIT 10", (platform,), ) ] live = next((v for v in versions if v["published"]), None) slots.append({ "platform": platform, "label": meta["label"], "covers": meta["covers"], # False means no device fetches this — the script belongs on the CheckMK # server instead. The UI says so rather than showing a slot that looks # broken because nothing ever reports against it. "runs_on_device": meta.get("runs_on_device", True), "published_version": live["version"] if live else None, "published_sha256": live["sha256"] if live else None, "versions": versions, }) reports = [ {k: r[k] for k in r.keys()} for r in conn.execute( "SELECT hostname, platform, version, sha256, ok, detail, MAX(reported_at) " "AS reported_at FROM fleet_reports GROUP BY hostname ORDER BY hostname" ) ] # The cross-check that makes the page worth opening: a host that ran an older # version than the one now published is drifted, and nothing else here would say so. live_by_platform = {s["platform"]: s["published_version"] for s in slots} for report in reports: expected = live_by_platform.get(report["platform"]) report["current"] = expected is not None and report["version"] == expected return {"slots": slots, "reports": reports} def report(payload: dict) -> dict: """An endpoint saying what it ran and how it went.""" hostname = str(payload.get("hostname") or "").strip() platform = str(payload.get("platform") or "").strip() if not hostname or platform not in PLATFORMS: return {"ok": False, "reason": "bad_field", "message": "'hostname' and a known 'platform' are both required."} try: version = int(payload.get("version")) except (TypeError, ValueError): version = None with sqlite3.connect(DB_PATH, timeout=10) as conn: conn.execute( "INSERT INTO fleet_reports (hostname, platform, version, sha256, ok, detail, " "reported_at) VALUES (?, ?, ?, ?, ?, ?, ?)", (hostname, platform, version, str(payload.get("sha256") or "") or None, 1 if payload.get("ok") else 0, str(payload.get("detail") or "")[:2000], _now()), ) LOG.info("workshop: %s reported %s v%s (ok=%s)", hostname, platform, version, bool(payload.get("ok"))) return {"ok": True}