321 lines
15 KiB
Python
321 lines
15 KiB
Python
"""Fleet scripts — one uploaded script per platform, fetched by the endpoints.
|
|
|
|
WHAT THIS IS FOR
|
|
----------------
|
|
Getting every machine in the house into CheckMK monitoring means running an agent
|
|
installer on each of them, and the installer is not the same file on a Debian x86 thin
|
|
client, an arm64 Raspberry Pi audio endpoint and the GPU host. So: one slot per
|
|
platform, uploaded through the admin UI, fetched by each endpoint over HTTP. Every
|
|
machine in the house gets a slot, including the two kinds that cannot run a script at
|
|
all — see PLATFORMS.
|
|
|
|
The slots are a FIXED SET, not a free-form list of files. A fixed set means every
|
|
endpoint knows which slot is its own without being told, the UI can say plainly which
|
|
platforms are covered and which are still empty, and nobody has to invent a naming
|
|
convention that then has to be kept in two places.
|
|
|
|
THIS SERVICE NEVER EXECUTES ANYTHING
|
|
-------------------------------------
|
|
It stores text and it serves text. Execution happens on the endpoint, by that
|
|
endpoint's own systemd unit, under that endpoint's own root — which is exactly where
|
|
the decision to run it belongs. Nothing here reaches out to a machine, and there is no
|
|
"deploy now" button, because a button that runs a script on eleven machines at once is
|
|
a button that breaks eleven machines at once.
|
|
|
|
UPLOAD IS NOT DEPLOY
|
|
--------------------
|
|
An upload creates a new **draft** version. Endpoints only ever fetch the **published**
|
|
one, and publishing is a second, deliberate action. This is the same propose-then-
|
|
confirm rule pantry-vision applies to stock writes and identity applies to merges, and
|
|
it matters more here than in either: the thing being confirmed will run as root on
|
|
every machine in the house.
|
|
|
|
WHY THE SCRIPTS LIVE IN SQLITE AND NOT ON THE SMB SHARE
|
|
--------------------------------------------------------
|
|
The workspace share is writable by anyone with the SMB password. A file that will be
|
|
executed as root on every endpoint must not be writable by that path — it would mean a
|
|
share credential is silently a whole-fleet code-execution credential, with no version
|
|
history and no record of who changed what. In the database each version is immutable,
|
|
checksummed, and attributed. This is the one place where "artefacts go on the share"
|
|
is the wrong rule.
|
|
|
|
WHAT AN ENDPOINT SEES, AND WHAT IT REPORTS
|
|
-------------------------------------------
|
|
`GET /fleet/script/<platform>` returns the published body plus its sha256 and version.
|
|
The endpoint runs it and POSTs back what it ran. That report is the only way to answer
|
|
"is this machine actually monitored yet", and it is deliberately reported by the
|
|
endpoint rather than assumed by this service: a script that was served is not a script
|
|
that succeeded.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import logging
|
|
import os
|
|
import sqlite3
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
|
|
LOG = logging.getLogger("workshop.fleet")
|
|
|
|
DB_PATH = Path(os.environ.get("WORKSHOP_KNOWLEDGE_DB_PATH", "/data/workshop-knowledge.db"))
|
|
|
|
# One slot per KIND OF MACHINE in the house — every machine gets monitored, so every
|
|
# machine needs a slot. Fixed set on purpose (see the module docstring): each endpoint
|
|
# knows which slot is its own without being told, and the UI can show which platforms
|
|
# are still uncovered, which is the thing somebody setting this up needs to see.
|
|
#
|
|
# The split is by "would the installer actually differ", not by role. A thin client and
|
|
# a touch panel are both Debian x86 running the same agent, so they share a slot; the
|
|
# container host does not, because a Docker host wants the agent plus container checks
|
|
# and its own plugin set. Splitting a slot later is easy; merging two that people have
|
|
# already written different scripts into is not.
|
|
PLATFORMS = {
|
|
"debian-x86": {
|
|
"label": "Debian / x86_64 kiosk endpoints",
|
|
"covers": "thin clients, touch panel, kitchen display, door panel, workshop display",
|
|
"runs_on_device": True,
|
|
},
|
|
"raspbian-arm": {
|
|
"label": "Raspberry Pi OS / arm64 endpoints",
|
|
"covers": "the arm64 audio endpoints, and any future Pi",
|
|
"runs_on_device": True,
|
|
},
|
|
"container-host": {
|
|
"label": "Docker / container host",
|
|
"covers": "the Phase 1 machine running the whole compose stack — the agent plus "
|
|
"container and disk checks, which is a different install from a kiosk's",
|
|
"runs_on_device": True,
|
|
},
|
|
"llm-host": {
|
|
"label": "LLM / GPU host",
|
|
"covers": "the Ollama machine — a different package set, and the one host where "
|
|
"GPU temperature and VRAM are worth checking at all",
|
|
"runs_on_device": True,
|
|
},
|
|
"esphome": {
|
|
"label": "ESP32 devices (voice satellites, BLE proxies, RuView nodes)",
|
|
"covers": "microcontrollers. THEY CANNOT RUN A SHELL SCRIPT — this slot holds the "
|
|
"CheckMK-side piece instead (an active check or a special agent that "
|
|
"polls their ESPHome/MQTT state from the monitoring server). The slot "
|
|
"exists so 'have we got the ESPs monitored?' has a visible answer "
|
|
"rather than being quietly out of scope",
|
|
# Nothing fetches this slot: no endpoint of this kind runs an agent.
|
|
"runs_on_device": False,
|
|
},
|
|
"network-appliance": {
|
|
"label": "Firewalls, switches and network cameras",
|
|
"covers": "OPNsense and anything else with no place to install an agent. Same "
|
|
"shape as the ESP slot: the script belongs on the CheckMK server "
|
|
"(SNMP or a special agent), not on the device",
|
|
# Nothing fetches this slot: no endpoint of this kind runs an agent.
|
|
"runs_on_device": False,
|
|
},
|
|
}
|
|
|
|
MAX_SCRIPT_BYTES = int(os.environ.get("WORKSHOP_MAX_SCRIPT_KB", "256")) * 1024
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def init_db() -> None:
|
|
with sqlite3.connect(DB_PATH, timeout=10) as conn:
|
|
conn.executescript(
|
|
"""
|
|
-- Every version ever uploaded, kept. Rolling back is picking an older row,
|
|
-- which is only possible if nothing deletes them — the same append-only
|
|
-- reasoning git_ops.py applies to history, for the same reason: what runs
|
|
-- as root on the fleet must always be recoverable.
|
|
CREATE TABLE IF NOT EXISTS fleet_scripts (
|
|
id INTEGER PRIMARY KEY,
|
|
platform TEXT NOT NULL,
|
|
version INTEGER NOT NULL,
|
|
body TEXT NOT NULL,
|
|
-- Shown in the UI and returned to the endpoint, which records what it
|
|
-- ran. This is how "what is actually on that machine" stops being a
|
|
-- guess.
|
|
sha256 TEXT NOT NULL,
|
|
note TEXT,
|
|
-- Upload is not deploy: a new row starts unpublished and endpoints
|
|
-- never see it until somebody publishes it deliberately.
|
|
published INTEGER NOT NULL DEFAULT 0,
|
|
uploaded_at TEXT NOT NULL,
|
|
published_at TEXT
|
|
);
|
|
CREATE UNIQUE INDEX IF NOT EXISTS fleet_scripts_version
|
|
ON fleet_scripts (platform, version);
|
|
|
|
-- What each endpoint says it ran. REPORTED, never assumed: a script that
|
|
-- was served is not a script that succeeded, and the difference is the
|
|
-- whole point of having this table.
|
|
CREATE TABLE IF NOT EXISTS fleet_reports (
|
|
id INTEGER PRIMARY KEY,
|
|
hostname TEXT NOT NULL,
|
|
platform TEXT NOT NULL,
|
|
version INTEGER,
|
|
sha256 TEXT,
|
|
ok INTEGER NOT NULL DEFAULT 0,
|
|
detail TEXT,
|
|
reported_at TEXT NOT NULL
|
|
);
|
|
CREATE INDEX IF NOT EXISTS fleet_reports_host ON fleet_reports (hostname, reported_at);
|
|
"""
|
|
)
|
|
|
|
|
|
def _sha256(body: str) -> str:
|
|
return hashlib.sha256(body.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def upload(platform: str, body: str, note: str = "") -> dict:
|
|
"""Store a new draft version. Never published by this call."""
|
|
if platform not in PLATFORMS:
|
|
return {"ok": False, "reason": "bad_platform",
|
|
"message": f"Unknown platform. Known: {', '.join(sorted(PLATFORMS))}."}
|
|
body = str(body or "")
|
|
if not body.strip():
|
|
return {"ok": False, "reason": "bad_field", "message": "The script is empty."}
|
|
if len(body.encode("utf-8")) > MAX_SCRIPT_BYTES:
|
|
return {"ok": False, "reason": "too_large",
|
|
"message": f"Scripts are capped at {MAX_SCRIPT_BYTES // 1024} KB. Something "
|
|
"bigger than that is a package, not a bootstrap script."}
|
|
|
|
digest = _sha256(body)
|
|
with sqlite3.connect(DB_PATH, timeout=10) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
existing = conn.execute(
|
|
"SELECT id, version, published FROM fleet_scripts WHERE platform = ? AND sha256 = ? "
|
|
"ORDER BY version DESC LIMIT 1", (platform, digest)
|
|
).fetchone()
|
|
if existing:
|
|
# Byte-identical to a version already stored. Storing it again would create
|
|
# two versions nobody can tell apart, and the honest answer to "did this
|
|
# change?" is no.
|
|
return {"ok": True, "unchanged": True, "version": existing["version"],
|
|
"sha256": digest, "published": bool(existing["published"]),
|
|
"message": f"Identical to version {existing['version']} — nothing stored."}
|
|
row = conn.execute(
|
|
"SELECT MAX(version) AS v FROM fleet_scripts WHERE platform = ?", (platform,)
|
|
).fetchone()
|
|
version = int((row["v"] or 0)) + 1
|
|
conn.execute(
|
|
"INSERT INTO fleet_scripts (platform, version, body, sha256, note, published, "
|
|
"uploaded_at) VALUES (?, ?, ?, ?, ?, 0, ?)",
|
|
(platform, version, body, digest, str(note or "").strip() or None, _now()),
|
|
)
|
|
LOG.info("workshop: stored %s script v%d (%s), unpublished", platform, version, digest[:12])
|
|
return {"ok": True, "version": version, "sha256": digest, "published": False,
|
|
"message": "Stored as a draft. Endpoints will not fetch it until you publish it."}
|
|
|
|
|
|
def publish(platform: str, version: int) -> dict:
|
|
"""Make one version the one endpoints fetch. Exactly one per platform is live."""
|
|
if platform not in PLATFORMS:
|
|
return {"ok": False, "reason": "bad_platform", "message": "Unknown platform."}
|
|
with sqlite3.connect(DB_PATH, timeout=10) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
row = conn.execute(
|
|
"SELECT id FROM fleet_scripts WHERE platform = ? AND version = ?", (platform, version)
|
|
).fetchone()
|
|
if row is None:
|
|
return {"ok": False, "reason": "no_such_version", "message": "No such version."}
|
|
# Unpublish the rest first: two published versions of one platform would make
|
|
# "what do the endpoints run" unanswerable, which is the question this exists
|
|
# to answer.
|
|
conn.execute("UPDATE fleet_scripts SET published = 0 WHERE platform = ?", (platform,))
|
|
conn.execute(
|
|
"UPDATE fleet_scripts SET published = 1, published_at = ? WHERE id = ?",
|
|
(_now(), row["id"]),
|
|
)
|
|
LOG.info("workshop: published %s script v%d", platform, version)
|
|
return {"ok": True, "platform": platform, "version": version}
|
|
|
|
|
|
def published(platform: str) -> dict | None:
|
|
"""What an endpoint of this platform should run, or None."""
|
|
if platform not in PLATFORMS:
|
|
return None
|
|
with sqlite3.connect(DB_PATH, timeout=10) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
row = conn.execute(
|
|
"SELECT version, body, sha256, note, published_at FROM fleet_scripts "
|
|
"WHERE platform = ? AND published = 1", (platform,)
|
|
).fetchone()
|
|
return {k: row[k] for k in row.keys()} if row else None
|
|
|
|
|
|
def overview() -> dict:
|
|
"""Every slot, whether it is filled, and which endpoints have reported.
|
|
|
|
Empty slots are listed rather than omitted — "no script for arm64 yet" is the thing
|
|
somebody setting this up needs to see, and a UI that only lists what exists cannot
|
|
show it.
|
|
"""
|
|
slots = []
|
|
with sqlite3.connect(DB_PATH, timeout=10) as conn:
|
|
conn.row_factory = sqlite3.Row
|
|
for platform, meta in PLATFORMS.items():
|
|
versions = [
|
|
{k: r[k] for k in r.keys()}
|
|
for r in conn.execute(
|
|
"SELECT version, sha256, note, published, uploaded_at, published_at "
|
|
"FROM fleet_scripts WHERE platform = ? ORDER BY version DESC LIMIT 10",
|
|
(platform,),
|
|
)
|
|
]
|
|
live = next((v for v in versions if v["published"]), None)
|
|
slots.append({
|
|
"platform": platform,
|
|
"label": meta["label"],
|
|
"covers": meta["covers"],
|
|
# False means no device fetches this — the script belongs on the CheckMK
|
|
# server instead. The UI says so rather than showing a slot that looks
|
|
# broken because nothing ever reports against it.
|
|
"runs_on_device": meta.get("runs_on_device", True),
|
|
"published_version": live["version"] if live else None,
|
|
"published_sha256": live["sha256"] if live else None,
|
|
"versions": versions,
|
|
})
|
|
|
|
reports = [
|
|
{k: r[k] for k in r.keys()}
|
|
for r in conn.execute(
|
|
"SELECT hostname, platform, version, sha256, ok, detail, MAX(reported_at) "
|
|
"AS reported_at FROM fleet_reports GROUP BY hostname ORDER BY hostname"
|
|
)
|
|
]
|
|
|
|
# The cross-check that makes the page worth opening: a host that ran an older
|
|
# version than the one now published is drifted, and nothing else here would say so.
|
|
live_by_platform = {s["platform"]: s["published_version"] for s in slots}
|
|
for report in reports:
|
|
expected = live_by_platform.get(report["platform"])
|
|
report["current"] = expected is not None and report["version"] == expected
|
|
return {"slots": slots, "reports": reports}
|
|
|
|
|
|
def report(payload: dict) -> dict:
|
|
"""An endpoint saying what it ran and how it went."""
|
|
hostname = str(payload.get("hostname") or "").strip()
|
|
platform = str(payload.get("platform") or "").strip()
|
|
if not hostname or platform not in PLATFORMS:
|
|
return {"ok": False, "reason": "bad_field",
|
|
"message": "'hostname' and a known 'platform' are both required."}
|
|
try:
|
|
version = int(payload.get("version"))
|
|
except (TypeError, ValueError):
|
|
version = None
|
|
with sqlite3.connect(DB_PATH, timeout=10) as conn:
|
|
conn.execute(
|
|
"INSERT INTO fleet_reports (hostname, platform, version, sha256, ok, detail, "
|
|
"reported_at) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
|
(hostname, platform, version, str(payload.get("sha256") or "") or None,
|
|
1 if payload.get("ok") else 0, str(payload.get("detail") or "")[:2000], _now()),
|
|
)
|
|
LOG.info("workshop: %s reported %s v%s (ok=%s)", hostname, platform, version,
|
|
bool(payload.get("ok")))
|
|
return {"ok": True}
|