97 lines
4.4 KiB
Python
97 lines
4.4 KiB
Python
"""Infrastructure health for the network digest — CheckMK and every firewall.
|
|
|
|
READS THE POLLER, DOES NOT POLL. `workshop/health.py` already asks CheckMK and each
|
|
OPNsense box every few minutes and keeps a month of samples; this module fetches that
|
|
service's `GET /health` and reshapes it for the network section.
|
|
|
|
That indirection is the whole design and it is worth one paragraph. The digest runs
|
|
four times a day. "Is the NAS disk failing right now" is not a question with a
|
|
six-hour answer, so polling from here would have produced a snapshot taken at 06:00
|
|
and quoted at 12:00. Reading the poller instead gives the digest something a snapshot
|
|
cannot have: **how long the state has held**. "Critical since Tuesday" is a different
|
|
sentence from "critical", and it is the one that tells you whether to get up.
|
|
|
|
It also means one set of credentials in one place. digest-engine never learns the
|
|
CheckMK secret or any firewall's API key for this — those live in workshop's env file,
|
|
and this module needs only workshop's own bearer token.
|
|
|
|
WHY THIS IS NOT IN opnsense_ids.py
|
|
-----------------------------------
|
|
That module reads Suricata's *alert log*, with paging and a time window, from one
|
|
firewall. This one reads *service state* from many. They answer different questions
|
|
("what fired" vs "is it running and is anything broken"), and the alert query is
|
|
deliberately left where it already works rather than reimplemented here.
|
|
|
|
DEGRADES TO NOTHING, LOUDLY
|
|
----------------------------
|
|
An unreachable workshop service returns `None` and the run continues — same rule as
|
|
every other ingestion module here. But when the poller *is* reachable and reports a
|
|
target as `unreachable`, that is passed through as a finding rather than dropped: a
|
|
firewall nobody can reach is exactly the thing a network digest exists to mention.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import os
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
WORKSHOP_URL = os.environ.get("WORKSHOP_URL", "").rstrip("/")
|
|
WORKSHOP_TOKEN = os.environ.get("WORKSHOP_TOKEN", "")
|
|
TIMEOUT = float(os.environ.get("WORKSHOP_TIMEOUT", "10"))
|
|
|
|
|
|
def fetch() -> list[dict]:
|
|
"""Health entries for the digest context, or [] when unavailable.
|
|
|
|
Every entry carries `target` — the firewall's name or the CheckMK site — because a
|
|
household with two firewalls must never be handed "the IDS is running" as if there
|
|
were one of them. The prompt is told to name the target in anything it says.
|
|
"""
|
|
if not (WORKSHOP_URL and WORKSHOP_TOKEN):
|
|
LOG.info("infra_health: WORKSHOP_URL/WORKSHOP_TOKEN unset, skipping")
|
|
return []
|
|
|
|
try:
|
|
req = urllib.request.Request(f"{WORKSHOP_URL}/health")
|
|
req.add_header("Authorization", f"Bearer {WORKSHOP_TOKEN}")
|
|
with urllib.request.urlopen(req, timeout=TIMEOUT) as resp:
|
|
payload = json.loads(resp.read() or b"{}")
|
|
except (urllib.error.URLError, urllib.error.HTTPError, TimeoutError, OSError, ValueError):
|
|
LOG.warning("infra_health: could not reach the workshop health poller", exc_info=True)
|
|
return []
|
|
|
|
if not payload.get("configured"):
|
|
# Nothing is being watched. Distinct from "everything is fine", and the prompt
|
|
# must never render the second when the truth is the first.
|
|
return []
|
|
|
|
entries = []
|
|
for row in payload.get("results", []) or []:
|
|
problems = row.get("detail") or []
|
|
entries.append({
|
|
"category": "infra_health",
|
|
"source": row.get("source"),
|
|
"target": row.get("target"),
|
|
"state": row.get("state"),
|
|
"title": f"{row.get('target')}: {row.get('state')}",
|
|
"summary": row.get("summary", ""),
|
|
"problem_count": row.get("problem_count", 0),
|
|
# When this state started. None means it has held for the whole retention
|
|
# window, which the prompt should read as "long-standing", not "just now".
|
|
"since": row.get("since"),
|
|
"checked_at": row.get("checked_at"),
|
|
# Capped: a site with 200 failing services is a real state, and putting all
|
|
# 200 in an LLM context is not how you say so.
|
|
"problems": problems[:12],
|
|
"problems_truncated": max(0, len(problems) - 12),
|
|
})
|
|
|
|
if entries:
|
|
LOG.info("infra_health: %d target(s), overall %s", len(entries), payload.get("overall"))
|
|
return entries
|