diff --git a/docs/md/freeipa-ansible.md b/docs/md/freeipa-ansible.md index c678a6e..b449958 100644 --- a/docs/md/freeipa-ansible.md +++ b/docs/md/freeipa-ansible.md @@ -348,6 +348,46 @@ CheckMK agent (installed by dev_mon_base) Leaving `dev_mon_clamscan` removes the cron, scan script, log, and local check on the next enforcer run. +### Proxmox Infra Monitoring (PVE / PBS) → CheckMK + +The Proxmox VE hypervisor and Proxmox Backup Server are infrastructure +appliances, not FreeIPA clients, so they sit outside the `dev_mon_base` +host-group flow above. Instead, `image/ansipa-infra-checkmk-install.sh` +wires them into the same CheckMK instance directly: + +``` +ansipa-infra-checkmk-install.sh [HOST_FQDN] [CMK_USER] +``` + +Run as root on the PVE or PBS host itself (copy the `image/` directory over +first, or `scp` just the three files). The script: +- auto-detects the role (`pveversion` → PVE, `proxmox-backup-manager` → PBS), +- installs the CheckMK agent (`.deb`, falling back to the shell agent), +- drops the matching local check into `/usr/lib/check_mk_agent/local/`, +- registers `cmk-agent-ctl` push mode if available (useful behind NAT), +- registers the host in CheckMK under folder `/ansipa/infra` and triggers + service discovery + activate-changes. + +It is idempotent — safe to re-run after a secret rotation or hostname change. + +**`ansipa_pve`** (`image/ansipa-pve-monitor.sh`) reports, via `pvesh`: +`Ansipa_PVE_Cluster` (quorum/standalone), `Ansipa_PVE_Storage` (worst +utilisation across node storages), `Ansipa_PVE_Guests` (VM/CT running vs +stopped counts), `Ansipa_PVE_Backup` (freshness + outcome of the last +vzdump job). + +**`ansipa_pbs`** (`image/ansipa-pbs-monitor.sh`) reports, via +`proxmox-backup-manager`: `Ansipa_PBS_Datastores` (worst datastore disk +usage via `df`), `Ansipa_PBS_LastBackup` (freshness + outcome of the last +backup task), `Ansipa_PBS_TaskFailures` (any failed GC/verify/sync/prune/ +backup task in the last 24h). + +Both scripts have no dependency beyond the Proxmox CLI tooling itself (no +`jq`/`python3` required) and degrade a given service to CheckMK state 3 +(UNKNOWN) rather than failing outright when a command is unavailable. CPU/ +RAM/root-disk are already covered by CheckMK's stock Linux checks, so they +aren't duplicated here. + ### Local User Lockdown Adding a host to the `no_local_users` group locks the password of every local account — root (UID 0) and all regular users (UID ≥ 1000) — using `passwd -l`. Accounts whose passwords are already locked (`!` or `*` prefix in shadow) are left untouched and are not tracked. diff --git a/setup/modules/FreeipaAnsible/image/ansipa-infra-checkmk-install.sh b/setup/modules/FreeipaAnsible/image/ansipa-infra-checkmk-install.sh new file mode 100755 index 0000000..bf3c528 --- /dev/null +++ b/setup/modules/FreeipaAnsible/image/ansipa-infra-checkmk-install.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# ansipa-infra-checkmk-install.sh — wire a Proxmox VE hypervisor or a Proxmox +# Backup Server into the ansipa CheckMK instance. +# +# Unlike regular ansipa clients (enrolled in FreeIPA, driven by the +# dev_mon_base host-group via ansipa-enforce-policies.sh), PVE/PBS hosts are +# infrastructure appliances that are not FreeIPA members — so this script +# does the same job (install agent, drop the local check, register with +# CheckMK) standalone, taking the CheckMK automation credentials directly +# instead of reading them from an IPA hostgroup description. +# +# Usage: +# ansipa-infra-checkmk-install.sh [HOST_FQDN] [CMK_USER] +# +# CMK_URL e.g. http://mon.corp.example.com:8090 (client-routable — see +# CMK_ADVERTISED_URL in image/.env.example) +# CMK_SITE OMD site name, e.g. "cmk" +# CMK_SECRET automation user's secret (CheckMK → Users → automation → API secret) +# HOST_FQDN defaults to `hostname -f` +# CMK_USER defaults to "automation" +# +# Run as root on the PVE or PBS host itself. Re-run any time to pick up a new +# secret or to re-register after a host rename — it is idempotent. + +set -euo pipefail + +LOG_TAG="ansipa-infra-checkmk" +log() { echo "[$LOG_TAG] $*"; } +warn() { echo "[$LOG_TAG][WARN] $*" >&2; } +die() { echo "[$LOG_TAG][ERROR] $*" >&2; exit 1; } + +[[ $EUID -eq 0 ]] || die "Run as root." +[[ $# -ge 3 ]] || die "Usage: $0 [HOST_FQDN] [CMK_USER]" + +CMK_URL="${1%/}" +CMK_SITE="$2" +CMK_SECRET="$3" +HOST_FQDN="${4:-$(hostname -f 2>/dev/null || hostname)}" +CMK_USER="${5:-automation}" +CMK_API="${CMK_URL}/${CMK_SITE}/check_mk/api/1.0" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LOCAL_DIR="/usr/lib/check_mk_agent/local" + +# ── Detect role ──────────────────────────────────────────────────────────── +ROLE="" +if command -v pveversion &>/dev/null; then + ROLE="pve" +elif command -v proxmox-backup-manager &>/dev/null; then + ROLE="pbs" +else + die "Neither pveversion nor proxmox-backup-manager found — is this a PVE/PBS host?" +fi +MONITOR_SRC="$SCRIPT_DIR/ansipa-${ROLE}-monitor.sh" +[[ -f "$MONITOR_SRC" ]] || die "Companion script not found: $MONITOR_SRC" +log "Detected role: $ROLE (${HOST_FQDN})" + +# ── Install CheckMK agent (Debian .deb — PVE/PBS are Debian-based) ────────── +_AGENT_INSTALLED=false +if dpkg -s check-mk-agent &>/dev/null || command -v check_mk_agent &>/dev/null; then + _AGENT_INSTALLED=true + log "CheckMK agent already installed" +else + log "Downloading CheckMK agent from ${CMK_URL}" + _DEB=$(curl -sf -u "${CMK_USER}:${CMK_SECRET}" \ + "${CMK_URL}/${CMK_SITE}/check_mk/agents/" 2>/dev/null \ + | grep -oE 'check-mk-agent_[^"]*_all\.deb' | head -1 || true) + if [[ -n "$_DEB" ]]; then + curl -sf -u "${CMK_USER}:${CMK_SECRET}" \ + "${CMK_URL}/${CMK_SITE}/check_mk/agents/${_DEB}" \ + -o /tmp/cmk-agent.deb 2>/dev/null \ + && (dpkg -i /tmp/cmk-agent.deb 2>/dev/null || apt-get install -y -f 2>/dev/null) \ + && dpkg -s check-mk-agent &>/dev/null \ + && _AGENT_INSTALLED=true \ + || warn "CheckMK .deb install failed" + rm -f /tmp/cmk-agent.deb + fi + if [[ "$_AGENT_INSTALLED" == false ]]; then + log "Falling back to the shell agent" + curl -sf -u "${CMK_USER}:${CMK_SECRET}" \ + "${CMK_URL}/${CMK_SITE}/check_mk/agents/check_mk_agent.linux" \ + -o /usr/local/bin/check_mk_agent 2>/dev/null \ + && chmod 755 /usr/local/bin/check_mk_agent \ + && _AGENT_INSTALLED=true \ + || die "Shell agent download failed — check CMK_URL/CMK_SITE/CMK_SECRET" + fi +fi + +# ── Serve the agent: enable the package's own socket, or roll our own ─────── +if systemctl list-unit-files 2>/dev/null | grep -q '^check-mk-agent\.socket'; then + systemctl enable --now check-mk-agent.socket 2>/dev/null \ + && log "check-mk-agent.socket enabled" \ + || warn "could not enable check-mk-agent.socket" +elif [[ ! -f /etc/systemd/system/check_mk.socket ]]; then + cat > /etc/systemd/system/check_mk.socket <<'UNIT' +[Unit] +Description=Check_MK agent socket (ansipa-managed) + +[Socket] +ListenStream=6556 +Accept=yes + +[Install] +WantedBy=sockets.target +UNIT + cat > /etc/systemd/system/check_mk@.service <<'UNIT' +[Unit] +Description=Check_MK per-connection agent (ansipa-managed) + +[Service] +ExecStart=/usr/local/bin/check_mk_agent +StandardInput=socket +StandardOutput=socket +StandardError=null +UNIT + systemctl daemon-reload + systemctl enable --now check_mk.socket 2>/dev/null \ + && log "check_mk.socket enabled (port 6556)" \ + || warn "could not enable check_mk.socket" +fi + +# ── Drop in the local check ────────────────────────────────────────────────── +mkdir -p "$LOCAL_DIR" +install -m 755 "$MONITOR_SRC" "$LOCAL_DIR/ansipa_${ROLE}" +log "Installed local check: $LOCAL_DIR/ansipa_${ROLE}" + +# ── cmk-agent-ctl: register for TLS/push transport (useful behind NAT) ────── +if command -v cmk-agent-ctl &>/dev/null; then + _RECV_HOST=$(echo "$CMK_URL" | sed 's|https\?://||;s|:.*||') + _ALREADY_CTL=$(cmk-agent-ctl status 2>/dev/null | grep -c "${_RECV_HOST}/${CMK_SITE}" || true) + if [[ "${_ALREADY_CTL:-0}" -eq 0 ]]; then + cmk-agent-ctl register \ + --server "${_RECV_HOST}:8000" \ + --site "${CMK_SITE}" \ + --user "${CMK_USER}" \ + --password "${CMK_SECRET}" \ + --hostname "${HOST_FQDN}" \ + --trust-cert &>/dev/null \ + && log "cmk-agent-ctl registered with ${_RECV_HOST}:8000" \ + || warn "cmk-agent-ctl registration failed (non-fatal — pull mode still works if reachable)" + fi + _CTL_MODE=$(cmk-agent-ctl status 2>/dev/null | awk '/Connection mode:/{print $NF; exit}') + if [[ "$_CTL_MODE" == "push-agent" ]]; then + cmk-agent-ctl push 2>/dev/null || true + if [[ ! -f /etc/systemd/system/cmk-agent-push.timer ]]; then + cat > /etc/systemd/system/cmk-agent-push.service <<'UNIT' +[Unit] +Description=CheckMK agent push (ansipa-managed) +[Service] +Type=oneshot +ExecStart=/usr/bin/cmk-agent-ctl push +UNIT + cat > /etc/systemd/system/cmk-agent-push.timer <<'UNIT' +[Unit] +Description=CheckMK agent push timer (ansipa-managed) +[Timer] +OnBootSec=30s +OnUnitActiveSec=60s +[Install] +WantedBy=timers.target +UNIT + systemctl daemon-reload + systemctl enable --now cmk-agent-push.timer 2>/dev/null \ + && log "cmk-agent-push.timer enabled (push every 60s)" \ + || warn "could not enable push timer" + fi + fi +fi + +# ── Register the host in CheckMK (folder /ansipa/infra) ───────────────────── +_folder_result=$(curl -sf -o /dev/null -w '%{http_code}' \ + -X POST "${CMK_API}/domain-types/folder_config/collections/all" \ + -H "Authorization: Bearer ${CMK_USER} ${CMK_SECRET}" \ + -H "Content-Type: application/json" -H "Accept: application/json" \ + -d '{"name":"infra","title":"Ansipa Infra (PVE/PBS)","parent":"/ansipa"}' \ + 2>/dev/null || echo "000") +[[ "$_folder_result" =~ ^2 || "$_folder_result" == "422" ]] \ + || warn "Unexpected response creating /ansipa/infra folder: HTTP $_folder_result" + +_HOST_IP=$(hostname -I 2>/dev/null | awk '{print $1}' || echo "") +_ALREADY_REG=false +curl -sf -o /dev/null \ + -H "Authorization: Bearer ${CMK_USER} ${CMK_SECRET}" \ + "${CMK_API}/objects/host_config/${HOST_FQDN}" 2>/dev/null && _ALREADY_REG=true || true + +if [[ "$_ALREADY_REG" == false ]]; then + _reg_body="{\"host_name\":\"${HOST_FQDN}\",\"folder\":\"/ansipa/infra\"" + [[ -n "$_HOST_IP" ]] && _reg_body+=",\"attributes\":{\"ipaddress\":\"${_HOST_IP}\",\"tag_role\":\"${ROLE}\"}" + _reg_body+="}" + _reg_http=$(curl -sf -o /dev/null -w '%{http_code}' \ + -X POST "${CMK_API}/domain-types/host_config/collections/all" \ + -H "Authorization: Bearer ${CMK_USER} ${CMK_SECRET}" \ + -H "Content-Type: application/json" -H "Accept: application/json" \ + -d "$_reg_body" 2>/dev/null || echo "000") + if [[ "$_reg_http" =~ ^2 ]]; then + log "Registered ${HOST_FQDN} in CheckMK (/ansipa/infra)" + else + warn "Host registration failed (HTTP ${_reg_http}) — add it manually in the CheckMK UI" + fi +else + log "${HOST_FQDN} already registered in CheckMK" +fi + +curl -sf -X POST "${CMK_API}/domain-types/service_discovery_run/actions/start/invoke" \ + -H "Authorization: Bearer ${CMK_USER} ${CMK_SECRET}" \ + -H "Content-Type: application/json" -H "Accept: application/json" \ + -d "{\"host_name\":\"${HOST_FQDN}\",\"mode\":\"refresh\"}" >/dev/null 2>&1 || true +curl -sf -X POST "${CMK_API}/domain-types/activation_run/actions/activate-changes/invoke" \ + -H "Authorization: Bearer ${CMK_USER} ${CMK_SECRET}" \ + -H "Content-Type: application/json" -H "Accept: application/json" \ + -d '{"redirect":false,"sites":[],"force_foreign_changes":true}' >/dev/null 2>&1 || true + +log "Done. ${HOST_FQDN} (${ROLE}) should appear under CheckMK folder /ansipa/infra within a minute." diff --git a/setup/modules/FreeipaAnsible/image/ansipa-pbs-monitor.sh b/setup/modules/FreeipaAnsible/image/ansipa-pbs-monitor.sh new file mode 100755 index 0000000..820e1ab --- /dev/null +++ b/setup/modules/FreeipaAnsible/image/ansipa-pbs-monitor.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# ansipa_pbs — CheckMK local check for a Proxmox Backup Server. +# +# Install: copy to /usr/lib/check_mk_agent/local/ansipa_pbs and chmod 755 +# (ansipa-infra-checkmk-install.sh does this for you, along with installing +# the CheckMK agent itself and registering the host). +# +# Requires only `proxmox-backup-manager` (ships with proxmox-backup-server) +# and `df` — no jq/python3 dependency. JSON is requested as "json-pretty" +# (one key per line) and scraped with a small grep/awk parser. Any command +# that fails or returns nothing degrades that one service to CheckMK state 3 +# (UNKNOWN) rather than aborting the others. +# +# Emits (CheckMK local-check format: " "): +# Ansipa_PBS_Datastores worst datastore disk utilisation (via df on path) +# Ansipa_PBS_LastBackup freshness + outcome of the last backup task +# Ansipa_PBS_TaskFailures any failed task (GC/verify/sync/prune/backup) in 24h +# +# CPU/RAM/disk-root are already covered by CheckMK's stock Linux checks — +# not duplicated here. + +command -v proxmox-backup-manager >/dev/null 2>&1 \ + || { echo "3 Ansipa_PBS - proxmox-backup-manager not found (not a PBS host?)"; exit 0; } + +# Extract every value of a JSON key from json-pretty text (one "key": value per line). +_jp() { grep -oP "^\s*\"$1\"\s*:\s*\"?\K[^\",]+" || true; } + +# ── Datastore disk usage (df on each datastore path — version-independent) ─── +DSTAT=$(proxmox-backup-manager datastore list --output-format json-pretty 2>/dev/null || true) +if [[ -z "$DSTAT" ]]; then + echo "3 Ansipa_PBS_Datastores - could not query datastore list" +else + mapfile -t DNAMES < <(_jp name <<< "$DSTAT") + mapfile -t DPATHS < <(_jp path <<< "$DSTAT") + WORST=0; WORST_NAME=""; PERF="" + for i in "${!DNAMES[@]}"; do + p="${DPATHS[$i]:-}" + [[ -d "$p" ]] || continue + pct=$(df -P "$p" 2>/dev/null | awk 'NR==2{gsub("%","",$5); print $5}') + [[ "$pct" =~ ^[0-9]+$ ]] || continue + PERF+=" ${DNAMES[$i]}_pct=${pct}%;85;95" + (( pct > WORST )) && { WORST=$pct; WORST_NAME="${DNAMES[$i]}"; } + done + if [[ -z "$WORST_NAME" ]]; then + echo "3 Ansipa_PBS_Datastores - no readable datastore paths found" + elif (( WORST >= 95 )); then + echo "2 Ansipa_PBS_Datastores${PERF} CRIT — ${WORST_NAME} at ${WORST}%" + elif (( WORST >= 85 )); then + echo "1 Ansipa_PBS_Datastores${PERF} WARN — ${WORST_NAME} at ${WORST}%" + else + echo "0 Ansipa_PBS_Datastores${PERF} worst: ${WORST_NAME} at ${WORST}%" + fi +fi + +# ── Recent task history (shared by the next two checks) ────────────────────── +TSTAT=$(proxmox-backup-manager task list --output-format json-pretty --limit 500 2>/dev/null || true) +if [[ -z "$TSTAT" ]]; then + echo "3 Ansipa_PBS_LastBackup - could not query task list" + echo "3 Ansipa_PBS_TaskFailures - could not query task list" +else + mapfile -t WTYPE < <(_jp worker_type <<< "$TSTAT") + mapfile -t TSTATUS < <(_jp status <<< "$TSTAT") + mapfile -t TSTART < <(_jp starttime <<< "$TSTAT") + + # ── Most recent backup task ────────────────────────────────────────────── + LAST_TS=0; LAST_STATUS="" + for i in "${!WTYPE[@]}"; do + [[ "${WTYPE[$i]}" == "backup" ]] || continue + st="${TSTART[$i]:-0}" + [[ "$st" =~ ^[0-9]+$ ]] || continue + if (( st > LAST_TS )); then LAST_TS=$st; LAST_STATUS="${TSTATUS[$i]:-}"; fi + done + if [[ "$LAST_TS" -eq 0 ]]; then + echo "1 Ansipa_PBS_LastBackup - no backup task found in recent history" + else + AGE_H=$(( ( $(date +%s) - LAST_TS ) / 3600 )) + PERF="last_backup_hours=${AGE_H}" + if [[ "$LAST_STATUS" != "OK" ]]; then + echo "2 Ansipa_PBS_LastBackup ${PERF} Last backup task FAILED (${AGE_H}h ago): ${LAST_STATUS}" + elif (( AGE_H >= 48 )); then + echo "1 Ansipa_PBS_LastBackup ${PERF} Last successful backup ${AGE_H}h ago (stale)" + else + echo "0 Ansipa_PBS_LastBackup ${PERF} Last backup OK, ${AGE_H}h ago" + fi + fi + + # ── Any failed task in the last 24h (GC / verify / sync / prune / backup) ─ + NOW=$(date +%s); FAILED=0; TOTAL24=0 + for i in "${!TSTART[@]}"; do + st="${TSTART[$i]:-0}" + [[ "$st" =~ ^[0-9]+$ ]] || continue + (( NOW - st <= 86400 )) || continue + TOTAL24=$((TOTAL24+1)) + [[ "${TSTATUS[$i]:-}" == "OK" ]] || FAILED=$((FAILED+1)) + done + PERF="failed=${FAILED};1;3 total=${TOTAL24}" + if (( FAILED >= 3 )); then + echo "2 Ansipa_PBS_TaskFailures ${PERF} ${FAILED}/${TOTAL24} tasks failed in the last 24h" + elif (( FAILED >= 1 )); then + echo "1 Ansipa_PBS_TaskFailures ${PERF} ${FAILED}/${TOTAL24} task(s) failed in the last 24h" + else + echo "0 Ansipa_PBS_TaskFailures ${PERF} ${TOTAL24} task(s) in last 24h, none failed" + fi +fi diff --git a/setup/modules/FreeipaAnsible/image/ansipa-pve-monitor.sh b/setup/modules/FreeipaAnsible/image/ansipa-pve-monitor.sh new file mode 100755 index 0000000..75b0076 --- /dev/null +++ b/setup/modules/FreeipaAnsible/image/ansipa-pve-monitor.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# ansipa_pve — CheckMK local check for a Proxmox VE hypervisor node. +# +# Install: copy to /usr/lib/check_mk_agent/local/ansipa_pve and chmod 755 +# (ansipa-infra-checkmk-install.sh does this for you, along with installing +# the CheckMK agent itself and registering the host). +# +# Requires only `pvesh` (ships with pve-manager on every PVE node) — no jq +# or python3 dependency. JSON is requested as "json-pretty" (one key per +# line) and scraped with a small grep/awk parser so the script works on a +# bare PVE install. Any command that fails or returns nothing degrades that +# one service to CheckMK state 3 (UNKNOWN) rather than aborting the others. +# +# Emits (CheckMK local-check format: " "): +# Ansipa_PVE_Cluster quorum / standalone status +# Ansipa_PVE_Storage worst storage utilisation on this node +# Ansipa_PVE_Guests VM/CT counts (running vs stopped) +# Ansipa_PVE_Backup freshness + outcome of the last vzdump job +# +# CPU/RAM/disk-root are already covered by CheckMK's stock Linux checks — +# not duplicated here. + +command -v pvesh >/dev/null 2>&1 \ + || { echo "3 Ansipa_PVE - pvesh not found (not a Proxmox VE host?)"; exit 0; } + +NODE=$(hostname -s 2>/dev/null || echo "") + +# Extract every value of a JSON key from json-pretty text (one "key": value per line). +_jp() { grep -oP "^\s*\"$1\"\s*:\s*\"?\K[^\",]+" || true; } + +# ── Cluster / quorum ──────────────────────────────────────────────────────── +CSTAT=$(pvesh get /cluster/status --output-format json-pretty 2>/dev/null || true) +if [[ -z "$CSTAT" ]]; then + echo "3 Ansipa_PVE_Cluster - could not query /cluster/status" +else + mapfile -t CTYPES < <(_jp type <<< "$CSTAT") + QUORATE=$(_jp quorate <<< "$CSTAT" | head -1) + _clustered=false + for t in "${CTYPES[@]}"; do [[ "$t" == "cluster" ]] && _clustered=true && break; done + if [[ "$_clustered" == false ]]; then + echo "0 Ansipa_PVE_Cluster - standalone node (no cluster configured)" + elif [[ "$QUORATE" == "1" ]]; then + echo "0 Ansipa_PVE_Cluster - cluster quorate" + else + echo "2 Ansipa_PVE_Cluster - cluster NOT quorate" + fi +fi + +# ── Storage usage (worst-case % across all storages on this node) ──────────── +SSTAT="" +[[ -n "$NODE" ]] && SSTAT=$(pvesh get "/nodes/${NODE}/storage" --output-format json-pretty 2>/dev/null || true) +if [[ -z "$SSTAT" ]]; then + echo "3 Ansipa_PVE_Storage - could not query node storage" +else + mapfile -t SNAMES < <(_jp storage <<< "$SSTAT") + mapfile -t STOTAL < <(_jp total <<< "$SSTAT") + mapfile -t SUSED < <(_jp used <<< "$SSTAT") + WORST=0; WORST_NAME=""; PERF="" + for i in "${!SNAMES[@]}"; do + t="${STOTAL[$i]:-0}"; u="${SUSED[$i]:-0}" + [[ "$t" =~ ^[0-9]+$ && "$u" =~ ^[0-9]+$ && "$t" -gt 0 ]] || continue + pct=$(( u * 100 / t )) + PERF+=" ${SNAMES[$i]}_pct=${pct}%;80;90" + (( pct > WORST )) && { WORST=$pct; WORST_NAME="${SNAMES[$i]}"; } + done + if [[ -z "$WORST_NAME" ]]; then + echo "3 Ansipa_PVE_Storage - no usable storage entries found" + elif (( WORST >= 90 )); then + echo "2 Ansipa_PVE_Storage${PERF} CRIT — ${WORST_NAME} at ${WORST}%" + elif (( WORST >= 80 )); then + echo "1 Ansipa_PVE_Storage${PERF} WARN — ${WORST_NAME} at ${WORST}%" + else + echo "0 Ansipa_PVE_Storage${PERF} worst: ${WORST_NAME} at ${WORST}%" + fi +fi + +# ── Guests (VMs + containers) ───────────────────────────────────────────────── +RSTAT=$(pvesh get /cluster/resources --output-format json-pretty 2>/dev/null || true) +if [[ -z "$RSTAT" ]]; then + echo "3 Ansipa_PVE_Guests - could not query /cluster/resources" +else + mapfile -t RTYPES < <(_jp type <<< "$RSTAT") + mapfile -t RSTATUS < <(_jp status <<< "$RSTAT") + VMS=0; CTS=0; RUNNING=0; STOPPED=0 + for i in "${!RTYPES[@]}"; do + case "${RTYPES[$i]}" in + qm) VMS=$((VMS+1)) ;; + lxc) CTS=$((CTS+1)) ;; + *) continue ;; + esac + if [[ "${RSTATUS[$i]:-}" == "running" ]]; then RUNNING=$((RUNNING+1)); else STOPPED=$((STOPPED+1)); fi + done + TOTAL=$((VMS + CTS)) + PERF="vms=${VMS};; cts=${CTS};; running=${RUNNING};; stopped=${STOPPED};;" + echo "0 Ansipa_PVE_Guests ${PERF} ${TOTAL} guest(s): ${RUNNING} running, ${STOPPED} stopped" +fi + +# ── Most recent vzdump backup job (from cluster task history) ─────────────── +TSTAT=$(pvesh get /cluster/tasks --output-format json-pretty 2>/dev/null || true) +if [[ -z "$TSTAT" ]]; then + echo "3 Ansipa_PVE_Backup - could not query /cluster/tasks" +else + mapfile -t TTYPE < <(_jp type <<< "$TSTAT") + mapfile -t TSTATUS < <(_jp status <<< "$TSTAT") + mapfile -t TSTART < <(_jp starttime <<< "$TSTAT") + LAST_TS=0; LAST_STATUS="" + for i in "${!TTYPE[@]}"; do + [[ "${TTYPE[$i]}" == "vzdump" ]] || continue + st="${TSTART[$i]:-0}" + [[ "$st" =~ ^[0-9]+$ ]] || continue + if (( st > LAST_TS )); then LAST_TS=$st; LAST_STATUS="${TSTATUS[$i]:-}"; fi + done + if [[ "$LAST_TS" -eq 0 ]]; then + echo "1 Ansipa_PVE_Backup - no vzdump backup job found in recent task history" + else + AGE_H=$(( ( $(date +%s) - LAST_TS ) / 3600 )) + PERF="last_backup_hours=${AGE_H}" + if [[ "$LAST_STATUS" != "OK" ]]; then + echo "2 Ansipa_PVE_Backup ${PERF} Last vzdump job FAILED (${AGE_H}h ago): ${LAST_STATUS}" + elif (( AGE_H >= 48 )); then + echo "1 Ansipa_PVE_Backup ${PERF} Last successful backup ${AGE_H}h ago (stale)" + else + echo "0 Ansipa_PVE_Backup ${PERF} Last backup OK, ${AGE_H}h ago" + fi + fi +fi