Dotfiles/setup/modules/FreeipaAnsible/ansible/policies.d/dev_mon.sh

397 lines
18 KiB
Bash

#!/usr/bin/env bash
# policy: dev_mon — CheckMK monitoring agent + local checks.
# dev_mon_base install agent, register host, installed-packages check
# dev_mon_malware ClamAV scan result check
# dev_mon_timeshift Timeshift snapshot age check
# dev_mon_power CPU package power via RAPL / lm-sensors
# usr_mon_logins SSH login audit check (user group, but requires dev_mon_base)
# Revert: deregisters the host and removes the agent socket when dev_mon_base leaves.
MON_LOCAL_DIR="/usr/lib/check_mk_agent/local"
MON_STATE="$STATE_DIR/mon-registered"
MON_CHECKS_STATE="$STATE_DIR/mon-checks"
MON_CMK_CREDS="$STATE_DIR/mon-cmk-creds"
[[ -f "$MON_CHECKS_STATE" ]] || touch "$MON_CHECKS_STATE"
_mon_dev_active() { local n="$1"; for _m in "${ACTIVE_MON_DEV[@]+"${ACTIVE_MON_DEV[@]}"}"; do [[ "$_m" == "$n" ]] && return 0; done; return 1; }
_mon_usr_active() { local n="$1"; for _m in "${ACTIVE_MON_USR[@]+"${ACTIVE_MON_USR[@]}"}"; do [[ "$_m" == "$n" ]] && return 0; done; return 1; }
CMK_URL="" CMK_SITE="" CMK_USER="" CMK_SECRET="" CMK_API="" CMK_HOST_PORT=""
_cmk_parse_creds() {
local _desc
_desc=$(ipa hostgroup-show dev_mon_base --all 2>/dev/null \
| awk -F': ' '/Description:/{print $2; exit}' || true)
if [[ "$_desc" =~ ^cmk://([^/]+)(/[^:]+):([^:]+):(.+)$ ]]; then
CMK_HOST_PORT="${BASH_REMATCH[1]}"
CMK_SITE="${BASH_REMATCH[2]#/}"
CMK_USER="${BASH_REMATCH[3]}"
CMK_SECRET="${BASH_REMATCH[4]}"
CMK_URL="http://${CMK_HOST_PORT}"
CMK_API="${CMK_URL}/${CMK_SITE}/check_mk/api/1.0"
fi
}
_WANT_CHECKS=()
_cmk_write_check() {
local _name="$1" _body="$2"
mkdir -p "$MON_LOCAL_DIR"
printf '%s\n' "$_body" > "$MON_LOCAL_DIR/$_name"
chmod 755 "$MON_LOCAL_DIR/$_name"
_WANT_CHECKS+=("$_name")
}
if _mon_dev_active "base"; then
_cmk_parse_creds
if [[ -z "$CMK_SECRET" ]]; then
warn "dev_mon_base: no CheckMK credentials in IPA dev_mon_base description"
warn " — run ansipa-checkmk-setup on the FreeIPA container first"
else
# ── Install CheckMK agent ─────────────────────────────────────────────
_AGENT_INSTALLED=false
if rpm -q check-mk-agent &>/dev/null || command -v check_mk_agent &>/dev/null; then
_AGENT_INSTALLED=true
else
log "dev_mon_base: downloading CheckMK agent from ${CMK_URL}"
_RPM=$(curl -sf -u "${CMK_USER}:${CMK_SECRET}" \
"${CMK_URL}/${CMK_SITE}/check_mk/agents/" 2>/dev/null \
| grep -oE 'check-mk-agent-[0-9][^"]*\.noarch\.rpm' | head -1 || true)
if [[ -n "$_RPM" ]]; then
curl -sf -u "${CMK_USER}:${CMK_SECRET}" \
"${CMK_URL}/${CMK_SITE}/check_mk/agents/${_RPM}" \
-o /tmp/cmk-agent.rpm 2>/dev/null \
&& (rpm -ivh --nodeps /tmp/cmk-agent.rpm 2>/dev/null \
|| dnf install -y /tmp/cmk-agent.rpm 2>/dev/null) \
&& _AGENT_INSTALLED=true \
|| warn "dev_mon_base: CheckMK RPM install failed"
rm -f /tmp/cmk-agent.rpm
fi
if [[ "$_AGENT_INSTALLED" == false ]]; then
curl -sf -u "${CMK_USER}:${CMK_SECRET}" \
"${CMK_URL}/${CMK_SITE}/check_mk/agents/check_mk_agent.linux" \
-o /usr/local/bin/check_mk_agent 2>/dev/null \
&& chmod 755 /usr/local/bin/check_mk_agent \
&& _AGENT_INSTALLED=true \
|| warn "dev_mon_base: shell agent download failed"
fi
fi
if [[ "$_AGENT_INSTALLED" == true ]] && [[ ! -f /etc/systemd/system/check_mk.socket ]]; then
cat > /etc/systemd/system/check_mk.socket <<'UNIT'
[Unit]
Description=Check_MK agent socket (ansipa-managed)
[Socket]
ListenStream=6556
Accept=yes
[Install]
WantedBy=sockets.target
UNIT
cat > /etc/systemd/system/check_mk@.service <<'UNIT'
[Unit]
Description=Check_MK per-connection agent (ansipa-managed)
[Service]
ExecStart=/usr/local/bin/check_mk_agent
StandardInput=socket
StandardOutput=socket
StandardError=null
UNIT
systemctl daemon-reload
fi
if [[ "$_AGENT_INSTALLED" == true ]]; then
systemctl enable --now check_mk.socket 2>/dev/null \
&& log "dev_mon_base: check_mk.socket enabled (port 6556)" \
|| warn "dev_mon_base: could not enable check_mk.socket"
fi
# ── Register host in CheckMK ──────────────────────────────────────────
_HOST_IP=$(hostname -I 2>/dev/null | awk '{print $1}' || echo "")
_CMK_REG_URL="${CMK_API}/objects/host_config/${HOST_FQDN}"
_ALREADY_REG=false
curl -sf -o /dev/null \
-H "Authorization: Bearer ${CMK_USER} ${CMK_SECRET}" \
"$_CMK_REG_URL" 2>/dev/null && _ALREADY_REG=true || true
if [[ "$_ALREADY_REG" == false ]]; then
_reg_body="{\"host_name\":\"${HOST_FQDN}\",\"folder\":\"/ansipa\""
[[ -n "$_HOST_IP" ]] && _reg_body+=",\"attributes\":{\"ipaddress\":\"${_HOST_IP}\"}"
_reg_body+="}"
_reg_http=$(curl -sf -o /dev/null -w '%{http_code}' \
-X POST "${CMK_API}/domain-types/host_config/collections/all" \
-H "Authorization: Bearer ${CMK_USER} ${CMK_SECRET}" \
-H "Content-Type: application/json" \
-H "Accept: application/json" \
-d "$_reg_body" 2>/dev/null || echo "000")
if [[ "$_reg_http" =~ ^2 ]]; then
log "dev_mon_base: host ${HOST_FQDN} registered in CheckMK"
curl -sf \
-X POST "${CMK_API}/domain-types/service_discovery_run/actions/start/invoke" \
-H "Authorization: Bearer ${CMK_USER} ${CMK_SECRET}" \
-H "Content-Type: application/json" \
-H "Accept: application/json" \
-d "{\"host_name\":\"${HOST_FQDN}\",\"mode\":\"refresh\"}" \
>/dev/null 2>&1 || true
curl -sf \
-X POST "${CMK_API}/domain-types/activation_run/actions/activate-changes/invoke" \
-H "Authorization: Bearer ${CMK_USER} ${CMK_SECRET}" \
-H "Content-Type: application/json" \
-H "Accept: application/json" \
-d '{"redirect":false,"sites":[],"force_foreign_changes":true}' \
>/dev/null 2>&1 || true
printf '%s\n' "${CMK_API}" > "$MON_STATE"
else
warn "dev_mon_base: host registration failed (HTTP ${_reg_http})"
fi
else
log "dev_mon_base: ${HOST_FQDN} already registered in CheckMK"
printf '%s\n' "${CMK_API}" > "$MON_STATE"
fi
unset _HOST_IP _CMK_REG_URL _ALREADY_REG _reg_body _reg_http
# ── cmk-agent-ctl: register for TLS/push transport ───────────────────
if command -v cmk-agent-ctl &>/dev/null; then
_CMK_RECV_HOST=$(echo "$CMK_URL" | sed 's|http://||;s|:.*||')
_CMK_CTL_STATUS_FILE="$STATE_DIR/mon-cmk-ctl-mode"
_ALREADY_CTL=$(cmk-agent-ctl status 2>/dev/null | grep -c "${_CMK_RECV_HOST}/${CMK_SITE}" || true)
if [[ "${_ALREADY_CTL:-0}" -eq 0 ]]; then
cmk-agent-ctl register \
--server "${_CMK_RECV_HOST}:8000" \
--site "${CMK_SITE}" \
--user "${CMK_USER}" \
--password "${CMK_SECRET}" \
--hostname "${HOST_FQDN}" \
--trust-cert &>/dev/null \
&& log "dev_mon_base: cmk-agent-ctl registered with ${_CMK_RECV_HOST}:8000" \
|| warn "dev_mon_base: cmk-agent-ctl registration failed (non-fatal)"
fi
_CTL_MODE=$(cmk-agent-ctl status 2>/dev/null | awk '/Connection mode:/{print $NF; exit}')
echo "$_CTL_MODE" > "$_CMK_CTL_STATUS_FILE" 2>/dev/null || true
if [[ "$_CTL_MODE" == "push-agent" ]]; then
log "dev_mon_base: push mode active — setting up push timer"
cmk-agent-ctl push 2>/dev/null || true
if [[ ! -f /etc/systemd/system/cmk-agent-push.timer ]]; then
cat > /etc/systemd/system/cmk-agent-push.service <<'UNIT'
[Unit]
Description=CheckMK agent push (ansipa-managed)
[Service]
Type=oneshot
ExecStart=/usr/bin/cmk-agent-ctl push
UNIT
cat > /etc/systemd/system/cmk-agent-push.timer <<'UNIT'
[Unit]
Description=CheckMK agent push timer (ansipa-managed)
[Timer]
OnBootSec=30s
OnUnitActiveSec=60s
[Install]
WantedBy=timers.target
UNIT
systemctl daemon-reload
systemctl enable --now cmk-agent-push.timer 2>/dev/null \
&& log "dev_mon_base: cmk-agent-push.timer enabled (push every 60s)" \
|| warn "dev_mon_base: could not enable push timer"
fi
fi
unset _CMK_RECV_HOST _ALREADY_CTL _CTL_MODE _CMK_CTL_STATUS_FILE
fi
# ── Persist CMK credentials for push scripts ──────────────────────────
printf 'CMK_API=%q\nCMK_USER=%q\nCMK_SECRET=%q\n' \
"$CMK_API" "$CMK_USER" "$CMK_SECRET" > "$MON_CMK_CREDS"
chmod 600 "$MON_CMK_CREDS"
# ── dev_mon_base: installed-packages local check ──────────────────────
_cmk_write_check "ansipa_packages" '#!/bin/bash
PKG_COUNT=0
RECENT="n/a"
if command -v rpm &>/dev/null; then
PKG_COUNT=$(rpm -qa --qf "%{NAME}\n" 2>/dev/null | wc -l)
RECENT=$(rpm -qa --qf "%{INSTALLTIME} %{NAME}\n" 2>/dev/null | sort -rn | head -5 | awk "{print \$2}" | tr "\n" "," | sed "s/,\$//")
elif command -v dpkg &>/dev/null; then
PKG_COUNT=$(dpkg --list 2>/dev/null | grep -c "^ii")
RECENT=$(grep " install " /var/log/dpkg.log 2>/dev/null | tail -5 | awk "{print \$4}" | tr "\n" "," | sed "s/,\$//")
elif command -v pacman &>/dev/null; then
PKG_COUNT=$(pacman -Qq 2>/dev/null | wc -l)
RECENT=$(grep "\[ALPM\] installed" /var/log/pacman.log 2>/dev/null | tail -5 | awk "{print \$4}" | tr "\n" "," | sed "s/,\$//")
fi
echo "0 Ansipa_Packages packages=${PKG_COUNT} ${PKG_COUNT} packages installed. Recent: ${RECENT}"'
# ── dev_mon_malware: ClamAV scan result check ─────────────────────────
if _mon_dev_active "malware"; then
_cmk_write_check "ansipa_clamav" '#!/bin/bash
LOG=/var/log/ansipa-security-scan.log
CLAMDB=/var/lib/clamav/main.cvd
[[ -f /var/lib/clamav/main.cld ]] && CLAMDB=/var/lib/clamav/main.cld
if [[ ! -f "$LOG" ]]; then
echo "3 Ansipa_ClamAV - No scan log found. Add host to dev_security-scan first."
exit 0
fi
LAST_DATE=$(grep "^=== ansipa-security-scan:" "$LOG" 2>/dev/null | tail -1 \
| grep -oE "[0-9]{4}-[0-9]{2}-[0-9]{2}" || echo "")
INFECTED=$(grep -c "FOUND$" "$LOG" 2>/dev/null) || INFECTED=0
DB_DAYS=999
if [[ -f "$CLAMDB" ]]; then
DB_DAYS=$(( ( $(date +%s) - $(stat -c %Y "$CLAMDB" 2>/dev/null || echo 0) ) / 86400 ))
fi
PERF="infected=${INFECTED};0;0 db_age_days=${DB_DAYS};7;30"
if [[ -z "$LAST_DATE" ]]; then
echo "1 Ansipa_ClamAV ${PERF} No scan found yet"
elif [[ "$INFECTED" -gt 0 ]]; then
echo "2 Ansipa_ClamAV ${PERF} INFECTED: ${INFECTED} threat(s) found — last scan: ${LAST_DATE}"
elif [[ "$DB_DAYS" -gt 7 ]]; then
echo "1 Ansipa_ClamAV ${PERF} DB stale (${DB_DAYS}d). Last scan: ${LAST_DATE}"
else
echo "0 Ansipa_ClamAV ${PERF} Clean — last scan: ${LAST_DATE}, DB age: ${DB_DAYS}d"
fi'
fi
# ── dev_mon_timeshift: snapshot age check ─────────────────────────────
if _mon_dev_active "timeshift"; then
_cmk_write_check "ansipa_timeshift" '#!/bin/bash
if ! command -v timeshift &>/dev/null; then
echo "3 Ansipa_Timeshift - Timeshift not installed on this host"
exit 0
fi
LAST=$(timeshift --list 2>/dev/null | grep -E "^\s*[0-9]+" \
| awk "{print \$3, \$4}" | tail -1 || echo "")
if [[ -z "$LAST" ]]; then
echo "2 Ansipa_Timeshift last_backup_days=9999;5;10 No Timeshift snapshots found"
exit 0
fi
LAST_TS=$(date -d "$LAST" +%s 2>/dev/null || echo 0)
DAYS=$(( ( $(date +%s) - LAST_TS ) / 86400 ))
PERF="last_backup_days=${DAYS};5;10"
if [[ "$DAYS" -ge 10 ]]; then
echo "2 Ansipa_Timeshift ${PERF} Last backup: ${DAYS}d ago (${LAST}) — CRIT"
elif [[ "$DAYS" -ge 5 ]]; then
echo "1 Ansipa_Timeshift ${PERF} Last backup: ${DAYS}d ago (${LAST}) — WARN"
else
echo "0 Ansipa_Timeshift ${PERF} Last backup: ${DAYS}d ago (${LAST})"
fi'
fi
# ── dev_mon_power: CPU package power via RAPL / lm-sensors ──────────
if _mon_dev_active "power"; then
_cmk_write_check "ansipa_power" '#!/bin/bash
RAPL=/sys/class/powercap/intel-rapl/intel-rapl:0/energy_uj
POWER_W=0; SOURCE=""
if [[ -r "$RAPL" ]]; then
E1=$(cat "$RAPL" 2>/dev/null || echo 0)
sleep 1
E2=$(cat "$RAPL" 2>/dev/null || echo 0)
if [[ "$E2" -gt "$E1" ]]; then
POWER_W=$(( (E2 - E1) / 1000000 ))
SOURCE="RAPL"
fi
fi
if [[ -z "$SOURCE" ]] && command -v sensors &>/dev/null; then
W=$(sensors 2>/dev/null | grep -iE "package.*power|cpu.*power" \
| grep -oP "[0-9]+\.[0-9]+" | head -1 | cut -d. -f1 || echo 0)
[[ "${W:-0}" -gt 0 ]] && POWER_W="$W" && SOURCE="lm-sensors"
fi
if [[ -z "$SOURCE" ]]; then
echo "3 Ansipa_Power power_w=0 Power monitoring unavailable (needs Intel RAPL or lm-sensors)"
exit 0
fi
PERF="power_w=${POWER_W};65;95"
if [[ "$POWER_W" -ge 95 ]]; then
echo "2 Ansipa_Power ${PERF} CPU power: ${POWER_W}W via ${SOURCE} — CRIT"
elif [[ "$POWER_W" -ge 65 ]]; then
echo "1 Ansipa_Power ${PERF} CPU power: ${POWER_W}W via ${SOURCE} — WARN"
else
echo "0 Ansipa_Power ${PERF} CPU power: ${POWER_W}W via ${SOURCE}"
fi'
fi
# ── usr_mon_logins: SSH login audit (requires dev_mon_base agent) ─────
if _mon_usr_active "logins"; then
_cmk_write_check "ansipa_logins" '#!/bin/bash
count_journal() {
local pat="$1"
journalctl -u sshd --since "24 hours ago" --no-pager -q 2>/dev/null \
| grep -c "$pat" 2>/dev/null || true
}
count_secure() {
local pat="$1"
{ grep "$(date "+%b %e")" /var/log/secure 2>/dev/null
grep "$(date -d yesterday "+%b %e" 2>/dev/null || true)" /var/log/secure 2>/dev/null; } \
| grep -c "$pat" 2>/dev/null || true
}
if command -v journalctl &>/dev/null && journalctl -u sshd --since "24 hours ago" -q &>/dev/null; then
ACCEPTED=$(count_journal "Accepted ")
FAILED=$(count_journal "Failed ")
INVALID=$(count_journal "Invalid user ")
else
ACCEPTED=$(count_secure "sshd.*Accepted")
FAILED=$(count_secure "sshd.*Failed")
INVALID=$(count_secure "sshd.*Invalid user")
fi
PERF="successful=${ACCEPTED};; failed=${FAILED};10;50 invalid=${INVALID};5;20"
MSG="Logins 24h: ${ACCEPTED} ok / ${FAILED} failed / ${INVALID} invalid user"
if [[ "$FAILED" -ge 50 ]] || [[ "$INVALID" -ge 20 ]]; then
echo "2 Ansipa_Logins ${PERF} HIGH ACTIVITY — ${MSG}"
elif [[ "$FAILED" -ge 10 ]] || [[ "$INVALID" -ge 5 ]]; then
echo "1 Ansipa_Logins ${PERF} ELEVATED — ${MSG}"
else
echo "0 Ansipa_Logins ${PERF} ${MSG}"
fi'
fi
fi # CMK_SECRET non-empty
else
# ── Revert: host left dev_mon_base → deregister and clean up ─────────────
if [[ -f "$MON_STATE" ]]; then
_OLD_CMK_API=$(cat "$MON_STATE" 2>/dev/null || true)
if [[ -n "$_OLD_CMK_API" ]] && [[ -f "$MON_CMK_CREDS" ]]; then
# shellcheck source=/dev/null
source "$MON_CMK_CREDS" 2>/dev/null || true
_del_http=$(curl -sf -o /dev/null -w '%{http_code}' \
-X DELETE "${_OLD_CMK_API}/objects/host_config/${HOST_FQDN}" \
-H "Authorization: Bearer ${CMK_USER:-} ${CMK_SECRET:-}" \
-H "Accept: application/json" 2>/dev/null || echo "000")
if [[ "$_del_http" =~ ^(2|404) ]]; then
log "Deregistered ${HOST_FQDN} from CheckMK (HTTP ${_del_http})"
curl -sf \
-X POST "${_OLD_CMK_API}/domain-types/activation_run/actions/activate-changes/invoke" \
-H "Authorization: Bearer ${CMK_USER:-} ${CMK_SECRET:-}" \
-H "Content-Type: application/json" \
-H "Accept: application/json" \
-d '{"redirect":false,"sites":[],"force_foreign_changes":true}' \
>/dev/null 2>&1 || true
else
warn "CheckMK deregistration returned HTTP ${_del_http} — may need manual cleanup"
fi
fi
rm -f "$MON_STATE" "$MON_CMK_CREDS"
systemctl disable --now check_mk.socket 2>/dev/null || true
systemctl disable --now cmk-agent-push.timer 2>/dev/null || true
command -v cmk-agent-ctl &>/dev/null && cmk-agent-ctl delete-all 2>/dev/null || true
log "dev_mon_base reverted: check_mk.socket disabled, host deregistered"
unset _OLD_CMK_API _del_http
fi
fi
# ── Cleanup: remove local check scripts for groups no longer active ───────────
while IFS= read -r _old_check; do
[[ -z "$_old_check" ]] && continue
_still_wanted=false
for _wc in "${_WANT_CHECKS[@]+"${_WANT_CHECKS[@]}"}"; do
[[ "$_wc" == "$_old_check" ]] && _still_wanted=true && break
done
if [[ "$_still_wanted" == false ]] && [[ -f "$MON_LOCAL_DIR/$_old_check" ]]; then
rm -f "$MON_LOCAL_DIR/$_old_check"
log "Removed CheckMK local check: $_old_check (group left)"
fi
done < "$MON_CHECKS_STATE"
unset _old_check _still_wanted _wc
if [[ ${#_WANT_CHECKS[@]} -gt 0 ]]; then
printf '%s\n' "${_WANT_CHECKS[@]+"${_WANT_CHECKS[@]}"}" | sort -u > "$MON_CHECKS_STATE"
else
> "$MON_CHECKS_STATE"
fi
unset _WANT_CHECKS _AGENT_INSTALLED