#!/usr/bin/env bash # # Spaceport Monitoring Bot # # Collects host metrics + Docker container health and pushes them as an # HMAC-signed JSON POST to the Spaceport monitoring API. # # Commands: # spaceport-bot run collect + send one report (default, used by the timer) # spaceport-bot update download the newest bot from the dashboard # spaceport-bot version print the installed version # spaceport-bot status show config + last run # # Config: /etc/spaceport-bot/config # set -uo pipefail BOT_VERSION="2.13.0" REPORT_VERSION=2 CONFIG_FILE="${CONFIG_FILE:-/etc/spaceport-bot/config}" BOT_PATH="${BOT_PATH:-/usr/local/sbin/spaceport-bot}" UNIT_DIR="${UNIT_DIR:-/etc/systemd/system}" AUTO_UPDATE_CONF="${AUTO_UPDATE_CONF:-/etc/spaceport-bot/auto-update.conf}" # Unter /var/lib statt /etc: die systemd-Sandbox (ProtectSystem=full) macht /etc # schreibgeschützt, /var bleibt schreibbar. COOLIFY_CIDRS_FILE="${COOLIFY_CIDRS_FILE:-/var/lib/spaceport-bot/coolify-admin-cidrs}" COOLIFY_FIREWALL_BIN="${COOLIFY_FIREWALL_BIN:-/usr/local/sbin/spaceport-coolify-firewall}" TIMEOUT_SECONDS=25 log() { printf '[spaceport-bot] %s\n' "$*"; } fail() { printf '[spaceport-bot] %s\n' "$*" >&2; exit 1; } # Antwort des Dashboards. Eigene Datei per mktemp statt eines festen Namens in # /tmp: Den könnte ein lokaler Benutzer vorher anlegen und so die Antwort — # und damit Firewall-Whitelist und Reboot-Fenster — selbst bestimmen. RESP_FILE="" HDR_FILE="" cleanup_tmp() { rm -f "$RESP_FILE" "$HDR_FILE"; } trap cleanup_tmp EXIT # hmac_sha256 KEY (Daten auf stdin) -> Hex-Digest # # HMAC nach RFC 2104, aus zwei SHA-256-Läufen zusammengesetzt. Grund: Bei # `openssl dgst -hmac KEY` stünde das Secret in der Kommandozeile, und die ist # über /proc//cmdline für jeden lokalen Benutzer lesbar. Hier läuft der # Schlüssel nur durch Shell-Builtins und Pipes. hmac_sha256() { local key_hex ipad="" opad="" i byte key_hex="$(printf '%s' "$1" | od -An -v -tx1 | tr -d ' \n')" if (( ${#key_hex} > 128 )); then key_hex="$(printf '%s' "$1" | openssl dgst -sha256 -binary | od -An -v -tx1 | tr -d ' \n')" fi while (( ${#key_hex} < 128 )); do key_hex+="00"; done for (( i = 0; i < 128; i += 2 )); do byte=$(( 16#${key_hex:i:2} )) ipad+="$(printf '\\x%02x' $(( byte ^ 0x36 )))" opad+="$(printf '\\x%02x' $(( byte ^ 0x5c )))" done # shellcheck disable=SC2059 { printf "$opad"; { printf "$ipad"; cat; } | openssl dgst -sha256 -binary; } \ | openssl dgst -sha256 | awk '{print $NF}' } # Öffentliche Schlüssel der Release-Signatur (ECDSA P-256, PEM). Ein Update — # Script wie systemd-Unit — wird nur installiert, wenn es mit einem dieser # Schlüssel signiert ist. Der private Schlüssel liegt NICHT beim Dashboard: # Wer das Dashboard oder seine Datenbank übernimmt, kann damit keine Updates # auf die Server bringen. Mehrere Blöcke erlauben einen Schlüsselwechsel. # Signieren: npx tsx scripts/bot-signing.ts sign SIGNING_PUBKEYS='-----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEz1G2ccK4hBKi4SrLua3P/N4kpjwR FORgJWkU5IIugPCPxPcgfiuKdJMYrHCKMhJ+rlIru5Z2fTWPMQOX7HlQ8w== -----END PUBLIC KEY-----' # release_signature_ok DATEI HEADERDATEI -> 0, wenn die Release-Signatur aus # dem Header zu einem der Schlüssel oben passt. Ohne Schlüssel: nie. release_signature_ok() { local file="$1" hdr="$2" sig_b64 dir key ok=1 sig_b64="$(awk 'tolower($0) ~ /^x-spaceport-release-signature:/ {gsub(/\r/, ""); print $2}' "$hdr" | tail -n 1)" [[ -n "$sig_b64" ]] || return 1 dir="$(mktemp -d)" if ! printf '%s' "$sig_b64" | openssl base64 -d -A > "$dir/sig" 2>/dev/null || [[ ! -s "$dir/sig" ]]; then rm -rf "$dir" return 1 fi printf '%s\n' "$SIGNING_PUBKEYS" \ | awk -v d="$dir" '/-----BEGIN PUBLIC KEY-----/ { n++ } n > 0 { print > (d "/key" n ".pem") }' for key in "$dir"/key*.pem; do [[ -s "$key" ]] || continue if openssl dgst -sha256 -verify "$key" -signature "$dir/sig" "$file" >/dev/null 2>&1; then ok=0 break fi done rm -rf "$dir" return "$ok" } # Header-Datei mit dem Bot-Token für curl (-H @datei). Als Argument stünde das # Token in der Prozessliste. write_auth_headers() { local file="$1"; shift ( umask 077; printf 'X-Spaceport-Token: %s\n' "$BOT_TOKEN" > "$file" ) local line for line in "$@"; do printf '%s\n' "$line" >> "$file"; done } load_config() { [[ -r "$CONFIG_FILE" ]] || fail "config not readable: $CONFIG_FILE" # shellcheck source=/dev/null . "$CONFIG_FILE" : "${API_URL:?API_URL missing in $CONFIG_FILE}" # Self-update is on unless the config turns it off. SELF_UPDATE="${SELF_UPDATE:-yes}" : "${BOT_TOKEN:?BOT_TOKEN missing in $CONFIG_FILE}" : "${BOT_SECRET:?BOT_SECRET missing in $CONFIG_FILE}" API_URL="${API_URL%/}" # Transportverschluesselung erzwingen. Der Bericht enthaelt Hostnamen, IPs, # offene Ports und die letzten Anmeldungen — nichts davon gehoert # unverschluesselt ins Netz. Eine Fehlkonfiguration darf das nicht still # aushebeln, deshalb Abbruch statt Warnung. Ausgenommen ist nur der # Entwicklungsfall auf dem eigenen Rechner. case "$API_URL" in https://*) ;; http://localhost*|http://127.0.0.1*|http://\[::1\]*) ;; *) fail "API_URL muss https sein (ist: $API_URL)" ;; esac } require_tools() { for t in jq curl openssl awk; do command -v "$t" >/dev/null 2>&1 || fail "missing tool: $t (apt-get install -y $t)" done } trim() { sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//'; } # coolify_port_blocked PORT -> true when a DROP/REJECT rule in DOCKER-USER (or # our own chain hooked into it) covers PORT. Docker-published ports bypass UFW, # so this is the only place where blocking them actually works. coolify_port_blocked() { local rules rules="$(iptables -w 5 -S DOCKER-USER 2>/dev/null)" if iptables -w 5 -C DOCKER-USER -j SPACEPORT-COOLIFY 2>/dev/null; then rules+=$'\n'"$(iptables -w 5 -S SPACEPORT-COOLIFY 2>/dev/null)" fi printf '%s\n' "$rules" | grep -E -- '-j (DROP|REJECT)' | awk -v p="$1" ' { for (i = 1; i < NF; i++) if ($i == "--ctorigdstport" || $i == "--dport" || $i == "--dports") { n = split($(i + 1), parts, ",") for (j = 1; j <= n; j++) { split(parts[j], r, ":"); lo = r[1] + 0; hi = (r[2] == "" ? lo : r[2] + 0) if (p + 0 >= lo && p + 0 <= hi) found = 1 } } } END { exit found ? 0 : 1 }' } # version_gt A B -> true when A is strictly newer than B # Ressourcen eines laufenden Containers als eine JSON-Zeile — nur Dateien aus # cgroup v2 und /proc gelesen, kein `docker stats` (das hält dockerd pro Aufruf # sekundenlang beschäftigt). Alles kumulierte Zähler; die Raten rechnet das # Dashboard aus der Differenz zweier Reports. Ohne cgroup (v1, rootless): nichts. container_stats() { local id="$1" pid="$2" dir="" d f for d in "/sys/fs/cgroup/system.slice/docker-$id.scope" "/sys/fs/cgroup/docker/$id"; do [[ -r "$d/cpu.stat" ]] && { dir="$d"; break; } done [[ -n "$dir" ]] || return 0 # Netz nur mit eigenem Namespace — bei network_mode host wären es die # Zähler des ganzen Hosts. local net="null null" if [[ "$pid" -gt 0 && -r "/proc/$pid/net/dev" ]] \ && [[ "$(readlink "/proc/$pid/ns/net" 2>/dev/null)" != "$(readlink /proc/1/ns/net 2>/dev/null)" ]]; then net="$(awk 'NR > 2 { sub(/^ +/, ""); split($0, f, /[: ]+/); if (f[1] != "lo") { rx += f[2]; tx += f[10] } } END { printf "%.0f %.0f", rx, tx }' "/proc/$pid/net/dev" 2>/dev/null)" fi # Fehlt eine Datei (Controller nicht aktiv), bricht mawk sonst ganz ab. local files=() for f in cpu.stat memory.current memory.max memory.stat io.stat pids.current; do [[ -r "$dir/$f" ]] && files+=("$dir/$f") done # %.0f statt %d: mawk kappt %d bei 2^31, Byte-Zähler sind größer. awk -v id="$id" -v net="$net" ' FILENAME ~ /cpu\.stat$/ && $1 == "usage_usec" { cpu = $2 } FILENAME ~ /memory\.current$/ { mem = $1 } FILENAME ~ /memory\.max$/ { limit = $1 } FILENAME ~ /memory\.stat$/ && $1 == "inactive_file" { inactive = $2 } FILENAME ~ /io\.stat$/ { for (i = 2; i <= NF; i++) { split($i, kv, "=") if (kv[1] == "rbytes") rd += kv[2]; else if (kv[1] == "wbytes") wr += kv[2] } } FILENAME ~ /pids\.current$/ { pids = $1 } END { split(net, n, " ") # Wie docker stats: Page-Cache, den der Kernel jederzeit freigeben kann, zählt nicht. used = mem - inactive; if (used < 0) used = mem printf "{\"id\":\"%s\",\"cpu_usec\":%.0f,\"mem_bytes\":%.0f,\"mem_limit_bytes\":%s,", id, cpu, used, (limit ~ /^[0-9]+$/ ? limit : "null") printf "\"net_rx_bytes\":%s,\"net_tx_bytes\":%s,\"blk_read_bytes\":%.0f,\"blk_write_bytes\":%.0f,\"pids\":%.0f}\n", n[1], n[2], rd, wr, pids }' "${files[@]}" 2>/dev/null } version_gt() { [[ "$1" != "$2" ]] && [[ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -n 1)" == "$1" ]] } # --------------------------------------------------------------------------- # collect # --------------------------------------------------------------------------- collect() { HOSTNAME_FQDN="$(hostname -f 2>/dev/null || hostname)" KERNEL="$(uname -r)" ARCH="$(uname -m)" OS_PRETTY="unknown" if [[ -r /etc/os-release ]]; then # shellcheck disable=SC1091 OS_PRETTY="$(. /etc/os-release; echo "${PRETTY_NAME:-unknown}")" fi UPTIME_SECONDS="$(awk '{print int($1)}' /proc/uptime)" BOOT_TIME="$(date -u -d "$(uptime -s) UTC" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")" PRIMARY_IPV4="$(ip -4 -o route get 1.1.1.1 2>/dev/null | awk '{for(i=1;i<=NF;i++) if ($i=="src") {print $(i+1); exit}}')" [[ -z "$PRIMARY_IPV4" ]] && PRIMARY_IPV4="$(hostname -I 2>/dev/null | awk '{print $1}')" PRIMARY_IPV4="${PRIMARY_IPV4:-}" CPU_CORES="$(nproc 2>/dev/null || echo 1)" read -r LOAD_1 LOAD_5 LOAD_15 _ < /proc/loadavg # Instantaneous CPU usage over a 1s sample of /proc/stat. read_cpu_stat() { awk '/^cpu /{idle=$5+$6; total=0; for(i=2;i<=NF;i++) total+=$i; print total, idle}' /proc/stat; } read -r CPU_T1 CPU_I1 < <(read_cpu_stat) sleep 1 read -r CPU_T2 CPU_I2 < <(read_cpu_stat) CPU_USAGE="$(awk -v t1="$CPU_T1" -v i1="$CPU_I1" -v t2="$CPU_T2" -v i2="$CPU_I2" \ 'BEGIN { dt = t2 - t1; di = i2 - i1; if (dt <= 0) { print 0 } else { printf "%.1f", (1 - di/dt) * 100 } }')" read -r MEM_TOTAL MEM_USED MEM_FREE MEM_AVAIL < <(free -m | awk '/^Mem:/{print $2,$3,$4,$7}') MEM_TOTAL="${MEM_TOTAL:-0}"; MEM_USED="${MEM_USED:-0}"; MEM_FREE="${MEM_FREE:-0}"; MEM_AVAIL="${MEM_AVAIL:-0}" if [[ "$MEM_TOTAL" -gt 0 ]]; then MEM_PCT=$(( MEM_USED * 100 / MEM_TOTAL )); else MEM_PCT=0; fi read -r SWAP_TOTAL SWAP_USED < <(free -m | awk '/^Swap:/{print $2,$3}') SWAP_TOTAL="${SWAP_TOTAL:-0}"; SWAP_USED="${SWAP_USED:-0}" if [[ "$SWAP_TOTAL" -gt 0 ]]; then SWAP_PCT=$(( SWAP_USED * 100 / SWAP_TOTAL )); else SWAP_PCT=0; fi read -r DISK_TOTAL DISK_USED DISK_AVAIL DISK_PCT < <(df -h / | awk 'NR==2{gsub("%","",$5); print $2,$3,$4,$5}') DISK_TOTAL="${DISK_TOTAL:-?}"; DISK_USED="${DISK_USED:-?}"; DISK_AVAIL="${DISK_AVAIL:-?}"; DISK_PCT="${DISK_PCT:-0}" # Every real (non-tmpfs/overlay) mount, so a full /var or /data is visible too. # Werte als TSV, das JSON baut jq — ein Mountpoint mit Anführungszeichen # oder Backslash darf die Liste nicht kaputt machen. DISKS_JSON="$(df -h -x tmpfs -x devtmpfs -x overlay -x squashfs --output=target,size,used,pcent 2>/dev/null \ | awk 'NR>1 {gsub("%","",$4); printf "%s\t%s\t%s\t%s\n", $1, $2, $3, ($4==""?0:$4)}' \ | jq -R 'split("\t") | {mount: .[0], total: .[1], used: .[2], percent: ((.[3] | tonumber?) // 0)}' \ | jq -s . 2>/dev/null)" DISKS_JSON="${DISKS_JSON:-[]}" # ---------- packages ---------- INSTALLED_COUNT="$(dpkg-query -f '.\n' -W 2>/dev/null | wc -l | trim)" INSTALLED_COUNT="${INSTALLED_COUNT:-0}" PENDING_LIST_FILE="$(mktemp)" PENDING_SECURITY_COUNT=0 if command -v apt-get >/dev/null 2>&1; then apt-get -s upgrade 2>/dev/null | awk '/^Inst /{print $2}' > "$PENDING_LIST_FILE" PENDING_SECURITY_COUNT="$(apt-get -s upgrade 2>/dev/null | grep -cE '^Inst.*-security' || true)" fi PENDING_COUNT="$(wc -l < "$PENDING_LIST_FILE" | trim)" PENDING_LIST_JSON="$(jq -R . < "$PENDING_LIST_FILE" | jq -s .)" rm -f "$PENDING_LIST_FILE" PENDING_COUNT="${PENDING_COUNT:-0}" PENDING_SECURITY_COUNT="${PENDING_SECURITY_COUNT:-0}" LAST_APT_UPDATE="" [[ -f /var/cache/apt/pkgcache.bin ]] && \ LAST_APT_UPDATE="$(date -u -d "@$(stat -c %Y /var/cache/apt/pkgcache.bin)" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")" LAST_APT_UPGRADE="" if [[ -r /var/log/apt/history.log ]]; then LAST_UPGRADE_RAW="$(awk 'BEGIN{RS="";FS="\n"} /Commandline:.*(upgrade|dist-upgrade)/ {block=$0} END { for (i=1; i<=split(block, lines, "\n"); i++) if (lines[i] ~ /^End-Date:/) print lines[i] }' \ /var/log/apt/history.log 2>/dev/null | sed 's/^End-Date:[[:space:]]*//')" [[ -n "$LAST_UPGRADE_RAW" ]] && \ LAST_APT_UPGRADE="$(date -u -d "$LAST_UPGRADE_RAW" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")" fi REBOOT_REQUIRED=false REBOOT_PKGS_JSON='[]' if [[ -f /var/run/reboot-required ]]; then REBOOT_REQUIRED=true [[ -r /var/run/reboot-required.pkgs ]] && REBOOT_PKGS_JSON="$(jq -R . < /var/run/reboot-required.pkgs | jq -s .)" fi # Three different things, deliberately reported separately — conflating them # hides which kind of updating a server actually does: # unattended-upgrades daily security patches (reported under security) # spaceport-auto-update daily full apt upgrade via our own timer # auto reboot whether that timer may reboot, and when AUTO_UPDATE_ENABLED=false systemctl is-enabled --quiet spaceport-auto-update.timer 2>/dev/null && AUTO_UPDATE_ENABLED=true AUTO_UPDATE_LAST_RUN="" if [[ "$AUTO_UPDATE_ENABLED" == "true" ]]; then AUTO_UPDATE_RAW="$(systemctl show spaceport-auto-update.service -p ExecMainStartTimestamp --value 2>/dev/null)" if [[ -n "$AUTO_UPDATE_RAW" ]]; then AUTO_UPDATE_LAST_RUN="$(date -u -d "$AUTO_UPDATE_RAW" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")" fi fi AUTO_REBOOT="no"; AUTO_REBOOT_DAY=""; AUTO_REBOOT_HOUR="" if [[ -r /etc/spaceport-bot/auto-update.conf ]]; then # shellcheck disable=SC1091 AUTO_REBOOT="$(. /etc/spaceport-bot/auto-update.conf 2>/dev/null; echo "${AUTO_REBOOT:-no}")" AUTO_REBOOT_DAY="$(. /etc/spaceport-bot/auto-update.conf 2>/dev/null; echo "${REBOOT_DAY:-}")" AUTO_REBOOT_HOUR="$(. /etc/spaceport-bot/auto-update.conf 2>/dev/null; echo "${REBOOT_HOUR:-}")" fi # ---------- auth ---------- LOGINS_RECENT_JSON='[]' if command -v last >/dev/null 2>&1; then # --time-format iso liefert den Zeitpunkt fertig — früher baute awk # daraus einen `date`-Aufruf für die Shell, mit Feldern aus wtmp. LOGINS_RECENT_JSON="$(last -i -n 50 --time-format iso 2>/dev/null | awk ' /^reboot/ || /^wtmp/ || NF < 4 { next } $4 !~ /^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]+([+-][0-9:]+|Z)?$/ { next } { valid++; if (valid > 25) exit printf "%s\t%s\t%s\n", $1, $3, $4 }' \ | jq -R 'split("\t") | {user: .[0], ip: .[1], ts: .[2]}' | jq -s . 2>/dev/null)" LOGINS_RECENT_JSON="${LOGINS_RECENT_JSON:-[]}" fi FAILED_LOGINS_24H=0 if command -v journalctl >/dev/null 2>&1; then FAILED_LOGINS_24H="$(journalctl -u ssh --since "24 hours ago" --no-pager 2>/dev/null \ | grep -cE "(Failed password|Invalid user|authentication failure)" || true)" FAILED_LOGINS_24H="${FAILED_LOGINS_24H:-0}" fi ACTIVE_SESSIONS_JSON='[]' if command -v who >/dev/null 2>&1; then ACTIVE_SESSIONS_JSON="$(who 2>/dev/null | awk ' { ip="-"; for (i=NF;i>=1;i--) if ($i ~ /^\([0-9.:a-fA-F]+\)$/) { ip=$i; gsub(/[()]/,"",ip); break } printf "%s\t%s\t%s\n", $1, $2, ip }' \ | jq -R 'split("\t") | {user: .[0], tty: .[1], ip: .[2]}' | jq -s . 2>/dev/null)" ACTIVE_SESSIONS_JSON="${ACTIVE_SESSIONS_JSON:-[]}" fi # ---------- services ---------- ACTIVE_UNIT_COUNT=0; FAILED_UNIT_COUNT=0; FAILED_UNITS_JSON='[]' UNIT_RESTARTS_JSON='[]'; STOPPED_UNITS_JSON='[]' if command -v systemctl >/dev/null 2>&1; then ACTIVE_UNIT_COUNT="$(systemctl list-units --state=active --no-legend --type=service 2>/dev/null | wc -l | trim)" FAILED_UNITS_RAW="$(systemctl list-units --state=failed --no-legend --type=service --plain 2>/dev/null | awk '{print $1}')" if [[ -n "$FAILED_UNITS_RAW" ]]; then FAILED_UNITS_JSON="$(printf '%s\n' "$FAILED_UNITS_RAW" | jq -R . | jq -s .)" FAILED_UNIT_COUNT="$(printf '%s\n' "$FAILED_UNITS_RAW" | wc -l | trim)" fi # Neustartzähler je Unit. Ein Dienst in der Neustartschleife erreicht # nie den Zustand "failed" — er steht dauerhaft auf "activating" und # wäre über failed_units unsichtbar. Der Zähler verrät ihn. # # Blockweise auswerten: systemctl show gibt die Eigenschaften NICHT in # der angeforderten Reihenfolge aus, zeilenweises Paaren ordnet die # Zahlen der falschen Unit zu. UNIT_RESTARTS_JSON="$(systemctl show '*.service' --property=Id --property=NRestarts 2>/dev/null \ | awk 'BEGIN { RS = "" } { id = ""; n = 0 for (i = 1; i <= NF; i++) { if ($i ~ /^Id=/) id = substr($i, 4) if ($i ~ /^NRestarts=/) n = substr($i, 11) + 0 } if (id != "" && n > 0) printf "%s\t%d\n", id, n }' \ | jq -R 'split("\t") | {unit: .[0], restarts: (.[1] | tonumber)}' \ | jq -s 'sort_by(-.restarts) | .[0:50]' 2>/dev/null)" [[ -z "$UNIT_RESTARTS_JSON" ]] && UNIT_RESTARTS_JSON='[]' # Aktivierte Dienste, die gerade nicht laufen. Ein gestoppter Dienst ist # "inactive", nicht "failed" — über failed_units also unsichtbar. Genau # so verschwand einmal ein sshd, ohne dass irgendwo etwas rot wurde. # # Ausgenommen: oneshot und idle (laufen einmal und enden planmäßig) # sowie Units, deren Startbedingung nicht zutraf. Ohne das stünde auf # jedem Ubuntu dauerhaft dmesg.service in der Liste. ENABLED_UNITS="$(systemctl list-unit-files --type=service --state=enabled --no-legend --plain 2>/dev/null | awk '{print $1}')" if [[ -n "$ENABLED_UNITS" ]]; then # shellcheck disable=SC2086 STOPPED_UNITS_JSON="$(systemctl show $ENABLED_UNITS \ --property=Id --property=ActiveState --property=SubState \ --property=Type --property=ConditionResult \ --property=Result --property=InactiveEnterTimestampMonotonic 2>/dev/null \ | awk 'BEGIN { RS = ""; BOOT_TASK_US = 300000000 } { # "sub" waere hier die eingebaute awk-Funktion — der Name ist belegt. id = ""; state = ""; substate = ""; type = ""; cond = "" result = ""; ended = 0 for (i = 1; i <= NF; i++) { if ($i ~ /^Id=/) id = substr($i, 4) if ($i ~ /^ActiveState=/) state = substr($i, 13) if ($i ~ /^SubState=/) substate = substr($i, 10) if ($i ~ /^Type=/) type = substr($i, 6) if ($i ~ /^ConditionResult=/) cond = substr($i, 17) if ($i ~ /^Result=/) result = substr($i, 8) # "InactiveEnterTimestampMonotonic=" ist 32 Zeichen lang, # der Wert beginnt also bei 33. if ($i ~ /^InactiveEnterTimestampMonotonic=/) ended = substr($i, 33) + 0 } # Startaufgaben aussortieren, die planmaessig enden, sich aber # nicht als oneshot deklarieren: cloud-init-main ist "notify" # und endet trotzdem nach 15 Sekunden. Der Zeitstempel zaehlt # ab Boot, ist also gegen Uhrzeitspruenge immun. Wer in den # ersten fuenf Minuten sauber endet, hat seine Arbeit getan. boot_task = (ended > 0 && ended < BOOT_TASK_US && result == "success") # oneshot laeuft einmal und endet planmaessig; idle ist fuer # Boot-Ausgabe gedacht (dmesg.service) und steht danach # dauerhaft auf inactive — beides ist kein Ausfall. # Genau "inactive": "failed" meldet bereits failed_units, # "activating" ist die Neustartschleife. Sonst gaebe es # denselben Vorfall zweimal. if (id != "" && state == "inactive" && type != "oneshot" && type != "idle" \ && cond != "no" && !boot_task) printf "%s\t%s\t%s\n", id, state, substate }' \ | jq -R 'split("\t") | {unit: .[0], state: .[1], sub: .[2]}' \ | jq -s '.[0:30]' 2>/dev/null)" fi [[ -z "$STOPPED_UNITS_JSON" ]] && STOPPED_UNITS_JSON='[]' fi ACTIVE_UNIT_COUNT="${ACTIVE_UNIT_COUNT:-0}" # ---------- network ---------- LISTEN_PORTS_JSON='[]' if command -v ss >/dev/null 2>&1; then LISTEN_PORTS_JSON="$(ss -tlnpH 2>/dev/null | awk ' { n = split($4, a, ":"); port = a[n]; process = "?" if (match($0, /users:\(\("[^"]+"/)) process = substr($0, RSTART+8, RLENGTH-9) if (port == "" || port !~ /^[0-9]+$/) next if (!(port in seen)) { seen[port]=1; printf "%s\t%s\n", port, process } }' \ | jq -R 'split("\t") | {port: (.[0] | tonumber), proto: "tcp", process: .[1]}' \ | jq -s . 2>/dev/null)" LISTEN_PORTS_JSON="${LISTEN_PORTS_JSON:-[]}" fi # ---------- security ---------- svc_active() { systemctl is-active --quiet "$1" 2>/dev/null && echo true || echo false; } svc_enabled() { systemctl is-enabled --quiet "$1" 2>/dev/null && echo true || echo false; } UFW_ACTIVE=false if command -v ufw >/dev/null 2>&1 && ufw status 2>/dev/null | head -1 | grep -q "Status: active"; then UFW_ACTIVE=true; fi FAIL2BAN_ACTIVE="$(svc_active fail2ban)" CROWDSEC_ACTIVE="$(svc_active crowdsec)" UNATT_ACTIVE="$(svc_enabled unattended-upgrades)" CROWDSEC_DECISIONS=0 if [[ "$CROWDSEC_ACTIVE" == "true" ]] && command -v cscli >/dev/null 2>&1; then CROWDSEC_DECISIONS="$(cscli decisions list -o json 2>/dev/null | jq 'length' 2>/dev/null || echo 0)" CROWDSEC_DECISIONS="${CROWDSEC_DECISIONS:-0}" fi # Can this process replace its own binary? Inside the systemd sandbox # ProtectSystem=full mounts /usr read-only, so an outdated unit silently # blocks every self-update. Reporting it makes the dashboard able to say # why a bot stays behind instead of leaving it a mystery. SELF_UPDATE_BLOCKED=false [[ -w "$(dirname "$BOT_PATH")" ]] || SELF_UPDATE_BLOCKED=true # Läuft der alte Report-Bot hier noch? Während der Migration hilft das, # weil sonst zwei Bots parallel funken und niemand es merkt. LEGACY_BOT=false [[ -e /usr/local/sbin/spaceport-report || -e /etc/spaceport-report ]] && LEGACY_BOT=true systemctl list-unit-files 2>/dev/null | grep -q '^spaceport-report\.' && LEGACY_BOT=true HARDENED=false [[ -f /etc/spaceport-bot/hardened ]] && HARDENED=true [[ "$UFW_ACTIVE" == "true" && "$UNATT_ACTIVE" == "true" ]] && HARDENED=true # ---------- docker ---------- DOCKER_INSTALLED=false DOCKER_RUNNING=0; DOCKER_TOTAL=0; DOCKER_IMAGES=0; DOCKER_UNHEALTHY=0 CONTAINERS_JSON='[]' if command -v docker >/dev/null 2>&1 && docker info >/dev/null 2>&1; then DOCKER_INSTALLED=true DOCKER_RUNNING="$(docker ps -q 2>/dev/null | wc -l | trim)" DOCKER_TOTAL="$(docker ps -aq 2>/dev/null | wc -l | trim)" DOCKER_IMAGES="$(docker images -q 2>/dev/null | sort -u | wc -l | trim)" # Health comes from the container's healthcheck; "none" when it has none. # # Every value goes through the template function `json`, which quotes and # escapes it — container names and image tags must not be able to break # the payload. Do NOT use `trimPrefix` here: it is not a Docker template # function and makes the whole inspect call fail. The leading slash of # .Name is stripped in jq instead. CONTAINERS_JSON="$(docker ps -aq 2>/dev/null \ | head -200 \ | xargs -r docker inspect --format \ '{"id":{{json .Id}},"pid":{{.State.Pid}},"name":{{json .Name}},"image":{{json .Config.Image}},"state":{{json .State.Status}},"status":{{json .State.Status}},"health":{{if .State.Health}}{{json .State.Health.Status}}{{else}}"none"{{end}},"health_detail":{{if .State.Health}}{{json .State.Health}}{{else}}null{{end}},"restart_count":{{.RestartCount}},"started_at":{{json .State.StartedAt}},"exit_code":{{.State.ExitCode}},"env":{{json .Config.Env}},"labels":{{json .Config.Labels}}}' \ 2>/dev/null \ | jq -s '[.[] | .name |= ltrimstr("/") # Keep only what makes an unhealthy container actionable: how # often the check failed in a row and what it last printed. | .health_failing_streak = (.health_detail.FailingStreak // 0) | .health_output = ( (.health_detail.Log // []) | last | (.Output // "") | gsub("\\s+$"; "") | .[0:400] ) | del(.health_detail) # Runtime-Versionen aus den Env-Variablen, die offizielle Images # setzen (php, node, python, wordpress …). Die Env enthält auch # Secrets — deshalb nur bekannte *_VERSION-Schlüssel mit # harmlosem Wert übernehmen und .env danach restlos löschen. # Nichts davon verlässt den Server ungefiltert. | .runtimes = ([(.env // [])[] | strings | capture("^(?PHP|NODE|PYTHON|RUBY|GOLANG|JAVA|DOTNET|ASPNET|BUN|DENO|NGINX|HTTPD|CADDY|PG|MYSQL|MARIADB|REDIS|WORDPRESS|COMPOSER)_VERSION=(?[0-9A-Za-z.+~:_-]{1,40})$") | {key: ({PHP: "php", NODE: "node", PYTHON: "python", RUBY: "ruby", GOLANG: "go", JAVA: "java", DOTNET: "dotnet", ASPNET: "aspnet", BUN: "bun", DENO: "deno", NGINX: "nginx", HTTPD: "httpd", CADDY: "caddy", PG: "postgres", MYSQL: "mysql", MARIADB: "mariadb", REDIS: "redis", WORDPRESS: "wordpress", COMPOSER: "composer"}[.k]), value: .v}] | from_entries) | del(.env) # Wer den Container deployt hat: Coolify (App, Service, # Datenbank) oder ein eigenes Compose-Projekt. Nur diese # Schlüssel — Traefik-Labels & Co. bleiben draußen. | .labels = ((.labels // {}) | with_entries( select(.key | test("^coolify\\.(managed|type|name|resourceName|projectName|environmentName|serviceName|applicationId|applicationUuid)$|^com\\.docker\\.compose\\.(project|service)$")) | .value |= (tostring | .[0:120])))]' 2>/dev/null)" # Fallback: if inspect gave us nothing (permissions, template trouble on # an old Docker), fall back to `docker ps`, whose Status text carries the # health in parentheses. if [[ -z "$CONTAINERS_JSON" || "$CONTAINERS_JSON" == "[]" ]] && [[ "$DOCKER_TOTAL" -gt 0 ]]; then CONTAINERS_JSON="$(docker ps -a --format '{{json .}}' 2>/dev/null \ | jq -s '[.[] | { name: .Names, image: .Image, state: (.State // ""), status: (.Status // ""), health: (if (.Status // "") | test("\\(healthy\\)") then "healthy" elif (.Status // "") | test("\\(unhealthy\\)") then "unhealthy" else "none" end), restart_count: 0, started_at: "", exit_code: 0, health_failing_streak: 0, health_output: "" }]' 2>/dev/null)" fi CONTAINERS_JSON="${CONTAINERS_JSON:-[]}" # Ressourcen pro laufendem Container (ab Bot 2.11.0), als .stats angehängt. CONTAINER_STATS="$(printf '%s' "$CONTAINERS_JSON" \ | jq -r '.[] | select(.state == "running" and (.id // "") != "") | "\(.id) \(.pid // 0)"' 2>/dev/null \ | while read -r cid cpid; do container_stats "$cid" "$cpid"; done \ | jq -s 'map({key: .id, value: del(.id)}) | from_entries' 2>/dev/null)" [[ -n "$CONTAINER_STATS" ]] || CONTAINER_STATS='{}' CONTAINERS_JSON="$(printf '%s' "$CONTAINERS_JSON" \ | jq --argjson stats "$CONTAINER_STATS" '[.[] | .stats = ($stats[.id // ""] // null) | del(.pid)]' 2>/dev/null \ || printf '%s' "$CONTAINERS_JSON")" # Unhealthy = failing healthcheck, restart-looping, dead, or exited non-zero. # Gestoppte buildx-Builder nicht: Coolify stoppt sie nach jedem Cleanup # selbst, buildkitd exitet dabei mit 1. DOCKER_UNHEALTHY="$(printf '%s' "$CONTAINERS_JSON" | jq '[.[] | select( .health == "unhealthy" or .state == "restarting" or .state == "dead" or (.state == "exited" and .exit_code != 0 and (.name | startswith("buildx_buildkit_") | not)) )] | length' 2>/dev/null || echo 0)" DOCKER_UNHEALTHY="${DOCKER_UNHEALTHY:-0}" fi # ---------- OOM-Kills ---------- # Aus dem Kernel-Log der letzten Stunde (ab Bot 2.12.0). Nur dort stehen auch # Kills des Hauptprozesses, nach denen Docker den Container still neu startet # — dessen cgroup samt Zählern ist dann schon weg, und Dockers eigene Events # liegen nur in einem kleinen Ringpuffer. Jeder Report schickt die ganze # Stunde, das Dashboard muss nichts zusammenzählen. # oom-kill:constraint=CONSTRAINT_MEMCG,…,task_memcg=/system.slice/docker-.scope,task=node,pid=… # CONSTRAINT_MEMCG: Limit der cgroup erreicht, CONSTRAINT_NONE: Host ohne RAM. OOM_KILLS_JSON="$(journalctl -k -q --since "-60min" -o short-unix --no-pager -g 'oom-kill:' 2>/dev/null \ | awk '{ ts = $1; sub(/.*oom-kill:/, "") n = split($0, kv, ","); c = ""; m = ""; t = "" for (i = 1; i <= n; i++) { split(kv[i], p, "=") if (p[1] == "constraint") c = p[2]; else if (p[1] == "task_memcg") m = p[2]; else if (p[1] == "task") t = p[2] } printf "%s\t%s\t%s\t%s\n", ts, c, m, t }' \ | tail -50 \ | jq -R -s --argjson containers "$CONTAINERS_JSON" ' ($containers | map({key: (.id // ""), value: .name}) | from_entries) as $names | [split("\n")[] | select(length > 0) | split("\t") | {at: (.[0] | tonumber | floor), constraint: .[1], cgroup: .[2][0:200], process: .[3][0:64]} | .container_id = ((.cgroup | capture("(?[0-9a-f]{64})") | .id) // null) | .container = (if .container_id then $names[.container_id] else null end)]' 2>/dev/null)" [[ -n "$OOM_KILLS_JSON" ]] || OOM_KILLS_JSON='[]' # ---------- coolify ---------- COOLIFY_INSTALLED=false; ACME_TOTAL=0; ACME_BAD=0; TRAEFIK_ERRORS=0 if [[ -d /data/coolify ]]; then COOLIFY_INSTALLED=true ACME_TOTAL="$(find /data/coolify -name acme.json 2>/dev/null | wc -l | trim)" ACME_BAD="$(find /data/coolify -name acme.json -printf '%m\n' 2>/dev/null | awk '$1 != "600"' | wc -l | trim)" if docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^coolify-proxy$'; then TRAEFIK_ERRORS="$(docker logs coolify-proxy --since 1h 2>&1 \ | grep -cE 'permissions [0-9]+ for .* too open|nonexistent certificate resolver|ACME resolve is skipped' || true)" fi fi ACME_TOTAL="${ACME_TOTAL:-0}"; ACME_BAD="${ACME_BAD:-0}"; TRAEFIK_ERRORS="${TRAEFIK_ERRORS:-0}" # Rolle: "dashboard" = hier läuft Coolify selbst (Container "coolify", PHP), # "target" = nur Deploy-Ziel (/data/coolify ohne Dashboard). Das Dashboard # ist die Angriffsfläche für PHP-Lücken wie PHP2Shell — sein PHP steckt im # Container, apt auf dem Host patcht es nicht. COOLIFY_ROLE="none"; COOLIFY_VERSION=""; COOLIFY_PHP="" COOLIFY_PUBLISHED_JSON='[]'; COOLIFY_PUBLIC_JSON='[]'; COOLIFY_FIREWALL="null" COOLIFY_ADMIN_CIDRS_JSON="null" [[ "$COOLIFY_INSTALLED" == "true" ]] && COOLIFY_ROLE="target" if [[ "$DOCKER_INSTALLED" == "true" ]] && docker inspect coolify >/dev/null 2>&1; then COOLIFY_INSTALLED=true; COOLIFY_ROLE="dashboard" COOLIFY_VERSION="$(docker inspect coolify --format '{{.Config.Image}}' 2>/dev/null \ | sed -n 's/.*:v\{0,1\}\([0-9][0-9A-Za-z.+-]*\)$/\1/p')" COOLIFY_PHP="$(timeout 15 docker exec coolify php -r 'echo PHP_VERSION;' 2>/dev/null)" [[ "$COOLIFY_PHP" =~ ^[0-9]+\.[0-9]+\.[0-9]+ ]] || COOLIFY_PHP="" # Ports, die Docker für Dashboard/Realtime auf allen Interfaces # veröffentlicht. Die gehen an UFW vorbei — siehe coolify_port_blocked. COOLIFY_PUBLISHED_JSON="$(docker inspect coolify coolify-realtime \ --format '{{json .NetworkSettings.Ports}}' 2>/dev/null \ | jq -s '[.[] | select(. != null) | to_entries[] | (.value // [])[] | select(.HostIp == "" or .HostIp == "0.0.0.0" or .HostIp == "::") | .HostPort | tonumber] | unique' 2>/dev/null)" COOLIFY_PUBLISHED_JSON="${COOLIFY_PUBLISHED_JSON:-[]}" if command -v iptables >/dev/null 2>&1 && iptables -w 5 -S DOCKER-USER >/dev/null 2>&1; then COOLIFY_FIREWALL=false if iptables -w 5 -C DOCKER-USER -j SPACEPORT-COOLIFY 2>/dev/null; then COOLIFY_FIREWALL=true # Freigegebene Netze, wie die Firewall sie tatsächlich anwendet — # ohne die immer erlaubten privaten Netze. COOLIFY_ADMIN_CIDRS_JSON="$( { iptables -w 5 -S SPACEPORT-COOLIFY 2>/dev/null ip6tables -w 5 -S SPACEPORT-COOLIFY-IN 2>/dev/null } | awk '$0 ~ /-j RETURN/ { for (i = 1; i < NF; i++) if ($i == "-s") print $(i + 1) }' \ | grep -vxE '10\.0\.0\.0/8|172\.16\.0\.0/12|192\.168\.0\.0/16|fc00::/7' \ | sort -u | jq -R . | jq -s . 2>/dev/null)" COOLIFY_ADMIN_CIDRS_JSON="${COOLIFY_ADMIN_CIDRS_JSON:-[]}" fi local public=() port for port in $(printf '%s' "$COOLIFY_PUBLISHED_JSON" | jq -r '.[]' 2>/dev/null); do coolify_port_blocked "$port" || public+=("$port") done COOLIFY_PUBLIC_JSON="$(printf '%s\n' "${public[@]}" | jq -R 'select(. != "") | tonumber' | jq -s .)" COOLIFY_PUBLIC_JSON="${COOLIFY_PUBLIC_JSON:-[]}" else # Ohne lesbare Firewall wissen wir es nicht — dann gilt, was Docker # veröffentlicht, als offen. COOLIFY_PUBLIC_JSON="$COOLIFY_PUBLISHED_JSON" fi fi GENERATED_AT="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" PAYLOAD="$(jq -n \ --argjson version "$REPORT_VERSION" \ --arg generated_at "$GENERATED_AT" \ --arg hostname "$HOSTNAME_FQDN" --arg kernel "$KERNEL" --arg os "$OS_PRETTY" --arg arch "$ARCH" \ --argjson uptime_seconds "$UPTIME_SECONDS" --arg boot_time "$BOOT_TIME" --arg primary_ipv4 "$PRIMARY_IPV4" \ --argjson cpu_cores "$CPU_CORES" --argjson load_1 "$LOAD_1" --argjson load_5 "$LOAD_5" --argjson load_15 "$LOAD_15" \ --argjson cpu_usage "$CPU_USAGE" \ --argjson mem_total "$MEM_TOTAL" --argjson mem_used "$MEM_USED" --argjson mem_free "$MEM_FREE" \ --argjson mem_avail "$MEM_AVAIL" --argjson mem_pct "$MEM_PCT" \ --arg disk_total "$DISK_TOTAL" --arg disk_used "$DISK_USED" --arg disk_avail "$DISK_AVAIL" --argjson disk_pct "$DISK_PCT" \ --argjson disks "$DISKS_JSON" \ --argjson swap_total "$SWAP_TOTAL" --argjson swap_used "$SWAP_USED" --argjson swap_pct "$SWAP_PCT" \ --argjson installed_count "$INSTALLED_COUNT" --argjson pending_count "$PENDING_COUNT" \ --argjson pending_security_count "$PENDING_SECURITY_COUNT" --argjson pending_list "$PENDING_LIST_JSON" \ --arg last_apt_update "$LAST_APT_UPDATE" --arg last_apt_upgrade "$LAST_APT_UPGRADE" \ --argjson reboot_required "$REBOOT_REQUIRED" --argjson reboot_required_packages "$REBOOT_PKGS_JSON" \ --argjson auto_update_enabled "$AUTO_UPDATE_ENABLED" \ --arg auto_update_last_run "$AUTO_UPDATE_LAST_RUN" \ --arg auto_reboot "$AUTO_REBOOT" \ --arg auto_reboot_day "$AUTO_REBOOT_DAY" \ --arg auto_reboot_hour "$AUTO_REBOOT_HOUR" \ --argjson logins_recent "$LOGINS_RECENT_JSON" --argjson failed_logins_24h "$FAILED_LOGINS_24H" \ --argjson active_sessions "$ACTIVE_SESSIONS_JSON" \ --argjson active_unit_count "$ACTIVE_UNIT_COUNT" --argjson failed_unit_count "$FAILED_UNIT_COUNT" \ --argjson failed_units "$FAILED_UNITS_JSON" --argjson unit_restarts "$UNIT_RESTARTS_JSON" \ --argjson stopped_units "$STOPPED_UNITS_JSON" \ --argjson listen_ports "$LISTEN_PORTS_JSON" \ --argjson ufw_active "$UFW_ACTIVE" --argjson fail2ban_active "$FAIL2BAN_ACTIVE" \ --argjson crowdsec_active "$CROWDSEC_ACTIVE" --argjson crowdsec_decisions "$CROWDSEC_DECISIONS" \ --argjson unatt_active "$UNATT_ACTIVE" --argjson hardened "$HARDENED" \ --argjson docker_installed "$DOCKER_INSTALLED" --argjson docker_running "$DOCKER_RUNNING" \ --argjson docker_total "$DOCKER_TOTAL" --argjson docker_images "$DOCKER_IMAGES" \ --argjson docker_unhealthy "$DOCKER_UNHEALTHY" --argjson containers "$CONTAINERS_JSON" --argjson oom_kills "$OOM_KILLS_JSON" \ --argjson coolify_installed "$COOLIFY_INSTALLED" --argjson acme_total "$ACME_TOTAL" \ --argjson acme_bad "$ACME_BAD" --argjson traefik_errors "$TRAEFIK_ERRORS" \ --arg coolify_role "$COOLIFY_ROLE" --arg coolify_version "$COOLIFY_VERSION" \ --arg coolify_php "$COOLIFY_PHP" --argjson coolify_published "$COOLIFY_PUBLISHED_JSON" \ --argjson coolify_public "$COOLIFY_PUBLIC_JSON" --argjson coolify_firewall "$COOLIFY_FIREWALL" \ --argjson coolify_admin_cidrs "$COOLIFY_ADMIN_CIDRS_JSON" \ --arg bot_version "$BOT_VERSION" --argjson interval "${INTERVAL_SECONDS:-300}" \ --arg self_update "$SELF_UPDATE" \ --argjson self_update_blocked "$SELF_UPDATE_BLOCKED" \ --argjson legacy_bot "$LEGACY_BOT" \ '{ version: $version, generated_at: $generated_at, server: { hostname: $hostname, kernel: $kernel, os: $os, arch: $arch, uptime_seconds: $uptime_seconds, boot_time: $boot_time, primary_ipv4: $primary_ipv4 }, system: { cpu: { cores: $cpu_cores, load_1: $load_1, load_5: $load_5, load_15: $load_15, usage_percent: $cpu_usage }, memory: { total_mb: $mem_total, used_mb: $mem_used, free_mb: $mem_free, available_mb: $mem_avail, percent: $mem_pct }, disk_root: { total: $disk_total, used: $disk_used, available: $disk_avail, percent: $disk_pct }, disks: $disks, swap: { total_mb: $swap_total, used_mb: $swap_used, percent: $swap_pct } }, packages: { installed_count: $installed_count, pending_count: $pending_count, pending_security_count: $pending_security_count, pending_list: $pending_list, last_apt_update: ($last_apt_update | if . == "" then null else . end), last_apt_upgrade: ($last_apt_upgrade | if . == "" then null else . end), reboot_required: $reboot_required, reboot_required_packages: $reboot_required_packages, auto_update_enabled: $auto_update_enabled, auto_update_last_run: ($auto_update_last_run | if . == "" then null else . end), auto_reboot: ($auto_reboot == "yes"), auto_reboot_day: ($auto_reboot_day | if . == "" then null else . end), auto_reboot_hour: ($auto_reboot_hour | if . == "" then null else . end) }, auth: { logins_recent: $logins_recent, failed_logins_24h: $failed_logins_24h, active_sessions: $active_sessions }, services: { active_count: $active_unit_count, failed_count: $failed_unit_count, failed_units: $failed_units, unit_restarts: $unit_restarts, stopped_units: $stopped_units }, network: { listening_ports: $listen_ports }, security: { ufw_active: $ufw_active, fail2ban_active: $fail2ban_active, crowdsec_active: $crowdsec_active, crowdsec_decisions_count: $crowdsec_decisions, unattended_upgrades_active: $unatt_active, hardened: $hardened }, docker: { installed: $docker_installed, running_containers: $docker_running, total_containers: $docker_total, images_count: $docker_images, unhealthy_containers: $docker_unhealthy, containers: $containers }, oom_kills: $oom_kills, coolify: { installed: $coolify_installed, acme_files_total: $acme_total, acme_files_bad_perms: $acme_bad, traefik_acme_errors_recent: $traefik_errors, role: $coolify_role, version: ($coolify_version | if . == "" then null else . end), php_version: ($coolify_php | if . == "" then null else . end), published_ports: $coolify_published, public_ports: $coolify_public, firewall: $coolify_firewall, admin_cidrs: $coolify_admin_cidrs }, report_meta: { report_version: $version, bot_version: $bot_version, interval_seconds: $interval, self_update: ($self_update == "yes"), self_update_blocked: $self_update_blocked, legacy_bot: $legacy_bot } }')" [[ -n "$PAYLOAD" ]] || fail "failed to build payload" } # --------------------------------------------------------------------------- # send # --------------------------------------------------------------------------- send() { local ts sig code ts="$(date +%s)" sig="$(printf '%s' "${ts}.${PAYLOAD}" | hmac_sha256 "$BOT_SECRET")" RESP_FILE="$(mktemp)" HDR_FILE="$(mktemp)" # Token per Header-Datei, Payload per stdin: nichts davon in der Prozessliste. write_auth_headers "$HDR_FILE" \ 'Content-Type: application/json' \ "X-Spaceport-Timestamp: $ts" \ "X-Spaceport-Signature: $sig" \ "User-Agent: spaceport-bot/$BOT_VERSION" code="$(printf '%s' "$PAYLOAD" | curl -sS -o "$RESP_FILE" -w '%{http_code}' \ --max-time "$TIMEOUT_SECONDS" \ -X POST "${API_URL}/api/bot/report" \ -H @"$HDR_FILE" \ --data-binary @- || echo 000)" if [[ "$code" =~ ^2[0-9][0-9]$ ]]; then # The API answers with the newest bot version so we can warn about drift. local latest latest="$(jq -r '.latestBotVersion // empty' < "$RESP_FILE" 2>/dev/null)" log "report sent (HTTP $code)" apply_remote_config apply_coolify_cidrs if [[ -n "$latest" ]] && version_gt "$latest" "$BOT_VERSION"; then if [[ "$SELF_UPDATE" == "yes" ]]; then log "newer bot available: $BOT_VERSION -> $latest — updating myself" # A failed self-update must not fail the report that just worked. cmd_update || log "self-update failed — fix it with: sudo spaceport-bot update" else log "update available: $BOT_VERSION -> $latest (run: sudo spaceport-bot update)" fi fi return 0 fi log "send failed (HTTP $code): $(head -c 300 "$RESP_FILE" 2>/dev/null)" return 2 } # Übernimmt die Admin-Whitelist der Coolify-Firewall aus dem Dashboard. # # Die Liste öffnet Firewall-Ports, deshalb streng: nur Einträge, die wie eine # IP mit Präfix aussehen, keine zu weiten Netze, höchstens 50. Angewendet wird # nur, wo die Firewall bereits läuft — der Bot schaltet sie nie selbst ein. apply_coolify_cidrs() { local has want have entry addr prefix has="$(jq -r 'if (.config | type) == "object" and (.config | has("coolifyAdminCidrs")) then "yes" else "no" end' \ < "$RESP_FILE" 2>/dev/null)" [[ "$has" == "yes" ]] || return 0 want="$(jq -r '.config.coolifyAdminCidrs | if type == "array" then .[] else empty end | strings' \ < "$RESP_FILE" 2>/dev/null \ | while IFS= read -r entry; do addr="${entry%/*}"; prefix="${entry##*/}" [[ "$entry" == */* && "$prefix" =~ ^[0-9]{1,3}$ ]] || continue if [[ "$addr" =~ ^[0-9]{1,3}(\.[0-9]{1,3}){3}$ ]]; then (( prefix >= 8 && prefix <= 32 )) && printf '%s\n' "$entry" elif [[ "$addr" =~ ^[0-9a-fA-F:]+$ && "$addr" == *:* ]]; then (( prefix >= 16 && prefix <= 128 )) && printf '%s\n' "$entry" fi done | sort -u | head -50)" have="$(cat "$COOLIFY_CIDRS_FILE" 2>/dev/null)" [[ "$want" == "$have" ]] && return 0 if ! install -d -m 0700 "$(dirname "$COOLIFY_CIDRS_FILE")" 2>/dev/null; then log "coolify whitelist differs from the dashboard, but $(dirname "$COOLIFY_CIDRS_FILE") is not writable" return 0 fi umask 077 printf '%s\n' "$want" | grep -v '^$' > "$COOLIFY_CIDRS_FILE" || true log "coolify whitelist from dashboard: ${want//$'\n'/, }" if [[ -x "$COOLIFY_FIREWALL_BIN" ]] && iptables -w 5 -C DOCKER-USER -j SPACEPORT-COOLIFY 2>/dev/null; then "$COOLIFY_FIREWALL_BIN" apply >/dev/null 2>&1 \ && log "coolify firewall re-applied" \ || log "coolify firewall apply failed — check: sudo spaceport-coolify-firewall apply" fi } # Apply the reboot window the dashboard asks for. The bot only ever pushes # measurements; this is the one piece of desired state it pulls back, so the # window can be changed in the UI instead of over SSH. Only the three known # keys are written — the answer never becomes a way to run code. apply_remote_config() { local want_reboot want_day want_hour have_reboot have_day have_hour want_reboot="$(jq -r 'if .config.autoReboot == true then "yes" elif .config.autoReboot == false then "no" else empty end' < "$RESP_FILE" 2>/dev/null)" [[ -z "$want_reboot" ]] && return 0 want_day="$(jq -r '.config.rebootDay // 0' < "$RESP_FILE" 2>/dev/null)" want_hour="$(jq -r '.config.rebootHour // 4' < "$RESP_FILE" 2>/dev/null)" [[ "$want_day" =~ ^[0-6]$ ]] || want_day=0 [[ "$want_hour" =~ ^([0-9]|1[0-9]|2[0-3])$ ]] || want_hour=4 if [[ -r "$AUTO_UPDATE_CONF" ]]; then have_reboot="$(. "$AUTO_UPDATE_CONF" 2>/dev/null; echo "${AUTO_REBOOT:-no}")" have_day="$(. "$AUTO_UPDATE_CONF" 2>/dev/null; echo "${REBOOT_DAY:-0}")" have_hour="$(. "$AUTO_UPDATE_CONF" 2>/dev/null; echo "${REBOOT_HOUR:-4}")" [[ "$have_reboot" == "$want_reboot" && "$have_day" == "$want_day" && "$have_hour" == "$want_hour" ]] && return 0 fi if [[ ! -w "$(dirname "$AUTO_UPDATE_CONF")" ]]; then log "reboot window differs from the dashboard, but $(dirname "$AUTO_UPDATE_CONF") is read-only" return 0 fi umask 077 cat > "$AUTO_UPDATE_CONF" </dev/null || fail "downloaded script has syntax errors — not installing" # Replace by rename: atomic, and the currently running script keeps its # inode instead of being rewritten underneath itself. if ! install -m 0755 "$tmp" "${BOT_PATH}.new"; then fail "could not write ${BOT_PATH}.new — read-only or not root. From a shell: sudo spaceport-bot update" fi [[ $EUID -eq 0 ]] && chown root:root "${BOT_PATH}.new" 2>/dev/null if ! mv -f "${BOT_PATH}.new" "$BOT_PATH"; then rm -f "${BOT_PATH}.new" fail "could not replace $BOT_PATH" fi new_version="$("$BOT_PATH" version 2>/dev/null || echo unknown)" log "updated $BOT_VERSION -> $new_version" update_units systemctl restart spaceport-bot.timer 2>/dev/null || true # Report right away with the new binary, otherwise the dashboard keeps # showing the old version until the next timer tick. Only when the version # really changed, so a no-op update cannot cause a report loop. if [[ "$new_version" != "$BOT_VERSION" && "$new_version" != "unknown" ]]; then log "sending a report with the new version" "$BOT_PATH" run >/dev/null 2>&1 || log "first report after update failed — check: spaceport-bot run" fi } # Keep the systemd units in sync with the dashboard. Matters because the units # carry the sandbox rules: an old unit without ReadWritePaths=/usr/local/sbin # mounts /usr read-only and makes every self-update fail. Only reachable when # run outside that sandbox, i.e. from a shell as root. # # Units sind genauso signiert wie das Script: Eine Unit mit beliebigem # ExecStart= wäre sonst ein Weg, an der Signaturprüfung vorbei Code als root # auszuführen. update_units() { local changed=0 unit tmp patched uhdr sig_remote [[ -w "$UNIT_DIR" ]] || { log "no write permission on $UNIT_DIR — systemd units unchanged"; return 0; } [[ -n "$HDR_FILE" && -s "$HDR_FILE" ]] || { HDR_FILE="$(mktemp)"; write_auth_headers "$HDR_FILE"; } for unit in spaceport-bot.service spaceport-bot.timer; do tmp="$(mktemp)" uhdr="$(mktemp)" if ! curl -fsSL "${API_URL}/api/bot/script?name=${unit}" -H @"$HDR_FILE" -D "$uhdr" -o "$tmp"; then rm -f "$tmp" "$uhdr" log "could not fetch $unit — keeping the installed one" continue fi sig_remote="$(awk 'BEGIN{IGNORECASE=1} /^x-spaceport-signature:/ {gsub(/\r/,""); print $2}' "$uhdr" | tail -n 1)" if [[ -z "$sig_remote" || "$sig_remote" != "$(hmac_sha256 "$BOT_SECRET" < "$tmp")" ]] \ || ! release_signature_ok "$tmp" "$uhdr"; then rm -f "$tmp" "$uhdr" log "$unit has no valid signature — keeping the installed one" continue fi rm -f "$uhdr" # A unit must at least look like one before it replaces a working file. if ! grep -q '^\[Unit\]' "$tmp"; then rm -f "$tmp" log "downloaded $unit looks wrong — keeping the installed one" continue fi # The interval lives in the config, the shipped timer only carries a # default. Patch before comparing, otherwise every run would rewrite the # file and reload systemd for nothing. A temp file instead of `sed -i`, # which takes an argument on BSD and none on GNU. if [[ "$unit" == "spaceport-bot.timer" && -n "${INTERVAL_SECONDS:-}" ]]; then patched="$(mktemp)" if sed "s/^OnUnitActiveSec=.*/OnUnitActiveSec=${INTERVAL_SECONDS}s/" "$tmp" > "$patched" 2>/dev/null; then mv -f "$patched" "$tmp" else rm -f "$patched" fi fi if ! cmp -s "$tmp" "${UNIT_DIR}/${unit}"; then install -m 0644 "$tmp" "${UNIT_DIR}/${unit}" && changed=1 && log "updated $unit" fi rm -f "$tmp" done [[ "$changed" == "1" ]] && systemctl daemon-reload 2>/dev/null || true } cmd_status() { load_config echo "version: $BOT_VERSION" echo "api: $API_URL" echo "token: ${BOT_TOKEN:0:8}…" echo "interval: ${INTERVAL_SECONDS:-300}s" echo "self-update: ${SELF_UPDATE}" systemctl status spaceport-bot.timer --no-pager 2>/dev/null | head -5 || true } case "${1:-run}" in run) cmd_run ;; update) cmd_update ;; status) cmd_status ;; version) echo "$BOT_VERSION" ;; *) fail "unknown command: $1 (run|update|status|version)" ;; esac