lanky-alert-monitor.sh

Watches for alert conditions and fires notifications when thresholds trip.

#!/usr/bin/env bash
set -u
STATE_DIR=/var/lib/lanky-alerts
REPORT_DIR=/home/lanky/reports
mkdir -p "$STATE_DIR" "$REPORT_DIR"
STATUS_FILE="$STATE_DIR/last-alert-status.txt"
CURRENT=$(mktemp)
trap 'rm -f "$CURRENT"' EXIT

add() { echo "$*" >> "$CURRENT"; }


power_state="unknown"
if command -v on_ac_power >/dev/null 2>&1; then
  if on_ac_power >/dev/null 2>&1; then
    power_state="ac"
  else
    rc=$?
    [ "$rc" -eq 1 ] && power_state="battery"
  fi
fi
if [ "$power_state" = unknown ]; then
  for supply in /sys/class/power_supply/*; do
    [ -d "$supply" ] || continue
    type=$(cat "$supply/type" 2>/dev/null || echo unknown)
    if [ "$type" = Mains ] || [ "$type" = USB ] || [ "$type" = USB_C ] || [ "$type" = USB_PD ]; then
      online=$(cat "$supply/online" 2>/dev/null || echo unknown)
      [ "$online" = 1 ] && power_state="ac"
    elif [ "$type" = Battery ]; then
      status=$(cat "$supply/status" 2>/dev/null || echo unknown)
      printf '%s' "$status" | grep -qi '^Discharging$' && power_state="battery"
    fi
  done
fi
if [ "$power_state" = battery ]; then
  battery_detail=$(for b in /sys/class/power_supply/*; do [ -d "$b" ] || continue; [ "$(cat "$b/type" 2>/dev/null || echo unknown)" = Battery ] || continue; printf '%s %s %s%%; ' "$(basename "$b")" "$(cat "$b/status" 2>/dev/null || echo unknown)" "$(cat "$b/capacity" 2>/dev/null || echo unknown)"; done)
  add "MAJOR: server is running on battery power; faulty charge port risk; server may shut down${battery_detail:+ ($battery_detail)}"
fi

for m in /mnt/nas /mnt/jelly; do
  timeout 5s mountpoint -q "$m" || add "CRITICAL: $m is not mounted"
done

if ! systemctl is-active --quiet docker; then
  add "CRITICAL: docker service is not active"
fi

failed_units=$(systemctl --failed --plain --no-legend 2>/dev/null | awk '{print $1}' | paste -sd ', ' -)
[ -n "$failed_units" ] && add "WARNING: failed systemd units: $failed_units"

unhealthy=$(timeout 10s docker ps --filter health=unhealthy --format '{{.Names}} ({{.Status}})' 2>/dev/null | paste -sd '; ' -)
[ -n "$unhealthy" ] && add "CRITICAL: unhealthy containers: $unhealthy"

bad_containers=$(timeout 10s docker ps -a --filter status=exited --filter status=dead --filter status=restarting --format '{{.Names}} ({{.Status}})' 2>/dev/null | paste -sd '; ' -)
[ -n "$bad_containers" ] && add "WARNING: stopped/restarting containers: $bad_containers"

root_pct=$(df -P / 2>/dev/null | awk 'NR==2 {gsub(/%/,"",$5); print $5}')
[ -n "${root_pct:-}" ] && [ "$root_pct" -ge 85 ] && add "WARNING: root disk is ${root_pct}% full"

for m in /mnt/nas /mnt/jelly; do
  if timeout 5s mountpoint -q "$m"; then
    pct=$(timeout 10s df -P "$m" 2>/dev/null | awk 'NR==2 {gsub(/%/,"",$5); print $5}')
    [ -n "${pct:-}" ] && [ "$pct" -ge 90 ] && add "WARNING: $m is ${pct}% full"
  fi
done

[ -f /var/run/reboot-required ] && add "INFO: reboot required after updates"

if [ ! -s "$CURRENT" ]; then
  echo "OK $(date -Is)" > "$REPORT_DIR/lanky-alert-monitor-latest.txt"
  if [ -s "$STATUS_FILE" ]; then
    /home/lanky/scripts/notify-admin.sh "RESOLVED: no active server alerts" || true
    : > "$STATUS_FILE"
  fi
  exit 0
fi

cp "$CURRENT" "$REPORT_DIR/lanky-alert-monitor-latest.txt"
if ! cmp -s "$CURRENT" "$STATUS_FILE"; then
  msg=$(sed ':a;N;$!ba;s/\n/ | /g' "$CURRENT")
  /home/lanky/scripts/notify-admin.sh "$msg" || true
  cp "$CURRENT" "$STATUS_FILE"
fi

back to scripts