docker-watchdog.sh

Detects stale NAS/CIFS mounts, stops affected containers, remounts, and only restarts once storage is readable again.

#!/usr/bin/env bash
set -u
REPORT_DIR=/home/lanky/reports
LOG="$REPORT_DIR/docker-watchdog.log"
LOCK=/run/lanky-docker-watchdog.lock
PROTECTED_RESTART=" homeassistant cloudflared pihole-edge wireguard "
NAS_BACKED_CONTAINERS=" jellyfin jellyseerr qbittorrent radarr sonarr "
mkdir -p "$REPORT_DIR"
exec >> "$LOG" 2>&1
exec 9>"$LOCK"
if ! flock -n 9; then
  echo "[$(date -Is)] watchdog already running; skipping overlapping run"
  exit 0
fi

is_protected_restart() {
  case "$PROTECTED_RESTART" in
    *" $1 "*) return 0 ;;
    *) return 1 ;;
  esac
}

container_state() {
  docker inspect "$1" --format '{{.State.Status}} exit={{.State.ExitCode}} {{if .State.Health}}{{.State.Health.Status}}{{end}}' 2>/dev/null || echo unknown
}

readable_mount() {
  local mount_path=$1
  local probe_path=$mount_path
  case "$mount_path" in
    /mnt/jelly) probe_path=/mnt/jelly/Cartoons ;;
    /mnt/nas) probe_path=/mnt/nas/Backups ;;
  esac
  timeout 25s find "$probe_path" -maxdepth 1 -mindepth 1 -print -quit >/dev/null 2>&1
}

jellyfin_media_visible() {
  docker inspect jellyfin >/dev/null 2>&1 || return 0
  [ "$(docker inspect jellyfin --format '{{.State.Status}}' 2>/dev/null)" = "running" ] || return 0
  timeout 8s docker exec jellyfin sh -c 'test -d /data/media/Movies && test -d /data/media/Cartoons && test -d "/data/media/TV Shows"' >/dev/null 2>&1
}

recover_mounts() {
  echo "[$(date -Is)] stopping NAS-backed containers for mount recovery"
  for c in $NAS_BACKED_CONTAINERS; do
    if docker inspect "$c" >/dev/null 2>&1; then
      docker stop -t 15 "$c" >/dev/null || docker kill "$c" >/dev/null || true
    fi
  done

  echo "[$(date -Is)] refreshing CIFS mounts"
  systemctl stop mnt-jelly.mount mnt-nas.mount 2>/dev/null || true
  umount -l /mnt/jelly 2>/dev/null || true
  umount -l /mnt/nas 2>/dev/null || true
  sleep 3
  systemctl daemon-reload || true
  timeout 30s mount /mnt/jelly || echo "[$(date -Is)] failed to mount /mnt/jelly"
  timeout 30s mount /mnt/nas || echo "[$(date -Is)] failed to mount /mnt/nas"

  mounts_recovered=1
  for m in /mnt/nas /mnt/jelly; do
    if readable_mount "$m"; then
      echo "[$(date -Is)] $m readable after recovery"
    else
      echo "[$(date -Is)] $m still unreadable after recovery"
      mounts_recovered=0
    fi
  done

  if [ "$mounts_recovered" -ne 1 ]; then
    echo "[$(date -Is)] leaving NAS-backed containers stopped until mounts recover"
    return
  fi

  echo "[$(date -Is)] starting NAS-backed containers after mount recovery"
  for c in $NAS_BACKED_CONTAINERS; do
    if docker inspect "$c" >/dev/null 2>&1; then
      docker start "$c" >/dev/null || echo "[$(date -Is)] failed to start $c"
    fi
  done
}

echo "[$(date -Is)] watchdog start"
needs_mount_recovery=0
for m in /mnt/nas /mnt/jelly; do
  if ! mountpoint -q "$m"; then
    echo "[$(date -Is)] $m missing; attempting mount"
    needs_mount_recovery=1
    mount "$m" || echo "[$(date -Is)] failed to mount $m"
  fi
  if mountpoint -q "$m" && readable_mount "$m"; then
    echo "[$(date -Is)] $m mounted and readable"
  elif mountpoint -q "$m"; then
    echo "[$(date -Is)] $m mounted but unreadable/stale"
    needs_mount_recovery=1
  else
    echo "[$(date -Is)] $m still missing"
    needs_mount_recovery=1
  fi
done

if ! systemctl is-active --quiet docker; then
  echo "[$(date -Is)] docker inactive; starting docker"
  systemctl start docker || echo "[$(date -Is)] failed to start docker"
fi

if ! command -v docker >/dev/null 2>&1; then
  echo "[$(date -Is)] docker command missing"
  exit 0
fi

if [ "$needs_mount_recovery" -eq 1 ]; then
  recover_mounts
fi

problem_containers=$(docker ps -a --filter status=exited --filter status=dead --filter status=restarting --format '{{.Names}}' 2>/dev/null; docker ps --filter health=unhealthy --format '{{.Names}}' 2>/dev/null)
echo "$problem_containers" | awk 'NF' | sort -u | while read -r c; do
  case "$c" in
    '') continue ;;
    *)
      if is_protected_restart "$c"; then
        state=$(container_state "$c")
        echo "[$(date -Is)] protected problem container, report only: $c state=$state"
        continue
      fi
      state=$(container_state "$c")
      case "$state" in
        "exited exit=0 "*)
          echo "[$(date -Is)] ignoring successful exited container: $c state=$state"
          ;;
        *unhealthy*|exited*|dead*|restarting*)
          echo "[$(date -Is)] restarting problem container: $c state=$state"
          docker restart "$c" >/dev/null || echo "[$(date -Is)] failed to restart $c"
          ;;
        *)
          echo "[$(date -Is)] skip restart, state no longer failed: $c state=$state"
          ;;
      esac
      ;;
  esac
done

if readable_mount /mnt/jelly && ! jellyfin_media_visible; then
  echo "[$(date -Is)] jellyfin running without visible media paths; restarting"
  docker restart jellyfin >/dev/null || echo "[$(date -Is)] failed to restart jellyfin"
fi

echo "[$(date -Is)] watchdog done"

back to scripts