nas-duplicate-cleanup.sh

Removes duplicates safely (via safe-recycle, never a hard delete).

#!/usr/bin/env bash
set -u
MODE="${1:---dry-run}"
if [ "$MODE" != "--dry-run" ] && [ "$MODE" != "--apply" ]; then
  echo "Usage: $0 [--dry-run|--apply]" >&2
  exit 2
fi
REPORT_DIR=/home/lanky/reports
mkdir -p "$REPORT_DIR"
STAMP=$(date +%Y%m%d-%H%M%S)
OUT="$REPORT_DIR/nas-duplicate-cleanup-$STAMP.txt"
LATEST="$REPORT_DIR/nas-duplicate-cleanup-latest.txt"
python3 - "$MODE" <<'PY' > "$OUT" 2>&1
import hashlib, os, subprocess, sys
from collections import defaultdict
from datetime import datetime

mode = sys.argv[1]
roots = ['/mnt/jelly', '/mnt/nas']
min_size = 50 * 1024 * 1024
skip_parts = {'@Recycle', '@Recently-Snapshot', '#recycle', '.Trash', '.TemporaryItems'}
protected_root = '/mnt/jelly'
preferred_delete_root = '/mnt/nas'

print('NAS Duplicate Cleanup')
print('Generated:', datetime.now().astimezone().isoformat())
print('Mode:', mode)
print('Policy: delete exact SHA256 duplicates from /mnt/nas; preserve /mnt/jelly copies')
print('Minimum size:', min_size, 'bytes')
print()

def mounted(path):
    return subprocess.run(['mountpoint', '-q', path]).returncode == 0

def should_skip(path):
    parts = set(path.split(os.sep))
    return bool(parts & skip_parts)

def under(path, root):
    try:
        return os.path.commonpath([os.path.realpath(path), root]) == root
    except ValueError:
        return False

for root in roots:
    if not mounted(root):
        print(f'BAD: {root} is not mounted; refusing cleanup')
        sys.exit(1)

by_size = defaultdict(list)
for root in roots:
    print(f'Scanning {root} ...', file=sys.stderr)
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = [d for d in dirnames if d not in skip_parts]
        if should_skip(dirpath):
            continue
        for name in filenames:
            path = os.path.join(dirpath, name)
            try:
                st = os.stat(path)
            except OSError:
                continue
            if st.st_size >= min_size:
                by_size[st.st_size].append(path)

hash_groups = defaultdict(list)
for size, paths in by_size.items():
    if len(paths) < 2:
        continue
    for path in paths:
        h = hashlib.sha256()
        try:
            with open(path, 'rb', buffering=1024*1024) as f:
                for chunk in iter(lambda: f.read(1024*1024), b''):
                    h.update(chunk)
        except OSError as exc:
            print(f'WARN: could not hash {path}: {exc}', file=sys.stderr)
            continue
        hash_groups[(size, h.hexdigest())].append(path)

deleted = 0
would_delete = 0
bytes_freed = 0
errors = 0
for (size, digest), paths in sorted(hash_groups.items(), key=lambda item: (-item[0][0], item[0][1])):
    if len(paths) < 2:
        continue
    paths = sorted(paths, key=lambda p: (0 if under(p, protected_root) else 1, len(p), p))
    jelly_paths = [p for p in paths if under(p, protected_root)]
    nas_paths = [p for p in paths if under(p, preferred_delete_root)]
    if jelly_paths:
        keep = jelly_paths[0]
        candidates = [p for p in nas_paths if p != keep]
    else:
        keep = nas_paths[0] if nas_paths else paths[0]
        candidates = [p for p in nas_paths[1:] if p != keep]
    if not candidates:
        continue
    print(f'== Duplicate group: {len(paths)} files, size each {size:,} bytes ==')
    print(f'sha256: {digest}')
    print(f'KEEP: {keep}')
    for p in candidates:
        print(('DELETE: ' if mode == '--apply' else 'WOULD DELETE: ') + p)
        if mode == '--apply':
            try:
                result = subprocess.run(
                    ['/home/lanky/scripts/safe-recycle.py',
                     'exact duplicate selected by NAS cleanup', p],
                    text=True,
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT,
                )
                print(result.stdout.rstrip())
                if result.returncode != 0:
                    raise OSError(f'safe recycle failed with exit code {result.returncode}')
                deleted += 1
                bytes_freed += size
            except OSError as exc:
                errors += 1
                print(f'ERROR deleting {p}: {exc}')
        else:
            would_delete += 1
            bytes_freed += size
    print()

print('Summary')
print('Deleted files:' if mode == '--apply' else 'Would delete files:', deleted if mode == '--apply' else would_delete)
print('Bytes freed estimate:', bytes_freed)
print('GiB freed estimate:', round(bytes_freed / (1024**3), 2))
print('Errors:', errors)
PY
status=$?
chmod 0644 "$OUT"
cp "$OUT" "$LATEST"
echo "$OUT"
exit $status

back to scripts