nas-duplicate-report.sh

Finds and reports duplicate files across the NAS.

#!/usr/bin/env bash
set -u
REPORT_DIR=/home/lanky/reports
mkdir -p "$REPORT_DIR"
STAMP=$(date +%Y%m%d-%H%M%S)
OUT="$REPORT_DIR/nas-duplicates-$STAMP.txt"
LATEST="$REPORT_DIR/nas-duplicates-latest.txt"
python3 - <<'PY' > "$OUT" 2>&1
import hashlib, os, subprocess, sys
from collections import defaultdict
from datetime import datetime

roots = ['/mnt/jelly', '/mnt/nas']
min_size = 50 * 1024 * 1024
skip_parts = {'@Recycle', '@Recently-Snapshot', '#recycle', '.Trash', '.TemporaryItems'}
print('NAS Duplicate Candidate Report')
print('Generated:', datetime.now().astimezone().isoformat())
print('Mode: report only; no files are deleted')
print('Minimum size:', min_size, 'bytes')
print()

def mounted(path):
    return subprocess.run(['mountpoint', '-q', path]).returncode == 0

def should_skip(path):
    parts = set(path.split(os.sep))
    return bool(parts & skip_parts)

by_size = defaultdict(list)
for root in roots:
    if not mounted(root):
        print(f'BAD: {root} is not mounted; skipped')
        continue
    print(f'Scanning {root} ...', file=sys.stderr)
    for dirpath, dirnames, filenames in os.walk(root):
        dirnames[:] = [d for d in dirnames if d not in skip_parts]
        if should_skip(dirpath):
            continue
        for name in filenames:
            path = os.path.join(dirpath, name)
            try:
                st = os.stat(path)
            except OSError:
                continue
            if st.st_size >= min_size:
                by_size[st.st_size].append(path)

hash_groups = defaultdict(list)
for size, paths in by_size.items():
    if len(paths) < 2:
        continue
    for path in paths:
        h = hashlib.sha256()
        try:
            with open(path, 'rb', buffering=1024*1024) as f:
                for chunk in iter(lambda: f.read(1024*1024), b''):
                    h.update(chunk)
        except OSError as exc:
            print(f'WARN: could not hash {path}: {exc}', file=sys.stderr)
            continue
        hash_groups[(size, h.hexdigest())].append(path)

dupe_count = 0
wasted = 0
for (size, digest), paths in sorted(hash_groups.items(), key=lambda item: (-item[0][0], item[0][1])):
    if len(paths) < 2:
        continue
    dupe_count += 1
    wasted += size * (len(paths) - 1)
    print(f'== Duplicate group: {len(paths)} files, size each {size:,} bytes, possible wasted {(size*(len(paths)-1)):,} bytes ==')
    print(f'sha256: {digest}')
    for p in paths:
        print(p)
    print()

print('Summary')
print('Duplicate groups:', dupe_count)
print('Possible wasted bytes:', wasted)
print('Possible wasted GiB:', round(wasted / (1024**3), 2))
PY
chmod 0644 "$OUT"
cp "$OUT" "$LATEST"
echo "$OUT"

back to scripts