diff --git a/backup-to-drive.sh b/backup-to-drive.sh index f4f4a6f..ba5d4bc 100755 --- a/backup-to-drive.sh +++ b/backup-to-drive.sh @@ -112,6 +112,7 @@ if [ -z "$MISSING" ]; then say "✓ Drive already has every NAS backup — nothing to do" else FAILED=0 + OVERDUE="" COUNT="$(echo "$MISSING" | wc -l | tr -d ' ')" say "$COUNT missing from Drive; copying up to $MAX_PER_RUN" n=0 @@ -119,7 +120,16 @@ else [ -z "$f" ] && continue n=$((n+1)); [ $n -gt $MAX_PER_RUN ] && { say "· stopping at $MAX_PER_RUN this run"; break; } TMP="$(mktemp -t checklist-backup)"; OUT="$(mktemp -t checklist-backup-json)" - run_remote "cat $REMOTE_DIR/$f" > "$TMP" + # Retried, because the fetch is the fragile part. Each of these files is ~275 KB and it + # comes back through an expect pseudo-terminal, which does line discipline and has finite + # buffers — so a capture occasionally arrives truncated and fails to parse. That is a + # transport hiccup, not a corrupt backup: every file that failed this way between + # 2026-09-29 and 2026-10-07 copied cleanly on a later night. Retrying in the same run + # turns a nightly email into a non-event. + ok=0 + for try in 1 2 3; do + : > "$TMP" + run_remote "cat $REMOTE_DIR/$f" > "$TMP" # expect echoes its own `spawn …` banner and the password prompt onto stdout before the # file's contents, so the capture is never pure JSON. Slice from the first { to the last } # and only then validate — a truncated or half-read file that *looks* like a backup is @@ -133,12 +143,27 @@ else if(!Array.isArray(j.lists)) process.exit(1); // shape check, not just valid JSON fs.writeFileSync(process.argv[2], JSON.stringify(j)); ' "$TMP" "$OUT" 2>/dev/null; then + ok=1; break + fi + [ "$try" -lt 3 ] && say " · $f — incomplete capture (attempt $try), retrying" + sleep 3 + done + + if [ "$ok" = 1 ]; then mv "$OUT" "$DRIVE_DIR/$f"; rm -f "$TMP" say " ✓ $f ($(du -k "$DRIVE_DIR/$f" | cut -f1) KB)" else rm -f "$TMP" "$OUT" FAILED=$((FAILED+1)) - say " ✗ $f — no valid checklist JSON in the response, skipped" + say " ✗ $f — no valid checklist JSON after 3 attempts, skipped" + # Severity depends on the file's AGE, not on this run. Today's backup failing is a + # retry tomorrow; one still missing days later means the retries are not working. + FDATE="$(printf '%s' "$f" | sed -E 's/^checklists-([0-9]{4}-[0-9]{2}-[0-9]{2})\.json$/\1/')" + FEPOCH="$(date -j -f "%Y-%m-%d" "$FDATE" +%s 2>/dev/null || echo 0)" + if [ "$FEPOCH" -gt 0 ]; then + FAGE=$(( ( $(date +%s) - FEPOCH ) / 86400 )) + [ "$FAGE" -gt "$STALE_DAYS" ] && OVERDUE="$OVERDUE$f ($FAGE days old)"$'\n' + fi fi done <<< "$MISSING" fi @@ -171,10 +196,18 @@ say "✓ done — $TOTAL copies off-site, newest ${NEWEST:-none}" # Some files were fetched but were not usable. The run "succeeded" in the sense that it did # not crash, which is exactly the kind of success that hides a broken backup. -if [ "${FAILED:-0}" -gt 0 ]; then - alert "$FAILED backup file(s) could not be copied" \ - "The run finished, but $FAILED file(s) came back without valid checklist JSON and -were skipped. See the log for which ones." +# Only files that are genuinely overdue raise mail. A fetch that fails tonight and succeeds +# tomorrow is noise, and nightly noise is how an alert stops being read — which is the whole +# failure mode this script exists to prevent. +if [ -n "${OVERDUE:-}" ]; then + alert "backup file(s) still missing off-site after several days" \ + "These NAS backups have failed to copy for more than $STALE_DAYS days, so the +retries are not recovering on their own: + +$OVERDUE +Log: $LOG" +elif [ "${FAILED:-0}" -gt 0 ]; then + say "($FAILED file(s) failed this run but are recent — they will be retried tomorrow, not alerting)" fi # The check that would have caught both of the incidents in the header comment. Every