From 5db4aada74140331db36956fe4b9f492a3c603b7 Mon Sep 17 00:00:00 2001 From: Waters Church Date: Thu, 8 Oct 2026 09:11:21 -0400 Subject: [PATCH] Retry a failed backup fetch, and only alert when a file is actually overdue MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The nightly backup has emailed "1 backup file(s) could not be copied" most nights since 29 September. Nothing was wrong with the backups: every file that failed that way copied cleanly on a later night, and the only one still outstanding is today's. The alert was reporting a transport hiccup as if it were data loss. The fetch is the fragile part. Each file is ~275 KB and arrives through an expect pseudo-terminal, which does line discipline and has finite buffers, so a capture occasionally comes back truncated and fails to parse. It is now retried three times within the same run, which should turn most of these into a non-event. Alert severity now depends on the file's AGE rather than on this run. Today's backup failing is a retry tomorrow and is logged, not mailed. A file still missing after STALE_DAYS means the retries are not recovering and that does raise mail, naming the files and their ages. A nightly email that never needed action is how an alert stops being read, which is the exact failure this script was written to prevent — it was training its reader to ignore it. The real cure is to stop pulling files through a pty at all: with key-based SSH to the NAS this could be a plain scp and be byte-exact. That needs a key installed on the NAS and is left for a deliberate change. Co-Authored-By: Claude Opus 5 --- backup-to-drive.sh | 45 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/backup-to-drive.sh b/backup-to-drive.sh index f4f4a6f..ba5d4bc 100755 --- a/backup-to-drive.sh +++ b/backup-to-drive.sh @@ -112,6 +112,7 @@ if [ -z "$MISSING" ]; then say "✓ Drive already has every NAS backup — nothing to do" else FAILED=0 + OVERDUE="" COUNT="$(echo "$MISSING" | wc -l | tr -d ' ')" say "$COUNT missing from Drive; copying up to $MAX_PER_RUN" n=0 @@ -119,7 +120,16 @@ else [ -z "$f" ] && continue n=$((n+1)); [ $n -gt $MAX_PER_RUN ] && { say "· stopping at $MAX_PER_RUN this run"; break; } TMP="$(mktemp -t checklist-backup)"; OUT="$(mktemp -t checklist-backup-json)" - run_remote "cat $REMOTE_DIR/$f" > "$TMP" + # Retried, because the fetch is the fragile part. Each of these files is ~275 KB and it + # comes back through an expect pseudo-terminal, which does line discipline and has finite + # buffers — so a capture occasionally arrives truncated and fails to parse. That is a + # transport hiccup, not a corrupt backup: every file that failed this way between + # 2026-09-29 and 2026-10-07 copied cleanly on a later night. Retrying in the same run + # turns a nightly email into a non-event. + ok=0 + for try in 1 2 3; do + : > "$TMP" + run_remote "cat $REMOTE_DIR/$f" > "$TMP" # expect echoes its own `spawn …` banner and the password prompt onto stdout before the # file's contents, so the capture is never pure JSON. Slice from the first { to the last } # and only then validate — a truncated or half-read file that *looks* like a backup is @@ -133,12 +143,27 @@ else if(!Array.isArray(j.lists)) process.exit(1); // shape check, not just valid JSON fs.writeFileSync(process.argv[2], JSON.stringify(j)); ' "$TMP" "$OUT" 2>/dev/null; then + ok=1; break + fi + [ "$try" -lt 3 ] && say " · $f — incomplete capture (attempt $try), retrying" + sleep 3 + done + + if [ "$ok" = 1 ]; then mv "$OUT" "$DRIVE_DIR/$f"; rm -f "$TMP" say " ✓ $f ($(du -k "$DRIVE_DIR/$f" | cut -f1) KB)" else rm -f "$TMP" "$OUT" FAILED=$((FAILED+1)) - say " ✗ $f — no valid checklist JSON in the response, skipped" + say " ✗ $f — no valid checklist JSON after 3 attempts, skipped" + # Severity depends on the file's AGE, not on this run. Today's backup failing is a + # retry tomorrow; one still missing days later means the retries are not working. + FDATE="$(printf '%s' "$f" | sed -E 's/^checklists-([0-9]{4}-[0-9]{2}-[0-9]{2})\.json$/\1/')" + FEPOCH="$(date -j -f "%Y-%m-%d" "$FDATE" +%s 2>/dev/null || echo 0)" + if [ "$FEPOCH" -gt 0 ]; then + FAGE=$(( ( $(date +%s) - FEPOCH ) / 86400 )) + [ "$FAGE" -gt "$STALE_DAYS" ] && OVERDUE="$OVERDUE$f ($FAGE days old)"$'\n' + fi fi done <<< "$MISSING" fi @@ -171,10 +196,18 @@ say "✓ done — $TOTAL copies off-site, newest ${NEWEST:-none}" # Some files were fetched but were not usable. The run "succeeded" in the sense that it did # not crash, which is exactly the kind of success that hides a broken backup. -if [ "${FAILED:-0}" -gt 0 ]; then - alert "$FAILED backup file(s) could not be copied" \ - "The run finished, but $FAILED file(s) came back without valid checklist JSON and -were skipped. See the log for which ones." +# Only files that are genuinely overdue raise mail. A fetch that fails tonight and succeeds +# tomorrow is noise, and nightly noise is how an alert stops being read — which is the whole +# failure mode this script exists to prevent. +if [ -n "${OVERDUE:-}" ]; then + alert "backup file(s) still missing off-site after several days" \ + "These NAS backups have failed to copy for more than $STALE_DAYS days, so the +retries are not recovering on their own: + +$OVERDUE +Log: $LOG" +elif [ "${FAILED:-0}" -gt 0 ]; then + say "($FAILED file(s) failed this run but are recent — they will be retried tomorrow, not alerting)" fi # The check that would have caught both of the incidents in the header comment. Every