Waters ChurchandClaude Opus 5
cf3c496c17
Each nightly backup is ~275 KB and was pulled with `cat` through an expect
pseudo-terminal, then recovered by slicing from the first { to the last } of the
transcript. A pty does line discipline and has finite buffers — it is not a file
transfer — so captures arrived truncated often enough to fail the JSON check most
nights between 29 September and 8 October, emailing each time. No backup was ever
lost; every file copied on a later attempt.
The ingest Mac now has a key authorised on the NAS, so there is no password prompt
to automate and the file comes over scp, byte for byte. expect is gone from this
script, and so is the NAS password: nothing here reads the keychain any more,
which also removes the reason it could not be run by hand over SSH.
The JSON check stays, but it only verifies now — there is no transcript to repair.
A backup that is present but unreadable is still worse than one that is obviously
missing.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
228 lines
11 KiB
Bash
Executable File
228 lines
11 KiB
Bash
Executable File
#!/bin/bash
|
|
# backup-to-drive.sh — copy the checklist's nightly backups off the NAS into Google Drive.
|
|
#
|
|
# WHY THIS EXISTS
|
|
# The NAS writes data/backups/checklists-<date>.json every night at 03:30 and keeps 30.
|
|
# That is a good backup of the data and a poor backup of the building: if the NAS dies,
|
|
# every one of those 30 files dies with it. The Google Drive folder was meant to be the
|
|
# off-site copy, but it was hand-made — and on 2026-09-16 it was discovered to have
|
|
# stopped in July, ten weeks earlier, with nobody noticing. This closes that loop.
|
|
#
|
|
# Run by a LaunchAgent at 04:15 daily (after the NAS job). Also safe to run by hand.
|
|
#
|
|
# WHAT IT DOES NOT DO
|
|
# It never writes to the NAS and never deletes anything there. The only thing it removes
|
|
# is old copies in Drive beyond KEEP, and only ones matching the backup naming pattern.
|
|
|
|
set -uo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")" && pwd)"
|
|
REMOTE_DIR="/volume2/docker/recurring-checklist/data/backups"
|
|
NAS_HOST="10.1.1.5"
|
|
NAS_USER="Christian"
|
|
LOG="$HOME/Library/Logs/checklist-drive-backup.log"
|
|
MAX_PER_RUN=10 # so a first run with a big gap can't hang for ages; it catches up next time
|
|
STALE_DAYS=2 # newest off-site copy older than this => alert, even on a "clean" run
|
|
|
|
mkdir -p "$(dirname "$LOG")"
|
|
log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" >> "$LOG"; }
|
|
say() { log "$*"; echo "$*"; }
|
|
|
|
# Send an email and stop. The log alone was never enough: this script was written in
|
|
# September 2026 precisely because the Drive copies had gone ten weeks stale unnoticed —
|
|
# and then it failed silently for another nine days itself, because a wrong path only ever
|
|
# printed into a logfile. A failure that nobody is told about is the same as no backup.
|
|
# A failed alert must not mask the original problem, so alert.js's own exit code is ignored.
|
|
alert() {
|
|
say "$1"
|
|
[ -n "${2:-}" ] && say "$2"
|
|
node "$ROOT/scripts/alert.js" "Checklist off-site backup: $1" "${2:-}$(printf '\n\nLog: %s' "$LOG")" >> "$LOG" 2>&1 || \
|
|
log " (could not send the alert email — see above)"
|
|
}
|
|
die() { alert "$1" "${2:-}"; exit 1; }
|
|
|
|
say "── run start ──"
|
|
|
|
# Where Drive is mounted is not stable, and getting it wrong is silent: the guard below
|
|
# just exits, so a stale path looks exactly like a healthy no-op from the outside. Two
|
|
# things have moved since this was written — the folder lost its "Backups" level, and the
|
|
# laptop mounts christian@ where the Studio mounts media@ — so probe instead of hard-coding.
|
|
DRIVE_DIR=""
|
|
for acct in "$HOME/Library/CloudStorage"/GoogleDrive-*; do
|
|
[ -d "$acct" ] || continue
|
|
for sub in \
|
|
"Shared drives/Waters Creative/Apps/Checklist Backups" \
|
|
"Shared drives/Waters Creative/Apps/Backups/Checklist Backups"; do
|
|
if [ -d "$acct/$sub" ]; then DRIVE_DIR="$acct/$sub"; break 2; fi
|
|
done
|
|
done
|
|
|
|
# Key-based SSH. This used to pull each file with `cat` through an expect pseudo-terminal,
|
|
# which is how a ~275 KB JSON arrived truncated often enough to email a false alarm most
|
|
# nights between 29 September and 8 October 2026. A pty does line discipline and has finite
|
|
# buffers; it is not a file transfer. With a key there is no password prompt to automate, so
|
|
# the file can come over scp and be byte-exact.
|
|
#
|
|
# The key lives on this machine only and is authorised on the NAS for this user. There is no
|
|
# password here any more, and nothing to read out of the keychain.
|
|
SSH_KEY="$HOME/.ssh/id_ed25519_checklist_nas"
|
|
SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -i $SSH_KEY"
|
|
|
|
if [ ! -f "$SSH_KEY" ]; then
|
|
die "no SSH key for the NAS at $SSH_KEY" \
|
|
"Generate one and authorise it on the NAS:
|
|
ssh-keygen -t ed25519 -f $SSH_KEY -N ''
|
|
ssh-copy-id -i $SSH_KEY $NAS_USER@$NAS_HOST"
|
|
fi
|
|
if ! ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" true 2>/dev/null; then
|
|
die "cannot reach the NAS over SSH" \
|
|
"Key auth to $NAS_USER@$NAS_HOST failed. Either the NAS is down, or the key at
|
|
$SSH_KEY is no longer in its authorized_keys."
|
|
fi
|
|
|
|
# Google Drive streams files on demand; if the app is signed out or not running the path
|
|
# simply is not there. Writing to a path Drive is not watching would look like it worked and
|
|
# sync nothing, so refuse rather than pretend.
|
|
if [ -z "$DRIVE_DIR" ]; then
|
|
die "cannot find the Drive backup folder" \
|
|
"Looked under every ~/Library/CloudStorage/GoogleDrive-* account for
|
|
Shared drives/Waters Creative/Apps/Checklist Backups
|
|
and found none. Is Google Drive for Desktop running and signed in, and does that
|
|
folder still exist under that name?"
|
|
fi
|
|
say "Drive folder: $DRIVE_DIR"
|
|
|
|
run_remote() { # $1 = remote command
|
|
ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" "$1"
|
|
}
|
|
|
|
# 1. What does the NAS have?
|
|
REMOTE_LIST="$(run_remote "ls -1 $REMOTE_DIR/checklists-*.json 2>/dev/null | xargs -n1 basename" \
|
|
| grep -E '^checklists-[0-9]{4}-[0-9]{2}-[0-9]{2}\.json$' | sort)"
|
|
if [ -z "$REMOTE_LIST" ]; then
|
|
die "no backups found on the NAS, or SSH failed" \
|
|
"Expected $REMOTE_DIR/checklists-<date>.json on $NAS_HOST. Either the NAS's own
|
|
nightly job has stopped writing them, or this machine can no longer reach the NAS."
|
|
fi
|
|
say "NAS has $(echo "$REMOTE_LIST" | wc -l | tr -d ' ') backup(s)"
|
|
|
|
# 2. Which are missing from Drive?
|
|
MISSING=""
|
|
while IFS= read -r f; do
|
|
[ -s "$DRIVE_DIR/$f" ] || MISSING="$MISSING$f"$'\n'
|
|
done <<< "$REMOTE_LIST"
|
|
MISSING="$(printf '%s' "$MISSING" | sed '/^$/d')"
|
|
|
|
if [ -z "$MISSING" ]; then
|
|
say "✓ Drive already has every NAS backup — nothing to do"
|
|
else
|
|
FAILED=0
|
|
OVERDUE=""
|
|
COUNT="$(echo "$MISSING" | wc -l | tr -d ' ')"
|
|
say "$COUNT missing from Drive; copying up to $MAX_PER_RUN"
|
|
n=0
|
|
while IFS= read -r f; do
|
|
[ -z "$f" ] && continue
|
|
n=$((n+1)); [ $n -gt $MAX_PER_RUN ] && { say "· stopping at $MAX_PER_RUN this run"; break; }
|
|
TMP="$(mktemp -t checklist-backup)"; OUT="$(mktemp -t checklist-backup-json)"
|
|
# Still retried, though the reason it used to be needed is gone. The truncation that
|
|
# failed a file most nights between 29 September and 7 October 2026 came from pulling it
|
|
# through a pty; scp cannot truncate silently. A network blip mid-copy is still possible,
|
|
# and retrying costs a few seconds against an email nobody needed.
|
|
ok=0
|
|
for try in 1 2 3; do
|
|
: > "$TMP"
|
|
scp -q $SSH_OPTS "$NAS_USER@$NAS_HOST:$REMOTE_DIR/$f" "$TMP" 2>/dev/null || true
|
|
# scp delivers the file byte for byte, so there is nothing to slice out of a transcript
|
|
# any more. The check stays because a backup that is present but unreadable is worse
|
|
# than one that is obviously missing — it just verifies rather than repairs now.
|
|
if node -e '
|
|
const fs=require("fs");
|
|
const j=JSON.parse(fs.readFileSync(process.argv[1],"utf8")); // throws => non-zero
|
|
if(!Array.isArray(j.lists)) process.exit(1); // shape, not just syntax
|
|
fs.copyFileSync(process.argv[1], process.argv[2]);
|
|
' "$TMP" "$OUT" 2>/dev/null; then
|
|
ok=1; break
|
|
fi
|
|
[ "$try" -lt 3 ] && say " · $f — incomplete capture (attempt $try), retrying"
|
|
sleep 3
|
|
done
|
|
|
|
if [ "$ok" = 1 ]; then
|
|
mv "$OUT" "$DRIVE_DIR/$f"; rm -f "$TMP"
|
|
say " ✓ $f ($(du -k "$DRIVE_DIR/$f" | cut -f1) KB)"
|
|
else
|
|
rm -f "$TMP" "$OUT"
|
|
FAILED=$((FAILED+1))
|
|
say " ✗ $f — no valid checklist JSON after 3 attempts, skipped"
|
|
# Severity depends on the file's AGE, not on this run. Today's backup failing is a
|
|
# retry tomorrow; one still missing days later means the retries are not working.
|
|
FDATE="$(printf '%s' "$f" | sed -E 's/^checklists-([0-9]{4}-[0-9]{2}-[0-9]{2})\.json$/\1/')"
|
|
FEPOCH="$(date -j -f "%Y-%m-%d" "$FDATE" +%s 2>/dev/null || echo 0)"
|
|
if [ "$FEPOCH" -gt 0 ]; then
|
|
FAGE=$(( ( $(date +%s) - FEPOCH ) / 86400 ))
|
|
[ "$FAGE" -gt "$STALE_DAYS" ] && OVERDUE="$OVERDUE$f ($FAGE days old)"$'\n'
|
|
fi
|
|
fi
|
|
done <<< "$MISSING"
|
|
fi
|
|
|
|
# 3. Report only. This script DELIBERATELY DELETES NOTHING in Drive.
|
|
#
|
|
# An earlier version trimmed Drive to the newest 30 to mirror the NAS, and promptly destroyed
|
|
# the June snapshots — which existed ONLY in Drive, because the NAS had already rotated them
|
|
# out. That is backwards: the whole value of the off-site copy is that it outlives the NAS's
|
|
# 30-day window. Each file is ~260 KB, so a decade of them is under a gigabyte. If it ever
|
|
# genuinely needs trimming, do it by hand, and never below what the NAS still holds.
|
|
# Reporting and freshness.
|
|
#
|
|
# This deliberately does NOT rely on listing the Drive folder. Under launchd, macOS lets a
|
|
# named path be stat'd but blocks enumerating ~/Library/CloudStorage, so a glob quietly
|
|
# expands to nothing — which on 2026-09-24 made this script report "0 copies off-site" and
|
|
# email an alert while 57 files sat there. Checking explicit filenames works in both
|
|
# contexts, so freshness is derived that way and enumeration is treated as a bonus.
|
|
NEWEST=""
|
|
for i in $(seq 0 60); do
|
|
d="$(date -v-${i}d +%Y-%m-%d 2>/dev/null)" || break
|
|
if [ -s "$DRIVE_DIR/checklists-$d.json" ]; then NEWEST="checklists-$d.json"; NEWEST_DATE="$d"; AGE_DAYS=$i; break; fi
|
|
done
|
|
|
|
# Enumeration is allowed when run by hand; when it is blocked the count is unknown rather
|
|
# than zero, and an unknown count is never treated as a missing backup.
|
|
TOTAL="$(ls -1 "$DRIVE_DIR"/checklists-????-??-??.json 2>/dev/null | wc -l | tr -d ' ')"
|
|
if [ "$TOTAL" = "0" ]; then TOTAL="unknown (cannot list the folder from here)"; fi
|
|
say "✓ done — $TOTAL copies off-site, newest ${NEWEST:-none}"
|
|
|
|
# Some files were fetched but were not usable. The run "succeeded" in the sense that it did
|
|
# not crash, which is exactly the kind of success that hides a broken backup.
|
|
# Only files that are genuinely overdue raise mail. A fetch that fails tonight and succeeds
|
|
# tomorrow is noise, and nightly noise is how an alert stops being read — which is the whole
|
|
# failure mode this script exists to prevent.
|
|
if [ -n "${OVERDUE:-}" ]; then
|
|
alert "backup file(s) still missing off-site after several days" \
|
|
"These NAS backups have failed to copy for more than $STALE_DAYS days, so the
|
|
retries are not recovering on their own:
|
|
|
|
$OVERDUE
|
|
Log: $LOG"
|
|
elif [ "${FAILED:-0}" -gt 0 ]; then
|
|
say "($FAILED file(s) failed this run but are recent — they will be retried tomorrow, not alerting)"
|
|
fi
|
|
|
|
# The check that would have caught both of the incidents in the header comment. Every
|
|
# failure path above assumes this script RAN; if the Mac is asleep or the LaunchAgent is
|
|
# unloaded, nothing fires at all.
|
|
if [ -z "$NEWEST" ]; then
|
|
alert "no recent off-site copy" \
|
|
"No checklists-<date>.json for any of the last 60 days was found in
|
|
$DRIVE_DIR
|
|
Either copying has stopped, or this machine cannot read that folder."
|
|
elif [ "$AGE_DAYS" -gt "$STALE_DAYS" ]; then
|
|
alert "off-site copy is $AGE_DAYS days old" \
|
|
"Newest file off-site is $NEWEST ($AGE_DAYS days old); more than $STALE_DAYS days
|
|
means the chain from the NAS to Drive is not working, even though this run did
|
|
not error."
|
|
else
|
|
say "freshness OK — newest off-site copy is $AGE_DAYS day(s) old"
|
|
fi
|