recurring-checklist/backup-to-drive.sh

#!/bin/bash
# backup-to-drive.sh — copy the checklist's nightly backups off the NAS into Google Drive.
#
# WHY THIS EXISTS
#   The NAS writes data/backups/checklists-<date>.json every night at 03:30 and keeps 30.
#   That is a good backup of the data and a poor backup of the building: if the NAS dies,
#   every one of those 30 files dies with it. The Google Drive folder was meant to be the
#   off-site copy, but it was hand-made — and on 2026-09-16 it was discovered to have
#   stopped in July, ten weeks earlier, with nobody noticing. This closes that loop.
#
#   Run by a LaunchAgent at 04:15 daily (after the NAS job). Also safe to run by hand.
#
# WHAT IT DOES NOT DO
#   It never writes to the NAS and never deletes anything there. The only thing it removes
#   is old copies in Drive beyond KEEP, and only ones matching the backup naming pattern.

set -uo pipefail

ROOT="$(cd "$(dirname "$0")" && pwd)"
REMOTE_DIR="/volume2/docker/recurring-checklist/data/backups"
NAS_HOST="10.1.1.5"
NAS_USER="Christian"
LOG="$HOME/Library/Logs/checklist-drive-backup.log"
MAX_PER_RUN=10       # so a first run with a big gap can't hang for ages; it catches up next time
STALE_DAYS=2         # newest off-site copy older than this => alert, even on a "clean" run

mkdir -p "$(dirname "$LOG")"
log() { printf '%s  %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" >> "$LOG"; }
say() { log "$*"; echo "$*"; }

# Send an email and stop. The log alone was never enough: this script was written in
# September 2026 precisely because the Drive copies had gone ten weeks stale unnoticed —
# and then it failed silently for another nine days itself, because a wrong path only ever
# printed into a logfile. A failure that nobody is told about is the same as no backup.
# A failed alert must not mask the original problem, so alert.js's own exit code is ignored.
alert() {
  say "$1"
  [ -n "${2:-}" ] && say "$2"
  node "$ROOT/scripts/alert.js" "Checklist off-site backup: $1" "${2:-}$(printf '\n\nLog: %s' "$LOG")" >> "$LOG" 2>&1 || \
    log "  (could not send the alert email — see above)"
}
die() { alert "$1" "${2:-}"; exit 1; }

say "── run start ──"

# Where Drive is mounted is not stable, and getting it wrong is silent: the guard below
# just exits, so a stale path looks exactly like a healthy no-op from the outside. Two
# things have moved since this was written — the folder lost its "Backups" level, and the
# laptop mounts christian@ where the Studio mounts media@ — so probe instead of hard-coding.
DRIVE_DIR=""
for acct in "$HOME/Library/CloudStorage"/GoogleDrive-*; do
  [ -d "$acct" ] || continue
  for sub in \
    "Shared drives/Waters Creative/Apps/Checklist Backups" \
    "Shared drives/Waters Creative/Apps/Backups/Checklist Backups"; do
    if [ -d "$acct/$sub" ]; then DRIVE_DIR="$acct/$sub"; break 2; fi
  done
done

# Key-based SSH. This used to pull each file with `cat` through an expect pseudo-terminal,
# which is how a ~275 KB JSON arrived truncated often enough to email a false alarm most
# nights between 29 September and 8 October 2026. A pty does line discipline and has finite
# buffers; it is not a file transfer. With a key there is no password prompt to automate, so
# the file can be streamed over a plain ssh session and be byte-exact.
#
# The key lives on this machine only and is authorised on the NAS for this user. There is no
# password here any more, and nothing to read out of the keychain.
SSH_KEY="$HOME/.ssh/id_ed25519_checklist_nas"
SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -i $SSH_KEY"

if [ ! -f "$SSH_KEY" ]; then
  die "no SSH key for the NAS at $SSH_KEY" \
      "Generate one and authorise it on the NAS:
  ssh-keygen -t ed25519 -f $SSH_KEY -N ''
  ssh-copy-id -i $SSH_KEY $NAS_USER@$NAS_HOST"
fi
if ! ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" true 2>/dev/null; then
  die "cannot reach the NAS over SSH" \
      "Key auth to $NAS_USER@$NAS_HOST failed. Either the NAS is down, or the key at
$SSH_KEY is no longer in its authorized_keys."
fi

# Google Drive streams files on demand; if the app is signed out or not running the path
# simply is not there. Writing to a path Drive is not watching would look like it worked and
# sync nothing, so refuse rather than pretend.
if [ -z "$DRIVE_DIR" ]; then
  die "cannot find the Drive backup folder" \
      "Looked under every ~/Library/CloudStorage/GoogleDrive-* account for
  Shared drives/Waters Creative/Apps/Checklist Backups
and found none. Is Google Drive for Desktop running and signed in, and does that
folder still exist under that name?"
fi
say "Drive folder: $DRIVE_DIR"

run_remote() {   # $1 = remote command
  ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" "$1"
}

# 1. What does the NAS have?
REMOTE_LIST="$(run_remote "ls -1 $REMOTE_DIR/checklists-*.json 2>/dev/null | xargs -n1 basename" \
              | grep -E '^checklists-[0-9]{4}-[0-9]{2}-[0-9]{2}\.json$' | sort)"
if [ -z "$REMOTE_LIST" ]; then
  die "no backups found on the NAS, or SSH failed" \
      "Expected $REMOTE_DIR/checklists-<date>.json on $NAS_HOST. Either the NAS's own
nightly job has stopped writing them, or this machine can no longer reach the NAS."
fi
say "NAS has $(echo "$REMOTE_LIST" | wc -l | tr -d ' ') backup(s)"

# 2. Which are missing from Drive?
MISSING=""
while IFS= read -r f; do
  [ -s "$DRIVE_DIR/$f" ] || MISSING="$MISSING$f"$'\n'
done <<< "$REMOTE_LIST"
MISSING="$(printf '%s' "$MISSING" | sed '/^$/d')"

if [ -z "$MISSING" ]; then
  say "✓ Drive already has every NAS backup — nothing to do"
else
  FAILED=0
  OVERDUE=""
  COUNT="$(echo "$MISSING" | wc -l | tr -d ' ')"
  say "$COUNT missing from Drive; copying up to $MAX_PER_RUN"
  n=0
  while IFS= read -r f; do
    [ -z "$f" ] && continue
    n=$((n+1)); [ $n -gt $MAX_PER_RUN ] && { say "· stopping at $MAX_PER_RUN this run"; break; }
    TMP="$(mktemp -t checklist-backup)"; OUT="$(mktemp -t checklist-backup-json)"
    # Still retried, though the reason it used to be needed is gone. The truncation that
    # failed a file most nights between 29 September and 7 October 2026 came from pulling it
    # through a pty; scp cannot truncate silently. A network blip mid-copy is still possible,
    # and retrying costs a few seconds against an email nobody needed.
    ok=0
    for try in 1 2 3; do
      : > "$TMP"
      # `ssh cat`, not scp: this NAS has no SFTP subsystem, and modern scp speaks SFTP, so
      # scp fails on every file with "subsystem request failed on channel 0". `scp -O` would
      # work too but the legacy protocol is on its way out of OpenSSH. With key auth there
      # is no pty in the way, so cat is byte-exact — verified against the NAS's own byte
      # count. It was only ever the pty that corrupted this, never cat.
      ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" "cat $REMOTE_DIR/$f" > "$TMP" 2>/dev/null || true
    # scp delivers the file byte for byte, so there is nothing to slice out of a transcript
    # any more. The check stays because a backup that is present but unreadable is worse
    # than one that is obviously missing — it just verifies rather than repairs now.
    if node -e '
        const fs=require("fs");
        const j=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));  // throws => non-zero
        if(!Array.isArray(j.lists)) process.exit(1);                  // shape, not just syntax
        fs.copyFileSync(process.argv[1], process.argv[2]);
      ' "$TMP" "$OUT" 2>/dev/null; then
        ok=1; break
      fi
      [ "$try" -lt 3 ] && say "  · $f — incomplete capture (attempt $try), retrying"
      sleep 3
    done

    if [ "$ok" = 1 ]; then
      mv "$OUT" "$DRIVE_DIR/$f"; rm -f "$TMP"
      say "  ✓ $f ($(du -k "$DRIVE_DIR/$f" | cut -f1) KB)"
    else
      rm -f "$TMP" "$OUT"
      FAILED=$((FAILED+1))
      say "  ✗ $f — no valid checklist JSON after 3 attempts, skipped"
      # Severity depends on the file's AGE, not on this run. Today's backup failing is a
      # retry tomorrow; one still missing days later means the retries are not working.
      FDATE="$(printf '%s' "$f" | sed -E 's/^checklists-([0-9]{4}-[0-9]{2}-[0-9]{2})\.json$/\1/')"
      FEPOCH="$(date -j -f "%Y-%m-%d" "$FDATE" +%s 2>/dev/null || echo 0)"
      if [ "$FEPOCH" -gt 0 ]; then
        FAGE=$(( ( $(date +%s) - FEPOCH ) / 86400 ))
        [ "$FAGE" -gt "$STALE_DAYS" ] && OVERDUE="$OVERDUE$f ($FAGE days old)"$'\n'
      fi
    fi
  done <<< "$MISSING"
fi

# 3. Report only. This script DELIBERATELY DELETES NOTHING in Drive.
#
# An earlier version trimmed Drive to the newest 30 to mirror the NAS, and promptly destroyed
# the June snapshots — which existed ONLY in Drive, because the NAS had already rotated them
# out. That is backwards: the whole value of the off-site copy is that it outlives the NAS's
# 30-day window. Each file is ~260 KB, so a decade of them is under a gigabyte. If it ever
# genuinely needs trimming, do it by hand, and never below what the NAS still holds.
# Reporting and freshness.
#
# This deliberately does NOT rely on listing the Drive folder. Under launchd, macOS lets a
# named path be stat'd but blocks enumerating ~/Library/CloudStorage, so a glob quietly
# expands to nothing — which on 2026-09-24 made this script report "0 copies off-site" and
# email an alert while 57 files sat there. Checking explicit filenames works in both
# contexts, so freshness is derived that way and enumeration is treated as a bonus.
NEWEST=""
for i in $(seq 0 60); do
  d="$(date -v-${i}d +%Y-%m-%d 2>/dev/null)" || break
  if [ -s "$DRIVE_DIR/checklists-$d.json" ]; then NEWEST="checklists-$d.json"; NEWEST_DATE="$d"; AGE_DAYS=$i; break; fi
done

# Enumeration is allowed when run by hand; when it is blocked the count is unknown rather
# than zero, and an unknown count is never treated as a missing backup.
TOTAL="$(ls -1 "$DRIVE_DIR"/checklists-????-??-??.json 2>/dev/null | wc -l | tr -d ' ')"
if [ "$TOTAL" = "0" ]; then TOTAL="unknown (cannot list the folder from here)"; fi
say "✓ done — $TOTAL copies off-site, newest ${NEWEST:-none}"

# Some files were fetched but were not usable. The run "succeeded" in the sense that it did
# not crash, which is exactly the kind of success that hides a broken backup.
# Only files that are genuinely overdue raise mail. A fetch that fails tonight and succeeds
# tomorrow is noise, and nightly noise is how an alert stops being read — which is the whole
# failure mode this script exists to prevent.
if [ -n "${OVERDUE:-}" ]; then
  alert "backup file(s) still missing off-site after several days" \
        "These NAS backups have failed to copy for more than $STALE_DAYS days, so the
retries are not recovering on their own:

$OVERDUE
Log: $LOG"
elif [ "${FAILED:-0}" -gt 0 ]; then
  say "($FAILED file(s) failed this run but are recent — they will be retried tomorrow, not alerting)"
fi

# The check that would have caught both of the incidents in the header comment. Every
# failure path above assumes this script RAN; if the Mac is asleep or the LaunchAgent is
# unloaded, nothing fires at all.
if [ -z "$NEWEST" ]; then
  alert "no recent off-site copy" \
        "No checklists-<date>.json for any of the last 60 days was found in
$DRIVE_DIR
Either copying has stopped, or this machine cannot read that folder."
elif [ "$AGE_DAYS" -gt "$STALE_DAYS" ]; then
  alert "off-site copy is $AGE_DAYS days old" \
        "Newest file off-site is $NEWEST ($AGE_DAYS days old); more than $STALE_DAYS days
means the chain from the NAS to Drive is not working, even though this run did
not error."
else
  say "freshness OK — newest off-site copy is $AGE_DAYS day(s) old"
fi