recurring-checklist/backup-to-drive.sh

#!/bin/bash
# backup-to-drive.sh — copy the checklist's nightly backups off the NAS into Google Drive.
#
# WHY THIS EXISTS
#   The NAS writes data/backups/checklists-<date>.json every night at 03:30 and keeps 30.
#   That is a good backup of the data and a poor backup of the building: if the NAS dies,
#   every one of those 30 files dies with it. The Google Drive folder was meant to be the
#   off-site copy, but it was hand-made — and on 2026-09-16 it was discovered to have
#   stopped in July, ten weeks earlier, with nobody noticing. This closes that loop.
#
#   Run by a LaunchAgent at 04:15 daily (after the NAS job). Also safe to run by hand.
#
# WHAT IT DOES NOT DO
#   It never writes to the NAS and never deletes anything there. The only thing it removes
#   is old copies in Drive beyond KEEP, and only ones matching the backup naming pattern.

set -uo pipefail

ROOT="$(cd "$(dirname "$0")" && pwd)"
REMOTE_DIR="/volume2/docker/recurring-checklist/data/backups"
NAS_HOST="10.1.1.5"
NAS_USER="Christian"
LOG="$HOME/Library/Logs/checklist-drive-backup.log"
MAX_PER_RUN=10       # so a first run with a big gap can't hang for ages; it catches up next time
STALE_DAYS=2         # newest off-site copy older than this => alert, even on a "clean" run

mkdir -p "$(dirname "$LOG")"
log() { printf '%s  %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" >> "$LOG"; }
say() { log "$*"; echo "$*"; }

# Send an email and stop. The log alone was never enough: this script was written in
# September 2026 precisely because the Drive copies had gone ten weeks stale unnoticed —
# and then it failed silently for another nine days itself, because a wrong path only ever
# printed into a logfile. A failure that nobody is told about is the same as no backup.
# A failed alert must not mask the original problem, so alert.js's own exit code is ignored.
alert() {
  say "$1"
  [ -n "${2:-}" ] && say "$2"
  node "$ROOT/scripts/alert.js" "Checklist off-site backup: $1" "${2:-}$(printf '\n\nLog: %s' "$LOG")" >> "$LOG" 2>&1 || \
    log "  (could not send the alert email — see above)"
}
die() { alert "$1" "${2:-}"; exit 1; }

say "── run start ──"

# Where Drive is mounted is not stable, and getting it wrong is silent: the guard below
# just exits, so a stale path looks exactly like a healthy no-op from the outside. Two
# things have moved since this was written — the folder lost its "Backups" level, and the
# laptop mounts christian@ where the Studio mounts media@ — so probe instead of hard-coding.
DRIVE_DIR=""
for acct in "$HOME/Library/CloudStorage"/GoogleDrive-*; do
  [ -d "$acct" ] || continue
  for sub in \
    "Shared drives/Waters Creative/Apps/Checklist Backups" \
    "Shared drives/Waters Creative/Apps/Backups/Checklist Backups"; do
    if [ -d "$acct/$sub" ]; then DRIVE_DIR="$acct/$sub"; break 2; fi
  done
done

# Key-based SSH. This used to pull each file with `cat` through an expect pseudo-terminal,
# which is how a ~275 KB JSON arrived truncated often enough to email a false alarm most
# nights between 29 September and 8 October 2026. A pty does line discipline and has finite
# buffers; it is not a file transfer. With a key there is no password prompt to automate, so
# the file can be streamed over a plain ssh session and be byte-exact.
#
# The key lives on this machine only and is authorised on the NAS for this user. There is no
# password here any more, and nothing to read out of the keychain.
SSH_KEY="$HOME/.ssh/id_ed25519_checklist_nas"
SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -i $SSH_KEY"

if [ ! -f "$SSH_KEY" ]; then
  die "tonight's backup did not run — the key for the NAS is missing" \
      "Nothing was copied off-site tonight. Previous backups are untouched and safe.

This machine signs in to the NAS with a key file, and that file is gone from
  $SSH_KEY

To fix it, on this Mac:
  ssh-keygen -t ed25519 -f $SSH_KEY -N ''
  ssh-copy-id -i $SSH_KEY $NAS_USER@$NAS_HOST"
fi
if ! ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" true 2>/dev/null; then
  die "tonight's backup did not run — could not reach the NAS" \
      "Nothing was copied off-site tonight. Previous backups are untouched and safe.

Either the NAS is off or off the network, or this machine's key is no longer
accepted by it. Worth checking the NAS is up first; that is the usual cause."
fi

# Google Drive streams files on demand; if the app is signed out or not running the path
# simply is not there. Writing to a path Drive is not watching would look like it worked and
# sync nothing, so refuse rather than pretend.
if [ -z "$DRIVE_DIR" ]; then
  die "tonight's backup did not run — the Google Drive folder is missing" \
      "Nothing was copied off-site tonight. Previous backups are untouched and safe.

The folder it copies into could not be found:
  Shared drives > Waters Creative > Apps > Checklist Backups

Usually this means Google Drive is not running or has signed out on this Mac.
It can also mean the folder was renamed or moved."
fi
say "Drive folder: $DRIVE_DIR"

run_remote() {   # $1 = remote command
  ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" "$1"
}

# 1. What does the NAS have?
REMOTE_LIST="$(run_remote "ls -1 $REMOTE_DIR/checklists-*.json 2>/dev/null | xargs -n1 basename" \
              | grep -E '^checklists-[0-9]{4}-[0-9]{2}-[0-9]{2}\.json$' | sort)"
if [ -z "$REMOTE_LIST" ]; then
  die "the NAS has no backups to copy" \
      "Copies already in Google Drive are untouched and safe, but no new ones were found.

The NAS writes a backup of the checklists every night at 03:30, and none are
there. That is a problem on the NAS itself rather than with this copy job — the
checklist app may have stopped writing them."
fi
say "NAS has $(echo "$REMOTE_LIST" | wc -l | tr -d ' ') backup(s)"

# 2. Which are missing from Drive?
MISSING=""
while IFS= read -r f; do
  [ -s "$DRIVE_DIR/$f" ] || MISSING="$MISSING$f"$'\n'
done <<< "$REMOTE_LIST"
MISSING="$(printf '%s' "$MISSING" | sed '/^$/d')"

if [ -z "$MISSING" ]; then
  say "✓ Drive already has every NAS backup — nothing to do"
else
  FAILED=0
  OVERDUE=""
  COUNT="$(echo "$MISSING" | wc -l | tr -d ' ')"
  say "$COUNT missing from Drive; copying up to $MAX_PER_RUN"
  n=0
  while IFS= read -r f; do
    [ -z "$f" ] && continue
    n=$((n+1)); [ $n -gt $MAX_PER_RUN ] && { say "· stopping at $MAX_PER_RUN this run"; break; }
    TMP="$(mktemp -t checklist-backup)"; OUT="$(mktemp -t checklist-backup-json)"
    # Still retried, though the reason it used to be needed is gone. The truncation that
    # failed a file most nights between 29 September and 7 October 2026 came from pulling it
    # through a pty; scp cannot truncate silently. A network blip mid-copy is still possible,
    # and retrying costs a few seconds against an email nobody needed.
    ok=0
    for try in 1 2 3; do
      : > "$TMP"
      # `ssh cat`, not scp: this NAS has no SFTP subsystem, and modern scp speaks SFTP, so
      # scp fails on every file with "subsystem request failed on channel 0". `scp -O` would
      # work too but the legacy protocol is on its way out of OpenSSH. With key auth there
      # is no pty in the way, so cat is byte-exact — verified against the NAS's own byte
      # count. It was only ever the pty that corrupted this, never cat.
      ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" "cat $REMOTE_DIR/$f" > "$TMP" 2>/dev/null || true
    # scp delivers the file byte for byte, so there is nothing to slice out of a transcript
    # any more. The check stays because a backup that is present but unreadable is worse
    # than one that is obviously missing — it just verifies rather than repairs now.
    if node -e '
        const fs=require("fs");
        const j=JSON.parse(fs.readFileSync(process.argv[1],"utf8"));  // throws => non-zero
        if(!Array.isArray(j.lists)) process.exit(1);                  // shape, not just syntax
        fs.copyFileSync(process.argv[1], process.argv[2]);
      ' "$TMP" "$OUT" 2>/dev/null; then
        ok=1; break
      fi
      [ "$try" -lt 3 ] && say "  · $f — incomplete capture (attempt $try), retrying"
      sleep 3
    done

    if [ "$ok" = 1 ]; then
      mv "$OUT" "$DRIVE_DIR/$f"; rm -f "$TMP"
      say "  ✓ $f ($(du -k "$DRIVE_DIR/$f" | cut -f1) KB)"
    else
      rm -f "$TMP" "$OUT"
      FAILED=$((FAILED+1))
      say "  ✗ $f — no valid checklist JSON after 3 attempts, skipped"
      # Severity depends on the file's AGE, not on this run. Today's backup failing is a
      # retry tomorrow; one still missing days later means the retries are not working.
      FDATE="$(printf '%s' "$f" | sed -E 's/^checklists-([0-9]{4}-[0-9]{2}-[0-9]{2})\.json$/\1/')"
      FEPOCH="$(date -j -f "%Y-%m-%d" "$FDATE" +%s 2>/dev/null || echo 0)"
      if [ "$FEPOCH" -gt 0 ]; then
        FAGE=$(( ( $(date +%s) - FEPOCH ) / 86400 ))
        [ "$FAGE" -gt "$STALE_DAYS" ] && OVERDUE="$OVERDUE$f ($FAGE days old)"$'\n'
      fi
    fi
  done <<< "$MISSING"
fi

# 3. Report only. This script DELIBERATELY DELETES NOTHING in Drive.
#
# An earlier version trimmed Drive to the newest 30 to mirror the NAS, and promptly destroyed
# the June snapshots — which existed ONLY in Drive, because the NAS had already rotated them
# out. That is backwards: the whole value of the off-site copy is that it outlives the NAS's
# 30-day window. Each file is ~260 KB, so a decade of them is under a gigabyte. If it ever
# genuinely needs trimming, do it by hand, and never below what the NAS still holds.
# Reporting and freshness.
#
# This deliberately does NOT rely on listing the Drive folder. Under launchd, macOS lets a
# named path be stat'd but blocks enumerating ~/Library/CloudStorage, so a glob quietly
# expands to nothing — which on 2026-09-24 made this script report "0 copies off-site" and
# email an alert while 57 files sat there. Checking explicit filenames works in both
# contexts, so freshness is derived that way and enumeration is treated as a bonus.
NEWEST=""
for i in $(seq 0 60); do
  d="$(date -v-${i}d +%Y-%m-%d 2>/dev/null)" || break
  if [ -s "$DRIVE_DIR/checklists-$d.json" ]; then NEWEST="checklists-$d.json"; NEWEST_DATE="$d"; AGE_DAYS=$i; break; fi
done

# Enumeration is allowed when run by hand; when it is blocked the count is unknown rather
# than zero, and an unknown count is never treated as a missing backup.
TOTAL="$(ls -1 "$DRIVE_DIR"/checklists-????-??-??.json 2>/dev/null | wc -l | tr -d ' ')"
if [ "$TOTAL" = "0" ]; then TOTAL="unknown (cannot list the folder from here)"; fi
say "✓ done — $TOTAL copies off-site, newest ${NEWEST:-none}"

# Some files were fetched but were not usable. The run "succeeded" in the sense that it did
# not crash, which is exactly the kind of success that hides a broken backup.
# Only files that are genuinely overdue raise mail. A fetch that fails tonight and succeeds
# tomorrow is noise, and nightly noise is how an alert stops being read — which is the whole
# failure mode this script exists to prevent.
if [ -n "${OVERDUE:-}" ]; then
  alert "some backups are not reaching Google Drive" \
        "The website and the checklists themselves are fine. This is about the off-site
copies only.

These have failed to copy for more than $STALE_DAYS days, so the automatic retries
are not fixing it:

$OVERDUE
Everything else copied normally. Worth a look when convenient — it is not urgent
unless it keeps growing."
elif [ "${FAILED:-0}" -gt 0 ]; then
  say "($FAILED file(s) failed this run but are recent — they will be retried tomorrow, not alerting)"
fi

# The check that would have caught both of the incidents in the header comment. Every
# failure path above assumes this script RAN; if the Mac is asleep or the LaunchAgent is
# unloaded, nothing fires at all.
if [ -z "$NEWEST" ]; then
  alert "no recent backup in Google Drive" \
        "The checklists themselves are fine — this is about the off-site copies.

Nothing from the last 60 days is in the Drive folder. Either the copying has
stopped, or this Mac can no longer read that folder."
elif [ "$AGE_DAYS" -gt "$STALE_DAYS" ]; then
  alert "backups have stopped reaching Google Drive" \
        "The checklists themselves are fine — this is about the off-site copies.

The newest one in Drive is $AGE_DAYS days old ($NEWEST). Tonight's run did not
report an error, which means something is failing quietly rather than loudly."
else
  say "freshness OK — newest off-site copy is $AGE_DAYS day(s) old"
fi