Make the off-site backup find Drive, and say so when it cannot

The nightly copy to Drive had been dead since 15 September. DRIVE_DIR pointed at
Apps/Backups/Checklist Backups; the folder is Apps/Checklist Backups, with no
Backups level. The guard for a missing folder did its job and exited — into a
logfile nobody reads, so from the outside a broken backup and a healthy no-op
looked identical. Eight days were missing; this run copied them.

That is the second time the same failure has hidden in the same place. The
header comment already records the first: ten weeks stale in July, found in
September. Correcting the path alone would leave the mechanism that hid it
untouched, so:

- The Drive folder is probed rather than hard-coded, across every mounted
  GoogleDrive-* account and both known folder layouts. The laptop mounts
  christian@ where the Studio mounts media@, and the folder has moved once
  already, so a literal path is the wrong thing to depend on.
- Every failure now emails as well as logs, via scripts/alert.js, which reuses
  the server's own SMTP settings. A failed alert never masks the failure it was
  reporting.
- A run that does not error but leaves the newest off-site copy more than
  STALE_DAYS old now alerts too. That is the check that would have caught both
  incidents; the error paths only fire when the script actually runs.

What this still cannot catch is the script never running at all — a sleeping
Mac or an unloaded LaunchAgent is silent by construction.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backup-to-drive.sh

@@ -17,37 +17,66 @@
 set -uo pipefail
 
 ROOT="$(cd "$(dirname "$0")" && pwd)"
-DRIVE_DIR="$HOME/Library/CloudStorage/GoogleDrive-media@waterschurch.org/Shared drives/Waters Creative/Apps/Backups/Checklist Backups"
 REMOTE_DIR="/volume2/docker/recurring-checklist/data/backups"
 NAS_HOST="10.1.1.5"
 NAS_USER="Christian"
 LOG="$HOME/Library/Logs/checklist-drive-backup.log"
 MAX_PER_RUN=10       # so a first run with a big gap can't hang for ages; it catches up next time
+STALE_DAYS=2         # newest off-site copy older than this => alert, even on a "clean" run
 
 mkdir -p "$(dirname "$LOG")"
 log() { printf '%s  %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" >> "$LOG"; }
 say() { log "$*"; echo "$*"; }
 
+# Send an email and stop. The log alone was never enough: this script was written in
+# September 2026 precisely because the Drive copies had gone ten weeks stale unnoticed —
+# and then it failed silently for another nine days itself, because a wrong path only ever
+# printed into a logfile. A failure that nobody is told about is the same as no backup.
+# A failed alert must not mask the original problem, so alert.js's own exit code is ignored.
+alert() {
+  say "$1"
+  [ -n "${2:-}" ] && say "$2"
+  node "$ROOT/scripts/alert.js" "Checklist off-site backup: $1" "${2:-}$(printf '\n\nLog: %s' "$LOG")" >> "$LOG" 2>&1 || \
+    log "  (could not send the alert email — see above)"
+}
+die() { alert "$1" "${2:-}"; exit 1; }
+
 say "── run start ──"
 
+# Where Drive is mounted is not stable, and getting it wrong is silent: the guard below
+# just exits, so a stale path looks exactly like a healthy no-op from the outside. Two
+# things have moved since this was written — the folder lost its "Backups" level, and the
+# laptop mounts christian@ where the Studio mounts media@ — so probe instead of hard-coding.
+DRIVE_DIR=""
+for acct in "$HOME/Library/CloudStorage"/GoogleDrive-*; do
+  [ -d "$acct" ] || continue
+  for sub in \
+    "Shared drives/Waters Creative/Apps/Checklist Backups" \
+    "Shared drives/Waters Creative/Apps/Backups/Checklist Backups"; do
+    if [ -d "$acct/$sub" ]; then DRIVE_DIR="$acct/$sub"; break 2; fi
+  done
+done
+
 # The SSH password comes from the login keychain — the same entry build-deploy.sh reads, so
 # there is still one copy of the credential to rotate, and it is not in the project at all.
 # Store it with:  security add-generic-password -U -s checklist-nas -a Christian -w
 NAS_PASS="${NAS_PASS:-$(security find-generic-password -s checklist-nas -a Christian -w 2>/dev/null || true)}"
 if [ -z "$NAS_PASS" ]; then
-  say "✗ no NAS password in the keychain (service checklist-nas, account Christian)"
+  die "no NAS password in the keychain" \
-  say "  store it with: security add-generic-password -U -s checklist-nas -a Christian -w"
+      "Store it with: security add-generic-password -U -s checklist-nas -a Christian -w"
-  exit 1
 fi
 
 # Google Drive streams files on demand; if the app is signed out or not running the path
 # simply is not there. Writing to a path Drive is not watching would look like it worked and
 # sync nothing, so refuse rather than pretend.
-if [ ! -d "$DRIVE_DIR" ]; then
+if [ -z "$DRIVE_DIR" ]; then
-  say "✗ Drive folder not mounted: $DRIVE_DIR"
+  die "cannot find the Drive backup folder" \
-  say "  Is Google Drive for Desktop running and signed in as media@waterschurch.org?"
+      "Looked under every ~/Library/CloudStorage/GoogleDrive-* account for
-  exit 1
+  Shared drives/Waters Creative/Apps/Checklist Backups
+and found none. Is Google Drive for Desktop running and signed in, and does that
+folder still exist under that name?"
 fi
+say "Drive folder: $DRIVE_DIR"
 
 run_remote() {   # $1 = remote command; prints stdout, strips the pty's CRs
   local cmd="$1"
@@ -65,7 +94,11 @@ EXP
 # 1. What does the NAS have?
 REMOTE_LIST="$(run_remote "ls -1 $REMOTE_DIR/checklists-*.json 2>/dev/null | xargs -n1 basename" \
               | grep -E '^checklists-[0-9]{4}-[0-9]{2}-[0-9]{2}\.json$' | sort)"
-if [ -z "$REMOTE_LIST" ]; then say "✗ no backups found on the NAS (or SSH failed)"; exit 1; fi
+if [ -z "$REMOTE_LIST" ]; then
+  die "no backups found on the NAS, or SSH failed" \
+      "Expected $REMOTE_DIR/checklists-<date>.json on $NAS_HOST. Either the NAS's own
+nightly job has stopped writing them, or this machine can no longer reach the NAS."
+fi
 say "NAS has $(echo "$REMOTE_LIST" | wc -l | tr -d ' ') backup(s)"
 
 # 2. Which are missing from Drive?
@@ -78,6 +111,7 @@ MISSING="$(printf '%s' "$MISSING" | sed '/^$/d')"
 if [ -z "$MISSING" ]; then
   say "✓ Drive already has every NAS backup — nothing to do"
 else
+  FAILED=0
   COUNT="$(echo "$MISSING" | wc -l | tr -d ' ')"
   say "$COUNT missing from Drive; copying up to $MAX_PER_RUN"
   n=0
@@ -103,6 +137,7 @@ else
       say "  ✓ $f ($(du -k "$DRIVE_DIR/$f" | cut -f1) KB)"
     else
       rm -f "$TMP" "$OUT"
+      FAILED=$((FAILED+1))
       say "  ✗ $f — no valid checklist JSON in the response, skipped"
     fi
   done <<< "$MISSING"
@@ -115,8 +150,38 @@ fi
 # out. That is backwards: the whole value of the off-site copy is that it outlives the NAS's
 # 30-day window. Each file is ~260 KB, so a decade of them is under a gigabyte. If it ever
 # genuinely needs trimming, do it by hand, and never below what the NAS still holds.
-cd "$DRIVE_DIR" || exit 1
+cd "$DRIVE_DIR" || die "cannot enter $DRIVE_DIR"
 TOTAL="$(ls -1 checklists-????-??-??.json 2>/dev/null | wc -l | tr -d ' ')"
 OLDEST="$(ls -1 checklists-????-??-??.json 2>/dev/null | sort | head -1)"
 NEWEST="$(ls -1 checklists-????-??-??.json 2>/dev/null | sort | tail -1)"
 say "✓ done — $TOTAL copies off-site, ${OLDEST:-none} → ${NEWEST:-none}"
+
+# Some files were fetched but were not usable. The run "succeeded" in the sense that it did
+# not crash, which is exactly the kind of success that hides a broken backup.
+if [ "${FAILED:-0}" -gt 0 ]; then
+  alert "$FAILED backup file(s) could not be copied" \
+        "The run finished, but $FAILED file(s) came back without valid checklist JSON and
+were skipped. See the log for which ones."
+fi
+
+# The staleness check is the one that would have caught both of the incidents in the header
+# comment. Every failure path above assumes this script RAN; if the Mac is asleep, or the
+# LaunchAgent is unloaded, nothing above ever fires. This at least turns a quietly ageing
+# off-site copy into mail the next time it does run.
+if [ -z "$NEWEST" ]; then
+  alert "no off-site copies at all" "$DRIVE_DIR contains no checklists-<date>.json files."
+else
+  NEWEST_DATE="$(printf '%s' "$NEWEST" | sed -E 's/^checklists-([0-9]{4}-[0-9]{2}-[0-9]{2})\.json$/\1/')"
+  NEWEST_EPOCH="$(date -j -f "%Y-%m-%d" "$NEWEST_DATE" +%s 2>/dev/null || echo 0)"
+  if [ "$NEWEST_EPOCH" -gt 0 ]; then
+    AGE_DAYS=$(( ( $(date +%s) - NEWEST_EPOCH ) / 86400 ))
+    if [ "$AGE_DAYS" -gt "$STALE_DAYS" ]; then
+      alert "off-site copy is $AGE_DAYS days old" \
+            "Newest file off-site is $NEWEST ($AGE_DAYS days old); anything more than
+$STALE_DAYS days means the chain from the NAS to Drive is not working, even though
+this run did not error."
+    else
+      say "freshness OK — newest off-site copy is $AGE_DAYS day(s) old"
+    fi
+  fi
+fi

scripts/alert.js

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+'use strict';
+// alert.js — send one alert email, reusing the server's own SMTP settings.
+//
+// Exists because backup-to-drive.sh used to report failure only into
+// ~/Library/Logs/checklist-drive-backup.log, where nobody was ever going to look. That is
+// how the Drive copies went ten weeks stale in July 2026, and then — despite the script
+// written to prevent exactly that — another nine days in September 2026.
+//
+// Usage:  node scripts/alert.js "<subject>" "<body>"
+// Exit 0 if the mail went out, 1 otherwise. Callers should NOT fail on a failed alert:
+// the thing being reported matters more than the reporting of it.
+
+const fs = require('fs');
+const path = require('path');
+
+const ROOT = path.resolve(__dirname, '..');
+const ENV_FILE = path.join(ROOT, 'server', '.env');
+
+// Minimal .env reader — the server uses dotenv, but this script has to run from a plain
+// LaunchAgent with no cwd guarantees and no node_modules of its own.
+function readEnv(file) {
+  const out = {};
+  let text;
+  try { text = fs.readFileSync(file, 'utf8'); }
+  catch { return out; }
+  for (const raw of text.split('\n')) {
+    const line = raw.trim();
+    if (!line || line.startsWith('#')) continue;
+    const eq = line.indexOf('=');
+    if (eq < 0) continue;
+    let v = line.slice(eq + 1).trim();
+    if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
+    out[line.slice(0, eq).trim()] = v;
+  }
+  return out;
+}
+
+async function main() {
+  const subject = process.argv[2];
+  const body = process.argv[3] || '';
+  if (!subject) { console.error('alert.js: no subject given'); process.exit(1); }
+
+  const env = readEnv(ENV_FILE);
+  const to = env.REPORT_EMAIL;
+  if (!to) { console.error(`alert.js: no REPORT_EMAIL in ${ENV_FILE}`); process.exit(1); }
+  if (!env.SMTP_HOST || !env.SMTP_USER || !env.SMTP_PASS) {
+    console.error(`alert.js: SMTP is not configured in ${ENV_FILE}`); process.exit(1);
+  }
+
+  // nodemailer lives in server/node_modules; this script has none of its own.
+  let nodemailer;
+  try { nodemailer = require(path.join(ROOT, 'server', 'node_modules', 'nodemailer')); }
+  catch { console.error('alert.js: nodemailer not installed — run `npm install` in server/'); process.exit(1); }
+
+  const transport = nodemailer.createTransport({
+    host: env.SMTP_HOST,
+    port: Number(env.SMTP_PORT || 587),
+    secure: String(env.SMTP_SECURE || 'false') === 'true',
+    auth: { user: env.SMTP_USER, pass: env.SMTP_PASS },
+  });
+
+  await transport.sendMail({
+    from: env.SMTP_FROM || env.SMTP_USER,
+    to,
+    subject,
+    text: `${body}\n\n-- \nSent by scripts/alert.js on ${require('os').hostname()} at ${new Date().toISOString()}\n`,
+  });
+  console.log(`alert.js: sent "${subject}" to ${to}`);
+}
+
+main().catch(err => { console.error('alert.js:', err && err.message ? err.message : err); process.exit(1); });