diff --git a/backup-to-drive.sh b/backup-to-drive.sh index 5dabe1d..3c3acc7 100755 --- a/backup-to-drive.sh +++ b/backup-to-drive.sh @@ -17,37 +17,66 @@ set -uo pipefail ROOT="$(cd "$(dirname "$0")" && pwd)" -DRIVE_DIR="$HOME/Library/CloudStorage/GoogleDrive-media@waterschurch.org/Shared drives/Waters Creative/Apps/Backups/Checklist Backups" REMOTE_DIR="/volume2/docker/recurring-checklist/data/backups" NAS_HOST="10.1.1.5" NAS_USER="Christian" LOG="$HOME/Library/Logs/checklist-drive-backup.log" MAX_PER_RUN=10 # so a first run with a big gap can't hang for ages; it catches up next time +STALE_DAYS=2 # newest off-site copy older than this => alert, even on a "clean" run mkdir -p "$(dirname "$LOG")" log() { printf '%s %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" >> "$LOG"; } say() { log "$*"; echo "$*"; } +# Send an email and stop. The log alone was never enough: this script was written in +# September 2026 precisely because the Drive copies had gone ten weeks stale unnoticed — +# and then it failed silently for another nine days itself, because a wrong path only ever +# printed into a logfile. A failure that nobody is told about is the same as no backup. +# A failed alert must not mask the original problem, so alert.js's own exit code is ignored. +alert() { + say "$1" + [ -n "${2:-}" ] && say "$2" + node "$ROOT/scripts/alert.js" "Checklist off-site backup: $1" "${2:-}$(printf '\n\nLog: %s' "$LOG")" >> "$LOG" 2>&1 || \ + log " (could not send the alert email — see above)" +} +die() { alert "$1" "${2:-}"; exit 1; } + say "── run start ──" +# Where Drive is mounted is not stable, and getting it wrong is silent: the guard below +# just exits, so a stale path looks exactly like a healthy no-op from the outside. Two +# things have moved since this was written — the folder lost its "Backups" level, and the +# laptop mounts christian@ where the Studio mounts media@ — so probe instead of hard-coding. +DRIVE_DIR="" +for acct in "$HOME/Library/CloudStorage"/GoogleDrive-*; do + [ -d "$acct" ] || continue + for sub in \ + "Shared drives/Waters Creative/Apps/Checklist Backups" \ + "Shared drives/Waters Creative/Apps/Backups/Checklist Backups"; do + if [ -d "$acct/$sub" ]; then DRIVE_DIR="$acct/$sub"; break 2; fi + done +done + # The SSH password comes from the login keychain — the same entry build-deploy.sh reads, so # there is still one copy of the credential to rotate, and it is not in the project at all. # Store it with: security add-generic-password -U -s checklist-nas -a Christian -w NAS_PASS="${NAS_PASS:-$(security find-generic-password -s checklist-nas -a Christian -w 2>/dev/null || true)}" if [ -z "$NAS_PASS" ]; then - say "✗ no NAS password in the keychain (service checklist-nas, account Christian)" - say " store it with: security add-generic-password -U -s checklist-nas -a Christian -w" - exit 1 + die "no NAS password in the keychain" \ + "Store it with: security add-generic-password -U -s checklist-nas -a Christian -w" fi # Google Drive streams files on demand; if the app is signed out or not running the path # simply is not there. Writing to a path Drive is not watching would look like it worked and # sync nothing, so refuse rather than pretend. -if [ ! -d "$DRIVE_DIR" ]; then - say "✗ Drive folder not mounted: $DRIVE_DIR" - say " Is Google Drive for Desktop running and signed in as media@waterschurch.org?" - exit 1 +if [ -z "$DRIVE_DIR" ]; then + die "cannot find the Drive backup folder" \ + "Looked under every ~/Library/CloudStorage/GoogleDrive-* account for + Shared drives/Waters Creative/Apps/Checklist Backups +and found none. Is Google Drive for Desktop running and signed in, and does that +folder still exist under that name?" fi +say "Drive folder: $DRIVE_DIR" run_remote() { # $1 = remote command; prints stdout, strips the pty's CRs local cmd="$1" @@ -65,7 +94,11 @@ EXP # 1. What does the NAS have? REMOTE_LIST="$(run_remote "ls -1 $REMOTE_DIR/checklists-*.json 2>/dev/null | xargs -n1 basename" \ | grep -E '^checklists-[0-9]{4}-[0-9]{2}-[0-9]{2}\.json$' | sort)" -if [ -z "$REMOTE_LIST" ]; then say "✗ no backups found on the NAS (or SSH failed)"; exit 1; fi +if [ -z "$REMOTE_LIST" ]; then + die "no backups found on the NAS, or SSH failed" \ + "Expected $REMOTE_DIR/checklists-.json on $NAS_HOST. Either the NAS's own +nightly job has stopped writing them, or this machine can no longer reach the NAS." +fi say "NAS has $(echo "$REMOTE_LIST" | wc -l | tr -d ' ') backup(s)" # 2. Which are missing from Drive? @@ -78,6 +111,7 @@ MISSING="$(printf '%s' "$MISSING" | sed '/^$/d')" if [ -z "$MISSING" ]; then say "✓ Drive already has every NAS backup — nothing to do" else + FAILED=0 COUNT="$(echo "$MISSING" | wc -l | tr -d ' ')" say "$COUNT missing from Drive; copying up to $MAX_PER_RUN" n=0 @@ -103,6 +137,7 @@ else say " ✓ $f ($(du -k "$DRIVE_DIR/$f" | cut -f1) KB)" else rm -f "$TMP" "$OUT" + FAILED=$((FAILED+1)) say " ✗ $f — no valid checklist JSON in the response, skipped" fi done <<< "$MISSING" @@ -115,8 +150,38 @@ fi # out. That is backwards: the whole value of the off-site copy is that it outlives the NAS's # 30-day window. Each file is ~260 KB, so a decade of them is under a gigabyte. If it ever # genuinely needs trimming, do it by hand, and never below what the NAS still holds. -cd "$DRIVE_DIR" || exit 1 +cd "$DRIVE_DIR" || die "cannot enter $DRIVE_DIR" TOTAL="$(ls -1 checklists-????-??-??.json 2>/dev/null | wc -l | tr -d ' ')" OLDEST="$(ls -1 checklists-????-??-??.json 2>/dev/null | sort | head -1)" NEWEST="$(ls -1 checklists-????-??-??.json 2>/dev/null | sort | tail -1)" say "✓ done — $TOTAL copies off-site, ${OLDEST:-none} → ${NEWEST:-none}" + +# Some files were fetched but were not usable. The run "succeeded" in the sense that it did +# not crash, which is exactly the kind of success that hides a broken backup. +if [ "${FAILED:-0}" -gt 0 ]; then + alert "$FAILED backup file(s) could not be copied" \ + "The run finished, but $FAILED file(s) came back without valid checklist JSON and +were skipped. See the log for which ones." +fi + +# The staleness check is the one that would have caught both of the incidents in the header +# comment. Every failure path above assumes this script RAN; if the Mac is asleep, or the +# LaunchAgent is unloaded, nothing above ever fires. This at least turns a quietly ageing +# off-site copy into mail the next time it does run. +if [ -z "$NEWEST" ]; then + alert "no off-site copies at all" "$DRIVE_DIR contains no checklists-.json files." +else + NEWEST_DATE="$(printf '%s' "$NEWEST" | sed -E 's/^checklists-([0-9]{4}-[0-9]{2}-[0-9]{2})\.json$/\1/')" + NEWEST_EPOCH="$(date -j -f "%Y-%m-%d" "$NEWEST_DATE" +%s 2>/dev/null || echo 0)" + if [ "$NEWEST_EPOCH" -gt 0 ]; then + AGE_DAYS=$(( ( $(date +%s) - NEWEST_EPOCH ) / 86400 )) + if [ "$AGE_DAYS" -gt "$STALE_DAYS" ]; then + alert "off-site copy is $AGE_DAYS days old" \ + "Newest file off-site is $NEWEST ($AGE_DAYS days old); anything more than +$STALE_DAYS days means the chain from the NAS to Drive is not working, even though +this run did not error." + else + say "freshness OK — newest off-site copy is $AGE_DAYS day(s) old" + fi + fi +fi diff --git a/scripts/alert.js b/scripts/alert.js new file mode 100755 index 0000000..31ed036 --- /dev/null +++ b/scripts/alert.js @@ -0,0 +1,72 @@ +#!/usr/bin/env node +'use strict'; +// alert.js — send one alert email, reusing the server's own SMTP settings. +// +// Exists because backup-to-drive.sh used to report failure only into +// ~/Library/Logs/checklist-drive-backup.log, where nobody was ever going to look. That is +// how the Drive copies went ten weeks stale in July 2026, and then — despite the script +// written to prevent exactly that — another nine days in September 2026. +// +// Usage: node scripts/alert.js "" "" +// Exit 0 if the mail went out, 1 otherwise. Callers should NOT fail on a failed alert: +// the thing being reported matters more than the reporting of it. + +const fs = require('fs'); +const path = require('path'); + +const ROOT = path.resolve(__dirname, '..'); +const ENV_FILE = path.join(ROOT, 'server', '.env'); + +// Minimal .env reader — the server uses dotenv, but this script has to run from a plain +// LaunchAgent with no cwd guarantees and no node_modules of its own. +function readEnv(file) { + const out = {}; + let text; + try { text = fs.readFileSync(file, 'utf8'); } + catch { return out; } + for (const raw of text.split('\n')) { + const line = raw.trim(); + if (!line || line.startsWith('#')) continue; + const eq = line.indexOf('='); + if (eq < 0) continue; + let v = line.slice(eq + 1).trim(); + if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1); + out[line.slice(0, eq).trim()] = v; + } + return out; +} + +async function main() { + const subject = process.argv[2]; + const body = process.argv[3] || ''; + if (!subject) { console.error('alert.js: no subject given'); process.exit(1); } + + const env = readEnv(ENV_FILE); + const to = env.REPORT_EMAIL; + if (!to) { console.error(`alert.js: no REPORT_EMAIL in ${ENV_FILE}`); process.exit(1); } + if (!env.SMTP_HOST || !env.SMTP_USER || !env.SMTP_PASS) { + console.error(`alert.js: SMTP is not configured in ${ENV_FILE}`); process.exit(1); + } + + // nodemailer lives in server/node_modules; this script has none of its own. + let nodemailer; + try { nodemailer = require(path.join(ROOT, 'server', 'node_modules', 'nodemailer')); } + catch { console.error('alert.js: nodemailer not installed — run `npm install` in server/'); process.exit(1); } + + const transport = nodemailer.createTransport({ + host: env.SMTP_HOST, + port: Number(env.SMTP_PORT || 587), + secure: String(env.SMTP_SECURE || 'false') === 'true', + auth: { user: env.SMTP_USER, pass: env.SMTP_PASS }, + }); + + await transport.sendMail({ + from: env.SMTP_FROM || env.SMTP_USER, + to, + subject, + text: `${body}\n\n-- \nSent by scripts/alert.js on ${require('os').hostname()} at ${new Date().toISOString()}\n`, + }); + console.log(`alert.js: sent "${subject}" to ${to}`); +} + +main().catch(err => { console.error('alert.js:', err && err.message ? err.message : err); process.exit(1); });