diff --git a/backup-to-drive.sh b/backup-to-drive.sh index abae2c7..3a5891d 100755 --- a/backup-to-drive.sh +++ b/backup-to-drive.sh @@ -69,26 +69,36 @@ SSH_KEY="$HOME/.ssh/id_ed25519_checklist_nas" SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=20 -i $SSH_KEY" if [ ! -f "$SSH_KEY" ]; then - die "no SSH key for the NAS at $SSH_KEY" \ - "Generate one and authorise it on the NAS: + die "tonight's backup did not run — the key for the NAS is missing" \ + "Nothing was copied off-site tonight. Previous backups are untouched and safe. + +This machine signs in to the NAS with a key file, and that file is gone from + $SSH_KEY + +To fix it, on this Mac: ssh-keygen -t ed25519 -f $SSH_KEY -N '' ssh-copy-id -i $SSH_KEY $NAS_USER@$NAS_HOST" fi if ! ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" true 2>/dev/null; then - die "cannot reach the NAS over SSH" \ - "Key auth to $NAS_USER@$NAS_HOST failed. Either the NAS is down, or the key at -$SSH_KEY is no longer in its authorized_keys." + die "tonight's backup did not run — could not reach the NAS" \ + "Nothing was copied off-site tonight. Previous backups are untouched and safe. + +Either the NAS is off or off the network, or this machine's key is no longer +accepted by it. Worth checking the NAS is up first; that is the usual cause." fi # Google Drive streams files on demand; if the app is signed out or not running the path # simply is not there. Writing to a path Drive is not watching would look like it worked and # sync nothing, so refuse rather than pretend. if [ -z "$DRIVE_DIR" ]; then - die "cannot find the Drive backup folder" \ - "Looked under every ~/Library/CloudStorage/GoogleDrive-* account for - Shared drives/Waters Creative/Apps/Checklist Backups -and found none. Is Google Drive for Desktop running and signed in, and does that -folder still exist under that name?" + die "tonight's backup did not run — the Google Drive folder is missing" \ + "Nothing was copied off-site tonight. Previous backups are untouched and safe. + +The folder it copies into could not be found: + Shared drives > Waters Creative > Apps > Checklist Backups + +Usually this means Google Drive is not running or has signed out on this Mac. +It can also mean the folder was renamed or moved." fi say "Drive folder: $DRIVE_DIR" @@ -100,9 +110,12 @@ run_remote() { # $1 = remote command REMOTE_LIST="$(run_remote "ls -1 $REMOTE_DIR/checklists-*.json 2>/dev/null | xargs -n1 basename" \ | grep -E '^checklists-[0-9]{4}-[0-9]{2}-[0-9]{2}\.json$' | sort)" if [ -z "$REMOTE_LIST" ]; then - die "no backups found on the NAS, or SSH failed" \ - "Expected $REMOTE_DIR/checklists-.json on $NAS_HOST. Either the NAS's own -nightly job has stopped writing them, or this machine can no longer reach the NAS." + die "the NAS has no backups to copy" \ + "Copies already in Google Drive are untouched and safe, but no new ones were found. + +The NAS writes a backup of the checklists every night at 03:30, and none are +there. That is a problem on the NAS itself rather than with this copy job — the +checklist app may have stopped writing them." fi say "NAS has $(echo "$REMOTE_LIST" | wc -l | tr -d ' ') backup(s)" @@ -204,12 +217,16 @@ say "✓ done — $TOTAL copies off-site, newest ${NEWEST:-none}" # tomorrow is noise, and nightly noise is how an alert stops being read — which is the whole # failure mode this script exists to prevent. if [ -n "${OVERDUE:-}" ]; then - alert "backup file(s) still missing off-site after several days" \ - "These NAS backups have failed to copy for more than $STALE_DAYS days, so the -retries are not recovering on their own: + alert "some backups are not reaching Google Drive" \ + "The website and the checklists themselves are fine. This is about the off-site +copies only. + +These have failed to copy for more than $STALE_DAYS days, so the automatic retries +are not fixing it: $OVERDUE -Log: $LOG" +Everything else copied normally. Worth a look when convenient — it is not urgent +unless it keeps growing." elif [ "${FAILED:-0}" -gt 0 ]; then say "($FAILED file(s) failed this run but are recent — they will be retried tomorrow, not alerting)" fi @@ -218,15 +235,17 @@ fi # failure path above assumes this script RAN; if the Mac is asleep or the LaunchAgent is # unloaded, nothing fires at all. if [ -z "$NEWEST" ]; then - alert "no recent off-site copy" \ - "No checklists-.json for any of the last 60 days was found in -$DRIVE_DIR -Either copying has stopped, or this machine cannot read that folder." + alert "no recent backup in Google Drive" \ + "The checklists themselves are fine — this is about the off-site copies. + +Nothing from the last 60 days is in the Drive folder. Either the copying has +stopped, or this Mac can no longer read that folder." elif [ "$AGE_DAYS" -gt "$STALE_DAYS" ]; then - alert "off-site copy is $AGE_DAYS days old" \ - "Newest file off-site is $NEWEST ($AGE_DAYS days old); more than $STALE_DAYS days -means the chain from the NAS to Drive is not working, even though this run did -not error." + alert "backups have stopped reaching Google Drive" \ + "The checklists themselves are fine — this is about the off-site copies. + +The newest one in Drive is $AGE_DAYS days old ($NEWEST). Tonight's run did not +report an error, which means something is failing quietly rather than loudly." else say "freshness OK — newest off-site copy is $AGE_DAYS day(s) old" fi diff --git a/deploy/org.waterschurch.checklist-weekly-health.plist b/deploy/org.waterschurch.checklist-weekly-health.plist new file mode 100644 index 0000000..122ffac --- /dev/null +++ b/deploy/org.waterschurch.checklist-weekly-health.plist @@ -0,0 +1,39 @@ + + + + + Label + org.waterschurch.checklist-weekly-health + + + ProgramArguments + + /Users/Studio/checklist-app/scripts/weekly-health.sh + + + StartCalendarInterval + + Weekday1 + Hour8 + Minute30 + + + RunAtLoad + + + StandardOutPath + /Users/Studio/Library/Logs/checklist-weekly-health.out.log + StandardErrorPath + /Users/Studio/Library/Logs/checklist-weekly-health.err.log + + + EnvironmentVariables + + PATH + /usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin + + +