diff --git a/scripts/weekly-health.sh b/scripts/weekly-health.sh new file mode 100755 index 0000000..7011117 --- /dev/null +++ b/scripts/weekly-health.sh @@ -0,0 +1,126 @@ +#!/bin/bash +# weekly-health.sh — one email a week saying whether everything is actually fine. +# +# WHY THIS EXISTS +# Every other alert here only fires when something breaks, which means silence has two +# meanings: all is well, or the thing that was supposed to warn you is itself broken. +# That ambiguity is what let the off-site backup sit dead for ten weeks, and then for +# another eight days after it was "fixed". A weekly note that always arrives collapses +# the two: if it stops coming, that is itself the signal. +# +# It is deliberately the ONLY scheduled mail. Everything else stays silent unless it +# needs something from you. +# +# Run weekly by org.waterschurch.checklist-weekly-health. Safe to run by hand. + +set -uo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +NAS_HOST="10.1.1.5" +NAS_USER="Christian" +SSH_KEY="$HOME/.ssh/id_ed25519_checklist_nas" +SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -i $SSH_KEY" +SITE="https://check.watersmedia.org" +STALE_DAYS=2 + +PROBLEMS="" +LINES="" +note() { LINES="$LINES$1"$'\n'; } +problem() { PROBLEMS="$PROBLEMS - $1"$'\n'; } + +# ── 1. Is the checklist actually serving? ───────────────────────────────────────────── +CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 "$SITE" 2>/dev/null || echo 000)" +if [ "$CODE" = "200" ]; then + note "The checklist website is up." +else + note "The checklist website did NOT respond normally (got $CODE)." + problem "$SITE returned $CODE instead of 200. Nobody can use the checklist if this is right." +fi + +# ── 2. Off-site backups ─────────────────────────────────────────────────────────────── +DRIVE_DIR="" +for acct in "$HOME/Library/CloudStorage"/GoogleDrive-*; do + [ -d "$acct" ] || continue + for sub in "Shared drives/Waters Creative/Apps/Checklist Backups" \ + "Shared drives/Waters Creative/Apps/Backups/Checklist Backups"; do + [ -d "$acct/$sub" ] && { DRIVE_DIR="$acct/$sub"; break 2; } + done +done + +if [ -z "$DRIVE_DIR" ]; then + note "Could not find the Google Drive backup folder." + problem "The Drive folder is missing, so nothing is being copied off-site." +else + NEWEST=""; AGE="" + for i in $(seq 0 60); do + d="$(date -v-${i}d +%Y-%m-%d 2>/dev/null)" || break + if [ -s "$DRIVE_DIR/checklists-$d.json" ]; then NEWEST="$d"; AGE=$i; break; fi + done + TOTAL="$(ls -1 "$DRIVE_DIR"/checklists-????-??-??.json 2>/dev/null | wc -l | tr -d ' ')" + if [ -z "$NEWEST" ]; then + note "No recent off-site backup could be found." + problem "Nothing dated in the last 60 days is in Drive. The nightly copy is not working." + elif [ "$AGE" -gt "$STALE_DAYS" ]; then + note "The newest off-site backup is $AGE days old." + problem "Backups have stopped reaching Drive — newest is $AGE days old." + else + if [ "$TOTAL" = "0" ]; then + note "Off-site backups are current (newest is from $NEWEST)." + else + note "Off-site backups are current: $TOTAL copies saved, newest from $NEWEST." + fi + fi +fi + +# ── 3. Can we still reach the NAS? ──────────────────────────────────────────────────── +if ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" true 2>/dev/null; then + note "The NAS is reachable." +else + note "The NAS could NOT be reached." + problem "SSH to the NAS failed, so tonight's backup will not run." +fi + +# ── 4. Are the phone apps behind the website? ───────────────────────────────────────── +# The apps bundle a copy of the UI at build time, so a website change does not reach them +# until someone ships a release. This is the thing most easily forgotten. +AAB="$ROOT/mobile/android/app/build/outputs/bundle/release/app-release.aab" +if [ -d "$ROOT/.git" ]; then + LAST_UI="$(cd "$ROOT" && git log -1 --format=%ct -- server/public mobile/src 2>/dev/null || echo 0)" + LAST_BUILD=0 + [ -f "$AAB" ] && LAST_BUILD="$(stat -f %m "$AAB" 2>/dev/null || echo 0)" + if [ "$LAST_UI" -gt 0 ] && [ "$LAST_BUILD" -gt 0 ] && [ "$LAST_UI" -gt "$LAST_BUILD" ]; then + DAYS=$(( ( $(date +%s) - LAST_UI ) / 86400 )) + note "The phone apps are behind the website by a change made $DAYS day(s) ago." + problem "To put that change on phones: cd ~/Documents/recurring-checklist-app && ./release.sh --ios --android" + elif [ "$LAST_BUILD" -gt 0 ]; then + note "The phone apps are up to date with the website." + fi +fi + +# ── Compose ─────────────────────────────────────────────────────────────────────────── +if [ -z "$PROBLEMS" ]; then + SUBJECT="Checklist weekly check — all good" + BODY_FILE="$(mktemp -t checklist-health)" + cat > "$BODY_FILE" < "$BODY_FILE" <