#!/bin/bash # weekly-health.sh — one email a week saying whether everything is actually fine. # # WHY THIS EXISTS # Every other alert here only fires when something breaks, which means silence has two # meanings: all is well, or the thing that was supposed to warn you is itself broken. # That ambiguity is what let the off-site backup sit dead for ten weeks, and then for # another eight days after it was "fixed". A weekly note that always arrives collapses # the two: if it stops coming, that is itself the signal. # # It is deliberately the ONLY scheduled mail. Everything else stays silent unless it # needs something from you. # # Run weekly by org.waterschurch.checklist-weekly-health. Safe to run by hand. set -uo pipefail ROOT="$(cd "$(dirname "$0")/.." && pwd)" NAS_HOST="10.1.1.5" NAS_USER="Christian" SSH_KEY="$HOME/.ssh/id_ed25519_checklist_nas" SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -i $SSH_KEY" SITE="https://check.watersmedia.org" STALE_DAYS=2 PROBLEMS="" LINES="" note() { LINES="$LINES$1"$'\n'; } problem() { PROBLEMS="$PROBLEMS - $1"$'\n'; } # ── 1. Is the checklist actually serving? ───────────────────────────────────────────── CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 "$SITE" 2>/dev/null || echo 000)" if [ "$CODE" = "200" ]; then note "The checklist website is up." else note "The checklist website did NOT respond normally (got $CODE)." problem "$SITE returned $CODE instead of 200. Nobody can use the checklist if this is right." fi # ── 2. Off-site backups ─────────────────────────────────────────────────────────────── DRIVE_DIR="" for acct in "$HOME/Library/CloudStorage"/GoogleDrive-*; do [ -d "$acct" ] || continue for sub in "Shared drives/Waters Creative/Apps/Checklist Backups" \ "Shared drives/Waters Creative/Apps/Backups/Checklist Backups"; do [ -d "$acct/$sub" ] && { DRIVE_DIR="$acct/$sub"; break 2; } done done if [ -z "$DRIVE_DIR" ]; then note "Could not find the Google Drive backup folder." problem "The Drive folder is missing, so nothing is being copied off-site." else NEWEST=""; AGE="" for i in $(seq 0 60); do d="$(date -v-${i}d +%Y-%m-%d 2>/dev/null)" || break if [ -s "$DRIVE_DIR/checklists-$d.json" ]; then NEWEST="$d"; AGE=$i; break; fi done TOTAL="$(ls -1 "$DRIVE_DIR"/checklists-????-??-??.json 2>/dev/null | wc -l | tr -d ' ')" if [ -z "$NEWEST" ]; then note "No recent off-site backup could be found." problem "Nothing dated in the last 60 days is in Drive. The nightly copy is not working." elif [ "$AGE" -gt "$STALE_DAYS" ]; then note "The newest off-site backup is $AGE days old." problem "Backups have stopped reaching Drive — newest is $AGE days old." else if [ "$TOTAL" = "0" ]; then note "Off-site backups are current (newest is from $NEWEST)." else note "Off-site backups are current: $TOTAL copies saved, newest from $NEWEST." fi fi fi # ── 3. Can we still reach the NAS? ──────────────────────────────────────────────────── if ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" true 2>/dev/null; then note "The NAS is reachable." else note "The NAS could NOT be reached." problem "SSH to the NAS failed, so tonight's backup will not run." fi # ── 4. Are the phone apps behind the website? ───────────────────────────────────────── # The apps bundle a copy of the UI at build time, so a website change does not reach them # until someone ships a release. This is the thing most easily forgotten. AAB="$ROOT/mobile/android/app/build/outputs/bundle/release/app-release.aab" if [ -d "$ROOT/.git" ]; then LAST_UI="$(cd "$ROOT" && git log -1 --format=%ct -- server/public mobile/src 2>/dev/null || echo 0)" LAST_BUILD=0 [ -f "$AAB" ] && LAST_BUILD="$(stat -f %m "$AAB" 2>/dev/null || echo 0)" if [ "$LAST_UI" -gt 0 ] && [ "$LAST_BUILD" -gt 0 ] && [ "$LAST_UI" -gt "$LAST_BUILD" ]; then DAYS=$(( ( $(date +%s) - LAST_UI ) / 86400 )) note "The phone apps are behind the website by a change made $DAYS day(s) ago." problem "To put that change on phones: cd ~/Documents/recurring-checklist-app && ./release.sh --ios --android" elif [ "$LAST_BUILD" -gt 0 ]; then note "The phone apps are up to date with the website." fi fi # ── Compose ─────────────────────────────────────────────────────────────────────────── if [ -z "$PROBLEMS" ]; then SUBJECT="Checklist weekly check — all good" BODY_FILE="$(mktemp -t checklist-health)" cat > "$BODY_FILE" < "$BODY_FILE" <