recurring-checklist/scripts/weekly-health.sh

#!/bin/bash
# weekly-health.sh — one email a week saying whether everything is actually fine.
#
# WHY THIS EXISTS
#   Every other alert here only fires when something breaks, which means silence has two
#   meanings: all is well, or the thing that was supposed to warn you is itself broken.
#   That ambiguity is what let the off-site backup sit dead for ten weeks, and then for
#   another eight days after it was "fixed". A weekly note that always arrives collapses
#   the two: if it stops coming, that is itself the signal.
#
#   It is deliberately the ONLY scheduled mail. Everything else stays silent unless it
#   needs something from you.
#
# Run weekly by org.waterschurch.checklist-weekly-health. Safe to run by hand.

set -uo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
NAS_HOST="10.1.1.5"
NAS_USER="Christian"
SSH_KEY="$HOME/.ssh/id_ed25519_checklist_nas"
SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -i $SSH_KEY"
SITE="https://check.watersmedia.org"
STALE_DAYS=2

PROBLEMS=""
LINES=""
note()    { LINES="$LINES$1"$'\n'; }
problem() { PROBLEMS="$PROBLEMS  - $1"$'\n'; }

# ── 1. Is the checklist actually serving? ─────────────────────────────────────────────
CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 "$SITE" 2>/dev/null || echo 000)"
if [ "$CODE" = "200" ]; then
  note "The checklist website is up."
else
  note "The checklist website did NOT respond normally (got $CODE)."
  problem "$SITE returned $CODE instead of 200. Nobody can use the checklist if this is right."
fi

# ── 2. Off-site backups ───────────────────────────────────────────────────────────────
DRIVE_DIR=""
for acct in "$HOME/Library/CloudStorage"/GoogleDrive-*; do
  [ -d "$acct" ] || continue
  for sub in "Shared drives/Waters Creative/Apps/Checklist Backups" \
             "Shared drives/Waters Creative/Apps/Backups/Checklist Backups"; do
    [ -d "$acct/$sub" ] && { DRIVE_DIR="$acct/$sub"; break 2; }
  done
done

if [ -z "$DRIVE_DIR" ]; then
  note "Could not find the Google Drive backup folder."
  problem "The Drive folder is missing, so nothing is being copied off-site."
else
  NEWEST=""; AGE=""
  for i in $(seq 0 60); do
    d="$(date -v-${i}d +%Y-%m-%d 2>/dev/null)" || break
    if [ -s "$DRIVE_DIR/checklists-$d.json" ]; then NEWEST="$d"; AGE=$i; break; fi
  done
  TOTAL="$(ls -1 "$DRIVE_DIR"/checklists-????-??-??.json 2>/dev/null | wc -l | tr -d ' ')"
  if [ -z "$NEWEST" ]; then
    note "No recent off-site backup could be found."
    problem "Nothing dated in the last 60 days is in Drive. The nightly copy is not working."
  elif [ "$AGE" -gt "$STALE_DAYS" ]; then
    note "The newest off-site backup is $AGE days old."
    problem "Backups have stopped reaching Drive — newest is $AGE days old."
  else
    if [ "$TOTAL" = "0" ]; then
      note "Off-site backups are current (newest is from $NEWEST)."
    else
      note "Off-site backups are current: $TOTAL copies saved, newest from $NEWEST."
    fi
  fi
fi

# ── 3. Can we still reach the NAS? ────────────────────────────────────────────────────
if ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" true 2>/dev/null; then
  note "The NAS is reachable."
else
  note "The NAS could NOT be reached."
  problem "SSH to the NAS failed, so tonight's backup will not run."
fi

# ── 4. Are the phone apps behind the website? ─────────────────────────────────────────
# The apps bundle a copy of the UI at build time, so a website change does not reach them
# until someone ships a release. This is the thing most easily forgotten.
AAB="$ROOT/mobile/android/app/build/outputs/bundle/release/app-release.aab"
if [ -d "$ROOT/.git" ]; then
  LAST_UI="$(cd "$ROOT" && git log -1 --format=%ct -- server/public mobile/src 2>/dev/null || echo 0)"
  LAST_BUILD=0
  [ -f "$AAB" ] && LAST_BUILD="$(stat -f %m "$AAB" 2>/dev/null || echo 0)"
  if [ "$LAST_UI" -gt 0 ] && [ "$LAST_BUILD" -gt 0 ] && [ "$LAST_UI" -gt "$LAST_BUILD" ]; then
    DAYS=$(( ( $(date +%s) - LAST_UI ) / 86400 ))
    note "The phone apps are behind the website by a change made $DAYS day(s) ago."
    problem "To put that change on phones: cd ~/Documents/recurring-checklist-app && ./release.sh --ios --android"
  elif [ "$LAST_BUILD" -gt 0 ]; then
    note "The phone apps are up to date with the website."
  fi
fi

# ── Compose ───────────────────────────────────────────────────────────────────────────
if [ -z "$PROBLEMS" ]; then
  SUBJECT="Checklist weekly check — all good"
  BODY_FILE="$(mktemp -t checklist-health)"
  cat > "$BODY_FILE" <<MSG
Everything is working. Nothing needs doing.

$LINES
This note arrives once a week whether or not anything is wrong, so that you can
tell the difference between "all is well" and "the thing that warns me is broken".
If a week goes by with no email at all, that is worth a look.
MSG
else
  SUBJECT="Checklist weekly check — $(printf '%s' "$PROBLEMS" | grep -c '^  - ') thing(s) need attention"
  BODY_FILE="$(mktemp -t checklist-health)"
  cat > "$BODY_FILE" <<MSG
Some things need attention:

$PROBLEMS
Everything that was checked:

$LINES
MSG
fi

node "$ROOT/scripts/alert.js" "$SUBJECT" "$(cat "$BODY_FILE")"
rm -f "$BODY_FILE"