Add a weekly health email so silence means something
Every alert here only fires on failure, which leaves silence ambiguous: either all is well, or the thing meant to warn you is itself broken. That ambiguity is exactly what let the off-site backup sit dead for ten weeks, and then for another eight days after it was supposedly fixed. weekly-health.sh sends one note a week whether or not anything is wrong. It checks the four things worth knowing: the website answers, off-site backups are current, the NAS is reachable, and whether the phone apps have fallen behind the website. If everything is fine it says so in a sentence and lists what it checked. If something is not, it leads with what needs doing. It is deliberately the only scheduled mail. Everything else stays quiet unless it needs something, so a message in the inbox always means either the weekly note or a real problem. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Waters Church
co-authored by
Claude Opus 5
Claude Opus 5
parent
f52c3c17ba
commit
8db4b5bb02
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/bin/bash
|
||||
# weekly-health.sh — one email a week saying whether everything is actually fine.
|
||||
#
|
||||
# WHY THIS EXISTS
|
||||
# Every other alert here only fires when something breaks, which means silence has two
|
||||
# meanings: all is well, or the thing that was supposed to warn you is itself broken.
|
||||
# That ambiguity is what let the off-site backup sit dead for ten weeks, and then for
|
||||
# another eight days after it was "fixed". A weekly note that always arrives collapses
|
||||
# the two: if it stops coming, that is itself the signal.
|
||||
#
|
||||
# It is deliberately the ONLY scheduled mail. Everything else stays silent unless it
|
||||
# needs something from you.
|
||||
#
|
||||
# Run weekly by org.waterschurch.checklist-weekly-health. Safe to run by hand.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
||||
NAS_HOST="10.1.1.5"
|
||||
NAS_USER="Christian"
|
||||
SSH_KEY="$HOME/.ssh/id_ed25519_checklist_nas"
|
||||
SSH_OPTS="-o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout=15 -i $SSH_KEY"
|
||||
SITE="https://check.watersmedia.org"
|
||||
STALE_DAYS=2
|
||||
|
||||
PROBLEMS=""
|
||||
LINES=""
|
||||
note() { LINES="$LINES$1"$'\n'; }
|
||||
problem() { PROBLEMS="$PROBLEMS - $1"$'\n'; }
|
||||
|
||||
# ── 1. Is the checklist actually serving? ─────────────────────────────────────────────
|
||||
CODE="$(curl -s -o /dev/null -w '%{http_code}' --max-time 20 "$SITE" 2>/dev/null || echo 000)"
|
||||
if [ "$CODE" = "200" ]; then
|
||||
note "The checklist website is up."
|
||||
else
|
||||
note "The checklist website did NOT respond normally (got $CODE)."
|
||||
problem "$SITE returned $CODE instead of 200. Nobody can use the checklist if this is right."
|
||||
fi
|
||||
|
||||
# ── 2. Off-site backups ───────────────────────────────────────────────────────────────
|
||||
DRIVE_DIR=""
|
||||
for acct in "$HOME/Library/CloudStorage"/GoogleDrive-*; do
|
||||
[ -d "$acct" ] || continue
|
||||
for sub in "Shared drives/Waters Creative/Apps/Checklist Backups" \
|
||||
"Shared drives/Waters Creative/Apps/Backups/Checklist Backups"; do
|
||||
[ -d "$acct/$sub" ] && { DRIVE_DIR="$acct/$sub"; break 2; }
|
||||
done
|
||||
done
|
||||
|
||||
if [ -z "$DRIVE_DIR" ]; then
|
||||
note "Could not find the Google Drive backup folder."
|
||||
problem "The Drive folder is missing, so nothing is being copied off-site."
|
||||
else
|
||||
NEWEST=""; AGE=""
|
||||
for i in $(seq 0 60); do
|
||||
d="$(date -v-${i}d +%Y-%m-%d 2>/dev/null)" || break
|
||||
if [ -s "$DRIVE_DIR/checklists-$d.json" ]; then NEWEST="$d"; AGE=$i; break; fi
|
||||
done
|
||||
TOTAL="$(ls -1 "$DRIVE_DIR"/checklists-????-??-??.json 2>/dev/null | wc -l | tr -d ' ')"
|
||||
if [ -z "$NEWEST" ]; then
|
||||
note "No recent off-site backup could be found."
|
||||
problem "Nothing dated in the last 60 days is in Drive. The nightly copy is not working."
|
||||
elif [ "$AGE" -gt "$STALE_DAYS" ]; then
|
||||
note "The newest off-site backup is $AGE days old."
|
||||
problem "Backups have stopped reaching Drive — newest is $AGE days old."
|
||||
else
|
||||
if [ "$TOTAL" = "0" ]; then
|
||||
note "Off-site backups are current (newest is from $NEWEST)."
|
||||
else
|
||||
note "Off-site backups are current: $TOTAL copies saved, newest from $NEWEST."
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── 3. Can we still reach the NAS? ────────────────────────────────────────────────────
|
||||
if ssh $SSH_OPTS "$NAS_USER@$NAS_HOST" true 2>/dev/null; then
|
||||
note "The NAS is reachable."
|
||||
else
|
||||
note "The NAS could NOT be reached."
|
||||
problem "SSH to the NAS failed, so tonight's backup will not run."
|
||||
fi
|
||||
|
||||
# ── 4. Are the phone apps behind the website? ─────────────────────────────────────────
|
||||
# The apps bundle a copy of the UI at build time, so a website change does not reach them
|
||||
# until someone ships a release. This is the thing most easily forgotten.
|
||||
AAB="$ROOT/mobile/android/app/build/outputs/bundle/release/app-release.aab"
|
||||
if [ -d "$ROOT/.git" ]; then
|
||||
LAST_UI="$(cd "$ROOT" && git log -1 --format=%ct -- server/public mobile/src 2>/dev/null || echo 0)"
|
||||
LAST_BUILD=0
|
||||
[ -f "$AAB" ] && LAST_BUILD="$(stat -f %m "$AAB" 2>/dev/null || echo 0)"
|
||||
if [ "$LAST_UI" -gt 0 ] && [ "$LAST_BUILD" -gt 0 ] && [ "$LAST_UI" -gt "$LAST_BUILD" ]; then
|
||||
DAYS=$(( ( $(date +%s) - LAST_UI ) / 86400 ))
|
||||
note "The phone apps are behind the website by a change made $DAYS day(s) ago."
|
||||
problem "To put that change on phones: cd ~/Documents/recurring-checklist-app && ./release.sh --ios --android"
|
||||
elif [ "$LAST_BUILD" -gt 0 ]; then
|
||||
note "The phone apps are up to date with the website."
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Compose ───────────────────────────────────────────────────────────────────────────
|
||||
if [ -z "$PROBLEMS" ]; then
|
||||
SUBJECT="Checklist weekly check — all good"
|
||||
BODY_FILE="$(mktemp -t checklist-health)"
|
||||
cat > "$BODY_FILE" <<MSG
|
||||
Everything is working. Nothing needs doing.
|
||||
|
||||
$LINES
|
||||
This note arrives once a week whether or not anything is wrong, so that you can
|
||||
tell the difference between "all is well" and "the thing that warns me is broken".
|
||||
If a week goes by with no email at all, that is worth a look.
|
||||
MSG
|
||||
else
|
||||
SUBJECT="Checklist weekly check — $(printf '%s' "$PROBLEMS" | grep -c '^ - ') thing(s) need attention"
|
||||
BODY_FILE="$(mktemp -t checklist-health)"
|
||||
cat > "$BODY_FILE" <<MSG
|
||||
Some things need attention:
|
||||
|
||||
$PROBLEMS
|
||||
Everything that was checked:
|
||||
|
||||
$LINES
|
||||
MSG
|
||||
fi
|
||||
|
||||
node "$ROOT/scripts/alert.js" "$SUBJECT" "$(cat "$BODY_FILE")"
|
||||
rm -f "$BODY_FILE"
|
||||
Reference in New Issue
Block a user