From 39c65cf9def21a2f6f0a972436e4499394d44108 Mon Sep 17 00:00:00 2001
From: Waters Church <info@waterschurch.org>
Date: Thu, 24 Sep 2026 16:11:22 -0400
Subject: [PATCH] Make the website deploy report its own failures
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The deploy was the one pipeline with no failure notification. It either worked or
you found out by noticing the site was stale — the same shape as the off-site
backup before today.

alert.js now takes CHECKLIST_ENV_FILE and CHECKLIST_NODE_MODULES, because the
deploy job runs in a container that has the repo but no server/.env and no
node_modules; both live on the NAS at the mounted deploy directory.

What this still does not catch, and the reason it is written down in the workflow:
a runner that wedges. Today the NAS runner claimed run 7, executed none of its six
steps, and Gitea timed it out fifteen minutes later — every step was marked failed
without one having run, so no in-job step could have reported it. The container
said "Up 2 hours" throughout and admitted on restart that it had been holding the
job. A stuck runner looks exactly like a slow one, and catching it needs something
watching from outside the run.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
---
 .gitea/workflows/deploy.yml | 25 +++++++++++++++++++++++++
 scripts/alert.js            | 10 +++++++---
 2 files changed, 32 insertions(+), 3 deletions(-)

diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml
index 414a3b4..4fe4c42 100644
--- a/.gitea/workflows/deploy.yml
+++ b/.gitea/workflows/deploy.yml
@@ -88,3 +88,28 @@ jobs:
           docker ps --filter name=recurring-checklist --format '{{.Names}} {{.Status}}'
           docker logs --tail 30 recurring-checklist
           exit 1
+
+      - name: Tell Christian the deploy broke
+        # The web deploy had no failure notification at all until now, which is the same
+        # silent-failure shape as the backup: it either worked or you found out by looking.
+        #
+        # NOT covered by this: a runner that wedges. On 2026-09-24 the NAS runner claimed
+        # this job, ran none of its six steps, and Gitea timed it out — no step ran, so no
+        # step could report. Catching that needs something watching from outside the run.
+        if: failure()
+        run: |
+          D=/volume2/docker/recurring-checklist
+          BODY_FILE="$(mktemp)"
+          cat > "$BODY_FILE" <<MSG
+          A push to main failed to deploy the website to the NAS.
+
+          The running container was left alone — the deploy recreates it in place and only
+          after the files are copied, so the previous version should still be serving. Worth
+          confirming: https://check.watersmedia.org
+
+          Full log:
+          http://chronicles-ab.waterschurch.org:3000/christian/recurring-checklist/actions
+          MSG
+          CHECKLIST_ENV_FILE="$D/.env" CHECKLIST_NODE_MODULES="$D/node_modules" \
+            node scripts/alert.js "Website deploy FAILED" "$(cat "$BODY_FILE")" || true
+          rm -f "$BODY_FILE"
diff --git a/scripts/alert.js b/scripts/alert.js
index 642eb92..a926e69 100755
--- a/scripts/alert.js
+++ b/scripts/alert.js
@@ -15,7 +15,11 @@ const fs = require('fs');
 const path = require('path');
 
 const ROOT = path.resolve(__dirname, '..');
-const ENV_FILE = path.join(ROOT, 'server', '.env');
+// Overridable because the deploy job runs inside a container that has the repo checked out
+// but no server/.env and no node_modules — those live on the NAS at the mounted deploy
+// directory instead. Without this the deploy could not report its own failures.
+const ENV_FILE = process.env.CHECKLIST_ENV_FILE || path.join(ROOT, 'server', '.env');
+const NODE_MODULES = process.env.CHECKLIST_NODE_MODULES || path.join(ROOT, 'server', 'node_modules');
 
 // Minimal .env reader — the server uses dotenv, but this script has to run from a plain
 // LaunchAgent with no cwd guarantees and no node_modules of its own.
@@ -56,8 +60,8 @@ async function main() {
 
   // nodemailer lives in server/node_modules; this script has none of its own.
   let nodemailer;
-  try { nodemailer = require(path.join(ROOT, 'server', 'node_modules', 'nodemailer')); }
-  catch { console.error('alert.js: nodemailer not installed — run `npm install` in server/'); process.exit(1); }
+  try { nodemailer = require(path.join(NODE_MODULES, 'nodemailer')); }
+  catch { console.error(`alert.js: nodemailer not found under ${NODE_MODULES} — run \`npm install\` in server/`); process.exit(1); }
 
   const transport = nodemailer.createTransport({
     host: env.SMTP_HOST,