Make the website deploy report its own failures
The deploy was the one pipeline with no failure notification. It either worked or you found out by noticing the site was stale — the same shape as the off-site backup before today. alert.js now takes CHECKLIST_ENV_FILE and CHECKLIST_NODE_MODULES, because the deploy job runs in a container that has the repo but no server/.env and no node_modules; both live on the NAS at the mounted deploy directory. What this still does not catch, and the reason it is written down in the workflow: a runner that wedges. Today the NAS runner claimed run 7, executed none of its six steps, and Gitea timed it out fifteen minutes later — every step was marked failed without one having run, so no in-job step could have reported it. The container said "Up 2 hours" throughout and admitted on restart that it had been holding the job. A stuck runner looks exactly like a slow one, and catching it needs something watching from outside the run. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
17254696e7
commit
39c65cf9de
@@ -88,3 +88,28 @@ jobs:
|
||||
docker ps --filter name=recurring-checklist --format '{{.Names}} {{.Status}}'
|
||||
docker logs --tail 30 recurring-checklist
|
||||
exit 1
|
||||
|
||||
- name: Tell Christian the deploy broke
|
||||
# The web deploy had no failure notification at all until now, which is the same
|
||||
# silent-failure shape as the backup: it either worked or you found out by looking.
|
||||
#
|
||||
# NOT covered by this: a runner that wedges. On 2026-09-24 the NAS runner claimed
|
||||
# this job, ran none of its six steps, and Gitea timed it out — no step ran, so no
|
||||
# step could report. Catching that needs something watching from outside the run.
|
||||
if: failure()
|
||||
run: |
|
||||
D=/volume2/docker/recurring-checklist
|
||||
BODY_FILE="$(mktemp)"
|
||||
cat > "$BODY_FILE" <<MSG
|
||||
A push to main failed to deploy the website to the NAS.
|
||||
|
||||
The running container was left alone — the deploy recreates it in place and only
|
||||
after the files are copied, so the previous version should still be serving. Worth
|
||||
confirming: https://check.watersmedia.org
|
||||
|
||||
Full log:
|
||||
http://chronicles-ab.waterschurch.org:3000/christian/recurring-checklist/actions
|
||||
MSG
|
||||
CHECKLIST_ENV_FILE="$D/.env" CHECKLIST_NODE_MODULES="$D/node_modules" \
|
||||
node scripts/alert.js "Website deploy FAILED" "$(cat "$BODY_FILE")" || true
|
||||
rm -f "$BODY_FILE"
|
||||
|
||||
+7
-3
@@ -15,7 +15,11 @@ const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const ROOT = path.resolve(__dirname, '..');
|
||||
const ENV_FILE = path.join(ROOT, 'server', '.env');
|
||||
// Overridable because the deploy job runs inside a container that has the repo checked out
|
||||
// but no server/.env and no node_modules — those live on the NAS at the mounted deploy
|
||||
// directory instead. Without this the deploy could not report its own failures.
|
||||
const ENV_FILE = process.env.CHECKLIST_ENV_FILE || path.join(ROOT, 'server', '.env');
|
||||
const NODE_MODULES = process.env.CHECKLIST_NODE_MODULES || path.join(ROOT, 'server', 'node_modules');
|
||||
|
||||
// Minimal .env reader — the server uses dotenv, but this script has to run from a plain
|
||||
// LaunchAgent with no cwd guarantees and no node_modules of its own.
|
||||
@@ -56,8 +60,8 @@ async function main() {
|
||||
|
||||
// nodemailer lives in server/node_modules; this script has none of its own.
|
||||
let nodemailer;
|
||||
try { nodemailer = require(path.join(ROOT, 'server', 'node_modules', 'nodemailer')); }
|
||||
catch { console.error('alert.js: nodemailer not installed — run `npm install` in server/'); process.exit(1); }
|
||||
try { nodemailer = require(path.join(NODE_MODULES, 'nodemailer')); }
|
||||
catch { console.error(`alert.js: nodemailer not found under ${NODE_MODULES} — run \`npm install\` in server/`); process.exit(1); }
|
||||
|
||||
const transport = nodemailer.createTransport({
|
||||
host: env.SMTP_HOST,
|
||||
|
||||
Reference in New Issue
Block a user