Merge pull request 'Document the LAN-only translate.watersmedia.org setup at North Attleboro' (#1) from docs/waters-na-lan-proxy into main

PUBLIC-ACCESS.md now describes what actually runs: a local UniFi DNS
record at Apollo Beach and North Attleboro pointing the name at the
Boba-Fett NAS, a DSM reverse proxy rule from port 80 to the booth Mac at
10.1.7.39:3100 with WebSocket headers, X-Forwarded-Host and a 3600 s
read timeout, and an nginx block that redirects https:// to http://.
The hand-written nginx example is kept for non-DSM proxies, with the
booth Mac address corrected and proxy_buffering off for live audio.

docs/examples/waters-church-na/ holds the runbook, the script run on
Boba-Fett, and the UniFi DNS record script used to create the records.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

app/engine.js

@@ -190,7 +190,9 @@ const state = {
 
   /* health counters, reported to the operator page */
   degradedFinals: 0,         // finals that shipped with fewer languages (stale)
-  captionOnlyFinals: 0       // finals that shipped with no translations at all
+  captionOnlyFinals: 0,      // finals that shipped with no translations at all
+  captionOnlyAt: 0,          // when the last one happened
+  degradedAt: 0
 };
 
 // ────────────────────────────────────────────────────────── logging
@@ -891,12 +893,54 @@ async function respawnWorker(worker) {
         log('the speech model is too heavy for this machine — switching to ' +
             lighter.split('/').pop(), 'err');
       }
+      /*
+       * A LIGHTER MODEL GETS THE PREVIEW'S DTYPE, NOT THE BIG ONE'S.
+       *
+       * asrDtype is chosen for the accurate model and its weights are cached against
+       * it. Reusing it when stepping down asks for a variant that was never
+       * downloaded — and those weights live in the browser cache, with no network at
+       * a venue — so the step-down failed and the ladder could not move at all.
+       *
+       * MEASURED on a campus Mac: the cache held whisper-large-v3-turbo as
+       * encoder_model_q4f16.onnx (its asrDtype) and whisper-small as
+       * encoder_model_fp16.onnx (the preview's). Stepping down asked for
+       * whisper-small at q4f16, a file that is not there; it went to the network,
+       * failed, and logged "Unsupported model type: whisper" three times a minute
+       * for ever while the service sat dead.
+       *
+       * The preview runs whisper-small on exactly these settings every service, so
+       * they are the one combination this machine is known to be able to load.
+       */
+      const stepDtype = lighter
+        ? resolveDtype(cfg.asrFastDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' })
+        : resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' });
       state.asrWorker = await loadWorkerWithFallback('speech', 'automatic-speech-recognition',
         state.asrModelInUse || cfg.asrModel,
-        resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' }),
+        stepDtype,
         { encoder_model: 'q4', decoder_model_merged: 'q4' });
       log('speech model restarted' + (lighter ? ' on ' + lighter.split('/').pop() : ''), 'ok');
       state.slowRun = 0;
+      if (lighter) {
+        /*
+         * THROW AWAY THE BACKLOG THAT CAUSED THIS, AND LET THE NEW MODEL SETTLE.
+         *
+         * Swapping the model does nothing about the queue that triggered the swap.
+         * Those chunks are already past the caption-only cutoff, so they can no
+         * longer be translated in time and are worth nothing to the room — but the
+         * first of them gets handed to a model that has just loaded, takes the full
+         * worker timeout, and trips the absolute rule. MEASURED: a campus Mac fell
+         * large-v3-turbo -> small -> base in 34 seconds exactly that way, landing on
+         * a model SLOWER than the one it started from.
+         *
+         * So drop them, and refuse to step down again for a grace window, so the new
+         * model is judged on sentences it was actually given.
+         */
+        const dropped = queue.length;
+        queue.length = 0;
+        state.steppedDownAt = Date.now();
+        if (dropped) log('dropped ' + dropped + ' stale sentence' + (dropped === 1 ? '' : 's') +
+                         ' the old model never reached', 'warn');
+      }
     }
   } catch (err) {
     log('model restart failed: ' + err.message, 'err');
@@ -1794,6 +1838,7 @@ async function pump() {
 
   state.busy = true;
   const started = Date.now();
+  let jobAudioMs = 0;
   try {
     /* job.at is when the audio ENDED. Live translation of stale audio is
        worthless, but the WORDS are not: however old the job, it still gets
@@ -1805,6 +1850,9 @@ async function pump() {
     /* Near-silence telemetry (log-only this release): a chunk whose average
        level barely clears the noise floor is hallucination fuel. */
     const audio = flatten(job.frames);
+    /* How much speech this chunk actually holds. The step-down below compares
+       against it, because "slow" only means anything relative to the audio. */
+    jobAudioMs = (audio.length / SAMPLE_RATE) * 1000;
     let sum = 0;
     for (let i = 0; i < audio.length; i += 4) sum += audio[i] * audio[i];
     const meanRms = Math.sqrt(sum / Math.ceil(audio.length / 4));
@@ -1845,15 +1893,33 @@ async function pump() {
       const sentences = collapseLoops(splitSentences(text));
 
       let targets = orderedTargets();
-      if (ageSec > 12) {
+      /*
+       * SKIPPING A TRANSLATION ONLY BUYS SOMETHING IF THERE IS A QUEUE TO DRAIN.
+       *
+       * Dropping the translations of stale audio is how the app catches up: the
+       * listener has moved on, so the work is wasted and the time is better spent on
+       * the sentences behind it. That reasoning needs a backlog. With nothing waiting
+       * behind it, skipping costs the room a sentence and saves nothing at all.
+       *
+       * MEASURED: a campus Mac took one 16 s stall — two Screen Sharing sessions
+       * competing for the machine — and skipped that sentence's translations with an
+       * EMPTY queue. The room lost a line of the sermon to a catch-up that had
+       * nothing to catch up on.
+       */
+      if (ageSec > 12 && queue.length > 0) {
         targets = [];
         state.captionOnlyFinals++;
-        log('audio ' + ageSec.toFixed(0) + 's stale — caption only, translations skipped', 'err');
+        state.captionOnlyAt = Date.now();
+        log('audio ' + ageSec.toFixed(0) + 's stale with ' + queue.length + ' waiting — ' +
+            'caption only, translations skipped', 'err');
+      } else if (ageSec > 12) {
+        /* Late, but nothing is behind it: say so, and translate it anyway. */
+        log('audio ' + ageSec.toFixed(0) + 's late, but nothing is queued — translating it anyway', 'warn');
       } else if (ageSec > 6) {
         const listening = new Set((state.byLanguage || []).map((r) => r.lang));
         const before = targets.length;
         targets = targets.filter((t) => listening.has(t));
-        if (targets.length < before) state.degradedFinals++;
+        if (targets.length < before) { state.degradedFinals++; state.degradedAt = Date.now(); }
       }
 
       /* The segment's own loudness, measured once and shared by every sentence
@@ -1908,11 +1974,56 @@ async function pump() {
      * talk, and the backlog only grows — step down before the service is lost
      * rather than after. Measured on the Intel laptop: 12-25 s, every sentence.
      */
-    if (took > 8000) {
+    /*
+     * SLOWER THAN THE AUDIO IT COVERS — not slower than some constant.
+     *
+     * This used to fire only past 8 s, on the reasoning that 8 s meant slower
+     * than people talk. That was true when a chunk could be 8 s long. Chunks
+     * are capped at HARD_CUT_MAX, 5.5 s, so a 4.5 s chunk taking 4.3 s is
+     * already losing ground and never came near the old threshold.
+     *
+     * MEASURED on a campus M4 Pro: every sentence 4.2-4.7 s against roughly
+     * 4-5 s of audio, a queue 46 deep and a lag of 152 s still climbing — with
+     * nobody listening and no language being translated, so this was
+     * transcription alone. The one mechanism built to catch exactly that could
+     * not fire, because 4.4 s is not 8 s.
+     *
+     * The queue test is what separates "this machine cannot keep up" from one
+     * slow sentence: a backlog that is three deep and being fed faster than it
+     * drains does not recover on its own.
+     */
+    /*
+     * SLOWER THAN THE AUDIO IT COVERS, AND LANDING LATE.
+     *
+     * This used to fire only past 8 s, on the reasoning that 8 s meant slower
+     * than people talk. That held when a chunk could be 8 s long; chunks are
+     * capped at HARD_CUT_MAX, 5.5 s, so a 4.4 s sentence is already losing and
+     * never came near the old threshold.
+     *
+     * Both numbers below were measured on a campus M4 Pro, and the second one
+     * only after a first attempt got it wrong. 0.85, because that Mac ran 4.4 s
+     * against 4.5 s of audio — nominally FASTER than real time while losing
+     * badly, since the work outside this timer (segmenting, the preview model
+     * sharing the GPU, the scheduler) loses on its own.
+     *
+     * The second test was a queue depth, and watching that Mac go red disproved
+     * it: the queue sat at 2-3 while the lag held at 15 s and the verdict was
+     * already red. A machine can be persistently late without ever stacking work
+     * up — it finishes each sentence just slightly too slowly, for ever. So the
+     * test is how far behind the speaker the sentence actually landed, which is
+     * what the operator is complaining about in the first place.
+     */
+    /* A model that loaded seconds ago has not had a fair sentence yet. */
+    const settling = state.steppedDownAt && (Date.now() - state.steppedDownAt) < STEP_SETTLE_MS;
+    const behindMs = Date.now() - job.at;
+    const losingGround = jobAudioMs > 0 && took > jobAudioMs * 0.85 && behindMs > 8000;
+    if (!settling && (took > 8000 || losingGround)) {
       state.slowRun = (state.slowRun || 0) + 1;
       if (state.slowRun >= 3 && !respawning && lighterAsr(state.asrModelInUse || state.cfg.asrModel)) {
         state.slowRun = 0;
-        log('transcription is slower than people are speaking — stepping down to a lighter model', 'err');
+        log('transcription is slower than people are speaking (' + took + ' ms for ' +
+            Math.round(jobAudioMs) + ' ms of audio, ' + Math.round(behindMs / 1000) + 's behind) — ' +
+            'stepping down to a lighter model', 'err');
         respawnWorker(state.asrWorker);
       }
     } else state.slowRun = 0;
@@ -2423,6 +2534,11 @@ function monitorTranslation(cid, lang, text) {
  * It says it once per episode, and names both numbers so the operator can act
  * on it rather than go looking.
  */
+/* How long a freshly swapped speech model is left alone. Long enough to cover a
+   cold first inference, short enough that a genuinely hopeless model still falls
+   through the ladder within a sentence or two. */
+const STEP_SETTLE_MS = 20000;
+
 const DEAF_AFTER_MS = 90000;
 setInterval(() => {
   try {
@@ -2662,6 +2778,7 @@ function goLive(on) {
   state.live = on;
   if (on) {
     state.degradedFinals = 0; state.captionOnlyFinals = 0; state.fragmentFinals = 0;
+    state.captionOnlyAt = 0; state.degradedAt = 0;
     /* The watchdog measures from the start of the service, not from the last
        sentence of the previous one. */
     state.lastFinalAt = Date.now();
@@ -2736,6 +2853,10 @@ function reportStatus() {
       queue: queue.length,
       degradedFinals: state.degradedFinals,
       captionOnlyFinals: state.captionOnlyFinals,
+      /* WHEN, not just how many — the verdict needs to know whether this is
+         happening now or happened once an hour ago. */
+      captionOnlyAt: state.captionOnlyAt || 0,
+      degradedAt: state.degradedAt || 0,
       fragmentFinals: state.fragmentFinals || 0,
       cloud: state.cloudStats || null
     }

app/package.json

@@ -2,7 +2,7 @@
   "name": "church-translate-engine",
   "productName": "Church Translate",
   "version": "3.23.2",
-  "buildNumber": 116,
+  "buildNumber": 121,
   "private": true,
   "description": "Live sermon translation — self-contained: captures audio, transcribes, translates, and serves the congregation page itself",
   "main": "main.js",
@@ -106,7 +106,7 @@
       ]
     },
     "asar": false,
-    "buildVersion": "116",
+    "buildVersion": "121",
     "win": {
       "target": [
         {

app/server/server.js

@@ -510,6 +510,11 @@ const ENGINE_STALE_MS = 12000;
  * A campus reported "constantly behind, in the red" and the cause took an
  * afternoon to find; it was eight languages on hardware tuned against one.
  */
+/* How long a stumble keeps colouring the verdict. Long enough that a real
+   problem stays visible while someone walks to the desk, short enough that a
+   one-off does not outlive the sentence after it. */
+const TROUBLE_WINDOW_MS = 120000;
+
 function loadNote() {
   const n = activeLanguages().length;
   if (n < 2) return '';
@@ -523,10 +528,27 @@ function healthVerdict(connected, stale) {
   if (stale)                    return { level: 'red',   detail: 'The translation app has stopped responding.' };
   if (!state.live)              return { level: 'green', detail: 'Ready.' };
   if (!es.modelsReady)          return { level: 'amber', detail: 'Models are still loading.' };
-  if ((es.lagMs || 0) > 12000 || (es.captionOnlyFinals || 0) > 0)
+  /*
+   * RECENTLY, not ever.
+   *
+   * captionOnlyFinals is a running total for the whole service, and testing it
+   * for "more than zero" meant a single stale sentence pinned the desk's light
+   * red until the next restart. MEASURED: a campus Mac took one 16 s stall at
+   * 14:25 — caused by two Screen Sharing sessions competing for the machine —
+   * recovered immediately to a 421 ms lag with an empty queue, and still showed
+   * red half an hour later. An operator who looks up and sees red wants to know
+   * what is wrong NOW; a light that cannot go back to green teaches them to
+   * ignore it.
+   *
+   * So the totals still travel, for the record and for the operator page, but
+   * the verdict asks when it last happened.
+   */
+  const recentCaptionOnly = es.captionOnlyAt && (Date.now() - es.captionOnlyAt) < TROUBLE_WINDOW_MS;
+  if ((es.lagMs || 0) > 12000 || recentCaptionOnly)
     return { level: 'red',
              detail: 'Falling badly behind — some sentences are captions only.' + loadNote() };
-  if ((es.lagMs || 0) > 6000 || (es.queue || 0) >= 2 || (es.degradedFinals || 0) > 0)
+  const recentDegraded = es.degradedAt && (Date.now() - es.degradedAt) < TROUBLE_WINDOW_MS;
+  if ((es.lagMs || 0) > 6000 || (es.queue || 0) >= 2 || recentDegraded)
     return { level: 'amber',
              detail: 'Running behind — idle languages may be trimmed to catch up.' + loadNote() };
   if (es.cloud && es.cloud.parked)

docs/PUBLIC-ACCESS.md

@@ -57,16 +57,45 @@ For a second campus, repeat with its own name — `translate-north`,
 
 ---
 
-## The other way: your existing nginx proxy
+## The other way: a name on your existing reverse proxy
 
-You already run a reverse proxy at `10.1.1.4` fronting `check.watersmedia.org`,
+You already run a reverse proxy on the NAS at `10.1.1.4` (Boba-Fett, Synology
-with split-horizon DNS. It can front this too, though it means the traffic goes
+DSM) fronting `check.watersmedia.org`. It can front this too, though it means
-through the NAS network to reach the booth Mac.
+the traffic goes through the NAS network to reach the booth Mac.
 
-Add a DNS record for `translate.watersmedia.org` — public to your WAN address,
+This is what North Attleboro runs as of 2026-10-15, in the **LAN-only** form:
-internal to `10.1.1.4` — and a server block. Copy the one that already serves
+`http://translate.watersmedia.org` works on the building's WiFi at both the
-the checklist app, change the name, and point it at the **booth Mac**, not the
+Apollo Beach and North Attleboro sites and nowhere else. Nothing was opened to
-NAS:
+the WAN. The pieces, with the exact scripts and a step-by-step runbook under
+[`examples/waters-church-na/`](examples/waters-church-na/README.md):
+
+| Piece | Where | What |
+| --- | --- | --- |
+| Local DNS | UniFi consoles at AB and NA (Settings → Policy Table → DNS) | A record `translate.watersmedia.org` → `10.1.1.4`, TTL 300, created with `unifi-dns-record.py` and a per-console Network API key |
+| Proxy rule | DSM → Control Panel → Login Portal → Advanced → Reverse Proxy | HTTP `translate.watersmedia.org:80` → HTTP `10.1.7.39:3100`, WebSocket headers, `X-Forwarded-Host: $http_host`, read timeout 3600 s |
+| HTTPS | `/usr/local/etc/nginx/sites-enabled/translate-redirect.conf` on the NAS | `https://` answers a 301 to `http://`; the name has no certificate, so browsers warn before the redirect |
+
+Both NAS pieces come from `boba-fett-translate-proxy.sh`, run as
+`ssh -t boba-fett sudo bash translate-proxy.sh` because DSM's reverse-proxy API
+is root-only. Going through DSM rather than a hand-edited nginx file means the
+rule shows up in the DSM UI and survives DSM updates. DSM adds `Host`,
+`X-Real-IP`, `X-Forwarded-For` and `X-Forwarded-Proto` on its own.
+
+Then: Operator page → **Public address** → `http://translate.watersmedia.org`
+(no trailing slash), so the QR code and poster use the name instead of the
+IP and port.
+
+To let people off-site use the same name later: add a public DNS record for
+`translate.watersmedia.org` pointing at the NA WAN address, get a certificate
+in DSM (Control Panel → Security → Certificate → Let's Encrypt), replace the
+redirect with an HTTPS rule to the same destination, and switch Public address
+to `https://`. The app behaves the same either way, including the `/operator`
+404 through a proxy.
+
+### If the proxy is plain nginx instead of DSM
+
+DSM writes most of this for you. On a hand-managed nginx the server block
+needs all of it:
 
 ```nginx
 server {
@@ -77,7 +106,7 @@ server {
     ssl_certificate_key /path/to/privkey.pem;
 
     location / {
-        proxy_pass http://10.6.3.222:3100;      # the booth Mac
+        proxy_pass http://10.1.7.39:3100;      # the booth Mac
 
         proxy_set_header Host              $host;
         proxy_set_header X-Real-IP         $remote_addr;
@@ -95,21 +124,36 @@ server {
         # the room reconnects at once.
         proxy_read_timeout  3600s;
         proxy_send_timeout  3600s;
+
+        # Live audio: do not hold HLS segments back.
+        proxy_buffering off;
     }
 }
 ```
 
-This has one real drawback the tunnel does not: the booth Mac needs a fixed
+Either way the booth Mac needs a fixed address or a DHCP reservation, or the
-address on the network, or the proxy breaks the next time DHCP moves it.
+proxy breaks the next time DHCP moves it.
 
 ---
 
 ## Checking it worked
 
 ```bash
-curl -s https://translate.watersmedia.org/healthz
+curl -s http://translate.watersmedia.org/healthz      # https:// once published
 ```
 
-Should return `{"ok":true,…}`. Then open the page on a phone **with WiFi turned
+Should return `{"ok":true,…}`, the same as `curl -s http://10.1.7.39:3100/healthz`.
-off** — that is the real test. If the page loads but captions never appear, the
+A proxy that passes the WebSocket answers `HTTP/1.1 101 Switching Protocols` to:
-WebSocket headers are missing.
+
+```bash
+curl -si -N -H 'Connection: Upgrade' -H 'Upgrade: websocket' \
+  -H 'Sec-WebSocket-Version: 13' -H 'Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==' \
+  http://translate.watersmedia.org/ws | head -1
+```
+
+Then open the page on a phone: for the LAN-only setup on the guest WiFi, for a
+published one **with WiFi turned off**. That is the real test. If the page loads
+but captions never appear, the WebSocket headers are missing. If phones on
+guest WiFi cannot connect at all, look at client isolation on that network
+before anything else. `/operator` returning 404 through the proxy is by design;
+staff use `http://10.1.7.39:3100/operator` on the LAN.

docs/examples/waters-church-na/README.md

@@ -0,0 +1,93 @@
+# translate.watersmedia.org — local DNS and reverse proxy
+
+Puts the Church Translate congregation page on a church hostname inside the
+Apollo Beach and North Attleboro networks. LAN only, plain HTTP, no WAN access.
+The app's own requirements are in [PUBLIC-ACCESS.md](../../PUBLIC-ACCESS.md). This
+was done on 2026-10-15 against app 3.23.2 build 121.
+
+| What | Value |
+| --- | --- |
+| App host | Translate Mac, North Attleboro LAN `10.1.7.39` |
+| App port | TCP 3100, plain HTTP + WebSocket on `/ws` (the only port to expose) |
+| Reverse proxy | Boba-Fett (Synology DSM 7.0.1), `10.1.1.4`, `ssh boba-fett` (port 202, user Michael) |
+| Hostname | `translate.watersmedia.org` |
+| Local DNS | A record on the AB and NA UDM Pro consoles → `10.1.1.4`, TTL 300 |
+| HTTPS | no certificate; `https://translate.watersmedia.org` answers 301 to `http://` |
+
+`curl -s http://10.1.7.39:3100/healthz` should return `{"ok":true,"live":true,...}`
+before anything below is tested.
+
+## 1. Local DNS on UniFi (AB and NA) — done 2026-10-15
+
+Each console has its own Network API key saved as
+`~/.config/unifi/<site>-network-api-key` (created in Settings → Control Plane →
+Integrations → Create API Key; all five sites are saved on Michael's Mac). The
+record was created with:
+
+```bash
+python3 docs/examples/waters-church-na/unifi-dns-record.py translate.watersmedia.org 10.1.1.4 --sites apollo-beach,north-attleboro
+```
+
+Resulting policy ids: AB `379f868d-a956-4ec1-acef-13557153ff3f`, NA
+`16bf7109-ec68-41f9-9c16-fa6a62471430`. Both gateways answer `10.1.1.4`.
+`--list` shows every DNS policy per console; `--dry-run` previews.
+
+Why Boba-Fett and not the Mac: the proxy owns the hostname so the port never
+appears in the URL, and AB clients reach `10.1.1.4` over the site-to-site link
+exactly as they already do for `chronicles-na.waterschurch.org`.
+
+## 2. Reverse proxy on Boba-Fett — done 2026-10-15
+
+[boba-fett-translate-proxy.sh](boba-fett-translate-proxy.sh) does the whole
+job and is idempotent. It was run on 2026-10-15; afterwards `/healthz`,
+`/api/languages`, the `/ws` upgrade (101) and the HTTPS 301 all checked out
+from both the AB and NA LANs, and `/operator` returned 404 as designed. A copy lives at `~/translate-proxy.sh` on Boba-Fett
+(Michael's home). DSM's reverse-proxy API and nginx reload both need root, and
+sudo there asks for Michael's password, so run it interactively:
+
+```bash
+ssh -t boba-fett sudo bash translate-proxy.sh
+```
+
+What it creates:
+
+| Piece | Where | Content |
+| --- | --- | --- |
+| DSM reverse proxy rule | Control Panel → Login Portal → Advanced → Reverse Proxy, "translate.watersmedia.org" | HTTP `translate.watersmedia.org:80` → HTTP `10.1.7.39:3100`; custom headers `Upgrade: $http_upgrade`, `Connection: $connection_upgrade` (WebSocket), `X-Forwarded-Host: $http_host`; read timeout 3600 s; HTTP/1.1 |
+| HTTPS redirect | `/usr/local/etc/nginx/sites-enabled/translate-redirect.conf` | `server { listen 443 ssl; server_name translate.watersmedia.org; return 301 http://translate.watersmedia.org$request_uri; }` |
+
+DSM already sends `Host`, `X-Real-IP`, `X-Forwarded-For` and
+`X-Forwarded-Proto` on every rule. The whole host lands on `/`, which the app
+requires. `/operator` returning 404 through the proxy is intended; staff use
+`http://10.1.7.39:3100/operator` on the LAN.
+
+To undo: delete the rule in the DSM Reverse Proxy list, remove the redirect
+file, and `sudo synosystemctl reload nginx`.
+
+Known limits:
+
+- The HTTPS redirect only runs after a TLS handshake with Boba-Fett's default
+  certificate, so a browser shows a certificate warning first. A trusted
+  certificate for this name would need DNS-01 validation against the
+  watersmedia.org zone (cPanel), which is not set up.
+- DSM's reverse proxy cannot turn `proxy_buffering` off. HLS segments under
+  `/live/` are small discrete files and the WebSocket is a tunnel after upgrade,
+  so the effect is minor; if audio lag is reported, that is the first thing to
+  revisit with a hand-written nginx include.
+
+## 3. After the proxy answers
+
+1. On the NA LAN open `http://10.1.7.39:3100/operator`, enter the operator PIN,
+   and set **Public address** to `http://translate.watersmedia.org` (no trailing
+   slash). Otherwise the QR code and poster keep advertising `10.1.7.39:3100`.
+2. Verify from a machine at AB and one at NA:
+
+```bash
+curl -s http://translate.watersmedia.org/healthz
+```
+
+   It must return the same JSON as the direct `10.1.7.39:3100` check.
+3. Open the page on a phone on guest wifi and confirm captions appear
+   (WebSocket) and spoken audio plays (`/live/`). If phones cannot connect at
+   all, look at client isolation on the guest network and the guest → `10.1.1.4`
+   firewall rule before anything else.

docs/examples/waters-church-na/boba-fett-translate-proxy.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+# translate.watersmedia.org on Boba-Fett (Synology DSM 7.0.1).
+#
+# 1. Creates the DSM reverse-proxy rule  http://translate.watersmedia.org:80
+#    -> http://10.1.7.39:3100 (Church Translate) with WebSocket headers,
+#    X-Forwarded-Host, and a 3600 s read timeout, through DSM's own API so it
+#    shows up in Control Panel -> Login Portal -> Advanced -> Reverse Proxy.
+# 2. Adds an nginx server block that answers https://translate.watersmedia.org
+#    with a 301 to the http:// address (the name is LAN-only and has no
+#    certificate, so browsers warn before the redirect; that is expected).
+# 3. Tests the nginx config, reloads it, and prints a local check.
+#
+# Idempotent: re-running skips the rule if it exists and rewrites the redirect.
+# Run on Boba-Fett as root:   sudo bash ~/translate-proxy.sh
+set -euo pipefail
+
+NAME=translate.watersmedia.org
+BACKEND_IP=10.1.7.39
+BACKEND_PORT=3100
+RP_JSON=/usr/syno/etc/www/ReverseProxy.json
+REDIRECT_CONF=/usr/local/etc/nginx/sites-enabled/translate-redirect.conf
+
+if [ "$(id -u)" != 0 ]; then
+  echo "Run with sudo: sudo bash $0" >&2
+  exit 1
+fi
+
+echo "== 1. DSM reverse proxy rule for $NAME"
+if grep -q "\"fqdn\" : \"$NAME\"" "$RP_JSON"; then
+  echo "   rule already present in $RP_JSON, skipping create"
+else
+  # $http_upgrade etc. are nginx variables and must reach DSM literally.
+  ENTRY='{"description":"translate.watersmedia.org","frontend":{"acl":null,"fqdn":"translate.watersmedia.org","https":{"hsts":false},"port":80,"protocol":0},"backend":{"fqdn":"10.1.7.39","port":3100,"protocol":0},"customize_headers":[{"name":"Upgrade","value":"$http_upgrade"},{"name":"Connection","value":"$connection_upgrade"},{"name":"X-Forwarded-Host","value":"$http_host"}],"proxy_connect_timeout":60,"proxy_read_timeout":3600,"proxy_send_timeout":60,"proxy_http_version":1,"proxy_intercept_errors":false}'
+  OUT=$(/usr/syno/bin/synowebapi --exec api=SYNO.Core.AppPortal.ReverseProxy method=create version=1 entry="$ENTRY" 2>&1 || true)
+  echo "   $OUT"
+  if ! echo "$OUT" | grep -q '"success" *: *true'; then
+    echo "   synowebapi did not report success; stopping before the redirect." >&2
+    exit 1
+  fi
+fi
+
+echo "== 2. HTTPS -> HTTP redirect block"
+cat > "$REDIRECT_CONF" <<'EOF'
+# translate.watersmedia.org is served over plain HTTP on the LAN (no certificate).
+# Anyone who types https:// is sent to the http:// address. Added 2026-10-15.
+server {
+    listen 443 ssl;
+    server_name translate.watersmedia.org;
+    return 301 http://translate.watersmedia.org$request_uri;
+}
+EOF
+echo "   wrote $REDIRECT_CONF"
+
+echo "== 3. nginx test + reload"
+/usr/bin/nginx -t
+/usr/syno/bin/synosystemctl reload nginx 2>/dev/null || /usr/bin/nginx -s reload
+echo "   reloaded"
+
+echo "== 4. Local checks from Boba-Fett"
+echo -n "   http  /healthz : "
+curl -s -m 5 --resolve "$NAME:80:127.0.0.1" "http://$NAME/healthz"; echo
+echo -n "   https /        : "
+curl -sk -m 5 -o /dev/null -w '%{http_code} -> %{redirect_url}\n' --resolve "$NAME:443:127.0.0.1" "https://$NAME/"
+echo "done"

docs/examples/waters-church-na/unifi-dns-record.py

@@ -0,0 +1,211 @@
+#!/usr/bin/env python3
+"""Create or update a local DNS A record on every Waters Church UniFi console.
+
+Usage:
+    docs/examples/waters-church-na/unifi-dns-record.py repo.watersmedia.org 10.3.8.90
+    docs/examples/waters-church-na/unifi-dns-record.py repo.watersmedia.org 10.3.8.90 --sites apollo-beach,norwood
+    docs/examples/waters-church-na/unifi-dns-record.py repo.watersmedia.org 10.3.8.90 --dry-run
+    docs/examples/waters-church-na/unifi-dns-record.py --list                # show every DNS policy per site
+
+Each console needs its own Network API key (Site Manager keys from api.ui.com are
+rejected locally). Create one per console in the UniFi UI:
+    Settings -> Control Plane -> Integrations -> Create API Key
+and save it, one line, mode 600, as:
+    ~/.config/unifi/<site>-network-api-key
+where <site> is one of the names in SITES below.
+
+The script talks to the console's Network Integration API
+(/proxy/network/integration/v1/sites/{siteId}/dns/policies) over the LAN /
+site-to-site links, so it must run from inside the church network. Gateways use
+self-signed certificates, so certificate verification is disabled for them.
+Keys are read from disk and sent only in the X-API-KEY header; they are never
+printed.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import ssl
+import subprocess
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+# Console name -> gateway LAN address (UDM Pro at each site, 2026-10).
+SITES = {
+    "apollo-beach": "10.3.0.1",
+    "north-attleboro": "10.1.0.1",
+    "woonsocket": "10.2.0.1",
+    "norwood": "10.4.0.1",
+    "southcoast": "10.5.0.1",
+}
+
+KEY_DIR = os.path.expanduser("~/.config/unifi")
+TIMEOUT = 15
+
+
+class ApiError(RuntimeError):
+    pass
+
+
+class Console:
+    def __init__(self, name: str, gateway: str):
+        self.name = name
+        self.gateway = gateway
+        self.base = f"https://{gateway}/proxy/network/integration/v1"
+        self.key_path = os.path.join(KEY_DIR, f"{name}-network-api-key")
+        self.key = self._load_key()
+        self.ctx = ssl.create_default_context()
+        self.ctx.check_hostname = False
+        self.ctx.verify_mode = ssl.CERT_NONE
+
+    def _load_key(self) -> str | None:
+        try:
+            with open(self.key_path) as fh:
+                key = fh.read().strip()
+        except FileNotFoundError:
+            return None
+        return key or None
+
+    def request(self, method: str, path: str, body: dict | None = None, query: dict | None = None):
+        url = self.base + path
+        if query:
+            url += "?" + urllib.parse.urlencode(query)
+        data = json.dumps(body).encode() if body is not None else None
+        req = urllib.request.Request(url, data=data, method=method)
+        req.add_header("X-API-KEY", self.key or "")
+        req.add_header("Accept", "application/json")
+        if data is not None:
+            req.add_header("Content-Type", "application/json")
+        try:
+            with urllib.request.urlopen(req, timeout=TIMEOUT, context=self.ctx) as resp:
+                raw = resp.read()
+                return json.loads(raw) if raw else None
+        except urllib.error.HTTPError as e:
+            detail = e.read().decode(errors="replace")[:300]
+            raise ApiError(f"{method} {path} -> HTTP {e.code}: {detail}") from None
+        except (urllib.error.URLError, TimeoutError, OSError) as e:
+            raise ApiError(f"{method} {path} -> {e}") from None
+
+    def site_id(self) -> str:
+        sites = self.request("GET", "/sites").get("data", [])
+        if not sites:
+            raise ApiError("console reports no sites")
+        for s in sites:
+            if s.get("internalReference") == "default" or s.get("name") == "default":
+                return s["id"]
+        return sites[0]["id"]
+
+    def dns_policies(self, site_id: str) -> list[dict]:
+        out: list[dict] = []
+        offset, limit = 0, 100
+        while True:
+            page = self.request("GET", f"/sites/{site_id}/dns/policies", query={"offset": offset, "limit": limit})
+            data = page.get("data", [])
+            out.extend(data)
+            total = page.get("totalCount")
+            offset += len(data)
+            if not data or (total is not None and offset >= total) or len(data) < limit:
+                return out
+
+    def ensure_a_record(self, site_id: str, fqdn: str, ipv4: str, ttl: int | None, dry_run: bool) -> str:
+        existing = [p for p in self.dns_policies(site_id) if p.get("domain", "").lower() == fqdn.lower()]
+        a_records = [p for p in existing if p.get("type") == "A_RECORD"]
+        other = [p for p in existing if p.get("type") != "A_RECORD"]
+        if other:
+            kinds = ", ".join(sorted({p.get("type", "?") for p in other}))
+            return f"SKIPPED: {fqdn} already has a non-A policy here ({kinds}); fix by hand"
+        if len(a_records) > 1:
+            return f"SKIPPED: {len(a_records)} A records already exist for {fqdn}; fix by hand"
+
+        body = {"type": "A_RECORD", "enabled": True, "domain": fqdn, "ipv4Address": ipv4}
+        if ttl:
+            body["ttlSeconds"] = ttl
+
+        if a_records:
+            cur = a_records[0]
+            same = cur.get("ipv4Address") == ipv4 and cur.get("enabled", True) and (ttl is None or cur.get("ttlSeconds") == ttl)
+            if same:
+                return f"unchanged: {fqdn} -> {ipv4} already present (id {cur.get('id')})"
+            if dry_run:
+                return f"would UPDATE {fqdn}: {cur.get('ipv4Address')} -> {ipv4} (id {cur.get('id')})"
+            self.request("PUT", f"/sites/{site_id}/dns/policies/{cur['id']}", body=body)
+            return f"UPDATED {fqdn}: {cur.get('ipv4Address')} -> {ipv4}"
+
+        if dry_run:
+            return f"would CREATE {fqdn} -> {ipv4}"
+        created = self.request("POST", f"/sites/{site_id}/dns/policies", body=body)
+        return f"CREATED {fqdn} -> {ipv4} (id {created.get('id') if isinstance(created, dict) else '?'})"
+
+
+def resolve_via(gateway: str, fqdn: str) -> str:
+    """Ask the gateway's resolver directly; returns the answer or a short status."""
+    dig = shutil.which("dig")
+    if not dig:
+        return "(dig not installed)"
+    try:
+        res = subprocess.run([dig, "+short", "+time=3", "+tries=1", f"@{gateway}", fqdn, "A"],
+                             capture_output=True, text=True, timeout=10)
+    except subprocess.TimeoutExpired:
+        return "(timeout)"
+    answer = res.stdout.strip().replace("\n", " ")
+    return answer or "(no answer)"
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("fqdn", nargs="?", help="host name, e.g. repo.watersmedia.org")
+    ap.add_argument("ipv4", nargs="?", help="IPv4 address the name should resolve to")
+    ap.add_argument("--sites", default=",".join(SITES), help="comma-separated subset of: " + ", ".join(SITES))
+    ap.add_argument("--ttl", type=int, default=300, help="TTL in seconds (default 300; the API rejects a null TTL)")
+    ap.add_argument("--dry-run", action="store_true", help="show what would change without writing")
+    ap.add_argument("--list", action="store_true", help="print every DNS policy on each console and exit")
+    ap.add_argument("--no-verify", action="store_true", help="skip the dig check after writing")
+    args = ap.parse_args()
+
+    if not args.list and (not args.fqdn or not args.ipv4):
+        ap.error("fqdn and ipv4 are required unless --list is given")
+
+    names = [s.strip() for s in args.sites.split(",") if s.strip()]
+    unknown = [n for n in names if n not in SITES]
+    if unknown:
+        ap.error(f"unknown site(s): {', '.join(unknown)}")
+
+    failures = 0
+    for name in names:
+        console = Console(name, SITES[name])
+        label = f"{name:16s} {console.gateway:10s}"
+        if not console.key:
+            print(f"{label} NO KEY: save the console's API key to {console.key_path}")
+            failures += 1
+            continue
+        try:
+            site_id = console.site_id()
+            if args.list:
+                policies = console.dns_policies(site_id)
+                print(f"{label} {len(policies)} DNS policies")
+                for p in sorted(policies, key=lambda p: p.get("domain", "")):
+                    target = p.get("ipv4Address") or p.get("ipv6Address") or p.get("targetDomain") or p.get("text") or ""
+                    flag = "" if p.get("enabled", True) else " (disabled)"
+                    print(f"    {p.get('type', '?'):12s} {p.get('domain', ''):40s} -> {target}{flag}")
+                continue
+            result = console.ensure_a_record(site_id, args.fqdn, args.ipv4, args.ttl, args.dry_run)
+            line = f"{label} {result}"
+            if not args.dry_run and not args.no_verify and not result.startswith("SKIPPED"):
+                line += f" | dig @{console.gateway}: {resolve_via(console.gateway, args.fqdn)}"
+            print(line)
+            if result.startswith("SKIPPED"):
+                failures += 1
+        except ApiError as e:
+            print(f"{label} FAILED: {e}")
+            failures += 1
+
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())