A stumble should not colour the desk red for the rest of the service

Build 121. Three things a single 16 s stall did that it should not have.

captionOnlyFinals is a running total for the whole service, and the verdict
tested it for more than zero — so one stale sentence pinned the light red until
the next restart. MEASURED: a campus Mac stalled once at 14:25, recovered
immediately to a 421 ms lag with an empty queue, and still showed red half an
hour later, with the operator quite reasonably reporting it as behind. The
totals still travel for the record; the verdict now asks when it last happened,
inside a two-minute window. A light that cannot go back to green teaches the
desk to ignore it.

That sentence also had its translations skipped. Dropping them is how the app
catches up — the listener has moved on and the time is better spent on what is
behind it. But that reasoning needs a backlog, and the queue was EMPTY: the room
lost a line of the sermon to a catch-up that had nothing to catch up on. It now
skips only when something is actually waiting, and says plainly when it chooses
to translate a late sentence anyway.

The stall itself was not the app. Two Screen Sharing sessions were taking as
much of the machine as the app was, with WindowServer on top — watching a campus
Mac during a service measurably degrades it.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/engine.js

@@ -190,7 +190,9 @@ const state = {
 
   /* health counters, reported to the operator page */
   degradedFinals: 0,         // finals that shipped with fewer languages (stale)
-  captionOnlyFinals: 0       // finals that shipped with no translations at all
+  captionOnlyFinals: 0,      // finals that shipped with no translations at all
+  captionOnlyAt: 0,          // when the last one happened
+  degradedAt: 0
 };
 
 // ────────────────────────────────────────────────────────── logging
@@ -891,12 +893,54 @@ async function respawnWorker(worker) {
         log('the speech model is too heavy for this machine — switching to ' +
             lighter.split('/').pop(), 'err');
       }
+      /*
+       * A LIGHTER MODEL GETS THE PREVIEW'S DTYPE, NOT THE BIG ONE'S.
+       *
+       * asrDtype is chosen for the accurate model and its weights are cached against
+       * it. Reusing it when stepping down asks for a variant that was never
+       * downloaded — and those weights live in the browser cache, with no network at
+       * a venue — so the step-down failed and the ladder could not move at all.
+       *
+       * MEASURED on a campus Mac: the cache held whisper-large-v3-turbo as
+       * encoder_model_q4f16.onnx (its asrDtype) and whisper-small as
+       * encoder_model_fp16.onnx (the preview's). Stepping down asked for
+       * whisper-small at q4f16, a file that is not there; it went to the network,
+       * failed, and logged "Unsupported model type: whisper" three times a minute
+       * for ever while the service sat dead.
+       *
+       * The preview runs whisper-small on exactly these settings every service, so
+       * they are the one combination this machine is known to be able to load.
+       */
+      const stepDtype = lighter
+        ? resolveDtype(cfg.asrFastDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' })
+        : resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' });
       state.asrWorker = await loadWorkerWithFallback('speech', 'automatic-speech-recognition',
         state.asrModelInUse || cfg.asrModel,
-        resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' }),
+        stepDtype,
         { encoder_model: 'q4', decoder_model_merged: 'q4' });
       log('speech model restarted' + (lighter ? ' on ' + lighter.split('/').pop() : ''), 'ok');
       state.slowRun = 0;
+      if (lighter) {
+        /*
+         * THROW AWAY THE BACKLOG THAT CAUSED THIS, AND LET THE NEW MODEL SETTLE.
+         *
+         * Swapping the model does nothing about the queue that triggered the swap.
+         * Those chunks are already past the caption-only cutoff, so they can no
+         * longer be translated in time and are worth nothing to the room — but the
+         * first of them gets handed to a model that has just loaded, takes the full
+         * worker timeout, and trips the absolute rule. MEASURED: a campus Mac fell
+         * large-v3-turbo -> small -> base in 34 seconds exactly that way, landing on
+         * a model SLOWER than the one it started from.
+         *
+         * So drop them, and refuse to step down again for a grace window, so the new
+         * model is judged on sentences it was actually given.
+         */
+        const dropped = queue.length;
+        queue.length = 0;
+        state.steppedDownAt = Date.now();
+        if (dropped) log('dropped ' + dropped + ' stale sentence' + (dropped === 1 ? '' : 's') +
+                         ' the old model never reached', 'warn');
+      }
     }
   } catch (err) {
     log('model restart failed: ' + err.message, 'err');
@@ -1794,6 +1838,7 @@ async function pump() {
 
   state.busy = true;
   const started = Date.now();
+  let jobAudioMs = 0;
   try {
     /* job.at is when the audio ENDED. Live translation of stale audio is
        worthless, but the WORDS are not: however old the job, it still gets
@@ -1805,6 +1850,9 @@ async function pump() {
     /* Near-silence telemetry (log-only this release): a chunk whose average
        level barely clears the noise floor is hallucination fuel. */
     const audio = flatten(job.frames);
+    /* How much speech this chunk actually holds. The step-down below compares
+       against it, because "slow" only means anything relative to the audio. */
+    jobAudioMs = (audio.length / SAMPLE_RATE) * 1000;
     let sum = 0;
     for (let i = 0; i < audio.length; i += 4) sum += audio[i] * audio[i];
     const meanRms = Math.sqrt(sum / Math.ceil(audio.length / 4));
@@ -1845,15 +1893,33 @@ async function pump() {
       const sentences = collapseLoops(splitSentences(text));
 
       let targets = orderedTargets();
-      if (ageSec > 12) {
+      /*
+       * SKIPPING A TRANSLATION ONLY BUYS SOMETHING IF THERE IS A QUEUE TO DRAIN.
+       *
+       * Dropping the translations of stale audio is how the app catches up: the
+       * listener has moved on, so the work is wasted and the time is better spent on
+       * the sentences behind it. That reasoning needs a backlog. With nothing waiting
+       * behind it, skipping costs the room a sentence and saves nothing at all.
+       *
+       * MEASURED: a campus Mac took one 16 s stall — two Screen Sharing sessions
+       * competing for the machine — and skipped that sentence's translations with an
+       * EMPTY queue. The room lost a line of the sermon to a catch-up that had
+       * nothing to catch up on.
+       */
+      if (ageSec > 12 && queue.length > 0) {
         targets = [];
         state.captionOnlyFinals++;
-        log('audio ' + ageSec.toFixed(0) + 's stale — caption only, translations skipped', 'err');
+        state.captionOnlyAt = Date.now();
+        log('audio ' + ageSec.toFixed(0) + 's stale with ' + queue.length + ' waiting — ' +
+            'caption only, translations skipped', 'err');
+      } else if (ageSec > 12) {
+        /* Late, but nothing is behind it: say so, and translate it anyway. */
+        log('audio ' + ageSec.toFixed(0) + 's late, but nothing is queued — translating it anyway', 'warn');
       } else if (ageSec > 6) {
         const listening = new Set((state.byLanguage || []).map((r) => r.lang));
         const before = targets.length;
         targets = targets.filter((t) => listening.has(t));
-        if (targets.length < before) state.degradedFinals++;
+        if (targets.length < before) { state.degradedFinals++; state.degradedAt = Date.now(); }
       }
 
       /* The segment's own loudness, measured once and shared by every sentence
@@ -1908,11 +1974,56 @@ async function pump() {
      * talk, and the backlog only grows — step down before the service is lost
      * rather than after. Measured on the Intel laptop: 12-25 s, every sentence.
      */
-    if (took > 8000) {
+    /*
+     * SLOWER THAN THE AUDIO IT COVERS — not slower than some constant.
+     *
+     * This used to fire only past 8 s, on the reasoning that 8 s meant slower
+     * than people talk. That was true when a chunk could be 8 s long. Chunks
+     * are capped at HARD_CUT_MAX, 5.5 s, so a 4.5 s chunk taking 4.3 s is
+     * already losing ground and never came near the old threshold.
+     *
+     * MEASURED on a campus M4 Pro: every sentence 4.2-4.7 s against roughly
+     * 4-5 s of audio, a queue 46 deep and a lag of 152 s still climbing — with
+     * nobody listening and no language being translated, so this was
+     * transcription alone. The one mechanism built to catch exactly that could
+     * not fire, because 4.4 s is not 8 s.
+     *
+     * The queue test is what separates "this machine cannot keep up" from one
+     * slow sentence: a backlog that is three deep and being fed faster than it
+     * drains does not recover on its own.
+     */
+    /*
+     * SLOWER THAN THE AUDIO IT COVERS, AND LANDING LATE.
+     *
+     * This used to fire only past 8 s, on the reasoning that 8 s meant slower
+     * than people talk. That held when a chunk could be 8 s long; chunks are
+     * capped at HARD_CUT_MAX, 5.5 s, so a 4.4 s sentence is already losing and
+     * never came near the old threshold.
+     *
+     * Both numbers below were measured on a campus M4 Pro, and the second one
+     * only after a first attempt got it wrong. 0.85, because that Mac ran 4.4 s
+     * against 4.5 s of audio — nominally FASTER than real time while losing
+     * badly, since the work outside this timer (segmenting, the preview model
+     * sharing the GPU, the scheduler) loses on its own.
+     *
+     * The second test was a queue depth, and watching that Mac go red disproved
+     * it: the queue sat at 2-3 while the lag held at 15 s and the verdict was
+     * already red. A machine can be persistently late without ever stacking work
+     * up — it finishes each sentence just slightly too slowly, for ever. So the
+     * test is how far behind the speaker the sentence actually landed, which is
+     * what the operator is complaining about in the first place.
+     */
+    /* A model that loaded seconds ago has not had a fair sentence yet. */
+    const settling = state.steppedDownAt && (Date.now() - state.steppedDownAt) < STEP_SETTLE_MS;
+    const behindMs = Date.now() - job.at;
+    const losingGround = jobAudioMs > 0 && took > jobAudioMs * 0.85 && behindMs > 8000;
+    if (!settling && (took > 8000 || losingGround)) {
       state.slowRun = (state.slowRun || 0) + 1;
       if (state.slowRun >= 3 && !respawning && lighterAsr(state.asrModelInUse || state.cfg.asrModel)) {
         state.slowRun = 0;
-        log('transcription is slower than people are speaking — stepping down to a lighter model', 'err');
+        log('transcription is slower than people are speaking (' + took + ' ms for ' +
+            Math.round(jobAudioMs) + ' ms of audio, ' + Math.round(behindMs / 1000) + 's behind) — ' +
+            'stepping down to a lighter model', 'err');
         respawnWorker(state.asrWorker);
       }
     } else state.slowRun = 0;
@@ -2423,6 +2534,11 @@ function monitorTranslation(cid, lang, text) {
  * It says it once per episode, and names both numbers so the operator can act
  * on it rather than go looking.
  */
+/* How long a freshly swapped speech model is left alone. Long enough to cover a
+   cold first inference, short enough that a genuinely hopeless model still falls
+   through the ladder within a sentence or two. */
+const STEP_SETTLE_MS = 20000;
+
 const DEAF_AFTER_MS = 90000;
 setInterval(() => {
   try {
@@ -2662,6 +2778,7 @@ function goLive(on) {
   state.live = on;
   if (on) {
     state.degradedFinals = 0; state.captionOnlyFinals = 0; state.fragmentFinals = 0;
+    state.captionOnlyAt = 0; state.degradedAt = 0;
     /* The watchdog measures from the start of the service, not from the last
        sentence of the previous one. */
     state.lastFinalAt = Date.now();
@@ -2736,6 +2853,10 @@ function reportStatus() {
       queue: queue.length,
       degradedFinals: state.degradedFinals,
       captionOnlyFinals: state.captionOnlyFinals,
+      /* WHEN, not just how many — the verdict needs to know whether this is
+         happening now or happened once an hour ago. */
+      captionOnlyAt: state.captionOnlyAt || 0,
+      degradedAt: state.degradedAt || 0,
       fragmentFinals: state.fragmentFinals || 0,
       cloud: state.cloudStats || null
     }

app/package.json

@@ -2,7 +2,7 @@
   "name": "church-translate-engine",
   "productName": "Church Translate",
   "version": "3.23.2",
-  "buildNumber": 116,
+  "buildNumber": 121,
   "private": true,
   "description": "Live sermon translation — self-contained: captures audio, transcribes, translates, and serves the congregation page itself",
   "main": "main.js",
@@ -106,7 +106,7 @@
       ]
     },
     "asar": false,
-    "buildVersion": "116",
+    "buildVersion": "121",
     "win": {
       "target": [
         {

app/server/server.js

@@ -510,6 +510,11 @@ const ENGINE_STALE_MS = 12000;
  * A campus reported "constantly behind, in the red" and the cause took an
  * afternoon to find; it was eight languages on hardware tuned against one.
  */
+/* How long a stumble keeps colouring the verdict. Long enough that a real
+   problem stays visible while someone walks to the desk, short enough that a
+   one-off does not outlive the sentence after it. */
+const TROUBLE_WINDOW_MS = 120000;
+
 function loadNote() {
   const n = activeLanguages().length;
   if (n < 2) return '';
@@ -523,10 +528,27 @@ function healthVerdict(connected, stale) {
   if (stale)                    return { level: 'red',   detail: 'The translation app has stopped responding.' };
   if (!state.live)              return { level: 'green', detail: 'Ready.' };
   if (!es.modelsReady)          return { level: 'amber', detail: 'Models are still loading.' };
-  if ((es.lagMs || 0) > 12000 || (es.captionOnlyFinals || 0) > 0)
+  /*
+   * RECENTLY, not ever.
+   *
+   * captionOnlyFinals is a running total for the whole service, and testing it
+   * for "more than zero" meant a single stale sentence pinned the desk's light
+   * red until the next restart. MEASURED: a campus Mac took one 16 s stall at
+   * 14:25 — caused by two Screen Sharing sessions competing for the machine —
+   * recovered immediately to a 421 ms lag with an empty queue, and still showed
+   * red half an hour later. An operator who looks up and sees red wants to know
+   * what is wrong NOW; a light that cannot go back to green teaches them to
+   * ignore it.
+   *
+   * So the totals still travel, for the record and for the operator page, but
+   * the verdict asks when it last happened.
+   */
+  const recentCaptionOnly = es.captionOnlyAt && (Date.now() - es.captionOnlyAt) < TROUBLE_WINDOW_MS;
+  if ((es.lagMs || 0) > 12000 || recentCaptionOnly)
     return { level: 'red',
              detail: 'Falling badly behind — some sentences are captions only.' + loadNote() };
-  if ((es.lagMs || 0) > 6000 || (es.queue || 0) >= 2 || (es.degradedFinals || 0) > 0)
+  const recentDegraded = es.degradedAt && (Date.now() - es.degradedAt) < TROUBLE_WINDOW_MS;
+  if ((es.lagMs || 0) > 6000 || (es.queue || 0) >= 2 || recentDegraded)
     return { level: 'amber',
              detail: 'Running behind — idle languages may be trimmed to catch up.' + loadNote() };
   if (es.cloud && es.cloud.parked)