Step down when losing ground, not when slower than a constant

Build 117.

A campus M4 Pro sat in the red with nobody listening and no language being
translated — so transcription alone. Measured over SSH: sentences 4.2-4.7 s
against roughly 4-5 s of audio, queue 46 deep, lag 152 s and still climbing.

The mechanism built to catch exactly this could not fire. It triggers past 8 s,
on the reasoning that 8 s means slower than people talk. That held when a chunk
could be 8 s long; chunks are capped at HARD_CUT_MAX, 5.5 s. A 4.4 s sentence is
already losing and never comes near 8 s, so the ladder never stepped down and
the backlog grew without bound.

It now compares against the audio the chunk actually held. Two details are
measured rather than chosen: the margin is 0.85, because the Mini's 4.4 s
against 4.5 s was NOMINALLY faster than real time while still losing — the work
outside that timer (segmenting, the preview model sharing the GPU, the
scheduler) loses on its own. And it requires a backlog of five, so one slow
sentence on a healthy machine touches nothing. Checked against eight cases
including the Mini's real numbers and a healthy M2 Max.

The old absolute rule stays for a model that is genuinely hung.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/engine.js

@@ -1794,6 +1794,7 @@ async function pump() {
 
   state.busy = true;
   const started = Date.now();
+  let jobAudioMs = 0;
   try {
     /* job.at is when the audio ENDED. Live translation of stale audio is
        worthless, but the WORDS are not: however old the job, it still gets
@@ -1805,6 +1806,9 @@ async function pump() {
     /* Near-silence telemetry (log-only this release): a chunk whose average
        level barely clears the noise floor is hallucination fuel. */
     const audio = flatten(job.frames);
+    /* How much speech this chunk actually holds. The step-down below compares
+       against it, because "slow" only means anything relative to the audio. */
+    jobAudioMs = (audio.length / SAMPLE_RATE) * 1000;
     let sum = 0;
     for (let i = 0; i < audio.length; i += 4) sum += audio[i] * audio[i];
     const meanRms = Math.sqrt(sum / Math.ceil(audio.length / 4));
@@ -1908,11 +1912,43 @@ async function pump() {
      * talk, and the backlog only grows — step down before the service is lost
      * rather than after. Measured on the Intel laptop: 12-25 s, every sentence.
      */
-    if (took > 8000) {
+    /*
+     * SLOWER THAN THE AUDIO IT COVERS — not slower than some constant.
+     *
+     * This used to fire only past 8 s, on the reasoning that 8 s meant slower
+     * than people talk. That was true when a chunk could be 8 s long. Chunks
+     * are capped at HARD_CUT_MAX, 5.5 s, so a 4.5 s chunk taking 4.3 s is
+     * already losing ground and never came near the old threshold.
+     *
+     * MEASURED on a campus M4 Pro: every sentence 4.2-4.7 s against roughly
+     * 4-5 s of audio, a queue 46 deep and a lag of 152 s still climbing — with
+     * nobody listening and no language being translated, so this was
+     * transcription alone. The one mechanism built to catch exactly that could
+     * not fire, because 4.4 s is not 8 s.
+     *
+     * The queue test is what separates "this machine cannot keep up" from one
+     * slow sentence: a backlog that is three deep and being fed faster than it
+     * drains does not recover on its own.
+     */
+    /*
+     * 0.85 and a backlog of five, both measured rather than chosen.
+     *
+     * The Mini's sentences ran 4.4 s against 4.5 s of audio — nominally FASTER
+     * than real time, while the queue sat at 46 and the lag climbed past 150 s.
+     * Per-sentence time alone therefore misses it: the work outside this timer
+     * (segmenting, the preview model sharing the GPU, the scheduler) is enough
+     * to lose on its own. The margin catches "near enough to real time to be
+     * losing"; the queue depth is the proof that it actually is, and keeps one
+     * slow sentence on a healthy machine from touching anything.
+     */
+    const losingGround = jobAudioMs > 0 && took > jobAudioMs * 0.85 && queue.length >= 5;
+    if (took > 8000 || losingGround) {
       state.slowRun = (state.slowRun || 0) + 1;
       if (state.slowRun >= 3 && !respawning && lighterAsr(state.asrModelInUse || state.cfg.asrModel)) {
         state.slowRun = 0;
-        log('transcription is slower than people are speaking — stepping down to a lighter model', 'err');
+        log('transcription is slower than people are speaking (' + took + ' ms for ' +
+            Math.round(jobAudioMs) + ' ms of audio, ' + queue.length + ' waiting) — ' +
+            'stepping down to a lighter model', 'err');
         respawnWorker(state.asrWorker);
       }
     } else state.slowRun = 0;
@@ -2983,14 +3019,26 @@ async function downloadUpdate() {
   try {
     const r = await window.engineHost.updateDownload(updFound.asset.url, updFound.asset.name);
     if (!r.ok) { $('updProgress').textContent = r.error || 'Download failed.'; btn.disabled = false; return; }
-    $('updProgress').textContent = 'Saved to Downloads.';
-    /* Say what is left to do. Nothing installs itself here on purpose — see
-       the note on the main-process side. */
-    $('updWhat').textContent = 'Downloaded ' + updFound.tag + ' (' + fmtMB(r.bytes) + '). ' +
-      'Quit Church Translate, open the disk image, and drag the app over the one in ' +
-      'Applications. Your settings, models and voices are untouched.';
+      $('updProgress').textContent = 'Downloaded. Installing\u2026';
       log('update ' + updFound.tag + ' downloaded to ' + r.path, 'ok');
+
+      /*
+       * Straight into the install. The app quits, a helper swaps the bundle and
+       * reopens it — so from here the window closing IS the update working, and
+       * saying so before it happens stops that looking like a crash.
+       */
+      $('updWhat').textContent = 'Installing ' + updFound.tag + '. Church Translate will close and ' +
+        'reopen by itself in a few seconds. Your settings, models and voices are untouched.';
+      const ins = await window.engineHost.updateInstall(r.path);
+      if (!ins || !ins.ok) {
+        /* Still useful: the file is downloaded, so fall back to the manual route
+           rather than leaving them with nothing. */
+        $('updProgress').textContent = (ins && ins.error) || 'Could not install.';
+        $('updWhat').textContent = 'Downloaded, but it could not install itself. Quit Church ' +
+          'Translate, open the disk image in Downloads, and drag the app over the one in Applications.';
         try { await window.engineHost.updateReveal(r.path); } catch (_) {}
+        btn.disabled = false;
+      }
   } catch (e) {
     $('updProgress').textContent = 'Download failed: ' + e.message;
     btn.disabled = false;

app/index.html

@@ -391,7 +391,7 @@
           <div id="updFound" hidden>
             <p class="hint" id="updWhat"></p>
             <div class="row">
-              <button class="btn" id="updGet">Download it</button>
+              <button class="btn" id="updGet">Download and install</button>
               <span class="hint" id="updProgress"></span>
             </div>
             <details id="updNotesBox"><summary class="hint">What changed</summary>

app/main.js

@@ -440,6 +440,11 @@ function startServer() {
     execPath: nodeBin,
     env: Object.assign({}, process.env, {
       CT_FFMPEG: ffmpegPathForChildren(),
+      /* So /healthz can report the APP’s version rather than only an asset
+         hash. A campus Mac’s build was otherwise unknowable from the network,
+         which made confirming an update a matter of inference. */
+      CT_APP_VERSION: appBuild().version,
+      CT_APP_BUILD: appBuild().build,
       PORT: String(port),
       DATA_DIR: dataDir(),
       /* Where the spoken clips land, so the server can hand them to phones. */
@@ -2259,6 +2264,97 @@ ipcMain.handle('update:download', async (_e, url, name) => {
   });
 });
 
+/*
+ * ─────────────────────────── installing it, in place ──────────────────────
+ *
+ * An app cannot replace itself while it is running, so the swap is done by a
+ * small script that outlives us: it waits for this process to exit, moves the
+ * old bundle aside, copies the new one in, and relaunches.
+ *
+ * MOVED ASIDE, NOT DELETED. If the copy fails half way — a full disk, a bad
+ * download, the volume going away — the old app is put straight back and
+ * relaunched. The failure mode this is written against is a campus Mac with no
+ * working app and a service in an hour, which is far worse than not updating.
+ *
+ * No sudo: /Applications is admin-writable on macOS and the bundle belongs to
+ * the operator, so nothing here needs a password. If that is ever not true the
+ * copy fails, the old app comes back, and the log says so.
+ */
+ipcMain.handle('update:install', async (_e, dmgPath) => {
+  const dmg = String(dmgPath || '');
+  if (!dmg || !fs.existsSync(dmg)) return { ok: false, error: 'That installer is no longer on disk.' };
+
+  /* Never mid-service. Quitting the app stops every listener's captions, and an
+     update is never worth that — the operator stops translation first. */
+  if (engineIsLive) {
+    return { ok: false, error: 'Translation is live. Press Stop translating first, then install.' };
+  }
+
+  const appPath = app.getPath('exe').replace(/\/Contents\/MacOS\/[^/]+$/, '');
+  if (!/\.app$/.test(appPath)) {
+    return { ok: false, error: 'This copy is not running from an .app bundle, so it cannot replace itself.' };
+  }
+
+  const tmp = path.join(require('os').tmpdir(), 'ct-install-' + process.pid + '.sh');
+  const mount = '/Volumes/ChurchTranslateUpdate';
+  const aside = appPath + '.replacing-' + process.pid;
+  const script = [
+    '#!/bin/bash',
+    '# Written by Church Translate to swap itself. Safe to delete.',
+    'set -u',
+    'DMG=' + JSON.stringify(dmg),
+    'APP=' + JSON.stringify(appPath),
+    'ASIDE=' + JSON.stringify(aside),
+    'MNT=' + JSON.stringify(mount),
+    'PID=' + process.pid,
+    'LOG="$HOME/Library/Application Support/Church Translate/data/logs/install.log"',
+    'mkdir -p "$(dirname "$LOG")"',
+    'say(){ echo "$(date "+%Y-%m-%d %H:%M:%S") $*" >> "$LOG"; }',
+    'say "installing $DMG into $APP"',
+    '# wait for the app to actually be gone (up to 30s)',
+    'for i in $(seq 1 60); do kill -0 "$PID" 2>/dev/null || break; sleep 0.5; done',
+    'if kill -0 "$PID" 2>/dev/null; then say "app did not exit; aborting"; exit 1; fi',
+    'hdiutil detach "$MNT" -quiet 2>/dev/null',
+    'if ! hdiutil attach "$DMG" -nobrowse -quiet -mountpoint "$MNT"; then say "could not mount"; open -a "$APP"; exit 1; fi',
+    'NEW="$MNT/Church Translate.app"',
+    '# refuse to touch the installed app unless the replacement is really there',
+    'if [ ! -f "$NEW/Contents/Info.plist" ]; then say "disk image has no app"; hdiutil detach "$MNT" -quiet; open -a "$APP"; exit 1; fi',
+    'rm -rf "$ASIDE"',
+    'if ! mv "$APP" "$ASIDE"; then say "could not move the old app aside"; hdiutil detach "$MNT" -quiet; open -a "$APP"; exit 1; fi',
+    'if cp -R "$NEW" "$APP"; then',
+    '  xattr -dr com.apple.quarantine "$APP" 2>/dev/null',
+    '  say "installed $( /usr/libexec/PlistBuddy -c "Print :CFBundleVersion" "$APP/Contents/Info.plist" 2>/dev/null )"',
+    '  rm -rf "$ASIDE"',
+    'else',
+    '  say "copy failed — putting the old app back"',
+    '  rm -rf "$APP"; mv "$ASIDE" "$APP"',
+    'fi',
+    'hdiutil detach "$MNT" -quiet 2>/dev/null',
+    'open -a "$APP"',
+    'say "relaunched"',
+    'rm -f "$0"'
+  ].join('\n');
+
+  try {
+    fs.writeFileSync(tmp, script, { mode: 0o755 });
+  } catch (e) {
+    return { ok: false, error: 'Could not prepare the installer: ' + e.message };
+  }
+
+  /* Detached and unref'd, or it dies with us the moment we quit. */
+  try {
+    const child = spawn('/bin/bash', [tmp], { detached: true, stdio: 'ignore' });
+    child.unref();
+  } catch (e) {
+    return { ok: false, error: 'Could not start the installer: ' + e.message };
+  }
+
+  /* Give the renderer a moment to show the message before we go. */
+  forceQuit = true;
+  setTimeout(() => { try { app.quit(); } catch (_) {} }, 600);
+  return { ok: true };
+});
+
 ipcMain.handle('update:reveal', (_e, p2) => { try { shell.showItemInFolder(String(p2)); return true; } catch (_) { return false; } });
 
 ipcMain.handle('config:get', () => loadConfig());

app/package.json

@@ -2,7 +2,7 @@
   "name": "church-translate-engine",
   "productName": "Church Translate",
   "version": "3.23.2",
-  "buildNumber": 113,
+  "buildNumber": 117,
   "private": true,
   "description": "Live sermon translation — self-contained: captures audio, transcribes, translates, and serves the congregation page itself",
   "main": "main.js",
@@ -106,7 +106,7 @@
       ]
     },
     "asar": false,
-    "buildVersion": "113",
+    "buildVersion": "117",
     "win": {
       "target": [
         {

app/preload.js

@@ -24,6 +24,7 @@ contextBridge.exposeInMainWorld('engineHost', {
   updateCheck:  ()      => ipcRenderer.invoke('update:check'),
   updateDownload: (u, n) => ipcRenderer.invoke('update:download', u, n),
   updateReveal: (p)     => ipcRenderer.invoke('update:reveal', p),
+  updateInstall: (p)    => ipcRenderer.invoke('update:install', p),
   onUpdateProgress: (fn) => ipcRenderer.on('update:progress', (_e, p) => fn(p)),
   settingsExport: ()    => ipcRenderer.invoke('settings:export'),
   settingsImport: ()    => ipcRenderer.invoke('settings:import'),

app/server/server.js

@@ -500,6 +500,23 @@ const ENGINE_STALE_MS = 12000;
  * with one plain sentence does. Computed here so every operator surface —
  * in-app pane, browser page — shows the same verdict.
  */
+/*
+ * WHEN IT IS BEHIND, SAY WHAT IS LOADING IT.
+ *
+ * Translation cost is roughly linear in the number of active languages, through
+ * ONE NLLB process — measured, five languages in 1.9 s. So the single most
+ * useful fact about a machine that is falling behind is how many languages it
+ * is carrying, and that number was computed one function away and never shown.
+ * A campus reported "constantly behind, in the red" and the cause took an
+ * afternoon to find; it was eight languages on hardware tuned against one.
+ */
+function loadNote() {
+  const n = activeLanguages().length;
+  if (n < 2) return '';
+  return ' Carrying ' + n + ' languages' +
+         (config.allowAllLanguages ? ', and listeners may add more' : '') + '.';
+}
+
 function healthVerdict(connected, stale) {
   const es = state.engineStatus || {};
   if (!connected)               return { level: 'red',   detail: 'The translation app is not connected.' };
@@ -507,9 +524,11 @@ function healthVerdict(connected, stale) {
   if (!state.live)              return { level: 'green', detail: 'Ready.' };
   if (!es.modelsReady)          return { level: 'amber', detail: 'Models are still loading.' };
   if ((es.lagMs || 0) > 12000 || (es.captionOnlyFinals || 0) > 0)
-    return { level: 'red',   detail: 'Falling badly behind — some sentences are captions only.' };
+    return { level: 'red',
+             detail: 'Falling badly behind — some sentences are captions only.' + loadNote() };
   if ((es.lagMs || 0) > 6000 || (es.queue || 0) >= 2 || (es.degradedFinals || 0) > 0)
-    return { level: 'amber', detail: 'Running behind — idle languages may be trimmed to catch up.' };
+    return { level: 'amber',
+             detail: 'Running behind — idle languages may be trimmed to catch up.' + loadNote() };
   if (es.cloud && es.cloud.parked)
     return { level: 'amber', detail: 'Natural voices are down — the built-in voices have taken over.' };
   /* A fallback in the last minute means one language just changed voice
@@ -1076,7 +1095,22 @@ const server = http.createServer(async (req, res) => {
   }
   if (req.method === 'OPTIONS') { res.writeHead(origin ? 204 : 405).end(); return; }
 
-  if (p === '/healthz') { sendJSON(res, 200, { ok: true, live: state.live, build: BUILD }); return; }
+  /*
+   * The APP's version, plainly, to anyone who can reach this port.
+   *
+   * `build` here has always been a hash of the listener-page assets — a
+   * cache-busting token, not a version — so the only way to tell what a campus
+   * Mac was running was to install something and watch the hash move. With an
+   * updater in the field that is the first question anyone asks, so answer it.
+   */
+  if (p === '/healthz') {
+    sendJSON(res, 200, {
+      ok: true, live: state.live, build: BUILD,
+      version: process.env.CT_APP_VERSION || '',
+      appBuild: process.env.CT_APP_BUILD || ''
+    });
+    return;
+  }
 
   // --- Companion / Stream Deck control -------------------------------
   //  GET or POST, token in ?key= or the X-Control-Token header.