From 39dea804334baf780e90646fc169b9ab57e95d20 Mon Sep 17 00:00:00 2001 From: Waters Church Date: Thu, 24 Sep 2026 13:54:26 -0400 Subject: [PATCH] Do not step down twice for the same backlog MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build 120. Swapping the speech model did nothing about the queue that caused the swap. The first of those chunks went straight to a model that had just loaded, took the full 30 s worker timeout, and tripped the absolute rule — so the ladder fell again. MEASURED on a campus Mac: large-v3-turbo to small to base in 34 seconds, landing on a model measurably SLOWER than the one it started from, which is the opposite of what stepping down is for. Two changes. A successful step-down now discards the queued chunks: they are already past the caption-only cutoff, so they cannot be translated in time and are worth nothing to the room, and it says how many it dropped. And no further step-down is allowed for twenty seconds, so the new model is judged on sentences it was actually given rather than on the wreckage of the old one. Twenty seconds covers a cold first inference while still letting a genuinely hopeless model fall through the ladder within a sentence or two. Checked against the real cascade from the log, including that a hung model long after a swap is still caught. Co-Authored-By: Claude Opus 5 --- app/engine.js | 53 ++++++++++++++++++++++++++++++++++++++++++++++-- app/package.json | 4 ++-- 2 files changed, 53 insertions(+), 4 deletions(-) diff --git a/app/engine.js b/app/engine.js index d7e5559..7b5b7a6 100644 --- a/app/engine.js +++ b/app/engine.js @@ -891,12 +891,54 @@ async function respawnWorker(worker) { log('the speech model is too heavy for this machine — switching to ' + lighter.split('/').pop(), 'err'); } + /* + * A LIGHTER MODEL GETS THE PREVIEW'S DTYPE, NOT THE BIG ONE'S. + * + * asrDtype is chosen for the accurate model and its weights are cached against + * it. Reusing it when stepping down asks for a variant that was never + * downloaded — and those weights live in the browser cache, with no network at + * a venue — so the step-down failed and the ladder could not move at all. + * + * MEASURED on a campus Mac: the cache held whisper-large-v3-turbo as + * encoder_model_q4f16.onnx (its asrDtype) and whisper-small as + * encoder_model_fp16.onnx (the preview's). Stepping down asked for + * whisper-small at q4f16, a file that is not there; it went to the network, + * failed, and logged "Unsupported model type: whisper" three times a minute + * for ever while the service sat dead. + * + * The preview runs whisper-small on exactly these settings every service, so + * they are the one combination this machine is known to be able to load. + */ + const stepDtype = lighter + ? resolveDtype(cfg.asrFastDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' }) + : resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' }); state.asrWorker = await loadWorkerWithFallback('speech', 'automatic-speech-recognition', state.asrModelInUse || cfg.asrModel, - resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' }), + stepDtype, { encoder_model: 'q4', decoder_model_merged: 'q4' }); log('speech model restarted' + (lighter ? ' on ' + lighter.split('/').pop() : ''), 'ok'); state.slowRun = 0; + if (lighter) { + /* + * THROW AWAY THE BACKLOG THAT CAUSED THIS, AND LET THE NEW MODEL SETTLE. + * + * Swapping the model does nothing about the queue that triggered the swap. + * Those chunks are already past the caption-only cutoff, so they can no + * longer be translated in time and are worth nothing to the room — but the + * first of them gets handed to a model that has just loaded, takes the full + * worker timeout, and trips the absolute rule. MEASURED: a campus Mac fell + * large-v3-turbo -> small -> base in 34 seconds exactly that way, landing on + * a model SLOWER than the one it started from. + * + * So drop them, and refuse to step down again for a grace window, so the new + * model is judged on sentences it was actually given. + */ + const dropped = queue.length; + queue.length = 0; + state.steppedDownAt = Date.now(); + if (dropped) log('dropped ' + dropped + ' stale sentence' + (dropped === 1 ? '' : 's') + + ' the old model never reached', 'warn'); + } } } catch (err) { log('model restart failed: ' + err.message, 'err'); @@ -1951,9 +1993,11 @@ async function pump() { * test is how far behind the speaker the sentence actually landed, which is * what the operator is complaining about in the first place. */ + /* A model that loaded seconds ago has not had a fair sentence yet. */ + const settling = state.steppedDownAt && (Date.now() - state.steppedDownAt) < STEP_SETTLE_MS; const behindMs = Date.now() - job.at; const losingGround = jobAudioMs > 0 && took > jobAudioMs * 0.85 && behindMs > 8000; - if (took > 8000 || losingGround) { + if (!settling && (took > 8000 || losingGround)) { state.slowRun = (state.slowRun || 0) + 1; if (state.slowRun >= 3 && !respawning && lighterAsr(state.asrModelInUse || state.cfg.asrModel)) { state.slowRun = 0; @@ -2470,6 +2514,11 @@ function monitorTranslation(cid, lang, text) { * It says it once per episode, and names both numbers so the operator can act * on it rather than go looking. */ +/* How long a freshly swapped speech model is left alone. Long enough to cover a + cold first inference, short enough that a genuinely hopeless model still falls + through the ladder within a sentence or two. */ +const STEP_SETTLE_MS = 20000; + const DEAF_AFTER_MS = 90000; setInterval(() => { try { diff --git a/app/package.json b/app/package.json index 74df696..f818dc7 100644 --- a/app/package.json +++ b/app/package.json @@ -2,7 +2,7 @@ "name": "church-translate-engine", "productName": "Church Translate", "version": "3.23.2", - "buildNumber": 118, + "buildNumber": 120, "private": true, "description": "Live sermon translation — self-contained: captures audio, transcribes, translates, and serves the congregation page itself", "main": "main.js", @@ -106,7 +106,7 @@ ] }, "asar": false, - "buildVersion": "118", + "buildVersion": "120", "win": { "target": [ {