Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
a06cf35cbb | ||
|
|
39dea80433 | ||
|
|
6f91bc1357 | ||
|
|
92e86e415f | ||
|
|
73ec7f1436 | ||
|
|
4aa30699c9 | ||
|
|
6753a94e1c |
+148
-15
@@ -190,7 +190,9 @@ const state = {
|
||||
|
||||
/* health counters, reported to the operator page */
|
||||
degradedFinals: 0, // finals that shipped with fewer languages (stale)
|
||||
captionOnlyFinals: 0 // finals that shipped with no translations at all
|
||||
captionOnlyFinals: 0, // finals that shipped with no translations at all
|
||||
captionOnlyAt: 0, // when the last one happened
|
||||
degradedAt: 0
|
||||
};
|
||||
|
||||
// ────────────────────────────────────────────────────────── logging
|
||||
@@ -891,12 +893,54 @@ async function respawnWorker(worker) {
|
||||
log('the speech model is too heavy for this machine — switching to ' +
|
||||
lighter.split('/').pop(), 'err');
|
||||
}
|
||||
/*
|
||||
* A LIGHTER MODEL GETS THE PREVIEW'S DTYPE, NOT THE BIG ONE'S.
|
||||
*
|
||||
* asrDtype is chosen for the accurate model and its weights are cached against
|
||||
* it. Reusing it when stepping down asks for a variant that was never
|
||||
* downloaded — and those weights live in the browser cache, with no network at
|
||||
* a venue — so the step-down failed and the ladder could not move at all.
|
||||
*
|
||||
* MEASURED on a campus Mac: the cache held whisper-large-v3-turbo as
|
||||
* encoder_model_q4f16.onnx (its asrDtype) and whisper-small as
|
||||
* encoder_model_fp16.onnx (the preview's). Stepping down asked for
|
||||
* whisper-small at q4f16, a file that is not there; it went to the network,
|
||||
* failed, and logged "Unsupported model type: whisper" three times a minute
|
||||
* for ever while the service sat dead.
|
||||
*
|
||||
* The preview runs whisper-small on exactly these settings every service, so
|
||||
* they are the one combination this machine is known to be able to load.
|
||||
*/
|
||||
const stepDtype = lighter
|
||||
? resolveDtype(cfg.asrFastDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' })
|
||||
: resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' });
|
||||
state.asrWorker = await loadWorkerWithFallback('speech', 'automatic-speech-recognition',
|
||||
state.asrModelInUse || cfg.asrModel,
|
||||
resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' }),
|
||||
stepDtype,
|
||||
{ encoder_model: 'q4', decoder_model_merged: 'q4' });
|
||||
log('speech model restarted' + (lighter ? ' on ' + lighter.split('/').pop() : ''), 'ok');
|
||||
state.slowRun = 0;
|
||||
if (lighter) {
|
||||
/*
|
||||
* THROW AWAY THE BACKLOG THAT CAUSED THIS, AND LET THE NEW MODEL SETTLE.
|
||||
*
|
||||
* Swapping the model does nothing about the queue that triggered the swap.
|
||||
* Those chunks are already past the caption-only cutoff, so they can no
|
||||
* longer be translated in time and are worth nothing to the room — but the
|
||||
* first of them gets handed to a model that has just loaded, takes the full
|
||||
* worker timeout, and trips the absolute rule. MEASURED: a campus Mac fell
|
||||
* large-v3-turbo -> small -> base in 34 seconds exactly that way, landing on
|
||||
* a model SLOWER than the one it started from.
|
||||
*
|
||||
* So drop them, and refuse to step down again for a grace window, so the new
|
||||
* model is judged on sentences it was actually given.
|
||||
*/
|
||||
const dropped = queue.length;
|
||||
queue.length = 0;
|
||||
state.steppedDownAt = Date.now();
|
||||
if (dropped) log('dropped ' + dropped + ' stale sentence' + (dropped === 1 ? '' : 's') +
|
||||
' the old model never reached', 'warn');
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
log('model restart failed: ' + err.message, 'err');
|
||||
@@ -1794,6 +1838,7 @@ async function pump() {
|
||||
|
||||
state.busy = true;
|
||||
const started = Date.now();
|
||||
let jobAudioMs = 0;
|
||||
try {
|
||||
/* job.at is when the audio ENDED. Live translation of stale audio is
|
||||
worthless, but the WORDS are not: however old the job, it still gets
|
||||
@@ -1805,6 +1850,9 @@ async function pump() {
|
||||
/* Near-silence telemetry (log-only this release): a chunk whose average
|
||||
level barely clears the noise floor is hallucination fuel. */
|
||||
const audio = flatten(job.frames);
|
||||
/* How much speech this chunk actually holds. The step-down below compares
|
||||
against it, because "slow" only means anything relative to the audio. */
|
||||
jobAudioMs = (audio.length / SAMPLE_RATE) * 1000;
|
||||
let sum = 0;
|
||||
for (let i = 0; i < audio.length; i += 4) sum += audio[i] * audio[i];
|
||||
const meanRms = Math.sqrt(sum / Math.ceil(audio.length / 4));
|
||||
@@ -1845,15 +1893,33 @@ async function pump() {
|
||||
const sentences = collapseLoops(splitSentences(text));
|
||||
|
||||
let targets = orderedTargets();
|
||||
if (ageSec > 12) {
|
||||
/*
|
||||
* SKIPPING A TRANSLATION ONLY BUYS SOMETHING IF THERE IS A QUEUE TO DRAIN.
|
||||
*
|
||||
* Dropping the translations of stale audio is how the app catches up: the
|
||||
* listener has moved on, so the work is wasted and the time is better spent on
|
||||
* the sentences behind it. That reasoning needs a backlog. With nothing waiting
|
||||
* behind it, skipping costs the room a sentence and saves nothing at all.
|
||||
*
|
||||
* MEASURED: a campus Mac took one 16 s stall — two Screen Sharing sessions
|
||||
* competing for the machine — and skipped that sentence's translations with an
|
||||
* EMPTY queue. The room lost a line of the sermon to a catch-up that had
|
||||
* nothing to catch up on.
|
||||
*/
|
||||
if (ageSec > 12 && queue.length > 0) {
|
||||
targets = [];
|
||||
state.captionOnlyFinals++;
|
||||
log('audio ' + ageSec.toFixed(0) + 's stale — caption only, translations skipped', 'err');
|
||||
state.captionOnlyAt = Date.now();
|
||||
log('audio ' + ageSec.toFixed(0) + 's stale with ' + queue.length + ' waiting — ' +
|
||||
'caption only, translations skipped', 'err');
|
||||
} else if (ageSec > 12) {
|
||||
/* Late, but nothing is behind it: say so, and translate it anyway. */
|
||||
log('audio ' + ageSec.toFixed(0) + 's late, but nothing is queued — translating it anyway', 'warn');
|
||||
} else if (ageSec > 6) {
|
||||
const listening = new Set((state.byLanguage || []).map((r) => r.lang));
|
||||
const before = targets.length;
|
||||
targets = targets.filter((t) => listening.has(t));
|
||||
if (targets.length < before) state.degradedFinals++;
|
||||
if (targets.length < before) { state.degradedFinals++; state.degradedAt = Date.now(); }
|
||||
}
|
||||
|
||||
/* The segment's own loudness, measured once and shared by every sentence
|
||||
@@ -1908,11 +1974,56 @@ async function pump() {
|
||||
* talk, and the backlog only grows — step down before the service is lost
|
||||
* rather than after. Measured on the Intel laptop: 12-25 s, every sentence.
|
||||
*/
|
||||
if (took > 8000) {
|
||||
/*
|
||||
* SLOWER THAN THE AUDIO IT COVERS — not slower than some constant.
|
||||
*
|
||||
* This used to fire only past 8 s, on the reasoning that 8 s meant slower
|
||||
* than people talk. That was true when a chunk could be 8 s long. Chunks
|
||||
* are capped at HARD_CUT_MAX, 5.5 s, so a 4.5 s chunk taking 4.3 s is
|
||||
* already losing ground and never came near the old threshold.
|
||||
*
|
||||
* MEASURED on a campus M4 Pro: every sentence 4.2-4.7 s against roughly
|
||||
* 4-5 s of audio, a queue 46 deep and a lag of 152 s still climbing — with
|
||||
* nobody listening and no language being translated, so this was
|
||||
* transcription alone. The one mechanism built to catch exactly that could
|
||||
* not fire, because 4.4 s is not 8 s.
|
||||
*
|
||||
* The queue test is what separates "this machine cannot keep up" from one
|
||||
* slow sentence: a backlog that is three deep and being fed faster than it
|
||||
* drains does not recover on its own.
|
||||
*/
|
||||
/*
|
||||
* SLOWER THAN THE AUDIO IT COVERS, AND LANDING LATE.
|
||||
*
|
||||
* This used to fire only past 8 s, on the reasoning that 8 s meant slower
|
||||
* than people talk. That held when a chunk could be 8 s long; chunks are
|
||||
* capped at HARD_CUT_MAX, 5.5 s, so a 4.4 s sentence is already losing and
|
||||
* never came near the old threshold.
|
||||
*
|
||||
* Both numbers below were measured on a campus M4 Pro, and the second one
|
||||
* only after a first attempt got it wrong. 0.85, because that Mac ran 4.4 s
|
||||
* against 4.5 s of audio — nominally FASTER than real time while losing
|
||||
* badly, since the work outside this timer (segmenting, the preview model
|
||||
* sharing the GPU, the scheduler) loses on its own.
|
||||
*
|
||||
* The second test was a queue depth, and watching that Mac go red disproved
|
||||
* it: the queue sat at 2-3 while the lag held at 15 s and the verdict was
|
||||
* already red. A machine can be persistently late without ever stacking work
|
||||
* up — it finishes each sentence just slightly too slowly, for ever. So the
|
||||
* test is how far behind the speaker the sentence actually landed, which is
|
||||
* what the operator is complaining about in the first place.
|
||||
*/
|
||||
/* A model that loaded seconds ago has not had a fair sentence yet. */
|
||||
const settling = state.steppedDownAt && (Date.now() - state.steppedDownAt) < STEP_SETTLE_MS;
|
||||
const behindMs = Date.now() - job.at;
|
||||
const losingGround = jobAudioMs > 0 && took > jobAudioMs * 0.85 && behindMs > 8000;
|
||||
if (!settling && (took > 8000 || losingGround)) {
|
||||
state.slowRun = (state.slowRun || 0) + 1;
|
||||
if (state.slowRun >= 3 && !respawning && lighterAsr(state.asrModelInUse || state.cfg.asrModel)) {
|
||||
state.slowRun = 0;
|
||||
log('transcription is slower than people are speaking — stepping down to a lighter model', 'err');
|
||||
log('transcription is slower than people are speaking (' + took + ' ms for ' +
|
||||
Math.round(jobAudioMs) + ' ms of audio, ' + Math.round(behindMs / 1000) + 's behind) — ' +
|
||||
'stepping down to a lighter model', 'err');
|
||||
respawnWorker(state.asrWorker);
|
||||
}
|
||||
} else state.slowRun = 0;
|
||||
@@ -2423,6 +2534,11 @@ function monitorTranslation(cid, lang, text) {
|
||||
* It says it once per episode, and names both numbers so the operator can act
|
||||
* on it rather than go looking.
|
||||
*/
|
||||
/* How long a freshly swapped speech model is left alone. Long enough to cover a
|
||||
cold first inference, short enough that a genuinely hopeless model still falls
|
||||
through the ladder within a sentence or two. */
|
||||
const STEP_SETTLE_MS = 20000;
|
||||
|
||||
const DEAF_AFTER_MS = 90000;
|
||||
setInterval(() => {
|
||||
try {
|
||||
@@ -2662,6 +2778,7 @@ function goLive(on) {
|
||||
state.live = on;
|
||||
if (on) {
|
||||
state.degradedFinals = 0; state.captionOnlyFinals = 0; state.fragmentFinals = 0;
|
||||
state.captionOnlyAt = 0; state.degradedAt = 0;
|
||||
/* The watchdog measures from the start of the service, not from the last
|
||||
sentence of the previous one. */
|
||||
state.lastFinalAt = Date.now();
|
||||
@@ -2736,6 +2853,10 @@ function reportStatus() {
|
||||
queue: queue.length,
|
||||
degradedFinals: state.degradedFinals,
|
||||
captionOnlyFinals: state.captionOnlyFinals,
|
||||
/* WHEN, not just how many — the verdict needs to know whether this is
|
||||
happening now or happened once an hour ago. */
|
||||
captionOnlyAt: state.captionOnlyAt || 0,
|
||||
degradedAt: state.degradedAt || 0,
|
||||
fragmentFinals: state.fragmentFinals || 0,
|
||||
cloud: state.cloudStats || null
|
||||
}
|
||||
@@ -2983,14 +3104,26 @@ async function downloadUpdate() {
|
||||
try {
|
||||
const r = await window.engineHost.updateDownload(updFound.asset.url, updFound.asset.name);
|
||||
if (!r.ok) { $('updProgress').textContent = r.error || 'Download failed.'; btn.disabled = false; return; }
|
||||
$('updProgress').textContent = 'Saved to Downloads.';
|
||||
/* Say what is left to do. Nothing installs itself here on purpose — see
|
||||
the note on the main-process side. */
|
||||
$('updWhat').textContent = 'Downloaded ' + updFound.tag + ' (' + fmtMB(r.bytes) + '). ' +
|
||||
'Quit Church Translate, open the disk image, and drag the app over the one in ' +
|
||||
'Applications. Your settings, models and voices are untouched.';
|
||||
log('update ' + updFound.tag + ' downloaded to ' + r.path, 'ok');
|
||||
try { await window.engineHost.updateReveal(r.path); } catch (_) {}
|
||||
$('updProgress').textContent = 'Downloaded. Installing\u2026';
|
||||
log('update ' + updFound.tag + ' downloaded to ' + r.path, 'ok');
|
||||
|
||||
/*
|
||||
* Straight into the install. The app quits, a helper swaps the bundle and
|
||||
* reopens it — so from here the window closing IS the update working, and
|
||||
* saying so before it happens stops that looking like a crash.
|
||||
*/
|
||||
$('updWhat').textContent = 'Installing ' + updFound.tag + '. Church Translate will close and ' +
|
||||
'reopen by itself in a few seconds. Your settings, models and voices are untouched.';
|
||||
const ins = await window.engineHost.updateInstall(r.path);
|
||||
if (!ins || !ins.ok) {
|
||||
/* Still useful: the file is downloaded, so fall back to the manual route
|
||||
rather than leaving them with nothing. */
|
||||
$('updProgress').textContent = (ins && ins.error) || 'Could not install.';
|
||||
$('updWhat').textContent = 'Downloaded, but it could not install itself. Quit Church ' +
|
||||
'Translate, open the disk image in Downloads, and drag the app over the one in Applications.';
|
||||
try { await window.engineHost.updateReveal(r.path); } catch (_) {}
|
||||
btn.disabled = false;
|
||||
}
|
||||
} catch (e) {
|
||||
$('updProgress').textContent = 'Download failed: ' + e.message;
|
||||
btn.disabled = false;
|
||||
|
||||
+1
-1
@@ -391,7 +391,7 @@
|
||||
<div id="updFound" hidden>
|
||||
<p class="hint" id="updWhat"></p>
|
||||
<div class="row">
|
||||
<button class="btn" id="updGet">Download it</button>
|
||||
<button class="btn" id="updGet">Download and install</button>
|
||||
<span class="hint" id="updProgress"></span>
|
||||
</div>
|
||||
<details id="updNotesBox"><summary class="hint">What changed</summary>
|
||||
|
||||
+96
@@ -440,6 +440,11 @@ function startServer() {
|
||||
execPath: nodeBin,
|
||||
env: Object.assign({}, process.env, {
|
||||
CT_FFMPEG: ffmpegPathForChildren(),
|
||||
/* So /healthz can report the APP’s version rather than only an asset
|
||||
hash. A campus Mac’s build was otherwise unknowable from the network,
|
||||
which made confirming an update a matter of inference. */
|
||||
CT_APP_VERSION: appBuild().version,
|
||||
CT_APP_BUILD: appBuild().build,
|
||||
PORT: String(port),
|
||||
DATA_DIR: dataDir(),
|
||||
/* Where the spoken clips land, so the server can hand them to phones. */
|
||||
@@ -2259,6 +2264,97 @@ ipcMain.handle('update:download', async (_e, url, name) => {
|
||||
});
|
||||
});
|
||||
|
||||
/*
|
||||
* ─────────────────────────── installing it, in place ──────────────────────
|
||||
*
|
||||
* An app cannot replace itself while it is running, so the swap is done by a
|
||||
* small script that outlives us: it waits for this process to exit, moves the
|
||||
* old bundle aside, copies the new one in, and relaunches.
|
||||
*
|
||||
* MOVED ASIDE, NOT DELETED. If the copy fails half way — a full disk, a bad
|
||||
* download, the volume going away — the old app is put straight back and
|
||||
* relaunched. The failure mode this is written against is a campus Mac with no
|
||||
* working app and a service in an hour, which is far worse than not updating.
|
||||
*
|
||||
* No sudo: /Applications is admin-writable on macOS and the bundle belongs to
|
||||
* the operator, so nothing here needs a password. If that is ever not true the
|
||||
* copy fails, the old app comes back, and the log says so.
|
||||
*/
|
||||
ipcMain.handle('update:install', async (_e, dmgPath) => {
|
||||
const dmg = String(dmgPath || '');
|
||||
if (!dmg || !fs.existsSync(dmg)) return { ok: false, error: 'That installer is no longer on disk.' };
|
||||
|
||||
/* Never mid-service. Quitting the app stops every listener's captions, and an
|
||||
update is never worth that — the operator stops translation first. */
|
||||
if (engineIsLive) {
|
||||
return { ok: false, error: 'Translation is live. Press Stop translating first, then install.' };
|
||||
}
|
||||
|
||||
const appPath = app.getPath('exe').replace(/\/Contents\/MacOS\/[^/]+$/, '');
|
||||
if (!/\.app$/.test(appPath)) {
|
||||
return { ok: false, error: 'This copy is not running from an .app bundle, so it cannot replace itself.' };
|
||||
}
|
||||
|
||||
const tmp = path.join(require('os').tmpdir(), 'ct-install-' + process.pid + '.sh');
|
||||
const mount = '/Volumes/ChurchTranslateUpdate';
|
||||
const aside = appPath + '.replacing-' + process.pid;
|
||||
const script = [
|
||||
'#!/bin/bash',
|
||||
'# Written by Church Translate to swap itself. Safe to delete.',
|
||||
'set -u',
|
||||
'DMG=' + JSON.stringify(dmg),
|
||||
'APP=' + JSON.stringify(appPath),
|
||||
'ASIDE=' + JSON.stringify(aside),
|
||||
'MNT=' + JSON.stringify(mount),
|
||||
'PID=' + process.pid,
|
||||
'LOG="$HOME/Library/Application Support/Church Translate/data/logs/install.log"',
|
||||
'mkdir -p "$(dirname "$LOG")"',
|
||||
'say(){ echo "$(date "+%Y-%m-%d %H:%M:%S") $*" >> "$LOG"; }',
|
||||
'say "installing $DMG into $APP"',
|
||||
'# wait for the app to actually be gone (up to 30s)',
|
||||
'for i in $(seq 1 60); do kill -0 "$PID" 2>/dev/null || break; sleep 0.5; done',
|
||||
'if kill -0 "$PID" 2>/dev/null; then say "app did not exit; aborting"; exit 1; fi',
|
||||
'hdiutil detach "$MNT" -quiet 2>/dev/null',
|
||||
'if ! hdiutil attach "$DMG" -nobrowse -quiet -mountpoint "$MNT"; then say "could not mount"; open -a "$APP"; exit 1; fi',
|
||||
'NEW="$MNT/Church Translate.app"',
|
||||
'# refuse to touch the installed app unless the replacement is really there',
|
||||
'if [ ! -f "$NEW/Contents/Info.plist" ]; then say "disk image has no app"; hdiutil detach "$MNT" -quiet; open -a "$APP"; exit 1; fi',
|
||||
'rm -rf "$ASIDE"',
|
||||
'if ! mv "$APP" "$ASIDE"; then say "could not move the old app aside"; hdiutil detach "$MNT" -quiet; open -a "$APP"; exit 1; fi',
|
||||
'if cp -R "$NEW" "$APP"; then',
|
||||
' xattr -dr com.apple.quarantine "$APP" 2>/dev/null',
|
||||
' say "installed $( /usr/libexec/PlistBuddy -c "Print :CFBundleVersion" "$APP/Contents/Info.plist" 2>/dev/null )"',
|
||||
' rm -rf "$ASIDE"',
|
||||
'else',
|
||||
' say "copy failed — putting the old app back"',
|
||||
' rm -rf "$APP"; mv "$ASIDE" "$APP"',
|
||||
'fi',
|
||||
'hdiutil detach "$MNT" -quiet 2>/dev/null',
|
||||
'open -a "$APP"',
|
||||
'say "relaunched"',
|
||||
'rm -f "$0"'
|
||||
].join('\n');
|
||||
|
||||
try {
|
||||
fs.writeFileSync(tmp, script, { mode: 0o755 });
|
||||
} catch (e) {
|
||||
return { ok: false, error: 'Could not prepare the installer: ' + e.message };
|
||||
}
|
||||
|
||||
/* Detached and unref'd, or it dies with us the moment we quit. */
|
||||
try {
|
||||
const child = spawn('/bin/bash', [tmp], { detached: true, stdio: 'ignore' });
|
||||
child.unref();
|
||||
} catch (e) {
|
||||
return { ok: false, error: 'Could not start the installer: ' + e.message };
|
||||
}
|
||||
|
||||
/* Give the renderer a moment to show the message before we go. */
|
||||
forceQuit = true;
|
||||
setTimeout(() => { try { app.quit(); } catch (_) {} }, 600);
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
ipcMain.handle('update:reveal', (_e, p2) => { try { shell.showItemInFolder(String(p2)); return true; } catch (_) { return false; } });
|
||||
|
||||
ipcMain.handle('config:get', () => loadConfig());
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@
|
||||
"name": "church-translate-engine",
|
||||
"productName": "Church Translate",
|
||||
"version": "3.23.2",
|
||||
"buildNumber": 113,
|
||||
"buildNumber": 121,
|
||||
"private": true,
|
||||
"description": "Live sermon translation — self-contained: captures audio, transcribes, translates, and serves the congregation page itself",
|
||||
"main": "main.js",
|
||||
@@ -106,7 +106,7 @@
|
||||
]
|
||||
},
|
||||
"asar": false,
|
||||
"buildVersion": "113",
|
||||
"buildVersion": "121",
|
||||
"win": {
|
||||
"target": [
|
||||
{
|
||||
|
||||
@@ -24,6 +24,7 @@ contextBridge.exposeInMainWorld('engineHost', {
|
||||
updateCheck: () => ipcRenderer.invoke('update:check'),
|
||||
updateDownload: (u, n) => ipcRenderer.invoke('update:download', u, n),
|
||||
updateReveal: (p) => ipcRenderer.invoke('update:reveal', p),
|
||||
updateInstall: (p) => ipcRenderer.invoke('update:install', p),
|
||||
onUpdateProgress: (fn) => ipcRenderer.on('update:progress', (_e, p) => fn(p)),
|
||||
settingsExport: () => ipcRenderer.invoke('settings:export'),
|
||||
settingsImport: () => ipcRenderer.invoke('settings:import'),
|
||||
|
||||
+61
-5
@@ -500,16 +500,57 @@ const ENGINE_STALE_MS = 12000;
|
||||
* with one plain sentence does. Computed here so every operator surface —
|
||||
* in-app pane, browser page — shows the same verdict.
|
||||
*/
|
||||
/*
|
||||
* WHEN IT IS BEHIND, SAY WHAT IS LOADING IT.
|
||||
*
|
||||
* Translation cost is roughly linear in the number of active languages, through
|
||||
* ONE NLLB process — measured, five languages in 1.9 s. So the single most
|
||||
* useful fact about a machine that is falling behind is how many languages it
|
||||
* is carrying, and that number was computed one function away and never shown.
|
||||
* A campus reported "constantly behind, in the red" and the cause took an
|
||||
* afternoon to find; it was eight languages on hardware tuned against one.
|
||||
*/
|
||||
/* How long a stumble keeps colouring the verdict. Long enough that a real
|
||||
problem stays visible while someone walks to the desk, short enough that a
|
||||
one-off does not outlive the sentence after it. */
|
||||
const TROUBLE_WINDOW_MS = 120000;
|
||||
|
||||
function loadNote() {
|
||||
const n = activeLanguages().length;
|
||||
if (n < 2) return '';
|
||||
return ' Carrying ' + n + ' languages' +
|
||||
(config.allowAllLanguages ? ', and listeners may add more' : '') + '.';
|
||||
}
|
||||
|
||||
function healthVerdict(connected, stale) {
|
||||
const es = state.engineStatus || {};
|
||||
if (!connected) return { level: 'red', detail: 'The translation app is not connected.' };
|
||||
if (stale) return { level: 'red', detail: 'The translation app has stopped responding.' };
|
||||
if (!state.live) return { level: 'green', detail: 'Ready.' };
|
||||
if (!es.modelsReady) return { level: 'amber', detail: 'Models are still loading.' };
|
||||
if ((es.lagMs || 0) > 12000 || (es.captionOnlyFinals || 0) > 0)
|
||||
return { level: 'red', detail: 'Falling badly behind — some sentences are captions only.' };
|
||||
if ((es.lagMs || 0) > 6000 || (es.queue || 0) >= 2 || (es.degradedFinals || 0) > 0)
|
||||
return { level: 'amber', detail: 'Running behind — idle languages may be trimmed to catch up.' };
|
||||
/*
|
||||
* RECENTLY, not ever.
|
||||
*
|
||||
* captionOnlyFinals is a running total for the whole service, and testing it
|
||||
* for "more than zero" meant a single stale sentence pinned the desk's light
|
||||
* red until the next restart. MEASURED: a campus Mac took one 16 s stall at
|
||||
* 14:25 — caused by two Screen Sharing sessions competing for the machine —
|
||||
* recovered immediately to a 421 ms lag with an empty queue, and still showed
|
||||
* red half an hour later. An operator who looks up and sees red wants to know
|
||||
* what is wrong NOW; a light that cannot go back to green teaches them to
|
||||
* ignore it.
|
||||
*
|
||||
* So the totals still travel, for the record and for the operator page, but
|
||||
* the verdict asks when it last happened.
|
||||
*/
|
||||
const recentCaptionOnly = es.captionOnlyAt && (Date.now() - es.captionOnlyAt) < TROUBLE_WINDOW_MS;
|
||||
if ((es.lagMs || 0) > 12000 || recentCaptionOnly)
|
||||
return { level: 'red',
|
||||
detail: 'Falling badly behind — some sentences are captions only.' + loadNote() };
|
||||
const recentDegraded = es.degradedAt && (Date.now() - es.degradedAt) < TROUBLE_WINDOW_MS;
|
||||
if ((es.lagMs || 0) > 6000 || (es.queue || 0) >= 2 || recentDegraded)
|
||||
return { level: 'amber',
|
||||
detail: 'Running behind — idle languages may be trimmed to catch up.' + loadNote() };
|
||||
if (es.cloud && es.cloud.parked)
|
||||
return { level: 'amber', detail: 'Natural voices are down — the built-in voices have taken over.' };
|
||||
/* A fallback in the last minute means one language just changed voice
|
||||
@@ -1076,7 +1117,22 @@ const server = http.createServer(async (req, res) => {
|
||||
}
|
||||
if (req.method === 'OPTIONS') { res.writeHead(origin ? 204 : 405).end(); return; }
|
||||
|
||||
if (p === '/healthz') { sendJSON(res, 200, { ok: true, live: state.live, build: BUILD }); return; }
|
||||
/*
|
||||
* The APP's version, plainly, to anyone who can reach this port.
|
||||
*
|
||||
* `build` here has always been a hash of the listener-page assets — a
|
||||
* cache-busting token, not a version — so the only way to tell what a campus
|
||||
* Mac was running was to install something and watch the hash move. With an
|
||||
* updater in the field that is the first question anyone asks, so answer it.
|
||||
*/
|
||||
if (p === '/healthz') {
|
||||
sendJSON(res, 200, {
|
||||
ok: true, live: state.live, build: BUILD,
|
||||
version: process.env.CT_APP_VERSION || '',
|
||||
appBuild: process.env.CT_APP_BUILD || ''
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// --- Companion / Stream Deck control -------------------------------
|
||||
// GET or POST, token in ?key= or the X-Control-Token header.
|
||||
|
||||
Reference in New Issue
Block a user