christian/church-translate
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases 11 Wiki Activity
Files
73ec7f1436fab0b280a8e485e8a756116dad8b2e
church-translate/app/tts-service.js
T

1273 lines
59 KiB
JavaScript
Raw Normal View History

Church Translate, as it runs at the campuses
2026-09-23 15:41:21 -04:00
/*
* tts-service.js — the built-in voices, in a process of its own.
*
* Listeners used to hear their own device's voice, which is a lottery: a Mac has
* ~180 and Chrome adds Google's, an iPad ships roughly one per language until
* someone digs through Settings, and Apple's novelty voices sort first and sound
* like cartoons. The church's requirement was voices that are built in and ready
* for anyone, so the Mac speaks and the phones just play what it sends.
*
* Runs as a forked PLAIN NODE child for the same reason the translator does:
* Chromium's PartitionAlloc traps on the allocations a model load makes, killing
* any process inside Electron. See ensureTranslator() in main.js.
*
* Measured on this M2 Max, already under load:
* synthesis ~394 ms for 7.8 s of speech (20x real time)
* encoding ~67 ms — afconvert on macOS, bundled ffmpeg on Windows
* size 51 KB of AAC for 7.8 s, 6.6x smaller than the WAV
*
* Threads are capped on purpose. Unbounded, each session grabs all 12 cores and
* extra workers buy nothing — the same flat curve NLLB showed. Capped at two,
* synthesis scales properly across languages, because Piper is compute-bound on
* a 63 MB model that stays in cache rather than DRAM-bandwidth-bound like NLLB.
*/
'use strict';
const fs = require('fs');
const path = require('path');
const os = require('os');
const { execFile, execFileSync } = require('child_process');
const voices = require('./voices.js');
const catalog = require('./voice-catalog.js');
const { normalizeForSpeech } = require('./speech-text.js');
let sherpa = null;
/* Overridden by main.js's {type:'config'} message in the packaged app; this is
only the default for running this process standalone. Windows keeps the big
directories in LOCALAPPDATA — %APPDATA% is the ROAMING profile and a domain
PC would try to sync tens of gigabytes of voice bundles at every logon. */
let VOICE_DIR = process.platform === 'win32'
? path.join(process.env.LOCALAPPDATA || path.join(os.homedir(), 'AppData', 'Local'), 'Church Translate Voices')
: path.join(os.homedir(), 'Library/Application Support/Church Translate Voices');
let CLIP_DIR = path.join(os.tmpdir(), 'church-translate-audio');
/* Loaded engines, one per language. Kept small: each costs ~330-400 MB of ONNX
arena regardless of the 63 MB model, so we do not hold voices nobody is
listening to. */
/* What each voice says when auditioned — a real line of scripture, so the
operator hears the voice on the material it will actually read. */
const SAMPLE_TEXT = {
eng_Latn: 'The Lord is my shepherd; I shall not want.',
spa_Latn: 'El Señor es mi pastor, nada me faltará.',
por_Latn: 'O Senhor é o meu pastor, nada me faltará.',
por_PT: 'O Senhor é o meu pastor, nada me faltará.',
spa_ES: 'El Señor es mi pastor; nada me faltará.',
spa_MX: 'El Señor es mi pastor; nada me faltará.',
fra_Latn: 'Le Seigneur est mon berger, je ne manquerai de rien.',
deu_Latn: 'Der Herr ist mein Hirte, mir wird nichts mangeln.',
ita_Latn: 'Il Signore è il mio pastore, non manco di nulla.',
zho_Hans: '耶和华是我的牧者，我必不致缺乏。',
kor_Hang: '여호와는 나의 목자시니 내게 부족함이 없으리로다.',
vie_Latn: 'Chúa là Đấng chăn giữ tôi, tôi sẽ chẳng thiếu thốn gì.',
arb_Arab: 'الرب راعيّ فلا يعوزني شيء.',
rus_Cyrl: 'Господь — Пастырь мой; я ни в чём не буду нуждаться.',
hat_Latn: 'Senyè a se gadò mwen, mwen p ap manke anyen.'
};
const loaded = new Map();
const MAX_LOADED = 6;
/*
* The operator's per-language voice choices, made by ear in the Voices
* card: { spa_Latn: { engine: "local", speaker: "ryan" },
* kor_Hang: { engine: "builtin", bundle: "...", model: "...onnx" } }.
* A language with no choice uses the shipped default in voices.js.
*/
let voiceChoice = {};
/* The effective built-in voice for a language: the operator's pick if they
made one, otherwise the default. Cloud/local picks are handled earlier. */
function builtinVoiceFor(lang) {
const c = voiceChoice[lang];
if (c && c.engine === "builtin" && c.bundle) {
const base = voices.voiceFor(lang) || {};
/* Keep the default's tuning (speed/gain/sid) only when the operator
stayed on the same bundle; another bundle has its own character. */
return c.bundle === base.bundle ? base
: { bundle: c.bundle, model: c.model, speed: c.speed, gain: c.gain, sid: c.sid };
}
return voices.voiceFor(lang);
}
/*
* Resolve a listener's chosen voice KEY into a voice the synthesiser can use.
*
* The key is whatever identifies a voice uniquely for its engine: a `speaker`
* for Kokoro and the local sidecar, a `bundle` for a built-in.
*
* Looked up in the CATALOG rather than trusted. The key arrives from a phone
* on the other side of a WebSocket, and `forLanguage()` is where the no-female
* rule lives — resolving through it means a listener cannot name a voice the
* catalog filters out, cannot reach another language's voice, and cannot
* inject a path. An unknown key resolves to null and the caller falls back to
* the operator's own choice, which is the right failure.
*/
function voiceByKey(lang, key) {
if (!key) return null;
for (const v of catalog.forLanguage(lang) || []) {
const k = (v.engine === 'local' || v.engine === 'kokoro') ? v.speaker : v.bundle;
if (k && k === String(key)) return v;
}
return null;
}
function reply(msg) { try { if (process.send) process.send(msg); } catch (_) {} }
function log(msg) { process.stdout.write(msg + '\n'); }
// ---------------------------------------------------------------- voices
function voicePath(v) { return path.join(VOICE_DIR, v.bundle); }
function isInstalled(v) {
try { return fs.existsSync(path.join(voicePath(v), v.model)); } catch (_) { return false; }
}
/* Fetch and unpack one voice bundle. ~60-80 MB each, so this is deliberately on
demand rather than shipped in the app: a campus that serves Spanish and
Korean should not download thirty languages it will never speak. */
/*
* Fetch and unpack one voice bundle, WITHOUT shelling out.
*
* This used to be `curl` piped into `tar xjf`, which is fine on macOS and
* fatal on Windows: System32's tar.exe is a libarchive build linked against
* zlib ONLY — it cannot read bzip2 — and every one of the 100+ bundles is
* .tar.bz2 with no .tar.gz published. On Windows that meant no voice could
* ever install, and with the natural-voice sidecar being Apple-Silicon-only
* too, a Windows build would have had no voices at all.
*
* So: global fetch (follows GitHub's redirect) into unbzip2-stream into
* tar-stream. Verified byte-identical to `tar xjf` on a real 64 MB bundle —
* 359 files, same total bytes — and it is now the same code on both platforms.
*/
function installVoice(v) {
return new Promise((resolve, reject) => {
fs.mkdirSync(VOICE_DIR, { recursive: true });
const url = voices.bundleUrl(v.bundle);
reply({ type: 'voiceProgress', bundle: v.bundle, status: 'downloading' });
(async () => {
const res = await fetch(url, { redirect: 'follow' });
if (!res.ok) throw new Error('HTTP ' + res.status);
const body = Buffer.from(await res.arrayBuffer());
/* A proxy or a captive portal returns an HTML error page with HTTP 200;
the bzip2 magic is the cheapest way to notice before unpacking it. */
if (body.length < 1024 || body.toString('latin1', 0, 3) !== 'BZh') {
throw new Error('that download was not a voice bundle');
}
reply({ type: 'voiceProgress', bundle: v.bundle, status: 'installing' });
const bunzip = require('unbzip2-stream');
const tar = require('tar-stream');
const { Readable } = require('stream');
await new Promise((res2, rej2) => {
const extract = tar.extract();
extract.on('entry', (header, stream, next) => {
/* Never write outside VOICE_DIR, whatever the archive claims. */
const dest = path.resolve(VOICE_DIR, header.name);
if (dest !== VOICE_DIR && !dest.startsWith(VOICE_DIR + path.sep)) {
stream.resume(); return next();
}
if (header.type === 'directory') {
fs.mkdirSync(dest, { recursive: true });
stream.resume(); return next();
}
fs.mkdirSync(path.dirname(dest), { recursive: true });
stream.pipe(fs.createWriteStream(dest)).on('finish', next).on('error', rej2);
});
extract.on('finish', res2);
extract.on('error', rej2);
Readable.from(body).pipe(bunzip()).on('error', rej2).pipe(extract);
});
reply({ type: 'voiceProgress', bundle: v.bundle, status: 'ready' });
})().then(resolve, (err) => reject(new Error('could not install ' + v.bundle + ': ' + err.message)));
});
}
async function engineFor(lang, explicit) {
/* Auditions key the cache by bundle so trying five voices for one language
does not evict the voice that language is actually using. */
const v = explicit || builtinVoiceFor(lang);
const key = explicit ? "audition:" + v.bundle : lang;
if (loaded.has(key)) {
const e = loaded.get(key);
e.usedAt = Date.now();
return e.tts;
}
if (!v) throw new Error('no built-in voice for ' + lang);
if (!isInstalled(v)) await installVoice(v);
if (!sherpa) sherpa = require('sherpa-onnx-node');
const dir = voicePath(v);
const tts = new sherpa.OfflineTts(voices.ttsConfig(dir, v));
/* Evict the least recently used rather than growing without bound. */
if (loaded.size >= MAX_LOADED) {
let oldest = null;
for (const [k, e] of loaded) if (!oldest || e.usedAt < loaded.get(oldest).usedAt) oldest = k;
if (oldest) { loaded.delete(oldest); log('evicted voice ' + oldest); }
}
loaded.set(key, { tts, usedAt: Date.now() });
log('voice ready: ' + key + ' (' + v.bundle + ')');
return tts;
}
// ------------------------------------------------------------- cloud voices
/*
* Premium cloud voices, strictly optional and strictly guarded.
*
* The contract: when a provider is configured and healthy, a sentence is
* synthesized in the cloud and sounds like a person; when ANYTHING goes wrong
* — no key, network down, wallet drained, one slow request — the same sentence
* comes out of the local Piper voice instead, a beat later, and the service
* never goes silent. A circuit breaker stops the flip-flop failure mode:
* three consecutive failures park that provider for five minutes rather than
* letting every sentence race a 2.5s timeout and alternate voices.
*
* Keys live in the APP's config (never the congregation server's config,
* which is readable over the operator API and the internet tunnel).
*/
let cloud = { provider: 'off' };
let cloudFailStreak = 0; // = strikes.length, kept by name for the stats
let cloudParkedUntil = 0;
let parkedAt = 0; // when the park was set — a success from a request in flight then unparks
/* Each strike remembers when its request was SENT, so a later success can
prove the sidecar was alive while it waited (contention, not death). */
let strikes = [];
let cloudStats = { ok: 0, failed: 0, fallback: 0 };
let cloudInFlight = 0;
let lastCloudError = '';
/*
* Languages the local engine has already spoken once. Measured: the first
* sentence in a language costs ~6s (the sidecar loads that language lazily)
* while every later one lands under 1s. A single timeout for both cases
* either fell back on every first sentence or waited absurdly long on
* every later one, so the first call in a language gets a generous budget
* and the rest get a strict one.
*/
const localWarm = new Set();
const CLOUD_MAX_INFLIGHT = 4; // fish's starter tier allows 5 concurrent
const CLOUD_TIMEOUT_MS = 2500; // beyond this, Piper is the better answer
/*
* THE SIDECAR'S OWN CLOCK, and the morning it cost a service its voice.
*
* MEASURED on the shipped build: the very first Kokoro answer after the sidecar
* starts takes ~4.3 s (load_model 0.2 s + the first generate for a language
* 4.1 s). Its budget was the fixed 2.5 s above. That one sentence is spoken in
* three byte-identical Spanish variants, so ONE cold sentence produced three
* aborts, three aborts tripped the breaker, and every premium voice was parked
* for five minutes — the congregation heard the Piper default, then Alex came
* back "on its own" when the clock ran out. The app had been up 92 minutes; only
* the Qwen3 model had ever been warmed.
*
* So budgets are now per (provider, language) PAIR and the first attempt on a
* cold pair is generous, once. Everything stays under main.js's 15 s tts:speak
* wall INCLUDING a Piper fallback (~1-2.5 s under load): past that wall a
* sentence is not slow, it is SILENT, which is worse than the wrong voice.
*/
const COLD_FIRST_MS = { kokoro: 8000, local: 10000 };
const WARM_BASE_MS = { kokoro: 4000, local: 6000 };
const WARM_MAX_MS = { kokoro: 7000, local: 9000 };
const warmPairs = new Set(); // 'kokoro:e', 'local:spanish' — answered at least once
const coldTried = new Set(); // pairs whose one generous attempt is spent
let sidecarReady = false; // main.js tells us; nothing counts against the breaker before this
let lastSidecarOkAt = 0; // liveness: a later request succeeding proves an abort was contention
let localInFlight = 0; // Qwen3 requests only — Kokoro must not gate German onto Piper
let fallbackAt = 0; // when a premium sentence last fell back to Piper (for the operator)
let parkedReason = '';
function pairKey(pick, lang) {
return pick.provider === 'kokoro' ? 'kokoro:' + (KOKORO_LANG[lang] || lang)
: pick.provider === 'local' ? 'local:' + (LOCAL_LANG[lang] || lang)
: pick.provider;
}
/* How long this request may take before Piper is the better answer. */
function premiumBudget(pick, lang, explicit) {
if (explicit) return 90000; // an audition is the operator waiting on purpose
const p = pick.provider;
if (p !== 'kokoro' && p !== 'local') return CLOUD_TIMEOUT_MS;
/*
* While main.js has not declared the sidecar ready, FAIL FAST to Piper.
* Nothing counts as a strike in that window anyway, and the alternative —
* the generous cold budget on every sentence — made each one wait 8 s and
* left the whole language chain 8-9 s per sentence behind while the
* sidecar was still loading (reviewer's repro: 8547 / 8123 / 8115 ms).
*/
if (!sidecarReady) return CLOUD_TIMEOUT_MS;
const pair = pairKey(pick, lang);
if (!warmPairs.has(pair) && !coldTried.has(pair)) return COLD_FIRST_MS[p];
/* Warm, but the sidecar serialises: cover the other requests already in it
(~175 ms each for Kokoro) and a Qwen3 job (2-8 s at RTF 0.5-1.5) if one
is running. The per-job deadline in speak() is what keeps the sum under
main.js's 15 s wall. */
const others = Math.max(0, cloudInFlight - 1);
const qwen = (p === 'kokoro' && localInFlight > 0) ? 4000 : 0;
const cap = (p === 'kokoro' && localInFlight > 0) ? 9000 : WARM_MAX_MS[p];
return Math.min(cap, WARM_BASE_MS[p] + others * 500 + qwen);
}
/* Does any configured voice actually use the sidecar? A cloud-only (fish /
google) setup must not be told its "voices are starting". */
function needsSidecar() {
if (cloud && (cloud.provider === 'kokoro' || cloud.provider === 'local')) return true;
return Object.keys(voiceChoice).some((l) => {
const e = (voiceChoice[l] || {}).engine;
return e === 'kokoro' || e === 'local';
});
}
function isConnRefused(err) {
const c = err && err.cause && err.cause.code;
return c === 'ECONNREFUSED' || c === 'ECONNRESET' || /ECONNREFUSED/.test(String(err && err.message || ''));
}
/*
* The park's clock ran out. Without this the streak stayed at 3 after the
* five minutes, so the very next miss re-parked everything on ONE strike —
* the same five-minute wrong voice, through a different door.
*/
function expireParkIfDue() {
if (cloudParkedUntil && Date.now() >= cloudParkedUntil) {
cloudParkedUntil = 0; parkedAt = 0; parkedReason = '';
strikes = []; cloudFailStreak = 0;
log('premium voices unparked — park expired');
pushStats('unparked');
}
}
/* Degradation pushed to the app the moment it happens, not on the next 20 s tick. */
function pushStats(why) {
reply({ type: 'cloudStats', stats: Object.assign({}, cloudStats, {
parked: Date.now() < cloudParkedUntil, parkedUntil: cloudParkedUntil, parkedReason,
strikes: cloudFailStreak, fallbackAt, sidecarReady: needsSidecar() ? sidecarReady : null,
lastError: lastCloudError, provider: (cloud && cloud.provider) || 'off', why: why || ''
}) });
}
/*
* The local provider speaks Qwen3-TTS through an mlx-audio sidecar on this
* Mac — bought once for $0 (Apache-2.0 weights), no network, no metering.
* Its language names must be FULL WORDS: short codes silently fall through
* to auto-detect (measured — the difference between clean Spanish and a
* guess). Languages outside this map stay on Piper.
*/
/*
* Kokoro speaks in SINGLE-LETTER language codes, not words — a request with
* lang_code 'spanish' is rejected outright with
* AssertionError: ('spanish', {'a': 'American English', ... 'e': 'es', ...}).
* Its own table, because it shares nothing with the one below.
*
* MEASURED on this M2 Max: 3.50s of Spanish in 0.16s — RTF 0.05, and 0.12 with
* three streams at once. Roughly 3x faster than Piper and 60x faster than
* Qwen3, which is why it is the only realistic voice here that can cover every
* language at once.
*/
/* One table for both processes — see sidecar-langs.js for why. */
const { KOKORO_MODEL, KOKORO_LANG, LOCAL_LANG } = require('./sidecar-langs.js');
function cloudVoiceFor(lang) {
/*
* A voice the operator chose BY EAR for this language wins over the global
* provider setting — that is the whole contract of the Voices card. A
* built-in pick means "speak this language locally, whatever the premium
* provider is doing"; a premium pick names the exact speaker to use.
*/
expireParkIfDue();
const choice = voiceChoice[lang];
if (choice && choice.engine === 'builtin') return null;
if (choice && choice.engine === 'kokoro' && choice.speaker) {
return (Date.now() < cloudParkedUntil || !KOKORO_LANG[lang]) ? null
: { provider: 'kokoro', voice: choice.speaker };
}
if (choice && choice.engine === 'local' && choice.speaker) {
/* Honoured even when the global provider is 'off': choosing a premium
voice for ONE language is a legitimate setup (Spanish premium, the rest
built-in), and the sidecar is running either way. */
return Date.now() < cloudParkedUntil ? null
: { provider: 'local', voice: choice.speaker };
}
if (choice && (choice.engine === 'fish' || choice.engine === 'google') && choice.voice) {
return Date.now() < cloudParkedUntil ? null
: { provider: choice.engine, voice: choice.voice };
}
if (!cloud || cloud.provider === 'off' || !cloud.provider) return null;
if (Date.now() < cloudParkedUntil) return null;
const per = (cloud.perLang || {})[lang];
if (cloud.provider === 'fish') return (per || cloud.fishVoice) ? { provider: 'fish', voice: per || cloud.fishVoice } : null;
if (cloud.provider === 'google') return per ? { provider: 'google', voice: per } : null;
if (cloud.provider === 'local') return LOCAL_LANG[lang] ? { provider: 'local', voice: per || cloud.localVoice || 'aiden' } : null;
if (cloud.provider === 'kokoro') return KOKORO_LANG[lang] ? { provider: 'kokoro', voice: per || cloud.kokoroVoice || 'em_alex' } : null;
return null;
}
/* Concatenate WAVs that share a format (they come from one engine). */
function joinWavs(parts) {
if (parts.length === 1) return parts[0];
const rate = parts[0].readUInt32LE(24) || 24000;
const bodies = parts.map((b) => b.subarray(44));
/* A breath between sentences, same as the built-in path. */
const gap = Buffer.alloc(Math.round(rate * 0.22) * 2);
const pieces = [];
bodies.forEach((b, i) => { if (i) pieces.push(gap); pieces.push(b); });
const data = Buffer.concat(pieces);
const out = Buffer.alloc(44 + data.length);
parts[0].copy(out, 0, 0, 44);
out.writeUInt32LE(36 + data.length, 4);
out.writeUInt32LE(data.length, 40);
data.copy(out, 44);
return out;
}
async function cloudSynth(lang, text, explicit, opts) {
opts = opts || {};
/*
* Multi-sentence text goes one sentence at a time. A premium speaker that
* stops cleanly on a single sentence can run away on a passage — measured:
* one sentence spoken, then 92 seconds of silence with no end token. This
* also keeps every runaway contained to the sentence that caused it.
*/
const sentences = String(text).split(/(?<=[.!?\u3002\uff01\uff1f])\s+/u).filter((s) => s.trim());
if (sentences.length > 1) {
const parts = [];
for (const s of sentences) {
const one = await cloudSynth(lang, s, explicit, opts);
if (!one) return null; // any failure falls back whole
parts.push(one.wav);
}
const who = explicit ? (explicit.engine || 'local') : ((cloudVoiceFor(lang) || {}).provider || 'local');
return { wav: joinWavs(parts), provider: who };
}
const pick = explicit && explicit.engine
? { provider: explicit.engine, voice: explicit.speaker || explicit.voice }
: cloudVoiceFor(lang);
const instruct = (explicit && explicit.instruct) ? String(explicit.instruct).slice(0, 300) : '';
/*
* Capacity is never a reason to change VOICE. The inflight cap exists for
* metered cloud tiers; with five pinned languages it sent the fifth one to
* Piper on every single sentence — a different voice for one language, all
* service long. Kokoro is exempt: the sidecar serialises at ~175 ms a
* request, so queueing costs a beat and keeps the voice.
*/
if (!pick) return null;
if (!explicit && pick.provider !== 'kokoro' && cloudInFlight >= CLOUD_MAX_INFLIGHT) return null;
/*
* Qwen3 only. MEASURED: RTF 0.48 alone but 1.5 at three concurrent, so
* parallel requests make every one slower than real time.
*
* Kokoro is explicitly NOT held to this — measured at RTF 0.05 alone and
* 0.12 with three at once, it has headroom the bigger model never had, and
* serialising it would throw away the entire reason to use it.
*/
/* Counted against OTHER Qwen3 jobs only. Counting Kokoro too meant German
went to Piper whenever any Spanish sentence was in flight — nearly always. */
if (pick.provider === 'local' && !explicit && localInFlight >= 1) return null;
cloudInFlight++;
if (pick.provider === 'local') localInFlight++;
const sentAt = Date.now();
const ctl = new AbortController();
/*
* Live speech is impatient (a late sentence is worse than a plain one),
* but an AUDITION is the operator deliberately waiting to hear one voice
* — cutting that off at six seconds just showed them "aborted" instead of
* the voice they asked for. Cold languages also need room to load.
*/
const fullBudget = premiumBudget(pick, lang, explicit);
let budget = fullBudget;
/*
* ONE deadline per speak() job, not one budget per sentence. A caption that
* splits into two sentences used to get two full budgets (cold 8 s, then
* warm 7 s) and THEN the Piper fallback — 15.4 s measured — past main.js's
* 15 s wall, where the sentence is not late but silent. Every sentence's
* budget is clamped to what is left, and with under 800 ms left the premium
* attempt is skipped so Piper still has room.
*
* Applied whenever a deadline is present — including a LISTENER'S ALTERNATE
* voice, which travels as `explicit` and used to keep the audition's 90 s
* (review measured a 16 s alternate sentence: silent on every phone on that
* voice). Auditions never carry a deadline, so they keep their 90 s.
*/
let clamped = false;
if (opts.deadline) {
const left = opts.deadline - Date.now();
if (left < 800) { cloudInFlight--; if (pick.provider === 'local') localInFlight--; return null; }
if (left < budget) { budget = left; clamped = true; }
}
const timer = setTimeout(() => ctl.abort(), budget);
try {
let body;
if (pick.provider === 'local' || pick.provider === 'kokoro') {
/* One sidecar, two models. Kokoro is small and fast enough to serve
every language; Qwen3 is the expressive one and serves about one. */
const isKokoro = pick.provider === 'kokoro';
const res = await fetch('http://127.0.0.1:' + (cloud.localPort || 43111) + '/v1/audio/speech', {
method: 'POST',
signal: ctl.signal,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: isKokoro ? (cloud.kokoroModel || KOKORO_MODEL)
: (cloud.localModel || 'mlx-community/Qwen3-TTS-12Hz-1.7B-CustomVoice-8bit'),
voice: pick.voice,
input: text,
lang_code: isKokoro ? KOKORO_LANG[lang] : LOCAL_LANG[lang],
response_format: 'wav',
/*
* The interpreter's reflex, for Kokoro too. Until now the lag-driven
* hurry only ever reached the Piper path; a Kokoro clip that was
* late could only be DROPPED by the stream, never quickened, so the
* dropper was the sole catch-up mechanism. Live speech only — an
* audition or a listener's chosen alternate must never speed up
* because the room's stream happens to be behind.
*/
...((isKokoro && !explicit && opts.speed && opts.speed !== 1) ? { speed: opts.speed } : {}),
/*
* Plain-English delivery direction, when one was given. This model
* takes it, and it changes the DELIVERY rather than the volume —
* MEASURED: "whisper this softly and intimately" made the same
* Spanish sentence 6.2s where the plain reading was 4.0s. Wording
* matters and is not predictable ("speak loudly and forcefully"
* came out QUIETER than plain), which is exactly why the operator
* needs to try wordings by ear rather than be handed one.
*/
...(instruct ? { instruct } : {}),
/* Both guards are measured, not theoretical: an uncapped generation
once ran to 96 SECONDS of audio for a one-line sentence, and the
default temperature made the same sentence vary 5-12s across
runs. */
max_tokens: Math.min(900, 80 + text.length * 4),
/*
* temperature 0, and it is not a style preference — it is the fix
* for "five voices randomly switching". This model samples its
* SPEAKER along with its audio, so any temperature above zero
* re-rolls who is talking on every sentence. Measured: at 0.2 the
* same sentence drifted 122-145 Hz (3 semitones) across takes; at
* 0 it is bit-identical every time. One language, one voice, all
* service long.
*/
temperature: 0
})
});
if (!res.ok) throw new Error('local ' + res.status);
body = Buffer.from(await res.arrayBuffer());
/* The sidecar returns 200 + audio even for garbage input — validate. */
if (body.length < 512 || body.toString('latin1', 0, 4) !== 'RIFF') throw new Error('local: not a wav');
const rate = body.readUInt32LE(24) || 24000;
const secs = (body.length - 44) / 2 / rate;
const maxSecs = 2 + text.length * 0.22;
if (secs > maxSecs) {
/*
* Some speakers never emit a stop token for some languages — they
* generate until whatever cap you give them. Unusable live, but the
* operator auditioning voices needs to HEAR that, not read a refusal:
* so an audition returns the audio with a warning attached, while
* live speech still refuses and falls back to the built-in voice.
*/
const why = 'never stops in ' + (LOCAL_LANG[lang] || lang) + ' — ' + secs.toFixed(0) +
's of audio for a ' + text.length + '-character line';
if (!explicit) throw new Error(why);
/* Trim the preview to something listenable and hand back the warning. */
const keep = Math.min(body.length, 44 + Math.round(rate * 6) * 2);
const head = body.subarray(0, keep);
head.writeUInt32LE(keep - 8, 4);
head.writeUInt32LE(keep - 44, 40);
cloudFailStreak = 0;
if (pick.provider === 'local') localWarm.add(lang);
warmPairs.add(pairKey(pick, lang));
lastSidecarOkAt = Date.now();
return { wav: head, provider: pick.provider, warning: why };
}
} else if (pick.provider === 'fish') {
const res = await fetch('https://api.fish.audio/v1/tts', {
method: 'POST',
signal: ctl.signal,
headers: {
'Authorization': 'Bearer ' + (cloud.fishKey || ''),
'Content-Type': 'application/json',
'model': cloud.fishModel || 's2.1-pro'
},
body: JSON.stringify({
text,
reference_id: pick.voice,
format: 'wav',
sample_rate: 24000,
latency: 'balanced'
})
});
if (!res.ok) throw new Error('fish ' + res.status);
body = Buffer.from(await res.arrayBuffer());
} else {
const res = await fetch('https://texttospeech.googleapis.com/v1/text:synthesize?key=' +
encodeURIComponent(cloud.googleKey || ''), {
method: 'POST',
signal: ctl.signal,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
input: { text },
voice: { languageCode: pick.voice.split('-').slice(0, 2).join('-'), name: pick.voice },
audioConfig: { audioEncoding: 'LINEAR16', sampleRateHertz: 24000 }
})
});
if (!res.ok) throw new Error('google ' + res.status);
const j = await res.json();
body = Buffer.from(j.audioContent || '', 'base64');
/* LINEAR16 from Google is raw PCM without a header — wrap it. */
const wav = Buffer.alloc(44 + body.length);
wav.write('RIFF', 0); wav.writeUInt32LE(36 + body.length, 4);
wav.write('WAVE', 8); wav.write('fmt ', 12);
wav.writeUInt32LE(16, 16); wav.writeUInt16LE(1, 20); wav.writeUInt16LE(1, 22);
wav.writeUInt32LE(24000, 24); wav.writeUInt32LE(48000, 28);
wav.writeUInt16LE(2, 32); wav.writeUInt16LE(16, 34);
wav.write('data', 36); wav.writeUInt32LE(body.length, 40);
body.copy(wav, 44);
body = wav;
}
if (!body || body.length < 512) throw new Error('empty audio');
cloudFailStreak = 0;
cloudStats.ok++;
if (pick.provider === 'local') localWarm.add(lang);
strikes = [];
if (pick.provider === 'local' || pick.provider === 'kokoro') {
warmPairs.add(pairKey(pick, lang));
lastSidecarOkAt = Date.now();
/* This request was already in the sidecar when the park was set, and it
came back fine: the sidecar was alive, the strikes were contention. */
if (cloudParkedUntil && parkedAt >= sentAt) {
cloudParkedUntil = 0; parkedAt = 0; parkedReason = '';
log('premium voices unparked — a request sent before the park succeeded');
pushStats('unparked');
}
}
/* The caller tags the clip with the engine that produced it (the
operator Test button reports it) and `pick` is local to this
function, so it travels with the audio. */
return { wav: body, provider: pick.provider };
} catch (err) {
lastCloudError = (err && err.message) || String(err);
cloudStats.failed++;
const isSidecar = pick.provider === 'local' || pick.provider === 'kokoro';
const pair = pairKey(pick, lang);
const wasCold = isSidecar && !warmPairs.has(pair) && !coldTried.has(pair);
/* The generous attempt is spent by an attempt that REACHED the sidecar
(an answer, an HTTP error, or an abort while it was working) — not by a
connection refused, which says nothing about this pair. */
if (isSidecar && !isConnRefused(err)) coldTried.add(pair);
/*
* What counts as a STRIKE. The breaker exists for a provider that is
* genuinely down, and four kinds of failure say nothing about that:
* - an AUDITION (explicit): the operator trying voices during a blip
* used to park the live service for five minutes;
* - the one cold-load attempt on a pair the sidecar has never spoken;
* - anything while main.js says the sidecar is not ready yet;
* - an abort while a request sent AFTER this one has since succeeded —
* the sidecar is demonstrably alive, this was contention.
*/
const provedAlive = isSidecar && lastSidecarOkAt > sentAt;
/* An abort because the JOB's deadline shortened this sentence's budget says
nothing about the provider — it answered the earlier sentences fine. */
const countable = !explicit && !wasCold && !(isSidecar && !sidecarReady) && !provedAlive && !clamped;
if (countable) { strikes.push(sentAt); cloudFailStreak = strikes.length; }
if (cloudFailStreak >= 3 && Date.now() >= cloudParkedUntil) {
cloudParkedUntil = Date.now() + 5 * 60 * 1000;
parkedAt = Date.now();
parkedReason = String(err && err.message || err).slice(0, 120);
log('premium voices parked for 5 min after 3 straight failures (' + parkedReason + ')');
pushStats('parked');
} else if (!countable) {
log('premium miss not counted (' + (explicit ? 'audition/alternate' : wasCold ? 'cold load' :
!sidecarReady ? 'sidecar starting' : clamped ? 'job deadline' : 'contention') + '): ' + lastCloudError);
}
return null;
} finally {
clearTimeout(timer);
cloudInFlight--;
if (pick.provider === 'local') localInFlight--;
}
}
// ---------------------------------------------------------------- synthesis
/*
* SEEDED FROM THE CLOCK, NOT FROM ZERO.
*
* A clip's name IS its URL: /audio/p3-spa_Latn.aac. Starting this counter at 0
* meant every restart minted the SAME names again — and the server serves that
* URL `public, max-age=3600, immutable` while the hardware output fetches it
* `force-cache`. Identical URL, different audio, declared permanent: the
* renderer and every listener's phone kept serving the PREVIOUS session's clip
* for an hour.
*
* MEASURED, and it is what "it said 'right now' when I said 'it is not
* translating'" was: the disk held the correct 'No está traduciendo.' as
* p2-spa_Latn.aac; what played was the p2 from the run before. Restarts are
* what surface it, which is why it looked intermittent and unrelated to
* anything the operator changed.
*
* MILLISECONDS times 1000, so each run owns a block of a thousand names per
* millisecond of its start time. A second-resolution seed was not enough: this
* app's own supervisor restarts the voice process on failure, and two starts
* inside one second would have collided again — the same bug, rarer, which is
* the worst kind. To collide now a run would have to emit more than a thousand
* clips within one millisecond of another run starting.
*
* Still plain digits, so the server's \d+ check is unchanged; nothing
* downstream parses the number, it only has to be unique. Stays inside
* Number.MAX_SAFE_INTEGER until the year 2255.
*/
let seq = Date.now() * 1000;
function encodeAac(wavPath, outPath, rate) {
/* ADTS, not MP4. Each ADTS frame carries its own header and length, so a
stream is just frames concatenated — which is what lets the live-audio
pump splice sentences into one continuous stream that survives a locked
screen. An .m4a cannot be spliced that way; its moov box describes the
whole file up front. The file is still a perfectly ordinary AAC that any
browser plays directly, so the per-clip path still works as a fallback.
One encoder module for both platforms — see server/aac-encode.js. */
/*
* The output rate is PINNED to the language's declared stream rate. Without
* this, a premium cloud clip (24k/44.1k WAV) landing next to a Piper clip
* (22.05k) makes the live stream detect a rate change and REBUILD — dumping
* every queued sentence. One language, one rate, forever, whatever produced
* the audio.
*/
require('./server/aac-encode.js').encodeAdts(wavPath, outPath, rate || 22050);
}
/*
* Below this, a clip is not speech. Ukrainian shipped broken for months at
* 0.006-0.019 s/char — a fraction of a second of noise per sentence, encoded
* and delivered to listeners with no error anywhere. Real speech in every
* language measured sits far above this; 33 characters per second is not a
* rate a human voice reaches.
*/
const MIN_SEC_PER_CHAR = 0.030;
/*
* Scale a finished 16-bit WAV in place.
*
* The premium path returns rendered WAV rather than samples, so it never went
* through the Piper gain block. Left alone, dynamics would apply to built-in
* voices and silently not to premium ones — the kind of split that costs
* somebody an afternoon a year from now.
*/
function applyWavGainDb(wav, gainDb) {
const db = Number(gainDb) || 0;
if (!db || !Buffer.isBuffer(wav) || wav.length < 44) return wav;
const g = Math.pow(10, Math.max(-12, Math.min(12, db)) / 20);
/* Walk the chunks rather than assuming a 44-byte header — plenty of encoders
emit LIST/fact chunks before the data. */
let off = 12;
while (off + 8 <= wav.length) {
const id = wav.toString('ascii', off, off + 4);
const size = wav.readUInt32LE(off + 4);
const body = off + 8;
if (id === 'data') {
const end = Math.min(wav.length, body + size);
const out = Buffer.from(wav);
for (let i = body; i + 1 < end; i += 2) {
let v = Math.round(out.readInt16LE(i) * g);
if (v > 32000) v = 32000; else if (v < -32000) v = -32000;
out.writeInt16LE(v, i);
}
return out;
}
off = body + size + (size % 2);
}
return wav;
}
/*
* Kokoro pads every clip with silence — MEASURED at -40 dBFS on the live
* sidecar: 0.18-0.21 s before the first word and 0.13-0.25 s after the last,
* so a 1.45 s "Te necesitamos." is 26% silence. Multiplied by every sentence
* of a sermon that is the difference between keeping up and dropping. The
* stream adds its own breath between sentences (SENTENCE_GAP_MS), so nothing
* natural is lost by trimming here; a little lead is kept so the first
* consonant is never clipped.
*/
function trimWavSilence(wav, keepLeadMs, keepTrailMs) {
try {
if (!Buffer.isBuffer(wav) || wav.length < 44 + 4800) return wav;
if (wav.toString('latin1', 0, 4) !== 'RIFF' || wav.toString('latin1', 36, 40) !== 'data') return wav;
if (wav.readUInt16LE(20) !== 1 || wav.readUInt16LE(22) !== 1 || wav.readUInt16LE(34) !== 16) return wav;
const rate = wav.readUInt32LE(24) || 24000;
const n = (wav.length - 44) >> 1;
const win = Math.max(1, Math.round(rate * 0.01));
const thr = 32768 * 0.01; // -40 dBFS
const peak = (from, to) => {
let m = 0;
for (let i = from; i < to; i++) { const v = wav.readInt16LE(44 + i * 2); const a = v < 0 ? -v : v; if (a > m) m = a; }
return m;
};
let lead = 0;
for (let i = 0; i < n; i += win) { if (peak(i, Math.min(n, i + win)) > thr) { lead = i; break; } if (i + win >= n) return wav; }
let trail = n;
for (let i = n - win; i >= 0; i -= win) { if (peak(i, Math.min(n, i + win)) > thr) { trail = Math.min(n, i + win); break; } }
const start = Math.max(0, lead - Math.round(rate * keepLeadMs / 1000));
const end = Math.min(n, trail + Math.round(rate * keepTrailMs / 1000));
if (end - start < win * 4 || (start === 0 && end === n)) return wav;
const body = wav.subarray(44 + start * 2, 44 + end * 2);
const out = Buffer.alloc(44 + body.length);
wav.copy(out, 0, 0, 44);
out.writeUInt32LE(36 + body.length, 4);
out.writeUInt32LE(body.length, 40);
body.copy(out, 44);
return out;
} catch (_) { return wav; }
}
/*
* Clip names now say WHICH ENGINE spoke: k = Kokoro, q = Qwen3 (local),
* c = a cloud provider, p = Piper. The transcript records these names, and the
* morning a voice switched mid-service the transcript could not tell us
* whether a single sentence had been Piper or Kokoro. Nothing parses the
* prefix (verified: the server serves /audio/<name> and the stream is keyed by
* language), so this is free.
*/
function clipPrefix(provider) {
return provider === 'kokoro' ? 'k' : provider === 'local' ? 'q' : provider === 'piper' ? 'p' : 'c';
}
/*
* SYNTHESISE ONCE, SPEAK IN EVERY VARIANT.
*
* spa_Latn, spa_ES and spa_MX carry byte-identical text and the same Kokoro
* voice, yet were three separate synth requests per sentence (the model side
* already dedupes; the voice side did not) — 5 requests where 2 would do,
* which is also what pushed the 5th language over the inflight cap onto
* Piper. The key is everything that changes the AUDIO. Never shared for a
* listener's chosen alternate (`alt`), and a failure is not cached: the
* follower re-synthesises itself.
*/
const synthCache = new Map(); // key -> { promise, at }
const SYNTH_CACHE_MS = 3 * 60 * 1000; // under sweepClips' 5 min, so the file is still there
setInterval(() => {
const cut = Date.now() - SYNTH_CACHE_MS;
for (const [k, e] of synthCache) if (e.at < cut) synthCache.delete(k);
}, 60 * 1000).unref();
function cloneClip(done, lang, started) {
const pfx = (done.name || 'c').charAt(0);
const name = pfx + (++seq) + '-' + lang + '.aac';
const file = path.join(CLIP_DIR, name);
fs.copyFileSync(done.file, file);
return Object.assign({}, done, { file, name, totalMs: Date.now() - started, shared: true });
}
async function speak(job) {
const started = Date.now();
/* The caption keeps the original text; only the spoken form is normalised. */
const text = normalizeForSpeech(job.text, job.lang);
if (!text) {
/* Nothing sayable. Asking the engine to say nothing emits an audible burst,
not silence, so do not ask it. */
return { empty: true };
}
/*
* A listener may be listening on an alternate voice the operator opened up.
* `alt` is null for everyone else, which is the whole room by default, so
* this path costs nothing until someone actually picks something.
*/
const alt = voiceByKey(job.lang, job.voice);
const voice = (alt && alt.engine === 'builtin')
? { bundle: alt.bundle, model: alt.model, speed: alt.speed, gain: alt.gain, sid: alt.sid }
: (builtinVoiceFor(job.lang) || {});
const pinRate = voices.sampleRateFor(job.lang);
/*
* PACE FIRST, for every engine. The interpreter's reflex: when this
* language's spoken audio has fallen behind its captions, deliver a little
* faster until it catches up. It used to be computed AFTER the premium
* branch had already returned, so a Kokoro clip that was late could only
* be dropped by the stream, never quickened.
*
* Piper: starts at 1 s behind, +5%/s, capped +16% — measured, a full hurry
* shortens a clip ~14%, and past that a voice stops sounding like someone
* talking. Kokoro's speed control is weaker — MEASURED on the live sidecar:
* 1.16 is only 4.7% shorter, 1.25 is 9.3%, and 1.35 falls off a cliff to
* 30% (rushed) — so it gets a longer lever, capped at 1.25, quantised to
* 0.05 so variants whose backlogs differ by a hair still share one
* synthesis (see the cache key below).
*/
const lag = Number(job.lagSec) || 0;
const hurry = Math.min(0.16, Math.max(0, (lag - 1) * 0.05));
const speed = (voice.speed || 1.0) * (1 + hurry);
/* Badly behind (past the stream's TARGET backlog) the 1.35 "cliff" — 30%
shorter, audibly rushed — is exactly the deficit, and a rushed sentence
beats a missing one. Otherwise the gentle curve. */
const kokoroSpeed = lag >= 6 ? 1.35
: Math.round((1 + Math.min(0.25, Math.max(0, (lag - 1) * 0.08))) * 20) / 20;
/* Leaves ~4 s for the Piper fallback (5 languages measured at 2.3 s) before
main.js's 15 s tts:speak wall. */
const deadline = started + 11000;
const premiumPick = (alt && alt.engine === 'builtin') ? null
: (alt && alt.engine !== 'builtin') ? { provider: alt.engine, voice: alt.speaker || alt.voice }
: cloudVoiceFor(job.lang);
/* Everything that changes the audio, and nothing that does not. */
const key = alt ? null : [
premiumPick ? (premiumPick.provider + '|' + premiumPick.voice + '|' +
(premiumPick.provider === 'kokoro' ? KOKORO_LANG[job.lang] : (LOCAL_LANG[job.lang] || job.lang)) +
'|' + kokoroSpeed)
: ('piper|' + (voice.bundle || '') + '|' + (voice.sid || 0) + '|' + speed.toFixed(2)),
text, Math.round((Number(job.gainDb) || 0) * 2) / 2, pinRate
].join('\u0001');
if (key) {
const hit = synthCache.get(key);
if (hit) {
try {
const done = await hit.promise;
if (done && !done.empty && done.file && fs.existsSync(done.file)) {
/* A follower that was already waiting shares the leader's result
even when that result is the Piper fallback — the variants of one
utterance switch together rather than each re-hitting a sidecar
that just missed. It is a fallback for THIS stream too, so it is
counted as one; the key is deleted below so the NEXT utterance
tries the natural voice again. */
if (premiumPick && done.source !== premiumPick.provider) { cloudStats.fallback++; fallbackAt = Date.now(); }
return cloneClip(done, job.lang, started);
}
} catch (_) { /* the first caller failed — make our own */ }
}
}
const produce = async () => {
/*
* Premium first, when configured: the sidecar or cloud returns finished
* WAV speech, so the whole Piper path is skipped — the clip goes straight
* to the encoder, pinned to this language's stream rate. Any failure
* falls through to Piper below. A listener who picked a BUILT-IN
* alternate skips the premium path entirely — otherwise cloudSynth would
* fall back to the operator's own pick and the page would lie.
*/
const cloudOut = (alt && alt.engine === 'builtin')
? null
: await cloudSynth(job.lang, text, (alt && alt.engine !== 'builtin') ? alt : null, { speed: kokoroSpeed, deadline });
if (cloudOut) {
let cloudWav = cloudOut.wav;
if (cloudOut.provider === 'kokoro' || cloudOut.provider === 'local') {
cloudWav = trimWavSilence(cloudWav, 40, 120);
}
fs.mkdirSync(CLIP_DIR, { recursive: true });
const cbase = clipPrefix(cloudOut.provider) + (++seq) + '-' + job.lang;
const cwav = path.join(CLIP_DIR, cbase + '.wavtmp');
const caac = path.join(CLIP_DIR, cbase + '.aac');
fs.writeFileSync(cwav, applyWavGainDb(cloudWav, job.gainDb));
encodeAac(cwav, caac, pinRate);
try { fs.unlinkSync(cwav); } catch (_) {}
const st = fs.statSync(caac);
/* Duration from the WAV header's data size (16-bit mono). */
const dataLen = cloudWav.length - 44;
const cloudRate = cloudWav.readUInt32LE(24) || 24000;
return {
file: caac, name: cbase + '.aac',
seconds: dataLen / 2 / cloudRate,
terminal: text.slice(-1),
sampleRate: pinRate,
synthMs: 0, totalMs: Date.now() - started, bytes: st.size,
source: cloudOut.provider
};
}
if (!(alt && alt.engine === 'builtin') && cloudVoiceFor(job.lang)) {
cloudStats.fallback++;
fallbackAt = Date.now();
if (cloudStats.fallback === 1 || cloudStats.fallback % 10 === 0) pushStats('fallback');
}
const tts = await engineFor(job.lang, voice.bundle ? voice : null);
if (!sherpa) sherpa = require('sherpa-onnx-node');
/*
* One caption can carry several sentences (the engine translates per source
* sentence and rejoins). sherpa splits them internally but butts the audio
* together with almost nothing between — measured 90-190 ms where real
* speech leaves 300+. The stream's inter-clip gap cannot reach inside a
* clip, so the pause is put in here, at the PCM level.
*/
const parts = text.split(/(?<=[.!?\u3002\uff01\uff1f])\s+/u).filter((p) => p.trim());
let audio;
if (parts.length <= 1) {
audio = tts.generate({ text, sid: voice.sid || 0, speed });
} else {
const chunks = [];
let sampleRate = 0;
for (const part of parts) {
const a = tts.generate({ text: part, sid: voice.sid || 0, speed });
sampleRate = a.sampleRate;
if (chunks.length) chunks.push(new Float32Array(Math.round(sampleRate * 0.28)));
chunks.push(a.samples);
}
const total = chunks.reduce((n, c) => n + c.length, 0);
const joined = new Float32Array(total);
let off = 0;
for (const c of chunks) { joined.set(c, off); off += c.length; }
audio = { samples: joined, sampleRate };
}
const synthMs = Date.now() - started;
const secs = audio.samples.length / audio.sampleRate;
if (text.length >= 8 && secs / text.length < MIN_SEC_PER_CHAR) {
/* A silent failure becomes a caption. speakAll() already treats a
per-language error as "no voice for this one" and the listener falls back
to their own device, which is worth far more than a burst of noise. */
log('voice output too short for ' + job.lang + ' (' + secs.toFixed(2) + 's for ' +
text.length + ' chars) — refusing the clip');
throw new Error('voice produced unintelligible audio for ' + job.lang);
}
/*
* Two gains, multiplied into one pass.
*
* voice.gain — per-voice correction, because models render at wildly
* different levels (the Italian miro peaks at ~0.2 where
* everyone else sits above 0.5) and a stream that changes
* loudness per language reads as broken.
*
* job.gainDb — how much louder or quieter the PREACHER was for this
* sentence than the room's normal, already compressed to
* about +/-7 dB by the engine. Off (0) unless the operator
* turned dynamics on.
*
* Hard-limited either way, so no combination can clip.
*/
const dynG = job.gainDb ? Math.pow(10, Math.max(-12, Math.min(12, job.gainDb)) / 20) : 1;
const totalG = (voice.gain || 1) * dynG;
if (totalG !== 1) {
for (let i = 0; i < audio.samples.length; i++) {
const x = audio.samples[i] * totalG;
audio.samples[i] = x > 0.98 ? 0.98 : (x < -0.98 ? -0.98 : x);
}
}
fs.mkdirSync(CLIP_DIR, { recursive: true });
const base = clipPrefix('piper') + (++seq) + '-' + job.lang;
const wav = path.join(CLIP_DIR, base + '.wav');
const aac = path.join(CLIP_DIR, base + '.aac');
sherpa.writeWave(wav, { samples: audio.samples, sampleRate: audio.sampleRate });
encodeAac(wav, aac, pinRate);
try { fs.unlinkSync(wav); } catch (_) {}
const seconds = secs;
return {
source: 'piper',
/* What the sentence ended on — the stream gives a full stop a full breath
and a mid-thought fragment only a beat. */
terminal: text.slice(-1),
file: aac,
name: base + '.aac',
sampleRate: pinRate,
seconds,
synthMs,
totalMs: Date.now() - started,
bytes: fs.statSync(aac).size
};
};
const promise = produce();
if (key) synthCache.set(key, { promise, at: Date.now() });
try {
const out = await promise;
/*
* NEVER cache a fallback. If the premium voice missed and Piper spoke this
* sentence, the next identical sentence must try the premium voice again —
* a worship chorus repeats the same line eleven times, and caching the
* miss would have re-created the very voice switch this file exists to
* prevent (measured in the harness before shipping, not after).
*/
if (key && premiumPick && out && out.source !== premiumPick.provider) synthCache.delete(key);
return out;
} catch (e) {
if (key) synthCache.delete(key);
throw e;
}
}
/* Clips are worth nothing once played. Sweep anything older than five minutes so
a two-hour service does not fill the disk with audio nobody will hear again. */
function sweepClips() {
try {
const cutoff = Date.now() - 5 * 60 * 1000;
for (const f of fs.readdirSync(CLIP_DIR)) {
const p = path.join(CLIP_DIR, f);
try { if (fs.statSync(p).mtimeMs < cutoff) fs.unlinkSync(p); } catch (_) {}
}
} catch (_) {}
}
setInterval(sweepClips, 60 * 1000).unref();
/* Degradation must be visible: while a premium provider is configured, its
health rides up to the app every 20s and lands on the operator's status. */
setInterval(() => {
/* Per-language Kokoro picks work with the global provider 'off', so this
cannot be gated on cloud.provider — it used to be, and a Kokoro-only
setup never reported anything. */
const anyPremium = (cloud && cloud.provider && cloud.provider !== 'off') ||
Object.keys(voiceChoice).some((l) => /^(kokoro|local|fish|google)$/.test((voiceChoice[l] || {}).engine));
if (anyPremium) pushStats();
}, 20000).unref();
// ---------------------------------------------------------------- protocol
process.on('message', async (msg) => {
msg = msg || {};
try {
if (msg.type === 'config') {
if (msg.voiceDir) VOICE_DIR = msg.voiceDir;
if (msg.clipDir) CLIP_DIR = msg.clipDir;
reply({
type: 'ready',
clipDir: CLIP_DIR,
spoken: voices.spokenLanguages(),
/* The shipped default bundle per language, so the Voices card can mark
which row is the default when the operator has not chosen one. */
defaults: Object.fromEntries(voices.spokenLanguages().map(
(l) => [l, (voices.voiceFor(l) || {}).bundle])),
installed: voices.spokenLanguages().filter((l) => isInstalled(voices.voiceFor(l)))
});
return;
}
if (msg.type === 'sidecar') {
/*
* main.js owns the sidecar process and tells us when it is genuinely
* ready (every configured voice has answered once). Coming up resets
* the breaker — the thing it was tripping on has been fixed — and going
* down forgets what was warm, because a restarted sidecar is cold again.
*/
const was = sidecarReady;
sidecarReady = !!msg.ready;
if (!sidecarReady) { warmPairs.clear(); coldTried.clear(); }
if (!was && sidecarReady) {
coldTried.clear(); // a fresh sidecar earns fresh first attempts
strikes = []; cloudFailStreak = 0; parkedAt = 0;
if (cloudParkedUntil) log('premium voices unparked — sidecar is ready');
cloudParkedUntil = 0; parkedReason = '';
pushStats('sidecar ready');
}
return;
}
if (msg.type === 'cloud') {
/* Premium-voice settings, pushed by the app whenever they change. A
distinct message type on purpose: replying 'ready' here would re-fire
the app's voices-ready path and re-preload everything. */
cloud = msg.cloud || { provider: 'off' };
strikes = []; cloudFailStreak = 0;
cloudParkedUntil = 0; parkedAt = 0; parkedReason = '';
reply({ type: 'cloudOk', provider: cloud.provider || 'off' });
return;
}
if (msg.type === 'stats') {
reply({ type: 'stats', id: msg.id, cloud: Object.assign({}, cloudStats, {
parked: Date.now() < cloudParkedUntil, parkedUntil: cloudParkedUntil, parkedReason,
strikes: cloudFailStreak, fallbackAt, sidecarReady: needsSidecar() ? sidecarReady : null,
lastError: lastCloudError, provider: (cloud && cloud.provider) || 'off', inFlight: cloudInFlight
}) });
return;
}
if (msg.type === 'voices') {
/* The operator changed a per-language voice choice. */
voiceChoice = msg.choices || {};
/* Drop cached engines for languages whose choice changed, so the next
sentence uses the new voice rather than the one already resident. */
for (const k of Array.from(loaded.keys())) {
if (!String(k).startsWith("audition:")) loaded.delete(k);
}
reply({ type: 'voicesOk', count: Object.keys(voiceChoice).length });
return;
}
if (msg.type === 'audition') {
/*
* Speak one sample sentence with a specific voice and hand the audio
* back as bytes — the point of the whole Voices card is that the
* operator judges by ear, so this must be able to play ANY candidate,
* including one that is not installed yet (it downloads first).
*/
const spec = msg.voice || {};
const text = normalizeForSpeech(msg.text || SAMPLE_TEXT[msg.lang] || SAMPLE_TEXT.eng_Latn, msg.lang);
const started = Date.now();
let wav;
let auditionWarning = '';
if (spec.engine === 'local' || spec.engine === 'kokoro' ||
spec.engine === 'fish' || spec.engine === 'google') {
const saved = voiceChoice;
try {
lastCloudError = '';
const out = await cloudSynth(msg.lang, text,
msg.instruct ? Object.assign({}, spec, { instruct: msg.instruct }) : spec);
if (!out) throw new Error(lastCloudError || 'that voice did not answer');
wav = out.wav;
auditionWarning = out.warning || '';
} finally { voiceChoice = saved; }
} else {
if (!sherpa) sherpa = require('sherpa-onnx-node');
const tts = await engineFor(msg.lang, spec.bundle ? spec : null);
const audio = tts.generate({ text, sid: spec.sid || 0, speed: spec.speed || 1.0 });
if (spec.gain && spec.gain !== 1) {
for (let i = 0; i < audio.samples.length; i++) {
const x = audio.samples[i] * spec.gain;
audio.samples[i] = x > 0.98 ? 0.98 : (x < -0.98 ? -0.98 : x);
}
}
fs.mkdirSync(CLIP_DIR, { recursive: true });
const tmp = path.join(CLIP_DIR, 'audition.wav');
sherpa.writeWave(tmp, { samples: audio.samples, sampleRate: audio.sampleRate });
wav = fs.readFileSync(tmp);
try { fs.unlinkSync(tmp); } catch (_) {}
}
const rate = wav.readUInt32LE(24) || 24000;
reply({ type: 'audition', id: msg.id, wav: wav.toString('base64'),
seconds: (wav.length - 44) / 2 / rate,
warning: auditionWarning,
ms: Date.now() - started });
return;
}
if (msg.type === 'speak') {
const out = await speak(msg);
if (out.empty) { reply({ type: 'empty', id: msg.id, lang: msg.lang }); return; }
reply({ type: 'audio', id: msg.id, lang: msg.lang, ...out });
return;
}
if (msg.type === 'preload') {
/* Warm the languages people are actually listening to, so the first
sentence of a service is not the one that pays for the download. */
for (const lang of msg.langs || []) {
try { await engineFor(lang); } catch (e) { log('preload failed for ' + lang + ': ' + e.message); }
}
reply({ type: 'preloaded', langs: msg.langs || [] });
return;
}
} catch (err) {
reply({ type: 'error', id: msg.id, lang: msg.lang, message: (err && err.message) || String(err) });
}
});
process.on('uncaughtException', (e) => {
reply({ type: 'fatal', message: e.message });
process.exit(1);
});
Reference in New Issue Copy Permalink