Church Translate, as it runs at the campuses
Live sermon translation into 100+ languages, running entirely on one Mac per campus: audio in, voice detection, Whisper on the GPU, NLLB-200 in its own process, and a built-in web server the congregation reads from on their phones. No audio leaves the building. This is the source only. What used to sit beside it — 14 GB of speech and translation models, 8 GB of voice bundles, 6 GB of natural voices, the Electron caches, and the app data holding the operator PIN and every service transcript — is machine state and downloaded assets, and stays in the NAS backup where it belongs. The vendored node, cloudflared and Windows ffmpeg binaries are out too (533 MB, re-fetched for packaging); the browser-side runtime beside them is kept, so a checkout still serves the congregation page. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,231 @@
|
||||
/*
|
||||
* kokoro-sidecar.js — the natural voices, without Python.
|
||||
*
|
||||
* WHY THIS EXISTS. On macOS the natural voices come from `mlx_audio.server`, a
|
||||
* Python sidecar that runs Kokoro on MLX. MLX is Apple-Silicon-only, so on
|
||||
* Windows that process cannot exist at all — and the built-in Piper voices are
|
||||
* a quality drop the church rejected by name.
|
||||
*
|
||||
* So this serves the SAME HTTP API on the SAME port using the ONNX runtime the
|
||||
* app already ships. Everything upstream — sidecarWanted(), the warm-up, the
|
||||
* readiness handshake, the circuit breaker, the per-language voice choices —
|
||||
* is untouched, because from tts-service.js's point of view nothing changed.
|
||||
*
|
||||
* MEASURED against the live MLX sidecar, same text and voice:
|
||||
* duration ratio 1.0000, 1.0000, 1.0000 (identical audio length)
|
||||
* rms ratio 1.476 / 1.471 / 1.457 -> this build is ~3.3 dB HOTTER
|
||||
* phonemes byte-identical to misaki's EspeakG2P
|
||||
* The gain below cancels that level difference so a Windows campus and a Mac
|
||||
* campus sound the same. Duration being exactly equal is what matters most:
|
||||
* the HLS window pins a language's clips to one rate and length, and a
|
||||
* different duration would change the stream's pacing.
|
||||
*
|
||||
* dtype MUST be fp32. fp16 returns 100% NaN on the CPU execution provider —
|
||||
* silent audio with a flattering speed number — and q8 changes clip length.
|
||||
*/
|
||||
'use strict';
|
||||
const http = require('http');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const PORT = Number(process.argv[2] || process.env.CT_KOKORO_PORT || 43111);
|
||||
const MODEL_ID = 'onnx-community/Kokoro-82M-v1.0-ONNX';
|
||||
const RATE = 24000;
|
||||
|
||||
/* Cancels the level difference against MLX (measured 1.457-1.476, mean 1.468). */
|
||||
const MLX_MATCH_GAIN = 1 / 1.468;
|
||||
|
||||
/* The app speaks Kokoro's single-letter codes; espeak-ng wants its own names. */
|
||||
const ESPEAK_VOICE = {
|
||||
a: 'en-us', b: 'en-gb', e: 'es', p: 'pt-br', f: 'fr-fr',
|
||||
i: 'it', h: 'hi', j: 'ja', z: 'cmn'
|
||||
};
|
||||
|
||||
function log(msg) { process.stdout.write('[kokoro] ' + msg + '\n'); }
|
||||
|
||||
/* ---------------------------------------------------------------- G2P
|
||||
* A reimplementation of misaki.espeak.EspeakG2P — the exact preprocessing the
|
||||
* Python side does — on top of the espeak-ng WASM build. Verified phoneme-for-
|
||||
* phoneme identical to misaki across punctuation, «», numbers and Devanagari.
|
||||
*/
|
||||
const ZWJ = ''; // espeak --ipa=3 tie character
|
||||
const PUNCT = ';:,.!?¡¿—…"«»“”()';
|
||||
const E2M = [
|
||||
['a^ɪ', 'I'], ['a^ʊ', 'W'], ['d^z', 'ʣ'], ['d^ʒ', 'ʤ'], ['e^ɪ', 'A'],
|
||||
['o^ʊ', 'O'], ['ə^ʊ', 'Q'], ['s^s', 'S'], ['t^s', 'ʦ'], ['t^ʃ', 'ʧ'], ['ɔ^ɪ', 'Y']
|
||||
];
|
||||
|
||||
let ESpeakNg = null;
|
||||
async function espeakIPA(text, voice) {
|
||||
if (!ESpeakNg) ESpeakNg = (await import('espeak-ng')).default;
|
||||
const e = await ESpeakNg({ arguments: ['--phonout', 'out', '-q', '--ipa=3', '-v', voice, text] });
|
||||
return e.FS.readFile('out', { encoding: 'utf8' });
|
||||
}
|
||||
|
||||
async function g2p(text, espeakVoice) {
|
||||
text = String(text).replace(/«/g, '“').replace(/»/g, '”').replace(/\(/g, '«').replace(/\)/g, '»');
|
||||
const re = new RegExp('([' + PUNCT.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '])');
|
||||
let out = '';
|
||||
for (const p of text.split(re)) {
|
||||
if (!p) continue;
|
||||
if (p.length === 1 && PUNCT.includes(p)) { out += p; continue; }
|
||||
if (!p.trim()) { out += p; continue; }
|
||||
out += p.match(/^\s*/)[0] + (await espeakIPA(p.trim(), espeakVoice)).trim() + p.match(/\s*$/)[0];
|
||||
}
|
||||
let ps = out.replaceAll(ZWJ, '^');
|
||||
for (const [o, n] of E2M) ps = ps.replaceAll(o, n);
|
||||
return ps.replaceAll('^', '').replaceAll('-', '')
|
||||
.replace(/«/g, '(').replace(/»/g, ')').replace(/\s+/g, ' ').trim();
|
||||
}
|
||||
|
||||
/* ------------------------------------------------------------- the model */
|
||||
let model = null, tokenizer = null, Tensor = null, loading = null;
|
||||
const styleCache = {};
|
||||
|
||||
function voicesDir() {
|
||||
const candidates = [
|
||||
process.env.CT_KOKORO_VOICES,
|
||||
path.join(__dirname, 'node_modules', 'kokoro-js', 'voices'),
|
||||
path.join(__dirname, '..', 'node_modules', 'kokoro-js', 'voices')
|
||||
].filter(Boolean);
|
||||
for (const c of candidates) { try { if (fs.existsSync(c)) return c; } catch (_) {} }
|
||||
throw new Error('voice style vectors not found');
|
||||
}
|
||||
|
||||
/*
|
||||
* One 256-float style vector per voice, chosen by TOKEN COUNT — Kokoro stores
|
||||
* 510 of them per voice and picks by length, which is how it varies delivery
|
||||
* with sentence length. Same indexing as the reference implementation.
|
||||
*/
|
||||
function styleFor(voice, nTokens) {
|
||||
if (!styleCache[voice]) {
|
||||
const buf = fs.readFileSync(path.join(voicesDir(), voice + '.bin'));
|
||||
styleCache[voice] = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.length));
|
||||
}
|
||||
const off = 256 * Math.min(Math.max(nTokens - 2, 0), 509);
|
||||
return styleCache[voice].slice(off, off + 256);
|
||||
}
|
||||
|
||||
async function ensureModel() {
|
||||
if (model) return;
|
||||
if (loading) return loading;
|
||||
loading = (async () => {
|
||||
const tf = await import('@huggingface/transformers');
|
||||
Tensor = tf.Tensor;
|
||||
tf.env.allowLocalModels = false;
|
||||
if (process.env.CT_MODELS_DIR) tf.env.cacheDir = process.env.CT_MODELS_DIR;
|
||||
const t0 = Date.now();
|
||||
const [m, tk] = await Promise.all([
|
||||
tf.StyleTextToSpeech2Model.from_pretrained(MODEL_ID, { dtype: 'fp32', device: 'cpu' }),
|
||||
tf.AutoTokenizer.from_pretrained(MODEL_ID)
|
||||
]);
|
||||
model = m; tokenizer = tk;
|
||||
log('model ready in ' + ((Date.now() - t0) / 1000).toFixed(1) + 's');
|
||||
})();
|
||||
return loading;
|
||||
}
|
||||
|
||||
/*
|
||||
* ONE generation at a time.
|
||||
*
|
||||
* ONNX inference saturates the CPU, and the app fires several languages at
|
||||
* once per sentence. Run concurrently they all slow down together and every
|
||||
* one of them risks blowing its budget and falling back to a built-in voice —
|
||||
* which is the voice-switching failure this whole engine exists to avoid. A
|
||||
* queue makes the first clip fast and the rest predictable.
|
||||
*/
|
||||
let chain = Promise.resolve();
|
||||
function serialize(fn) {
|
||||
const run = chain.then(fn, fn);
|
||||
chain = run.then(() => {}, () => {});
|
||||
return run;
|
||||
}
|
||||
|
||||
function wavOf(float32) {
|
||||
const n = float32.length, b = Buffer.alloc(44 + n * 2);
|
||||
b.write('RIFF', 0); b.writeUInt32LE(36 + n * 2, 4); b.write('WAVE', 8); b.write('fmt ', 12);
|
||||
b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22);
|
||||
b.writeUInt32LE(RATE, 24); b.writeUInt32LE(RATE * 2, 28);
|
||||
b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34);
|
||||
b.write('data', 36); b.writeUInt32LE(n * 2, 40);
|
||||
for (let i = 0; i < n; i++) {
|
||||
let v = float32[i] * MLX_MATCH_GAIN;
|
||||
v = v > 0.999 ? 0.999 : (v < -0.999 ? -0.999 : v);
|
||||
b.writeInt16LE(Math.round(v * 32767), 44 + i * 2);
|
||||
}
|
||||
return b;
|
||||
}
|
||||
|
||||
async function synth(voice, text, langCode, speed) {
|
||||
await ensureModel();
|
||||
const ev = ESPEAK_VOICE[langCode] || langCode || 'en-us';
|
||||
const phonemes = await g2p(text, ev);
|
||||
if (!phonemes) throw new Error('nothing to say');
|
||||
const { input_ids } = await tokenizer(phonemes, { truncation: true });
|
||||
const n = input_ids.dims.at(-1);
|
||||
const out = await model({
|
||||
input_ids,
|
||||
style: new Tensor('float32', styleFor(voice, n), [1, 256]),
|
||||
speed: new Tensor('float32', [Number(speed) || 1], [1])
|
||||
});
|
||||
return wavOf(out.waveform.data);
|
||||
}
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
const url = (req.url || '').split('?')[0];
|
||||
|
||||
if (req.method === 'GET' && (url === '/healthz' || url === '/')) {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ ok: true, engine: 'kokoro-onnx', ready: !!model }));
|
||||
return;
|
||||
}
|
||||
/* Same shape mlx_audio returns, so anything probing it sees what it expects. */
|
||||
if (req.method === 'GET' && url === '/v1/models') {
|
||||
res.writeHead(200, { 'Content-Type': 'application/json' });
|
||||
res.end(JSON.stringify({ object: 'list', data: [{ id: MODEL_ID, object: 'model', owned_by: 'system' }] }));
|
||||
return;
|
||||
}
|
||||
if (req.method === 'POST' && url === '/v1/audio/speech') {
|
||||
let body = '';
|
||||
req.on('data', (d) => { body += d; if (body.length > 1 << 20) req.destroy(); });
|
||||
req.on('end', () => {
|
||||
let j;
|
||||
try { j = JSON.parse(body); } catch (_) { res.writeHead(400).end('bad json'); return; }
|
||||
const voice = String(j.voice || '');
|
||||
const text = String(j.input || '');
|
||||
const lang = String(j.lang_code || 'a');
|
||||
if (!voice || !text) { res.writeHead(400).end('voice and input are required'); return; }
|
||||
/*
|
||||
* Only Kokoro lives here. A request for the Qwen3 expressive model (an
|
||||
* operator picking a "Local" voice) is refused plainly rather than
|
||||
* answered with the wrong voice — tts-service then falls back to the
|
||||
* built-in voice for that language, which is the honest outcome on a
|
||||
* platform where Qwen3/MLX cannot run.
|
||||
*/
|
||||
if (j.model && String(j.model).indexOf('Kokoro') === -1) {
|
||||
res.writeHead(501, { 'Content-Type': 'text/plain' });
|
||||
res.end('this build serves Kokoro only');
|
||||
return;
|
||||
}
|
||||
serialize(() => synth(voice, text, lang, j.speed))
|
||||
.then((wav) => {
|
||||
res.writeHead(200, { 'Content-Type': 'audio/wav', 'Content-Length': wav.length });
|
||||
res.end(wav);
|
||||
})
|
||||
.catch((err) => {
|
||||
log('synth failed (' + voice + '/' + lang + '): ' + err.message);
|
||||
res.writeHead(500, { 'Content-Type': 'text/plain' });
|
||||
res.end(String(err.message || err));
|
||||
});
|
||||
});
|
||||
return;
|
||||
}
|
||||
res.writeHead(404).end('not found');
|
||||
});
|
||||
|
||||
server.listen(PORT, '127.0.0.1', () => {
|
||||
log('listening on 127.0.0.1:' + PORT + ' (Kokoro via ONNX, no Python)');
|
||||
/* Load now rather than on the first sentence of the service. */
|
||||
ensureModel().catch((e) => log('model load failed: ' + e.message));
|
||||
});
|
||||
server.on('error', (e) => { log('cannot listen on ' + PORT + ': ' + e.message); process.exit(1); });
|
||||
Reference in New Issue
Block a user