church-translate/app/kokoro-sidecar.js

/*
 * kokoro-sidecar.js — the natural voices, without Python.
 *
 * WHY THIS EXISTS. On macOS the natural voices come from `mlx_audio.server`, a
 * Python sidecar that runs Kokoro on MLX. MLX is Apple-Silicon-only, so on
 * Windows that process cannot exist at all — and the built-in Piper voices are
 * a quality drop the church rejected by name.
 *
 * So this serves the SAME HTTP API on the SAME port using the ONNX runtime the
 * app already ships. Everything upstream — sidecarWanted(), the warm-up, the
 * readiness handshake, the circuit breaker, the per-language voice choices —
 * is untouched, because from tts-service.js's point of view nothing changed.
 *
 * MEASURED against the live MLX sidecar, same text and voice:
 *   duration ratio  1.0000, 1.0000, 1.0000  (identical audio length)
 *   rms ratio       1.476 / 1.471 / 1.457   -> this build is ~3.3 dB HOTTER
 *   phonemes        byte-identical to misaki's EspeakG2P
 * The gain below cancels that level difference so a Windows campus and a Mac
 * campus sound the same. Duration being exactly equal is what matters most:
 * the HLS window pins a language's clips to one rate and length, and a
 * different duration would change the stream's pacing.
 *
 * dtype MUST be fp32. fp16 returns 100% NaN on the CPU execution provider —
 * silent audio with a flattering speed number — and q8 changes clip length.
 */
'use strict';
const http = require('http');
const fs = require('fs');
const path = require('path');

const PORT = Number(process.argv[2] || process.env.CT_KOKORO_PORT || 43111);
const MODEL_ID = 'onnx-community/Kokoro-82M-v1.0-ONNX';
const RATE = 24000;

/* Cancels the level difference against MLX (measured 1.457-1.476, mean 1.468). */
const MLX_MATCH_GAIN = 1 / 1.468;

/* The app speaks Kokoro's single-letter codes; espeak-ng wants its own names. */
const ESPEAK_VOICE = {
  a: 'en-us', b: 'en-gb', e: 'es', p: 'pt-br', f: 'fr-fr',
  i: 'it', h: 'hi', j: 'ja', z: 'cmn'
};

function log(msg) { process.stdout.write('[kokoro] ' + msg + '\n'); }

/* ---------------------------------------------------------------- G2P
 * A reimplementation of misaki.espeak.EspeakG2P — the exact preprocessing the
 * Python side does — on top of the espeak-ng WASM build. Verified phoneme-for-
 * phoneme identical to misaki across punctuation, «», numbers and Devanagari.
 */
const ZWJ = '‍';                       // espeak --ipa=3 tie character
const PUNCT = ';:,.!?¡¿—…"«»“”()';
const E2M = [
  ['a^ɪ', 'I'], ['a^ʊ', 'W'], ['d^z', 'ʣ'], ['d^ʒ', 'ʤ'], ['e^ɪ', 'A'],
  ['o^ʊ', 'O'], ['ə^ʊ', 'Q'], ['s^s', 'S'], ['t^s', 'ʦ'], ['t^ʃ', 'ʧ'], ['ɔ^ɪ', 'Y']
];

let ESpeakNg = null;
async function espeakIPA(text, voice) {
  if (!ESpeakNg) ESpeakNg = (await import('espeak-ng')).default;
  const e = await ESpeakNg({ arguments: ['--phonout', 'out', '-q', '--ipa=3', '-v', voice, text] });
  return e.FS.readFile('out', { encoding: 'utf8' });
}

async function g2p(text, espeakVoice) {
  text = String(text).replace(/«/g, '“').replace(/»/g, '”').replace(/\(/g, '«').replace(/\)/g, '»');
  const re = new RegExp('([' + PUNCT.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '])');
  let out = '';
  for (const p of text.split(re)) {
    if (!p) continue;
    if (p.length === 1 && PUNCT.includes(p)) { out += p; continue; }
    if (!p.trim()) { out += p; continue; }
    out += p.match(/^\s*/)[0] + (await espeakIPA(p.trim(), espeakVoice)).trim() + p.match(/\s*$/)[0];
  }
  let ps = out.replaceAll(ZWJ, '^');
  for (const [o, n] of E2M) ps = ps.replaceAll(o, n);
  return ps.replaceAll('^', '').replaceAll('-', '')
           .replace(/«/g, '(').replace(/»/g, ')').replace(/\s+/g, ' ').trim();
}

/* ------------------------------------------------------------- the model */
let model = null, tokenizer = null, Tensor = null, loading = null;
const styleCache = {};

function voicesDir() {
  const candidates = [
    process.env.CT_KOKORO_VOICES,
    path.join(__dirname, 'node_modules', 'kokoro-js', 'voices'),
    path.join(__dirname, '..', 'node_modules', 'kokoro-js', 'voices')
  ].filter(Boolean);
  for (const c of candidates) { try { if (fs.existsSync(c)) return c; } catch (_) {} }
  throw new Error('voice style vectors not found');
}

/*
 * One 256-float style vector per voice, chosen by TOKEN COUNT — Kokoro stores
 * 510 of them per voice and picks by length, which is how it varies delivery
 * with sentence length. Same indexing as the reference implementation.
 */
function styleFor(voice, nTokens) {
  if (!styleCache[voice]) {
    const buf = fs.readFileSync(path.join(voicesDir(), voice + '.bin'));
    styleCache[voice] = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.length));
  }
  const off = 256 * Math.min(Math.max(nTokens - 2, 0), 509);
  return styleCache[voice].slice(off, off + 256);
}

async function ensureModel() {
  if (model) return;
  if (loading) return loading;
  loading = (async () => {
    const tf = await import('@huggingface/transformers');
    Tensor = tf.Tensor;
    tf.env.allowLocalModels = false;
    if (process.env.CT_MODELS_DIR) tf.env.cacheDir = process.env.CT_MODELS_DIR;
    const t0 = Date.now();
    const [m, tk] = await Promise.all([
      tf.StyleTextToSpeech2Model.from_pretrained(MODEL_ID, { dtype: 'fp32', device: 'cpu' }),
      tf.AutoTokenizer.from_pretrained(MODEL_ID)
    ]);
    model = m; tokenizer = tk;
    log('model ready in ' + ((Date.now() - t0) / 1000).toFixed(1) + 's');
  })();
  return loading;
}

/*
 * ONE generation at a time.
 *
 * ONNX inference saturates the CPU, and the app fires several languages at
 * once per sentence. Run concurrently they all slow down together and every
 * one of them risks blowing its budget and falling back to a built-in voice —
 * which is the voice-switching failure this whole engine exists to avoid. A
 * queue makes the first clip fast and the rest predictable.
 */
let chain = Promise.resolve();
function serialize(fn) {
  const run = chain.then(fn, fn);
  chain = run.then(() => {}, () => {});
  return run;
}

function wavOf(float32) {
  const n = float32.length, b = Buffer.alloc(44 + n * 2);
  b.write('RIFF', 0); b.writeUInt32LE(36 + n * 2, 4); b.write('WAVE', 8); b.write('fmt ', 12);
  b.writeUInt32LE(16, 16); b.writeUInt16LE(1, 20); b.writeUInt16LE(1, 22);
  b.writeUInt32LE(RATE, 24); b.writeUInt32LE(RATE * 2, 28);
  b.writeUInt16LE(2, 32); b.writeUInt16LE(16, 34);
  b.write('data', 36); b.writeUInt32LE(n * 2, 40);
  for (let i = 0; i < n; i++) {
    let v = float32[i] * MLX_MATCH_GAIN;
    v = v > 0.999 ? 0.999 : (v < -0.999 ? -0.999 : v);
    b.writeInt16LE(Math.round(v * 32767), 44 + i * 2);
  }
  return b;
}

async function synth(voice, text, langCode, speed) {
  await ensureModel();
  const ev = ESPEAK_VOICE[langCode] || langCode || 'en-us';
  const phonemes = await g2p(text, ev);
  if (!phonemes) throw new Error('nothing to say');
  const { input_ids } = await tokenizer(phonemes, { truncation: true });
  const n = input_ids.dims.at(-1);
  const out = await model({
    input_ids,
    style: new Tensor('float32', styleFor(voice, n), [1, 256]),
    speed: new Tensor('float32', [Number(speed) || 1], [1])
  });
  return wavOf(out.waveform.data);
}

const server = http.createServer((req, res) => {
  const url = (req.url || '').split('?')[0];

  if (req.method === 'GET' && (url === '/healthz' || url === '/')) {
    res.writeHead(200, { 'Content-Type': 'application/json' });
    res.end(JSON.stringify({ ok: true, engine: 'kokoro-onnx', ready: !!model }));
    return;
  }
  /* Same shape mlx_audio returns, so anything probing it sees what it expects. */
  if (req.method === 'GET' && url === '/v1/models') {
    res.writeHead(200, { 'Content-Type': 'application/json' });
    res.end(JSON.stringify({ object: 'list', data: [{ id: MODEL_ID, object: 'model', owned_by: 'system' }] }));
    return;
  }
  if (req.method === 'POST' && url === '/v1/audio/speech') {
    let body = '';
    req.on('data', (d) => { body += d; if (body.length > 1 << 20) req.destroy(); });
    req.on('end', () => {
      let j;
      try { j = JSON.parse(body); } catch (_) { res.writeHead(400).end('bad json'); return; }
      const voice = String(j.voice || '');
      const text = String(j.input || '');
      const lang = String(j.lang_code || 'a');
      if (!voice || !text) { res.writeHead(400).end('voice and input are required'); return; }
      /*
       * Only Kokoro lives here. A request for the Qwen3 expressive model (an
       * operator picking a "Local" voice) is refused plainly rather than
       * answered with the wrong voice — tts-service then falls back to the
       * built-in voice for that language, which is the honest outcome on a
       * platform where Qwen3/MLX cannot run.
       */
      if (j.model && String(j.model).indexOf('Kokoro') === -1) {
        res.writeHead(501, { 'Content-Type': 'text/plain' });
        res.end('this build serves Kokoro only');
        return;
      }
      serialize(() => synth(voice, text, lang, j.speed))
        .then((wav) => {
          res.writeHead(200, { 'Content-Type': 'audio/wav', 'Content-Length': wav.length });
          res.end(wav);
        })
        .catch((err) => {
          log('synth failed (' + voice + '/' + lang + '): ' + err.message);
          res.writeHead(500, { 'Content-Type': 'text/plain' });
          res.end(String(err.message || err));
        });
    });
    return;
  }
  res.writeHead(404).end('not found');
});

server.listen(PORT, '127.0.0.1', () => {
  log('listening on 127.0.0.1:' + PORT + ' (Kokoro via ONNX, no Python)');
  /* Load now rather than on the first sentence of the service. */
  ensureModel().catch((e) => log('model load failed: ' + e.message));
});
server.on('error', (e) => { log('cannot listen on ' + PORT + ': ' + e.message); process.exit(1); });