church-translate/app/selftest.js

/*
 * Proves the model half of the engine works without needing any audio hardware:
 * a known WAV goes in, a transcript and translations come out. When something
 * is wrong on a Sunday, this separates "the AI is broken" from "the audio
 * routing is wrong" in about a minute.
 */
const out = document.getElementById('out');
const lines = [];
function say(msg, cls) {
  lines.push(msg);
  out.innerHTML = lines.map((l) => l.startsWith('FAIL') ? '<span class="bad">' + l + '</span>'
                                : l.startsWith('PASS') ? '<span class="ok">' + l + '</span>' : l).join('\n');
  console.log('[selftest] ' + msg);
}

function makeWorker(label, task, model, dtype, fallback) {
  return new Promise((resolve, reject) => {
    const w = new Worker(new URL('./model-worker.js', import.meta.url), { type: 'module' });
    w.onerror = (e) => reject(new Error(e.message || 'worker error'));
    w.onmessage = (e) => {
      const m = e.data;
      if (m.type === 'gpu') { say('  gpu: ' + (m.features.includes('shader-f16') ? 'shader-f16 present' : 'no shader-f16')); return; }
      if (m.type === 'ready') { resolve(w); return; }
      if (m.type === 'loadError') {
        if (fallback && /f16|fp16/i.test(m.message)) {
          say('  ' + label + ': no fp16, retrying with integer weights');
          w.terminate();
          makeWorker(label, task, model, fallback, null).then(resolve, reject);
          return;
        }
        reject(new Error(m.message));
      }
    };
    w.postMessage({ type: 'load', label, task, model, dtype });
  });
}

let seq = 0;
const pending = new Map();
function run(worker, input, options, transfer) {
  return new Promise((resolve, reject) => {
    const id = ++seq;
    pending.set(id, { resolve, reject });
    const prev = worker.onmessage;
    worker.onmessage = (e) => {
      const m = e.data;
      if (m.type === 'result' || m.type === 'error') {
        const p = pending.get(m.id);
        if (p) { pending.delete(m.id); m.type === 'result' ? p.resolve(m.out) : p.reject(new Error(m.message)); }
        return;
      }
      if (prev) prev(e);
    };
    worker.postMessage({ type: 'run', id, input, options }, transfer || []);
  });
}

const ARGS = new URLSearchParams(location.search);

(async () => {
  let failures = 0;
  try {
    const cfg = await window.engineHost.getConfig();
    /* ?mt=q4f16 overrides the configured weights, and ?skipAsr=1 skips the
       speech model — loading it costs ~50s and it is not what we are testing
       when we are iterating on translation. */
    const mtDtype = ARGS.get('mt') || cfg.mtDtype || 'fp16';
    const skipAsr = ARGS.get('skipAsr') === '1';
    say('translation weights under test: ' + mtDtype + (skipAsr ? '  (speech model skipped)' : ''));

    say('loading audio…');
    const buf = await (await fetch('./selftest.wav')).arrayBuffer();
    const ctx = new OfflineAudioContext(1, 1, 16000);
    const decoded = await ctx.decodeAudioData(buf);
    const audio = decoded.getChannelData(0);
    say('  ' + decoded.duration.toFixed(1) + 's at ' + decoded.sampleRate + ' Hz');

    let text = 'Good morning, church. Turn with me to Habakkuk, chapter one. ' +
               'How long, O Lord, must I call for help, and you will not listen?';
    if (!skipAsr) {
    say('loading speech model…');
    const t0 = performance.now();
    const asr = await makeWorker('speech', 'automatic-speech-recognition', cfg.asrModel,
      { encoder_model: 'fp16', decoder_model_merged: 'q4' },
      { encoder_model: 'q4', decoder_model_merged: 'q4' });
    say('  ready in ' + ((performance.now() - t0) / 1000).toFixed(1) + 's');

    say('transcribing…');
    const t1 = performance.now();
    const res = await run(asr, audio.slice(), { language: 'en', task: 'transcribe', return_timestamps: false });
    const elapsed = (performance.now() - t1) / 1000;
    text = (Array.isArray(res) ? res.map((r) => r.text).join(' ') : res.text).trim();
    say('  "' + text + '"');
    say('  ' + elapsed.toFixed(1) + 's for ' + decoded.duration.toFixed(1) + 's of audio  ('
        + (decoded.duration / elapsed).toFixed(1) + '× real time)');

    const lower = text.toLowerCase();
    if (lower.includes('habakkuk')) say('PASS  heard "Habakkuk"');
    else { say('FAIL  did not hear "Habakkuk" — got: ' + text); failures++; }
    if (lower.includes('how long')) say('PASS  heard "how long"');
    else { say('FAIL  did not hear "how long"'); failures++; }
    if (decoded.duration / elapsed > 1) say('PASS  faster than real time');
    else { say('FAIL  slower than real time — will fall behind a preacher'); failures++; }
    }

    say('loading translation model…');
    const t2 = performance.now();
    window.engineHost.onMtProgress(() => {});
    await window.engineHost.mtEnsure();
    say('  ready in ' + ((performance.now() - t2) / 1000).toFixed(1) + 's');

    /* Catch the failure mode that quantized NLLB produces: fluent-looking
       output made of tokens from a dozen unrelated languages. Checking the
       script is cheap and catches it every time — an empty-or-identical check
       does not. */
    const SCRIPT = {
      spa_Latn: { want: /[a-z\u00c0-\u024f]/i, banned: /[\u4e00-\u9fff\u0600-\u06ff\uac00-\ud7af\u0400-\u04ff\u0900-\u097f]/ },
      hat_Latn: { want: /[a-z\u00c0-\u024f]/i, banned: /[\u4e00-\u9fff\u0600-\u06ff\uac00-\ud7af\u0400-\u04ff\u0900-\u097f]/ },
      zho_Hans: { want: /[\u4e00-\u9fff]/,     banned: /[\uac00-\ud7af\u0600-\u06ff]/ },
      kor_Hang: { want: /[\uac00-\ud7af]/,     banned: /[\u0600-\u06ff]/ },
      arb_Arab: { want: /[\u0600-\u06ff]/,     banned: /[\u4e00-\u9fff\uac00-\ud7af]/ }
    };

    function scriptOk(text, lang) {
      const rule = SCRIPT[lang];
      if (!rule) return true;
      if (!rule.want.test(text)) return false;
      if (rule.banned.test(text)) return false;
      return true;
    }

    const targets = ['spa_Latn', 'zho_Hans', 'kor_Hang', 'hat_Latn', 'arb_Arab'];
    say('translating into ' + targets.length + ' languages…');
    const t3 = performance.now();
    for (const tgt of targets) {
      const one = performance.now();
      const got = (await window.engineHost.mtTranslate({ text, src: 'eng_Latn', tgt })).trim();
      const name = window.Languages.get(tgt).name;
      say('  ' + name.padEnd(20) + ((performance.now() - one) / 1000).toFixed(2) + 's  ' + got.slice(0, 120));
      if (!got || got === text) { say('FAIL  ' + name + ' came back empty or untranslated'); failures++; }
      else if (!scriptOk(got, tgt)) { say('FAIL  ' + name + ' is not in the right script — quantized-model gibberish'); failures++; }
      else say('PASS  ' + name + ' is real ' + name);
    }
    const total = (performance.now() - t3) / 1000;
    say('  ' + total.toFixed(1) + 's for all ' + targets.length);
    if (total < 5) say('PASS  all languages inside 5s');
    else { say('FAIL  ' + total.toFixed(1) + 's is too slow for live use'); failures++; }

  } catch (e) {
    say('FAIL  ' + (e && e.message || e));
    failures++;
  }
  say(failures ? ('SELFTEST: ' + failures + ' FAILED') : 'SELFTEST: all passed');
})();