church-translate/app/_ctxprobe.mjs

import { pipeline, env } from '@huggingface/transformers';
env.allowLocalModels = false;
env.cacheDir = '/Users/Studio/Library/Application Support/Church Translate Models';

// --- chrF (character n-gram F-score, beta=2), written from scratch: no npm pkg exists for node
function ngrams(s, n) {
  const t = s.replace(/\s+/g, '');
  const m = new Map();
  for (let i = 0; i + n <= t.length; i++) { const g = t.slice(i, i + n); m.set(g, (m.get(g) || 0) + 1); }
  return m;
}
function chrF(hyp, ref, N = 6, beta = 2) {
  let pS = 0, rS = 0, cnt = 0;
  for (let n = 1; n <= N; n++) {
    const H = ngrams(hyp, n), R = ngrams(ref, n);
    let hs = 0, rs = 0, ov = 0;
    for (const [g, c] of H) { hs += c; ov += Math.min(c, R.get(g) || 0); }
    for (const [, c] of R) rs += c;
    if (hs === 0 || rs === 0) continue;
    pS += ov / hs; rS += ov / rs; cnt++;
  }
  if (!cnt) return 0;
  const p = pS / cnt, r = rS / cnt;
  if (p + r === 0) return 0;
  return (1 + beta * beta) * p * r / (beta * beta * p + r);
}

const PAIRS = [
  ["Habakkuk stood upon his watchtower and waited to see what the Lord would say.", "And the Lord answered him and told him to write the vision plainly upon tablets."],
  ["Turn with me to Second Chronicles chapter seven verse fourteen.", "If my people who are called by my name will humble themselves and pray, I will heal their land."],
  ["For by grace you have been saved through faith, and that not of yourselves.", "It is the gift of God, not of works, so that no one may boast."],
  ["Melchizedek, king of Salem, brought out bread and wine.", "He was priest of God Most High, and he blessed Abram that day."],
  ["The Lord is my shepherd; I shall not want.", "He maketh me to lie down in green pastures and leadeth me beside the still waters."],
  ["Jesus wept at the tomb of Lazarus.", "Then he cried with a loud voice and called his friend out of the grave."],
  ["Paul wrote to the church at Philippi from a prison cell.", "He told them to rejoice in the Lord always, and again he said rejoice."],
  ["The elders fell down before the Lamb with harps and golden bowls of incense.", "And they sang a new song, saying, worthy art thou to take the scroll."],
];
const LANGS = [['spa_Latn','Spanish'],['hat_Latn','Haitian'],['arb_Arab','Arabic'],['kor_Hang','Korean'],['vie_Latn','Vietnamese']];

const t0 = Date.now();
const pipe = await pipeline('translation', 'Xenova/nllb-200-distilled-600M', { dtype: 'fp32' });
console.log(`MODEL LOAD: ${((Date.now()-t0)/1000).toFixed(1)}s`);

const tr = async (text, tgt) => {
  const t = Date.now();
  const r = await pipe(text, { src_lang: 'eng_Latn', tgt_lang: tgt });
  return { text: (Array.isArray(r)?r[0]:r).translation_text.trim(), ms: Date.now()-t };
};
await tr(PAIRS[0][0], 'spa_Latn'); // warmup

const rows = [];
const tStart = Date.now();
for (const [code, name] of LANGS) {
  for (const [s1, s2] of PAIRS) {
    const a = await tr(s1, code);
    const b = await tr(s2, code);
    const c = await tr(s1 + ' ' + s2, code);
    const simA = chrF(c.text, a.text);
    const covB = chrF(c.text, b.text);
    const lenRatio = c.text.length / a.text.length;
    // clean-prefix test: does a.text appear verbatim at the head of c.text?
    const prefixExact = c.text.startsWith(a.text);
    // fuzzy prefix: chrF of the first a.text.length chars of c vs a
    const prefixFuzzy = chrF(c.text.slice(0, a.text.length), a.text);
    const dropped = simA > 0.85 && lenRatio < 1.35;
    rows.push({ name, code, s1: s1.slice(0,28), a: a.text, b: b.text, c: c.text,
      simA:+simA.toFixed(3), covB:+covB.toFixed(3), lenRatio:+lenRatio.toFixed(2),
      prefixExact, prefixFuzzy:+prefixFuzzy.toFixed(3), dropped,
      msA:a.ms, msB:b.ms, msC:c.ms });
  }
}
const wall = (Date.now()-tStart)/1000;

console.log(`\nWALL CLOCK for ${rows.length*3} calls (${PAIRS.length} pairs x ${LANGS.length} langs x 3): ${wall.toFixed(1)}s`);
console.log(`=> per-call avg ${(wall*1000/(rows.length*3)).toFixed(0)}ms\n`);

console.log('lang        dropRate  meanCovB  prefixExact  prefixFuzzy  meanLenRatio  msA   msC   ratio');
for (const [code,name] of LANGS) {
  const R = rows.filter(r=>r.code===code);
  const f=(k)=> (R.reduce((s,r)=>s+(typeof r[k]==='boolean'? (r[k]?1:0) : r[k]),0)/R.length);
  console.log(
    name.padEnd(12) +
    (f('dropped')*100).toFixed(0).padStart(6)+'%  ' +
    f('covB').toFixed(3).padStart(8) + '  ' +
    (f('prefixExact')*100).toFixed(0).padStart(10)+'%  ' +
    f('prefixFuzzy').toFixed(3).padStart(10) + '  ' +
    f('lenRatio').toFixed(2).padStart(12) + '  ' +
    f('msA').toFixed(0).padStart(4) + '  ' + f('msC').toFixed(0).padStart(4) + '  ' +
    (f('msC')/f('msA')).toFixed(2)
  );
}
console.log('\n--- samples ---');
for (const r of rows.filter(r=>r.code==='hat_Latn'||r.code==='arb_Arab').slice(0,6)) {
  console.log(`\n[${r.name}] ${r.s1}...  dropped=${r.dropped} covB=${r.covB} prefixExact=${r.prefixExact}`);
  console.log('  a:', r.a); console.log('  b:', r.b); console.log('  c:', r.c);
}
import fs from 'fs';
fs.writeFileSync('/private/tmp/claude-502/-Users-Studio-Documents-BandLineupDisplay/cee248f6-d861-4845-bd13-03867cf46596/scratchpad/rows.json', JSON.stringify(rows,null,1));