tools/interlinear/gen-stats.cjs

/*
 * gen-stats.cjs — rebuild interlinear/_strongs-stats.json from the tagged corpus.
 *
 * Powers the popup's "Used: 175× in Scripture — as “angel” (168×), “messenger” (7×)" line.
 * Shape: { "G0032": { n: <total occurrences>, g: [[gloss, count], ...] } }
 *
 * The generator that produced the shipped file is not in the repo, so the normalisation below is
 * reverse-engineered from it and then MEASURED against it: Greek numbers are validated exactly,
 * because the NT corpus did not change in this fix, so any disagreement there is my algorithm
 * differing rather than the data.
 */
'use strict';

const fs = require('fs');
const path = require('path');

const DIR = process.argv[2] || 'C:/Users/media/Downloads/bible-app/interlinear';
const MODE = process.argv[3] || 'compare';   // 'compare' | 'write'

/*
 * Fold the per-verse gloss into the form the card shows.
 * Deliberately light: "create" and "created" stay DISTINCT (the shipped data keeps both), so this
 * is not a stemmer. Only a final plural -s is folded, which is what turns "angels"/"angels,"
 * into "angel" while leaving "brothers —" alone (its last character is the dash, not the s).
 */
function normGloss(raw) {
  let s = String(raw || '').toLowerCase();
  s = s.split('[').join('').split(']').join('')
       .split('<').join('').split('>').join('');
  s = s.replace(/["'“”]/g, '');
  s = s.replace(/[.,;:?!]+/g, ' ');
  s = s.replace(/\s+/g, ' ').trim();

  /*
   * Strip the leading words that come from GRAMMAR rather than from the choice of English word.
   *
   * A Hebrew verb's gloss carries its subject ("and he said", "am about to create") and a noun's
   * carries its case ("of Paul", "in Israel"). Those are properties of the occurrence, not
   * different translations of the lexeme, and leaving them in fragments the count: without this,
   * H1254 reports «"he created" (4×)» instead of «"created" (21×)», which is the wrong answer to
   * "how is this word translated?".
   *
   * The list is derived from the shipped file's own output: it folds and/in/of/to and subject
   * pronouns ("said" 2912, "saying" 940 for H0559; "israel" 2357 absorbing "in israel") while
   * keeping meaning-bearing prepositions distinct ("o israel", "against israel", "among israel").
   * So `o`, `against`, `among`, `from`, `by`, `on`, `for`, `with`, `after`, `like` stay.
   *
   * The trailing space in the pattern is what makes this safe for the function words' OWN
   * entries: "was" alone never matches, so H1961 still reports "was" and "were".
   */
  const LEAD = /^(?:a|an|the|of|to|in|and|i|you|he|she|it|we|they|me|him|her|them|us|am|is|are|was|were|be|been|being|has|have|had|will|shall|do|does|did|let) /;
  for (;;) {
    const next = s.replace(LEAD, '');
    if (next === s || !next) break;
    s = next;
  }

  /*
   * Fold a final plural -s, guarded on -ss/-us/-is/-as.
   * The guard is the point: without it the shipped generator turned every "Jesus" into "jesu",
   * so the card read «Used: 907× in Scripture — as "jesu" (899×)».
   */
  if (/s$/.test(s) && s.length > 3 && !/(ss|us|is|as)$/.test(s)) s = s.slice(0, -1);
  return s.trim();
}

const counts = new Map();   // strong -> { n, glosses: Map }
for (const f of fs.readdirSync(DIR).filter((x) => x.endsWith('.json') && !x.startsWith('_'))) {
  const d = JSON.parse(fs.readFileSync(path.join(DIR, f), 'utf-8'));
  for (const k of Object.keys(d)) {
    for (const w of (d[k] || [])) {
      const s = w && w.s;
      if (!s) continue;
      if (!counts.has(s)) counts.set(s, { n: 0, glosses: new Map() });
      const rec = counts.get(s);
      rec.n++;
      const g = normGloss(w.g);
      if (g) rec.glosses.set(g, (rec.glosses.get(g) || 0) + 1);
    }
  }
}

const out = {};
for (const [s, rec] of counts) {
  const g = [...rec.glosses.entries()]
    .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
    .slice(0, 4);
  out[s] = { n: rec.n, g };
}

if (MODE === 'write') {
  const p = path.join(DIR, '_strongs-stats.json');
  fs.writeFileSync(p, JSON.stringify(out));
  console.log('wrote', p, '| entries:', Object.keys(out).length);
} else {
  const shipped = JSON.parse(fs.readFileSync(path.join(DIR, '_strongs-stats.json'), 'utf-8'));
  let gTotal = 0, gExactN = 0, gExactAll = 0;
  const bad = [];
  for (const k of Object.keys(shipped)) {
    if (k[0] !== 'G') continue;               // Greek only: that corpus is unchanged
    gTotal++;
    const a = shipped[k], b = out[k];
    if (!b) { bad.push([k, 'absent now']); continue; }
    if (a.n === b.n) gExactN++;
    if (JSON.stringify(a) === JSON.stringify(b)) gExactAll++;
    else if (bad.length < 6) bad.push([k, 'shipped=' + JSON.stringify(a), 'mine=' + JSON.stringify(b)]);
  }
  console.log('Greek entries checked:', gTotal);
  console.log('  same occurrence count n :', gExactN, '(' + (100 * gExactN / gTotal).toFixed(1) + '%)');
  console.log('  identical n AND glosses :', gExactAll, '(' + (100 * gExactAll / gTotal).toFixed(1) + '%)');

  /*
   * The real quality signal: does the PRIMARY gloss — the one the card leads with — still agree?
   * Counts must move (2,083 verses were added, and the shipped `n` matches neither word nor verse
   * occurrences), but the headline answer to "how is this word translated?" should not.
   */
  let topSame = 0, topTotal = 0, topDiff = [];
  for (const k of Object.keys(shipped)) {
    const a = shipped[k], b = out[k];
    if (!a || !b || !a.g || !a.g.length || !b.g || !b.g.length) continue;
    topTotal++;
    if (a.g[0][0] === b.g[0][0]) topSame++;
    else if (topDiff.length < 8) topDiff.push('  ' + k + ' shipped="' + a.g[0][0] + '" mine="' + b.g[0][0] + '"');
  }
  console.log('\nprimary gloss agreement (all', topTotal, 'entries):', topSame,
              '(' + (100 * topSame / topTotal).toFixed(1) + '%)');
  if (topDiff.length) console.log('sample primary-gloss changes:\n' + topDiff.join('\n'));
  console.log('\n  new total entries:', Object.keys(out).length, 'vs shipped', Object.keys(shipped).length);
  if (bad.length) console.log('\nexamples of n/gloss disagreement:\n' + bad.map((x) => '  ' + x.join('\n    ')).join('\n'));
}