bible-app/tools/interlinear/tahot-convert.cjs

/*
 * tahot-convert.cjs — parse STEPBible TAHOT into the Bible app's interlinear/<Book>.json shape.
 *
 * The shipped data was produced by a converter that is NOT in the repo, and whose reference regex
 * silently dropped every verse whose STEP ref carries a Hebrew-versification parenthetical
 * ("Gen.32.1(32.2)"). That lost 96 whole OT chapters — 62 of 150 Psalms — plus scattered single
 * verses like Gen.31.55(32.1).
 *
 * So this converter is validated by REPRODUCTION: it regenerates verses that already ship and
 * diffs them against the shipped JSON. Only once it matches byte-for-byte on the existing corpus
 * is it trusted to emit the missing verses.
 */
'use strict';

const fs = require('fs');
const path = require('path');

// TAHOT book abbreviation -> the app's interlinear filename (no extension).
const BOOKS = {
  Gen: 'Genesis', Exo: 'Exodus', Lev: 'Leviticus', Num: 'Numbers', Deu: 'Deuteronomy',
  Jos: 'Joshua', Jdg: 'Judges', Rut: 'Ruth', '1Sa': '1 Samuel', '2Sa': '2 Samuel',
  '1Ki': '1 Kings', '2Ki': '2 Kings', '1Ch': '1 Chronicles', '2Ch': '2 Chronicles',
  Ezr: 'Ezra', Neh: 'Nehemiah', Est: 'Esther', Job: 'Job', Psa: 'Psalms', Pro: 'Proverbs',
  Ecc: 'Ecclesiastes', Sng: 'Song of Solomon', Isa: 'Isaiah', Jer: 'Jeremiah',
  Lam: 'Lamentations', Ezk: 'Ezekiel', Dan: 'Daniel', Hos: 'Hosea', Jol: 'Joel',
  Amo: 'Amos', Oba: 'Obadiah', Jon: 'Jonah', Mic: 'Micah', Nam: 'Nahum', Hab: 'Habakkuk',
  Zep: 'Zephaniah', Hag: 'Haggai', Zec: 'Zechariah', Mal: 'Malachi'
};

/*
 * Column 1: "<Book>.<ch>.<v>[(<hebCh>.<hebV>)]#<word>[.<sub>]=<textType>"
 * The optional parenthetical is the Hebrew reference where it differs from the English one, and
 * allowing it is the entire bug fix. The reference used for keying is always the ENGLISH one
 * (field spec: "Bible reference in English Bibles, as defined by the NRSV"), which is what the
 * app's chapter|verse keys are.
 */
const REF_RE = /^([1-3]?[A-Za-z]{2,3})\.(\d+)\.(\d+)(?:\(([^)]*)\))?#(\d+)(?:\.(\d+))?=(.*)$/;

const stripSlashes = (s) => String(s || '').split('/').join('').split('\\').join('');

/*
 * The Hebrew word.
 *
 * TAHOT separates prefixes/suffixes with "/" and punctuation with "\". The shipped data drops a
 * TRAILING sof pasuq (verse-end "׃") and nothing else — the maqqef "־" and paseq "׀" survive, and
 * so does a sof pasuq that is followed by a paragraph marker ("אֶחָֽד\׃\ \פ" ships as "אֶחָֽד׃ פ",
 * 90 such words in Genesis alone). Matching that exactly is what makes the regenerated verses
 * byte-identical to the ones already shipping, which is the only evidence that this converter
 * agrees with the one that produced them.
 */
const SOF_PASUQ = '׃';
function hebrewOf(field2) {
  let s = String(field2 || '');
  if (s.endsWith('\\' + SOF_PASUQ)) s = s.slice(0, -2);
  return stripSlashes(s).trim();
}

// English gloss: "/" mirrors the Hebrew morpheme breaks ("to/ way/ his" -> "to way his").
const glossOf = (s) => String(s || '').split('/').join(' ').replace(/\s+/g, ' ').trim();

// Transliteration: "be./re.Shit" -> "be.re.Shit"
const translitOf = (s) => stripSlashes(s).trim();

/*
 * The root word's Strong's number sits in {curly brackets} among the prefix/suffix tags.
 * The trailing disambiguation letter (H7225G, H1254A) is dropped: the shipped data stores the
 * plain 4-digit form, and the app's dictionaries are keyed that way.
 */
function strongOf(dStrongs) {
  const m = String(dStrongs || '').match(/\{\s*([HA])0*(\d{1,4})[A-Za-z]*\s*\}/);
  if (m) return m[1] + m[2].padStart(4, '0');
  const bare = String(dStrongs || '').match(/([HA])0*(\d{1,4})[A-Za-z]*/);
  return bare ? bare[1] + bare[2].padStart(4, '0') : '';
}

/*
 * Expanded tags: "H9003=בְּ=in/{H7225G=רֵאשִׁית=: beginning»first:1_beginning}"
 * The {..} group carries the root's lemma and its gloss. In the shipped data the gloss has its
 * leading ": " removed, "»" spaced out, and "_" turned into a space.
 */
function lemmaOf(expanded) {
  const m = String(expanded || '').match(/\{([^{}]*)\}/);
  if (!m) return { l: '', lg: '' };
  const parts = m[1].split('=');
  if (parts.length < 2) return { l: '', lg: '' };
  const l = (parts[1] || '').trim();
  let lg = parts.slice(2).join('=');
  lg = lg.replace(/^\s*:\s*/, '')
         .split('»').join(' » ')
         .split('_').join(' ')
         .replace(/\s+/g, ' ')
         .trim();
  return { l, lg };
}

function parseFile(file) {
  const out = new Map();     // "Book|ch|v" -> [word, ...]
  const text = fs.readFileSync(file, 'utf-8');
  const lines = text.split(/\r?\n/);
  let dataLines = 0, skipped = 0;

  for (const raw of lines) {
    if (!raw || raw[0] === '#' || raw[0] === '\t') continue;   // notes + interlinear display rows
    const cols = raw.split('\t');
    const m = REF_RE.exec(cols[0] || '');
    if (!m) { if ((cols[0] || '').includes('#')) skipped++; continue; }

    const book = BOOKS[m[1]];
    if (!book) { skipped++; continue; }
    const chapter = Number(m[2]);
    const verse = Number(m[3]);
    /*
     * Word numbers are TWO digits, optionally followed by two more for a sub-position:
     * "#05" is word 5; "#0501"/"#0502" are the two LXX-restored words that sit BETWEEN words 5 and
     * 6 (Gen.4.8, "let us go [into] the field"). Reading "0501" as the number 501 sorted those two
     * words to the end of the verse — the only ordering error left after the punctuation fix.
     */
    const digits = m[5];
    const wordNo = parseInt(digits.length > 2 ? digits.slice(0, 2) : digits, 10);
    const sub = m[6] ? Number(m[6]) : (digits.length > 2 ? parseInt(digits.slice(2), 10) : 0);

    const word = {
      w: hebrewOf(cols[1]),
      tr: translitOf(cols[2]),
      g: glossOf(cols[3]),
      s: strongOf(cols[4]),
      m: String(cols[5] || '').trim()
    };
    const { l, lg } = lemmaOf(cols[11]);
    if (l) word.l = l;
    if (lg) word.lg = lg;

    if (!word.w) continue;

    const key = book + '|' + chapter + '|' + verse;
    if (!out.has(key)) out.set(key, []);
    out.get(key).push({ order: wordNo * 100 + sub, word });
    dataLines++;
  }

  for (const [k, arr] of out) {
    arr.sort((a, b) => a.order - b.order);
    out.set(k, arr.map((x) => x.word));
  }
  return { verses: out, dataLines, skipped };
}

module.exports = { parseFile, BOOKS, strongOf, lemmaOf, glossOf, translitOf, stripSlashes };

// ---- CLI: validate against the shipped data ----
if (require.main === module) {
  const [, , tahotFile, repoDir, onlyBook] = process.argv;
  const { verses, dataLines, skipped } = parseFile(tahotFile);
  console.log('parsed word rows:', dataLines, '| unparsed ref-ish rows:', skipped);
  console.log('verses found:', verses.size);

  const byBook = {};
  for (const k of verses.keys()) {
    const b = k.split('|')[0];
    byBook[b] = (byBook[b] || 0) + 1;
  }

  let exact = 0, differ = 0, absent = 0;
  const examples = [];
  for (const book of Object.keys(byBook)) {
    if (onlyBook && book !== onlyBook) continue;
    const p = path.join(repoDir, 'interlinear', book + '.json');
    if (!fs.existsSync(p)) continue;
    const shipped = JSON.parse(fs.readFileSync(p, 'utf-8'));
    for (const [k, mine] of verses) {
      const [b, ch, v] = k.split('|');
      if (b !== book) continue;
      const have = shipped[ch + '|' + v];
      if (!have || !have.length) { absent++; continue; }
      if (JSON.stringify(have) === JSON.stringify(mine)) exact++;
      else {
        differ++;
        if (examples.length < 4) examples.push({ ref: k, shipped: have[0], mine: mine[0], nShipped: have.length, nMine: mine.length });
      }
    }
  }
  console.log('\nvs shipped data  exact:', exact, '| differ:', differ, '| missing from shipped:', absent);
  if (examples.length) console.log('\nfirst differences:\n' + JSON.stringify(examples, null, 1));
}