bible-app/tools/convert-message-txt-to-json.cjs

// convert-message-txt-to-json.js
// Purpose: Convert a plain text export of The Message (MSG) into a local JSON file
// compatible with this app's loaders (Variant B: { Book: { Chapter: { Verse: "text" }}} like ESV/NIV/NLT JSON in this repo).
//
// Notes:
// - The input text groups content in verse ranges (e.g., "1–4  34 text...") rather
//   than strict per-verse lines. This converter maps each range to all covered verse
//   numbers by duplicating the range's text to each verse. While this isn't as precise
//   as official per-verse segmentation, it produces a structurally valid local JSON
//   that the app can render offline.
// - The converter validates chapter verse counts against the project's bible-structure.js
//   and will pad missing verses with empty strings or truncate excess if necessary.
// - If you later want exact per-verse MSG text, prefer building from an official API
//   and writing the same JSON shape.
//
// Usage (Windows CMD or PowerShell):
//   node "C:\\Users\\media\\Downloads\\bible-app\\tools\\convert-message-txt-to-json.js" \
//        "C:\\Users\\media\\Downloads\\bible-app\\message translation.txt" \
//        "C:\\Users\\media\\Downloads\\bible-app\\MSG_bible.json"

const fs = require('fs');
const path = require('path');

function readFileUtf8(filePath) {
  return fs.readFileSync(filePath, { encoding: 'utf8' });
}

function writeFileUtf8(filePath, data) {
  fs.mkdirSync(path.dirname(filePath), { recursive: true });
  fs.writeFileSync(filePath, data, { encoding: 'utf8' });
}

function loadBibleStructure(structureFilePath) {
  const raw = readFileUtf8(structureFilePath);
  // Extract the JS object literal assigned to `export const bibleStructure = { ... }`.
  const startToken = 'export const bibleStructure =';
  const startIdx = raw.indexOf(startToken);
  if (startIdx === -1) {
    throw new Error('Could not find bibleStructure export in bible-structure.js');
  }
  const afterStart = raw.slice(startIdx + startToken.length);
  // Find the terminating semicolon after the object literal
  // by tracking braces.
  let brace = 0;
  let objStart = -1;
  let objEnd = -1;
  for (let i = 0; i < afterStart.length; i++) {
    const ch = afterStart[i];
    if (objStart === -1) {
      if (ch === '{') { objStart = i; brace = 1; }
      continue;
    } else {
      if (ch === '{') brace++;
      if (ch === '}') {
        brace--;
        if (brace === 0) { objEnd = i; break; }
      }
    }
  }
  if (objStart === -1 || objEnd === -1) {
    throw new Error('Failed to extract bibleStructure object literal');
  }
  const objectLiteral = afterStart.slice(objStart, objEnd + 1);
  // Evaluate in a sandboxed Function to obtain the object.
  // The literal is plain JS with numeric arrays; no functions.
  // eslint-disable-next-line no-new-func
  const bibleStructure = new Function(`return (${objectLiteral});`)();
  if (!bibleStructure || typeof bibleStructure !== 'object') {
    throw new Error('bibleStructure parsed as invalid object');
  }
  return bibleStructure;
}

function normalizeSpace(s) {
  return String(s || '')
    .replace(/\u00A0/g, ' ') // non-breaking spaces -> space
    .replace(/[\t ]+/g, ' ') // collapse whitespace
    .trim();
}

function isBookHeader(line, knownBooksSet) {
  const t = normalizeSpace(line);
  if (!t) return null;
  // Lines like "Exodus" / "Leviticus" / "Introduction".
  if (t.toLowerCase() === 'introduction') return { type: 'intro' };
  // Case-insensitive match to tolerate capitalization variants in source
  for (const b of knownBooksSet) {
    if (String(b).toLowerCase() === t.toLowerCase()) return { type: 'book', name: b };
  }
  return null;
}

function isNoiseLine(line) {
  const t = normalizeSpace(line).toLowerCase();
  if (!t) return true;
  // Skip site chrome and non-content lines common in scraped exports
  const noiseSnippets = [
    'link panels', 'go to table of contents', 'single column', 'stretch across both columns',
    'show reading view', 'learn more', 'home', 'topic or passage', 'reading plans', 'about biblia',
    'a service of logos', 'restore columns', 'share', 'the message: the bible in contemporary language',
    'aa', 'christian bridges', 'the future of bible study is here.'
  ];
  return noiseSnippets.some(sn => t.includes(sn));
}

// Match lines that begin a new (verse-range, chapter) chunk, e.g.:
//   "1–4  34 One day Dinah, ..."
//   "5–7  Jacob heard ..." (sometimes chapter implied by prior? Not in this export)
//   "1  36 This is the family tree of Esau, ..."
// We capture: verseStart, verseEnd (optional), chapterNumber, and the first text segment.
const RANGE_DASH = '[\u2013\u2014\-]'; // en dash/em dash/hyphen
const CHUNK_RE = new RegExp(`^(\\d+)(?:\\s*${RANGE_DASH}\\s*(\\d+))?\\s+(\\d+)\\s+(.*)$`);

function appendTextToVerses(versesArray, indices, extraText) {
  const add = normalizeSpace(extraText);
  if (!add) return;
  for (const idx of indices) {
    if (!versesArray[idx]) {
      versesArray[idx] = { verse: idx + 1, text: add };
    } else {
      const prev = String(versesArray[idx].text || '').trim();
      versesArray[idx].text = prev ? (prev + ' ' + add) : add;
    }
  }
}

function convertTextToJson(inputText, bibleStructure) {
  const knownBooks = Object.keys(bibleStructure);
  const knownBooksSet = new Set(knownBooks);
  // Heuristic: If the input starts mid-book without a header, assume Genesis until a header tells us otherwise.
  let currentBook = 'Genesis';
  let currentChapter = null; // number
  let currentVerses = []; // array of { verse, text }
  let lastTargetIndices = []; // verse indices (0-based) most recently written to
  const booksOut = new Map(); // bookName -> Map(chapterNumber -> versesArray)

  function ensureBook(book) {
    if (!booksOut.has(book)) booksOut.set(book, new Map());
  }
  function flushChapter() {
    if (currentBook && currentChapter != null) {
      ensureBook(currentBook);
      const chapters = booksOut.get(currentBook);
      // Validate/pad according to structure counts where available
      const chapterIdx = Number(currentChapter) - 1;
      const expectedCount = Array.isArray(bibleStructure[currentBook]) ? (bibleStructure[currentBook][chapterIdx] || 0) : 0;
      if (expectedCount > 0) {
        if (currentVerses.length > expectedCount) {
          currentVerses = currentVerses.slice(0, expectedCount);
        } else if (currentVerses.length < expectedCount) {
          for (let v = currentVerses.length; v < expectedCount; v++) {
            currentVerses[v] = { verse: v + 1, text: '' };
          }
        }
      }
      chapters.set(Number(currentChapter), currentVerses);
    }
  }

  const lines = inputText.split(/\r?\n/);
  for (let rawLine of lines) {
    const line = normalizeSpace(rawLine);
    if (!line) continue;
    const header = isBookHeader(line, knownBooksSet);
    if (header) {
      if (header.type === 'book') {
        // Switching books: flush any prior chapter
        flushChapter();
        currentBook = header.name;
        currentChapter = null;
        currentVerses = [];
        lastTargetIndices = [];
      }
      // Skip intros explicitly
      continue;
    }

    if (isNoiseLine(line)) continue;

    const m = CHUNK_RE.exec(line);
    if (m) {
      const verseStart = Number(m[1]);
      const verseEnd = m[2] ? Number(m[2]) : verseStart;
      const chapterNum = Number(m[3]);
      const firstText = normalizeSpace(m[4] || '');

      if (Number.isNaN(verseStart) || Number.isNaN(verseEnd) || Number.isNaN(chapterNum)) {
        continue; // malformed
      }

      // New chapter encountered
      if (currentChapter === null || Number(chapterNum) !== Number(currentChapter)) {
        // flush previous chapter
        flushChapter();
        currentChapter = chapterNum;
        currentVerses = [];
        lastTargetIndices = [];
      }

      const startIdx = Math.max(1, verseStart) - 1;
      const endIdx = Math.max(startIdx, verseEnd - 1);
      const targetIndices = [];
      for (let idx = startIdx; idx <= endIdx; idx++) targetIndices.push(idx);

      appendTextToVerses(currentVerses, targetIndices, firstText);
      lastTargetIndices = targetIndices;
      continue;
    }

    // Continuation line: append to last target verses
    if (lastTargetIndices.length) {
      appendTextToVerses(currentVerses, lastTargetIndices, line);
    }
  }

  // Flush trailing chapter
  flushChapter();

  // Build final JSON structure matching ESV/NIV/NLT shape: { Book: { Chapter: { Verse: text } } }
  const out = {};
  for (const book of knownBooks) {
    const chaptersMap = booksOut.get(book);
    if (!chaptersMap || chaptersMap.size === 0) continue; // only include books present in the input
    const chapterObj = {};
    const chapterNumbers = Array.from(chaptersMap.keys()).sort((a, b) => a - b);
    for (const chNum of chapterNumbers) {
      const verses = chaptersMap.get(chNum) || [];
      const verseObj = {};
      for (const v of verses) {
        if (!v) continue;
        verseObj[String(v.verse)] = String(v.text == null ? '' : v.text);
      }
      chapterObj[String(chNum)] = verseObj;
    }
    out[book] = chapterObj;
  }

  return out;
}

function main() {
  const [, , inputPathArg, outputPathArg] = process.argv;
  if (!inputPathArg || !outputPathArg) {
    console.error('Usage:\n  node \'tools/convert-message-txt-to-json.js\' "<INPUT_TXT_ABSOLUTE_PATH>" "<OUTPUT_JSON_ABSOLUTE_PATH>"');
    process.exit(1);
  }

  const inputPath = path.resolve(inputPathArg);
  const outputPath = path.resolve(outputPathArg);
  const structurePath = path.resolve(__dirname, '..', 'bible-structure.js');

  console.log('[MSG Convert] Reading input:', inputPath);
  const inputText = readFileUtf8(inputPath);
  console.log('[MSG Convert] Loading verse structure:', structurePath);
  const bibleStructure = loadBibleStructure(structurePath);
  console.log('[MSG Convert] Converting...');
  const json = convertTextToJson(inputText, bibleStructure);
  console.log('[MSG Convert] Writing output:', outputPath);
  writeFileUtf8(outputPath, JSON.stringify(json, null, 2));
  console.log('[MSG Convert] Done.');
}

if (require.main === module) {
  try { main(); } catch (err) { console.error(err); process.exit(1); }
}