bible-app/tools/fetch-greek-wordnet.js

// Fetch and locally materialize the Greek WordNet lexicon
// Output: ../dict/greek-wordnet.json (lemma-keyed)
// Docs: https://greekwordnet.chs.harvard.edu/api

import fs from 'fs';
import path from 'path';
import https from 'https';
import { fileURLToPath } from 'url';
import os from 'os';

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const REMOTE_HOST = 'greekwordnet.chs.harvard.edu';

function sleep(ms) {
	return new Promise(r => setTimeout(r, ms));
}

function getJson(pathname, { timeoutMs = 20000 } = {}) {
	return new Promise((resolve, reject) => {
		const options = {
			hostname: REMOTE_HOST,
			port: 443,
			path: pathname,
			method: 'GET',
			headers: {
				'Accept': 'application/json, text/plain, */*',
				'User-Agent': 'BibleApp-WordNet-Ingester/1.0',
				'Accept-Language': 'en-US,en;q=0.9',
				'Connection': 'close',
			}
		};
		const req = https.request(options, (res) => {
			const status = res.statusCode || 0;
			if (status >= 300 && status < 400 && res.headers.location) {
				// follow simple redirects
				try {
					const next = new URL(res.headers.location, `https://${REMOTE_HOST}`);
					resolve(getJson(next.pathname + (next.search || ''), { timeoutMs }));
					res.resume();
					return;
				} catch (e) {
					reject(e);
					res.resume();
					return;
				}
			}
			if (status < 200 || status > 299) {
				reject(new Error(`HTTP ${status} for ${pathname}`));
				res.resume();
				return;
			}
			const chunks = [];
			res.on('data', (d) => chunks.push(d));
			res.on('end', () => {
				const buf = Buffer.concat(chunks);
				const ct = (res.headers['content-type'] || '').toLowerCase();
				if (ct.includes('application/json')) {
					try { resolve(JSON.parse(buf.toString('utf-8'))); }
					catch (e) { reject(e); }
				} else {
					// Occasionally text is returned; try to parse as JSON anyway
					const s = buf.toString('utf-8');
					try { resolve(JSON.parse(s)); }
					catch (_) { resolve(s); }
				}
			});
		});
		req.setTimeout(timeoutMs, () => {
			try { req.destroy(new Error('timeout')); } catch {}
		});
		req.on('error', reject);
		req.end();
	});
}

async function getJsonWithRetries(pathname, tries = 3) {
	let attempt = 0;
	while (attempt < tries) {
		try {
			return await getJson(pathname);
		} catch (e) {
			attempt++;
			if (attempt >= tries) throw e;
			await sleep(300 + attempt * 500);
		}
	}
}

async function fetchIndexByPos(pos) {
	const path = `/api/index/${encodeURIComponent(pos)}/`;
	const data = await getJsonWithRetries(path).catch(() => []);
	return Array.isArray(data) ? data : [];
}

function pick(obj, keys) {
	for (const k of keys) {
		if (obj && Object.prototype.hasOwnProperty.call(obj, k) && obj[k]) return obj[k];
	}
	return '';
}

function uniqueKeyFromIndexItem(it) {
	const uri = pick(it, ['uri', 'URI', 'id']);
	if (uri) return `uri:${uri}`;
	const lemma = (pick(it, ['lemma', 'headword', 'form', 'text']) || '').toLowerCase();
	const pos = (pick(it, ['pos', 'part_of_speech', 'partOfSpeech']) || '').toLowerCase();
	const morpho = (pick(it, ['morpho', 'morphology', 'morph']) || '').toLowerCase();
	return `lp:${lemma}|${pos}|${morpho}`;
}

async function fetchLemmaSynsetsByGuess(headword, pos, morpho) {
	const parts = [encodeURIComponent(headword)];
	const attempts = [];
	// Most specific first
	if (pos && morpho) attempts.push(`/api/lemmas/${parts[0]}/${encodeURIComponent(pos)}/${encodeURIComponent(morpho)}/synsets`);
	if (pos) attempts.push(`/api/lemmas/${parts[0]}/${encodeURIComponent(pos)}/synsets`);
	attempts.push(`/api/lemmas/${parts[0]}/synsets`);
	for (const p of attempts) {
		try {
			const resp = await getJsonWithRetries(p);
			if (resp) return resp;
		} catch (_) {}
	}
	return [];
}

async function deriveFromUri(uri) {
	if (!uri) return { lemmas: [], synsets: [] };
	// Get lemma record(s) via URI
	const data = await getJsonWithRetries(`/api/lemmas?uri=${encodeURIComponent(uri)}`).catch(() => []);
	const list = Array.isArray(data) ? data : (data ? [data] : []);
	let synsets = [];
	// Try to fetch synsets via guessed route for each returned lemma
	for (const it of list) {
		const hw = pick(it, ['lemma', 'headword', 'form', 'text']);
		const pos = pick(it, ['pos', 'part_of_speech', 'partOfSpeech']);
		const morpho = pick(it, ['morpho', 'morphology', 'morph']);
		if (!hw) continue;
		try {
			const s = await fetchLemmaSynsetsByGuess(hw, pos, morpho);
			if (Array.isArray(s) && s.length) synsets = synsets.concat(s);
		} catch (_) {}
	}
	return { lemmas: list, synsets };
}

async function mapLimit(items, limit, iterator) {
	const ret = [];
	let idx = 0;
	let active = 0;
	return await new Promise((resolve, reject) => {
		const next = () => {
			if (idx >= items.length && active === 0) return resolve(ret);
			while (active < limit && idx < items.length) {
				const cur = items[idx++];
				active++;
				Promise.resolve()
					.then(() => iterator(cur))
					.then((val) => { ret.push(val); })
					.catch((e) => { ret.push({ __error: String(e && e.message || e) }); })
					.finally(() => { active--; next(); });
			}
		};
		next();
	});
}

function ensureDirSync(dir) {
	try { fs.mkdirSync(dir, { recursive: true }); } catch {}
}

async function main() {
	console.log('[WordNet] Fetching complete lemma index by part of speech...');
	const [n, v, a, r] = await Promise.all([
		fetchIndexByPos('n'), fetchIndexByPos('v'), fetchIndexByPos('a'), fetchIndexByPos('r')
	]);
	let index = [...n, ...v, ...a, ...r];
	console.log(`[WordNet] Index sizes: n=${n.length} v=${v.length} a=${a.length} r=${r.length} total=${index.length}`);

	// Fallback: try complete index if POS-filtered queries returned nothing (upstream variance)
	if (index.length === 0) {
		try {
			const full = await getJsonWithRetries('/api/index/');
			if (Array.isArray(full) && full.length) {
				index = full;
				console.log(`[WordNet] Fallback index: total=${index.length}`);
			}
		} catch (e) {
			console.warn('[WordNet] Fallback /api/index/ failed:', e && e.message || e);
		}
	}

	// Deduplicate
	const seen = new Set();
	const unique = [];
	for (const it of index) {
		const key = uniqueKeyFromIndexItem(it);
		if (!seen.has(key)) { seen.add(key); unique.push(it); }
	}
	console.log(`[WordNet] Unique entries: ${unique.length}`);

	const out = {};
	let processed = 0;
	const startedAt = Date.now();

	const consume = async (item) => {
		const headword = pick(item, ['lemma', 'headword', 'form', 'text']);
		const pos = pick(item, ['pos', 'part_of_speech', 'partOfSpeech']);
		const morpho = pick(item, ['morpho', 'morphology', 'morph']);
		const uri = pick(item, ['uri', 'URI', 'id']);

		let synsets = [];
		try {
			if (headword) {
				const s = await fetchLemmaSynsetsByGuess(headword, pos, morpho);
				if (Array.isArray(s)) synsets = s;
			}
			if (!synsets || synsets.length === 0) {
				const via = await deriveFromUri(uri);
				if (via && Array.isArray(via.synsets) && via.synsets.length) synsets = via.synsets;
			}
		} catch (_) {}

		const senses = Array.isArray(synsets) ? synsets : [];
		const glosses = [];
		for (const s of senses) {
			const g = pick(s, ['gloss', 'definition', 'label', 'text']);
			if (g) glosses.push(String(g));
		}
		const meaning = glosses.slice(0, 3).join('; ');
		const lemmaText = (headword || '').toString().trim();
		if (!lemmaText) return;
		const key = lemmaText.toLowerCase();
		if (!out[key]) {
			out[key] = {
				text: lemmaText,
				transliteration: '',
				meaning: meaning || 'WordNet: lemma',
				pos: pos || '',
				morphoTags: morpho ? [morpho] : [],
				senses: senses.map(s => ({
					pos: pick(s, ['pos', 'part_of_speech', 'partOfSpeech']) || pos || '',
					offset: pick(s, ['offset', 'id']) || '',
					gloss: pick(s, ['gloss', 'definition', 'label', 'text']) || ''
				}))
			};
		} else {
			// Merge morpho tags and append senses
			const cur = out[key];
			if (morpho && Array.isArray(cur.morphoTags) && !cur.morphoTags.includes(morpho)) cur.morphoTags.push(morpho);
			if (meaning && !cur.meaning) cur.meaning = meaning;
			if (Array.isArray(senses) && senses.length) {
				const newSenses = senses.map(s => ({
					pos: pick(s, ['pos', 'part_of_speech', 'partOfSpeech']) || pos || '',
					offset: pick(s, ['offset', 'id']) || '',
					gloss: pick(s, ['gloss', 'definition', 'label', 'text']) || ''
				}));
				cur.senses = Array.isArray(cur.senses) ? cur.senses.concat(newSenses) : newSenses;
			}
		}
	};

	const concurrency = Math.max(4, Math.min(12, Math.floor((os.cpus()?.length) || 4)));
	console.log(`[WordNet] Ingesting ${unique.length} lemmas with concurrency=${concurrency}...`);

	await mapLimit(unique, concurrency, async (item) => {
		try { await consume(item); } catch (_) {}
		processed++;
		if (processed % 500 === 0) {
			const elapsed = Math.max(1, Date.now() - startedAt);
			const rate = Math.round(processed * 1000 / elapsed);
			console.log(`[WordNet] Processed ${processed}/${unique.length} (~${rate}/s)`);
		}
	});

	const meta = {
		source: 'Greek WordNet API',
		base: `https://${REMOTE_HOST}/api`,
		fetchedAt: new Date().toISOString(),
		lemmaCount: Object.keys(out).length
	};
	out.__meta = meta;

	const targetDir = path.join(__dirname, '..', 'dict');
	const targetFile = path.join(targetDir, 'greek-wordnet.json');
	ensureDirSync(targetDir);

	fs.writeFileSync(targetFile, JSON.stringify(out), 'utf-8');
	console.log(`[WordNet] Wrote ${Object.keys(out).length} lemmas to ${path.relative(process.cwd(), targetFile)}`);
}

main().catch((e) => {
	console.error('[WordNet] Ingestion failed:', e && e.message || e);
	process.exitCode = 1;
});