300 lines
9.7 KiB
JavaScript
300 lines
9.7 KiB
JavaScript
 
|
// Fetch and locally materialize the Greek WordNet lexicon
|
||
|
|
// Output: ../dict/greek-wordnet.json (lemma-keyed)
|
||
|
|
// Docs: https://greekwordnet.chs.harvard.edu/api
|
||
|
|
|
||
|
|
import fs from 'fs';
|
||
|
|
import path from 'path';
|
||
|
|
import https from 'https';
|
||
|
|
import { fileURLToPath } from 'url';
|
||
|
|
import os from 'os';
|
||
|
|
|
||
|
|
const __filename = fileURLToPath(import.meta.url);
|
||
|
|
const __dirname = path.dirname(__filename);
|
||
|
|
|
||
|
|
const REMOTE_HOST = 'greekwordnet.chs.harvard.edu';
|
||
|
|
|
||
|
|
function sleep(ms) {
|
||
|
|
return new Promise(r => setTimeout(r, ms));
|
||
|
|
}
|
||
|
|
|
||
|
|
function getJson(pathname, { timeoutMs = 20000 } = {}) {
|
||
|
|
return new Promise((resolve, reject) => {
|
||
|
|
const options = {
|
||
|
|
hostname: REMOTE_HOST,
|
||
|
|
port: 443,
|
||
|
|
path: pathname,
|
||
|
|
method: 'GET',
|
||
|
|
headers: {
|
||
|
|
'Accept': 'application/json, text/plain, */*',
|
||
|
|
'User-Agent': 'BibleApp-WordNet-Ingester/1.0',
|
||
|
|
'Accept-Language': 'en-US,en;q=0.9',
|
||
|
|
'Connection': 'close',
|
||
|
|
}
|
||
|
|
};
|
||
|
|
const req = https.request(options, (res) => {
|
||
|
|
const status = res.statusCode || 0;
|
||
|
|
if (status >= 300 && status < 400 && res.headers.location) {
|
||
|
|
// follow simple redirects
|
||
|
|
try {
|
||
|
|
const next = new URL(res.headers.location, `https://${REMOTE_HOST}`);
|
||
|
|
resolve(getJson(next.pathname + (next.search || ''), { timeoutMs }));
|
||
|
|
res.resume();
|
||
|
|
return;
|
||
|
|
} catch (e) {
|
||
|
|
reject(e);
|
||
|
|
res.resume();
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
if (status < 200 || status > 299) {
|
||
|
|
reject(new Error(`HTTP ${status} for ${pathname}`));
|
||
|
|
res.resume();
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
const chunks = [];
|
||
|
|
res.on('data', (d) => chunks.push(d));
|
||
|
|
res.on('end', () => {
|
||
|
|
const buf = Buffer.concat(chunks);
|
||
|
|
const ct = (res.headers['content-type'] || '').toLowerCase();
|
||
|
|
if (ct.includes('application/json')) {
|
||
|
|
try { resolve(JSON.parse(buf.toString('utf-8'))); }
|
||
|
|
catch (e) { reject(e); }
|
||
|
|
} else {
|
||
|
|
// Occasionally text is returned; try to parse as JSON anyway
|
||
|
|
const s = buf.toString('utf-8');
|
||
|
|
try { resolve(JSON.parse(s)); }
|
||
|
|
catch (_) { resolve(s); }
|
||
|
|
}
|
||
|
|
});
|
||
|
|
});
|
||
|
|
req.setTimeout(timeoutMs, () => {
|
||
|
|
try { req.destroy(new Error('timeout')); } catch {}
|
||
|
|
});
|
||
|
|
req.on('error', reject);
|
||
|
|
req.end();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
async function getJsonWithRetries(pathname, tries = 3) {
|
||
|
|
let attempt = 0;
|
||
|
|
while (attempt < tries) {
|
||
|
|
try {
|
||
|
|
return await getJson(pathname);
|
||
|
|
} catch (e) {
|
||
|
|
attempt++;
|
||
|
|
if (attempt >= tries) throw e;
|
||
|
|
await sleep(300 + attempt * 500);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
async function fetchIndexByPos(pos) {
|
||
|
|
const path = `/api/index/${encodeURIComponent(pos)}/`;
|
||
|
|
const data = await getJsonWithRetries(path).catch(() => []);
|
||
|
|
return Array.isArray(data) ? data : [];
|
||
|
|
}
|
||
|
|
|
||
|
|
function pick(obj, keys) {
|
||
|
|
for (const k of keys) {
|
||
|
|
if (obj && Object.prototype.hasOwnProperty.call(obj, k) && obj[k]) return obj[k];
|
||
|
|
}
|
||
|
|
return '';
|
||
|
|
}
|
||
|
|
|
||
|
|
function uniqueKeyFromIndexItem(it) {
|
||
|
|
const uri = pick(it, ['uri', 'URI', 'id']);
|
||
|
|
if (uri) return `uri:${uri}`;
|
||
|
|
const lemma = (pick(it, ['lemma', 'headword', 'form', 'text']) || '').toLowerCase();
|
||
|
|
const pos = (pick(it, ['pos', 'part_of_speech', 'partOfSpeech']) || '').toLowerCase();
|
||
|
|
const morpho = (pick(it, ['morpho', 'morphology', 'morph']) || '').toLowerCase();
|
||
|
|
return `lp:${lemma}|${pos}|${morpho}`;
|
||
|
|
}
|
||
|
|
|
||
|
|
async function fetchLemmaSynsetsByGuess(headword, pos, morpho) {
|
||
|
|
const parts = [encodeURIComponent(headword)];
|
||
|
|
const attempts = [];
|
||
|
|
// Most specific first
|
||
|
|
if (pos && morpho) attempts.push(`/api/lemmas/${parts[0]}/${encodeURIComponent(pos)}/${encodeURIComponent(morpho)}/synsets`);
|
||
|
|
if (pos) attempts.push(`/api/lemmas/${parts[0]}/${encodeURIComponent(pos)}/synsets`);
|
||
|
|
attempts.push(`/api/lemmas/${parts[0]}/synsets`);
|
||
|
|
for (const p of attempts) {
|
||
|
|
try {
|
||
|
|
const resp = await getJsonWithRetries(p);
|
||
|
|
if (resp) return resp;
|
||
|
|
} catch (_) {}
|
||
|
|
}
|
||
|
|
return [];
|
||
|
|
}
|
||
|
|
|
||
|
|
async function deriveFromUri(uri) {
|
||
|
|
if (!uri) return { lemmas: [], synsets: [] };
|
||
|
|
// Get lemma record(s) via URI
|
||
|
|
const data = await getJsonWithRetries(`/api/lemmas?uri=${encodeURIComponent(uri)}`).catch(() => []);
|
||
|
|
const list = Array.isArray(data) ? data : (data ? [data] : []);
|
||
|
|
let synsets = [];
|
||
|
|
// Try to fetch synsets via guessed route for each returned lemma
|
||
|
|
for (const it of list) {
|
||
|
|
const hw = pick(it, ['lemma', 'headword', 'form', 'text']);
|
||
|
|
const pos = pick(it, ['pos', 'part_of_speech', 'partOfSpeech']);
|
||
|
|
const morpho = pick(it, ['morpho', 'morphology', 'morph']);
|
||
|
|
if (!hw) continue;
|
||
|
|
try {
|
||
|
|
const s = await fetchLemmaSynsetsByGuess(hw, pos, morpho);
|
||
|
|
if (Array.isArray(s) && s.length) synsets = synsets.concat(s);
|
||
|
|
} catch (_) {}
|
||
|
|
}
|
||
|
|
return { lemmas: list, synsets };
|
||
|
|
}
|
||
|
|
|
||
|
|
async function mapLimit(items, limit, iterator) {
|
||
|
|
const ret = [];
|
||
|
|
let idx = 0;
|
||
|
|
let active = 0;
|
||
|
|
return await new Promise((resolve, reject) => {
|
||
|
|
const next = () => {
|
||
|
|
if (idx >= items.length && active === 0) return resolve(ret);
|
||
|
|
while (active < limit && idx < items.length) {
|
||
|
|
const cur = items[idx++];
|
||
|
|
active++;
|
||
|
|
Promise.resolve()
|
||
|
|
.then(() => iterator(cur))
|
||
|
|
.then((val) => { ret.push(val); })
|
||
|
|
.catch((e) => { ret.push({ __error: String(e && e.message || e) }); })
|
||
|
|
.finally(() => { active--; next(); });
|
||
|
|
}
|
||
|
|
};
|
||
|
|
next();
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
function ensureDirSync(dir) {
|
||
|
|
try { fs.mkdirSync(dir, { recursive: true }); } catch {}
|
||
|
|
}
|
||
|
|
|
||
|
|
async function main() {
|
||
|
|
console.log('[WordNet] Fetching complete lemma index by part of speech...');
|
||
|
|
const [n, v, a, r] = await Promise.all([
|
||
|
|
fetchIndexByPos('n'), fetchIndexByPos('v'), fetchIndexByPos('a'), fetchIndexByPos('r')
|
||
|
|
]);
|
||
|
|
let index = [...n, ...v, ...a, ...r];
|
||
|
|
console.log(`[WordNet] Index sizes: n=${n.length} v=${v.length} a=${a.length} r=${r.length} total=${index.length}`);
|
||
|
|
|
||
|
|
// Fallback: try complete index if POS-filtered queries returned nothing (upstream variance)
|
||
|
|
if (index.length === 0) {
|
||
|
|
try {
|
||
|
|
const full = await getJsonWithRetries('/api/index/');
|
||
|
|
if (Array.isArray(full) && full.length) {
|
||
|
|
index = full;
|
||
|
|
console.log(`[WordNet] Fallback index: total=${index.length}`);
|
||
|
|
}
|
||
|
|
} catch (e) {
|
||
|
|
console.warn('[WordNet] Fallback /api/index/ failed:', e && e.message || e);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
// Deduplicate
|
||
|
|
const seen = new Set();
|
||
|
|
const unique = [];
|
||
|
|
for (const it of index) {
|
||
|
|
const key = uniqueKeyFromIndexItem(it);
|
||
|
|
if (!seen.has(key)) { seen.add(key); unique.push(it); }
|
||
|
|
}
|
||
|
|
console.log(`[WordNet] Unique entries: ${unique.length}`);
|
||
|
|
|
||
|
|
const out = {};
|
||
|
|
let processed = 0;
|
||
|
|
const startedAt = Date.now();
|
||
|
|
|
||
|
|
const consume = async (item) => {
|
||
|
|
const headword = pick(item, ['lemma', 'headword', 'form', 'text']);
|
||
|
|
const pos = pick(item, ['pos', 'part_of_speech', 'partOfSpeech']);
|
||
|
|
const morpho = pick(item, ['morpho', 'morphology', 'morph']);
|
||
|
|
const uri = pick(item, ['uri', 'URI', 'id']);
|
||
|
|
|
||
|
|
let synsets = [];
|
||
|
|
try {
|
||
|
|
if (headword) {
|
||
|
|
const s = await fetchLemmaSynsetsByGuess(headword, pos, morpho);
|
||
|
|
if (Array.isArray(s)) synsets = s;
|
||
|
|
}
|
||
|
|
if (!synsets || synsets.length === 0) {
|
||
|
|
const via = await deriveFromUri(uri);
|
||
|
|
if (via && Array.isArray(via.synsets) && via.synsets.length) synsets = via.synsets;
|
||
|
|
}
|
||
|
|
} catch (_) {}
|
||
|
|
|
||
|
|
const senses = Array.isArray(synsets) ? synsets : [];
|
||
|
|
const glosses = [];
|
||
|
|
for (const s of senses) {
|
||
|
|
const g = pick(s, ['gloss', 'definition', 'label', 'text']);
|
||
|
|
if (g) glosses.push(String(g));
|
||
|
|
}
|
||
|
|
const meaning = glosses.slice(0, 3).join('; ');
|
||
|
|
const lemmaText = (headword || '').toString().trim();
|
||
|
|
if (!lemmaText) return;
|
||
|
|
const key = lemmaText.toLowerCase();
|
||
|
|
if (!out[key]) {
|
||
|
|
out[key] = {
|
||
|
|
text: lemmaText,
|
||
|
|
transliteration: '',
|
||
|
|
meaning: meaning || 'WordNet: lemma',
|
||
|
|
pos: pos || '',
|
||
|
|
morphoTags: morpho ? [morpho] : [],
|
||
|
|
senses: senses.map(s => ({
|
||
|
|
pos: pick(s, ['pos', 'part_of_speech', 'partOfSpeech']) || pos || '',
|
||
|
|
offset: pick(s, ['offset', 'id']) || '',
|
||
|
|
gloss: pick(s, ['gloss', 'definition', 'label', 'text']) || ''
|
||
|
|
}))
|
||
|
|
};
|
||
|
|
} else {
|
||
|
|
// Merge morpho tags and append senses
|
||
|
|
const cur = out[key];
|
||
|
|
if (morpho && Array.isArray(cur.morphoTags) && !cur.morphoTags.includes(morpho)) cur.morphoTags.push(morpho);
|
||
|
|
if (meaning && !cur.meaning) cur.meaning = meaning;
|
||
|
|
if (Array.isArray(senses) && senses.length) {
|
||
|
|
const newSenses = senses.map(s => ({
|
||
|
|
pos: pick(s, ['pos', 'part_of_speech', 'partOfSpeech']) || pos || '',
|
||
|
|
offset: pick(s, ['offset', 'id']) || '',
|
||
|
|
gloss: pick(s, ['gloss', 'definition', 'label', 'text']) || ''
|
||
|
|
}));
|
||
|
|
cur.senses = Array.isArray(cur.senses) ? cur.senses.concat(newSenses) : newSenses;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
};
|
||
|
|
|
||
|
|
const concurrency = Math.max(4, Math.min(12, Math.floor((os.cpus()?.length) || 4)));
|
||
|
|
console.log(`[WordNet] Ingesting ${unique.length} lemmas with concurrency=${concurrency}...`);
|
||
|
|
|
||
|
|
await mapLimit(unique, concurrency, async (item) => {
|
||
|
|
try { await consume(item); } catch (_) {}
|
||
|
|
processed++;
|
||
|
|
if (processed % 500 === 0) {
|
||
|
|
const elapsed = Math.max(1, Date.now() - startedAt);
|
||
|
|
const rate = Math.round(processed * 1000 / elapsed);
|
||
|
|
console.log(`[WordNet] Processed ${processed}/${unique.length} (~${rate}/s)`);
|
||
|
|
}
|
||
|
|
});
|
||
|
|
|
||
|
|
const meta = {
|
||
|
|
source: 'Greek WordNet API',
|
||
|
|
base: `https://${REMOTE_HOST}/api`,
|
||
|
|
fetchedAt: new Date().toISOString(),
|
||
|
|
lemmaCount: Object.keys(out).length
|
||
|
|
};
|
||
|
|
out.__meta = meta;
|
||
|
|
|
||
|
|
const targetDir = path.join(__dirname, '..', 'dict');
|
||
|
|
const targetFile = path.join(targetDir, 'greek-wordnet.json');
|
||
|
|
ensureDirSync(targetDir);
|
||
|
|
|
||
|
|
fs.writeFileSync(targetFile, JSON.stringify(out), 'utf-8');
|
||
|
|
console.log(`[WordNet] Wrote ${Object.keys(out).length} lemmas to ${path.relative(process.cwd(), targetFile)}`);
|
||
|
|
}
|
||
|
|
|
||
|
|
main().catch((e) => {
|
||
|
|
console.error('[WordNet] Ingestion failed:', e && e.message || e);
|
||
|
|
process.exitCode = 1;
|
||
|
|
});
|
||
|
|
|
||
|
|
|