89 lines
3.2 KiB
JavaScript
89 lines
3.2 KiB
JavaScript
 
|
/*
|
||
|
|
* model-worker.js — one model, one worker, one WASM heap.
|
||
|
|
*
|
||
|
|
* This file exists because of a hard limit: ONNX Runtime stages model weights
|
||
|
|
* through a 32-bit WASM heap capped at 4 GB, even when inference itself runs on
|
||
|
|
* the GPU. Whisper's fp16 encoder is 1.2 GB and NLLB is another 0.9 GB, and
|
||
|
|
* loading both into one heap fails — first as an opaque `Aborted()`, then, with
|
||
|
|
* smaller weights, as a load that simply never returns.
|
||
|
|
*
|
||
|
|
* A Web Worker gets its own ONNX Runtime instance and therefore its own heap.
|
||
|
|
* Two workers means two ceilings instead of one, so both models can run at the
|
||
|
|
* quality we actually want. Keeping inference off the main thread also stops
|
||
|
|
* the level meter from stuttering every time a sentence is decoded.
|
||
|
|
*/
|
||
|
|
|
||
|
|
import { pipeline, env } from './vendor/transformers.min.js';
|
||
|
|
|
||
|
|
env.allowLocalModels = false;
|
||
|
|
env.backends.onnx.wasm.wasmPaths = new URL('./vendor/', import.meta.url).href;
|
||
|
|
env.backends.onnx.wasm.numThreads = 1;
|
||
|
|
|
||
|
|
let pipe = null;
|
||
|
|
let label = 'model';
|
||
|
|
|
||
|
|
/* Report the adapter's features once. Whether this worker's WebGPU exposes
|
||
|
|
shader-f16 decides which weights we can use, and a bare "does not support
|
||
|
|
fp16" from deep inside ONNX is not much to debug with. */
|
||
|
|
/*
|
||
|
|
* WebGPU when there is a real adapter, WASM when there is not.
|
||
|
|
*
|
||
|
|
* device:'webgpu' was hard-coded, which is safe on Apple Silicon where Metal
|
||
|
|
* is always there. On Windows Chromium disables WebGPU on blocklisted drivers,
|
||
|
|
* software adapters, old Intel iGPUs and over Remote Desktop — and with no
|
||
|
|
* fallback that is not "slower speech recognition", it is NO speech
|
||
|
|
* recognition, which is the whole app. WASM is several times slower but it
|
||
|
|
* runs; the choice is reported so the operator log says which one is in use.
|
||
|
|
*/
|
||
|
|
async function bestDevice() {
|
||
|
|
try {
|
||
|
|
if (!navigator.gpu) return 'wasm';
|
||
|
|
const adapter = await navigator.gpu.requestAdapter();
|
||
|
|
return adapter ? 'webgpu' : 'wasm';
|
||
|
|
} catch (_) { return 'wasm'; }
|
||
|
|
}
|
||
|
|
|
||
|
|
async function reportGpu() {
|
||
|
|
try {
|
||
|
|
if (!navigator.gpu) return 'no navigator.gpu';
|
||
|
|
const adapter = await navigator.gpu.requestAdapter();
|
||
|
|
if (!adapter) return 'no adapter';
|
||
|
|
return Array.from(adapter.features || []).join(',') || '(no optional features)';
|
||
|
|
} catch (e) {
|
||
|
|
return 'adapter error: ' + e.message;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
self.onmessage = async (e) => {
|
||
|
|
const msg = e.data || {};
|
||
|
|
|
||
|
|
try {
|
||
|
|
if (msg.type === 'load') {
|
||
|
|
label = msg.label || msg.task;
|
||
|
|
self.postMessage({ type: 'gpu', label, features: await reportGpu() });
|
||
|
|
const device = await bestDevice();
|
||
|
|
pipe = await pipeline(msg.task, msg.model, {
|
||
|
|
device,
|
||
|
|
dtype: msg.dtype,
|
||
|
|
progress_callback: (p) => self.postMessage({ type: 'progress', label, p })
|
||
|
|
});
|
||
|
|
self.postMessage({ type: 'ready', label, device });
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (msg.type === 'run') {
|
||
|
|
if (!pipe) throw new Error(label + ' is not loaded yet');
|
||
|
|
const out = await pipe(msg.input, msg.options || {});
|
||
|
|
self.postMessage({ type: 'result', id: msg.id, out });
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
} catch (err) {
|
||
|
|
self.postMessage({
|
||
|
|
type: msg.type === 'load' ? 'loadError' : 'error',
|
||
|
|
id: msg.id,
|
||
|
|
label,
|
||
|
|
message: (err && err.message) || String(err)
|
||
|
|
});
|
||
|
|
}
|
||
|
|
};
|