Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
825a25b626 | ||
|
|
81da657094 |
+73
-9
@@ -195,6 +195,23 @@ const state = {
|
||||
degradedAt: 0
|
||||
};
|
||||
|
||||
/*
|
||||
* THE SHAPE OF A LISTENER ROW. The server sends { code, count } — see
|
||||
* listenerCounts() in server.js — and this file read { lang, n } in five
|
||||
* places, so every set built from the rows was EMPTY: no language ever counted
|
||||
* as being listened to. Mostly harmless, except in the one place it mattered.
|
||||
* When a sentence arrives late, speech is kept only for the languages someone
|
||||
* is actually on, and "someone is on" was never true for any of them — so
|
||||
* under load the app dropped the speech of EVERY language, for everyone,
|
||||
* instead of only the languages nobody had chosen. Late became silent.
|
||||
* MEASURED on a campus Mac, 2026-10-15 15:11-15:12: 26 sentences dropped
|
||||
* across five languages during a stall, every one of them with listeners
|
||||
* present. Both spellings are read, so a server one build older or newer
|
||||
* still counts.
|
||||
*/
|
||||
function rowLang(r) { return r ? (r.code || r.lang || '') : ''; }
|
||||
function rowCount(r) { return r ? (Number(r.count != null ? r.count : r.n) || 0) : 0; }
|
||||
|
||||
/*
|
||||
* FOLLOW THE BOTTOM ONLY WHILE THE OPERATOR IS AT THE BOTTOM.
|
||||
*
|
||||
@@ -1070,10 +1087,15 @@ function preloadNewVoices() {
|
||||
const want = state.targets.filter((l) =>
|
||||
state.spokenLanguages.indexOf(l) !== -1 && !preloadedVoices.has(l));
|
||||
if (!want.length) return;
|
||||
want.forEach((l) => preloadedVoices.add(l));
|
||||
/* One at a time in the child anyway; cap the request so a burst of joins
|
||||
cannot queue up a re-download marathon. */
|
||||
window.engineHost.ttsPreload(want.slice(0, 4));
|
||||
cannot queue up a re-download marathon.
|
||||
Only what is SENT counts as warmed. Every wanted language used to be
|
||||
marked and only the first four sent, so a fifth was never preloaded and
|
||||
built its engine on its first sentence instead — por_PT, at all five
|
||||
starts on the Mini on 2026-10-15. Six is the voice process's own cap. */
|
||||
const batch = want.slice(0, 6);
|
||||
batch.forEach((l) => preloadedVoices.add(l));
|
||||
window.engineHost.ttsPreload(batch);
|
||||
}
|
||||
|
||||
/* The translation model reports its download from the main process. */
|
||||
@@ -1084,7 +1106,10 @@ if (window.engineHost && window.engineHost.onTtsReady) {
|
||||
state.spokenDefaults = info.defaults || state.spokenDefaults || {};
|
||||
log('built-in voices ready — ' + state.spokenLanguages.length + ' languages', 'ok');
|
||||
/* Warm whatever people are already reading, so the first sentence of a
|
||||
service is not the one that pays for the download. */
|
||||
service is not the one that pays for the download. A voice process
|
||||
that has just (re)started holds no engines, whatever this page warmed
|
||||
in the last one — so forget that first. */
|
||||
preloadedVoices.clear();
|
||||
preloadNewVoices();
|
||||
});
|
||||
window.engineHost.onTtsProgress((p) => log('voice ' + p.bundle + ': ' + p.status));
|
||||
@@ -1778,9 +1803,9 @@ async function pumpPartial() {
|
||||
* still gets a preview.
|
||||
*/
|
||||
const heard = (state.byLanguage || [])
|
||||
.filter((r) => r && r.lang && r.lang !== state.source && (r.n || 0) > 0)
|
||||
.filter((r) => r && rowLang(r) && rowLang(r) !== state.source && rowCount(r) > 0)
|
||||
.sort((a, b) => (b.n || 0) - (a.n || 0))
|
||||
.map((r) => r.lang)
|
||||
.map(rowLang)
|
||||
.filter((l) => state.targets.indexOf(l) !== -1);
|
||||
const want = (heard.length ? heard : orderedTargets()).slice(0, PARTIAL_MAX_LANGS);
|
||||
/* Never queue behind a finished sentence — a final in flight is worth more
|
||||
@@ -1806,7 +1831,7 @@ async function pumpPartial() {
|
||||
*/
|
||||
function orderedTargets() {
|
||||
const counts = {};
|
||||
for (const row of state.byLanguage || []) counts[row.lang] = row.n || 1;
|
||||
for (const row of state.byLanguage || []) counts[rowLang(row)] = rowCount(row) || 1;
|
||||
return state.targets
|
||||
.filter((t) => t !== state.source)
|
||||
.sort((a, b) => (counts[b] || 0) - (counts[a] || 0));
|
||||
@@ -1969,7 +1994,7 @@ async function pump() {
|
||||
/* Late, but nothing is behind it: say so, and translate it anyway. */
|
||||
log('audio ' + ageSec.toFixed(0) + 's late, but nothing is queued — translating it anyway', 'warn');
|
||||
} else if (ageSec > 6) {
|
||||
const listening = new Set((state.byLanguage || []).map((r) => r.lang));
|
||||
const listening = new Set((state.byLanguage || []).map(rowLang));
|
||||
const before = targets.length;
|
||||
targets = targets.filter((t) => listening.has(t));
|
||||
if (targets.length < before) { state.degradedFinals++; state.degradedAt = Date.now(); }
|
||||
@@ -2169,7 +2194,7 @@ setInterval(() => {
|
||||
}, 30000);
|
||||
|
||||
function translateSentence(cid, sentence, targets, gainDb, silent) {
|
||||
const listening = new Set((state.byLanguage || []).map((r) => r.lang));
|
||||
const listening = new Set((state.byLanguage || []).map(rowLang));
|
||||
for (const lang of targets) {
|
||||
if ((mtDepth[lang] || 0) > 6 && !listening.has(lang)) {
|
||||
state.degradedFinals++;
|
||||
@@ -4643,6 +4668,40 @@ function renderVoiceList() {
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* WHERE AN AUDITION PLAYS: never on the card that feeds the room.
|
||||
*
|
||||
* An <audio> element with no sink plays on the system default output, and on
|
||||
* the production Mini that IS the Dante Virtual Soundcard (system_profiler:
|
||||
* "Default Output Device: Yes") — the same card that carries Spanish on its
|
||||
* third channel (outputMap spa_Latn -> [2]). So ▶ there put the sample line
|
||||
* onto the Dante network, service or no service. The phones were never
|
||||
* exposed: an audition comes back to this page as bytes and never becomes a
|
||||
* clip the server can see.
|
||||
*
|
||||
* Which of the card's channels it reached is Chromium's mono up-mix and the
|
||||
* Dante routing, neither of which this page controls, so it does not try to
|
||||
* pick a "safe" channel. It plays on another output, or not at all, and says
|
||||
* why.
|
||||
*/
|
||||
async function auditionSink() {
|
||||
const routedId = ($('outDevice') && $('outDevice').value) || '';
|
||||
const routedLabel = (state.cfg && state.cfg.outputDeviceLabel) || '';
|
||||
if (!routedId && !routedLabel) return ''; // no hardware output in use
|
||||
const strip = (l) => String(l || '').toLowerCase().replace(/^default\s*-\s*/, '').trim();
|
||||
let outs = [];
|
||||
try {
|
||||
outs = (await navigator.mediaDevices.enumerateDevices()).filter((d) => d.kind === 'audiooutput');
|
||||
} catch (_) { /* refused below rather than guessed */ }
|
||||
const card = strip(((outs.find((d) => d.deviceId === routedId) || {}).label) || routedLabel);
|
||||
const isCard = (d) => (!!routedId && d.deviceId === routedId) || (!!card && strip(d.label) === card);
|
||||
const def = outs.find((d) => d.deviceId === 'default');
|
||||
if (def && !isCard(def)) return ''; // the system default is somewhere else
|
||||
const other = outs.find((d) => d.deviceId !== 'default' && d.deviceId !== 'communications' && !isCard(d));
|
||||
if (other) return other.deviceId;
|
||||
throw new Error('the only output here is the one feeding the hardware channels — auditions stay off it');
|
||||
}
|
||||
|
||||
async function auditionVoice(lang, v, btn, msg) {
|
||||
if (auditionBusy) return;
|
||||
auditionBusy = true;
|
||||
@@ -4667,9 +4726,14 @@ async function auditionVoice(lang, v, btn, msg) {
|
||||
msg.className = 'vmsg';
|
||||
}
|
||||
if (instruct) msg.textContent = 'with your direction…';
|
||||
/* Where it will play is settled BEFORE the voice process builds anything:
|
||||
an audition that cannot play should cost the service nothing. */
|
||||
const sink = await auditionSink();
|
||||
const out = await window.engineHost.ttsAudition({ lang, voice: v, instruct });
|
||||
if (!auditionAudio) auditionAudio = new Audio();
|
||||
if (!out.wav) throw new Error('that voice produced no audio');
|
||||
if (typeof auditionAudio.setSinkId === 'function') await auditionAudio.setSinkId(sink);
|
||||
else if (sink) throw new Error('this build cannot keep auditions off the hardware output');
|
||||
auditionAudio.src = 'data:audio/wav;base64,' + out.wav;
|
||||
/* play() rejects on autoplay policy or a decode failure; either way the
|
||||
operator needs the reason, not a silent button. */
|
||||
|
||||
+3
-2
@@ -2,7 +2,7 @@
|
||||
"name": "church-translate-engine",
|
||||
"productName": "Church Translate",
|
||||
"version": "3.23.2",
|
||||
"buildNumber": 126,
|
||||
"buildNumber": 127,
|
||||
"private": true,
|
||||
"description": "Live sermon translation — self-contained: captures audio, transcribes, translates, and serves the congregation page itself",
|
||||
"main": "main.js",
|
||||
@@ -50,6 +50,7 @@
|
||||
"vendor/ort-wasm-simd-threaded.jsep.mjs",
|
||||
"server/**/*",
|
||||
"tts-service.js",
|
||||
"voice-install.js",
|
||||
"kokoro-sidecar.js",
|
||||
"voices.js",
|
||||
"pt-variants.js",
|
||||
@@ -106,7 +107,7 @@
|
||||
]
|
||||
},
|
||||
"asar": false,
|
||||
"buildVersion": "126",
|
||||
"buildVersion": "127",
|
||||
"win": {
|
||||
"target": [
|
||||
{
|
||||
|
||||
@@ -219,7 +219,21 @@ class LanguageStream {
|
||||
this.dropTimes = []; // when, for a "recently" view on the operator page
|
||||
this.building = []; // frames of the segment being filled
|
||||
this.segments = []; // { seq, body, frames }
|
||||
this.nextSeq = 1;
|
||||
/*
|
||||
* SEGMENT NUMBERS MUST NEVER REPEAT, and starting at 1 made them repeat.
|
||||
* A stream object is built when a language's first listener arrives and
|
||||
* dropped when the last one leaves, so within one service the same
|
||||
* language can get a fresh stream several times — and a phone that came
|
||||
* back from the lock screen, or a second listener, asked for
|
||||
* spa_Latn/1.aac again. The segments are served 'max-age=60, immutable',
|
||||
* which is the right header for a segment that is never reused and the
|
||||
* wrong one for a number that is: for up to a minute the browser answered
|
||||
* from its cache with the PREVIOUS stream's audio under the new number —
|
||||
* a sentence from earlier in the sermon, replayed. Numbering from the
|
||||
* clock makes every incarnation's numbers new; HLS only asks that they
|
||||
* increase by one within a playlist, which they still do.
|
||||
*/
|
||||
this.nextSeq = Date.now();
|
||||
this.timer = null;
|
||||
this.lastTick = 0;
|
||||
this.carry = 0; // fractional frames owed from the last tick
|
||||
|
||||
@@ -475,6 +475,7 @@
|
||||
drift: null,
|
||||
watchdog: null, // catches engines that stall on HLS without erroring
|
||||
everPlayed: false, // this engine has PROVEN it can play HLS
|
||||
clipsForVoice: false, // on clips because of an alternate voice, not because HLS failed
|
||||
// clip-fallback state
|
||||
ctx: null,
|
||||
keepAlive: null,
|
||||
@@ -594,9 +595,13 @@
|
||||
/* Give up on streaming and use per-sentence clips. The clip path was already
|
||||
unlocked inside the tap precisely so this is possible after the gesture has
|
||||
been spent — otherwise a browser that lies about HLS would end up silent. */
|
||||
function fallBackToClips() {
|
||||
function fallBackToClips(forVoice) {
|
||||
if (srv.mode === 'clips') return;
|
||||
srv.mode = 'clips';
|
||||
/* Remember WHY. Only a page that left the stream for an alternate voice
|
||||
returns to it when the listener goes back to the default — see
|
||||
applyVoiceMode(). */
|
||||
srv.clipsForVoice = !!forVoice;
|
||||
stopStream();
|
||||
if (srv.keepAlive) srv.keepAlive.play().catch(function () {});
|
||||
try { console.warn('live stream unsupported here — using per-sentence clips'); } catch (_) {}
|
||||
@@ -868,14 +873,27 @@
|
||||
*/
|
||||
function applyVoiceMode() {
|
||||
if (srv.voice) {
|
||||
fallBackToClips();
|
||||
fallBackToClips(true);
|
||||
return;
|
||||
}
|
||||
/* Back on the default: allow streaming again. fallBackToClips latches
|
||||
mode to 'clips' and maybeStartStream refuses to act on that, so the
|
||||
latch has to be released here or the listener stays on clips for the
|
||||
rest of the service. */
|
||||
if (srv.mode === 'clips' && hlsSupported() && srv.unlocked && srv.serverHello) {
|
||||
/*
|
||||
* Only when the alternate is what put us on clips. Every engine reconnect
|
||||
* sends every listener a 'voices' frame, and an engine that cannot play
|
||||
* HLS used to be handed back to the stream by it — eight seconds of
|
||||
* dropped clips until the watchdog demoted it again.
|
||||
*
|
||||
* And silence the alternate's clips that are still scheduled. Left to
|
||||
* play out they overlapped the shared stream as it came up: the voice
|
||||
* the listener left and the default together, for up to the six seconds
|
||||
* the clip queue holds.
|
||||
*/
|
||||
if (srv.mode === 'clips' && srv.clipsForVoice && hlsSupported() && srv.unlocked && srv.serverHello) {
|
||||
stopClips();
|
||||
srv.clipsForVoice = false;
|
||||
srv.mode = '';
|
||||
maybeStartStream();
|
||||
}
|
||||
@@ -909,22 +927,52 @@
|
||||
function switchAudioLanguage(lang) {
|
||||
stopClips(); // the old language stops NOW, mid-word
|
||||
if (srv.mode === 'stream' && srv.el) {
|
||||
/* Pause the old stream; the new one starts when the server's hello
|
||||
confirms the new language's stream exists. */
|
||||
try { srv.el.pause(); } catch (_) {}
|
||||
if (srv.retry) { clearTimeout(srv.retry); srv.retry = null; }
|
||||
/*
|
||||
* STOP the old stream, do not merely pause it; the new one starts when
|
||||
* the server's hello confirms the new language's stream exists.
|
||||
*
|
||||
* Paused, the element still pointed at the OLD playlist until that
|
||||
* hello, and three things restart a paused element without asking: the
|
||||
* 8 s watchdog's retry (startStream(srv.lang, true), and srv.lang was
|
||||
* still the old language), coming back from a locked screen
|
||||
* (visibilitychange plays it), and the lock-screen play button. Each
|
||||
* put the previous language's voice back in the listener's ear.
|
||||
* stopStream() is what the back button already does, and the hello's
|
||||
* maybeStartStream() brings the new language up from it.
|
||||
*/
|
||||
stopStream();
|
||||
}
|
||||
setMediaSession();
|
||||
}
|
||||
|
||||
// ------------------------------------------------- per-clip fallback path
|
||||
|
||||
/*
|
||||
* Is this clip for the language AND voice this page is on right now?
|
||||
*
|
||||
* Asked twice — when the clip is announced and again when its bytes land —
|
||||
* because the fetch and the decode take a few hundred milliseconds, and
|
||||
* stopClips() can only silence what is already scheduled, not what is still
|
||||
* downloading. A listener who switched language in that window heard the
|
||||
* previous language's sentence AFTER the switch; for Spanish that is one
|
||||
* sentence of the Mexican voice in a Spaniard's ear, or the reverse. The
|
||||
* name carries the language (p<seq>-spa_ES.aac); the server also sends lang
|
||||
* and voice in the frame, and a frame without them is judged on the name.
|
||||
*/
|
||||
function clipIsOurs(frame) {
|
||||
var lang = frame.lang || (String(frame.name).match(/-([a-z]{3}_[A-Za-z]+)\.aac$/) || [])[1];
|
||||
if (lang !== app.lang) return false;
|
||||
if ('voice' in frame && String(frame.voice || '') !== String(srv.voice || '')) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/* Fetch, decode and schedule one clip so it follows the previous one. Only
|
||||
used where native HLS is missing. */
|
||||
function playClip(frame) {
|
||||
var name = frame && frame.name;
|
||||
if (srv.mode === 'stream') return; // the stream already carries it
|
||||
if (!tts.on || !srv.ctx || !name) return;
|
||||
if (!clipIsOurs(frame)) return;
|
||||
|
||||
fetch('/audio/' + encodeURIComponent(name))
|
||||
.then(function (r) { return r.ok ? r.arrayBuffer() : Promise.reject(new Error('missing')); })
|
||||
@@ -936,6 +984,8 @@
|
||||
})
|
||||
.then(function (audio) {
|
||||
if (!tts.on || !srv.ctx) return;
|
||||
/* Landed after a switch of language, voice or mode: not ours any more. */
|
||||
if (srv.mode === 'stream' || !clipIsOurs(frame)) return;
|
||||
var src = srv.ctx.createBufferSource();
|
||||
src.buffer = audio;
|
||||
src.connect(srv.ctx.destination);
|
||||
@@ -1424,7 +1474,11 @@
|
||||
}
|
||||
/* The playlist is guaranteed to exist from this moment — the server
|
||||
creates the stream when this listener registers. */
|
||||
maybeStartStream();
|
||||
/* Not for a listener on an alternate: the shared stream is the
|
||||
operator's default voice, and starting it here played that voice
|
||||
until the server's 'voices' reply moved the page to clips. */
|
||||
if (srv.voice) fallBackToClips(true);
|
||||
else maybeStartStream();
|
||||
/* The picker was rendered before this socket existed, when we still had
|
||||
to assume the device's own voice. Now we know better — re-render, or
|
||||
a listener keeps staring at a voice chooser that controls nothing. */
|
||||
|
||||
+22
-6
@@ -1763,12 +1763,20 @@ wss.on('connection', (ws, req) => {
|
||||
if (!m.voice) row.audio[m.lang] = m.name;
|
||||
|
||||
delayed((who) => {
|
||||
const frame = JSON.stringify({
|
||||
t: 'audio', id: row.id, name: m.name, seconds: m.seconds || 0
|
||||
});
|
||||
/* Which voice this clip IS. '' is the operator's default, which is
|
||||
what a listener who never opened the chooser is on. */
|
||||
const clipVoice = String(m.voice || '');
|
||||
/*
|
||||
* The frame says which language and voice it carries, and the page
|
||||
* checks both again when the clip's bytes land (clipIsOurs() in
|
||||
* listen.js). The routing below is decided at SEND time; a listener
|
||||
* who switches language or voice while a clip is still downloading
|
||||
* would otherwise hear it after the switch, in the voice they left.
|
||||
*/
|
||||
const frame = JSON.stringify({
|
||||
t: 'audio', id: row.id, name: m.name, seconds: m.seconds || 0,
|
||||
lang: m.lang, voice: clipVoice
|
||||
});
|
||||
for (const [ws, meta] of state.listeners) {
|
||||
if (ws.readyState !== 1) continue;
|
||||
if (!who(meta)) continue;
|
||||
@@ -2054,14 +2062,22 @@ wss.on('connection', (ws, req) => {
|
||||
if (m.t === 'lang') {
|
||||
const meta = state.listeners.get(ws);
|
||||
if (!meta) return;
|
||||
meta.lang = String(m.lang || config.sourceLanguage);
|
||||
const next = String(m.lang || config.sourceLanguage);
|
||||
/*
|
||||
* A voice belongs to a language. Carrying a Spanish speaker across to
|
||||
* Portuguese would ask the engine for a voice that language does not
|
||||
* have, so switching language always returns the listener to the
|
||||
* default and hands them the new language's options.
|
||||
*
|
||||
* Only a CHANGE of language, though. The page re-sends the language it
|
||||
* is already on to have its captions re-rendered (the original-text
|
||||
* toggle does exactly that), and resetting the voice there put a
|
||||
* listener on an alternate back on the default for a round trip —
|
||||
* until the page re-asserted its pick, they were sent the default
|
||||
* voice's clips: one voice, then the other, then theirs again.
|
||||
*/
|
||||
meta.voice = '';
|
||||
if (next !== meta.lang) meta.voice = '';
|
||||
meta.lang = next;
|
||||
pushLanguagesToEngine();
|
||||
/*
|
||||
* builtInVoice MUST be here, not just on the first hello. The page does
|
||||
@@ -2073,7 +2089,7 @@ wss.on('connection', (ws, req) => {
|
||||
*/
|
||||
send(ws, { t: 'hello', live: state.live, lang: meta.lang, serviceName: config.serviceName, build: BUILD,
|
||||
builtInVoice: Voices.hasVoice(meta.lang),
|
||||
voiceOptions: voiceOptionsFor(meta.lang), voice: '',
|
||||
voiceOptions: voiceOptionsFor(meta.lang), voice: meta.voice || '',
|
||||
scrollback: scrollbackFor(meta.lang, meta.proxied) });
|
||||
broadcastOperators();
|
||||
}
|
||||
|
||||
+170
-81
@@ -42,9 +42,6 @@ let VOICE_DIR = process.platform === 'win32'
|
||||
: path.join(os.homedir(), 'Library/Application Support/Church Translate Voices');
|
||||
let CLIP_DIR = path.join(os.tmpdir(), 'church-translate-audio');
|
||||
|
||||
/* Loaded engines, one per language. Kept small: each costs ~330-400 MB of ONNX
|
||||
arena regardless of the 63 MB model, so we do not hold voices nobody is
|
||||
listening to. */
|
||||
/* What each voice says when auditioned — a real line of scripture, so the
|
||||
operator hears the voice on the material it will actually read. */
|
||||
const SAMPLE_TEXT = {
|
||||
@@ -65,8 +62,25 @@ const SAMPLE_TEXT = {
|
||||
hat_Latn: 'Senyè a se gadò mwen, mwen p ap manke anyen.'
|
||||
};
|
||||
|
||||
const loaded = new Map();
|
||||
const MAX_LOADED = 6;
|
||||
/*
|
||||
* Loaded engines, ONE PER BUNDLE, shared by every language, variant and
|
||||
* audition that speaks with it. Kept bounded, not small: MEASURED on an M2
|
||||
* Max, an engine costs ~75 MB at construction and ~165 MB once it has spoken
|
||||
* (8 engines, 1.3 GB after one sentence each), not the 330-400 MB this used
|
||||
* to assume. The cap is sized to the largest campus — eight languages — so a
|
||||
* live engine is never evicted to make room for another live one in normal
|
||||
* service; below the cap an eviction means a reload of ~0.3-0.5 s on this
|
||||
* process's only thread, in front of the next sentence, every time.
|
||||
*
|
||||
* `live` marks an engine a sentence or the preload asked for. Auditions keep
|
||||
* to MAX_AUDITION slots of their own, always go first, and can never push a
|
||||
* live engine out: with every slot speaking, an auditioned voice is built,
|
||||
* played once and dropped. See engineFor() for why this is keyed by bundle.
|
||||
*/
|
||||
const loaded = new Map(); // 'bundle/model' -> { tts, usedAt, live }
|
||||
const installing = new Map(); // bundle -> Promise: one download, however many ask
|
||||
const MAX_LOADED = 8;
|
||||
const MAX_AUDITION = 2;
|
||||
|
||||
/*
|
||||
* The operator's per-language voice choices, made by ear in the Voices
|
||||
@@ -142,81 +156,120 @@ function isInstalled(v) {
|
||||
* 359 files, same total bytes — and it is now the same code on both platforms.
|
||||
*/
|
||||
function installVoice(v) {
|
||||
/*
|
||||
* OFF THIS THREAD. The fetch, the bzip2 and the unpack all ran here, on the
|
||||
* voice process's only thread, from inside engineFor() — so the first
|
||||
* sentence of a language whose voice was not installed yet stalled every
|
||||
* OTHER language's sentences for as long as the install took. MEASURED on a
|
||||
* campus Mac, 2026-10-15 15:11: a listener tapped Chinese, the 121 MB zh
|
||||
* model was fetched and unpacked three times at once on this thread, every
|
||||
* stream stalled, 26 sentences were dropped and the catch-up rushed the
|
||||
* Spanish voices to their +16% cap. The work now happens in a worker thread
|
||||
* (voice-install.js); this thread only relays its progress and keeps
|
||||
* speaking. The worker unpacks into a staging folder and renames it into
|
||||
* place whole, so isInstalled() never sees a half-written bundle.
|
||||
*/
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.mkdirSync(VOICE_DIR, { recursive: true });
|
||||
const url = voices.bundleUrl(v.bundle);
|
||||
|
||||
const fail = (why) => reject(new Error('could not install ' + v.bundle + ': ' + why));
|
||||
reply({ type: 'voiceProgress', bundle: v.bundle, status: 'downloading' });
|
||||
|
||||
(async () => {
|
||||
const res = await fetch(url, { redirect: 'follow' });
|
||||
if (!res.ok) throw new Error('HTTP ' + res.status);
|
||||
const body = Buffer.from(await res.arrayBuffer());
|
||||
/* A proxy or a captive portal returns an HTML error page with HTTP 200;
|
||||
the bzip2 magic is the cheapest way to notice before unpacking it. */
|
||||
if (body.length < 1024 || body.toString('latin1', 0, 3) !== 'BZh') {
|
||||
throw new Error('that download was not a voice bundle');
|
||||
}
|
||||
reply({ type: 'voiceProgress', bundle: v.bundle, status: 'installing' });
|
||||
|
||||
const bunzip = require('unbzip2-stream');
|
||||
const tar = require('tar-stream');
|
||||
const { Readable } = require('stream');
|
||||
|
||||
await new Promise((res2, rej2) => {
|
||||
const extract = tar.extract();
|
||||
extract.on('entry', (header, stream, next) => {
|
||||
/* Never write outside VOICE_DIR, whatever the archive claims. */
|
||||
const dest = path.resolve(VOICE_DIR, header.name);
|
||||
if (dest !== VOICE_DIR && !dest.startsWith(VOICE_DIR + path.sep)) {
|
||||
stream.resume(); return next();
|
||||
}
|
||||
if (header.type === 'directory') {
|
||||
fs.mkdirSync(dest, { recursive: true });
|
||||
stream.resume(); return next();
|
||||
}
|
||||
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
||||
stream.pipe(fs.createWriteStream(dest)).on('finish', next).on('error', rej2);
|
||||
let w;
|
||||
try {
|
||||
const { Worker } = require('worker_threads');
|
||||
w = new Worker(path.join(__dirname, 'voice-install.js'), {
|
||||
workerData: { url: voices.bundleUrl(v.bundle), voiceDir: VOICE_DIR, bundle: v.bundle }
|
||||
});
|
||||
extract.on('finish', res2);
|
||||
extract.on('error', rej2);
|
||||
Readable.from(body).pipe(bunzip()).on('error', rej2).pipe(extract);
|
||||
} catch (e) { return fail(e.message); }
|
||||
let settled = false;
|
||||
const done = (err) => { if (settled) return; settled = true; err ? fail(err) : resolve(); };
|
||||
w.on('message', (m) => {
|
||||
if (!m || !m.status) return;
|
||||
if (m.status === 'error') { done(m.message || 'the installer failed'); return; }
|
||||
reply({ type: 'voiceProgress', bundle: v.bundle, status: m.status });
|
||||
if (m.status === 'ready') done();
|
||||
});
|
||||
|
||||
reply({ type: 'voiceProgress', bundle: v.bundle, status: 'ready' });
|
||||
})().then(resolve, (err) => reject(new Error('could not install ' + v.bundle + ': ' + err.message)));
|
||||
w.on('error', (e) => done((e && e.message) || String(e)));
|
||||
w.on('exit', (code) => { if (!settled) done('the installer exited (' + code + ')'); });
|
||||
});
|
||||
}
|
||||
|
||||
async function engineFor(lang, explicit) {
|
||||
/* Auditions key the cache by bundle so trying five voices for one language
|
||||
does not evict the voice that language is actually using. */
|
||||
const v = explicit || builtinVoiceFor(lang);
|
||||
const key = explicit ? "audition:" + v.bundle : lang;
|
||||
if (loaded.has(key)) {
|
||||
const e = loaded.get(key);
|
||||
e.usedAt = Date.now();
|
||||
return e.tts;
|
||||
/*
|
||||
* The engine for one voice, built at most once however many ask for it.
|
||||
*
|
||||
* KEYED BY BUNDLE — and the old keying is what put batches of four
|
||||
* "audition:" loads in the production log with nobody at the Voices card.
|
||||
* Engines were keyed two ways: by language ('spa_Latn') for the preload, and
|
||||
* 'audition:<bundle>' whenever a voice was passed in, which speak() did for
|
||||
* EVERY ordinary sentence because the resolved default always has a bundle.
|
||||
* So the live path was filed as an audition, every bundle was built at least
|
||||
* twice (five languages asked for nine engines against a cap of six), the
|
||||
* preload warmed copies no sentence ever read, and the first sentence of
|
||||
* every service built the real ones on the spot. MEASURED on the Mini,
|
||||
* 2026-10-15: the 14:26 service's first sentence was said at 14:54:26.291
|
||||
* and its four engines were ready at 26.995, 27.309, 27.833 and 28.344 —
|
||||
* ~0.3-0.5 s each on this process's only thread — while what got evicted
|
||||
* was the startup preload's unused copies.
|
||||
*
|
||||
* The engine's config is a function of the bundle alone (voices.ttsConfig;
|
||||
* speed, gain and sid are applied per sentence), so sharing is exact:
|
||||
* spa_Latn and spa_MX read one es_MX-ald engine, and auditioning a voice that
|
||||
* is already speaking reads the live engine instead of building a second.
|
||||
*/
|
||||
async function engineFor(v, live, who) {
|
||||
if (!v || !v.bundle) throw new Error('no built-in voice' + (who ? ' for ' + who : ''));
|
||||
const key = v.bundle + '/' + (v.model || '');
|
||||
|
||||
/*
|
||||
* One download per bundle. A language joining mid-service is asked for by
|
||||
* the preload AND by its first sentences at once, and each used to fetch
|
||||
* and unpack the whole bundle itself — MEASURED on the Mini: pt_BR-jeff
|
||||
* three times between 11:39:05 and 11:39:50, zh-ll three times between
|
||||
* 15:11:17 and 15:12:16, two 'voice timed out' each time. The unpack is staged and
|
||||
* renamed into place whole (voice-install.js), so isInstalled() stays false
|
||||
* until the bundle is complete — but a second asker must still wait on the
|
||||
* first install, not start its own.
|
||||
*/
|
||||
if (!loaded.has(key) && (installing.has(v.bundle) || !isInstalled(v))) {
|
||||
if (!installing.has(v.bundle)) {
|
||||
installing.set(v.bundle, installVoice(v).finally(() => installing.delete(v.bundle)));
|
||||
}
|
||||
await installing.get(v.bundle);
|
||||
}
|
||||
|
||||
if (!v) throw new Error('no built-in voice for ' + lang);
|
||||
if (!isInstalled(v)) await installVoice(v);
|
||||
|
||||
/* Looked up again: whoever finished waiting first has built it. Nothing can
|
||||
interleave from here to the set — construction is synchronous. */
|
||||
let e = loaded.get(key);
|
||||
if (!e) {
|
||||
if (!sherpa) sherpa = require('sherpa-onnx-node');
|
||||
|
||||
const dir = voicePath(v);
|
||||
const tts = new sherpa.OfflineTts(voices.ttsConfig(dir, v));
|
||||
|
||||
/* Evict the least recently used rather than growing without bound. */
|
||||
if (loaded.size >= MAX_LOADED) {
|
||||
let oldest = null;
|
||||
for (const [k, e] of loaded) if (!oldest || e.usedAt < loaded.get(oldest).usedAt) oldest = k;
|
||||
if (oldest) { loaded.delete(oldest); log('evicted voice ' + oldest); }
|
||||
e = { tts: new sherpa.OfflineTts(voices.ttsConfig(voicePath(v), v)), usedAt: 0, live: !!live };
|
||||
if (makeRoom(!!live)) loaded.set(key, e);
|
||||
log('voice ready: ' + v.bundle + (who ? ' for ' + who : '') +
|
||||
(live ? '' : loaded.get(key) === e ? ' (audition)' : ' (audition, not kept: every slot is speaking)'));
|
||||
}
|
||||
e.usedAt = Date.now();
|
||||
/* An audition never demotes a live engine; a sentence promotes an auditioned one. */
|
||||
if (live) e.live = true;
|
||||
return e.tts;
|
||||
}
|
||||
|
||||
loaded.set(key, { tts, usedAt: Date.now() });
|
||||
log('voice ready: ' + key + ' (' + v.bundle + ')');
|
||||
return tts;
|
||||
/*
|
||||
* Room for one more engine, or false when an audition should not take one.
|
||||
* Least recently used goes first, as before — auditioned engines before live
|
||||
* ones, and a live engine only ever gives way to another live one.
|
||||
*/
|
||||
function makeRoom(live) {
|
||||
const stalest = (wantLive) => {
|
||||
let k = null;
|
||||
for (const [key, x] of loaded) if (x.live === wantLive && (!k || x.usedAt < loaded.get(k).usedAt)) k = key;
|
||||
return k;
|
||||
};
|
||||
const evict = (k) => { log('evicted voice ' + k + (loaded.get(k).live ? '' : ' (audition)')); loaded.delete(k); };
|
||||
const auditions = Array.from(loaded.values()).filter((x) => !x.live).length;
|
||||
if (!live && auditions >= MAX_AUDITION) evict(stalest(false));
|
||||
if (loaded.size < MAX_LOADED) return true;
|
||||
const k = stalest(false) || (live ? stalest(true) : null);
|
||||
if (!k) return false;
|
||||
evict(k);
|
||||
return true;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------- cloud voices
|
||||
@@ -928,6 +981,9 @@ async function speak(job) {
|
||||
text, Math.round((Number(job.gainDb) || 0) * 2) / 2, pinRate
|
||||
].join('\u0001');
|
||||
|
||||
/* Set when the leader's result was a Piper fallback in ANOTHER variant's
|
||||
voice — this job then speaks its own fallback, straight to Piper. */
|
||||
let piperOnly = false;
|
||||
if (key) {
|
||||
const hit = synthCache.get(key);
|
||||
if (hit) {
|
||||
@@ -941,8 +997,23 @@ async function speak(job) {
|
||||
counted as one; the key is deleted below so the NEXT utterance
|
||||
tries the natural voice again. */
|
||||
if (premiumPick && done.source !== premiumPick.provider) { cloudStats.fallback++; fallbackAt = Date.now(); }
|
||||
/*
|
||||
* ...but only a fallback in THIS language's own Piper voice. The key
|
||||
* names the PREMIUM voice, and spa_Latn, spa_ES and spa_MX share it
|
||||
* (KOKORO_LANG maps all three to 'e', both Portugueses to 'p'). Their
|
||||
* Piper fallbacks they do not share: spa_ES falls back to davefx,
|
||||
* spa_Latn to ald, por_PT to miro, por_Latn to jeff. Cloning the
|
||||
* leader's fallback put the Mexican voice in the Castilian stream for
|
||||
* that sentence — measured in a harness against this file: spa_ES's
|
||||
* clip was the byte-identical ald file and davefx never even loaded.
|
||||
* So the follower still skips the sidecar that just missed (the
|
||||
* variants still switch together), but in its own voice.
|
||||
*/
|
||||
if (done.source !== 'piper' || done.bundle === (voice.bundle || '')) {
|
||||
return cloneClip(done, job.lang, started);
|
||||
}
|
||||
piperOnly = true;
|
||||
}
|
||||
} catch (_) { /* the first caller failed — make our own */ }
|
||||
}
|
||||
}
|
||||
@@ -956,7 +1027,7 @@ async function speak(job) {
|
||||
* alternate skips the premium path entirely — otherwise cloudSynth would
|
||||
* fall back to the operator's own pick and the page would lie.
|
||||
*/
|
||||
const cloudOut = (alt && alt.engine === 'builtin')
|
||||
const cloudOut = (piperOnly || (alt && alt.engine === 'builtin'))
|
||||
? null
|
||||
: await cloudSynth(job.lang, text, (alt && alt.engine !== 'builtin') ? alt : null, { speed: kokoroSpeed, deadline });
|
||||
if (cloudOut) {
|
||||
@@ -984,13 +1055,16 @@ async function speak(job) {
|
||||
source: cloudOut.provider
|
||||
};
|
||||
}
|
||||
if (!(alt && alt.engine === 'builtin') && cloudVoiceFor(job.lang)) {
|
||||
/* A piperOnly job's fallback was already counted at the cache hit. */
|
||||
if (!piperOnly && !(alt && alt.engine === 'builtin') && cloudVoiceFor(job.lang)) {
|
||||
cloudStats.fallback++;
|
||||
fallbackAt = Date.now();
|
||||
if (cloudStats.fallback === 1 || cloudStats.fallback % 10 === 0) pushStats('fallback');
|
||||
}
|
||||
|
||||
const tts = await engineFor(job.lang, voice.bundle ? voice : null);
|
||||
/* Live: a sentence is speaking with this voice, so no audition may take
|
||||
its engine. A listener's built-in alternate counts — someone is on it. */
|
||||
const tts = await engineFor(voice.bundle ? voice : null, true, job.lang);
|
||||
if (!sherpa) sherpa = require('sherpa-onnx-node');
|
||||
|
||||
/*
|
||||
@@ -1067,6 +1141,9 @@ async function speak(job) {
|
||||
const seconds = secs;
|
||||
return {
|
||||
source: 'piper',
|
||||
/* WHICH Piper voice spoke it. A follower may share a fallback only when
|
||||
it is its own voice — see the cache hit above. */
|
||||
bundle: voice.bundle || '',
|
||||
/* What the sentence ended on — the stream gives a full stop a full breath
|
||||
and a mid-thought fragment only a beat. */
|
||||
terminal: text.slice(-1),
|
||||
@@ -1080,8 +1157,12 @@ async function speak(job) {
|
||||
};
|
||||
};
|
||||
|
||||
/* A piperOnly job is a fallback by construction. It must not stand in the
|
||||
cache as a leader, or the next identical line would inherit the miss
|
||||
without trying the natural voice — the chorus case below. */
|
||||
const cacheKey = piperOnly ? null : key;
|
||||
const promise = produce();
|
||||
if (key) synthCache.set(key, { promise, at: Date.now() });
|
||||
if (cacheKey) synthCache.set(cacheKey, { promise, at: Date.now() });
|
||||
try {
|
||||
const out = await promise;
|
||||
/*
|
||||
@@ -1091,10 +1172,10 @@ async function speak(job) {
|
||||
* miss would have re-created the very voice switch this file exists to
|
||||
* prevent (measured in the harness before shipping, not after).
|
||||
*/
|
||||
if (key && premiumPick && out && out.source !== premiumPick.provider) synthCache.delete(key);
|
||||
if (cacheKey && premiumPick && out && out.source !== premiumPick.provider) synthCache.delete(cacheKey);
|
||||
return out;
|
||||
} catch (e) {
|
||||
if (key) synthCache.delete(key);
|
||||
if (cacheKey) synthCache.delete(cacheKey);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
@@ -1187,11 +1268,15 @@ process.on('message', async (msg) => {
|
||||
if (msg.type === 'voices') {
|
||||
/* The operator changed a per-language voice choice. */
|
||||
voiceChoice = msg.choices || {};
|
||||
/* Drop cached engines for languages whose choice changed, so the next
|
||||
sentence uses the new voice rather than the one already resident. */
|
||||
for (const k of Array.from(loaded.keys())) {
|
||||
if (!String(k).startsWith("audition:")) loaded.delete(k);
|
||||
}
|
||||
/*
|
||||
* Nothing to drop. Engines are keyed by bundle, so a changed choice
|
||||
* resolves to a different engine on the next sentence by itself, and
|
||||
* the one it replaced ages out like any other. The loop that was here
|
||||
* only ever deleted the preload's per-language copies (the live path was
|
||||
* filed under 'audition:'); under bundle keys it would delete the live
|
||||
* engines too, and every "Use it" would make the next sentence rebuild
|
||||
* every language's voice.
|
||||
*/
|
||||
reply({ type: 'voicesOk', count: Object.keys(voiceChoice).length });
|
||||
return;
|
||||
}
|
||||
@@ -1222,7 +1307,9 @@ process.on('message', async (msg) => {
|
||||
} finally { voiceChoice = saved; }
|
||||
} else {
|
||||
if (!sherpa) sherpa = require('sherpa-onnx-node');
|
||||
const tts = await engineFor(msg.lang, spec.bundle ? spec : null);
|
||||
/* Not live: shares the engine when this voice is already speaking,
|
||||
otherwise takes an audition slot — never a live one. */
|
||||
const tts = await engineFor(spec.bundle ? spec : builtinVoiceFor(msg.lang), false, 'audition');
|
||||
const audio = tts.generate({ text, sid: spec.sid || 0, speed: spec.speed || 1.0 });
|
||||
if (spec.gain && spec.gain !== 1) {
|
||||
for (let i = 0; i < audio.samples.length; i++) {
|
||||
@@ -1256,7 +1343,9 @@ process.on('message', async (msg) => {
|
||||
/* Warm the languages people are actually listening to, so the first
|
||||
sentence of a service is not the one that pays for the download. */
|
||||
for (const lang of msg.langs || []) {
|
||||
try { await engineFor(lang); } catch (e) { log('preload failed for ' + lang + ': ' + e.message); }
|
||||
/* Keyed exactly as speak() keys it, so this warms the engine the first
|
||||
sentence will read, not a copy beside it. */
|
||||
try { await engineFor(builtinVoiceFor(lang), true, lang); } catch (e) { log('preload failed for ' + lang + ': ' + e.message); }
|
||||
}
|
||||
reply({ type: 'preloaded', langs: msg.langs || [] });
|
||||
return;
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
* voice-install.js — download and unpack ONE voice bundle, in a worker thread.
|
||||
*
|
||||
* This work used to run on the voice process's own event loop, from inside
|
||||
* engineFor(), the moment a sentence arrived for a language whose voice was
|
||||
* not installed. MEASURED on a campus Mac, 2026-10-15 15:11: a listener tapped
|
||||
* Chinese, three parallel downloads and unpacks of the 121 MB zh model ran at
|
||||
* once on the thread that has to answer every sentence, every other language
|
||||
* stalled for about a minute, 26 sentences were dropped, and the catch-up
|
||||
* pushed the Spanish voices to their +16% rush cap — reported from the pews
|
||||
* as the voice "changing". bzip2 in pure JS is CPU work; it has no business on
|
||||
* that thread.
|
||||
*
|
||||
* So the whole fetch-check-unpack now happens here, off the loop, and
|
||||
* tts-service.js de-duplicates it per bundle. The archive is unpacked into a
|
||||
* staging folder and renamed into place only when complete, so a half-written
|
||||
* bundle can never pass isInstalled() and be handed to sherpa.
|
||||
*/
|
||||
'use strict';
|
||||
const { parentPort, workerData } = require('worker_threads');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const { url, voiceDir, bundle } = workerData;
|
||||
|
||||
(async () => {
|
||||
fs.mkdirSync(voiceDir, { recursive: true });
|
||||
const res = await fetch(url, { redirect: 'follow' });
|
||||
if (!res.ok) throw new Error('HTTP ' + res.status);
|
||||
const body = Buffer.from(await res.arrayBuffer());
|
||||
/* A proxy or a captive portal returns an HTML error page with HTTP 200;
|
||||
the bzip2 magic is the cheapest way to notice before unpacking it. */
|
||||
if (body.length < 1024 || body.toString('latin1', 0, 3) !== 'BZh') {
|
||||
throw new Error('that download was not a voice bundle');
|
||||
}
|
||||
parentPort.postMessage({ status: 'installing' });
|
||||
|
||||
const bunzip = require('unbzip2-stream');
|
||||
const tar = require('tar-stream');
|
||||
const { Readable } = require('stream');
|
||||
|
||||
const staging = path.join(voiceDir, '.installing-' + bundle + '-' + process.pid);
|
||||
fs.rmSync(staging, { recursive: true, force: true });
|
||||
fs.mkdirSync(staging, { recursive: true });
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
const extract = tar.extract();
|
||||
extract.on('entry', (header, stream, next) => {
|
||||
/* Never write outside the staging folder, whatever the archive claims. */
|
||||
const dest = path.resolve(staging, header.name);
|
||||
if (dest !== staging && !dest.startsWith(staging + path.sep)) {
|
||||
stream.resume(); return next();
|
||||
}
|
||||
if (header.type === 'directory') {
|
||||
fs.mkdirSync(dest, { recursive: true });
|
||||
stream.resume(); return next();
|
||||
}
|
||||
fs.mkdirSync(path.dirname(dest), { recursive: true });
|
||||
stream.pipe(fs.createWriteStream(dest)).on('finish', next).on('error', reject);
|
||||
});
|
||||
extract.on('finish', resolve);
|
||||
extract.on('error', reject);
|
||||
Readable.from(body).pipe(bunzip()).on('error', reject).pipe(extract);
|
||||
});
|
||||
|
||||
/* The archive's top-level folder is the bundle name — that is what
|
||||
voicePath() opens. Swap it into place in one rename. */
|
||||
const unpacked = path.join(staging, bundle);
|
||||
if (!fs.existsSync(unpacked)) throw new Error('the archive did not contain ' + bundle);
|
||||
const final = path.join(voiceDir, bundle);
|
||||
fs.rmSync(final, { recursive: true, force: true });
|
||||
fs.renameSync(unpacked, final);
|
||||
fs.rmSync(staging, { recursive: true, force: true });
|
||||
|
||||
parentPort.postMessage({ status: 'ready' });
|
||||
})().catch((err) => {
|
||||
parentPort.postMessage({ status: 'error', message: (err && err.message) || String(err) });
|
||||
});
|
||||
+12
-4
@@ -153,7 +153,15 @@ curl -si -N -H 'Connection: Upgrade' -H 'Upgrade: websocket' \
|
||||
|
||||
Then open the page on a phone: for the LAN-only setup on the guest WiFi, for a
|
||||
published one **with WiFi turned off**. That is the real test. If the page loads
|
||||
but captions never appear, the WebSocket headers are missing. If phones on
|
||||
guest WiFi cannot connect at all, look at client isolation on that network
|
||||
before anything else. `/operator` returning 404 through the proxy is by design;
|
||||
staff use `http://10.1.7.39:3100/operator` on the LAN.
|
||||
but captions never appear, the WebSocket headers are missing. `/operator`
|
||||
returning 404 through the proxy is by design; staff use
|
||||
`http://10.1.7.39:3100/operator` on the LAN.
|
||||
|
||||
**Guest WiFi on UniFi.** Guest networks block every private range by default,
|
||||
so the proxy is unreachable from them until two things change: a firewall
|
||||
policy that allows the guest zone to reach the proxy on 80/443, and the
|
||||
proxy's range removed from the guest hotspot's *Post-Authorization
|
||||
Restrictions*. The second one is easy to miss: the access points enforce that
|
||||
list themselves, so with only the firewall policy a phone works for a minute or
|
||||
two and then times out. North Attleboro's fix, with scripts, is in
|
||||
[`examples/waters-church-na/`](examples/waters-church-na/README.md) section 3.
|
||||
|
||||
@@ -75,7 +75,33 @@ Known limits:
|
||||
so the effect is minor; if audio lag is reported, that is the first thing to
|
||||
revisit with a hand-written nginx include.
|
||||
|
||||
## 3. After the proxy answers
|
||||
## 3. Waters Guest WiFi at North Attleboro — done 2026-10-15
|
||||
|
||||
The Guest network (VLAN 25, 192.168.16.0/24) is in the UniFi **Hotspot** zone.
|
||||
Two things stood between a phone on Waters Guest and `10.1.1.4`:
|
||||
|
||||
| Blocker | Fix |
|
||||
| --- | --- |
|
||||
| Zone default "Block All Traffic" Hotspot → Internal | User policy "Guest to translate.watersmedia.org (Boba-Fett web)", ALLOW Hotspot → Internal, destination `10.1.1.4` TCP 80 and 443, index 10000 (evaluated first). Created with [`zone-allow-policy.py`](zone-allow-policy.py). |
|
||||
| Guest "Post-Authorization Restrictions" listed 10.0.0.0/8. The list is enforced twice: as system policies on the gateway (index 30001, which the allow rule sits above) **and on the access points**, which no gateway policy can override. Symptom: a fresh guest client worked for about two minutes, then timed out even to `http://10.1.1.4/`. | Removed 10.0.0.0/8 from the list with [`guest-restrictions.py`](guest-restrictions.py). The Integration API refuses to edit the system policies; the script edits the legacy `guest_access` setting (`restricted_subnet_1..3`) and the four gateway policies regenerate from it. 192.168.0.0/16 and 172.16.0.0/12 stay. Guests are still cut off from every other 10.x host by the zone default block. |
|
||||
| The Waters Guest SSID only broadcast from the AP "The Vault" | Added Auditorium Left, Auditorium Right, Main Lobby Entrance, Admin Offices - U7 and Creative Offices - U7 with [`wifi-broadcast-devices.py`](wifi-broadcast-devices.py). |
|
||||
|
||||
Guest clients get DNS from the gateway (DHCP hands out no custom DNS), so the
|
||||
local record resolves for them. The SSID keeps client isolation on, which only
|
||||
separates guest devices from each other, not from the gateway.
|
||||
|
||||
```bash
|
||||
python3 docs/examples/waters-church-na/zone-allow-policy.py north-attleboro Hotspot 10.1.1.4 80,443 --name "Guest to translate.watersmedia.org (Boba-Fett web)" --apply
|
||||
python3 docs/examples/waters-church-na/guest-restrictions.py north-attleboro --remove 10.0.0.0/8 --apply
|
||||
python3 docs/examples/waters-church-na/wifi-broadcast-devices.py north-attleboro "Waters Guest" --list
|
||||
```
|
||||
|
||||
To undo: delete the allow policy in Settings → Policy Engine (or Firewall) on
|
||||
the NA console, put 10.0.0.0/8 back under the guest hotspot's
|
||||
Post-Authorization Restrictions, and remove the APs with
|
||||
`wifi-broadcast-devices.py --remove`.
|
||||
|
||||
## 4. After the proxy answers
|
||||
|
||||
1. On the NA LAN open `http://10.1.7.39:3100/operator`, enter the operator PIN,
|
||||
and set **Public address** to `http://translate.watersmedia.org` (no trailing
|
||||
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Remove a subnet from the guest "Post-Authorization Restrictions" on one UniFi console.
|
||||
|
||||
UniFi blocks guest (Hotspot zone) clients from the private ranges listed in the
|
||||
guest hotspot's Post-Authorization Restrictions. That list is enforced in two
|
||||
places: as system firewall policies on the gateway (one per zone pair, which a
|
||||
user policy can sit above) and on the access points themselves, which a gateway
|
||||
policy cannot override. To let guests reach one internal host you therefore
|
||||
remove its range from the restriction list and rely on the zone firewall's
|
||||
default Hotspot -> Internal block plus an explicit allow (zone-allow-policy.py).
|
||||
|
||||
Usage:
|
||||
docs/examples/waters-church-na/guest-restrictions.py north-attleboro --show
|
||||
docs/examples/waters-church-na/guest-restrictions.py north-attleboro --remove 10.0.0.0/8 # dry run
|
||||
docs/examples/waters-church-na/guest-restrictions.py north-attleboro --remove 10.0.0.0/8 --apply
|
||||
|
||||
It updates every "Post-Authorization Restrictions" policy whose source is the
|
||||
Hotspot zone through the Network Integration API, then the legacy
|
||||
guest_access setting (restricted_subnet_1..3) that the access points read.
|
||||
|
||||
Needs the per-console Network API key at ~/.config/unifi/<site>-network-api-key
|
||||
(see unifi-dns-record.py). The key is only ever sent in the X-API-KEY header.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
SITES = {
|
||||
"apollo-beach": "10.3.0.1",
|
||||
"north-attleboro": "10.1.0.1",
|
||||
"woonsocket": "10.2.0.1",
|
||||
"norwood": "10.4.0.1",
|
||||
"southcoast": "10.5.0.1",
|
||||
}
|
||||
KEY_DIR = os.path.expanduser("~/.config/unifi")
|
||||
TIMEOUT = 20
|
||||
POLICY_NAME = "Post-Authorization Restrictions"
|
||||
|
||||
|
||||
class Console:
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.gateway = SITES[name]
|
||||
with open(os.path.join(KEY_DIR, f"{name}-network-api-key")) as f:
|
||||
self.key = f.read().strip()
|
||||
self.ctx = ssl.create_default_context()
|
||||
self.ctx.check_hostname = False
|
||||
self.ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def request(self, method: str, path: str, body: dict | None = None):
|
||||
url = f"https://{self.gateway}/proxy/network{path}"
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request(url, data=data, method=method, headers={
|
||||
"X-API-KEY": self.key, "Accept": "application/json",
|
||||
"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT, context=self.ctx) as r:
|
||||
raw = r.read().decode()
|
||||
except urllib.error.HTTPError as e:
|
||||
raise SystemExit(f"{method} {path} -> HTTP {e.code}: {e.read().decode(errors='replace')[:600]}")
|
||||
return json.loads(raw) if raw else None
|
||||
|
||||
def site_id(self) -> str:
|
||||
return self.request("GET", "/integration/v1/sites")["data"][0]["id"]
|
||||
|
||||
def zones(self, sid: str) -> dict[str, str]:
|
||||
return {z["id"]: z["name"] for z in self.request("GET", f"/integration/v1/sites/{sid}/firewall/zones?limit=200")["data"]}
|
||||
|
||||
def restriction_policies(self, sid: str, zones: dict[str, str]) -> list[dict]:
|
||||
pols = self.request("GET", f"/integration/v1/sites/{sid}/firewall/policies?limit=500")["data"]
|
||||
return [p for p in pols if p.get("name") == POLICY_NAME and zones.get(p["source"]["zoneId"]) == "Hotspot"]
|
||||
|
||||
def guest_access(self) -> dict:
|
||||
for s in self.request("GET", "/api/s/default/get/setting")["data"]:
|
||||
if s.get("key") == "guest_access":
|
||||
return s
|
||||
raise SystemExit("no guest_access setting found")
|
||||
|
||||
|
||||
def items_of(p: dict) -> list[str]:
|
||||
f = (p.get("destination") or {}).get("trafficFilter") or {}
|
||||
return [i["value"] for i in (f.get("ipAddressFilter") or {}).get("items", [])]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("site", choices=sorted(SITES))
|
||||
ap.add_argument("--show", action="store_true", help="print the current restrictions and exit")
|
||||
ap.add_argument("--remove", help="subnet to remove, e.g. 10.0.0.0/8")
|
||||
ap.add_argument("--apply", action="store_true", help="actually write the change")
|
||||
a = ap.parse_args()
|
||||
|
||||
c = Console(a.site)
|
||||
sid = c.site_id()
|
||||
zones = c.zones(sid)
|
||||
pols = c.restriction_policies(sid, zones)
|
||||
ga = c.guest_access()
|
||||
legacy = {k: v for k, v in ga.items() if k.startswith("restricted_subnet_")}
|
||||
|
||||
print(f"{a.site} ({c.gateway}) guest Post-Authorization Restrictions:")
|
||||
for p in pols:
|
||||
print(f" gateway policy Hotspot -> {zones.get(p['destination']['zoneId']):8} index {p['index']}: {', '.join(items_of(p))}")
|
||||
print(f" legacy guest_access (access points): {json.dumps(legacy)}")
|
||||
if a.show or not a.remove:
|
||||
return 0
|
||||
|
||||
targets = [p for p in pols if a.remove in items_of(p)]
|
||||
legacy_hits = {k: v for k, v in legacy.items() if v == a.remove}
|
||||
if not targets and not legacy_hits:
|
||||
print(f"\n{a.remove} is not in any restriction list; nothing to do")
|
||||
return 0
|
||||
print(f"\nwould remove {a.remove} from legacy field(s) {list(legacy_hits)}; the {len(targets)} gateway policy(ies) listing it regenerate from that setting")
|
||||
if not a.apply:
|
||||
print("DRY RUN, add --apply to write it")
|
||||
return 0
|
||||
|
||||
# The gateway policies are SYSTEM_DEFINED and the Integration API refuses to
|
||||
# edit them (api.firewall.policy.operation.system-defined-firewall-policy-update-forbidden).
|
||||
# They are generated from the legacy guest_access restricted_subnet_* fields, so
|
||||
# that setting is the one lever; the policies and the APs follow it.
|
||||
if not legacy_hits:
|
||||
print(" no legacy restricted_subnet field holds it; the gateway policies cannot be edited directly")
|
||||
return 1
|
||||
# Re-pack the remaining subnets into slots 1..n and blank the rest.
|
||||
remaining = [v for k, v in sorted(legacy.items()) if v and v != a.remove]
|
||||
payload = {f"restricted_subnet_{i}": (remaining[i - 1] if i <= len(remaining) else "") for i in range(1, 4)}
|
||||
print(f" POST /api/s/default/set/setting/guest_access {json.dumps(payload)}")
|
||||
c.request("POST", "/api/s/default/set/setting/guest_access", payload)
|
||||
ga2 = c.guest_access()
|
||||
print(" legacy guest_access now:", json.dumps({k: v for k, v in ga2.items() if k.startswith("restricted_subnet_")}))
|
||||
for p in c.restriction_policies(sid, zones):
|
||||
print(f" gateway policy Hotspot -> {zones.get(p['destination']['zoneId']):8} now: {', '.join(items_of(p))}")
|
||||
print("done; access points re-provision within a minute, guests should reconnect")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Change which access points broadcast a WiFi network (SSID) on one UniFi console.
|
||||
|
||||
Usage:
|
||||
docs/examples/waters-church-na/wifi-broadcast-devices.py north-attleboro "Waters Guest" --list
|
||||
docs/examples/waters-church-na/wifi-broadcast-devices.py north-attleboro "Waters Guest" \
|
||||
--add "Auditorium Left,Auditorium Right,Main Lobby Entrance" # dry run
|
||||
docs/examples/waters-church-na/wifi-broadcast-devices.py north-attleboro "Waters Guest" \
|
||||
--add "Auditorium Left,Auditorium Right,Main Lobby Entrance" --apply # write it
|
||||
docs/examples/waters-church-na/wifi-broadcast-devices.py north-attleboro "Waters Guest" --remove "The Vault" --apply
|
||||
docs/examples/waters-church-na/wifi-broadcast-devices.py north-attleboro "Waters Guest" --all --apply # every AP
|
||||
|
||||
Names are matched exactly against the console's device names (--list prints
|
||||
them). Without --apply nothing is written: the script prints the APs the SSID
|
||||
is on now and the list it would send. The whole broadcast object is PUT back
|
||||
with only broadcastingDeviceFilter changed.
|
||||
|
||||
Needs the per-console Network API key at ~/.config/unifi/<site>-network-api-key
|
||||
(see unifi-dns-record.py). The key is only ever sent in the X-API-KEY header.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
SITES = {
|
||||
"apollo-beach": "10.3.0.1",
|
||||
"north-attleboro": "10.1.0.1",
|
||||
"woonsocket": "10.2.0.1",
|
||||
"norwood": "10.4.0.1",
|
||||
"southcoast": "10.5.0.1",
|
||||
}
|
||||
KEY_DIR = os.path.expanduser("~/.config/unifi")
|
||||
TIMEOUT = 20
|
||||
|
||||
|
||||
class Console:
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.gateway = SITES[name]
|
||||
with open(os.path.join(KEY_DIR, f"{name}-network-api-key")) as f:
|
||||
self.key = f.read().strip()
|
||||
self.ctx = ssl.create_default_context()
|
||||
self.ctx.check_hostname = False
|
||||
self.ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def request(self, method: str, path: str, body: dict | None = None):
|
||||
url = f"https://{self.gateway}/proxy/network/integration/v1{path}"
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request(url, data=data, method=method, headers={
|
||||
"X-API-KEY": self.key, "Accept": "application/json",
|
||||
"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT, context=self.ctx) as r:
|
||||
raw = r.read().decode()
|
||||
except urllib.error.HTTPError as e:
|
||||
raise SystemExit(f"{method} {path} -> HTTP {e.code}: {e.read().decode(errors='replace')[:600]}")
|
||||
return json.loads(raw) if raw else None
|
||||
|
||||
def site_id(self) -> str:
|
||||
return self.request("GET", "/sites")["data"][0]["id"]
|
||||
|
||||
def access_points(self, sid: str) -> dict[str, str]:
|
||||
devs = self.request("GET", f"/sites/{sid}/devices?limit=200")["data"]
|
||||
return {d["name"]: d["id"] for d in devs if "accessPoint" in (d.get("features") or [])}
|
||||
|
||||
def broadcast(self, sid: str, ssid: str) -> dict:
|
||||
for b in self.request("GET", f"/sites/{sid}/wifi/broadcasts?limit=200")["data"]:
|
||||
if b.get("name") == ssid:
|
||||
return b
|
||||
raise SystemExit(f"no WiFi network named {ssid!r} on {self.name}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("site", choices=sorted(SITES))
|
||||
ap.add_argument("ssid", help="WiFi network name as shown in the console")
|
||||
ap.add_argument("--add", default="", help="comma-separated AP names to add")
|
||||
ap.add_argument("--remove", default="", help="comma-separated AP names to remove")
|
||||
ap.add_argument("--all", action="store_true", help="broadcast on every AP (clears the device filter)")
|
||||
ap.add_argument("--list", action="store_true", help="print the APs and where the SSID broadcasts, then exit")
|
||||
ap.add_argument("--apply", action="store_true", help="actually write the change")
|
||||
a = ap.parse_args()
|
||||
|
||||
c = Console(a.site)
|
||||
sid = c.site_id()
|
||||
aps = c.access_points(sid)
|
||||
by_id = {v: k for k, v in aps.items()}
|
||||
b = c.broadcast(sid, a.ssid)
|
||||
filt = b.get("broadcastingDeviceFilter") or {}
|
||||
current = list(filt.get("deviceIds", [])) if filt.get("type") == "DEVICES" else None
|
||||
|
||||
def names(ids):
|
||||
return ", ".join(by_id.get(i, i) for i in ids)
|
||||
|
||||
print(f"{a.site}: {a.ssid!r} broadcasts on " + ("ALL access points" if current is None else f"{len(current)} AP(s): {names(current)}"))
|
||||
if a.list:
|
||||
print("\nAccess points on this console:")
|
||||
for n in sorted(aps):
|
||||
print(f" {'*' if current is None or aps[n] in current else ' '} {n}")
|
||||
return 0
|
||||
|
||||
add = [s.strip() for s in a.add.split(",") if s.strip()]
|
||||
rem = [s.strip() for s in a.remove.split(",") if s.strip()]
|
||||
unknown = [n for n in add + rem if n not in aps]
|
||||
if unknown:
|
||||
raise SystemExit(f"unknown AP name(s): {', '.join(unknown)}; use --list")
|
||||
if not (add or rem or a.all):
|
||||
ap.error("nothing to change: give --add, --remove or --all")
|
||||
|
||||
new_b = {k: v for k, v in b.items() if k not in ("id", "metadata")}
|
||||
if a.all:
|
||||
new_b.pop("broadcastingDeviceFilter", None)
|
||||
summary = "ALL access points"
|
||||
else:
|
||||
ids = list(current or [])
|
||||
for n in add:
|
||||
if aps[n] not in ids:
|
||||
ids.append(aps[n])
|
||||
ids = [i for i in ids if i not in {aps[n] for n in rem}]
|
||||
new_b["broadcastingDeviceFilter"] = {"type": "DEVICES", "deviceIds": ids}
|
||||
summary = f"{len(ids)} AP(s): {names(ids)}"
|
||||
print(f"would broadcast on {summary}")
|
||||
if not a.apply:
|
||||
print("DRY RUN, add --apply to write it")
|
||||
return 0
|
||||
out = c.request("PUT", f"/sites/{sid}/wifi/broadcasts/{b['id']}", new_b)
|
||||
f2 = (out or {}).get("broadcastingDeviceFilter") or {}
|
||||
print("UPDATED: now on " + ("ALL access points" if f2.get("type") != "DEVICES" else names(f2.get("deviceIds", []))))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create a zone-based firewall policy on one UniFi console that allows TCP from a
|
||||
source zone to one destination IP on given ports (e.g. guest WiFi -> a LAN web host).
|
||||
|
||||
Usage:
|
||||
docs/examples/waters-church-na/zone-allow-policy.py north-attleboro Hotspot 10.1.1.4 80,443 \
|
||||
--name "Guest to translate.watersmedia.org (Boba-Fett web)" # dry run
|
||||
docs/examples/waters-church-na/zone-allow-policy.py north-attleboro Hotspot 10.1.1.4 80,443 \
|
||||
--name "Guest to translate.watersmedia.org (Boba-Fett web)" --apply # create it
|
||||
docs/examples/waters-church-na/zone-allow-policy.py north-attleboro --list-zones
|
||||
|
||||
Without --apply nothing is written: the script prints the zones it resolved, any
|
||||
existing policy with the same name, and the exact JSON it would POST. The
|
||||
destination zone is found from the console's networks (the one whose subnet
|
||||
contains the IP). User-defined policies are evaluated before the system ones
|
||||
(index >= 30000), so the allow lands above "Post-Authorization Restrictions".
|
||||
|
||||
Needs the per-console Network API key at ~/.config/unifi/<site>-network-api-key
|
||||
(see unifi-dns-record.py). The key is only ever sent in the X-API-KEY header.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import ipaddress
|
||||
import json
|
||||
import os
|
||||
import ssl
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
SITES = {
|
||||
"apollo-beach": "10.3.0.1",
|
||||
"north-attleboro": "10.1.0.1",
|
||||
"woonsocket": "10.2.0.1",
|
||||
"norwood": "10.4.0.1",
|
||||
"southcoast": "10.5.0.1",
|
||||
}
|
||||
KEY_DIR = os.path.expanduser("~/.config/unifi")
|
||||
TIMEOUT = 20
|
||||
|
||||
|
||||
class Console:
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
self.gateway = SITES[name]
|
||||
with open(os.path.join(KEY_DIR, f"{name}-network-api-key")) as f:
|
||||
self.key = f.read().strip()
|
||||
self.ctx = ssl.create_default_context()
|
||||
self.ctx.check_hostname = False
|
||||
self.ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def request(self, method: str, path: str, body: dict | None = None):
|
||||
url = f"https://{self.gateway}/proxy/network{path}"
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request(url, data=data, method=method, headers={
|
||||
"X-API-KEY": self.key, "Accept": "application/json",
|
||||
"Content-Type": "application/json"})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT, context=self.ctx) as r:
|
||||
raw = r.read().decode()
|
||||
except urllib.error.HTTPError as e:
|
||||
raise SystemExit(f"{method} {path} -> HTTP {e.code}: {e.read().decode(errors='replace')[:600]}")
|
||||
return json.loads(raw) if raw else None
|
||||
|
||||
def site_id(self) -> str:
|
||||
return self.request("GET", "/integration/v1/sites")["data"][0]["id"]
|
||||
|
||||
def zones(self, sid: str) -> dict[str, str]:
|
||||
data = self.request("GET", f"/integration/v1/sites/{sid}/firewall/zones?limit=200")["data"]
|
||||
return {z["name"]: z["id"] for z in data}
|
||||
|
||||
def zone_for_ip(self, sid: str, ip: str) -> tuple[str, str]:
|
||||
"""Zone id and network name whose subnet contains ip (via the legacy networkconf)."""
|
||||
nets = {n["id"]: n for n in self.request("GET", f"/integration/v1/sites/{sid}/networks?limit=200")["data"]}
|
||||
addr = ipaddress.ip_address(ip)
|
||||
for n in self.request("GET", "/api/s/default/rest/networkconf")["data"]:
|
||||
subnet = n.get("ip_subnet")
|
||||
if not subnet:
|
||||
continue
|
||||
if addr in ipaddress.ip_interface(subnet).network:
|
||||
zone_id = nets.get(n.get("external_id", ""), {}).get("zoneId")
|
||||
if zone_id:
|
||||
return zone_id, n["name"]
|
||||
raise SystemExit(f"no gateway network on {self.name} contains {ip}")
|
||||
|
||||
def policies(self, sid: str) -> list[dict]:
|
||||
return self.request("GET", f"/integration/v1/sites/{sid}/firewall/policies?limit=500")["data"]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("site", choices=sorted(SITES))
|
||||
ap.add_argument("source_zone", nargs="?", help="source zone name, e.g. Hotspot")
|
||||
ap.add_argument("dest_ip", nargs="?", help="destination IPv4 address")
|
||||
ap.add_argument("ports", nargs="?", help="comma-separated TCP ports, e.g. 80,443")
|
||||
ap.add_argument("--name", help="policy name (default is generated)")
|
||||
ap.add_argument("--apply", action="store_true", help="actually create the policy")
|
||||
ap.add_argument("--list-zones", action="store_true", help="print the console's zones and exit")
|
||||
a = ap.parse_args()
|
||||
|
||||
c = Console(a.site)
|
||||
sid = c.site_id()
|
||||
zones = c.zones(sid)
|
||||
if a.list_zones:
|
||||
for name, zid in sorted(zones.items()):
|
||||
print(f"{name:12} {zid}")
|
||||
return 0
|
||||
if not (a.source_zone and a.dest_ip and a.ports):
|
||||
ap.error("source_zone, dest_ip and ports are required unless --list-zones")
|
||||
if a.source_zone not in zones:
|
||||
raise SystemExit(f"zone {a.source_zone!r} not on {a.site}; have: {', '.join(sorted(zones))}")
|
||||
ports = [int(p) for p in a.ports.split(",")]
|
||||
dest_zone, dest_net = c.zone_for_ip(sid, a.dest_ip)
|
||||
dest_zone_name = next(n for n, z in zones.items() if z == dest_zone)
|
||||
name = a.name or f"{a.source_zone} to {a.dest_ip} tcp {a.ports}"
|
||||
|
||||
body = {
|
||||
"name": name,
|
||||
"enabled": True,
|
||||
"action": {"type": "ALLOW", "allowReturnTraffic": True},
|
||||
"source": {"zoneId": zones[a.source_zone]},
|
||||
"destination": {
|
||||
"zoneId": dest_zone,
|
||||
"trafficFilter": {
|
||||
"type": "IP_ADDRESS",
|
||||
"ipAddressFilter": {"type": "IP_ADDRESSES", "matchOpposite": False,
|
||||
"items": [{"type": "IP_ADDRESS", "value": a.dest_ip}]},
|
||||
"portFilter": {"type": "PORTS", "matchOpposite": False,
|
||||
"items": [{"type": "PORT_NUMBER", "value": p} for p in ports]},
|
||||
},
|
||||
},
|
||||
"ipProtocolScope": {"ipVersion": "IPV4",
|
||||
"protocolFilter": {"type": "NAMED_PROTOCOL", "protocol": {"name": "TCP"},
|
||||
"matchOpposite": False}},
|
||||
"loggingEnabled": False,
|
||||
}
|
||||
|
||||
print(f"{a.site} ({c.gateway}): {a.source_zone} -> {dest_zone_name} ({dest_net}) {a.dest_ip} tcp {a.ports}")
|
||||
existing = [p for p in c.policies(sid) if p.get("name") == name]
|
||||
if existing:
|
||||
print(f"a policy named {name!r} already exists (id {existing[0]['id']}, index {existing[0].get('index')}); nothing to do")
|
||||
return 0
|
||||
if not a.apply:
|
||||
print("DRY RUN, would POST /integration/v1/sites/<site>/firewall/policies:")
|
||||
print(json.dumps(body, indent=2))
|
||||
return 0
|
||||
created = c.request("POST", f"/integration/v1/sites/{sid}/firewall/policies", body)
|
||||
print(f"CREATED id {created.get('id')} index {created.get('index')} enabled {created.get('enabled')}")
|
||||
blockers = [p for p in c.policies(sid) if p.get("name") == "Post-Authorization Restrictions"
|
||||
and p["source"]["zoneId"] == zones[a.source_zone] and p["destination"]["zoneId"] == dest_zone]
|
||||
for b in blockers:
|
||||
order = "above" if created.get("index", 0) < b.get("index", 0) else "BELOW"
|
||||
print(f"new policy sits {order} 'Post-Authorization Restrictions' (index {b.get('index')})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user