Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
826c471ed7 | ||
|
|
817fb743de | ||
|
|
d289d33aab | ||
|
|
d785ebf979 | ||
|
|
a06cf35cbb | ||
|
|
39dea80433 | ||
|
|
6f91bc1357 |
+178
-20
@@ -190,7 +190,9 @@ const state = {
|
||||
|
||||
/* health counters, reported to the operator page */
|
||||
degradedFinals: 0, // finals that shipped with fewer languages (stale)
|
||||
captionOnlyFinals: 0 // finals that shipped with no translations at all
|
||||
captionOnlyFinals: 0, // finals that shipped with no translations at all
|
||||
captionOnlyAt: 0, // when the last one happened
|
||||
degradedAt: 0
|
||||
};
|
||||
|
||||
// ────────────────────────────────────────────────────────── logging
|
||||
@@ -891,12 +893,54 @@ async function respawnWorker(worker) {
|
||||
log('the speech model is too heavy for this machine — switching to ' +
|
||||
lighter.split('/').pop(), 'err');
|
||||
}
|
||||
/*
|
||||
* A LIGHTER MODEL GETS THE PREVIEW'S DTYPE, NOT THE BIG ONE'S.
|
||||
*
|
||||
* asrDtype is chosen for the accurate model and its weights are cached against
|
||||
* it. Reusing it when stepping down asks for a variant that was never
|
||||
* downloaded — and those weights live in the browser cache, with no network at
|
||||
* a venue — so the step-down failed and the ladder could not move at all.
|
||||
*
|
||||
* MEASURED on a campus Mac: the cache held whisper-large-v3-turbo as
|
||||
* encoder_model_q4f16.onnx (its asrDtype) and whisper-small as
|
||||
* encoder_model_fp16.onnx (the preview's). Stepping down asked for
|
||||
* whisper-small at q4f16, a file that is not there; it went to the network,
|
||||
* failed, and logged "Unsupported model type: whisper" three times a minute
|
||||
* for ever while the service sat dead.
|
||||
*
|
||||
* The preview runs whisper-small on exactly these settings every service, so
|
||||
* they are the one combination this machine is known to be able to load.
|
||||
*/
|
||||
const stepDtype = lighter
|
||||
? resolveDtype(cfg.asrFastDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' })
|
||||
: resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' });
|
||||
state.asrWorker = await loadWorkerWithFallback('speech', 'automatic-speech-recognition',
|
||||
state.asrModelInUse || cfg.asrModel,
|
||||
resolveDtype(cfg.asrDtype, { encoder_model: 'fp16', decoder_model_merged: 'q4' }),
|
||||
stepDtype,
|
||||
{ encoder_model: 'q4', decoder_model_merged: 'q4' });
|
||||
log('speech model restarted' + (lighter ? ' on ' + lighter.split('/').pop() : ''), 'ok');
|
||||
state.slowRun = 0;
|
||||
if (lighter) {
|
||||
/*
|
||||
* THROW AWAY THE BACKLOG THAT CAUSED THIS, AND LET THE NEW MODEL SETTLE.
|
||||
*
|
||||
* Swapping the model does nothing about the queue that triggered the swap.
|
||||
* Those chunks are already past the caption-only cutoff, so they can no
|
||||
* longer be translated in time and are worth nothing to the room — but the
|
||||
* first of them gets handed to a model that has just loaded, takes the full
|
||||
* worker timeout, and trips the absolute rule. MEASURED: a campus Mac fell
|
||||
* large-v3-turbo -> small -> base in 34 seconds exactly that way, landing on
|
||||
* a model SLOWER than the one it started from.
|
||||
*
|
||||
* So drop them, and refuse to step down again for a grace window, so the new
|
||||
* model is judged on sentences it was actually given.
|
||||
*/
|
||||
const dropped = queue.length;
|
||||
queue.length = 0;
|
||||
state.steppedDownAt = Date.now();
|
||||
if (dropped) log('dropped ' + dropped + ' stale sentence' + (dropped === 1 ? '' : 's') +
|
||||
' the old model never reached', 'warn');
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
log('model restart failed: ' + err.message, 'err');
|
||||
@@ -1849,15 +1893,33 @@ async function pump() {
|
||||
const sentences = collapseLoops(splitSentences(text));
|
||||
|
||||
let targets = orderedTargets();
|
||||
if (ageSec > 12) {
|
||||
/*
|
||||
* SKIPPING A TRANSLATION ONLY BUYS SOMETHING IF THERE IS A QUEUE TO DRAIN.
|
||||
*
|
||||
* Dropping the translations of stale audio is how the app catches up: the
|
||||
* listener has moved on, so the work is wasted and the time is better spent on
|
||||
* the sentences behind it. That reasoning needs a backlog. With nothing waiting
|
||||
* behind it, skipping costs the room a sentence and saves nothing at all.
|
||||
*
|
||||
* MEASURED: a campus Mac took one 16 s stall — two Screen Sharing sessions
|
||||
* competing for the machine — and skipped that sentence's translations with an
|
||||
* EMPTY queue. The room lost a line of the sermon to a catch-up that had
|
||||
* nothing to catch up on.
|
||||
*/
|
||||
if (ageSec > 12 && queue.length > 0) {
|
||||
targets = [];
|
||||
state.captionOnlyFinals++;
|
||||
log('audio ' + ageSec.toFixed(0) + 's stale — caption only, translations skipped', 'err');
|
||||
state.captionOnlyAt = Date.now();
|
||||
log('audio ' + ageSec.toFixed(0) + 's stale with ' + queue.length + ' waiting — ' +
|
||||
'caption only, translations skipped', 'err');
|
||||
} else if (ageSec > 12) {
|
||||
/* Late, but nothing is behind it: say so, and translate it anyway. */
|
||||
log('audio ' + ageSec.toFixed(0) + 's late, but nothing is queued — translating it anyway', 'warn');
|
||||
} else if (ageSec > 6) {
|
||||
const listening = new Set((state.byLanguage || []).map((r) => r.lang));
|
||||
const before = targets.length;
|
||||
targets = targets.filter((t) => listening.has(t));
|
||||
if (targets.length < before) state.degradedFinals++;
|
||||
if (targets.length < before) { state.degradedFinals++; state.degradedAt = Date.now(); }
|
||||
}
|
||||
|
||||
/* The segment's own loudness, measured once and shared by every sentence
|
||||
@@ -1931,23 +1993,36 @@ async function pump() {
|
||||
* drains does not recover on its own.
|
||||
*/
|
||||
/*
|
||||
* 0.85 and a backlog of five, both measured rather than chosen.
|
||||
* SLOWER THAN THE AUDIO IT COVERS, AND LANDING LATE.
|
||||
*
|
||||
* The Mini's sentences ran 4.4 s against 4.5 s of audio — nominally FASTER
|
||||
* than real time, while the queue sat at 46 and the lag climbed past 150 s.
|
||||
* Per-sentence time alone therefore misses it: the work outside this timer
|
||||
* (segmenting, the preview model sharing the GPU, the scheduler) is enough
|
||||
* to lose on its own. The margin catches "near enough to real time to be
|
||||
* losing"; the queue depth is the proof that it actually is, and keeps one
|
||||
* slow sentence on a healthy machine from touching anything.
|
||||
* This used to fire only past 8 s, on the reasoning that 8 s meant slower
|
||||
* than people talk. That held when a chunk could be 8 s long; chunks are
|
||||
* capped at HARD_CUT_MAX, 5.5 s, so a 4.4 s sentence is already losing and
|
||||
* never came near the old threshold.
|
||||
*
|
||||
* Both numbers below were measured on a campus M4 Pro, and the second one
|
||||
* only after a first attempt got it wrong. 0.85, because that Mac ran 4.4 s
|
||||
* against 4.5 s of audio — nominally FASTER than real time while losing
|
||||
* badly, since the work outside this timer (segmenting, the preview model
|
||||
* sharing the GPU, the scheduler) loses on its own.
|
||||
*
|
||||
* The second test was a queue depth, and watching that Mac go red disproved
|
||||
* it: the queue sat at 2-3 while the lag held at 15 s and the verdict was
|
||||
* already red. A machine can be persistently late without ever stacking work
|
||||
* up — it finishes each sentence just slightly too slowly, for ever. So the
|
||||
* test is how far behind the speaker the sentence actually landed, which is
|
||||
* what the operator is complaining about in the first place.
|
||||
*/
|
||||
const losingGround = jobAudioMs > 0 && took > jobAudioMs * 0.85 && queue.length >= 5;
|
||||
if (took > 8000 || losingGround) {
|
||||
/* A model that loaded seconds ago has not had a fair sentence yet. */
|
||||
const settling = state.steppedDownAt && (Date.now() - state.steppedDownAt) < STEP_SETTLE_MS;
|
||||
const behindMs = Date.now() - job.at;
|
||||
const losingGround = jobAudioMs > 0 && took > jobAudioMs * 0.85 && behindMs > 8000;
|
||||
if (!settling && (took > 8000 || losingGround)) {
|
||||
state.slowRun = (state.slowRun || 0) + 1;
|
||||
if (state.slowRun >= 3 && !respawning && lighterAsr(state.asrModelInUse || state.cfg.asrModel)) {
|
||||
state.slowRun = 0;
|
||||
log('transcription is slower than people are speaking (' + took + ' ms for ' +
|
||||
Math.round(jobAudioMs) + ' ms of audio, ' + queue.length + ' waiting) — ' +
|
||||
Math.round(jobAudioMs) + ' ms of audio, ' + Math.round(behindMs / 1000) + 's behind) — ' +
|
||||
'stepping down to a lighter model', 'err');
|
||||
respawnWorker(state.asrWorker);
|
||||
}
|
||||
@@ -2459,6 +2534,11 @@ function monitorTranslation(cid, lang, text) {
|
||||
* It says it once per episode, and names both numbers so the operator can act
|
||||
* on it rather than go looking.
|
||||
*/
|
||||
/* How long a freshly swapped speech model is left alone. Long enough to cover a
|
||||
cold first inference, short enough that a genuinely hopeless model still falls
|
||||
through the ladder within a sentence or two. */
|
||||
const STEP_SETTLE_MS = 20000;
|
||||
|
||||
const DEAF_AFTER_MS = 90000;
|
||||
setInterval(() => {
|
||||
try {
|
||||
@@ -2698,6 +2778,7 @@ function goLive(on) {
|
||||
state.live = on;
|
||||
if (on) {
|
||||
state.degradedFinals = 0; state.captionOnlyFinals = 0; state.fragmentFinals = 0;
|
||||
state.captionOnlyAt = 0; state.degradedAt = 0;
|
||||
/* The watchdog measures from the start of the service, not from the last
|
||||
sentence of the previous one. */
|
||||
state.lastFinalAt = Date.now();
|
||||
@@ -2772,6 +2853,10 @@ function reportStatus() {
|
||||
queue: queue.length,
|
||||
degradedFinals: state.degradedFinals,
|
||||
captionOnlyFinals: state.captionOnlyFinals,
|
||||
/* WHEN, not just how many — the verdict needs to know whether this is
|
||||
happening now or happened once an hour ago. */
|
||||
captionOnlyAt: state.captionOnlyAt || 0,
|
||||
degradedAt: state.degradedAt || 0,
|
||||
fragmentFinals: state.fragmentFinals || 0,
|
||||
cloud: state.cloudStats || null
|
||||
}
|
||||
@@ -3423,14 +3508,50 @@ function shareChoice() {
|
||||
};
|
||||
}
|
||||
|
||||
function shareBase() {
|
||||
/* This Mac on the church network. The operator page lives here and NOWHERE
|
||||
else: it returns 404 to anything arriving through a proxy, by design. */
|
||||
function lanBase() {
|
||||
const c = shareChoice();
|
||||
if (!c || c.none) return '';
|
||||
return 'http://' + c.addr.address + ':' + state.local.port;
|
||||
}
|
||||
|
||||
/*
|
||||
* WHAT THE CONGREGATION OPENS.
|
||||
*
|
||||
* A church that has put this on a domain has said so in the operator page's
|
||||
* Public address, and from that moment the LAN address is the wrong thing to
|
||||
* print: it is not what is on the poster, not what the QR should encode, and
|
||||
* not what someone off the building's wifi can reach. The app window went on
|
||||
* showing the IP anyway, because this half of it never read that setting --
|
||||
* the operator page honoured it and the main window did not, so the two
|
||||
* disagreed about the one fact they both exist to display.
|
||||
*/
|
||||
function publicBaseConfigured() {
|
||||
const u = opConfig && typeof opConfig.publicUrl === 'string' ? opConfig.publicUrl.trim() : '';
|
||||
return u ? u.replace(/\/+$/, '') : '';
|
||||
}
|
||||
|
||||
function shareBase() {
|
||||
return publicBaseConfigured() || lanBase();
|
||||
}
|
||||
|
||||
function renderQrWarning() {
|
||||
const el = $('qrWarn');
|
||||
/*
|
||||
* On a domain the warnings below are about an address nobody is being
|
||||
* given any more. Saying "Wi-Fi is down, phones cannot reach the code"
|
||||
* while the code points at a proxy would send the desk after the wrong
|
||||
* thing entirely.
|
||||
*/
|
||||
const pub = publicBaseConfigured();
|
||||
if (pub) {
|
||||
el.hidden = false;
|
||||
el.textContent = 'The code points at ' + pub + ', set as the public address. ' +
|
||||
'This Mac still has to be reachable from there; its own address is ' +
|
||||
(lanBase() || 'unavailable') + '.';
|
||||
return;
|
||||
}
|
||||
const c = shareChoice();
|
||||
if (!c) { el.hidden = true; return; }
|
||||
|
||||
@@ -3481,6 +3602,26 @@ function renderShare() {
|
||||
const list = state.local.addresses || [];
|
||||
const box = $('addrList');
|
||||
box.textContent = '';
|
||||
/*
|
||||
* WHICH INTERFACE THE QR USES IS A QUESTION ONLY A LAN SETUP HAS.
|
||||
*
|
||||
* These rows let the operator pick the address the code points at. Once a
|
||||
* public address is configured the code points at the domain whatever they
|
||||
* pick, so the list is a row of numbers that do nothing — and the one
|
||||
* marked IN USE is not the address anyone is being given. Hide it.
|
||||
*/
|
||||
const addrBox = $('addrList');
|
||||
addrBox.hidden = !!publicBaseConfigured();
|
||||
if (addrBox.hidden) {
|
||||
/* Still draw the code itself — it is the only thing left in this panel. */
|
||||
const only = shareBase();
|
||||
$('qr').src = only
|
||||
? 'http://127.0.0.1:' + state.local.port + '/qr.svg?url=' + encodeURIComponent(only + '/')
|
||||
: '';
|
||||
renderQrWarning();
|
||||
renderOperatorUrl();
|
||||
return;
|
||||
}
|
||||
|
||||
/* Mark the row the QR is actually using, not the row that was saved — when
|
||||
a lease moves those are two different things and the saved one is gone. */
|
||||
@@ -3522,8 +3663,22 @@ $('copyAddr').addEventListener('click', () => {
|
||||
.catch((err) => log('could not copy: ' + err.message, 'err'));
|
||||
});
|
||||
|
||||
$('openPoster').addEventListener('click', () => {
|
||||
window.open('http://127.0.0.1:' + state.local.port + '/poster', '_blank');
|
||||
/*
|
||||
* The QR itself, saved as a file, because the poster people actually print
|
||||
* is made in their own design tool. The built-in poster page is still there
|
||||
* at /poster for anyone who wants it; it is just not what this button does.
|
||||
*/
|
||||
$('saveQr').addEventListener('click', () => {
|
||||
const base = shareBase();
|
||||
if (!base) { log('no address to encode yet', 'err'); return; }
|
||||
const a = document.createElement('a');
|
||||
a.href = 'http://127.0.0.1:' + state.local.port + '/qr.svg?url=' +
|
||||
encodeURIComponent(base + '/');
|
||||
a.download = 'church-translate-qr.svg';
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
a.remove();
|
||||
log('saved the QR for ' + base + '/', 'ok');
|
||||
});
|
||||
$('openOperator').addEventListener('click', () => {
|
||||
window.open('http://127.0.0.1:' + state.local.port + '/operator', '_blank');
|
||||
@@ -3580,7 +3735,8 @@ function renderBuildTag() {
|
||||
}
|
||||
|
||||
function renderOperatorUrl() {
|
||||
const base = shareBase();
|
||||
/* Deliberately lanBase, not shareBase: over a proxy /operator is a 404. */
|
||||
const base = lanBase();
|
||||
$('opPageUrl').textContent = base ? base + '/operator' : 'no network';
|
||||
}
|
||||
|
||||
@@ -3883,6 +4039,7 @@ async function loadOperator() {
|
||||
try {
|
||||
const j = await opApi('/api/config');
|
||||
opConfig = j.config;
|
||||
renderShare(); /* publicUrl may have just arrived */
|
||||
clearTimeout(opLoadTimer); opLoadTimer = null; // it answered; stop retrying
|
||||
opRetryMs = 0;
|
||||
renderOpLanguages();
|
||||
@@ -3937,6 +4094,7 @@ async function opSave(patch, okMsg) {
|
||||
try {
|
||||
const j = await opApi('/api/config', { method: 'POST', body: JSON.stringify(patch) });
|
||||
opConfig = j.config;
|
||||
renderShare(); /* publicUrl may have just arrived */
|
||||
if (okMsg) log(okMsg, 'ok');
|
||||
return j;
|
||||
} catch (e) { log('could not save: ' + e.message, 'err'); throw e; }
|
||||
|
||||
+1
-1
@@ -433,7 +433,7 @@
|
||||
<div id="addrList" class="addrlist"></div>
|
||||
<div class="railbtns">
|
||||
<button class="btn btn-primary" id="copyAddr">Copy address</button>
|
||||
<button class="btn" id="openPoster">Printable poster</button>
|
||||
<button class="btn" id="saveQr">Save QR image</button>
|
||||
</div>
|
||||
|
||||
<h3 class="subhead">Operator page</h3>
|
||||
|
||||
+2
-2
@@ -2,7 +2,7 @@
|
||||
"name": "church-translate-engine",
|
||||
"productName": "Church Translate",
|
||||
"version": "3.23.2",
|
||||
"buildNumber": 117,
|
||||
"buildNumber": 123,
|
||||
"private": true,
|
||||
"description": "Live sermon translation — self-contained: captures audio, transcribes, translates, and serves the congregation page itself",
|
||||
"main": "main.js",
|
||||
@@ -106,7 +106,7 @@
|
||||
]
|
||||
},
|
||||
"asar": false,
|
||||
"buildVersion": "117",
|
||||
"buildVersion": "123",
|
||||
"win": {
|
||||
"target": [
|
||||
{
|
||||
|
||||
+24
-2
@@ -510,6 +510,11 @@ const ENGINE_STALE_MS = 12000;
|
||||
* A campus reported "constantly behind, in the red" and the cause took an
|
||||
* afternoon to find; it was eight languages on hardware tuned against one.
|
||||
*/
|
||||
/* How long a stumble keeps colouring the verdict. Long enough that a real
|
||||
problem stays visible while someone walks to the desk, short enough that a
|
||||
one-off does not outlive the sentence after it. */
|
||||
const TROUBLE_WINDOW_MS = 120000;
|
||||
|
||||
function loadNote() {
|
||||
const n = activeLanguages().length;
|
||||
if (n < 2) return '';
|
||||
@@ -523,10 +528,27 @@ function healthVerdict(connected, stale) {
|
||||
if (stale) return { level: 'red', detail: 'The translation app has stopped responding.' };
|
||||
if (!state.live) return { level: 'green', detail: 'Ready.' };
|
||||
if (!es.modelsReady) return { level: 'amber', detail: 'Models are still loading.' };
|
||||
if ((es.lagMs || 0) > 12000 || (es.captionOnlyFinals || 0) > 0)
|
||||
/*
|
||||
* RECENTLY, not ever.
|
||||
*
|
||||
* captionOnlyFinals is a running total for the whole service, and testing it
|
||||
* for "more than zero" meant a single stale sentence pinned the desk's light
|
||||
* red until the next restart. MEASURED: a campus Mac took one 16 s stall at
|
||||
* 14:25 — caused by two Screen Sharing sessions competing for the machine —
|
||||
* recovered immediately to a 421 ms lag with an empty queue, and still showed
|
||||
* red half an hour later. An operator who looks up and sees red wants to know
|
||||
* what is wrong NOW; a light that cannot go back to green teaches them to
|
||||
* ignore it.
|
||||
*
|
||||
* So the totals still travel, for the record and for the operator page, but
|
||||
* the verdict asks when it last happened.
|
||||
*/
|
||||
const recentCaptionOnly = es.captionOnlyAt && (Date.now() - es.captionOnlyAt) < TROUBLE_WINDOW_MS;
|
||||
if ((es.lagMs || 0) > 12000 || recentCaptionOnly)
|
||||
return { level: 'red',
|
||||
detail: 'Falling badly behind — some sentences are captions only.' + loadNote() };
|
||||
if ((es.lagMs || 0) > 6000 || (es.queue || 0) >= 2 || (es.degradedFinals || 0) > 0)
|
||||
const recentDegraded = es.degradedAt && (Date.now() - es.degradedAt) < TROUBLE_WINDOW_MS;
|
||||
if ((es.lagMs || 0) > 6000 || (es.queue || 0) >= 2 || recentDegraded)
|
||||
return { level: 'amber',
|
||||
detail: 'Running behind — idle languages may be trimmed to catch up.' + loadNote() };
|
||||
if (es.cloud && es.cloud.parked)
|
||||
|
||||
+59
-15
@@ -57,16 +57,45 @@ For a second campus, repeat with its own name — `translate-north`,
|
||||
|
||||
---
|
||||
|
||||
## The other way: your existing nginx proxy
|
||||
## The other way: a name on your existing reverse proxy
|
||||
|
||||
You already run a reverse proxy at `10.1.1.4` fronting `check.watersmedia.org`,
|
||||
with split-horizon DNS. It can front this too, though it means the traffic goes
|
||||
through the NAS network to reach the booth Mac.
|
||||
You already run a reverse proxy on the NAS at `10.1.1.4` (Boba-Fett, Synology
|
||||
DSM) fronting `check.watersmedia.org`. It can front this too, though it means
|
||||
the traffic goes through the NAS network to reach the booth Mac.
|
||||
|
||||
Add a DNS record for `translate.watersmedia.org` — public to your WAN address,
|
||||
internal to `10.1.1.4` — and a server block. Copy the one that already serves
|
||||
the checklist app, change the name, and point it at the **booth Mac**, not the
|
||||
NAS:
|
||||
This is what North Attleboro runs as of 2026-10-15, in the **LAN-only** form:
|
||||
`http://translate.watersmedia.org` works on the building's WiFi at both the
|
||||
Apollo Beach and North Attleboro sites and nowhere else. Nothing was opened to
|
||||
the WAN. The pieces, with the exact scripts and a step-by-step runbook under
|
||||
[`examples/waters-church-na/`](examples/waters-church-na/README.md):
|
||||
|
||||
| Piece | Where | What |
|
||||
| --- | --- | --- |
|
||||
| Local DNS | UniFi consoles at AB and NA (Settings → Policy Table → DNS) | A record `translate.watersmedia.org` → `10.1.1.4`, TTL 300, created with `unifi-dns-record.py` and a per-console Network API key |
|
||||
| Proxy rule | DSM → Control Panel → Login Portal → Advanced → Reverse Proxy | HTTP `translate.watersmedia.org:80` → HTTP `10.1.7.39:3100`, WebSocket headers, `X-Forwarded-Host: $http_host`, read timeout 3600 s |
|
||||
| HTTPS | `/usr/local/etc/nginx/sites-enabled/translate-redirect.conf` on the NAS | `https://` answers a 301 to `http://`; the name has no certificate, so browsers warn before the redirect |
|
||||
|
||||
Both NAS pieces come from `boba-fett-translate-proxy.sh`, run as
|
||||
`ssh -t boba-fett sudo bash translate-proxy.sh` because DSM's reverse-proxy API
|
||||
is root-only. Going through DSM rather than a hand-edited nginx file means the
|
||||
rule shows up in the DSM UI and survives DSM updates. DSM adds `Host`,
|
||||
`X-Real-IP`, `X-Forwarded-For` and `X-Forwarded-Proto` on its own.
|
||||
|
||||
Then: Operator page → **Public address** → `http://translate.watersmedia.org`
|
||||
(no trailing slash), so the QR code and poster use the name instead of the
|
||||
IP and port.
|
||||
|
||||
To let people off-site use the same name later: add a public DNS record for
|
||||
`translate.watersmedia.org` pointing at the NA WAN address, get a certificate
|
||||
in DSM (Control Panel → Security → Certificate → Let's Encrypt), replace the
|
||||
redirect with an HTTPS rule to the same destination, and switch Public address
|
||||
to `https://`. The app behaves the same either way, including the `/operator`
|
||||
404 through a proxy.
|
||||
|
||||
### If the proxy is plain nginx instead of DSM
|
||||
|
||||
DSM writes most of this for you. On a hand-managed nginx the server block
|
||||
needs all of it:
|
||||
|
||||
```nginx
|
||||
server {
|
||||
@@ -77,7 +106,7 @@ server {
|
||||
ssl_certificate_key /path/to/privkey.pem;
|
||||
|
||||
location / {
|
||||
proxy_pass http://10.6.3.222:3100; # the booth Mac
|
||||
proxy_pass http://10.1.7.39:3100; # the booth Mac
|
||||
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
@@ -95,21 +124,36 @@ server {
|
||||
# the room reconnects at once.
|
||||
proxy_read_timeout 3600s;
|
||||
proxy_send_timeout 3600s;
|
||||
|
||||
# Live audio: do not hold HLS segments back.
|
||||
proxy_buffering off;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This has one real drawback the tunnel does not: the booth Mac needs a fixed
|
||||
address on the network, or the proxy breaks the next time DHCP moves it.
|
||||
Either way the booth Mac needs a fixed address or a DHCP reservation, or the
|
||||
proxy breaks the next time DHCP moves it.
|
||||
|
||||
---
|
||||
|
||||
## Checking it worked
|
||||
|
||||
```bash
|
||||
curl -s https://translate.watersmedia.org/healthz
|
||||
curl -s http://translate.watersmedia.org/healthz # https:// once published
|
||||
```
|
||||
|
||||
Should return `{"ok":true,…}`. Then open the page on a phone **with WiFi turned
|
||||
off** — that is the real test. If the page loads but captions never appear, the
|
||||
WebSocket headers are missing.
|
||||
Should return `{"ok":true,…}`, the same as `curl -s http://10.1.7.39:3100/healthz`.
|
||||
A proxy that passes the WebSocket answers `HTTP/1.1 101 Switching Protocols` to:
|
||||
|
||||
```bash
|
||||
curl -si -N -H 'Connection: Upgrade' -H 'Upgrade: websocket' \
|
||||
-H 'Sec-WebSocket-Version: 13' -H 'Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==' \
|
||||
http://translate.watersmedia.org/ws | head -1
|
||||
```
|
||||
|
||||
Then open the page on a phone: for the LAN-only setup on the guest WiFi, for a
|
||||
published one **with WiFi turned off**. That is the real test. If the page loads
|
||||
but captions never appear, the WebSocket headers are missing. If phones on
|
||||
guest WiFi cannot connect at all, look at client isolation on that network
|
||||
before anything else. `/operator` returning 404 through the proxy is by design;
|
||||
staff use `http://10.1.7.39:3100/operator` on the LAN.
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
# translate.watersmedia.org — local DNS and reverse proxy
|
||||
|
||||
Puts the Church Translate congregation page on a church hostname inside the
|
||||
Apollo Beach and North Attleboro networks. LAN only, plain HTTP, no WAN access.
|
||||
The app's own requirements are in [PUBLIC-ACCESS.md](../../PUBLIC-ACCESS.md). This
|
||||
was done on 2026-10-15 against app 3.23.2 build 121.
|
||||
|
||||
| What | Value |
|
||||
| --- | --- |
|
||||
| App host | Translate Mac, North Attleboro LAN `10.1.7.39` |
|
||||
| App port | TCP 3100, plain HTTP + WebSocket on `/ws` (the only port to expose) |
|
||||
| Reverse proxy | Boba-Fett (Synology DSM 7.0.1), `10.1.1.4`, `ssh boba-fett` (port 202, user Michael) |
|
||||
| Hostname | `translate.watersmedia.org` |
|
||||
| Local DNS | A record on the AB and NA UDM Pro consoles → `10.1.1.4`, TTL 300 |
|
||||
| HTTPS | no certificate; `https://translate.watersmedia.org` answers 301 to `http://` |
|
||||
|
||||
`curl -s http://10.1.7.39:3100/healthz` should return `{"ok":true,"live":true,...}`
|
||||
before anything below is tested.
|
||||
|
||||
## 1. Local DNS on UniFi (AB and NA) — done 2026-10-15
|
||||
|
||||
Each console has its own Network API key saved as
|
||||
`~/.config/unifi/<site>-network-api-key` (created in Settings → Control Plane →
|
||||
Integrations → Create API Key; all five sites are saved on Michael's Mac). The
|
||||
record was created with:
|
||||
|
||||
```bash
|
||||
python3 docs/examples/waters-church-na/unifi-dns-record.py translate.watersmedia.org 10.1.1.4 --sites apollo-beach,north-attleboro
|
||||
```
|
||||
|
||||
Resulting policy ids: AB `379f868d-a956-4ec1-acef-13557153ff3f`, NA
|
||||
`16bf7109-ec68-41f9-9c16-fa6a62471430`. Both gateways answer `10.1.1.4`.
|
||||
`--list` shows every DNS policy per console; `--dry-run` previews.
|
||||
|
||||
Why Boba-Fett and not the Mac: the proxy owns the hostname so the port never
|
||||
appears in the URL, and AB clients reach `10.1.1.4` over the site-to-site link
|
||||
exactly as they already do for `chronicles-na.waterschurch.org`.
|
||||
|
||||
## 2. Reverse proxy on Boba-Fett — done 2026-10-15
|
||||
|
||||
[boba-fett-translate-proxy.sh](boba-fett-translate-proxy.sh) does the whole
|
||||
job and is idempotent. It was run on 2026-10-15; afterwards `/healthz`,
|
||||
`/api/languages`, the `/ws` upgrade (101) and the HTTPS 301 all checked out
|
||||
from both the AB and NA LANs, and `/operator` returned 404 as designed. A copy lives at `~/translate-proxy.sh` on Boba-Fett
|
||||
(Michael's home). DSM's reverse-proxy API and nginx reload both need root, and
|
||||
sudo there asks for Michael's password, so run it interactively:
|
||||
|
||||
```bash
|
||||
ssh -t boba-fett sudo bash translate-proxy.sh
|
||||
```
|
||||
|
||||
What it creates:
|
||||
|
||||
| Piece | Where | Content |
|
||||
| --- | --- | --- |
|
||||
| DSM reverse proxy rule | Control Panel → Login Portal → Advanced → Reverse Proxy, "translate.watersmedia.org" | HTTP `translate.watersmedia.org:80` → HTTP `10.1.7.39:3100`; custom headers `Upgrade: $http_upgrade`, `Connection: $connection_upgrade` (WebSocket), `X-Forwarded-Host: $http_host`; read timeout 3600 s; HTTP/1.1 |
|
||||
| HTTPS redirect | `/usr/local/etc/nginx/sites-enabled/translate-redirect.conf` | `server { listen 443 ssl; server_name translate.watersmedia.org; return 301 http://translate.watersmedia.org$request_uri; }` |
|
||||
|
||||
DSM already sends `Host`, `X-Real-IP`, `X-Forwarded-For` and
|
||||
`X-Forwarded-Proto` on every rule. The whole host lands on `/`, which the app
|
||||
requires. `/operator` returning 404 through the proxy is intended; staff use
|
||||
`http://10.1.7.39:3100/operator` on the LAN.
|
||||
|
||||
To undo: delete the rule in the DSM Reverse Proxy list, remove the redirect
|
||||
file, and `sudo synosystemctl reload nginx`.
|
||||
|
||||
Known limits:
|
||||
|
||||
- The HTTPS redirect only runs after a TLS handshake with Boba-Fett's default
|
||||
certificate, so a browser shows a certificate warning first. A trusted
|
||||
certificate for this name would need DNS-01 validation against the
|
||||
watersmedia.org zone (cPanel), which is not set up.
|
||||
- DSM's reverse proxy cannot turn `proxy_buffering` off. HLS segments under
|
||||
`/live/` are small discrete files and the WebSocket is a tunnel after upgrade,
|
||||
so the effect is minor; if audio lag is reported, that is the first thing to
|
||||
revisit with a hand-written nginx include.
|
||||
|
||||
## 3. After the proxy answers
|
||||
|
||||
1. On the NA LAN open `http://10.1.7.39:3100/operator`, enter the operator PIN,
|
||||
and set **Public address** to `http://translate.watersmedia.org` (no trailing
|
||||
slash). Otherwise the QR code and poster keep advertising `10.1.7.39:3100`.
|
||||
2. Verify from a machine at AB and one at NA:
|
||||
|
||||
```bash
|
||||
curl -s http://translate.watersmedia.org/healthz
|
||||
```
|
||||
|
||||
It must return the same JSON as the direct `10.1.7.39:3100` check.
|
||||
3. Open the page on a phone on guest wifi and confirm captions appear
|
||||
(WebSocket) and spoken audio plays (`/live/`). If phones cannot connect at
|
||||
all, look at client isolation on the guest network and the guest → `10.1.1.4`
|
||||
firewall rule before anything else.
|
||||
@@ -0,0 +1,64 @@
|
||||
#!/bin/bash
|
||||
# translate.watersmedia.org on Boba-Fett (Synology DSM 7.0.1).
|
||||
#
|
||||
# 1. Creates the DSM reverse-proxy rule http://translate.watersmedia.org:80
|
||||
# -> http://10.1.7.39:3100 (Church Translate) with WebSocket headers,
|
||||
# X-Forwarded-Host, and a 3600 s read timeout, through DSM's own API so it
|
||||
# shows up in Control Panel -> Login Portal -> Advanced -> Reverse Proxy.
|
||||
# 2. Adds an nginx server block that answers https://translate.watersmedia.org
|
||||
# with a 301 to the http:// address (the name is LAN-only and has no
|
||||
# certificate, so browsers warn before the redirect; that is expected).
|
||||
# 3. Tests the nginx config, reloads it, and prints a local check.
|
||||
#
|
||||
# Idempotent: re-running skips the rule if it exists and rewrites the redirect.
|
||||
# Run on Boba-Fett as root: sudo bash ~/translate-proxy.sh
|
||||
set -euo pipefail
|
||||
|
||||
NAME=translate.watersmedia.org
|
||||
BACKEND_IP=10.1.7.39
|
||||
BACKEND_PORT=3100
|
||||
RP_JSON=/usr/syno/etc/www/ReverseProxy.json
|
||||
REDIRECT_CONF=/usr/local/etc/nginx/sites-enabled/translate-redirect.conf
|
||||
|
||||
if [ "$(id -u)" != 0 ]; then
|
||||
echo "Run with sudo: sudo bash $0" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "== 1. DSM reverse proxy rule for $NAME"
|
||||
if grep -q "\"fqdn\" : \"$NAME\"" "$RP_JSON"; then
|
||||
echo " rule already present in $RP_JSON, skipping create"
|
||||
else
|
||||
# $http_upgrade etc. are nginx variables and must reach DSM literally.
|
||||
ENTRY='{"description":"translate.watersmedia.org","frontend":{"acl":null,"fqdn":"translate.watersmedia.org","https":{"hsts":false},"port":80,"protocol":0},"backend":{"fqdn":"10.1.7.39","port":3100,"protocol":0},"customize_headers":[{"name":"Upgrade","value":"$http_upgrade"},{"name":"Connection","value":"$connection_upgrade"},{"name":"X-Forwarded-Host","value":"$http_host"}],"proxy_connect_timeout":60,"proxy_read_timeout":3600,"proxy_send_timeout":60,"proxy_http_version":1,"proxy_intercept_errors":false}'
|
||||
OUT=$(/usr/syno/bin/synowebapi --exec api=SYNO.Core.AppPortal.ReverseProxy method=create version=1 entry="$ENTRY" 2>&1 || true)
|
||||
echo " $OUT"
|
||||
if ! echo "$OUT" | grep -q '"success" *: *true'; then
|
||||
echo " synowebapi did not report success; stopping before the redirect." >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "== 2. HTTPS -> HTTP redirect block"
|
||||
cat > "$REDIRECT_CONF" <<'EOF'
|
||||
# translate.watersmedia.org is served over plain HTTP on the LAN (no certificate).
|
||||
# Anyone who types https:// is sent to the http:// address. Added 2026-10-15.
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name translate.watersmedia.org;
|
||||
return 301 http://translate.watersmedia.org$request_uri;
|
||||
}
|
||||
EOF
|
||||
echo " wrote $REDIRECT_CONF"
|
||||
|
||||
echo "== 3. nginx test + reload"
|
||||
/usr/bin/nginx -t
|
||||
/usr/syno/bin/synosystemctl reload nginx 2>/dev/null || /usr/bin/nginx -s reload
|
||||
echo " reloaded"
|
||||
|
||||
echo "== 4. Local checks from Boba-Fett"
|
||||
echo -n " http /healthz : "
|
||||
curl -s -m 5 --resolve "$NAME:80:127.0.0.1" "http://$NAME/healthz"; echo
|
||||
echo -n " https / : "
|
||||
curl -sk -m 5 -o /dev/null -w '%{http_code} -> %{redirect_url}\n' --resolve "$NAME:443:127.0.0.1" "https://$NAME/"
|
||||
echo "done"
|
||||
+211
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Create or update a local DNS A record on every Waters Church UniFi console.
|
||||
|
||||
Usage:
|
||||
docs/examples/waters-church-na/unifi-dns-record.py repo.watersmedia.org 10.3.8.90
|
||||
docs/examples/waters-church-na/unifi-dns-record.py repo.watersmedia.org 10.3.8.90 --sites apollo-beach,norwood
|
||||
docs/examples/waters-church-na/unifi-dns-record.py repo.watersmedia.org 10.3.8.90 --dry-run
|
||||
docs/examples/waters-church-na/unifi-dns-record.py --list # show every DNS policy per site
|
||||
|
||||
Each console needs its own Network API key (Site Manager keys from api.ui.com are
|
||||
rejected locally). Create one per console in the UniFi UI:
|
||||
Settings -> Control Plane -> Integrations -> Create API Key
|
||||
and save it, one line, mode 600, as:
|
||||
~/.config/unifi/<site>-network-api-key
|
||||
where <site> is one of the names in SITES below.
|
||||
|
||||
The script talks to the console's Network Integration API
|
||||
(/proxy/network/integration/v1/sites/{siteId}/dns/policies) over the LAN /
|
||||
site-to-site links, so it must run from inside the church network. Gateways use
|
||||
self-signed certificates, so certificate verification is disabled for them.
|
||||
Keys are read from disk and sent only in the X-API-KEY header; they are never
|
||||
printed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
# Console name -> gateway LAN address (UDM Pro at each site, 2026-10).
|
||||
SITES = {
|
||||
"apollo-beach": "10.3.0.1",
|
||||
"north-attleboro": "10.1.0.1",
|
||||
"woonsocket": "10.2.0.1",
|
||||
"norwood": "10.4.0.1",
|
||||
"southcoast": "10.5.0.1",
|
||||
}
|
||||
|
||||
KEY_DIR = os.path.expanduser("~/.config/unifi")
|
||||
TIMEOUT = 15
|
||||
|
||||
|
||||
class ApiError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
class Console:
|
||||
def __init__(self, name: str, gateway: str):
|
||||
self.name = name
|
||||
self.gateway = gateway
|
||||
self.base = f"https://{gateway}/proxy/network/integration/v1"
|
||||
self.key_path = os.path.join(KEY_DIR, f"{name}-network-api-key")
|
||||
self.key = self._load_key()
|
||||
self.ctx = ssl.create_default_context()
|
||||
self.ctx.check_hostname = False
|
||||
self.ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
def _load_key(self) -> str | None:
|
||||
try:
|
||||
with open(self.key_path) as fh:
|
||||
key = fh.read().strip()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
return key or None
|
||||
|
||||
def request(self, method: str, path: str, body: dict | None = None, query: dict | None = None):
|
||||
url = self.base + path
|
||||
if query:
|
||||
url += "?" + urllib.parse.urlencode(query)
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request(url, data=data, method=method)
|
||||
req.add_header("X-API-KEY", self.key or "")
|
||||
req.add_header("Accept", "application/json")
|
||||
if data is not None:
|
||||
req.add_header("Content-Type", "application/json")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=TIMEOUT, context=self.ctx) as resp:
|
||||
raw = resp.read()
|
||||
return json.loads(raw) if raw else None
|
||||
except urllib.error.HTTPError as e:
|
||||
detail = e.read().decode(errors="replace")[:300]
|
||||
raise ApiError(f"{method} {path} -> HTTP {e.code}: {detail}") from None
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as e:
|
||||
raise ApiError(f"{method} {path} -> {e}") from None
|
||||
|
||||
def site_id(self) -> str:
|
||||
sites = self.request("GET", "/sites").get("data", [])
|
||||
if not sites:
|
||||
raise ApiError("console reports no sites")
|
||||
for s in sites:
|
||||
if s.get("internalReference") == "default" or s.get("name") == "default":
|
||||
return s["id"]
|
||||
return sites[0]["id"]
|
||||
|
||||
def dns_policies(self, site_id: str) -> list[dict]:
|
||||
out: list[dict] = []
|
||||
offset, limit = 0, 100
|
||||
while True:
|
||||
page = self.request("GET", f"/sites/{site_id}/dns/policies", query={"offset": offset, "limit": limit})
|
||||
data = page.get("data", [])
|
||||
out.extend(data)
|
||||
total = page.get("totalCount")
|
||||
offset += len(data)
|
||||
if not data or (total is not None and offset >= total) or len(data) < limit:
|
||||
return out
|
||||
|
||||
def ensure_a_record(self, site_id: str, fqdn: str, ipv4: str, ttl: int | None, dry_run: bool) -> str:
|
||||
existing = [p for p in self.dns_policies(site_id) if p.get("domain", "").lower() == fqdn.lower()]
|
||||
a_records = [p for p in existing if p.get("type") == "A_RECORD"]
|
||||
other = [p for p in existing if p.get("type") != "A_RECORD"]
|
||||
if other:
|
||||
kinds = ", ".join(sorted({p.get("type", "?") for p in other}))
|
||||
return f"SKIPPED: {fqdn} already has a non-A policy here ({kinds}); fix by hand"
|
||||
if len(a_records) > 1:
|
||||
return f"SKIPPED: {len(a_records)} A records already exist for {fqdn}; fix by hand"
|
||||
|
||||
body = {"type": "A_RECORD", "enabled": True, "domain": fqdn, "ipv4Address": ipv4}
|
||||
if ttl:
|
||||
body["ttlSeconds"] = ttl
|
||||
|
||||
if a_records:
|
||||
cur = a_records[0]
|
||||
same = cur.get("ipv4Address") == ipv4 and cur.get("enabled", True) and (ttl is None or cur.get("ttlSeconds") == ttl)
|
||||
if same:
|
||||
return f"unchanged: {fqdn} -> {ipv4} already present (id {cur.get('id')})"
|
||||
if dry_run:
|
||||
return f"would UPDATE {fqdn}: {cur.get('ipv4Address')} -> {ipv4} (id {cur.get('id')})"
|
||||
self.request("PUT", f"/sites/{site_id}/dns/policies/{cur['id']}", body=body)
|
||||
return f"UPDATED {fqdn}: {cur.get('ipv4Address')} -> {ipv4}"
|
||||
|
||||
if dry_run:
|
||||
return f"would CREATE {fqdn} -> {ipv4}"
|
||||
created = self.request("POST", f"/sites/{site_id}/dns/policies", body=body)
|
||||
return f"CREATED {fqdn} -> {ipv4} (id {created.get('id') if isinstance(created, dict) else '?'})"
|
||||
|
||||
|
||||
def resolve_via(gateway: str, fqdn: str) -> str:
|
||||
"""Ask the gateway's resolver directly; returns the answer or a short status."""
|
||||
dig = shutil.which("dig")
|
||||
if not dig:
|
||||
return "(dig not installed)"
|
||||
try:
|
||||
res = subprocess.run([dig, "+short", "+time=3", "+tries=1", f"@{gateway}", fqdn, "A"],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
return "(timeout)"
|
||||
answer = res.stdout.strip().replace("\n", " ")
|
||||
return answer or "(no answer)"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
ap.add_argument("fqdn", nargs="?", help="host name, e.g. repo.watersmedia.org")
|
||||
ap.add_argument("ipv4", nargs="?", help="IPv4 address the name should resolve to")
|
||||
ap.add_argument("--sites", default=",".join(SITES), help="comma-separated subset of: " + ", ".join(SITES))
|
||||
ap.add_argument("--ttl", type=int, default=300, help="TTL in seconds (default 300; the API rejects a null TTL)")
|
||||
ap.add_argument("--dry-run", action="store_true", help="show what would change without writing")
|
||||
ap.add_argument("--list", action="store_true", help="print every DNS policy on each console and exit")
|
||||
ap.add_argument("--no-verify", action="store_true", help="skip the dig check after writing")
|
||||
args = ap.parse_args()
|
||||
|
||||
if not args.list and (not args.fqdn or not args.ipv4):
|
||||
ap.error("fqdn and ipv4 are required unless --list is given")
|
||||
|
||||
names = [s.strip() for s in args.sites.split(",") if s.strip()]
|
||||
unknown = [n for n in names if n not in SITES]
|
||||
if unknown:
|
||||
ap.error(f"unknown site(s): {', '.join(unknown)}")
|
||||
|
||||
failures = 0
|
||||
for name in names:
|
||||
console = Console(name, SITES[name])
|
||||
label = f"{name:16s} {console.gateway:10s}"
|
||||
if not console.key:
|
||||
print(f"{label} NO KEY: save the console's API key to {console.key_path}")
|
||||
failures += 1
|
||||
continue
|
||||
try:
|
||||
site_id = console.site_id()
|
||||
if args.list:
|
||||
policies = console.dns_policies(site_id)
|
||||
print(f"{label} {len(policies)} DNS policies")
|
||||
for p in sorted(policies, key=lambda p: p.get("domain", "")):
|
||||
target = p.get("ipv4Address") or p.get("ipv6Address") or p.get("targetDomain") or p.get("text") or ""
|
||||
flag = "" if p.get("enabled", True) else " (disabled)"
|
||||
print(f" {p.get('type', '?'):12s} {p.get('domain', ''):40s} -> {target}{flag}")
|
||||
continue
|
||||
result = console.ensure_a_record(site_id, args.fqdn, args.ipv4, args.ttl, args.dry_run)
|
||||
line = f"{label} {result}"
|
||||
if not args.dry_run and not args.no_verify and not result.startswith("SKIPPED"):
|
||||
line += f" | dig @{console.gateway}: {resolve_via(console.gateway, args.fqdn)}"
|
||||
print(line)
|
||||
if result.startswith("SKIPPED"):
|
||||
failures += 1
|
||||
except ApiError as e:
|
||||
print(f"{label} FAILED: {e}")
|
||||
failures += 1
|
||||
|
||||
return 1 if failures else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user