diff --git a/README.md b/README.md
index dbbcd16..0b74c76 100644
--- a/README.md
+++ b/README.md
@@ -358,7 +358,13 @@ below. Nothing is installed without someone clicking.
at ProdCom (the tile's bottom edge lights up in the accent color), and the
channel the text goes out *as* is picked from their Send as menu — only
channels ProdCom lets text be typed onto are offered.
- Admin config: ProdCom server (host : port), API key.
+ Admin config: ProdCom server (host : port), API key, and two switches for
+ a ProdCom on this Mac (macOS only): **Start ProdCom when it is offline**
+ opens it when it isn't running (up to three tries, five seconds apart), and
+ **Restart ProdCom to turn its API on** turns the Application API on and
+ restarts ProdCom when it was running with the API off — ProdCom reads that
+ switch only at launch, so the restart costs about ten seconds without
+ intercom. Neither does anything for a ProdCom on another machine.
Tile settings: channel icons, direction and timestamps.
- **ProPresenter** — what's live and what's next, with slide progress,
driven by the shared `propresenter-core` client (ProPresenter 7/20).
diff --git a/launcher/macos/Resources/Info.plist b/launcher/macos/Resources/Info.plist
index 9108e0b..0517773 100644
--- a/launcher/macos/Resources/Info.plist
+++ b/launcher/macos/Resources/Info.plist
@@ -33,6 +33,8 @@
verbatim, so they say what ProdDash does with each permission. -->
NSMicrophoneUsageDescription
ProdDash reads an audio input on this Mac to decode SMPTE LTC timecode for the Timers module.
+ NSAppleEventsUsageDescription
+ ProdDash asks ProdCom to quit and relaunches it when its Application API has been switched off.
NSLocalNetworkUsageDescription
ProdDash connects to production gear on this network — ProdCom, ProPresenter and anything else its modules are pointed at.
NSBonjourServices
diff --git a/launcher/macos/Resources/ProdDash.entitlements b/launcher/macos/Resources/ProdDash.entitlements
index 0cf521e..9545ab0 100644
--- a/launcher/macos/Resources/ProdDash.entitlements
+++ b/launcher/macos/Resources/ProdDash.entitlements
@@ -6,5 +6,10 @@
the hardened runtime that needs saying out loud. -->
com.apple.security.device.audio-input
+
+ com.apple.security.automation.apple-events
+
diff --git a/modules/prodcom-transcript/module.json b/modules/prodcom-transcript/module.json
index af43546..ebc5a39 100644
--- a/modules/prodcom-transcript/module.json
+++ b/modules/prodcom-transcript/module.json
@@ -1,7 +1,7 @@
{
"id": "prodcom-transcript",
"name": "ProdCom",
- "version": "1.4.0",
+ "version": "1.5.0",
"proddash": ">=1.4.0",
"description": "Live ProdCom transcript; typed text to a ProdCom channel through the Send Message module",
"permissions": [
@@ -38,6 +38,19 @@
"type": "password",
"label": "API key (only if PSK auth is enabled in ProdCom)",
"default": ""
+ },
+ "autoStart": {
+ "type": "switch",
+ "label": "Start ProdCom when it is offline",
+ "default": true,
+ "help": "Only when ProdCom runs on this machine (the server above is 127.0.0.1 or this Mac’s own address). If ProdCom is not running, ProdDash opens it — up to three tries, five seconds apart. macOS only."
+ },
+ "apiRestart": {
+ "type": "switch",
+ "label": "Restart ProdCom to turn its API on",
+ "default": true,
+ "showWhen": "autoStart",
+ "help": "When ProdCom is running but its Application API is switched off, ProdDash turns the API on and restarts ProdCom to apply it (about ten seconds without intercom). Off: the API is turned on for ProdCom’s next start and the tile says to restart it."
}
},
"instanceSchema": {
diff --git a/modules/prodcom-transcript/server.js b/modules/prodcom-transcript/server.js
index 93dd26e..b71c7fc 100644
--- a/modules/prodcom-transcript/server.js
+++ b/modules/prodcom-transcript/server.js
@@ -32,9 +32,15 @@
* extractEntries). POST /api/v1/channels/{id}/send with { text } puts typed
* text on a channel (it comes back on the stream with source "typed"). A
* bearer API key is sent when one is configured.
+ *
+ * When ProdCom runs on this very Mac and stops answering, supervisor.js opens
+ * it again — or turns its Application API on and restarts it, the API being
+ * read only at launch. Every failed connection is reported to it (fail()) and
+ * every good one (connect()); it does nothing for a ProdCom on another machine.
*/
const http = require('http');
+const { createSupervisor } = require('./supervisor');
/** The module that owns the PIN and the unlock tokens — see verifyUnlock(). */
const SENDER_ID = 'send-message';
@@ -60,6 +66,19 @@ function resolveTarget(config) {
return null;
}
+/** Codes that mean ProdCom could not be reached at all — not that it answered badly. */
+const CONNECTION_CODES = new Set(['ECONNREFUSED', 'ECONNRESET', 'ETIMEDOUT', 'EHOSTUNREACH', 'ENOTFOUND']);
+
+/** The code of an error that is a failed (or timed-out) connection to ProdCom, else ''.
+ An HTTP status from ProdCom is not one: its API is up, so there is nothing to start. */
+function connectionCode(err) {
+ const code = err && (err.cause?.code || err.code);
+ if (CONNECTION_CODES.has(code)) return code;
+ if (code === 'UND_ERR_SOCKET') return 'ECONNRESET'; // fetch's spelling of a connection dropped under it
+ if (err && (err.name === 'TimeoutError' || err.name === 'AbortError')) return 'ETIMEDOUT';
+ return '';
+}
+
/** Entries inside whatever ProdCom sent: a bare entry, or one wrapped in data / payload / entry / entries / transcript. */
function extractEntries(obj) {
if (!obj || typeof obj !== 'object') return [];
@@ -181,6 +200,7 @@ module.exports = {
let upstream = null;
let generation = 0; // callbacks from a superseded connection are ignored
let retryTimer = null;
+ let retryPending = false; // waiting out the backoff, with no connect() under way
let retryMs = RETRY_MIN_MS;
const authHeaders = () => (apiKey ? { authorization: `Bearer ${apiKey}` } : {});
@@ -190,7 +210,11 @@ module.exports = {
headers: { accept: 'application/json', ...authHeaders() },
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
- if (!res.ok) throw new Error(`HTTP ${res.status} for ${pathname}`);
+ if (!res.ok) {
+ const err = new Error(`HTTP ${res.status} for ${pathname}`);
+ err.httpStatus = res.status; // ProdCom answered, so it is up
+ throw err;
+ }
return res.json();
}
@@ -220,6 +244,22 @@ module.exports = {
broadcast('status', status);
}
+ // Keeps ProdCom itself up when it is on this machine (supervisor.js); a
+ // no-op everywhere else. Its words reach the tile through setStatus. This
+ // module is re-init'ed whenever its config is saved, so `config` is current.
+ const supervisor = createSupervisor({
+ settings: () => ({ autoStart: config.autoStart !== false, apiRestart: config.apiRestart !== false }),
+ target: () => ({ host: target ? target.hostname : '', port: target ? Number(target.port) || 80 : 0 }),
+ log,
+ status: setStatus,
+ // ProdCom answers again: connect now rather than when the backoff runs out — unless a connect is already under way.
+ onUp: () => {
+ if (stopped || !retryPending) return;
+ clearTimeout(retryTimer);
+ connect();
+ },
+ });
+
function trim() {
while (entries.size > ENTRIES_MAX) entries.delete(entries.keys().next().value);
}
@@ -281,22 +321,29 @@ module.exports = {
function scheduleRetry() {
if (stopped) return;
clearTimeout(retryTimer);
+ retryPending = true;
retryTimer = setTimeout(connect, retryMs);
retryMs = Math.min(RETRY_MAX_MS, retryMs * 2);
}
- function fail(gen, why) {
+ /** `code` is set when the connection itself failed (see connectionCode), not when ProdCom answered badly. */
+ function fail(gen, why, code = '') {
if (stopped || gen !== generation) return;
generation += 1; // whatever else this connection reports is stale now
try { upstream?.destroy(); } catch { /* already gone */ }
upstream = null;
- setStatus('error', `ProdCom unreachable (${why}) — retrying`);
- log(`ProdCom ${target.origin}: ${why} — retrying in ${retryMs / 1000} s`);
+ if (code) supervisor.kick(code);
+ // While the supervisor is starting ProdCom — or has given up and says why —
+ // its words are the ones worth reading, not "unreachable" on every retry.
+ const shown = code || why; // the normalised code where there is one (a timeout reads ETIMEDOUT, not 23)
+ if (!(code && supervisor.message())) setStatus('error', `ProdCom unreachable (${shown}) — retrying`);
+ log(`ProdCom ${target.origin}: ${shown} — retrying in ${retryMs / 1000} s`);
scheduleRetry();
}
/** Lists and history first (a fresh snapshot for the tiles), then the live stream. */
async function connect() {
+ retryPending = false;
if (stopped || !target) return;
const gen = ++generation;
try {
@@ -304,7 +351,8 @@ module.exports = {
await loadHistory();
} catch (err) {
if (gen !== generation || stopped) return;
- return fail(gen, err.cause?.code || err.code || err.message);
+ if (err.httpStatus) supervisor.noteReachable(); // 401, 5xx…: ProdCom is up, whatever it said
+ return fail(gen, err.cause?.code || err.code || err.message, connectionCode(err));
}
if (gen !== generation || stopped) return;
broadcast('state', snapshot());
@@ -319,9 +367,11 @@ module.exports = {
if (gen !== generation) return void res.destroy();
if (res.statusCode !== 200) {
res.resume();
+ supervisor.noteReachable(); // ProdCom answered
return fail(gen, `HTTP ${res.statusCode} from the transcript stream`);
}
retryMs = RETRY_MIN_MS;
+ supervisor.noteReachable();
setStatus('ok', 'Live');
log(`following ${target.origin} — ${channels.length} channels, ${entries.size} entries`);
let buf = '';
@@ -337,9 +387,9 @@ module.exports = {
if (buf.length > 1e6) buf = ''; // a frame that never ends is dropped
});
res.on('end', () => fail(gen, 'stream ended'));
- res.on('error', (err) => fail(gen, err.code || err.message));
+ res.on('error', (err) => fail(gen, err.code || err.message, connectionCode(err)));
});
- req.on('error', (err) => fail(gen, err.code || err.message));
+ req.on('error', (err) => fail(gen, err.code || err.message, connectionCode(err)));
req.end();
upstream = req;
}
@@ -449,6 +499,7 @@ module.exports = {
stop() {
stopped = true;
generation += 1;
+ supervisor.stop();
clearTimeout(retryTimer);
clearInterval(listsTimer);
clearInterval(heartbeat);
diff --git a/modules/prodcom-transcript/supervisor.js b/modules/prodcom-transcript/supervisor.js
new file mode 100644
index 0000000..0d2cbcd
--- /dev/null
+++ b/modules/prodcom-transcript/supervisor.js
@@ -0,0 +1,437 @@
+'use strict';
+
+/**
+ * ProdCom Transcript — keeping ProdCom itself running.
+ *
+ * When ProdCom runs on the same Mac as ProdDash (the booth Mac) and its API
+ * stops answering, the usual reason is that nobody has opened ProdCom, or that
+ * its Application API is switched off — ProdCom reads that switch only at
+ * launch. This watches for that and puts it right: open ProdCom if it is not
+ * running; if it is running with the API off, turn the API on and restart it.
+ * Nothing here does anything on another OS, or when ProdCom is on another
+ * machine — a remote ProdCom is somebody else's to look after.
+ *
+ * The module calls kick(code) every time a connection to ProdCom fails and
+ * noteReachable() when one succeeds; the rest is decided here. A "run" is up to
+ * three attempts, each: act, wait five seconds, ask the API whether it is up.
+ * When a run cannot fix it, the supervisor gives up for that outage — kick()
+ * then returns at once, so the module's reconnect loop can call it every few
+ * seconds without anything happening — until a connection succeeds again.
+ *
+ * Every touch of the machine (is ProdCom running, open it, read/write its
+ * preference, quit it) and of the network (the ping) is injected, so
+ * supervisor.test.js drives the decisions with fakes; the real ones are
+ * defaultOs() and httpProbe() below, all child_process.execFile with an
+ * argument list — never a shell string — and a timeout.
+ */
+
+const http = require('http');
+const nodeOs = require('os');
+const { execFile } = require('child_process');
+
+const BUNDLE_ID = 'com.prodcom.prodcom';
+const PROCESS_NAME = 'ProdCom';
+const API_PREF = 'applicationAPIEnabled'; // a UserDefaults bool in the app's own domain
+
+const MAX_ATTEMPTS = 3;
+const ATTEMPT_WAIT_MS = 5000; // after acting, before asking whether the API answers
+const PROBE_TIMEOUT_MS = 3000;
+const QUICK_TOOL_MS = 5000; // pgrep, defaults, pkill
+const SLOW_TOOL_MS = 10000; // open
+const QUIT_ASK_MS = 5000; // osascript's patience: a ProdCom that will not answer an Apple Event is told with SIGTERM
+const QUIT_WAIT_MS = 10000; // for the process to be gone after asking it to quit
+const QUIT_POLL_MS = 500;
+
+/* ── the real machine ───────────────────────────────────────────────── */
+
+/** Run a program with an argument list (`execFileImpl` is child_process.execFile, or a fake in tests).
+ Never rejects: { ok, exitCode, error, stdout, stderr } —
+ exitCode is the program's own exit status, or null when it never got to exit
+ normally (not found, timed out, killed). */
+function runProgram(execFileImpl, file, args, timeout) {
+ return new Promise((resolve) => {
+ execFileImpl(file, args, { timeout, windowsHide: true }, (error, stdout, stderr) => {
+ resolve({
+ ok: !error,
+ exitCode: error ? (typeof error.code === 'number' ? error.code : null) : 0,
+ error,
+ stdout: String(stdout || ''),
+ stderr: String(stderr || ''),
+ });
+ });
+ });
+}
+
+/** What went wrong with an exec() result, for a message: how it failed, then the first line the program said about it. */
+function why(r) {
+ let how = 'unknown';
+ if (r.exitCode !== null) how = `exit ${r.exitCode}`;
+ else if (r.error) how = r.error.killed ? 'timed out' : (r.error.code || r.error.message);
+ const said = r.stderr.split('\n').map((l) => l.trim()).find(Boolean);
+ if (!said) return how;
+ return `${how}: ${said.length > 120 ? `${said.slice(0, 120)}…` : said}`;
+}
+
+/** A sleep the supervisor can cut short: resolves true after `ms`, false at once
+ when `signal` aborts (its timer is cleared, so nothing is left behind). */
+function abortableSleep(ms, signal) {
+ return new Promise((resolve) => {
+ if (signal && signal.aborted) return resolve(false);
+ let timer = null;
+ const onAbort = () => {
+ clearTimeout(timer);
+ resolve(false);
+ };
+ timer = setTimeout(() => {
+ if (signal) signal.removeEventListener('abort', onAbort);
+ resolve(true);
+ }, ms);
+ timer.unref?.();
+ if (signal) signal.addEventListener('abort', onAbort, { once: true });
+ });
+}
+
+/** `defaults read` prints a bool as 1 / 0; anything else is not an answer. */
+function parseApiPref(stdout) {
+ const s = String(stdout || '').trim().toLowerCase();
+ if (s === '1' || s === 'true' || s === 'yes') return true;
+ if (s === '0' || s === 'false' || s === 'no') return false;
+ return null;
+}
+
+/** The address as one canonical spelling, so a configured "::1" matches the interface's. */
+function canonicalAddress(a) {
+ let s = String(a ?? '').trim().toLowerCase();
+ if (s.startsWith('[') && s.endsWith(']')) s = s.slice(1, -1);
+ s = s.replace(/%.*$/, ''); // a scope id (fe80::1%en0)
+ if (s.includes(':')) {
+ try { s = new URL(`http://[${s}]`).hostname.slice(1, -1); } catch { /* not an address */ }
+ }
+ return s;
+}
+
+/** Is `host` this machine? `hostname` and `addresses` are this machine's own
+ (os.hostname(), every address of os.networkInterfaces()) — passed in so this
+ is a pure function. */
+function isLocalHostName(host, hostname, addresses) {
+ const h = canonicalAddress(host).replace(/\.$/, '');
+ if (h === '' || h === 'localhost' || h === '127.0.0.1' || h === '::1' || h === '0.0.0.0') return true;
+ const bare = (s) => String(s).toLowerCase().replace(/\.$/, '').replace(/\.local$/, '');
+ if (hostname && bare(hostname) === bare(h)) return true;
+ return (addresses || []).some((a) => canonicalAddress(a) === h);
+}
+
+/** Is anything answering HTTP at host:port? Any response counts, whatever its
+ status — the API's ping needs no key, and a 401 is still a live server. */
+function httpProbe(host, port) {
+ return new Promise((resolve) => {
+ let done = false;
+ const finish = (up) => {
+ if (done) return;
+ done = true;
+ resolve(up);
+ };
+ let h = canonicalAddress(host);
+ if (!h || h === '0.0.0.0') h = '127.0.0.1';
+ try {
+ const req = http.get({
+ host: h,
+ port,
+ path: '/api/v1/ping',
+ agent: false,
+ timeout: PROBE_TIMEOUT_MS,
+ headers: { accept: 'application/json' },
+ }, (res) => {
+ res.resume();
+ finish(true);
+ });
+ req.on('timeout', () => {
+ req.destroy();
+ finish(false);
+ });
+ req.on('error', () => finish(false));
+ } catch {
+ finish(false);
+ }
+ });
+}
+
+/**
+ * The operating-system layer: macOS commands, each run with execFile.
+ * `deps` is for tests only — { execFile, sleep, now } replace the real
+ * child_process.execFile, abortableSleep and Date.now.
+ */
+function defaultOs(deps = {}) {
+ const exec = (file, args, timeout) => runProgram(deps.execFile || execFile, file, args, timeout);
+ const wait = deps.sleep || abortableSleep;
+ const now = deps.now || Date.now;
+
+ async function isRunning() {
+ const r = await exec('/usr/bin/pgrep', ['-x', PROCESS_NAME], QUICK_TOOL_MS);
+ if (r.ok) return true;
+ if (r.exitCode === 1) return false; // pgrep's "nothing matched"
+ throw new Error(`Could not check whether ProdCom is running (${why(r)})`);
+ }
+
+ /** Open ProdCom in the background (-g): a booth Mac must not lose the keyboard from ProPresenter or OBS mid-service. */
+ async function launch() {
+ const r = await exec('/usr/bin/open', ['-g', '-b', BUNDLE_ID], SLOW_TOOL_MS);
+ if (r.ok) return;
+ if (r.exitCode !== null) throw new Error('ProdCom is not installed on this machine');
+ throw new Error(`Could not open ProdCom (${why(r)})`);
+ }
+
+ async function readApiPref() {
+ const r = await exec('/usr/bin/defaults', ['read', BUNDLE_ID, API_PREF], QUICK_TOOL_MS);
+ if (r.ok) return parseApiPref(r.stdout);
+ if (r.exitCode !== null) return null; // the key does not exist yet
+ // Not knowing is not the same as "off": it would restart a ProdCom that is fine.
+ throw new Error(`Could not read ProdCom's settings (${why(r)})`);
+ }
+
+ async function writeApiPref() {
+ const r = await exec('/usr/bin/defaults', ['write', BUNDLE_ID, API_PREF, '-bool', 'true'], QUICK_TOOL_MS);
+ if (!r.ok) throw new Error(`Could not turn on ProdCom's API (${why(r)})`);
+ }
+
+ /** Ask ProdCom to quit; if it will not be asked, terminate it. Resolves when it is gone.
+ Once `signal` aborts it does nothing more — no SIGTERM, no further look at the process list. */
+ async function quit(signal) {
+ const cancelled = () => Boolean(signal && signal.aborted);
+ if (cancelled()) throw new Error('cancelled');
+ const asked = await exec('/usr/bin/osascript', ['-e', `tell application id "${BUNDLE_ID}" to quit`], QUIT_ASK_MS);
+ if (cancelled()) throw new Error('cancelled');
+ if (!asked.ok) await exec('/usr/bin/pkill', ['-x', '-TERM', PROCESS_NAME], QUICK_TOOL_MS);
+ const until = now() + QUIT_WAIT_MS;
+ for (;;) {
+ if (cancelled()) throw new Error('cancelled');
+ if (!(await isRunning())) return;
+ if (now() >= until) throw new Error('ProdCom would not quit');
+ await wait(QUIT_POLL_MS, signal);
+ }
+ }
+
+ return {
+ platform: process.platform,
+ isLocalHost(host) {
+ const addresses = [];
+ for (const list of Object.values(nodeOs.networkInterfaces())) {
+ for (const a of list || []) addresses.push(a.address);
+ }
+ return isLocalHostName(host, nodeOs.hostname(), addresses);
+ },
+ isRunning,
+ launch,
+ readApiPref,
+ writeApiPref,
+ quit,
+ };
+}
+
+/* ── the decisions ──────────────────────────────────────────────────── */
+
+/**
+ * @param opts.settings () => { autoStart, apiRestart } read live, every time
+ * @param opts.target () => { host, port } where the module points ProdCom
+ * @param opts.probe () => Promise optional; the API answers (default: GET /api/v1/ping)
+ * @param opts.os the machine layer (see defaultOs); optional, defaults to the real one
+ * @param opts.sleep (ms, signal) => Promise optional; the default is cut short by `signal`
+ * @param opts.log (line) => void
+ * @param opts.status (state, message) => void the tile's status; state is 'connecting' | 'error'
+ * @param opts.onUp () => void optional; ProdCom answers again — the module may reconnect now
+ */
+function createSupervisor(opts) {
+ const { settings, target, log, status } = opts;
+ const onUp = opts.onUp || (() => {});
+ const os = opts.os || defaultOs();
+ const probe = opts.probe || (() => { const t = target(); return httpProbe(t.host, t.port); });
+ const sleep = opts.sleep || abortableSleep;
+
+ let stopped = false;
+ let busy = false; // a run is in progress
+ let gaveUp = false; // this outage is beyond the supervisor until a connection succeeds
+ let msg = null; // the supervisor's words for the tile
+ let epoch = 0; // a run belongs to the epoch it started in; a new one ends it
+ let abort = null; // cuts the running run's sleep (and quit) short
+ let current = Promise.resolve();
+ const said = new Set(); // no-op reasons already logged
+
+ /** End the run in progress, if any: it unwinds without another word. */
+ function cancelRun() {
+ epoch += 1;
+ if (abort) {
+ abort.abort();
+ abort = null;
+ }
+ }
+
+ /** Nothing to do, and why — logged once per distinct reason, not on every reconnect tick. */
+ function skip(reason) {
+ if (!said.has(reason)) {
+ said.add(reason);
+ log(reason);
+ }
+ return reason;
+ }
+
+ /** Wait `ms`, but never past stop() or the end of the run, whatever `sleep` does with the signal. */
+ function pause(ms, signal) {
+ return new Promise((resolve) => {
+ if (signal.aborted) return resolve();
+ const done = () => {
+ signal.removeEventListener('abort', done);
+ resolve();
+ };
+ signal.addEventListener('abort', done, { once: true });
+ Promise.resolve().then(() => sleep(ms, signal)).then(done, done);
+ });
+ }
+
+ async function run(myEpoch, signal) {
+ const live = () => !stopped && epoch === myEpoch;
+ const say = (state, text) => {
+ msg = text;
+ status(state, text);
+ };
+ const giveUp = (text) => {
+ gaveUp = true;
+ say('error', text);
+ log(text);
+ };
+
+ let restarted = false; // ProdCom was quit and opened again in this run — once is all it gets
+ let opened = false; // this run opened ProdCom itself: if it is running now, it may simply still be starting up
+ let apiOnSilent = false; // the latest look found ProdCom running, API on, port silent, and not of our opening
+ try {
+ for (let n = 1; n <= MAX_ATTEMPTS; n += 1) {
+ const s = settings();
+ apiOnSilent = false;
+ // Look afresh every attempt: one sample proves little (a ProdCom that is quitting still shows as running).
+ const running = await os.isRunning();
+ if (!live()) return;
+ if (!running) {
+ // Not there — never opened, quit by hand, or gone since the restart. A launch, never a second quit.
+ await os.launch();
+ if (!live()) return;
+ opened = true;
+ say('connecting', `ProdCom is not running — starting it (${n}/${MAX_ATTEMPTS})`);
+ } else if (!restarted) {
+ const pref = await os.readApiPref();
+ if (!live()) return;
+ if (pref === true) {
+ // Running with its API on and the port still silent. Slow to start, quitting, or the wrong port —
+ // one look cannot tell, so wait and look again; only the last attempt blames the port.
+ apiOnSilent = !opened;
+ } else {
+ // The API is off (or the key was never written): switch it on for ProdCom's next start…
+ await os.writeApiPref();
+ if (!live()) return;
+ // …and restart ProdCom only if the setting really holds: that restart cuts its intercom.
+ const now = await os.readApiPref();
+ if (!live()) return;
+ if (now !== true) {
+ return giveUp(`Could not turn on ProdCom's API — it still reads ${now === false ? 'off' : 'unset'}`);
+ }
+ if (!s.apiRestart) {
+ return giveUp("ProdCom's API is off — restart ProdCom to turn it on");
+ }
+ say('connecting', "ProdCom's API is off — turning it on and restarting ProdCom");
+ await os.quit(signal);
+ if (!live()) return;
+ await os.launch();
+ if (!live()) return;
+ restarted = true;
+ }
+ }
+ await pause(ATTEMPT_WAIT_MS, signal);
+ if (!live()) return;
+ const up = await Promise.resolve().then(probe).catch(() => false);
+ if (!live()) return;
+ if (up) {
+ log(`ProdCom is up after ${n} attempt(s)`);
+ say('connecting', 'ProdCom is back — reconnecting');
+ try {
+ onUp();
+ } catch (err) {
+ log(`ProdCom auto-start: reconnect nudge failed (${err && err.message ? err.message : err})`);
+ }
+ return;
+ }
+ }
+ giveUp(apiOnSilent
+ ? `ProdCom is running with its API on, but port ${target().port} does not answer — check its API port`
+ : 'ProdCom would not start — open it by hand');
+ } catch (err) {
+ if (!live()) return;
+ giveUp(err && err.message ? err.message : String(err));
+ }
+ }
+
+ /** Decide whether a run is called for, and start it: the reason it did not, or null. */
+ function start(code) {
+ if (stopped) return 'stopped';
+ if (!settings().autoStart) return skip('ProdCom auto-start is off');
+ if (os.platform !== 'darwin') return skip('ProdCom auto-start is macOS only');
+ const { host } = target();
+ if (!os.isLocalHost(host)) return skip(`ProdCom at ${host} is not on this machine`);
+ if (busy) return skip('ProdCom auto-start is already working on it');
+ if (gaveUp) return skip('ProdCom auto-start has given up until ProdCom answers');
+
+ busy = true;
+ said.clear(); // a new outage says its reasons again
+ const myEpoch = epoch;
+ const controller = new AbortController();
+ abort = controller;
+ log(`ProdCom did not answer (${code}) — checking whether to start it`);
+ current = run(myEpoch, controller.signal)
+ .catch((err) => log(`ProdCom auto-start failed: ${err && err.message ? err.message : err}`))
+ .then(() => {
+ busy = false;
+ if (abort === controller) abort = null;
+ });
+ return null;
+ }
+
+ return {
+ /**
+ * A connection to ProdCom failed (`code` is the error code). Starts a run
+ * unless there is nothing to do; answers the reason when it does nothing,
+ * null when it started one.
+ */
+ kick(code) {
+ try {
+ return start(code);
+ } catch (err) {
+ // Called from the module's reconnect path: whatever goes wrong here must not stop it reconnecting.
+ return skip(`ProdCom auto-start could not decide (${err && err.message ? err.message : err})`);
+ }
+ },
+
+ /** A connection to ProdCom succeeded: this outage is over, the next gets a fresh run. */
+ noteReachable() {
+ if (busy) log('ProdCom answered on its own — ending the start-up run');
+ gaveUp = false;
+ msg = null;
+ said.clear();
+ cancelRun();
+ },
+
+ /** The supervisor's words for the tile while it is working or has given up, else null. */
+ message() {
+ return busy || gaveUp ? msg : null;
+ },
+
+ /** Resolves when no run is in progress. */
+ idle() {
+ return current;
+ },
+
+ stop() {
+ stopped = true;
+ msg = null;
+ cancelRun();
+ },
+ };
+}
+
+module.exports = { createSupervisor, defaultOs, isLocalHostName, parseApiPref, abortableSleep };
diff --git a/modules/prodcom-transcript/supervisor.test.js b/modules/prodcom-transcript/supervisor.test.js
new file mode 100644
index 0000000..a2b2356
--- /dev/null
+++ b/modules/prodcom-transcript/supervisor.test.js
@@ -0,0 +1,832 @@
+'use strict';
+
+/**
+ * Tests for the ProdCom supervisor (supervisor.js).
+ *
+ * node --test modules/prodcom-transcript/supervisor.test.js
+ *
+ * Fakes only. The decisions run on a fake machine layer (is ProdCom running,
+ * open it, its preference, quit it), a fake ping, a fake sleep, log and tile
+ * status; the real machine layer (defaultOs) runs only on a fake execFile, a
+ * fake sleep and a fake clock. Nothing here starts, quits or configures a real
+ * ProdCom, runs a command or waits on a timer. (One test aborts the real
+ * abortableSleep with an hour on it: the timer is cleared at once.)
+ */
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const {
+ createSupervisor, defaultOs, isLocalHostName, parseApiPref, abortableSleep,
+} = require('./supervisor');
+
+const START = (n) => ['connecting', `ProdCom is not running — starting it (${n}/3)`];
+const BACK = ['connecting', 'ProdCom is back — reconnecting'];
+const NO_START = ['error', 'ProdCom would not start — open it by hand'];
+const API_OFF_RESTART = ['connecting', "ProdCom's API is off — turning it on and restarting ProdCom"];
+const API_OFF_STAY = ['error', "ProdCom's API is off — restart ProdCom to turn it on"];
+const API_PORT = ['error', 'ProdCom is running with its API on, but port 24480 does not answer — check its API port'];
+const NOT_STUCK = (what) => ['error', `Could not turn on ProdCom's API — it still reads ${what}`];
+
+/**
+ * A supervisor on fakes. `calls` is every machine action in order, `sleeps` the
+ * waits asked for, `statuses` what the tile was told, `logs` what the log got,
+ * `ups` how often onUp fired. `hold` makes the Nth call of a fake hold until
+ * the test lets it go — `hold: { 'launch#1': true }`, then `r.release('launch#1')`
+ * — so a run can be looked at, or ended, in the middle of a machine call.
+ */
+function rig(o = {}) {
+ const settings = { autoStart: true, apiRestart: true, ...(o.settings || {}) };
+ const machine = { running: false, pref: null, local: true, ...(o.machine || {}) };
+ const calls = [];
+ const sleeps = [];
+ const statuses = [];
+ const logs = [];
+ const order = []; // statuses and onUp, in the order they happened
+ const probes = [...(o.probes || [])];
+ const counts = {};
+ const gates = {};
+ let probed = 0;
+ let ups = 0;
+
+ const held = (name) => {
+ counts[name] = (counts[name] || 0) + 1;
+ const key = `${name}#${counts[name]}`;
+ if (!(o.hold && o.hold[key])) return null;
+ return new Promise((resolve) => { gates[key] = resolve; });
+ };
+
+ const os = {
+ platform: o.platform || 'darwin',
+ isLocalHost: (host) => {
+ calls.push(`isLocalHost:${host}`);
+ return machine.local;
+ },
+ isRunning: async () => {
+ calls.push('isRunning');
+ const g = held('isRunning');
+ if (g) await g;
+ if (o.runningError) throw new Error(o.runningError);
+ // `runningSeq` scripts it per attempt (the last answer repeats); otherwise it is whatever `machine.running` says.
+ if (o.runningSeq) return o.runningSeq.length > 1 ? o.runningSeq.shift() : o.runningSeq[0];
+ return machine.running;
+ },
+ launch: async () => {
+ calls.push('launch');
+ const g = held('launch');
+ if (g) await g;
+ if (o.launchError) throw new Error(o.launchError);
+ },
+ readApiPref: async () => {
+ calls.push('readApiPref');
+ const g = held('readApiPref');
+ if (g) await g;
+ if (o.readError) throw new Error(o.readError);
+ return machine.pref;
+ },
+ writeApiPref: async () => {
+ calls.push('writeApiPref');
+ const g = held('writeApiPref');
+ if (g) await g;
+ if (o.writeError) throw new Error(o.writeError);
+ if (o.writeSticks !== false) machine.pref = true; // the preference now reads on, as `defaults write` makes it
+ },
+ quit: async () => {
+ calls.push('quit');
+ const g = held('quit');
+ if (g) await g;
+ if (o.quitError) throw new Error(o.quitError);
+ },
+ };
+
+ const sup = createSupervisor({
+ settings: () => settings,
+ target: () => ({ host: o.host ?? '127.0.0.1', port: 24480 }),
+ probe: async () => {
+ probed += 1;
+ const g = held('probe');
+ if (g) await g;
+ return probes.length ? probes.shift() : false;
+ },
+ os,
+ sleep: o.sleep || ((ms) => {
+ sleeps.push(ms);
+ return Promise.resolve();
+ }),
+ log: (line) => logs.push(line),
+ status: (state, message) => {
+ statuses.push([state, message]);
+ order.push(`status:${message}`);
+ },
+ onUp: () => {
+ ups += 1;
+ order.push('onUp');
+ if (o.upError) throw new Error(o.upError);
+ },
+ });
+
+ return {
+ sup, settings, machine, calls, sleeps, statuses, logs, order,
+ probed: () => probed,
+ ups: () => ups,
+ isHeld: (key) => typeof gates[key] === 'function',
+ release: (key) => gates[key](),
+ /** The machine actions, without the host check every kick starts with. */
+ actions: () => calls.filter((c) => !c.startsWith('isLocalHost')),
+ };
+}
+
+/** A sleep that waits until the test lets it go — to look at a run mid-flight. */
+function heldSleep() {
+ const held = [];
+ const sleep = (ms, signal) => new Promise((resolve) => held.push({ ms, signal, release: resolve }));
+ return { sleep, held, releaseAll: () => held.splice(0).forEach((h) => h.release()) };
+}
+
+const tick = () => new Promise((resolve) => setImmediate(resolve));
+
+/* ── a run, start to finish ─────────────────────────────────────────── */
+
+test('not running: opens ProdCom, and the first probe answers', async () => {
+ const r = rig({ probes: [true] });
+ assert.equal(r.sup.kick('ECONNREFUSED'), null);
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'launch']);
+ assert.deepEqual(r.statuses, [START(1), BACK]);
+ assert.deepEqual(r.sleeps, [5000]);
+ assert.equal(r.probed(), 1);
+ assert.ok(r.logs.includes('ProdCom is up after 1 attempt(s)'));
+ assert.equal(r.ups(), 1, 'the module is told to reconnect now');
+ assert.equal(r.order[r.order.length - 1], 'onUp', '…after the tile has been told');
+ // The run ended well: the module's own reconnect does the rest, and fail() may speak for itself again.
+ assert.equal(r.sup.message(), null);
+});
+
+test('not running: three tries that never answer, then it gives up and stays quiet', async () => {
+ const r = rig({ probes: [false, false, false] });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'launch', 'isRunning', 'launch', 'isRunning', 'launch']);
+ assert.deepEqual(r.sleeps, [5000, 5000, 5000]);
+ assert.equal(r.probed(), 3);
+ assert.deepEqual(r.statuses, [START(1), START(2), START(3), NO_START]);
+ assert.ok(r.logs.includes(NO_START[1]));
+ assert.equal(r.sup.message(), NO_START[1]); // fail() keeps this instead of "unreachable"
+ assert.equal(r.ups(), 0, 'no reconnect nudge for a run that failed');
+
+ const before = { statuses: r.statuses.length, sleeps: r.sleeps.length, probed: r.probed() };
+ for (let i = 0; i < 5; i += 1) assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
+ await r.sup.idle();
+ assert.equal(r.actions().length, 6, 'no machine action once given up');
+ assert.equal(r.statuses.length, before.statuses);
+ assert.equal(r.sleeps.length, before.sleeps);
+ assert.equal(r.probed(), before.probed);
+});
+
+test('a reconnect nudge that throws does not spoil the run', async () => {
+ const r = rig({ probes: [true], upError: 'boom' });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.statuses, [START(1), BACK]);
+ assert.ok(r.logs.some((l) => l.includes('reconnect nudge failed (boom)')));
+ assert.equal(r.sup.message(), null);
+});
+
+/* ── the API is off ─────────────────────────────────────────────────── */
+
+test('running with the API off: writes the preference, checks it took, restarts ProdCom once, then only looks and probes', async () => {
+ const r = rig({ machine: { running: true, pref: null }, probes: [false, false, true] });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), [
+ 'isRunning', 'readApiPref', 'writeApiPref', 'readApiPref', 'quit', 'launch',
+ 'isRunning', 'isRunning', // attempts 2 and 3: still there, so nothing but the probe
+ ]);
+ assert.deepEqual(r.statuses, [API_OFF_RESTART, BACK]);
+ assert.deepEqual(r.sleeps, [5000, 5000, 5000]);
+ assert.equal(r.probed(), 3);
+ assert.equal(r.calls.filter((c) => c === 'quit').length, 1);
+ assert.ok(r.logs.includes('ProdCom is up after 3 attempt(s)'));
+});
+
+test('the restart waits for the preference to read on: when it does not stick, nothing is quit', async () => {
+ for (const [pref, what] of [[null, 'unset'], [false, 'off']]) {
+ for (const apiRestart of [true, false]) {
+ const r = rig({ machine: { running: true, pref }, writeSticks: false, settings: { apiRestart } });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'writeApiPref', 'readApiPref'], `pref ${pref}, apiRestart ${apiRestart}`);
+ assert.deepEqual(r.statuses, [NOT_STUCK(what)]);
+ assert.equal(r.sleeps.length, 0);
+ assert.equal(r.probed(), 0);
+ assert.equal(r.sup.message(), NOT_STUCK(what)[1]);
+ assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
+ assert.equal(r.actions().length, 4);
+ }
+ }
+});
+
+test('a preference that cannot be written: says so, restarts nothing', async () => {
+ const r = rig({ machine: { running: true, pref: false }, writeError: "Could not turn on ProdCom's API (exit 1: no)" });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'writeApiPref']);
+ assert.deepEqual(r.statuses, [['error', "Could not turn on ProdCom's API (exit 1: no)"]]);
+});
+
+test('running with the API off and restarting not allowed: writes the preference and says so', async () => {
+ const r = rig({ machine: { running: true, pref: false }, settings: { apiRestart: false } });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'writeApiPref', 'readApiPref']);
+ assert.deepEqual(r.statuses, [API_OFF_STAY]);
+ assert.equal(r.sleeps.length, 0);
+ assert.equal(r.probed(), 0);
+ assert.equal(r.sup.message(), API_OFF_STAY[1]);
+ assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
+ assert.equal(r.actions().length, 4);
+});
+
+test('after the one restart, a ProdCom that is not running is opened again — never quit a second time', async () => {
+ const r = rig({ runningSeq: [true, false, true], machine: { pref: null }, probes: [false, false, true] });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), [
+ 'isRunning', 'readApiPref', 'writeApiPref', 'readApiPref', 'quit', 'launch',
+ 'isRunning', 'launch', // attempt 2: gone again — a launch
+ 'isRunning', // attempt 3: there; just the probe
+ ]);
+ assert.deepEqual(r.statuses, [API_OFF_RESTART, START(2), BACK]);
+ assert.equal(r.calls.filter((c) => c === 'quit').length, 1);
+ assert.ok(r.logs.includes('ProdCom is up after 3 attempt(s)'));
+});
+
+/* ── the API is on, and the port is silent ──────────────────────────── */
+
+test('running with the API on: it is looked at again each attempt, and only the last blames the port', async () => {
+ const r = rig({ machine: { running: true, pref: true } });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'isRunning', 'readApiPref', 'isRunning', 'readApiPref']);
+ assert.deepEqual(r.statuses, [API_PORT], 'no verdict before the third look');
+ assert.deepEqual(r.sleeps, [5000, 5000, 5000]);
+ assert.equal(r.probed(), 3);
+ assert.equal(r.sup.message(), API_PORT[1]);
+ assert.equal(r.calls.includes('writeApiPref'), false);
+ assert.equal(r.calls.includes('quit'), false);
+ assert.equal(r.calls.includes('launch'), false);
+});
+
+test('running with the API on, and it starts answering on the second look: no blame at all', async () => {
+ const r = rig({ machine: { running: true, pref: true }, probes: [false, true] });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'isRunning', 'readApiPref']);
+ assert.deepEqual(r.statuses, [BACK]);
+ assert.ok(r.logs.includes('ProdCom is up after 2 attempt(s)'));
+});
+
+test('running with the API on, then gone on the second look (it was quitting): it is opened', async () => {
+ const r = rig({ runningSeq: [true, false], machine: { pref: true }, probes: [false, true] });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'isRunning', 'launch']);
+ assert.deepEqual(r.statuses, [START(2), BACK]);
+ assert.ok(r.logs.includes('ProdCom is up after 2 attempt(s)'));
+});
+
+test('gone only on the last look: opened, and if it still will not answer that is "would not start"', async () => {
+ const r = rig({ runningSeq: [true, true, false], machine: { pref: true } });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'isRunning', 'readApiPref', 'isRunning', 'launch']);
+ assert.deepEqual(r.statuses, [START(3), NO_START]);
+});
+
+test('a ProdCom this run just opened, still starting up with its API on, is waited for — not blamed on the port', async () => {
+ // Attempt 1: not running → opened; the 5 s probe is false. Attempt 2: it is running now, its pref is on, the port is
+ // still silent — that is a slow start, not a wrong port.
+ const r = rig({ runningSeq: [false, true], machine: { pref: true }, probes: [false, true] });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'launch', 'isRunning', 'readApiPref']);
+ assert.deepEqual(r.statuses, [START(1), BACK], 'no error status on the way');
+ assert.deepEqual(r.sleeps, [5000, 5000]);
+ assert.equal(r.probed(), 2);
+ assert.ok(r.logs.includes('ProdCom is up after 2 attempt(s)'));
+ assert.equal(r.calls.includes('quit'), false);
+ assert.equal(r.calls.includes('writeApiPref'), false);
+});
+
+test('a ProdCom this run just opened that turns out to have its API off still gets the one restart', async () => {
+ const r = rig({ runningSeq: [false, true], machine: { pref: null }, probes: [false, false, true] });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), [
+ 'isRunning', 'launch',
+ 'isRunning', 'readApiPref', 'writeApiPref', 'readApiPref', 'quit', 'launch',
+ 'isRunning',
+ ]);
+ assert.deepEqual(r.statuses, [START(1), API_OFF_RESTART, BACK]);
+ assert.equal(r.calls.filter((c) => c === 'quit').length, 1);
+ assert.ok(r.logs.includes('ProdCom is up after 3 attempt(s)'));
+});
+
+test('a ProdCom this run opened that never answers, API on: three tries, then "would not start"', async () => {
+ const r = rig({ runningSeq: [false, true, true], machine: { pref: true }, probes: [false, false, false] });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'launch', 'isRunning', 'readApiPref', 'isRunning', 'readApiPref']);
+ assert.deepEqual(r.statuses, [START(1), NO_START]);
+});
+
+/* ── when there is nothing to do ────────────────────────────────────── */
+
+test('nothing to do for a ProdCom on another machine', async () => {
+ const r = rig({ host: '10.1.1.50', machine: { local: false } });
+ const why = r.sup.kick('ECONNREFUSED');
+ assert.equal(why, 'ProdCom at 10.1.1.50 is not on this machine');
+ for (let i = 0; i < 4; i += 1) r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), []);
+ assert.equal(r.statuses.length, 0);
+ assert.equal(r.logs.filter((l) => l === why).length, 1, 'the reason is logged once, not on every tick');
+ assert.equal(r.sup.message(), null);
+});
+
+test('nothing to do when switched off', async () => {
+ const r = rig({ settings: { autoStart: false } });
+ assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.calls, []);
+ assert.equal(r.statuses.length, 0);
+ assert.equal(r.logs.length, 1);
+});
+
+test('the setting is read live: switching it on later takes effect without a new supervisor', async () => {
+ const r = rig({ settings: { autoStart: false }, probes: [true] });
+ assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
+ r.settings.autoStart = true;
+ assert.equal(r.sup.kick('ECONNREFUSED'), null);
+ await r.sup.idle();
+ assert.deepEqual(r.statuses, [START(1), BACK]);
+});
+
+test('nothing to do off macOS', async () => {
+ const r = rig({ platform: 'win32' });
+ assert.equal(r.sup.kick('ECONNREFUSED'), 'ProdCom auto-start is macOS only');
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), []);
+ assert.equal(r.statuses.length, 0);
+ assert.equal(r.logs.filter((l) => l === 'ProdCom auto-start is macOS only').length, 1);
+});
+
+test('a kick while a run is in progress does nothing', async () => {
+ const h = heldSleep();
+ const r = rig({ sleep: h.sleep, probes: [true] });
+ assert.equal(r.sup.kick('ECONNREFUSED'), null);
+ await tick();
+ assert.equal(h.held.length, 1, 'the run is waiting out its five seconds');
+ assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
+ assert.equal(typeof r.sup.kick('ETIMEDOUT'), 'string');
+ assert.equal(r.sup.message(), START(1)[1], 'while it works, its words are the tile\'s');
+ assert.deepEqual(r.actions(), ['isRunning', 'launch'], 'no second run started');
+ h.releaseAll();
+ await r.sup.idle();
+ assert.deepEqual(r.statuses, [START(1), BACK]);
+});
+
+test('kick() never throws into the reconnect loop, whatever the machine layer does', async () => {
+ const r = rig();
+ const sup = createSupervisor({
+ settings: () => r.settings,
+ target: () => ({ host: '127.0.0.1', port: 24480 }),
+ os: { platform: 'darwin', isLocalHost: () => { throw new Error('no network interfaces'); } },
+ probe: async () => true,
+ sleep: () => Promise.resolve(),
+ log: (line) => r.logs.push(line),
+ status: (s, m) => r.statuses.push([s, m]),
+ });
+ assert.equal(sup.kick('ECONNREFUSED'), 'ProdCom auto-start could not decide (no network interfaces)');
+ assert.equal(r.statuses.length, 0);
+});
+
+/* ── an outage ends, a new one begins ───────────────────────────────── */
+
+test('after giving up, a successful connection re-arms it for the next outage', async () => {
+ const r = rig({ machine: { running: true, pref: true } });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ const first = r.actions().length;
+ assert.equal(first, 6);
+ assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
+ assert.equal(r.actions().length, first);
+
+ r.sup.noteReachable();
+ assert.equal(r.sup.message(), null);
+ assert.equal(r.sup.kick('ECONNREFUSED'), null);
+ await r.sup.idle();
+ assert.equal(r.actions().length, first * 2);
+ assert.deepEqual(r.statuses, [API_PORT, API_PORT]);
+});
+
+test('a second outage logs its reasons again', async () => {
+ const r = rig({ host: '10.1.1.50', machine: { local: false } });
+ const why = 'ProdCom at 10.1.1.50 is not on this machine';
+ r.sup.kick('ECONNREFUSED');
+ r.sup.kick('ECONNREFUSED');
+ assert.equal(r.logs.filter((l) => l === why).length, 1);
+ r.sup.noteReachable(); // the first outage is over
+ r.sup.kick('ECONNREFUSED');
+ r.sup.kick('ECONNREFUSED');
+ assert.equal(r.logs.filter((l) => l === why).length, 2);
+
+ // And a run that starts clears them too: "has given up" is said once per outage, not once ever.
+ const g = rig({ machine: { running: true, pref: true } });
+ const gaveUp = 'ProdCom auto-start has given up until ProdCom answers';
+ g.sup.kick('ECONNREFUSED');
+ await g.sup.idle();
+ g.sup.kick('ECONNREFUSED');
+ g.sup.kick('ECONNREFUSED');
+ assert.equal(g.logs.filter((l) => l === gaveUp).length, 1);
+ g.sup.noteReachable();
+ g.sup.kick('ECONNREFUSED');
+ await g.sup.idle();
+ g.sup.kick('ECONNREFUSED');
+ assert.equal(g.logs.filter((l) => l === gaveUp).length, 2);
+});
+
+test('a run that starts clears the reasons already logged, even with no connection in between', async () => {
+ const h = heldSleep();
+ const r = rig({ sleep: h.sleep, probes: [true, true] });
+ const busy = 'ProdCom auto-start is already working on it';
+ const said = () => r.logs.filter((l) => l === busy).length;
+ r.sup.kick('ECONNREFUSED');
+ await tick();
+ r.sup.kick('ECONNREFUSED');
+ r.sup.kick('ECONNREFUSED');
+ assert.equal(said(), 1);
+ h.releaseAll();
+ await r.sup.idle(); // the run ended well; the module has not connected (noteReachable) yet
+ assert.equal(r.sup.kick('ECONNREFUSED'), null, 'a new run');
+ await tick();
+ r.sup.kick('ECONNREFUSED');
+ assert.equal(said(), 2, 'its reason is logged again');
+ h.releaseAll();
+ await r.sup.idle();
+});
+
+test('a connection that succeeds mid-run ends the run without another word', async () => {
+ const h = heldSleep();
+ const r = rig({ sleep: h.sleep, probes: [true] });
+ r.sup.kick('ECONNREFUSED');
+ await tick();
+ assert.deepEqual(r.statuses, [START(1)]);
+ r.sup.noteReachable(); // the module got through on its own while we waited
+ h.releaseAll();
+ await r.sup.idle();
+ assert.equal(r.probed(), 0, 'no probe after the outage was over');
+ assert.deepEqual(r.statuses, [START(1)], 'nothing overwrites the module\'s own "Live"');
+ assert.equal(r.ups(), 0);
+ assert.equal(r.sup.message(), null);
+ assert.equal(r.sup.kick('ECONNREFUSED'), null, 'and the next outage is a fresh run');
+ await tick();
+ assert.deepEqual(r.statuses, [START(1), START(1)]);
+ r.sup.stop();
+ await r.sup.idle();
+});
+
+/* ── stop() and noteReachable() land mid-run ────────────────────────── */
+
+test('stop() during the sleep ends the run: no more launches, no probes', async () => {
+ const h = heldSleep();
+ const r = rig({ sleep: h.sleep, probes: [true] });
+ r.sup.kick('ECONNREFUSED');
+ await tick();
+ assert.equal(h.held.length, 1);
+ assert.equal(h.held[0].signal.aborted, false);
+ r.sup.stop();
+ assert.equal(h.held[0].signal.aborted, true, 'the sleep was told to cut itself short');
+ await r.sup.idle(); // settles even though the fake sleep never does
+ assert.deepEqual(r.actions(), ['isRunning', 'launch']);
+ assert.equal(r.probed(), 0);
+ assert.deepEqual(r.statuses, [START(1)]);
+ assert.equal(r.sup.message(), null);
+ assert.equal(r.sup.kick('ECONNREFUSED'), 'stopped');
+ h.releaseAll();
+ await tick();
+ assert.deepEqual(r.actions(), ['isRunning', 'launch']);
+ assert.equal(r.probed(), 0);
+ assert.equal(r.ups(), 0);
+});
+
+// Each machine call a run makes, held in flight; the run is ended (by the module
+// connecting, or by stop()) while the call is out, and nothing may follow it:
+// no status, no log, no further call, no reconnect nudge.
+const MID_CALL = [
+ ['isRunning', 'isRunning#1', { machine: { running: false } }],
+ ['launch', 'launch#1', { machine: { running: false } }],
+ ['readApiPref', 'readApiPref#1', { machine: { running: true, pref: null } }],
+ ['writeApiPref', 'writeApiPref#1', { machine: { running: true, pref: null } }],
+ ['readApiPref (the check that it took)', 'readApiPref#2', { machine: { running: true, pref: null } }],
+ ['quit', 'quit#1', { machine: { running: true, pref: null } }],
+ ['launch (after the quit)', 'launch#1', { machine: { running: true, pref: null } }],
+ ['probe', 'probe#1', { machine: { running: false }, probes: [true] }],
+];
+for (const [label, key, setup] of MID_CALL) {
+ for (const how of ['noteReachable', 'stop']) {
+ test(`${how}() while ${label} is in flight: nothing follows`, async () => {
+ const r = rig({ ...setup, hold: { [key]: true } });
+ r.sup.kick('ECONNREFUSED');
+ await tick();
+ assert.ok(r.isHeld(key), `${key} is in flight`);
+ r.sup[how]();
+ const snapshot = { calls: [...r.calls], statuses: [...r.statuses], logs: [...r.logs], order: [...r.order], sleeps: [...r.sleeps] };
+ r.release(key);
+ await r.sup.idle();
+ await tick();
+ assert.deepEqual(r.calls, snapshot.calls, 'no further machine call');
+ assert.deepEqual(r.statuses, snapshot.statuses, 'no status');
+ assert.deepEqual(r.logs, snapshot.logs, 'no log line');
+ assert.deepEqual(r.order, snapshot.order, 'no nudge');
+ assert.deepEqual(r.sleeps, snapshot.sleeps, 'no further wait');
+ assert.equal(r.probed(), key === 'probe#1' ? 1 : 0, 'no further probe');
+ assert.equal(r.ups(), 0);
+ });
+ }
+}
+
+// …and the same when the call that was in flight ends in an error: the catch
+// must not report a failure of a run that is already over.
+const MID_CALL_ERROR = [
+ ['isRunning', 'isRunning#1', { machine: { running: false }, runningError: 'pgrep failed' }],
+ ['launch', 'launch#1', { machine: { running: false }, launchError: 'ProdCom is not installed on this machine' }],
+ ['readApiPref', 'readApiPref#1', { machine: { running: true, pref: null }, readError: 'defaults failed' }],
+ ['writeApiPref', 'writeApiPref#1', { machine: { running: true, pref: null }, writeError: 'write failed' }],
+ ['quit', 'quit#1', { machine: { running: true, pref: null }, quitError: 'ProdCom would not quit' }],
+ ['launch (after the quit)', 'launch#1', { machine: { running: true, pref: null }, launchError: 'launch failed' }],
+];
+for (const [label, key, setup] of MID_CALL_ERROR) {
+ for (const how of ['noteReachable', 'stop']) {
+ test(`${how}() while ${label} is in flight and then fails: no error is reported`, async () => {
+ const r = rig({ ...setup, hold: { [key]: true } });
+ r.sup.kick('ECONNREFUSED');
+ await tick();
+ assert.ok(r.isHeld(key), `${key} is in flight`);
+ r.sup[how]();
+ const snapshot = { calls: [...r.calls], statuses: [...r.statuses], logs: [...r.logs] };
+ r.release(key);
+ await r.sup.idle();
+ await tick();
+ assert.deepEqual(r.calls, snapshot.calls);
+ assert.deepEqual(r.statuses, snapshot.statuses, 'no error status');
+ assert.deepEqual(r.logs, snapshot.logs, 'no error log');
+ assert.equal(r.sup.message(), null, 'and nothing latched as given up');
+ if (how === 'noteReachable') assert.equal(r.sup.kick('ECONNREFUSED'), null, 'the next outage still gets a run');
+ r.sup.stop();
+ await r.sup.idle();
+ });
+ }
+}
+
+test('ProdCom is not installed: says so and gives up', async () => {
+ const r = rig({ launchError: 'ProdCom is not installed on this machine' });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'launch']);
+ assert.deepEqual(r.statuses, [['error', 'ProdCom is not installed on this machine']]);
+ assert.equal(r.sleeps.length, 0);
+ assert.equal(r.probed(), 0);
+ assert.equal(r.sup.message(), 'ProdCom is not installed on this machine');
+ assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
+ assert.equal(r.actions().length, 2);
+});
+
+test('ProdCom will not quit: says so, does not open it again, gives up', async () => {
+ const r = rig({ machine: { running: true, pref: null }, quitError: 'ProdCom would not quit' });
+ r.sup.kick('ECONNREFUSED');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'writeApiPref', 'readApiPref', 'quit']);
+ assert.deepEqual(r.statuses, [API_OFF_RESTART, ['error', 'ProdCom would not quit']]);
+ assert.equal(r.sleeps.length, 0);
+ assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
+});
+
+test('a machine call that fails for another reason gives up with its message', async () => {
+ const r = rig({ machine: { running: true }, readError: "Could not read ProdCom's settings (timed out)" });
+ r.sup.kick('ETIMEDOUT');
+ await r.sup.idle();
+ assert.deepEqual(r.actions(), ['isRunning', 'readApiPref'], 'not knowing the setting is not a reason to restart ProdCom');
+ assert.deepEqual(r.statuses, [['error', "Could not read ProdCom's settings (timed out)"]]);
+});
+
+/* ── the real machine layer, on a fake execFile ─────────────────────── */
+
+const ok = (stdout = '') => ({ stdout });
+const exited = (code, stderr = '') => ({ error: Object.assign(new Error('Command failed'), { code }), stderr });
+const timedOut = () => ({ error: Object.assign(new Error('timed out'), { killed: true, signal: 'SIGTERM', code: null }) });
+const BUNDLE = 'com.prodcom.prodcom';
+
+/** A fake child_process.execFile answering from `script(file, args, n)` (a result, or a promise of one). */
+function fakeExecFile(script) {
+ const calls = [];
+ const execFile = (file, args, opts, cb) => {
+ calls.push({ file, args, opts, line: [file, ...args].join(' ') });
+ Promise.resolve(script(file, args, calls.length)).then((res) => {
+ const r = res || {};
+ cb(r.error || null, r.stdout || '', r.stderr || '');
+ });
+ };
+ return { execFile, calls, lines: () => calls.map((c) => c.line) };
+}
+
+/** A fake clock: sleeping advances it, so a wait that would last ten seconds takes no time. */
+function fakeClock() {
+ let t = 0;
+ const slept = [];
+ return {
+ now: () => t,
+ sleep: (ms) => {
+ slept.push(ms);
+ t += ms;
+ return Promise.resolve(true);
+ },
+ slept,
+ };
+}
+
+test('default os: reading the preference — 1, 0, a missing key, a timeout', async () => {
+ const answers = [ok('1\n'), ok('0\n'), exited(1, 'The domain/default pair of (com.prodcom.prodcom, applicationAPIEnabled) does not exist'), timedOut()];
+ const x = fakeExecFile(() => answers.shift());
+ const os = defaultOs({ execFile: x.execFile });
+ assert.equal(await os.readApiPref(), true);
+ assert.equal(await os.readApiPref(), false);
+ assert.equal(await os.readApiPref(), null, 'a missing key is "not set", not an error');
+ await assert.rejects(os.readApiPref(), /^Error: Could not read ProdCom's settings \(timed out\)$/);
+ assert.equal(x.lines()[0], `/usr/bin/defaults read ${BUNDLE} applicationAPIEnabled`);
+ assert.equal(x.calls[0].opts.timeout, 5000);
+});
+
+test('default os: writing the preference, and what a failure says', async () => {
+ const x = fakeExecFile((file, args, n) => (n === 1 ? ok() : exited(1, '\n Could not write domain com.prodcom.prodcom; exiting\nmore')));
+ const os = defaultOs({ execFile: x.execFile });
+ await os.writeApiPref();
+ assert.equal(x.lines()[0], `/usr/bin/defaults write ${BUNDLE} applicationAPIEnabled -bool true`);
+ await assert.rejects(os.writeApiPref(), /^Error: Could not turn on ProdCom's API \(exit 1: Could not write domain com\.prodcom\.prodcom; exiting\)$/);
+});
+
+test('default os: opening ProdCom is done in the background (-g), and a failed open reads as not installed', async () => {
+ const answers = [ok(), exited(1, 'Unable to find application'), timedOut()];
+ const x = fakeExecFile(() => answers.shift());
+ const os = defaultOs({ execFile: x.execFile });
+ await os.launch();
+ assert.equal(x.lines()[0], `/usr/bin/open -g -b ${BUNDLE}`);
+ assert.ok(x.calls[0].args.includes('-g'));
+ await assert.rejects(os.launch(), /^Error: ProdCom is not installed on this machine$/);
+ await assert.rejects(os.launch(), /^Error: Could not open ProdCom \(timed out\)$/);
+});
+
+test('default os: is ProdCom running — pgrep 0 / 1, anything else is an error', async () => {
+ const answers = [ok('123\n'), exited(1), exited(2, 'pgrep: bad'), timedOut()];
+ const x = fakeExecFile(() => answers.shift());
+ const os = defaultOs({ execFile: x.execFile });
+ assert.equal(await os.isRunning(), true);
+ assert.equal(await os.isRunning(), false);
+ await assert.rejects(os.isRunning(), /Could not check whether ProdCom is running \(exit 2: pgrep: bad\)/);
+ await assert.rejects(os.isRunning(), /Could not check whether ProdCom is running \(timed out\)/);
+ assert.equal(x.lines()[0], '/usr/bin/pgrep -x ProdCom');
+});
+
+test('default os: quit, the polite way — asked by Apple Event, then waited for', async () => {
+ let pgreps = 0;
+ const x = fakeExecFile((file) => {
+ if (file === '/usr/bin/osascript') return ok();
+ pgreps += 1;
+ return pgreps < 3 ? ok('123\n') : exited(1); // still there twice, then gone
+ });
+ const clock = fakeClock();
+ const os = defaultOs({ execFile: x.execFile, sleep: clock.sleep, now: clock.now });
+ await os.quit(new AbortController().signal);
+ assert.deepEqual(x.lines(), [
+ `/usr/bin/osascript -e tell application id "${BUNDLE}" to quit`,
+ '/usr/bin/pgrep -x ProdCom', '/usr/bin/pgrep -x ProdCom', '/usr/bin/pgrep -x ProdCom',
+ ]);
+ assert.equal(x.calls[0].opts.timeout, 5000, 'the Apple Event gets five seconds');
+ assert.deepEqual(clock.slept, [500, 500]);
+});
+
+test('default os: quit, when it will not be asked — SIGTERM, then waited for', async () => {
+ let pgreps = 0;
+ const x = fakeExecFile((file) => {
+ if (file === '/usr/bin/osascript') return exited(1, 'execution error: Not authorized to send Apple events (-1743)');
+ if (file === '/usr/bin/pkill') return ok();
+ pgreps += 1;
+ return pgreps < 2 ? ok('123\n') : exited(1);
+ });
+ const clock = fakeClock();
+ const os = defaultOs({ execFile: x.execFile, sleep: clock.sleep, now: clock.now });
+ await os.quit();
+ assert.deepEqual(x.lines().map((l) => l.split(' ').slice(0, 3).join(' ')), [
+ '/usr/bin/osascript -e tell',
+ '/usr/bin/pkill -x -TERM',
+ '/usr/bin/pgrep -x ProdCom',
+ '/usr/bin/pgrep -x ProdCom',
+ ]);
+ assert.deepEqual(x.calls[1].args, ['-x', '-TERM', 'ProdCom']);
+});
+
+test('default os: quit, and it will not go — "would not quit" after ten seconds', async () => {
+ const x = fakeExecFile((file) => (file === '/usr/bin/pgrep' ? ok('123\n') : ok()));
+ const clock = fakeClock();
+ const os = defaultOs({ execFile: x.execFile, sleep: clock.sleep, now: clock.now });
+ await assert.rejects(os.quit(), /^Error: ProdCom would not quit$/);
+ assert.equal(clock.now(), 10000);
+ assert.equal(x.calls.filter((c) => c.file === '/usr/bin/pkill').length, 0, 'the Apple Event worked, so no SIGTERM');
+});
+
+test('default os: quit, ended while the Apple Event is out: no SIGTERM, no look at the process list', async () => {
+ const ctl = new AbortController();
+ let finish;
+ const x = fakeExecFile((file) => {
+ if (file === '/usr/bin/osascript') return new Promise((resolve) => { finish = () => resolve(exited(1, 'denied')); });
+ return ok('123\n');
+ });
+ const os = defaultOs({ execFile: x.execFile, sleep: fakeClock().sleep });
+ const quitting = os.quit(ctl.signal);
+ await tick();
+ ctl.abort();
+ finish();
+ await assert.rejects(quitting, /^Error: cancelled$/);
+ assert.equal(x.calls.length, 1, 'only the osascript was ever run');
+});
+
+test('default os: quit, ended while SIGTERM is out: no look at the process list', async () => {
+ const ctl = new AbortController();
+ let finish;
+ const x = fakeExecFile((file) => {
+ if (file === '/usr/bin/osascript') return exited(1, 'denied');
+ if (file === '/usr/bin/pkill') return new Promise((resolve) => { finish = () => resolve(ok()); });
+ return ok('123\n');
+ });
+ const os = defaultOs({ execFile: x.execFile, sleep: fakeClock().sleep });
+ const quitting = os.quit(ctl.signal);
+ await tick();
+ ctl.abort();
+ finish();
+ await assert.rejects(quitting, /^Error: cancelled$/);
+ assert.deepEqual(x.calls.map((c) => c.file), ['/usr/bin/osascript', '/usr/bin/pkill']);
+});
+
+test('default os: quit, ended while it waits for the process to go: no further look', async () => {
+ const ctl = new AbortController();
+ const x = fakeExecFile((file) => (file === '/usr/bin/pgrep' ? ok('123\n') : ok()));
+ const os = defaultOs({
+ execFile: x.execFile,
+ sleep: () => { ctl.abort(); return Promise.resolve(false); }, // the run ends during the first wait
+ });
+ await assert.rejects(os.quit(ctl.signal), /^Error: cancelled$/);
+ assert.equal(x.calls.filter((c) => c.file === '/usr/bin/pgrep').length, 1);
+});
+
+test('default os: quit, already ended before it starts: it does nothing at all', async () => {
+ const ctl = new AbortController();
+ ctl.abort();
+ const x = fakeExecFile(() => ok());
+ const os = defaultOs({ execFile: x.execFile });
+ await assert.rejects(os.quit(ctl.signal), /^Error: cancelled$/);
+ assert.equal(x.calls.length, 0);
+});
+
+/* ── the pieces that need no machine ────────────────────────────────── */
+
+test('isLocalHostName: this machine by every name it has, and no other', () => {
+ const addrs = ['127.0.0.1', '::1', 'fe80::1', '10.3.11.152', 'fe80::aede:48ff:fe00:1122'];
+ const here = 'AB-ProPresenter.local';
+ for (const host of ['', 'localhost', 'LocalHost', '127.0.0.1', '::1', '[::1]', '0.0.0.0',
+ '10.3.11.152', 'FE80::1', 'fe80:0:0:0:0:0:0:1', 'fe80::1%lo0',
+ 'ab-propresenter.local', 'AB-PROPRESENTER', 'ab-propresenter', 'ab-propresenter.local.']) {
+ assert.equal(isLocalHostName(host, here, addrs), true, `"${host}" is this machine`);
+ }
+ for (const host of ['10.1.1.50', 'prodcom.local', 'na-ingest', 'example.com', '10.3.11.153']) {
+ assert.equal(isLocalHostName(host, here, addrs), false, `"${host}" is not`);
+ }
+ // A hostname without .local matches the same machine reached as .local.
+ assert.equal(isLocalHostName('studio.local', 'Studio', []), true);
+});
+
+test('parseApiPref: defaults prints a bool as 1 or 0', () => {
+ assert.equal(parseApiPref('1\n'), true);
+ assert.equal(parseApiPref('0\n'), false);
+ assert.equal(parseApiPref(' true '), true);
+ assert.equal(parseApiPref(''), null);
+ assert.equal(parseApiPref('banana'), null);
+});
+
+test('the default sleep is cut short by its signal and leaves no timer behind', async () => {
+ const ctl = new AbortController();
+ const slept = abortableSleep(60 * 60 * 1000, ctl.signal); // an hour — would hold the test open if it were left running
+ ctl.abort();
+ assert.equal(await slept, false);
+ assert.equal(await abortableSleep(60 * 60 * 1000, ctl.signal), false, 'already aborted: returns at once');
+});