ProdDash/modules/prodcom-transcript/supervisor.test.js

'use strict';

/**
 * Tests for the ProdCom supervisor (supervisor.js).
 *
 *   node --test modules/prodcom-transcript/supervisor.test.js
 *
 * Fakes only. The decisions run on a fake machine layer (is ProdCom running,
 * open it, its preference, quit it), a fake ping, a fake sleep, log and tile
 * status; the real machine layer (defaultOs) runs only on a fake execFile, a
 * fake sleep and a fake clock. Nothing here starts, quits or configures a real
 * ProdCom, runs a command or waits on a timer. (One test aborts the real
 * abortableSleep with an hour on it: the timer is cleared at once.)
 */

const test = require('node:test');
const assert = require('node:assert/strict');
const {
  createSupervisor, defaultOs, isLocalHostName, parseApiPref, abortableSleep,
} = require('./supervisor');

const START = (n) => ['connecting', `ProdCom is not running — starting it (${n}/3)`];
const BACK = ['connecting', 'ProdCom is back — reconnecting'];
const NO_START = ['error', 'ProdCom would not start — open it by hand'];
const API_OFF_RESTART = ['connecting', "ProdCom's API is off — turning it on and restarting ProdCom"];
const API_OFF_STAY = ['error', "ProdCom's API is off — restart ProdCom to turn it on"];
const API_PORT = ['error', 'ProdCom is running with its API on, but port 24480 does not answer — check its API port'];
const NOT_STUCK = (what) => ['error', `Could not turn on ProdCom's API — it still reads ${what}`];

/**
 * A supervisor on fakes. `calls` is every machine action in order, `sleeps` the
 * waits asked for, `statuses` what the tile was told, `logs` what the log got,
 * `ups` how often onUp fired. `hold` makes the Nth call of a fake hold until
 * the test lets it go — `hold: { 'launch#1': true }`, then `r.release('launch#1')`
 * — so a run can be looked at, or ended, in the middle of a machine call.
 */
function rig(o = {}) {
  const settings = { autoStart: true, apiRestart: true, ...(o.settings || {}) };
  const machine = { running: false, pref: null, local: true, ...(o.machine || {}) };
  const calls = [];
  const sleeps = [];
  const statuses = [];
  const logs = [];
  const order = []; // statuses and onUp, in the order they happened
  const probes = [...(o.probes || [])];
  const counts = {};
  const gates = {};
  let probed = 0;
  let ups = 0;

  const held = (name) => {
    counts[name] = (counts[name] || 0) + 1;
    const key = `${name}#${counts[name]}`;
    if (!(o.hold && o.hold[key])) return null;
    return new Promise((resolve) => { gates[key] = resolve; });
  };

  const os = {
    platform: o.platform || 'darwin',
    isLocalHost: (host) => {
      calls.push(`isLocalHost:${host}`);
      return machine.local;
    },
    isRunning: async () => {
      calls.push('isRunning');
      const g = held('isRunning');
      if (g) await g;
      if (o.runningError) throw new Error(o.runningError);
      // `runningSeq` scripts it per attempt (the last answer repeats); otherwise it is whatever `machine.running` says.
      if (o.runningSeq) return o.runningSeq.length > 1 ? o.runningSeq.shift() : o.runningSeq[0];
      return machine.running;
    },
    launch: async () => {
      calls.push('launch');
      const g = held('launch');
      if (g) await g;
      if (o.launchError) throw new Error(o.launchError);
    },
    readApiPref: async () => {
      calls.push('readApiPref');
      const g = held('readApiPref');
      if (g) await g;
      if (o.readError) throw new Error(o.readError);
      return machine.pref;
    },
    writeApiPref: async () => {
      calls.push('writeApiPref');
      const g = held('writeApiPref');
      if (g) await g;
      if (o.writeError) throw new Error(o.writeError);
      if (o.writeSticks !== false) machine.pref = true; // the preference now reads on, as `defaults write` makes it
    },
    quit: async () => {
      calls.push('quit');
      const g = held('quit');
      if (g) await g;
      if (o.quitError) throw new Error(o.quitError);
    },
  };

  const sup = createSupervisor({
    settings: () => settings,
    target: () => ({ host: o.host ?? '127.0.0.1', port: 24480 }),
    probe: async () => {
      probed += 1;
      const g = held('probe');
      if (g) await g;
      return probes.length ? probes.shift() : false;
    },
    os,
    sleep: o.sleep || ((ms) => {
      sleeps.push(ms);
      return Promise.resolve();
    }),
    log: (line) => logs.push(line),
    status: (state, message) => {
      statuses.push([state, message]);
      order.push(`status:${message}`);
    },
    onUp: () => {
      ups += 1;
      order.push('onUp');
      if (o.upError) throw new Error(o.upError);
    },
  });

  return {
    sup, settings, machine, calls, sleeps, statuses, logs, order,
    probed: () => probed,
    ups: () => ups,
    isHeld: (key) => typeof gates[key] === 'function',
    release: (key) => gates[key](),
    /** The machine actions, without the host check every kick starts with. */
    actions: () => calls.filter((c) => !c.startsWith('isLocalHost')),
  };
}

/** A sleep that waits until the test lets it go — to look at a run mid-flight. */
function heldSleep() {
  const held = [];
  const sleep = (ms, signal) => new Promise((resolve) => held.push({ ms, signal, release: resolve }));
  return { sleep, held, releaseAll: () => held.splice(0).forEach((h) => h.release()) };
}

const tick = () => new Promise((resolve) => setImmediate(resolve));

/* ── a run, start to finish ─────────────────────────────────────────── */

test('not running: opens ProdCom, and the first probe answers', async () => {
  const r = rig({ probes: [true] });
  assert.equal(r.sup.kick('ECONNREFUSED'), null);
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'launch']);
  assert.deepEqual(r.statuses, [START(1), BACK]);
  assert.deepEqual(r.sleeps, [5000]);
  assert.equal(r.probed(), 1);
  assert.ok(r.logs.includes('ProdCom is up after 1 attempt(s)'));
  assert.equal(r.ups(), 1, 'the module is told to reconnect now');
  assert.equal(r.order[r.order.length - 1], 'onUp', '…after the tile has been told');
  // The run ended well: the module's own reconnect does the rest, and fail() may speak for itself again.
  assert.equal(r.sup.message(), null);
});

test('not running: three tries that never answer, then it gives up and stays quiet', async () => {
  const r = rig({ probes: [false, false, false] });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'launch', 'isRunning', 'launch', 'isRunning', 'launch']);
  assert.deepEqual(r.sleeps, [5000, 5000, 5000]);
  assert.equal(r.probed(), 3);
  assert.deepEqual(r.statuses, [START(1), START(2), START(3), NO_START]);
  assert.ok(r.logs.includes(NO_START[1]));
  assert.equal(r.sup.message(), NO_START[1]); // fail() keeps this instead of "unreachable"
  assert.equal(r.ups(), 0, 'no reconnect nudge for a run that failed');

  const before = { statuses: r.statuses.length, sleeps: r.sleeps.length, probed: r.probed() };
  for (let i = 0; i < 5; i += 1) assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
  await r.sup.idle();
  assert.equal(r.actions().length, 6, 'no machine action once given up');
  assert.equal(r.statuses.length, before.statuses);
  assert.equal(r.sleeps.length, before.sleeps);
  assert.equal(r.probed(), before.probed);
});

test('a reconnect nudge that throws does not spoil the run', async () => {
  const r = rig({ probes: [true], upError: 'boom' });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.statuses, [START(1), BACK]);
  assert.ok(r.logs.some((l) => l.includes('reconnect nudge failed (boom)')));
  assert.equal(r.sup.message(), null);
});

/* ── the API is off ─────────────────────────────────────────────────── */

test('running with the API off: writes the preference, checks it took, restarts ProdCom once, then only looks and probes', async () => {
  const r = rig({ machine: { running: true, pref: null }, probes: [false, false, true] });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), [
    'isRunning', 'readApiPref', 'writeApiPref', 'readApiPref', 'quit', 'launch',
    'isRunning', 'isRunning', // attempts 2 and 3: still there, so nothing but the probe
  ]);
  assert.deepEqual(r.statuses, [API_OFF_RESTART, BACK]);
  assert.deepEqual(r.sleeps, [5000, 5000, 5000]);
  assert.equal(r.probed(), 3);
  assert.equal(r.calls.filter((c) => c === 'quit').length, 1);
  assert.ok(r.logs.includes('ProdCom is up after 3 attempt(s)'));
});

test('the restart waits for the preference to read on: when it does not stick, nothing is quit', async () => {
  for (const [pref, what] of [[null, 'unset'], [false, 'off']]) {
    for (const apiRestart of [true, false]) {
      const r = rig({ machine: { running: true, pref }, writeSticks: false, settings: { apiRestart } });
      r.sup.kick('ECONNREFUSED');
      await r.sup.idle();
      assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'writeApiPref', 'readApiPref'], `pref ${pref}, apiRestart ${apiRestart}`);
      assert.deepEqual(r.statuses, [NOT_STUCK(what)]);
      assert.equal(r.sleeps.length, 0);
      assert.equal(r.probed(), 0);
      assert.equal(r.sup.message(), NOT_STUCK(what)[1]);
      assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
      assert.equal(r.actions().length, 4);
    }
  }
});

test('a preference that cannot be written: says so, restarts nothing', async () => {
  const r = rig({ machine: { running: true, pref: false }, writeError: "Could not turn on ProdCom's API (exit 1: no)" });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'writeApiPref']);
  assert.deepEqual(r.statuses, [['error', "Could not turn on ProdCom's API (exit 1: no)"]]);
});

test('running with the API off and restarting not allowed: writes the preference and says so', async () => {
  const r = rig({ machine: { running: true, pref: false }, settings: { apiRestart: false } });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'writeApiPref', 'readApiPref']);
  assert.deepEqual(r.statuses, [API_OFF_STAY]);
  assert.equal(r.sleeps.length, 0);
  assert.equal(r.probed(), 0);
  assert.equal(r.sup.message(), API_OFF_STAY[1]);
  assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
  assert.equal(r.actions().length, 4);
});

test('after the one restart, a ProdCom that is not running is opened again — never quit a second time', async () => {
  const r = rig({ runningSeq: [true, false, true], machine: { pref: null }, probes: [false, false, true] });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), [
    'isRunning', 'readApiPref', 'writeApiPref', 'readApiPref', 'quit', 'launch',
    'isRunning', 'launch', // attempt 2: gone again — a launch
    'isRunning',           // attempt 3: there; just the probe
  ]);
  assert.deepEqual(r.statuses, [API_OFF_RESTART, START(2), BACK]);
  assert.equal(r.calls.filter((c) => c === 'quit').length, 1);
  assert.ok(r.logs.includes('ProdCom is up after 3 attempt(s)'));
});

/* ── the API is on, and the port is silent ──────────────────────────── */

test('running with the API on: it is looked at again each attempt, and only the last blames the port', async () => {
  const r = rig({ machine: { running: true, pref: true } });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'isRunning', 'readApiPref', 'isRunning', 'readApiPref']);
  assert.deepEqual(r.statuses, [API_PORT], 'no verdict before the third look');
  assert.deepEqual(r.sleeps, [5000, 5000, 5000]);
  assert.equal(r.probed(), 3);
  assert.equal(r.sup.message(), API_PORT[1]);
  assert.equal(r.calls.includes('writeApiPref'), false);
  assert.equal(r.calls.includes('quit'), false);
  assert.equal(r.calls.includes('launch'), false);
});

test('running with the API on, and it starts answering on the second look: no blame at all', async () => {
  const r = rig({ machine: { running: true, pref: true }, probes: [false, true] });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'isRunning', 'readApiPref']);
  assert.deepEqual(r.statuses, [BACK]);
  assert.ok(r.logs.includes('ProdCom is up after 2 attempt(s)'));
});

test('running with the API on, then gone on the second look (it was quitting): it is opened', async () => {
  const r = rig({ runningSeq: [true, false], machine: { pref: true }, probes: [false, true] });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'isRunning', 'launch']);
  assert.deepEqual(r.statuses, [START(2), BACK]);
  assert.ok(r.logs.includes('ProdCom is up after 2 attempt(s)'));
});

test('gone only on the last look: opened, and if it still will not answer that is "would not start"', async () => {
  const r = rig({ runningSeq: [true, true, false], machine: { pref: true } });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'isRunning', 'readApiPref', 'isRunning', 'launch']);
  assert.deepEqual(r.statuses, [START(3), NO_START]);
});

test('a ProdCom this run just opened, still starting up with its API on, is waited for — not blamed on the port', async () => {
  // Attempt 1: not running → opened; the 5 s probe is false. Attempt 2: it is running now, its pref is on, the port is
  // still silent — that is a slow start, not a wrong port.
  const r = rig({ runningSeq: [false, true], machine: { pref: true }, probes: [false, true] });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'launch', 'isRunning', 'readApiPref']);
  assert.deepEqual(r.statuses, [START(1), BACK], 'no error status on the way');
  assert.deepEqual(r.sleeps, [5000, 5000]);
  assert.equal(r.probed(), 2);
  assert.ok(r.logs.includes('ProdCom is up after 2 attempt(s)'));
  assert.equal(r.calls.includes('quit'), false);
  assert.equal(r.calls.includes('writeApiPref'), false);
});

test('a ProdCom this run just opened that turns out to have its API off still gets the one restart', async () => {
  const r = rig({ runningSeq: [false, true], machine: { pref: null }, probes: [false, false, true] });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), [
    'isRunning', 'launch',
    'isRunning', 'readApiPref', 'writeApiPref', 'readApiPref', 'quit', 'launch',
    'isRunning',
  ]);
  assert.deepEqual(r.statuses, [START(1), API_OFF_RESTART, BACK]);
  assert.equal(r.calls.filter((c) => c === 'quit').length, 1);
  assert.ok(r.logs.includes('ProdCom is up after 3 attempt(s)'));
});

test('a ProdCom this run opened that never answers, API on: three tries, then "would not start"', async () => {
  const r = rig({ runningSeq: [false, true, true], machine: { pref: true }, probes: [false, false, false] });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'launch', 'isRunning', 'readApiPref', 'isRunning', 'readApiPref']);
  assert.deepEqual(r.statuses, [START(1), NO_START]);
});

/* ── when there is nothing to do ────────────────────────────────────── */

test('nothing to do for a ProdCom on another machine', async () => {
  const r = rig({ host: '10.1.1.50', machine: { local: false } });
  const why = r.sup.kick('ECONNREFUSED');
  assert.equal(why, 'ProdCom at 10.1.1.50 is not on this machine');
  for (let i = 0; i < 4; i += 1) r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), []);
  assert.equal(r.statuses.length, 0);
  assert.equal(r.logs.filter((l) => l === why).length, 1, 'the reason is logged once, not on every tick');
  assert.equal(r.sup.message(), null);
});

test('nothing to do when switched off', async () => {
  const r = rig({ settings: { autoStart: false } });
  assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.calls, []);
  assert.equal(r.statuses.length, 0);
  assert.equal(r.logs.length, 1);
});

test('the setting is read live: switching it on later takes effect without a new supervisor', async () => {
  const r = rig({ settings: { autoStart: false }, probes: [true] });
  assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
  r.settings.autoStart = true;
  assert.equal(r.sup.kick('ECONNREFUSED'), null);
  await r.sup.idle();
  assert.deepEqual(r.statuses, [START(1), BACK]);
});

test('nothing to do off macOS', async () => {
  const r = rig({ platform: 'win32' });
  assert.equal(r.sup.kick('ECONNREFUSED'), 'ProdCom auto-start is macOS only');
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), []);
  assert.equal(r.statuses.length, 0);
  assert.equal(r.logs.filter((l) => l === 'ProdCom auto-start is macOS only').length, 1);
});

test('a kick while a run is in progress does nothing', async () => {
  const h = heldSleep();
  const r = rig({ sleep: h.sleep, probes: [true] });
  assert.equal(r.sup.kick('ECONNREFUSED'), null);
  await tick();
  assert.equal(h.held.length, 1, 'the run is waiting out its five seconds');
  assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
  assert.equal(typeof r.sup.kick('ETIMEDOUT'), 'string');
  assert.equal(r.sup.message(), START(1)[1], 'while it works, its words are the tile\'s');
  assert.deepEqual(r.actions(), ['isRunning', 'launch'], 'no second run started');
  h.releaseAll();
  await r.sup.idle();
  assert.deepEqual(r.statuses, [START(1), BACK]);
});

test('kick() never throws into the reconnect loop, whatever the machine layer does', async () => {
  const r = rig();
  const sup = createSupervisor({
    settings: () => r.settings,
    target: () => ({ host: '127.0.0.1', port: 24480 }),
    os: { platform: 'darwin', isLocalHost: () => { throw new Error('no network interfaces'); } },
    probe: async () => true,
    sleep: () => Promise.resolve(),
    log: (line) => r.logs.push(line),
    status: (s, m) => r.statuses.push([s, m]),
  });
  assert.equal(sup.kick('ECONNREFUSED'), 'ProdCom auto-start could not decide (no network interfaces)');
  assert.equal(r.statuses.length, 0);
});

/* ── an outage ends, a new one begins ───────────────────────────────── */

test('after giving up, a successful connection re-arms it for the next outage', async () => {
  const r = rig({ machine: { running: true, pref: true } });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  const first = r.actions().length;
  assert.equal(first, 6);
  assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
  assert.equal(r.actions().length, first);

  r.sup.noteReachable();
  assert.equal(r.sup.message(), null);
  assert.equal(r.sup.kick('ECONNREFUSED'), null);
  await r.sup.idle();
  assert.equal(r.actions().length, first * 2);
  assert.deepEqual(r.statuses, [API_PORT, API_PORT]);
});

test('a second outage logs its reasons again', async () => {
  const r = rig({ host: '10.1.1.50', machine: { local: false } });
  const why = 'ProdCom at 10.1.1.50 is not on this machine';
  r.sup.kick('ECONNREFUSED');
  r.sup.kick('ECONNREFUSED');
  assert.equal(r.logs.filter((l) => l === why).length, 1);
  r.sup.noteReachable(); // the first outage is over
  r.sup.kick('ECONNREFUSED');
  r.sup.kick('ECONNREFUSED');
  assert.equal(r.logs.filter((l) => l === why).length, 2);

  // And a run that starts clears them too: "has given up" is said once per outage, not once ever.
  const g = rig({ machine: { running: true, pref: true } });
  const gaveUp = 'ProdCom auto-start has given up until ProdCom answers';
  g.sup.kick('ECONNREFUSED');
  await g.sup.idle();
  g.sup.kick('ECONNREFUSED');
  g.sup.kick('ECONNREFUSED');
  assert.equal(g.logs.filter((l) => l === gaveUp).length, 1);
  g.sup.noteReachable();
  g.sup.kick('ECONNREFUSED');
  await g.sup.idle();
  g.sup.kick('ECONNREFUSED');
  assert.equal(g.logs.filter((l) => l === gaveUp).length, 2);
});

test('a run that starts clears the reasons already logged, even with no connection in between', async () => {
  const h = heldSleep();
  const r = rig({ sleep: h.sleep, probes: [true, true] });
  const busy = 'ProdCom auto-start is already working on it';
  const said = () => r.logs.filter((l) => l === busy).length;
  r.sup.kick('ECONNREFUSED');
  await tick();
  r.sup.kick('ECONNREFUSED');
  r.sup.kick('ECONNREFUSED');
  assert.equal(said(), 1);
  h.releaseAll();
  await r.sup.idle(); // the run ended well; the module has not connected (noteReachable) yet
  assert.equal(r.sup.kick('ECONNREFUSED'), null, 'a new run');
  await tick();
  r.sup.kick('ECONNREFUSED');
  assert.equal(said(), 2, 'its reason is logged again');
  h.releaseAll();
  await r.sup.idle();
});

test('a connection that succeeds mid-run ends the run without another word', async () => {
  const h = heldSleep();
  const r = rig({ sleep: h.sleep, probes: [true] });
  r.sup.kick('ECONNREFUSED');
  await tick();
  assert.deepEqual(r.statuses, [START(1)]);
  r.sup.noteReachable(); // the module got through on its own while we waited
  h.releaseAll();
  await r.sup.idle();
  assert.equal(r.probed(), 0, 'no probe after the outage was over');
  assert.deepEqual(r.statuses, [START(1)], 'nothing overwrites the module\'s own "Live"');
  assert.equal(r.ups(), 0);
  assert.equal(r.sup.message(), null);
  assert.equal(r.sup.kick('ECONNREFUSED'), null, 'and the next outage is a fresh run');
  await tick();
  assert.deepEqual(r.statuses, [START(1), START(1)]);
  r.sup.stop();
  await r.sup.idle();
});

/* ── stop() and noteReachable() land mid-run ────────────────────────── */

test('stop() during the sleep ends the run: no more launches, no probes', async () => {
  const h = heldSleep();
  const r = rig({ sleep: h.sleep, probes: [true] });
  r.sup.kick('ECONNREFUSED');
  await tick();
  assert.equal(h.held.length, 1);
  assert.equal(h.held[0].signal.aborted, false);
  r.sup.stop();
  assert.equal(h.held[0].signal.aborted, true, 'the sleep was told to cut itself short');
  await r.sup.idle(); // settles even though the fake sleep never does
  assert.deepEqual(r.actions(), ['isRunning', 'launch']);
  assert.equal(r.probed(), 0);
  assert.deepEqual(r.statuses, [START(1)]);
  assert.equal(r.sup.message(), null);
  assert.equal(r.sup.kick('ECONNREFUSED'), 'stopped');
  h.releaseAll();
  await tick();
  assert.deepEqual(r.actions(), ['isRunning', 'launch']);
  assert.equal(r.probed(), 0);
  assert.equal(r.ups(), 0);
});

// Each machine call a run makes, held in flight; the run is ended (by the module
// connecting, or by stop()) while the call is out, and nothing may follow it:
// no status, no log, no further call, no reconnect nudge.
const MID_CALL = [
  ['isRunning', 'isRunning#1', { machine: { running: false } }],
  ['launch', 'launch#1', { machine: { running: false } }],
  ['readApiPref', 'readApiPref#1', { machine: { running: true, pref: null } }],
  ['writeApiPref', 'writeApiPref#1', { machine: { running: true, pref: null } }],
  ['readApiPref (the check that it took)', 'readApiPref#2', { machine: { running: true, pref: null } }],
  ['quit', 'quit#1', { machine: { running: true, pref: null } }],
  ['launch (after the quit)', 'launch#1', { machine: { running: true, pref: null } }],
  ['probe', 'probe#1', { machine: { running: false }, probes: [true] }],
];
for (const [label, key, setup] of MID_CALL) {
  for (const how of ['noteReachable', 'stop']) {
    test(`${how}() while ${label} is in flight: nothing follows`, async () => {
      const r = rig({ ...setup, hold: { [key]: true } });
      r.sup.kick('ECONNREFUSED');
      await tick();
      assert.ok(r.isHeld(key), `${key} is in flight`);
      r.sup[how]();
      const snapshot = { calls: [...r.calls], statuses: [...r.statuses], logs: [...r.logs], order: [...r.order], sleeps: [...r.sleeps] };
      r.release(key);
      await r.sup.idle();
      await tick();
      assert.deepEqual(r.calls, snapshot.calls, 'no further machine call');
      assert.deepEqual(r.statuses, snapshot.statuses, 'no status');
      assert.deepEqual(r.logs, snapshot.logs, 'no log line');
      assert.deepEqual(r.order, snapshot.order, 'no nudge');
      assert.deepEqual(r.sleeps, snapshot.sleeps, 'no further wait');
      assert.equal(r.probed(), key === 'probe#1' ? 1 : 0, 'no further probe');
      assert.equal(r.ups(), 0);
    });
  }
}

// …and the same when the call that was in flight ends in an error: the catch
// must not report a failure of a run that is already over.
const MID_CALL_ERROR = [
  ['isRunning', 'isRunning#1', { machine: { running: false }, runningError: 'pgrep failed' }],
  ['launch', 'launch#1', { machine: { running: false }, launchError: 'ProdCom is not installed on this machine' }],
  ['readApiPref', 'readApiPref#1', { machine: { running: true, pref: null }, readError: 'defaults failed' }],
  ['writeApiPref', 'writeApiPref#1', { machine: { running: true, pref: null }, writeError: 'write failed' }],
  ['quit', 'quit#1', { machine: { running: true, pref: null }, quitError: 'ProdCom would not quit' }],
  ['launch (after the quit)', 'launch#1', { machine: { running: true, pref: null }, launchError: 'launch failed' }],
];
for (const [label, key, setup] of MID_CALL_ERROR) {
  for (const how of ['noteReachable', 'stop']) {
    test(`${how}() while ${label} is in flight and then fails: no error is reported`, async () => {
      const r = rig({ ...setup, hold: { [key]: true } });
      r.sup.kick('ECONNREFUSED');
      await tick();
      assert.ok(r.isHeld(key), `${key} is in flight`);
      r.sup[how]();
      const snapshot = { calls: [...r.calls], statuses: [...r.statuses], logs: [...r.logs] };
      r.release(key);
      await r.sup.idle();
      await tick();
      assert.deepEqual(r.calls, snapshot.calls);
      assert.deepEqual(r.statuses, snapshot.statuses, 'no error status');
      assert.deepEqual(r.logs, snapshot.logs, 'no error log');
      assert.equal(r.sup.message(), null, 'and nothing latched as given up');
      if (how === 'noteReachable') assert.equal(r.sup.kick('ECONNREFUSED'), null, 'the next outage still gets a run');
      r.sup.stop();
      await r.sup.idle();
    });
  }
}

test('ProdCom is not installed: says so and gives up', async () => {
  const r = rig({ launchError: 'ProdCom is not installed on this machine' });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'launch']);
  assert.deepEqual(r.statuses, [['error', 'ProdCom is not installed on this machine']]);
  assert.equal(r.sleeps.length, 0);
  assert.equal(r.probed(), 0);
  assert.equal(r.sup.message(), 'ProdCom is not installed on this machine');
  assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
  assert.equal(r.actions().length, 2);
});

test('ProdCom will not quit: says so, does not open it again, gives up', async () => {
  const r = rig({ machine: { running: true, pref: null }, quitError: 'ProdCom would not quit' });
  r.sup.kick('ECONNREFUSED');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'readApiPref', 'writeApiPref', 'readApiPref', 'quit']);
  assert.deepEqual(r.statuses, [API_OFF_RESTART, ['error', 'ProdCom would not quit']]);
  assert.equal(r.sleeps.length, 0);
  assert.equal(typeof r.sup.kick('ECONNREFUSED'), 'string');
});

test('a machine call that fails for another reason gives up with its message', async () => {
  const r = rig({ machine: { running: true }, readError: "Could not read ProdCom's settings (timed out)" });
  r.sup.kick('ETIMEDOUT');
  await r.sup.idle();
  assert.deepEqual(r.actions(), ['isRunning', 'readApiPref'], 'not knowing the setting is not a reason to restart ProdCom');
  assert.deepEqual(r.statuses, [['error', "Could not read ProdCom's settings (timed out)"]]);
});

/* ── the real machine layer, on a fake execFile ─────────────────────── */

const ok = (stdout = '') => ({ stdout });
const exited = (code, stderr = '') => ({ error: Object.assign(new Error('Command failed'), { code }), stderr });
const timedOut = () => ({ error: Object.assign(new Error('timed out'), { killed: true, signal: 'SIGTERM', code: null }) });
const BUNDLE = 'com.prodcom.prodcom';

/** A fake child_process.execFile answering from `script(file, args, n)` (a result, or a promise of one). */
function fakeExecFile(script) {
  const calls = [];
  const execFile = (file, args, opts, cb) => {
    calls.push({ file, args, opts, line: [file, ...args].join(' ') });
    Promise.resolve(script(file, args, calls.length)).then((res) => {
      const r = res || {};
      cb(r.error || null, r.stdout || '', r.stderr || '');
    });
  };
  return { execFile, calls, lines: () => calls.map((c) => c.line) };
}

/** A fake clock: sleeping advances it, so a wait that would last ten seconds takes no time. */
function fakeClock() {
  let t = 0;
  const slept = [];
  return {
    now: () => t,
    sleep: (ms) => {
      slept.push(ms);
      t += ms;
      return Promise.resolve(true);
    },
    slept,
  };
}

test('default os: reading the preference — 1, 0, a missing key, a timeout', async () => {
  const answers = [ok('1\n'), ok('0\n'), exited(1, 'The domain/default pair of (com.prodcom.prodcom, applicationAPIEnabled) does not exist'), timedOut()];
  const x = fakeExecFile(() => answers.shift());
  const os = defaultOs({ execFile: x.execFile });
  assert.equal(await os.readApiPref(), true);
  assert.equal(await os.readApiPref(), false);
  assert.equal(await os.readApiPref(), null, 'a missing key is "not set", not an error');
  await assert.rejects(os.readApiPref(), /^Error: Could not read ProdCom's settings \(timed out\)$/);
  assert.equal(x.lines()[0], `/usr/bin/defaults read ${BUNDLE} applicationAPIEnabled`);
  assert.equal(x.calls[0].opts.timeout, 5000);
});

test('default os: writing the preference, and what a failure says', async () => {
  const x = fakeExecFile((file, args, n) => (n === 1 ? ok() : exited(1, '\n  Could not write domain com.prodcom.prodcom; exiting\nmore')));
  const os = defaultOs({ execFile: x.execFile });
  await os.writeApiPref();
  assert.equal(x.lines()[0], `/usr/bin/defaults write ${BUNDLE} applicationAPIEnabled -bool true`);
  await assert.rejects(os.writeApiPref(), /^Error: Could not turn on ProdCom's API \(exit 1: Could not write domain com\.prodcom\.prodcom; exiting\)$/);
});

test('default os: opening ProdCom is done in the background (-g), and a failed open reads as not installed', async () => {
  const answers = [ok(), exited(1, 'Unable to find application'), timedOut()];
  const x = fakeExecFile(() => answers.shift());
  const os = defaultOs({ execFile: x.execFile });
  await os.launch();
  assert.equal(x.lines()[0], `/usr/bin/open -g -b ${BUNDLE}`);
  assert.ok(x.calls[0].args.includes('-g'));
  await assert.rejects(os.launch(), /^Error: ProdCom is not installed on this machine$/);
  await assert.rejects(os.launch(), /^Error: Could not open ProdCom \(timed out\)$/);
});

test('default os: is ProdCom running — pgrep 0 / 1, anything else is an error', async () => {
  const answers = [ok('123\n'), exited(1), exited(2, 'pgrep: bad'), timedOut()];
  const x = fakeExecFile(() => answers.shift());
  const os = defaultOs({ execFile: x.execFile });
  assert.equal(await os.isRunning(), true);
  assert.equal(await os.isRunning(), false);
  await assert.rejects(os.isRunning(), /Could not check whether ProdCom is running \(exit 2: pgrep: bad\)/);
  await assert.rejects(os.isRunning(), /Could not check whether ProdCom is running \(timed out\)/);
  assert.equal(x.lines()[0], '/usr/bin/pgrep -x ProdCom');
});

test('default os: quit, the polite way — asked by Apple Event, then waited for', async () => {
  let pgreps = 0;
  const x = fakeExecFile((file) => {
    if (file === '/usr/bin/osascript') return ok();
    pgreps += 1;
    return pgreps < 3 ? ok('123\n') : exited(1); // still there twice, then gone
  });
  const clock = fakeClock();
  const os = defaultOs({ execFile: x.execFile, sleep: clock.sleep, now: clock.now });
  await os.quit(new AbortController().signal);
  assert.deepEqual(x.lines(), [
    `/usr/bin/osascript -e tell application id "${BUNDLE}" to quit`,
    '/usr/bin/pgrep -x ProdCom', '/usr/bin/pgrep -x ProdCom', '/usr/bin/pgrep -x ProdCom',
  ]);
  assert.equal(x.calls[0].opts.timeout, 5000, 'the Apple Event gets five seconds');
  assert.deepEqual(clock.slept, [500, 500]);
});

test('default os: quit, when it will not be asked — SIGTERM, then waited for', async () => {
  let pgreps = 0;
  const x = fakeExecFile((file) => {
    if (file === '/usr/bin/osascript') return exited(1, 'execution error: Not authorized to send Apple events (-1743)');
    if (file === '/usr/bin/pkill') return ok();
    pgreps += 1;
    return pgreps < 2 ? ok('123\n') : exited(1);
  });
  const clock = fakeClock();
  const os = defaultOs({ execFile: x.execFile, sleep: clock.sleep, now: clock.now });
  await os.quit();
  assert.deepEqual(x.lines().map((l) => l.split(' ').slice(0, 3).join(' ')), [
    '/usr/bin/osascript -e tell',
    '/usr/bin/pkill -x -TERM',
    '/usr/bin/pgrep -x ProdCom',
    '/usr/bin/pgrep -x ProdCom',
  ]);
  assert.deepEqual(x.calls[1].args, ['-x', '-TERM', 'ProdCom']);
});

test('default os: quit, and it will not go — "would not quit" after ten seconds', async () => {
  const x = fakeExecFile((file) => (file === '/usr/bin/pgrep' ? ok('123\n') : ok()));
  const clock = fakeClock();
  const os = defaultOs({ execFile: x.execFile, sleep: clock.sleep, now: clock.now });
  await assert.rejects(os.quit(), /^Error: ProdCom would not quit$/);
  assert.equal(clock.now(), 10000);
  assert.equal(x.calls.filter((c) => c.file === '/usr/bin/pkill').length, 0, 'the Apple Event worked, so no SIGTERM');
});

test('default os: quit, ended while the Apple Event is out: no SIGTERM, no look at the process list', async () => {
  const ctl = new AbortController();
  let finish;
  const x = fakeExecFile((file) => {
    if (file === '/usr/bin/osascript') return new Promise((resolve) => { finish = () => resolve(exited(1, 'denied')); });
    return ok('123\n');
  });
  const os = defaultOs({ execFile: x.execFile, sleep: fakeClock().sleep });
  const quitting = os.quit(ctl.signal);
  await tick();
  ctl.abort();
  finish();
  await assert.rejects(quitting, /^Error: cancelled$/);
  assert.equal(x.calls.length, 1, 'only the osascript was ever run');
});

test('default os: quit, ended while SIGTERM is out: no look at the process list', async () => {
  const ctl = new AbortController();
  let finish;
  const x = fakeExecFile((file) => {
    if (file === '/usr/bin/osascript') return exited(1, 'denied');
    if (file === '/usr/bin/pkill') return new Promise((resolve) => { finish = () => resolve(ok()); });
    return ok('123\n');
  });
  const os = defaultOs({ execFile: x.execFile, sleep: fakeClock().sleep });
  const quitting = os.quit(ctl.signal);
  await tick();
  ctl.abort();
  finish();
  await assert.rejects(quitting, /^Error: cancelled$/);
  assert.deepEqual(x.calls.map((c) => c.file), ['/usr/bin/osascript', '/usr/bin/pkill']);
});

test('default os: quit, ended while it waits for the process to go: no further look', async () => {
  const ctl = new AbortController();
  const x = fakeExecFile((file) => (file === '/usr/bin/pgrep' ? ok('123\n') : ok()));
  const os = defaultOs({
    execFile: x.execFile,
    sleep: () => { ctl.abort(); return Promise.resolve(false); }, // the run ends during the first wait
  });
  await assert.rejects(os.quit(ctl.signal), /^Error: cancelled$/);
  assert.equal(x.calls.filter((c) => c.file === '/usr/bin/pgrep').length, 1);
});

test('default os: quit, already ended before it starts: it does nothing at all', async () => {
  const ctl = new AbortController();
  ctl.abort();
  const x = fakeExecFile(() => ok());
  const os = defaultOs({ execFile: x.execFile });
  await assert.rejects(os.quit(ctl.signal), /^Error: cancelled$/);
  assert.equal(x.calls.length, 0);
});

/* ── the pieces that need no machine ────────────────────────────────── */

test('isLocalHostName: this machine by every name it has, and no other', () => {
  const addrs = ['127.0.0.1', '::1', 'fe80::1', '10.3.11.152', 'fe80::aede:48ff:fe00:1122'];
  const here = 'AB-ProPresenter.local';
  for (const host of ['', 'localhost', 'LocalHost', '127.0.0.1', '::1', '[::1]', '0.0.0.0',
    '10.3.11.152', 'FE80::1', 'fe80:0:0:0:0:0:0:1', 'fe80::1%lo0',
    'ab-propresenter.local', 'AB-PROPRESENTER', 'ab-propresenter', 'ab-propresenter.local.']) {
    assert.equal(isLocalHostName(host, here, addrs), true, `"${host}" is this machine`);
  }
  for (const host of ['10.1.1.50', 'prodcom.local', 'na-ingest', 'example.com', '10.3.11.153']) {
    assert.equal(isLocalHostName(host, here, addrs), false, `"${host}" is not`);
  }
  // A hostname without .local matches the same machine reached as .local.
  assert.equal(isLocalHostName('studio.local', 'Studio', []), true);
});

test('parseApiPref: defaults prints a bool as 1 or 0', () => {
  assert.equal(parseApiPref('1\n'), true);
  assert.equal(parseApiPref('0\n'), false);
  assert.equal(parseApiPref(' true '), true);
  assert.equal(parseApiPref(''), null);
  assert.equal(parseApiPref('banana'), null);
});

test('the default sleep is cut short by its signal and leaves no timer behind', async () => {
  const ctl = new AbortController();
  const slept = abortableSleep(60 * 60 * 1000, ctl.signal); // an hour — would hold the test open if it were left running
  ctl.abort();
  assert.equal(await slept, false);
  assert.equal(await abortableSleep(60 * 60 * 1000, ctl.signal), false, 'already aborted: returns at once');
});