Audio failover: press FOH Backup by itself when Studio B dies; Companion button presses as fixes

A new fix kind "companion_press" presses any Companion button over its HTTP API (page, row, col).
A fix marked "failover" runs by itself within seconds of its check failing: inside the "failover"
windows (Sunday 06:30-13:30), even while ALARMS is muted, once per fault, never back the other way.
The hidden STB AUDIO key (inside AUDIO) watches Studio B's ping every second, Pro Tools over SSH
every 2 s and its Dante device; each check's fix is the FOH Backup press. The keys say ON FOH BACKUP
until STREAM STB on the Restart page switches back. Settings page: the press builder, the failover
tick and undo, the Failover calendar, help entry "A fallback button".

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
2026-10-15 12:40:24 -04:00
co-authored by Claude Fable 5.1
parent 422e92f814
commit 14fce2cf9a
13 changed files with 855 additions and 67 deletions
+303 -1
View File
@@ -71,6 +71,7 @@ def healthy_snapshot(now):
"org.waterschurch.lobby-captions-display": 901, "org.waterschurch.digico-relay": 902}
snap.launchd_time = now
snap.procs = {"/Applications/ProdDash.app/Contents/MacOS/ProdDash": True}
snap.remote = {"studiob\0pgrep -xq 'Pro Tools'": True} # the failover's check has a verdict
snap.http = {"https://central.resi.io/": True, "http://10.6.7.13:8080/": True, "http://10.1.14.225:1025/": True}
snap.disk = {"/": 97}
return snap
@@ -1521,6 +1522,284 @@ class TestFixes(unittest.TestCase):
self.assertIn(f.get("kind", "ssh"), sp.FIX_KINDS)
def run_failover(engine, snap, start, seconds, step=0.5):
"""Like run_steps, but the failover looks after every step, as the run loop does."""
tiles, t = None, start
while t <= start + seconds:
snap.conns_time = t
tiles = engine.step(snap, t)
engine.failover(snap, None, t)
t += step
return tiles, t
STB_PING, STB_PT = "10.1.13.175", "studiob\0pgrep -xq 'Pro Tools'"
class TestFailover(unittest.TestCase):
"""Studio B dies on a Sunday: the FOH Backup button is pressed by itself, once, and the keys say so.
Nothing here presses a real button, posts a real alert or writes a real log."""
def setUp(self):
self.snap = healthy_snapshot(SUNDAY)
self.engine = sp.Engine(CONFIG)
self.ran, self.sent = [], []
self.engine.executor = lambda cfg, fix_id, chk: self.ran.append(fix_id) or True
self.engine.poe_executor = lambda *a: True
self.engine.notify = lambda url, text: self.sent.append(text)
self.engine.autolog = sp.AutoLog(tempfile.mkdtemp())
def fired_at(self):
self.engine.fix_threads and [t.join(2) for t in self.engine.fix_threads]
rec = self.engine.failed.get("stream_foh_backup")
return rec and rec["at"]
def test_config_wires_studio_b_to_foh_backup(self):
tile = next(t for t in CONFIG["tiles"] if t["id"] == "stb_audio")
self.assertNotIn("pos", tile) # hidden: AUDIO shows it
self.assertIn("stb_audio", next(t for t in CONFIG["tiles"] if t["id"] == "audio")["members"])
for c in tile["checks"]:
fixes = set(c["fix"].values()) if isinstance(c["fix"], dict) else {c["fix"]}
self.assertEqual(fixes, {"stream_foh_backup"}, c["type"])
fix = CONFIG["fixes"]["stream_foh_backup"]
self.assertEqual((fix["kind"], fix["page"], fix["row"], fix["col"]), ("companion_press", 4, 0, 5))
self.assertTrue(fix["failover"])
self.assertEqual(fix["undo"], "stream_studio_b")
self.assertFalse(CONFIG["fixes"]["stream_studio_b"].get("failover"))
self.assertNotIn("stream_foh_backup", CONFIG["auto_fix"]["fixes"])
keys = {k["id"]: k["fix"] for k in CONFIG["restart_page"]["keys"]}
self.assertEqual((keys["rs_stream_stb"], keys["rs_stream_foh"]), ("stream_studio_b", "stream_foh_backup"))
self.assertEqual(CONFIG["failover"]["after_seconds"], 2)
self.assertEqual(self.engine.failover_windows, [{"days": ["sun"], "start": "06:30", "end": "13:30"}])
def test_studio_b_dies_foh_backup_is_pressed_once(self):
t0 = SUNDAY
run_failover(self.engine, self.snap, t0 - 5, 4) # all fine for a while
self.snap.ping[STB_PING] = 2 # two 1-second pings missed
tiles, t = run_failover(self.engine, self.snap, t0, 6)
at = self.fired_at()
self.assertEqual(self.ran, ["stream_foh_backup"])
key = next(k for k, (tid, chk, sub) in self.engine.meta.items() if chk.get("host") == STB_PING)
since = self.engine.trackers[key].since
self.assertEqual(since, t0 + 1) # 1 s debounce
self.assertEqual(at, since + 2) # + 2 s after_seconds
self.assertLessEqual(at - t0, 5)
tiles, t = run_failover(self.engine, self.snap, t, 60) # the fault lasts a minute
self.assertEqual(self.ran, ["stream_foh_backup"]) # still once
self.assertNotIn("stream_studio_b", self.ran) # never switched back by itself
# the keys say where the stream is
self.assertEqual(tiles["audio"]["x"], "AUDIO\nSTUDIO B NO PING\nON FOH BACKUP")
self.assertEqual(level(tiles["audio"]), "err")
self.engine.notes.pop("stb_audio", None) # under the fix's own feedback
tiles = self.engine.step(self.snap, t)
self.assertEqual(tiles["stb_audio"]["x"], "STB AUDIO\nSTUDIO B NO PING\nON FOH BACKUP")
self.assertEqual(tiles["all"]["x"], "1 DOWN\nSTUDIO B NO PING")
# the log, the alert, and a hold right after
body = [l for l in open(self.engine.autolog.path(t0)).read().splitlines() if l[:2].isdigit()]
self.assertTrue(any("STB AUDIO FAILOVER: STUDIO B NO PING - press the FOH Backup button" in l for l in body), body)
self.assertTrue(any("-> done, checking it comes back" in l for l in body), body)
self.assertEqual(len(self.sent), 1)
self.assertTrue(self.sent[0].startswith("NA Ingest: STB AUDIO STUDIO B NO PING - switched over by itself: press the FOH Backup"))
self.assertIn("hold STREAM STB on the Restart page", self.sent[0])
self.engine.run_fixes("audio", self.snap, None, at + 10)
self.assertEqual(self.engine.notes["audio"]["text"], "JUST TRIED\nWAIT 1 MIN")
self.assertIsNone(self.engine.run_force("rs_stream_foh", self.snap, None, at + 11))
self.assertEqual(self.engine.notes["rs_stream_foh"]["text"], "JUST DONE\nWAIT 1 MIN")
out = sp.snapshot_summary(self.snap, self.engine, t)
json.dumps(out)
self.assertEqual(out["failover"]["stream_foh_backup"]["note"], "ON FOH BACKUP")
self.assertEqual(out["failover"]["stream_foh_backup"]["faults"], ["STUDIO B NO PING"])
def test_pro_tools_crash_is_as_fast(self):
t0 = SUNDAY
self.snap.remote[STB_PT] = False
run_failover(self.engine, self.snap, t0, 6)
self.assertEqual(self.ran, ["stream_foh_backup"])
self.assertLessEqual(self.fired_at() - t0, 5)
tiles = self.engine.step(self.snap, t0 + 7)
self.assertEqual(tiles["audio"]["x"], "AUDIO\nPRO TOOLS NOT RUNNING\nON FOH BACKUP")
def test_fires_with_alarms_muted(self):
self.engine.service.set(False, SUNDAY) # maintenance: auto-fix stops
self.snap.ping[STB_PING] = 2
run_failover(self.engine, self.snap, SUNDAY, 6)
self.assertEqual(self.ran, ["stream_foh_backup"]) # the failover does not
def test_only_inside_the_window(self):
for when in (WEDNESDAY, datetime(2026, 10, 11, 15, 0).timestamp()): # a weekday; Sunday after the window
snap = healthy_snapshot(when)
snap.ping[STB_PING] = 2
run_failover(self.engine, snap, when, 10)
self.assertEqual(self.ran, [], datetime.fromtimestamp(when))
self.assertEqual(self.engine.failed, {})
def test_rearms_when_the_fault_clears_and_the_note_fades(self):
t0 = SUNDAY
self.snap.ping[STB_PING] = 2
tiles, t = run_failover(self.engine, self.snap, t0, 6)
self.snap.ping[STB_PING] = 0 # Studio B answers again
tiles, t = run_failover(self.engine, self.snap, t, 5)
self.assertEqual(tiles["audio"]["x"], "AUDIO\nOK\nON FOH BACKUP") # still on the backup
self.assertEqual(level(tiles["audio"]), "ok")
self.assertIsNotNone(self.engine.failed["stream_foh_backup"]["cleared"])
self.snap.ping[STB_PING] = 2 # ...and dies again 5 s later
tiles, t = run_failover(self.engine, self.snap, t, 4)
self.assertEqual(self.ran, ["stream_foh_backup"]) # was clear under 10 s: not armed yet
tiles, t = run_failover(self.engine, self.snap, t, 3) # 10 s since it cleared: armed, fires
self.assertEqual(self.ran, ["stream_foh_backup", "stream_foh_backup"])
self.snap.ping[STB_PING] = 0
tiles, t = run_failover(self.engine, self.snap, t, 2)
tiles, t = run_failover(self.engine, self.snap, t + 1000, 1) # note_seconds later: forgotten
self.assertEqual(tiles["audio"]["x"], "AUDIO\nOK")
self.assertEqual(self.engine.failed, {})
def test_a_check_that_cannot_ask_is_not_a_clear(self):
t0 = SUNDAY
self.snap.remote[STB_PT] = False # Pro Tools gone
tiles, t = run_failover(self.engine, self.snap, t0, 6)
self.assertEqual(self.ran, ["stream_foh_backup"])
self.snap.remote[STB_PT] = None # ssh could not ask for a while
tiles, t = run_failover(self.engine, self.snap, t, 30)
self.assertIsNone(self.engine.failed["stream_foh_backup"]["cleared"]) # not re-armed on a hiccup
self.snap.remote[STB_PT] = False # the answer is back: still gone
tiles, t = run_failover(self.engine, self.snap, t, 10)
self.assertEqual(self.ran, ["stream_foh_backup"]) # no second press
self.assertEqual(tiles["rs_stream_foh"]["x"], "STREAM FOH\nON FOH BACKUP") # pressed already: no HOLD TO FIX
self.assertEqual(level(tiles["rs_stream_foh"]), "ok")
self.assertEqual(tiles["rs_stream_stb"]["x"], "STREAM STB\nHOLD 2s")
def test_studio_b_rednet_is_never_power_cycled_by_itself(self):
"""Only STUDIO-B-RedNet-TNX vanishes: the failover presses FOH Backup; Sunday auto-fix on the DANTE
key must not power-cycle the RedNet's PoE port (that stays a hold), nor touch anything else."""
comp = mock_comp()
self.snap.dante_map = {"STUDIO-B-RedNet-TNX": {"mac": "00:1d:c1:aa:bb:cc", "poe": True, "switch": "10.1.60.2", "port": "0/14"}}
devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")])
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps([d for d in devices if d != "STUDIO-B-RedNet-TNX"])
poe = []
self.engine.poe_executor = lambda cfg, name, entry: poe.append(name) or True
tiles, t = run_failover(self.engine, self.snap, SUNDAY, 30)
self.assertEqual(self.ran, ["stream_foh_backup"])
self.assertEqual(self.engine.auto_fix(self.snap, comp, t), [])
time.sleep(0.2)
self.assertEqual((poe, comp.restarted, self.ran), ([], [], ["stream_foh_backup"]))
self.assertEqual([(k, w) for _, k, w, _ in self.engine.fix_plan("dante", self.snap)], [("poe", "STUDIO-B-RedNet-TNX")]) # a hold would
def test_switching_back_by_hand_clears_the_note(self):
t0 = SUNDAY
gate = threading.Event() # the Studio B press takes a moment
def press(cfg, fix_id, chk):
if fix_id == "stream_studio_b":
gate.wait(2)
self.ran.append(fix_id)
return True
self.engine.executor = press
self.snap.ping[STB_PING] = 2
tiles, t = run_failover(self.engine, self.snap, t0, 6)
self.assertIn("ON FOH BACKUP", tiles["audio"]["x"])
self.assertEqual(self.engine.run_force("rs_stream_stb", self.snap, None, t), "stream_studio_b")
self.assertEqual(self.engine.notes["rs_stream_stb"]["text"], "PRESSING")
gate.set()
time.sleep(0.2)
self.assertEqual(self.engine.notes["rs_stream_stb"]["text"], "DONE")
self.assertEqual(self.engine.failed, {})
tiles = self.engine.step(self.snap, t + 0.5)
self.assertEqual(tiles["audio"]["x"], "AUDIO\nSTUDIO B NO PING")
tiles, t = run_failover(self.engine, self.snap, t + 1, 4) # still dead: it fires again
self.assertEqual(self.ran, ["stream_foh_backup", "stream_studio_b", "stream_foh_backup"])
self.assertIn("ON FOH BACKUP", tiles["audio"]["x"])
def test_held_back_when_several_dante_devices_vanish(self):
t0 = SUNDAY
devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")])
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(
[d for d in devices if d not in ("STUDIO-B-RedNet-TNX", "NA-Tracks", "BOH-WING")])
tiles, t = run_failover(self.engine, self.snap, t0, 12)
self.assertEqual(self.ran, []) # Ingest's view of Dante: hands off
self.assertEqual(tiles["stb_audio"]["x"], "STB AUDIO\nSTB TNX GONE") # seen, no note
body = open(self.engine.autolog.path(t0)).read()
self.assertIn("failover held back: 3 Dante devices gone at once", body)
self.assertEqual(body.count("held back"), 1) # said once, not every half second
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(
[d for d in devices if d != "STUDIO-B-RedNet-TNX"]) # only Studio B's own box
run_failover(self.engine, self.snap, t, 3)
self.assertEqual(self.ran, ["stream_foh_backup"])
def test_state_survives_a_restart(self):
state = os.path.join(tempfile.mkdtemp(prefix="sp-state-"), "state.json")
engine = sp.Engine(CONFIG, state)
engine.executor = lambda cfg, fix_id, chk: self.ran.append(fix_id) or True
engine.notify = lambda url, text: None
engine.autolog = sp.AutoLog(tempfile.mkdtemp())
self.snap.ping[STB_PING] = 2
tiles, t = run_failover(engine, self.snap, SUNDAY, 6)
self.assertEqual(self.ran, ["stream_foh_backup"])
self.assertEqual(json.load(open(state))["failover"]["stream_foh_backup"]["tile"], "stb_audio")
again = sp.Engine(CONFIG, state) # the watchdog restarts mid-fault
again.executor = lambda cfg, fix_id, chk: self.ran.append(fix_id) or True
again.notify = lambda url, text: None
again.autolog = sp.AutoLog(tempfile.mkdtemp())
tiles, t = run_failover(again, self.snap, t, 10)
self.assertEqual(self.ran, ["stream_foh_backup"]) # not pressed a second time
self.assertEqual(tiles["audio"]["x"], "AUDIO\nSTUDIO B NO PING\nON FOH BACKUP")
def test_nothing_new_starts_while_stopping(self):
self.snap.ping[STB_PING] = 2
self.engine.stopping = True
run_failover(self.engine, self.snap, SUNDAY, 6)
self.assertEqual(self.ran, [])
def test_load_config_checks_the_button_and_the_undo(self):
def load(change):
cfg = json.loads(json.dumps(CONFIG))
change(cfg)
path = os.path.join(tempfile.mkdtemp(prefix="sp-cfg-"), "config.json")
with open(path, "w") as f:
json.dump(cfg, f)
return sp.load_config(path)
load(lambda c: None)
for bad in (lambda c: c["fixes"]["stream_foh_backup"].update(page=0),
lambda c: c["fixes"]["stream_foh_backup"].update(row=True),
lambda c: c["fixes"]["stream_foh_backup"].update(col="5"),
lambda c: c["fixes"]["stream_foh_backup"].pop("col"),
lambda c: c["fixes"]["stream_foh_backup"].update(never_auto=True),
lambda c: c["fixes"]["stream_foh_backup"].update(undo="nope"),
lambda c: c["failover"]["windows"].append({"days": ["sun"], "start": "6:30", "end": "13:30"})):
with self.assertRaises(ValueError):
load(bad)
def test_studio_b_is_pinged_every_second(self):
self.assertEqual(sp.ping_hosts(CONFIG, fast=True), [STB_PING])
slow = sp.ping_hosts(CONFIG)
self.assertNotIn(STB_PING, slow)
self.assertIn("10.1.0.1", slow) # the gateway stays in the slow list
self.assertIn("10.1.3.227", slow)
self.assertEqual(sp.PING_INTERVAL, 3)
def test_remote_groups_run_when_due_each_on_its_own(self):
from unittest import mock
groups = sp.remote_groups(CONFIG)
self.assertEqual(groups[("studiob", 2)], ["pgrep -xq 'Pro Tools'"])
self.assertTrue(all(k[1] == 10 for k in groups if k != ("studiob", 2)))
calls = []
fake = lambda argv, **kw: calls.append(argv) or mock.Mock(returncode=0, stdout="SP0=0\nSP1=0\n", stderr="")
t = SUNDAY + 100
with mock.patch.object(sp.subprocess, "run", fake):
last = {k: t - 3 for k in groups} # every group ran 3 s ago
for th in sp.poll_remote(self.snap, CONFIG, now=t, last=last):
th.join(2)
self.assertEqual(len(calls), 1) # only the 2-second group is due
self.assertIn("studiob@10.1.13.175", calls[0])
self.assertIn("pgrep -xq 'Pro Tools'", calls[0][-1])
self.assertTrue(self.snap.remote[STB_PT])
self.assertEqual(last[("studiob", 2)], t)
self.assertEqual(sp.poll_remote(self.snap, CONFIG, now=t + 1, last=last), []) # nothing due yet
calls.clear()
sp.poll_remote(self.snap, CONFIG) # run_once: everything, and waits
self.assertEqual(len(calls), len(groups))
self.assertEqual(sp.REMOTE_INTERVAL, 10)
class TestNetgearParsing(unittest.TestCase):
BIG = """ VLAN ID MAC Address Interface IfIndex Status
66 00:1D:C1:51:34:C8 0/14 14 Learned
@@ -1603,6 +1882,13 @@ class FakeCompanion(BaseHTTPRequestHandler):
if m:
s["restarts"].append(m.group(1))
return self._send(200, "{}")
m = re.match(r"^/api/location/(\d+)/(\d+)/(\d+)/press$", u.path)
if m:
where = tuple(int(x) for x in m.groups())
s["presses"].append(where)
if where in s["buttons"]:
return self._send(200, "ok")
return self._send(204, "No control") # like Companion 5.0.7 for an empty cell
self._send(404, "")
@@ -1615,6 +1901,8 @@ class TestAgainstFakeCompanion(unittest.TestCase):
"vars": dict(snap.vars),
"custom": {"sp_state": "", "sp_hb": "", "sp_cmd": "", "flash_tick": "0"},
"restarts": [],
"presses": [],
"buttons": {(4, 0, 5), (4, 0, 6)}, # the Audio page's FOH Backup and Studio B
}
cls.server = ThreadingHTTPServer(("127.0.0.1", 0), FakeCompanion)
threading.Thread(target=cls.server.serve_forever, daemon=True).start()
@@ -1637,6 +1925,20 @@ class TestAgainstFakeCompanion(unittest.TestCase):
self.assertEqual(comp.set_custom("missing", "x"), (False, 404))
print("\n sp_state is %d bytes, %d in the URL" % (len(js), len(urllib.parse.quote(js, safe=""))))
def test_press_a_button(self):
comp = sp.Companion("127.0.0.1", self.port)
store = FakeCompanion.store
self.assertEqual(comp.press(4, 0, 5), 200)
self.assertEqual(comp.press(4, 1, 1), 204) # an empty cell
self.assertEqual(store["presses"][-2:], [(4, 0, 5), (4, 1, 1)])
cfg = json.loads(json.dumps(CONFIG))
cfg["companion"]["port"] = self.port
self.assertTrue(sp.execute_fix(cfg, "stream_foh_backup", {}))
self.assertEqual(store["presses"][-1], (4, 0, 5))
cfg["fixes"]["stream_foh_backup"]["col"] = 7
self.assertEqual(sp.execute_fix(cfg, "stream_foh_backup", {}), "NO BUTTON\nTHERE")
self.assertEqual(store["presses"][-1], (4, 0, 7))
def test_commands(self):
comp = sp.Companion("127.0.0.1", self.port)
store = FakeCompanion.store
@@ -1679,7 +1981,7 @@ class TestAgainstFakeCompanion(unittest.TestCase):
sp.poll_mdns = lambda snap, seconds=3: None
sp.poll_system = lambda snap, cfg: None
sp.poll_midi = lambda snap, cfg: None # CoreMIDI has its own test
sp.poll_remote = lambda snap, cfg: None # never SSH anywhere from a test
sp.poll_remote = lambda snap, cfg, **k: [] # never SSH anywhere from a test
sp.start_remote_midi_streams = lambda snap, cfg, stop: []
stop = threading.Event()
loop = threading.Thread(target=sp.run, args=(cfg, data_dir), kwargs={"stop": stop}, daemon=True)