Audio failover: press FOH Backup by itself when Studio B dies; Companion button presses as fixes
A new fix kind "companion_press" presses any Companion button over its HTTP API (page, row, col). A fix marked "failover" runs by itself within seconds of its check failing: inside the "failover" windows (Sunday 06:30-13:30), even while ALARMS is muted, once per fault, never back the other way. The hidden STB AUDIO key (inside AUDIO) watches Studio B's ping every second, Pro Tools over SSH every 2 s and its Dante device; each check's fix is the FOH Backup press. The keys say ON FOH BACKUP until STREAM STB on the Restart page switches back. Settings page: the press builder, the failover tick and undo, the Failover calendar, help entry "A fallback button". Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5.1
+22
-1
@@ -35,7 +35,7 @@ class TestTodaysFixes(unittest.TestCase):
|
||||
if c.get("template"):
|
||||
self.assertEqual(b.build_check(CFG, c)["cmd"], c["cmd"])
|
||||
n += 1
|
||||
self.assertEqual(n, 5)
|
||||
self.assertEqual(n, 6)
|
||||
|
||||
def test_a_changed_command_is_refused_by_validate(self):
|
||||
cfg = json.loads(json.dumps(CFG))
|
||||
@@ -126,6 +126,27 @@ if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class TestCompanionPress(unittest.TestCase):
|
||||
def test_a_press_builds_from_its_fields(self):
|
||||
f = b.build(CFG, {"template": "companion_press", "params": {"page": 4, "row": 0, "col": "5", "button": "FOH Backup"}, "short": "FOH BACKUP"})
|
||||
self.assertEqual((f["kind"], f["page"], f["row"], f["col"], f["short"]), ("companion_press", 4, 0, 5, "FOH BACKUP"))
|
||||
self.assertEqual(f["params"]["col"], 5) # copied as a number
|
||||
self.assertNotIn("run", f); self.assertNotIn("host", f)
|
||||
self.assertEqual(b.suggest_label(CFG, f), "press the FOH Backup button in Companion (page 4, row 0, column 5)")
|
||||
self.assertEqual(b.suggest_label(CFG, {"template": "companion_press", "params": {"page": 4, "row": 0, "col": 5}}),
|
||||
"press the button in Companion (page 4, row 0, column 5)")
|
||||
for bad in ({"page": 4, "row": 0}, {"page": "four", "row": 0, "col": 5}, {"page": 4, "row": 1.5, "col": 5},
|
||||
{"page": True, "row": 0, "col": 5}):
|
||||
with self.assertRaises(ValueError):
|
||||
b.build(CFG, {"template": "companion_press", "params": bad})
|
||||
|
||||
def test_a_changed_button_is_caught(self):
|
||||
cfg = json.loads(json.dumps(CFG))
|
||||
cfg["fixes"]["stream_foh_backup"]["col"] = 7 # the field moved, the params did not
|
||||
self.assertTrue(any("stream_foh_backup" in e and "its col no longer matches" in e for e in b.mismatches(cfg)), b.mismatches(cfg))
|
||||
self.assertEqual([e for e in b.mismatches(CFG) if "stream_" in e], [])
|
||||
|
||||
|
||||
class TestOutlet(unittest.TestCase):
|
||||
def test_outlet_cycle_renders_and_is_local(self):
|
||||
f = b.build(CFG, {"template": "outlet_cycle", "params": {"outlet": "tascam", "address": "10.1.11.74", "wait_seconds": 8}})
|
||||
|
||||
@@ -23,17 +23,17 @@ NAMES = {"cfg": "top", "config": "top", "chk": "check", "c": "check", "fix": "fi
|
||||
"tile": "tile", "k": "restart_key", "key": "restart_key", "rp": "restart_page", "w": "window",
|
||||
"names": "variables", "host": "host", "h": "host", "windows": "windows", "site": "site", "rc": "ready"}
|
||||
ATTRS = {"self.cfg": "top", "engine.cfg": "top", "self.net": "network", "self.auto": "auto", "self.site": "site",
|
||||
"self.atem": "atem", "self.ready_cfg": "ready",
|
||||
"self.atem": "atem", "self.ready_cfg": "ready", "self.failover_cfg": "failover",
|
||||
"self.tiles": "tiles", "engine.tiles": "tiles", "self.restart_keys": "restart_keys",
|
||||
"engine.restart_keys": "restart_keys"}
|
||||
# A key of a top-level dict that holds a record of another kind.
|
||||
TOP = {"companion": "companion", "network": "network", "variables": "variables", "auto_fix": "auto", "site": "site",
|
||||
"atem": "atem", "spotify": "spotify", "timing": "timing", "flap": "flap", "hosts": "hosts", "fixes": "fixes",
|
||||
"tiles": "tiles", "restart_page": "restart_page", "gear": "gears", "service_windows": "windows",
|
||||
"ready": "ready", "ready_page": "ready_page"}
|
||||
"ready": "ready", "ready_page": "ready_page", "failover": "failover"}
|
||||
SUB = {("auto", "windows"): "windows", ("site", "live_services"): "windows", ("tile", "checks"): "checks",
|
||||
("restart_page", "keys"): "restart_keys", ("tile", "members"): "names", ("ready", "due"): "windows",
|
||||
("ready_page", "keys"): "tiles"}
|
||||
("ready_page", "keys"): "tiles", ("failover", "windows"): "windows"}
|
||||
ELEMENT = {"tiles": "tile", "checks": "check", "restart_keys": "restart_key", "windows": "window", "fixes": "fix",
|
||||
"hosts": "host", "gears": "gear"}
|
||||
# Keys the code reads that are the watchdog's own working data, not settings.
|
||||
@@ -145,7 +145,8 @@ def path_of(kind, key):
|
||||
"restart_key": "restart_page.keys[].", "restart_page": "restart_page.", "auto": "auto_fix.",
|
||||
"network": "network.", "variables": "variables.", "site": "site.", "atem": "atem.", "spotify": "spotify.",
|
||||
"companion": "companion.", "timing": "timing.", "flap": "flap.", "gear": "gear.*.",
|
||||
"window": "auto_fix.windows[].", "ready": "ready.", "ready_page": "ready_page."}.get(kind)
|
||||
"window": "auto_fix.windows[].", "ready": "ready.", "ready_page": "ready_page.",
|
||||
"failover": "failover."}.get(kind)
|
||||
return None if prefix is None else prefix + key
|
||||
|
||||
|
||||
@@ -188,7 +189,8 @@ class TestSchemaMatchesCode(unittest.TestCase):
|
||||
"site.live_services[].days", "site.live_services[].start", "site.live_services[].end"}
|
||||
for w in ("days", "start", "end"):
|
||||
if "auto_fix.windows[]." + w in read:
|
||||
read |= {"service_windows[].%s" % w, "site.live_services[].%s" % w, "ready.due[].%s" % w}
|
||||
read |= {"service_windows[].%s" % w, "site.live_services[].%s" % w, "ready.due[].%s" % w,
|
||||
"failover.windows[].%s" % w}
|
||||
dead = sorted(p for p in schema.F if p not in read and p not in ui_only)
|
||||
self.assertEqual(dead, [], "declared settings nothing reads")
|
||||
|
||||
|
||||
+303
-1
@@ -71,6 +71,7 @@ def healthy_snapshot(now):
|
||||
"org.waterschurch.lobby-captions-display": 901, "org.waterschurch.digico-relay": 902}
|
||||
snap.launchd_time = now
|
||||
snap.procs = {"/Applications/ProdDash.app/Contents/MacOS/ProdDash": True}
|
||||
snap.remote = {"studiob\0pgrep -xq 'Pro Tools'": True} # the failover's check has a verdict
|
||||
snap.http = {"https://central.resi.io/": True, "http://10.6.7.13:8080/": True, "http://10.1.14.225:1025/": True}
|
||||
snap.disk = {"/": 97}
|
||||
return snap
|
||||
@@ -1521,6 +1522,284 @@ class TestFixes(unittest.TestCase):
|
||||
self.assertIn(f.get("kind", "ssh"), sp.FIX_KINDS)
|
||||
|
||||
|
||||
def run_failover(engine, snap, start, seconds, step=0.5):
|
||||
"""Like run_steps, but the failover looks after every step, as the run loop does."""
|
||||
tiles, t = None, start
|
||||
while t <= start + seconds:
|
||||
snap.conns_time = t
|
||||
tiles = engine.step(snap, t)
|
||||
engine.failover(snap, None, t)
|
||||
t += step
|
||||
return tiles, t
|
||||
|
||||
|
||||
STB_PING, STB_PT = "10.1.13.175", "studiob\0pgrep -xq 'Pro Tools'"
|
||||
|
||||
|
||||
class TestFailover(unittest.TestCase):
|
||||
"""Studio B dies on a Sunday: the FOH Backup button is pressed by itself, once, and the keys say so.
|
||||
Nothing here presses a real button, posts a real alert or writes a real log."""
|
||||
|
||||
def setUp(self):
|
||||
self.snap = healthy_snapshot(SUNDAY)
|
||||
self.engine = sp.Engine(CONFIG)
|
||||
self.ran, self.sent = [], []
|
||||
self.engine.executor = lambda cfg, fix_id, chk: self.ran.append(fix_id) or True
|
||||
self.engine.poe_executor = lambda *a: True
|
||||
self.engine.notify = lambda url, text: self.sent.append(text)
|
||||
self.engine.autolog = sp.AutoLog(tempfile.mkdtemp())
|
||||
|
||||
def fired_at(self):
|
||||
self.engine.fix_threads and [t.join(2) for t in self.engine.fix_threads]
|
||||
rec = self.engine.failed.get("stream_foh_backup")
|
||||
return rec and rec["at"]
|
||||
|
||||
def test_config_wires_studio_b_to_foh_backup(self):
|
||||
tile = next(t for t in CONFIG["tiles"] if t["id"] == "stb_audio")
|
||||
self.assertNotIn("pos", tile) # hidden: AUDIO shows it
|
||||
self.assertIn("stb_audio", next(t for t in CONFIG["tiles"] if t["id"] == "audio")["members"])
|
||||
for c in tile["checks"]:
|
||||
fixes = set(c["fix"].values()) if isinstance(c["fix"], dict) else {c["fix"]}
|
||||
self.assertEqual(fixes, {"stream_foh_backup"}, c["type"])
|
||||
fix = CONFIG["fixes"]["stream_foh_backup"]
|
||||
self.assertEqual((fix["kind"], fix["page"], fix["row"], fix["col"]), ("companion_press", 4, 0, 5))
|
||||
self.assertTrue(fix["failover"])
|
||||
self.assertEqual(fix["undo"], "stream_studio_b")
|
||||
self.assertFalse(CONFIG["fixes"]["stream_studio_b"].get("failover"))
|
||||
self.assertNotIn("stream_foh_backup", CONFIG["auto_fix"]["fixes"])
|
||||
keys = {k["id"]: k["fix"] for k in CONFIG["restart_page"]["keys"]}
|
||||
self.assertEqual((keys["rs_stream_stb"], keys["rs_stream_foh"]), ("stream_studio_b", "stream_foh_backup"))
|
||||
self.assertEqual(CONFIG["failover"]["after_seconds"], 2)
|
||||
self.assertEqual(self.engine.failover_windows, [{"days": ["sun"], "start": "06:30", "end": "13:30"}])
|
||||
|
||||
def test_studio_b_dies_foh_backup_is_pressed_once(self):
|
||||
t0 = SUNDAY
|
||||
run_failover(self.engine, self.snap, t0 - 5, 4) # all fine for a while
|
||||
self.snap.ping[STB_PING] = 2 # two 1-second pings missed
|
||||
tiles, t = run_failover(self.engine, self.snap, t0, 6)
|
||||
at = self.fired_at()
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"])
|
||||
key = next(k for k, (tid, chk, sub) in self.engine.meta.items() if chk.get("host") == STB_PING)
|
||||
since = self.engine.trackers[key].since
|
||||
self.assertEqual(since, t0 + 1) # 1 s debounce
|
||||
self.assertEqual(at, since + 2) # + 2 s after_seconds
|
||||
self.assertLessEqual(at - t0, 5)
|
||||
tiles, t = run_failover(self.engine, self.snap, t, 60) # the fault lasts a minute
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"]) # still once
|
||||
self.assertNotIn("stream_studio_b", self.ran) # never switched back by itself
|
||||
# the keys say where the stream is
|
||||
self.assertEqual(tiles["audio"]["x"], "AUDIO\nSTUDIO B NO PING\nON FOH BACKUP")
|
||||
self.assertEqual(level(tiles["audio"]), "err")
|
||||
self.engine.notes.pop("stb_audio", None) # under the fix's own feedback
|
||||
tiles = self.engine.step(self.snap, t)
|
||||
self.assertEqual(tiles["stb_audio"]["x"], "STB AUDIO\nSTUDIO B NO PING\nON FOH BACKUP")
|
||||
self.assertEqual(tiles["all"]["x"], "1 DOWN\nSTUDIO B NO PING")
|
||||
# the log, the alert, and a hold right after
|
||||
body = [l for l in open(self.engine.autolog.path(t0)).read().splitlines() if l[:2].isdigit()]
|
||||
self.assertTrue(any("STB AUDIO FAILOVER: STUDIO B NO PING - press the FOH Backup button" in l for l in body), body)
|
||||
self.assertTrue(any("-> done, checking it comes back" in l for l in body), body)
|
||||
self.assertEqual(len(self.sent), 1)
|
||||
self.assertTrue(self.sent[0].startswith("NA Ingest: STB AUDIO STUDIO B NO PING - switched over by itself: press the FOH Backup"))
|
||||
self.assertIn("hold STREAM STB on the Restart page", self.sent[0])
|
||||
self.engine.run_fixes("audio", self.snap, None, at + 10)
|
||||
self.assertEqual(self.engine.notes["audio"]["text"], "JUST TRIED\nWAIT 1 MIN")
|
||||
self.assertIsNone(self.engine.run_force("rs_stream_foh", self.snap, None, at + 11))
|
||||
self.assertEqual(self.engine.notes["rs_stream_foh"]["text"], "JUST DONE\nWAIT 1 MIN")
|
||||
out = sp.snapshot_summary(self.snap, self.engine, t)
|
||||
json.dumps(out)
|
||||
self.assertEqual(out["failover"]["stream_foh_backup"]["note"], "ON FOH BACKUP")
|
||||
self.assertEqual(out["failover"]["stream_foh_backup"]["faults"], ["STUDIO B NO PING"])
|
||||
|
||||
def test_pro_tools_crash_is_as_fast(self):
|
||||
t0 = SUNDAY
|
||||
self.snap.remote[STB_PT] = False
|
||||
run_failover(self.engine, self.snap, t0, 6)
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"])
|
||||
self.assertLessEqual(self.fired_at() - t0, 5)
|
||||
tiles = self.engine.step(self.snap, t0 + 7)
|
||||
self.assertEqual(tiles["audio"]["x"], "AUDIO\nPRO TOOLS NOT RUNNING\nON FOH BACKUP")
|
||||
|
||||
def test_fires_with_alarms_muted(self):
|
||||
self.engine.service.set(False, SUNDAY) # maintenance: auto-fix stops
|
||||
self.snap.ping[STB_PING] = 2
|
||||
run_failover(self.engine, self.snap, SUNDAY, 6)
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"]) # the failover does not
|
||||
|
||||
def test_only_inside_the_window(self):
|
||||
for when in (WEDNESDAY, datetime(2026, 10, 11, 15, 0).timestamp()): # a weekday; Sunday after the window
|
||||
snap = healthy_snapshot(when)
|
||||
snap.ping[STB_PING] = 2
|
||||
run_failover(self.engine, snap, when, 10)
|
||||
self.assertEqual(self.ran, [], datetime.fromtimestamp(when))
|
||||
self.assertEqual(self.engine.failed, {})
|
||||
|
||||
def test_rearms_when_the_fault_clears_and_the_note_fades(self):
|
||||
t0 = SUNDAY
|
||||
self.snap.ping[STB_PING] = 2
|
||||
tiles, t = run_failover(self.engine, self.snap, t0, 6)
|
||||
self.snap.ping[STB_PING] = 0 # Studio B answers again
|
||||
tiles, t = run_failover(self.engine, self.snap, t, 5)
|
||||
self.assertEqual(tiles["audio"]["x"], "AUDIO\nOK\nON FOH BACKUP") # still on the backup
|
||||
self.assertEqual(level(tiles["audio"]), "ok")
|
||||
self.assertIsNotNone(self.engine.failed["stream_foh_backup"]["cleared"])
|
||||
self.snap.ping[STB_PING] = 2 # ...and dies again 5 s later
|
||||
tiles, t = run_failover(self.engine, self.snap, t, 4)
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"]) # was clear under 10 s: not armed yet
|
||||
tiles, t = run_failover(self.engine, self.snap, t, 3) # 10 s since it cleared: armed, fires
|
||||
self.assertEqual(self.ran, ["stream_foh_backup", "stream_foh_backup"])
|
||||
self.snap.ping[STB_PING] = 0
|
||||
tiles, t = run_failover(self.engine, self.snap, t, 2)
|
||||
tiles, t = run_failover(self.engine, self.snap, t + 1000, 1) # note_seconds later: forgotten
|
||||
self.assertEqual(tiles["audio"]["x"], "AUDIO\nOK")
|
||||
self.assertEqual(self.engine.failed, {})
|
||||
|
||||
def test_a_check_that_cannot_ask_is_not_a_clear(self):
|
||||
t0 = SUNDAY
|
||||
self.snap.remote[STB_PT] = False # Pro Tools gone
|
||||
tiles, t = run_failover(self.engine, self.snap, t0, 6)
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"])
|
||||
self.snap.remote[STB_PT] = None # ssh could not ask for a while
|
||||
tiles, t = run_failover(self.engine, self.snap, t, 30)
|
||||
self.assertIsNone(self.engine.failed["stream_foh_backup"]["cleared"]) # not re-armed on a hiccup
|
||||
self.snap.remote[STB_PT] = False # the answer is back: still gone
|
||||
tiles, t = run_failover(self.engine, self.snap, t, 10)
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"]) # no second press
|
||||
self.assertEqual(tiles["rs_stream_foh"]["x"], "STREAM FOH\nON FOH BACKUP") # pressed already: no HOLD TO FIX
|
||||
self.assertEqual(level(tiles["rs_stream_foh"]), "ok")
|
||||
self.assertEqual(tiles["rs_stream_stb"]["x"], "STREAM STB\nHOLD 2s")
|
||||
|
||||
def test_studio_b_rednet_is_never_power_cycled_by_itself(self):
|
||||
"""Only STUDIO-B-RedNet-TNX vanishes: the failover presses FOH Backup; Sunday auto-fix on the DANTE
|
||||
key must not power-cycle the RedNet's PoE port (that stays a hold), nor touch anything else."""
|
||||
comp = mock_comp()
|
||||
self.snap.dante_map = {"STUDIO-B-RedNet-TNX": {"mac": "00:1d:c1:aa:bb:cc", "poe": True, "switch": "10.1.60.2", "port": "0/14"}}
|
||||
devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")])
|
||||
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps([d for d in devices if d != "STUDIO-B-RedNet-TNX"])
|
||||
poe = []
|
||||
self.engine.poe_executor = lambda cfg, name, entry: poe.append(name) or True
|
||||
tiles, t = run_failover(self.engine, self.snap, SUNDAY, 30)
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"])
|
||||
self.assertEqual(self.engine.auto_fix(self.snap, comp, t), [])
|
||||
time.sleep(0.2)
|
||||
self.assertEqual((poe, comp.restarted, self.ran), ([], [], ["stream_foh_backup"]))
|
||||
self.assertEqual([(k, w) for _, k, w, _ in self.engine.fix_plan("dante", self.snap)], [("poe", "STUDIO-B-RedNet-TNX")]) # a hold would
|
||||
|
||||
def test_switching_back_by_hand_clears_the_note(self):
|
||||
t0 = SUNDAY
|
||||
gate = threading.Event() # the Studio B press takes a moment
|
||||
|
||||
def press(cfg, fix_id, chk):
|
||||
if fix_id == "stream_studio_b":
|
||||
gate.wait(2)
|
||||
self.ran.append(fix_id)
|
||||
return True
|
||||
self.engine.executor = press
|
||||
self.snap.ping[STB_PING] = 2
|
||||
tiles, t = run_failover(self.engine, self.snap, t0, 6)
|
||||
self.assertIn("ON FOH BACKUP", tiles["audio"]["x"])
|
||||
self.assertEqual(self.engine.run_force("rs_stream_stb", self.snap, None, t), "stream_studio_b")
|
||||
self.assertEqual(self.engine.notes["rs_stream_stb"]["text"], "PRESSING")
|
||||
gate.set()
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(self.engine.notes["rs_stream_stb"]["text"], "DONE")
|
||||
self.assertEqual(self.engine.failed, {})
|
||||
tiles = self.engine.step(self.snap, t + 0.5)
|
||||
self.assertEqual(tiles["audio"]["x"], "AUDIO\nSTUDIO B NO PING")
|
||||
tiles, t = run_failover(self.engine, self.snap, t + 1, 4) # still dead: it fires again
|
||||
self.assertEqual(self.ran, ["stream_foh_backup", "stream_studio_b", "stream_foh_backup"])
|
||||
self.assertIn("ON FOH BACKUP", tiles["audio"]["x"])
|
||||
|
||||
def test_held_back_when_several_dante_devices_vanish(self):
|
||||
t0 = SUNDAY
|
||||
devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")])
|
||||
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(
|
||||
[d for d in devices if d not in ("STUDIO-B-RedNet-TNX", "NA-Tracks", "BOH-WING")])
|
||||
tiles, t = run_failover(self.engine, self.snap, t0, 12)
|
||||
self.assertEqual(self.ran, []) # Ingest's view of Dante: hands off
|
||||
self.assertEqual(tiles["stb_audio"]["x"], "STB AUDIO\nSTB TNX GONE") # seen, no note
|
||||
body = open(self.engine.autolog.path(t0)).read()
|
||||
self.assertIn("failover held back: 3 Dante devices gone at once", body)
|
||||
self.assertEqual(body.count("held back"), 1) # said once, not every half second
|
||||
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(
|
||||
[d for d in devices if d != "STUDIO-B-RedNet-TNX"]) # only Studio B's own box
|
||||
run_failover(self.engine, self.snap, t, 3)
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"])
|
||||
|
||||
def test_state_survives_a_restart(self):
|
||||
state = os.path.join(tempfile.mkdtemp(prefix="sp-state-"), "state.json")
|
||||
engine = sp.Engine(CONFIG, state)
|
||||
engine.executor = lambda cfg, fix_id, chk: self.ran.append(fix_id) or True
|
||||
engine.notify = lambda url, text: None
|
||||
engine.autolog = sp.AutoLog(tempfile.mkdtemp())
|
||||
self.snap.ping[STB_PING] = 2
|
||||
tiles, t = run_failover(engine, self.snap, SUNDAY, 6)
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"])
|
||||
self.assertEqual(json.load(open(state))["failover"]["stream_foh_backup"]["tile"], "stb_audio")
|
||||
again = sp.Engine(CONFIG, state) # the watchdog restarts mid-fault
|
||||
again.executor = lambda cfg, fix_id, chk: self.ran.append(fix_id) or True
|
||||
again.notify = lambda url, text: None
|
||||
again.autolog = sp.AutoLog(tempfile.mkdtemp())
|
||||
tiles, t = run_failover(again, self.snap, t, 10)
|
||||
self.assertEqual(self.ran, ["stream_foh_backup"]) # not pressed a second time
|
||||
self.assertEqual(tiles["audio"]["x"], "AUDIO\nSTUDIO B NO PING\nON FOH BACKUP")
|
||||
|
||||
def test_nothing_new_starts_while_stopping(self):
|
||||
self.snap.ping[STB_PING] = 2
|
||||
self.engine.stopping = True
|
||||
run_failover(self.engine, self.snap, SUNDAY, 6)
|
||||
self.assertEqual(self.ran, [])
|
||||
|
||||
def test_load_config_checks_the_button_and_the_undo(self):
|
||||
def load(change):
|
||||
cfg = json.loads(json.dumps(CONFIG))
|
||||
change(cfg)
|
||||
path = os.path.join(tempfile.mkdtemp(prefix="sp-cfg-"), "config.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump(cfg, f)
|
||||
return sp.load_config(path)
|
||||
load(lambda c: None)
|
||||
for bad in (lambda c: c["fixes"]["stream_foh_backup"].update(page=0),
|
||||
lambda c: c["fixes"]["stream_foh_backup"].update(row=True),
|
||||
lambda c: c["fixes"]["stream_foh_backup"].update(col="5"),
|
||||
lambda c: c["fixes"]["stream_foh_backup"].pop("col"),
|
||||
lambda c: c["fixes"]["stream_foh_backup"].update(never_auto=True),
|
||||
lambda c: c["fixes"]["stream_foh_backup"].update(undo="nope"),
|
||||
lambda c: c["failover"]["windows"].append({"days": ["sun"], "start": "6:30", "end": "13:30"})):
|
||||
with self.assertRaises(ValueError):
|
||||
load(bad)
|
||||
|
||||
def test_studio_b_is_pinged_every_second(self):
|
||||
self.assertEqual(sp.ping_hosts(CONFIG, fast=True), [STB_PING])
|
||||
slow = sp.ping_hosts(CONFIG)
|
||||
self.assertNotIn(STB_PING, slow)
|
||||
self.assertIn("10.1.0.1", slow) # the gateway stays in the slow list
|
||||
self.assertIn("10.1.3.227", slow)
|
||||
self.assertEqual(sp.PING_INTERVAL, 3)
|
||||
|
||||
def test_remote_groups_run_when_due_each_on_its_own(self):
|
||||
from unittest import mock
|
||||
groups = sp.remote_groups(CONFIG)
|
||||
self.assertEqual(groups[("studiob", 2)], ["pgrep -xq 'Pro Tools'"])
|
||||
self.assertTrue(all(k[1] == 10 for k in groups if k != ("studiob", 2)))
|
||||
calls = []
|
||||
fake = lambda argv, **kw: calls.append(argv) or mock.Mock(returncode=0, stdout="SP0=0\nSP1=0\n", stderr="")
|
||||
t = SUNDAY + 100
|
||||
with mock.patch.object(sp.subprocess, "run", fake):
|
||||
last = {k: t - 3 for k in groups} # every group ran 3 s ago
|
||||
for th in sp.poll_remote(self.snap, CONFIG, now=t, last=last):
|
||||
th.join(2)
|
||||
self.assertEqual(len(calls), 1) # only the 2-second group is due
|
||||
self.assertIn("studiob@10.1.13.175", calls[0])
|
||||
self.assertIn("pgrep -xq 'Pro Tools'", calls[0][-1])
|
||||
self.assertTrue(self.snap.remote[STB_PT])
|
||||
self.assertEqual(last[("studiob", 2)], t)
|
||||
self.assertEqual(sp.poll_remote(self.snap, CONFIG, now=t + 1, last=last), []) # nothing due yet
|
||||
calls.clear()
|
||||
sp.poll_remote(self.snap, CONFIG) # run_once: everything, and waits
|
||||
self.assertEqual(len(calls), len(groups))
|
||||
self.assertEqual(sp.REMOTE_INTERVAL, 10)
|
||||
|
||||
|
||||
class TestNetgearParsing(unittest.TestCase):
|
||||
BIG = """ VLAN ID MAC Address Interface IfIndex Status
|
||||
66 00:1D:C1:51:34:C8 0/14 14 Learned
|
||||
@@ -1603,6 +1882,13 @@ class FakeCompanion(BaseHTTPRequestHandler):
|
||||
if m:
|
||||
s["restarts"].append(m.group(1))
|
||||
return self._send(200, "{}")
|
||||
m = re.match(r"^/api/location/(\d+)/(\d+)/(\d+)/press$", u.path)
|
||||
if m:
|
||||
where = tuple(int(x) for x in m.groups())
|
||||
s["presses"].append(where)
|
||||
if where in s["buttons"]:
|
||||
return self._send(200, "ok")
|
||||
return self._send(204, "No control") # like Companion 5.0.7 for an empty cell
|
||||
self._send(404, "")
|
||||
|
||||
|
||||
@@ -1615,6 +1901,8 @@ class TestAgainstFakeCompanion(unittest.TestCase):
|
||||
"vars": dict(snap.vars),
|
||||
"custom": {"sp_state": "", "sp_hb": "", "sp_cmd": "", "flash_tick": "0"},
|
||||
"restarts": [],
|
||||
"presses": [],
|
||||
"buttons": {(4, 0, 5), (4, 0, 6)}, # the Audio page's FOH Backup and Studio B
|
||||
}
|
||||
cls.server = ThreadingHTTPServer(("127.0.0.1", 0), FakeCompanion)
|
||||
threading.Thread(target=cls.server.serve_forever, daemon=True).start()
|
||||
@@ -1637,6 +1925,20 @@ class TestAgainstFakeCompanion(unittest.TestCase):
|
||||
self.assertEqual(comp.set_custom("missing", "x"), (False, 404))
|
||||
print("\n sp_state is %d bytes, %d in the URL" % (len(js), len(urllib.parse.quote(js, safe=""))))
|
||||
|
||||
def test_press_a_button(self):
|
||||
comp = sp.Companion("127.0.0.1", self.port)
|
||||
store = FakeCompanion.store
|
||||
self.assertEqual(comp.press(4, 0, 5), 200)
|
||||
self.assertEqual(comp.press(4, 1, 1), 204) # an empty cell
|
||||
self.assertEqual(store["presses"][-2:], [(4, 0, 5), (4, 1, 1)])
|
||||
cfg = json.loads(json.dumps(CONFIG))
|
||||
cfg["companion"]["port"] = self.port
|
||||
self.assertTrue(sp.execute_fix(cfg, "stream_foh_backup", {}))
|
||||
self.assertEqual(store["presses"][-1], (4, 0, 5))
|
||||
cfg["fixes"]["stream_foh_backup"]["col"] = 7
|
||||
self.assertEqual(sp.execute_fix(cfg, "stream_foh_backup", {}), "NO BUTTON\nTHERE")
|
||||
self.assertEqual(store["presses"][-1], (4, 0, 7))
|
||||
|
||||
def test_commands(self):
|
||||
comp = sp.Companion("127.0.0.1", self.port)
|
||||
store = FakeCompanion.store
|
||||
@@ -1679,7 +1981,7 @@ class TestAgainstFakeCompanion(unittest.TestCase):
|
||||
sp.poll_mdns = lambda snap, seconds=3: None
|
||||
sp.poll_system = lambda snap, cfg: None
|
||||
sp.poll_midi = lambda snap, cfg: None # CoreMIDI has its own test
|
||||
sp.poll_remote = lambda snap, cfg: None # never SSH anywhere from a test
|
||||
sp.poll_remote = lambda snap, cfg, **k: [] # never SSH anywhere from a test
|
||||
sp.start_remote_midi_streams = lambda snap, cfg, stop: []
|
||||
stop = threading.Event()
|
||||
loop = threading.Thread(target=sp.run, args=(cfg, data_dir), kwargs={"stop": stop}, daemon=True)
|
||||
|
||||
Reference in New Issue
Block a user