Restart page (23), force restarts, Tascam reconnect, ProdCom, Resi encoder
- Page 23 "Restart": hold a key 2 s to restart DVS (4 Macs), ProPresenter,
Ableton, Church Translate, ProdCom, ProdDash, captions, the bridge or the
Tascam connection even when the status page says it is fine ("force",
shared 60 s guard). RESTART PAGE key on page 19, STATUS PAGE back.
- Holds carry Companion's clock; ones queued while the watchdog was down
are dropped instead of firing together later.
- TASCAM hold always reconnects and checks again after 20 s: HUNG - POWER
CYCLE IT (pings, control port hung), NO NETWORK, or turns a disabled
connection on. Tascam pinged at its Companion address.
- INGEST: ProdCom (:24480), ProdDash's page (:24500), the caption screen;
DVS on Ingest restarts ProdCom afterwards. RESI: the encoder (10.1.10.202).
- Windows starts check the app really started (FIX FAILED otherwise); OBS's
start task runs elevated (OBS is "Run as administrator"); setup script
does that for any such app and gives sshd restart-on-failure.
- push_page.py checks what is on the target page, not its name (pages 22
and 97 are named "PAGE" but hold buttons); --which restart.
- Companion custom variables are written in the request body (no URL limit).
Reviewed by a 3-lens review with per-finding verification; 7 confirmed
issues fixed. Tested live: OBS hold opens OBS; Restart page TASCAM key.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+152
-6
@@ -66,7 +66,8 @@ def healthy_snapshot(now):
|
||||
snap.mdns, snap.mdns_time = {"NA Tracks", "Lighting PC", "NA Ingest"}, now
|
||||
snap.ping = {"10.1.3.227": 0, "10.6.7.13": 0, "10.1.0.1": 0}
|
||||
snap.link = True
|
||||
snap.launchd = {"org.waterschurch.resi-propresenter": 884, "org.waterschurch.lobby-captions": 900}
|
||||
snap.launchd = {"org.waterschurch.resi-propresenter": 884, "org.waterschurch.lobby-captions": 900,
|
||||
"org.waterschurch.lobby-captions-display": 901}
|
||||
snap.launchd_time = now
|
||||
snap.procs = {"/Applications/ProdDash.app/Contents/MacOS/ProdDash": True}
|
||||
snap.http = {"https://central.resi.io/": True, "http://10.6.7.13:8080/": True, "http://10.1.14.225:1025/": True}
|
||||
@@ -98,8 +99,10 @@ class TestChecks(unittest.TestCase):
|
||||
|
||||
def test_all_green_when_healthy(self):
|
||||
tiles = sp.Engine(CONFIG).step(self.snap, self.now)
|
||||
bad = {k: v["x"] for k, v in tiles.items() if level(v) not in ("ok", "service", "off")}
|
||||
bad = {k: v["x"] for k, v in tiles.items() if level(v) not in ("ok", "service", "off")
|
||||
and not k.startswith("rs_")} # Restart page keys sit grey
|
||||
self.assertEqual(bad, {})
|
||||
self.assertEqual((tiles["rs_dvs_tracks"]["x"], level(tiles["rs_dvs_tracks"])), ("DVS TRACKS\nHOLD 2s", "idle"))
|
||||
self.assertEqual(tiles["all"]["x"], "ALL OK")
|
||||
self.assertEqual(tiles["hd1"]["x"], "HD1\n28h")
|
||||
self.assertEqual(tiles["dante"]["x"], "DANTE\n19/19")
|
||||
@@ -608,6 +611,116 @@ class TestFixes(unittest.TestCase):
|
||||
self.assertEqual(self.plan("rf"), []) # nothing left to "fix"
|
||||
self.assertFalse(self.engine.failing("rf"))
|
||||
|
||||
def test_restart_page_keys_run_even_when_all_is_fine(self):
|
||||
ran = []
|
||||
self.engine.executor = lambda cfg, fix_id, chk: ran.append(fix_id) or True
|
||||
run_steps(self.engine, self.snap, SUNDAY, 4) # everything green
|
||||
self.assertEqual(self.engine.run_force("rs_dvs_tracks", self.snap, None, SUNDAY + 5), "dvs_tracks")
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(ran, ["dvs_tracks"])
|
||||
self.assertEqual(self.engine.notes["rs_dvs_tracks"]["text"], "DONE")
|
||||
self.assertIsNone(self.engine.run_force("rs_dvs_tracks", self.snap, None, SUNDAY + 20)) # 60 s guard
|
||||
self.assertEqual(self.engine.notes["rs_dvs_tracks"]["text"], "JUST DONE\nWAIT 1 MIN")
|
||||
self.assertIsNone(self.engine.run_force("rs_nope", self.snap, None, SUNDAY + 90))
|
||||
# the hold on the status page and the Restart key share the guard
|
||||
self.engine.run_force("rs_prodcom", self.snap, None, SUNDAY + 100)
|
||||
self.engine.notes.clear()
|
||||
self.snap.http["http://127.0.0.1:24480/api/v1/ping"] = False
|
||||
run_steps(self.engine, self.snap, SUNDAY + 100, 4)
|
||||
self.engine.run_fixes("ingest", self.snap, None, SUNDAY + 106)
|
||||
self.assertEqual(self.engine.notes["ingest"]["text"], "JUST TRIED\nWAIT 1 MIN")
|
||||
# what each key restarts
|
||||
keys = {k["id"]: k["fix"] for k in CONFIG["restart_page"]["keys"]}
|
||||
self.assertEqual(keys["rs_ableton"], "ableton_restart")
|
||||
self.assertIn("pkill -TERM -x 'Live'", CONFIG["fixes"]["ableton_restart"]["run"])
|
||||
self.assertIn("kickstart -k system/com.audinate.dante.DanteVirtualSoundcard",
|
||||
open(os.path.join(os.path.dirname(os.path.abspath(sp.__file__)), "remote-access", "mac-dvs-helper.sh")).read())
|
||||
self.assertIn("open -g /Applications/ProdCom.app", CONFIG["fixes"]["dvs_ingest"]["run"]) # ProdCom after DVS
|
||||
|
||||
def test_tascam_hold_always_reconnects(self):
|
||||
ran = []
|
||||
self.engine.executor = lambda cfg, fix_id, chk: ran.append(fix_id) or "HUNG - POWER\nCYCLE IT"
|
||||
run_steps(self.engine, self.snap, SUNDAY, 4) # TASCAM looks fine
|
||||
self.engine.run_fixes("tascam", self.snap, None, SUNDAY + 5)
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(ran, ["tascam_reconnect"])
|
||||
tiles = self.engine.step(self.snap, time.time())
|
||||
self.assertEqual((tiles["tascam"]["x"], level(tiles["tascam"])), ("TASCAM\nHUNG - POWER\nCYCLE IT", "failed"))
|
||||
|
||||
def test_reconnect_and_verify(self):
|
||||
from unittest import mock
|
||||
fix = CONFIG["fixes"]["tascam_reconnect"]
|
||||
conn = lambda cat, enabled=True: [{"label": "tascam", "id": "t1", "enabled": enabled, "status": {"category": cat}}]
|
||||
|
||||
def run(before, after, pings=True, restart_ok=True, enable_ok=True):
|
||||
comp = mock.Mock()
|
||||
comp.connections.side_effect = [before, after]
|
||||
comp.restart_connection.return_value = restart_ok
|
||||
comp.enable_connection.return_value = enable_ok
|
||||
with mock.patch.object(sp, "Companion", return_value=comp), \
|
||||
mock.patch.object(sp, "companion_conn_hosts", return_value={"tascam": "10.1.11.74"}), \
|
||||
mock.patch.object(sp, "ping_once", return_value=pings):
|
||||
return sp.reconnect_and_verify(CONFIG, fix, "x", sleep=lambda s: None), comp
|
||||
r, comp = run(conn("error"), conn("good"))
|
||||
self.assertTrue(r)
|
||||
comp.restart_connection.assert_called_once_with("t1")
|
||||
self.assertEqual(run(conn("error"), conn("error"))[0], "HUNG - POWER\nCYCLE IT") # pings: hung
|
||||
self.assertEqual(run(conn("error"), conn("error"), pings=False)[0], "NO NETWORK\nCHECK POWER")
|
||||
self.assertEqual(run(conn("good"), conn(None))[0], "STILL\nCONNECTING") # no verdict yet
|
||||
self.assertFalse(run(conn("error"), conn("error"), restart_ok=False)[0])
|
||||
r, comp = run(conn(None, enabled=False), conn("good")) # switched off
|
||||
self.assertTrue(r)
|
||||
comp.enable_connection.assert_called_once_with("t1")
|
||||
comp.restart_connection.assert_not_called()
|
||||
self.assertEqual(run(conn(None, enabled=False), conn(None), enable_ok=False)[0], "DISABLED IN\nCOMPANION")
|
||||
self.assertFalse(run([], [])[0])
|
||||
|
||||
def test_stale_or_unstamped_holds_are_dropped(self):
|
||||
ran = []
|
||||
self.engine.executor = lambda cfg, fix_id, chk: ran.append(fix_id) or True
|
||||
now = SUNDAY + 100
|
||||
sp.handle_commands(";force rs_bridge %d;force rs_prodcom %d;force rs_proddash" % (now - 300, now - 2),
|
||||
self.engine, None, self.snap, now)
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(ran, ["prodcom"]) # only the fresh one: not the old or unstamped
|
||||
|
||||
def test_a_second_press_keeps_saying_restarting(self):
|
||||
gate = threading.Event()
|
||||
self.engine.executor = lambda cfg, fix_id, chk: gate.wait(2) or True
|
||||
self.engine.run_force("rs_bridge", self.snap, None, SUNDAY)
|
||||
self.engine.run_force("rs_bridge", self.snap, None, SUNDAY + 3)
|
||||
self.assertEqual(self.engine.notes["rs_bridge"]["text"], "RESTARTING")
|
||||
gate.set()
|
||||
|
||||
def test_group_button_names_the_device_that_needs_a_power_cycle(self):
|
||||
self.engine.executor = lambda cfg, fix_id, chk: "HUNG - POWER\nCYCLE IT"
|
||||
self.snap.conns["tascam"]["category"] = "error"
|
||||
_, t = run_steps(self.engine, self.snap, SUNDAY, 4)
|
||||
self.engine.run_fixes("rec", self.snap, None, t)
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(self.engine.notes["rec"]["text"], "TASCAM HUNG - POWER\nCYCLE IT")
|
||||
|
||||
def test_windows_starts_check_the_app_really_started(self):
|
||||
for fix_id, exe in (("obs", "obs64.exe"), ("arena", "Arena.exe"), ("vista", "Vista.exe"), ("bibleapp", "Bible App.exe")):
|
||||
run = CONFIG["fixes"][fix_id]["run"]
|
||||
self.assertTrue(run.rstrip(")").endswith('find /I "%s" >nul' % exe), fix_id)
|
||||
self.assertIn("ping -n 9 127.0.0.1", run)
|
||||
|
||||
def test_ingest_resi_and_tascam_checks(self):
|
||||
self.snap.http["http://127.0.0.1:24480/api/v1/ping"] = False
|
||||
self.assertEqual(self.plan("ingest"), [("fix", "prodcom")])
|
||||
self.snap.http["http://127.0.0.1:24480/api/v1/ping"] = True
|
||||
self.snap.launchd.pop("org.waterschurch.lobby-captions-display")
|
||||
self.assertEqual(self.plan("ingest"), [("fix", "caption_screen")]) # the screen only
|
||||
self.snap.launchd["org.waterschurch.lobby-captions-display"] = 1
|
||||
self.snap.ping["10.1.10.202"] = 3
|
||||
tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, SUNDAY, 4)
|
||||
self.assertEqual(tiles["resi"]["x"], "RESI\nENCODER NO PING")
|
||||
self.snap.conn_hosts = {"tascam": "10.1.11.74"}
|
||||
self.snap.ping["10.1.11.74"] = 3
|
||||
tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, SUNDAY, 4)
|
||||
self.assertEqual(tiles["tascam"]["x"], "TASCAM\nTASCAM NO PING")
|
||||
|
||||
def test_midi_reconnect_where_the_peer_was_last_seen(self):
|
||||
others = [{"name": "SuperRack-LiveBox", "address": "10.1.13.140:5004", "state": "running"},
|
||||
{"name": "Studio B Mac Studio", "address": "\tStudio B Mac Studio\tlocal.", "state": "running"}]
|
||||
@@ -853,7 +966,7 @@ class TestFixes(unittest.TestCase):
|
||||
|
||||
def test_every_fix_is_valid(self):
|
||||
for f in CONFIG["fixes"].values():
|
||||
self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck", "midi_connect"))
|
||||
self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck", "midi_connect", "reconnect"))
|
||||
|
||||
|
||||
class FakeDeck:
|
||||
@@ -1000,7 +1113,11 @@ class FakeCompanion(BaseHTTPRequestHandler):
|
||||
if m:
|
||||
if m.group(1) not in s["custom"]:
|
||||
return self._send(404, "Not found")
|
||||
s["custom"][m.group(1)] = urllib.parse.parse_qs(u.query, keep_blank_values=True)["value"][0]
|
||||
body = self.rfile.read(int(self.headers.get("Content-Length") or 0)).decode()
|
||||
query = urllib.parse.parse_qs(u.query, keep_blank_values=True)
|
||||
if not body and "value" not in query:
|
||||
return self._send(400, "No value") # like Companion 5.0.7 with an empty body
|
||||
s["custom"][m.group(1)] = body if body else query["value"][0]
|
||||
return self._send(200, "ok")
|
||||
m = re.match(r"^/api/connections/([^/]+)/restart$", u.path)
|
||||
if m:
|
||||
@@ -1134,10 +1251,38 @@ class TestPage(unittest.TestCase):
|
||||
self.assertIn("'$.all.b'", box["color"]["value"]) # follows the ALL button
|
||||
positioned = [t for t in CONFIG["tiles"] if "pos" in t]
|
||||
n = sum(1 for r in self.controls.values() for c in r.values() if c["type"] == "button-layered")
|
||||
self.assertEqual(n, len(positioned) + 1) # + the page number key
|
||||
self.assertEqual(n, len(positioned) + 2) # + the page number and RESTART keys
|
||||
jump = self.controls["2"]["6"]["steps"]["0"]["action_sets"]["up"][0]
|
||||
self.assertEqual((jump["definitionId"], jump["options"]["page"]["value"]), ("set_page", 23))
|
||||
# The master is where the smallest Stream Deck can see it.
|
||||
self.assertEqual(CONFIG["tiles"][0]["pos"], [0, 1])
|
||||
|
||||
def test_restart_page(self):
|
||||
export = make_page.build_restart(CONFIG)
|
||||
self.assertEqual((export["oldPageNumber"], export["page"]["name"]), (23, "23\\nRestart"))
|
||||
c = export["page"]["controls"]
|
||||
holds = {}
|
||||
for r in c.values():
|
||||
for ctl in r.values():
|
||||
sets = ctl.get("steps", {}).get("0", {}).get("action_sets", {})
|
||||
if "2000" in sets:
|
||||
holds[sets["2000"][0]["options"]["value"]["value"]] = sets["up"]
|
||||
self.assertEqual(len(holds), len(CONFIG["restart_page"]["keys"]))
|
||||
self.assertIn("$(custom:sp_cmd);force rs_tascam $(internal:time_unix)", holds)
|
||||
self.assertTrue(all(up == [] for up in holds.values())) # a tap does nothing
|
||||
back = c["3"]["0"]["steps"]["0"]["action_sets"]["up"][0]
|
||||
self.assertEqual((back["definitionId"], back["options"]["page"]["value"]), ("set_page", 19))
|
||||
|
||||
def test_push_page_looks_at_what_is_on_the_page(self):
|
||||
import push_page
|
||||
nav = {"0": {"0": {"type": "pageup"}}, "1": {"0": {"type": "button-layered",
|
||||
"options": {"notes": "STATUS nav key (pagenum): x"}}}}
|
||||
self.assertTrue(push_page.safe_target({"page": {"name": "PAGE", "controls": nav}}, "23\\nRestart")[0])
|
||||
busy = {"page": {"name": "PAGE", "controls": dict(nav, **{"2": {"3": {"type": "button"}}})}}
|
||||
self.assertFalse(push_page.safe_target(busy, "23\\nRestart")[0]) # pages 22 and 97
|
||||
busy["page"]["name"] = "23\\nRestart"
|
||||
self.assertTrue(push_page.safe_target(busy, "23\\nRestart")[0]) # our own page again
|
||||
|
||||
def test_each_button_reads_its_own_tile(self):
|
||||
for t in CONFIG["tiles"]:
|
||||
if "pos" not in t:
|
||||
@@ -1155,7 +1300,8 @@ class TestPage(unittest.TestCase):
|
||||
hd1 = self.controls["1"]["1"]["steps"]["0"]
|
||||
up = hd1["action_sets"]["up"][0]["options"]["value"]
|
||||
self.assertEqual(up, {"value": "$(custom:sp_cmd);ack hd1", "isExpression": False})
|
||||
self.assertEqual(hd1["action_sets"]["2000"][0]["options"]["value"]["value"], "$(custom:sp_cmd);restart hd1")
|
||||
self.assertEqual(hd1["action_sets"]["2000"][0]["options"]["value"]["value"],
|
||||
"$(custom:sp_cmd);restart hd1 $(internal:time_unix)")
|
||||
self.assertEqual(hd1["options"]["runWhileHeld"], [2000])
|
||||
master = self.controls["0"]["1"]["steps"]["0"]
|
||||
self.assertIn("ackall", master["action_sets"]["up"][0]["options"]["value"]["value"])
|
||||
|
||||
Reference in New Issue
Block a user