From df4da8f42ee6ae84044e006b5d788edc469962d8 Mon Sep 17 00:00:00 2001 From: christian Date: Thu, 8 Oct 2026 12:23:05 -0400 Subject: [PATCH] Never reboot a HyperDeck to fix a connection that is switched off; turn connections back on instead. On 2026-10-08 a hold on HD1, whose Companion connection had been disabled, sent the deck a reboot. Now a disabled connection on any button is fixed by enabling it, before any other fix is considered. The HyperDeck fix asks the deck itself (transport info on port 9993): if it answers - recording or not - only Companion's connection is restarted; only a deck that accepts a connection and says nothing at all is rebooted; an unreachable deck is left alone and the button shows FIX FAILED. Tested against a fake deck. Co-Authored-By: Claude Opus 5.5 --- config.json | 14 +++--- status_panel.py | 87 ++++++++++++++++++++++++++++++++------ tests/test_status_panel.py | 84 +++++++++++++++++++++++++++++++++++- 3 files changed, 164 insertions(+), 21 deletions(-) diff --git a/config.json b/config.json index be7d822..83dec45 100644 --- a/config.json +++ b/config.json @@ -48,7 +48,7 @@ "captions": { "kind": "local", "short": "CAPTIONS", "label": "restart lobby captions", "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" }, "proddash": { "kind": "local", "short": "PRODDASH", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" }, - "hd_reboot": { "kind": "hyperdeck_reboot", "short": "REBOOT", "label": "reboot HyperDeck" } + "hyperdeck": { "kind": "hyperdeck", "short": "HYPERDECK", "label": "reconnect Companion to the HyperDeck, or reboot the deck only if it is hung" } }, "tiles": [ @@ -60,12 +60,12 @@ { "id": "video", "label": "VIDEO", "pos": [0, 6], "members": ["atem", "led", "arena"] }, { "id": "stream", "label": "STREAM", "pos": [0, 7], "members": ["obs", "resi"] }, - { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hd_reboot" } }] }, - { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hd_reboot" } }] }, - { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hd_reboot" } }] }, - { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hd_reboot" } }] }, - { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hd_reboot" } }] }, - { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hyperdeck" } }] }, + { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hyperdeck" } }] }, + { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hyperdeck" } }] }, + { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hyperdeck" } }] }, + { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hyperdeck" } }] }, + { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hyperdeck" } }] }, { "id": "tascam", "label": "TASCAM", "pos": [1, 7], "checks": [{ "type": "tascam", "conn": "tascam" }] }, { "id": "x32", "label": "X32", "pos": [2, 1], "checks": [{ "type": "conn", "conn": "x32" }] }, diff --git a/status_panel.py b/status_panel.py index 16e3e85..aba0e4a 100644 --- a/status_panel.py +++ b/status_panel.py @@ -120,6 +120,10 @@ class Companion: status, _ = self._request("POST", "/api/connections/%s/restart" % urllib.parse.quote(conn_id, safe="")) return status == 200 + def enable_connection(self, conn_id): + status, _ = self._request("POST", "/api/connections/%s/enable" % urllib.parse.quote(conn_id, safe="")) + return status == 200 + # --- Shared snapshot written by the pollers -------------------------------- @@ -735,6 +739,15 @@ class Engine: tr = self.trackers.get(key) if tid not in ids or not tr or not tr.confirmed or tr.result.level not in (WARN, ERR): continue + # A connection switched off in Companion is fixed by switching it on - never by + # acting on the device (2026-10-08: a disabled HyperDeck connection got a deck reboot). + label = chk.get("conn") + if label and label in snap.conns and not snap.conns[label].get("enabled"): + ident = ("enable", label) + if ident not in seen: + seen.add(ident) + plan.append((ident, "enable", label, chk)) + continue fix_id = chk.get("fix") if isinstance(fix_id, dict): fix_id = fix_id.get(sub) @@ -742,7 +755,7 @@ class Engine: if chk["type"] == "hyperdeck" and hyperdeck_recording(snap, chk["conn"]): continue # One run per fix, except a HyperDeck reboot, which is per deck. - per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck_reboot" + per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck" ident = ("fix", fix_id, chk.get("ip", "") if per_deck else "") if ident not in seen: seen.add(ident) @@ -783,7 +796,13 @@ class Engine: waited = True continue self.fix_last[ident] = now - if kind == "restart": + if kind == "enable": + c = snap.conns.get(what) + if c: + ok = comp.enable_connection(c["id"]) + log.info("ENABLE %s -> %s", what, "ok" if ok else "failed") + names.append("TURN ON") + elif kind == "restart": c = snap.conns.get(what) if c and c.get("enabled"): ok = comp.restart_connection(c["id"]) @@ -1100,6 +1119,41 @@ def poll_remote(snap, cfg): snap.remote = result +def hyperdeck_probe(ip, timeout=3, port=9993): + """Ask a HyperDeck directly what it is doing, on its control port 9993. + + Returns its transport status ("record", "preview", "stopped", ...), "busy" if it + refused another client, "answered" if it greeted but did not report a status, + "hung" if it accepted the connection and said nothing at all, or "unreachable". + Only "hung" justifies a reboot. + """ + try: + s = socket.create_connection((ip, port), timeout=timeout) + except OSError: + return "unreachable" + buf = b"" + with s: + s.settimeout(timeout) + try: + s.sendall(b"transport info\r\n") + end = time.time() + timeout + while time.time() < end: + chunk = s.recv(4096) + if not chunk: + break + buf += chunk + m = re.search(rb"208 transport info:.*?status: (\w+)", buf, re.S) + if m: + return m.group(1).decode().lower() + if b"120 connection rejected" in buf: + return "busy" + except socket.timeout: + pass + except OSError: + return "unreachable" if not buf else "answered" + return "answered" if buf else "hung" + + def execute_fix(cfg, fix_id, chk): """Run one entry from config "fixes" (in its own thread). Only listed commands ever run. Returns True if the command ran and succeeded.""" @@ -1113,16 +1167,25 @@ def execute_fix(cfg, fix_id, chk): log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode, (": " + detail) if detail else "") return r.returncode == 0 - if kind == "hyperdeck_reboot": - with socket.create_connection((chk["ip"], 9993), timeout=3) as s: - s.settimeout(3) - try: - s.recv(1024) # the deck's greeting - except socket.timeout: - pass - s.sendall(b"reboot\n") - log.info("FIX %s %s -> reboot sent", label, chk["conn"]) - return True + if kind == "hyperdeck": + port = chk.get("port", 9993) + verdict = hyperdeck_probe(chk["ip"], port=port) + if verdict == "hung": + with socket.create_connection((chk["ip"], port), timeout=3) as s: + s.sendall(b"reboot\r\n") + log.info("FIX %s %s -> deck accepted a connection but did not answer: reboot sent", + label, chk["conn"]) + return True + if verdict == "unreachable": + log.info("FIX %s %s -> deck does not answer on the network: nothing sent", label, chk["conn"]) + return False + # The deck answers (and may be recording): leave it alone, reconnect Companion instead. + comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000)) + ids = {c["label"]: c["id"] for c in comp.connections()} + ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]]) + log.info("FIX %s %s -> deck is fine (%s): Companion connection restarted %s", + label, chk["conn"], verdict, "ok" if ok else "FAILED") + return bool(ok) log.warning("FIX %s: unknown kind %r", label, kind) except Exception as e: log.warning("FIX %s failed: %s", label, e) diff --git a/tests/test_status_panel.py b/tests/test_status_panel.py index 601d139..f7785ca 100644 --- a/tests/test_status_panel.py +++ b/tests/test_status_panel.py @@ -469,11 +469,19 @@ class TestFixes(unittest.TestCase): for i in (2, 3, 5): self.snap.conns["HyperDeck_%d" % i]["category"] = "error" self.snap.vars[("HyperDeck_3", "status")] = "Record" # last known state: recording - self.assertEqual(self.plan("rec"), [("fix", "hd_reboot"), ("fix", "hd_reboot")]) + self.assertEqual(self.plan("rec"), [("fix", "hyperdeck"), ("fix", "hyperdeck")]) decks = [chk["conn"] for _, _, _, chk in self.engine.fix_plan("rec", self.snap)] self.assertEqual(decks, ["HyperDeck_2", "HyperDeck_5"]) self.assertEqual(self.plan("hd3"), []) + def test_disabled_connection_is_switched_on_never_rebooted(self): + # 2026-10-08: HyperDeck_1 disabled in Companion, a hold sent the deck a reboot. + self.snap.conns["HyperDeck_1"]["enabled"] = False + self.snap.conns["x32"]["enabled"] = False + self.assertEqual(self.plan("hd1"), [("enable", "HyperDeck_1")]) + self.assertEqual(self.plan("rec"), [("enable", "HyperDeck_1")]) + self.assertEqual(self.plan("x32"), [("enable", "x32")]) + def test_not_recording_is_not_fixed_by_a_reboot(self): for i in (1, 3): self.snap.vars[("HyperDeck_%d" % i, "status")] = "Record" @@ -559,7 +567,79 @@ class TestFixes(unittest.TestCase): def test_every_fix_is_valid(self): for f in CONFIG["fixes"].values(): - self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck_reboot")) + self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck")) + + +class FakeDeck: + """A pretend HyperDeck on a local port: answers like a real one, stays silent, or greets only.""" + + def __init__(self, mode): + import socket as _s + self.mode, self.received = mode, b"" + self.srv = _s.socket() + self.srv.bind(("127.0.0.1", 0)) + self.srv.listen(5) + self.port = self.srv.getsockname()[1] + threading.Thread(target=self.serve, daemon=True).start() + + def serve(self): + while True: + try: + conn, _ = self.srv.accept() + except OSError: + return + with conn: + conn.settimeout(4) + if self.mode != "silent": + conn.sendall(b"500 connection info:\r\nprotocol version: 1.11\r\nmodel: HyperDeck Studio 4K Pro\r\n\r\n") + try: + data = conn.recv(1024) + self.received += data + if self.mode in ("record", "preview") and b"transport info" in data: + conn.sendall(b"208 transport info:\r\nstatus: %s\r\nspeed: 100\r\n\r\n" % self.mode.encode()) + more = conn.recv(1024) + self.received += more + except OSError: + pass + + def close(self): + self.srv.close() + + +class TestHyperDeckProbe(unittest.TestCase): + def test_probe_answers(self): + for mode, expected in (("record", "record"), ("preview", "preview"), ("greet", "answered"), ("silent", "hung")): + deck = FakeDeck(mode) + try: + self.assertEqual(sp.hyperdeck_probe("127.0.0.1", timeout=1, port=deck.port), expected, mode) + finally: + deck.close() + deck = FakeDeck("preview") + port = deck.port + deck.close() + self.assertEqual(sp.hyperdeck_probe("127.0.0.1", timeout=1, port=port), "unreachable") + + def test_only_a_silent_deck_is_rebooted(self): + deck = FakeDeck("silent") + try: + chk = {"conn": "HyperDeck_1", "ip": "127.0.0.1", "port": deck.port} + self.assertTrue(sp.execute_fix(CONFIG, "hyperdeck", chk)) + time.sleep(0.3) + self.assertIn(b"reboot", deck.received) + finally: + deck.close() + + def test_a_recording_deck_is_never_rebooted(self): + deck = FakeDeck("record") + try: + cfg = json.loads(json.dumps(CONFIG)) + cfg["companion"]["port"] = 1 # no Companion here: the reconnect just fails + chk = {"conn": "HyperDeck_1", "ip": "127.0.0.1", "port": deck.port} + sp.execute_fix(cfg, "hyperdeck", chk) + time.sleep(0.3) + self.assertNotIn(b"reboot", deck.received) + finally: + deck.close() class FakeCompanion(BaseHTTPRequestHandler):