Never reboot a HyperDeck to fix a connection that is switched off; turn connections back on instead.

On 2026-10-08 a hold on HD1, whose Companion connection had been disabled,
sent the deck a reboot. Now a disabled connection on any button is fixed by
enabling it, before any other fix is considered. The HyperDeck fix asks the
deck itself (transport info on port 9993): if it answers - recording or not -
only Companion's connection is restarted; only a deck that accepts a
connection and says nothing at all is rebooted; an unreachable deck is left
alone and the button shows FIX FAILED. Tested against a fake deck.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

config.json

@@ -48,7 +48,7 @@
     "captions":        { "kind": "local", "short": "CAPTIONS", "label": "restart lobby captions",
                          "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" },
     "proddash":        { "kind": "local", "short": "PRODDASH", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" },
-    "hd_reboot":       { "kind": "hyperdeck_reboot", "short": "REBOOT", "label": "reboot HyperDeck" }
+    "hyperdeck":       { "kind": "hyperdeck", "short": "HYPERDECK", "label": "reconnect Companion to the HyperDeck, or reboot the deck only if it is hung" }
   },
 
   "tiles": [
@@ -60,12 +60,12 @@
     { "id": "video",  "label": "VIDEO",  "pos": [0, 6], "members": ["atem", "led", "arena"] },
     { "id": "stream", "label": "STREAM", "pos": [0, 7], "members": ["obs", "resi"] },
 
-    { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hyperdeck" } }] },
-    { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hyperdeck" } }] },
-    { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hyperdeck" } }] },
-    { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hyperdeck" } }] },
-    { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hyperdeck" } }] },
-    { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hyperdeck" } }] },
     { "id": "tascam", "label": "TASCAM", "pos": [1, 7], "checks": [{ "type": "tascam", "conn": "tascam" }] },
 
     { "id": "x32", "label": "X32", "pos": [2, 1], "checks": [{ "type": "conn", "conn": "x32" }] },

status_panel.py

@@ -120,6 +120,10 @@ class Companion:
         status, _ = self._request("POST", "/api/connections/%s/restart" % urllib.parse.quote(conn_id, safe=""))
         return status == 200
 
+    def enable_connection(self, conn_id):
+        status, _ = self._request("POST", "/api/connections/%s/enable" % urllib.parse.quote(conn_id, safe=""))
+        return status == 200
+
 
 # --- Shared snapshot written by the pollers --------------------------------
 
@@ -735,6 +739,15 @@ class Engine:
             tr = self.trackers.get(key)
             if tid not in ids or not tr or not tr.confirmed or tr.result.level not in (WARN, ERR):
                 continue
+            # A connection switched off in Companion is fixed by switching it on - never by
+            # acting on the device (2026-10-08: a disabled HyperDeck connection got a deck reboot).
+            label = chk.get("conn")
+            if label and label in snap.conns and not snap.conns[label].get("enabled"):
+                ident = ("enable", label)
+                if ident not in seen:
+                    seen.add(ident)
+                    plan.append((ident, "enable", label, chk))
+                continue
             fix_id = chk.get("fix")
             if isinstance(fix_id, dict):
                 fix_id = fix_id.get(sub)
@@ -742,7 +755,7 @@ class Engine:
                 if chk["type"] == "hyperdeck" and hyperdeck_recording(snap, chk["conn"]):
                     continue
                 # One run per fix, except a HyperDeck reboot, which is per deck.
-                per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck_reboot"
+                per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck"
                 ident = ("fix", fix_id, chk.get("ip", "") if per_deck else "")
                 if ident not in seen:
                     seen.add(ident)
@@ -783,7 +796,13 @@ class Engine:
                 waited = True
                 continue
             self.fix_last[ident] = now
-            if kind == "restart":
+            if kind == "enable":
+                c = snap.conns.get(what)
+                if c:
+                    ok = comp.enable_connection(c["id"])
+                    log.info("ENABLE %s -> %s", what, "ok" if ok else "failed")
+                    names.append("TURN ON")
+            elif kind == "restart":
                 c = snap.conns.get(what)
                 if c and c.get("enabled"):
                     ok = comp.restart_connection(c["id"])
@@ -1100,6 +1119,41 @@ def poll_remote(snap, cfg):
     snap.remote = result
 
 
+def hyperdeck_probe(ip, timeout=3, port=9993):
+    """Ask a HyperDeck directly what it is doing, on its control port 9993.
+
+    Returns its transport status ("record", "preview", "stopped", ...), "busy" if it
+    refused another client, "answered" if it greeted but did not report a status,
+    "hung" if it accepted the connection and said nothing at all, or "unreachable".
+    Only "hung" justifies a reboot.
+    """
+    try:
+        s = socket.create_connection((ip, port), timeout=timeout)
+    except OSError:
+        return "unreachable"
+    buf = b""
+    with s:
+        s.settimeout(timeout)
+        try:
+            s.sendall(b"transport info\r\n")
+            end = time.time() + timeout
+            while time.time() < end:
+                chunk = s.recv(4096)
+                if not chunk:
+                    break
+                buf += chunk
+                m = re.search(rb"208 transport info:.*?status: (\w+)", buf, re.S)
+                if m:
+                    return m.group(1).decode().lower()
+                if b"120 connection rejected" in buf:
+                    return "busy"
+        except socket.timeout:
+            pass
+        except OSError:
+            return "unreachable" if not buf else "answered"
+    return "answered" if buf else "hung"
+
+
 def execute_fix(cfg, fix_id, chk):
     """Run one entry from config "fixes" (in its own thread). Only listed commands ever run.
     Returns True if the command ran and succeeded."""
@@ -1113,16 +1167,25 @@ def execute_fix(cfg, fix_id, chk):
             log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode,
                      (": " + detail) if detail else "")
             return r.returncode == 0
-        if kind == "hyperdeck_reboot":
+        if kind == "hyperdeck":
-            with socket.create_connection((chk["ip"], 9993), timeout=3) as s:
+            port = chk.get("port", 9993)
-                s.settimeout(3)
+            verdict = hyperdeck_probe(chk["ip"], port=port)
-                try:
+            if verdict == "hung":
-                    s.recv(1024)                       # the deck's greeting
+                with socket.create_connection((chk["ip"], port), timeout=3) as s:
-                except socket.timeout:
+                    s.sendall(b"reboot\r\n")
-                    pass
+                log.info("FIX %s %s -> deck accepted a connection but did not answer: reboot sent",
-                s.sendall(b"reboot\n")
+                         label, chk["conn"])
-            log.info("FIX %s %s -> reboot sent", label, chk["conn"])
                 return True
+            if verdict == "unreachable":
+                log.info("FIX %s %s -> deck does not answer on the network: nothing sent", label, chk["conn"])
+                return False
+            # The deck answers (and may be recording): leave it alone, reconnect Companion instead.
+            comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
+            ids = {c["label"]: c["id"] for c in comp.connections()}
+            ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]])
+            log.info("FIX %s %s -> deck is fine (%s): Companion connection restarted %s",
+                     label, chk["conn"], verdict, "ok" if ok else "FAILED")
+            return bool(ok)
         log.warning("FIX %s: unknown kind %r", label, kind)
     except Exception as e:
         log.warning("FIX %s failed: %s", label, e)

tests/test_status_panel.py

@@ -469,11 +469,19 @@ class TestFixes(unittest.TestCase):
         for i in (2, 3, 5):
             self.snap.conns["HyperDeck_%d" % i]["category"] = "error"
         self.snap.vars[("HyperDeck_3", "status")] = "Record"          # last known state: recording
-        self.assertEqual(self.plan("rec"), [("fix", "hd_reboot"), ("fix", "hd_reboot")])
+        self.assertEqual(self.plan("rec"), [("fix", "hyperdeck"), ("fix", "hyperdeck")])
         decks = [chk["conn"] for _, _, _, chk in self.engine.fix_plan("rec", self.snap)]
         self.assertEqual(decks, ["HyperDeck_2", "HyperDeck_5"])
         self.assertEqual(self.plan("hd3"), [])
 
+    def test_disabled_connection_is_switched_on_never_rebooted(self):
+        # 2026-10-08: HyperDeck_1 disabled in Companion, a hold sent the deck a reboot.
+        self.snap.conns["HyperDeck_1"]["enabled"] = False
+        self.snap.conns["x32"]["enabled"] = False
+        self.assertEqual(self.plan("hd1"), [("enable", "HyperDeck_1")])
+        self.assertEqual(self.plan("rec"), [("enable", "HyperDeck_1")])
+        self.assertEqual(self.plan("x32"), [("enable", "x32")])
+
     def test_not_recording_is_not_fixed_by_a_reboot(self):
         for i in (1, 3):
             self.snap.vars[("HyperDeck_%d" % i, "status")] = "Record"
@@ -559,7 +567,79 @@ class TestFixes(unittest.TestCase):
 
     def test_every_fix_is_valid(self):
         for f in CONFIG["fixes"].values():
-            self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck_reboot"))
+            self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck"))
+
+
+class FakeDeck:
+    """A pretend HyperDeck on a local port: answers like a real one, stays silent, or greets only."""
+
+    def __init__(self, mode):
+        import socket as _s
+        self.mode, self.received = mode, b""
+        self.srv = _s.socket()
+        self.srv.bind(("127.0.0.1", 0))
+        self.srv.listen(5)
+        self.port = self.srv.getsockname()[1]
+        threading.Thread(target=self.serve, daemon=True).start()
+
+    def serve(self):
+        while True:
+            try:
+                conn, _ = self.srv.accept()
+            except OSError:
+                return
+            with conn:
+                conn.settimeout(4)
+                if self.mode != "silent":
+                    conn.sendall(b"500 connection info:\r\nprotocol version: 1.11\r\nmodel: HyperDeck Studio 4K Pro\r\n\r\n")
+                try:
+                    data = conn.recv(1024)
+                    self.received += data
+                    if self.mode in ("record", "preview") and b"transport info" in data:
+                        conn.sendall(b"208 transport info:\r\nstatus: %s\r\nspeed: 100\r\n\r\n" % self.mode.encode())
+                    more = conn.recv(1024)
+                    self.received += more
+                except OSError:
+                    pass
+
+    def close(self):
+        self.srv.close()
+
+
+class TestHyperDeckProbe(unittest.TestCase):
+    def test_probe_answers(self):
+        for mode, expected in (("record", "record"), ("preview", "preview"), ("greet", "answered"), ("silent", "hung")):
+            deck = FakeDeck(mode)
+            try:
+                self.assertEqual(sp.hyperdeck_probe("127.0.0.1", timeout=1, port=deck.port), expected, mode)
+            finally:
+                deck.close()
+        deck = FakeDeck("preview")
+        port = deck.port
+        deck.close()
+        self.assertEqual(sp.hyperdeck_probe("127.0.0.1", timeout=1, port=port), "unreachable")
+
+    def test_only_a_silent_deck_is_rebooted(self):
+        deck = FakeDeck("silent")
+        try:
+            chk = {"conn": "HyperDeck_1", "ip": "127.0.0.1", "port": deck.port}
+            self.assertTrue(sp.execute_fix(CONFIG, "hyperdeck", chk))
+            time.sleep(0.3)
+            self.assertIn(b"reboot", deck.received)
+        finally:
+            deck.close()
+
+    def test_a_recording_deck_is_never_rebooted(self):
+        deck = FakeDeck("record")
+        try:
+            cfg = json.loads(json.dumps(CONFIG))
+            cfg["companion"]["port"] = 1                      # no Companion here: the reconnect just fails
+            chk = {"conn": "HyperDeck_1", "ip": "127.0.0.1", "port": deck.port}
+            sp.execute_fix(cfg, "hyperdeck", chk)
+            time.sleep(0.3)
+            self.assertNotIn(b"reboot", deck.received)
+        finally:
+            deck.close()
 
 
 class FakeCompanion(BaseHTTPRequestHandler):