christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Never reboot a HyperDeck to fix a connection that is switched off; turn connections back on instead.

Browse Source 
On 2026-10-08 a hold on HD1, whose Companion connection had been disabled,
sent the deck a reboot. Now a disabled connection on any button is fixed by
enabling it, before any other fix is considered. The HyperDeck fix asks the
deck itself (transport info on port 9993): if it answers - recording or not -
only Companion's connection is restarted; only a deck that accepts a
connection and says nothing at all is rebooted; an unreachable deck is left
alone and the button shows FIX FAILED. Tested against a fake deck.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-08 12:23:05 -04:00
co-authored by Claude Opus 5.5
parent b8748be655
commit df4da8f42e
3 changed files with 164 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config.json
+7 -7
View File
@@ -48,7 +48,7 @@
"captions": { "kind": "local", "short": "CAPTIONS", "label": "restart lobby captions",
"run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" },
"proddash": { "kind": "local", "short": "PRODDASH", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" },
"hd_reboot": { "kind": "hyperdeck_reboot", "short": "REBOOT", "label": "reboot HyperDeck" }
"hyperdeck": { "kind": "hyperdeck", "short": "HYPERDECK", "label": "reconnect Companion to the HyperDeck, or reboot the deck only if it is hung" }
},
"tiles": [
@@ -60,12 +60,12 @@
{ "id": "video", "label": "VIDEO", "pos": [0, 6], "members": ["atem", "led", "arena"] },
{ "id": "stream", "label": "STREAM", "pos": [0, 7], "members": ["obs", "resi"] },
{ "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hyperdeck" } }] },
{ "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hyperdeck" } }] },
{ "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hyperdeck" } }] },
{ "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hyperdeck" } }] },
{ "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hyperdeck" } }] },
{ "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hyperdeck" } }] },
{ "id": "tascam", "label": "TASCAM", "pos": [1, 7], "checks": [{ "type": "tascam", "conn": "tascam" }] },
{ "id": "x32", "label": "X32", "pos": [2, 1], "checks": [{ "type": "conn", "conn": "x32" }] },
status_panel.py
+75 -12
View File
@@ -120,6 +120,10 @@ class Companion:
status, _ = self._request("POST", "/api/connections/%s/restart" % urllib.parse.quote(conn_id, safe=""))
return status == 200
def enable_connection(self, conn_id):
status, _ = self._request("POST", "/api/connections/%s/enable" % urllib.parse.quote(conn_id, safe=""))
return status == 200
# --- Shared snapshot written by the pollers --------------------------------
@@ -735,6 +739,15 @@ class Engine:
tr = self.trackers.get(key)
if tid not in ids or not tr or not tr.confirmed or tr.result.level not in (WARN, ERR):
continue
# A connection switched off in Companion is fixed by switching it on - never by
# acting on the device (2026-10-08: a disabled HyperDeck connection got a deck reboot).
label = chk.get("conn")
if label and label in snap.conns and not snap.conns[label].get("enabled"):
ident = ("enable", label)
if ident not in seen:
seen.add(ident)
plan.append((ident, "enable", label, chk))
continue
fix_id = chk.get("fix")
if isinstance(fix_id, dict):
fix_id = fix_id.get(sub)
@@ -742,7 +755,7 @@ class Engine:
if chk["type"] == "hyperdeck" and hyperdeck_recording(snap, chk["conn"]):
continue
# One run per fix, except a HyperDeck reboot, which is per deck.
per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck_reboot"
per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck"
ident = ("fix", fix_id, chk.get("ip", "") if per_deck else "")
if ident not in seen:
seen.add(ident)
@@ -783,7 +796,13 @@ class Engine:
waited = True
continue
self.fix_last[ident] = now
if kind == "restart":
if kind == "enable":
c = snap.conns.get(what)
if c:
ok = comp.enable_connection(c["id"])
log.info("ENABLE %s -> %s", what, "ok" if ok else "failed")
names.append("TURN ON")
elif kind == "restart":
c = snap.conns.get(what)
if c and c.get("enabled"):
ok = comp.restart_connection(c["id"])
@@ -1100,6 +1119,41 @@ def poll_remote(snap, cfg):
snap.remote = result
def hyperdeck_probe(ip, timeout=3, port=9993):
"""Ask a HyperDeck directly what it is doing, on its control port 9993.
Returns its transport status ("record", "preview", "stopped", ...), "busy" if it
refused another client, "answered" if it greeted but did not report a status,
"hung" if it accepted the connection and said nothing at all, or "unreachable".
Only "hung" justifies a reboot.
"""
try:
s = socket.create_connection((ip, port), timeout=timeout)
except OSError:
return "unreachable"
buf = b""
with s:
s.settimeout(timeout)
try:
s.sendall(b"transport info\r\n")
end = time.time() + timeout
while time.time() < end:
chunk = s.recv(4096)
if not chunk:
break
buf += chunk
m = re.search(rb"208 transport info:.*?status: (\w+)", buf, re.S)
if m:
return m.group(1).decode().lower()
if b"120 connection rejected" in buf:
return "busy"
except socket.timeout:
pass
except OSError:
return "unreachable" if not buf else "answered"
return "answered" if buf else "hung"
def execute_fix(cfg, fix_id, chk):
"""Run one entry from config "fixes" (in its own thread). Only listed commands ever run.
Returns True if the command ran and succeeded."""
@@ -1113,16 +1167,25 @@ def execute_fix(cfg, fix_id, chk):
log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode,
(": " + detail) if detail else "")
return r.returncode == 0
if kind == "hyperdeck_reboot":
with socket.create_connection((chk["ip"], 9993), timeout=3) as s:
s.settimeout(3)
try:
s.recv(1024) # the deck's greeting
except socket.timeout:
pass
s.sendall(b"reboot\n")
log.info("FIX %s %s -> reboot sent", label, chk["conn"])
return True
if kind == "hyperdeck":
port = chk.get("port", 9993)
verdict = hyperdeck_probe(chk["ip"], port=port)
if verdict == "hung":
with socket.create_connection((chk["ip"], port), timeout=3) as s:
s.sendall(b"reboot\r\n")
log.info("FIX %s %s -> deck accepted a connection but did not answer: reboot sent",
label, chk["conn"])
return True
if verdict == "unreachable":
log.info("FIX %s %s -> deck does not answer on the network: nothing sent", label, chk["conn"])
return False
# The deck answers (and may be recording): leave it alone, reconnect Companion instead.
comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
ids = {c["label"]: c["id"] for c in comp.connections()}
ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]])
log.info("FIX %s %s -> deck is fine (%s): Companion connection restarted %s",
label, chk["conn"], verdict, "ok" if ok else "FAILED")
return bool(ok)
log.warning("FIX %s: unknown kind %r", label, kind)
except Exception as e:
log.warning("FIX %s failed: %s", label, e)
tests/test_status_panel.py
+82 -2
View File
@@ -469,11 +469,19 @@ class TestFixes(unittest.TestCase):
for i in (2, 3, 5):
self.snap.conns["HyperDeck_%d" % i]["category"] = "error"
self.snap.vars[("HyperDeck_3", "status")] = "Record" # last known state: recording
self.assertEqual(self.plan("rec"), [("fix", "hd_reboot"), ("fix", "hd_reboot")])
self.assertEqual(self.plan("rec"), [("fix", "hyperdeck"), ("fix", "hyperdeck")])
decks = [chk["conn"] for _, _, _, chk in self.engine.fix_plan("rec", self.snap)]
self.assertEqual(decks, ["HyperDeck_2", "HyperDeck_5"])
self.assertEqual(self.plan("hd3"), [])
def test_disabled_connection_is_switched_on_never_rebooted(self):
# 2026-10-08: HyperDeck_1 disabled in Companion, a hold sent the deck a reboot.
self.snap.conns["HyperDeck_1"]["enabled"] = False
self.snap.conns["x32"]["enabled"] = False
self.assertEqual(self.plan("hd1"), [("enable", "HyperDeck_1")])
self.assertEqual(self.plan("rec"), [("enable", "HyperDeck_1")])
self.assertEqual(self.plan("x32"), [("enable", "x32")])
def test_not_recording_is_not_fixed_by_a_reboot(self):
for i in (1, 3):
self.snap.vars[("HyperDeck_%d" % i, "status")] = "Record"
@@ -559,7 +567,79 @@ class TestFixes(unittest.TestCase):
def test_every_fix_is_valid(self):
for f in CONFIG["fixes"].values():
self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck_reboot"))
self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck"))
class FakeDeck:
"""A pretend HyperDeck on a local port: answers like a real one, stays silent, or greets only."""
def __init__(self, mode):
import socket as _s
self.mode, self.received = mode, b""
self.srv = _s.socket()
self.srv.bind(("127.0.0.1", 0))
self.srv.listen(5)
self.port = self.srv.getsockname()[1]
threading.Thread(target=self.serve, daemon=True).start()
def serve(self):
while True:
try:
conn, _ = self.srv.accept()
except OSError:
return
with conn:
conn.settimeout(4)
if self.mode != "silent":
conn.sendall(b"500 connection info:\r\nprotocol version: 1.11\r\nmodel: HyperDeck Studio 4K Pro\r\n\r\n")
try:
data = conn.recv(1024)
self.received += data
if self.mode in ("record", "preview") and b"transport info" in data:
conn.sendall(b"208 transport info:\r\nstatus: %s\r\nspeed: 100\r\n\r\n" % self.mode.encode())
more = conn.recv(1024)
self.received += more
except OSError:
pass
def close(self):
self.srv.close()
class TestHyperDeckProbe(unittest.TestCase):
def test_probe_answers(self):
for mode, expected in (("record", "record"), ("preview", "preview"), ("greet", "answered"), ("silent", "hung")):
deck = FakeDeck(mode)
try:
self.assertEqual(sp.hyperdeck_probe("127.0.0.1", timeout=1, port=deck.port), expected, mode)
finally:
deck.close()
deck = FakeDeck("preview")
port = deck.port
deck.close()
self.assertEqual(sp.hyperdeck_probe("127.0.0.1", timeout=1, port=port), "unreachable")
def test_only_a_silent_deck_is_rebooted(self):
deck = FakeDeck("silent")
try:
chk = {"conn": "HyperDeck_1", "ip": "127.0.0.1", "port": deck.port}
self.assertTrue(sp.execute_fix(CONFIG, "hyperdeck", chk))
time.sleep(0.3)
self.assertIn(b"reboot", deck.received)
finally:
deck.close()
def test_a_recording_deck_is_never_rebooted(self):
deck = FakeDeck("record")
try:
cfg = json.loads(json.dumps(CONFIG))
cfg["companion"]["port"] = 1 # no Companion here: the reconnect just fails
chk = {"conn": "HyperDeck_1", "ip": "127.0.0.1", "port": deck.port}
sp.execute_fix(cfg, "hyperdeck", chk)
time.sleep(0.3)
self.assertNotIn(b"reboot", deck.received)
finally:
deck.close()
class FakeCompanion(BaseHTTPRequestHandler):