christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

HyperDeck hold switches its Companion connection off and on

Browse Source 
After a deck restarts, Companion's HyperDeck connection does not come back
by itself and a plain restart of the connection does not help; switching
it off and on does. Each HD key's hold now does exactly that (even when it
looks green), waits 15 s and checks: DONE, STILL OFFLINE (deck pings) or
NO NETWORK. Decks are no longer rebooted by the panel; a recording deck is
left alone. Tested live on HD2 (off/on, reconnected in 17 s).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-11 05:20:07 -04:00
co-authored by Claude Opus 5.5
parent 0b38e9247b
commit ae28bb3c6d
4 changed files with 91 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
status_panel.py
+38 -29
View File
@@ -127,6 +127,10 @@ class Companion:
status, _ = self._request("POST", "/api/connections/%s/restart" % urllib.parse.quote(conn_id, safe=""))
return status == 200
def disable_connection(self, conn_id):
status, _ = self._request("POST", "/api/connections/%s/disable" % urllib.parse.quote(conn_id, safe=""))
return status == 200
def enable_connection(self, conn_id):
status, _ = self._request("POST", "/api/connections/%s/enable" % urllib.parse.quote(conn_id, safe=""))
return status == 200
@@ -906,8 +910,15 @@ class Engine:
plan = self.fix_plan(tile_id, snap)
always = self.tiles.get(tile_id, {}).get("hold_always")
if not plan and always:
# e.g. TASCAM: its hold reconnects even when it looks fine (its control port hangs).
return self.run_force(tile_id, snap, comp, now, cooldown, fix_id=always)
# e.g. TASCAM, HD1-6: the hold reconnects even when the button looks fine.
chk = next((c for c in self.tiles[tile_id].get("checks", []) if c.get("conn")), {})
if self.cfg["fixes"][always].get("kind") == "hyperdeck":
if hyperdeck_recording(snap, chk["conn"]):
log.info("HOLD %s: %s is recording, left alone", tile_id, chk["conn"])
self.note(tile_id, "RECORDING\nLEFT ALONE", "idle", now, 4)
return []
chk = dict(chk, ip=chk.get("ip") or snap.var(chk["conn"], "ip") or self.deck_ips.get(chk["conn"]))
return self.run_force(tile_id, snap, comp, now, cooldown, fix_id=always, chk=chk)
if not plan:
if self.failing(tile_id):
log.info("HOLD %s: failing, but there is no automatic fix for it", tile_id)
@@ -976,7 +987,7 @@ class Engine:
threading.Thread(target=work, args=(fn, args, short), daemon=True).start()
return plan
def run_force(self, key_id, snap, comp, now, cooldown=60, fix_id=None):
def run_force(self, key_id, snap, comp, now, cooldown=60, fix_id=None, chk=None):
"""Run one fix whatever the status says: a Restart page key, or a button's "hold_always".
Shares the 60 s guard with holds, so a hold and a restart key never double up."""
if fix_id is None:
@@ -985,7 +996,8 @@ class Engine:
log.warning("FORCE %s: no such restart key", key_id)
return None
fix_id = key["fix"]
ident = ("fix", fix_id, "")
per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck" and chk and chk.get("conn")
ident = ("fix", fix_id, chk["conn"] if per_deck else "") # the same guard a failing hold uses
if now - self.fix_last.get(ident, 0) < cooldown:
log.info("FORCE %s: %s ran less than %ds ago, skipped", key_id, fix_id, cooldown)
cur = self.notes.get(key_id)
@@ -998,7 +1010,7 @@ class Engine:
self.note(key_id, "RESTARTING", "fixing", now, 180)
def work():
r = self.executor(self.cfg, fix_id, {})
r = self.executor(self.cfg, fix_id, chk or {})
done = time.time()
if r == WAKING:
self.note(key_id, "WAKING PC", "fixing", done, 120)
@@ -1578,40 +1590,37 @@ def execute_fix(cfg, fix_id, chk):
log.info("FIX %s -> switch %s port %s power-cycled", label, fix["switch"], fix["port"])
return True
if kind == "hyperdeck":
port = chk.get("port", 9993)
verdict = hyperdeck_probe(chk["ip"], port=port)
if verdict == "hung":
with socket.create_connection((chk["ip"], port), timeout=3) as s:
s.sendall(b"reboot\r\n")
log.info("FIX %s %s -> deck accepted a connection but did not answer: reboot sent",
label, chk["conn"])
return True
if verdict == "unreachable":
log.info("FIX %s %s -> deck does not answer on the network: nothing sent", label, chk["conn"])
return False
# The deck answers (and may be recording): leave it alone, reconnect Companion instead.
comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
ids = {c["label"]: c["id"] for c in comp.connections()}
ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]])
log.info("FIX %s %s -> deck is fine (%s): Companion connection restarted %s",
label, chk["conn"], verdict, "ok" if ok else "FAILED")
return bool(ok)
# Turn Companion's connection to the deck off and on: after a deck restarts, that is what
# brings it back green (a plain reconnect does not, 2026-10-11). The deck is never rebooted.
return reconnect_and_verify(cfg, fix, label, conn=chk["conn"], host=chk.get("ip"), toggle=True)
log.warning("FIX %s: unknown kind %r", label, kind)
except Exception as e:
log.warning("FIX %s failed: %s", label, e)
return False
def reconnect_and_verify(cfg, fix, label, sleep=time.sleep):
"""Restart one Companion connection, then look again after "verify_seconds". Returns True if it
came back; otherwise what to tell the operator: "hung_note" when the device still answers ping
(its control side has hung - only a power-cycle helps), else that it is not on the network."""
def reconnect_and_verify(cfg, fix, label, sleep=None, conn=None, host=None, toggle=False):
"""Restart one Companion connection (toggle=True: switch it off and on again), then look again
after "verify_seconds". Returns True if it came back; otherwise what to tell the operator:
"hung_note" when the device still answers ping, else that it is not on the network."""
sleep = sleep or time.sleep
label_conn = conn or fix["conn"]
fix = dict(fix, conn=label_conn)
comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
conn = {c["label"]: c for c in comp.connections()}.get(fix["conn"])
if conn is None:
log.info("FIX %s -> no Companion connection %s", label, fix["conn"])
return False
if not conn.get("enabled"):
if toggle:
if conn.get("enabled") and not comp.disable_connection(conn["id"]):
log.info("FIX %s -> Companion refused to switch %s off", label, fix["conn"])
return False
sleep(fix.get("off_seconds", 2))
if not comp.enable_connection(conn["id"]):
log.info("FIX %s -> Companion did not switch %s back on", label, fix["conn"])
return "STILL OFF IN\nCOMPANION"
log.info("FIX %s -> %s switched off and on", label, fix["conn"])
elif not conn.get("enabled"):
# Switched off in Companion: turn it on (never call that a hung device).
ok = comp.enable_connection(conn["id"])
log.info("FIX %s -> %s was disabled in Companion: turned on %s", label, fix["conn"], "ok" if ok else "FAILED")
@@ -1629,7 +1638,7 @@ def reconnect_and_verify(cfg, fix, label, sleep=time.sleep):
if status not in ("warning", "error"): # still starting up, or unknown: no verdict yet
log.info("FIX %s -> %s is %s after %ss", label, fix["conn"], status, fix.get("verify_seconds", 20))
return "STILL\nCONNECTING"
host = companion_conn_hosts(companion_db_path(cfg)).get(fix["conn"])
host = host or companion_conn_hosts(companion_db_path(cfg)).get(fix["conn"])
if host and ping_once(host):
log.info("FIX %s -> %s still %s but %s answers ping: control port hung", label, fix["conn"], status, host)
return fix.get("hung_note", "STILL DOWN")