christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Restart page (23), force restarts, Tascam reconnect, ProdCom, Resi encoder

Browse Source 
- Page 23 "Restart": hold a key 2 s to restart DVS (4 Macs), ProPresenter,
  Ableton, Church Translate, ProdCom, ProdDash, captions, the bridge or the
  Tascam connection even when the status page says it is fine ("force",
  shared 60 s guard). RESTART PAGE key on page 19, STATUS PAGE back.
- Holds carry Companion's clock; ones queued while the watchdog was down
  are dropped instead of firing together later.
- TASCAM hold always reconnects and checks again after 20 s: HUNG - POWER
  CYCLE IT (pings, control port hung), NO NETWORK, or turns a disabled
  connection on. Tascam pinged at its Companion address.
- INGEST: ProdCom (:24480), ProdDash's page (:24500), the caption screen;
  DVS on Ingest restarts ProdCom afterwards. RESI: the encoder (10.1.10.202).
- Windows starts check the app really started (FIX FAILED otherwise); OBS's
  start task runs elevated (OBS is "Run as administrator"); setup script
  does that for any such app and gives sshd restart-on-failure.
- push_page.py checks what is on the target page, not its name (pages 22
  and 97 are named "PAGE" but hold buttons); --which restart.
- Companion custom variables are written in the request body (no URL limit).
Reviewed by a 3-lens review with per-finding verification; 7 confirmed
issues fixed. Tested live: OBS hold opens OBS; Restart page TASCAM key.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-08 18:08:29 -04:00
co-authored by Claude Opus 5.5
parent 17347a5d9a
commit a0c72183bd
7 changed files with 544 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files
status_panel.py
+131 -13
View File
@@ -111,9 +111,14 @@ class Companion:
return text if status == 200 else None
def set_custom(self, name, value):
path = "/api/custom-variable/%s/value?value=%s" % (
urllib.parse.quote(name, safe=""), urllib.parse.quote(value, safe=""))
status, text = self._request("POST", path)
path = "/api/custom-variable/%s/value" % urllib.parse.quote(name, safe="")
if value == "":
# Companion answers 400 to an empty body; an empty ?value= clears it.
status, text = self._request("POST", path + "?value=")
else:
# In the body, so sp_state (several KB) never runs into a URL length limit.
status, text = self._request("POST", path, body=value.encode("utf-8"),
headers={"Content-Type": "text/plain; charset=utf-8"})
return status == 200, status
def restart_connection(self, conn_id):
@@ -641,6 +646,8 @@ class Engine:
self.executor = execute_fix
self.poe_executor = execute_poe
self.notes = {} # tile id -> hold feedback {"text", "level", "until"}
# The Restart page: one key per thing, hold 2 s = restart it even when it looks fine.
self.restart_keys = {k["id"]: k for k in cfg.get("restart_page", {}).get("keys", [])}
self.deck_ips = {} # HyperDeck connection -> last address Companion reported
self.now = 0.0
@@ -867,6 +874,10 @@ class Engine:
def run_fixes(self, tile_id, snap, comp, now, cooldown=60):
plan = self.fix_plan(tile_id, snap)
always = self.tiles.get(tile_id, {}).get("hold_always")
if not plan and always:
# e.g. TASCAM: its hold reconnects even when it looks fine (its control port hangs).
return self.run_force(tile_id, snap, comp, now, cooldown, fix_id=always)
if not plan:
if self.failing(tile_id):
log.info("HOLD %s: failing, but there is no automatic fix for it", tile_id)
@@ -897,12 +908,12 @@ class Engine:
elif kind == "poe":
log.info("FIX %s: power-cycle %s's PoE port (found by MAC)", tile_id, what)
names.append("POWER CYCLE")
started.append((self.poe_executor, (self.cfg, what, dict(snap.dante_map.get(what, {})))))
started.append((self.poe_executor, (self.cfg, what, dict(snap.dante_map.get(what, {}))), what))
else:
fix = self.cfg["fixes"][what]
log.info("FIX %s: %s", tile_id, fix.get("label", what))
names.append(fix.get("short", what.upper()))
started.append((self.executor, (self.cfg, what, chk)))
started.append((self.executor, (self.cfg, what, chk), fix.get("short", what.upper())))
if not names:
self.note(tile_id, "JUST TRIED\nWAIT 1 MIN" if waited else "NOTHING\nTO FIX", "idle", now, 4)
return plan
@@ -913,21 +924,63 @@ class Engine:
self.note(tile_id, "FIXING\n" + summary, "fixing", now, 180)
results = []
def work(fn, args):
group = "members" in self.tiles.get(tile_id, {}) or len(started) > 1
def work(fn, args, short):
ok = fn(*args)
results.append(ok)
if isinstance(ok, str) and ok != WAKING:
said.append(((short + " ") if group else "") + ok) # on a group button, say which
if len(results) == len(started): # the last one to finish reports for all
done = time.time()
if WAKING in results:
self.note(tile_id, "WAKING PC\n" + summary, "fixing", done, 120)
elif said: # the fix says what to do next
self.note(tile_id, said[0], "failed", done, 60)
elif all(r is not False for r in results):
self.note(tile_id, "CHECKING\n" + summary, "fixing", done, 20)
else:
self.note(tile_id, "FIX FAILED\n" + summary, "failed", done, 20)
for fn, args in started:
threading.Thread(target=work, args=(fn, args), daemon=True).start()
said = []
for fn, args, short in started:
threading.Thread(target=work, args=(fn, args, short), daemon=True).start()
return plan
def run_force(self, key_id, snap, comp, now, cooldown=60, fix_id=None):
"""Run one fix whatever the status says: a Restart page key, or a button's "hold_always".
Shares the 60 s guard with holds, so a hold and a restart key never double up."""
if fix_id is None:
key = self.restart_keys.get(key_id)
if not key:
log.warning("FORCE %s: no such restart key", key_id)
return None
fix_id = key["fix"]
ident = ("fix", fix_id, "")
if now - self.fix_last.get(ident, 0) < cooldown:
log.info("FORCE %s: %s ran less than %ds ago, skipped", key_id, fix_id, cooldown)
cur = self.notes.get(key_id)
if not (cur and cur["level"] == "fixing" and now < cur["until"]): # still running: keep saying so
self.note(key_id, "JUST DONE\nWAIT 1 MIN", "idle", now, 4)
return None
self.fix_last[ident] = now
fix = self.cfg["fixes"][fix_id]
log.info("FORCE %s: %s", key_id, fix.get("label", fix_id))
self.note(key_id, "RESTARTING", "fixing", now, 180)
def work():
r = self.executor(self.cfg, fix_id, {})
done = time.time()
if r == WAKING:
self.note(key_id, "WAKING PC", "fixing", done, 120)
elif isinstance(r, str):
self.note(key_id, r, "failed", done, 60)
elif r is False:
self.note(key_id, "FAILED", "failed", done, 30)
else:
self.note(key_id, "DONE", "ok", done, 20)
threading.Thread(target=work, daemon=True).start()
return fix_id
def display_level(res, acked, service_on):
if res.level == OFF:
@@ -996,6 +1049,12 @@ def colored(text, level):
def render(engine, service_on):
out = {}
for kid, key in engine.restart_keys.items():
note = engine.notes.get(kid)
if note and engine.now < note["until"]:
out[kid] = colored(key["label"] + "\n" + note["text"], note["level"])
else:
out[kid] = colored(key["label"] + "\nHOLD 2s", "idle")
for tile in engine.cfg["tiles"]:
tid, kind = tile["id"], tile.get("kind")
note = engine.notes.get(tid)
@@ -1449,13 +1508,16 @@ def execute_fix(cfg, fix_id, chk):
return WAKING
log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode,
(": " + detail) if detail else "")
if r.returncode == 0 and fix.get("reconnect") and chk.get("conn"):
conn = chk.get("conn") or fix.get("conn")
if r.returncode == 0 and fix.get("reconnect") and conn:
# The app is running now: make Companion reconnect to it instead of waiting for its retry.
comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
ids = {c["label"]: c["id"] for c in comp.connections()}
ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]])
log.info("FIX %s -> Companion connection %s restarted %s", label, chk["conn"], "ok" if ok else "FAILED")
ok = conn in ids and comp.restart_connection(ids[conn])
log.info("FIX %s -> Companion connection %s restarted %s", label, conn, "ok" if ok else "FAILED")
return r.returncode == 0
if kind == "reconnect":
return reconnect_and_verify(cfg, fix, label)
if kind == "midi_connect":
# Like Connect in Audio MIDI Setup: add the peer back to the session it dropped out of.
args = (chk["session"], chk["peer"], chk["address"])
@@ -1505,6 +1567,41 @@ def execute_fix(cfg, fix_id, chk):
return False
def reconnect_and_verify(cfg, fix, label, sleep=time.sleep):
"""Restart one Companion connection, then look again after "verify_seconds". Returns True if it
came back; otherwise what to tell the operator: "hung_note" when the device still answers ping
(its control side has hung - only a power-cycle helps), else that it is not on the network."""
comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
conn = {c["label"]: c for c in comp.connections()}.get(fix["conn"])
if conn is None:
log.info("FIX %s -> no Companion connection %s", label, fix["conn"])
return False
if not conn.get("enabled"):
# Switched off in Companion: turn it on (never call that a hung device).
ok = comp.enable_connection(conn["id"])
log.info("FIX %s -> %s was disabled in Companion: turned on %s", label, fix["conn"], "ok" if ok else "FAILED")
if not ok:
return "DISABLED IN\nCOMPANION"
elif not comp.restart_connection(conn["id"]):
log.info("FIX %s -> Companion refused to restart %s", label, fix["conn"])
return False
sleep(fix.get("verify_seconds", 20))
after = {c["label"]: c for c in comp.connections()}.get(fix["conn"]) or {}
status = (after.get("status") or {}).get("category")
if status == "good":
log.info("FIX %s -> %s reconnected", label, fix["conn"])
return True
if status not in ("warning", "error"): # still starting up, or unknown: no verdict yet
log.info("FIX %s -> %s is %s after %ss", label, fix["conn"], status, fix.get("verify_seconds", 20))
return "STILL\nCONNECTING"
host = companion_conn_hosts(companion_db_path(cfg)).get(fix["conn"])
if host and ping_once(host):
log.info("FIX %s -> %s still %s but %s answers ping: control port hung", label, fix["conn"], status, host)
return fix.get("hung_note", "STILL DOWN")
log.info("FIX %s -> %s still %s and %s does not answer ping", label, fix["conn"], status, host or "its address")
return "NO NETWORK\nCHECK POWER"
def start_artnet_listener(snap, stop, port=6454):
"""Note when each source last sent Art-Net. Shares the port (SO_REUSEPORT) so it never
keeps another program from listening too. Passive: nothing is ever sent."""
@@ -1580,6 +1677,13 @@ def load_config(path):
raise ValueError("fix %s: unknown host" % f)
if c["type"] == "remote" and c["host"] not in cfg.get("hosts", {}):
raise ValueError("tile %s: unknown host %s" % (t["id"], c["host"]))
if t.get("hold_always") and t["hold_always"] not in cfg.get("fixes", {}):
raise ValueError("tile %s: unknown hold_always fix %s" % (t["id"], t["hold_always"]))
for k in cfg.get("restart_page", {}).get("keys", []):
if k["id"] in ids:
raise ValueError("restart key %s has a tile's id" % k["id"])
if k["fix"] not in cfg.get("fixes", {}):
raise ValueError("restart key %s: unknown fix %s" % (k["id"], k["fix"]))
return cfg
@@ -1641,6 +1745,9 @@ def post_webhook(url, text):
log.warning("webhook failed: %s", e)
HOLD_MAX_AGE = 15 # seconds: a hold older than this (queued while the watchdog was down) is dropped
def handle_commands(text, engine, comp, snap, now):
for part in text.split(";"):
words = part.strip().split()
@@ -1655,8 +1762,19 @@ def handle_commands(text, engine, comp, snap, now):
target = {"on": True, "off": False}.get(arg, not engine.service.on(now))
engine.service.set(target, now)
log.info("SERVICE %s (button)", "ON" if target else "OFF")
elif verb == "restart": # sent by a 2 second hold
engine.run_fixes(arg, snap, comp, now)
elif verb in ("restart", "force"):
# Holds carry Companion's clock ("force rs_x 1791496225"). One pressed while the watchdog
# was down must not fire minutes later in a burst with the others.
stamp = words[2] if len(words) > 2 else None
if stamp is not None and (not stamp.isdigit() or now - int(stamp) > HOLD_MAX_AGE):
log.info("%s %s: pressed %s, too long ago - ignored", verb.upper(), arg,
"at %s" % stamp if stamp.isdigit() else "at an unknown time")
elif stamp is None and verb == "force":
log.info("FORCE %s: no time on it - ignored", arg)
elif verb == "restart": # a 2 second hold on a status button
engine.run_fixes(arg, snap, comp, now)
else: # a Restart page key, held 2 seconds
engine.run_force(arg, snap, comp, now)
else:
log.warning("unknown command %r", part)