christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Hold a failing button to fix what it is failing on.

Browse Source 
Fixes come from config.json only and run over SSH with NA Ingest's
status-panel key (ProPresenter restart, Ableton / Church Translate start,
reopen DVS), locally (Resi bridge, captions, ProdDash) or as a HyperDeck
reboot. A green button does nothing, each fix waits 60 s before it can run
again, and a recording deck is never rebooted. New remote checks ask the Macs
every 10 s whether their apps are running. Group buttons and INGEST now have a
hold too; tests never run a real fix or open an SSH connection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-08 11:34:44 -04:00
co-authored by Claude Opus 5.5
parent e873d1aa40
commit 51f09900ab
6 changed files with 310 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
status_panel.py
+150 -7
View File
@@ -22,6 +22,7 @@ import logging.handlers
import os
import re
import shutil
import socket
import ssl
import subprocess
import sys
@@ -137,6 +138,7 @@ class Snapshot:
self.http = {} # url -> reachable
self.disk = {} # path -> free GB
self.listeners = {} # MIDI source name -> coremidi.Listener
self.remote = {} # (host, command) -> True / False / None (could not ask)
def var(self, label, name):
return self.vars.get((label, name))
@@ -424,11 +426,22 @@ def check_midi_rx(chk, snap, ctx):
return [("rx", Result(OK, info="%s %s" % (name, fmt_age(age))))]
def remote_key(chk):
return "%s\0%s" % (chk["host"], chk["cmd"])
def check_remote(chk, snap, ctx):
"""A command run over SSH on another machine, e.g. "is ProPresenter running?"."""
if snap.remote.get(remote_key(chk)) is False:
return [("remote", Result(ERR, chk["msg"], named=True))]
return [("remote", Result(OK))]
CHECKS = {
"conn": check_conn, "hyperdeck": check_hyperdeck, "dante": check_dante, "shure": check_shure,
"psm": check_psm, "tascam": check_tascam, "ping": check_ping, "launchd": check_launchd,
"process": check_process, "http": check_http, "disk": check_disk, "midi_peer": check_midi_peer,
"mdns": check_mdns, "midi_rx": check_midi_rx,
"mdns": check_mdns, "midi_rx": check_midi_rx, "remote": check_remote,
}
# Checks whose connection a long press may restart (Dante is shared, so not that one).
@@ -574,6 +587,9 @@ class Engine:
self.conn_drops = deque() # (time, conn label, group)
self.burst = None # {"start", "conns", "cleared"}
self.events = []
self.meta = {} # key -> (tile id, check, sub key)
self.fix_last = {} # fix identity -> last run time
self.executor = execute_fix
# raw results for every check key
def evaluate(self, snap, now, service_on):
@@ -600,6 +616,7 @@ class Engine:
if chk.get("tag") and res.level in (WARN, ERR) and not res.named:
res.msg, res.named = "%s %s" % (chk["tag"], res.msg), True
raw["%s:%d:%s" % (t["id"], i, sub)] = (t["id"], res, chk)
self.meta["%s:%d:%s" % (t["id"], i, sub)] = (t["id"], chk, sub)
for sub, res in self.network_results(snap, now):
raw["net:%s" % sub] = ("net", res, {"type": "network"})
return raw
@@ -698,6 +715,59 @@ class Engine:
tile = self.tiles.get(tile_id, {})
return [c["conn"] for c in tile.get("checks", []) if c["type"] in RESTARTABLE and c.get("conn")]
def fix_plan(self, tile_id, snap):
"""What a hold on this button would do: fix only what is failing right now.
[(identity, "fix", fix id, check) or (identity, "restart", connection label, check)]
A check's own "fix" wins; otherwise a failing Companion connection is restarted.
Nothing is done for a healthy button, and a recording HyperDeck is never touched.
"""
tile = self.tiles.get(tile_id, {})
ids = tile.get("members") or [tile_id]
plan, seen = [], set()
for key in sorted(self.meta):
tid, chk, sub = self.meta[key]
tr = self.trackers.get(key)
if tid not in ids or not tr or not tr.confirmed or tr.result.level not in (WARN, ERR):
continue
fix_id = chk.get("fix")
if isinstance(fix_id, dict):
fix_id = fix_id.get(sub)
if fix_id:
if chk["type"] == "hyperdeck" and hyperdeck_recording(snap, chk["conn"]):
continue
# One run per fix, except a HyperDeck reboot, which is per deck.
per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck_reboot"
ident = ("fix", fix_id, chk.get("ip", "") if per_deck else "")
if ident not in seen:
seen.add(ident)
plan.append((ident, "fix", fix_id, chk))
elif chk["type"] in RESTARTABLE and sub == "conn" and chk.get("conn"):
ident = ("restart", chk["conn"])
if ident not in seen:
seen.add(ident)
plan.append((ident, "restart", chk["conn"], chk))
return plan
def run_fixes(self, tile_id, snap, comp, now, cooldown=60):
plan = self.fix_plan(tile_id, snap)
if not plan:
log.info("HOLD %s: nothing failing, nothing to fix", tile_id)
for ident, kind, what, chk in plan:
if now - self.fix_last.get(ident, 0) < cooldown:
log.info("HOLD %s: %s %s ran less than %ds ago, skipped", tile_id, kind, what, cooldown)
continue
self.fix_last[ident] = now
if kind == "restart":
c = snap.conns.get(what)
if c and c.get("enabled"):
ok = comp.restart_connection(c["id"])
log.info("RESTART %s -> %s", what, "ok" if ok else "failed")
else:
log.info("FIX %s: %s", tile_id, self.cfg["fixes"][what].get("label", what))
threading.Thread(target=self.executor, args=(self.cfg, what, chk), daemon=True).start()
return plan
def display_level(res, acked, service_on):
if res.level == OFF:
@@ -937,6 +1007,72 @@ def poll_midi(snap, cfg):
lis.poll()
# --- Other machines: SSH checks and fixes ------------------------------------
def ssh_cmd(cfg, host):
"""ssh to a machine in config "hosts", with NA Ingest's status-panel key; one shared connection per host."""
return ["/usr/bin/ssh", "-i", os.path.expanduser(cfg.get("ssh_key", "~/.ssh/id_status_panel")),
"-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "-o", "IdentitiesOnly=yes",
"-o", "StrictHostKeyChecking=accept-new", "-o", "ControlMaster=auto",
"-o", "ControlPath=/tmp/status-panel-ssh-%C", "-o", "ControlPersist=120", cfg["hosts"][host]["ssh"]]
def remote_script(cmds):
"""One shell line that runs every check and prints SP<n>=<exit code>."""
return "; ".join("(%s) >/dev/null 2>&1; echo SP%d=$?" % (cmd, i) for i, cmd in enumerate(cmds))
def parse_remote(out, n):
codes = dict(re.findall(r"SP(\d+)=(\d+)", out or ""))
return [None if str(i) not in codes else codes[str(i)] == "0" for i in range(n)]
def poll_remote(snap, cfg):
by_host = {}
for t in cfg["tiles"]:
for c in t.get("checks", []):
if c["type"] == "remote":
by_host.setdefault(c["host"], [])
if c["cmd"] not in by_host[c["host"]]:
by_host[c["host"]].append(c["cmd"])
result = dict(snap.remote)
for host, cmds in by_host.items():
try:
out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds)], capture_output=True,
text=True, timeout=20).stdout
except subprocess.TimeoutExpired:
out = ""
for cmd, ok in zip(cmds, parse_remote(out, len(cmds))):
result["%s\0%s" % (host, cmd)] = ok # None = could not reach it: no verdict
snap.remote = result
def execute_fix(cfg, fix_id, chk):
"""Run one entry from config "fixes" (in its own thread). Only listed commands ever run."""
fix = cfg["fixes"][fix_id]
kind, label = fix.get("kind", "ssh"), fix.get("label", fix_id)
try:
if kind in ("ssh", "local"):
argv = ssh_cmd(cfg, fix["host"]) + [fix["run"]] if kind == "ssh" else ["/bin/zsh", "-c", fix["run"]]
r = subprocess.run(argv, capture_output=True, text=True, timeout=fix.get("timeout", 60))
detail = (r.stdout + r.stderr).strip().replace("\n", " ")[-200:]
log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode,
(": " + detail) if detail else "")
elif kind == "hyperdeck_reboot":
with socket.create_connection((chk["ip"], 9993), timeout=3) as s:
s.settimeout(3)
try:
s.recv(1024) # the deck's greeting
except socket.timeout:
pass
s.sendall(b"reboot\n")
log.info("FIX %s %s -> reboot sent", label, chk["conn"])
else:
log.warning("FIX %s: unknown kind %r", label, kind)
except Exception as e:
log.warning("FIX %s failed: %s", label, e)
class Periodic(threading.Thread):
def __init__(self, name, interval, fn, stop):
super().__init__(name=name, daemon=True)
@@ -975,6 +1111,15 @@ def load_config(path):
for m in t.get("members", []):
if m not in ids:
raise ValueError("tile %s: unknown member %s" % (t["id"], m))
for c in t.get("checks", []):
fixes = c.get("fix") if isinstance(c.get("fix"), dict) else {"": c.get("fix")}
for f in fixes.values():
if f and f not in cfg.get("fixes", {}):
raise ValueError("tile %s: unknown fix %s" % (t["id"], f))
if f and cfg["fixes"][f].get("kind", "ssh") == "ssh" and cfg["fixes"][f]["host"] not in cfg.get("hosts", {}):
raise ValueError("fix %s: unknown host" % f)
if c["type"] == "remote" and c["host"] not in cfg.get("hosts", {}):
raise ValueError("tile %s: unknown host %s" % (t["id"], c["host"]))
return cfg
@@ -1011,6 +1156,7 @@ def start_pollers(cfg, snap, stop):
Periodic("system", 10.0, lambda: poll_system(snap, cfg), stop),
Periodic("bonjour", 15.0, lambda: poll_mdns(snap), stop),
Periodic("http", 5.0, lambda: poll_http(snap, http_checks(cfg)), stop),
Periodic("remote", 10.0, lambda: poll_remote(snap, cfg), stop),
]
for t in threads:
t.start()
@@ -1042,12 +1188,8 @@ def handle_commands(text, engine, comp, snap, now):
target = {"on": True, "off": False}.get(arg, not engine.service.on(now))
engine.service.set(target, now)
log.info("SERVICE %s (button)", "ON" if target else "OFF")
elif verb == "restart":
for label in engine.restart_targets(arg):
c = snap.conns.get(label)
if c and c.get("enabled"):
ok = comp.restart_connection(c["id"])
log.info("RESTART %s -> %s", label, "ok" if ok else "failed")
elif verb == "restart": # sent by a 2 second hold
engine.run_fixes(arg, snap, comp, now)
else:
log.warning("unknown command %r", part)
@@ -1123,6 +1265,7 @@ def run_once(cfg):
poll_system(snap, cfg)
poll_mdns(snap)
poll_http(snap, http_checks(cfg))
poll_remote(snap, cfg)
try:
poll_midi(snap, cfg)
except Exception as e: