Hold-to-fix for the Lighting PC and the LiveStream (OBS) PC

- OBS: start OBS on the LiveStream PC if it is not running, then reconnect
  Companion. If the PC does not answer, send Wake-on-LAN and show WAKING PC.
- ARENA: start Resolume Arena on the Lighting PC if it is not running, then
  reconnect Companion. VISTA SILENT: start Vista if it is not running.
  A running OBS, Arena or Vista is never closed.
- MIDI (LIGHT PC): restart the rtpMIDI service; it reconnects to its saved
  peers (NA Ingest, NA ProP, NA Tracks).
- Fixes may set "reconnect" (restart the check's Companion connection after
  the command succeeds); hosts may set "mac"/"wake_broadcast" for Wake-on-LAN.
- windows-setup.ps1 finds OBS on any drive and Vista, and leaves Wake-on-LAN
  alone when it is already on (changing it can reset the adapter).
- Tests stub the fix executors so a hold in a test never reaches a machine.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-08 15:29:54 -04:00
co-authored by Claude Opus 5.5
parent 7cc73c253d
commit d6f54685bc
5 changed files with 118 additions and 10 deletions
+33 -2
View File
@@ -877,10 +877,12 @@ class Engine:
def work(fn, args):
ok = fn(*args)
results.append(ok is not False)
results.append(ok)
if len(results) == len(started): # the last one to finish reports for all
done = time.time()
if all(results):
if WAKING in results:
self.note(tile_id, "WAKING PC\n" + summary, "fixing", done, 120)
elif all(r is not False for r in results):
self.note(tile_id, "CHECKING\n" + summary, "fixing", done, 20)
else:
self.note(tile_id, "FIX FAILED\n" + summary, "failed", done, 20)
@@ -1141,6 +1143,22 @@ def ssh_cmd(cfg, host):
"-o", "ControlPath=/tmp/status-panel-ssh-%C", "-o", "ControlPersist=120", cfg["hosts"][host]["ssh"]]
WAKING = "waking" # execute_fix result: the PC was off, a Wake-on-LAN packet went out
def wake_on_lan(mac, broadcast=None, port=9):
"""Send a Wake-on-LAN magic packet (to the PC's subnet broadcast as well, when given)."""
raw = bytes.fromhex(re.sub(r"[^0-9A-Fa-f]", "", mac))
if len(raw) != 6:
raise ValueError("not a MAC address: %r" % mac)
packet = b"\xff" * 6 + raw * 16
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
for addr in [a for a in (broadcast, "255.255.255.255") if a]:
s.sendto(packet, (addr, port))
return packet
def remote_script(cmds):
"""One shell line that runs every check and prints SP<n>=<exit code>."""
return "; ".join("(%s) >/dev/null 2>&1; echo SP%d=$?" % (cmd, i) for i, cmd in enumerate(cmds))
@@ -1300,8 +1318,21 @@ def execute_fix(cfg, fix_id, chk):
argv = ssh_cmd(cfg, fix["host"]) + [fix["run"]] if kind == "ssh" else ["/bin/zsh", "-c", fix["run"]]
r = subprocess.run(argv, capture_output=True, text=True, timeout=fix.get("timeout", 60))
detail = (r.stdout + r.stderr).strip().replace("\n", " ")[-200:]
host = cfg.get("hosts", {}).get(fix.get("host"), {}) if kind == "ssh" else {}
if r.returncode == 255 and host.get("mac"):
# ssh could not reach the PC: it is probably off or asleep. Wake it; hold again
# once it is up (the app starts only after someone is logged in).
wake_on_lan(host["mac"], host.get("wake_broadcast"))
log.info("FIX %s -> %s does not answer: Wake-on-LAN sent to %s", label, fix["host"], host["mac"])
return WAKING
log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode,
(": " + detail) if detail else "")
if r.returncode == 0 and fix.get("reconnect") and chk.get("conn"):
# The app is running now: make Companion reconnect to it instead of waiting for its retry.
comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
ids = {c["label"]: c["id"] for c in comp.connections()}
ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]])
log.info("FIX %s -> Companion connection %s restarted %s", label, chk["conn"], "ok" if ok else "FAILED")
return r.returncode == 0
if kind == "netgear_poe":
# Netgear AV switch on the Dante network: power-cycle one PoE port (netgear.py).