From d6f54685bcc3b3cb147e7c116f0b6f5e0630364e Mon Sep 17 00:00:00 2001 From: christian Date: Thu, 8 Oct 2026 15:29:54 -0400 Subject: [PATCH] Hold-to-fix for the Lighting PC and the LiveStream (OBS) PC - OBS: start OBS on the LiveStream PC if it is not running, then reconnect Companion. If the PC does not answer, send Wake-on-LAN and show WAKING PC. - ARENA: start Resolume Arena on the Lighting PC if it is not running, then reconnect Companion. VISTA SILENT: start Vista if it is not running. A running OBS, Arena or Vista is never closed. - MIDI (LIGHT PC): restart the rtpMIDI service; it reconnects to its saved peers (NA Ingest, NA ProP, NA Tracks). - Fixes may set "reconnect" (restart the check's Companion connection after the command succeeds); hosts may set "mac"/"wake_broadcast" for Wake-on-LAN. - windows-setup.ps1 finds OBS on any drive and Vista, and leaves Wake-on-LAN alone when it is already on (changing it can reset the adapter). - Tests stub the fix executors so a hold in a test never reaches a machine. Co-Authored-By: Claude Opus 5.5 --- README.md | 4 +++ config.json | 22 +++++++++---- remote-access/windows-setup.ps1 | 9 ++++- status_panel.py | 35 ++++++++++++++++++-- tests/test_status_panel.py | 58 ++++++++++++++++++++++++++++++++- 5 files changed, 118 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 43d6ac4..fd8b764 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,10 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl | APPS: PROPRES OFFLINE / NOT RUNNING | restarts ProPresenter on its Mac (the newest version installed) | | APPS: ABLETON / TRANSLATE NOT RUNNING | starts Ableton on the Tracks Mac / Church Translate on NA Translation | | APPS: BIBLE APP DOWN | restarts the Bible app on the Sanctuary TV PC (SSH + its "Start BibleApp" task from `remote-access/windows-setup.ps1`) | + | APPS: VISTA SILENT | starts Vista on the Lighting PC if it is not running. A running Vista is never closed. | + | ARENA | starts Resolume Arena on the Lighting PC if it is not running, then reconnects Companion | + | OBS | starts OBS on the LiveStream PC if it is not running, then reconnects Companion. If the PC is off, it is woken (Wake-on-LAN) and the button shows WAKING PC; hold again once someone has logged in. | + | MIDI: LIGHT PC | restarts rtpMIDI on the Lighting PC; it reconnects to NA Ingest, NA ProP and NA Tracks by itself. Lighting MIDI drops for a few seconds. | | DANTE: PROP MAC / TRACKS / TRANSL / INGEST GONE | starts Dante Virtual Soundcard's engine on that Mac (`remote-access/mac-dvs-helper.sh`) | | DANTE: any device powered by its switch port (today the four Studio B boxes) | power-cycles the port it is on **now**, found by its MAC on the Netgear Dante switches | | DANTE: any other device GONE | restarts Companion's Dante module (hardware: if still gone, it is off or unplugged) | diff --git a/config.json b/config.json index 13f4ecd..b44ce16 100644 --- a/config.json +++ b/config.json @@ -32,7 +32,9 @@ "prop": { "ssh": "napropresenter@10.1.14.225" }, "tracks": { "ssh": "waterschurchmedia@10.1.3.227" }, "translation": { "ssh": "studio@10.1.7.39" }, - "bibletv": { "ssh": "media@10.6.7.13" } + "bibletv": { "ssh": "media@10.6.7.13" }, + "lighting": { "ssh": "media@10.1.10.206" }, + "livestream": { "ssh": "media@10.1.9.126", "mac": "6c:24:08:d7:fb:57", "wake_broadcast": "10.1.255.255" } }, "fixes": { @@ -52,6 +54,14 @@ "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" }, "bibleapp": { "host": "bibletv", "short": "BIBLE APP", "label": "restart the Bible app on the Sanctuary TV PC (Windows: cmd.exe)", "run": "taskkill /F /T /IM \"Bible App.exe\" >nul 2>&1 & ping -n 3 127.0.0.1 >nul & schtasks /Run /TN \"\\StatusPanel\\Start BibleApp\"" }, + "obs": { "host": "livestream", "short": "OBS", "label": "start OBS on the LiveStream PC if it is not running (wakes the PC if it is off), then reconnect Companion", "reconnect": true, + "run": "tasklist /NH /FI \"IMAGENAME eq obs64.exe\" | find /I \"obs64.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start OBS\"" }, + "arena": { "host": "lighting", "short": "ARENA", "label": "start Resolume Arena on the Lighting PC if it is not running, then reconnect Companion", "reconnect": true, + "run": "tasklist /NH /FI \"IMAGENAME eq Arena.exe\" | find /I \"Arena.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Arena\"" }, + "vista": { "host": "lighting", "short": "VISTA", "label": "start Vista on the Lighting PC if it is not running (a running Vista is never closed)", + "run": "tasklist /NH /FI \"IMAGENAME eq Vista.exe\" | find /I \"Vista.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Vista\"" }, + "rtpmidi": { "host": "lighting", "short": "RTPMIDI", "label": "restart rtpMIDI on the Lighting PC: it reconnects to NA Ingest, NA ProP and NA Tracks (MIDI drops for a few seconds)", + "run": "powershell -NoProfile -NonInteractive -Command \"Restart-Service rtpMIDIService -Force\"" }, "proddash": { "kind": "local", "short": "PRODDASH", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" }, "hyperdeck": { "kind": "hyperdeck", "short": "HYPERDECK", "label": "reconnect Companion to the HyperDeck, or reboot the deck only if it is hung" } }, @@ -88,9 +98,9 @@ "NA-Tracks": "TRACKS", "FOH-SuperRack-LiveBox": "SUPERRACK", "NA-ProPresenter-Mac": "PROP MAC" } } ] }, { "id": "midi", "label": "MIDI", "pos": [2, 3], "checks": [ - { "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC" }, + { "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC", "fix": "rtpmidi" }, { "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox", "tag": "SUPERRACK" }, - { "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI" }, + { "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI", "fix": "rtpmidi" }, { "type": "mdns", "instance": "NA Tracks", "name": "TRACKS MIDI" }, { "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" } ] }, @@ -101,14 +111,14 @@ { "type": "remote", "host": "tracks", "cmd": "pgrep -xq Live", "msg": "ABLETON NOT RUNNING", "fix": "ableton" }, { "type": "remote", "host": "translation", "cmd": "pgrep -xq 'Church Translate'", "msg": "TRANSLATE NOT RUNNING", "fix": "translate" }, { "type": "ping", "host": "10.1.3.227", "name": "TRACKS MAC" }, - { "type": "artnet", "source": "10.1.10.206", "max_silence_seconds": 5, "msg": "VISTA SILENT" }, + { "type": "artnet", "source": "10.1.10.206", "max_silence_seconds": 5, "msg": "VISTA SILENT", "fix": "vista" }, { "type": "http", "url": "http://10.6.7.13:8080/", "msg": "BIBLE APP DOWN", "interval_seconds": 5, "fix": "bibleapp" }, { "type": "conn", "conn": "BibleTV", "tag": "BIBLE OSC" } ] }, { "id": "service", "label": "ALARMS", "pos": [3, 0], "kind": "service" }, { "id": "atem", "label": "ATEM", "pos": [3, 1], "checks": [{ "type": "conn", "conn": "ATEM" }] }, - { "id": "obs", "label": "OBS", "pos": [3, 2], "checks": [{ "type": "conn", "conn": "OBS-Youtube", "service_only": true }] }, + { "id": "obs", "label": "OBS", "pos": [3, 2], "checks": [{ "type": "conn", "conn": "OBS-Youtube", "service_only": true, "fix": "obs" }] }, { "id": "resi", "label": "RESI", "pos": [3, 3], "checks": [ { "type": "conn", "conn": "RESI_Decoder", "tag": "DECODER" }, { "type": "ping", "host": "10.1.10.203", "name": "DECODER" }, @@ -128,7 +138,7 @@ { "type": "ping", "host": "10.1.14.116", "name": "VX1000" }, { "type": "ping", "host": "10.1.14.115", "name": "UHDJR" } ] }, - { "id": "arena", "label": "ARENA", "pos": [3, 7], "checks": [{ "type": "conn", "conn": "arena" }] }, + { "id": "arena", "label": "ARENA", "pos": [3, 7], "checks": [{ "type": "conn", "conn": "arena", "fix": "arena" }] }, { "id": "axient1", "label": "AXIENT 1", "checks": [{ "type": "shure", "conn": "Axient_1", "channels": 4 }] }, { "id": "axient2", "label": "AXIENT 2", "checks": [{ "type": "shure", "conn": "Axient_2", "channels": 4 }] }, diff --git a/remote-access/windows-setup.ps1 b/remote-access/windows-setup.ps1 index 2ba52aa..f0593b0 100644 --- a/remote-access/windows-setup.ps1 +++ b/remote-access/windows-setup.ps1 @@ -68,8 +68,13 @@ $roots = @($env:ProgramFiles, ${env:ProgramFiles(x86)}, (Join-Path $env:LOCALAPP $bible = Get-ChildItem -Path $roots -Filter '*.exe' -Recurse -Depth 3 -ErrorAction SilentlyContinue | Where-Object { $_.FullName -match 'bible' -and $_.Name -notmatch 'uninst|update|crash|helper' } | Select-Object -First 1 -ExpandProperty FullName +# Apps can live on any local drive (the LiveStream PC has OBS on D:, the Lighting PC has Vista on G:). +$drives = Get-PSDrive -PSProvider FileSystem | Where-Object { $_.Free -ne $null } | ForEach-Object { $_.Root.TrimEnd('\') } +$vista = Find-First (@($drives | ForEach-Object { "$_\Vista3\Vista.exe" }) + + @(Get-ChildItem -Path $roots -Filter 'Vista.exe' -Recurse -Depth 3 -ErrorAction SilentlyContinue | ForEach-Object FullName)) $apps = @( - @{ Name = 'OBS'; Exe = (Find-First @("$env:ProgramFiles\obs-studio\bin\64bit\obs64.exe")); Args = '--disable-shutdown-check' }, + @{ Name = 'OBS'; Exe = (Find-First @($drives | ForEach-Object { "$_\Program Files\obs-studio\bin\64bit\obs64.exe" })); Args = '--disable-shutdown-check' }, + @{ Name = 'Vista'; Exe = $vista; Args = '' }, @{ Name = 'Arena'; Exe = (Find-First (Get-ChildItem "$env:ProgramFiles\Resolume Arena*\Arena.exe" -ErrorAction SilentlyContinue | ForEach-Object FullName)); Args = '' }, @{ Name = 'rtpMIDI'; Exe = (Find-First @("${env:ProgramFiles(x86)}\Tobias Erichsen\rtpMIDI\rtpMIDI.exe", "$env:ProgramFiles\Tobias Erichsen\rtpMIDI\rtpMIDI.exe")); Args = '' }, @{ Name = 'BibleApp'; Exe = $bible; Args = ''; AtLogon = $true } # also opens it 15 s after login @@ -96,6 +101,8 @@ foreach ($app in $apps) { # 4. Wake-on-LAN: let the wired adapter wake the PC (the BIOS setting must be on too) $nics = Get-NetAdapter -Physical -ErrorAction SilentlyContinue | Where-Object Status -eq 'Up' foreach ($nic in $nics) { + # Only when it is off: changing it can briefly reset the adapter (and the PC's network). + if ((Get-NetAdapterPowerManagement -Name $nic.Name -ErrorAction SilentlyContinue).WakeOnMagicPacket -eq 'Enabled') { continue } try { Set-NetAdapterPowerManagement -Name $nic.Name -WakeOnMagicPacket Enabled -ErrorAction Stop } catch { $notes += "Wake-on-LAN not settable on $($nic.Name)" } } diff --git a/status_panel.py b/status_panel.py index 0ed5a0e..19c49f4 100644 --- a/status_panel.py +++ b/status_panel.py @@ -877,10 +877,12 @@ class Engine: def work(fn, args): ok = fn(*args) - results.append(ok is not False) + results.append(ok) if len(results) == len(started): # the last one to finish reports for all done = time.time() - if all(results): + if WAKING in results: + self.note(tile_id, "WAKING PC\n" + summary, "fixing", done, 120) + elif all(r is not False for r in results): self.note(tile_id, "CHECKING\n" + summary, "fixing", done, 20) else: self.note(tile_id, "FIX FAILED\n" + summary, "failed", done, 20) @@ -1141,6 +1143,22 @@ def ssh_cmd(cfg, host): "-o", "ControlPath=/tmp/status-panel-ssh-%C", "-o", "ControlPersist=120", cfg["hosts"][host]["ssh"]] +WAKING = "waking" # execute_fix result: the PC was off, a Wake-on-LAN packet went out + + +def wake_on_lan(mac, broadcast=None, port=9): + """Send a Wake-on-LAN magic packet (to the PC's subnet broadcast as well, when given).""" + raw = bytes.fromhex(re.sub(r"[^0-9A-Fa-f]", "", mac)) + if len(raw) != 6: + raise ValueError("not a MAC address: %r" % mac) + packet = b"\xff" * 6 + raw * 16 + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) + for addr in [a for a in (broadcast, "255.255.255.255") if a]: + s.sendto(packet, (addr, port)) + return packet + + def remote_script(cmds): """One shell line that runs every check and prints SP=.""" return "; ".join("(%s) >/dev/null 2>&1; echo SP%d=$?" % (cmd, i) for i, cmd in enumerate(cmds)) @@ -1300,8 +1318,21 @@ def execute_fix(cfg, fix_id, chk): argv = ssh_cmd(cfg, fix["host"]) + [fix["run"]] if kind == "ssh" else ["/bin/zsh", "-c", fix["run"]] r = subprocess.run(argv, capture_output=True, text=True, timeout=fix.get("timeout", 60)) detail = (r.stdout + r.stderr).strip().replace("\n", " ")[-200:] + host = cfg.get("hosts", {}).get(fix.get("host"), {}) if kind == "ssh" else {} + if r.returncode == 255 and host.get("mac"): + # ssh could not reach the PC: it is probably off or asleep. Wake it; hold again + # once it is up (the app starts only after someone is logged in). + wake_on_lan(host["mac"], host.get("wake_broadcast")) + log.info("FIX %s -> %s does not answer: Wake-on-LAN sent to %s", label, fix["host"], host["mac"]) + return WAKING log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode, (": " + detail) if detail else "") + if r.returncode == 0 and fix.get("reconnect") and chk.get("conn"): + # The app is running now: make Companion reconnect to it instead of waiting for its retry. + comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000)) + ids = {c["label"]: c["id"] for c in comp.connections()} + ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]]) + log.info("FIX %s -> Companion connection %s restarted %s", label, chk["conn"], "ok" if ok else "FAILED") return r.returncode == 0 if kind == "netgear_poe": # Netgear AV switch on the Dante network: power-cycle one PoE port (netgear.py). diff --git a/tests/test_status_panel.py b/tests/test_status_panel.py index 740c1fc..0f1ea55 100644 --- a/tests/test_status_panel.py +++ b/tests/test_status_panel.py @@ -465,6 +465,8 @@ class TestFixes(unittest.TestCase): def setUp(self): self.snap = healthy_snapshot(SUNDAY) self.engine = sp.Engine(CONFIG) + # Safety net: a hold in these tests must never reach a real machine. + self.engine.executor = self.engine.poe_executor = lambda *a: True def plan(self, tile): run_steps(self.engine, self.snap, SUNDAY, 4) @@ -492,6 +494,59 @@ class TestFixes(unittest.TestCase): self.assertEqual(CONFIG["hosts"][fix["host"]]["ssh"], "media@10.6.7.13") self.assertIn('schtasks /Run /TN "\\StatusPanel\\Start BibleApp"', fix["run"]) # cmd.exe, not sh + def test_lighting_and_livestream_pcs(self): + self.snap.conns["OBS-Youtube"]["category"] = "error" + self.snap.conns["arena"]["category"] = "error" + self.snap.artnet_since, self.snap.artnet_last = SUNDAY - 100, {"10.1.10.206": SUNDAY - 100} + self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"] + if p["name"] != "Lighting PC"] + self.snap.mdns = self.snap.mdns - {"Lighting PC"} + self.assertEqual(self.plan("obs"), [("fix", "obs")]) + self.assertEqual(self.plan("arena"), [("fix", "arena")]) + self.assertEqual(self.plan("apps"), [("fix", "vista")]) + self.assertEqual(self.plan("midi"), [("fix", "rtpmidi")]) # two checks, one restart + for fix_id in ("obs", "arena", "vista"): # never closes a running app + self.assertTrue(CONFIG["fixes"][fix_id]["run"].startswith("tasklist "), fix_id) + + def test_an_off_pc_is_woken(self): + from unittest import mock + woke = [] + done = mock.Mock(returncode=255, stdout="", stderr="ssh: connect to host 10.1.9.126 port 22: timed out") + with mock.patch.object(sp.subprocess, "run", return_value=done), \ + mock.patch.object(sp, "wake_on_lan", side_effect=lambda mac, bc=None: woke.append((mac, bc))): + self.assertEqual(sp.execute_fix(CONFIG, "obs", {"conn": "OBS-Youtube"}), sp.WAKING) + self.assertEqual(woke, [("6c:24:08:d7:fb:57", "10.1.255.255")]) + self.assertFalse(sp.execute_fix(CONFIG, "arena", {"conn": "arena"})) # no MAC: just fails + self.assertEqual(len(woke), 1) + # the button says so + self.snap.conns["OBS-Youtube"]["category"] = "error" + self.engine.executor = lambda cfg, fix_id, chk: sp.WAKING + _, t = run_steps(self.engine, self.snap, SUNDAY, 4) + self.engine.run_fixes("obs", self.snap, None, t) + time.sleep(0.2) + self.assertEqual(self.engine.step(self.snap, time.time())["obs"]["x"], "OBS\nWAKING PC\nOBS") + + def test_magic_packet(self): + from unittest import mock + with mock.patch.object(sp.socket, "socket") as sock: + packet = sp.wake_on_lan("6c:24:08:d7:fb:57", "10.1.255.255") + self.assertEqual(packet, b"\xff" * 6 + bytes.fromhex("6c2408d7fb57") * 16) + sent = [c.args[1] for c in sock.return_value.__enter__.return_value.sendto.call_args_list] + self.assertEqual(sent, [("10.1.255.255", 9), ("255.255.255.255", 9)]) + + def test_companion_reconnects_after_the_app_starts(self): + from unittest import mock + comp = mock.Mock() + comp.connections.return_value = [{"label": "arena", "id": "abc"}] + ok = mock.Mock(returncode=0, stdout="", stderr="") + with mock.patch.object(sp.subprocess, "run", return_value=ok), \ + mock.patch.object(sp, "Companion", return_value=comp): + self.assertTrue(sp.execute_fix(CONFIG, "arena", {"conn": "arena"})) + comp.restart_connection.assert_called_once_with("abc") + comp.reset_mock() + self.assertTrue(sp.execute_fix(CONFIG, "vista", {"type": "artnet"})) # no "reconnect" + comp.restart_connection.assert_not_called() + def test_unreachable_mac_is_not_a_verdict(self): self.snap.remote["tracks\0pgrep -xq Live"] = None self.assertEqual(self.plan("apps"), []) @@ -632,7 +687,8 @@ class TestFixes(unittest.TestCase): self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices) self.assertEqual(self.plan("dante"), [("fix", "dvs_ingest")]) # A failure with no fix must not claim there is nothing wrong. - self.snap.midi["NA Ingest"]["peers"] = [] + self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"] + if p["name"] != "SuperRack-LiveBox"] _, t = run_steps(self.engine, self.snap, SUNDAY, 4) self.engine.run_fixes("midi", self.snap, None, t) self.assertEqual(self.engine.notes["midi"]["text"], "NO AUTO\nFIX")