Hold-to-fix for the Lighting PC and the LiveStream (OBS) PC

- OBS: start OBS on the LiveStream PC if it is not running, then reconnect
  Companion. If the PC does not answer, send Wake-on-LAN and show WAKING PC.
- ARENA: start Resolume Arena on the Lighting PC if it is not running, then
  reconnect Companion. VISTA SILENT: start Vista if it is not running.
  A running OBS, Arena or Vista is never closed.
- MIDI (LIGHT PC): restart the rtpMIDI service; it reconnects to its saved
  peers (NA Ingest, NA ProP, NA Tracks).
- Fixes may set "reconnect" (restart the check's Companion connection after
  the command succeeds); hosts may set "mac"/"wake_broadcast" for Wake-on-LAN.
- windows-setup.ps1 finds OBS on any drive and Vista, and leaves Wake-on-LAN
  alone when it is already on (changing it can reset the adapter).
- Tests stub the fix executors so a hold in a test never reaches a machine.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -47,6 +47,10 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl
   | APPS: PROPRES OFFLINE / NOT RUNNING | restarts ProPresenter on its Mac (the newest version installed) |
   | APPS: ABLETON / TRANSLATE NOT RUNNING | starts Ableton on the Tracks Mac / Church Translate on NA Translation |
   | APPS: BIBLE APP DOWN | restarts the Bible app on the Sanctuary TV PC (SSH + its "Start BibleApp" task from `remote-access/windows-setup.ps1`) |
+  | APPS: VISTA SILENT | starts Vista on the Lighting PC if it is not running. A running Vista is never closed. |
+  | ARENA | starts Resolume Arena on the Lighting PC if it is not running, then reconnects Companion |
+  | OBS | starts OBS on the LiveStream PC if it is not running, then reconnects Companion. If the PC is off, it is woken (Wake-on-LAN) and the button shows WAKING PC; hold again once someone has logged in. |
+  | MIDI: LIGHT PC | restarts rtpMIDI on the Lighting PC; it reconnects to NA Ingest, NA ProP and NA Tracks by itself. Lighting MIDI drops for a few seconds. |
   | DANTE: PROP MAC / TRACKS / TRANSL / INGEST GONE | starts Dante Virtual Soundcard's engine on that Mac (`remote-access/mac-dvs-helper.sh`) |
   | DANTE: any device powered by its switch port (today the four Studio B boxes) | power-cycles the port it is on **now**, found by its MAC on the Netgear Dante switches |
   | DANTE: any other device GONE | restarts Companion's Dante module (hardware: if still gone, it is off or unplugged) |

config.json

@@ -32,7 +32,9 @@
     "prop":        { "ssh": "napropresenter@10.1.14.225" },
     "tracks":      { "ssh": "waterschurchmedia@10.1.3.227" },
     "translation": { "ssh": "studio@10.1.7.39" },
-    "bibletv":     { "ssh": "media@10.6.7.13" }
+    "bibletv":     { "ssh": "media@10.6.7.13" },
+    "lighting":    { "ssh": "media@10.1.10.206" },
+    "livestream":  { "ssh": "media@10.1.9.126", "mac": "6c:24:08:d7:fb:57", "wake_broadcast": "10.1.255.255" }
   },
 
   "fixes": {
@@ -52,6 +54,14 @@
                          "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" },
     "bibleapp":        { "host": "bibletv", "short": "BIBLE APP", "label": "restart the Bible app on the Sanctuary TV PC (Windows: cmd.exe)",
                          "run": "taskkill /F /T /IM \"Bible App.exe\" >nul 2>&1 & ping -n 3 127.0.0.1 >nul & schtasks /Run /TN \"\\StatusPanel\\Start BibleApp\"" },
+    "obs":             { "host": "livestream", "short": "OBS", "label": "start OBS on the LiveStream PC if it is not running (wakes the PC if it is off), then reconnect Companion", "reconnect": true,
+                         "run": "tasklist /NH /FI \"IMAGENAME eq obs64.exe\" | find /I \"obs64.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start OBS\"" },
+    "arena":           { "host": "lighting", "short": "ARENA", "label": "start Resolume Arena on the Lighting PC if it is not running, then reconnect Companion", "reconnect": true,
+                         "run": "tasklist /NH /FI \"IMAGENAME eq Arena.exe\" | find /I \"Arena.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Arena\"" },
+    "vista":           { "host": "lighting", "short": "VISTA", "label": "start Vista on the Lighting PC if it is not running (a running Vista is never closed)",
+                         "run": "tasklist /NH /FI \"IMAGENAME eq Vista.exe\" | find /I \"Vista.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Vista\"" },
+    "rtpmidi":         { "host": "lighting", "short": "RTPMIDI", "label": "restart rtpMIDI on the Lighting PC: it reconnects to NA Ingest, NA ProP and NA Tracks (MIDI drops for a few seconds)",
+                         "run": "powershell -NoProfile -NonInteractive -Command \"Restart-Service rtpMIDIService -Force\"" },
     "proddash":        { "kind": "local", "short": "PRODDASH", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" },
     "hyperdeck":       { "kind": "hyperdeck", "short": "HYPERDECK", "label": "reconnect Companion to the HyperDeck, or reboot the deck only if it is hung" }
   },
@@ -88,9 +98,9 @@
         "NA-Tracks": "TRACKS", "FOH-SuperRack-LiveBox": "SUPERRACK", "NA-ProPresenter-Mac": "PROP MAC" } }
     ] },
     { "id": "midi", "label": "MIDI", "pos": [2, 3], "checks": [
-      { "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC" },
+      { "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC", "fix": "rtpmidi" },
       { "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox", "tag": "SUPERRACK" },
-      { "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI" },
+      { "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI", "fix": "rtpmidi" },
       { "type": "mdns", "instance": "NA Tracks", "name": "TRACKS MIDI" },
       { "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" }
     ] },
@@ -101,14 +111,14 @@
       { "type": "remote", "host": "tracks", "cmd": "pgrep -xq Live", "msg": "ABLETON NOT RUNNING", "fix": "ableton" },
       { "type": "remote", "host": "translation", "cmd": "pgrep -xq 'Church Translate'", "msg": "TRANSLATE NOT RUNNING", "fix": "translate" },
       { "type": "ping", "host": "10.1.3.227", "name": "TRACKS MAC" },
-      { "type": "artnet", "source": "10.1.10.206", "max_silence_seconds": 5, "msg": "VISTA SILENT" },
+      { "type": "artnet", "source": "10.1.10.206", "max_silence_seconds": 5, "msg": "VISTA SILENT", "fix": "vista" },
       { "type": "http", "url": "http://10.6.7.13:8080/", "msg": "BIBLE APP DOWN", "interval_seconds": 5, "fix": "bibleapp" },
       { "type": "conn", "conn": "BibleTV", "tag": "BIBLE OSC" }
     ] },
 
     { "id": "service", "label": "ALARMS", "pos": [3, 0], "kind": "service" },
     { "id": "atem", "label": "ATEM", "pos": [3, 1], "checks": [{ "type": "conn", "conn": "ATEM" }] },
-    { "id": "obs", "label": "OBS", "pos": [3, 2], "checks": [{ "type": "conn", "conn": "OBS-Youtube", "service_only": true }] },
+    { "id": "obs", "label": "OBS", "pos": [3, 2], "checks": [{ "type": "conn", "conn": "OBS-Youtube", "service_only": true, "fix": "obs" }] },
     { "id": "resi", "label": "RESI", "pos": [3, 3], "checks": [
       { "type": "conn", "conn": "RESI_Decoder", "tag": "DECODER" },
       { "type": "ping", "host": "10.1.10.203", "name": "DECODER" },
@@ -128,7 +138,7 @@
       { "type": "ping", "host": "10.1.14.116", "name": "VX1000" },
       { "type": "ping", "host": "10.1.14.115", "name": "UHDJR" }
     ] },
-    { "id": "arena", "label": "ARENA", "pos": [3, 7], "checks": [{ "type": "conn", "conn": "arena" }] },
+    { "id": "arena", "label": "ARENA", "pos": [3, 7], "checks": [{ "type": "conn", "conn": "arena", "fix": "arena" }] },
 
     { "id": "axient1", "label": "AXIENT 1", "checks": [{ "type": "shure", "conn": "Axient_1", "channels": 4 }] },
     { "id": "axient2", "label": "AXIENT 2", "checks": [{ "type": "shure", "conn": "Axient_2", "channels": 4 }] },

remote-access/windows-setup.ps1

@@ -68,8 +68,13 @@ $roots = @($env:ProgramFiles, ${env:ProgramFiles(x86)}, (Join-Path $env:LOCALAPP
 $bible = Get-ChildItem -Path $roots -Filter '*.exe' -Recurse -Depth 3 -ErrorAction SilentlyContinue |
     Where-Object { $_.FullName -match 'bible' -and $_.Name -notmatch 'uninst|update|crash|helper' } |
     Select-Object -First 1 -ExpandProperty FullName
+# Apps can live on any local drive (the LiveStream PC has OBS on D:, the Lighting PC has Vista on G:).
+$drives = Get-PSDrive -PSProvider FileSystem | Where-Object { $_.Free -ne $null } | ForEach-Object { $_.Root.TrimEnd('\') }
+$vista = Find-First (@($drives | ForEach-Object { "$_\Vista3\Vista.exe" }) +
+    @(Get-ChildItem -Path $roots -Filter 'Vista.exe' -Recurse -Depth 3 -ErrorAction SilentlyContinue | ForEach-Object FullName))
 $apps = @(
-    @{ Name = 'OBS';      Exe = (Find-First @("$env:ProgramFiles\obs-studio\bin\64bit\obs64.exe")); Args = '--disable-shutdown-check' },
+    @{ Name = 'OBS';      Exe = (Find-First @($drives | ForEach-Object { "$_\Program Files\obs-studio\bin\64bit\obs64.exe" })); Args = '--disable-shutdown-check' },
+    @{ Name = 'Vista';    Exe = $vista; Args = '' },
     @{ Name = 'Arena';    Exe = (Find-First (Get-ChildItem "$env:ProgramFiles\Resolume Arena*\Arena.exe" -ErrorAction SilentlyContinue | ForEach-Object FullName)); Args = '' },
     @{ Name = 'rtpMIDI';  Exe = (Find-First @("${env:ProgramFiles(x86)}\Tobias Erichsen\rtpMIDI\rtpMIDI.exe", "$env:ProgramFiles\Tobias Erichsen\rtpMIDI\rtpMIDI.exe")); Args = '' },
     @{ Name = 'BibleApp'; Exe = $bible; Args = ''; AtLogon = $true }   # also opens it 15 s after login
@@ -96,6 +101,8 @@ foreach ($app in $apps) {
 # 4. Wake-on-LAN: let the wired adapter wake the PC (the BIOS setting must be on too)
 $nics = Get-NetAdapter -Physical -ErrorAction SilentlyContinue | Where-Object Status -eq 'Up'
 foreach ($nic in $nics) {
+    # Only when it is off: changing it can briefly reset the adapter (and the PC's network).
+    if ((Get-NetAdapterPowerManagement -Name $nic.Name -ErrorAction SilentlyContinue).WakeOnMagicPacket -eq 'Enabled') { continue }
     try { Set-NetAdapterPowerManagement -Name $nic.Name -WakeOnMagicPacket Enabled -ErrorAction Stop }
     catch { $notes += "Wake-on-LAN not settable on $($nic.Name)" }
 }

status_panel.py

@@ -877,10 +877,12 @@ class Engine:
 
         def work(fn, args):
             ok = fn(*args)
-            results.append(ok is not False)
+            results.append(ok)
             if len(results) == len(started):          # the last one to finish reports for all
                 done = time.time()
-                if all(results):
+                if WAKING in results:
+                    self.note(tile_id, "WAKING PC\n" + summary, "fixing", done, 120)
+                elif all(r is not False for r in results):
                     self.note(tile_id, "CHECKING\n" + summary, "fixing", done, 20)
                 else:
                     self.note(tile_id, "FIX FAILED\n" + summary, "failed", done, 20)
@@ -1141,6 +1143,22 @@ def ssh_cmd(cfg, host):
             "-o", "ControlPath=/tmp/status-panel-ssh-%C", "-o", "ControlPersist=120", cfg["hosts"][host]["ssh"]]
 
 
+WAKING = "waking"     # execute_fix result: the PC was off, a Wake-on-LAN packet went out
+
+
+def wake_on_lan(mac, broadcast=None, port=9):
+    """Send a Wake-on-LAN magic packet (to the PC's subnet broadcast as well, when given)."""
+    raw = bytes.fromhex(re.sub(r"[^0-9A-Fa-f]", "", mac))
+    if len(raw) != 6:
+        raise ValueError("not a MAC address: %r" % mac)
+    packet = b"\xff" * 6 + raw * 16
+    with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
+        s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
+        for addr in [a for a in (broadcast, "255.255.255.255") if a]:
+            s.sendto(packet, (addr, port))
+    return packet
+
+
 def remote_script(cmds):
     """One shell line that runs every check and prints SP<n>=<exit code>."""
     return "; ".join("(%s) >/dev/null 2>&1; echo SP%d=$?" % (cmd, i) for i, cmd in enumerate(cmds))
@@ -1300,8 +1318,21 @@ def execute_fix(cfg, fix_id, chk):
             argv = ssh_cmd(cfg, fix["host"]) + [fix["run"]] if kind == "ssh" else ["/bin/zsh", "-c", fix["run"]]
             r = subprocess.run(argv, capture_output=True, text=True, timeout=fix.get("timeout", 60))
             detail = (r.stdout + r.stderr).strip().replace("\n", " ")[-200:]
+            host = cfg.get("hosts", {}).get(fix.get("host"), {}) if kind == "ssh" else {}
+            if r.returncode == 255 and host.get("mac"):
+                # ssh could not reach the PC: it is probably off or asleep. Wake it; hold again
+                # once it is up (the app starts only after someone is logged in).
+                wake_on_lan(host["mac"], host.get("wake_broadcast"))
+                log.info("FIX %s -> %s does not answer: Wake-on-LAN sent to %s", label, fix["host"], host["mac"])
+                return WAKING
             log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode,
                      (": " + detail) if detail else "")
+            if r.returncode == 0 and fix.get("reconnect") and chk.get("conn"):
+                # The app is running now: make Companion reconnect to it instead of waiting for its retry.
+                comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
+                ids = {c["label"]: c["id"] for c in comp.connections()}
+                ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]])
+                log.info("FIX %s -> Companion connection %s restarted %s", label, chk["conn"], "ok" if ok else "FAILED")
             return r.returncode == 0
         if kind == "netgear_poe":
             # Netgear AV switch on the Dante network: power-cycle one PoE port (netgear.py).

tests/test_status_panel.py

@@ -465,6 +465,8 @@ class TestFixes(unittest.TestCase):
     def setUp(self):
         self.snap = healthy_snapshot(SUNDAY)
         self.engine = sp.Engine(CONFIG)
+        # Safety net: a hold in these tests must never reach a real machine.
+        self.engine.executor = self.engine.poe_executor = lambda *a: True
 
     def plan(self, tile):
         run_steps(self.engine, self.snap, SUNDAY, 4)
@@ -492,6 +494,59 @@ class TestFixes(unittest.TestCase):
         self.assertEqual(CONFIG["hosts"][fix["host"]]["ssh"], "media@10.6.7.13")
         self.assertIn('schtasks /Run /TN "\\StatusPanel\\Start BibleApp"', fix["run"])   # cmd.exe, not sh
 
+    def test_lighting_and_livestream_pcs(self):
+        self.snap.conns["OBS-Youtube"]["category"] = "error"
+        self.snap.conns["arena"]["category"] = "error"
+        self.snap.artnet_since, self.snap.artnet_last = SUNDAY - 100, {"10.1.10.206": SUNDAY - 100}
+        self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"]
+                                                if p["name"] != "Lighting PC"]
+        self.snap.mdns = self.snap.mdns - {"Lighting PC"}
+        self.assertEqual(self.plan("obs"), [("fix", "obs")])
+        self.assertEqual(self.plan("arena"), [("fix", "arena")])
+        self.assertEqual(self.plan("apps"), [("fix", "vista")])
+        self.assertEqual(self.plan("midi"), [("fix", "rtpmidi")])           # two checks, one restart
+        for fix_id in ("obs", "arena", "vista"):                             # never closes a running app
+            self.assertTrue(CONFIG["fixes"][fix_id]["run"].startswith("tasklist "), fix_id)
+
+    def test_an_off_pc_is_woken(self):
+        from unittest import mock
+        woke = []
+        done = mock.Mock(returncode=255, stdout="", stderr="ssh: connect to host 10.1.9.126 port 22: timed out")
+        with mock.patch.object(sp.subprocess, "run", return_value=done), \
+                mock.patch.object(sp, "wake_on_lan", side_effect=lambda mac, bc=None: woke.append((mac, bc))):
+            self.assertEqual(sp.execute_fix(CONFIG, "obs", {"conn": "OBS-Youtube"}), sp.WAKING)
+            self.assertEqual(woke, [("6c:24:08:d7:fb:57", "10.1.255.255")])
+            self.assertFalse(sp.execute_fix(CONFIG, "arena", {"conn": "arena"}))   # no MAC: just fails
+        self.assertEqual(len(woke), 1)
+        # the button says so
+        self.snap.conns["OBS-Youtube"]["category"] = "error"
+        self.engine.executor = lambda cfg, fix_id, chk: sp.WAKING
+        _, t = run_steps(self.engine, self.snap, SUNDAY, 4)
+        self.engine.run_fixes("obs", self.snap, None, t)
+        time.sleep(0.2)
+        self.assertEqual(self.engine.step(self.snap, time.time())["obs"]["x"], "OBS\nWAKING PC\nOBS")
+
+    def test_magic_packet(self):
+        from unittest import mock
+        with mock.patch.object(sp.socket, "socket") as sock:
+            packet = sp.wake_on_lan("6c:24:08:d7:fb:57", "10.1.255.255")
+        self.assertEqual(packet, b"\xff" * 6 + bytes.fromhex("6c2408d7fb57") * 16)
+        sent = [c.args[1] for c in sock.return_value.__enter__.return_value.sendto.call_args_list]
+        self.assertEqual(sent, [("10.1.255.255", 9), ("255.255.255.255", 9)])
+
+    def test_companion_reconnects_after_the_app_starts(self):
+        from unittest import mock
+        comp = mock.Mock()
+        comp.connections.return_value = [{"label": "arena", "id": "abc"}]
+        ok = mock.Mock(returncode=0, stdout="", stderr="")
+        with mock.patch.object(sp.subprocess, "run", return_value=ok), \
+                mock.patch.object(sp, "Companion", return_value=comp):
+            self.assertTrue(sp.execute_fix(CONFIG, "arena", {"conn": "arena"}))
+            comp.restart_connection.assert_called_once_with("abc")
+            comp.reset_mock()
+            self.assertTrue(sp.execute_fix(CONFIG, "vista", {"type": "artnet"}))   # no "reconnect"
+            comp.restart_connection.assert_not_called()
+
     def test_unreachable_mac_is_not_a_verdict(self):
         self.snap.remote["tracks\0pgrep -xq Live"] = None
         self.assertEqual(self.plan("apps"), [])
@@ -632,7 +687,8 @@ class TestFixes(unittest.TestCase):
         self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices)
         self.assertEqual(self.plan("dante"), [("fix", "dvs_ingest")])
         # A failure with no fix must not claim there is nothing wrong.
-        self.snap.midi["NA Ingest"]["peers"] = []
+        self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"]
+                                                if p["name"] != "SuperRack-LiveBox"]
         _, t = run_steps(self.engine, self.snap, SUNDAY, 4)
         self.engine.run_fixes("midi", self.snap, None, t)
         self.assertEqual(self.engine.notes["midi"]["text"], "NO AUTO\nFIX")