Hold-to-fix for the Lighting PC and the LiveStream (OBS) PC
- OBS: start OBS on the LiveStream PC if it is not running, then reconnect Companion. If the PC does not answer, send Wake-on-LAN and show WAKING PC. - ARENA: start Resolume Arena on the Lighting PC if it is not running, then reconnect Companion. VISTA SILENT: start Vista if it is not running. A running OBS, Arena or Vista is never closed. - MIDI (LIGHT PC): restart the rtpMIDI service; it reconnects to its saved peers (NA Ingest, NA ProP, NA Tracks). - Fixes may set "reconnect" (restart the check's Companion connection after the command succeeds); hosts may set "mac"/"wake_broadcast" for Wake-on-LAN. - windows-setup.ps1 finds OBS on any drive and Vista, and leaves Wake-on-LAN alone when it is already on (changing it can reset the adapter). - Tests stub the fix executors so a hold in a test never reaches a machine. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -47,6 +47,10 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl
|
||||
| APPS: PROPRES OFFLINE / NOT RUNNING | restarts ProPresenter on its Mac (the newest version installed) |
|
||||
| APPS: ABLETON / TRANSLATE NOT RUNNING | starts Ableton on the Tracks Mac / Church Translate on NA Translation |
|
||||
| APPS: BIBLE APP DOWN | restarts the Bible app on the Sanctuary TV PC (SSH + its "Start BibleApp" task from `remote-access/windows-setup.ps1`) |
|
||||
| APPS: VISTA SILENT | starts Vista on the Lighting PC if it is not running. A running Vista is never closed. |
|
||||
| ARENA | starts Resolume Arena on the Lighting PC if it is not running, then reconnects Companion |
|
||||
| OBS | starts OBS on the LiveStream PC if it is not running, then reconnects Companion. If the PC is off, it is woken (Wake-on-LAN) and the button shows WAKING PC; hold again once someone has logged in. |
|
||||
| MIDI: LIGHT PC | restarts rtpMIDI on the Lighting PC; it reconnects to NA Ingest, NA ProP and NA Tracks by itself. Lighting MIDI drops for a few seconds. |
|
||||
| DANTE: PROP MAC / TRACKS / TRANSL / INGEST GONE | starts Dante Virtual Soundcard's engine on that Mac (`remote-access/mac-dvs-helper.sh`) |
|
||||
| DANTE: any device powered by its switch port (today the four Studio B boxes) | power-cycles the port it is on **now**, found by its MAC on the Netgear Dante switches |
|
||||
| DANTE: any other device GONE | restarts Companion's Dante module (hardware: if still gone, it is off or unplugged) |
|
||||
|
||||
+16
-6
@@ -32,7 +32,9 @@
|
||||
"prop": { "ssh": "napropresenter@10.1.14.225" },
|
||||
"tracks": { "ssh": "waterschurchmedia@10.1.3.227" },
|
||||
"translation": { "ssh": "studio@10.1.7.39" },
|
||||
"bibletv": { "ssh": "media@10.6.7.13" }
|
||||
"bibletv": { "ssh": "media@10.6.7.13" },
|
||||
"lighting": { "ssh": "media@10.1.10.206" },
|
||||
"livestream": { "ssh": "media@10.1.9.126", "mac": "6c:24:08:d7:fb:57", "wake_broadcast": "10.1.255.255" }
|
||||
},
|
||||
|
||||
"fixes": {
|
||||
@@ -52,6 +54,14 @@
|
||||
"run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" },
|
||||
"bibleapp": { "host": "bibletv", "short": "BIBLE APP", "label": "restart the Bible app on the Sanctuary TV PC (Windows: cmd.exe)",
|
||||
"run": "taskkill /F /T /IM \"Bible App.exe\" >nul 2>&1 & ping -n 3 127.0.0.1 >nul & schtasks /Run /TN \"\\StatusPanel\\Start BibleApp\"" },
|
||||
"obs": { "host": "livestream", "short": "OBS", "label": "start OBS on the LiveStream PC if it is not running (wakes the PC if it is off), then reconnect Companion", "reconnect": true,
|
||||
"run": "tasklist /NH /FI \"IMAGENAME eq obs64.exe\" | find /I \"obs64.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start OBS\"" },
|
||||
"arena": { "host": "lighting", "short": "ARENA", "label": "start Resolume Arena on the Lighting PC if it is not running, then reconnect Companion", "reconnect": true,
|
||||
"run": "tasklist /NH /FI \"IMAGENAME eq Arena.exe\" | find /I \"Arena.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Arena\"" },
|
||||
"vista": { "host": "lighting", "short": "VISTA", "label": "start Vista on the Lighting PC if it is not running (a running Vista is never closed)",
|
||||
"run": "tasklist /NH /FI \"IMAGENAME eq Vista.exe\" | find /I \"Vista.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Vista\"" },
|
||||
"rtpmidi": { "host": "lighting", "short": "RTPMIDI", "label": "restart rtpMIDI on the Lighting PC: it reconnects to NA Ingest, NA ProP and NA Tracks (MIDI drops for a few seconds)",
|
||||
"run": "powershell -NoProfile -NonInteractive -Command \"Restart-Service rtpMIDIService -Force\"" },
|
||||
"proddash": { "kind": "local", "short": "PRODDASH", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" },
|
||||
"hyperdeck": { "kind": "hyperdeck", "short": "HYPERDECK", "label": "reconnect Companion to the HyperDeck, or reboot the deck only if it is hung" }
|
||||
},
|
||||
@@ -88,9 +98,9 @@
|
||||
"NA-Tracks": "TRACKS", "FOH-SuperRack-LiveBox": "SUPERRACK", "NA-ProPresenter-Mac": "PROP MAC" } }
|
||||
] },
|
||||
{ "id": "midi", "label": "MIDI", "pos": [2, 3], "checks": [
|
||||
{ "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC" },
|
||||
{ "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC", "fix": "rtpmidi" },
|
||||
{ "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox", "tag": "SUPERRACK" },
|
||||
{ "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI" },
|
||||
{ "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI", "fix": "rtpmidi" },
|
||||
{ "type": "mdns", "instance": "NA Tracks", "name": "TRACKS MIDI" },
|
||||
{ "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" }
|
||||
] },
|
||||
@@ -101,14 +111,14 @@
|
||||
{ "type": "remote", "host": "tracks", "cmd": "pgrep -xq Live", "msg": "ABLETON NOT RUNNING", "fix": "ableton" },
|
||||
{ "type": "remote", "host": "translation", "cmd": "pgrep -xq 'Church Translate'", "msg": "TRANSLATE NOT RUNNING", "fix": "translate" },
|
||||
{ "type": "ping", "host": "10.1.3.227", "name": "TRACKS MAC" },
|
||||
{ "type": "artnet", "source": "10.1.10.206", "max_silence_seconds": 5, "msg": "VISTA SILENT" },
|
||||
{ "type": "artnet", "source": "10.1.10.206", "max_silence_seconds": 5, "msg": "VISTA SILENT", "fix": "vista" },
|
||||
{ "type": "http", "url": "http://10.6.7.13:8080/", "msg": "BIBLE APP DOWN", "interval_seconds": 5, "fix": "bibleapp" },
|
||||
{ "type": "conn", "conn": "BibleTV", "tag": "BIBLE OSC" }
|
||||
] },
|
||||
|
||||
{ "id": "service", "label": "ALARMS", "pos": [3, 0], "kind": "service" },
|
||||
{ "id": "atem", "label": "ATEM", "pos": [3, 1], "checks": [{ "type": "conn", "conn": "ATEM" }] },
|
||||
{ "id": "obs", "label": "OBS", "pos": [3, 2], "checks": [{ "type": "conn", "conn": "OBS-Youtube", "service_only": true }] },
|
||||
{ "id": "obs", "label": "OBS", "pos": [3, 2], "checks": [{ "type": "conn", "conn": "OBS-Youtube", "service_only": true, "fix": "obs" }] },
|
||||
{ "id": "resi", "label": "RESI", "pos": [3, 3], "checks": [
|
||||
{ "type": "conn", "conn": "RESI_Decoder", "tag": "DECODER" },
|
||||
{ "type": "ping", "host": "10.1.10.203", "name": "DECODER" },
|
||||
@@ -128,7 +138,7 @@
|
||||
{ "type": "ping", "host": "10.1.14.116", "name": "VX1000" },
|
||||
{ "type": "ping", "host": "10.1.14.115", "name": "UHDJR" }
|
||||
] },
|
||||
{ "id": "arena", "label": "ARENA", "pos": [3, 7], "checks": [{ "type": "conn", "conn": "arena" }] },
|
||||
{ "id": "arena", "label": "ARENA", "pos": [3, 7], "checks": [{ "type": "conn", "conn": "arena", "fix": "arena" }] },
|
||||
|
||||
{ "id": "axient1", "label": "AXIENT 1", "checks": [{ "type": "shure", "conn": "Axient_1", "channels": 4 }] },
|
||||
{ "id": "axient2", "label": "AXIENT 2", "checks": [{ "type": "shure", "conn": "Axient_2", "channels": 4 }] },
|
||||
|
||||
@@ -68,8 +68,13 @@ $roots = @($env:ProgramFiles, ${env:ProgramFiles(x86)}, (Join-Path $env:LOCALAPP
|
||||
$bible = Get-ChildItem -Path $roots -Filter '*.exe' -Recurse -Depth 3 -ErrorAction SilentlyContinue |
|
||||
Where-Object { $_.FullName -match 'bible' -and $_.Name -notmatch 'uninst|update|crash|helper' } |
|
||||
Select-Object -First 1 -ExpandProperty FullName
|
||||
# Apps can live on any local drive (the LiveStream PC has OBS on D:, the Lighting PC has Vista on G:).
|
||||
$drives = Get-PSDrive -PSProvider FileSystem | Where-Object { $_.Free -ne $null } | ForEach-Object { $_.Root.TrimEnd('\') }
|
||||
$vista = Find-First (@($drives | ForEach-Object { "$_\Vista3\Vista.exe" }) +
|
||||
@(Get-ChildItem -Path $roots -Filter 'Vista.exe' -Recurse -Depth 3 -ErrorAction SilentlyContinue | ForEach-Object FullName))
|
||||
$apps = @(
|
||||
@{ Name = 'OBS'; Exe = (Find-First @("$env:ProgramFiles\obs-studio\bin\64bit\obs64.exe")); Args = '--disable-shutdown-check' },
|
||||
@{ Name = 'OBS'; Exe = (Find-First @($drives | ForEach-Object { "$_\Program Files\obs-studio\bin\64bit\obs64.exe" })); Args = '--disable-shutdown-check' },
|
||||
@{ Name = 'Vista'; Exe = $vista; Args = '' },
|
||||
@{ Name = 'Arena'; Exe = (Find-First (Get-ChildItem "$env:ProgramFiles\Resolume Arena*\Arena.exe" -ErrorAction SilentlyContinue | ForEach-Object FullName)); Args = '' },
|
||||
@{ Name = 'rtpMIDI'; Exe = (Find-First @("${env:ProgramFiles(x86)}\Tobias Erichsen\rtpMIDI\rtpMIDI.exe", "$env:ProgramFiles\Tobias Erichsen\rtpMIDI\rtpMIDI.exe")); Args = '' },
|
||||
@{ Name = 'BibleApp'; Exe = $bible; Args = ''; AtLogon = $true } # also opens it 15 s after login
|
||||
@@ -96,6 +101,8 @@ foreach ($app in $apps) {
|
||||
# 4. Wake-on-LAN: let the wired adapter wake the PC (the BIOS setting must be on too)
|
||||
$nics = Get-NetAdapter -Physical -ErrorAction SilentlyContinue | Where-Object Status -eq 'Up'
|
||||
foreach ($nic in $nics) {
|
||||
# Only when it is off: changing it can briefly reset the adapter (and the PC's network).
|
||||
if ((Get-NetAdapterPowerManagement -Name $nic.Name -ErrorAction SilentlyContinue).WakeOnMagicPacket -eq 'Enabled') { continue }
|
||||
try { Set-NetAdapterPowerManagement -Name $nic.Name -WakeOnMagicPacket Enabled -ErrorAction Stop }
|
||||
catch { $notes += "Wake-on-LAN not settable on $($nic.Name)" }
|
||||
}
|
||||
|
||||
+33
-2
@@ -877,10 +877,12 @@ class Engine:
|
||||
|
||||
def work(fn, args):
|
||||
ok = fn(*args)
|
||||
results.append(ok is not False)
|
||||
results.append(ok)
|
||||
if len(results) == len(started): # the last one to finish reports for all
|
||||
done = time.time()
|
||||
if all(results):
|
||||
if WAKING in results:
|
||||
self.note(tile_id, "WAKING PC\n" + summary, "fixing", done, 120)
|
||||
elif all(r is not False for r in results):
|
||||
self.note(tile_id, "CHECKING\n" + summary, "fixing", done, 20)
|
||||
else:
|
||||
self.note(tile_id, "FIX FAILED\n" + summary, "failed", done, 20)
|
||||
@@ -1141,6 +1143,22 @@ def ssh_cmd(cfg, host):
|
||||
"-o", "ControlPath=/tmp/status-panel-ssh-%C", "-o", "ControlPersist=120", cfg["hosts"][host]["ssh"]]
|
||||
|
||||
|
||||
WAKING = "waking" # execute_fix result: the PC was off, a Wake-on-LAN packet went out
|
||||
|
||||
|
||||
def wake_on_lan(mac, broadcast=None, port=9):
|
||||
"""Send a Wake-on-LAN magic packet (to the PC's subnet broadcast as well, when given)."""
|
||||
raw = bytes.fromhex(re.sub(r"[^0-9A-Fa-f]", "", mac))
|
||||
if len(raw) != 6:
|
||||
raise ValueError("not a MAC address: %r" % mac)
|
||||
packet = b"\xff" * 6 + raw * 16
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s:
|
||||
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
||||
for addr in [a for a in (broadcast, "255.255.255.255") if a]:
|
||||
s.sendto(packet, (addr, port))
|
||||
return packet
|
||||
|
||||
|
||||
def remote_script(cmds):
|
||||
"""One shell line that runs every check and prints SP<n>=<exit code>."""
|
||||
return "; ".join("(%s) >/dev/null 2>&1; echo SP%d=$?" % (cmd, i) for i, cmd in enumerate(cmds))
|
||||
@@ -1300,8 +1318,21 @@ def execute_fix(cfg, fix_id, chk):
|
||||
argv = ssh_cmd(cfg, fix["host"]) + [fix["run"]] if kind == "ssh" else ["/bin/zsh", "-c", fix["run"]]
|
||||
r = subprocess.run(argv, capture_output=True, text=True, timeout=fix.get("timeout", 60))
|
||||
detail = (r.stdout + r.stderr).strip().replace("\n", " ")[-200:]
|
||||
host = cfg.get("hosts", {}).get(fix.get("host"), {}) if kind == "ssh" else {}
|
||||
if r.returncode == 255 and host.get("mac"):
|
||||
# ssh could not reach the PC: it is probably off or asleep. Wake it; hold again
|
||||
# once it is up (the app starts only after someone is logged in).
|
||||
wake_on_lan(host["mac"], host.get("wake_broadcast"))
|
||||
log.info("FIX %s -> %s does not answer: Wake-on-LAN sent to %s", label, fix["host"], host["mac"])
|
||||
return WAKING
|
||||
log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode,
|
||||
(": " + detail) if detail else "")
|
||||
if r.returncode == 0 and fix.get("reconnect") and chk.get("conn"):
|
||||
# The app is running now: make Companion reconnect to it instead of waiting for its retry.
|
||||
comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
|
||||
ids = {c["label"]: c["id"] for c in comp.connections()}
|
||||
ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]])
|
||||
log.info("FIX %s -> Companion connection %s restarted %s", label, chk["conn"], "ok" if ok else "FAILED")
|
||||
return r.returncode == 0
|
||||
if kind == "netgear_poe":
|
||||
# Netgear AV switch on the Dante network: power-cycle one PoE port (netgear.py).
|
||||
|
||||
@@ -465,6 +465,8 @@ class TestFixes(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.snap = healthy_snapshot(SUNDAY)
|
||||
self.engine = sp.Engine(CONFIG)
|
||||
# Safety net: a hold in these tests must never reach a real machine.
|
||||
self.engine.executor = self.engine.poe_executor = lambda *a: True
|
||||
|
||||
def plan(self, tile):
|
||||
run_steps(self.engine, self.snap, SUNDAY, 4)
|
||||
@@ -492,6 +494,59 @@ class TestFixes(unittest.TestCase):
|
||||
self.assertEqual(CONFIG["hosts"][fix["host"]]["ssh"], "media@10.6.7.13")
|
||||
self.assertIn('schtasks /Run /TN "\\StatusPanel\\Start BibleApp"', fix["run"]) # cmd.exe, not sh
|
||||
|
||||
def test_lighting_and_livestream_pcs(self):
|
||||
self.snap.conns["OBS-Youtube"]["category"] = "error"
|
||||
self.snap.conns["arena"]["category"] = "error"
|
||||
self.snap.artnet_since, self.snap.artnet_last = SUNDAY - 100, {"10.1.10.206": SUNDAY - 100}
|
||||
self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"]
|
||||
if p["name"] != "Lighting PC"]
|
||||
self.snap.mdns = self.snap.mdns - {"Lighting PC"}
|
||||
self.assertEqual(self.plan("obs"), [("fix", "obs")])
|
||||
self.assertEqual(self.plan("arena"), [("fix", "arena")])
|
||||
self.assertEqual(self.plan("apps"), [("fix", "vista")])
|
||||
self.assertEqual(self.plan("midi"), [("fix", "rtpmidi")]) # two checks, one restart
|
||||
for fix_id in ("obs", "arena", "vista"): # never closes a running app
|
||||
self.assertTrue(CONFIG["fixes"][fix_id]["run"].startswith("tasklist "), fix_id)
|
||||
|
||||
def test_an_off_pc_is_woken(self):
|
||||
from unittest import mock
|
||||
woke = []
|
||||
done = mock.Mock(returncode=255, stdout="", stderr="ssh: connect to host 10.1.9.126 port 22: timed out")
|
||||
with mock.patch.object(sp.subprocess, "run", return_value=done), \
|
||||
mock.patch.object(sp, "wake_on_lan", side_effect=lambda mac, bc=None: woke.append((mac, bc))):
|
||||
self.assertEqual(sp.execute_fix(CONFIG, "obs", {"conn": "OBS-Youtube"}), sp.WAKING)
|
||||
self.assertEqual(woke, [("6c:24:08:d7:fb:57", "10.1.255.255")])
|
||||
self.assertFalse(sp.execute_fix(CONFIG, "arena", {"conn": "arena"})) # no MAC: just fails
|
||||
self.assertEqual(len(woke), 1)
|
||||
# the button says so
|
||||
self.snap.conns["OBS-Youtube"]["category"] = "error"
|
||||
self.engine.executor = lambda cfg, fix_id, chk: sp.WAKING
|
||||
_, t = run_steps(self.engine, self.snap, SUNDAY, 4)
|
||||
self.engine.run_fixes("obs", self.snap, None, t)
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(self.engine.step(self.snap, time.time())["obs"]["x"], "OBS\nWAKING PC\nOBS")
|
||||
|
||||
def test_magic_packet(self):
|
||||
from unittest import mock
|
||||
with mock.patch.object(sp.socket, "socket") as sock:
|
||||
packet = sp.wake_on_lan("6c:24:08:d7:fb:57", "10.1.255.255")
|
||||
self.assertEqual(packet, b"\xff" * 6 + bytes.fromhex("6c2408d7fb57") * 16)
|
||||
sent = [c.args[1] for c in sock.return_value.__enter__.return_value.sendto.call_args_list]
|
||||
self.assertEqual(sent, [("10.1.255.255", 9), ("255.255.255.255", 9)])
|
||||
|
||||
def test_companion_reconnects_after_the_app_starts(self):
|
||||
from unittest import mock
|
||||
comp = mock.Mock()
|
||||
comp.connections.return_value = [{"label": "arena", "id": "abc"}]
|
||||
ok = mock.Mock(returncode=0, stdout="", stderr="")
|
||||
with mock.patch.object(sp.subprocess, "run", return_value=ok), \
|
||||
mock.patch.object(sp, "Companion", return_value=comp):
|
||||
self.assertTrue(sp.execute_fix(CONFIG, "arena", {"conn": "arena"}))
|
||||
comp.restart_connection.assert_called_once_with("abc")
|
||||
comp.reset_mock()
|
||||
self.assertTrue(sp.execute_fix(CONFIG, "vista", {"type": "artnet"})) # no "reconnect"
|
||||
comp.restart_connection.assert_not_called()
|
||||
|
||||
def test_unreachable_mac_is_not_a_verdict(self):
|
||||
self.snap.remote["tracks\0pgrep -xq Live"] = None
|
||||
self.assertEqual(self.plan("apps"), [])
|
||||
@@ -632,7 +687,8 @@ class TestFixes(unittest.TestCase):
|
||||
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices)
|
||||
self.assertEqual(self.plan("dante"), [("fix", "dvs_ingest")])
|
||||
# A failure with no fix must not claim there is nothing wrong.
|
||||
self.snap.midi["NA Ingest"]["peers"] = []
|
||||
self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"]
|
||||
if p["name"] != "SuperRack-LiveBox"]
|
||||
_, t = run_steps(self.engine, self.snap, SUNDAY, 4)
|
||||
self.engine.run_fixes("midi", self.snap, None, t)
|
||||
self.assertEqual(self.engine.notes["midi"]["text"], "NO AUTO\nFIX")
|
||||
|
||||
Reference in New Issue
Block a user