MIDI alarm when the Lighting PC drops out of NA Tracks' session (FX lights)
Tracks sends the FX lighting cues to the Lighting PC through NA Tracks' network MIDI session. A midi_peer check may now name a "host": the watchdog reads that Mac's sessions over SSH every 5 s, sending this folder's coremidi.py over stdin (nothing installed there). Unreachable = no verdict; the Tracks Mac's own ping check covers that. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -84,7 +84,7 @@ The **page number key** on every page follows the ALL button. It's green when ev
|
||||
| X32, ATEM, ARENA, OBS | Companion connection |
|
||||
| LED | VX1000 and UHD Jr: Companion connection **and** each one answers ping (the NovaStar module keeps saying "Connected" after a processor is switched off). "UHDJR NO PING" = processor off or unreachable; "UHDJR OFFLINE" = Companion's connection failed; "UHDJR DISABLED" (amber) = connection turned off in Companion. |
|
||||
| DANTE | All 19 Dante devices ("19/19"), named when one drops: "WING GONE", "TRACKS GONE", "PROP MAC GONE". |
|
||||
| MIDI | Lighting PC and SuperRack still joined to NA Ingest's network MIDI session; the Lighting PC and NA Tracks MIDI sessions still on the network. A clean disconnect shows within seconds; a crash or pulled cable takes about a minute (macOS's own timeout, measured). |
|
||||
| MIDI | Lighting PC and SuperRack still joined to NA Ingest's network MIDI session; **FX LIGHTS**: the Lighting PC still joined to NA Tracks' session, which carries the FX lighting cues from tracks (read on the Tracks Mac over SSH every 5 s); the Lighting PC and NA Tracks MIDI sessions still on the network. A clean disconnect shows within seconds; a crash or pulled cable takes about a minute (macOS's own timeout, measured). |
|
||||
| APPS | ProPresenter API; Ableton OSC; tracks Mac pings; Bible TV app answering on :8080 (every 5 s); Bible TV OSC; **Vista** on the Lighting PC: its Art-Net poll (sent every 0.5 s) must be heard within 5 s, otherwise "VISTA SILENT" (Vista closed, crashed, or the Lighting PC is off the network). Resolume is checked by the ARENA button. Messages name the app ("PROPRES OFFLINE", "BIBLE APP DOWN"). Hold 2 s restarts the ProPresenter, Ableton and Bible TV Companion connections. |
|
||||
| RESI | decoder (RossTalk connection and ping); resi-studio-cues; Resi cloud reachable |
|
||||
| NET | NA Ingest's link; gateway 10.1.0.1 pings; **4+ connections from 3+ groups dropping within 10 s** (what happened Sunday 10/5 at 10:21) |
|
||||
|
||||
@@ -98,6 +98,7 @@
|
||||
{ "id": "midi", "label": "MIDI", "pos": [2, 3], "checks": [
|
||||
{ "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC" },
|
||||
{ "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox", "tag": "SUPERRACK" },
|
||||
{ "type": "midi_peer", "host": "tracks", "session": "NA Tracks", "peer": "Lighting PC", "tag": "FX LIGHTS" },
|
||||
{ "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI" },
|
||||
{ "type": "mdns", "instance": "NA Tracks", "name": "TRACKS MIDI" },
|
||||
{ "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" }
|
||||
|
||||
+40
-2
@@ -133,6 +133,7 @@ class Snapshot:
|
||||
self.conns_time = 0.0
|
||||
self.vars = {} # (label, name) -> str or None
|
||||
self.midi = {} # session name -> {"peers": [...]}
|
||||
self.remote_midi = {} # host -> the same, read on that Mac over SSH (None: unreachable)
|
||||
self.midi_time = 0.0
|
||||
self.mdns = set() # Bonjour _apple-midi._udp instance names
|
||||
self.mdns_time = 0.0
|
||||
@@ -400,9 +401,12 @@ def check_disk(chk, snap, ctx):
|
||||
|
||||
|
||||
def check_midi_peer(chk, snap, ctx):
|
||||
if not snap.midi_time:
|
||||
# "host": a session on another Mac (e.g. NA Tracks -> Lighting PC, the FX lighting cues),
|
||||
# read over SSH. Unreachable or not read yet = no verdict (that Mac's own ping says if it is down).
|
||||
sessions = snap.remote_midi.get(chk["host"]) if chk.get("host") else (snap.midi if snap.midi_time else None)
|
||||
if sessions is None:
|
||||
return [("peer", Result(OK))]
|
||||
sess = snap.midi.get(chk["session"])
|
||||
sess = sessions.get(chk["session"])
|
||||
if sess is None:
|
||||
return [("peer", Result(ERR, "NO SESSION"))]
|
||||
for p in sess["peers"]:
|
||||
@@ -1135,6 +1139,39 @@ def poll_midi(snap, cfg):
|
||||
|
||||
# --- Other machines: SSH checks and fixes ------------------------------------
|
||||
|
||||
def remote_midi_hosts(cfg):
|
||||
return sorted({c["host"] for t in cfg["tiles"] for c in t.get("checks", [])
|
||||
if c["type"] == "midi_peer" and c.get("host")})
|
||||
|
||||
|
||||
def remote_midi_script():
|
||||
"""Python for /usr/bin/python3 on another Mac: this folder's coremidi.py, then print its sessions.
|
||||
Nothing is installed there; the code goes over stdin each time."""
|
||||
src = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "coremidi.py"), encoding="utf-8").read()
|
||||
return ("import json\nns = {'__name__': 'spcm'}\nexec(compile(%r, 'coremidi.py', 'exec'), ns)\n"
|
||||
"print('SPMIDI' + json.dumps(ns['sessions']()))\n" % src)
|
||||
|
||||
|
||||
def parse_remote_midi(out):
|
||||
for line in (out or "").splitlines():
|
||||
if line.startswith("SPMIDI"):
|
||||
try:
|
||||
return json.loads(line[6:])
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def poll_remote_midi(snap, cfg):
|
||||
script = remote_midi_script()
|
||||
for host in remote_midi_hosts(cfg):
|
||||
try:
|
||||
r = subprocess.run(ssh_cmd(cfg, host) + ["/usr/bin/python3", "-"], input=script,
|
||||
capture_output=True, text=True, timeout=20)
|
||||
snap.remote_midi[host] = parse_remote_midi(r.stdout) if r.returncode == 0 else None
|
||||
except (OSError, subprocess.SubprocessError):
|
||||
snap.remote_midi[host] = None
|
||||
|
||||
def ssh_cmd(cfg, host):
|
||||
"""ssh to a machine in config "hosts", with NA Ingest's status-panel key; one shared connection per host."""
|
||||
return ["/usr/bin/ssh", "-i", os.path.expanduser(cfg.get("ssh_key", "~/.ssh/id_status_panel")),
|
||||
@@ -1477,6 +1514,7 @@ def start_pollers(cfg, snap, stop, data_dir=None):
|
||||
Periodic("bonjour", 15.0, lambda: poll_mdns(snap), stop),
|
||||
Periodic("http", 5.0, lambda: poll_http(snap, http_checks(cfg)), stop),
|
||||
Periodic("remote", 10.0, lambda: poll_remote(snap, cfg), stop),
|
||||
Periodic("remote-midi", 5.0, lambda: poll_remote_midi(snap, cfg), stop),
|
||||
Periodic("dante-map", 30.0, lambda: poll_dante_map(snap, cfg, c4, dante_map_path(data_dir)), stop),
|
||||
]
|
||||
for t in threads:
|
||||
|
||||
@@ -4,6 +4,7 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
@@ -244,6 +245,28 @@ class TestChecks(unittest.TestCase):
|
||||
finally:
|
||||
stop.set()
|
||||
|
||||
def test_fx_lights_midi_on_the_tracks_mac(self):
|
||||
engine = sp.Engine(CONFIG)
|
||||
self.assertEqual(engine.step(self.snap, self.now)["midi"]["x"], "MIDI\nOK") # not read yet: no verdict
|
||||
self.snap.remote_midi["tracks"] = {"NA Tracks": {"port": 5004, "peers": [
|
||||
{"name": "SuperRack-LiveBox", "address": "", "state": "running"},
|
||||
{"name": "Lighting PC", "address": "", "state": "running"}]}}
|
||||
self.assertEqual(engine.step(self.snap, self.now)["midi"]["x"], "MIDI\nOK")
|
||||
self.snap.remote_midi["tracks"]["NA Tracks"]["peers"].pop()
|
||||
tiles, _ = run_steps(engine, self.snap, self.now, 4)
|
||||
self.assertEqual(tiles["midi"]["x"], "MIDI\nFX LIGHTS MIDI LOST")
|
||||
self.snap.remote_midi["tracks"] = None # Tracks Mac unreachable: its ping says so
|
||||
tiles, _ = run_steps(engine, self.snap, self.now + 5, 1)
|
||||
self.assertEqual(tiles["midi"]["x"], "MIDI\nOK")
|
||||
|
||||
def test_remote_midi_script_runs_coremidi(self):
|
||||
r = subprocess.run(["/usr/bin/python3", "-"], input=sp.remote_midi_script(),
|
||||
capture_output=True, text=True, timeout=30)
|
||||
self.assertEqual(r.returncode, 0, r.stderr)
|
||||
self.assertIsInstance(sp.parse_remote_midi(r.stdout), dict)
|
||||
self.assertIsNone(sp.parse_remote_midi("ssh: connect to host timed out"))
|
||||
self.assertEqual(sp.remote_midi_hosts(CONFIG), ["tracks"])
|
||||
|
||||
def test_midi_peer_joining_is_not_an_error(self):
|
||||
for p in self.snap.midi["NA Ingest"]["peers"]:
|
||||
p["state"] = "connected" # joined, clock sync not finished yet
|
||||
@@ -932,12 +955,14 @@ class TestAgainstFakeCompanion(unittest.TestCase):
|
||||
store["conns"]["ATEM"]["category"] = "error"
|
||||
store["custom"]["sp_cmd"] = ";ack atem"
|
||||
data_dir = tempfile.mkdtemp()
|
||||
originals = sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote, sp.poll_dante_map
|
||||
originals = (sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote, sp.poll_dante_map,
|
||||
sp.poll_remote_midi)
|
||||
sp.poll_dante_map = lambda *a, **k: None # never log in to a switch from a test
|
||||
sp.poll_mdns = lambda snap, seconds=3: None
|
||||
sp.poll_system = lambda snap, cfg: None
|
||||
sp.poll_midi = lambda snap, cfg: None # CoreMIDI has its own test
|
||||
sp.poll_remote = lambda snap, cfg: None # never SSH anywhere from a test
|
||||
sp.poll_remote_midi = lambda snap, cfg: None
|
||||
stop = threading.Event()
|
||||
loop = threading.Thread(target=sp.run, args=(cfg, data_dir), kwargs={"stop": stop}, daemon=True)
|
||||
try:
|
||||
@@ -953,7 +978,8 @@ class TestAgainstFakeCompanion(unittest.TestCase):
|
||||
finally:
|
||||
stop.set()
|
||||
loop.join(timeout=15)
|
||||
sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote, sp.poll_dante_map = originals
|
||||
(sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote, sp.poll_dante_map,
|
||||
sp.poll_remote_midi) = originals
|
||||
store["conns"]["ATEM"]["category"] = "good"
|
||||
self.assertFalse(loop.is_alive()) # it shuts down cleanly
|
||||
self.assertTrue(store["custom"]["sp_hb"].isdigit())
|
||||
|
||||
Reference in New Issue
Block a user