From 80fd89d03bdaa14d4fba868fdac60254b46ea090 Mon Sep 17 00:00:00 2001 From: christian Date: Thu, 8 Oct 2026 15:54:53 -0400 Subject: [PATCH] Hold MIDI reconnects a dropped peer from the Mac side macOS's MIDINetworkSession API only reaches the default session (blank and disabled on NA Ingest and NA Tracks), so coremidi.connect_peer writes the named session's "apple.midirtp.session" settings with the peer added, as Audio MIDI Setup's Connect does. Existing peers are passed back unchanged. The watchdog remembers each peer's last address (config "address" is the fallback) and runs it locally for NA Ingest's session or over SSH on the Tracks Mac for NA Tracks' (FX lights). Checked live: both Macs answer "already" while the Lighting PC is connected. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- config.json | 7 ++-- coremidi.py | 66 ++++++++++++++++++++++++++++++++++++++ status_panel.py | 50 ++++++++++++++++++++++++++--- tests/test_status_panel.py | 50 ++++++++++++++++++++++++++--- 5 files changed, 161 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 1374753..4aa901f 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl | APPS: VISTA SILENT | starts Vista on the Lighting PC if it is not running. A running Vista is never closed. | | ARENA | starts Resolume Arena on the Lighting PC if it is not running, then reconnects Companion | | OBS | starts OBS on the LiveStream PC if it is not running, then reconnects Companion. If the PC is off, it is woken (Wake-on-LAN) and the button shows WAKING PC; hold again once someone has logged in. | - | MIDI: LIGHT PC | no automatic fix. Restarting rtpMIDI drops the PC from every MIDI session and it does not rejoin them, and macOS gives scripts no way to reconnect a named session. Reconnect it by hand in rtpMIDI (session "Lighting PC" → Directory → Connect). **Keep the Lighting PC's network on Public:** its Bonjour firewall rules only allow Public, and on Private it vanishes from the MIDI directory and cannot rejoin a session (2026-10-08). | + | MIDI: LIGHT PC / FX LIGHTS / SUPERRACK MIDI LOST | reconnects that peer to the session it dropped out of, from the Mac's side (NA Ingest's own session, or NA Tracks' over SSH), like Connect in Audio MIDI Setup. Other peers are left alone and nothing is restarted. The address is where the peer was last seen, else `"address"` on the check. Not for a missing Bonjour advert ("... MIDI GONE"). **Keep the Lighting PC's network on Public:** its Bonjour firewall rules only allow Public, and on Private it vanishes from the MIDI directory and cannot rejoin a session (2026-10-08). Never restart rtpMIDI as a fix: it drops every session and does not rejoin them. | | DANTE: PROP MAC / TRACKS / TRANSL / INGEST GONE | starts Dante Virtual Soundcard's engine on that Mac (`remote-access/mac-dvs-helper.sh`) | | DANTE: any device powered by its switch port (today the four Studio B boxes) | power-cycles the port it is on **now**, found by its MAC on the Netgear Dante switches | | DANTE: any other device GONE | restarts Companion's Dante module (hardware: if still gone, it is off or unplugged) | diff --git a/config.json b/config.json index 3ae1cec..7a8f1d6 100644 --- a/config.json +++ b/config.json @@ -60,6 +60,7 @@ "run": "tasklist /NH /FI \"IMAGENAME eq Arena.exe\" | find /I \"Arena.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Arena\"" }, "vista": { "host": "lighting", "short": "VISTA", "label": "start Vista on the Lighting PC if it is not running (a running Vista is never closed)", "run": "tasklist /NH /FI \"IMAGENAME eq Vista.exe\" | find /I \"Vista.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Vista\"" }, + "midi_reconnect": { "kind": "midi_connect", "short": "RECONNECT", "label": "reconnect the peer to the network MIDI session it dropped out of (like Connect in Audio MIDI Setup); the address is where it was last seen, else the check's \"address\"" }, "proddash": { "kind": "local", "short": "PRODDASH", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" }, "hyperdeck": { "kind": "hyperdeck", "short": "HYPERDECK", "label": "reconnect Companion to the HyperDeck, or reboot the deck only if it is hung" } }, @@ -96,9 +97,9 @@ "NA-Tracks": "TRACKS", "FOH-SuperRack-LiveBox": "SUPERRACK", "NA-ProPresenter-Mac": "PROP MAC" } } ] }, { "id": "midi", "label": "MIDI", "pos": [2, 3], "checks": [ - { "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC" }, - { "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox", "tag": "SUPERRACK" }, - { "type": "midi_peer", "host": "tracks", "session": "NA Tracks", "peer": "Lighting PC", "tag": "FX LIGHTS" }, + { "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC", "fix": "midi_reconnect", "address": "10.1.10.206:5004" }, + { "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox", "tag": "SUPERRACK", "fix": "midi_reconnect", "address": "10.1.13.140:5004" }, + { "type": "midi_peer", "host": "tracks", "session": "NA Tracks", "peer": "Lighting PC", "tag": "FX LIGHTS", "fix": "midi_reconnect", "address": "10.1.10.206:5004" }, { "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI" }, { "type": "mdns", "instance": "NA Tracks", "name": "TRACKS MIDI" }, { "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" } diff --git a/coremidi.py b/coremidi.py index ce70553..c6e51e8 100644 --- a/coremidi.py +++ b/coremidi.py @@ -7,6 +7,9 @@ Two things the status panel needs that Companion cannot see: "apple.midirtp.session" entity property. * Listener - timestamps the last message that arrived on a named MIDI source, so a tile can say "last MIDI 3s ago". +* connect_peer() - re-add a peer to a named session, the way Audio MIDI Setup's + "Connect" does (macOS's MIDINetworkSession API only reaches the + default session, not named ones like "NA Tracks"). """ import ctypes import ctypes.util @@ -33,6 +36,7 @@ _cm.MIDIGetSource.argtypes = [ctypes.c_ulong] _cm.MIDIObjectGetStringProperty.argtypes = [_U32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)] _cm.MIDIObjectGetDictionaryProperty.argtypes = [_U32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)] _cm.MIDIObjectGetIntegerProperty.argtypes = [_U32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_int32)] +_cm.MIDIObjectSetDictionaryProperty.argtypes = [_U32, ctypes.c_void_p, ctypes.c_void_p] _cf.CFStringCreateWithCString.restype = ctypes.c_void_p _cf.CFStringCreateWithCString.argtypes = [ctypes.c_void_p, ctypes.c_char_p, _U32] @@ -44,6 +48,11 @@ _cf.CFDataGetLength.restype = ctypes.c_long _cf.CFDataGetLength.argtypes = [ctypes.c_void_p] _cf.CFDataGetBytePtr.restype = ctypes.c_void_p _cf.CFDataGetBytePtr.argtypes = [ctypes.c_void_p] +_cf.CFDataCreate.restype = ctypes.c_void_p +_cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long] +_cf.CFPropertyListCreateWithData.restype = ctypes.c_void_p +_cf.CFPropertyListCreateWithData.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_ulong, + ctypes.c_void_p, ctypes.c_void_p] # One lock around every CoreMIDI call: concurrent first calls from two threads # (session reads vs. MIDIClientCreate) intermittently fail with -2. @@ -81,6 +90,63 @@ def _dict_prop(obj, key): return plistlib.loads(raw) +def _set_dict_prop(obj, key, value): + raw = plistlib.dumps(value) + data = _cf.CFDataCreate(None, raw, len(raw)) + plist = _cf.CFPropertyListCreateWithData(None, data, 0, None, None) + _cf.CFRelease(data) + if not plist: + raise OSError("could not build the property list") + k = _cfstr(key) + err = _cm.MIDIObjectSetDictionaryProperty(obj, k, plist) + _cf.CFRelease(k) + _cf.CFRelease(plist) + return err + + +def _session_entities(): + """[(entity, its "apple.midirtp.session" dict)] for every network MIDI session on this Mac.""" + out = [] + for i in range(_cm.MIDIGetNumberOfDevices()): + dev = _cm.MIDIGetDevice(i) + if _string_prop(dev, "driver") != "com.apple.AppleMIDINetworkDriver": + continue + for j in range(_cm.MIDIDeviceGetNumberOfEntities(dev)): + ent = _cm.MIDIDeviceGetEntity(dev, j) + sess = _dict_prop(ent, "apple.midirtp.session") + if sess is not None: + out.append((ent, sess)) + return out + + +def with_peer(sess, peer_name, address): + """The session settings with this peer added, or None if it is already in the session. + Peers already there are passed back exactly as read, so they are left alone.""" + peers = list(sess.get("peers", [])) + if any(p.get("name") == peer_name or p.get("address") == address for p in peers): + return None + new = dict(sess) + new["peers"] = peers + [{"address": address, "name": peer_name}] + return new + + +def connect_peer(session_name, peer_name, address): + """Ask macOS to connect `peer_name` at `address` ("10.1.10.206:5004") to the named session, + like Audio MIDI Setup's Connect. Returns "already" (it is in the session) or "asked".""" + with _lock: + for ent, sess in _session_entities(): + if (sess.get("dns-sd-name") or _string_prop(ent, "name")) != session_name: + continue + new = with_peer(sess, peer_name, address) + if new is None: + return "already" + err = _set_dict_prop(ent, "apple.midirtp.session", new) + if err: + raise OSError("MIDIObjectSetDictionaryProperty failed: %d" % err) + return "asked" + raise LookupError("no network MIDI session named %r on this Mac" % session_name) + + def sessions(): """{session name: {"port": int, "peers": [{"name", "address", "state"}]}}""" found = {} diff --git a/status_panel.py b/status_panel.py index e70772b..e572823 100644 --- a/status_panel.py +++ b/status_panel.py @@ -134,6 +134,7 @@ class Snapshot: self.vars = {} # (label, name) -> str or None self.midi = {} # session name -> {"peers": [...]} self.remote_midi = {} # host -> the same, read on that Mac over SSH (None: unreachable) + self.peer_addr = {} # (host or "", session, peer) -> address it was last seen at self.midi_time = 0.0 self.mdns = set() # Bonjour _apple-midi._udp instance names self.mdns_time = 0.0 @@ -801,14 +802,25 @@ class Engine: continue # One run per fix, except a HyperDeck fix, which is per deck. The deck's # address comes from its Companion connection (remembered if it goes away). - per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck" + fix_kind = self.cfg["fixes"][fix_id].get("kind") + per_deck = fix_kind == "hyperdeck" + target = "" if per_deck: ip = chk.get("ip") or snap.var(chk["conn"], "ip") or self.deck_ips.get(chk["conn"]) if not ip: continue self.deck_ips[chk["conn"]] = ip chk = dict(chk, ip=ip) - ident = ("fix", fix_id, chk["conn"] if per_deck else "") + target = chk["conn"] + if fix_kind == "midi_connect": + # Reconnect the peer where it was last seen (or the config's address). + key = (chk.get("host", ""), chk["session"], chk["peer"]) + address = snap.peer_addr.get(key) or chk.get("address") + if not address: + continue + chk = dict(chk, address=address) + target = "|".join(key) + ident = ("fix", fix_id, target) if ident not in seen: seen.add(ident) plan.append((ident, "fix", fix_id, chk)) @@ -1125,9 +1137,18 @@ def poll_http(snap, checks): snap.http = result +def remember_peers(snap, host, sessions): + """Keep each peer's last address, so a hold can reconnect it after it has gone.""" + for name, sess in (sessions or {}).items(): + for p in sess.get("peers", []): + if p.get("address"): + snap.peer_addr[(host, name, p["name"])] = p["address"] + + def poll_midi(snap, cfg): import coremidi snap.midi, snap.midi_time = coremidi.sessions(), time.time() + remember_peers(snap, "", snap.midi) for t in cfg["tiles"]: for c in t.get("checks", []): if c["type"] == "midi_rx": @@ -1144,12 +1165,12 @@ def remote_midi_hosts(cfg): if c["type"] == "midi_peer" and c.get("host")}) -def remote_midi_script(): - """Python for /usr/bin/python3 on another Mac: this folder's coremidi.py, then print its sessions. +def remote_midi_script(call="ns['sessions']()"): + """Python for /usr/bin/python3 on another Mac: this folder's coremidi.py, then print `call`'s result. Nothing is installed there; the code goes over stdin each time.""" src = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "coremidi.py"), encoding="utf-8").read() return ("import json\nns = {'__name__': 'spcm'}\nexec(compile(%r, 'coremidi.py', 'exec'), ns)\n" - "print('SPMIDI' + json.dumps(ns['sessions']()))\n" % src) + "print('SPMIDI' + json.dumps(%s))\n" % (src, call)) def parse_remote_midi(out): @@ -1169,6 +1190,7 @@ def poll_remote_midi(snap, cfg): r = subprocess.run(ssh_cmd(cfg, host) + ["/usr/bin/python3", "-"], input=script, capture_output=True, text=True, timeout=20) snap.remote_midi[host] = parse_remote_midi(r.stdout) if r.returncode == 0 else None + remember_peers(snap, host, snap.remote_midi[host]) except (OSError, subprocess.SubprocessError): snap.remote_midi[host] = None @@ -1371,6 +1393,24 @@ def execute_fix(cfg, fix_id, chk): ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]]) log.info("FIX %s -> Companion connection %s restarted %s", label, chk["conn"], "ok" if ok else "FAILED") return r.returncode == 0 + if kind == "midi_connect": + # Like Connect in Audio MIDI Setup: add the peer back to the session it dropped out of. + args = (chk["session"], chk["peer"], chk["address"]) + if chk.get("host"): + r = subprocess.run(ssh_cmd(cfg, chk["host"]) + ["/usr/bin/python3", "-"], + input=remote_midi_script("ns['connect_peer'](%r, %r, %r)" % args), + capture_output=True, text=True, timeout=30) + result = parse_remote_midi(r.stdout) if r.returncode == 0 else None + if result is None: + log.info("FIX %s -> %s on %s failed: %s", label, chk["peer"], chk["host"], + (r.stdout + r.stderr).strip().replace("\n", " ")[-200:]) + return False + else: + import coremidi + result = coremidi.connect_peer(*args) + log.info("FIX %s -> %s at %s, session %s%s: %s", label, chk["peer"], chk["address"], chk["session"], + (" on " + chk["host"]) if chk.get("host") else "", result) + return result in ("asked", "already") if kind == "netgear_poe": # Netgear AV switch on the Dante network: power-cycle one PoE port (netgear.py). import netgear diff --git a/tests/test_status_panel.py b/tests/test_status_panel.py index 3152ddf..87ef39d 100644 --- a/tests/test_status_panel.py +++ b/tests/test_status_panel.py @@ -527,11 +527,52 @@ class TestFixes(unittest.TestCase): self.assertEqual(self.plan("obs"), [("fix", "obs")]) self.assertEqual(self.plan("arena"), [("fix", "arena")]) self.assertEqual(self.plan("apps"), [("fix", "vista")]) - # Restarting rtpMIDI drops every session and it does not rejoin them (2026-10-08): no auto fix. - self.assertEqual(self.plan("midi"), []) + # Restarting rtpMIDI drops every session and does not rejoin them (2026-10-08): the hold + # reconnects the peer from the Mac's side instead, like Connect in Audio MIDI Setup. + self.assertEqual(self.plan("midi"), [("fix", "midi_reconnect")]) for fix_id in ("obs", "arena", "vista"): # never closes a running app self.assertTrue(CONFIG["fixes"][fix_id]["run"].startswith("tasklist "), fix_id) + def test_midi_reconnect_where_the_peer_was_last_seen(self): + tracks = {"NA Tracks": {"port": 5004, "peers": [ + {"name": "Lighting PC", "address": "10.1.10.77:5004", "state": "running"}]}} + sp.remember_peers(self.snap, "tracks", tracks) # seen at a new address earlier + tracks["NA Tracks"]["peers"] = [] + self.snap.remote_midi["tracks"] = tracks + self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"] + if p["name"] != "Lighting PC"] + run_steps(self.engine, self.snap, SUNDAY, 4) + plan = self.engine.fix_plan("midi", self.snap) + got = sorted((c.get("host", ""), c["session"], c["address"]) for _, kind, what, c in plan) + self.assertEqual(got, [("", "NA Ingest", "10.1.10.206:5004"), # never seen: config address + ("tracks", "NA Tracks", "10.1.10.77:5004")]) # seen: last address + self.assertEqual(self.plan("show"), [("fix", "midi_reconnect")] * 2) # one per session + + def test_midi_reconnect_runs_on_the_right_mac(self): + from unittest import mock + chk = {"type": "midi_peer", "host": "tracks", "session": "NA Tracks", "peer": "Lighting PC", + "address": "10.1.10.206:5004"} + done = mock.Mock(returncode=0, stdout='SPMIDI"asked"\n', stderr="") + with mock.patch.object(sp.subprocess, "run", return_value=done) as run: + self.assertTrue(sp.execute_fix(CONFIG, "midi_reconnect", chk)) + argv, kw = run.call_args.args[0], run.call_args.kwargs + self.assertIn("waterschurchmedia@10.1.3.227", argv) + self.assertIn("ns['connect_peer']('NA Tracks', 'Lighting PC', '10.1.10.206:5004')", kw["input"]) + failed = mock.Mock(returncode=255, stdout="", stderr="ssh: timed out") + with mock.patch.object(sp.subprocess, "run", return_value=failed): + self.assertFalse(sp.execute_fix(CONFIG, "midi_reconnect", chk)) + + def test_reconnect_leaves_other_peers_alone(self): + import coremidi + sess = {"dns-sd-name": "NA Tracks", "flags": 1, "join-policy": 2, "port": 5004, "peers": [ + {"address": "10.1.13.140:5004", "name": "SuperRack-LiveBox", "state": "running"}, + {"address": "\tStudio B Mac Studio\tlocal.", "name": "Studio B Mac Studio", "state": "running"}]} + new = coremidi.with_peer(sess, "Lighting PC", "10.1.10.206:5004") + self.assertEqual(new["peers"][:2], sess["peers"]) + self.assertEqual(new["peers"][2], {"address": "10.1.10.206:5004", "name": "Lighting PC"}) + self.assertEqual({k: v for k, v in new.items() if k != "peers"}, {k: v for k, v in sess.items() if k != "peers"}) + self.assertIsNone(coremidi.with_peer(new, "Lighting PC", "10.1.10.206:5004")) # already there + def test_an_off_pc_is_woken(self): from unittest import mock woke = [] @@ -711,8 +752,7 @@ class TestFixes(unittest.TestCase): self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices) self.assertEqual(self.plan("dante"), [("fix", "dvs_ingest")]) # A failure with no fix must not claim there is nothing wrong. - self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"] - if p["name"] != "SuperRack-LiveBox"] + self.snap.mdns = self.snap.mdns - {"NA Tracks"} # an advert gone: nothing to reconnect _, t = run_steps(self.engine, self.snap, SUNDAY, 4) self.engine.run_fixes("midi", self.snap, None, t) self.assertEqual(self.engine.notes["midi"]["text"], "NO AUTO\nFIX") @@ -733,7 +773,7 @@ class TestFixes(unittest.TestCase): def test_every_fix_is_valid(self): for f in CONFIG["fixes"].values(): - self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck")) + self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck", "midi_connect")) class FakeDeck: