christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Hold MIDI reconnects a dropped peer from the Mac side

Browse Source 
macOS's MIDINetworkSession API only reaches the default session (blank and
disabled on NA Ingest and NA Tracks), so coremidi.connect_peer writes the
named session's "apple.midirtp.session" settings with the peer added, as
Audio MIDI Setup's Connect does. Existing peers are passed back unchanged.
The watchdog remembers each peer's last address (config "address" is the
fallback) and runs it locally for NA Ingest's session or over SSH on the
Tracks Mac for NA Tracks' (FX lights). Checked live: both Macs answer
"already" while the Lighting PC is connected.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-08 15:54:53 -04:00
co-authored by Claude Opus 5.5
parent 74fbe73a4b
commit 80fd89d03b
5 changed files with 161 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+1 -1
View File
@@ -50,7 +50,7 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl
| APPS: VISTA SILENT | starts Vista on the Lighting PC if it is not running. A running Vista is never closed. |
| ARENA | starts Resolume Arena on the Lighting PC if it is not running, then reconnects Companion |
| OBS | starts OBS on the LiveStream PC if it is not running, then reconnects Companion. If the PC is off, it is woken (Wake-on-LAN) and the button shows WAKING PC; hold again once someone has logged in. |
| MIDI: LIGHT PC | no automatic fix. Restarting rtpMIDI drops the PC from every MIDI session and it does not rejoin them, and macOS gives scripts no way to reconnect a named session. Reconnect it by hand in rtpMIDI (session "Lighting PC" → Directory → Connect). **Keep the Lighting PC's network on Public:** its Bonjour firewall rules only allow Public, and on Private it vanishes from the MIDI directory and cannot rejoin a session (2026-10-08). |
| MIDI: LIGHT PC / FX LIGHTS / SUPERRACK MIDI LOST | reconnects that peer to the session it dropped out of, from the Mac's side (NA Ingest's own session, or NA Tracks' over SSH), like Connect in Audio MIDI Setup. Other peers are left alone and nothing is restarted. The address is where the peer was last seen, else `"address"` on the check. Not for a missing Bonjour advert ("... MIDI GONE"). **Keep the Lighting PC's network on Public:** its Bonjour firewall rules only allow Public, and on Private it vanishes from the MIDI directory and cannot rejoin a session (2026-10-08). Never restart rtpMIDI as a fix: it drops every session and does not rejoin them. |
| DANTE: PROP MAC / TRACKS / TRANSL / INGEST GONE | starts Dante Virtual Soundcard's engine on that Mac (`remote-access/mac-dvs-helper.sh`) |
| DANTE: any device powered by its switch port (today the four Studio B boxes) | power-cycles the port it is on **now**, found by its MAC on the Netgear Dante switches |
| DANTE: any other device GONE | restarts Companion's Dante module (hardware: if still gone, it is off or unplugged) |
config.json
+4 -3
View File
@@ -60,6 +60,7 @@
"run": "tasklist /NH /FI \"IMAGENAME eq Arena.exe\" | find /I \"Arena.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Arena\"" },
"vista": { "host": "lighting", "short": "VISTA", "label": "start Vista on the Lighting PC if it is not running (a running Vista is never closed)",
"run": "tasklist /NH /FI \"IMAGENAME eq Vista.exe\" | find /I \"Vista.exe\" >nul || schtasks /Run /TN \"\\StatusPanel\\Start Vista\"" },
"midi_reconnect": { "kind": "midi_connect", "short": "RECONNECT", "label": "reconnect the peer to the network MIDI session it dropped out of (like Connect in Audio MIDI Setup); the address is where it was last seen, else the check's \"address\"" },
"proddash": { "kind": "local", "short": "PRODDASH", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" },
"hyperdeck": { "kind": "hyperdeck", "short": "HYPERDECK", "label": "reconnect Companion to the HyperDeck, or reboot the deck only if it is hung" }
},
@@ -96,9 +97,9 @@
"NA-Tracks": "TRACKS", "FOH-SuperRack-LiveBox": "SUPERRACK", "NA-ProPresenter-Mac": "PROP MAC" } }
] },
{ "id": "midi", "label": "MIDI", "pos": [2, 3], "checks": [
{ "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC" },
{ "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox", "tag": "SUPERRACK" },
{ "type": "midi_peer", "host": "tracks", "session": "NA Tracks", "peer": "Lighting PC", "tag": "FX LIGHTS" },
{ "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC", "fix": "midi_reconnect", "address": "10.1.10.206:5004" },
{ "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox", "tag": "SUPERRACK", "fix": "midi_reconnect", "address": "10.1.13.140:5004" },
{ "type": "midi_peer", "host": "tracks", "session": "NA Tracks", "peer": "Lighting PC", "tag": "FX LIGHTS", "fix": "midi_reconnect", "address": "10.1.10.206:5004" },
{ "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI" },
{ "type": "mdns", "instance": "NA Tracks", "name": "TRACKS MIDI" },
{ "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" }
coremidi.py
+66
View File
@@ -7,6 +7,9 @@ Two things the status panel needs that Companion cannot see:
"apple.midirtp.session" entity property.
* Listener - timestamps the last message that arrived on a named MIDI
source, so a tile can say "last MIDI 3s ago".
* connect_peer() - re-add a peer to a named session, the way Audio MIDI Setup's
"Connect" does (macOS's MIDINetworkSession API only reaches the
default session, not named ones like "NA Tracks").
"""
import ctypes
import ctypes.util
@@ -33,6 +36,7 @@ _cm.MIDIGetSource.argtypes = [ctypes.c_ulong]
_cm.MIDIObjectGetStringProperty.argtypes = [_U32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
_cm.MIDIObjectGetDictionaryProperty.argtypes = [_U32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_void_p)]
_cm.MIDIObjectGetIntegerProperty.argtypes = [_U32, ctypes.c_void_p, ctypes.POINTER(ctypes.c_int32)]
_cm.MIDIObjectSetDictionaryProperty.argtypes = [_U32, ctypes.c_void_p, ctypes.c_void_p]
_cf.CFStringCreateWithCString.restype = ctypes.c_void_p
_cf.CFStringCreateWithCString.argtypes = [ctypes.c_void_p, ctypes.c_char_p, _U32]
@@ -44,6 +48,11 @@ _cf.CFDataGetLength.restype = ctypes.c_long
_cf.CFDataGetLength.argtypes = [ctypes.c_void_p]
_cf.CFDataGetBytePtr.restype = ctypes.c_void_p
_cf.CFDataGetBytePtr.argtypes = [ctypes.c_void_p]
_cf.CFDataCreate.restype = ctypes.c_void_p
_cf.CFDataCreate.argtypes = [ctypes.c_void_p, ctypes.c_char_p, ctypes.c_long]
_cf.CFPropertyListCreateWithData.restype = ctypes.c_void_p
_cf.CFPropertyListCreateWithData.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_ulong,
ctypes.c_void_p, ctypes.c_void_p]
# One lock around every CoreMIDI call: concurrent first calls from two threads
# (session reads vs. MIDIClientCreate) intermittently fail with -2.
@@ -81,6 +90,63 @@ def _dict_prop(obj, key):
return plistlib.loads(raw)
def _set_dict_prop(obj, key, value):
raw = plistlib.dumps(value)
data = _cf.CFDataCreate(None, raw, len(raw))
plist = _cf.CFPropertyListCreateWithData(None, data, 0, None, None)
_cf.CFRelease(data)
if not plist:
raise OSError("could not build the property list")
k = _cfstr(key)
err = _cm.MIDIObjectSetDictionaryProperty(obj, k, plist)
_cf.CFRelease(k)
_cf.CFRelease(plist)
return err
def _session_entities():
"""[(entity, its "apple.midirtp.session" dict)] for every network MIDI session on this Mac."""
out = []
for i in range(_cm.MIDIGetNumberOfDevices()):
dev = _cm.MIDIGetDevice(i)
if _string_prop(dev, "driver") != "com.apple.AppleMIDINetworkDriver":
continue
for j in range(_cm.MIDIDeviceGetNumberOfEntities(dev)):
ent = _cm.MIDIDeviceGetEntity(dev, j)
sess = _dict_prop(ent, "apple.midirtp.session")
if sess is not None:
out.append((ent, sess))
return out
def with_peer(sess, peer_name, address):
"""The session settings with this peer added, or None if it is already in the session.
Peers already there are passed back exactly as read, so they are left alone."""
peers = list(sess.get("peers", []))
if any(p.get("name") == peer_name or p.get("address") == address for p in peers):
return None
new = dict(sess)
new["peers"] = peers + [{"address": address, "name": peer_name}]
return new
def connect_peer(session_name, peer_name, address):
"""Ask macOS to connect `peer_name` at `address` ("10.1.10.206:5004") to the named session,
like Audio MIDI Setup's Connect. Returns "already" (it is in the session) or "asked"."""
with _lock:
for ent, sess in _session_entities():
if (sess.get("dns-sd-name") or _string_prop(ent, "name")) != session_name:
continue
new = with_peer(sess, peer_name, address)
if new is None:
return "already"
err = _set_dict_prop(ent, "apple.midirtp.session", new)
if err:
raise OSError("MIDIObjectSetDictionaryProperty failed: %d" % err)
return "asked"
raise LookupError("no network MIDI session named %r on this Mac" % session_name)
def sessions():
"""{session name: {"port": int, "peers": [{"name", "address", "state"}]}}"""
found = {}
status_panel.py
+45 -5
View File
@@ -134,6 +134,7 @@ class Snapshot:
self.vars = {} # (label, name) -> str or None
self.midi = {} # session name -> {"peers": [...]}
self.remote_midi = {} # host -> the same, read on that Mac over SSH (None: unreachable)
self.peer_addr = {} # (host or "", session, peer) -> address it was last seen at
self.midi_time = 0.0
self.mdns = set() # Bonjour _apple-midi._udp instance names
self.mdns_time = 0.0
@@ -801,14 +802,25 @@ class Engine:
continue
# One run per fix, except a HyperDeck fix, which is per deck. The deck's
# address comes from its Companion connection (remembered if it goes away).
per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck"
fix_kind = self.cfg["fixes"][fix_id].get("kind")
per_deck = fix_kind == "hyperdeck"
target = ""
if per_deck:
ip = chk.get("ip") or snap.var(chk["conn"], "ip") or self.deck_ips.get(chk["conn"])
if not ip:
continue
self.deck_ips[chk["conn"]] = ip
chk = dict(chk, ip=ip)
ident = ("fix", fix_id, chk["conn"] if per_deck else "")
target = chk["conn"]
if fix_kind == "midi_connect":
# Reconnect the peer where it was last seen (or the config's address).
key = (chk.get("host", ""), chk["session"], chk["peer"])
address = snap.peer_addr.get(key) or chk.get("address")
if not address:
continue
chk = dict(chk, address=address)
target = "|".join(key)
ident = ("fix", fix_id, target)
if ident not in seen:
seen.add(ident)
plan.append((ident, "fix", fix_id, chk))
@@ -1125,9 +1137,18 @@ def poll_http(snap, checks):
snap.http = result
def remember_peers(snap, host, sessions):
"""Keep each peer's last address, so a hold can reconnect it after it has gone."""
for name, sess in (sessions or {}).items():
for p in sess.get("peers", []):
if p.get("address"):
snap.peer_addr[(host, name, p["name"])] = p["address"]
def poll_midi(snap, cfg):
import coremidi
snap.midi, snap.midi_time = coremidi.sessions(), time.time()
remember_peers(snap, "", snap.midi)
for t in cfg["tiles"]:
for c in t.get("checks", []):
if c["type"] == "midi_rx":
@@ -1144,12 +1165,12 @@ def remote_midi_hosts(cfg):
if c["type"] == "midi_peer" and c.get("host")})
def remote_midi_script():
"""Python for /usr/bin/python3 on another Mac: this folder's coremidi.py, then print its sessions.
def remote_midi_script(call="ns['sessions']()"):
"""Python for /usr/bin/python3 on another Mac: this folder's coremidi.py, then print `call`'s result.
Nothing is installed there; the code goes over stdin each time."""
src = open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "coremidi.py"), encoding="utf-8").read()
return ("import json\nns = {'__name__': 'spcm'}\nexec(compile(%r, 'coremidi.py', 'exec'), ns)\n"
"print('SPMIDI' + json.dumps(ns['sessions']()))\n" % src)
"print('SPMIDI' + json.dumps(%s))\n" % (src, call))
def parse_remote_midi(out):
@@ -1169,6 +1190,7 @@ def poll_remote_midi(snap, cfg):
r = subprocess.run(ssh_cmd(cfg, host) + ["/usr/bin/python3", "-"], input=script,
capture_output=True, text=True, timeout=20)
snap.remote_midi[host] = parse_remote_midi(r.stdout) if r.returncode == 0 else None
remember_peers(snap, host, snap.remote_midi[host])
except (OSError, subprocess.SubprocessError):
snap.remote_midi[host] = None
@@ -1371,6 +1393,24 @@ def execute_fix(cfg, fix_id, chk):
ok = chk["conn"] in ids and comp.restart_connection(ids[chk["conn"]])
log.info("FIX %s -> Companion connection %s restarted %s", label, chk["conn"], "ok" if ok else "FAILED")
return r.returncode == 0
if kind == "midi_connect":
# Like Connect in Audio MIDI Setup: add the peer back to the session it dropped out of.
args = (chk["session"], chk["peer"], chk["address"])
if chk.get("host"):
r = subprocess.run(ssh_cmd(cfg, chk["host"]) + ["/usr/bin/python3", "-"],
input=remote_midi_script("ns['connect_peer'](%r, %r, %r)" % args),
capture_output=True, text=True, timeout=30)
result = parse_remote_midi(r.stdout) if r.returncode == 0 else None
if result is None:
log.info("FIX %s -> %s on %s failed: %s", label, chk["peer"], chk["host"],
(r.stdout + r.stderr).strip().replace("\n", " ")[-200:])
return False
else:
import coremidi
result = coremidi.connect_peer(*args)
log.info("FIX %s -> %s at %s, session %s%s: %s", label, chk["peer"], chk["address"], chk["session"],
(" on " + chk["host"]) if chk.get("host") else "", result)
return result in ("asked", "already")
if kind == "netgear_poe":
# Netgear AV switch on the Dante network: power-cycle one PoE port (netgear.py).
import netgear
tests/test_status_panel.py
+45 -5
View File
@@ -527,11 +527,52 @@ class TestFixes(unittest.TestCase):
self.assertEqual(self.plan("obs"), [("fix", "obs")])
self.assertEqual(self.plan("arena"), [("fix", "arena")])
self.assertEqual(self.plan("apps"), [("fix", "vista")])
# Restarting rtpMIDI drops every session and it does not rejoin them (2026-10-08): no auto fix.
self.assertEqual(self.plan("midi"), [])
# Restarting rtpMIDI drops every session and does not rejoin them (2026-10-08): the hold
# reconnects the peer from the Mac's side instead, like Connect in Audio MIDI Setup.
self.assertEqual(self.plan("midi"), [("fix", "midi_reconnect")])
for fix_id in ("obs", "arena", "vista"): # never closes a running app
self.assertTrue(CONFIG["fixes"][fix_id]["run"].startswith("tasklist "), fix_id)
def test_midi_reconnect_where_the_peer_was_last_seen(self):
tracks = {"NA Tracks": {"port": 5004, "peers": [
{"name": "Lighting PC", "address": "10.1.10.77:5004", "state": "running"}]}}
sp.remember_peers(self.snap, "tracks", tracks) # seen at a new address earlier
tracks["NA Tracks"]["peers"] = []
self.snap.remote_midi["tracks"] = tracks
self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"]
if p["name"] != "Lighting PC"]
run_steps(self.engine, self.snap, SUNDAY, 4)
plan = self.engine.fix_plan("midi", self.snap)
got = sorted((c.get("host", ""), c["session"], c["address"]) for _, kind, what, c in plan)
self.assertEqual(got, [("", "NA Ingest", "10.1.10.206:5004"), # never seen: config address
("tracks", "NA Tracks", "10.1.10.77:5004")]) # seen: last address
self.assertEqual(self.plan("show"), [("fix", "midi_reconnect")] * 2) # one per session
def test_midi_reconnect_runs_on_the_right_mac(self):
from unittest import mock
chk = {"type": "midi_peer", "host": "tracks", "session": "NA Tracks", "peer": "Lighting PC",
"address": "10.1.10.206:5004"}
done = mock.Mock(returncode=0, stdout='SPMIDI"asked"\n', stderr="")
with mock.patch.object(sp.subprocess, "run", return_value=done) as run:
self.assertTrue(sp.execute_fix(CONFIG, "midi_reconnect", chk))
argv, kw = run.call_args.args[0], run.call_args.kwargs
self.assertIn("waterschurchmedia@10.1.3.227", argv)
self.assertIn("ns['connect_peer']('NA Tracks', 'Lighting PC', '10.1.10.206:5004')", kw["input"])
failed = mock.Mock(returncode=255, stdout="", stderr="ssh: timed out")
with mock.patch.object(sp.subprocess, "run", return_value=failed):
self.assertFalse(sp.execute_fix(CONFIG, "midi_reconnect", chk))
def test_reconnect_leaves_other_peers_alone(self):
import coremidi
sess = {"dns-sd-name": "NA Tracks", "flags": 1, "join-policy": 2, "port": 5004, "peers": [
{"address": "10.1.13.140:5004", "name": "SuperRack-LiveBox", "state": "running"},
{"address": "\tStudio B Mac Studio\tlocal.", "name": "Studio B Mac Studio", "state": "running"}]}
new = coremidi.with_peer(sess, "Lighting PC", "10.1.10.206:5004")
self.assertEqual(new["peers"][:2], sess["peers"])
self.assertEqual(new["peers"][2], {"address": "10.1.10.206:5004", "name": "Lighting PC"})
self.assertEqual({k: v for k, v in new.items() if k != "peers"}, {k: v for k, v in sess.items() if k != "peers"})
self.assertIsNone(coremidi.with_peer(new, "Lighting PC", "10.1.10.206:5004")) # already there
def test_an_off_pc_is_woken(self):
from unittest import mock
woke = []
@@ -711,8 +752,7 @@ class TestFixes(unittest.TestCase):
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices)
self.assertEqual(self.plan("dante"), [("fix", "dvs_ingest")])
# A failure with no fix must not claim there is nothing wrong.
self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"]
if p["name"] != "SuperRack-LiveBox"]
self.snap.mdns = self.snap.mdns - {"NA Tracks"} # an advert gone: nothing to reconnect
_, t = run_steps(self.engine, self.snap, SUNDAY, 4)
self.engine.run_fixes("midi", self.snap, None, t)
self.assertEqual(self.engine.notes["midi"]["text"], "NO AUTO\nFIX")
@@ -733,7 +773,7 @@ class TestFixes(unittest.TestCase):
def test_every_fix_is_valid(self):
for f in CONFIG["fixes"].values():
self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck"))
self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck", "midi_connect"))
class FakeDeck: