Hold Woonsocket's keys from NA; MIDI links moved and fixed on Macs without python; one check per link

- NA's CAMPUS page (99): tap = acknowledge, hold = fix, at the campus shown. NA's settings page turns the cell into that campus's key and writes the command into that campus's Companion (sp_cmd), exactly as a press on its own deck does. Logged on both sides. - coremidi.JXA_CONNECT: connect_peer in JavaScript for Automation, used when a Mac has no python (no Command Line Tools): a MIDI hold now reconnects Woonsocket's Tracks, FOH and Lights. - Woonsocket's MIDI links are on wired addresses now (Tracks to Lighting 10.2.2.2, Tracks to FOH 10.2.1.12; the other ends list Tracks at 10.2.1.219), and the wired-network rule is back on there. - Auto-setup keeps one check per MIDI link (the end that reports the link's state): two pulled links read 4 DOWN before. Tried live: a hold sent from NA's page 99 reconnected the FOH link at Woonsocket. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-20 17:00:31 -04:00
parent 457335bc96
commit b793490d90
5 changed files with 119 additions and 7 deletions
@@ -229,6 +229,49 @@ for (var i = 0; i < n; i++) {
 """


+
+# Connect, for a Mac without python: what connect_peer does, in JavaScript for Automation. Filled in
+# with jxa_connect(); prints SPMIDI"already" / "asked" / "kicked" / "moved", like connect_peer.
+JXA_CONNECT = r"""
+ObjC.import('CoreMIDI'); ObjC.import('Foundation');
+var SESSION = __SESSION__, PEER = __PEER__, ADDRESS = __ADDRESS__, REPOINT = __REPOINT__;
+function str(o, k) { var r = Ref(); if ($.MIDIObjectGetStringProperty(o, k, r) !== 0) return null; return ObjC.unwrap(ObjC.castRefToObject(r[0])); }
+function dict(o, k) { var r = Ref(); if ($.MIDIObjectGetDictionaryProperty(o, k, r) !== 0) return null; return ObjC.deepUnwrap(ObjC.castRefToObject(r[0])); }
+function put(ent, s) { var e = $.MIDIObjectSetDictionaryProperty(ent, $('apple.midirtp.session'), $(s)); if (e !== 0) throw new Error('MIDIObjectSetDictionaryProperty failed: ' + e); }
+var found = null, n = $.MIDIGetNumberOfDevices();
+for (var i = 0; i < n && !found; i++) {
+  var dev = $.MIDIGetDevice(i), drv = str(dev, $('driver')) || '';
+  if (drv.indexOf('Network') < 0 && drv.toLowerCase().indexOf('rtp') < 0) continue;
+  for (var j = 0; j < $.MIDIDeviceGetNumberOfEntities(dev); j++) {
+    var ent = $.MIDIDeviceGetEntity(dev, j), s = dict(ent, $('apple.midirtp.session'));
+    if (s && (s['dns-sd-name'] || str(ent, $('name'))) === SESSION) { found = [ent, s]; break; }
+  }
+}
+if (!found) throw new Error('no network MIDI session named ' + SESSION + ' on this Mac');
+var ent = found[0], s = found[1], peers = s.peers || [], result;
+var mine = peers.filter(function (p) { return p.name === PEER; });
+var stuck = mine.some(function (p) { return p.state !== undefined && p.state !== 'running' && p.state !== 'connected'; });
+var elsewhere = mine.some(function (p) { return p.address !== ADDRESS; });
+if (mine.length && (stuck || (REPOINT && elsewhere))) {
+  s.peers = peers.filter(function (p) { return p.name !== PEER; });
+  put(ent, s); delay(1);
+  s.peers = s.peers.concat([{address: ADDRESS, name: PEER}]); put(ent, s);
+  result = (REPOINT && elsewhere) ? 'moved' : 'kicked';
+} else if (mine.length || peers.some(function (p) { return p.address === ADDRESS; })) {
+  result = 'already';
+} else {
+  s.peers = peers.concat([{address: ADDRESS, name: PEER}]); put(ent, s); result = 'asked';
+}
+'SPMIDI' + JSON.stringify(result);
+"""
+
+
+def jxa_connect(session_name, peer_name, address, repoint=False):
+    """The JXA_CONNECT script for one peer (values go in as JSON, never as code)."""
+    import json as _json
+    return (JXA_CONNECT.replace("__SESSION__", _json.dumps(session_name)).replace("__PEER__", _json.dumps(peer_name))
+            .replace("__ADDRESS__", _json.dumps(address)).replace("__REPOINT__", "true" if repoint else "false"))
+
 def sessions():
    """{session name: {"port": int, "peers": [{"name", "address", "state"}]}}"""
    found = {}
@@ -287,6 +287,21 @@ def items(inv, nets):
                if host:
                    chk["host"] = host
                midi_checks.append(("midi:%s|%s|%s" % (host, sname, p["name"]), chk))
+    # One check per link, not one per end (2026-10-20: two links pulled at Woonsocket read "4 DOWN"):
+    # keep the end whose Mac says whether the link is live (newer macOS reports a peer's state).
+    def end_of(host):
+        return host or "ingest"
+
+    def reports_state(host, sname):
+        return any("state" in p for p in (((inv.get("midi") or {}).get(host) or {}).get(sname) or {}).get("peers") or [])
+    by_pair = {}
+    for ident, c in midi_checks:
+        other = owner.get(c["peer"]) or computer_by_ip(inv, ip_of(c["address"])) or c["peer"]
+        pair = frozenset((end_of(c.get("host", "")), other))
+        best = by_pair.get(pair)
+        if best is None or (reports_state(c.get("host", ""), c["session"]) and not reports_state(best[1].get("host", ""), best[1]["session"])):
+            by_pair[pair] = (ident, c)
+    midi_checks = [x for x in midi_checks if x in by_pair.values()]
    if midi_checks:
        checks = [c for _, c in midi_checks] + [{"type": "midi_wired", "nets": list(nets), "fix": "midi_reconnect"}]
        out.append({"ident": "midi_wired", "group": "SHOW", "label": "MIDI", "checks": checks,
@@ -18,12 +18,14 @@ where nothing a person made sits (push_page.safe_target): an empty page, else on
 end. State: campuses.json in the data folder.
 """
 import gzip
+import logging
 import hashlib
 import json
 import os
 import tempfile
 import threading
 import time
+import urllib.parse
 import urllib.request

 from settings import PARENT  # noqa: F401  (puts the project folder on sys.path)
@@ -33,6 +35,7 @@ import push_page
 import status_panel as sp

 FILE = "campuses.json"
+log = logging.getLogger("status-settings")    # the settings page's own log (status-settings.log)
 POLL, REFRESH = 2.0, 60.0
 SHORT = {"na": "NA", "ab": "AB", "sc": "SC", "ws": "WS"}
 MIRROR_MARK = "STATUS tile"            # copied keys keep the tile marker, so a later rebuild may overwrite them
@@ -90,6 +93,7 @@ def http_json(url, body=None, timeout=3):

 VIEW = {"state": "sp_state_view", "heartbeat": "sp_hb_view"}
 SELECT = "sp_view"
+VIEW_CMD = "sp_view_cmd"       # a tap or hold on the CAMPUS page, passed on to that campus's own watchdog
 CELLS = [(r, c) for r in range(4) for c in range(1, 8)]


@@ -108,16 +112,22 @@ def view_state(state, keys):


 def build_view(local_cfg, page, overview_page):
-    """The CAMPUS page: a key per cell, drawn from sp_state_view; the ALARMS corner goes back."""
+    """The CAMPUS page: a key per cell, drawn from sp_state_view; tap = acknowledge, hold = fix, at
+    that campus (2026-10-20: the user wants to test and fix Woonsocket from NA). The ALARMS corner goes back."""
    opts = make_page.settings(local_cfg)
    controls = make_page.nav_controls(local_cfg)
+    hold_ms = opts["hold_ms"]
    for r, c in CELLS:
        layers = make_page.layers(cell_id(r, c), {}, VIEW, "''", opts)
+        tap = make_page.command_action("view_%d_%d" % (r, c), "press", VIEW_CMD, "tap %d %d $(internal:time_unix)" % (r, c))
+        hold = make_page.command_action("view_%d_%d" % (r, c), "hold", VIEW_CMD, "hold %d %d $(internal:time_unix)" % (r, c))
        make_page.place(controls, (r, c), {
            "type": "button-layered", "style": {"layers": layers},
            "options": {"stepProgression": "auto", "stepExpression": "", "rotaryActions": False, "canModifyStyleInApis": True,
-                        "notes": "STATUS tile 'campus view %d/%d' - the tapped campus's key here, view only (hold it on that campus's own deck)." % (r, c)},
-            "feedbacks": [], "steps": {"0": {"action_sets": {"down": [], "up": []}, "options": {"runWhileHeld": []}}}, "localVariables": []})
+                        "notes": "STATUS tile 'campus view %d/%d' - the tapped campus's key here. Press: acknowledge it there. "
+                                 "Hold %g s: fix it there, like a hold on that campus's own deck." % (r, c, hold_ms / 1000.0)},
+            "feedbacks": [], "steps": {"0": {"action_sets": {"down": [], "up": [tap], str(hold_ms): [hold]},
+                                              "options": {"runWhileHeld": [hold_ms]}}}, "localVariables": []})
    back = make_page.static_button("campus_back", "ALL\\nSITES", 0x37474F, 0xFFFFFF, overview_page,
                                   "Tap: back to the CAMPUSES page (page %d)." % overview_page, "STATUS jump key")
    for layer in back["style"]["layers"]:
@@ -169,7 +179,7 @@ def ensure_variables(cfg, ids):
    from settings import campus
    have, _ = campus.companion_state(cfg)
    want = [n for cid in ids for n in ("sp_state_" + cid, "sp_hb_" + cid) if n not in have]
-    want += [n for n in (VIEW["state"], VIEW["heartbeat"], SELECT) if n not in have]
+    want += [n for n in (VIEW["state"], VIEW["heartbeat"], SELECT, VIEW_CMD) if n not in have]
    if not want:
        return
    comp = cfg.get("companion") or {}
@@ -303,6 +313,24 @@ class Mirror:
        save(self.site.data_dir, data)
        return data

+    def pass_on(self, pick, others):
+        """A tap or hold on the CAMPUS page goes to the shown campus's watchdog, through its Companion."""
+        text = (self.site.comp.get_custom(VIEW_CMD) or "").strip()
+        if not text:
+            return
+        self.site.comp.set_custom(VIEW_CMD, "")
+        cmds = campus_command(self.keys.get(pick), text, time.time()) if pick else []
+        if not cmds:
+            return
+        host = urllib.parse.urlparse(others[pick]["url"]).hostname
+        try:
+            remote = sp.Companion(host, 8000, timeout=3)
+            before = (remote.get_custom("sp_cmd") or "").strip()
+            remote.set_custom("sp_cmd", ";".join([before] + cmds if before else cmds))
+            log.info("CAMPUS %s: %s (from %s's CAMPUS page)", pick, "; ".join(cmds), (self.site.cfg.get("site") or {}).get("name", "here"))
+        except OSError as e:
+            log.warning("CAMPUS %s: could not pass on %s: %s", pick, cmds, e)
+
    def poll(self):
        """The hub: copy each other campus's key states into this Companion."""
        cfg = self.site.load()
@@ -334,6 +362,7 @@ class Mirror:
            # Nothing tapped yet (or a campus that left): show the first one rather than a purple page.
            pick = sorted(others)[0]
            self.site.comp.set_custom(SELECT, pick.upper())
+        self.pass_on(pick if pick in others else None, others)
        st = self.remote.get(pick)
        if st and pick in self.keys and (data.get("pages") or {}).get("view"):
            try:
@@ -344,6 +373,22 @@ class Mirror:
            self.site.comp.set_custom(VIEW["heartbeat"], str(st.get("hb") or "0"))


+def campus_command(keys, text, now):
+    """The CAMPUS page's taps and holds ("tap 1 3 <time>;hold 0 2 <time>") as that campus's own
+    commands ("ack hd1", "restart hd1 <time>"), for the keys at those cells. Stale ones are dropped."""
+    at = {tuple(k["pos"]): k["id"] for k in keys or []}
+    out = []
+    for part in (text or "").split(";"):
+        w = part.split()
+        if len(w) != 4 or w[0] not in ("tap", "hold") or not (w[1].isdigit() and w[2].isdigit() and w[3].isdigit()):
+            continue
+        tid = at.get((int(w[1]), int(w[2])))
+        if not tid or now - int(w[3]) > 15:
+            continue
+        out.append("ack %s" % tid if w[0] == "tap" else "restart %s %s" % (tid, w[3]))
+    return out
+
+
 def loop(site, stop):
    m = Mirror(site)
    site.mirror = m
@@ -2806,6 +2806,10 @@ def execute_fix(cfg, fix_id, chk):
                r = subprocess.run(ssh_cmd(cfg, chk["host"]) + remote_python("-"),
                                   input=remote_midi_script("ns['connect_peer'](%r, %r, %r, repoint=%r)" % (args + (repoint,))),
                                   capture_output=True, text=True, timeout=30)
+                if r.returncode == 97:                        # no python on that Mac: the same, in JavaScript for Automation
+                    import coremidi
+                    r = subprocess.run(ssh_cmd(cfg, chk["host"]) + ["/usr/bin/osascript -l JavaScript -"],
+                                       input=coremidi.jxa_connect(*args, repoint=repoint), capture_output=True, text=True, timeout=30)
                result = parse_remote_midi(r.stdout) if r.returncode == 0 else None
                if result is None:
                    log.info("FIX %s -> %s on %s failed: %s", label, chk["peer"], chk["host"],
@@ -133,8 +133,8 @@ class TestBuild(unittest.TestCase):
        cfg, _ = ac.build(BASE, inv, {"id": "ws", "name": "Woonsocket"}, "1234")
        midi = next(t for t in cfg["tiles"] if t["label"] == "MIDI")
        got = sorted((c.get("host"), c["peer"], c["address"]) for c in midi["checks"] if c["type"] == "midi_peer")
-        self.assertEqual(got, [("foh", "WS Tracks", "10.4.1.219:5004"), ("lights", "WS Tracks", "10.4.1.219:5004"),
-                               ("tracks", "Lighting", "10.4.2.2:5006"), ("tracks", "WS FOH Mac mini", "10.4.1.12:5004")])
+        # One check per link: Tracks-Lights from Lights (it reports the state), Tracks-FOH from one end only.
+        self.assertEqual(got, [("lights", "WS Tracks", "10.4.1.219:5004"), ("tracks", "WS FOH Mac mini", "10.4.1.12:5004")])
        self.assertEqual(schema.validate(cfg)[0], [])
        import status_panel as sp
        snap = sp.Snapshot()
@@ -312,7 +312,12 @@ class TestCampusesView(unittest.TestCase):
        self.assertIn("$(custom:sp_view)", json.dumps(ctrls["3"]["0"]["style"]))       # it says which campus is shown
        cells = [ctl for row in ctrls.values() for ctl in row.values() if "campus view" in (ctl.get("options") or {}).get("notes", "")]
        self.assertEqual(len(cells), 28)
-        self.assertTrue(all(c["steps"]["0"]["action_sets"] == {"down": [], "up": []} for c in cells))   # no press, no hold
+        acts = cells[0]["steps"]["0"]["action_sets"]
+        self.assertEqual(sorted(acts), ["2000", "down", "up"])                       # tap = acknowledge, hold = fix, there
+        self.assertIn("sp_view_cmd", json.dumps(acts))
+        self.assertEqual(cv.campus_command([{"id": "hd1", "pos": [0, 2]}, {"id": "midi", "pos": [1, 3]}],
+                                           "tap 0 2 1000;hold 1 3 1000;hold 3 3 1000;hold 1 3 900", 1005),
+                         ["ack hd1", "restart midi 1000"])                         # an empty cell and a stale hold are dropped
        self.assertIn("sp_state_view", json.dumps(cells[0]["style"]))
        self.assertTrue(push_page.safe_target(export)[0])                          # a rebuild may overwrite it