christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ALARMS and AUTO-FIX as two keys; LIGHTS at Southcoast hands the lights to an operator

Browse Source 
- kind autofix: its own switch on the auto-fix hours; ALARMS then only silences (NA keeps the old
  combined key until it has room for one)
- kind bypass: hold takes every network MIDI link to a peer out (here and on listed Macs), checks stop
  calling it lost, re-taken every 10 s; hold again puts back exactly what was taken plus every watched link
- a MIDI peer with no address is never called (it connected SC Tracks to itself)
- holds from NA's CAMPUS page flip switch keys; a rebuild keeps them; new campuses get AUTO-FIX
- key kind changes restart the watchdog; settings page has separate alarm / auto-fix buttons
- WS and SC configs: AUTO-FIX at 3,1; SC LIGHTS at 1,3; NA config synced from live

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-21 13:33:48 -04:00
co-authored by Claude Opus 5.5
parent 2f8e3d47a3
commit d0c755824b
15 changed files with 605 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
status_panel.py
+236 -29
View File
@@ -494,6 +494,8 @@ def check_midi_peer(chk, snap, ctx):
sessions = None # that Mac stopped reporting: no verdict
if sessions is None:
return [("peer", Result(OK))]
if chk["peer"] in ctx.get("bypassed", ()) or chk["session"] in ctx.get("bypassed", ()):
return [("peer", Result(OK))] # an operator has it (LIGHTS key): taken out on purpose
sess = sessions.get(chk["session"])
if sess is None:
return [("peer", Result(ERR, "NO SESSION"))]
@@ -508,6 +510,101 @@ def check_midi_peer(chk, snap, ctx):
return [("peer", Result(ERR, "MIDI LOST"))]
def midi_on(cfg, host, call, jxa):
"""Run a coremidi call on another Mac (python there, else the JavaScript for Automation `jxa`):
(result, error text)."""
r = subprocess.run(ssh_cmd(cfg, host) + remote_python("-"), input=remote_midi_script(call),
capture_output=True, text=True, timeout=30)
if r.returncode == 97: # no python on that Mac
r = subprocess.run(ssh_cmd(cfg, host) + ["/usr/bin/osascript -l JavaScript -"], input=jxa,
capture_output=True, text=True, timeout=30)
result = parse_remote_midi(r.stdout) if r.returncode == 0 else None
return result, (None if result is not None else (r.stdout + r.stderr).strip().replace("\n", " ")[-200:])
def midi_peer(cfg, host, action, session, peer, address=None, repoint=False):
"""Connect ("connect") or drop ("disconnect") one peer of a network MIDI session, on this Mac
(host "") or another: (result word or None, error text)."""
import coremidi
if action == "connect" and not str(address or "").rsplit(":", 1)[0].strip("[]"):
# A peer that called in keeps no address (":5004"): adding it back connects the session to
# itself (SC Tracks, 2026-10-21). It comes back when it calls again.
return None, "no address to call (%r): it reconnects when %s calls in" % (address, peer)
if not host:
if action == "connect":
return coremidi.connect_peer(session, peer, address, repoint=repoint), None
return coremidi.disconnect_peer(session, peer), None
if action == "connect":
return midi_on(cfg, host, "ns['connect_peer'](%r, %r, %r, repoint=%r)" % (session, peer, address, repoint),
coremidi.jxa_connect(session, peer, address, repoint=repoint))
return midi_on(cfg, host, "ns['disconnect_peer'](%r, %r)" % (session, peer), coremidi.jxa_disconnect(session, peer))
def midi_sessions(cfg, host):
"""The network MIDI sessions on this Mac (host "") or another, or None."""
import coremidi
if not host:
return coremidi.sessions()
return midi_on(cfg, host, "ns['sessions']()", coremidi.JXA_SESSIONS)[0]
def bypass_apply(cfg, tile, on, entry, comp, names_ips, lock=None):
"""Hand a bypass key's device to an operator, or take it back. `entry["dropped"]` keeps every peer
taken out ({host, session, peer, address}), so taking it back puts exactly those back; the MIDI
checks naming it are reconnected too. Its Companion connections are switched off, then on again."""
b = tile.get("bypass") or {}
names, ips = set(b.get("peers") or []), set(names_ips[1]) if on else set()
hosts = b.get("hosts") or [""] + sorted({c["host"] for t in cfg["tiles"] for c in t.get("checks", [])
if c.get("type") == "midi_peer" and c.get("peer") in names and c.get("host")})
if not on:
for c in [x for t in cfg["tiles"] for x in t.get("checks", [])
if x.get("type") == "midi_peer" and x.get("peer") in names]:
ip = peer_ip(c.get("address"))
if ip:
ips.add(str(ip))
conns = {c["label"]: c for c in comp.connections()} if b.get("conns") else {}
for label in b.get("conns") or []:
conn = conns.get(label)
if not conn:
log.info("BYPASS %s: Companion has no connection %s", tile["label"], label)
elif on and conn.get("enabled"):
log.info("BYPASS %s: Companion %s off -> %s", tile["label"], label, comp.disable_connection(conn["id"]))
elif not on and not conn.get("enabled"):
log.info("BYPASS %s: Companion %s on -> %s", tile["label"], label, comp.enable_connection(conn["id"]))
if on:
for host in hosts:
try:
sessions = midi_sessions(cfg, host)
except Exception as e:
sessions, err = None, str(e)
if sessions is None:
continue # that Mac is off or unreachable: tried again in 10 s
for sname, sess in sessions.items():
for p in sess.get("peers") or []:
ip = peer_ip(p.get("address"))
if p.get("name") in names or (ip and str(ip) in ips):
res, err = midi_peer(cfg, host, "disconnect", sname, p["name"])
log.info("BYPASS %s: %s out of %s%s -> %s", tile["label"], p["name"], sname,
" on " + host if host else "", res or err)
if res:
rec = {"host": host, "session": sname, "peer": p["name"], "address": p.get("address") or ""}
with (lock or threading.Lock()):
if rec not in entry["dropped"]:
entry["dropped"].append(rec)
return
done = set()
for rec in entry.get("dropped") or []:
res, err = midi_peer(cfg, rec["host"], "connect", rec["session"], rec["peer"], rec["address"])
done.add((rec["host"], rec["session"], rec["peer"]))
log.info("BYPASS %s: %s back into %s%s -> %s", tile["label"], rec["peer"], rec["session"],
" on " + rec["host"] if rec["host"] else "", res or err)
for t in cfg["tiles"]:
for c in t.get("checks", []):
if c.get("type") == "midi_peer" and c.get("peer") in names and (c.get("host", ""), c["session"], c["peer"]) not in done:
res, err = midi_peer(cfg, c.get("host", ""), "connect", c["session"], c["peer"], c["address"])
log.info("BYPASS %s: %s back into %s -> %s", tile["label"], c["peer"], c["session"], res or err)
def peer_ip(address):
"""The IPv4 address of a MIDI peer ("10.1.1.1:5040"), or None for a Bonjour name or IPv6."""
host = str(address or "").strip().rsplit(":", 1)[0]
@@ -967,10 +1064,11 @@ class ServiceMode:
edge is worked out at the press and kept, so the override ends there even when the watchdog
was not running at that minute (a restart over 23:59 left the alarms muted for good)."""
def __init__(self, windows, state):
def __init__(self, windows, state, name="service"):
self.windows = windows
self.name = name # "service" (ALARMS) or "autofix" (AUTO-FIX)
self.state = state if isinstance(state, State) else State(state)
saved = self.state.get("service") or {}
saved = self.state.get(name) or {}
self.override, self.last_sched = saved.get("override"), saved.get("last_sched")
self.edge = saved.get("edge")
@@ -1027,7 +1125,18 @@ class ServiceMode:
self._save()
def _save(self):
self.state.set("service", {"override": self.override, "last_sched": self.last_sched, "edge": self.edge})
self.state.set(self.name, {"override": self.override, "last_sched": self.last_sched, "edge": self.edge})
def until(self, now):
"""When the state shown now next changes by itself (an override's end, else the schedule's next edge),
looked up once a minute."""
minute = int(now // 60)
cached = getattr(self, "_until", None)
if cached and cached[0] == (minute, self.override):
return cached[1]
found = self.back_at(now) if self.override is not None else self.next_change(now)
self._until = ((minute, self.override), found)
return found
# --- Engine ------------------------------------------------------------------
@@ -1050,6 +1159,16 @@ class Engine:
self.hold_ms = cfg.get("page_hold_ms", 2000) # how long a key is held to run its fix
self.state = State(state_path)
self.service = ServiceMode(cfg.get("service_windows", []), self.state)
# With an AUTO-FIX key, ALARMS and auto-fix are two switches (2026-10-21, the user: "seperatly
# silence alarms and auto fixes"); without one, ALARMS stops both, as before.
self.split = any(t.get("kind") == "autofix" for t in cfg["tiles"])
self.autofix = ServiceMode((cfg.get("auto_fix") or {}).get("windows", []), self.state, "autofix")
# Keys that hand a device to an operator (Southcoast's LIGHTS: a lighting op runs the desk, so
# nothing may send it MIDI): tile id -> {"since", "dropped": [peers taken out, to put back]}.
self.bypass = {k: v for k, v in (self.state.get("bypass") or {}).items() if k in self.tiles}
self.bypass_lock = threading.Lock()
self.bypass_busy = False
self.bypass_last = 0.0
self.data_dir = data_dir
self.stopping = False # SIGTERM arrived: finish what runs, start nothing new
self.running = {} # fix thread name -> {"key", "what", "since"} (published as fixes-running.json)
@@ -1184,7 +1303,8 @@ class Engine:
# raw results for every check key
def evaluate(self, snap, now, service_on):
ctx = {"now": now, "service": service_on, "any_recording": False, "dante_all": {},
"ready_due": in_windows(self.ready_cfg.get("due", []), now), "ready_done": self.ready_done(now)}
"ready_due": in_windows(self.ready_cfg.get("due", []), now), "ready_done": self.ready_done(now),
"bypassed": self.bypass_names()[0]}
for t in all_tiles(self.cfg):
for chk in t.get("checks", []):
if chk["type"] == "dante":
@@ -1500,11 +1620,77 @@ class Engine:
"""Show `text` on the button for a while (hold feedback), instead of its usual text."""
self.notes[tile_id] = {"text": text, "level": level, "until": now + seconds}
# --- operator bypass (a LIGHTS key) ---------------------------------------------------------
def bypass_names(self):
"""(peer and session names an operator has, their IPv4 addresses as the MIDI checks know them)."""
names, ips = set(), set()
for tid in self.bypass:
names |= set(((self.tiles.get(tid) or {}).get("bypass") or {}).get("peers") or [])
if names:
for t in self.cfg["tiles"]:
for c in t.get("checks", []):
if c.get("type") == "midi_peer" and (c.get("peer") in names):
ip = peer_ip(c.get("address"))
if ip:
ips.add(str(ip))
return names, ips
def toggle_bypass(self, tile_id, comp, now):
"""A hold on a bypass key: hand it to an operator (take its MIDI links out, switch its Companion
connections off), or take it back (put everything back, as it was)."""
if self.bypass_busy:
self.note(tile_id, "BUSY\n" + self.wait_text, "idle", now, 4)
return
tile = self.tiles[tile_id]
with self.bypass_lock:
if tile_id in self.bypass:
entry = self.bypass.pop(tile_id)
on = False
else:
entry = self.bypass[tile_id] = {"since": now, "dropped": []}
on = True
self.state.set("bypass", self.bypass)
log.info("BYPASS %s: %s", tile["label"], "handed to an operator" if on else "back to the panel")
self.note(tile_id, "TAKING\nMIDI OUT" if on else "PUTTING\nMIDI BACK", "fixing", now, 6)
self.bypass_last = now
self.start_bypass_job(tile_id, on, entry, comp)
def start_bypass_job(self, tile_id, on, entry, comp):
self.bypass_busy = True
def job():
try:
bypass_apply(self.cfg, self.tiles[tile_id], on, entry, comp, self.bypass_names(), self.bypass_lock)
if tile_id in self.bypass:
self.state.set("bypass", self.bypass) # the peers taken out, to put back later
except Exception:
log.exception("bypass %s failed", tile_id)
finally:
self.bypass_busy = False
threading.Thread(target=job, name="bypass-" + tile_id, daemon=True).start()
def bypass_tick(self, comp, now):
"""While an operator has it, every 10 s: a link that came back (rtpMIDI asking again) goes out again."""
if not self.bypass or self.bypass_busy or self.stopping or now - self.bypass_last < 10:
return
self.bypass_last = now
tid = next(iter(self.bypass))
self.start_bypass_job(tid, True, self.bypass[tid], comp)
def run_fixes(self, tile_id, snap, comp, now, cooldown=None):
cooldown = self.cooldown if cooldown is None else cooldown
if self.stopping:
self.note(tile_id, "RESTARTING\nSOON", "idle", now, 4)
return []
kind = self.tiles.get(tile_id, {}).get("kind")
if kind in ("service", "autofix"): # from North Attleboro's CAMPUS page: the hold is the press
mode = self.service if kind == "service" else self.autofix
mode.set(not mode.on(now), now)
log.info("%s %s (hold)", "SERVICE" if kind == "service" else "AUTO-FIX", "ON" if mode.on(now) else "OFF")
return []
if kind == "bypass":
self.toggle_bypass(tile_id, comp, now)
return []
if self.tiles.get(tile_id, {}).get("kind") == "network":
# NET: a hold clears a remembered network blip (link and gateway clear themselves).
if self.burst is None:
@@ -1631,17 +1817,23 @@ class Engine:
if not a or now - self.auto_last < 5 or self.stopping:
return []
self.auto_last = now
# The ALARMS key is the on-the-fly switch: muted, nothing is fixed by itself at all (holds
# still work). It comes back by itself with the alarms (2026-10-19: the user's choice).
muted = not self.service.on(now)
hours = in_windows(a.get("windows", []), now)
if self.split:
# The AUTO-FIX key: paused, nothing is fixed by itself (holds still work); pressed outside
# the hours, the gentle fixes run until the hours next start or end.
hours = self.autofix.on(now)
muted = self.autofix.override is False
else:
# The ALARMS key is the on-the-fly switch: muted, nothing is fixed by itself at all (holds
# still work). It comes back by itself with the alarms (2026-10-19: the user's choice).
muted = not self.service.on(now)
hours = in_windows(a.get("windows", []), now)
# A fix marked "anytime" (writing the timecode into Planning Center) runs by itself at any
# hour, not only in the auto-fix hours - but never while muted, like every other.
anytime = {f for f in a.get("fixes", []) if self.cfg["fixes"].get(f, {}).get("anytime")
and not self.cfg["fixes"].get(f, {}).get("never_auto") and not self.cfg["fixes"].get(f, {}).get("disabled")}
if muted:
if self.autolog and (self.auto_on or (anytime and not self.auto_muted)):
self.autolog.write(now, "PANEL", "auto-fix off (ALARMS muted)")
self.autolog.write(now, "PANEL", "auto-fix off (%s)" % ("AUTO-FIX key" if self.split else "ALARMS muted"))
self.auto_on, self.auto_muted = False, True
self.auto_tries.clear()
self.auto_seen.clear()
@@ -2070,12 +2262,30 @@ def render(engine, service_on):
# they come back by themselves after a press.
word = "ON" if service_on else "OFF"
fix_word = "OFF" if (not service_on or (engine.cfg.get("auto_fix") or {}).get("disabled")) else "ON"
text = "%s %s\nAUTO-FIX %s" % (tile["label"], word, fix_word)
text = ("%s\n%s" % (tile["label"], word)) if engine.split else "%s %s\nAUTO-FIX %s" % (tile["label"], word, fix_word)
back = engine.service.back_at(engine.now)
if back:
text += "\nTIL " + clock12(back)
out[tid] = colored(text, "service" if service_on else "off")
continue
if kind == "autofix":
# Its own switch: on in the auto-fix hours, a press pauses it (or runs it outside them).
if (engine.cfg.get("auto_fix") or {}).get("disabled"):
out[tid] = colored(tile["label"] + "\nOFF IN\nSETTINGS", "off")
continue
on = engine.autofix.on(engine.now)
until = engine.autofix.until(engine.now)
text = "%s\n%s" % (tile["label"], "ON" if on else "OFF")
if until and (on or engine.autofix.override is not None):
same_day = datetime.fromtimestamp(until).date() == datetime.fromtimestamp(engine.now).date()
text += "\nTIL " + ("" if same_day else datetime.fromtimestamp(until).strftime("%a ").upper()) + clock12(until)
out[tid] = colored(text, "service" if on else "off")
continue
if kind == "bypass":
# Handed to an operator: steady blue, says so; otherwise the panel runs it.
on = tid in engine.bypass
out[tid] = colored("%s\n%s" % (tile["label"], "OPERATOR\nMIDI OFF" if on else "AUTO"), "service" if on else "ok")
continue
if kind == "master":
# Idle faults are expected outside service; they do not stop "ALL OK".
items = [("ok" if lv == "idle" else lv, res, since, engine.tiles[t]) for t in engine.detail_ids
@@ -2856,24 +3066,11 @@ def execute_fix(cfg, fix_id, chk):
return propresenter_port(cfg, fix, label)
if kind == "midi_connect":
# Like Connect in Audio MIDI Setup: add the peer back to the session it dropped out of.
args = (chk["session"], chk["peer"], chk["address"])
repoint = bool(chk.get("repoint"))
if chk.get("host"):
r = subprocess.run(ssh_cmd(cfg, chk["host"]) + remote_python("-"),
input=remote_midi_script("ns['connect_peer'](%r, %r, %r, repoint=%r)" % (args + (repoint,))),
capture_output=True, text=True, timeout=30)
if r.returncode == 97: # no python on that Mac: the same, in JavaScript for Automation
import coremidi
r = subprocess.run(ssh_cmd(cfg, chk["host"]) + ["/usr/bin/osascript -l JavaScript -"],
input=coremidi.jxa_connect(*args, repoint=repoint), capture_output=True, text=True, timeout=30)
result = parse_remote_midi(r.stdout) if r.returncode == 0 else None
if result is None:
log.info("FIX %s -> %s on %s failed: %s", label, chk["peer"], chk["host"],
(r.stdout + r.stderr).strip().replace("\n", " ")[-200:])
return False
else:
import coremidi
result = coremidi.connect_peer(*args, repoint=repoint)
result, err = midi_peer(cfg, chk.get("host", ""), "connect", chk["session"], chk["peer"], chk["address"],
repoint=bool(chk.get("repoint")))
if result is None:
log.info("FIX %s -> %s on %s failed: %s", label, chk["peer"], chk.get("host"), err)
return False
log.info("FIX %s -> %s at %s, session %s%s: %s", label, chk["peer"], chk["address"], chk["session"],
(" on " + chk["host"]) if chk.get("host") else "", result)
return result in ("asked", "already", "kicked", "moved")
@@ -3401,7 +3598,9 @@ def snapshot_summary(snap, engine, now):
for name, l in snap.listeners.items()},
"service_on": engine.service.on(now),
"alarms_override": engine.service.override,
"auto_fix": {"on": engine.auto_on, "paused": engine.auto_paused},
"auto_fix": {"on": engine.auto_on, "paused": engine.auto_paused, "split": engine.split,
"switch": engine.autofix.on(now), "override": engine.autofix.override},
"bypass": {tid: {"since": b.get("since")} for tid, b in engine.bypass.items()},
"failover": dict(engine.failed),
"burst": engine.burst and {"start": engine.burst["start"], "conns": sorted(engine.burst["conns"]),
"cleared": engine.burst["cleared"]},
@@ -3446,6 +3645,10 @@ def handle_commands(text, engine, comp, snap, now):
target = {"on": True, "off": False}.get(arg, not engine.service.on(now))
engine.service.set(target, now)
log.info("SERVICE %s (button)", "ON" if target else "OFF")
elif verb == "autofix":
target = {"on": True, "off": False}.get(arg, not engine.autofix.on(now))
engine.autofix.set(target, now)
log.info("AUTO-FIX %s (button)", "ON" if target else "OFF")
elif verb in ("restart", "force"):
# Holds carry Companion's clock ("force rs_x 1791496225"). One pressed while the watchdog
# was down must not fire minutes later in a burst with the others.
@@ -3540,6 +3743,10 @@ def run(cfg, data_dir, dry_run=False, stop=None, config_sha=None):
engine.auto_fix(snap, comp, now)
except Exception: # a bug here must never take the panel down
log.exception("auto-fix failed; the panel carries on without it this round")
try:
engine.bypass_tick(comp, now)
except Exception:
log.exception("operator bypass failed; the panel carries on without it this round")
try:
engine.failover(snap, comp, now)
except Exception: