Southcoast matches NA, per its startup checklist: X32s, ProPresenter to Lighting MIDI, pages 19/20

- An explicit look or rebuild may apply inside the auto-fix hours (never while live)
- A page move clears the page left behind when only our keys are on it
- A rebuild keeps MIDI links that are down now; first-setup import waits for the keys to land
- ProPresenter port drift fixed automatically (kind propresenter_port)
- sites/sc/config.json kept with the others

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-21 13:07:15 -04:00
co-authored by Claude Opus 5.5
parent ee4c97c699
commit bea587cfad
8 changed files with 554 additions and 20 deletions
+2 -1
View File
@@ -219,6 +219,7 @@ field("fixes.*.label", "str", None, "Plain words: what it does.", "Fixes", requi
field("fixes.*.reconnect", "bool", False, "Reconnect the Companion connection afterwards.", "Fixes") field("fixes.*.reconnect", "bool", False, "Reconnect the Companion connection afterwards.", "Fixes")
field("fixes.*.conn", "str", None, "The Companion connection it reconnects.", "Fixes") field("fixes.*.conn", "str", None, "The Companion connection it reconnects.", "Fixes")
field("fixes.*.restart_ok", "bool", False, "May close a running program (and still run by itself).", "Fixes") field("fixes.*.restart_ok", "bool", False, "May close a running program (and still run by itself).", "Fixes")
field("fixes.*.on", "host", None, "The computer ProPresenter runs on, when not this Mac (follow its network port).", "Fixes")
field("fixes.*.never_auto", "bool", False, "Never runs by itself.", "Fixes") field("fixes.*.never_auto", "bool", False, "Never runs by itself.", "Fixes")
field("fixes.*.anytime", "bool", False, "Ticked for auto-fix, runs by itself at any hour, not only in the auto-fix hours, unless ALARMS is muted (a READY fix that only writes to Planning Center).", "Fixes") field("fixes.*.anytime", "bool", False, "Ticked for auto-fix, runs by itself at any hour, not only in the auto-fix hours, unless ALARMS is muted (a READY fix that only writes to Planning Center).", "Fixes")
field("fixes.*.part", "enum", None, "Which part of the switcher.", "Fixes", options=["audio", "camera"]) field("fixes.*.part", "enum", None, "Which part of the switcher.", "Fixes", options=["audio", "camera"])
@@ -453,7 +454,7 @@ COMMON_CHECK_FIELDS = ["type", "fix", "tag", "service_only", "debounce_seconds",
FIX_KINDS = sp.FIX_KINDS FIX_KINDS = sp.FIX_KINDS
FIX_KIND_FIELDS = { FIX_KIND_FIELDS = {
"ssh": ["host", "run", "timeout", "reconnect", "conn"], "local": ["run", "timeout", "reconnect", "conn"], "ssh": ["host", "run", "timeout", "reconnect", "conn"], "local": ["run", "timeout", "reconnect", "conn"],
"atem": ["part"], "reconnect": ["conn", "verify_seconds", "hung_note"], "midi_connect": [], "atem": ["part"], "reconnect": ["conn", "verify_seconds", "hung_note"], "midi_connect": [], "propresenter_port": ["conn", "on"],
"netgear_poe": ["switch", "port", "mac"], "hyperdeck": ["verify_seconds", "off_seconds", "hung_note"], "netgear_poe": ["switch", "port", "mac"], "hyperdeck": ["verify_seconds", "off_seconds", "hung_note"],
"spotify": ["playlist"], "ready": ["action"], "companion_press": ["page", "row", "col"], "spotify": ["playlist"], "ready": ["action"], "companion_press": ["page", "row", "col"],
} }
+8 -2
View File
@@ -443,9 +443,15 @@ class Applier:
if not ok: if not ok:
return False, "page %d: %s" % (page, why) return False, "page %d: %s" % (page, why)
push_page.push(host, port, page, path, expected) push_page.push(host, port, page, path, expected)
after = push_page.fetch_page(host, port, page)
want = {(r, c) for r, row in export["page"]["controls"].items() for c, ctl in row.items() if ctl and ctl.get("type") == "button-layered"} want = {(r, c) for r, row in export["page"]["controls"].items() for c, ctl in row.items() if ctl and ctl.get("type") == "button-layered"}
have = {(r, c) for r, row in (after.get("page") or {}).get("controls", {}).items() for c, ctl in (row or {}).items() if ctl and ctl.get("type") == "button-layered"} # Companion places the keys a moment after the import call returns (Southcoast's first setup
# was judged too early, 2026-10-21): look again for up to 10 s before calling it a failure.
for _ in range(20):
after = push_page.fetch_page(host, port, page)
have = {(r, c) for r, row in (after.get("page") or {}).get("controls", {}).items() for c, ctl in (row or {}).items() if ctl and ctl.get("type") == "button-layered"}
if want == have:
break
time.sleep(0.5)
if want != have: if want != have:
return False, "page %d: the import did not land as built (%d keys built, %d on the page)" % (page, len(want), len(have)) return False, "page %d: the import did not land as built (%d keys built, %d on the page)" % (page, len(want), len(have))
done.append(name) done.append(name)
+54 -2
View File
@@ -283,14 +283,23 @@ def items(inv, nets):
"checks": [{"type": spec["kind"], "conn": label, "channels": n}, {"type": "ping", "conn": label}], "fixes": {}}) "checks": [{"type": spec["kind"], "conn": label, "channels": n}, {"type": "ping", "conn": label}], "fixes": {}})
continue continue
checks = [{"type": "conn", "conn": label}] checks = [{"type": "conn", "conn": label}]
fixes = {}
if conn.get("module") == "renewedvision-propresenter-api":
# ProPresenter changes its network port now and then: the fix finds the new one.
fid = slug("prop_port_" + label)
where = computer_by_ip(inv, ip_of(conn.get("host")) or conn.get("host"))
fixes[fid] = dict({"kind": "propresenter_port", "template": "propresenter_port", "params": {}, "conn": label, "short": "PROP PORT",
"label": "find ProPresenter's network port and reconnect Companion's %s to it" % label}, **({"on": where} if where else {}))
checks[0]["fix"] = fid
if spec.get("ping") and conn.get("host"): if spec.get("ping") and conn.get("host"):
checks.append({"type": "ping", "conn": label}) checks.append({"type": "ping", "conn": label})
app_item = apps_at.get((spec.get("app"), ip_of(conn.get("host")))) if spec.get("app") else None app_item = apps_at.get((spec.get("app"), ip_of(conn.get("host")))) if spec.get("app") else None
if app_item: # the program's own key carries its connection too if app_item: # the program's own key carries its connection too
app_item["checks"] = checks + app_item["checks"] app_item["checks"] = checks + app_item["checks"]
app_item["fixes"].update(fixes)
app_item["also"].append(ident) app_item["also"].append(ident)
continue continue
out.append({"ident": ident, "group": spec["group"], "label": text, "checks": checks, "fixes": {}, "restart": [], "also": []}) out.append({"ident": ident, "group": spec["group"], "label": text, "checks": checks, "fixes": fixes, "restart": [], "also": []})
# Network MIDI: every peer in the sessions here and on the other Macs, at its wired address. # Network MIDI: every peer in the sessions here and on the other Macs, at its wired address.
midi_checks = [] midi_checks = []
# Whose session a peer is, by its name: a peer joined by name or by an IPv6 address gets that Mac's wired address. # Whose session a peer is, by its name: a peer joined by name or by an IPv6 address gets that Mac's wired address.
@@ -555,6 +564,49 @@ def merge_parts(cfg, it, fresh):
return words return words
def keep_from(new, old):
"""A rebuild keeps what is down right now and so was not found: MIDI links that dropped (Southcoast's
ProPresenter to Lighting lost its MIDI key, 2026-10-21) and computers that are off, with their keys."""
cfg = copy.deepcopy(new)
have = referenced(cfg)
kept = []
# Computers in the old setup that did not answer this time, and the keys that check them.
gone = {a: h for a, h in (old.get("hosts") or {}).items() if not h.get("self") and a not in cfg["hosts"]}
for alias, h in gone.items():
cfg["hosts"][alias] = h
for t in old.get("tiles", []):
if t.get("checks") and all(c.get("host") == alias or (c.get("params") or {}).get("host") == alias for c in t["checks"]) \
and t["id"] not in {x["id"] for x in cfg["tiles"]}:
for c in t["checks"]:
for fid in ([c["fix"]] if isinstance(c.get("fix"), str) else []):
if fid in old.get("fixes", {}):
cfg["fixes"].setdefault(fid, old["fixes"][fid])
place_new(cfg, {k: v for k, v in t.items() if k != "pos"})
kept.append(t["label"])
for k in (old.get("restart_page") or {}).get("keys", []):
f = old.get("fixes", {}).get(k.get("fix"), {})
if (f.get("host") == alias or (f.get("params") or {}).get("host") == alias) and cfg.get("restart_page"):
cfg["fixes"].setdefault(k["fix"], f)
restart_layout(cfg, [{"label": k["label"], "fix": k["fix"]}])
# MIDI links that are down now.
lost = [c for t in old.get("tiles", []) for c in t.get("checks", []) if c.get("type") == "midi_peer"
and "midi:%s|%s|%s" % (c.get("host", ""), c["session"], c["peer"]) not in have and (not c.get("host") or c["host"] in cfg["hosts"])]
if lost:
midi = next((t for t in cfg["tiles"] if any(c.get("type") in ("midi_peer", "midi_wired") for c in t.get("checks", []))), None)
if midi is None:
wired = [c for t in old.get("tiles", []) for c in t.get("checks", []) if c.get("type") == "midi_wired"]
midi = {"id": slug("midi", {t["id"] for t in cfg["tiles"]}), "label": "MIDI", "checks": wired[:1], "_group": "SHOW"}
place_new(cfg, midi)
midi = cfg["tiles"][-1]
at = next((i for i, c in enumerate(midi["checks"]) if c.get("type") == "midi_wired"), len(midi["checks"]))
midi["checks"][at:at] = lost
cfg["fixes"].setdefault("midi_reconnect", dict(SHARED_FIXES["midi_reconnect"]))
if "midi_reconnect" not in cfg["auto_fix"]["fixes"]:
cfg["auto_fix"]["fixes"].append("midi_reconnect")
kept += ["MIDI link %s" % c["peer"] for c in lost]
return cfg, kept
def finish(cfg): def finish(cfg):
"""Fixes and remote checks built by their builders, words filled in, as the page would.""" """Fixes and remote checks built by their builders, words filled in, as the page would."""
for fid, f in list(cfg["fixes"].items()): for fid, f in list(cfg["fixes"].items()):
@@ -578,7 +630,7 @@ def gentle(cfg):
for fid, f in cfg["fixes"].items(): for fid, f in cfg["fixes"].items():
if fid.startswith("restart_") or f.get("restart_ok") or f.get("never_auto"): if fid.startswith("restart_") or f.get("restart_ok") or f.get("never_auto"):
continue continue
if f.get("kind") in ("hyperdeck", "midi_connect", "reconnect") or (f.get("template") in ("mac_start", "win_start", "ableton") if f.get("kind") in ("hyperdeck", "midi_connect", "reconnect", "propresenter_port") or (f.get("template") in ("mac_start", "win_start", "ableton")
and not (f.get("params") or {}).get("restart")): and not (f.get("params") or {}).get("restart")):
out.append(fid) out.append(fid)
return out return out
+2
View File
@@ -219,6 +219,8 @@ TEMPLATES = {
"suggest": lambda p, cfg: "reconnect Companion to it"}, "suggest": lambda p, cfg: "reconnect Companion to it"},
"hyperdeck": {"words": "Switch a HyperDeck's connection off and on", "kind": "hyperdeck", "start_only": True, "params": [], "render": None, "hyperdeck": {"words": "Switch a HyperDeck's connection off and on", "kind": "hyperdeck", "start_only": True, "params": [], "render": None,
"suggest": lambda p, cfg: "turn Companion's connection to that HyperDeck off and on; never reboots the deck"}, "suggest": lambda p, cfg: "turn Companion's connection to that HyperDeck off and on; never reboots the deck"},
"propresenter_port": {"words": "Follow ProPresenter's network port (when it changes) and reconnect Companion", "kind": "propresenter_port",
"start_only": True, "params": [], "render": None, "suggest": lambda p, cfg: "find ProPresenter's network port and reconnect Companion to it"},
"midi_connect": {"words": "Reconnect a network MIDI peer", "kind": "midi_connect", "start_only": True, "params": [], "render": None, "midi_connect": {"words": "Reconnect a network MIDI peer", "kind": "midi_connect", "start_only": True, "params": [], "render": None,
"suggest": lambda p, cfg: "reconnect the peer to the network MIDI session it dropped out of"}, "suggest": lambda p, cfg: "reconnect the peer to the network MIDI session it dropped out of"},
"atem": {"words": "Put the switcher back to the saved file", "kind": "atem", "params": [], "render": None, "atem": {"words": "Put the switcher back to the saved file", "kind": "atem", "params": [], "render": None,
+56 -13
View File
@@ -211,7 +211,7 @@ def setup(data_dir, cid, name=None, say=print, home=None):
return new return new
def rebuild(data_dir, say=print): def rebuild(data_dir, say=print, pages=None):
"""A campus's setup made again from what is found now (today's recipes), keeping its name, PIN, """A campus's setup made again from what is found now (today's recipes), keeping its name, PIN,
pages and site details. Applied like any Apply (it waits for a recording; history keeps the old one).""" pages and site details. Applied like any Apply (it waits for a recording; history keeps the old one)."""
from settings import server from settings import server
@@ -224,7 +224,17 @@ def rebuild(data_dir, say=print):
rp = (cfg.get("restart_page") or {}).get("page") rp = (cfg.get("restart_page") or {}).get("page")
if not rp: if not rp:
rp = next(p for p in pick_pages(cfg, 3) if p != cfg["page"]) rp = next(p for p in pick_pages(cfg, 3) if p != cfg["page"])
new, seen = autoconf.build(template(data_dir), inv, {"id": s.get("id"), "name": s.get("name")}, s.get("editing_code"), (cfg["page"], rp)) pages = tuple(pages or (cfg["page"], rp))
if pages != (cfg["page"], rp): # moving (19/20 like every campus): only onto pages that are free or ours
comp = cfg.get("companion") or {}
for p in pages:
ok, why = push_page.safe_target(push_page.fetch_page(comp.get("host", "127.0.0.1"), int(comp.get("port", 8000)), p))
if not ok:
raise RuntimeError("page %d: %s" % (p, why))
new, seen = autoconf.build(template(data_dir), inv, {"id": s.get("id"), "name": s.get("name")}, s.get("editing_code"), pages)
new, kept = autoconf.keep_from(new, cfg)
if kept:
say("Kept (down right now, so not found): %s" % ", ".join(kept))
for k in ("contacts", "live_services", "ingest_name", "settings_port"): for k in ("contacts", "live_services", "ingest_name", "settings_port"):
if k in s: if k in s:
new["site"][k] = s[k] new["site"][k] = s[k]
@@ -238,11 +248,29 @@ def rebuild(data_dir, say=print):
st = wait_applied(site.applier) st = wait_applied(site.applier)
save_seen(data_dir, load_seen(data_dir) | set(seen)) save_seen(data_dir, load_seen(data_dir) | set(seen))
say("%s: %s" % (st.get("state"), st.get("outcome", ""))) say("%s: %s" % (st.get("state"), st.get("outcome", "")))
if st.get("state") == "done":
for p in sorted({cfg["page"], rp} - set(pages)):
say("Old page %d: %s" % (p, clear_old_page(cfg, p)))
say("Keys: %s" % ", ".join(t["label"] for t in new["tiles"] if not t.get("kind"))) say("Keys: %s" % ", ".join(t["label"] for t in new["tiles"] if not t.get("kind")))
say("Restart page %s: %s" % (rp, ", ".join(k["label"] for k in new.get("restart_page", {}).get("keys", [])) or "no keys yet")) say("Restart page %s: %s" % (new.get("restart_page", {}).get("page"), ", ".join(k["label"] for k in new.get("restart_page", {}).get("keys", [])) or "no keys yet"))
return new return new
def clear_old_page(cfg, page):
"""A page left behind by a move is emptied, but only when it holds nothing a person made."""
comp = cfg.get("companion") or {}
host, port = comp.get("host", "127.0.0.1"), int(comp.get("port", 8000))
ok, why = push_page.safe_target(push_page.fetch_page(host, port, page))
if not ok:
return "left as it is (%s)" % why
ws = push_page.WebSocket(host, port, "/trpc")
try:
push_page.Trpc(ws).mutation("pages.clearPage", {"pageNumber": page})
finally:
ws.close()
return "cleared"
def upgrade_companion(base, say=print): def upgrade_companion(base, say=print):
"""Companion 5.0.7 from `base` (North Attleboro's settings page), as Woonsocket was upgraded on 2026-10-20: """Companion 5.0.7 from `base` (North Attleboro's settings page), as Woonsocket was upgraded on 2026-10-20:
the running Companion is quit, the newest settings folder is copied for 5.0 (and marked newest, or the the running Companion is quit, the newest settings folder is copied for 5.0 (and marked newest, or the
@@ -305,13 +333,14 @@ def record(data_dir, result):
# --- every ten minutes: what is new ------------------------------------------------------ # --- every ten minutes: what is new ------------------------------------------------------
def blocked(cfg, applier, now=None): def blocked(cfg, applier, now=None, asked=False):
"""Why auto-setup may not apply right now, or None: never during a service or a recording.""" """Why auto-setup may not apply right now, or None: never during a recording; by itself, never in the
auto-fix hours either (a person asking - Look now, setting up on site - may)."""
now = now or time.time() now = now or time.time()
live, why = applier.live() live, why = applier.live()
if live: if live:
return "live now (%s)" % why return "live now (%s)" % why
if sp.in_windows((cfg.get("auto_fix") or {}).get("windows") or [], now): if not asked and sp.in_windows((cfg.get("auto_fix") or {}).get("windows") or [], now):
return "inside the auto-fix hours (a service day): waits until they end" return "inside the auto-fix hours (a service day): waits until they end"
if applier.busy(): if applier.busy():
return "an Apply is running" return "an Apply is running"
@@ -327,7 +356,7 @@ def look_and_add(site, say=lambda s: None, force=False):
result = {"at": int(time.time()), "added": [], "waiting": [], "inventory": summary(inv)} result = {"at": int(time.time()), "added": [], "waiting": [], "inventory": summary(inv)}
if added: if added:
auto = (cfg.get("discover") or {}).get("auto") auto = (cfg.get("discover") or {}).get("auto")
why = blocked(cfg, site.applier) why = blocked(cfg, site.applier, asked=force)
if not (auto or force): if not (auto or force):
result["waiting"] = added result["waiting"] = added
result["why"] = "adding by itself is off (Settings > Site)" result["why"] = "adding by itself is off (Settings > Site)"
@@ -449,10 +478,14 @@ if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then
echo echo
echo "One more thing: turn on Remote Login (System Settings > General > Sharing), so North Attleboro can help remotely." echo "One more thing: turn on Remote Login (System Settings > General > Sharing), so North Attleboro can help remotely."
fi fi
# This Mac's wired address (the default route's port, not en0: at Southcoast en0 is a Dante-style link).
IF=$(/sbin/route -n get default 2>/dev/null | awk '/interface:/{print $2}')
ME=$(/usr/sbin/ipconfig getifaddr "$IF" 2>/dev/null || hostname)
echo echo
echo "Next, on each other computer here (ProPresenter, Tracks, Lighting...):" echo "Next, on each other computer here (ProPresenter, Tracks, Lighting...):"
echo " Mac: open Terminal and paste curl -fsSL http://$(ipconfig getifaddr en0 2>/dev/null || hostname):8020/join | sh" echo " Mac: open Terminal and paste curl -fsSL http://$ME:8020/join | sh"
echo " PC: open PowerShell as administrator and paste irm http://$(ipconfig getifaddr en0 2>/dev/null || hostname):8020/join.ps1 | iex" echo " (or send it to all of them at once in Apple Remote Desktop: Send UNIX Command, as root)"
echo " PC: open PowerShell as administrator and paste irm http://$ME:8020/join.ps1 | iex"
""" % {"name": name, "base": shq(base), "key": shq(key), "cid": cid, "qname": shq(name)} """ % {"name": name, "base": shq(base), "key": shq(key), "cid": cid, "qname": shq(name)}
@@ -464,8 +497,17 @@ def join_mac(cfg, base):
return """#!/bin/sh return """#!/bin/sh
# Joins this Mac to %(ingest)s's status panel: lets it log in with its key (no password) and says this Mac is here. # Joins this Mac to %(ingest)s's status panel: lets it log in with its key (no password) and says this Mac is here.
KEY=%(key)s KEY=%(key)s
mkdir -p "$HOME/.ssh" && touch "$HOME/.ssh/authorized_keys" && chmod 700 "$HOME/.ssh" && chmod 600 "$HOME/.ssh/authorized_keys" # Run as root (Apple Remote Desktop > Send UNIX Command): it is for whoever is logged in on this Mac.
grep -qF "$KEY" "$HOME/.ssh/authorized_keys" || echo "$KEY" >> "$HOME/.ssh/authorized_keys" U=$(id -un); H="$HOME"
if [ "$(id -u)" = 0 ]; then
U=$(stat -f%%Su /dev/console)
[ "$U" = root ] && { echo "Nobody is logged in on $(scutil --get ComputerName): log in, then send this again."; exit 1; }
H=$(dscl . -read "/Users/$U" NFSHomeDirectory | awk '{print $2}')
fi
mkdir -p "$H/.ssh" && touch "$H/.ssh/authorized_keys" && chmod 700 "$H/.ssh" && chmod 600 "$H/.ssh/authorized_keys"
grep -qF "$KEY" "$H/.ssh/authorized_keys" || echo "$KEY" >> "$H/.ssh/authorized_keys"
[ "$(id -u)" = 0 ] && chown -R "$U:staff" "$H/.ssh"
[ "$(id -u)" = 0 ] && { /usr/sbin/systemsetup -setremotelogin on >/dev/null 2>&1 || /bin/launchctl load -w /System/Library/LaunchDaemons/ssh.plist >/dev/null 2>&1; }
if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then
echo "Remote Login is off. Turning it on (it asks for this Mac's administrator password):" echo "Remote Login is off. Turning it on (it asks for this Mac's administrator password):"
if ! sudo /usr/sbin/systemsetup -setremotelogin on >/dev/null 2>&1; then if ! sudo /usr/sbin/systemsetup -setremotelogin on >/dev/null 2>&1; then
@@ -476,7 +518,7 @@ if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then
fi fi
esc() { printf '%%s' "$1" | sed 's/[\\\\"]//g'; } esc() { printf '%%s' "$1" | sed 's/[\\\\"]//g'; }
IPS=$(/sbin/ifconfig | awk '/inet /{print $2}' | grep -v '^127\\.' | sed 's/.*/"&"/' | paste -sd, -) IPS=$(/sbin/ifconfig | awk '/inet /{print $2}' | grep -v '^127\\.' | sed 's/.*/"&"/' | paste -sd, -)
BODY="{\\"user\\":\\"$(esc "$(id -un)")\\",\\"name\\":\\"$(esc "$(scutil --get ComputerName)")\\",\\"hostname\\":\\"$(esc "$(scutil --get LocalHostName).local")\\",\\"os\\":\\"mac\\",\\"ips\\":[$IPS]}" BODY="{\\"user\\":\\"$(esc "$U")\\",\\"name\\":\\"$(esc "$(scutil --get ComputerName)")\\",\\"hostname\\":\\"$(esc "$(scutil --get LocalHostName).local")\\",\\"os\\":\\"mac\\",\\"ips\\":[$IPS]}"
if /usr/bin/curl -fsS -X POST -H 'Content-Type: application/json' -d "$BODY" %(hello)s >/dev/null; then if /usr/bin/curl -fsS -X POST -H 'Content-Type: application/json' -d "$BODY" %(hello)s >/dev/null; then
echo "Joined. %(ingest)s sets this Mac up in about a minute: watch its status page." echo "Joined. %(ingest)s sets this Mac up in about a minute: watch its status page."
else else
@@ -521,11 +563,12 @@ def main(argv=None):
ap.add_argument("--name") ap.add_argument("--name")
ap.add_argument("--data-dir", default=PARENT) ap.add_argument("--data-dir", default=PARENT)
ap.add_argument("--home", help="the settings page this campus was installed from (the campus list lives there)") ap.add_argument("--home", help="the settings page this campus was installed from (the campus list lives there)")
ap.add_argument("--pages", help="rebuild onto these Status,Restart pages (19,20 on every campus)")
a = ap.parse_args(argv) a = ap.parse_args(argv)
if a.what == "setup": if a.what == "setup":
setup(a.data_dir, a.id, a.name, home=a.home) setup(a.data_dir, a.id, a.name, home=a.home)
elif a.what == "rebuild": elif a.what == "rebuild":
rebuild(a.data_dir) rebuild(a.data_dir, pages=tuple(int(p) for p in a.pages.split(",")) if a.pages else None)
elif a.what == "upgrade-companion": elif a.what == "upgrade-companion":
upgrade_companion(a.home or (load_campuses_home(a.data_dir) or "http://10.1.1.1:8020")) upgrade_companion(a.home or (load_campuses_home(a.data_dir) or "http://10.1.1.1:8020"))
elif a.what == "look": elif a.what == "look":
+359
View File
@@ -0,0 +1,359 @@
{
"site": {
"id": "sc",
"name": "Southcoast",
"ingest_name": "SC ProPresenter",
"ingest_hostname": "SC-ProPresenter-423.local",
"ingest_ips": [
"10.5.2.60",
"192.168.2.1",
"169.254.161.214",
"169.254.108.73"
],
"contacts": "",
"live_services": [],
"editing_code": "5690"
},
"companion": {
"host": "127.0.0.1",
"port": 8000
},
"page": 19,
"page_name": "19\\nStatus",
"page_hold_ms": 2000,
"variables": {
"state": "sp_state",
"heartbeat": "sp_hb",
"command": "sp_cmd"
},
"timing": {
"fix_cooldown_seconds": 60,
"alert_after_seconds": 30
},
"auto_fix": {
"after_seconds": 20,
"max_tries": 2,
"retry_seconds": 90,
"pause_after_network_seconds": 60,
"restart_connections": true,
"windows": [
{
"days": [
"tue"
],
"start": "14:00",
"end": "22:00"
},
{
"days": [
"wed"
],
"start": "09:00",
"end": "21:00"
},
{
"days": [
"sun"
],
"start": "05:00",
"end": "14:00"
}
],
"fixes": [
"propresenter_ingest",
"prop_port_propresenter_api",
"hyperdeck",
"midi_reconnect"
]
},
"tiles": [
{
"id": "all",
"label": "ALL",
"pos": [
0,
1
],
"kind": "master"
},
{
"id": "service",
"label": "ALARMS",
"pos": [
3,
0
],
"kind": "service"
},
{
"id": "net",
"label": "NET",
"pos": [
3,
7
],
"kind": "network"
},
{
"id": "hd",
"label": "HD",
"checks": [
{
"type": "hyperdeck",
"conn": "hyperdeck",
"fix": {
"conn": "hyperdeck"
}
}
],
"pos": [
0,
2
]
},
{
"id": "foh_x32",
"label": "FOH X32",
"checks": [
{
"type": "conn",
"conn": "FOH_X32"
}
],
"pos": [
0,
3
]
},
{
"id": "iem_x32",
"label": "IEM X32",
"checks": [
{
"type": "conn",
"conn": "IEM_X32"
}
],
"pos": [
0,
4
]
},
{
"id": "prop",
"label": "PROP",
"checks": [
{
"type": "conn",
"conn": "Propresenter-API",
"fix": "prop_port_propresenter_api"
},
{
"type": "process",
"match": "/Applications/ProPresenter 20.0.1.app/Contents/MacOS/ProPresenter",
"name": "PROP",
"fix": "propresenter_ingest"
}
],
"pos": [
0,
5
]
},
{
"id": "atem",
"label": "ATEM",
"checks": [
{
"type": "conn",
"conn": "atem"
}
],
"pos": [
0,
6
]
},
{
"id": "resi",
"label": "RESI",
"checks": [
{
"type": "conn",
"conn": "resi"
},
{
"type": "ping",
"conn": "resi"
}
],
"pos": [
0,
7
]
},
{
"id": "midi",
"label": "MIDI",
"checks": [
{
"type": "midi_peer",
"session": "ProPresenter",
"peer": "Lighting",
"address": "10.5.1.168:5004",
"fix": "midi_reconnect",
"tag": "PROP LIGHTS"
},
{
"type": "midi_wired",
"nets": [
"10.5.0.0/16"
],
"fix": "midi_reconnect"
}
],
"pos": [
1,
1
]
}
],
"fixes": {
"propresenter_ingest": {
"template": "mac_start",
"params": {
"host": "ingest",
"process": "ProPresenter",
"app": "/Applications/ProPresenter 20.0.1.app"
},
"label": "start ProPresenter on this Mac if it is not running",
"short": "PROP",
"kind": "local",
"run": "pgrep -xq ProPresenter || open '/Applications/ProPresenter 20.0.1.app'"
},
"restart_propresenter_ingest": {
"template": "mac_restart",
"params": {
"host": "ingest",
"process": "ProPresenter",
"app": "/Applications/ProPresenter 20.0.1.app",
"wait_seconds": 10,
"force": true,
"background": false,
"stop_children": false,
"newest_of": ""
},
"label": "restart ProPresenter on this Mac (closes it first)",
"short": "PROP",
"kind": "local",
"run": "pkill -TERM -x ProPresenter; i=0; while pgrep -xq ProPresenter && [ $i -lt 20 ]; do sleep 0.5; i=$((i+1)); done; pgrep -xq ProPresenter && pkill -KILL -x ProPresenter; sleep 1; open '/Applications/ProPresenter 20.0.1.app'"
},
"prop_port_propresenter_api": {
"kind": "propresenter_port",
"template": "propresenter_port",
"params": {},
"conn": "Propresenter-API",
"short": "PROP PORT",
"label": "find ProPresenter's network port and reconnect Companion's Propresenter-API to it"
},
"hyperdeck": {
"kind": "hyperdeck",
"template": "hyperdeck",
"params": {},
"short": "HYPERDECK",
"verify_seconds": 15,
"hung_note": "STILL OFFLINE\nHOLD AGAIN",
"label": "turn Companion's connection to that HyperDeck off and on (what gets a restarted deck green); never reboots the deck"
},
"midi_reconnect": {
"kind": "midi_connect",
"template": "midi_connect",
"params": {},
"short": "RECONNECT",
"label": "reconnect the peer to the network MIDI session it dropped out of (like Connect in Audio MIDI Setup), at its wired address"
}
},
"hosts": {
"ingest": {
"self": true,
"name": "SC ProPresenter",
"role": "the watchdog, Companion, the settings page",
"os": "mac",
"networks": [],
"runs": [
{
"kind": "launchagent",
"label": "org.waterschurch.status-panel",
"name": "the watchdog"
},
{
"kind": "launchagent",
"label": "org.waterschurch.status-settings",
"name": "the settings page"
},
{
"kind": "app",
"process": "Companion",
"name": "Companion",
"http": "http://127.0.0.1:8000/api/connections"
}
]
}
},
"gear": {},
"debounce_seconds": 3,
"network": {
"interface": "en11",
"gateway": "10.5.0.1",
"burst_connections": 4,
"burst_groups": 3,
"burst_window_seconds": 10,
"remember_seconds": 300
},
"nav_keys": [
"pagenum"
],
"nav_hold_ms": 1000,
"nav_tap_page": 1,
"discover": {
"auto": true
},
"restart_page": {
"page": 20,
"page_name": "20\\nRestart",
"jump_pos": [
3,
6
],
"back_pos": [
3,
0
],
"keys": [
{
"id": "rs_prop",
"label": "PROP",
"fix": "restart_propresenter_ingest",
"pos": [
0,
1
]
}
]
},
"service_windows": [
{
"days": [
"mon",
"tue",
"wed",
"thu",
"fri",
"sat",
"sun"
],
"start": "00:00",
"end": "23:59"
}
]
}
+58 -2
View File
@@ -44,7 +44,7 @@ log = logging.getLogger("status-panel")
VERSION = "2026.10.15" # logged at start ("status panel running (code ...)") and written to started.json VERSION = "2026.10.15" # logged at start ("status panel running (code ...)") and written to started.json
# Every kind of fix the watchdog can run (validated by load_config). # Every kind of fix the watchdog can run (validated by load_config).
FIX_KINDS = ("ssh", "local", "atem", "reconnect", "midi_connect", "netgear_poe", "hyperdeck", "spotify", "ready", FIX_KINDS = ("ssh", "local", "atem", "reconnect", "midi_connect", "netgear_poe", "hyperdeck", "spotify", "ready",
"companion_press") "companion_press", "propresenter_port")
# Failover: a fix marked "failover" (pressing the FOH Backup button when Studio B dies) runs by itself # Failover: a fix marked "failover" (pressing the FOH Backup button when Studio B dies) runs by itself
# inside these windows when the "failover" block names none, "after_seconds" after its fault is confirmed. # inside these windows when the "failover" block names none, "after_seconds" after its fault is confirmed.
FAILOVER_WINDOWS = [{"days": ["sun"], "start": "06:30", "end": "13:30"}] FAILOVER_WINDOWS = [{"days": ["sun"], "start": "06:30", "end": "13:30"}]
@@ -502,7 +502,7 @@ def check_midi_peer(chk, snap, ctx):
# Measured 2026-10-07: a joining peer reads "connected" for ~6 s until # Measured 2026-10-07: a joining peer reads "connected" for ~6 s until
# clock sync, then "running". A peer that dies is removed from the list: # clock sync, then "running". A peer that dies is removed from the list:
# at once if it says goodbye, after ~60 s if it crashes or its cable is pulled. # at once if it says goodbye, after ~60 s if it crashes or its cable is pulled.
if p.get("state", "running") in ("running", "connected"): # no state: older macOS, configured if (p.get("state") or "running") in ("running", "connected"): # no state: older macOS, configured
return [("peer", Result(OK))] return [("peer", Result(OK))]
return [("peer", Result(ERR, "MIDI " + (p["state"] or "?").upper()[:8]))] return [("peer", Result(ERR, "MIDI " + (p["state"] or "?").upper()[:8]))]
return [("peer", Result(ERR, "MIDI LOST"))] return [("peer", Result(ERR, "MIDI LOST"))]
@@ -2755,6 +2755,60 @@ def execute_poe(cfg, name, entry):
return False return False
def propresenter_ports(cfg, fix):
"""The TCP ports ProPresenter listens on, on the Mac it runs on (this one, or fix "on")."""
cmd = "/usr/sbin/lsof -nP -iTCP -sTCP:LISTEN -a -c ProPresen 2>/dev/null"
argv = ssh_cmd(cfg, fix["on"]) + [cmd] if fix.get("on") else ["/bin/zsh", "-c", cmd]
out = subprocess.run(argv, capture_output=True, text=True, timeout=20).stdout
return sorted({int(m.group(1)) for m in re.finditer(r":(\d+) \(LISTEN\)", out)})
def propresenter_port(cfg, fix, label):
"""ProPresenter picked another network port (it does, now and then: Southcoast's checklist says to
copy it by hand; Woonsocket's had moved from 50822 to 49285): find the port that answers its API
and put it into Companion's connection, then let Companion reconnect. Never touches ProPresenter."""
import sqlite3
comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
conns = {c["label"]: c for c in comp.connections()}
conn = conns.get(fix["conn"])
if not conn:
log.info("FIX %s -> no Companion connection %s", label, fix["conn"])
return False
path = companion_db_path(cfg)
db = sqlite3.connect("file:%s?mode=ro" % urllib.parse.quote(path), uri=True, timeout=2)
try:
conf = next((json.loads(v).get("config") or {} for (v,) in db.execute("SELECT value FROM instances")
if json.loads(v).get("label") == fix["conn"]), {})
finally:
db.close()
host = conf.get("host") or "127.0.0.1"
found = None
for port in propresenter_ports(cfg, fix):
try:
with urllib.request.urlopen("http://%s:%d/version" % (host, port), timeout=2) as r:
if "api_version" in r.read().decode("utf-8", "replace"):
found = port
break
except (OSError, ValueError):
continue
if not found:
log.info("FIX %s -> ProPresenter's network API answers on no port (is ProPresenter open, Network on?)", label)
return "PROP API\nNOT ON"
if int(conf.get("port") or 0) == found:
ok = comp.restart_connection(conn["id"])
log.info("FIX %s -> port %d is already right; connection restarted %s", label, found, "ok" if ok else "FAILED")
return ok
import push_page
ws = push_page.WebSocket(cfg["companion"].get("host", "127.0.0.1"), int(cfg["companion"].get("port", 8000)), "/trpc")
try:
push_page.Trpc(ws).mutation("instances.connections.setConfig", {"connectionId": conn["id"], "label": fix["conn"],
"config": dict(conf, port=found)})
finally:
ws.close()
log.info("FIX %s -> ProPresenter's API moved from port %s to %d: Companion's %s now uses %d", label, conf.get("port"), found, fix["conn"], found)
return True
def execute_fix(cfg, fix_id, chk): def execute_fix(cfg, fix_id, chk):
"""Run one entry from config "fixes" (in its own thread). Only listed commands ever run. """Run one entry from config "fixes" (in its own thread). Only listed commands ever run.
Returns True if the command ran and succeeded.""" Returns True if the command ran and succeeded."""
@@ -2798,6 +2852,8 @@ def execute_fix(cfg, fix_id, chk):
return ok return ok
if kind == "reconnect": if kind == "reconnect":
return reconnect_and_verify(cfg, fix, label) return reconnect_and_verify(cfg, fix, label)
if kind == "propresenter_port":
return propresenter_port(cfg, fix, label)
if kind == "midi_connect": if kind == "midi_connect":
# Like Connect in Audio MIDI Setup: add the peer back to the session it dropped out of. # Like Connect in Audio MIDI Setup: add the peer back to the session it dropped out of.
args = (chk["session"], chk["peer"], chk["address"]) args = (chk["session"], chk["peer"], chk["address"])
+15
View File
@@ -241,6 +241,21 @@ class TestExtend(unittest.TestCase):
self.assertIn("PROP checks on the PROP key", added) self.assertIn("PROP checks on the PROP key", added)
self.assertEqual(schema.validate(new)[0], []) self.assertEqual(schema.validate(new)[0], [])
def test_a_rebuild_keeps_what_is_down_right_now(self):
"""Southcoast's rebuild dropped its MIDI key: the ProPresenter-to-Lighting link was down at that moment."""
old, _ = ac.build(BASE, small(), {"id": "sc", "name": "Southcoast"}, "1234")
now = small()
now["midi"] = {"": {"SC Ingest": {"port": 5004, "peers": []}}} # the link dropped
now["computers"]["lighting_pc"]["reachable"] = False # and the Lighting PC is off
new, _ = ac.build(BASE, now, {"id": "sc", "name": "Southcoast"}, "1234")
self.assertNotIn("MIDI", [t["label"] for t in new["tiles"]])
kept, words = ac.keep_from(new, old)
self.assertIn("MIDI", [t["label"] for t in kept["tiles"]])
self.assertIn("VISTA", [t["label"] for t in kept["tiles"]])
self.assertIn("lighting_pc", kept["hosts"])
self.assertIn("MIDI link Lighting PC", words)
self.assertEqual(schema.validate(kept)[0], [])
def test_a_key_someone_took_off_is_not_added_back(self): def test_a_key_someone_took_off_is_not_added_back(self):
cfg = json.loads(json.dumps(self.cfg)) cfg = json.loads(json.dumps(self.cfg))
cfg["tiles"] = [t for t in cfg["tiles"] if t["label"] != "X32"] cfg["tiles"] = [t for t in cfg["tiles"] if t["label"] != "X32"]