Southcoast matches NA, per its startup checklist: X32s, ProPresenter to Lighting MIDI, pages 19/20
- An explicit look or rebuild may apply inside the auto-fix hours (never while live) - A page move clears the page left behind when only our keys are on it - A rebuild keeps MIDI links that are down now; first-setup import waits for the keys to land - ProPresenter port drift fixed automatically (kind propresenter_port) - sites/sc/config.json kept with the others Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -219,6 +219,7 @@ field("fixes.*.label", "str", None, "Plain words: what it does.", "Fixes", requi
|
||||
field("fixes.*.reconnect", "bool", False, "Reconnect the Companion connection afterwards.", "Fixes")
|
||||
field("fixes.*.conn", "str", None, "The Companion connection it reconnects.", "Fixes")
|
||||
field("fixes.*.restart_ok", "bool", False, "May close a running program (and still run by itself).", "Fixes")
|
||||
field("fixes.*.on", "host", None, "The computer ProPresenter runs on, when not this Mac (follow its network port).", "Fixes")
|
||||
field("fixes.*.never_auto", "bool", False, "Never runs by itself.", "Fixes")
|
||||
field("fixes.*.anytime", "bool", False, "Ticked for auto-fix, runs by itself at any hour, not only in the auto-fix hours, unless ALARMS is muted (a READY fix that only writes to Planning Center).", "Fixes")
|
||||
field("fixes.*.part", "enum", None, "Which part of the switcher.", "Fixes", options=["audio", "camera"])
|
||||
@@ -453,7 +454,7 @@ COMMON_CHECK_FIELDS = ["type", "fix", "tag", "service_only", "debounce_seconds",
|
||||
FIX_KINDS = sp.FIX_KINDS
|
||||
FIX_KIND_FIELDS = {
|
||||
"ssh": ["host", "run", "timeout", "reconnect", "conn"], "local": ["run", "timeout", "reconnect", "conn"],
|
||||
"atem": ["part"], "reconnect": ["conn", "verify_seconds", "hung_note"], "midi_connect": [],
|
||||
"atem": ["part"], "reconnect": ["conn", "verify_seconds", "hung_note"], "midi_connect": [], "propresenter_port": ["conn", "on"],
|
||||
"netgear_poe": ["switch", "port", "mac"], "hyperdeck": ["verify_seconds", "off_seconds", "hung_note"],
|
||||
"spotify": ["playlist"], "ready": ["action"], "companion_press": ["page", "row", "col"],
|
||||
}
|
||||
|
||||
+8
-2
@@ -443,9 +443,15 @@ class Applier:
|
||||
if not ok:
|
||||
return False, "page %d: %s" % (page, why)
|
||||
push_page.push(host, port, page, path, expected)
|
||||
after = push_page.fetch_page(host, port, page)
|
||||
want = {(r, c) for r, row in export["page"]["controls"].items() for c, ctl in row.items() if ctl and ctl.get("type") == "button-layered"}
|
||||
have = {(r, c) for r, row in (after.get("page") or {}).get("controls", {}).items() for c, ctl in (row or {}).items() if ctl and ctl.get("type") == "button-layered"}
|
||||
# Companion places the keys a moment after the import call returns (Southcoast's first setup
|
||||
# was judged too early, 2026-10-21): look again for up to 10 s before calling it a failure.
|
||||
for _ in range(20):
|
||||
after = push_page.fetch_page(host, port, page)
|
||||
have = {(r, c) for r, row in (after.get("page") or {}).get("controls", {}).items() for c, ctl in (row or {}).items() if ctl and ctl.get("type") == "button-layered"}
|
||||
if want == have:
|
||||
break
|
||||
time.sleep(0.5)
|
||||
if want != have:
|
||||
return False, "page %d: the import did not land as built (%d keys built, %d on the page)" % (page, len(want), len(have))
|
||||
done.append(name)
|
||||
|
||||
+54
-2
@@ -283,14 +283,23 @@ def items(inv, nets):
|
||||
"checks": [{"type": spec["kind"], "conn": label, "channels": n}, {"type": "ping", "conn": label}], "fixes": {}})
|
||||
continue
|
||||
checks = [{"type": "conn", "conn": label}]
|
||||
fixes = {}
|
||||
if conn.get("module") == "renewedvision-propresenter-api":
|
||||
# ProPresenter changes its network port now and then: the fix finds the new one.
|
||||
fid = slug("prop_port_" + label)
|
||||
where = computer_by_ip(inv, ip_of(conn.get("host")) or conn.get("host"))
|
||||
fixes[fid] = dict({"kind": "propresenter_port", "template": "propresenter_port", "params": {}, "conn": label, "short": "PROP PORT",
|
||||
"label": "find ProPresenter's network port and reconnect Companion's %s to it" % label}, **({"on": where} if where else {}))
|
||||
checks[0]["fix"] = fid
|
||||
if spec.get("ping") and conn.get("host"):
|
||||
checks.append({"type": "ping", "conn": label})
|
||||
app_item = apps_at.get((spec.get("app"), ip_of(conn.get("host")))) if spec.get("app") else None
|
||||
if app_item: # the program's own key carries its connection too
|
||||
app_item["checks"] = checks + app_item["checks"]
|
||||
app_item["fixes"].update(fixes)
|
||||
app_item["also"].append(ident)
|
||||
continue
|
||||
out.append({"ident": ident, "group": spec["group"], "label": text, "checks": checks, "fixes": {}, "restart": [], "also": []})
|
||||
out.append({"ident": ident, "group": spec["group"], "label": text, "checks": checks, "fixes": fixes, "restart": [], "also": []})
|
||||
# Network MIDI: every peer in the sessions here and on the other Macs, at its wired address.
|
||||
midi_checks = []
|
||||
# Whose session a peer is, by its name: a peer joined by name or by an IPv6 address gets that Mac's wired address.
|
||||
@@ -555,6 +564,49 @@ def merge_parts(cfg, it, fresh):
|
||||
return words
|
||||
|
||||
|
||||
def keep_from(new, old):
|
||||
"""A rebuild keeps what is down right now and so was not found: MIDI links that dropped (Southcoast's
|
||||
ProPresenter to Lighting lost its MIDI key, 2026-10-21) and computers that are off, with their keys."""
|
||||
cfg = copy.deepcopy(new)
|
||||
have = referenced(cfg)
|
||||
kept = []
|
||||
# Computers in the old setup that did not answer this time, and the keys that check them.
|
||||
gone = {a: h for a, h in (old.get("hosts") or {}).items() if not h.get("self") and a not in cfg["hosts"]}
|
||||
for alias, h in gone.items():
|
||||
cfg["hosts"][alias] = h
|
||||
for t in old.get("tiles", []):
|
||||
if t.get("checks") and all(c.get("host") == alias or (c.get("params") or {}).get("host") == alias for c in t["checks"]) \
|
||||
and t["id"] not in {x["id"] for x in cfg["tiles"]}:
|
||||
for c in t["checks"]:
|
||||
for fid in ([c["fix"]] if isinstance(c.get("fix"), str) else []):
|
||||
if fid in old.get("fixes", {}):
|
||||
cfg["fixes"].setdefault(fid, old["fixes"][fid])
|
||||
place_new(cfg, {k: v for k, v in t.items() if k != "pos"})
|
||||
kept.append(t["label"])
|
||||
for k in (old.get("restart_page") or {}).get("keys", []):
|
||||
f = old.get("fixes", {}).get(k.get("fix"), {})
|
||||
if (f.get("host") == alias or (f.get("params") or {}).get("host") == alias) and cfg.get("restart_page"):
|
||||
cfg["fixes"].setdefault(k["fix"], f)
|
||||
restart_layout(cfg, [{"label": k["label"], "fix": k["fix"]}])
|
||||
# MIDI links that are down now.
|
||||
lost = [c for t in old.get("tiles", []) for c in t.get("checks", []) if c.get("type") == "midi_peer"
|
||||
and "midi:%s|%s|%s" % (c.get("host", ""), c["session"], c["peer"]) not in have and (not c.get("host") or c["host"] in cfg["hosts"])]
|
||||
if lost:
|
||||
midi = next((t for t in cfg["tiles"] if any(c.get("type") in ("midi_peer", "midi_wired") for c in t.get("checks", []))), None)
|
||||
if midi is None:
|
||||
wired = [c for t in old.get("tiles", []) for c in t.get("checks", []) if c.get("type") == "midi_wired"]
|
||||
midi = {"id": slug("midi", {t["id"] for t in cfg["tiles"]}), "label": "MIDI", "checks": wired[:1], "_group": "SHOW"}
|
||||
place_new(cfg, midi)
|
||||
midi = cfg["tiles"][-1]
|
||||
at = next((i for i, c in enumerate(midi["checks"]) if c.get("type") == "midi_wired"), len(midi["checks"]))
|
||||
midi["checks"][at:at] = lost
|
||||
cfg["fixes"].setdefault("midi_reconnect", dict(SHARED_FIXES["midi_reconnect"]))
|
||||
if "midi_reconnect" not in cfg["auto_fix"]["fixes"]:
|
||||
cfg["auto_fix"]["fixes"].append("midi_reconnect")
|
||||
kept += ["MIDI link %s" % c["peer"] for c in lost]
|
||||
return cfg, kept
|
||||
|
||||
|
||||
def finish(cfg):
|
||||
"""Fixes and remote checks built by their builders, words filled in, as the page would."""
|
||||
for fid, f in list(cfg["fixes"].items()):
|
||||
@@ -578,7 +630,7 @@ def gentle(cfg):
|
||||
for fid, f in cfg["fixes"].items():
|
||||
if fid.startswith("restart_") or f.get("restart_ok") or f.get("never_auto"):
|
||||
continue
|
||||
if f.get("kind") in ("hyperdeck", "midi_connect", "reconnect") or (f.get("template") in ("mac_start", "win_start", "ableton")
|
||||
if f.get("kind") in ("hyperdeck", "midi_connect", "reconnect", "propresenter_port") or (f.get("template") in ("mac_start", "win_start", "ableton")
|
||||
and not (f.get("params") or {}).get("restart")):
|
||||
out.append(fid)
|
||||
return out
|
||||
|
||||
@@ -219,6 +219,8 @@ TEMPLATES = {
|
||||
"suggest": lambda p, cfg: "reconnect Companion to it"},
|
||||
"hyperdeck": {"words": "Switch a HyperDeck's connection off and on", "kind": "hyperdeck", "start_only": True, "params": [], "render": None,
|
||||
"suggest": lambda p, cfg: "turn Companion's connection to that HyperDeck off and on; never reboots the deck"},
|
||||
"propresenter_port": {"words": "Follow ProPresenter's network port (when it changes) and reconnect Companion", "kind": "propresenter_port",
|
||||
"start_only": True, "params": [], "render": None, "suggest": lambda p, cfg: "find ProPresenter's network port and reconnect Companion to it"},
|
||||
"midi_connect": {"words": "Reconnect a network MIDI peer", "kind": "midi_connect", "start_only": True, "params": [], "render": None,
|
||||
"suggest": lambda p, cfg: "reconnect the peer to the network MIDI session it dropped out of"},
|
||||
"atem": {"words": "Put the switcher back to the saved file", "kind": "atem", "params": [], "render": None,
|
||||
|
||||
+56
-13
@@ -211,7 +211,7 @@ def setup(data_dir, cid, name=None, say=print, home=None):
|
||||
return new
|
||||
|
||||
|
||||
def rebuild(data_dir, say=print):
|
||||
def rebuild(data_dir, say=print, pages=None):
|
||||
"""A campus's setup made again from what is found now (today's recipes), keeping its name, PIN,
|
||||
pages and site details. Applied like any Apply (it waits for a recording; history keeps the old one)."""
|
||||
from settings import server
|
||||
@@ -224,7 +224,17 @@ def rebuild(data_dir, say=print):
|
||||
rp = (cfg.get("restart_page") or {}).get("page")
|
||||
if not rp:
|
||||
rp = next(p for p in pick_pages(cfg, 3) if p != cfg["page"])
|
||||
new, seen = autoconf.build(template(data_dir), inv, {"id": s.get("id"), "name": s.get("name")}, s.get("editing_code"), (cfg["page"], rp))
|
||||
pages = tuple(pages or (cfg["page"], rp))
|
||||
if pages != (cfg["page"], rp): # moving (19/20 like every campus): only onto pages that are free or ours
|
||||
comp = cfg.get("companion") or {}
|
||||
for p in pages:
|
||||
ok, why = push_page.safe_target(push_page.fetch_page(comp.get("host", "127.0.0.1"), int(comp.get("port", 8000)), p))
|
||||
if not ok:
|
||||
raise RuntimeError("page %d: %s" % (p, why))
|
||||
new, seen = autoconf.build(template(data_dir), inv, {"id": s.get("id"), "name": s.get("name")}, s.get("editing_code"), pages)
|
||||
new, kept = autoconf.keep_from(new, cfg)
|
||||
if kept:
|
||||
say("Kept (down right now, so not found): %s" % ", ".join(kept))
|
||||
for k in ("contacts", "live_services", "ingest_name", "settings_port"):
|
||||
if k in s:
|
||||
new["site"][k] = s[k]
|
||||
@@ -238,11 +248,29 @@ def rebuild(data_dir, say=print):
|
||||
st = wait_applied(site.applier)
|
||||
save_seen(data_dir, load_seen(data_dir) | set(seen))
|
||||
say("%s: %s" % (st.get("state"), st.get("outcome", "")))
|
||||
if st.get("state") == "done":
|
||||
for p in sorted({cfg["page"], rp} - set(pages)):
|
||||
say("Old page %d: %s" % (p, clear_old_page(cfg, p)))
|
||||
say("Keys: %s" % ", ".join(t["label"] for t in new["tiles"] if not t.get("kind")))
|
||||
say("Restart page %s: %s" % (rp, ", ".join(k["label"] for k in new.get("restart_page", {}).get("keys", [])) or "no keys yet"))
|
||||
say("Restart page %s: %s" % (new.get("restart_page", {}).get("page"), ", ".join(k["label"] for k in new.get("restart_page", {}).get("keys", [])) or "no keys yet"))
|
||||
return new
|
||||
|
||||
|
||||
def clear_old_page(cfg, page):
|
||||
"""A page left behind by a move is emptied, but only when it holds nothing a person made."""
|
||||
comp = cfg.get("companion") or {}
|
||||
host, port = comp.get("host", "127.0.0.1"), int(comp.get("port", 8000))
|
||||
ok, why = push_page.safe_target(push_page.fetch_page(host, port, page))
|
||||
if not ok:
|
||||
return "left as it is (%s)" % why
|
||||
ws = push_page.WebSocket(host, port, "/trpc")
|
||||
try:
|
||||
push_page.Trpc(ws).mutation("pages.clearPage", {"pageNumber": page})
|
||||
finally:
|
||||
ws.close()
|
||||
return "cleared"
|
||||
|
||||
|
||||
def upgrade_companion(base, say=print):
|
||||
"""Companion 5.0.7 from `base` (North Attleboro's settings page), as Woonsocket was upgraded on 2026-10-20:
|
||||
the running Companion is quit, the newest settings folder is copied for 5.0 (and marked newest, or the
|
||||
@@ -305,13 +333,14 @@ def record(data_dir, result):
|
||||
|
||||
# --- every ten minutes: what is new ------------------------------------------------------
|
||||
|
||||
def blocked(cfg, applier, now=None):
|
||||
"""Why auto-setup may not apply right now, or None: never during a service or a recording."""
|
||||
def blocked(cfg, applier, now=None, asked=False):
|
||||
"""Why auto-setup may not apply right now, or None: never during a recording; by itself, never in the
|
||||
auto-fix hours either (a person asking - Look now, setting up on site - may)."""
|
||||
now = now or time.time()
|
||||
live, why = applier.live()
|
||||
if live:
|
||||
return "live now (%s)" % why
|
||||
if sp.in_windows((cfg.get("auto_fix") or {}).get("windows") or [], now):
|
||||
if not asked and sp.in_windows((cfg.get("auto_fix") or {}).get("windows") or [], now):
|
||||
return "inside the auto-fix hours (a service day): waits until they end"
|
||||
if applier.busy():
|
||||
return "an Apply is running"
|
||||
@@ -327,7 +356,7 @@ def look_and_add(site, say=lambda s: None, force=False):
|
||||
result = {"at": int(time.time()), "added": [], "waiting": [], "inventory": summary(inv)}
|
||||
if added:
|
||||
auto = (cfg.get("discover") or {}).get("auto")
|
||||
why = blocked(cfg, site.applier)
|
||||
why = blocked(cfg, site.applier, asked=force)
|
||||
if not (auto or force):
|
||||
result["waiting"] = added
|
||||
result["why"] = "adding by itself is off (Settings > Site)"
|
||||
@@ -449,10 +478,14 @@ if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then
|
||||
echo
|
||||
echo "One more thing: turn on Remote Login (System Settings > General > Sharing), so North Attleboro can help remotely."
|
||||
fi
|
||||
# This Mac's wired address (the default route's port, not en0: at Southcoast en0 is a Dante-style link).
|
||||
IF=$(/sbin/route -n get default 2>/dev/null | awk '/interface:/{print $2}')
|
||||
ME=$(/usr/sbin/ipconfig getifaddr "$IF" 2>/dev/null || hostname)
|
||||
echo
|
||||
echo "Next, on each other computer here (ProPresenter, Tracks, Lighting...):"
|
||||
echo " Mac: open Terminal and paste curl -fsSL http://$(ipconfig getifaddr en0 2>/dev/null || hostname):8020/join | sh"
|
||||
echo " PC: open PowerShell as administrator and paste irm http://$(ipconfig getifaddr en0 2>/dev/null || hostname):8020/join.ps1 | iex"
|
||||
echo " Mac: open Terminal and paste curl -fsSL http://$ME:8020/join | sh"
|
||||
echo " (or send it to all of them at once in Apple Remote Desktop: Send UNIX Command, as root)"
|
||||
echo " PC: open PowerShell as administrator and paste irm http://$ME:8020/join.ps1 | iex"
|
||||
""" % {"name": name, "base": shq(base), "key": shq(key), "cid": cid, "qname": shq(name)}
|
||||
|
||||
|
||||
@@ -464,8 +497,17 @@ def join_mac(cfg, base):
|
||||
return """#!/bin/sh
|
||||
# Joins this Mac to %(ingest)s's status panel: lets it log in with its key (no password) and says this Mac is here.
|
||||
KEY=%(key)s
|
||||
mkdir -p "$HOME/.ssh" && touch "$HOME/.ssh/authorized_keys" && chmod 700 "$HOME/.ssh" && chmod 600 "$HOME/.ssh/authorized_keys"
|
||||
grep -qF "$KEY" "$HOME/.ssh/authorized_keys" || echo "$KEY" >> "$HOME/.ssh/authorized_keys"
|
||||
# Run as root (Apple Remote Desktop > Send UNIX Command): it is for whoever is logged in on this Mac.
|
||||
U=$(id -un); H="$HOME"
|
||||
if [ "$(id -u)" = 0 ]; then
|
||||
U=$(stat -f%%Su /dev/console)
|
||||
[ "$U" = root ] && { echo "Nobody is logged in on $(scutil --get ComputerName): log in, then send this again."; exit 1; }
|
||||
H=$(dscl . -read "/Users/$U" NFSHomeDirectory | awk '{print $2}')
|
||||
fi
|
||||
mkdir -p "$H/.ssh" && touch "$H/.ssh/authorized_keys" && chmod 700 "$H/.ssh" && chmod 600 "$H/.ssh/authorized_keys"
|
||||
grep -qF "$KEY" "$H/.ssh/authorized_keys" || echo "$KEY" >> "$H/.ssh/authorized_keys"
|
||||
[ "$(id -u)" = 0 ] && chown -R "$U:staff" "$H/.ssh"
|
||||
[ "$(id -u)" = 0 ] && { /usr/sbin/systemsetup -setremotelogin on >/dev/null 2>&1 || /bin/launchctl load -w /System/Library/LaunchDaemons/ssh.plist >/dev/null 2>&1; }
|
||||
if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then
|
||||
echo "Remote Login is off. Turning it on (it asks for this Mac's administrator password):"
|
||||
if ! sudo /usr/sbin/systemsetup -setremotelogin on >/dev/null 2>&1; then
|
||||
@@ -476,7 +518,7 @@ if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then
|
||||
fi
|
||||
esc() { printf '%%s' "$1" | sed 's/[\\\\"]//g'; }
|
||||
IPS=$(/sbin/ifconfig | awk '/inet /{print $2}' | grep -v '^127\\.' | sed 's/.*/"&"/' | paste -sd, -)
|
||||
BODY="{\\"user\\":\\"$(esc "$(id -un)")\\",\\"name\\":\\"$(esc "$(scutil --get ComputerName)")\\",\\"hostname\\":\\"$(esc "$(scutil --get LocalHostName).local")\\",\\"os\\":\\"mac\\",\\"ips\\":[$IPS]}"
|
||||
BODY="{\\"user\\":\\"$(esc "$U")\\",\\"name\\":\\"$(esc "$(scutil --get ComputerName)")\\",\\"hostname\\":\\"$(esc "$(scutil --get LocalHostName).local")\\",\\"os\\":\\"mac\\",\\"ips\\":[$IPS]}"
|
||||
if /usr/bin/curl -fsS -X POST -H 'Content-Type: application/json' -d "$BODY" %(hello)s >/dev/null; then
|
||||
echo "Joined. %(ingest)s sets this Mac up in about a minute: watch its status page."
|
||||
else
|
||||
@@ -521,11 +563,12 @@ def main(argv=None):
|
||||
ap.add_argument("--name")
|
||||
ap.add_argument("--data-dir", default=PARENT)
|
||||
ap.add_argument("--home", help="the settings page this campus was installed from (the campus list lives there)")
|
||||
ap.add_argument("--pages", help="rebuild onto these Status,Restart pages (19,20 on every campus)")
|
||||
a = ap.parse_args(argv)
|
||||
if a.what == "setup":
|
||||
setup(a.data_dir, a.id, a.name, home=a.home)
|
||||
elif a.what == "rebuild":
|
||||
rebuild(a.data_dir)
|
||||
rebuild(a.data_dir, pages=tuple(int(p) for p in a.pages.split(",")) if a.pages else None)
|
||||
elif a.what == "upgrade-companion":
|
||||
upgrade_companion(a.home or (load_campuses_home(a.data_dir) or "http://10.1.1.1:8020"))
|
||||
elif a.what == "look":
|
||||
|
||||
@@ -0,0 +1,359 @@
|
||||
{
|
||||
"site": {
|
||||
"id": "sc",
|
||||
"name": "Southcoast",
|
||||
"ingest_name": "SC ProPresenter",
|
||||
"ingest_hostname": "SC-ProPresenter-423.local",
|
||||
"ingest_ips": [
|
||||
"10.5.2.60",
|
||||
"192.168.2.1",
|
||||
"169.254.161.214",
|
||||
"169.254.108.73"
|
||||
],
|
||||
"contacts": "",
|
||||
"live_services": [],
|
||||
"editing_code": "5690"
|
||||
},
|
||||
"companion": {
|
||||
"host": "127.0.0.1",
|
||||
"port": 8000
|
||||
},
|
||||
"page": 19,
|
||||
"page_name": "19\\nStatus",
|
||||
"page_hold_ms": 2000,
|
||||
"variables": {
|
||||
"state": "sp_state",
|
||||
"heartbeat": "sp_hb",
|
||||
"command": "sp_cmd"
|
||||
},
|
||||
"timing": {
|
||||
"fix_cooldown_seconds": 60,
|
||||
"alert_after_seconds": 30
|
||||
},
|
||||
"auto_fix": {
|
||||
"after_seconds": 20,
|
||||
"max_tries": 2,
|
||||
"retry_seconds": 90,
|
||||
"pause_after_network_seconds": 60,
|
||||
"restart_connections": true,
|
||||
"windows": [
|
||||
{
|
||||
"days": [
|
||||
"tue"
|
||||
],
|
||||
"start": "14:00",
|
||||
"end": "22:00"
|
||||
},
|
||||
{
|
||||
"days": [
|
||||
"wed"
|
||||
],
|
||||
"start": "09:00",
|
||||
"end": "21:00"
|
||||
},
|
||||
{
|
||||
"days": [
|
||||
"sun"
|
||||
],
|
||||
"start": "05:00",
|
||||
"end": "14:00"
|
||||
}
|
||||
],
|
||||
"fixes": [
|
||||
"propresenter_ingest",
|
||||
"prop_port_propresenter_api",
|
||||
"hyperdeck",
|
||||
"midi_reconnect"
|
||||
]
|
||||
},
|
||||
"tiles": [
|
||||
{
|
||||
"id": "all",
|
||||
"label": "ALL",
|
||||
"pos": [
|
||||
0,
|
||||
1
|
||||
],
|
||||
"kind": "master"
|
||||
},
|
||||
{
|
||||
"id": "service",
|
||||
"label": "ALARMS",
|
||||
"pos": [
|
||||
3,
|
||||
0
|
||||
],
|
||||
"kind": "service"
|
||||
},
|
||||
{
|
||||
"id": "net",
|
||||
"label": "NET",
|
||||
"pos": [
|
||||
3,
|
||||
7
|
||||
],
|
||||
"kind": "network"
|
||||
},
|
||||
{
|
||||
"id": "hd",
|
||||
"label": "HD",
|
||||
"checks": [
|
||||
{
|
||||
"type": "hyperdeck",
|
||||
"conn": "hyperdeck",
|
||||
"fix": {
|
||||
"conn": "hyperdeck"
|
||||
}
|
||||
}
|
||||
],
|
||||
"pos": [
|
||||
0,
|
||||
2
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "foh_x32",
|
||||
"label": "FOH X32",
|
||||
"checks": [
|
||||
{
|
||||
"type": "conn",
|
||||
"conn": "FOH_X32"
|
||||
}
|
||||
],
|
||||
"pos": [
|
||||
0,
|
||||
3
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "iem_x32",
|
||||
"label": "IEM X32",
|
||||
"checks": [
|
||||
{
|
||||
"type": "conn",
|
||||
"conn": "IEM_X32"
|
||||
}
|
||||
],
|
||||
"pos": [
|
||||
0,
|
||||
4
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "prop",
|
||||
"label": "PROP",
|
||||
"checks": [
|
||||
{
|
||||
"type": "conn",
|
||||
"conn": "Propresenter-API",
|
||||
"fix": "prop_port_propresenter_api"
|
||||
},
|
||||
{
|
||||
"type": "process",
|
||||
"match": "/Applications/ProPresenter 20.0.1.app/Contents/MacOS/ProPresenter",
|
||||
"name": "PROP",
|
||||
"fix": "propresenter_ingest"
|
||||
}
|
||||
],
|
||||
"pos": [
|
||||
0,
|
||||
5
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "atem",
|
||||
"label": "ATEM",
|
||||
"checks": [
|
||||
{
|
||||
"type": "conn",
|
||||
"conn": "atem"
|
||||
}
|
||||
],
|
||||
"pos": [
|
||||
0,
|
||||
6
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "resi",
|
||||
"label": "RESI",
|
||||
"checks": [
|
||||
{
|
||||
"type": "conn",
|
||||
"conn": "resi"
|
||||
},
|
||||
{
|
||||
"type": "ping",
|
||||
"conn": "resi"
|
||||
}
|
||||
],
|
||||
"pos": [
|
||||
0,
|
||||
7
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "midi",
|
||||
"label": "MIDI",
|
||||
"checks": [
|
||||
{
|
||||
"type": "midi_peer",
|
||||
"session": "ProPresenter",
|
||||
"peer": "Lighting",
|
||||
"address": "10.5.1.168:5004",
|
||||
"fix": "midi_reconnect",
|
||||
"tag": "PROP LIGHTS"
|
||||
},
|
||||
{
|
||||
"type": "midi_wired",
|
||||
"nets": [
|
||||
"10.5.0.0/16"
|
||||
],
|
||||
"fix": "midi_reconnect"
|
||||
}
|
||||
],
|
||||
"pos": [
|
||||
1,
|
||||
1
|
||||
]
|
||||
}
|
||||
],
|
||||
"fixes": {
|
||||
"propresenter_ingest": {
|
||||
"template": "mac_start",
|
||||
"params": {
|
||||
"host": "ingest",
|
||||
"process": "ProPresenter",
|
||||
"app": "/Applications/ProPresenter 20.0.1.app"
|
||||
},
|
||||
"label": "start ProPresenter on this Mac if it is not running",
|
||||
"short": "PROP",
|
||||
"kind": "local",
|
||||
"run": "pgrep -xq ProPresenter || open '/Applications/ProPresenter 20.0.1.app'"
|
||||
},
|
||||
"restart_propresenter_ingest": {
|
||||
"template": "mac_restart",
|
||||
"params": {
|
||||
"host": "ingest",
|
||||
"process": "ProPresenter",
|
||||
"app": "/Applications/ProPresenter 20.0.1.app",
|
||||
"wait_seconds": 10,
|
||||
"force": true,
|
||||
"background": false,
|
||||
"stop_children": false,
|
||||
"newest_of": ""
|
||||
},
|
||||
"label": "restart ProPresenter on this Mac (closes it first)",
|
||||
"short": "PROP",
|
||||
"kind": "local",
|
||||
"run": "pkill -TERM -x ProPresenter; i=0; while pgrep -xq ProPresenter && [ $i -lt 20 ]; do sleep 0.5; i=$((i+1)); done; pgrep -xq ProPresenter && pkill -KILL -x ProPresenter; sleep 1; open '/Applications/ProPresenter 20.0.1.app'"
|
||||
},
|
||||
"prop_port_propresenter_api": {
|
||||
"kind": "propresenter_port",
|
||||
"template": "propresenter_port",
|
||||
"params": {},
|
||||
"conn": "Propresenter-API",
|
||||
"short": "PROP PORT",
|
||||
"label": "find ProPresenter's network port and reconnect Companion's Propresenter-API to it"
|
||||
},
|
||||
"hyperdeck": {
|
||||
"kind": "hyperdeck",
|
||||
"template": "hyperdeck",
|
||||
"params": {},
|
||||
"short": "HYPERDECK",
|
||||
"verify_seconds": 15,
|
||||
"hung_note": "STILL OFFLINE\nHOLD AGAIN",
|
||||
"label": "turn Companion's connection to that HyperDeck off and on (what gets a restarted deck green); never reboots the deck"
|
||||
},
|
||||
"midi_reconnect": {
|
||||
"kind": "midi_connect",
|
||||
"template": "midi_connect",
|
||||
"params": {},
|
||||
"short": "RECONNECT",
|
||||
"label": "reconnect the peer to the network MIDI session it dropped out of (like Connect in Audio MIDI Setup), at its wired address"
|
||||
}
|
||||
},
|
||||
"hosts": {
|
||||
"ingest": {
|
||||
"self": true,
|
||||
"name": "SC ProPresenter",
|
||||
"role": "the watchdog, Companion, the settings page",
|
||||
"os": "mac",
|
||||
"networks": [],
|
||||
"runs": [
|
||||
{
|
||||
"kind": "launchagent",
|
||||
"label": "org.waterschurch.status-panel",
|
||||
"name": "the watchdog"
|
||||
},
|
||||
{
|
||||
"kind": "launchagent",
|
||||
"label": "org.waterschurch.status-settings",
|
||||
"name": "the settings page"
|
||||
},
|
||||
{
|
||||
"kind": "app",
|
||||
"process": "Companion",
|
||||
"name": "Companion",
|
||||
"http": "http://127.0.0.1:8000/api/connections"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"gear": {},
|
||||
"debounce_seconds": 3,
|
||||
"network": {
|
||||
"interface": "en11",
|
||||
"gateway": "10.5.0.1",
|
||||
"burst_connections": 4,
|
||||
"burst_groups": 3,
|
||||
"burst_window_seconds": 10,
|
||||
"remember_seconds": 300
|
||||
},
|
||||
"nav_keys": [
|
||||
"pagenum"
|
||||
],
|
||||
"nav_hold_ms": 1000,
|
||||
"nav_tap_page": 1,
|
||||
"discover": {
|
||||
"auto": true
|
||||
},
|
||||
"restart_page": {
|
||||
"page": 20,
|
||||
"page_name": "20\\nRestart",
|
||||
"jump_pos": [
|
||||
3,
|
||||
6
|
||||
],
|
||||
"back_pos": [
|
||||
3,
|
||||
0
|
||||
],
|
||||
"keys": [
|
||||
{
|
||||
"id": "rs_prop",
|
||||
"label": "PROP",
|
||||
"fix": "restart_propresenter_ingest",
|
||||
"pos": [
|
||||
0,
|
||||
1
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"service_windows": [
|
||||
{
|
||||
"days": [
|
||||
"mon",
|
||||
"tue",
|
||||
"wed",
|
||||
"thu",
|
||||
"fri",
|
||||
"sat",
|
||||
"sun"
|
||||
],
|
||||
"start": "00:00",
|
||||
"end": "23:59"
|
||||
}
|
||||
]
|
||||
}
|
||||
+58
-2
@@ -44,7 +44,7 @@ log = logging.getLogger("status-panel")
|
||||
VERSION = "2026.10.15" # logged at start ("status panel running (code ...)") and written to started.json
|
||||
# Every kind of fix the watchdog can run (validated by load_config).
|
||||
FIX_KINDS = ("ssh", "local", "atem", "reconnect", "midi_connect", "netgear_poe", "hyperdeck", "spotify", "ready",
|
||||
"companion_press")
|
||||
"companion_press", "propresenter_port")
|
||||
# Failover: a fix marked "failover" (pressing the FOH Backup button when Studio B dies) runs by itself
|
||||
# inside these windows when the "failover" block names none, "after_seconds" after its fault is confirmed.
|
||||
FAILOVER_WINDOWS = [{"days": ["sun"], "start": "06:30", "end": "13:30"}]
|
||||
@@ -502,7 +502,7 @@ def check_midi_peer(chk, snap, ctx):
|
||||
# Measured 2026-10-07: a joining peer reads "connected" for ~6 s until
|
||||
# clock sync, then "running". A peer that dies is removed from the list:
|
||||
# at once if it says goodbye, after ~60 s if it crashes or its cable is pulled.
|
||||
if p.get("state", "running") in ("running", "connected"): # no state: older macOS, configured
|
||||
if (p.get("state") or "running") in ("running", "connected"): # no state: older macOS, configured
|
||||
return [("peer", Result(OK))]
|
||||
return [("peer", Result(ERR, "MIDI " + (p["state"] or "?").upper()[:8]))]
|
||||
return [("peer", Result(ERR, "MIDI LOST"))]
|
||||
@@ -2755,6 +2755,60 @@ def execute_poe(cfg, name, entry):
|
||||
return False
|
||||
|
||||
|
||||
def propresenter_ports(cfg, fix):
|
||||
"""The TCP ports ProPresenter listens on, on the Mac it runs on (this one, or fix "on")."""
|
||||
cmd = "/usr/sbin/lsof -nP -iTCP -sTCP:LISTEN -a -c ProPresen 2>/dev/null"
|
||||
argv = ssh_cmd(cfg, fix["on"]) + [cmd] if fix.get("on") else ["/bin/zsh", "-c", cmd]
|
||||
out = subprocess.run(argv, capture_output=True, text=True, timeout=20).stdout
|
||||
return sorted({int(m.group(1)) for m in re.finditer(r":(\d+) \(LISTEN\)", out)})
|
||||
|
||||
|
||||
def propresenter_port(cfg, fix, label):
|
||||
"""ProPresenter picked another network port (it does, now and then: Southcoast's checklist says to
|
||||
copy it by hand; Woonsocket's had moved from 50822 to 49285): find the port that answers its API
|
||||
and put it into Companion's connection, then let Companion reconnect. Never touches ProPresenter."""
|
||||
import sqlite3
|
||||
comp = Companion(cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000))
|
||||
conns = {c["label"]: c for c in comp.connections()}
|
||||
conn = conns.get(fix["conn"])
|
||||
if not conn:
|
||||
log.info("FIX %s -> no Companion connection %s", label, fix["conn"])
|
||||
return False
|
||||
path = companion_db_path(cfg)
|
||||
db = sqlite3.connect("file:%s?mode=ro" % urllib.parse.quote(path), uri=True, timeout=2)
|
||||
try:
|
||||
conf = next((json.loads(v).get("config") or {} for (v,) in db.execute("SELECT value FROM instances")
|
||||
if json.loads(v).get("label") == fix["conn"]), {})
|
||||
finally:
|
||||
db.close()
|
||||
host = conf.get("host") or "127.0.0.1"
|
||||
found = None
|
||||
for port in propresenter_ports(cfg, fix):
|
||||
try:
|
||||
with urllib.request.urlopen("http://%s:%d/version" % (host, port), timeout=2) as r:
|
||||
if "api_version" in r.read().decode("utf-8", "replace"):
|
||||
found = port
|
||||
break
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if not found:
|
||||
log.info("FIX %s -> ProPresenter's network API answers on no port (is ProPresenter open, Network on?)", label)
|
||||
return "PROP API\nNOT ON"
|
||||
if int(conf.get("port") or 0) == found:
|
||||
ok = comp.restart_connection(conn["id"])
|
||||
log.info("FIX %s -> port %d is already right; connection restarted %s", label, found, "ok" if ok else "FAILED")
|
||||
return ok
|
||||
import push_page
|
||||
ws = push_page.WebSocket(cfg["companion"].get("host", "127.0.0.1"), int(cfg["companion"].get("port", 8000)), "/trpc")
|
||||
try:
|
||||
push_page.Trpc(ws).mutation("instances.connections.setConfig", {"connectionId": conn["id"], "label": fix["conn"],
|
||||
"config": dict(conf, port=found)})
|
||||
finally:
|
||||
ws.close()
|
||||
log.info("FIX %s -> ProPresenter's API moved from port %s to %d: Companion's %s now uses %d", label, conf.get("port"), found, fix["conn"], found)
|
||||
return True
|
||||
|
||||
|
||||
def execute_fix(cfg, fix_id, chk):
|
||||
"""Run one entry from config "fixes" (in its own thread). Only listed commands ever run.
|
||||
Returns True if the command ran and succeeded."""
|
||||
@@ -2798,6 +2852,8 @@ def execute_fix(cfg, fix_id, chk):
|
||||
return ok
|
||||
if kind == "reconnect":
|
||||
return reconnect_and_verify(cfg, fix, label)
|
||||
if kind == "propresenter_port":
|
||||
return propresenter_port(cfg, fix, label)
|
||||
if kind == "midi_connect":
|
||||
# Like Connect in Audio MIDI Setup: add the peer back to the session it dropped out of.
|
||||
args = (chk["session"], chk["peer"], chk["address"])
|
||||
|
||||
@@ -241,6 +241,21 @@ class TestExtend(unittest.TestCase):
|
||||
self.assertIn("PROP checks on the PROP key", added)
|
||||
self.assertEqual(schema.validate(new)[0], [])
|
||||
|
||||
def test_a_rebuild_keeps_what_is_down_right_now(self):
|
||||
"""Southcoast's rebuild dropped its MIDI key: the ProPresenter-to-Lighting link was down at that moment."""
|
||||
old, _ = ac.build(BASE, small(), {"id": "sc", "name": "Southcoast"}, "1234")
|
||||
now = small()
|
||||
now["midi"] = {"": {"SC Ingest": {"port": 5004, "peers": []}}} # the link dropped
|
||||
now["computers"]["lighting_pc"]["reachable"] = False # and the Lighting PC is off
|
||||
new, _ = ac.build(BASE, now, {"id": "sc", "name": "Southcoast"}, "1234")
|
||||
self.assertNotIn("MIDI", [t["label"] for t in new["tiles"]])
|
||||
kept, words = ac.keep_from(new, old)
|
||||
self.assertIn("MIDI", [t["label"] for t in kept["tiles"]])
|
||||
self.assertIn("VISTA", [t["label"] for t in kept["tiles"]])
|
||||
self.assertIn("lighting_pc", kept["hosts"])
|
||||
self.assertIn("MIDI link Lighting PC", words)
|
||||
self.assertEqual(schema.validate(kept)[0], [])
|
||||
|
||||
def test_a_key_someone_took_off_is_not_added_back(self):
|
||||
cfg = json.loads(json.dumps(self.cfg))
|
||||
cfg["tiles"] = [t for t in cfg["tiles"] if t["label"] != "X32"]
|
||||
|
||||
Reference in New Issue
Block a user