christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Audio failover: press FOH Backup by itself when Studio B dies; Companion button presses as fixes

Browse Source 
A new fix kind "companion_press" presses any Companion button over its HTTP API (page, row, col).
A fix marked "failover" runs by itself within seconds of its check failing: inside the "failover"
windows (Sunday 06:30-13:30), even while ALARMS is muted, once per fault, never back the other way.
The hidden STB AUDIO key (inside AUDIO) watches Studio B's ping every second, Pro Tools over SSH
every 2 s and its Dante device; each check's fix is the FOH Backup press. The keys say ON FOH BACKUP
until STREAM STB on the Restart page switches back. Settings page: the press builder, the failover
tick and undo, the Failover calendar, help entry "A fallback button".

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-15 12:40:24 -04:00
co-authored by Claude Fable 5.1
parent 422e92f814
commit 14fce2cf9a
13 changed files with 855 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files
schema.py
+52 -6
View File
@@ -128,6 +128,17 @@ field("auto_fix.log_dir", "path", "~/Desktop/Auto-Fix Logs", "One file per day o
"Schedules", advanced=True)
field("auto_fix.keep_months", "int", 1, "Daily log files are deleted after this many months.", "Schedules",
advanced=True, lo=1, hi=12)
field("failover", "dict", {}, "Automatic fallback: when a fix marked failover may act (it acts even while ALARMS is "
"muted).", "Schedules")
field("failover.windows", "list", [{"days": ["sun"], "start": "06:30", "end": "13:30"}], "When a failover may switch "
"by itself.", "Schedules")
field("failover.windows[].days", "days", None, "", "Schedules", required=True)
field("failover.windows[].start", "hhmm", None, "", "Schedules", required=True)
field("failover.windows[].end", "hhmm", None, "", "Schedules", required=True)
field("failover.after_seconds", "int", 2, "The fault must have lasted this long before the failover switches.",
"Schedules", lo=0, hi=60)
field("failover.note_seconds", "int", 900, "After the fault clears, the key keeps saying ON <the fallback> this long "
"(until the fix that switches back runs).", "Schedules", advanced=True, lo=60, hi=86400)
# -- network and Dante --
field("network", "dict", {}, "The NET key.", "Network & Dante")
@@ -219,6 +230,13 @@ field("fixes.*.playlist", "str", None, "A playlist other than the site's (rare).
field("fixes.*.template", "str", None, "The builder this fix was made with.", "Fixes", advanced=True, apply="none")
field("fixes.*.params", "dict", None, "The builder's fields.", "Fixes", advanced=True, apply="none")
field("fixes.*.action", "enum", None, "What a READY page fix does.", "Fixes", options=list(sp.READY_ACTIONS))
field("fixes.*.page", "int", None, "The Companion page the button is on.", "Fixes", lo=1)
field("fixes.*.row", "int", None, "The button's row on that page (the top row is 0).", "Fixes", lo=0)
field("fixes.*.col", "int", None, "The button's column (the left column is 0).", "Fixes", lo=0)
field("fixes.*.failover", "bool", False, "An automatic fallback: when a check that names this fix fails, the watchdog "
"runs it by itself within seconds (inside the failover hours, even while ALARMS is muted, once per fault). It "
"never switches back by itself.", "Fixes")
field("fixes.*.undo", "str", None, "The fix that switches back by hand (its key takes the ON ... note off).", "Fixes")
# -- tiles (keys) --
field("tiles", "list", [], "The keys on the Status page.", "Keys", required=True)
@@ -262,7 +280,8 @@ field("tiles[].checks[].failures", "int", 2, "Pings missed before NO PING.", "Ke
field("tiles[].checks[].label", "str", None, "The launchd helper.", "Keys")
field("tiles[].checks[].match", "str", None, "The program (pgrep -f pattern).", "Keys")
field("tiles[].checks[].url", "url", None, "The web address to fetch.", "Keys")
field("tiles[].checks[].interval_seconds", "int", 30, "Fetch it this often.", "Keys", advanced=True, lo=5, hi=3600)
field("tiles[].checks[].interval_seconds", "int", 30, "Look this often, in seconds: a web address every 30, a command "
"on another computer every 10, a ping every 3 (1 for a fast fallback check).", "Keys", advanced=True, lo=1, hi=3600)
field("tiles[].checks[].path", "path", None, "The disk to watch.", "Keys")
field("tiles[].checks[].warn_gb", "int", 20, "Amber below this many GB free.", "Keys", advanced=True)
field("tiles[].checks[].err_gb", "int", 5, "Red below this many GB free.", "Keys", advanced=True)
@@ -371,7 +390,7 @@ CHECKS = {
"module": "shure-psm1000", "question": "Is a device answering?", "words": "A PSM transmitter: RF mute."},
"tascam": {"fields": ["conn"], "subs": ["conn", "error", "caution"], "identity": ["conn"], "module": "tascam-da-6400",
"question": "Is a recorder OK?", "words": "The Tascam recorder reports no error or caution."},
"ping": {"fields": ["host", "conn", "name", "msg", "failures", "service_only"], "subs": ["ping"], "identity": ["conn", "host"],
"ping": {"fields": ["host", "conn", "name", "msg", "failures", "interval_seconds", "service_only"], "subs": ["ping"], "identity": ["conn", "host"],
"module": None, "question": "Is a device answering?", "words": "An address answers ping."},
"launchd": {"fields": ["label", "name"], "subs": ["svc"], "identity": ["label"], "module": None,
"question": "Is a program running?", "words": "A helper (LaunchAgent) on this Mac is loaded and running."},
@@ -391,7 +410,7 @@ CHECKS = {
"question": "Is a MIDI link up?", "words": "A network MIDI session is advertised on Bonjour."},
"midi_rx": {"fields": ["source", "name", "max_silence_seconds"], "subs": ["rx"], "identity": ["source"], "module": None,
"question": "Is a MIDI link up?", "words": "MIDI is arriving from a source."},
"remote": {"fields": ["host", "cmd", "msg", "shell", "template", "params"], "subs": ["remote"], "identity": ["host", "cmd"],
"remote": {"fields": ["host", "cmd", "msg", "shell", "interval_seconds", "template", "params"], "subs": ["remote"], "identity": ["host", "cmd"],
"module": None, "question": "Is a program running?", "words": "A command on another computer succeeds."},
"artnet": {"fields": ["source", "max_silence_seconds", "msg"], "subs": ["artnet"], "identity": ["source"], "module": None,
"question": "Is the lighting console sending?", "words": "Art-Net is arriving from the lighting program."},
@@ -408,7 +427,7 @@ FIX_KIND_FIELDS = {
"ssh": ["host", "run", "timeout", "reconnect", "conn"], "local": ["run", "timeout", "reconnect", "conn"],
"atem": ["part"], "reconnect": ["conn", "verify_seconds", "hung_note"], "midi_connect": [],
"netgear_poe": ["switch", "port", "mac"], "hyperdeck": ["verify_seconds", "off_seconds", "hung_note"],
"spotify": ["playlist"], "ready": ["action"],
"spotify": ["playlist"], "ready": ["action"], "companion_press": ["page", "row", "col"],
}
EXPECTED_MODULE = {t: spec["module"] for t, spec in CHECKS.items() if spec["module"]}
@@ -442,7 +461,8 @@ GLOSSARY = {
"STILL OFF IN COMPANION", "DISABLED IN COMPANION", "TURN ON", "RECONNECT", "POWER CYCLE", "FIXES", "HOLD TO FIX",
"HOLD", "REC", "PLAYING", "OK", "OFF", "ON", "ALL OK", "WARN", "TRS", "GB", "NOT", "REACHABLE", "RECORDING",
"LEFT ALONE", "TO FIX", "TO CLEAR", "NO AUTO", "FIX", "AUTO", "IN", "COMPANION", "POWER", "CYCLE",
"STILL", "OFF IN", "DISABLED IN", "CONNECTING", "CHECK", "NETWORK", "NO",
"STILL", "OFF IN", "DISABLED IN", "CONNECTING", "CHECK", "NETWORK", "NO", "PRESSING", "NO BUTTON THERE",
"ON FOH BACKUP",
],
}
@@ -478,6 +498,8 @@ def key_record():
kind, parts = "ready", parts[1:]
elif p.startswith("auto_fix."):
kind, parts = "auto", parts[1:]
elif p.startswith("failover."):
kind, parts = "failover", parts[1:]
elif p.startswith("network."):
kind, parts = "network", parts[1:]
elif p.startswith("variables."):
@@ -621,10 +643,24 @@ def validate(cfg, live=None):
errors.append("%s: needs the connection to reconnect" % where)
if kind == "atem" and fix.get("part") not in ("audio", "camera"):
errors.append("%s: part must be audio or camera" % where)
if kind == "companion_press" and not (_whole(fix.get("page"), 1) and _whole(fix.get("row"), 0) and _whole(fix.get("col"), 0)):
errors.append("%s: a Companion button needs its page, row and column" % where)
if fix.get("failover") and fix.get("never_auto"):
errors.append("%s: is a failover (runs by itself) and marked never to run by itself - one or the other" % where)
if fix.get("undo") and fix["undo"] not in fixes:
errors.append("%s: the fix that switches back (undo) is unknown: %s" % (where, fix["undo"]))
for k in fix:
if k not in FIX_KIND_FIELDS.get(kind, []) and k not in ("kind", "short", "label", "restart_ok", "never_auto",
"template", "params", "timeout"):
"template", "params", "timeout", "failover", "undo"):
warnings.append("%s: %s is not a field of a %s fix" % (where, k, kind))
named = set()
for t in cfg["tiles"] + ((cfg.get("ready_page") or {}).get("keys") or []):
for c in t.get("checks", []):
fx = c.get("fix")
named |= set(fx.values()) if isinstance(fx, dict) else ({fx} if fx else set())
for fid, fix in fixes.items():
if fix.get("failover") and fid not in named:
warnings.append("fix %s is a failover but no check runs it" % fid)
for alias, h in hosts.items():
if not re.fullmatch(r"[a-z0-9_]+", alias):
errors.append("computer %s: the short name may only hold lowercase letters, digits and _" % alias)
@@ -648,11 +684,14 @@ def validate(cfg, live=None):
errors.append("auto-fix: %s is marked never to run by itself" % f)
elif re.search(r"pkill|taskkill|killall", fixes[f].get("run", "")) and not fixes[f].get("restart_ok"):
errors.append("auto-fix: %s closes a running program; tick 'may close a running program' or take it out" % f)
elif fixes[f].get("failover"):
warnings.append("auto-fix: %s is a failover - it already runs by itself as one" % f)
for f in a.get("while_muted", []):
if f not in a.get("fixes", []):
errors.append("auto-fix: %s runs while ALARMS is muted but is not an auto-fix" % f)
_windows(cfg.get("service_windows"), "alarms window", errors)
_windows(a.get("windows"), "auto-fix window", errors)
_windows((cfg.get("failover") or {}).get("windows"), "failover window", errors)
_windows((cfg.get("site") or {}).get("live_services"), "live services", errors)
rp = cfg.get("restart_page")
if rp:
@@ -752,6 +791,11 @@ def validate(cfg, live=None):
return errors, warnings
def _whole(v, lo):
"""A whole number of at least lo (a bool is not one)."""
return isinstance(v, int) and not isinstance(v, bool) and v >= lo
def _get(cfg, path):
"""The value at a plain path (no [] or *); None when absent."""
if "[]" in path or ".*." in path:
@@ -804,8 +848,10 @@ def phrases(cfg):
"""The sentences the page and the logs build from the config."""
a = cfg.get("auto_fix") or {}
site = cfg.get("site") or {}
fo_windows = (cfg.get("failover") or {}).get("windows", F["failover.windows"]["default"])
return {
"auto_fix": "may run by itself %s" % sp.windows_phrase(a.get("windows", [])) if a.get("windows") else "never runs by itself",
"failover": "switches by itself %s" % sp.windows_phrase(fo_windows) if fo_windows else "never switches by itself",
"live": ("changes wait while a HyperDeck is recording" + (" or %s" % sp.windows_phrase(site["live_services"])
if site.get("live_services") else "")),
"alarms": "Alarms are on every day. The ALARMS key silences them until midnight, then they come back by themselves.",