From b7af31ea6f71885f664511d56c450995da55b795 Mon Sep 17 00:00:00 2001 From: christian Date: Mon, 19 Oct 2026 14:08:33 -0400 Subject: [PATCH] The LTC write happens by itself: READY fixes ticked for auto-fix act on the checklist, any hour for ready_ltc Auto-fix now looks at the READY page's keys too, for the READY fixes ticked under Settings > Auto-fix, and acts on a checklist gap while it is still amber. A fix marked anytime (only a READY fix may be) runs outside the auto-fix hours as well, never while ALARMS is muted; the window's own fixes keep their hours and start fresh each window. ready_ltc is ticked and marked anytime at North Attleboro: once the weekend's set is on Tracks, every song's timecode goes into all four campuses' plans. Co-Authored-By: Claude Fable 5.1 --- README.md | 1 + config.json | 4 ++- schema.py | 7 +++++- settings/ui/app.js | 2 +- status_panel.py | 51 ++++++++++++++++++++++++-------------- tests/test_status_panel.py | 39 +++++++++++++++++++++++++++++ 6 files changed, 82 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 6ed17ec..0a641ec 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ During that window, with ALARMS on, the panel fixes a problem by itself once it - Never automatic: restarting a running ProPresenter, Bible app, ProdCom (or DVS on NA Ingest, which restarts ProdCom), captions or the bridge, power-cycling a switch port, or turning on a connection someone switched off. `load_config` refuses an auto fix that closes a program. - **Daily log:** everything auto-fix does is written as it happens to `~/Desktop/Auto-Fix Logs/ Auto-Fix.log` on NA Ingest (e.g. `2026-10-11 Sunday Auto-Fix.log`): the problem, what it did, whether it worked, when the fault cleared, when it gave up or held back. Double-click it to open it in Console, which shows new lines live. One file per day; each is deleted a calendar month after its date (the 11 Oct file on 11 Nov). - Press ALARMS to mute the panel for maintenance: that also stops auto-fix. The window and the list are `"auto_fix"` in `config.json`. +- **READY page fixes** ticked under Settings › Auto-fix act on the checklist's gaps too, amber or red. A fix marked `"anytime": true` (only a READY fix may be; today the LTC write, `ready_ltc`) runs at any hour, not only in the auto-fix hours: once the weekend's set is on Tracks, every song's timecode is written into Planning Center for all four campuses by itself. ALARMS muted still stops it. ## Audio failover (Studio B -> FOH Backup) diff --git a/config.json b/config.json index 8acc426..1cfe6c4 100644 --- a/config.json +++ b/config.json @@ -26,7 +26,8 @@ "atem_audio", "atem_camera", "atem_midi", - "spotify_play" + "spotify_play", + "ready_ltc" ], "max_tries": 2, "pause_after_network_seconds": 60, @@ -355,6 +356,7 @@ }, "ready_ltc": { "action": "ltc_to_pco", + "anytime": true, "kind": "ready", "label": "write each song's timecode from the tracks into its LTC note in Planning Center", "short": "LTC" diff --git a/schema.py b/schema.py index 81878f4..903351b 100644 --- a/schema.py +++ b/schema.py @@ -218,6 +218,7 @@ field("fixes.*.reconnect", "bool", False, "Reconnect the Companion connection af field("fixes.*.conn", "str", None, "The Companion connection it reconnects.", "Fixes") field("fixes.*.restart_ok", "bool", False, "May close a running program (and still run by itself).", "Fixes") field("fixes.*.never_auto", "bool", False, "Never runs by itself.", "Fixes") +field("fixes.*.anytime", "bool", False, "Ticked for auto-fix, runs by itself at any hour, not only in the auto-fix hours (a READY fix that only writes to Planning Center).", "Fixes") field("fixes.*.part", "enum", None, "Which part of the switcher.", "Fixes", options=["audio", "camera"]) field("fixes.*.verify_seconds", "int", 20, "Wait this long before judging a reconnect.", "Fixes", advanced=True, lo=5, hi=120) @@ -669,10 +670,14 @@ def validate(cfg, live=None): errors.append("%s: a Companion button needs its page, row and column" % where) if fix.get("failover") and fix.get("never_auto"): errors.append("%s: is a failover (runs by itself) and marked never to run by itself - one or the other" % where) + if fix.get("anytime") and kind != "ready": + errors.append("%s: only a READY page fix may run at any hour (anytime)" % where) + if fix.get("anytime") and fix.get("never_auto"): + errors.append("%s: runs at any hour (anytime) and never by itself - one or the other" % where) if fix.get("undo") and fix["undo"] not in fixes: errors.append("%s: the fix that switches back (undo) is unknown: %s" % (where, fix["undo"])) for k in fix: - if k not in FIX_KIND_FIELDS.get(kind, []) and k not in ("kind", "short", "label", "restart_ok", "never_auto", "disabled", + if k not in FIX_KIND_FIELDS.get(kind, []) and k not in ("kind", "short", "label", "restart_ok", "never_auto", "anytime", "disabled", "template", "params", "timeout", "failover", "undo"): warnings.append("%s: %s is not a field of a %s fix" % (where, k, kind)) named = set() diff --git a/settings/ui/app.js b/settings/ui/app.js index c77bb73..f90c551 100644 --- a/settings/ui/app.js +++ b/settings/ui/app.js @@ -939,7 +939,7 @@
name${hint("As the keys say it")}

` + fieldsHtml("Spotify"); } function autofixHtml(c) { - const fixes = Object.entries(c.fixes).map(([id, f]) => { const cl = closes(f); const on = (c.auto_fix.fixes || []).includes(id); return ``; }).join(" "); + const fixes = Object.entries(c.fixes).map(([id, f]) => { const cl = closes(f); const on = (c.auto_fix.fixes || []).includes(id); return ``; }).join(" "); return `

${onOff("auto_fix", c.auto_fix, "auto-fix")} Auto-fix ${c.auto_fix.disabled ? "is off: nothing is fixed by itself (holds still work)." : "fixes the ticked fixes by itself in the painted hours."}

${calendarHtml("auto_fix.windows", c.auto_fix.windows)}

Fixes it may run${hint("Ticked fixes run by themselves inside the painted hours. ⚠ = it closes a running program to do its job; ticking it allows that without a hold.")}

${fixes}

Pressing a Companion button by itself when something dies is under Settings › Auto switch.

` + fieldsHtml("Schedules"); } diff --git a/status_panel.py b/status_panel.py index 6ea1d00..376d5e6 100644 --- a/status_panel.py +++ b/status_panel.py @@ -645,7 +645,8 @@ def effective_config(cfg): def all_tiles(cfg): """Every key the watchdog judges: the Status page's tiles and the READY page's keys. The READY - page's keys are not in ALL (nor in auto-fix): a weekday gap in the checklist is not an alarm.""" + page's keys are not in ALL: a weekday gap in the checklist is not an alarm. Auto-fix does look + at them, for the READY fixes ticked under Settings > Auto-fix.""" return cfg["tiles"] + ready_keys(cfg) # Checks whose connection a long press may restart (Dante is shared, so not that one). @@ -1233,12 +1234,13 @@ class Engine: tile = self.tiles.get(tile_id, {}) return [c["conn"] for c in tile.get("checks", []) if c["type"] in RESTARTABLE and c.get("conn")] - def fix_plan(self, tile_id, snap, now=None, min_age=0, errors_only=False): + def fix_plan(self, tile_id, snap, now=None, min_age=0, errors_only=False, levels=(ERR,)): """What a hold on this button would do: fix only what is failing right now. [(identity, "fix", fix id, check) or (identity, "restart", connection label, check)] A check's own "fix" wins; otherwise a failing Companion connection is restarted. Nothing is done for a healthy button, and a recording HyperDeck is never touched. + With errors_only, only faults of these `levels` that have lasted min_age count (auto-fix). """ tile = self.tiles.get(tile_id, {}) ids = self.ready_ids if tile.get("kind") == "ready" else (tile.get("members") or [tile_id]) @@ -1248,7 +1250,7 @@ class Engine: tr = self.trackers.get(key) if tid not in ids or not tr or not tr.confirmed or tr.result.level not in (WARN, ERR): continue - if errors_only and (tr.result.level != ERR or now - (tr.since or now) < min_age): + if errors_only and (tr.result.level not in levels or now - (tr.since or now) < min_age): continue # auto-fix: only real faults that have lasted # A connection switched off in Companion is fixed by switching it on - never by # acting on the device (2026-10-08: a disabled HyperDeck connection got a deck reboot). @@ -1500,19 +1502,26 @@ class Engine: # (playing the Spotify playlist): those keep running inside the window. muted = not self.service.on(now) keep = set(a.get("while_muted", [])) if muted else None - if not in_windows(a.get("windows", []), now) or (muted and not keep): + hours = in_windows(a.get("windows", []), now) + # A fix marked "anytime" (writing the timecode into Planning Center) runs by itself at any + # hour, not only in the auto-fix hours - still never while ALARMS is muted. + anytime = {f for f in a.get("fixes", []) if self.cfg["fixes"].get(f, {}).get("anytime") + and not self.cfg["fixes"].get(f, {}).get("never_auto") and not self.cfg["fixes"].get(f, {}).get("disabled")} + if not hours or (muted and not keep): if self.auto_on and self.autolog: - self.autolog.write(now, "PANEL", "auto-fix off (%s)" % ("ALARMS muted" if in_windows( - a.get("windows", []), now) else "end of the window")) + self.autolog.write(now, "PANEL", "auto-fix off (%s)" % ("ALARMS muted" if hours else "end of the window")) self.auto_on = False - self.auto_tries.clear() - self.auto_seen.clear() - return [] - if (not self.auto_on or self.auto_muted != muted) and self.autolog: # opens today's file, so it can be watched from the start - ends = [w["end"] for w in a.get("windows", []) if in_windows([w], now)] - self.autolog.write(now, "PANEL", "auto-fix on%s%s" % (" until " + ends[0] if ends else "", - " (ALARMS muted: only %s)" % ", ".join(sorted(keep)) if muted else "")) - self.auto_on, self.auto_muted = True, muted + for ident in [i for i in self.auto_tries if not (i[0] == "fix" and i[1] in anytime)]: + del self.auto_tries[ident] # the window's fixes start fresh next time + self.auto_seen.pop(ident, None) + if muted or not anytime: + return [] + else: + if (not self.auto_on or self.auto_muted != muted) and self.autolog: # opens today's file, so it can be watched from the start + ends = [w["end"] for w in a.get("windows", []) if in_windows([w], now)] + self.autolog.write(now, "PANEL", "auto-fix on%s%s" % (" until " + ends[0] if ends else "", + " (ALARMS muted: only %s)" % ", ".join(sorted(keep)) if muted else "")) + self.auto_on, self.auto_muted = True, muted net_down = any(self.trackers.get("net:" + k) and self.trackers["net:" + k].confirmed and self.trackers["net:" + k].result.level == ERR for k in ("link", "gateway")) if net_down or (self.burst and (self.burst["cleared"] is None @@ -1525,9 +1534,11 @@ class Engine: self.autolog.write(now, "NET", "network settled: auto-fix is back on") self.auto_paused = False gentle = {f for f in a.get("fixes", []) if not self.cfg["fixes"].get(f, {}).get("never_auto") - and not self.cfg["fixes"].get(f, {}).get("disabled")} + and not self.cfg["fixes"].get(f, {}).get("disabled")} if hours else anytime + restarts = bool(hours and a.get("restart_connections")) + ready_ids = {k["id"] for k in ready_keys(self.cfg)} done = [] - for t in self.cfg["tiles"]: + for t in all_tiles(self.cfg): if "checks" not in t: continue # groups repeat their members gone = [k for k in self.keys_for(t["id"]) if self.meta.get(k) and self.meta[k][1]["type"] == "dante" @@ -1544,8 +1555,10 @@ class Engine: self.auto_seen[("dante-wide", t["id"])] = now continue plan = [] - for ident, kind, what, chk in self.fix_plan(t["id"], snap, now, a.get("after_seconds", 20), True): - if not ((kind == "fix" and what in gentle) or (kind == "restart" and a.get("restart_connections"))): + # A READY key's gap is amber until the checklist is due: a ticked READY fix acts on it anyway. + levels = (WARN, ERR) if t["id"] in ready_ids else (ERR,) + for ident, kind, what, chk in self.fix_plan(t["id"], snap, now, a.get("after_seconds", 20), True, levels): + if not ((kind == "fix" and what in gentle) or (kind == "restart" and restarts)): continue # not gentle: stays a hold if keep is not None and not (kind == "fix" and what in keep): continue # ALARMS muted: only the "while_muted" fixes @@ -1567,7 +1580,7 @@ class Engine: try_no = max(self.auto_tries[i][0] for i, _, _, _ in plan) log.info("AUTO %s: fixing %s (try %d)", t["id"], ", ".join("%s %s" % (k, w) for _, k, w, _ in plan), try_no) faults = [r.msg for k in self.keys_for(t["id"]) for r in [self.trackers[k].result] - if self.trackers[k].confirmed and r.level == ERR] + if self.trackers[k].confirmed and r.level in levels] for f in faults: self.auto_watch["%s %s" % (t["label"], f)] = now label = t["label"] diff --git a/tests/test_status_panel.py b/tests/test_status_panel.py index 68017f2..26db56b 100644 --- a/tests/test_status_panel.py +++ b/tests/test_status_panel.py @@ -975,6 +975,45 @@ class TestFixes(unittest.TestCase): self.assertEqual(self.engine.auto_fix(self.snap, comp, weekday + 30), []) self.assertEqual(ran, []) + def test_a_ready_fix_marked_anytime_runs_on_a_weekday(self): + """The LTC write, ticked for auto-fix and marked anytime, fills an amber TIMECODE gap on a + Thursday morning; the window's fixes (Ableton closed) still wait for the window.""" + cfg = json.loads(json.dumps(CONFIG)) + cfg["auto_fix"]["fixes"] = cfg["auto_fix"]["fixes"] + ["ready_ltc"] + cfg["fixes"]["ready_ltc"]["anytime"] = True + thursday = SUNDAY - 3 * 86400 + self.engine = sp.Engine(cfg) + self.engine.autolog = sp.AutoLog(tempfile.mkdtemp()) + self.engine.service.set(True, thursday) # ALARMS on all week, as at NA + ran, comp = self.auto_setup() + self.snap.ready = {"items": {"timecode": {"faults": [[sp.ERR, "NO LTC JEHOVAH"]], "info": ""}}} + self.snap.remote["tracks\0pgrep -xq Live"] = False + run_steps(self.engine, self.snap, thursday, 10) + self.assertEqual(self.engine.auto_fix(self.snap, comp, thursday + 10), []) # not 20 s yet + done = self.engine.auto_fix(self.snap, comp, thursday + 30) + time.sleep(0.2) + self.assertEqual((done, ran), (["rd_timecode"], ["ready_ltc"])) # never "ableton" + self.assertEqual(self.engine.auto_fix(self.snap, comp, thursday + 60), []) # 90 s between tries + self.engine.service.set(False, thursday + 61) # ALARMS muted: nothing + self.assertEqual(self.engine.auto_fix(self.snap, comp, thursday + 200), []) + self.engine.service.set(True, thursday + 201) + del cfg["fixes"]["ready_ltc"]["anytime"] # not marked: the window only + self.engine = sp.Engine(cfg) + self.engine.autolog = sp.AutoLog(tempfile.mkdtemp()) + self.engine.service.set(True, thursday) + ran, comp = self.auto_setup() + run_steps(self.engine, self.snap, thursday, 25) + self.assertEqual(self.engine.auto_fix(self.snap, comp, thursday + 30), []) + run_steps(self.engine, self.snap, SUNDAY, 25) + done = self.engine.auto_fix(self.snap, comp, SUNDAY + 30) + time.sleep(0.2) + self.assertIn("rd_timecode", done) # in the window it acts + self.assertIn("ready_ltc", ran) + import schema + bad = json.loads(json.dumps(CONFIG)) + bad["fixes"]["ableton"]["anytime"] = True + self.assertTrue(any("anytime" in e for e in schema.validate(bad)[0])) # only a READY fix may + def test_restart_keys_flash_when_their_thing_is_down(self): devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")]) devices.remove("NA-Tracks") # DVS on the Tracks Mac gone