christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sunday auto-fix (gentle fixes only), reviewed; PROPRESENTER key label

Browse Source 
Sundays 06:30-13:30, with ALARMS on, a fault that has lasted 20 s is fixed
by itself if the fix is gentle (reconnects, HyperDeck off/on, Tascam, MIDI,
rtpMIDI, starting a closed Vista/Resolume/OBS/Ableton/Translate, DVS on one
Mac). At most 2 tries 90 s apart; key shows AUTO-FIXING. Anything that
closes a running program stays a hold (load_config refuses it).

A review before going live (2 lenses, each finding verified) confirmed and
this fixes: a Dante-wide "gone" (Ingest's view) would restart DVS on every
Mac every minute - several devices gone at once now means hands off; the
try limit reset whenever a fix made its fault blink - tries are kept until
the fault has been gone 10 min; DVS on Ingest kills ProdCom - removed from
the auto list; restarting dead connections looked like a network burst -
only a connection good in the last 30 s counts; a restart key could kill
an app auto-fix was starting; concurrent MIDI reconnects overwrote each
other - now one worker. Auto-fix errors are caught so the panel keeps
running. 105 tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-11 07:05:05 -04:00
co-authored by Claude Opus 5.5
parent 3417f00f1b
commit 560e0401a3
4 changed files with 274 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
status_panel.py
+116 -6
View File
@@ -580,6 +580,13 @@ class Tracker:
DAYS = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]
def in_windows(windows, now):
"""Is `now` inside one of [{"days": [...], "start": "HH:MM", "end": "HH:MM"}]?"""
dt = datetime.fromtimestamp(now)
hm, day = dt.strftime("%H:%M"), DAYS[dt.weekday()]
return any(day in w["days"] and w["start"] <= hm < w["end"] for w in windows)
class ServiceMode:
"""On inside a scheduled window; a button press overrides until the next window edge."""
@@ -644,6 +651,7 @@ class Engine:
self.group_of[m] = t["id"]
self.detail_ids = [t["id"] for t in cfg["tiles"] if "checks" in t or t.get("kind") == "network"]
self.prev_conn_bad = {}
self.conn_good_at = {} # connection -> last time it was "good"
self.conn_drops = deque() # (time, conn label, group)
self.burst = None # {"start", "conns", "cleared"}
self.events = []
@@ -654,6 +662,10 @@ class Engine:
self.notes = {} # tile id -> hold feedback {"text", "level", "until"}
# The Restart page: one key per thing, hold 2 s = restart it even when it looks fine.
self.restart_keys = {k["id"]: k for k in cfg.get("restart_page", {}).get("keys", [])}
self.auto = cfg.get("auto_fix") or {}
self.auto_tries = {} # fix identity -> [tries, last try time] for the fault now going on
self.auto_seen = {} # fix identity -> last time it was in an auto plan
self.auto_last = 0.0
self.deck_ips = {} # HyperDeck connection -> last address Companion reported
self.now = 0.0
@@ -700,7 +712,11 @@ class Engine:
bad = bool(c.get("enabled")) and c.get("category") == "error"
if bad:
bad_now.add(label)
if bad and not self.prev_conn_bad.get(label):
if c.get("category") == "good":
self.conn_good_at[label] = now
# Only a connection that was fine moments ago counts as dropping: one that a fix just
# restarted goes error -> starting -> error, and is not the network (2026-10-11 review).
if bad and not self.prev_conn_bad.get(label) and now - self.conn_good_at.get(label, -1e9) <= 30:
self.conn_drops.append((now, label, self.group_for_conn(label)))
self.prev_conn_bad[label] = bad
window = self.net.get("burst_window_seconds", 10)
@@ -791,7 +807,7 @@ class Engine:
tile = self.tiles.get(tile_id, {})
return [c["conn"] for c in tile.get("checks", []) if c["type"] in RESTARTABLE and c.get("conn")]
def fix_plan(self, tile_id, snap):
def fix_plan(self, tile_id, snap, now=None, min_age=0, errors_only=False):
"""What a hold on this button would do: fix only what is failing right now.
[(identity, "fix", fix id, check) or (identity, "restart", connection label, check)]
@@ -806,6 +822,8 @@ class Engine:
tr = self.trackers.get(key)
if tid not in ids or not tr or not tr.confirmed or tr.result.level not in (WARN, ERR):
continue
if errors_only and (tr.result.level != ERR or now - (tr.since or now) < min_age):
continue # auto-fix: only real faults that have lasted
# A connection switched off in Companion is fixed by switching it on - never by
# acting on the device (2026-10-08: a disabled HyperDeck connection got a deck reboot).
label = chk.get("conn")
@@ -927,7 +945,11 @@ class Engine:
log.info("HOLD %s: nothing failing, nothing to fix", tile_id)
self.note(tile_id, "NOTHING\nTO FIX", "idle", now, 4)
return plan
started, names, waited = [], [], False
return self.run_plan(tile_id, plan, snap, comp, now, cooldown)
def run_plan(self, tile_id, plan, snap, comp, now, cooldown=60, auto=False):
"""Carry out fix_plan() items for one button (a hold, or auto-fix during service)."""
started, names, waited, midi = [], [], False, []
for ident, kind, what, chk in plan:
if now - self.fix_last.get(ident, 0) < cooldown:
log.info("HOLD %s: %s %s ran less than %ds ago, skipped", tile_id, kind, what, cooldown)
@@ -954,15 +976,25 @@ class Engine:
fix = self.cfg["fixes"][what]
log.info("FIX %s: %s", tile_id, fix.get("label", what))
names.append(fix.get("short", what.upper()))
started.append((self.executor, (self.cfg, what, chk), fix.get("short", what.upper())))
if fix.get("kind") == "midi_connect":
midi.append((what, chk)) # one worker for all of them (see below)
else:
started.append((self.executor, (self.cfg, what, chk), fix.get("short", what.upper())))
if midi:
# One after another: two reconnects writing the same session at once overwrite each other.
def midi_worker(batch):
rs = [self.executor(self.cfg, f, c) for f, c in batch]
return next((r for r in rs if isinstance(r, str)), False if False in rs else True)
started.append((midi_worker, (midi,), self.cfg["fixes"][midi[0][0]].get("short", "MIDI")))
if not names:
self.note(tile_id, "JUST TRIED\nWAIT 1 MIN" if waited else "NOTHING\nTO FIX", "idle", now, 4)
if not auto:
self.note(tile_id, "JUST TRIED\nWAIT 1 MIN" if waited else "NOTHING\nTO FIX", "idle", now, 4)
return plan
summary = names[0] if len(set(names)) == 1 else "%d FIXES" % len(names)
if not started: # only Companion reconnects: quick
self.note(tile_id, "CHECKING\n" + summary, "fixing", now, 15)
return plan
self.note(tile_id, "FIXING\n" + summary, "fixing", now, 180)
self.note(tile_id, ("AUTO-FIXING\n" if auto else "FIXING\n") + summary, "fixing", now, 180)
results = []
group = "members" in self.tiles.get(tile_id, {}) or len(started) > 1
@@ -987,6 +1019,68 @@ class Engine:
threading.Thread(target=work, args=(fn, args, short), daemon=True).start()
return plan
def auto_fix(self, snap, comp, now):
"""During the auto-fix windows (Sunday service), fix by itself what a hold would fix - but
only gentle fixes (reconnects, starting a closed app, DVS for a Mac that dropped), only for
real faults that have lasted "after_seconds", at most "max_tries" times each, never while
the network is having a blip, and never while ALARMS has muted the panel (maintenance).
Restarting a running program stays a hold."""
a = self.auto
if not a or now - self.auto_last < 5:
return []
self.auto_last = now
if not in_windows(a.get("windows", []), now) or not self.service.on(now):
self.auto_tries.clear()
self.auto_seen.clear()
return []
net_down = any(self.trackers.get("net:" + k) and self.trackers["net:" + k].confirmed
and self.trackers["net:" + k].result.level == ERR for k in ("link", "gateway"))
if net_down or (self.burst and (self.burst["cleared"] is None
or now - self.burst["cleared"] < a.get("pause_after_network_seconds", 60))):
return [] # everything looks broken at once: wait it out
gentle = set(a.get("fixes", []))
done = []
for t in self.cfg["tiles"]:
if "checks" not in t:
continue # groups repeat their members
gone = [k for k in self.keys_for(t["id"]) if self.meta.get(k) and self.meta[k][1]["type"] == "dante"
and self.meta[k][2] != "conn" and self.trackers[k].confirmed and self.trackers[k].result.level == ERR]
if len(gone) > 1:
# Several Dante devices "gone" at once is NA Ingest's view of Dante or the Dante network,
# not those Macs: restarting their DVS would cut audio that is working. Leave it to a hold.
if self.auto_seen.get(("dante-wide", t["id"]), 0) < now - 600:
log.info("AUTO %s: %d Dante devices gone at once - not touching them, hold DANTE if needed",
t["id"], len(gone))
self.auto_seen[("dante-wide", t["id"])] = now
continue
plan = []
for ident, kind, what, chk in self.fix_plan(t["id"], snap, now, a.get("after_seconds", 20), True):
if not ((kind == "fix" and what in gentle) or (kind == "restart" and a.get("restart_connections"))):
continue # not gentle: stays a hold
self.auto_seen[ident] = now
tries, last = self.auto_tries.get(ident, (0, 0.0))
if tries >= a.get("max_tries", 2):
if tries == a.get("max_tries", 2):
log.info("AUTO %s: gave up on %s %s after %d tries - hold it to try again", t["id"], kind, what, tries)
self.auto_tries[ident] = (tries + 1, last)
continue
if now - last < a.get("retry_seconds", 90) or now - self.fix_last.get(ident, 0) < 60:
continue
self.auto_tries[ident] = (tries + 1, now)
plan.append((ident, kind, what, chk))
if plan:
log.info("AUTO %s: fixing %s (try %d)", t["id"], ", ".join("%s %s" % (k, w) for _, k, w, _ in plan),
max(self.auto_tries[i][0] for i, _, _, _ in plan))
self.run_plan(t["id"], plan, snap, comp, now, auto=True)
done.append(t["id"])
forget = a.get("forget_seconds", 600)
for ident in list(self.auto_tries):
# Forget the tries only once the fault has been gone a long while: a fix makes its own
# fault blink out for a moment (restarting, re-confirming), which must not reset the count.
if now - self.auto_seen.get(ident, 0) >= forget:
del self.auto_tries[ident]
return done
def run_force(self, key_id, snap, comp, now, cooldown=60, fix_id=None, chk=None):
"""Run one fix whatever the status says: a Restart page key, or a button's "hold_always".
Shares the 60 s guard with holds, so a hold and a restart key never double up."""
@@ -996,6 +1090,12 @@ class Engine:
log.warning("FORCE %s: no such restart key", key_id)
return None
fix_id = key["fix"]
busy = [w for w in key.get("watch", []) if not w.startswith("conn:")
and now - self.fix_last.get(("fix", w, ""), 0) < cooldown]
if busy: # e.g. auto-fix is starting Ableton right now
log.info("FORCE %s: %s ran less than %ds ago, skipped", key_id, busy[0], cooldown)
self.note(key_id, "STARTING\nWAIT 1 MIN", "idle", now, 4)
return None
per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck" and chk and chk.get("conn")
ident = ("fix", fix_id, chk["conn"] if per_deck else "") # the same guard a failing hold uses
if now - self.fix_last.get(ident, 0) < cooldown:
@@ -1728,6 +1828,11 @@ def load_config(path):
raise ValueError("tile %s: unknown host %s" % (t["id"], c["host"]))
if t.get("hold_always") and t["hold_always"] not in cfg.get("fixes", {}):
raise ValueError("tile %s: unknown hold_always fix %s" % (t["id"], t["hold_always"]))
for f in (cfg.get("auto_fix") or {}).get("fixes", []):
if f not in cfg.get("fixes", {}):
raise ValueError("auto_fix: unknown fix %s" % f)
if re.search(r"pkill|taskkill|killall", cfg["fixes"][f].get("run", "")):
raise ValueError("auto_fix: %s closes a running program - that stays a hold" % f)
for k in cfg.get("restart_page", {}).get("keys", []):
if k["id"] in ids:
raise ValueError("restart key %s has a tile's id" % k["id"])
@@ -1855,6 +1960,11 @@ def run(cfg, data_dir, dry_run=False, stop=None):
if now - last_eval >= 1.0:
last_eval = now
tiles = engine.step(snap, now)
if not dry_run:
try:
engine.auto_fix(snap, comp, now)
except Exception: # a bug here must never take the panel down
log.exception("auto-fix failed; the panel carries on without it this round")
for kind, text in engine.events:
log.info("%s %s", kind, text)
engine.events = []