Clear a fault once its check stops reporting it

Found in today's power-off test: a Shure/PSM check reports "conn" while its
connection is down and per-channel results once it is back, so the
"AXIENT 1 OFFLINE" tracker never saw OK. The button looked green, but no
CLEAR was logged and a hold on RF would still have restarted the Shure
connections. A tracker missing from a round now gets an OK update.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-08 16:39:15 -04:00
co-authored by Claude Opus 5.5
parent e4b7d0d0ec
commit 46fb55533b
2 changed files with 20 additions and 0 deletions
+8
View File
@@ -724,6 +724,14 @@ class Engine:
self.events.append(("FAULT", "%s %s" % (self.tiles[tile_id]["label"], res.msg))) self.events.append(("FAULT", "%s %s" % (self.tiles[tile_id]["label"], res.msg)))
elif event == "clear": elif event == "clear":
self.events.append(("CLEAR", "%s %s" % (self.tiles[tile_id]["label"], tr.last_fault_msg))) self.events.append(("CLEAR", "%s %s" % (self.tiles[tile_id]["label"], tr.last_fault_msg)))
# A check may stop reporting a fault instead of reporting it OK: a Shure or PSM check says
# "conn" while its connection is down and per channel once it is back (2026-10-08). Such a
# fault is over - clear it, or it lingers unseen and a hold would still try to fix it.
for key, tr in self.trackers.items():
if key not in raw and (tr.confirmed or tr.bad_since is not None):
if tr.update(Result(OK), now, 0, self.flap_window) == "clear" and key in self.meta:
tile_id = self.meta[key][0]
self.events.append(("CLEAR", "%s %s" % (self.tiles[tile_id]["label"], tr.last_fault_msg)))
self.items = {} self.items = {}
for key, (tile_id, res, chk) in raw.items(): for key, (tile_id, res, chk) in raw.items():
tr = self.trackers[key] tr = self.trackers[key]
+12
View File
@@ -545,6 +545,18 @@ class TestFixes(unittest.TestCase):
for fix_id in ("obs", "arena", "vista"): # never closes a running app for fix_id in ("obs", "arena", "vista"): # never closes a running app
self.assertTrue(CONFIG["fixes"][fix_id]["run"].startswith("tasklist "), fix_id) self.assertTrue(CONFIG["fixes"][fix_id]["run"].startswith("tasklist "), fix_id)
def test_shure_back_online_clears_and_holds_do_nothing(self):
self.snap.conns["Axient_1"]["category"] = "error"
run_steps(self.engine, self.snap, SUNDAY, 4)
self.assertEqual(self.plan("rf"), [("restart", "Axient_1")])
self.engine.events.clear()
self.snap.conns["Axient_1"]["category"] = "good" # back: now reported per channel
tiles, _ = run_steps(self.engine, self.snap, SUNDAY + 5, 2)
self.assertEqual(tiles["rf"]["x"], "RF\nOK")
self.assertIn(("CLEAR", "AXIENT 1 OFFLINE"), self.engine.events)
self.assertEqual(self.plan("rf"), []) # nothing left to "fix"
self.assertFalse(self.engine.failing("rf"))
def test_midi_reconnect_where_the_peer_was_last_seen(self): def test_midi_reconnect_where_the_peer_was_last_seen(self):
others = [{"name": "SuperRack-LiveBox", "address": "10.1.13.140:5004", "state": "running"}, others = [{"name": "SuperRack-LiveBox", "address": "10.1.13.140:5004", "state": "running"},
{"name": "Studio B Mac Studio", "address": "\tStudio B Mac Studio\tlocal.", "state": "running"}] {"name": "Studio B Mac Studio", "address": "\tStudio B Mac Studio\tlocal.", "state": "running"}]