Sunday auto-fix (gentle fixes only), reviewed; PROPRESENTER key label
Sundays 06:30-13:30, with ALARMS on, a fault that has lasted 20 s is fixed by itself if the fix is gentle (reconnects, HyperDeck off/on, Tascam, MIDI, rtpMIDI, starting a closed Vista/Resolume/OBS/Ableton/Translate, DVS on one Mac). At most 2 tries 90 s apart; key shows AUTO-FIXING. Anything that closes a running program stays a hold (load_config refuses it). A review before going live (2 lenses, each finding verified) confirmed and this fixes: a Dante-wide "gone" (Ingest's view) would restart DVS on every Mac every minute - several devices gone at once now means hands off; the try limit reset whenever a fix made its fault blink - tries are kept until the fault has been gone 10 min; DVS on Ingest kills ProdCom - removed from the auto list; restarting dead connections looked like a network burst - only a connection good in the last 30 s counts; a restart key could kill an app auto-fix was starting; concurrent MIDI reconnects overwrote each other - now one worker. Auto-fix errors are caught so the panel keeps running. 105 tests. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+138
-1
@@ -566,6 +566,19 @@ class TestBehaviour(unittest.TestCase):
|
||||
self.assertEqual(engine.restart_targets("ingest"), [])
|
||||
|
||||
|
||||
def mock_comp():
|
||||
class Comp:
|
||||
def __init__(self):
|
||||
self.restarted, self.enabled = [], []
|
||||
def restart_connection(self, cid):
|
||||
self.restarted.append(cid)
|
||||
return True
|
||||
def enable_connection(self, cid):
|
||||
self.enabled.append(cid)
|
||||
return True
|
||||
return Comp()
|
||||
|
||||
|
||||
class TestFixes(unittest.TestCase):
|
||||
"""Hold = fix only what that button is failing on. Nothing here runs a real fix."""
|
||||
|
||||
@@ -660,6 +673,130 @@ class TestFixes(unittest.TestCase):
|
||||
open(os.path.join(os.path.dirname(os.path.abspath(sp.__file__)), "remote-access", "mac-dvs-helper.sh")).read())
|
||||
self.assertIn("open -g /Applications/ProdCom.app", CONFIG["fixes"]["dvs_ingest"]["run"]) # ProdCom after DVS
|
||||
|
||||
def auto_setup(self):
|
||||
ran = []
|
||||
self.engine.executor = lambda cfg, fix_id, chk: ran.append(fix_id) or True
|
||||
comp = mock_comp()
|
||||
return ran, comp
|
||||
|
||||
def test_auto_fix_sunday_service_gentle_fixes_only(self):
|
||||
ran, comp = self.auto_setup()
|
||||
self.snap.remote["tracks\0pgrep -xq Live"] = False # Ableton closed: gentle
|
||||
self.snap.http["http://10.6.7.13:8080/"] = False # Bible app: restart = a hold
|
||||
self.snap.conns["x32"]["category"] = "error" # a reconnect: gentle
|
||||
t = SUNDAY # Sunday 10:00, alarms on
|
||||
run_steps(self.engine, self.snap, t, 10)
|
||||
self.assertEqual(self.engine.auto_fix(self.snap, comp, t + 10), []) # not 20 s yet
|
||||
done = self.engine.auto_fix(self.snap, comp, t + 30)
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(sorted(done), ["apps", "x32"])
|
||||
self.assertEqual(ran, ["ableton"]) # never "bibleapp"
|
||||
self.assertEqual(comp.restarted, ["id-x32"])
|
||||
# while it works, the key says it is fixing itself
|
||||
gate = threading.Event()
|
||||
self.engine.executor = lambda cfg, fix_id, chk: gate.wait(2) or True
|
||||
self.engine.auto_fix(self.snap, comp, t + 130) # second try, 90 s later
|
||||
self.assertEqual(self.engine.notes["apps"]["text"], "AUTO-FIXING\nABLETON")
|
||||
gate.set()
|
||||
|
||||
def test_auto_fix_tries_twice_then_leaves_it_to_you(self):
|
||||
ran, comp = self.auto_setup()
|
||||
self.snap.remote["tracks\0pgrep -xq Live"] = False
|
||||
t = SUNDAY
|
||||
run_steps(self.engine, self.snap, t, 25)
|
||||
for dt in (25, 60, 120, 230, 400): # 90 s apart, at most twice
|
||||
self.engine.auto_fix(self.snap, comp, t + dt)
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(ran, ["ableton", "ableton"])
|
||||
self.snap.remote["tracks\0pgrep -xq Live"] = True # fixed (by hand)...
|
||||
run_steps(self.engine, self.snap, t + 401, 2)
|
||||
self.engine.auto_fix(self.snap, comp, t + 410)
|
||||
self.assertIn(("fix", "ableton", ""), self.engine.auto_tries) # a blink does not reset it
|
||||
self.engine.auto_fix(self.snap, comp, t + 1100)
|
||||
self.assertEqual(self.engine.auto_tries, {}) # gone 10 min: fresh tries
|
||||
|
||||
def test_auto_fix_count_survives_its_own_fault_blinking_out(self):
|
||||
ran, comp = self.auto_setup()
|
||||
t = SUNDAY
|
||||
self.snap.conns["x32"]["category"] = "error"
|
||||
run_steps(self.engine, self.snap, t, 25)
|
||||
for dt in range(25, 900, 30): # each reconnect: error -> starting -> error
|
||||
self.snap.conns["x32"]["category"] = None if dt % 60 == 55 else "error"
|
||||
run_steps(self.engine, self.snap, t + dt, 1)
|
||||
self.engine.auto_fix(self.snap, comp, t + dt + 1)
|
||||
self.assertEqual(comp.restarted, ["id-x32", "id-x32"]) # still only twice
|
||||
|
||||
def test_auto_fix_leaves_dante_alone_when_many_devices_vanish(self):
|
||||
ran, comp = self.auto_setup()
|
||||
t = SUNDAY
|
||||
devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")])
|
||||
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(
|
||||
[d for d in devices if d not in ("NA-Tracks", "NA-ProPresenter-Mac", "BOH-WING")])
|
||||
run_steps(self.engine, self.snap, t, 30)
|
||||
self.engine.auto_fix(self.snap, comp, t + 30)
|
||||
self.assertEqual((ran, comp.restarted), ([], [])) # Ingest's view of Dante: hands off
|
||||
self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(
|
||||
[d for d in devices if d != "NA-Tracks"]) # just the Tracks Mac's DVS
|
||||
run_steps(self.engine, self.snap, t + 31, 1)
|
||||
run_steps(self.engine, self.snap, t + 600, 30)
|
||||
self.engine.auto_fix(self.snap, comp, t + 630)
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(ran, ["dvs_tracks"])
|
||||
self.assertNotIn("dvs_ingest", CONFIG["auto_fix"]["fixes"]) # it closes ProdCom
|
||||
|
||||
def test_midi_reconnects_run_one_after_another(self):
|
||||
calls = []
|
||||
def slow(cfg, fix_id, chk):
|
||||
calls.append((threading.get_ident(), chk["peer"]))
|
||||
time.sleep(0.05)
|
||||
return True
|
||||
self.engine.executor = slow
|
||||
self.snap.midi["NA Ingest"]["peers"] = [] # Lighting PC and SuperRack gone
|
||||
_, t = run_steps(self.engine, self.snap, SUNDAY, 4)
|
||||
self.engine.run_fixes("midi", self.snap, None, t)
|
||||
time.sleep(0.4)
|
||||
self.assertEqual(sorted(p for _, p in calls), ["Lighting PC", "SuperRack-LiveBox"])
|
||||
self.assertEqual(len({tid for tid, _ in calls}), 1) # one worker, not two at once
|
||||
|
||||
def test_restart_key_waits_while_auto_fix_starts_that_app(self):
|
||||
ran, comp = self.auto_setup()
|
||||
self.snap.remote["tracks\0pgrep -xq Live"] = False
|
||||
run_steps(self.engine, self.snap, SUNDAY, 25)
|
||||
self.engine.auto_fix(self.snap, comp, SUNDAY + 25) # auto-fix starts Ableton
|
||||
self.assertIsNone(self.engine.run_force("rs_ableton", self.snap, None, SUNDAY + 30))
|
||||
self.assertEqual(self.engine.notes["rs_ableton"]["text"], "STARTING\nWAIT 1 MIN")
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(ran, ["ableton"]) # not killed while starting
|
||||
|
||||
def test_a_restarted_dead_connection_is_not_a_network_drop(self):
|
||||
t = SUNDAY
|
||||
run_steps(self.engine, self.snap, t, 2)
|
||||
for label in ("x32", "VX1000", "UHDJr", "RESI_Decoder", "tascam"): # dead for a long time
|
||||
self.snap.conns[label]["category"] = "error"
|
||||
run_steps(self.engine, self.snap, t + 100, 2)
|
||||
self.engine.burst, self.engine.conn_drops = None, sp.deque()
|
||||
for cat in (None, "error"): # auto restarts: starting, error
|
||||
for label in ("x32", "VX1000", "UHDJr", "RESI_Decoder", "tascam"):
|
||||
self.snap.conns[label]["category"] = cat
|
||||
tiles, t2 = run_steps(self.engine, self.snap, t + 200, 2)
|
||||
self.assertEqual(tiles["net"]["x"], "NET\nOK")
|
||||
|
||||
def test_auto_fix_waits_out_the_network_and_respects_alarms_off(self):
|
||||
ran, comp = self.auto_setup()
|
||||
self.snap.remote["tracks\0pgrep -xq Live"] = False
|
||||
t = SUNDAY
|
||||
run_steps(self.engine, self.snap, t, 25)
|
||||
self.engine.burst = {"start": t, "conns": set(), "cleared": None} # a network blip going on
|
||||
self.assertEqual(self.engine.auto_fix(self.snap, comp, t + 30), [])
|
||||
self.engine.burst = None
|
||||
self.engine.service.set(False, t + 31) # ALARMS pressed: maintenance
|
||||
self.assertEqual(self.engine.auto_fix(self.snap, comp, t + 40), [])
|
||||
self.engine.service.set(True, t + 41)
|
||||
weekday = SUNDAY - 3 * 86400 # Thursday: holds only
|
||||
run_steps(self.engine, self.snap, weekday, 25)
|
||||
self.assertEqual(self.engine.auto_fix(self.snap, comp, weekday + 30), [])
|
||||
self.assertEqual(ran, [])
|
||||
|
||||
def test_restart_keys_flash_when_their_thing_is_down(self):
|
||||
devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")])
|
||||
devices.remove("NA-Tracks") # DVS on the Tracks Mac gone
|
||||
@@ -1371,7 +1508,7 @@ class TestPage(unittest.TestCase):
|
||||
self.assertEqual(len(holds), len(CONFIG["restart_page"]["keys"]))
|
||||
self.assertIn("$(custom:sp_cmd);force rs_tascam $(internal:time_unix)", holds)
|
||||
self.assertTrue(all(up == [] for up in holds.values())) # a tap does nothing
|
||||
back = c["3"]["0"]["steps"]["0"]["action_sets"]["up"][0]
|
||||
back = c["2"]["6"]["steps"]["0"]["action_sets"]["up"][0] # where the user put it
|
||||
self.assertEqual((back["definitionId"], back["options"]["page"]["value"]), ("set_page", 19))
|
||||
|
||||
def test_push_page_looks_at_what_is_on_the_page(self):
|
||||
|
||||
Reference in New Issue
Block a user