diff --git a/config.json b/config.json index c0d7e67..395a77a 100644 --- a/config.json +++ b/config.json @@ -33,21 +33,21 @@ }, "fixes": { - "propresenter": { "host": "prop", "label": "restart ProPresenter (newest installed)", + "propresenter": { "host": "prop", "short": "PROPRES", "label": "restart ProPresenter (newest installed)", "run": "APP=$(ls -d /Applications/ProPresenter*.app | sort -V | tail -1); if pgrep -xq ProPresenter; then pkill -TERM -x ProPresenter; for i in 1 2 3 4 5 6 7 8 9 10; do pgrep -xq ProPresenter || break; sleep 1; done; pkill -KILL -x ProPresenter; sleep 1; fi; open \"$APP\"" }, - "ableton": { "host": "tracks", "label": "start Ableton if it is not running", + "ableton": { "host": "tracks", "short": "ABLETON", "label": "start Ableton if it is not running", "run": "pgrep -xq Live || open -a 'Ableton Live 11 Suite'" }, - "translate": { "host": "translation", "label": "start Church Translate if it is not running", + "translate": { "host": "translation", "short": "TRANSLATE", "label": "start Church Translate if it is not running", "run": "pgrep -xq 'Church Translate' || open -a 'Church Translate'" }, - "dvs_prop": { "host": "prop", "label": "open Dante Virtual Soundcard on the ProPresenter Mac", "run": "open -a 'Dante Virtual Soundcard'" }, - "dvs_tracks": { "host": "tracks", "label": "open Dante Virtual Soundcard on the Tracks Mac", "run": "open -a 'Dante Virtual Soundcard'" }, - "dvs_translation": { "host": "translation", "label": "open Dante Virtual Soundcard on NA Translation", "run": "open -a 'Dante Virtual Soundcard'" }, - "bridge": { "kind": "local", "label": "restart the Resi bridge", + "dvs_prop": { "host": "prop", "short": "DVS", "label": "open Dante Virtual Soundcard on the ProPresenter Mac", "run": "open -a 'Dante Virtual Soundcard'" }, + "dvs_tracks": { "host": "tracks", "short": "DVS", "label": "open Dante Virtual Soundcard on the Tracks Mac", "run": "open -a 'Dante Virtual Soundcard'" }, + "dvs_translation": { "host": "translation", "short": "DVS", "label": "open Dante Virtual Soundcard on NA Translation", "run": "open -a 'Dante Virtual Soundcard'" }, + "bridge": { "kind": "local", "short": "BRIDGE", "label": "restart the Resi bridge", "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.resi-propresenter || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.resi-propresenter.plist" }, - "captions": { "kind": "local", "label": "restart lobby captions", + "captions": { "kind": "local", "short": "CAPTIONS", "label": "restart lobby captions", "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" }, - "proddash": { "kind": "local", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" }, - "hd_reboot": { "kind": "hyperdeck_reboot", "label": "reboot HyperDeck" } + "proddash": { "kind": "local", "short": "PRODDASH", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" }, + "hd_reboot": { "kind": "hyperdeck_reboot", "short": "REBOOT", "label": "reboot HyperDeck" } }, "tiles": [ diff --git a/status_panel.py b/status_panel.py index 49be29e..b32a775 100644 --- a/status_panel.py +++ b/status_panel.py @@ -53,6 +53,8 @@ COLORS = { "off": ((0x262626, 0x8C8C8C), (0x262626, 0x8C8C8C)), "idle": ((0x262626, 0xD0D0D0), (0x262626, 0xD0D0D0)), "service": ((0x0B57D0, 0xFFFFFF), (0x0B57D0, 0xFFFFFF)), + "fixing": ((0x2962FF, 0xFFFFFF), (0x2962FF, 0xFFFFFF)), # a hold is being acted on + "failed": ((0xFF6D00, 0x000000), (0xFF6D00, 0x000000)), # the fix command itself failed } @@ -590,6 +592,8 @@ class Engine: self.meta = {} # key -> (tile id, check, sub key) self.fix_last = {} # fix identity -> last run time self.executor = execute_fix + self.notes = {} # tile id -> hold feedback {"text", "level", "until"} + self.now = 0.0 # raw results for every check key def evaluate(self, snap, now, service_on): @@ -672,6 +676,7 @@ class Engine: return label def step(self, snap, now, debounce=None): + self.now = now service_on = self.service.on(now) raw = self.evaluate(snap, now, service_on) deb = self.debounce if debounce is None else debounce @@ -749,13 +754,21 @@ class Engine: plan.append((ident, "restart", chk["conn"], chk)) return plan + def note(self, tile_id, text, level, now, seconds): + """Show `text` on the button for a while (hold feedback), instead of its usual text.""" + self.notes[tile_id] = {"text": text, "level": level, "until": now + seconds} + def run_fixes(self, tile_id, snap, comp, now, cooldown=60): plan = self.fix_plan(tile_id, snap) if not plan: log.info("HOLD %s: nothing failing, nothing to fix", tile_id) + self.note(tile_id, "NOTHING\nTO FIX", "idle", now, 4) + return plan + started, names, waited = [], [], False for ident, kind, what, chk in plan: if now - self.fix_last.get(ident, 0) < cooldown: log.info("HOLD %s: %s %s ran less than %ds ago, skipped", tile_id, kind, what, cooldown) + waited = True continue self.fix_last[ident] = now if kind == "restart": @@ -763,9 +776,33 @@ class Engine: if c and c.get("enabled"): ok = comp.restart_connection(c["id"]) log.info("RESTART %s -> %s", what, "ok" if ok else "failed") + names.append("RECONNECT") else: - log.info("FIX %s: %s", tile_id, self.cfg["fixes"][what].get("label", what)) - threading.Thread(target=self.executor, args=(self.cfg, what, chk), daemon=True).start() + fix = self.cfg["fixes"][what] + log.info("FIX %s: %s", tile_id, fix.get("label", what)) + names.append(fix.get("short", what.upper())) + started.append((what, chk)) + if not names: + self.note(tile_id, "JUST TRIED\nWAIT 1 MIN" if waited else "NOTHING\nTO FIX", "idle", now, 4) + return plan + summary = names[0] if len(set(names)) == 1 else "%d FIXES" % len(names) + if not started: # only Companion reconnects: quick + self.note(tile_id, "CHECKING\n" + summary, "fixing", now, 15) + return plan + self.note(tile_id, "FIXING\n" + summary, "fixing", now, 180) + results = [] + + def work(fix_id, chk): + ok = self.executor(self.cfg, fix_id, chk) + results.append(ok is not False) + if len(results) == len(started): # the last one to finish reports for all + done = time.time() + if all(results): + self.note(tile_id, "CHECKING\n" + summary, "fixing", done, 20) + else: + self.note(tile_id, "FIX FAILED\n" + summary, "failed", done, 20) + for fix_id, chk in started: + threading.Thread(target=work, args=(fix_id, chk), daemon=True).start() return plan @@ -838,6 +875,10 @@ def render(engine, service_on): out = {} for tile in engine.cfg["tiles"]: tid, kind = tile["id"], tile.get("kind") + note = engine.notes.get(tid) + if note and engine.now < note["until"]: # a hold is being acted on: say so + out[tid] = colored(tile["label"] + "\n" + note["text"], note["level"]) + continue if kind == "service": manual = engine.service.override is not None text = tile["label"] + "\n" + ("ON" if service_on else "OFF") + ("\nmanual" if manual else "") @@ -1048,7 +1089,8 @@ def poll_remote(snap, cfg): def execute_fix(cfg, fix_id, chk): - """Run one entry from config "fixes" (in its own thread). Only listed commands ever run.""" + """Run one entry from config "fixes" (in its own thread). Only listed commands ever run. + Returns True if the command ran and succeeded.""" fix = cfg["fixes"][fix_id] kind, label = fix.get("kind", "ssh"), fix.get("label", fix_id) try: @@ -1058,7 +1100,8 @@ def execute_fix(cfg, fix_id, chk): detail = (r.stdout + r.stderr).strip().replace("\n", " ")[-200:] log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode, (": " + detail) if detail else "") - elif kind == "hyperdeck_reboot": + return r.returncode == 0 + if kind == "hyperdeck_reboot": with socket.create_connection((chk["ip"], 9993), timeout=3) as s: s.settimeout(3) try: @@ -1067,10 +1110,11 @@ def execute_fix(cfg, fix_id, chk): pass s.sendall(b"reboot\n") log.info("FIX %s %s -> reboot sent", label, chk["conn"]) - else: - log.warning("FIX %s: unknown kind %r", label, kind) + return True + log.warning("FIX %s: unknown kind %r", label, kind) except Exception as e: log.warning("FIX %s failed: %s", label, e) + return False class Periodic(threading.Thread): @@ -1213,6 +1257,7 @@ def run(cfg, data_dir, dry_run=False, stop=None): if cmd and cmd.strip(): comp.set_custom(names["command"], "") handle_commands(cmd, engine, comp, snap, now) + last_eval = 0.0 # redraw now, so a press shows at once if now - last_eval >= 1.0: last_eval = now tiles = engine.step(snap, now) diff --git a/tests/test_status_panel.py b/tests/test_status_panel.py index 418848c..79e44ff 100644 --- a/tests/test_status_panel.py +++ b/tests/test_status_panel.py @@ -499,6 +499,45 @@ class TestFixes(unittest.TestCase): time.sleep(0.2) self.assertEqual(ran, ["ableton", "ableton"]) + def test_hold_feedback_on_the_button(self): + devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")]) + devices.remove("NA-ProPresenter-Mac") + self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices) + gate = threading.Event() + outcome = {"ok": True} + + def slow_fix(cfg, fix_id, chk): + gate.wait(2) + return outcome["ok"] + self.engine.executor = slow_fix + _, t = run_steps(self.engine, self.snap, SUNDAY, 4) + self.engine.run_fixes("dante", self.snap, None, t) + tiles = self.engine.step(self.snap, t) + self.assertEqual((tiles["dante"]["x"], level(tiles["dante"])), ("DANTE\nFIXING\nDVS", "fixing")) + gate.set() + time.sleep(0.2) + tiles = self.engine.step(self.snap, time.time()) + self.assertEqual((tiles["dante"]["x"], level(tiles["dante"])), ("DANTE\nCHECKING\nDVS", "fixing")) + tiles = self.engine.step(self.snap, time.time() + 25) # still gone: back to the alarm + self.assertEqual(tiles["dante"]["x"], "DANTE\nPROP MAC GONE") + # held again within a minute + self.engine.run_fixes("dante", self.snap, None, t + 30) + self.assertEqual(self.engine.notes["dante"]["text"], "JUST TRIED\nWAIT 1 MIN") + # a fix whose command fails says so + outcome["ok"] = False + self.engine.run_fixes("dante", self.snap, None, t + 100) + time.sleep(0.2) + tiles = self.engine.step(self.snap, time.time()) + self.assertEqual((tiles["dante"]["x"], level(tiles["dante"])), ("DANTE\nFIX FAILED\nDVS", "failed")) + + def test_hold_on_a_green_button_says_so(self): + _, t = run_steps(self.engine, self.snap, SUNDAY, 2) + self.engine.run_fixes("x32", self.snap, None, t) + tiles = self.engine.step(self.snap, t + 1) + self.assertEqual((tiles["x32"]["x"], level(tiles["x32"])), ("X32\nNOTHING\nTO FIX", "idle")) + tiles = self.engine.step(self.snap, t + 5) + self.assertEqual(tiles["x32"]["x"], "X32\nOK") + def test_remote_script_round_trip(self): script = sp.remote_script(["pgrep -xq Live", "pgrep -xq 'Church Translate'"]) self.assertIn("(pgrep -xq 'Church Translate') >/dev/null 2>&1; echo SP1=$?", script)