diff --git a/README.md b/README.md index 2d5e36c..5f07c15 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,18 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl ## Buttons - **Press any button:** acknowledge its alarms (stops the blinking). Press **ALL** to acknowledge everything. -- **Hold a device button for 2 seconds:** restart that button's *Companion connection*. This is the fix for a connection stuck after a network blip. It only restarts Companion's side and never reboots the device. +- **Hold any button for 2 seconds:** fix what *that button is failing on right now*. A green button does nothing, the same fix won't re-run within 60 s, and every command comes from `"fixes"` in `config.json`. Hold a group button to fix all its members. + + | Failing | Hold does | + |---|---| + | APPS: PROPRES OFFLINE / NOT RUNNING | restarts ProPresenter on its Mac (the newest version installed) | + | APPS: ABLETON / TRANSLATE NOT RUNNING | starts Ableton on the Tracks Mac / Church Translate on NA Translation | + | DANTE: PROP MAC / TRACKS / TRANSL GONE | reopens Dante Virtual Soundcard on that Mac (only helps if DVS starts its engine when opened) | + | INGEST: BRIDGE / CAPTIONS / PRODDASH | restarts that service on NA Ingest | + | HD1–6 lost by Companion | reboots that deck. Never one that is recording, and never for NO DISK or NOT REC | + | anything else with a Companion connection | restarts that Companion connection | + + It checks over SSH every 10 s that ProPresenter, Ableton and Church Translate are running, using NA Ingest's key `~/.ssh/id_status_panel`. That key was authorized on the Macs with `remote-access/mac-authorize.sh`. - **ALARMS:** alarms are armed all the time (`service_windows` in `config.json`: every day, 00:00–23:59). Press ALARMS to mute the blinking, for example during maintenance. It re-arms itself at midnight. ## Page keys on every page diff --git a/config.json b/config.json index 6b7fcc6..c0d7e67 100644 --- a/config.json +++ b/config.json @@ -25,6 +25,31 @@ "alert_webhook": "", + "ssh_key": "~/.ssh/id_status_panel", + "hosts": { + "prop": { "ssh": "napropresenter@10.1.14.225" }, + "tracks": { "ssh": "waterschurchmedia@10.1.3.227" }, + "translation": { "ssh": "studio@10.1.7.39" } + }, + + "fixes": { + "propresenter": { "host": "prop", "label": "restart ProPresenter (newest installed)", + "run": "APP=$(ls -d /Applications/ProPresenter*.app | sort -V | tail -1); if pgrep -xq ProPresenter; then pkill -TERM -x ProPresenter; for i in 1 2 3 4 5 6 7 8 9 10; do pgrep -xq ProPresenter || break; sleep 1; done; pkill -KILL -x ProPresenter; sleep 1; fi; open \"$APP\"" }, + "ableton": { "host": "tracks", "label": "start Ableton if it is not running", + "run": "pgrep -xq Live || open -a 'Ableton Live 11 Suite'" }, + "translate": { "host": "translation", "label": "start Church Translate if it is not running", + "run": "pgrep -xq 'Church Translate' || open -a 'Church Translate'" }, + "dvs_prop": { "host": "prop", "label": "open Dante Virtual Soundcard on the ProPresenter Mac", "run": "open -a 'Dante Virtual Soundcard'" }, + "dvs_tracks": { "host": "tracks", "label": "open Dante Virtual Soundcard on the Tracks Mac", "run": "open -a 'Dante Virtual Soundcard'" }, + "dvs_translation": { "host": "translation", "label": "open Dante Virtual Soundcard on NA Translation", "run": "open -a 'Dante Virtual Soundcard'" }, + "bridge": { "kind": "local", "label": "restart the Resi bridge", + "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.resi-propresenter || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.resi-propresenter.plist" }, + "captions": { "kind": "local", "label": "restart lobby captions", + "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" }, + "proddash": { "kind": "local", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" }, + "hd_reboot": { "kind": "hyperdeck_reboot", "label": "reboot HyperDeck" } + }, + "tiles": [ { "id": "all", "label": "ALL", "pos": [0, 1], "kind": "master" }, { "id": "rec", "label": "REC", "pos": [0, 2], "members": ["hd1", "hd2", "hd3", "hd4", "hd5", "hd6", "tascam"] }, @@ -34,17 +59,19 @@ { "id": "video", "label": "VIDEO", "pos": [0, 6], "members": ["atem", "led", "arena"] }, { "id": "stream", "label": "STREAM", "pos": [0, 7], "members": ["obs", "resi"] }, - { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1" }] }, - { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true }] }, - { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3" }] }, - { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true }] }, - { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5" }] }, - { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6" }] }, + { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hd_reboot" } }] }, { "id": "tascam", "label": "TASCAM", "pos": [1, 7], "checks": [{ "type": "tascam", "conn": "tascam" }] }, { "id": "x32", "label": "X32", "pos": [2, 1], "checks": [{ "type": "conn", "conn": "x32" }] }, { "id": "dante", "label": "DANTE", "pos": [2, 2], "checks": [ - { "type": "dante", "conn": "audinate-dantecontroller", "devices": { + { "type": "dante", "conn": "audinate-dantecontroller", + "fix": { "NA-ProPresenter-Mac": "dvs_prop", "NA-Tracks": "dvs_tracks", "NA-Translation": "dvs_translation" }, + "devices": { "FOH-DiGiCo-DMI-1": "DMI-1", "FOH-DiGiCo-DMI-2": "DMI-2", "BOH-WING": "WING", "BOH-DQ-Rack": "DQ RACK", "BOH-AD4Q-A": "AD4Q-A", "BOH-AD4Q-B": "AD4Q-B", "STUDIO-A-X32": "STA X32", "STUDIO-A-Resi-AJA-SDI": "RESI AJA", "STUDIO-A-Ingest": "INGEST", "NA-Translation": "TRANSL", @@ -60,8 +87,11 @@ { "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" } ] }, { "id": "apps", "label": "APPS", "pos": [2, 4], "checks": [ - { "type": "conn", "conn": "Propresenter-API", "tag": "PROPRES" }, + { "type": "conn", "conn": "Propresenter-API", "tag": "PROPRES", "fix": "propresenter" }, + { "type": "remote", "host": "prop", "cmd": "pgrep -xq ProPresenter", "msg": "PROPRES NOT RUNNING", "fix": "propresenter" }, { "type": "conn", "conn": "ableton", "tag": "ABLETON" }, + { "type": "remote", "host": "tracks", "cmd": "pgrep -xq Live", "msg": "ABLETON NOT RUNNING", "fix": "ableton" }, + { "type": "remote", "host": "translation", "cmd": "pgrep -xq 'Church Translate'", "msg": "TRANSLATE NOT RUNNING", "fix": "translate" }, { "type": "ping", "host": "10.1.3.227", "name": "TRACKS MAC" }, { "type": "http", "url": "http://10.6.7.13:8080/", "msg": "BIBLE APP DOWN", "interval_seconds": 5 }, { "type": "conn", "conn": "BibleTV", "tag": "BIBLE OSC" } @@ -78,9 +108,9 @@ ] }, { "id": "net", "label": "NET", "pos": [3, 4], "kind": "network" }, { "id": "ingest", "label": "INGEST", "pos": [3, 5], "checks": [ - { "type": "launchd", "label": "org.waterschurch.resi-propresenter", "name": "BRIDGE" }, - { "type": "launchd", "label": "org.waterschurch.lobby-captions", "name": "CAPTIONS" }, - { "type": "process", "match": "/Applications/ProdDash.app/Contents/MacOS/ProdDash", "name": "PRODDASH" }, + { "type": "launchd", "label": "org.waterschurch.resi-propresenter", "name": "BRIDGE", "fix": "bridge" }, + { "type": "launchd", "label": "org.waterschurch.lobby-captions", "name": "CAPTIONS", "fix": "captions" }, + { "type": "process", "match": "/Applications/ProdDash.app/Contents/MacOS/ProdDash", "name": "PRODDASH", "fix": "proddash" }, { "type": "disk", "path": "/", "warn_gb": 25, "err_gb": 8 } ] }, { "id": "led", "label": "LED", "pos": [3, 6], "checks": [ diff --git a/make_page.py b/make_page.py index f24ab03..d0215f0 100644 --- a/make_page.py +++ b/make_page.py @@ -112,9 +112,11 @@ def button(tid, tile, cfg): else: stale_text = "'%s\\n?'" % label press = "ack " + tid - restartable = any(c["type"] in RESTARTABLE and c.get("conn") for c in tile.get("checks", [])) - hold = ("restart " + tid) if restartable else None - note = "Press: acknowledge." + (" Hold 2 s: restart this tile's Companion connection(s)." if hold else "") + fixable = "members" in tile or any((c["type"] in RESTARTABLE and c.get("conn")) or c.get("fix") + for c in tile.get("checks", [])) + hold = ("restart " + tid) if fixable else None + note = "Press: acknowledge." + (" Hold 2 s: fix what this button is failing on (nothing if it is green)." + if hold else "") action_sets = {"down": [], "up": [command_action(tid, "press", names["command"], press)]} run_while_held = [] if hold: diff --git a/status-page.companionconfig b/status-page.companionconfig index 17ca892..645c1d4 100644 Binary files a/status-page.companionconfig and b/status-page.companionconfig differ diff --git a/status_panel.py b/status_panel.py index bc064c6..49be29e 100644 --- a/status_panel.py +++ b/status_panel.py @@ -22,6 +22,7 @@ import logging.handlers import os import re import shutil +import socket import ssl import subprocess import sys @@ -137,6 +138,7 @@ class Snapshot: self.http = {} # url -> reachable self.disk = {} # path -> free GB self.listeners = {} # MIDI source name -> coremidi.Listener + self.remote = {} # (host, command) -> True / False / None (could not ask) def var(self, label, name): return self.vars.get((label, name)) @@ -424,11 +426,22 @@ def check_midi_rx(chk, snap, ctx): return [("rx", Result(OK, info="%s %s" % (name, fmt_age(age))))] +def remote_key(chk): + return "%s\0%s" % (chk["host"], chk["cmd"]) + + +def check_remote(chk, snap, ctx): + """A command run over SSH on another machine, e.g. "is ProPresenter running?".""" + if snap.remote.get(remote_key(chk)) is False: + return [("remote", Result(ERR, chk["msg"], named=True))] + return [("remote", Result(OK))] + + CHECKS = { "conn": check_conn, "hyperdeck": check_hyperdeck, "dante": check_dante, "shure": check_shure, "psm": check_psm, "tascam": check_tascam, "ping": check_ping, "launchd": check_launchd, "process": check_process, "http": check_http, "disk": check_disk, "midi_peer": check_midi_peer, - "mdns": check_mdns, "midi_rx": check_midi_rx, + "mdns": check_mdns, "midi_rx": check_midi_rx, "remote": check_remote, } # Checks whose connection a long press may restart (Dante is shared, so not that one). @@ -574,6 +587,9 @@ class Engine: self.conn_drops = deque() # (time, conn label, group) self.burst = None # {"start", "conns", "cleared"} self.events = [] + self.meta = {} # key -> (tile id, check, sub key) + self.fix_last = {} # fix identity -> last run time + self.executor = execute_fix # raw results for every check key def evaluate(self, snap, now, service_on): @@ -600,6 +616,7 @@ class Engine: if chk.get("tag") and res.level in (WARN, ERR) and not res.named: res.msg, res.named = "%s %s" % (chk["tag"], res.msg), True raw["%s:%d:%s" % (t["id"], i, sub)] = (t["id"], res, chk) + self.meta["%s:%d:%s" % (t["id"], i, sub)] = (t["id"], chk, sub) for sub, res in self.network_results(snap, now): raw["net:%s" % sub] = ("net", res, {"type": "network"}) return raw @@ -698,6 +715,59 @@ class Engine: tile = self.tiles.get(tile_id, {}) return [c["conn"] for c in tile.get("checks", []) if c["type"] in RESTARTABLE and c.get("conn")] + def fix_plan(self, tile_id, snap): + """What a hold on this button would do: fix only what is failing right now. + + [(identity, "fix", fix id, check) or (identity, "restart", connection label, check)] + A check's own "fix" wins; otherwise a failing Companion connection is restarted. + Nothing is done for a healthy button, and a recording HyperDeck is never touched. + """ + tile = self.tiles.get(tile_id, {}) + ids = tile.get("members") or [tile_id] + plan, seen = [], set() + for key in sorted(self.meta): + tid, chk, sub = self.meta[key] + tr = self.trackers.get(key) + if tid not in ids or not tr or not tr.confirmed or tr.result.level not in (WARN, ERR): + continue + fix_id = chk.get("fix") + if isinstance(fix_id, dict): + fix_id = fix_id.get(sub) + if fix_id: + if chk["type"] == "hyperdeck" and hyperdeck_recording(snap, chk["conn"]): + continue + # One run per fix, except a HyperDeck reboot, which is per deck. + per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck_reboot" + ident = ("fix", fix_id, chk.get("ip", "") if per_deck else "") + if ident not in seen: + seen.add(ident) + plan.append((ident, "fix", fix_id, chk)) + elif chk["type"] in RESTARTABLE and sub == "conn" and chk.get("conn"): + ident = ("restart", chk["conn"]) + if ident not in seen: + seen.add(ident) + plan.append((ident, "restart", chk["conn"], chk)) + return plan + + def run_fixes(self, tile_id, snap, comp, now, cooldown=60): + plan = self.fix_plan(tile_id, snap) + if not plan: + log.info("HOLD %s: nothing failing, nothing to fix", tile_id) + for ident, kind, what, chk in plan: + if now - self.fix_last.get(ident, 0) < cooldown: + log.info("HOLD %s: %s %s ran less than %ds ago, skipped", tile_id, kind, what, cooldown) + continue + self.fix_last[ident] = now + if kind == "restart": + c = snap.conns.get(what) + if c and c.get("enabled"): + ok = comp.restart_connection(c["id"]) + log.info("RESTART %s -> %s", what, "ok" if ok else "failed") + else: + log.info("FIX %s: %s", tile_id, self.cfg["fixes"][what].get("label", what)) + threading.Thread(target=self.executor, args=(self.cfg, what, chk), daemon=True).start() + return plan + def display_level(res, acked, service_on): if res.level == OFF: @@ -937,6 +1007,72 @@ def poll_midi(snap, cfg): lis.poll() +# --- Other machines: SSH checks and fixes ------------------------------------ + +def ssh_cmd(cfg, host): + """ssh to a machine in config "hosts", with NA Ingest's status-panel key; one shared connection per host.""" + return ["/usr/bin/ssh", "-i", os.path.expanduser(cfg.get("ssh_key", "~/.ssh/id_status_panel")), + "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=accept-new", "-o", "ControlMaster=auto", + "-o", "ControlPath=/tmp/status-panel-ssh-%C", "-o", "ControlPersist=120", cfg["hosts"][host]["ssh"]] + + +def remote_script(cmds): + """One shell line that runs every check and prints SP=.""" + return "; ".join("(%s) >/dev/null 2>&1; echo SP%d=$?" % (cmd, i) for i, cmd in enumerate(cmds)) + + +def parse_remote(out, n): + codes = dict(re.findall(r"SP(\d+)=(\d+)", out or "")) + return [None if str(i) not in codes else codes[str(i)] == "0" for i in range(n)] + + +def poll_remote(snap, cfg): + by_host = {} + for t in cfg["tiles"]: + for c in t.get("checks", []): + if c["type"] == "remote": + by_host.setdefault(c["host"], []) + if c["cmd"] not in by_host[c["host"]]: + by_host[c["host"]].append(c["cmd"]) + result = dict(snap.remote) + for host, cmds in by_host.items(): + try: + out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds)], capture_output=True, + text=True, timeout=20).stdout + except subprocess.TimeoutExpired: + out = "" + for cmd, ok in zip(cmds, parse_remote(out, len(cmds))): + result["%s\0%s" % (host, cmd)] = ok # None = could not reach it: no verdict + snap.remote = result + + +def execute_fix(cfg, fix_id, chk): + """Run one entry from config "fixes" (in its own thread). Only listed commands ever run.""" + fix = cfg["fixes"][fix_id] + kind, label = fix.get("kind", "ssh"), fix.get("label", fix_id) + try: + if kind in ("ssh", "local"): + argv = ssh_cmd(cfg, fix["host"]) + [fix["run"]] if kind == "ssh" else ["/bin/zsh", "-c", fix["run"]] + r = subprocess.run(argv, capture_output=True, text=True, timeout=fix.get("timeout", 60)) + detail = (r.stdout + r.stderr).strip().replace("\n", " ")[-200:] + log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode, + (": " + detail) if detail else "") + elif kind == "hyperdeck_reboot": + with socket.create_connection((chk["ip"], 9993), timeout=3) as s: + s.settimeout(3) + try: + s.recv(1024) # the deck's greeting + except socket.timeout: + pass + s.sendall(b"reboot\n") + log.info("FIX %s %s -> reboot sent", label, chk["conn"]) + else: + log.warning("FIX %s: unknown kind %r", label, kind) + except Exception as e: + log.warning("FIX %s failed: %s", label, e) + + class Periodic(threading.Thread): def __init__(self, name, interval, fn, stop): super().__init__(name=name, daemon=True) @@ -975,6 +1111,15 @@ def load_config(path): for m in t.get("members", []): if m not in ids: raise ValueError("tile %s: unknown member %s" % (t["id"], m)) + for c in t.get("checks", []): + fixes = c.get("fix") if isinstance(c.get("fix"), dict) else {"": c.get("fix")} + for f in fixes.values(): + if f and f not in cfg.get("fixes", {}): + raise ValueError("tile %s: unknown fix %s" % (t["id"], f)) + if f and cfg["fixes"][f].get("kind", "ssh") == "ssh" and cfg["fixes"][f]["host"] not in cfg.get("hosts", {}): + raise ValueError("fix %s: unknown host" % f) + if c["type"] == "remote" and c["host"] not in cfg.get("hosts", {}): + raise ValueError("tile %s: unknown host %s" % (t["id"], c["host"])) return cfg @@ -1011,6 +1156,7 @@ def start_pollers(cfg, snap, stop): Periodic("system", 10.0, lambda: poll_system(snap, cfg), stop), Periodic("bonjour", 15.0, lambda: poll_mdns(snap), stop), Periodic("http", 5.0, lambda: poll_http(snap, http_checks(cfg)), stop), + Periodic("remote", 10.0, lambda: poll_remote(snap, cfg), stop), ] for t in threads: t.start() @@ -1042,12 +1188,8 @@ def handle_commands(text, engine, comp, snap, now): target = {"on": True, "off": False}.get(arg, not engine.service.on(now)) engine.service.set(target, now) log.info("SERVICE %s (button)", "ON" if target else "OFF") - elif verb == "restart": - for label in engine.restart_targets(arg): - c = snap.conns.get(label) - if c and c.get("enabled"): - ok = comp.restart_connection(c["id"]) - log.info("RESTART %s -> %s", label, "ok" if ok else "failed") + elif verb == "restart": # sent by a 2 second hold + engine.run_fixes(arg, snap, comp, now) else: log.warning("unknown command %r", part) @@ -1123,6 +1265,7 @@ def run_once(cfg): poll_system(snap, cfg) poll_mdns(snap) poll_http(snap, http_checks(cfg)) + poll_remote(snap, cfg) try: poll_midi(snap, cfg) except Exception as e: diff --git a/tests/test_status_panel.py b/tests/test_status_panel.py index 1385c70..418848c 100644 --- a/tests/test_status_panel.py +++ b/tests/test_status_panel.py @@ -428,6 +428,88 @@ class TestBehaviour(unittest.TestCase): self.assertEqual(engine.restart_targets("ingest"), []) +class TestFixes(unittest.TestCase): + """Hold = fix only what that button is failing on. Nothing here runs a real fix.""" + + def setUp(self): + self.snap = healthy_snapshot(SUNDAY) + self.engine = sp.Engine(CONFIG) + + def plan(self, tile): + run_steps(self.engine, self.snap, SUNDAY, 4) + return [(kind, what) for _, kind, what, _ in self.engine.fix_plan(tile, self.snap)] + + def test_green_buttons_do_nothing(self): + for tile in ("apps", "dante", "rec", "hd1", "x32", "ingest", "show"): + self.assertEqual(self.plan(tile), [], tile) + + def test_propresenter_is_restarted_once(self): + self.snap.conns["Propresenter-API"]["category"] = "error" + self.snap.remote["prop\0pgrep -xq ProPresenter"] = False + self.assertEqual(self.plan("apps"), [("fix", "propresenter")]) + self.assertEqual(self.plan("show"), [("fix", "propresenter")]) # the group does the same + + def test_apps_on_other_macs(self): + self.snap.remote["tracks\0pgrep -xq Live"] = False + self.snap.remote["translation\0pgrep -xq 'Church Translate'"] = False + self.assertEqual(self.plan("apps"), [("fix", "ableton"), ("fix", "translate")]) + + def test_unreachable_mac_is_not_a_verdict(self): + self.snap.remote["tracks\0pgrep -xq Live"] = None + self.assertEqual(self.plan("apps"), []) + + def test_dante_reopens_dvs_on_the_right_mac(self): + devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")]) + devices.remove("NA-ProPresenter-Mac") + devices.remove("BOH-WING") # no fix for a console + self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices) + self.assertEqual(self.plan("dante"), [("fix", "dvs_prop")]) + + def test_rec_reboots_only_hung_decks_not_recording_ones(self): + for i in (2, 3, 5): + self.snap.conns["HyperDeck_%d" % i]["category"] = "error" + self.snap.vars[("HyperDeck_3", "status")] = "Record" # last known state: recording + self.assertEqual(self.plan("rec"), [("fix", "hd_reboot"), ("fix", "hd_reboot")]) + decks = [chk["conn"] for _, _, _, chk in self.engine.fix_plan("rec", self.snap)] + self.assertEqual(decks, ["HyperDeck_2", "HyperDeck_5"]) + self.assertEqual(self.plan("hd3"), []) + + def test_not_recording_is_not_fixed_by_a_reboot(self): + for i in (1, 3): + self.snap.vars[("HyperDeck_%d" % i, "status")] = "Record" + self.assertEqual(self.plan("hd5"), []) # NOT REC: a person decides + + def test_connection_without_a_fix_is_restarted(self): + self.snap.conns["x32"]["category"] = "error" + self.assertEqual(self.plan("x32"), [("restart", "x32")]) + + def test_ingest_services(self): + del self.snap.launchd["org.waterschurch.resi-propresenter"] + self.snap.procs["/Applications/ProdDash.app/Contents/MacOS/ProdDash"] = False + self.assertEqual(self.plan("ingest"), [("fix", "bridge"), ("fix", "proddash")]) + + def test_cooldown_and_executor(self): + ran = [] + self.engine.executor = lambda cfg, fix_id, chk: ran.append(fix_id) # record, never run + self.snap.remote["tracks\0pgrep -xq Live"] = False + run_steps(self.engine, self.snap, SUNDAY, 4) + self.engine.run_fixes("apps", self.snap, None, SUNDAY + 10) + self.engine.run_fixes("apps", self.snap, None, SUNDAY + 20) # held again 10 s later: skipped + self.engine.run_fixes("apps", self.snap, None, SUNDAY + 80) + time.sleep(0.2) + self.assertEqual(ran, ["ableton", "ableton"]) + + def test_remote_script_round_trip(self): + script = sp.remote_script(["pgrep -xq Live", "pgrep -xq 'Church Translate'"]) + self.assertIn("(pgrep -xq 'Church Translate') >/dev/null 2>&1; echo SP1=$?", script) + self.assertEqual(sp.parse_remote("SP0=0\nSP1=1\n", 2), [True, False]) + self.assertEqual(sp.parse_remote("", 2), [None, None]) + + def test_every_fix_is_valid(self): + for f in CONFIG["fixes"].values(): + self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck_reboot")) + + class FakeCompanion(BaseHTTPRequestHandler): """Just enough of Companion 5's HTTP API.""" store = {} @@ -509,12 +591,21 @@ class TestAgainstFakeCompanion(unittest.TestCase): def test_commands(self): comp = sp.Companion("127.0.0.1", self.port) - snap = sp.Snapshot() - sp.poll_connections(comp, snap) + store = FakeCompanion.store + snap = healthy_snapshot(SUNDAY) engine = sp.Engine(CONFIG) + engine.executor = lambda cfg, fix_id, chk: self.fail("no real fix may run in a test") now = SUNDAY - sp.handle_commands(";restart hd3;restart led;service off;bogus", engine, comp, snap, now) - self.assertEqual(FakeCompanion.store["restarts"][-3:], ["id-HyperDeck_3", "id-VX1000", "id-UHDJr"]) + store["conns"]["VX1000"]["category"] = "error" + try: + sp.poll_connections(comp, snap) + run_steps(engine, snap, now, 4) + before = len(store["restarts"]) + # hd3 is green, so its hold does nothing; LED is failing on VX1000, so that restarts. + sp.handle_commands(";restart hd3;restart led;service off;bogus", engine, comp, snap, now + 5) + self.assertEqual(store["restarts"][before:], ["id-VX1000"]) + finally: + store["conns"]["VX1000"]["category"] = "good" self.assertFalse(engine.service.on(now)) sp.handle_commands("service toggle", engine, comp, snap, now) self.assertTrue(engine.service.on(now)) @@ -533,10 +624,11 @@ class TestAgainstFakeCompanion(unittest.TestCase): store["conns"]["ATEM"]["category"] = "error" store["custom"]["sp_cmd"] = ";ack atem" data_dir = tempfile.mkdtemp() - originals = sp.poll_mdns, sp.poll_system, sp.poll_midi + originals = sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote sp.poll_mdns = lambda snap, seconds=3: None sp.poll_system = lambda snap, cfg: None sp.poll_midi = lambda snap, cfg: None # CoreMIDI has its own test + sp.poll_remote = lambda snap, cfg: None # never SSH anywhere from a test stop = threading.Event() loop = threading.Thread(target=sp.run, args=(cfg, data_dir), kwargs={"stop": stop}, daemon=True) try: @@ -552,7 +644,7 @@ class TestAgainstFakeCompanion(unittest.TestCase): finally: stop.set() loop.join(timeout=15) - sp.poll_mdns, sp.poll_system, sp.poll_midi = originals + sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote = originals store["conns"]["ATEM"]["category"] = "good" self.assertFalse(loop.is_alive()) # it shuts down cleanly self.assertTrue(store["custom"]["sp_hb"].isdigit()) @@ -614,7 +706,10 @@ class TestPage(unittest.TestCase): self.assertIn("ackall", master["action_sets"]["up"][0]["options"]["value"]["value"]) self.assertNotIn("2000", master["action_sets"]) ingest = self.controls["3"]["5"]["steps"]["0"] - self.assertNotIn("2000", ingest["action_sets"]) # nothing to restart there + self.assertIn("2000", ingest["action_sets"]) # restarts its services + self.assertIn("2000", self.controls["0"]["2"]["steps"]["0"]["action_sets"]) # REC group + net = self.controls["3"]["4"]["steps"]["0"] + self.assertNotIn("2000", net["action_sets"]) # nothing to fix there ids = [a["id"] for r in self.controls.values() for c in r.values() for s in c.get("steps", {}).values() for al in s["action_sets"].values() for a in al] self.assertEqual(len(ids), len(set(ids)))