Hold a failing button to fix what it is failing on.

Fixes come from config.json only and run over SSH with NA Ingest's status-panel key (ProPresenter restart, Ableton / Church Translate start, reopen DVS), locally (Resi bridge, captions, ProdDash) or as a HyperDeck reboot. A green button does nothing, each fix waits 60 s before it can run again, and a recording deck is never rebooted. New remote checks ask the Macs every 10 s whether their apps are running. Group buttons and INGEST now have a hold too; tests never run a real fix or open an SSH connection. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 11:34:44 -04:00
parent e873d1aa40
commit 51f09900ab
6 changed files with 310 additions and 29 deletions
@@ -40,7 +40,18 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl
 ## Buttons

 - **Press any button:** acknowledge its alarms (stops the blinking). Press **ALL** to acknowledge everything.
- **Hold a device button for 2 seconds:** restart that button's *Companion connection*. This is the fix for a connection stuck after a network blip. It only restarts Companion's side and never reboots the device.
+- **Hold any button for 2 seconds:** fix what *that button is failing on right now*. A green button does nothing, the same fix won't re-run within 60 s, and every command comes from `"fixes"` in `config.json`. Hold a group button to fix all its members.
+
+  | Failing | Hold does |
+  |---|---|
+  | APPS: PROPRES OFFLINE / NOT RUNNING | restarts ProPresenter on its Mac (the newest version installed) |
+  | APPS: ABLETON / TRANSLATE NOT RUNNING | starts Ableton on the Tracks Mac / Church Translate on NA Translation |
+  | DANTE: PROP MAC / TRACKS / TRANSL GONE | reopens Dante Virtual Soundcard on that Mac (only helps if DVS starts its engine when opened) |
+  | INGEST: BRIDGE / CAPTIONS / PRODDASH | restarts that service on NA Ingest |
+  | HD1–6 lost by Companion | reboots that deck. Never one that is recording, and never for NO DISK or NOT REC |
+  | anything else with a Companion connection | restarts that Companion connection |
+
+  It checks over SSH every 10 s that ProPresenter, Ableton and Church Translate are running, using NA Ingest's key `~/.ssh/id_status_panel`. That key was authorized on the Macs with `remote-access/mac-authorize.sh`.
 - **ALARMS:** alarms are armed all the time (`service_windows` in `config.json`: every day, 00:00–23:59). Press ALARMS to mute the blinking, for example during maintenance. It re-arms itself at midnight.

 ## Page keys on every page
@@ -25,6 +25,31 @@

  "alert_webhook": "",

+  "ssh_key": "~/.ssh/id_status_panel",
+  "hosts": {
+    "prop":        { "ssh": "napropresenter@10.1.14.225" },
+    "tracks":      { "ssh": "waterschurchmedia@10.1.3.227" },
+    "translation": { "ssh": "studio@10.1.7.39" }
+  },
+
+  "fixes": {
+    "propresenter":    { "host": "prop", "label": "restart ProPresenter (newest installed)",
+                         "run": "APP=$(ls -d /Applications/ProPresenter*.app | sort -V | tail -1); if pgrep -xq ProPresenter; then pkill -TERM -x ProPresenter; for i in 1 2 3 4 5 6 7 8 9 10; do pgrep -xq ProPresenter || break; sleep 1; done; pkill -KILL -x ProPresenter; sleep 1; fi; open \"$APP\"" },
+    "ableton":         { "host": "tracks", "label": "start Ableton if it is not running",
+                         "run": "pgrep -xq Live || open -a 'Ableton Live 11 Suite'" },
+    "translate":       { "host": "translation", "label": "start Church Translate if it is not running",
+                         "run": "pgrep -xq 'Church Translate' || open -a 'Church Translate'" },
+    "dvs_prop":        { "host": "prop", "label": "open Dante Virtual Soundcard on the ProPresenter Mac", "run": "open -a 'Dante Virtual Soundcard'" },
+    "dvs_tracks":      { "host": "tracks", "label": "open Dante Virtual Soundcard on the Tracks Mac", "run": "open -a 'Dante Virtual Soundcard'" },
+    "dvs_translation": { "host": "translation", "label": "open Dante Virtual Soundcard on NA Translation", "run": "open -a 'Dante Virtual Soundcard'" },
+    "bridge":          { "kind": "local", "label": "restart the Resi bridge",
+                         "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.resi-propresenter || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.resi-propresenter.plist" },
+    "captions":        { "kind": "local", "label": "restart lobby captions",
+                         "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" },
+    "proddash":        { "kind": "local", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" },
+    "hd_reboot":       { "kind": "hyperdeck_reboot", "label": "reboot HyperDeck" }
+  },
+
  "tiles": [
    { "id": "all",    "label": "ALL",    "pos": [0, 1], "kind": "master" },
    { "id": "rec",    "label": "REC",    "pos": [0, 2], "members": ["hd1", "hd2", "hd3", "hd4", "hd5", "hd6", "tascam"] },
@@ -34,17 +59,19 @@
    { "id": "video",  "label": "VIDEO",  "pos": [0, 6], "members": ["atem", "led", "arena"] },
    { "id": "stream", "label": "STREAM", "pos": [0, 7], "members": ["obs", "resi"] },

-    { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1" }] },
-    { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true }] },
-    { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3" }] },
-    { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true }] },
-    { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5" }] },
-    { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6" }] },
+    { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hd_reboot" } }] },
+    { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hd_reboot" } }] },
    { "id": "tascam", "label": "TASCAM", "pos": [1, 7], "checks": [{ "type": "tascam", "conn": "tascam" }] },

    { "id": "x32", "label": "X32", "pos": [2, 1], "checks": [{ "type": "conn", "conn": "x32" }] },
    { "id": "dante", "label": "DANTE", "pos": [2, 2], "checks": [
-      { "type": "dante", "conn": "audinate-dantecontroller", "devices": {
+      { "type": "dante", "conn": "audinate-dantecontroller",
+        "fix": { "NA-ProPresenter-Mac": "dvs_prop", "NA-Tracks": "dvs_tracks", "NA-Translation": "dvs_translation" },
+        "devices": {
        "FOH-DiGiCo-DMI-1": "DMI-1", "FOH-DiGiCo-DMI-2": "DMI-2", "BOH-WING": "WING", "BOH-DQ-Rack": "DQ RACK",
        "BOH-AD4Q-A": "AD4Q-A", "BOH-AD4Q-B": "AD4Q-B", "STUDIO-A-X32": "STA X32",
        "STUDIO-A-Resi-AJA-SDI": "RESI AJA", "STUDIO-A-Ingest": "INGEST", "NA-Translation": "TRANSL",
@@ -60,8 +87,11 @@
      { "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" }
    ] },
    { "id": "apps", "label": "APPS", "pos": [2, 4], "checks": [
-      { "type": "conn", "conn": "Propresenter-API", "tag": "PROPRES" },
+      { "type": "conn", "conn": "Propresenter-API", "tag": "PROPRES", "fix": "propresenter" },
+      { "type": "remote", "host": "prop", "cmd": "pgrep -xq ProPresenter", "msg": "PROPRES NOT RUNNING", "fix": "propresenter" },
      { "type": "conn", "conn": "ableton", "tag": "ABLETON" },
+      { "type": "remote", "host": "tracks", "cmd": "pgrep -xq Live", "msg": "ABLETON NOT RUNNING", "fix": "ableton" },
+      { "type": "remote", "host": "translation", "cmd": "pgrep -xq 'Church Translate'", "msg": "TRANSLATE NOT RUNNING", "fix": "translate" },
      { "type": "ping", "host": "10.1.3.227", "name": "TRACKS MAC" },
      { "type": "http", "url": "http://10.6.7.13:8080/", "msg": "BIBLE APP DOWN", "interval_seconds": 5 },
      { "type": "conn", "conn": "BibleTV", "tag": "BIBLE OSC" }
@@ -78,9 +108,9 @@
    ] },
    { "id": "net", "label": "NET", "pos": [3, 4], "kind": "network" },
    { "id": "ingest", "label": "INGEST", "pos": [3, 5], "checks": [
-      { "type": "launchd", "label": "org.waterschurch.resi-propresenter", "name": "BRIDGE" },
-      { "type": "launchd", "label": "org.waterschurch.lobby-captions", "name": "CAPTIONS" },
-      { "type": "process", "match": "/Applications/ProdDash.app/Contents/MacOS/ProdDash", "name": "PRODDASH" },
+      { "type": "launchd", "label": "org.waterschurch.resi-propresenter", "name": "BRIDGE", "fix": "bridge" },
+      { "type": "launchd", "label": "org.waterschurch.lobby-captions", "name": "CAPTIONS", "fix": "captions" },
+      { "type": "process", "match": "/Applications/ProdDash.app/Contents/MacOS/ProdDash", "name": "PRODDASH", "fix": "proddash" },
      { "type": "disk", "path": "/", "warn_gb": 25, "err_gb": 8 }
    ] },
    { "id": "led", "label": "LED", "pos": [3, 6], "checks": [
@@ -112,9 +112,11 @@ def button(tid, tile, cfg):
    else:
        stale_text = "'%s\\n?'" % label
        press = "ack " + tid
-        restartable = any(c["type"] in RESTARTABLE and c.get("conn") for c in tile.get("checks", []))
-        hold = ("restart " + tid) if restartable else None
-        note = "Press: acknowledge." + (" Hold 2 s: restart this tile's Companion connection(s)." if hold else "")
+        fixable = "members" in tile or any((c["type"] in RESTARTABLE and c.get("conn")) or c.get("fix")
+                                            for c in tile.get("checks", []))
+        hold = ("restart " + tid) if fixable else None
+        note = "Press: acknowledge." + (" Hold 2 s: fix what this button is failing on (nothing if it is green)."
+                                        if hold else "")
    action_sets = {"down": [], "up": [command_action(tid, "press", names["command"], press)]}
    run_while_held = []
    if hold:
@@ -22,6 +22,7 @@ import logging.handlers
 import os
 import re
 import shutil
+import socket
 import ssl
 import subprocess
 import sys
@@ -137,6 +138,7 @@ class Snapshot:
        self.http = {}           # url -> reachable
        self.disk = {}           # path -> free GB
        self.listeners = {}      # MIDI source name -> coremidi.Listener
+        self.remote = {}         # (host, command) -> True / False / None (could not ask)

    def var(self, label, name):
        return self.vars.get((label, name))
@@ -424,11 +426,22 @@ def check_midi_rx(chk, snap, ctx):
    return [("rx", Result(OK, info="%s %s" % (name, fmt_age(age))))]


+def remote_key(chk):
+    return "%s\0%s" % (chk["host"], chk["cmd"])
+
+
+def check_remote(chk, snap, ctx):
+    """A command run over SSH on another machine, e.g. "is ProPresenter running?"."""
+    if snap.remote.get(remote_key(chk)) is False:
+        return [("remote", Result(ERR, chk["msg"], named=True))]
+    return [("remote", Result(OK))]
+
+
 CHECKS = {
    "conn": check_conn, "hyperdeck": check_hyperdeck, "dante": check_dante, "shure": check_shure,
    "psm": check_psm, "tascam": check_tascam, "ping": check_ping, "launchd": check_launchd,
    "process": check_process, "http": check_http, "disk": check_disk, "midi_peer": check_midi_peer,
-    "mdns": check_mdns, "midi_rx": check_midi_rx,
+    "mdns": check_mdns, "midi_rx": check_midi_rx, "remote": check_remote,
 }

 # Checks whose connection a long press may restart (Dante is shared, so not that one).
@@ -574,6 +587,9 @@ class Engine:
        self.conn_drops = deque()   # (time, conn label, group)
        self.burst = None           # {"start", "conns", "cleared"}
        self.events = []
+        self.meta = {}              # key -> (tile id, check, sub key)
+        self.fix_last = {}          # fix identity -> last run time
+        self.executor = execute_fix

    # raw results for every check key
    def evaluate(self, snap, now, service_on):
@@ -600,6 +616,7 @@ class Engine:
                    if chk.get("tag") and res.level in (WARN, ERR) and not res.named:
                        res.msg, res.named = "%s %s" % (chk["tag"], res.msg), True
                    raw["%s:%d:%s" % (t["id"], i, sub)] = (t["id"], res, chk)
+                    self.meta["%s:%d:%s" % (t["id"], i, sub)] = (t["id"], chk, sub)
        for sub, res in self.network_results(snap, now):
            raw["net:%s" % sub] = ("net", res, {"type": "network"})
        return raw
@@ -698,6 +715,59 @@ class Engine:
        tile = self.tiles.get(tile_id, {})
        return [c["conn"] for c in tile.get("checks", []) if c["type"] in RESTARTABLE and c.get("conn")]

+    def fix_plan(self, tile_id, snap):
+        """What a hold on this button would do: fix only what is failing right now.
+
+        [(identity, "fix", fix id, check) or (identity, "restart", connection label, check)]
+        A check's own "fix" wins; otherwise a failing Companion connection is restarted.
+        Nothing is done for a healthy button, and a recording HyperDeck is never touched.
+        """
+        tile = self.tiles.get(tile_id, {})
+        ids = tile.get("members") or [tile_id]
+        plan, seen = [], set()
+        for key in sorted(self.meta):
+            tid, chk, sub = self.meta[key]
+            tr = self.trackers.get(key)
+            if tid not in ids or not tr or not tr.confirmed or tr.result.level not in (WARN, ERR):
+                continue
+            fix_id = chk.get("fix")
+            if isinstance(fix_id, dict):
+                fix_id = fix_id.get(sub)
+            if fix_id:
+                if chk["type"] == "hyperdeck" and hyperdeck_recording(snap, chk["conn"]):
+                    continue
+                # One run per fix, except a HyperDeck reboot, which is per deck.
+                per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck_reboot"
+                ident = ("fix", fix_id, chk.get("ip", "") if per_deck else "")
+                if ident not in seen:
+                    seen.add(ident)
+                    plan.append((ident, "fix", fix_id, chk))
+            elif chk["type"] in RESTARTABLE and sub == "conn" and chk.get("conn"):
+                ident = ("restart", chk["conn"])
+                if ident not in seen:
+                    seen.add(ident)
+                    plan.append((ident, "restart", chk["conn"], chk))
+        return plan
+
+    def run_fixes(self, tile_id, snap, comp, now, cooldown=60):
+        plan = self.fix_plan(tile_id, snap)
+        if not plan:
+            log.info("HOLD %s: nothing failing, nothing to fix", tile_id)
+        for ident, kind, what, chk in plan:
+            if now - self.fix_last.get(ident, 0) < cooldown:
+                log.info("HOLD %s: %s %s ran less than %ds ago, skipped", tile_id, kind, what, cooldown)
+                continue
+            self.fix_last[ident] = now
+            if kind == "restart":
+                c = snap.conns.get(what)
+                if c and c.get("enabled"):
+                    ok = comp.restart_connection(c["id"])
+                    log.info("RESTART %s -> %s", what, "ok" if ok else "failed")
+            else:
+                log.info("FIX %s: %s", tile_id, self.cfg["fixes"][what].get("label", what))
+                threading.Thread(target=self.executor, args=(self.cfg, what, chk), daemon=True).start()
+        return plan
+

 def display_level(res, acked, service_on):
    if res.level == OFF:
@@ -937,6 +1007,72 @@ def poll_midi(snap, cfg):
                lis.poll()


+# --- Other machines: SSH checks and fixes ------------------------------------
+
+def ssh_cmd(cfg, host):
+    """ssh to a machine in config "hosts", with NA Ingest's status-panel key; one shared connection per host."""
+    return ["/usr/bin/ssh", "-i", os.path.expanduser(cfg.get("ssh_key", "~/.ssh/id_status_panel")),
+            "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "-o", "IdentitiesOnly=yes",
+            "-o", "StrictHostKeyChecking=accept-new", "-o", "ControlMaster=auto",
+            "-o", "ControlPath=/tmp/status-panel-ssh-%C", "-o", "ControlPersist=120", cfg["hosts"][host]["ssh"]]
+
+
+def remote_script(cmds):
+    """One shell line that runs every check and prints SP<n>=<exit code>."""
+    return "; ".join("(%s) >/dev/null 2>&1; echo SP%d=$?" % (cmd, i) for i, cmd in enumerate(cmds))
+
+
+def parse_remote(out, n):
+    codes = dict(re.findall(r"SP(\d+)=(\d+)", out or ""))
+    return [None if str(i) not in codes else codes[str(i)] == "0" for i in range(n)]
+
+
+def poll_remote(snap, cfg):
+    by_host = {}
+    for t in cfg["tiles"]:
+        for c in t.get("checks", []):
+            if c["type"] == "remote":
+                by_host.setdefault(c["host"], [])
+                if c["cmd"] not in by_host[c["host"]]:
+                    by_host[c["host"]].append(c["cmd"])
+    result = dict(snap.remote)
+    for host, cmds in by_host.items():
+        try:
+            out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds)], capture_output=True,
+                                 text=True, timeout=20).stdout
+        except subprocess.TimeoutExpired:
+            out = ""
+        for cmd, ok in zip(cmds, parse_remote(out, len(cmds))):
+            result["%s\0%s" % (host, cmd)] = ok        # None = could not reach it: no verdict
+    snap.remote = result
+
+
+def execute_fix(cfg, fix_id, chk):
+    """Run one entry from config "fixes" (in its own thread). Only listed commands ever run."""
+    fix = cfg["fixes"][fix_id]
+    kind, label = fix.get("kind", "ssh"), fix.get("label", fix_id)
+    try:
+        if kind in ("ssh", "local"):
+            argv = ssh_cmd(cfg, fix["host"]) + [fix["run"]] if kind == "ssh" else ["/bin/zsh", "-c", fix["run"]]
+            r = subprocess.run(argv, capture_output=True, text=True, timeout=fix.get("timeout", 60))
+            detail = (r.stdout + r.stderr).strip().replace("\n", " ")[-200:]
+            log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode,
+                     (": " + detail) if detail else "")
+        elif kind == "hyperdeck_reboot":
+            with socket.create_connection((chk["ip"], 9993), timeout=3) as s:
+                s.settimeout(3)
+                try:
+                    s.recv(1024)                       # the deck's greeting
+                except socket.timeout:
+                    pass
+                s.sendall(b"reboot\n")
+            log.info("FIX %s %s -> reboot sent", label, chk["conn"])
+        else:
+            log.warning("FIX %s: unknown kind %r", label, kind)
+    except Exception as e:
+        log.warning("FIX %s failed: %s", label, e)
+
+
 class Periodic(threading.Thread):
    def __init__(self, name, interval, fn, stop):
        super().__init__(name=name, daemon=True)
@@ -975,6 +1111,15 @@ def load_config(path):
        for m in t.get("members", []):
            if m not in ids:
                raise ValueError("tile %s: unknown member %s" % (t["id"], m))
+        for c in t.get("checks", []):
+            fixes = c.get("fix") if isinstance(c.get("fix"), dict) else {"": c.get("fix")}
+            for f in fixes.values():
+                if f and f not in cfg.get("fixes", {}):
+                    raise ValueError("tile %s: unknown fix %s" % (t["id"], f))
+                if f and cfg["fixes"][f].get("kind", "ssh") == "ssh" and cfg["fixes"][f]["host"] not in cfg.get("hosts", {}):
+                    raise ValueError("fix %s: unknown host" % f)
+            if c["type"] == "remote" and c["host"] not in cfg.get("hosts", {}):
+                raise ValueError("tile %s: unknown host %s" % (t["id"], c["host"]))
    return cfg


@@ -1011,6 +1156,7 @@ def start_pollers(cfg, snap, stop):
        Periodic("system", 10.0, lambda: poll_system(snap, cfg), stop),
        Periodic("bonjour", 15.0, lambda: poll_mdns(snap), stop),
        Periodic("http", 5.0, lambda: poll_http(snap, http_checks(cfg)), stop),
+        Periodic("remote", 10.0, lambda: poll_remote(snap, cfg), stop),
    ]
    for t in threads:
        t.start()
@@ -1042,12 +1188,8 @@ def handle_commands(text, engine, comp, snap, now):
            target = {"on": True, "off": False}.get(arg, not engine.service.on(now))
            engine.service.set(target, now)
            log.info("SERVICE %s (button)", "ON" if target else "OFF")
-        elif verb == "restart":
-            for label in engine.restart_targets(arg):
-                c = snap.conns.get(label)
-                if c and c.get("enabled"):
-                    ok = comp.restart_connection(c["id"])
-                    log.info("RESTART %s -> %s", label, "ok" if ok else "failed")
+        elif verb == "restart":                      # sent by a 2 second hold
+            engine.run_fixes(arg, snap, comp, now)
        else:
            log.warning("unknown command %r", part)

@@ -1123,6 +1265,7 @@ def run_once(cfg):
    poll_system(snap, cfg)
    poll_mdns(snap)
    poll_http(snap, http_checks(cfg))
+    poll_remote(snap, cfg)
    try:
        poll_midi(snap, cfg)
    except Exception as e:
@@ -428,6 +428,88 @@ class TestBehaviour(unittest.TestCase):
        self.assertEqual(engine.restart_targets("ingest"), [])


+class TestFixes(unittest.TestCase):
+    """Hold = fix only what that button is failing on. Nothing here runs a real fix."""
+
+    def setUp(self):
+        self.snap = healthy_snapshot(SUNDAY)
+        self.engine = sp.Engine(CONFIG)
+
+    def plan(self, tile):
+        run_steps(self.engine, self.snap, SUNDAY, 4)
+        return [(kind, what) for _, kind, what, _ in self.engine.fix_plan(tile, self.snap)]
+
+    def test_green_buttons_do_nothing(self):
+        for tile in ("apps", "dante", "rec", "hd1", "x32", "ingest", "show"):
+            self.assertEqual(self.plan(tile), [], tile)
+
+    def test_propresenter_is_restarted_once(self):
+        self.snap.conns["Propresenter-API"]["category"] = "error"
+        self.snap.remote["prop\0pgrep -xq ProPresenter"] = False
+        self.assertEqual(self.plan("apps"), [("fix", "propresenter")])
+        self.assertEqual(self.plan("show"), [("fix", "propresenter")])      # the group does the same
+
+    def test_apps_on_other_macs(self):
+        self.snap.remote["tracks\0pgrep -xq Live"] = False
+        self.snap.remote["translation\0pgrep -xq 'Church Translate'"] = False
+        self.assertEqual(self.plan("apps"), [("fix", "ableton"), ("fix", "translate")])
+
+    def test_unreachable_mac_is_not_a_verdict(self):
+        self.snap.remote["tracks\0pgrep -xq Live"] = None
+        self.assertEqual(self.plan("apps"), [])
+
+    def test_dante_reopens_dvs_on_the_right_mac(self):
+        devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")])
+        devices.remove("NA-ProPresenter-Mac")
+        devices.remove("BOH-WING")                                     # no fix for a console
+        self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices)
+        self.assertEqual(self.plan("dante"), [("fix", "dvs_prop")])
+
+    def test_rec_reboots_only_hung_decks_not_recording_ones(self):
+        for i in (2, 3, 5):
+            self.snap.conns["HyperDeck_%d" % i]["category"] = "error"
+        self.snap.vars[("HyperDeck_3", "status")] = "Record"          # last known state: recording
+        self.assertEqual(self.plan("rec"), [("fix", "hd_reboot"), ("fix", "hd_reboot")])
+        decks = [chk["conn"] for _, _, _, chk in self.engine.fix_plan("rec", self.snap)]
+        self.assertEqual(decks, ["HyperDeck_2", "HyperDeck_5"])
+        self.assertEqual(self.plan("hd3"), [])
+
+    def test_not_recording_is_not_fixed_by_a_reboot(self):
+        for i in (1, 3):
+            self.snap.vars[("HyperDeck_%d" % i, "status")] = "Record"
+        self.assertEqual(self.plan("hd5"), [])                         # NOT REC: a person decides
+
+    def test_connection_without_a_fix_is_restarted(self):
+        self.snap.conns["x32"]["category"] = "error"
+        self.assertEqual(self.plan("x32"), [("restart", "x32")])
+
+    def test_ingest_services(self):
+        del self.snap.launchd["org.waterschurch.resi-propresenter"]
+        self.snap.procs["/Applications/ProdDash.app/Contents/MacOS/ProdDash"] = False
+        self.assertEqual(self.plan("ingest"), [("fix", "bridge"), ("fix", "proddash")])
+
+    def test_cooldown_and_executor(self):
+        ran = []
+        self.engine.executor = lambda cfg, fix_id, chk: ran.append(fix_id)   # record, never run
+        self.snap.remote["tracks\0pgrep -xq Live"] = False
+        run_steps(self.engine, self.snap, SUNDAY, 4)
+        self.engine.run_fixes("apps", self.snap, None, SUNDAY + 10)
+        self.engine.run_fixes("apps", self.snap, None, SUNDAY + 20)    # held again 10 s later: skipped
+        self.engine.run_fixes("apps", self.snap, None, SUNDAY + 80)
+        time.sleep(0.2)
+        self.assertEqual(ran, ["ableton", "ableton"])
+
+    def test_remote_script_round_trip(self):
+        script = sp.remote_script(["pgrep -xq Live", "pgrep -xq 'Church Translate'"])
+        self.assertIn("(pgrep -xq 'Church Translate') >/dev/null 2>&1; echo SP1=$?", script)
+        self.assertEqual(sp.parse_remote("SP0=0\nSP1=1\n", 2), [True, False])
+        self.assertEqual(sp.parse_remote("", 2), [None, None])
+
+    def test_every_fix_is_valid(self):
+        for f in CONFIG["fixes"].values():
+            self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck_reboot"))
+
+
 class FakeCompanion(BaseHTTPRequestHandler):
    """Just enough of Companion 5's HTTP API."""
    store = {}
@@ -509,12 +591,21 @@ class TestAgainstFakeCompanion(unittest.TestCase):

    def test_commands(self):
        comp = sp.Companion("127.0.0.1", self.port)
-        snap = sp.Snapshot()
-        sp.poll_connections(comp, snap)
+        store = FakeCompanion.store
+        snap = healthy_snapshot(SUNDAY)
        engine = sp.Engine(CONFIG)
+        engine.executor = lambda cfg, fix_id, chk: self.fail("no real fix may run in a test")
        now = SUNDAY
-        sp.handle_commands(";restart hd3;restart led;service off;bogus", engine, comp, snap, now)
-        self.assertEqual(FakeCompanion.store["restarts"][-3:], ["id-HyperDeck_3", "id-VX1000", "id-UHDJr"])
+        store["conns"]["VX1000"]["category"] = "error"
+        try:
+            sp.poll_connections(comp, snap)
+            run_steps(engine, snap, now, 4)
+            before = len(store["restarts"])
+            # hd3 is green, so its hold does nothing; LED is failing on VX1000, so that restarts.
+            sp.handle_commands(";restart hd3;restart led;service off;bogus", engine, comp, snap, now + 5)
+            self.assertEqual(store["restarts"][before:], ["id-VX1000"])
+        finally:
+            store["conns"]["VX1000"]["category"] = "good"
        self.assertFalse(engine.service.on(now))
        sp.handle_commands("service toggle", engine, comp, snap, now)
        self.assertTrue(engine.service.on(now))
@@ -533,10 +624,11 @@ class TestAgainstFakeCompanion(unittest.TestCase):
        store["conns"]["ATEM"]["category"] = "error"
        store["custom"]["sp_cmd"] = ";ack atem"
        data_dir = tempfile.mkdtemp()
-        originals = sp.poll_mdns, sp.poll_system, sp.poll_midi
+        originals = sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote
        sp.poll_mdns = lambda snap, seconds=3: None
        sp.poll_system = lambda snap, cfg: None
        sp.poll_midi = lambda snap, cfg: None            # CoreMIDI has its own test
+        sp.poll_remote = lambda snap, cfg: None          # never SSH anywhere from a test
        stop = threading.Event()
        loop = threading.Thread(target=sp.run, args=(cfg, data_dir), kwargs={"stop": stop}, daemon=True)
        try:
@@ -552,7 +644,7 @@ class TestAgainstFakeCompanion(unittest.TestCase):
        finally:
            stop.set()
            loop.join(timeout=15)
-            sp.poll_mdns, sp.poll_system, sp.poll_midi = originals
+            sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote = originals
            store["conns"]["ATEM"]["category"] = "good"
        self.assertFalse(loop.is_alive())                 # it shuts down cleanly
        self.assertTrue(store["custom"]["sp_hb"].isdigit())
@@ -614,7 +706,10 @@ class TestPage(unittest.TestCase):
        self.assertIn("ackall", master["action_sets"]["up"][0]["options"]["value"]["value"])
        self.assertNotIn("2000", master["action_sets"])
        ingest = self.controls["3"]["5"]["steps"]["0"]
-        self.assertNotIn("2000", ingest["action_sets"])              # nothing to restart there
+        self.assertIn("2000", ingest["action_sets"])                 # restarts its services
+        self.assertIn("2000", self.controls["0"]["2"]["steps"]["0"]["action_sets"])   # REC group
+        net = self.controls["3"]["4"]["steps"]["0"]
+        self.assertNotIn("2000", net["action_sets"])                 # nothing to fix there
        ids = [a["id"] for r in self.controls.values() for c in r.values()
               for s in c.get("steps", {}).values() for al in s["action_sets"].values() for a in al]
        self.assertEqual(len(ids), len(set(ids)))