Hold a failing button to fix what it is failing on.

Fixes come from config.json only and run over SSH with NA Ingest's
status-panel key (ProPresenter restart, Ableton / Church Translate start,
reopen DVS), locally (Resi bridge, captions, ProdDash) or as a HyperDeck
reboot. A green button does nothing, each fix waits 60 s before it can run
again, and a recording deck is never rebooted. New remote checks ask the Macs
every 10 s whether their apps are running. Group buttons and INGEST now have a
hold too; tests never run a real fix or open an SSH connection.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-08 11:34:44 -04:00
co-authored by Claude Opus 5.5
parent e873d1aa40
commit 51f09900ab
6 changed files with 310 additions and 29 deletions
+41 -11
View File
@@ -25,6 +25,31 @@
"alert_webhook": "",
"ssh_key": "~/.ssh/id_status_panel",
"hosts": {
"prop": { "ssh": "napropresenter@10.1.14.225" },
"tracks": { "ssh": "waterschurchmedia@10.1.3.227" },
"translation": { "ssh": "studio@10.1.7.39" }
},
"fixes": {
"propresenter": { "host": "prop", "label": "restart ProPresenter (newest installed)",
"run": "APP=$(ls -d /Applications/ProPresenter*.app | sort -V | tail -1); if pgrep -xq ProPresenter; then pkill -TERM -x ProPresenter; for i in 1 2 3 4 5 6 7 8 9 10; do pgrep -xq ProPresenter || break; sleep 1; done; pkill -KILL -x ProPresenter; sleep 1; fi; open \"$APP\"" },
"ableton": { "host": "tracks", "label": "start Ableton if it is not running",
"run": "pgrep -xq Live || open -a 'Ableton Live 11 Suite'" },
"translate": { "host": "translation", "label": "start Church Translate if it is not running",
"run": "pgrep -xq 'Church Translate' || open -a 'Church Translate'" },
"dvs_prop": { "host": "prop", "label": "open Dante Virtual Soundcard on the ProPresenter Mac", "run": "open -a 'Dante Virtual Soundcard'" },
"dvs_tracks": { "host": "tracks", "label": "open Dante Virtual Soundcard on the Tracks Mac", "run": "open -a 'Dante Virtual Soundcard'" },
"dvs_translation": { "host": "translation", "label": "open Dante Virtual Soundcard on NA Translation", "run": "open -a 'Dante Virtual Soundcard'" },
"bridge": { "kind": "local", "label": "restart the Resi bridge",
"run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.resi-propresenter || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.resi-propresenter.plist" },
"captions": { "kind": "local", "label": "restart lobby captions",
"run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" },
"proddash": { "kind": "local", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" },
"hd_reboot": { "kind": "hyperdeck_reboot", "label": "reboot HyperDeck" }
},
"tiles": [
{ "id": "all", "label": "ALL", "pos": [0, 1], "kind": "master" },
{ "id": "rec", "label": "REC", "pos": [0, 2], "members": ["hd1", "hd2", "hd3", "hd4", "hd5", "hd6", "tascam"] },
@@ -34,17 +59,19 @@
{ "id": "video", "label": "VIDEO", "pos": [0, 6], "members": ["atem", "led", "arena"] },
{ "id": "stream", "label": "STREAM", "pos": [0, 7], "members": ["obs", "resi"] },
{ "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1" }] },
{ "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true }] },
{ "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3" }] },
{ "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true }] },
{ "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5" }] },
{ "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6" }] },
{ "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hd_reboot" } }] },
{ "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hd_reboot" } }] },
{ "id": "tascam", "label": "TASCAM", "pos": [1, 7], "checks": [{ "type": "tascam", "conn": "tascam" }] },
{ "id": "x32", "label": "X32", "pos": [2, 1], "checks": [{ "type": "conn", "conn": "x32" }] },
{ "id": "dante", "label": "DANTE", "pos": [2, 2], "checks": [
{ "type": "dante", "conn": "audinate-dantecontroller", "devices": {
{ "type": "dante", "conn": "audinate-dantecontroller",
"fix": { "NA-ProPresenter-Mac": "dvs_prop", "NA-Tracks": "dvs_tracks", "NA-Translation": "dvs_translation" },
"devices": {
"FOH-DiGiCo-DMI-1": "DMI-1", "FOH-DiGiCo-DMI-2": "DMI-2", "BOH-WING": "WING", "BOH-DQ-Rack": "DQ RACK",
"BOH-AD4Q-A": "AD4Q-A", "BOH-AD4Q-B": "AD4Q-B", "STUDIO-A-X32": "STA X32",
"STUDIO-A-Resi-AJA-SDI": "RESI AJA", "STUDIO-A-Ingest": "INGEST", "NA-Translation": "TRANSL",
@@ -60,8 +87,11 @@
{ "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" }
] },
{ "id": "apps", "label": "APPS", "pos": [2, 4], "checks": [
{ "type": "conn", "conn": "Propresenter-API", "tag": "PROPRES" },
{ "type": "conn", "conn": "Propresenter-API", "tag": "PROPRES", "fix": "propresenter" },
{ "type": "remote", "host": "prop", "cmd": "pgrep -xq ProPresenter", "msg": "PROPRES NOT RUNNING", "fix": "propresenter" },
{ "type": "conn", "conn": "ableton", "tag": "ABLETON" },
{ "type": "remote", "host": "tracks", "cmd": "pgrep -xq Live", "msg": "ABLETON NOT RUNNING", "fix": "ableton" },
{ "type": "remote", "host": "translation", "cmd": "pgrep -xq 'Church Translate'", "msg": "TRANSLATE NOT RUNNING", "fix": "translate" },
{ "type": "ping", "host": "10.1.3.227", "name": "TRACKS MAC" },
{ "type": "http", "url": "http://10.6.7.13:8080/", "msg": "BIBLE APP DOWN", "interval_seconds": 5 },
{ "type": "conn", "conn": "BibleTV", "tag": "BIBLE OSC" }
@@ -78,9 +108,9 @@
] },
{ "id": "net", "label": "NET", "pos": [3, 4], "kind": "network" },
{ "id": "ingest", "label": "INGEST", "pos": [3, 5], "checks": [
{ "type": "launchd", "label": "org.waterschurch.resi-propresenter", "name": "BRIDGE" },
{ "type": "launchd", "label": "org.waterschurch.lobby-captions", "name": "CAPTIONS" },
{ "type": "process", "match": "/Applications/ProdDash.app/Contents/MacOS/ProdDash", "name": "PRODDASH" },
{ "type": "launchd", "label": "org.waterschurch.resi-propresenter", "name": "BRIDGE", "fix": "bridge" },
{ "type": "launchd", "label": "org.waterschurch.lobby-captions", "name": "CAPTIONS", "fix": "captions" },
{ "type": "process", "match": "/Applications/ProdDash.app/Contents/MacOS/ProdDash", "name": "PRODDASH", "fix": "proddash" },
{ "type": "disk", "path": "/", "warn_gb": 25, "err_gb": 8 }
] },
{ "id": "led", "label": "LED", "pos": [3, 6], "checks": [