From 51f09900ab42100c7006e2cbecdfceac4676768d Mon Sep 17 00:00:00 2001 From: christian Date: Thu, 8 Oct 2026 11:34:44 -0400 Subject: [PATCH] Hold a failing button to fix what it is failing on. Fixes come from config.json only and run over SSH with NA Ingest's status-panel key (ProPresenter restart, Ableton / Church Translate start, reopen DVS), locally (Resi bridge, captions, ProdDash) or as a HyperDeck reboot. A green button does nothing, each fix waits 60 s before it can run again, and a recording deck is never rebooted. New remote checks ask the Macs every 10 s whether their apps are running. Group buttons and INGEST now have a hold too; tests never run a real fix or open an SSH connection. Co-Authored-By: Claude Opus 5.5 --- README.md | 13 ++- config.json | 52 +++++++++--- make_page.py | 8 +- status-page.companionconfig | Bin 5122 -> 5379 bytes status_panel.py | 157 ++++++++++++++++++++++++++++++++++-- tests/test_status_panel.py | 109 +++++++++++++++++++++++-- 6 files changed, 310 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 2d5e36c..5f07c15 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,18 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl ## Buttons - **Press any button:** acknowledge its alarms (stops the blinking). Press **ALL** to acknowledge everything. -- **Hold a device button for 2 seconds:** restart that button's *Companion connection*. This is the fix for a connection stuck after a network blip. It only restarts Companion's side and never reboots the device. +- **Hold any button for 2 seconds:** fix what *that button is failing on right now*. A green button does nothing, the same fix won't re-run within 60 s, and every command comes from `"fixes"` in `config.json`. Hold a group button to fix all its members. + + | Failing | Hold does | + |---|---| + | APPS: PROPRES OFFLINE / NOT RUNNING | restarts ProPresenter on its Mac (the newest version installed) | + | APPS: ABLETON / TRANSLATE NOT RUNNING | starts Ableton on the Tracks Mac / Church Translate on NA Translation | + | DANTE: PROP MAC / TRACKS / TRANSL GONE | reopens Dante Virtual Soundcard on that Mac (only helps if DVS starts its engine when opened) | + | INGEST: BRIDGE / CAPTIONS / PRODDASH | restarts that service on NA Ingest | + | HD1–6 lost by Companion | reboots that deck. Never one that is recording, and never for NO DISK or NOT REC | + | anything else with a Companion connection | restarts that Companion connection | + + It checks over SSH every 10 s that ProPresenter, Ableton and Church Translate are running, using NA Ingest's key `~/.ssh/id_status_panel`. That key was authorized on the Macs with `remote-access/mac-authorize.sh`. - **ALARMS:** alarms are armed all the time (`service_windows` in `config.json`: every day, 00:00–23:59). Press ALARMS to mute the blinking, for example during maintenance. It re-arms itself at midnight. ## Page keys on every page diff --git a/config.json b/config.json index 6b7fcc6..c0d7e67 100644 --- a/config.json +++ b/config.json @@ -25,6 +25,31 @@ "alert_webhook": "", + "ssh_key": "~/.ssh/id_status_panel", + "hosts": { + "prop": { "ssh": "napropresenter@10.1.14.225" }, + "tracks": { "ssh": "waterschurchmedia@10.1.3.227" }, + "translation": { "ssh": "studio@10.1.7.39" } + }, + + "fixes": { + "propresenter": { "host": "prop", "label": "restart ProPresenter (newest installed)", + "run": "APP=$(ls -d /Applications/ProPresenter*.app | sort -V | tail -1); if pgrep -xq ProPresenter; then pkill -TERM -x ProPresenter; for i in 1 2 3 4 5 6 7 8 9 10; do pgrep -xq ProPresenter || break; sleep 1; done; pkill -KILL -x ProPresenter; sleep 1; fi; open \"$APP\"" }, + "ableton": { "host": "tracks", "label": "start Ableton if it is not running", + "run": "pgrep -xq Live || open -a 'Ableton Live 11 Suite'" }, + "translate": { "host": "translation", "label": "start Church Translate if it is not running", + "run": "pgrep -xq 'Church Translate' || open -a 'Church Translate'" }, + "dvs_prop": { "host": "prop", "label": "open Dante Virtual Soundcard on the ProPresenter Mac", "run": "open -a 'Dante Virtual Soundcard'" }, + "dvs_tracks": { "host": "tracks", "label": "open Dante Virtual Soundcard on the Tracks Mac", "run": "open -a 'Dante Virtual Soundcard'" }, + "dvs_translation": { "host": "translation", "label": "open Dante Virtual Soundcard on NA Translation", "run": "open -a 'Dante Virtual Soundcard'" }, + "bridge": { "kind": "local", "label": "restart the Resi bridge", + "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.resi-propresenter || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.resi-propresenter.plist" }, + "captions": { "kind": "local", "label": "restart lobby captions", + "run": "launchctl kickstart -k gui/$(id -u)/org.waterschurch.lobby-captions || launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.waterschurch.lobby-captions.plist" }, + "proddash": { "kind": "local", "label": "start ProdDash", "run": "open /Applications/ProdDash.app" }, + "hd_reboot": { "kind": "hyperdeck_reboot", "label": "reboot HyperDeck" } + }, + "tiles": [ { "id": "all", "label": "ALL", "pos": [0, 1], "kind": "master" }, { "id": "rec", "label": "REC", "pos": [0, 2], "members": ["hd1", "hd2", "hd3", "hd4", "hd5", "hd6", "tascam"] }, @@ -34,17 +59,19 @@ { "id": "video", "label": "VIDEO", "pos": [0, 6], "members": ["atem", "led", "arena"] }, { "id": "stream", "label": "STREAM", "pos": [0, 7], "members": ["obs", "resi"] }, - { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1" }] }, - { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true }] }, - { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3" }] }, - { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true }] }, - { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5" }] }, - { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6" }] }, + { "id": "hd1", "label": "HD1", "pos": [1, 1], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_1", "ip": "10.1.10.11", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd2", "label": "HD2", "pos": [1, 2], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_2", "optional": true, "ip": "10.1.10.12", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd3", "label": "HD3", "pos": [1, 3], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_3", "ip": "10.1.10.13", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd4", "label": "HD4", "pos": [1, 4], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_4", "optional": true, "ip": "10.1.10.14", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd5", "label": "HD5", "pos": [1, 5], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_5", "ip": "10.1.10.15", "fix": { "conn": "hd_reboot" } }] }, + { "id": "hd6", "label": "HD6", "pos": [1, 6], "checks": [{ "type": "hyperdeck", "conn": "HyperDeck_6", "ip": "10.1.10.16", "fix": { "conn": "hd_reboot" } }] }, { "id": "tascam", "label": "TASCAM", "pos": [1, 7], "checks": [{ "type": "tascam", "conn": "tascam" }] }, { "id": "x32", "label": "X32", "pos": [2, 1], "checks": [{ "type": "conn", "conn": "x32" }] }, { "id": "dante", "label": "DANTE", "pos": [2, 2], "checks": [ - { "type": "dante", "conn": "audinate-dantecontroller", "devices": { + { "type": "dante", "conn": "audinate-dantecontroller", + "fix": { "NA-ProPresenter-Mac": "dvs_prop", "NA-Tracks": "dvs_tracks", "NA-Translation": "dvs_translation" }, + "devices": { "FOH-DiGiCo-DMI-1": "DMI-1", "FOH-DiGiCo-DMI-2": "DMI-2", "BOH-WING": "WING", "BOH-DQ-Rack": "DQ RACK", "BOH-AD4Q-A": "AD4Q-A", "BOH-AD4Q-B": "AD4Q-B", "STUDIO-A-X32": "STA X32", "STUDIO-A-Resi-AJA-SDI": "RESI AJA", "STUDIO-A-Ingest": "INGEST", "NA-Translation": "TRANSL", @@ -60,8 +87,11 @@ { "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" } ] }, { "id": "apps", "label": "APPS", "pos": [2, 4], "checks": [ - { "type": "conn", "conn": "Propresenter-API", "tag": "PROPRES" }, + { "type": "conn", "conn": "Propresenter-API", "tag": "PROPRES", "fix": "propresenter" }, + { "type": "remote", "host": "prop", "cmd": "pgrep -xq ProPresenter", "msg": "PROPRES NOT RUNNING", "fix": "propresenter" }, { "type": "conn", "conn": "ableton", "tag": "ABLETON" }, + { "type": "remote", "host": "tracks", "cmd": "pgrep -xq Live", "msg": "ABLETON NOT RUNNING", "fix": "ableton" }, + { "type": "remote", "host": "translation", "cmd": "pgrep -xq 'Church Translate'", "msg": "TRANSLATE NOT RUNNING", "fix": "translate" }, { "type": "ping", "host": "10.1.3.227", "name": "TRACKS MAC" }, { "type": "http", "url": "http://10.6.7.13:8080/", "msg": "BIBLE APP DOWN", "interval_seconds": 5 }, { "type": "conn", "conn": "BibleTV", "tag": "BIBLE OSC" } @@ -78,9 +108,9 @@ ] }, { "id": "net", "label": "NET", "pos": [3, 4], "kind": "network" }, { "id": "ingest", "label": "INGEST", "pos": [3, 5], "checks": [ - { "type": "launchd", "label": "org.waterschurch.resi-propresenter", "name": "BRIDGE" }, - { "type": "launchd", "label": "org.waterschurch.lobby-captions", "name": "CAPTIONS" }, - { "type": "process", "match": "/Applications/ProdDash.app/Contents/MacOS/ProdDash", "name": "PRODDASH" }, + { "type": "launchd", "label": "org.waterschurch.resi-propresenter", "name": "BRIDGE", "fix": "bridge" }, + { "type": "launchd", "label": "org.waterschurch.lobby-captions", "name": "CAPTIONS", "fix": "captions" }, + { "type": "process", "match": "/Applications/ProdDash.app/Contents/MacOS/ProdDash", "name": "PRODDASH", "fix": "proddash" }, { "type": "disk", "path": "/", "warn_gb": 25, "err_gb": 8 } ] }, { "id": "led", "label": "LED", "pos": [3, 6], "checks": [ diff --git a/make_page.py b/make_page.py index f24ab03..d0215f0 100644 --- a/make_page.py +++ b/make_page.py @@ -112,9 +112,11 @@ def button(tid, tile, cfg): else: stale_text = "'%s\\n?'" % label press = "ack " + tid - restartable = any(c["type"] in RESTARTABLE and c.get("conn") for c in tile.get("checks", [])) - hold = ("restart " + tid) if restartable else None - note = "Press: acknowledge." + (" Hold 2 s: restart this tile's Companion connection(s)." if hold else "") + fixable = "members" in tile or any((c["type"] in RESTARTABLE and c.get("conn")) or c.get("fix") + for c in tile.get("checks", [])) + hold = ("restart " + tid) if fixable else None + note = "Press: acknowledge." + (" Hold 2 s: fix what this button is failing on (nothing if it is green)." + if hold else "") action_sets = {"down": [], "up": [command_action(tid, "press", names["command"], press)]} run_while_held = [] if hold: diff --git a/status-page.companionconfig b/status-page.companionconfig index 17ca89297f105cd2968c83f46662d8d8a2df8061..645c1d46846eb2a519f90abe0a3b0fe9f753c36a 100644 GIT binary patch literal 5379 zcma)fc{tQ>^fr>EB3lVDb|Fj1#Mlza9z%ArGqwsd6UG_|CB~M03EB5`7-h|pUDh#{ zWS<$bkI9d|@9+KR{pbDbT<3GH>+_s*pXWLEeE`ukH1iET5Q;NzA6p+^Z?PA)&W_^t zULG%OJzc##?Y%smT%F0bG5+8s^JeOWGa}dlY-MjT2IuCtJQ-I4M}D~jY=WoFcfE1A z9Lf`7$vr%VCGnHrNO-Q+n{-r?I}IHd0`;Q${#zh5q&`5yL-3xe5XpkR~(%UZvTkr<*sa9@}y zzF~3S{q60b$q^$kqxF3sC@flnxyvjuw1h`|pJ{ZNZP*w|bqJ2Srk*@R_`C4*rO$h?F$Fc~la7UMG^FkO7_+^2 zd^l?dgnqpL{VxoH-y|36r`~Qwpg3&A)>}y=U)FMO;ZlWtGs`A(X;K>9dMS1;H3XsE zW^A1NQ(aqgL?EQaiywIcbvI}0zd|||8;9LOW(HD(TnpJ!ET`8K(sZilGC3?sscxC( zeo;NX)3N|x>iqs=6gl$K@1}s6cv0x2=@h=ryWXCm-;|A){eYt)ilG~Mom+)lrKMvU z)xUvY$~X%k)xi=ye(A-E2tgmhjG9Dd&l(Pin-$?7wm7Q32`jiV>Voeh&VY#(6MvVb zOR5fJ`(5XyGoppSMJyg?NA$<(YoIJ<;NU7$N@sSzO= zm%KB%Dr!Dq*NRhTUT=9HXRgVb$~*nqQ51R%LEa6*mJl};TKc`C5QoW~ni$5c(0I-# zCrHhAcZwgTv-0gft{*7{NANR4hO9aL4=9*rJWHQ$)8?c6Myuaue5Veh%@;9})4q{F zKl1)k@y4mA@|_QD;PEO162}k~6DgbX;M3R%b(~NbHxqB|c6a#$(*74ihe{dAs;<}X zUf|^FeeCWMb=98s^x0{6LhqS&mUbyP@^qx^H#P=vRhIbc2qDMx6lSZ%-y`9Z&8~t; z99Ps5SQ;or2b7kq?aLSoHT5!=PBY7^5a7CkAaF4;Zxb7YrUpqHClh*Xz5+*&HK zW=fWI?fg_SimO#d5=hl|%*Gr0*%C$)eISB52_)oq%>x#nbToRpO?~p%j7hln#eD7C zuscN5+p>IZurOzA;vgZiYQVfONUZ2?60JkK2~RytphHgr)RU-!u_sB6u=IxnZg{M2 z|KJH(yB9@`defA$UQz;N&re3asY#EcXpcs6^F0A)K5HD&R+h<1|D(Bmvhet5-BrC& z&dwrbd;c2XC?-fI(wS^07@kmYN4KMzXtjqrwxawNw!qlU(5-U*e@Yt4$lwDsfH;35 z9^tl%T7xa4XrS1?Xc*&fdhf%T196HuAkUO1si1q(>rd0wKc!hsy=mL7X_vm0Pw$K4 zit8tbq6ccdb4*;pCZN1ATOkZsazeu0k5yT^t|lA$6r}YuA)6-$x|+KGZo8&n0r~=@ z%GSs^NALL$fY8~^!bH)w1)P(xEidfwjoKL-`xIh!3r8<~juM)EK4XR458G+4IZ@07 zR1Y66Ix3n5q3_5*8VIIdez2sO@+Fk1Fwj2TN|hxwk7*Vqmov7^Bj~S)rdC`8yvHIJ zU<8nsVZvvg#ltv$hPI$rPV+a(t_N#KUix}zft*0YK*R8m?9_4b4xXq#IIx=_$#}XI z(`n-1s&eGPcj}&=LwI^LaRpy~UGh{=s=qha4g=LFa#0tJU&1`8)KmLj3hY3+d6;mb z6@_ga-=qoZIzLX%yB72*`3H-We`^P|(lUT<2^6xdEvd%3*XBw}cly{<)3 zck!i%EIr)G{?`VTgLWX~*uD4A^T@1#shC8l0W;Yki$azg&_aQ-!vr|IrKcPY3nBXj zJqIX5S&6xY)A(28!A`lKD9 zed(nZg7i{t;C%S(f}0#sx_ZP=b^O%Q01QY?fBdQ#q3*YGTl@b0rs}T!vkhaLfwm{b zldV!961vM-YS(IiFBinUzaoQu)H`YUY2Fa$`_iwFQ$lF9EsI4!RGe?UGRqBQVIGPNz*{ zo}$Ga(LKLbn>J{IhJ|uFM9AjBW8MS$R{!{eRE~DR9&>5K%x2OC_m=CNgm6PF-?3Vf zGn-4>_s?c^?JVWdx-ai;>>i1vou)|*uTC!qoe>QwPF7VZAPs@@U0F|5;(~GJ6k3yg z>x^=!{GnIjb}=AOOwthOB7znscZABEl{)F~K9PeduDF*(ARrdHa-WXKfADUeT2v3> z-Mm~LE54R582X%sT8CS%Pl8xGelvQy09GQpD^agC?58m;dA^Y^&#<{|q!@*SyUKi? zy?>ER%?JJMRKX7nMEa{ElKV&DK^I9N#krGWR#Acuom?K>j-{iGP& zb%^A>=HmDlA4aQVb3vdRc?LgRoww%n+D8(z_g{y|O|T#^Sm^rIZ39wh+Iw zr$|9bp+=^U&Eo1JDf!`l-Ldt<&C3BsV9*F5Tmz}IRk{y8#}zwU5jZz*%?bY*$ZW;w zl-7`Osw1m-!DF2jQ#B4^9hMz=6Q7Z#_NKu zE~y++4s)*I-r;DoiGmDiBQcOL&)A4E*LaRypZiH0HNjK9YnfBVKCHVt2N^$=Vufn+ z9y-Zv$(T=z4(w~MWgZQz4gO!!qr$LZ|8r>^XdF~v*`N!=pRNraX}|;>&dp;O)fyLB zv-EigC3J2Mb-AIhw0pF|l`3MEz#(jD$N-ty%bn$qU`{3@t)}vu3G4QGz?Ff4ahIPD z1Ybrp1=3Md@oJQt)KWv#7K<~^f>t<&nb zs#*R-;G)XGKqnu4ivOi{Dhv`mqDcGybkl!SxmoD^OwDuLDVwQ-ahlnnVGJ_{c~R!U zuiq~m1Wy`#>$`kXlav%VT%@?6oXtY8gJ>h|@bpoDjkaNf|GnG2H$2>hQ#O-_6^}oS zC|*84nIZ;D=>jT}PE{W2S2fc5JT3o8hniK9tPI#abaS@c zPa#>`l3rggoIm5%WJ%#OY_(@Z3T~eZaGDTwEBJO46!db&xuolMBMEua&KPmf&#fx_mz zEN~eHRI_2*CN-_C^3M;_xXX=hTcy1JgW`MG#9=mBYxJIVRK?P(rom*UJQR^`TSjG) zI%;+TPkqX^yw+^`)v|(`7`#f%^T;j?`QOVPilcA_j0>TLW+~>$@ia~sl52sJoPwN} z2Cm$f#PPaqPpKc={svzo&14goPrlGVN2qy8b*_m_m|IXv!GyH0?pu{94u)l#dHFGB zgr3bWew3(X{AN8hjSbsR6ZT6=@B5WY0X@`l(iE}wL`0-z|EB} ztTpygCBTM>KZ{=3?Mei72VX+S;y1$ce=N^^3vKj3ZsvO?F$qfe>Oa}b%K4_~$p+%)C)?2%7Omc>_E{iiU!JcW)1zWXFPnbba%{EPa`V;R2>AI{3 z9@K`#JuNvDN!i*bCWd7aVvkJF%$sL@Kp8If*ef?~Y=}cz^lIZeiKIuo*UM~oC0|p@ zq$ucc3C3P%_sCw>n!0$e#5R57cFI48tNdsYvr9u>X{&e*cGc@SHe!2!ZtVJf2I}bM z{LU&kfqL2ki2!33VQ`gyL5z3~3#i*r9xD*)imfWD`b@Kt@@ikN(C9sHR5m_yz{ZlM zfwe9VzA4KLp!4f;+tleLb2G8uCQ;lQ*V7Qpva^sut1&V}S5rD_y40(4e8$5pYG+v} zuZF7Mx%4gc`NPVK863jk&S$gRDmK$=s9GmBBgZom6hQ^UoYcEGr-YQNn8bRF9PECr zU(+QdZtu(2eb&+UZlaQ#*!u%L4!w*npT6lKQD=p@sat@YeYleAuaG#^Jd$HC!p!-+ z+Ggq|C*jTVCd)Vc0xaS4!r?XnHFH4J(40&93R?hCun{_N)R;deY%AUxi^$&zg^>UeJq&QtZzw zzrJ7kr~i++bGk$_lLufa9m457Xls4_Z@$M58qOLch9(kRo6EWKIXv?3FU;D>_APJ9+6*6N3!UcefY9%lPd~)7 z2)LRaS@A^ZBVZO3*bqF8!vz$C|6Xc<#41PNHm290T@C-=lJ}KZzldRTYy{0A>QJ*K z%N|>3Z)+u-PsJ#(@X>ug-+}Yi6|u*@SNSq;?I{tbzmwc#CMlf4(se27?MKNX8I&s# zIZEvQYWQ5Zovo6^^vYfJe+!uQ)fHp_rXhJyI#Dui{WPI-(g6}Y=FzbpkZoI7IW_^6 zx8Rj^e(EiOnpmY}zWOsu-JS5zcTlq90aUuM&TTVjy|d_k!U*u`1@bkVthkE_CKC?!<+yB literal 5122 zcmZ{mcRUq-^#3!mm03pDF4;Tf+9mrEvbT$jT*6gub`%-ch>UB*y|&zIlug;Py4Jmy zD9Iifm(1_w^Z5P#`98jXz5hAy_w$_B`z+qKG&I5mJ)Y!e{sB$_f&LP{PVTOfE|90b zPCgzG9~X#^n}_?Q^%)O~UsH|LTl=AL^5qj~+u|)vK88Sx*%~I}N4Lm&T%#5-TIMCrCipp2pRuge zJmvr0++NzET0W(fXrTKC@#+27UZ?H6Ww5}t3C8-H^iH6;Co=C?SUwyv@7Ji6I>6Z2 zQ>K%Oep!p=ma@yHzW(*d!}1T(M0q_8sYyrhzsg}aJ1Z6#rC<fyV~&a^i)RTx}8atD_2a}D_5xa6LQ`|8JLY?H)hK=hpyprPT?7h*pIa_ z8vJGH+PjkU*x0|auQj-5thj~6+3-FhZ2m8-r7HgV-N!Bo%UjC4(>(I#cZ#1rQm`NS zqv{eSrvlki(8x+Nsc+9hr_;P(=hvWP()8=f<(OwD{4ly`++e2WD&Y}`*`JWhPI zUN9Z@#H*}y%PSCJH*-7?`RkkBk3y{&H@eLg>AK-(DKf;8J6y5N<>mv0t^4D5PRH;M zSs($b4J`l-&!BjtDl^lB$BTVOq6=nLtmp$2T~#F-GO_eB z?_>8BOpNbV6#fj_+rRDPb=F=LQzFz7=3Bke83>9-8s>GsTV?-x)+&jZ$hI3r^Ok-p zTlrr0L`0;qlRl5TuP7Igo!*Vngg|L;UUB(8V8jRKhlh0II#x6VR1rx7tB(z=LLG#w zF=IsKpLaWAlQ;E^Vz|q#(!dQx+x8(&ZD)PBM1ITe?@XiBvzB6-98#*`mm)6xrL7Dq z9$yqvI{QcTI#a^-=Kc{CCPqqHG)NN82xlkjleSvK=qX2b&=a~Wl;>dwMAV`Dzkca8( z(Sh2jX|IP92Zs2Sn+``kz9^M1ZLU5Wmj~-In=dT-DP_i?1e48v12TGXyr+1ZyO@v$ zKLLER`m2EZOBM=sC$@{f6hExDr#mu^5qn-X_@9bIWr9p z`bIyU$B0d0s&fOi765mBxn7<2z6Q5x#W&BEuOn|kOqU}dxM(}S!&NKft5eOM2Y-5Q zb~ZlsUWr66De~I`M0VI4a9UwJHGg)}8W>!x=Z|+^e5o3eLFgFZM-8xzvY-!uRibRY zCH(pWUB3*+=w(aqo1UA0oBR_9|L5z>yFM4T9#QK)fY+YcIiP|r>dStl^~pzw?H%{D*)JC8 z!^Dd+`|Ip7eMLWK9c{MR=OWr13=FImEdP1ld3Cs>&6Dsh2K~hnz1`j~$k$y)d!qo| zc+;su+MH6BLoA-XJC2kbKcmnaliRo>*n-2+t9yFz&M}W9og9RQfuaNoBz50_A2rZ5 zmJ8rz!LFNr|NgEtEm2z!27hbgRyos@Y(}NGQd*Rzq6LlAb#XM27P)rSQkl|KtgKI8 zblGjp!^&8sgsp?Yj863OI{|e*^#cuL=74R1W*ic$eAAfQVCGt^7ubUjn*fmwJmP#p zC%-r4w(>0r;DY^Ny9N-}(^GgUcO!TDM^QZ3ljhEzn`uzW8f8v5S#h|j_|57xn*LLh zbj)Ltsl0cR9=u$oyIyd%2c0F_fi_c=;Yw1;_N{T zI>0>wmqpuvyVSQD`@=hs`dbp$)Ek{0N2gFD9_xhz6*hE+2)wV-)0EF36qV}C0SfD({nhAaF%JPWLyK_<5AL?66_Y^X zQ8xqLI>}7C^tfLdW9K%Sv10Mon1PA1$EgBbLQ_H|OM=Cg4#y7bvqj4Uyv+<}!@|4P z@*M5_tV-C941$ERZ-f~X$Kg7y{(wcQBt=Ec*vlD8%8tBYR<}+JZx;(G2hQjACp@z= zpIkZa`rFw|WAW80<>tfZ(-%ukb3DpLJdSsnTOT zYMRT@0A(Ty^?fokewKlbIpD~uF1OBC%7<8 zOshe1L9utTo^=<#mXE5~f%GYbRW#>w?sYD|QBC#Xy;b-KX#sIyko5nYAM%dgItMo- z#`V>Ek-buaBkAng$%*Q2Ex6%ogMRK2yzaBLmE6R9@IHH0Pz-){sVgy-_b{;X$gyIY z%<*48q2n5T%k))L_5BqM4%1o0rK0yOQF&>sY6b}`kKH|&(ngs}4A44;`)zlRzT{Y- z2wk@qO<#{8zmyMge=#2E-tZdJkyTPFJ)K%A(ff8!533rt;Vn-8TA?@x9eC6eGub>b ze{fJb!I80JTlj4J^F6DUux~i235Vw4g|CFJ08>^g(Cw{qVmvM8pabA?Vxed(ZaWb& z&2~pI5_*UE-w22Od)6pmm78`8ScN~YQxusBbh7A?Zn+FR$`$kP{sG*rf(-<)eKMF& z_we5M+Qvc&_IG%l#NxBzukY;78=Z}0?)9ya_et*_{C?4nro}_D-lqHzS^s@oqtOt5 z(O`d4EgR1At?$ZiBzDh?4s0gYyZ7|?B%nZ#GmD?}hvGu9rzunm$AWn_C#@h?-q-M8 z^Ulq~4&uxsm0sH)bvvJ8%{<{8U2L~R`WXR1X0X{uPlqc;W(5~_wHKNTFv*_B){nr2 zK_Q_5${Sb2*yH!M(V@z#Av~K&g20o~UJwN?V$Cwxd6|;~TpXK>pu$k#n0uLe!`IZI zlr)qyZE(abZK#rCFNjqT$Ozsi!+G!l;WMn~2W|7jb}8N!S#;%Lp&<6gHnUd3;-!z} zaKsODuv+==l%IYJ&!0@&yVjHrtUZ3P;!-iI2@TFjR-pn@V_cSFk?xLgm51?bGNcCq z5eyLwy;s26;p*XA^f;H_y*%_Bn4UN1#w4bTI+U+T7}FEJCbX3`5$_r0!$>`-8wv0Y zP)J#m37>r2%R?{+FYb&^L`^6qb=TUesobdL9{i@zeJBA=T9?K2M3CYUS3&PGFkpIO zlU1&eTo5EJRb-^-&%Y*=AC_|iH&1q-cvt4!Sd|ZUgXGTs1L9pcg3`VxD4@t}$**5p z>F2zlf6i5Rr3>b)Ian%v2=970wFI)1Q<0JM@>Yh%phM|#^G)$<3~)eiB2qX#H$np% z!-3=c-(hk-1G|&QtTD)7yOs@`ppVLecGof`kxrUJ=A!*MU`bOd^6j5um&S0ZU=iz~Az$E#A z=M&4JHcQJXCWQPD28?U19?-l%`QL!;R@)!Ntds{Ja?JW~&OyYY ztEzaHiGdx5(cyKw@CwCNBjL&VLtu7-9U}e{5&%H$VUN=-4Q%hE1tVU@QlyIyWjg3~Ba#bW2pmcp?})uhzZB3!ZNx z1w=)eth6=6jhj05o3+|PLqmLf+3CT>86CThhYJRf;VT>}w0OO)pV7FxVZDF*zm7B( zAU$j0IENESvC&j-AB-o=2Y_zdBiCJp#!)D2R? zK|!G}8D>6#h6S~|9w_5(C}#D)acW9sRl*KxnzigrBZLTZrMxj%jyX2pI3#5{3t0p< zw^RR>kCFr(X+sc5i!kg!Ihvl=v)UBtC)stz#|BLzU!i=W9&9r(f*U<#1DiNW8;6}F zYG`*#M%tAnweZnyD32D{rH;fUOzGv z`zMo{C~ynyUNQCe4-6j<_%HLyaL0?jRxxHk87a%F~&3byA0IOf^{7-e z^d`(Z^|LvKDsfi6M~x`|fJdOs_P4tnI3!Kcz_7qA4;2jcak_rQ`YyObXzxwPB;AkJP$N(GlsWQGMYdleeIQzz?c* zut)@+YF%DOXn16Fh|lGjqB7jAjrI9tr{B9Pl>L;ZI4ksrsDpI- z1Un&roun5bKf3Rr%AsNDhLI4;r*`=b*VTQQDTsFz@kRUJ9q1g?7nq(uM;!ER!V82f zy|H1Ly5gLcIQ*q!MGCk?DDGRrNxP5Y;s27EaD{nw6cglCHVQErHEVPHgPn;?& z#F~cZPsZT@wP>3E3L&4b>7-6KjLy%R?b{VM?b{SJQMiu(ny@rX!a?{{TThW+`-CaQbiR|Yge-U0s|7+ULnH`k-64yGs<-y$5|ef{gR$=!b9t<2GU z*)oQH^E2^}E)^DPLmi8MPd$zNb;$qU`oIKsUtbzvkv*yq&j*YoI#Q9k8wH-3kBRR% zoM<57Fa$R_k~gAdsW-9_s!B$ZO)D~PU_+@IqNX!an}BNJ8xMxZ4S(1M!4`uuG8PfZ zWol{J(Ogi4kLp1|K2OD(_CY(&N}{}h+v8`-f5 zp;px(_M%3fG`%7)p7P-P{ zb_L?fc7oG$*D+E4 reachable self.disk = {} # path -> free GB self.listeners = {} # MIDI source name -> coremidi.Listener + self.remote = {} # (host, command) -> True / False / None (could not ask) def var(self, label, name): return self.vars.get((label, name)) @@ -424,11 +426,22 @@ def check_midi_rx(chk, snap, ctx): return [("rx", Result(OK, info="%s %s" % (name, fmt_age(age))))] +def remote_key(chk): + return "%s\0%s" % (chk["host"], chk["cmd"]) + + +def check_remote(chk, snap, ctx): + """A command run over SSH on another machine, e.g. "is ProPresenter running?".""" + if snap.remote.get(remote_key(chk)) is False: + return [("remote", Result(ERR, chk["msg"], named=True))] + return [("remote", Result(OK))] + + CHECKS = { "conn": check_conn, "hyperdeck": check_hyperdeck, "dante": check_dante, "shure": check_shure, "psm": check_psm, "tascam": check_tascam, "ping": check_ping, "launchd": check_launchd, "process": check_process, "http": check_http, "disk": check_disk, "midi_peer": check_midi_peer, - "mdns": check_mdns, "midi_rx": check_midi_rx, + "mdns": check_mdns, "midi_rx": check_midi_rx, "remote": check_remote, } # Checks whose connection a long press may restart (Dante is shared, so not that one). @@ -574,6 +587,9 @@ class Engine: self.conn_drops = deque() # (time, conn label, group) self.burst = None # {"start", "conns", "cleared"} self.events = [] + self.meta = {} # key -> (tile id, check, sub key) + self.fix_last = {} # fix identity -> last run time + self.executor = execute_fix # raw results for every check key def evaluate(self, snap, now, service_on): @@ -600,6 +616,7 @@ class Engine: if chk.get("tag") and res.level in (WARN, ERR) and not res.named: res.msg, res.named = "%s %s" % (chk["tag"], res.msg), True raw["%s:%d:%s" % (t["id"], i, sub)] = (t["id"], res, chk) + self.meta["%s:%d:%s" % (t["id"], i, sub)] = (t["id"], chk, sub) for sub, res in self.network_results(snap, now): raw["net:%s" % sub] = ("net", res, {"type": "network"}) return raw @@ -698,6 +715,59 @@ class Engine: tile = self.tiles.get(tile_id, {}) return [c["conn"] for c in tile.get("checks", []) if c["type"] in RESTARTABLE and c.get("conn")] + def fix_plan(self, tile_id, snap): + """What a hold on this button would do: fix only what is failing right now. + + [(identity, "fix", fix id, check) or (identity, "restart", connection label, check)] + A check's own "fix" wins; otherwise a failing Companion connection is restarted. + Nothing is done for a healthy button, and a recording HyperDeck is never touched. + """ + tile = self.tiles.get(tile_id, {}) + ids = tile.get("members") or [tile_id] + plan, seen = [], set() + for key in sorted(self.meta): + tid, chk, sub = self.meta[key] + tr = self.trackers.get(key) + if tid not in ids or not tr or not tr.confirmed or tr.result.level not in (WARN, ERR): + continue + fix_id = chk.get("fix") + if isinstance(fix_id, dict): + fix_id = fix_id.get(sub) + if fix_id: + if chk["type"] == "hyperdeck" and hyperdeck_recording(snap, chk["conn"]): + continue + # One run per fix, except a HyperDeck reboot, which is per deck. + per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck_reboot" + ident = ("fix", fix_id, chk.get("ip", "") if per_deck else "") + if ident not in seen: + seen.add(ident) + plan.append((ident, "fix", fix_id, chk)) + elif chk["type"] in RESTARTABLE and sub == "conn" and chk.get("conn"): + ident = ("restart", chk["conn"]) + if ident not in seen: + seen.add(ident) + plan.append((ident, "restart", chk["conn"], chk)) + return plan + + def run_fixes(self, tile_id, snap, comp, now, cooldown=60): + plan = self.fix_plan(tile_id, snap) + if not plan: + log.info("HOLD %s: nothing failing, nothing to fix", tile_id) + for ident, kind, what, chk in plan: + if now - self.fix_last.get(ident, 0) < cooldown: + log.info("HOLD %s: %s %s ran less than %ds ago, skipped", tile_id, kind, what, cooldown) + continue + self.fix_last[ident] = now + if kind == "restart": + c = snap.conns.get(what) + if c and c.get("enabled"): + ok = comp.restart_connection(c["id"]) + log.info("RESTART %s -> %s", what, "ok" if ok else "failed") + else: + log.info("FIX %s: %s", tile_id, self.cfg["fixes"][what].get("label", what)) + threading.Thread(target=self.executor, args=(self.cfg, what, chk), daemon=True).start() + return plan + def display_level(res, acked, service_on): if res.level == OFF: @@ -937,6 +1007,72 @@ def poll_midi(snap, cfg): lis.poll() +# --- Other machines: SSH checks and fixes ------------------------------------ + +def ssh_cmd(cfg, host): + """ssh to a machine in config "hosts", with NA Ingest's status-panel key; one shared connection per host.""" + return ["/usr/bin/ssh", "-i", os.path.expanduser(cfg.get("ssh_key", "~/.ssh/id_status_panel")), + "-o", "BatchMode=yes", "-o", "ConnectTimeout=5", "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=accept-new", "-o", "ControlMaster=auto", + "-o", "ControlPath=/tmp/status-panel-ssh-%C", "-o", "ControlPersist=120", cfg["hosts"][host]["ssh"]] + + +def remote_script(cmds): + """One shell line that runs every check and prints SP=.""" + return "; ".join("(%s) >/dev/null 2>&1; echo SP%d=$?" % (cmd, i) for i, cmd in enumerate(cmds)) + + +def parse_remote(out, n): + codes = dict(re.findall(r"SP(\d+)=(\d+)", out or "")) + return [None if str(i) not in codes else codes[str(i)] == "0" for i in range(n)] + + +def poll_remote(snap, cfg): + by_host = {} + for t in cfg["tiles"]: + for c in t.get("checks", []): + if c["type"] == "remote": + by_host.setdefault(c["host"], []) + if c["cmd"] not in by_host[c["host"]]: + by_host[c["host"]].append(c["cmd"]) + result = dict(snap.remote) + for host, cmds in by_host.items(): + try: + out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds)], capture_output=True, + text=True, timeout=20).stdout + except subprocess.TimeoutExpired: + out = "" + for cmd, ok in zip(cmds, parse_remote(out, len(cmds))): + result["%s\0%s" % (host, cmd)] = ok # None = could not reach it: no verdict + snap.remote = result + + +def execute_fix(cfg, fix_id, chk): + """Run one entry from config "fixes" (in its own thread). Only listed commands ever run.""" + fix = cfg["fixes"][fix_id] + kind, label = fix.get("kind", "ssh"), fix.get("label", fix_id) + try: + if kind in ("ssh", "local"): + argv = ssh_cmd(cfg, fix["host"]) + [fix["run"]] if kind == "ssh" else ["/bin/zsh", "-c", fix["run"]] + r = subprocess.run(argv, capture_output=True, text=True, timeout=fix.get("timeout", 60)) + detail = (r.stdout + r.stderr).strip().replace("\n", " ")[-200:] + log.info("FIX %s -> %s%s", label, "ok" if r.returncode == 0 else "exit %d" % r.returncode, + (": " + detail) if detail else "") + elif kind == "hyperdeck_reboot": + with socket.create_connection((chk["ip"], 9993), timeout=3) as s: + s.settimeout(3) + try: + s.recv(1024) # the deck's greeting + except socket.timeout: + pass + s.sendall(b"reboot\n") + log.info("FIX %s %s -> reboot sent", label, chk["conn"]) + else: + log.warning("FIX %s: unknown kind %r", label, kind) + except Exception as e: + log.warning("FIX %s failed: %s", label, e) + + class Periodic(threading.Thread): def __init__(self, name, interval, fn, stop): super().__init__(name=name, daemon=True) @@ -975,6 +1111,15 @@ def load_config(path): for m in t.get("members", []): if m not in ids: raise ValueError("tile %s: unknown member %s" % (t["id"], m)) + for c in t.get("checks", []): + fixes = c.get("fix") if isinstance(c.get("fix"), dict) else {"": c.get("fix")} + for f in fixes.values(): + if f and f not in cfg.get("fixes", {}): + raise ValueError("tile %s: unknown fix %s" % (t["id"], f)) + if f and cfg["fixes"][f].get("kind", "ssh") == "ssh" and cfg["fixes"][f]["host"] not in cfg.get("hosts", {}): + raise ValueError("fix %s: unknown host" % f) + if c["type"] == "remote" and c["host"] not in cfg.get("hosts", {}): + raise ValueError("tile %s: unknown host %s" % (t["id"], c["host"])) return cfg @@ -1011,6 +1156,7 @@ def start_pollers(cfg, snap, stop): Periodic("system", 10.0, lambda: poll_system(snap, cfg), stop), Periodic("bonjour", 15.0, lambda: poll_mdns(snap), stop), Periodic("http", 5.0, lambda: poll_http(snap, http_checks(cfg)), stop), + Periodic("remote", 10.0, lambda: poll_remote(snap, cfg), stop), ] for t in threads: t.start() @@ -1042,12 +1188,8 @@ def handle_commands(text, engine, comp, snap, now): target = {"on": True, "off": False}.get(arg, not engine.service.on(now)) engine.service.set(target, now) log.info("SERVICE %s (button)", "ON" if target else "OFF") - elif verb == "restart": - for label in engine.restart_targets(arg): - c = snap.conns.get(label) - if c and c.get("enabled"): - ok = comp.restart_connection(c["id"]) - log.info("RESTART %s -> %s", label, "ok" if ok else "failed") + elif verb == "restart": # sent by a 2 second hold + engine.run_fixes(arg, snap, comp, now) else: log.warning("unknown command %r", part) @@ -1123,6 +1265,7 @@ def run_once(cfg): poll_system(snap, cfg) poll_mdns(snap) poll_http(snap, http_checks(cfg)) + poll_remote(snap, cfg) try: poll_midi(snap, cfg) except Exception as e: diff --git a/tests/test_status_panel.py b/tests/test_status_panel.py index 1385c70..418848c 100644 --- a/tests/test_status_panel.py +++ b/tests/test_status_panel.py @@ -428,6 +428,88 @@ class TestBehaviour(unittest.TestCase): self.assertEqual(engine.restart_targets("ingest"), []) +class TestFixes(unittest.TestCase): + """Hold = fix only what that button is failing on. Nothing here runs a real fix.""" + + def setUp(self): + self.snap = healthy_snapshot(SUNDAY) + self.engine = sp.Engine(CONFIG) + + def plan(self, tile): + run_steps(self.engine, self.snap, SUNDAY, 4) + return [(kind, what) for _, kind, what, _ in self.engine.fix_plan(tile, self.snap)] + + def test_green_buttons_do_nothing(self): + for tile in ("apps", "dante", "rec", "hd1", "x32", "ingest", "show"): + self.assertEqual(self.plan(tile), [], tile) + + def test_propresenter_is_restarted_once(self): + self.snap.conns["Propresenter-API"]["category"] = "error" + self.snap.remote["prop\0pgrep -xq ProPresenter"] = False + self.assertEqual(self.plan("apps"), [("fix", "propresenter")]) + self.assertEqual(self.plan("show"), [("fix", "propresenter")]) # the group does the same + + def test_apps_on_other_macs(self): + self.snap.remote["tracks\0pgrep -xq Live"] = False + self.snap.remote["translation\0pgrep -xq 'Church Translate'"] = False + self.assertEqual(self.plan("apps"), [("fix", "ableton"), ("fix", "translate")]) + + def test_unreachable_mac_is_not_a_verdict(self): + self.snap.remote["tracks\0pgrep -xq Live"] = None + self.assertEqual(self.plan("apps"), []) + + def test_dante_reopens_dvs_on_the_right_mac(self): + devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")]) + devices.remove("NA-ProPresenter-Mac") + devices.remove("BOH-WING") # no fix for a console + self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices) + self.assertEqual(self.plan("dante"), [("fix", "dvs_prop")]) + + def test_rec_reboots_only_hung_decks_not_recording_ones(self): + for i in (2, 3, 5): + self.snap.conns["HyperDeck_%d" % i]["category"] = "error" + self.snap.vars[("HyperDeck_3", "status")] = "Record" # last known state: recording + self.assertEqual(self.plan("rec"), [("fix", "hd_reboot"), ("fix", "hd_reboot")]) + decks = [chk["conn"] for _, _, _, chk in self.engine.fix_plan("rec", self.snap)] + self.assertEqual(decks, ["HyperDeck_2", "HyperDeck_5"]) + self.assertEqual(self.plan("hd3"), []) + + def test_not_recording_is_not_fixed_by_a_reboot(self): + for i in (1, 3): + self.snap.vars[("HyperDeck_%d" % i, "status")] = "Record" + self.assertEqual(self.plan("hd5"), []) # NOT REC: a person decides + + def test_connection_without_a_fix_is_restarted(self): + self.snap.conns["x32"]["category"] = "error" + self.assertEqual(self.plan("x32"), [("restart", "x32")]) + + def test_ingest_services(self): + del self.snap.launchd["org.waterschurch.resi-propresenter"] + self.snap.procs["/Applications/ProdDash.app/Contents/MacOS/ProdDash"] = False + self.assertEqual(self.plan("ingest"), [("fix", "bridge"), ("fix", "proddash")]) + + def test_cooldown_and_executor(self): + ran = [] + self.engine.executor = lambda cfg, fix_id, chk: ran.append(fix_id) # record, never run + self.snap.remote["tracks\0pgrep -xq Live"] = False + run_steps(self.engine, self.snap, SUNDAY, 4) + self.engine.run_fixes("apps", self.snap, None, SUNDAY + 10) + self.engine.run_fixes("apps", self.snap, None, SUNDAY + 20) # held again 10 s later: skipped + self.engine.run_fixes("apps", self.snap, None, SUNDAY + 80) + time.sleep(0.2) + self.assertEqual(ran, ["ableton", "ableton"]) + + def test_remote_script_round_trip(self): + script = sp.remote_script(["pgrep -xq Live", "pgrep -xq 'Church Translate'"]) + self.assertIn("(pgrep -xq 'Church Translate') >/dev/null 2>&1; echo SP1=$?", script) + self.assertEqual(sp.parse_remote("SP0=0\nSP1=1\n", 2), [True, False]) + self.assertEqual(sp.parse_remote("", 2), [None, None]) + + def test_every_fix_is_valid(self): + for f in CONFIG["fixes"].values(): + self.assertIn(f.get("kind", "ssh"), ("ssh", "local", "hyperdeck_reboot")) + + class FakeCompanion(BaseHTTPRequestHandler): """Just enough of Companion 5's HTTP API.""" store = {} @@ -509,12 +591,21 @@ class TestAgainstFakeCompanion(unittest.TestCase): def test_commands(self): comp = sp.Companion("127.0.0.1", self.port) - snap = sp.Snapshot() - sp.poll_connections(comp, snap) + store = FakeCompanion.store + snap = healthy_snapshot(SUNDAY) engine = sp.Engine(CONFIG) + engine.executor = lambda cfg, fix_id, chk: self.fail("no real fix may run in a test") now = SUNDAY - sp.handle_commands(";restart hd3;restart led;service off;bogus", engine, comp, snap, now) - self.assertEqual(FakeCompanion.store["restarts"][-3:], ["id-HyperDeck_3", "id-VX1000", "id-UHDJr"]) + store["conns"]["VX1000"]["category"] = "error" + try: + sp.poll_connections(comp, snap) + run_steps(engine, snap, now, 4) + before = len(store["restarts"]) + # hd3 is green, so its hold does nothing; LED is failing on VX1000, so that restarts. + sp.handle_commands(";restart hd3;restart led;service off;bogus", engine, comp, snap, now + 5) + self.assertEqual(store["restarts"][before:], ["id-VX1000"]) + finally: + store["conns"]["VX1000"]["category"] = "good" self.assertFalse(engine.service.on(now)) sp.handle_commands("service toggle", engine, comp, snap, now) self.assertTrue(engine.service.on(now)) @@ -533,10 +624,11 @@ class TestAgainstFakeCompanion(unittest.TestCase): store["conns"]["ATEM"]["category"] = "error" store["custom"]["sp_cmd"] = ";ack atem" data_dir = tempfile.mkdtemp() - originals = sp.poll_mdns, sp.poll_system, sp.poll_midi + originals = sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote sp.poll_mdns = lambda snap, seconds=3: None sp.poll_system = lambda snap, cfg: None sp.poll_midi = lambda snap, cfg: None # CoreMIDI has its own test + sp.poll_remote = lambda snap, cfg: None # never SSH anywhere from a test stop = threading.Event() loop = threading.Thread(target=sp.run, args=(cfg, data_dir), kwargs={"stop": stop}, daemon=True) try: @@ -552,7 +644,7 @@ class TestAgainstFakeCompanion(unittest.TestCase): finally: stop.set() loop.join(timeout=15) - sp.poll_mdns, sp.poll_system, sp.poll_midi = originals + sp.poll_mdns, sp.poll_system, sp.poll_midi, sp.poll_remote = originals store["conns"]["ATEM"]["category"] = "good" self.assertFalse(loop.is_alive()) # it shuts down cleanly self.assertTrue(store["custom"]["sp_hb"].isdigit()) @@ -614,7 +706,10 @@ class TestPage(unittest.TestCase): self.assertIn("ackall", master["action_sets"]["up"][0]["options"]["value"]["value"]) self.assertNotIn("2000", master["action_sets"]) ingest = self.controls["3"]["5"]["steps"]["0"] - self.assertNotIn("2000", ingest["action_sets"]) # nothing to restart there + self.assertIn("2000", ingest["action_sets"]) # restarts its services + self.assertIn("2000", self.controls["0"]["2"]["steps"]["0"]["action_sets"]) # REC group + net = self.controls["3"]["4"]["steps"]["0"] + self.assertNotIn("2000", net["action_sets"]) # nothing to fix there ids = [a["id"] for r in self.controls.values() for c in r.values() for s in c.get("steps", {}).values() for al in s["action_sets"].values() for a in al] self.assertEqual(len(ids), len(set(ids)))