From 14fce2cf9a73ef7ab105367adb2885f2805cb52e Mon Sep 17 00:00:00 2001 From: christian Date: Thu, 15 Oct 2026 12:40:24 -0400 Subject: [PATCH] Audio failover: press FOH Backup by itself when Studio B dies; Companion button presses as fixes A new fix kind "companion_press" presses any Companion button over its HTTP API (page, row, col). A fix marked "failover" runs by itself within seconds of its check failing: inside the "failover" windows (Sunday 06:30-13:30), even while ALARMS is muted, once per fault, never back the other way. The hidden STB AUDIO key (inside AUDIO) watches Studio B's ping every second, Pro Tools over SSH every 2 s and its Dante device; each check's fix is the FOH Backup press. The keys say ON FOH BACKUP until STREAM STB on the Restart page switches back. Settings page: the press builder, the failover tick and undo, the Failover calendar, help entry "A fallback button". Co-Authored-By: Claude Fable 5.1 --- README.md | 28 +++- config.json | 105 +++++++++++- ready-page.companionconfig | Bin 3986 -> 3986 bytes restart-page.companionconfig | Bin 3574 -> 3809 bytes schema.py | 58 ++++++- settings/builders.py | 28 +++- settings/keyguide.py | 3 + settings/ui/app.js | 50 ++++-- status-page.companionconfig | Bin 6225 -> 6225 bytes status_panel.py | 311 +++++++++++++++++++++++++++++++---- tests/test_builders.py | 23 ++- tests/test_schema.py | 12 +- tests/test_status_panel.py | 304 +++++++++++++++++++++++++++++++++- 13 files changed, 855 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 02b1410..0a9d777 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ There are two parts: Row 0 holds the group buttons. Each one shows the first problem in its group, plus "+N" if there are more: - REC: HD1–6 and the Tascam. -- AUDIO: X32, DiGiCo and Dante. +- AUDIO: X32, DiGiCo, Dante and STB AUDIO (the Studio B audio source for the stream; no key of its own). - RF: all six Shure units (AD4Q ×2, ULXD4, PSM1000 ×3). - SHOW: MIDI, APPS, VISTA and SPOTIFY. SPOTIFY has no key of its own any more; it shows inside SHOW, and its force-play key is on the Restart page. @@ -58,6 +58,7 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl | DANTE: PROP MAC / TRACKS / TRANSL / INGEST GONE | starts Dante Virtual Soundcard's engine on that Mac (`remote-access/mac-dvs-helper.sh`) | | DANTE: any device powered by its switch port (today the four Studio B boxes) | power-cycles the port it is on **now**, found by its MAC on the Netgear Dante switches | | DANTE: any other device GONE | restarts Companion's Dante module (hardware: if still gone, it is off or unplugged) | + | STB AUDIO (inside AUDIO): STUDIO B NO PING / PRO TOOLS NOT RUNNING / STB TNX GONE | presses FOH Backup on Companion's Audio page (the stream audio comes from the FOH console). The same press happens by itself on Sundays, see Audio failover. | | INGEST: BRIDGE / CAPTIONS / CAPTION SCREEN / PRODCOM / PRODDASH | restarts that one on NA Ingest | | TASCAM (always, even when it looks fine) | reconnects Companion to the recorder and looks again after 20 s. **HUNG - POWER CYCLE IT**: the recorder answers ping but its control port has hung (only a power-cycle at the rack fixes that; it has no network reboot). **NO NETWORK / CHECK POWER**: it does not answer at all. | | any button whose Companion connection is **disabled** | turns the connection back on (nothing else) | @@ -79,6 +80,24 @@ During that window, with ALARMS on, the panel fixes a problem by itself once it - **Daily log:** everything auto-fix does is written as it happens to `~/Desktop/Auto-Fix Logs/ Auto-Fix.log` on NA Ingest (e.g. `2026-10-11 Sunday Auto-Fix.log`): the problem, what it did, whether it worked, when the fault cleared, when it gave up or held back. Double-click it to open it in Console, which shows new lines live. One file per day; each is deleted a calendar month after its date (the 11 Oct file on 11 Nov). - Press ALARMS to mute the panel for maintenance: that also stops auto-fix. The window and the list are `"auto_fix"` in `config.json`. +## Audio failover (Studio B -> FOH Backup) + +The stream's audio comes in on the Dante device STUDIO-A-Resi-AJA-SDI, channels 1/2. Companion's Audio page (page 4) picks the source: **Studio B** (row 0, col 6: STUDIO-B-RedNet-TNX 63/64 to the AJA, X32 snippet 1) or **FOH Backup** (row 0, col 5: FOH-DiGiCo-DMI-2 63/64 to the AJA, X32 snippet 8). If Studio B crashes on a Sunday, the panel presses FOH Backup for you. + +**STB AUDIO** (inside AUDIO, no key of its own) watches three things: Studio B answers ping (10.1.13.175, pinged every second, **STUDIO B NO PING** after 2 misses), Pro Tools is running on the Studio B Mac Studio (`pgrep -xq 'Pro Tools'` over SSH every 2 s, **PRO TOOLS NOT RUNNING**), and the Dante device STUDIO-B-RedNet-TNX is there (**STB TNX GONE**). Each check's fix is the FOH Backup press (`stream_foh_backup`, a `companion_press` fix: `POST /api/location/4/0/5/press` on Companion's HTTP API). + +- **When:** inside the `"failover"` windows (Sunday 06:30–13:30 today), even while ALARMS is muted. Once per fault: it re-arms once the fault has been clear for 10 s (a check that cannot ask, say an SSH hiccup, does not count as clear). +- **How fast:** about 5–6 s from a crash to the press (2–3 s to notice, 1 s debounce, 2 s `after_seconds`). A vanished Dante device takes as long as Companion's Dante module needs to notice it. +- **Unlike auto-fix:** it does not wait 20 s, does not stop for a network blip, and is not limited to 2 tries per 10 minutes. The one guard: when several Dante devices vanish at once (that is NA Ingest's view of Dante, not Studio B) it holds back, like auto-fix. +- **What the keys show:** AUDIO (and the hidden STB AUDIO) carry an extra line **ON FOH BACKUP** while the fault lasts and for 15 minutes after (`note_seconds`), or until STREAM STB is held. Hold AUDIO right after and it says JUST TRIED. +- **Log and alert:** every press goes to the daily auto-fix log as `FAILOVER:` lines, to the watchdog log, and to `alert_webhook` if set. +- **Switching back is yours.** It never switches back by itself: when Studio B is up again, hold **STREAM STB** on the Restart page for 2 s (the fix's `"undo"`). That presses Studio B on the Audio page and clears ON FOH BACKUP. +- **Never:** a press outside the window, a second press for the same fault, or a press back to Studio B. + +In `config.json`: the `"failover"` block (`windows`, `after_seconds` 2, `note_seconds` 900); `"failover": true` and `"undo"` on the fix (`stream_foh_backup` undoes with `stream_studio_b`); `"interval_seconds"` on ping checks (default 3) and remote checks (default 10). The remote poll runs each computer's checks in its own thread, so a dead Studio B never delays the checks on another Mac. + +**Another fallback button:** any Companion button can be a fix. On the settings page, Keys › Fixes › + Fix › "Press a Companion button" (page, row, column and what the button says), tick failover under more and pick its undo; Settings › Auto-fix › Failover calendar holds the window and `after_seconds`; Keys › + Key / + check… adds the checks that call it (a ping with interval seconds and failures, a program on a Mac, a Dante device) with fix = the press; Keys › Restart page › + Restart key adds the keys that press it by hand. The Help page "How do I add…" has an entry "A fallback button". + ## Restart page (page 20) Tap **RESTART PAGE** on the status page (row 2). Each key restarts one thing **even when the status page says it is fine**; hold it 2 s. A key **flashes red (HOLD TO FIX)** while the thing it restarts is down, so you can see which one to hold. It shows RESTARTING, then DONE (or FAILED / what to do). The same thing will not run twice within a minute. **STATUS PAGE** (bottom left) goes back. @@ -93,6 +112,10 @@ Tap **RESTART PAGE** on the status page (row 2). Each key restarts one thing **e | PRODDASH / CAPTIONS / BRIDGE | that service on NA Ingest | | TASCAM | reconnects Companion to the recorder (see TASCAM above) | | SPOTIFY | plays the Christian EDM 2026 playlist on NA Ingest (opens Spotify first if it is closed) | +| STREAM STB | presses Studio B on Companion's Audio page: the stream audio back on Studio B (how you switch back after a failover, see Audio failover) | +| STREAM FOH | presses FOH Backup on Companion's Audio page: the stream audio from the FOH console. Flashes HOLD TO FIX while Studio B is down; once the failover has pressed it, it shows ON FOH BACKUP (green) instead. | + +A pressing key says PRESSING, then DONE; a press that finds no button there says NO BUTTON THERE. Add or move keys in `"restart_page"` in `config.json`, then `python3 make_page.py` and `python3 push_page.py --which restart`. `push_page.py` refuses any page that holds real buttons unless it already is that page. @@ -178,6 +201,7 @@ The **page number key** on every page follows the ALL button. It's green when ev | TASCAM | connection; the recorder's own Error / Caution. Use the **1.3.1-quiet** Tascam module (see below) so Companion's log isn't flooded.; pinged at the address on its Companion connection. | | X32, ARENA, OBS | Companion connection | | DIGICO (inside AUDIO) | Companion's `DiGiCo_FOH` connection; the relay helper `org.waterschurch.digico-relay` on NA Ingest (`~/digico-bridge/relay.py`: Companion and the DiGiCo tablets share the console's one External Control slot through it), **DIGICO RELAY STOPPED** / **NOT LOADED**, hold AUDIO restarts it; the console itself answers ping at 10.1.60.100, **DIGICO NO PING**. Companion's OSC connection reads "good" even with the console off, so the ping is what catches that. | +| STB AUDIO (inside AUDIO) | the Studio B audio source for the stream: Studio B answers ping (10.1.13.175, every second, **STUDIO B NO PING** after 2 misses); Pro Tools running on the Studio B Mac Studio (over SSH every 2 s, **PRO TOOLS NOT RUNNING**); the Dante device STUDIO-B-RedNet-TNX (**STB TNX GONE**). Hold 2 s presses FOH Backup; on Sundays that press happens by itself (see Audio failover). | | ATEM | Companion connection; the switcher's **audio** (every Fairlight input's on/off/AFV and fader, e.g. Resi on input 16 ON, TRS off) and **camera settings** (iris, white balance, shutter, gain, detail, ND, lift/gamma/gain/offset, contrast, saturation) must match the restore file `atem-reference.xml` (a copy of `4k plus 2026-10-07 14-59-55.xml` from Google Drive Backup/NA/ATEM): **ATEM AUDIO WRONG** / **ATEM CAMERA WRONG**; ATEM Software Control's *Enable MIDI control* must be off: **ATEM SW MIDI ON**. Hold 2 s (and Sunday auto-fix) sends back only what differs; MIDI is fixed by quitting the app, switching it off and reopening it (the switcher is not affected). `atem_watch.js` keeps one connection to the switcher. New restore file: copy it over `atem-reference.xml` and re-run install.sh. | | LED | VX1000 and UHD Jr: Companion connection **and** each one answers ping (the NovaStar module keeps saying "Connected" after a processor is switched off). "UHDJR NO PING" = processor off or unreachable; "UHDJR OFFLINE" = Companion's connection failed; "UHDJR DISABLED" (amber) = connection turned off in Companion. | | DANTE | All 19 Dante devices ("19/19"), named when one drops: "WING GONE", "TRACKS GONE", "PROP MAC GONE". | @@ -200,7 +224,7 @@ With Editing on every screen grows inputs. Changes go into one shared draft on t **Keys** draws both pages as the deck; with Editing on a key is a chip you drag to move (or drop on the tray to hide), + New key / + New restart key add one, Delete this key removes one, the page numbers and "this campus has a Restart page" sit under the tabs. **Overview** carries the jobs people actually do: Change the playlist (paste the link; the key, the play fix and the key guide follow), Add or remove a HyperDeck (tick the decks in use; the HD keys and the REC group follow, "may have no disk" per deck), Add a MIDI link (the Mac, its session and peer from the live list or typed, which key), and under More jobs: Add a computer, Make a PC program startable. Site setup › Companion offers "Add a key for it" on every connection nothing watches. **Computers › Setup** is the one-time checklist per computer with a light and a [how] per line and the set-up scripts generated from that computer's record (`settings/scripts.py` fills the templates in `remote-access/`: the login key, this Mac's addresses, one scheduled task per program listed); Remove this computer drops the fixes and checks that used it and says so. **Site setup › Site** shows the running and staged code with **Install the staged folder** (install.sh runs as a one-shot launchd job with the config left alone), **Export site bundle** (a zip to drop in as `sites//`) and **New campus from a bundle** (this campus's name, addresses and code stay; every copied computer is flagged until touched). -Fixes and the remote checks are built from fields (`settings/builders.py`): a builder (Start a Windows program, Restart a Mac program, Restart a helper, Reopen Ableton with its last set, Restart DVS...) and its fields make the command; the stored command must equal what the fields build or the config is refused. Every Mac value is shell-quoted, every Windows value is regex-checked. A Custom command (Advanced) is the one place a command is typed by hand; Review shows commands in red. +Fixes and the remote checks are built from fields (`settings/builders.py`): a builder (Start a Windows program, Restart a Mac program, Restart a helper, Reopen Ableton with its last set, Restart DVS, Press a Companion button (page, row, column)...) and its fields make the command; the stored command must equal what the fields build or the config is refused. Every Mac value is shell-quoted, every Windows value is regex-checked. A Custom command (Advanced) is the one place a command is typed by hand; Review shows commands in red. A fix built with Press a Companion button can be ticked failover (under more) and given an undo; the Failover calendar and its `after_seconds` sit under Settings › Auto-fix, under the auto-fix window. The page reads what the watchdog writes next to its config (`snapshot.json` every 5 s, `started.json`, `live-now.json`, `fixes-running.json`) instead of asking the gear again. Its own look at the computers (`settings/probes.py`) runs fixed read-only commands over the same login key, fetches whole lists and matches them here, never while a fix runs. diff --git a/config.json b/config.json index cf214ba..f68c079 100644 --- a/config.json +++ b/config.json @@ -809,6 +809,38 @@ "org.waterschurch.digico-relay" ] } + }, + "stream_foh_backup": { + "kind": "companion_press", + "page": 4, + "row": 0, + "col": 5, + "short": "FOH BACKUP", + "label": "press the FOH Backup button in Companion (page 4, row 0, column 5): the stream audio comes from the FOH DiGiCo (DMI-2 63/64) and the X32 takes snippet 8", + "template": "companion_press", + "params": { + "page": 4, + "row": 0, + "col": 5, + "button": "FOH Backup" + }, + "failover": true, + "undo": "stream_studio_b" + }, + "stream_studio_b": { + "kind": "companion_press", + "page": 4, + "row": 0, + "col": 6, + "short": "STUDIO B", + "label": "press the Studio B button in Companion (page 4, row 0, column 6): the stream audio comes from Studio B (RedNet TNX 63/64) and the X32 takes snippet 1", + "template": "companion_press", + "params": { + "page": 4, + "row": 0, + "col": 6, + "button": "Studio B" + } } }, "atem": { @@ -863,6 +895,19 @@ "spotify_play" ] }, + "failover": { + "windows": [ + { + "days": [ + "sun" + ], + "start": "06:30", + "end": "13:30" + } + ], + "after_seconds": 2, + "note_seconds": 900 + }, "restart_page": { "page": 20, "page_name": "20\\nRestart", @@ -1004,6 +1049,24 @@ "watch": [ "conn:tascam" ] + }, + { + "id": "rs_stream_stb", + "label": "STREAM STB", + "pos": [ + 3, + 2 + ], + "fix": "stream_studio_b" + }, + { + "id": "rs_stream_foh", + "label": "STREAM FOH", + "pos": [ + 3, + 3 + ], + "fix": "stream_foh_backup" } ] }, @@ -1350,7 +1413,8 @@ "members": [ "x32", "digico", - "dante" + "dante", + "stb_audio" ] }, { @@ -1577,6 +1641,45 @@ } ] }, + { + "id": "stb_audio", + "label": "STB AUDIO", + "checks": [ + { + "type": "ping", + "host": "10.1.13.175", + "name": "STUDIO B", + "interval_seconds": 1, + "failures": 2, + "debounce_seconds": 1, + "fix": "stream_foh_backup" + }, + { + "type": "remote", + "host": "studiob", + "cmd": "pgrep -xq 'Pro Tools'", + "msg": "PRO TOOLS NOT RUNNING", + "interval_seconds": 2, + "debounce_seconds": 1, + "fix": "stream_foh_backup", + "template": "mac_program", + "params": { + "host": "studiob", + "process": "Pro Tools" + } + }, + { + "type": "dante", + "conn": "audinate-dantecontroller", + "devices": { + "STUDIO-B-RedNet-TNX": "STB TNX" + }, + "fix": { + "STUDIO-B-RedNet-TNX": "stream_foh_backup" + } + } + ] + }, { "id": "dante", "label": "DANTE", diff --git a/ready-page.companionconfig b/ready-page.companionconfig index 66fb25d4ec94337128aa91d463c8cdd8a15badaa..20ab251fc9bebd2a90d4db93528e4ebe823eff95 100644 GIT binary patch delta 16 XcmbOvKS`clzMF&N0^`Mv?A`nTDfk5# delta 16 XcmbOvKS`clzMF%CN9n>w_HKRvC0zt` diff --git a/restart-page.companionconfig b/restart-page.companionconfig index 9a3311dc0d52ef75ae2a88ec6070063774fa294c..7c451322ef948334ccbc940dbde787b36b4e6be1 100644 GIT binary patch delta 3797 zcmYk8c{J2t+{XvSWKU{pBx{7EtXUhAC4^sDvzIkwN*H7Njy19*%95p!eILz;nL%X< z*|JQAv1Z?wu`>@n&mYfuopb-WpL_0ke?IrV&wGn!oR^6&8Vv5H>v_v04WPlZg^KzA zJ$P_Rcuo2W^1-b+n5=LL3uHT!7UaqB{VI%8#(w$`qaZ>lo^zt6VKe6pXhiiM-t4)A zqV8A5($xOFq1oyvDidWI8itcvbuI1?Gkq&$lyG`~;K<{N!hU(EyAwiN;j3us;%b;- zRL$kqNQ3kF%L3GGM$MiCz(}(-llSVMX-ho;QuqcB&bwkeZ^*IK(yIOG&G)_m#;%Q+ zf=kQ1HYTk4@4dUYHANXe9`Lo)V+Qjwitp6bP9?~+sisC`i7QK!M4@MA%SWD3X4@T% z)3Y<^eHLgNWvmFc{ic_?uX>q8uGu2v@|RO;m9NV>LyKt5WB&R)z&UWaYx?++T26DR zZET``wN#M^XNOtKo|9h|X&`d>utZRF!NGU`RafnZF+S$or97K25f&V-aDGAOHmyi` z`8;SE^s6W~IYrK0`!el!x3SS#I9rBeCy;s#XUR;bDlMB>B5UWGHhDFN&rUw|UWo%1 zVuz4(JH%Qn=!WzFRxg1x4OiJ%Ii7xF`w^8#9oNd+iY~}X*17AvR+kM={Ig60ca*TQd# zh{O2#Io*I+;-u-qnE-i1k4r@s#m1$Z%k^mj3`rx?uH$V=Le&l;X&rebZ`@cZjon#k zUp;5H#Lf@u9(vEkmduIXzY34T)n$*e)gf(?N{4F;b=ul%bN_eNl4D$F$pZDtxamuS-u} z@|9Z=29|s6F#fldm!=M*rjkZht^UFtt;GLoL(Q+H89Y!Tk(Rg;459}3jVzZtYN^F% zQPmV6-#A=+rZD?=MahE?g@2~}`lc&dm#qDsjA_8bMB{)Luzp*i6iu>yXzD60Np>bi z&swzk_Ey$uZ1E18qWV4mbzSkd_jYPh?li8Q7WI$_Qn-8yh9MY9)LpJxysKFoAos9z z62Cs^s~&7F^w$BQ3BfA9#Ep#0wz9uiT(Lm{yq+b{a&hKy?S4k3TQ;ubq2`H-%ILaX z$1{whT+R0-JrC9fr>ZaXMcQVT4B%*zMNq?QA15EkOi68$$l#sVvZL=D)P>?O`?|4e zM{>NAotGby*T-+KaJJ$3EwLzdw<*1--bWm{6 z+XMdHDBe92FGbls?|IN6{UIC}KG_Hk65!;F1ot3P#lp4geY&{}H{G`HTM2|Q?8UMl zk|)_VSvK+DLyI6T)jcfK+w0DagYDh1y0`W2@7fM@ls+0@J*$~f3^eOFnnfq*NVyg0 z9jhO!|A~&LYl^1n8o)T?XzcT7S_ln3+yHz#=H$MD9YiG%U~2a#{@_I4PvC#`jn$uB zJJxqnHM4%AZ@T_ipL#t)cl>C5S97_aJ&yUw?*iNYH0kaYu2~Y5;nS$V9_3=vI&lm+ ztsG2(hlH*@^VK3;+z~H=t8s6ajWqaXV>Fn(j7r{-_q(hI|AzM(mwBbv{QTGfeEvS^ z3%Pyq%-QD^y!oE%m}qoH0<4}dE@*eTRVLpp=6T?rlg)}TGLCw*{pf>FUZLs5^IHAT zM7ik&=wS<=EiNYSGaot2iQ<7w+uJZ~`%ll61obiywKO`7E4)+p;7w=yI^R?L=9e@1 z4auHIZ*5xNfBI0L>^fgAlK<$;XFyIU^hsQ$`}K6;H`%{G_5H|5;^^v7uIh26?u0Vz z=K2Lu+|N0wFo4I(UphJ63}yR}{_Szg#MnWLMxC@h^zVvq@Iv(k`cE5v5z|f$z5~oKbXVjI z@2*~O3N^Lm@sE--=LpMsc?DejMb7y%eT|FDS@zF4(L1wTl9Fd0at{o<}I|n^HrNpaxw9%Cf+DwxIVRa4Db|OIBbhFMZFe9vI+1(pg9zt509){p= zlf`W1InjckKr_&Xqt9#3asXfo`?))Kw^OvgqIN4wz*(s#jjY|bYn0tt{*upyF;Q_P z@5bj9DFnqMw@J3l7Gbao+c&x5GI!Xut1&c6OE>(&c#DyCfk?f_VU(mz)Mv#xXUr~bm6pEZF%Gh-U(DnNDwl8b&AGuFi)0`0(b%AlE&iOBrmT2xO zY~a1Yy~nQa3dm(y%I)3jxm;6aKqVwq|N8x*Tn}z_EFzx0XK(olw#-46qtIv6rKl~F zqDyhrtTroLT6FPIoOCYLutfDgnlb3MAjGSY{sh%DRK2&{3i_sVcT2o#YbPlkDJ9*j zum9KgA@_8ttPp0W6YzFgDkNU9e?tY6Uhf8d`|EGUZj**!1T0jSahUsBYD2|V8LIK4 zR0WkCW0s=?H_AdrqYYw_+B~6lAfXm^E8cs8zbk{yo@n_pQMi6YJmV*v}8A)h1SN}XvJF`E5IS#ey}MH5)~%V${8gv59x zM?54R(#oEYu>%|VhKhAS{aPrd?`G*{dCD+*`j~JSj6WMhbeFVs(naAo>8Ww>q?B(X z4_X>m$9d3*Qw>PjVOSlkF8C*MRs=W;AAOwm;zu1*zsJJX(PzI7hlFw@w%1E;sF6IP z`3E`64ysQaQ(#+VS&jb7)M^;eP(EVHCSm&M%09aUXpepmJ^(s)V!FVSZeRN(WmHtpHTiS#>eGUukBp-LK`ncNoJdtprHOloLx9Pi@ofHgk8tE0TuOuis>_Mna2w~U2x>=*oVZ) zHj}S1Zt2gj200Jh{x@^;4epLqxlTL#lLN;d*X)zv0(6zUu!w2(6rGgI@YQRb8@7(C z-}eX@^?=$Cuu7ap|1P5?kDkd|K1hTxz&=A8Yn{m(jZp~kGgq3bXLjKC?! zZ*#(lJmyA!V~Nc4aK*S`7SksYNTN-Yz+L%I6mWN$!T<;l!Ux-tsBgiQWhA@%pUQln zdp%I~JIqv+^0~U3m#lQhu8Yna0<$JDp}m|lv}G)a(aCEetAZpzz3iOmcem$CUE#iv zl}{>QDX^3VTNvbolSziGVo=k!`KI*=HWgel-xsL5td}sst6DH$VkP7F>2Wq;LfIRQ zd%hIL1?U>5la)E*FwbnntuiATzx*sO`(hL8Ph61WE<)r? z%!(MiTDa+#r!FO?)nl<)bmVC|kY51k-{32QR>n AQ2+n{ delta 3546 zcmV<04JGp79rhasABzYGW{u8~2Qq)$1oYILQ>cuh3?RhIc-%$Bqyy2AU=nWbJ^%gW z3JDM(h?z0-e%Tw7?$xBT`t7xz>8AhQqUb(RHBpi2BPyBtZ2YZ{y2SW{?y@aHn-@5x z(`99&$Z=1Uc=Oi5equlOuftq6y{GByqeS*N!li_Ko@529idFZB`g@C_XwiSFBInMI z6HNQHBa^G*`8CryOuuSoS>qPlu^LPz9v1$*Thm7pnG~mJtAN@1|C}r1TjS^ASh>$?t_jNX zI43iAdYauWG~3e^?+y2R%^H8>38$#6Zoi*Nky7Sse!1hyyXA==oQX>6wB9+hEYUbh{y+;dA+dNLoX z>kM<=%S1Nak`H(yUG0&zI!x)ZoT!fi%qA0wwX6;U?5$^Qv{7pVk=K9Q8-Ct45nFA2 z!$ws_Uo=kEGs-EFq6Q6%PdlQl6IEuVBVFu}W=|G}yVM@_X@~1+y3#q)y3O|EF7+QO zc|iS_${rr1^Z68YMD2`A2okHco4Uw7?^0jBP)VcMlcvaBR&Vck%@~@fliknM_NRSS zYw~@q*;SS9{^RzpVJ?4Ejnww;?xKNg4vo28Ve8E80(V%ii*36n3p)BOc58|zE>cfX z)7-jzmDa?je;Tw3_}Zi=_e@z-M+MHcYJHvYU5=HF311XO z4HNp5^YR()D74DbyM2nT)7ykC$v44%b#Bo=Q>L5vAJbiV}5-^odLj z+QxtMHlZ{Nu}~(RqBM4-nI%Mx5{yo!cI=x?mE z?QxU4`}9=D354*EEca}Is=p>Mi*$XRnee_#nzj0+sS(|eeP~`8$X-KM5YnB#l1d~J zUM}VhaXi6^-HO)@-2h0E_UfhQ6Nb63ikB~Xs&jH%IrA6fiP1w=_Exe{n>A*eeO49C z5Xb!PP1}DVebqQcmI*hh=IVsM)U3Dp;xOMu&-gL~gX{1>@ZV|vn*sj?|3wCb3@BdwSF{3x zaQxS;1O5yC3vmPg1^)&Ag+M?cqT;`$%2|K0!Gz?$PS6rV{_8xMuto?6{u>ehb^06p z7yNf2|F!)jV*cwi8WZ&UDi`lApPr>*{qVRbB=zREW?KUKT><^tegO14M!#F*Ql209 zo%@@>8zI5L_%=KvUJiU^MJDfxd)IHkpQx;m0$rD?r)CPa`^|g1kSKB+$M&IXY+L@{tVN@7Z zYfaZ!tZVcie|*}}+oE=4hMG;Y-CgR>KPg=)D+8m`S&i(>6xgNy_(DBcKbHSMo26$> zSOq>+3J>&Jt_0Qmy;M_s>9MMjKpKA|GAQQoWn{#<_yK*RQO z|J_@d4_ZBKq!Syk9tcAhObF|=wVY_~;5Ptq56WBG67blRr?ZnbKYP zS{MkLG`KBfe^LZG8$JH>&$0mYizfgzO3#}3f&df&=tm1cQ}s%|PcH6L7bm51g1tE2 zt6a7afFb~WQvjOzpz&v|a8`dr$obJj&PPQz9_?L2&P;MYb%3bzr$n8f7fxt;BYQqb zU5MYGUk8I~r_s84xjw~&_Nz>2nS+lxqa7T4%o(jkbobw)cOK|*%JM*4DbE8v^k*v} zwI1``uQ1=~O7~qSyMxSrRCC=MwemLrEWh%~G=HJ-(zFIsV6GdK%=Uk+)oDAX0Li`1 z?h$3`M^B{$#dHnLT836AmnnPwi@C^bYGK+wbuIIXd4u$cIy5>I_*a7X?DaO!^E1Bv>Knbs6|mHg!Y*&f8g_YOmp68K zLm(g!5$y7=)vL97u~Dor#d?T+-fqkivrXRaV+mnnH*D{YV0(YJ+u7LLjlJE0d%GPc ziEv}L)1oz=;Dq>Vt&Vxe6@Ynxc_C}SyuiG`ybuToL=>2}a9l1j)k+A$>xL^aBfRco31P!F z5Z(v~uiM!mydZzPfrQs_k_ZW}(_+(UBrcFQ`*M=YDv5gY>9XA@j?V7V+1oN8?-I!C zI0GQ>G~`VK@&fWA*+H@sAIR&T*GN{!yq+omy#T$CHh^A$UVvT*1Oy@q(96^dl}5S1 z6holiNyrj2*gJVJA$<4-;2Qz(opd*lFOY8_@*TTLgwQ&7+-BUc?_q7wxw$*7T{W`F zm)rW+OfA>O+TLZ@ckB)TztiA11Mmy*i{uB%Pkg|y)>U*PR#_eUj+0>u6Mt``R%J?G zzlT7+ZnzRN;Ojn?5H@TB?2Q2Sx}6Qw3)CBkdL1W;5cE1NHoFo#iVNetZM>9I%6(5v z3@(&!11%@Cn-^fbOBk=?2_U>&i_2Zk>HZB@cgI3chx_Ez}2vdSf_}Z!1@7M@rl`lZi6U zv-x!sWgJ|C#1@GyqE@VV^8LIvZ-hdj4z95)V9gWOJVDm5<_T+_uz%(W0s(=D=-`^u zlj^r{XVzHZO8m~Ou^vkZ8@6H1Q#5DRSe=bEPgwJ`=*$|^NuoWo#%eJp%boO>OyV^q zW%9Rq?)JO>J^eKN`hw+7%a5xuodC<7a9j=GE|MH1IWdR3yiqY6-L-1~?*i{a*noF| zcY${y5DArKIVC@}9yp~jS|l|~56I}TW4hIz-&C4>&$fPZ--z`Wz$2Id9k4aB^z zmqdtp$4$lr^FDU3hsUocm7I8a#VAkRcB)xz1M@Dyysj4j^Wu0KU|u9UNOoe3c^_4g zZ-qFN+$|5p3&aai1L6hZ1>%K3Kp>((yvOy@=~9IF5bWAM5`Q7qbsCHb)D;eoFKWjx-?QhXfhb)MQ|E`wIiT(msB8NGknX(mXeO(7 z@ngIoyvTKs>%@uh>a51GA%2V3Ef2yA!V6IY;RWFZ;e|jzAfh0=Ordd72=hz4POuU) z!s|Sh5He%~;f;XsI(-eo3&I;ncx@kvknq|KHdW#v%zwDB-sbhNlI}45i`&6XwkqqT zEHkQs^)6w(wl{$F`dROmQ<83}BH!>#9CSyN4KeF=0N^bF;Q#;t diff --git a/schema.py b/schema.py index 28f3380..2a56dde 100644 --- a/schema.py +++ b/schema.py @@ -128,6 +128,17 @@ field("auto_fix.log_dir", "path", "~/Desktop/Auto-Fix Logs", "One file per day o "Schedules", advanced=True) field("auto_fix.keep_months", "int", 1, "Daily log files are deleted after this many months.", "Schedules", advanced=True, lo=1, hi=12) +field("failover", "dict", {}, "Automatic fallback: when a fix marked failover may act (it acts even while ALARMS is " + "muted).", "Schedules") +field("failover.windows", "list", [{"days": ["sun"], "start": "06:30", "end": "13:30"}], "When a failover may switch " + "by itself.", "Schedules") +field("failover.windows[].days", "days", None, "", "Schedules", required=True) +field("failover.windows[].start", "hhmm", None, "", "Schedules", required=True) +field("failover.windows[].end", "hhmm", None, "", "Schedules", required=True) +field("failover.after_seconds", "int", 2, "The fault must have lasted this long before the failover switches.", + "Schedules", lo=0, hi=60) +field("failover.note_seconds", "int", 900, "After the fault clears, the key keeps saying ON this long " + "(until the fix that switches back runs).", "Schedules", advanced=True, lo=60, hi=86400) # -- network and Dante -- field("network", "dict", {}, "The NET key.", "Network & Dante") @@ -219,6 +230,13 @@ field("fixes.*.playlist", "str", None, "A playlist other than the site's (rare). field("fixes.*.template", "str", None, "The builder this fix was made with.", "Fixes", advanced=True, apply="none") field("fixes.*.params", "dict", None, "The builder's fields.", "Fixes", advanced=True, apply="none") field("fixes.*.action", "enum", None, "What a READY page fix does.", "Fixes", options=list(sp.READY_ACTIONS)) +field("fixes.*.page", "int", None, "The Companion page the button is on.", "Fixes", lo=1) +field("fixes.*.row", "int", None, "The button's row on that page (the top row is 0).", "Fixes", lo=0) +field("fixes.*.col", "int", None, "The button's column (the left column is 0).", "Fixes", lo=0) +field("fixes.*.failover", "bool", False, "An automatic fallback: when a check that names this fix fails, the watchdog " + "runs it by itself within seconds (inside the failover hours, even while ALARMS is muted, once per fault). It " + "never switches back by itself.", "Fixes") +field("fixes.*.undo", "str", None, "The fix that switches back by hand (its key takes the ON ... note off).", "Fixes") # -- tiles (keys) -- field("tiles", "list", [], "The keys on the Status page.", "Keys", required=True) @@ -262,7 +280,8 @@ field("tiles[].checks[].failures", "int", 2, "Pings missed before NO PING.", "Ke field("tiles[].checks[].label", "str", None, "The launchd helper.", "Keys") field("tiles[].checks[].match", "str", None, "The program (pgrep -f pattern).", "Keys") field("tiles[].checks[].url", "url", None, "The web address to fetch.", "Keys") -field("tiles[].checks[].interval_seconds", "int", 30, "Fetch it this often.", "Keys", advanced=True, lo=5, hi=3600) +field("tiles[].checks[].interval_seconds", "int", 30, "Look this often, in seconds: a web address every 30, a command " + "on another computer every 10, a ping every 3 (1 for a fast fallback check).", "Keys", advanced=True, lo=1, hi=3600) field("tiles[].checks[].path", "path", None, "The disk to watch.", "Keys") field("tiles[].checks[].warn_gb", "int", 20, "Amber below this many GB free.", "Keys", advanced=True) field("tiles[].checks[].err_gb", "int", 5, "Red below this many GB free.", "Keys", advanced=True) @@ -371,7 +390,7 @@ CHECKS = { "module": "shure-psm1000", "question": "Is a device answering?", "words": "A PSM transmitter: RF mute."}, "tascam": {"fields": ["conn"], "subs": ["conn", "error", "caution"], "identity": ["conn"], "module": "tascam-da-6400", "question": "Is a recorder OK?", "words": "The Tascam recorder reports no error or caution."}, - "ping": {"fields": ["host", "conn", "name", "msg", "failures", "service_only"], "subs": ["ping"], "identity": ["conn", "host"], + "ping": {"fields": ["host", "conn", "name", "msg", "failures", "interval_seconds", "service_only"], "subs": ["ping"], "identity": ["conn", "host"], "module": None, "question": "Is a device answering?", "words": "An address answers ping."}, "launchd": {"fields": ["label", "name"], "subs": ["svc"], "identity": ["label"], "module": None, "question": "Is a program running?", "words": "A helper (LaunchAgent) on this Mac is loaded and running."}, @@ -391,7 +410,7 @@ CHECKS = { "question": "Is a MIDI link up?", "words": "A network MIDI session is advertised on Bonjour."}, "midi_rx": {"fields": ["source", "name", "max_silence_seconds"], "subs": ["rx"], "identity": ["source"], "module": None, "question": "Is a MIDI link up?", "words": "MIDI is arriving from a source."}, - "remote": {"fields": ["host", "cmd", "msg", "shell", "template", "params"], "subs": ["remote"], "identity": ["host", "cmd"], + "remote": {"fields": ["host", "cmd", "msg", "shell", "interval_seconds", "template", "params"], "subs": ["remote"], "identity": ["host", "cmd"], "module": None, "question": "Is a program running?", "words": "A command on another computer succeeds."}, "artnet": {"fields": ["source", "max_silence_seconds", "msg"], "subs": ["artnet"], "identity": ["source"], "module": None, "question": "Is the lighting console sending?", "words": "Art-Net is arriving from the lighting program."}, @@ -408,7 +427,7 @@ FIX_KIND_FIELDS = { "ssh": ["host", "run", "timeout", "reconnect", "conn"], "local": ["run", "timeout", "reconnect", "conn"], "atem": ["part"], "reconnect": ["conn", "verify_seconds", "hung_note"], "midi_connect": [], "netgear_poe": ["switch", "port", "mac"], "hyperdeck": ["verify_seconds", "off_seconds", "hung_note"], - "spotify": ["playlist"], "ready": ["action"], + "spotify": ["playlist"], "ready": ["action"], "companion_press": ["page", "row", "col"], } EXPECTED_MODULE = {t: spec["module"] for t, spec in CHECKS.items() if spec["module"]} @@ -442,7 +461,8 @@ GLOSSARY = { "STILL OFF IN COMPANION", "DISABLED IN COMPANION", "TURN ON", "RECONNECT", "POWER CYCLE", "FIXES", "HOLD TO FIX", "HOLD", "REC", "PLAYING", "OK", "OFF", "ON", "ALL OK", "WARN", "TRS", "GB", "NOT", "REACHABLE", "RECORDING", "LEFT ALONE", "TO FIX", "TO CLEAR", "NO AUTO", "FIX", "AUTO", "IN", "COMPANION", "POWER", "CYCLE", - "STILL", "OFF IN", "DISABLED IN", "CONNECTING", "CHECK", "NETWORK", "NO", + "STILL", "OFF IN", "DISABLED IN", "CONNECTING", "CHECK", "NETWORK", "NO", "PRESSING", "NO BUTTON THERE", + "ON FOH BACKUP", ], } @@ -478,6 +498,8 @@ def key_record(): kind, parts = "ready", parts[1:] elif p.startswith("auto_fix."): kind, parts = "auto", parts[1:] + elif p.startswith("failover."): + kind, parts = "failover", parts[1:] elif p.startswith("network."): kind, parts = "network", parts[1:] elif p.startswith("variables."): @@ -621,10 +643,24 @@ def validate(cfg, live=None): errors.append("%s: needs the connection to reconnect" % where) if kind == "atem" and fix.get("part") not in ("audio", "camera"): errors.append("%s: part must be audio or camera" % where) + if kind == "companion_press" and not (_whole(fix.get("page"), 1) and _whole(fix.get("row"), 0) and _whole(fix.get("col"), 0)): + errors.append("%s: a Companion button needs its page, row and column" % where) + if fix.get("failover") and fix.get("never_auto"): + errors.append("%s: is a failover (runs by itself) and marked never to run by itself - one or the other" % where) + if fix.get("undo") and fix["undo"] not in fixes: + errors.append("%s: the fix that switches back (undo) is unknown: %s" % (where, fix["undo"])) for k in fix: if k not in FIX_KIND_FIELDS.get(kind, []) and k not in ("kind", "short", "label", "restart_ok", "never_auto", - "template", "params", "timeout"): + "template", "params", "timeout", "failover", "undo"): warnings.append("%s: %s is not a field of a %s fix" % (where, k, kind)) + named = set() + for t in cfg["tiles"] + ((cfg.get("ready_page") or {}).get("keys") or []): + for c in t.get("checks", []): + fx = c.get("fix") + named |= set(fx.values()) if isinstance(fx, dict) else ({fx} if fx else set()) + for fid, fix in fixes.items(): + if fix.get("failover") and fid not in named: + warnings.append("fix %s is a failover but no check runs it" % fid) for alias, h in hosts.items(): if not re.fullmatch(r"[a-z0-9_]+", alias): errors.append("computer %s: the short name may only hold lowercase letters, digits and _" % alias) @@ -648,11 +684,14 @@ def validate(cfg, live=None): errors.append("auto-fix: %s is marked never to run by itself" % f) elif re.search(r"pkill|taskkill|killall", fixes[f].get("run", "")) and not fixes[f].get("restart_ok"): errors.append("auto-fix: %s closes a running program; tick 'may close a running program' or take it out" % f) + elif fixes[f].get("failover"): + warnings.append("auto-fix: %s is a failover - it already runs by itself as one" % f) for f in a.get("while_muted", []): if f not in a.get("fixes", []): errors.append("auto-fix: %s runs while ALARMS is muted but is not an auto-fix" % f) _windows(cfg.get("service_windows"), "alarms window", errors) _windows(a.get("windows"), "auto-fix window", errors) + _windows((cfg.get("failover") or {}).get("windows"), "failover window", errors) _windows((cfg.get("site") or {}).get("live_services"), "live services", errors) rp = cfg.get("restart_page") if rp: @@ -752,6 +791,11 @@ def validate(cfg, live=None): return errors, warnings +def _whole(v, lo): + """A whole number of at least lo (a bool is not one).""" + return isinstance(v, int) and not isinstance(v, bool) and v >= lo + + def _get(cfg, path): """The value at a plain path (no [] or *); None when absent.""" if "[]" in path or ".*." in path: @@ -804,8 +848,10 @@ def phrases(cfg): """The sentences the page and the logs build from the config.""" a = cfg.get("auto_fix") or {} site = cfg.get("site") or {} + fo_windows = (cfg.get("failover") or {}).get("windows", F["failover.windows"]["default"]) return { "auto_fix": "may run by itself %s" % sp.windows_phrase(a.get("windows", [])) if a.get("windows") else "never runs by itself", + "failover": "switches by itself %s" % sp.windows_phrase(fo_windows) if fo_windows else "never switches by itself", "live": ("changes wait while a HyperDeck is recording" + (" or %s" % sp.windows_phrase(site["live_services"]) if site.get("live_services") else "")), "alarms": "Alarms are on every day. The ALARMS key silences them until midnight, then they come back by themselves.", diff --git a/settings/builders.py b/settings/builders.py index 6b75223..4d69a68 100644 --- a/settings/builders.py +++ b/settings/builders.py @@ -30,6 +30,13 @@ def label(value, what): return value +def whole(value, what): + """A whole number (0 counts; a bool, a blank or 1.5 does not).""" + if isinstance(value, bool) or not re.fullmatch(r"-?\d+", str(value).strip()): + raise ValueError("%s must be a whole number, not %r" % (what, value)) + return int(str(value).strip()) + + def home_dir(path): """~/x -> $HOME/x inside double quotes; refuses quotes and $ in the rest.""" path = str(path) @@ -221,6 +228,17 @@ TEMPLATES = { "spotify": {"words": "Play the service playlist in Spotify", "kind": "spotify", "start_only": True, "params": [], "render": None, "suggest": lambda p, cfg: "play the %s playlist in Spotify on %s (opens Spotify if it is closed)" % ((cfg.get("spotify") or {}).get("name") or "service", (cfg.get("site") or {}).get("ingest_name") or "this Mac")}, + # a press: "copy" names the params that are also fields of the fix itself (the watchdog reads those) + "companion_press": { + "words": "Press a Companion button (any page, row and column)", "kind": "companion_press", "start_only": True, + "params": [("page", "Page", "The Companion page the button is on (page 4 = Audio).", "int", None), + ("row", "Row", "Top row is 0.", "int", None), + ("col", "Column", "Left column is 0.", "int", None), + ("button", "What the button says", "Only for the words: FOH Backup, say.", "str", "")], + "copy": ["page", "row", "col"], "render": None, + "suggest": lambda p, cfg: "press the %sbutton in Companion (page %s, row %s, column %s)" + % (("%s " % p["button"]) if p.get("button") else "", p["page"], p["row"], p["col"]), + }, "custom": {"words": "Custom command (Advanced: typed by hand)", "params": [], "render": None, "suggest": lambda p, cfg: ""}, } @@ -259,6 +277,9 @@ def build(cfg, fix): raise ValueError("%s: %s is needed" % (t["words"], words.lower())) out = {k: v for k, v in fix.items() if k not in ("run", "host", "kind")} out["template"], out["params"] = name, params + for n in t.get("copy", []): + words = next(w for a, w, _, _, _ in t["params"] if a == n) + out[n] = params[n] = whole(params.get(n), "%s: %s" % (t["words"], words.lower())) if t.get("kind"): out["kind"] = t["kind"] return out @@ -362,8 +383,11 @@ def mismatches(cfg): except ValueError as e: out.append("fix %s: %s" % (fid, e)) continue - for k in ("run", "kind", "host"): - if (b.get(k) or None) != (fix.get(k) or None): + for k in ("run", "kind", "host") + tuple(TEMPLATES[fix["template"]].get("copy", [])): + bv, fv = b.get(k), fix.get(k) + if k in ("run", "kind", "host"): + bv, fv = bv or None, fv or None # a blank and an absent command are the same + if bv != fv: out.append("fix %s: its %s no longer matches its fields - open it on Fixes and press Save" % (fid, "command" if k == "run" else k)) break for t in cfg.get("tiles") or []: diff --git a/settings/keyguide.py b/settings/keyguide.py index ec84336..33e6981 100644 --- a/settings/keyguide.py +++ b/settings/keyguide.py @@ -185,6 +185,9 @@ def key_guide(cfg): byself = sorted(f for f in own if f in auto) if byself: parts.append("

Auto-fix %s: %s.

" % (e(ph["auto_fix"]), ", ".join(e(cfg["fixes"][f].get("short", f)) for f in byself))) + failover = sorted(f for f in own if f in cfg.get("fixes", {}) and cfg["fixes"][f].get("failover")) + if failover: + parts.append("

Failover %s: %s.

" % (e(ph["failover"]), ", ".join(e(cfg["fixes"][f].get("short", f)) for f in failover))) parts.append("") rp = cfg.get("restart_page") if rp: diff --git a/settings/ui/app.js b/settings/ui/app.js index afaa277..b8771bc 100644 --- a/settings/ui/app.js +++ b/settings/ui/app.js @@ -171,7 +171,7 @@ if (F[path]) return F[path]; const generic = path.replace(/^(tiles|ready_page\.keys)\.\d+\.checks\.\d+\./, "tiles[].checks[].").replace(/^(tiles|ready_page\.keys)\.\d+\./, "tiles[].") .replace(/^restart_page\.keys\.\d+\./, "restart_page.keys[].").replace(/^fixes\.[^.]+\./, "fixes.*.").replace(/^hosts\.[^.]+\./, "hosts.*.") - .replace(/^gear\.[^.]+\./, "gear.*.").replace(/^(auto_fix\.windows|site\.live_services|service_windows)\.\d+\./, "$1[]."); + .replace(/^gear\.[^.]+\./, "gear.*.").replace(/^(auto_fix\.windows|failover\.windows|site\.live_services|service_windows|ready\.due)\.\d+\./, "$1[]."); return F[generic] || null; } const F = (name) => state.schema.fields[name]; @@ -261,7 +261,7 @@ } function canRunNow(fixId) { const f = cfg().fixes[fixId]; if (!f) return false; - if (["reconnect", "hyperdeck", "midi_connect", "spotify"].includes(f.kind)) return true; + if (["reconnect", "hyperdeck", "midi_connect", "spotify", "companion_press"].includes(f.kind)) return true; if (f.kind === "ready") return !f.never_auto; // the READY page's fixes (not the tracks copy) if ((f.kind || "ssh") === "ssh" || f.kind === "local") return !closes(f) && !f.never_auto; return false; @@ -477,7 +477,7 @@ const ctpls = state.builders.checks.concat([{ name: "custom", words: "Custom command (typed by hand)", params: [] }]); const ctName = chk.template || "custom", ct = ctpls.find((x) => x.name === ctName); fields = `
built as
${paramsHtml(ct, chk.params || {}, "ck" + j)}
` - + (ctName === "custom" ? fields : inputHtml(base + "msg", F("tiles[].checks[].msg"), chk.msg) + (chk.cmd ? `
the command
${esc(chk.cmd)}
` : "")); + + (ctName === "custom" ? fields : inputHtml(base + "msg", F("tiles[].checks[].msg"), chk.msg) + inputHtml(base + "interval_seconds", F("tiles[].checks[].interval_seconds"), chk.interval_seconds) + (chk.cmd ? `
the command
${esc(chk.cmd)}
` : "")); } const fixSel = typeof chk.fix === "object" && chk.fix ? `
fix per device${hint('A table: device → fix, "*" for everything else')}
` : `
fix${hint("What holding the key runs for this check")}
`; @@ -802,7 +802,7 @@ const glossary = Object.entries(gl).filter(([g]) => !/while you hold/.test(g)).map(([g, words]) => `
${esc(g)}

${esc(words.join(" · "))}

`).join(""); main.innerHTML = `

Help

${cards}

What each key does Cheat sheet Where things are

-
How do I add… (everything is done here; nothing needs a terminal)
  • Watch a new Companion connection. Edit (top right, the PIN) › Settings › Companion. Find it marked “not watched” › + Key. When the Status page is full it asks which group key to put it in (AUDIO, SHOW…): its problems show on that key, and holding the key reconnects it. Then the badge › Apply.
  • Add a connection to a key that is already there. Keys › click the key › + check… › “A Companion connection is good” › pick the connection from the list › fix: “reconnect its Companion connection” › Apply.
  • Watch that a device is on the network. Keys › the key › + check… › “An address answers ping” › type the address and the name the key should show (it says “NAME NO PING”).
  • Watch a program on another computer, and restart it. Keys › the key › + check… › “A command on another computer succeeds” › built as “A program is running on a Mac” › pick the computer and the program (on a Windows PC: Custom, e.g. tasklist | find /I \"OBS.exe\"). Then fix › a fix that starts it (make one first: next line).
  • Make a new fix (something a hold can do). Keys › Fixes › + Fix › pick how it is built (start a Windows program, restart a Mac program, restart a helper, reopen Ableton…) and fill in the fields; the command is written for you. Then choose it as a check's “fix”, and use “Run the fix” to try it.
  • Let a fix run by itself on Sundays. Settings › Auto-fix › tick it, and paint the hours on the calendar. A fix that closes a running program asks first.
  • Add a line to the weekly checklist. Keys › READY page › + READY key › “Tap to confirm” (a person taps it done; resets Monday), “a checklist line the page checks by itself”, or “something else” (then + check… like any key: a connection, a ping, a program). Drag it where you want it.
  • Change where the READY page looks. Settings › READY page: the Planning Center service type, the ProPresenter and Resolume addresses, the Tracks Mac and its projects folder, which locations get tracks, the lineup TV, the bundle folder, and when gaps turn red.
  • A new computer. Computers › + Computer (its log-in address), then Computers › Setup › that computer › copy its set-up script and run it there once.
  • Made a mistake?. Nothing is live until Apply. After Apply: History › Undo, then Apply again.
+
How do I add… (everything is done here; nothing needs a terminal)
  • Watch a new Companion connection. Edit (top right, the PIN) › Settings › Companion. Find it marked “not watched” › + Key. When the Status page is full it asks which group key to put it in (AUDIO, SHOW…): its problems show on that key, and holding the key reconnects it. Then the badge › Apply.
  • Add a connection to a key that is already there. Keys › click the key › + check… › “A Companion connection is good” › pick the connection from the list › fix: “reconnect its Companion connection” › Apply.
  • Watch that a device is on the network. Keys › the key › + check… › “An address answers ping” › type the address and the name the key should show (it says “NAME NO PING”).
  • Watch a program on another computer, and restart it. Keys › the key › + check… › “A command on another computer succeeds” › built as “A program is running on a Mac” › pick the computer and the program (on a Windows PC: Custom, e.g. tasklist | find /I \"OBS.exe\"). Then fix › a fix that starts it (make one first: next line).
  • Make a new fix (something a hold can do). Keys › Fixes › + Fix › pick how it is built (start a Windows program, restart a Mac program, restart a helper, reopen Ableton…) and fill in the fields; the command is written for you. Then choose it as a check's “fix”, and use “Run the fix” to try it.
  • Let a fix run by itself on Sundays. Settings › Auto-fix › tick it, and paint the hours on the calendar. A fix that closes a running program asks first.
  • A fallback button (the stream audio switches to FOH Backup by itself when Studio B dies). Keys › Fixes › + Fix › built as “Press a Companion button” › page, row, column of the button in Companion (page 4, row 0, column 5 is FOH Backup) › under more, tick failover (and pick the fix that switches back as undo). Then Keys › + Key (or an existing key) › + check… for what must be alive (an address answers ping, a program is running on a Mac, a Dante device) › fix: that press. Settings › Auto-fix › Failover: paint the hours. A Restart page key with the press switches by hand. The watchdog presses once per fault, within about 5 s, even while ALARMS is muted, and never switches back by itself.
  • Add a line to the weekly checklist. Keys › READY page › + READY key › “Tap to confirm” (a person taps it done; resets Monday), “a checklist line the page checks by itself”, or “something else” (then + check… like any key: a connection, a ping, a program). Drag it where you want it.
  • Change where the READY page looks. Settings › READY page: the Planning Center service type, the ProPresenter and Resolume addresses, the Tracks Mac and its projects folder, which locations get tracks, the lineup TV, the bundle folder, and when gaps turn red.
  • A new computer. Computers › + Computer (its log-in address), then Computers › Setup › that computer › copy its set-up script and run it there once.
  • Made a mistake?. Nothing is live until Apply. After Apply: History › Undo, then Apply again.
Try it (ten minutes with Editing on; nothing is live until APPLY)
  1. Keys › HD2: untick "may have no disk"; the badge counts it; Apply reads it back. Throw away.
  2. Keys: drag LED onto the tray and back. + Key "HD7", add a check (Is a recorder OK?), delete it.
    3. @@ -821,7 +821,7 @@ // ---- Settings: one page, every section ------------------------------------------ const HAND_MADE = ["tiles", "fixes", "hosts", "gear", "auto_fix.windows", "service_windows", "site.live_services", "auto_fix.fixes", "atem.reference", "spotify.playlist", "spotify.name", "restart_page.keys", "site.editing_code", - "ready_page.keys", "ready_page.back_pos", "ready.prop_lists"]; + "ready_page.keys", "ready_page.back_pos", "ready.prop_lists", "failover.windows", "failover.after_seconds", "failover.note_seconds"]; const SECTIONS = [ ["playlist", "Playlist", "The Spotify playlist the SPOTIFY key and the play fix use"], ["autofix", "Auto-fix", "When the panel may fix things by itself, and which fixes"], @@ -897,7 +897,25 @@ function autofixHtml(c) { const fixes = Object.entries(c.fixes).map(([id, f]) => { const cl = closes(f); const on = (c.auto_fix.fixes || []).includes(id); return ``; }).join(" "); return `
    ${calendarHtml("auto_fix.windows", c.auto_fix.windows)}
    -

    Fixes it may run${hint("Ticked fixes run by themselves inside the painted hours. ⚠ = it closes a running program to do its job; ticking it allows that without a hold.")}

    ${fixes}
    ` + fieldsHtml("Schedules"); +

    Fixes it may run${hint("Ticked fixes run by themselves inside the painted hours. ⚠ = it closes a running program to do its job; ticking it allows that without a hold.")}

    ${fixes}
    ` + failoverHtml(c) + fieldsHtml("Schedules"); + } + // Which keys name each fix (on a check, as hold-also-when-green, or on a restart key). + function fixUsers(c) { + const used = {}; + const use = (f, who) => { if (f) (used[f] = used[f] || new Set()).add(who); }; + for (const t of c.tiles.concat((c.ready_page || { keys: [] }).keys)) { for (const ch of t.checks || []) for (const f of (typeof ch.fix === "object" && ch.fix ? Object.values(ch.fix) : [ch.fix])) use(f, t.label); use(t.hold_always, t.label); } + for (const k of (c.restart_page || { keys: [] }).keys) use(k.fix, "restart key " + k.label); + return used; + } + function failoverHtml(c) { + const fo = c.failover || {}; + const windows = "windows" in fo ? fo.windows : (F("failover.windows").default || []); // absent = the watchdog's default hours + const used = fixUsers(c); + const list = Object.entries(c.fixes).filter(([, f]) => f.failover).map(([id, f]) => { const keys = [...(used[id] || [])].filter((k) => !k.startsWith("restart key ")); return `${esc(f.short || id)} - ${keys.length ? "used by " + esc(keys.join(", ")) : "no check runs it"}`; }).join(" "); + return `

    Failover${hint("A fix marked failover runs by itself within seconds when its check fails, inside these hours, even while ALARMS is muted, once per fault; it never switches back by itself.")}

    + ${calendarHtml("failover.windows", windows)} + ${inputHtml("failover.after_seconds", F("failover.after_seconds"), fo.after_seconds)}${inputHtml("failover.note_seconds", F("failover.note_seconds"), fo.note_seconds)} +

    Fixes marked failover: ${list || "none yet (Keys › Fixes › a fix › more › failover)"}

    `; } // --- the weekly calendar: half-hour cells; click or drag to paint --- const DAYS = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]; @@ -1234,13 +1252,11 @@ } // ---- Fixes (a tab of Keys) ---------------------------------------------------------- + let tplMsg = ""; // a builder's complaint, shown once after the next paint async function fixes(rest) { const c = cfg(), auto = new Set(c.auto_fix.fixes || []); - const used = {}; - const use = (f, who) => { if (f) (used[f] = used[f] || new Set()).add(who); }; - for (const t of c.tiles) { for (const ch of t.checks || []) for (const f of (typeof ch.fix === "object" && ch.fix ? Object.values(ch.fix) : [ch.fix])) use(f, t.label); use(t.hold_always, t.label); } - for (const k of (c.restart_page || { keys: [] }).keys) use(k.fix, "restart key " + k.label); - const rows = Object.entries(c.fixes).map(([id, f]) => `${esc(f.short || id)}${esc(f.label || "")}${esc(f.kind || "ssh")}${f.host ? " on " + esc((c.hosts[f.host] || {}).name || f.host) : f.kind === "local" || f.kind === "spotify" ? " on this Mac" : ""}${[...(used[id] || [])].join(", ")}${auto.has(id) ? "by itself" : f.never_auto ? "never by itself" : "hold only"}`).join(""); + const used = fixUsers(c); + const rows = Object.entries(c.fixes).map(([id, f]) => `${esc(f.short || id)}${esc(f.label || "")}${esc(f.kind || "ssh")}${f.host ? " on " + esc((c.hosts[f.host] || {}).name || f.host) : f.kind === "local" || f.kind === "spotify" ? " on this Mac" : ""}${[...(used[id] || [])].join(", ")}${f.failover ? "by itself (failover)" : auto.has(id) ? "by itself" : f.never_auto ? "never by itself" : "hold only"}`).join(""); const id = rest[0], f = c.fixes[id]; let detail = ""; if (f) { @@ -1251,15 +1267,16 @@ let inputs = ""; if (editing()) { inputs += `
    built as
    `; - inputs += `
    ${paramsHtml(tpl, f.params || {}, "fx")}

    `; + inputs += `
    ${paramsHtml(tpl, f.params || {}, "fx")}

    ${esc(tplMsg)}

    `; tplMsg = ""; inputs += inputHtml(`fixes.${id}.label`, F("fixes.*.label"), f.label) + inputHtml(`fixes.${id}.short`, F("fixes.*.short"), f.short); - if (kindOnly) inputs += (state.schema.fix_kind_fields[tpl.kind] || []).map((fl) => inputHtml(`fixes.${id}.${fl}`, F("fixes.*." + fl), f[fl])).join(""); + if (kindOnly) inputs += (state.schema.fix_kind_fields[tpl.kind] || []).filter((fl) => !tpl.params.some((p) => p.name === fl)).map((fl) => inputHtml(`fixes.${id}.${fl}`, F("fixes.*." + fl), f[fl])).join(""); if (tplName === "custom") { inputs += `
    runs on
    `; if (kind === "ssh") inputs += `
    computer
    `; inputs += `
    command${hint("Typed by hand: zsh on a Mac, cmd.exe on a PC. Shown as a command at Apply; never run before Apply.")}
    `; } else if (f.run) inputs += `
    the command it makes${f.kind === "local" ? "" : " (on " + esc((c.hosts[f.host] || {}).name || f.host) + ")"}
    ${esc(f.run)}
    `; - inputs += `
    more${["reconnect", "conn", "restart_ok", "never_auto", "timeout"].map((fl) => inputHtml(`fixes.${id}.${fl}`, F("fixes.*." + fl), f[fl])).join("")}
    `; + const undoSel = `
    undo${hint(F("fixes.*.undo").help)}
    `; + inputs += `
    more${["reconnect", "conn", "restart_ok", "never_auto", "timeout", "failover"].map((fl) => inputHtml(`fixes.${id}.${fl}`, F("fixes.*." + fl), f[fl])).join("")}${undoSel}
    `; } else inputs = `

    Built as: ${esc(tpl.words)}

    ${f.run ? `
    the command
    ${esc(f.run)}
    ` : ""}`; detail = `

    ${esc(f.short || id)} ${esc(f.label || "")}

    ${inputs}${keys.length ? `

    Used by ${esc(keys.join(", "))}.

    ` : ""}
    everything stored
    ${Object.entries(f).map(([k, v]) => `
    ${esc(k)}
    ${esc(typeof v === "object" ? JSON.stringify(v) : v)}
    `).join("")}
    id
    ${esc(id)}
    ${editing() && !keys.length ? `` : ""}
    `; } @@ -1273,12 +1290,11 @@ const params = readParams(main.querySelector("#tplform"), tpl); if (tplName === "custom") { fx.template = "custom"; delete fx.params; saveDraft(); route(); return; } const r = await post("/api/build", { fix: Object.assign({}, fx, { template: tplName, params }) }); - const msg = main.querySelector("#tplmsg"); - if (!r.ok) { msg.textContent = r.error; fx.template = tplName; fx.params = params; saveDraft(); return; } + // Not buildable yet (a field is still blank): keep the choice, repaint so the builder's own fields show, say why. + if (!r.ok) { tplMsg = r.error; fx.template = tplName; fx.params = params; saveDraft(); route(); return; } const keep = { label: fx.label, short: fx.short }; Object.assign(fx, r.fix); for (const k of ["run", "host"]) if (!(k in r.fix)) delete fx[k]; if (!keep.label && r.label) fx.label = r.label; - msg.textContent = ""; saveDraft(); route(); }; const ts = main.querySelector("#tplsel"); if (ts) { ts.onchange = rebuild; for (const el of main.querySelectorAll("#tplform [data-param]")) el.onchange = rebuild; } diff --git a/status-page.companionconfig b/status-page.companionconfig index 79b5482589f01e5c9a5b7e2421d0d552e72148b1..c24f6299a1a57238d871ae15d5de18437d953041 100644 GIT binary patch delta 16 Xcmca;aM6HWzMF&N0^`Mv>|PQ8GQ9 {"tile", "label", "note", "at", "faults", "cleared"} + self.failover_held = {} # fix id -> when the Dante-wide hold-back was last logged + self.notify = post_webhook # the alert message (replaced in tests) self.autolog = AutoLog(self.auto.get("log_dir", "~/Desktop/Auto-Fix Logs"), self.site.get("ingest_name", "NA Ingest"), windows_phrase(self.auto.get("windows", [])), - self.auto.get("keep_months", 1)) if self.auto else None + self.auto.get("keep_months", 1)) if (self.auto or fo) else None if self.autolog: self.autolog.cleanup(time.time()) self.auto_watch = {} # "BUTTON FAULT" auto-fix went after -> when (to log when it clears) @@ -936,6 +974,7 @@ class Engine: self.state.set("fix_last", {"\0".join(k): v for k, v in self.fix_last.items()}) self.state.set("auto", {"tries": {"\0".join(k): v for k, v in self.auto_tries.items()}, "seen": {"\0".join(k): v for k, v in self.auto_seen.items()}}) + self.state.set("failover", self.failed) def live_now(self, now): """(live?, why): changes to the panel wait while this is true. A recording HyperDeck is @@ -1337,6 +1376,8 @@ class Engine: waited = True continue self.fix_last[ident] = now + if kind == "fix": + self.failover_undone(what) # switching back by hand ends the failover note self.remember() if kind == "enable": c = snap.conns.get(what) @@ -1512,6 +1553,124 @@ class Engine: self.remember() return done + def failover_fix(self, key): + """The failover fix a tracker key's check names (per sub-check, "*" for the rest), or None.""" + tile_id, chk, sub = self.meta[key] + fix_id = chk.get("fix") + if isinstance(fix_id, dict): + fix_id = fix_id.get(sub, fix_id.get("*")) + fix = self.cfg["fixes"].get(fix_id) if fix_id else None + return fix_id if fix and fix.get("failover") else None + + def dante_gone(self, conn): + """How many devices on this Dante connection are gone right now, counted over every dante + check on every key (the same device may be watched by two keys).""" + return len({sub for k, (tid, chk, sub) in self.meta.items() + if chk["type"] == "dante" and chk["conn"] == conn and sub != "conn" + and self.trackers.get(k) and self.trackers[k].confirmed and self.trackers[k].result.level == ERR}) + + def failover_note_for(self, fix_id): + """The note ("ON FOH BACKUP") of a failover that fired for this fix and is not switched back yet.""" + rec = self.failed.get(fix_id) + return rec.get("note") if rec else None + + def failover_undone(self, fix_id): + """A fix that switches back by hand (the "undo" of a failover fix) was started: drop the + failover's note. If the fault is still there the failover fires again after_seconds later, + which is right: the stream must never stay silent.""" + for fid in [f for f, r in self.failed.items() if self.cfg["fixes"].get(f, {}).get("undo") == fix_id]: + log.info("FAILOVER %s: switched back by hand (%s)", fid, fix_id) + del self.failed[fid] + + def failover(self, snap, comp, now): + """Run a "failover" fix by itself the moment its check fails for after_seconds (Studio B dies: + press FOH Backup, so the stream is not silent until someone notices). Once per fault episode, + only inside the failover windows, even while ALARMS is muted and even during a network blip + (a dead network is no reason to leave the stream silent). Never switches back by itself. + + Timing, from a crash to the press: the Studio B Mac dies -> two missed 1-second pings (2-3 s) + + 1 s debounce + 2 s after_seconds = about 5-6 s. Pro Tools crashes -> the 2-second ssh check + (2-3 s) + 1 s + 2 s = about 5-6 s. Its Dante device vanishing is seen as fast as Companion's + Dante module notices, then 3 s debounce + 2 s. Returns the fix ids it fired.""" + if self.stopping: + return [] + faults = {} # failover fix id -> [(tracker key, tracker)] in a confirmed fault + bad = set() # ...and those with a check that is bad at all (still debouncing) + for key in self.meta: + fix_id = self.failover_fix(key) + tr = self.trackers.get(key) + if not fix_id or not tr: + continue + if tr.confirmed and tr.result.level == ERR: + faults.setdefault(fix_id, []).append((key, tr)) + # A check with no verdict (ssh could not ask, no ping result yet) is not a clear: a blip in + # the asking must not re-arm the failover and press a second time. + if tr.confirmed or tr.bad_since is not None or no_verdict(self.meta[key][1], snap, now): + bad.add(fix_id) + changed = False + for fix_id, rec in list(self.failed.items()): + if fix_id in bad: + # Still there, or back inside its debounce (the first seconds after a restart): not clear. + continue + if rec.get("cleared") is None: # the fault went: armed again, the note stays + rec["cleared"] = now + log.info("FAILOVER %s: fault clear - armed again (the stream stays %s until switched back by hand)", + fix_id, rec.get("note", "").lower()) + changed = True + elif now - rec["cleared"] > self.failover_note: + del self.failed[fix_id] + changed = True + fired = [] + for fix_id, items in faults.items(): + rec = self.failed.get(fix_id) + if rec and (rec.get("cleared") is None or now - rec["cleared"] < REARM_SECONDS): + continue # fired for this fault already (or it only just cleared) + if not in_windows(self.failover_windows, now): + continue + lasting = [(k, tr) for k, tr in items if now - (tr.since or now) >= self.failover_after] + triggers = [] + for k, tr in lasting: + chk = self.meta[k][1] + if chk["type"] == "dante" and self.dante_gone(chk["conn"]) > 1: + # Several Dante devices gone at once is NA Ingest's view of Dante, not that device. + if self.failover_held.get(fix_id, 0) < now - 600: + log.info("FAILOVER %s: %d Dante devices gone at once - that is NA Ingest's view of Dante, " + "not %s; not switching", fix_id, self.dante_gone(chk["conn"]), self.meta[k][2]) + if self.autolog: + self.autolog.write(now, self.tiles[self.meta[k][0]]["label"], + "failover held back: %d Dante devices gone at once (NA Ingest's view of " + "Dante, not %s)" % (self.dante_gone(chk["conn"]), self.meta[k][2])) + self.failover_held[fix_id] = now + continue + triggers.append((k, tr)) + if not triggers: + continue + key, tr = triggers[0] + tile_id, chk, sub = self.meta[key] + fix, label = self.cfg["fixes"][fix_id], self.tiles[tile_id]["label"] + msgs = list(dict.fromkeys(t.result.msg for _, t in items)) + what = fix.get("label", fix_id) + self.failed[fix_id] = {"tile": tile_id, "label": label, "note": "ON " + fix.get("short", fix_id.upper()), + "at": now, "faults": msgs, "cleared": None} + self.fix_last[("fix", fix_id, "")] = now # a hold or Restart key right after says JUST TRIED / JUST DONE + log.info("FAILOVER %s: %s - %s", tile_id, ", ".join(msgs), what) + if self.autolog: + self.autolog.write(now, label, "FAILOVER: %s - %s" % (", ".join(msgs), what)) + back = next((k["label"] for k in self.restart_keys.values() if k["fix"] == fix.get("undo")), None) + text = "%s: %s %s - switched over by itself: %s. Switch back by hand%s once it is fixed." % ( + self.site.get("alert_prefix") or self.site.get("ingest_name") or "Status panel", label, ", ".join(msgs), + what, (" (hold %s on the Restart page)" % back) if back else "") + threading.Thread(target=self.notify, args=(self.cfg.get("alert_webhook"), text), daemon=True).start() + started = time.time() + self.run_plan(tile_id, [(("failover", fix_id), "fix", fix_id, chk)], snap, comp, now, cooldown=0, auto=True, + on_done=(lambda outcome, label=label, started=started: self.autolog.write( + now + (time.time() - started), label, "-> " + outcome)) if self.autolog else None) + fired.append(fix_id) + changed = True + if changed: + self.remember() + return fired + def describe(self, kind, what): """A plan item in words, for the auto-fix log.""" if kind == "restart": @@ -1548,10 +1707,11 @@ class Engine: self.note(key_id, "JUST DONE\n" + self.wait_text, "idle", now, 4) return None self.fix_last[ident] = now + self.failover_undone(fix_id) self.remember() fix = self.cfg["fixes"][fix_id] log.info("%s %s: %s", verb, key_id, fix.get("label", fix_id)) - self.note(key_id, "RESTARTING", "fixing", now, 180) + self.note(key_id, "PRESSING" if fix.get("kind") == "companion_press" else "RESTARTING", "fixing", now, 180) def work(): r = self.executor(self.cfg, fix_id, chk or {}) @@ -1675,6 +1835,13 @@ def colored(text, level): return {"x": text, "b": b, "f": f, "b2": b2, "f2": f2, "s": fit_size(text)} +def failover_line(engine, ids): + """An extra last line ("ON FOH BACKUP") for a key one of whose tiles had a failover fire, until + it is switched back by hand (or the fault has been gone a while).""" + notes = list(dict.fromkeys(r["note"] for r in engine.failed.values() if r.get("tile") in ids and r.get("note"))) + return ("\n" + ", ".join(notes)) if notes else "" + + def render(engine, service_on): out = {} down_fixes, down_conns = engine.failing_targets() @@ -1682,8 +1849,11 @@ def render(engine, service_on): note = engine.notes.get(kid) watch = [key["fix"]] + key.get("watch", []) down = any(w[5:] in down_conns if w.startswith("conn:") else w in down_fixes for w in watch) + pressed = engine.failover_note_for(key["fix"]) if note and engine.now < note["until"]: out[kid] = colored(key["label"] + "\n" + note["text"], note["level"]) + elif pressed: # the failover pressed it already: say so, no flashing + out[kid] = colored(key["label"] + "\n" + pressed, "ok") elif down: # flashes: this is the one to hold out[kid] = colored(key["label"] + "\nHOLD TO FIX", "err") else: @@ -1736,7 +1906,7 @@ def render(engine, service_on): text = tile["label"] + "\n" + first + ("\n+%d" % (len(faults) - 1) if len(faults) > 1 else "") else: text = tile["label"] + "\n" + ("OK" if level == "ok" else "OFF") - out[tid] = colored(text, level) + out[tid] = colored(text + failover_line(engine, tile["members"]), level) continue items = engine.items.get(tid, []) level, faults = worst(items) @@ -1748,7 +1918,7 @@ def render(engine, service_on): text = tile["label"] + "\n" + (infos[0] if infos else "OFF") else: text = tile["label"] + "\n" + (infos[0] if infos else "OK") - out[tid] = colored(text, level) + out[tid] = colored(text + failover_line(engine, [tid]), level) return out @@ -1779,6 +1949,9 @@ def ping_once(host): return False +_ping_lock = threading.Lock() # the 3-second and the 1-second ping pollers both update snap.ping + + def poll_pings(snap, hosts, interface): results = {} @@ -1789,10 +1962,11 @@ def poll_pings(snap, hosts, interface): t.start() for t in threads: t.join() - pings = dict(snap.ping) - for h, ok in results.items(): - pings[h] = 0 if ok else pings.get(h, 0) + 1 - snap.ping = pings + with _ping_lock: + pings = dict(snap.ping) + for h, ok in results.items(): + pings[h] = 0 if ok else pings.get(h, 0) + 1 + snap.ping = pings if interface: try: out = subprocess.run(["/sbin/ifconfig", interface], capture_output=True, text=True, timeout=3).stdout @@ -2149,24 +2323,61 @@ def parse_remote(out, n): return [None if str(i) not in codes else codes[str(i)] == "0" for i in range(n)] -def poll_remote(snap, cfg): - by_host = {} +def remote_groups(cfg): + """{(host, interval seconds): [commands]}: each machine's remote checks, grouped by how often they + ask to run ("interval_seconds"; Pro Tools on Studio B every 2 s, the rest every REMOTE_INTERVAL).""" + groups = {} for t in all_tiles(cfg): for c in t.get("checks", []): if c["type"] == "remote": - by_host.setdefault(c["host"], []) - if c["cmd"] not in by_host[c["host"]]: - by_host[c["host"]].append(c["cmd"]) - result = dict(snap.remote) - for host, cmds in by_host.items(): - try: - out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds, host_shell(cfg, host))], capture_output=True, - text=True, timeout=20).stdout - except subprocess.TimeoutExpired: - out = "" + key = (c["host"], c.get("interval_seconds", REMOTE_INTERVAL)) + groups.setdefault(key, []) + if c["cmd"] not in groups[key]: + groups[key].append(c["cmd"]) + return groups + + +_remote_lock = threading.Lock() +_remote_last = {} # (host, interval) -> when that group last ran +_remote_running = {} # (host, interval) -> the thread running it now + + +def run_remote_group(snap, cfg, host, cmds): + """One ssh for one group of commands; its verdicts go into snap.remote.""" + try: + out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds, host_shell(cfg, host))], capture_output=True, + text=True, timeout=20).stdout + except (OSError, subprocess.SubprocessError): + out = "" + with _remote_lock: + result = dict(snap.remote) for cmd, ok in zip(cmds, parse_remote(out, len(cmds))): result["%s\0%s" % (host, cmd)] = ok # None = could not reach it: no verdict - snap.remote = result + snap.remote = result + + +def poll_remote(snap, cfg, now=None, last=None): + """Run the remote check groups that are due, each in its own thread (one ssh process per group, + sharing the host's ControlMaster), so a dead machine (ssh gives up after 5 s) never delays the + 2-second group on another. A group still running is not started again. With no `last` every + group runs and the call waits for them (run_once, tests). Returns the threads it started.""" + now = time.time() if now is None else now + threads = [] + for key, cmds in remote_groups(cfg).items(): + if last is not None: + running = _remote_running.get(key) + if now - last.get(key, 0) < key[1] or (running and running.is_alive()): + continue + last[key] = now + t = threading.Thread(target=run_remote_group, args=(snap, cfg, key[0], cmds), name="remote-" + key[0], daemon=True) + t.start() + threads.append(t) + if last is not None: + _remote_running[key] = t + if last is None: + for t in threads: + t.join() + return threads # --- The self-updating Dante map: name -> IP -> MAC -> switch port -> PoE -------- @@ -2325,6 +2536,18 @@ def execute_fix(cfg, fix_id, chk): # Turn Companion's connection to the deck off and on: after a deck restarts, that is what # brings it back green (a plain reconnect does not, 2026-10-11). The deck is never rebooted. return reconnect_and_verify(cfg, fix, label, conn=chk["conn"], host=chk.get("ip"), toggle=True) + if kind == "companion_press": + # Press a button on one of Companion's own pages (the Audio page's FOH Backup), as a finger would. + status = make_companion(cfg).press(fix["page"], fix["row"], fix["col"]) + where = "page %s row %s col %s" % (fix["page"], fix["row"], fix["col"]) + if status == 200: + log.info("FIX %s -> pressed Companion %s", label, where) + return True + if status == 204: + log.info("FIX %s -> no button at %s (HTTP 204)", label, where) + return "NO BUTTON\nTHERE" + log.info("FIX %s -> HTTP %d", label, status) + return False log.warning("FIX %s: unknown kind %r", label, kind) except Exception as e: log.warning("FIX %s failed: %s", label, e) @@ -2653,8 +2876,18 @@ def load_config(path): raise ValueError("fix %s: nothing to run" % fid) if kind == "ready" and fix.get("action") not in READY_ACTIONS: raise ValueError("fix %s: unknown READY action %s" % (fid, fix.get("action"))) + if kind == "companion_press": + where = [fix.get("page"), fix.get("row"), fix.get("col")] + if (any(isinstance(v, bool) or not isinstance(v, int) for v in where) + or where[0] < 1 or where[1] < 0 or where[2] < 0): + raise ValueError("fix %s: a Companion button needs page, row and col (whole numbers)" % fid) + if fix.get("failover") and fix.get("never_auto"): + raise ValueError("fix %s: a failover fix runs by itself - it cannot be marked never to" % fid) + if fix.get("undo") and fix["undo"] not in cfg.get("fixes", {}): + raise ValueError("fix %s: unknown undo fix %s" % (fid, fix["undo"])) check_windows(cfg.get("service_windows"), "service_windows") check_windows((cfg.get("auto_fix") or {}).get("windows"), "auto_fix") + check_windows((cfg.get("failover") or {}).get("windows"), "failover") check_windows((cfg.get("site") or {}).get("live_services"), "site.live_services") ids = set() for t in all_tiles(cfg): @@ -2715,12 +2948,19 @@ def load_config(path): return cfg -def ping_hosts(cfg, snap=None): - hosts = [ping_target(c, snap) if snap else c.get("host") - for t in all_tiles(cfg) for c in t.get("checks", []) if c["type"] == "ping"] - hosts = [h for h in hosts if h] +def ping_hosts(cfg, snap=None, fast=False): + """The hosts to ping every PING_INTERVAL seconds (plus the gateway), or with fast=True the ones + whose check asks for less (Studio B every second, so the failover sees it die within 2-3 s).""" + quick, slow = [], [] + for t in all_tiles(cfg): + for c in t.get("checks", []): + if c["type"] == "ping": + host = ping_target(c, snap) if snap else c.get("host") + (quick if c.get("interval_seconds", PING_INTERVAL) < PING_INTERVAL else slow).append(host) + if fast: + return list(dict.fromkeys(h for h in quick if h)) gw = cfg.get("network", {}).get("gateway") - return list(dict.fromkeys(hosts + ([gw] if gw else []))) + return list(dict.fromkeys(h for h in slow + ([gw] if gw else []) if h and h not in quick)) def http_checks(cfg): @@ -2747,12 +2987,13 @@ def start_pollers(cfg, snap, stop, data_dir=None): Periodic("slow-variables", 3.0, lambda: poll_variables(c3, snap, slow), stop), Periodic("conn-hosts", 60.0, lambda: poll_conn_hosts(snap, cfg), stop), Periodic("spotify", 3.0, lambda: poll_spotify(snap, cfg), stop), - Periodic("pings", 3.0, lambda: poll_pings(snap, ping_hosts(cfg, snap), iface), stop), + Periodic("pings", float(PING_INTERVAL), lambda: poll_pings(snap, ping_hosts(cfg, snap), iface), stop), + Periodic("fast-pings", 1.0, lambda: poll_pings(snap, ping_hosts(cfg, snap, fast=True), None), stop), Periodic("midi", 0.5, lambda: poll_midi(snap, cfg), stop), Periodic("system", 10.0, lambda: poll_system(snap, cfg), stop), Periodic("bonjour", 15.0, lambda: poll_mdns(snap), stop), Periodic("http", 5.0, lambda: poll_http(snap, http_checks(cfg)), stop), - Periodic("remote", 10.0, lambda: poll_remote(snap, cfg), stop), + Periodic("remote", 1.0, lambda: poll_remote(snap, cfg, last=_remote_last), stop), # runs the groups that are due Periodic("dante-map", 30.0, lambda: poll_dante_map(snap, cfg, c4, dante_map_path(data_dir)), stop), ] for t in threads: @@ -2806,6 +3047,7 @@ def snapshot_summary(snap, engine, now): "service_on": engine.service.on(now), "alarms_override": engine.service.override, "auto_fix": {"on": engine.auto_on, "paused": engine.auto_paused}, + "failover": dict(engine.failed), "burst": engine.burst and {"start": engine.burst["start"], "conns": sorted(engine.burst["conns"]), "cleared": engine.burst["cleared"]}, "live": list(engine.live_now(now)), @@ -2937,6 +3179,10 @@ def run(cfg, data_dir, dry_run=False, stop=None, config_sha=None): engine.auto_fix(snap, comp, now) except Exception: # a bug here must never take the panel down log.exception("auto-fix failed; the panel carries on without it this round") + try: + engine.failover(snap, comp, now) + except Exception: + log.exception("failover failed; the panel carries on without it this round") for kind, text in engine.events: log.info("%s %s", kind, text) engine.events = [] @@ -2997,8 +3243,9 @@ def run_once(cfg): fast, slow = needed_variables(cfg) poll_connections(comp, snap) poll_variables(comp, snap, fast + slow) - poll_pings(snap, ping_hosts(cfg), cfg.get("network", {}).get("interface")) - poll_pings(snap, ping_hosts(cfg), None) # pings need two misses to count + everything = ping_hosts(cfg) + ping_hosts(cfg, fast=True) + poll_pings(snap, everything, cfg.get("network", {}).get("interface")) + poll_pings(snap, everything, None) # pings need two misses to count poll_system(snap, cfg) poll_mdns(snap) poll_http(snap, http_checks(cfg)) diff --git a/tests/test_builders.py b/tests/test_builders.py index f9d959a..900262f 100644 --- a/tests/test_builders.py +++ b/tests/test_builders.py @@ -35,7 +35,7 @@ class TestTodaysFixes(unittest.TestCase): if c.get("template"): self.assertEqual(b.build_check(CFG, c)["cmd"], c["cmd"]) n += 1 - self.assertEqual(n, 5) + self.assertEqual(n, 6) def test_a_changed_command_is_refused_by_validate(self): cfg = json.loads(json.dumps(CFG)) @@ -126,6 +126,27 @@ if __name__ == "__main__": unittest.main() +class TestCompanionPress(unittest.TestCase): + def test_a_press_builds_from_its_fields(self): + f = b.build(CFG, {"template": "companion_press", "params": {"page": 4, "row": 0, "col": "5", "button": "FOH Backup"}, "short": "FOH BACKUP"}) + self.assertEqual((f["kind"], f["page"], f["row"], f["col"], f["short"]), ("companion_press", 4, 0, 5, "FOH BACKUP")) + self.assertEqual(f["params"]["col"], 5) # copied as a number + self.assertNotIn("run", f); self.assertNotIn("host", f) + self.assertEqual(b.suggest_label(CFG, f), "press the FOH Backup button in Companion (page 4, row 0, column 5)") + self.assertEqual(b.suggest_label(CFG, {"template": "companion_press", "params": {"page": 4, "row": 0, "col": 5}}), + "press the button in Companion (page 4, row 0, column 5)") + for bad in ({"page": 4, "row": 0}, {"page": "four", "row": 0, "col": 5}, {"page": 4, "row": 1.5, "col": 5}, + {"page": True, "row": 0, "col": 5}): + with self.assertRaises(ValueError): + b.build(CFG, {"template": "companion_press", "params": bad}) + + def test_a_changed_button_is_caught(self): + cfg = json.loads(json.dumps(CFG)) + cfg["fixes"]["stream_foh_backup"]["col"] = 7 # the field moved, the params did not + self.assertTrue(any("stream_foh_backup" in e and "its col no longer matches" in e for e in b.mismatches(cfg)), b.mismatches(cfg)) + self.assertEqual([e for e in b.mismatches(CFG) if "stream_" in e], []) + + class TestOutlet(unittest.TestCase): def test_outlet_cycle_renders_and_is_local(self): f = b.build(CFG, {"template": "outlet_cycle", "params": {"outlet": "tascam", "address": "10.1.11.74", "wait_seconds": 8}}) diff --git a/tests/test_schema.py b/tests/test_schema.py index a85fdb6..855b5bf 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -23,17 +23,17 @@ NAMES = {"cfg": "top", "config": "top", "chk": "check", "c": "check", "fix": "fi "tile": "tile", "k": "restart_key", "key": "restart_key", "rp": "restart_page", "w": "window", "names": "variables", "host": "host", "h": "host", "windows": "windows", "site": "site", "rc": "ready"} ATTRS = {"self.cfg": "top", "engine.cfg": "top", "self.net": "network", "self.auto": "auto", "self.site": "site", - "self.atem": "atem", "self.ready_cfg": "ready", + "self.atem": "atem", "self.ready_cfg": "ready", "self.failover_cfg": "failover", "self.tiles": "tiles", "engine.tiles": "tiles", "self.restart_keys": "restart_keys", "engine.restart_keys": "restart_keys"} # A key of a top-level dict that holds a record of another kind. TOP = {"companion": "companion", "network": "network", "variables": "variables", "auto_fix": "auto", "site": "site", "atem": "atem", "spotify": "spotify", "timing": "timing", "flap": "flap", "hosts": "hosts", "fixes": "fixes", "tiles": "tiles", "restart_page": "restart_page", "gear": "gears", "service_windows": "windows", - "ready": "ready", "ready_page": "ready_page"} + "ready": "ready", "ready_page": "ready_page", "failover": "failover"} SUB = {("auto", "windows"): "windows", ("site", "live_services"): "windows", ("tile", "checks"): "checks", ("restart_page", "keys"): "restart_keys", ("tile", "members"): "names", ("ready", "due"): "windows", - ("ready_page", "keys"): "tiles"} + ("ready_page", "keys"): "tiles", ("failover", "windows"): "windows"} ELEMENT = {"tiles": "tile", "checks": "check", "restart_keys": "restart_key", "windows": "window", "fixes": "fix", "hosts": "host", "gears": "gear"} # Keys the code reads that are the watchdog's own working data, not settings. @@ -145,7 +145,8 @@ def path_of(kind, key): "restart_key": "restart_page.keys[].", "restart_page": "restart_page.", "auto": "auto_fix.", "network": "network.", "variables": "variables.", "site": "site.", "atem": "atem.", "spotify": "spotify.", "companion": "companion.", "timing": "timing.", "flap": "flap.", "gear": "gear.*.", - "window": "auto_fix.windows[].", "ready": "ready.", "ready_page": "ready_page."}.get(kind) + "window": "auto_fix.windows[].", "ready": "ready.", "ready_page": "ready_page.", + "failover": "failover."}.get(kind) return None if prefix is None else prefix + key @@ -188,7 +189,8 @@ class TestSchemaMatchesCode(unittest.TestCase): "site.live_services[].days", "site.live_services[].start", "site.live_services[].end"} for w in ("days", "start", "end"): if "auto_fix.windows[]." + w in read: - read |= {"service_windows[].%s" % w, "site.live_services[].%s" % w, "ready.due[].%s" % w} + read |= {"service_windows[].%s" % w, "site.live_services[].%s" % w, "ready.due[].%s" % w, + "failover.windows[].%s" % w} dead = sorted(p for p in schema.F if p not in read and p not in ui_only) self.assertEqual(dead, [], "declared settings nothing reads") diff --git a/tests/test_status_panel.py b/tests/test_status_panel.py index e36b023..5855b8f 100644 --- a/tests/test_status_panel.py +++ b/tests/test_status_panel.py @@ -71,6 +71,7 @@ def healthy_snapshot(now): "org.waterschurch.lobby-captions-display": 901, "org.waterschurch.digico-relay": 902} snap.launchd_time = now snap.procs = {"/Applications/ProdDash.app/Contents/MacOS/ProdDash": True} + snap.remote = {"studiob\0pgrep -xq 'Pro Tools'": True} # the failover's check has a verdict snap.http = {"https://central.resi.io/": True, "http://10.6.7.13:8080/": True, "http://10.1.14.225:1025/": True} snap.disk = {"/": 97} return snap @@ -1521,6 +1522,284 @@ class TestFixes(unittest.TestCase): self.assertIn(f.get("kind", "ssh"), sp.FIX_KINDS) +def run_failover(engine, snap, start, seconds, step=0.5): + """Like run_steps, but the failover looks after every step, as the run loop does.""" + tiles, t = None, start + while t <= start + seconds: + snap.conns_time = t + tiles = engine.step(snap, t) + engine.failover(snap, None, t) + t += step + return tiles, t + + +STB_PING, STB_PT = "10.1.13.175", "studiob\0pgrep -xq 'Pro Tools'" + + +class TestFailover(unittest.TestCase): + """Studio B dies on a Sunday: the FOH Backup button is pressed by itself, once, and the keys say so. + Nothing here presses a real button, posts a real alert or writes a real log.""" + + def setUp(self): + self.snap = healthy_snapshot(SUNDAY) + self.engine = sp.Engine(CONFIG) + self.ran, self.sent = [], [] + self.engine.executor = lambda cfg, fix_id, chk: self.ran.append(fix_id) or True + self.engine.poe_executor = lambda *a: True + self.engine.notify = lambda url, text: self.sent.append(text) + self.engine.autolog = sp.AutoLog(tempfile.mkdtemp()) + + def fired_at(self): + self.engine.fix_threads and [t.join(2) for t in self.engine.fix_threads] + rec = self.engine.failed.get("stream_foh_backup") + return rec and rec["at"] + + def test_config_wires_studio_b_to_foh_backup(self): + tile = next(t for t in CONFIG["tiles"] if t["id"] == "stb_audio") + self.assertNotIn("pos", tile) # hidden: AUDIO shows it + self.assertIn("stb_audio", next(t for t in CONFIG["tiles"] if t["id"] == "audio")["members"]) + for c in tile["checks"]: + fixes = set(c["fix"].values()) if isinstance(c["fix"], dict) else {c["fix"]} + self.assertEqual(fixes, {"stream_foh_backup"}, c["type"]) + fix = CONFIG["fixes"]["stream_foh_backup"] + self.assertEqual((fix["kind"], fix["page"], fix["row"], fix["col"]), ("companion_press", 4, 0, 5)) + self.assertTrue(fix["failover"]) + self.assertEqual(fix["undo"], "stream_studio_b") + self.assertFalse(CONFIG["fixes"]["stream_studio_b"].get("failover")) + self.assertNotIn("stream_foh_backup", CONFIG["auto_fix"]["fixes"]) + keys = {k["id"]: k["fix"] for k in CONFIG["restart_page"]["keys"]} + self.assertEqual((keys["rs_stream_stb"], keys["rs_stream_foh"]), ("stream_studio_b", "stream_foh_backup")) + self.assertEqual(CONFIG["failover"]["after_seconds"], 2) + self.assertEqual(self.engine.failover_windows, [{"days": ["sun"], "start": "06:30", "end": "13:30"}]) + + def test_studio_b_dies_foh_backup_is_pressed_once(self): + t0 = SUNDAY + run_failover(self.engine, self.snap, t0 - 5, 4) # all fine for a while + self.snap.ping[STB_PING] = 2 # two 1-second pings missed + tiles, t = run_failover(self.engine, self.snap, t0, 6) + at = self.fired_at() + self.assertEqual(self.ran, ["stream_foh_backup"]) + key = next(k for k, (tid, chk, sub) in self.engine.meta.items() if chk.get("host") == STB_PING) + since = self.engine.trackers[key].since + self.assertEqual(since, t0 + 1) # 1 s debounce + self.assertEqual(at, since + 2) # + 2 s after_seconds + self.assertLessEqual(at - t0, 5) + tiles, t = run_failover(self.engine, self.snap, t, 60) # the fault lasts a minute + self.assertEqual(self.ran, ["stream_foh_backup"]) # still once + self.assertNotIn("stream_studio_b", self.ran) # never switched back by itself + # the keys say where the stream is + self.assertEqual(tiles["audio"]["x"], "AUDIO\nSTUDIO B NO PING\nON FOH BACKUP") + self.assertEqual(level(tiles["audio"]), "err") + self.engine.notes.pop("stb_audio", None) # under the fix's own feedback + tiles = self.engine.step(self.snap, t) + self.assertEqual(tiles["stb_audio"]["x"], "STB AUDIO\nSTUDIO B NO PING\nON FOH BACKUP") + self.assertEqual(tiles["all"]["x"], "1 DOWN\nSTUDIO B NO PING") + # the log, the alert, and a hold right after + body = [l for l in open(self.engine.autolog.path(t0)).read().splitlines() if l[:2].isdigit()] + self.assertTrue(any("STB AUDIO FAILOVER: STUDIO B NO PING - press the FOH Backup button" in l for l in body), body) + self.assertTrue(any("-> done, checking it comes back" in l for l in body), body) + self.assertEqual(len(self.sent), 1) + self.assertTrue(self.sent[0].startswith("NA Ingest: STB AUDIO STUDIO B NO PING - switched over by itself: press the FOH Backup")) + self.assertIn("hold STREAM STB on the Restart page", self.sent[0]) + self.engine.run_fixes("audio", self.snap, None, at + 10) + self.assertEqual(self.engine.notes["audio"]["text"], "JUST TRIED\nWAIT 1 MIN") + self.assertIsNone(self.engine.run_force("rs_stream_foh", self.snap, None, at + 11)) + self.assertEqual(self.engine.notes["rs_stream_foh"]["text"], "JUST DONE\nWAIT 1 MIN") + out = sp.snapshot_summary(self.snap, self.engine, t) + json.dumps(out) + self.assertEqual(out["failover"]["stream_foh_backup"]["note"], "ON FOH BACKUP") + self.assertEqual(out["failover"]["stream_foh_backup"]["faults"], ["STUDIO B NO PING"]) + + def test_pro_tools_crash_is_as_fast(self): + t0 = SUNDAY + self.snap.remote[STB_PT] = False + run_failover(self.engine, self.snap, t0, 6) + self.assertEqual(self.ran, ["stream_foh_backup"]) + self.assertLessEqual(self.fired_at() - t0, 5) + tiles = self.engine.step(self.snap, t0 + 7) + self.assertEqual(tiles["audio"]["x"], "AUDIO\nPRO TOOLS NOT RUNNING\nON FOH BACKUP") + + def test_fires_with_alarms_muted(self): + self.engine.service.set(False, SUNDAY) # maintenance: auto-fix stops + self.snap.ping[STB_PING] = 2 + run_failover(self.engine, self.snap, SUNDAY, 6) + self.assertEqual(self.ran, ["stream_foh_backup"]) # the failover does not + + def test_only_inside_the_window(self): + for when in (WEDNESDAY, datetime(2026, 10, 11, 15, 0).timestamp()): # a weekday; Sunday after the window + snap = healthy_snapshot(when) + snap.ping[STB_PING] = 2 + run_failover(self.engine, snap, when, 10) + self.assertEqual(self.ran, [], datetime.fromtimestamp(when)) + self.assertEqual(self.engine.failed, {}) + + def test_rearms_when_the_fault_clears_and_the_note_fades(self): + t0 = SUNDAY + self.snap.ping[STB_PING] = 2 + tiles, t = run_failover(self.engine, self.snap, t0, 6) + self.snap.ping[STB_PING] = 0 # Studio B answers again + tiles, t = run_failover(self.engine, self.snap, t, 5) + self.assertEqual(tiles["audio"]["x"], "AUDIO\nOK\nON FOH BACKUP") # still on the backup + self.assertEqual(level(tiles["audio"]), "ok") + self.assertIsNotNone(self.engine.failed["stream_foh_backup"]["cleared"]) + self.snap.ping[STB_PING] = 2 # ...and dies again 5 s later + tiles, t = run_failover(self.engine, self.snap, t, 4) + self.assertEqual(self.ran, ["stream_foh_backup"]) # was clear under 10 s: not armed yet + tiles, t = run_failover(self.engine, self.snap, t, 3) # 10 s since it cleared: armed, fires + self.assertEqual(self.ran, ["stream_foh_backup", "stream_foh_backup"]) + self.snap.ping[STB_PING] = 0 + tiles, t = run_failover(self.engine, self.snap, t, 2) + tiles, t = run_failover(self.engine, self.snap, t + 1000, 1) # note_seconds later: forgotten + self.assertEqual(tiles["audio"]["x"], "AUDIO\nOK") + self.assertEqual(self.engine.failed, {}) + + def test_a_check_that_cannot_ask_is_not_a_clear(self): + t0 = SUNDAY + self.snap.remote[STB_PT] = False # Pro Tools gone + tiles, t = run_failover(self.engine, self.snap, t0, 6) + self.assertEqual(self.ran, ["stream_foh_backup"]) + self.snap.remote[STB_PT] = None # ssh could not ask for a while + tiles, t = run_failover(self.engine, self.snap, t, 30) + self.assertIsNone(self.engine.failed["stream_foh_backup"]["cleared"]) # not re-armed on a hiccup + self.snap.remote[STB_PT] = False # the answer is back: still gone + tiles, t = run_failover(self.engine, self.snap, t, 10) + self.assertEqual(self.ran, ["stream_foh_backup"]) # no second press + self.assertEqual(tiles["rs_stream_foh"]["x"], "STREAM FOH\nON FOH BACKUP") # pressed already: no HOLD TO FIX + self.assertEqual(level(tiles["rs_stream_foh"]), "ok") + self.assertEqual(tiles["rs_stream_stb"]["x"], "STREAM STB\nHOLD 2s") + + def test_studio_b_rednet_is_never_power_cycled_by_itself(self): + """Only STUDIO-B-RedNet-TNX vanishes: the failover presses FOH Backup; Sunday auto-fix on the DANTE + key must not power-cycle the RedNet's PoE port (that stays a hold), nor touch anything else.""" + comp = mock_comp() + self.snap.dante_map = {"STUDIO-B-RedNet-TNX": {"mac": "00:1d:c1:aa:bb:cc", "poe": True, "switch": "10.1.60.2", "port": "0/14"}} + devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")]) + self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps([d for d in devices if d != "STUDIO-B-RedNet-TNX"]) + poe = [] + self.engine.poe_executor = lambda cfg, name, entry: poe.append(name) or True + tiles, t = run_failover(self.engine, self.snap, SUNDAY, 30) + self.assertEqual(self.ran, ["stream_foh_backup"]) + self.assertEqual(self.engine.auto_fix(self.snap, comp, t), []) + time.sleep(0.2) + self.assertEqual((poe, comp.restarted, self.ran), ([], [], ["stream_foh_backup"])) + self.assertEqual([(k, w) for _, k, w, _ in self.engine.fix_plan("dante", self.snap)], [("poe", "STUDIO-B-RedNet-TNX")]) # a hold would + + def test_switching_back_by_hand_clears_the_note(self): + t0 = SUNDAY + gate = threading.Event() # the Studio B press takes a moment + + def press(cfg, fix_id, chk): + if fix_id == "stream_studio_b": + gate.wait(2) + self.ran.append(fix_id) + return True + self.engine.executor = press + self.snap.ping[STB_PING] = 2 + tiles, t = run_failover(self.engine, self.snap, t0, 6) + self.assertIn("ON FOH BACKUP", tiles["audio"]["x"]) + self.assertEqual(self.engine.run_force("rs_stream_stb", self.snap, None, t), "stream_studio_b") + self.assertEqual(self.engine.notes["rs_stream_stb"]["text"], "PRESSING") + gate.set() + time.sleep(0.2) + self.assertEqual(self.engine.notes["rs_stream_stb"]["text"], "DONE") + self.assertEqual(self.engine.failed, {}) + tiles = self.engine.step(self.snap, t + 0.5) + self.assertEqual(tiles["audio"]["x"], "AUDIO\nSTUDIO B NO PING") + tiles, t = run_failover(self.engine, self.snap, t + 1, 4) # still dead: it fires again + self.assertEqual(self.ran, ["stream_foh_backup", "stream_studio_b", "stream_foh_backup"]) + self.assertIn("ON FOH BACKUP", tiles["audio"]["x"]) + + def test_held_back_when_several_dante_devices_vanish(self): + t0 = SUNDAY + devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")]) + self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps( + [d for d in devices if d not in ("STUDIO-B-RedNet-TNX", "NA-Tracks", "BOH-WING")]) + tiles, t = run_failover(self.engine, self.snap, t0, 12) + self.assertEqual(self.ran, []) # Ingest's view of Dante: hands off + self.assertEqual(tiles["stb_audio"]["x"], "STB AUDIO\nSTB TNX GONE") # seen, no note + body = open(self.engine.autolog.path(t0)).read() + self.assertIn("failover held back: 3 Dante devices gone at once", body) + self.assertEqual(body.count("held back"), 1) # said once, not every half second + self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps( + [d for d in devices if d != "STUDIO-B-RedNet-TNX"]) # only Studio B's own box + run_failover(self.engine, self.snap, t, 3) + self.assertEqual(self.ran, ["stream_foh_backup"]) + + def test_state_survives_a_restart(self): + state = os.path.join(tempfile.mkdtemp(prefix="sp-state-"), "state.json") + engine = sp.Engine(CONFIG, state) + engine.executor = lambda cfg, fix_id, chk: self.ran.append(fix_id) or True + engine.notify = lambda url, text: None + engine.autolog = sp.AutoLog(tempfile.mkdtemp()) + self.snap.ping[STB_PING] = 2 + tiles, t = run_failover(engine, self.snap, SUNDAY, 6) + self.assertEqual(self.ran, ["stream_foh_backup"]) + self.assertEqual(json.load(open(state))["failover"]["stream_foh_backup"]["tile"], "stb_audio") + again = sp.Engine(CONFIG, state) # the watchdog restarts mid-fault + again.executor = lambda cfg, fix_id, chk: self.ran.append(fix_id) or True + again.notify = lambda url, text: None + again.autolog = sp.AutoLog(tempfile.mkdtemp()) + tiles, t = run_failover(again, self.snap, t, 10) + self.assertEqual(self.ran, ["stream_foh_backup"]) # not pressed a second time + self.assertEqual(tiles["audio"]["x"], "AUDIO\nSTUDIO B NO PING\nON FOH BACKUP") + + def test_nothing_new_starts_while_stopping(self): + self.snap.ping[STB_PING] = 2 + self.engine.stopping = True + run_failover(self.engine, self.snap, SUNDAY, 6) + self.assertEqual(self.ran, []) + + def test_load_config_checks_the_button_and_the_undo(self): + def load(change): + cfg = json.loads(json.dumps(CONFIG)) + change(cfg) + path = os.path.join(tempfile.mkdtemp(prefix="sp-cfg-"), "config.json") + with open(path, "w") as f: + json.dump(cfg, f) + return sp.load_config(path) + load(lambda c: None) + for bad in (lambda c: c["fixes"]["stream_foh_backup"].update(page=0), + lambda c: c["fixes"]["stream_foh_backup"].update(row=True), + lambda c: c["fixes"]["stream_foh_backup"].update(col="5"), + lambda c: c["fixes"]["stream_foh_backup"].pop("col"), + lambda c: c["fixes"]["stream_foh_backup"].update(never_auto=True), + lambda c: c["fixes"]["stream_foh_backup"].update(undo="nope"), + lambda c: c["failover"]["windows"].append({"days": ["sun"], "start": "6:30", "end": "13:30"})): + with self.assertRaises(ValueError): + load(bad) + + def test_studio_b_is_pinged_every_second(self): + self.assertEqual(sp.ping_hosts(CONFIG, fast=True), [STB_PING]) + slow = sp.ping_hosts(CONFIG) + self.assertNotIn(STB_PING, slow) + self.assertIn("10.1.0.1", slow) # the gateway stays in the slow list + self.assertIn("10.1.3.227", slow) + self.assertEqual(sp.PING_INTERVAL, 3) + + def test_remote_groups_run_when_due_each_on_its_own(self): + from unittest import mock + groups = sp.remote_groups(CONFIG) + self.assertEqual(groups[("studiob", 2)], ["pgrep -xq 'Pro Tools'"]) + self.assertTrue(all(k[1] == 10 for k in groups if k != ("studiob", 2))) + calls = [] + fake = lambda argv, **kw: calls.append(argv) or mock.Mock(returncode=0, stdout="SP0=0\nSP1=0\n", stderr="") + t = SUNDAY + 100 + with mock.patch.object(sp.subprocess, "run", fake): + last = {k: t - 3 for k in groups} # every group ran 3 s ago + for th in sp.poll_remote(self.snap, CONFIG, now=t, last=last): + th.join(2) + self.assertEqual(len(calls), 1) # only the 2-second group is due + self.assertIn("studiob@10.1.13.175", calls[0]) + self.assertIn("pgrep -xq 'Pro Tools'", calls[0][-1]) + self.assertTrue(self.snap.remote[STB_PT]) + self.assertEqual(last[("studiob", 2)], t) + self.assertEqual(sp.poll_remote(self.snap, CONFIG, now=t + 1, last=last), []) # nothing due yet + calls.clear() + sp.poll_remote(self.snap, CONFIG) # run_once: everything, and waits + self.assertEqual(len(calls), len(groups)) + self.assertEqual(sp.REMOTE_INTERVAL, 10) + + class TestNetgearParsing(unittest.TestCase): BIG = """ VLAN ID MAC Address Interface IfIndex Status 66 00:1D:C1:51:34:C8 0/14 14 Learned @@ -1603,6 +1882,13 @@ class FakeCompanion(BaseHTTPRequestHandler): if m: s["restarts"].append(m.group(1)) return self._send(200, "{}") + m = re.match(r"^/api/location/(\d+)/(\d+)/(\d+)/press$", u.path) + if m: + where = tuple(int(x) for x in m.groups()) + s["presses"].append(where) + if where in s["buttons"]: + return self._send(200, "ok") + return self._send(204, "No control") # like Companion 5.0.7 for an empty cell self._send(404, "") @@ -1615,6 +1901,8 @@ class TestAgainstFakeCompanion(unittest.TestCase): "vars": dict(snap.vars), "custom": {"sp_state": "", "sp_hb": "", "sp_cmd": "", "flash_tick": "0"}, "restarts": [], + "presses": [], + "buttons": {(4, 0, 5), (4, 0, 6)}, # the Audio page's FOH Backup and Studio B } cls.server = ThreadingHTTPServer(("127.0.0.1", 0), FakeCompanion) threading.Thread(target=cls.server.serve_forever, daemon=True).start() @@ -1637,6 +1925,20 @@ class TestAgainstFakeCompanion(unittest.TestCase): self.assertEqual(comp.set_custom("missing", "x"), (False, 404)) print("\n sp_state is %d bytes, %d in the URL" % (len(js), len(urllib.parse.quote(js, safe="")))) + def test_press_a_button(self): + comp = sp.Companion("127.0.0.1", self.port) + store = FakeCompanion.store + self.assertEqual(comp.press(4, 0, 5), 200) + self.assertEqual(comp.press(4, 1, 1), 204) # an empty cell + self.assertEqual(store["presses"][-2:], [(4, 0, 5), (4, 1, 1)]) + cfg = json.loads(json.dumps(CONFIG)) + cfg["companion"]["port"] = self.port + self.assertTrue(sp.execute_fix(cfg, "stream_foh_backup", {})) + self.assertEqual(store["presses"][-1], (4, 0, 5)) + cfg["fixes"]["stream_foh_backup"]["col"] = 7 + self.assertEqual(sp.execute_fix(cfg, "stream_foh_backup", {}), "NO BUTTON\nTHERE") + self.assertEqual(store["presses"][-1], (4, 0, 7)) + def test_commands(self): comp = sp.Companion("127.0.0.1", self.port) store = FakeCompanion.store @@ -1679,7 +1981,7 @@ class TestAgainstFakeCompanion(unittest.TestCase): sp.poll_mdns = lambda snap, seconds=3: None sp.poll_system = lambda snap, cfg: None sp.poll_midi = lambda snap, cfg: None # CoreMIDI has its own test - sp.poll_remote = lambda snap, cfg: None # never SSH anywhere from a test + sp.poll_remote = lambda snap, cfg, **k: [] # never SSH anywhere from a test sp.start_remote_midi_streams = lambda snap, cfg, stop: [] stop = threading.Event() loop = threading.Thread(target=sp.run, args=(cfg, data_dir), kwargs={"stop": stop}, daemon=True)