Settings page: ON/OFF switches everywhere, the Auto switch screen, a Companion button picker
- ON/OFF on every key, check, fix, Restart and READY key, computer, MIDI link, HyperDeck, Dante device, watched Companion connection, switcher and Spotify check, outlet fix, and one main switch each for auto-fix, the auto switch and the READY page. Switched off = kept in the settings, grey OFF on the deck, never checked or run (effective_config in the watchdog). - Settings › Auto switch: a plain summary, what counts as down (each trigger editable in place: address, program and computer, Dante device, the button it presses, timings), the hours folded away, the buttons; + Button, + What counts as down, Remove. - Pick a Companion button by clicking it on its page, drawn like Companion (read-only API: /api/companion/pages and /api/companion/page). - STREAM STB / STREAM FOH Restart keys removed (the Studio B page has its own); the alert names the switch-back button instead. - Dante screen: shows the DANTE key's device list again (it picked STB AUDIO's one device). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
+57
-4
@@ -608,6 +608,41 @@ def ready_keys(cfg):
|
||||
return (cfg.get("ready_page") or {}).get("keys", [])
|
||||
|
||||
|
||||
def effective_config(cfg):
|
||||
"""The config as the watchdog runs it: a key that is switched off ("disabled": true) keeps its
|
||||
place on the page but checks nothing, and a check that is switched off is left out. Like a
|
||||
connection switched off in Companion, everything stays in the settings, ready to switch on."""
|
||||
out = json.loads(json.dumps(cfg))
|
||||
# A computer switched off: nothing logs in to it, so its checks and fixes go too.
|
||||
off_hosts = {a for a, h in cfg.get("hosts", {}).items() if h.get("disabled")}
|
||||
off_ips = {str(h.get("ssh", "")).split("@")[-1] for a, h in cfg.get("hosts", {}).items() if a in off_hosts}
|
||||
for fid, fix in (out.get("fixes") or {}).items():
|
||||
if fix.get("host") in off_hosts:
|
||||
fix["disabled"] = True
|
||||
ready = out.get("ready") or {}
|
||||
if (cfg.get("ready") or {}).get("disabled"): # the READY page switched off as a whole
|
||||
for k in ready_keys(out):
|
||||
k["disabled"] = True
|
||||
for name in ("tracks_host", "prop_bundle_host"):
|
||||
if ready.get(name) in off_hosts:
|
||||
ready.pop(name)
|
||||
for t in all_tiles(out):
|
||||
if "checks" in t:
|
||||
t["checks"] = [] if t.get("disabled") else [
|
||||
c for c in t["checks"] if not c.get("disabled") and c.get("host") not in off_hosts
|
||||
and not (c.get("type") == "ping" and c.get("host") in off_ips)
|
||||
and not (c.get("type") == "http" and urllib.parse.urlparse(c.get("url", "")).hostname in off_ips)]
|
||||
for c in t["checks"]:
|
||||
if c.get("devices_off") and isinstance(c.get("devices"), dict): # Dante devices switched off
|
||||
c["devices"] = {n: v for n, v in c["devices"].items() if n not in c["devices_off"]}
|
||||
# Auto-fix or the auto switch switched off as a whole: their hours are emptied, so they never act.
|
||||
if (cfg.get("auto_fix") or {}).get("disabled"):
|
||||
out["auto_fix"]["windows"] = []
|
||||
if (cfg.get("failover") or {}).get("disabled"):
|
||||
out["failover"]["windows"] = []
|
||||
return out
|
||||
|
||||
|
||||
def all_tiles(cfg):
|
||||
"""Every key the watchdog judges: the Status page's tiles and the READY page's keys. The READY
|
||||
page's keys are not in ALL (nor in auto-fix): a weekday gap in the checklist is not an alarm."""
|
||||
@@ -882,6 +917,7 @@ class ServiceMode:
|
||||
|
||||
class Engine:
|
||||
def __init__(self, cfg, state_path=None, data_dir=None):
|
||||
cfg = effective_config(cfg)
|
||||
self.cfg = cfg
|
||||
self.tiles = {t["id"]: t for t in all_tiles(cfg)}
|
||||
# The READY page's checklist keys (a "ready" kind key sums them up, on either page).
|
||||
@@ -1488,7 +1524,8 @@ class Engine:
|
||||
if self.auto_paused and self.autolog:
|
||||
self.autolog.write(now, "NET", "network settled: auto-fix is back on")
|
||||
self.auto_paused = False
|
||||
gentle = {f for f in a.get("fixes", []) if not self.cfg["fixes"].get(f, {}).get("never_auto")}
|
||||
gentle = {f for f in a.get("fixes", []) if not self.cfg["fixes"].get(f, {}).get("never_auto")
|
||||
and not self.cfg["fixes"].get(f, {}).get("disabled")}
|
||||
done = []
|
||||
for t in self.cfg["tiles"]:
|
||||
if "checks" not in t:
|
||||
@@ -1560,7 +1597,7 @@ class Engine:
|
||||
if isinstance(fix_id, dict):
|
||||
fix_id = fix_id.get(sub, fix_id.get("*"))
|
||||
fix = self.cfg["fixes"].get(fix_id) if fix_id else None
|
||||
return fix_id if fix and fix.get("failover") else None
|
||||
return fix_id if fix and fix.get("failover") and not fix.get("disabled") else None
|
||||
|
||||
def dante_gone(self, conn):
|
||||
"""How many devices on this Dante connection are gone right now, counted over every dante
|
||||
@@ -1656,10 +1693,12 @@ class Engine:
|
||||
log.info("FAILOVER %s: %s - %s", tile_id, ", ".join(msgs), what)
|
||||
if self.autolog:
|
||||
self.autolog.write(now, label, "FAILOVER: %s - %s" % (", ".join(msgs), what))
|
||||
# How to switch back: a Restart key that runs the undo, else the undo fix's own words.
|
||||
back = next((k["label"] for k in self.restart_keys.values() if k["fix"] == fix.get("undo")), None)
|
||||
undo = self.cfg["fixes"].get(fix.get("undo") or "", {}).get("label")
|
||||
text = "%s: %s %s - switched over by itself: %s. Switch back by hand%s once it is fixed." % (
|
||||
self.site.get("alert_prefix") or self.site.get("ingest_name") or "Status panel", label, ", ".join(msgs),
|
||||
what, (" (hold %s on the Restart page)" % back) if back else "")
|
||||
what, (" (hold %s on the Restart page)" % back) if back else (" (%s)" % undo) if undo else "")
|
||||
threading.Thread(target=self.notify, args=(self.cfg.get("alert_webhook"), text), daemon=True).start()
|
||||
started = time.time()
|
||||
self.run_plan(tile_id, [(("failover", fix_id), "fix", fix_id, chk)], snap, comp, now, cooldown=0, auto=True,
|
||||
@@ -1692,6 +1731,9 @@ class Engine:
|
||||
log.warning("FORCE %s: no such restart key", key_id)
|
||||
return None
|
||||
fix_id = key["fix"]
|
||||
if key.get("disabled"):
|
||||
self.note(key_id, "TURNED\nOFF", "idle", now, 4)
|
||||
return None
|
||||
busy = [w for w in key.get("watch", []) if not w.startswith("conn:")
|
||||
and now - self.fix_last.get(("fix", w, ""), 0) < cooldown]
|
||||
if busy: # e.g. auto-fix is starting Ableton right now
|
||||
@@ -1847,6 +1889,9 @@ def render(engine, service_on):
|
||||
down_fixes, down_conns = engine.failing_targets()
|
||||
for kid, key in engine.restart_keys.items():
|
||||
note = engine.notes.get(kid)
|
||||
if key.get("disabled") or engine.cfg["fixes"].get(key["fix"], {}).get("disabled"):
|
||||
out[kid] = colored(key["label"] + "\nOFF", "off") # switched off in the settings
|
||||
continue
|
||||
watch = [key["fix"]] + key.get("watch", [])
|
||||
down = any(w[5:] in down_conns if w.startswith("conn:") else w in down_fixes for w in watch)
|
||||
pressed = engine.failover_note_for(key["fix"])
|
||||
@@ -1860,6 +1905,9 @@ def render(engine, service_on):
|
||||
out[kid] = colored(key["label"] + "\n" + engine.hold_text, "idle")
|
||||
for tile in all_tiles(engine.cfg):
|
||||
tid, kind = tile["id"], tile.get("kind")
|
||||
if tile.get("disabled"): # switched off in the settings: grey, checks nothing
|
||||
out[tid] = colored(tile["label"] + "\nOFF", "off")
|
||||
continue
|
||||
note = engine.notes.get(tid)
|
||||
if note and engine.now < note["until"]: # a hold is being acted on: say so
|
||||
out[tid] = colored(tile["label"] + "\n" + note["text"], note["level"])
|
||||
@@ -2469,6 +2517,9 @@ def execute_fix(cfg, fix_id, chk):
|
||||
Returns True if the command ran and succeeded."""
|
||||
fix = cfg["fixes"][fix_id]
|
||||
kind, label = fix.get("kind", "ssh"), fix.get("label", fix_id)
|
||||
if fix.get("disabled"): # switched off in the settings: never runs
|
||||
log.info("FIX %s: switched off in the settings, not run", label)
|
||||
return "TURNED\nOFF"
|
||||
try:
|
||||
if kind in ("ssh", "local"):
|
||||
argv = ssh_cmd(cfg, fix["host"]) + [fix["run"]] if kind == "ssh" else ["/bin/zsh", "-c", fix["run"]]
|
||||
@@ -3120,6 +3171,7 @@ def handle_commands(text, engine, comp, snap, now):
|
||||
|
||||
|
||||
def run(cfg, data_dir, dry_run=False, stop=None, config_sha=None):
|
||||
cfg = effective_config(cfg)
|
||||
stop = stop or threading.Event()
|
||||
names = cfg["variables"]
|
||||
comp = make_companion(cfg)
|
||||
@@ -3139,7 +3191,7 @@ def run(cfg, data_dir, dry_run=False, stop=None, config_sha=None):
|
||||
except ValueError: # not the main thread (tests)
|
||||
pass
|
||||
threads = start_pollers(cfg, snap, stop, data_dir)
|
||||
if cfg.get("ready_page"):
|
||||
if cfg.get("ready_page") and not (cfg.get("ready") or {}).get("disabled"):
|
||||
import readiness
|
||||
poller = readiness.Poller(cfg, snap, stop, data_dir, is_live=lambda: engine.live_now(time.time())[0])
|
||||
poller.start()
|
||||
@@ -3238,6 +3290,7 @@ def run(cfg, data_dir, dry_run=False, stop=None, config_sha=None):
|
||||
|
||||
def run_once(cfg):
|
||||
"""Poll everything synchronously, judge with no debounce, print the tiles."""
|
||||
cfg = effective_config(cfg)
|
||||
snap = Snapshot()
|
||||
comp = make_companion(cfg)
|
||||
fast, slow = needed_variables(cfg)
|
||||
|
||||
Reference in New Issue
Block a user