christian/status-indicators
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

READY page: the weekly worship checklist, checked against Planning Center

Browse Source 
- readiness.py looks every 2 minutes (never while a HyperDeck records) at this weekend's PCO plan
  and what is in place for it: the Ableton set (songs, mix, routing, lighting cues), ProPresenter
  (every PCO item, MC announcements, baptism names, timelines, backgrounds), the LTC note of every
  song against the timecode the tracks play, the Band Lineup TV, Resolume's LED cross, and the
  tracks and ProP bundles for the locations.
- READY key on the Status page (row 2, col 7; SPOTIFY now shows inside SHOW) opens page 24.
  Amber until done, blinking red from Saturday noon; not part of ALL. Tap keys for BOH mix,
  mics and stage reset every Monday.
- Holds fix what can be fixed: re-sync the lineup, put the cross back up, write LTC into PCO,
  copy tracks to locations, look again. Hold READY fixes everything.
- Settings page: a READY page tab (live keys, drag to move, why each key is amber), and Apply
  re-imports page 24.
- Auto-fix: "while_muted" fixes keep running with ALARMS muted (Spotify playlist).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
christian
2026-10-14 20:28:43 -04:00
co-authored by Claude Opus 5.5
parent 9b33abb570
commit d5ede0c91e
15 changed files with 2392 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
status_panel.py
+157 -27
View File
@@ -42,7 +42,7 @@ log = logging.getLogger("status-panel")
VERSION = "2026.10.13" # logged at start ("status panel running (code ...)") and written to started.json
# Every kind of fix the watchdog can run (validated by load_config).
FIX_KINDS = ("ssh", "local", "atem", "reconnect", "midi_connect", "netgear_poe", "hyperdeck", "spotify")
FIX_KINDS = ("ssh", "local", "atem", "reconnect", "midi_connect", "netgear_poe", "hyperdeck", "spotify", "ready")
OK, WARN, ERR, OFF = "ok", "warn", "err", "off"
@@ -179,6 +179,7 @@ class Snapshot:
self.dante_map_time = 0.0 # last successful survey of the Dante switches
self.artnet_last = {} # source IP -> time its last Art-Net packet arrived
self.artnet_since = None # when the Art-Net listener started (None: not listening)
self.ready = None # readiness.py's latest look at this weekend's checklist
def var(self, label, name):
return self.vars.get((label, name))
@@ -537,13 +538,54 @@ def check_remote(chk, snap, ctx):
return [("remote", Result(OK))]
def check_ready(chk, snap, ctx):
"""One line of the weekly checklist on the READY page; readiness.py does the looking. Until the
checklist is due (Saturday noon on) every gap is amber; from then on a real gap is red."""
if snap.ready is None:
return [("ready", Result(OFF, info="CHECKING"))]
item = snap.ready.get("items", {}).get(chk["item"])
if item is None:
return [("ready", Result(OFF, info="NOT SET UP"))]
if not item.get("faults"):
return [("ready", Result(OK, info=item.get("info", "")))]
return [(msg, Result(ERR if level == ERR and ctx.get("ready_due") else WARN, msg))
for level, msg in item["faults"]]
def check_ready_manual(chk, snap, ctx):
"""A checklist line only a person can see (BOH mix, stage): a tap marks it done for the week."""
done = ctx.get("ready_done", {}).get(chk["item"])
if done:
return [("done", Result(OK, info="DONE " + datetime.fromtimestamp(done).strftime("%a %H:%M").upper()))]
info = ((snap.ready or {}).get("items", {}).get(chk["item"]) or {}).get("info") # e.g. the mics PCO needs
msg = chk.get("msg", "TAP WHEN DONE")
return [("done", Result(ERR if ctx.get("ready_due") else WARN, (info + "\n" + msg) if info else msg))]
# What readiness.py looks at (check type "ready") and what a "ready" fix can do.
READY_ITEMS = ("tracks", "mix_foh", "lights", "prop", "timecode", "backgrounds", "led_cross", "lineup", "mics",
"track_bundle", "prop_bundle")
READY_ACTIONS = ("recheck", "lineup_sync", "led_cross", "copy_tracks", "ltc_to_pco")
CHECKS = {
"conn": check_conn, "hyperdeck": check_hyperdeck, "dante": check_dante, "shure": check_shure,
"psm": check_psm, "tascam": check_tascam, "ping": check_ping, "launchd": check_launchd,
"process": check_process, "http": check_http, "disk": check_disk, "midi_peer": check_midi_peer, "pref": check_pref, "spotify": check_spotify,
"mdns": check_mdns, "midi_rx": check_midi_rx, "remote": check_remote, "artnet": check_artnet,
"ready": check_ready, "ready_manual": check_ready_manual,
}
def ready_keys(cfg):
"""The READY page's keys (tile-shaped: label, pos, checks)."""
return (cfg.get("ready_page") or {}).get("keys", [])
def all_tiles(cfg):
"""Every key the watchdog judges: the Status page's tiles and the READY page's keys. The READY
page's keys are not in ALL (nor in auto-fix): a weekday gap in the checklist is not an alarm."""
return cfg["tiles"] + ready_keys(cfg)
# Checks whose connection a long press may restart (Dante is shared, so not that one).
RESTARTABLE = ("conn", "hyperdeck", "shure", "psm", "tascam")
# A check's "fix" may name this instead of an entry in "fixes": restart its own Companion
@@ -558,7 +600,7 @@ DANTE_AUTO = "dante-auto"
def needed_variables(cfg):
"""(fast, slow) lists of (connection label, variable name) to poll."""
fast, slow = [], []
for tile in cfg["tiles"]:
for tile in all_tiles(cfg):
for chk in tile.get("checks", []):
t, label = chk["type"], chk.get("conn")
if t == "hyperdeck":
@@ -733,7 +775,7 @@ def check_identity(chk):
if t == "pref":
return "%s/%s" % (chk["domain"], chk["key"])
single = {"http": "url", "mdns": "instance", "launchd": "label", "process": "match", "artnet": "source",
"atem": "part", "disk": "path", "midi_rx": "source"}
"atem": "part", "disk": "path", "midi_rx": "source", "ready": "item", "ready_manual": "item"}
if t in single:
return str(chk.get(single[t], ""))
return chk.get("conn") or t
@@ -814,7 +856,10 @@ class ServiceMode:
class Engine:
def __init__(self, cfg, state_path=None, data_dir=None):
self.cfg = cfg
self.tiles = {t["id"]: t for t in cfg["tiles"]}
self.tiles = {t["id"]: t for t in all_tiles(cfg)}
# The READY page's checklist keys (a "ready" kind key sums them up, on either page).
self.ready_ids = [r["id"] for r in ready_keys(cfg) if r.get("kind") != "ready"]
self.ready_cfg = cfg.get("ready") or {}
self.debounce = cfg.get("debounce_seconds", 3)
self.flap_count = cfg.get("flap", {}).get("count", 3)
self.flap_window = cfg.get("flap", {}).get("window_seconds", 600)
@@ -864,6 +909,7 @@ class Engine:
self.auto_watch = {} # "BUTTON FAULT" auto-fix went after -> when (to log when it clears)
self.auto_paused = False
self.auto_on = False # inside the window (for the "auto-fix on" line)
self.auto_muted = False # ...with ALARMS muted, so only the "while_muted" fixes run
self.auto_last = 0.0
self.deck_ips = {} # HyperDeck connection -> last address Companion reported
self.now = 0.0
@@ -945,19 +991,20 @@ class Engine:
# raw results for every check key
def evaluate(self, snap, now, service_on):
ctx = {"now": now, "service": service_on, "any_recording": False, "dante_all": {}}
for t in self.cfg["tiles"]:
ctx = {"now": now, "service": service_on, "any_recording": False, "dante_all": {},
"ready_due": in_windows(self.ready_cfg.get("due", []), now), "ready_done": self.ready_done(now)}
for t in all_tiles(self.cfg):
for chk in t.get("checks", []):
if chk["type"] == "dante":
ctx["dante_all"].setdefault(chk["conn"], set()).update(chk["devices"])
for t in self.cfg["tiles"]:
for t in all_tiles(self.cfg):
for chk in t.get("checks", []):
if chk["type"] == "hyperdeck" and chk.get("rec_sync", True):
if conn_result(snap, chk["conn"], now).level == OK and hyperdeck_recording(snap, chk["conn"]):
ctx["any_recording"] = True
self.any_recording = ctx["any_recording"]
raw = {}
for t in self.cfg["tiles"]:
for t in all_tiles(self.cfg):
for i, chk in enumerate(t.get("checks", [])):
try:
results = CHECKS[chk["type"]](chk, snap, ctx)
@@ -1087,6 +1134,8 @@ class Engine:
tile = self.tiles.get(tile_id, {})
if tile.get("kind") == "master":
ids = self.detail_ids
elif tile.get("kind") == "ready":
ids = self.ready_ids
elif "members" in tile:
ids = tile["members"]
else:
@@ -1117,7 +1166,7 @@ class Engine:
Nothing is done for a healthy button, and a recording HyperDeck is never touched.
"""
tile = self.tiles.get(tile_id, {})
ids = tile.get("members") or [tile_id]
ids = self.ready_ids if tile.get("kind") == "ready" else (tile.get("members") or [tile_id])
plan, seen = [], set()
for key in sorted(self.meta):
tid, chk, sub = self.meta[key]
@@ -1212,6 +1261,31 @@ class Engine:
return True
return False
def ready_done(self, now):
"""{checklist item: when it was tapped done} for this week (the week starts on "reset_day")."""
import readiness
since = readiness.week_start(now, self.ready_cfg.get("reset_day", "mon"))
return {k: v for k, v in (self.state.get("ready_done") or {}).items() if v >= since}
def toggle_done(self, key_id, now):
"""A tap on a READY page key that only a person can check: done for the week, or (tapped
again) not done after all."""
chk = next((c for c in self.tiles.get(key_id, {}).get("checks", []) if c["type"] == "ready_manual"), None)
if chk is None:
log.warning("DONE %s: not a tap-to-confirm key", key_id)
return None
done = self.ready_done(now)
if chk["item"] in done:
del done[chk["item"]]
log.info("DONE %s: taken back", key_id)
self.note(key_id, "NOT DONE", "idle", now, 2)
else:
done[chk["item"]] = now
log.info("DONE %s: %s", key_id, self.tiles[key_id]["label"])
self.note(key_id, "DONE", "ok", now, 2)
self.state.set("ready_done", done)
return chk["item"] in done
def note(self, tile_id, text, level, now, seconds):
"""Show `text` on the button for a while (hold feedback), instead of its usual text."""
self.notes[tile_id] = {"text": text, "level": level, "until": now + seconds}
@@ -1345,7 +1419,11 @@ class Engine:
if not a or now - self.auto_last < 5 or self.stopping:
return []
self.auto_last = now
if not in_windows(a.get("windows", []), now) or not self.service.on(now):
# ALARMS muted (maintenance) stops auto-fix, except the harmless fixes in "while_muted"
# (playing the Spotify playlist): those keep running inside the window.
muted = not self.service.on(now)
keep = set(a.get("while_muted", [])) if muted else None
if not in_windows(a.get("windows", []), now) or (muted and not keep):
if self.auto_on and self.autolog:
self.autolog.write(now, "PANEL", "auto-fix off (%s)" % ("ALARMS muted" if in_windows(
a.get("windows", []), now) else "end of the window"))
@@ -1353,10 +1431,11 @@ class Engine:
self.auto_tries.clear()
self.auto_seen.clear()
return []
if not self.auto_on and self.autolog: # opens today's file, so it can be watched from the start
if (not self.auto_on or self.auto_muted != muted) and self.autolog: # opens today's file, so it can be watched from the start
ends = [w["end"] for w in a.get("windows", []) if in_windows([w], now)]
self.autolog.write(now, "PANEL", "auto-fix on%s" % (" until " + ends[0] if ends else ""))
self.auto_on = True
self.autolog.write(now, "PANEL", "auto-fix on%s%s" % (" until " + ends[0] if ends else "",
" (ALARMS muted: only %s)" % ", ".join(sorted(keep)) if muted else ""))
self.auto_on, self.auto_muted = True, muted
net_down = any(self.trackers.get("net:" + k) and self.trackers["net:" + k].confirmed
and self.trackers["net:" + k].result.level == ERR for k in ("link", "gateway"))
if net_down or (self.burst and (self.burst["cleared"] is None
@@ -1390,6 +1469,8 @@ class Engine:
for ident, kind, what, chk in self.fix_plan(t["id"], snap, now, a.get("after_seconds", 20), True):
if not ((kind == "fix" and what in gentle) or (kind == "restart" and a.get("restart_connections"))):
continue # not gentle: stays a hold
if keep is not None and not (kind == "fix" and what in keep):
continue # ALARMS muted: only the "while_muted" fixes
self.auto_seen[ident] = now
tries, last = self.auto_tries.get(ident, (0, 0.0))
if tries >= a.get("max_tries", 2):
@@ -1607,7 +1688,7 @@ def render(engine, service_on):
out[kid] = colored(key["label"] + "\nHOLD TO FIX", "err")
else:
out[kid] = colored(key["label"] + "\n" + engine.hold_text, "idle")
for tile in engine.cfg["tiles"]:
for tile in all_tiles(engine.cfg):
tid, kind = tile["id"], tile.get("kind")
note = engine.notes.get(tid)
if note and engine.now < note["until"]: # a hold is being acted on: say so
@@ -1631,6 +1712,21 @@ def render(engine, service_on):
text = ("%d DOWN" % n_bad if n_bad else "%d WARN" % len(faults)) + "\n" + first
out[tid] = colored(text, level)
continue
if kind == "ready":
# The checklist: amber while things are still to do, red once it is due and they are not.
items = [(lv, res, since, engine.tiles[m]) for m in engine.ready_ids
for lv, res, since in engine.items.get(m, [])]
level, faults = worst([(lv, res, since) for lv, res, since, _ in items])
if faults:
first = next(t for lv, r, s, t in items if (lv, r, s) == faults[0])
todo = len({t["id"] for lv, r, s, t in items if lv not in ("ok", "off")}) # keys, not faults
text = "%s\n%d TO DO\n%s" % (tile["label"], todo, first["label"]) # the key to look at first
elif level == "ok":
text = tile["label"] + "\nALL DONE"
else:
text = tile["label"] + "\nCHECKING"
out[tid] = colored(text, level)
continue
if "members" in tile:
items = [(lv, res, since, engine.tiles[m]) for m in tile["members"]
for lv, res, since in engine.items.get(m, [])]
@@ -1706,7 +1802,7 @@ def poll_pings(snap, hosts, interface):
def poll_system(snap, cfg):
checks = [c for t in cfg["tiles"] for c in t.get("checks", [])]
checks = [c for t in all_tiles(cfg) for c in t.get("checks", [])]
if any(c["type"] == "launchd" for c in checks):
out = subprocess.run(["/bin/launchctl", "list"], capture_output=True, text=True, timeout=5).stdout
agents = {}
@@ -1841,7 +1937,7 @@ def companion_conn_hosts(path):
def poll_conn_hosts(snap, cfg):
if any(c["type"] == "ping" and c.get("conn") for t in cfg["tiles"] for c in t.get("checks", [])):
if any(c["type"] == "ping" and c.get("conn") for t in all_tiles(cfg) for c in t.get("checks", [])):
try:
snap.conn_hosts = companion_conn_hosts(companion_db_path(cfg)) or snap.conn_hosts
except Exception as e: # Companion's schema is its own business: fall back to "host"
@@ -1880,7 +1976,7 @@ def spotify_playlist(cfg, chk=None):
def poll_spotify(snap, cfg):
checks = [c for t in cfg["tiles"] for c in t.get("checks", []) if c["type"] == "spotify"]
checks = [c for t in all_tiles(cfg) for c in t.get("checks", []) if c["type"] == "spotify"]
if not checks:
return
def refresh():
@@ -1912,7 +2008,7 @@ def poll_midi(snap, cfg):
import coremidi
snap.midi, snap.midi_time = coremidi.sessions(), time.time()
remember_peers(snap, "", snap.midi)
for t in cfg["tiles"]:
for t in all_tiles(cfg):
for c in t.get("checks", []):
if c["type"] == "midi_rx":
lis = snap.listeners.get(c["source"])
@@ -1924,7 +2020,7 @@ def poll_midi(snap, cfg):
# --- Other machines: SSH checks and fixes ------------------------------------
def remote_midi_hosts(cfg):
return sorted({c["host"] for t in cfg["tiles"] for c in t.get("checks", [])
return sorted({c["host"] for t in all_tiles(cfg) for c in t.get("checks", [])
if c["type"] == "midi_peer" and c.get("host")})
@@ -2055,7 +2151,7 @@ def parse_remote(out, n):
def poll_remote(snap, cfg):
by_host = {}
for t in cfg["tiles"]:
for t in all_tiles(cfg):
for c in t.get("checks", []):
if c["type"] == "remote":
by_host.setdefault(c["host"], [])
@@ -2098,7 +2194,7 @@ def arp_table():
def dante_label(cfg):
for t in cfg["tiles"]:
for t in all_tiles(cfg):
for c in t.get("checks", []):
if c["type"] == "dante":
return c["conn"]
@@ -2221,6 +2317,10 @@ def execute_fix(cfg, fix_id, chk):
netgear.poe_cycle(fix["switch"], fix["port"], fix.get("mac"))
log.info("FIX %s -> switch %s port %s power-cycled", label, fix["switch"], fix["port"])
return True
if kind == "ready":
# The READY page: re-sync the lineup, put the cross back up, copy tracks out, or look again.
import readiness
return readiness.run_action(cfg, fix["action"], chk)
if kind == "hyperdeck":
# Turn Companion's connection to the deck off and on: after a deck restarts, that is what
# brings it back green (a plain reconnect does not, 2026-10-11). The deck is never rebooted.
@@ -2551,18 +2651,20 @@ def load_config(path):
check_host(cfg, fix.get("host"), "fix " + fid)
if kind in ("ssh", "local") and not fix.get("run"):
raise ValueError("fix %s: nothing to run" % fid)
if kind == "ready" and fix.get("action") not in READY_ACTIONS:
raise ValueError("fix %s: unknown READY action %s" % (fid, fix.get("action")))
check_windows(cfg.get("service_windows"), "service_windows")
check_windows((cfg.get("auto_fix") or {}).get("windows"), "auto_fix")
check_windows((cfg.get("site") or {}).get("live_services"), "site.live_services")
ids = set()
for t in cfg["tiles"]:
for t in all_tiles(cfg):
if t["id"] in ids:
raise ValueError("duplicate tile id %s" % t["id"])
ids.add(t["id"])
for c in t.get("checks", []):
if c["type"] not in CHECKS:
raise ValueError("tile %s: unknown check type %s" % (t["id"], c["type"]))
for t in cfg["tiles"]:
for t in all_tiles(cfg):
for m in t.get("members", []):
if m not in ids:
raise ValueError("tile %s: unknown member %s" % (t["id"], m))
@@ -2588,17 +2690,34 @@ def load_config(path):
for k in cfg.get("restart_page", {}).get("keys", []):
if k["id"] in ids:
raise ValueError("restart key %s has a tile's id" % k["id"])
if k["id"] in {r["id"] for r in ready_keys(cfg)}:
raise ValueError("restart key %s has a READY key's id" % k["id"])
if k["fix"] not in cfg.get("fixes", {}):
raise ValueError("restart key %s: unknown fix %s" % (k["id"], k["fix"]))
for w in k.get("watch", []):
if not w.startswith("conn:") and w not in cfg.get("fixes", {}):
raise ValueError("restart key %s: unknown fix %s in watch" % (k["id"], w))
rp = cfg.get("ready_page")
if rp:
check_windows((cfg.get("ready") or {}).get("due"), "ready.due")
if rp.get("page") in (cfg.get("page"), (cfg.get("restart_page") or {}).get("page")):
raise ValueError("ready_page: page %s is already the Status or Restart page" % rp.get("page"))
for rk in ready_keys(cfg):
if "pos" not in rk:
raise ValueError("READY key %s has no place on the READY page" % rk["id"])
for c in rk.get("checks", []):
if c["type"] in ("ready", "ready_manual") and not c.get("item"):
raise ValueError("READY key %s: a %s check needs an item" % (rk["id"], c["type"]))
if c["type"] == "ready" and c["item"] not in READY_ITEMS:
raise ValueError("READY key %s: unknown item %s" % (rk["id"], c["item"]))
if sum(1 for t in cfg["tiles"] if t.get("kind") == "ready") > 1:
raise ValueError("at most one READY key on the Status page")
return cfg
def ping_hosts(cfg, snap=None):
hosts = [ping_target(c, snap) if snap else c.get("host")
for t in cfg["tiles"] for c in t.get("checks", []) if c["type"] == "ping"]
for t in all_tiles(cfg) for c in t.get("checks", []) if c["type"] == "ping"]
hosts = [h for h in hosts if h]
gw = cfg.get("network", {}).get("gateway")
return list(dict.fromkeys(hosts + ([gw] if gw else [])))
@@ -2607,7 +2726,7 @@ def ping_hosts(cfg, snap=None):
def http_checks(cfg):
"""[(url, interval)]: the Resi cloud every 30 s, a LAN app as often as its check asks."""
found = {}
for t in cfg["tiles"]:
for t in all_tiles(cfg):
for c in t.get("checks", []):
if c["type"] == "http":
found[c["url"]] = min(found.get(c["url"], 999), c.get("interval_seconds", 30))
@@ -2639,7 +2758,7 @@ def start_pollers(cfg, snap, stop, data_dir=None):
for t in threads:
t.start()
threads += start_remote_midi_streams(snap, cfg, stop)
if cfg.get("atem") and any(c["type"] == "atem" for t in cfg["tiles"] for c in t.get("checks", [])):
if cfg.get("atem") and any(c["type"] == "atem" for t in all_tiles(cfg) for c in t.get("checks", [])):
global ATEM_WATCHER
try:
ATEM_WATCHER = AtemWatcher(cfg, snap, data_dir, stop)
@@ -2648,7 +2767,7 @@ def start_pollers(cfg, snap, stop, data_dir=None):
threads.append(t)
except (OSError, ValueError) as e:
log.warning("ATEM checks off: %s", e)
if any(c["type"] == "artnet" for t in cfg["tiles"] for c in t.get("checks", [])):
if any(c["type"] == "artnet" for t in all_tiles(cfg) for c in t.get("checks", [])):
start_artnet_listener(snap, stop)
return threads
@@ -2690,6 +2809,7 @@ def snapshot_summary(snap, engine, now):
"burst": engine.burst and {"start": engine.burst["start"], "conns": sorted(engine.burst["conns"]),
"cleared": engine.burst["cleared"]},
"live": list(engine.live_now(now)),
"ready": snap.ready,
"faults": {},
}
for key, tr in engine.trackers.items():
@@ -2748,6 +2868,8 @@ def handle_commands(text, engine, comp, snap, now):
log.info("TEST %s: no time on it, or too long ago - ignored", arg)
else:
engine.run_test(arg, words[2], words[3], snap, comp, now)
elif verb == "done": # a tap on a READY page key a person checks
engine.toggle_done(arg, now)
elif verb == "resurvey": # the settings page: look at the Dante switches now
snap.dante_map_time = 0.0
log.info("RESURVEY: Dante map survey requested")
@@ -2775,6 +2897,11 @@ def run(cfg, data_dir, dry_run=False, stop=None, config_sha=None):
except ValueError: # not the main thread (tests)
pass
threads = start_pollers(cfg, snap, stop, data_dir)
if cfg.get("ready_page"):
import readiness
poller = readiness.Poller(cfg, snap, stop, data_dir, is_live=lambda: engine.live_now(time.time())[0])
poller.start()
threads.append(poller)
if not dry_run:
try:
# A hold pressed while the watchdog was down must not fire on a config it was not meant for.
@@ -2880,6 +3007,9 @@ def run_once(cfg):
poll_midi(snap, cfg)
except Exception as e:
print("midi: %s" % e)
if cfg.get("ready_page"):
import readiness
snap.ready = readiness.look(cfg)
engine = Engine(cfg, None)
tiles = engine.step(snap, time.time(), debounce=0)
rows = {}