Step 0 of the settings page: registry, safe restart, remembered state, test verb, page markers
- schema.py: every setting with type, default, help, owner screen and the rules a config must pass; tests/test_schema.py keeps it in step with the code (every key the code reads is declared, every default matches, check types and fix kinds agree, the glossary covers every word a key can show, today's config still builds today's pages apart from the key notes) - status_panel.py: VERSION + started.json; SIGTERM lets a running fix finish and starts nothing new (RESTARTING SOON); fixes-running.json and live-now.json for the settings page; state.json keeps the ALARMS override, acknowledgements, auto-fix tries and fix times across restarts, keyed by what a check is about rather than its position; 'test' and 'resurvey' commands; queued commands dropped at start; overnight windows; a HyperDeck switched off is always switched back on; the spotify fix kind (playlist set once, in config 'spotify'); Windows remote checks batched for cmd.exe with conditional echoes; every fix's kind and host validated; never_auto; wording and numbers from config (site block, timing, page_hold_ms, auto-fix keep_months); hyperdeck_probe removed; --check and --exit-timeout - make_page.py: numbers from config, markers on every key, nav templates per Companion version, --out; push_page.py: a page is overwritten only when every key carries a marker (foreign keys named by position), secret-free fetch, host from the site; nav_keys.py: host from the site; netgear.py: switches and the shared-port limit from config - install.sh: validates with the new code first, keeps prev/ for --rollback, --keep-config, derived ExitTimeOut, wider deploy list, waits for the start stamp - config.json: site, timing, spotify sections; Windows hosts marked; deployed to NA Ingest and both pages re-imported once (--force) so every key carries its marker Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5.1
+458
-109
@@ -16,6 +16,7 @@ Stdlib only; runs on the macOS /usr/bin/python3 (3.9).
|
||||
"""
|
||||
import argparse
|
||||
import calendar
|
||||
import hashlib
|
||||
import http.client
|
||||
import json
|
||||
import logging
|
||||
@@ -24,6 +25,7 @@ import logging.handlers
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
@@ -38,6 +40,10 @@ from datetime import datetime
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
log = logging.getLogger("status-panel")
|
||||
|
||||
VERSION = "2026.10.13" # logged at start ("status panel running (code ...)") and written to started.json
|
||||
# Every kind of fix the watchdog can run (validated by load_config).
|
||||
FIX_KINDS = ("ssh", "local", "atem", "reconnect", "midi_connect", "netgear_poe", "hyperdeck", "spotify")
|
||||
|
||||
OK, WARN, ERR, OFF = "ok", "warn", "err", "off"
|
||||
|
||||
# Display levels, least to most urgent. "idle" is a service-only fault while
|
||||
@@ -629,13 +635,14 @@ class AutoLog:
|
||||
|
||||
NAME = re.compile(r"^(\d{4})-(\d{2})-(\d{2}) [A-Za-z]+ Auto-Fix\.log$")
|
||||
|
||||
def __init__(self, directory):
|
||||
def __init__(self, directory, who="NA Ingest", when="on Sundays 06:30-13:30", keep_months=1):
|
||||
self.dir = os.path.expanduser(directory)
|
||||
self.lock = threading.Lock()
|
||||
self.who, self.when, self.keep_months = who, when, max(1, int(keep_months))
|
||||
|
||||
def cleanup(self, now):
|
||||
"""Delete day files once they are a calendar month old (the 11 Oct file goes on 11 Nov).
|
||||
Only files named exactly like these logs are ever touched."""
|
||||
"""Delete day files once they are keep_months calendar months old (the 11 Oct file goes on
|
||||
11 Nov). Only files named exactly like these logs are ever touched."""
|
||||
today = datetime.fromtimestamp(now).date()
|
||||
removed = []
|
||||
try:
|
||||
@@ -647,7 +654,8 @@ class AutoLog:
|
||||
if not m:
|
||||
continue
|
||||
y, mo, d = (int(x) for x in m.groups())
|
||||
y2, mo2 = (y + 1, 1) if mo == 12 else (y, mo + 1)
|
||||
total = mo - 1 + self.keep_months
|
||||
y2, mo2 = y + total // 12, total % 12 + 1
|
||||
try:
|
||||
due = datetime(y2, mo2, min(d, calendar.monthrange(y2, mo2)[1])).date()
|
||||
except ValueError:
|
||||
@@ -678,41 +686,111 @@ class AutoLog:
|
||||
with open(path, "a", encoding="utf-8") as f:
|
||||
if new:
|
||||
f.write("Status panel auto-fixes - %s\n"
|
||||
"(NA Ingest fixes these by itself on Sundays 06:30-13:30; anything that closes a\n"
|
||||
"(%s fixes these by itself %s; anything that closes a\n"
|
||||
" running program stays a hold. Newest lines at the bottom.)\n\n"
|
||||
% dt.strftime("%A %-d %B %Y"))
|
||||
% (dt.strftime("%A %-d %B %Y"), self.who, self.when))
|
||||
f.write(line)
|
||||
except OSError as e:
|
||||
log.warning("auto-fix log %s: %s", self.dir, e)
|
||||
|
||||
|
||||
def window_holds(w, day, hm, prev_day):
|
||||
"""One window at one moment. "end" <= "start" means the window runs past midnight: it starts on
|
||||
one of its days and ends early the next morning (a Saturday night service: sat 18:00-00:30)."""
|
||||
if w["start"] < w["end"]:
|
||||
return day in w["days"] and w["start"] <= hm < w["end"]
|
||||
return (day in w["days"] and hm >= w["start"]) or (prev_day in w["days"] and hm < w["end"])
|
||||
|
||||
|
||||
def in_windows(windows, now):
|
||||
"""Is `now` inside one of [{"days": [...], "start": "HH:MM", "end": "HH:MM"}]?"""
|
||||
dt = datetime.fromtimestamp(now)
|
||||
hm, day = dt.strftime("%H:%M"), DAYS[dt.weekday()]
|
||||
return any(day in w["days"] and w["start"] <= hm < w["end"] for w in windows)
|
||||
return any(window_holds(w, day, hm, DAYS[dt.weekday() - 1]) for w in windows)
|
||||
|
||||
|
||||
def windows_phrase(windows):
|
||||
"""Windows in words, for log headers and notes: "on Sundays 06:30-13:30"."""
|
||||
parts = []
|
||||
for w in windows:
|
||||
days = [d for d in DAYS if d in w["days"]]
|
||||
names = [datetime(2024, 1, 1 + DAYS.index(d)).strftime("%As") for d in days] # 2024-01-01 is a Monday
|
||||
when = "every day" if len(days) == 7 else " and ".join(names)
|
||||
parts.append("%s %s-%s" % (when, w["start"], w["end"]))
|
||||
return ("on " + " and ".join(parts)) if parts else "never"
|
||||
|
||||
|
||||
def check_identity(chk):
|
||||
"""What a check is about, independent of its position in the list: the key that remembered
|
||||
state (acknowledgements, auto-fix tries) is filed under, so editing the list keeps it."""
|
||||
t = chk["type"]
|
||||
if t == "midi_peer":
|
||||
return "%s/%s/%s" % (chk.get("host", ""), chk["session"], chk["peer"])
|
||||
if t == "remote":
|
||||
return "%s/%s" % (chk["host"], hashlib.sha1(chk["cmd"].encode()).hexdigest()[:8])
|
||||
if t == "ping":
|
||||
return chk.get("conn") or chk.get("host", "")
|
||||
if t == "pref":
|
||||
return "%s/%s" % (chk["domain"], chk["key"])
|
||||
single = {"http": "url", "mdns": "instance", "launchd": "label", "process": "match", "artnet": "source",
|
||||
"atem": "part", "disk": "path", "midi_rx": "source"}
|
||||
if t in single:
|
||||
return str(chk.get(single[t], ""))
|
||||
return chk.get("conn") or t
|
||||
|
||||
|
||||
def write_json(path, obj):
|
||||
"""Write a small JSON file in one go (a reader never sees half of it)."""
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(obj, f, indent=1, sort_keys=True)
|
||||
os.replace(tmp, path)
|
||||
|
||||
|
||||
class State:
|
||||
"""One file (state.json) for everything the watchdog remembers across restarts: the ALARMS
|
||||
override, acknowledged faults, auto-fix tries and when each fix last ran. One writer; saved whole."""
|
||||
|
||||
def __init__(self, path):
|
||||
self.path, self.data, self.lock = path, {}, threading.Lock()
|
||||
if not path:
|
||||
return
|
||||
try:
|
||||
with open(path) as f:
|
||||
self.data = json.load(f)
|
||||
except (OSError, ValueError):
|
||||
legacy = os.path.join(os.path.dirname(path), "service-mode.json") # the file before state.json
|
||||
try:
|
||||
with open(legacy) as f:
|
||||
old = json.load(f)
|
||||
self.data = {"service": {"override": old.get("override"), "last_sched": old.get("last_sched")}}
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
|
||||
def get(self, key, default=None):
|
||||
return self.data.get(key, default)
|
||||
|
||||
def set(self, key, value):
|
||||
with self.lock:
|
||||
self.data[key] = value
|
||||
if self.path:
|
||||
try:
|
||||
write_json(self.path, self.data)
|
||||
except OSError as e:
|
||||
log.warning("could not save %s: %s", self.path, e)
|
||||
|
||||
|
||||
class ServiceMode:
|
||||
"""On inside a scheduled window; a button press overrides until the next window edge."""
|
||||
|
||||
def __init__(self, windows, path):
|
||||
self.windows, self.path = windows, path
|
||||
self.override, self.last_sched = None, None
|
||||
if not path:
|
||||
return
|
||||
try:
|
||||
with open(path) as f:
|
||||
saved = json.load(f)
|
||||
self.override, self.last_sched = saved.get("override"), saved.get("last_sched")
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
def __init__(self, windows, state):
|
||||
self.windows = windows
|
||||
self.state = state if isinstance(state, State) else State(state)
|
||||
saved = self.state.get("service") or {}
|
||||
self.override, self.last_sched = saved.get("override"), saved.get("last_sched")
|
||||
|
||||
def scheduled(self, now):
|
||||
dt = datetime.fromtimestamp(now)
|
||||
hm = dt.strftime("%H:%M")
|
||||
day = DAYS[dt.weekday()]
|
||||
return any(day in w["days"] and w["start"] <= hm < w["end"] for w in self.windows)
|
||||
return in_windows(self.windows, now)
|
||||
|
||||
def on(self, now):
|
||||
sched = self.scheduled(now)
|
||||
@@ -728,28 +806,33 @@ class ServiceMode:
|
||||
self._save()
|
||||
|
||||
def _save(self):
|
||||
if not self.path:
|
||||
return
|
||||
try:
|
||||
tmp = self.path + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump({"override": self.override, "last_sched": self.last_sched}, f)
|
||||
os.replace(tmp, self.path)
|
||||
except OSError as e:
|
||||
log.warning("could not save service mode: %s", e)
|
||||
self.state.set("service", {"override": self.override, "last_sched": self.last_sched})
|
||||
|
||||
|
||||
# --- Engine ------------------------------------------------------------------
|
||||
|
||||
class Engine:
|
||||
def __init__(self, cfg, state_path=None):
|
||||
def __init__(self, cfg, state_path=None, data_dir=None):
|
||||
self.cfg = cfg
|
||||
self.tiles = {t["id"]: t for t in cfg["tiles"]}
|
||||
self.debounce = cfg.get("debounce_seconds", 3)
|
||||
self.flap_count = cfg.get("flap", {}).get("count", 3)
|
||||
self.flap_window = cfg.get("flap", {}).get("window_seconds", 600)
|
||||
self.net = cfg.get("network", {})
|
||||
self.service = ServiceMode(cfg.get("service_windows", []), state_path)
|
||||
self.site = cfg.get("site", {})
|
||||
timing = cfg.get("timing", {})
|
||||
self.cooldown = timing.get("fix_cooldown_seconds", 60) # the same fix never re-runs within this
|
||||
self.hold_ms = cfg.get("page_hold_ms", 2000) # how long a key is held to run its fix
|
||||
self.state = State(state_path)
|
||||
self.service = ServiceMode(cfg.get("service_windows", []), self.state)
|
||||
self.data_dir = data_dir
|
||||
self.stopping = False # SIGTERM arrived: finish what runs, start nothing new
|
||||
self.running = {} # fix thread name -> {"key", "what", "since"} (published as fixes-running.json)
|
||||
self.fix_threads = [] # the threads behind self.running, joined at exit
|
||||
self.any_recording = False
|
||||
self.live = None # (live?, reason) last written to live-now.json
|
||||
self.live_written = 0.0
|
||||
self.restore_acks = set(self.state.get("acks") or []) # acknowledged before the last restart
|
||||
self.trackers = {}
|
||||
self.group_of = {}
|
||||
for t in cfg["tiles"]:
|
||||
@@ -763,16 +846,19 @@ class Engine:
|
||||
self.burst = None # {"start", "conns", "cleared"}
|
||||
self.events = []
|
||||
self.meta = {} # key -> (tile id, check, sub key)
|
||||
self.fix_last = {} # fix identity -> last run time
|
||||
self.fix_last = {tuple(k.split("\0")): v for k, v in (self.state.get("fix_last") or {}).items()}
|
||||
self.executor = execute_fix
|
||||
self.poe_executor = execute_poe
|
||||
self.notes = {} # tile id -> hold feedback {"text", "level", "until"}
|
||||
# The Restart page: one key per thing, hold 2 s = restart it even when it looks fine.
|
||||
self.restart_keys = {k["id"]: k for k in cfg.get("restart_page", {}).get("keys", [])}
|
||||
self.auto = cfg.get("auto_fix") or {}
|
||||
self.auto_tries = {} # fix identity -> [tries, last try time] for the fault now going on
|
||||
self.auto_seen = {} # fix identity -> last time it was in an auto plan
|
||||
self.autolog = AutoLog(self.auto.get("log_dir", "~/Desktop/Auto-Fix Logs")) if self.auto else None
|
||||
saved = self.state.get("auto") or {}
|
||||
self.auto_tries = {tuple(k.split("\0")): v for k, v in saved.get("tries", {}).items()}
|
||||
self.auto_seen = {tuple(k.split("\0")): v for k, v in saved.get("seen", {}).items()}
|
||||
self.autolog = AutoLog(self.auto.get("log_dir", "~/Desktop/Auto-Fix Logs"),
|
||||
self.site.get("ingest_name", "NA Ingest"), windows_phrase(self.auto.get("windows", [])),
|
||||
self.auto.get("keep_months", 1)) if self.auto else None
|
||||
if self.autolog:
|
||||
self.autolog.cleanup(time.time())
|
||||
self.auto_watch = {} # "BUTTON FAULT" auto-fix went after -> when (to log when it clears)
|
||||
@@ -782,6 +868,81 @@ class Engine:
|
||||
self.deck_ips = {} # HyperDeck connection -> last address Companion reported
|
||||
self.now = 0.0
|
||||
|
||||
@property
|
||||
def hold_text(self):
|
||||
return "HOLD %ss" % ("%g" % (self.hold_ms / 1000.0))
|
||||
|
||||
@property
|
||||
def wait_text(self):
|
||||
c = int(self.cooldown)
|
||||
return "WAIT %d MIN" % (c // 60) if c % 60 == 0 else "WAIT %d S" % c
|
||||
|
||||
def identity_key(self, key):
|
||||
"""A tracker key ("tile:index:sub") as the position-free name its state is remembered by."""
|
||||
tile_id, chk, sub = self.meta[key]
|
||||
return "%s|%s|%s" % (tile_id, check_identity(chk), sub)
|
||||
|
||||
def remember(self):
|
||||
"""Save what must survive a restart: acknowledgements, auto-fix tries, when fixes last ran."""
|
||||
acks = sorted({self.identity_key(k) for k, tr in self.trackers.items()
|
||||
if tr.confirmed and tr.acked and k in self.meta} | self.restore_acks)
|
||||
self.state.set("acks", acks)
|
||||
self.state.set("fix_last", {"\0".join(k): v for k, v in self.fix_last.items()})
|
||||
self.state.set("auto", {"tries": {"\0".join(k): v for k, v in self.auto_tries.items()},
|
||||
"seen": {"\0".join(k): v for k, v in self.auto_seen.items()}})
|
||||
|
||||
def live_now(self, now):
|
||||
"""(live?, why): changes to the panel wait while this is true. A recording HyperDeck is
|
||||
always live; a campus may add clock windows ("live_services" in "site")."""
|
||||
if self.any_recording:
|
||||
return True, "a HyperDeck is recording"
|
||||
if in_windows(self.site.get("live_services", []), now):
|
||||
return True, "a live service is scheduled"
|
||||
return False, ""
|
||||
|
||||
def publish(self, now):
|
||||
"""live-now.json and fixes-running.json in the data dir (the settings page reads them)."""
|
||||
if not self.data_dir:
|
||||
return
|
||||
live = self.live_now(now)
|
||||
if live != self.live or now - self.live_written > 5:
|
||||
try:
|
||||
write_json(os.path.join(self.data_dir, "live-now.json"),
|
||||
{"live": live[0], "reason": live[1], "time": now})
|
||||
except OSError as e:
|
||||
log.warning("live-now.json: %s", e)
|
||||
self.live, self.live_written = live, now
|
||||
|
||||
def fix_started(self, name, key_id, what):
|
||||
self.running[name] = {"key": key_id, "what": what, "since": time.time()}
|
||||
self.publish_running()
|
||||
|
||||
def fix_done(self, name):
|
||||
self.running.pop(name, None)
|
||||
self.publish_running()
|
||||
|
||||
def publish_running(self):
|
||||
if self.data_dir:
|
||||
try:
|
||||
write_json(os.path.join(self.data_dir, "fixes-running.json"), self.running)
|
||||
except OSError as e:
|
||||
log.warning("fixes-running.json: %s", e)
|
||||
|
||||
def start_fix_thread(self, target, key_id, what):
|
||||
"""Every fix runs in a thread the watchdog knows about: a restart waits for them."""
|
||||
name = "fix-%s-%d" % (key_id, int(time.time() * 1000) % 100000)
|
||||
|
||||
def wrapped():
|
||||
try:
|
||||
target()
|
||||
finally:
|
||||
self.fix_done(name)
|
||||
self.fix_started(name, key_id, what)
|
||||
t = threading.Thread(target=wrapped, name=name, daemon=True)
|
||||
self.fix_threads = [x for x in self.fix_threads if x.is_alive()] + [t]
|
||||
t.start()
|
||||
return t
|
||||
|
||||
# raw results for every check key
|
||||
def evaluate(self, snap, now, service_on):
|
||||
ctx = {"now": now, "service": service_on, "any_recording": False, "dante_all": {}}
|
||||
@@ -794,6 +955,7 @@ class Engine:
|
||||
if chk["type"] == "hyperdeck" and chk.get("rec_sync", True):
|
||||
if conn_result(snap, chk["conn"], now).level == OK and hyperdeck_recording(snap, chk["conn"]):
|
||||
ctx["any_recording"] = True
|
||||
self.any_recording = ctx["any_recording"]
|
||||
raw = {}
|
||||
for t in self.cfg["tiles"]:
|
||||
for i, chk in enumerate(t.get("checks", [])):
|
||||
@@ -878,13 +1040,23 @@ class Engine:
|
||||
deb = self.debounce if debounce is None else debounce
|
||||
for key, (tile_id, res, chk) in raw.items():
|
||||
tr = self.trackers.setdefault(key, Tracker())
|
||||
was_acked = tr.acked
|
||||
# A MIDI peer leaving a session is a clean event: show it after 1 s, not the general 3 s.
|
||||
event = tr.update(res, now, 0 if chk["type"] == "network" else
|
||||
chk.get("debounce_seconds", 1 if chk["type"] == "midi_peer" else deb),
|
||||
self.flap_window)
|
||||
if event == "fault":
|
||||
self.events.append(("FAULT", "%s %s" % (self.tiles[tile_id]["label"], res.msg)))
|
||||
ident = self.identity_key(key) if key in self.meta else None
|
||||
if ident in self.restore_acks: # acknowledged before the restart: still acknowledged
|
||||
self.restore_acks.discard(ident)
|
||||
tr.acked = True
|
||||
self.remember()
|
||||
elif event == "clear":
|
||||
ident = self.identity_key(key) if key in self.meta else None
|
||||
if was_acked or ident in self.restore_acks:
|
||||
self.restore_acks.discard(ident)
|
||||
self.remember() # an acknowledgement is for one fault only
|
||||
text = "%s %s" % (self.tiles[tile_id]["label"], tr.last_fault_msg)
|
||||
self.events.append(("CLEAR", text))
|
||||
fixed_at = self.auto_watch.pop(text, None)
|
||||
@@ -896,15 +1068,19 @@ class Engine:
|
||||
# fault is over - clear it, or it lingers unseen and a hold would still try to fix it.
|
||||
for key, tr in self.trackers.items():
|
||||
if key not in raw and (tr.confirmed or tr.bad_since is not None):
|
||||
was_acked = tr.acked
|
||||
if tr.update(Result(OK), now, 0, self.flap_window) == "clear" and key in self.meta:
|
||||
tile_id = self.meta[key][0]
|
||||
self.events.append(("CLEAR", "%s %s" % (self.tiles[tile_id]["label"], tr.last_fault_msg)))
|
||||
if was_acked:
|
||||
self.remember()
|
||||
self.items = {}
|
||||
for key, (tile_id, res, chk) in raw.items():
|
||||
tr = self.trackers[key]
|
||||
eff = tr.effective(self.flap_count)
|
||||
self.items.setdefault(tile_id, []).append(
|
||||
(display_level(eff, tr.acked, service_on), eff, tr.since or now))
|
||||
self.publish(now)
|
||||
return render(self, service_on)
|
||||
|
||||
def keys_for(self, tile_id):
|
||||
@@ -925,6 +1101,8 @@ class Engine:
|
||||
if tr.confirmed and not tr.acked:
|
||||
tr.acked = True
|
||||
n += 1
|
||||
if n:
|
||||
self.remember()
|
||||
return n
|
||||
|
||||
def restart_targets(self, tile_id):
|
||||
@@ -1038,7 +1216,11 @@ class Engine:
|
||||
"""Show `text` on the button for a while (hold feedback), instead of its usual text."""
|
||||
self.notes[tile_id] = {"text": text, "level": level, "until": now + seconds}
|
||||
|
||||
def run_fixes(self, tile_id, snap, comp, now, cooldown=60):
|
||||
def run_fixes(self, tile_id, snap, comp, now, cooldown=None):
|
||||
cooldown = self.cooldown if cooldown is None else cooldown
|
||||
if self.stopping:
|
||||
self.note(tile_id, "RESTARTING\nSOON", "idle", now, 4)
|
||||
return []
|
||||
if self.tiles.get(tile_id, {}).get("kind") == "network":
|
||||
# NET: a hold clears a remembered network blip (link and gateway clear themselves).
|
||||
if self.burst is None:
|
||||
@@ -1071,8 +1253,9 @@ class Engine:
|
||||
return plan
|
||||
return self.run_plan(tile_id, plan, snap, comp, now, cooldown)
|
||||
|
||||
def run_plan(self, tile_id, plan, snap, comp, now, cooldown=60, auto=False, on_done=None):
|
||||
def run_plan(self, tile_id, plan, snap, comp, now, cooldown=None, auto=False, on_done=None):
|
||||
"""Carry out fix_plan() items for one button (a hold, or auto-fix during service)."""
|
||||
cooldown = self.cooldown if cooldown is None else cooldown
|
||||
started, names, waited, midi = [], [], False, []
|
||||
for ident, kind, what, chk in plan:
|
||||
if now - self.fix_last.get(ident, 0) < cooldown:
|
||||
@@ -1080,6 +1263,7 @@ class Engine:
|
||||
waited = True
|
||||
continue
|
||||
self.fix_last[ident] = now
|
||||
self.remember()
|
||||
if kind == "enable":
|
||||
c = snap.conns.get(what)
|
||||
if c:
|
||||
@@ -1112,7 +1296,7 @@ class Engine:
|
||||
started.append((midi_worker, (midi,), self.cfg["fixes"][midi[0][0]].get("short", "MIDI")))
|
||||
if not names:
|
||||
if not auto:
|
||||
self.note(tile_id, "JUST TRIED\nWAIT 1 MIN" if waited else "NOTHING\nTO FIX", "idle", now, 4)
|
||||
self.note(tile_id, ("JUST TRIED\n" + self.wait_text) if waited else "NOTHING\nTO FIX", "idle", now, 4)
|
||||
return plan
|
||||
summary = names[0] if len(set(names)) == 1 else "%d FIXES" % len(names)
|
||||
if not started: # only Companion reconnects: quick
|
||||
@@ -1148,7 +1332,7 @@ class Engine:
|
||||
on_done(outcome)
|
||||
said = []
|
||||
for fn, args, short in started:
|
||||
threading.Thread(target=work, args=(fn, args, short), daemon=True).start()
|
||||
self.start_fix_thread(lambda fn=fn, args=args, short=short: work(fn, args, short), tile_id, short)
|
||||
return plan
|
||||
|
||||
def auto_fix(self, snap, comp, now):
|
||||
@@ -1158,7 +1342,7 @@ class Engine:
|
||||
the network is having a blip, and never while ALARMS has muted the panel (maintenance).
|
||||
Restarting a running program stays a hold."""
|
||||
a = self.auto
|
||||
if not a or now - self.auto_last < 5:
|
||||
if not a or now - self.auto_last < 5 or self.stopping:
|
||||
return []
|
||||
self.auto_last = now
|
||||
if not in_windows(a.get("windows", []), now) or not self.service.on(now):
|
||||
@@ -1184,7 +1368,7 @@ class Engine:
|
||||
if self.auto_paused and self.autolog:
|
||||
self.autolog.write(now, "NET", "network settled: auto-fix is back on")
|
||||
self.auto_paused = False
|
||||
gentle = set(a.get("fixes", []))
|
||||
gentle = {f for f in a.get("fixes", []) if not self.cfg["fixes"].get(f, {}).get("never_auto")}
|
||||
done = []
|
||||
for t in self.cfg["tiles"]:
|
||||
if "checks" not in t:
|
||||
@@ -1243,6 +1427,8 @@ class Engine:
|
||||
# fault blink out for a moment (restarting, re-confirming), which must not reset the count.
|
||||
if now - self.auto_seen.get(ident, 0) >= forget:
|
||||
del self.auto_tries[ident]
|
||||
if done or self.auto_tries:
|
||||
self.remember()
|
||||
return done
|
||||
|
||||
def describe(self, kind, what):
|
||||
@@ -1253,9 +1439,13 @@ class Engine:
|
||||
return self.cfg["fixes"][what].get("label", what)
|
||||
return "%s %s" % (kind, what)
|
||||
|
||||
def run_force(self, key_id, snap, comp, now, cooldown=60, fix_id=None, chk=None):
|
||||
def run_force(self, key_id, snap, comp, now, cooldown=None, fix_id=None, chk=None, verb="FORCE"):
|
||||
"""Run one fix whatever the status says: a Restart page key, or a button's "hold_always".
|
||||
Shares the 60 s guard with holds, so a hold and a restart key never double up."""
|
||||
Shares the cooldown guard with holds, so a hold and a restart key never double up."""
|
||||
cooldown = self.cooldown if cooldown is None else cooldown
|
||||
if self.stopping:
|
||||
self.note(key_id, "RESTARTING\nSOON", "idle", now, 4)
|
||||
return None
|
||||
if fix_id is None:
|
||||
key = self.restart_keys.get(key_id)
|
||||
if not key:
|
||||
@@ -1266,19 +1456,20 @@ class Engine:
|
||||
and now - self.fix_last.get(("fix", w, ""), 0) < cooldown]
|
||||
if busy: # e.g. auto-fix is starting Ableton right now
|
||||
log.info("FORCE %s: %s ran less than %ds ago, skipped", key_id, busy[0], cooldown)
|
||||
self.note(key_id, "STARTING\nWAIT 1 MIN", "idle", now, 4)
|
||||
self.note(key_id, "STARTING\n" + self.wait_text, "idle", now, 4)
|
||||
return None
|
||||
per_deck = self.cfg["fixes"][fix_id].get("kind") == "hyperdeck" and chk and chk.get("conn")
|
||||
ident = ("fix", fix_id, chk["conn"] if per_deck else "") # the same guard a failing hold uses
|
||||
if now - self.fix_last.get(ident, 0) < cooldown:
|
||||
log.info("FORCE %s: %s ran less than %ds ago, skipped", key_id, fix_id, cooldown)
|
||||
log.info("%s %s: %s ran less than %ds ago, skipped", verb, key_id, fix_id, cooldown)
|
||||
cur = self.notes.get(key_id)
|
||||
if not (cur and cur["level"] == "fixing" and now < cur["until"]): # still running: keep saying so
|
||||
self.note(key_id, "JUST DONE\nWAIT 1 MIN", "idle", now, 4)
|
||||
self.note(key_id, "JUST DONE\n" + self.wait_text, "idle", now, 4)
|
||||
return None
|
||||
self.fix_last[ident] = now
|
||||
self.remember()
|
||||
fix = self.cfg["fixes"][fix_id]
|
||||
log.info("FORCE %s: %s", key_id, fix.get("label", fix_id))
|
||||
log.info("%s %s: %s", verb, key_id, fix.get("label", fix_id))
|
||||
self.note(key_id, "RESTARTING", "fixing", now, 180)
|
||||
|
||||
def work():
|
||||
@@ -1292,9 +1483,51 @@ class Engine:
|
||||
self.note(key_id, "FAILED", "failed", done, 30)
|
||||
else:
|
||||
self.note(key_id, "DONE", "ok", done, 20)
|
||||
threading.Thread(target=work, daemon=True).start()
|
||||
self.start_fix_thread(work, key_id, fix.get("short", fix_id.upper()))
|
||||
return fix_id
|
||||
|
||||
def run_test(self, key_id, fix_id, which, snap, comp, now):
|
||||
"""The settings page's "Run this fix now": one check's fix, run exactly as a hold on that key
|
||||
would run it (deck address, MIDI peer address, recording guard, cooldown), logged as TEST.
|
||||
`which` picks the check: its number on the key, or its identity. Refused for anything a hold
|
||||
would not do from that key, and for fixes that act on the switcher, a PoE port or Dante."""
|
||||
tile = self.tiles.get(key_id)
|
||||
if not tile:
|
||||
log.warning("TEST %s: no such key", key_id)
|
||||
return None
|
||||
fix = self.cfg["fixes"].get(fix_id)
|
||||
if not fix or fix.get("kind") in ("atem", "netgear_poe") or fix_id in (RESTART_CONNECTION, DANTE_AUTO):
|
||||
log.warning("TEST %s: %s is not a fix the page may run", key_id, fix_id)
|
||||
return None
|
||||
checks = tile.get("checks", [])
|
||||
chk = None
|
||||
for i, c in enumerate(checks):
|
||||
if which in (str(i), check_identity(c), urllib.parse.quote(check_identity(c), safe="")):
|
||||
chk = c
|
||||
if chk is None and tile.get("hold_always") == fix_id:
|
||||
chk = next((c for c in checks if c.get("conn")), {})
|
||||
if chk is None:
|
||||
log.warning("TEST %s: no check %r on it", key_id, which)
|
||||
return None
|
||||
own = chk.get("fix")
|
||||
own = set(own.values()) if isinstance(own, dict) else {own}
|
||||
if fix_id not in own and tile.get("hold_always") != fix_id:
|
||||
log.warning("TEST %s: %s is not attached to that check", key_id, fix_id)
|
||||
return None
|
||||
if chk.get("type") == "hyperdeck":
|
||||
if hyperdeck_recording(snap, chk["conn"]):
|
||||
log.info("TEST %s: %s is recording, left alone", key_id, chk["conn"])
|
||||
self.note(key_id, "RECORDING\nLEFT ALONE", "idle", now, 4)
|
||||
return None
|
||||
chk = dict(chk, ip=chk.get("ip") or snap.var(chk["conn"], "ip") or self.deck_ips.get(chk["conn"]))
|
||||
if fix.get("kind") == "midi_connect":
|
||||
address = snap.peer_addr.get((chk.get("host", ""), chk["session"], chk["peer"])) or chk.get("address")
|
||||
if not address:
|
||||
log.info("TEST %s: no address known for %s", key_id, chk["peer"])
|
||||
return None
|
||||
chk = dict(chk, address=address)
|
||||
return self.run_force(key_id, snap, comp, now, fix_id=fix_id, chk=chk, verb="TEST")
|
||||
|
||||
|
||||
def display_level(res, acked, service_on):
|
||||
if res.level == OFF:
|
||||
@@ -1373,7 +1606,7 @@ def render(engine, service_on):
|
||||
elif down: # flashes: this is the one to hold
|
||||
out[kid] = colored(key["label"] + "\nHOLD TO FIX", "err")
|
||||
else:
|
||||
out[kid] = colored(key["label"] + "\nHOLD 2s", "idle")
|
||||
out[kid] = colored(key["label"] + "\n" + engine.hold_text, "idle")
|
||||
for tile in engine.cfg["tiles"]:
|
||||
tid, kind = tile["id"], tile.get("kind")
|
||||
note = engine.notes.get(tid)
|
||||
@@ -1616,13 +1849,18 @@ def playlist_tracks(playlist_uri):
|
||||
return tracks, False
|
||||
|
||||
|
||||
def spotify_playlist(cfg, chk=None):
|
||||
"""The service playlist: "spotify" in config is the one place it is set; a check may override it."""
|
||||
return (chk or {}).get("playlist") or (cfg.get("spotify") or {}).get("playlist")
|
||||
|
||||
|
||||
def poll_spotify(snap, cfg):
|
||||
checks = [c for t in cfg["tiles"] for c in t.get("checks", []) if c["type"] == "spotify"]
|
||||
if not checks:
|
||||
return
|
||||
def refresh():
|
||||
try:
|
||||
snap.spotify_tracks, snap.spotify_complete = playlist_tracks(checks[0]["playlist"])
|
||||
snap.spotify_tracks, snap.spotify_complete = playlist_tracks(spotify_playlist(cfg, checks[0]))
|
||||
except (OSError, ValueError, KeyError, AttributeError, TypeError) as e:
|
||||
log.warning("Spotify playlist list: %s", e)
|
||||
snap.spotify_tracks_time = time.time()
|
||||
@@ -1769,11 +2007,22 @@ def wake_on_lan(mac, broadcast=None, port=9):
|
||||
return packet
|
||||
|
||||
|
||||
def remote_script(cmds):
|
||||
"""One shell line that runs every check and prints SP<n>=<exit code>."""
|
||||
def remote_script(cmds, shell="sh"):
|
||||
"""One line that runs every check and prints SP<n>=<0 or 1>. On a Mac the shell evaluates $?
|
||||
after each command; cmd.exe expands %ERRORLEVEL% once before the line runs, so Windows gets a
|
||||
conditional echo per command instead (never a bare %ERRORLEVEL%)."""
|
||||
if shell == "cmd":
|
||||
return " & ".join("((%s) >nul 2>&1 && echo SP%d=0 || echo SP%d=1)" % (cmd, i, i) for i, cmd in enumerate(cmds))
|
||||
return "; ".join("(%s) >/dev/null 2>&1; echo SP%d=$?" % (cmd, i) for i, cmd in enumerate(cmds))
|
||||
|
||||
|
||||
def host_shell(cfg, host, chk=None):
|
||||
"""Which shell a remote check's command line is written for: "cmd" on a Windows PC, else "sh"."""
|
||||
if chk and chk.get("shell"):
|
||||
return chk["shell"]
|
||||
return "cmd" if cfg.get("hosts", {}).get(host, {}).get("os") == "windows" else "sh"
|
||||
|
||||
|
||||
def parse_remote(out, n):
|
||||
codes = dict(re.findall(r"SP(\d+)=(\d+)", out or ""))
|
||||
return [None if str(i) not in codes else codes[str(i)] == "0" for i in range(n)]
|
||||
@@ -1790,7 +2039,7 @@ def poll_remote(snap, cfg):
|
||||
result = dict(snap.remote)
|
||||
for host, cmds in by_host.items():
|
||||
try:
|
||||
out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds)], capture_output=True,
|
||||
out = subprocess.run(ssh_cmd(cfg, host) + [remote_script(cmds, host_shell(cfg, host))], capture_output=True,
|
||||
text=True, timeout=20).stdout
|
||||
except subprocess.TimeoutExpired:
|
||||
out = ""
|
||||
@@ -1799,41 +2048,6 @@ def poll_remote(snap, cfg):
|
||||
snap.remote = result
|
||||
|
||||
|
||||
def hyperdeck_probe(ip, timeout=3, port=9993):
|
||||
"""Ask a HyperDeck directly what it is doing, on its control port 9993.
|
||||
|
||||
Returns its transport status ("record", "preview", "stopped", ...), "busy" if it
|
||||
refused another client, "answered" if it greeted but did not report a status,
|
||||
"hung" if it accepted the connection and said nothing at all, or "unreachable".
|
||||
Only "hung" justifies a reboot.
|
||||
"""
|
||||
try:
|
||||
s = socket.create_connection((ip, port), timeout=timeout)
|
||||
except OSError:
|
||||
return "unreachable"
|
||||
buf = b""
|
||||
with s:
|
||||
s.settimeout(timeout)
|
||||
try:
|
||||
s.sendall(b"transport info\r\n")
|
||||
end = time.time() + timeout
|
||||
while time.time() < end:
|
||||
chunk = s.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
buf += chunk
|
||||
m = re.search(rb"208 transport info:.*?status: (\w+)", buf, re.S)
|
||||
if m:
|
||||
return m.group(1).decode().lower()
|
||||
if b"120 connection rejected" in buf:
|
||||
return "busy"
|
||||
except socket.timeout:
|
||||
pass
|
||||
except OSError:
|
||||
return "unreachable" if not buf else "answered"
|
||||
return "answered" if buf else "hung"
|
||||
|
||||
|
||||
# --- The self-updating Dante map: name -> IP -> MAC -> switch port -> PoE --------
|
||||
|
||||
def dante_map_path(data_dir):
|
||||
@@ -1950,6 +2164,8 @@ def execute_fix(cfg, fix_id, chk):
|
||||
ok = conn in ids and comp.restart_connection(ids[conn])
|
||||
log.info("FIX %s -> Companion connection %s restarted %s", label, conn, "ok" if ok else "FAILED")
|
||||
return r.returncode == 0
|
||||
if kind == "spotify":
|
||||
return play_spotify(cfg, fix, chk, label)
|
||||
if kind == "atem":
|
||||
ok = bool(ATEM_WATCHER) and ATEM_WATCHER.fix(fix["part"])
|
||||
log.info("FIX %s -> %s", label, "switcher matches the restore file again" if ok else "FAILED")
|
||||
@@ -1990,6 +2206,27 @@ def execute_fix(cfg, fix_id, chk):
|
||||
return False
|
||||
|
||||
|
||||
def play_spotify(cfg, fix, chk, label, run=None):
|
||||
"""Play the service playlist in Spotify on this Mac, opening Spotify first if it is closed."""
|
||||
run = run or subprocess.run
|
||||
sp_ = cfg.get("spotify") or {}
|
||||
playlist = spotify_playlist(cfg, chk) or fix.get("playlist")
|
||||
if not playlist:
|
||||
log.warning("FIX %s: no playlist set", label)
|
||||
return False
|
||||
if run(["/usr/bin/pgrep", "-xq", "Spotify"], timeout=5).returncode != 0:
|
||||
run(["/usr/bin/open", "-g", "-a", "Spotify"], timeout=15)
|
||||
time.sleep(sp_.get("open_wait", 8))
|
||||
run(["/usr/bin/osascript", "-e", 'tell application "Spotify" to play track "%s"' % playlist],
|
||||
capture_output=True, text=True, timeout=15)
|
||||
time.sleep(sp_.get("play_wait", 3))
|
||||
r = run(["/usr/bin/osascript", "-e", 'tell application "Spotify" to player state as text'],
|
||||
capture_output=True, text=True, timeout=10)
|
||||
ok = r.returncode == 0 and r.stdout.strip() == "playing"
|
||||
log.info("FIX %s -> %s", label, "playing" if ok else "not playing (%s)" % (r.stdout.strip() or r.stderr.strip()[-80:]))
|
||||
return ok
|
||||
|
||||
|
||||
def reconnect_and_verify(cfg, fix, label, sleep=None, conn=None, host=None, toggle=False):
|
||||
"""Restart one Companion connection (toggle=True: switch it off and on again), then look again
|
||||
after "verify_seconds". Returns True if it came back; otherwise what to tell the operator:
|
||||
@@ -2006,8 +2243,11 @@ def reconnect_and_verify(cfg, fix, label, sleep=None, conn=None, host=None, togg
|
||||
if conn.get("enabled") and not comp.disable_connection(conn["id"]):
|
||||
log.info("FIX %s -> Companion refused to switch %s off", label, fix["conn"])
|
||||
return False
|
||||
sleep(fix.get("off_seconds", 2))
|
||||
if not comp.enable_connection(conn["id"]):
|
||||
try:
|
||||
sleep(fix.get("off_seconds", 2))
|
||||
finally: # whatever happens, it is switched back on
|
||||
on = comp.enable_connection(conn["id"])
|
||||
if not on:
|
||||
log.info("FIX %s -> Companion did not switch %s back on", label, fix["conn"])
|
||||
return "STILL OFF IN\nCOMPANION"
|
||||
log.info("FIX %s -> %s switched off and on", label, fix["conn"])
|
||||
@@ -2058,10 +2298,14 @@ CCU_PARAMS = {
|
||||
}
|
||||
|
||||
|
||||
def atem_reference(xml_path):
|
||||
def atem_reference(xml_path, compare=None):
|
||||
"""The restore file (.xml from ATEM Software Control) as what atem_watch.js compares against:
|
||||
every Fairlight source's mix option and fader, and every camera's settings above."""
|
||||
every Fairlight source's mix option and fader, and every camera's settings above. `compare`
|
||||
("compare" in config "atem") narrows it: {"audio": ["mixOption", "faderGain"], "camera": [...]}."""
|
||||
import xml.etree.ElementTree as ET
|
||||
compare = compare or {}
|
||||
audio_fields = compare.get("audio", ["mixOption", "faderGain"])
|
||||
camera_params = compare.get("camera") # parameter names, e.g. ["ApertureFstop", ...]
|
||||
root = ET.parse(xml_path).getroot()
|
||||
audio = {}
|
||||
mixer = root.find("FairlightAudioMixer")
|
||||
@@ -2074,13 +2318,16 @@ def atem_reference(xml_path):
|
||||
gain = float(src.attrib.get("faderGain", "nan"))
|
||||
except ValueError:
|
||||
gain = float("nan")
|
||||
if math.isfinite(gain): # the file saves dB; the switcher uses 1/100 dB
|
||||
if math.isfinite(gain) and "faderGain" in audio_fields: # the file saves dB; the switcher 1/100 dB
|
||||
want["faderGain"] = int(round(gain * 100))
|
||||
audio["%s/%s" % (inp.attrib["id"], src.attrib["id"])] = want
|
||||
if "mixOption" in audio_fields or "faderGain" in want:
|
||||
audio["%s/%s" % (inp.attrib["id"], src.attrib["id"])] = want
|
||||
camera = {}
|
||||
cc = root.find("CameraControl")
|
||||
for p in (cc if cc is not None else []):
|
||||
spec = CCU_PARAMS.get((p.attrib.get("category"), p.attrib.get("parameter")))
|
||||
if camera_params is not None and p.attrib.get("parameter") not in camera_params:
|
||||
continue
|
||||
if spec and all(f in p.attrib for f in spec[2]):
|
||||
camera["%s/%d/%d" % (p.attrib["device"], spec[0], spec[1])] = [float(p.attrib[f]) for f in spec[2]]
|
||||
return {"audio": audio, "camera": camera}
|
||||
@@ -2100,7 +2347,7 @@ class AtemWatcher:
|
||||
|
||||
def __init__(self, cfg, snap, data_dir, stop):
|
||||
a = cfg["atem"]
|
||||
self.cfg, self.snap, self.stop = a, snap, stop
|
||||
self.atem, self.snap, self.stop = a, snap, stop
|
||||
self.proc, self.lock, self.fixed = None, threading.Lock(), {}
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
self.script = os.path.join(here, "atem_watch.js")
|
||||
@@ -2112,12 +2359,12 @@ class AtemWatcher:
|
||||
if not os.path.isabs(xml):
|
||||
xml = os.path.join(here, xml)
|
||||
with open(self.ref_path, "w") as f:
|
||||
json.dump(atem_reference(xml), f)
|
||||
json.dump(atem_reference(xml, a.get("compare")), f)
|
||||
|
||||
def run(self):
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
self.proc = subprocess.Popen([self.node, self.script, self.cfg["host"], self.library, self.ref_path],
|
||||
self.proc = subprocess.Popen([self.node, self.script, self.atem["host"], self.library, self.ref_path],
|
||||
stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
||||
text=True, bufsize=1)
|
||||
errors = deque(maxlen=5) # read all the time, so a chatty stderr never stalls it
|
||||
@@ -2231,9 +2478,53 @@ class Periodic(threading.Thread):
|
||||
|
||||
# --- Main --------------------------------------------------------------------
|
||||
|
||||
def check_windows(windows, where):
|
||||
for w in windows or []:
|
||||
if not isinstance(w, dict) or not isinstance(w.get("days"), list) or not w["days"]:
|
||||
raise ValueError("%s: a window needs days, start and end" % where)
|
||||
for d in w["days"]:
|
||||
if d not in DAYS:
|
||||
raise ValueError("%s: unknown day %r" % (where, d))
|
||||
for k in ("start", "end"):
|
||||
if not re.fullmatch(r"([01]\d|2[0-3]):[0-5]\d", str(w.get(k, ""))):
|
||||
raise ValueError("%s: %s must be HH:MM" % (where, k))
|
||||
|
||||
|
||||
def check_host(cfg, host, where):
|
||||
"""A machine the watchdog logs in to: listed in "hosts", with an ssh address, and not this Mac."""
|
||||
h = cfg.get("hosts", {}).get(host)
|
||||
if h is None:
|
||||
raise ValueError("%s: unknown host %s" % (where, host))
|
||||
if h.get("self"):
|
||||
raise ValueError("%s: %s is this Mac - use a local fix or check" % (where, host))
|
||||
if not h.get("ssh"):
|
||||
raise ValueError("%s: host %s has no ssh address" % (where, host))
|
||||
|
||||
|
||||
def exit_timeout(cfg):
|
||||
"""How long launchd must let the watchdog finish a running fix before killing it: the longest
|
||||
fix plus its verify time plus a margin, never under a minute (install.sh writes it into the plist)."""
|
||||
longest = 0
|
||||
for fix in cfg.get("fixes", {}).values():
|
||||
longest = max(longest, fix.get("timeout", 60) + (fix.get("verify_seconds", 20)
|
||||
if fix.get("kind") in ("reconnect", "hyperdeck") else 0))
|
||||
return max(60, longest + 10)
|
||||
|
||||
|
||||
def load_config(path):
|
||||
with open(path) as f:
|
||||
cfg = json.load(f)
|
||||
for fid, fix in cfg.get("fixes", {}).items():
|
||||
kind = fix.get("kind", "ssh")
|
||||
if kind not in FIX_KINDS:
|
||||
raise ValueError("fix %s: unknown kind %s" % (fid, kind))
|
||||
if kind == "ssh":
|
||||
check_host(cfg, fix.get("host"), "fix " + fid)
|
||||
if kind in ("ssh", "local") and not fix.get("run"):
|
||||
raise ValueError("fix %s: nothing to run" % fid)
|
||||
check_windows(cfg.get("service_windows"), "service_windows")
|
||||
check_windows((cfg.get("auto_fix") or {}).get("windows"), "auto_fix")
|
||||
check_windows((cfg.get("site") or {}).get("live_services"), "site.live_services")
|
||||
ids = set()
|
||||
for t in cfg["tiles"]:
|
||||
if t["id"] in ids:
|
||||
@@ -2254,13 +2545,15 @@ def load_config(path):
|
||||
if (f and f not in (RESTART_CONNECTION, DANTE_AUTO) and cfg["fixes"][f].get("kind", "ssh") == "ssh"
|
||||
and cfg["fixes"][f]["host"] not in cfg.get("hosts", {})):
|
||||
raise ValueError("fix %s: unknown host" % f)
|
||||
if c["type"] == "remote" and c["host"] not in cfg.get("hosts", {}):
|
||||
raise ValueError("tile %s: unknown host %s" % (t["id"], c["host"]))
|
||||
if c["type"] == "remote" or (c["type"] == "midi_peer" and c.get("host")):
|
||||
check_host(cfg, c["host"], "tile " + t["id"])
|
||||
if t.get("hold_always") and t["hold_always"] not in cfg.get("fixes", {}):
|
||||
raise ValueError("tile %s: unknown hold_always fix %s" % (t["id"], t["hold_always"]))
|
||||
for f in (cfg.get("auto_fix") or {}).get("fixes", []):
|
||||
if f not in cfg.get("fixes", {}):
|
||||
raise ValueError("auto_fix: unknown fix %s" % f)
|
||||
if cfg["fixes"][f].get("never_auto"):
|
||||
raise ValueError("auto_fix: %s is marked never to run by itself" % f)
|
||||
if re.search(r"pkill|taskkill|killall", cfg["fixes"][f].get("run", "")) and not cfg["fixes"][f].get("restart_ok"):
|
||||
raise ValueError("auto_fix: %s closes a running program - that stays a hold" % f)
|
||||
for k in cfg.get("restart_page", {}).get("keys", []):
|
||||
@@ -2372,22 +2665,57 @@ def handle_commands(text, engine, comp, snap, now):
|
||||
engine.run_fixes(arg, snap, comp, now)
|
||||
else: # a Restart page key, held 2 seconds
|
||||
engine.run_force(arg, snap, comp, now)
|
||||
elif verb == "test":
|
||||
# The settings page: "test <key> <fix> <check number or identity> <unix time>".
|
||||
if len(words) < 5 or not words[4].isdigit() or now - int(words[4]) > HOLD_MAX_AGE:
|
||||
log.info("TEST %s: no time on it, or too long ago - ignored", arg)
|
||||
else:
|
||||
engine.run_test(arg, words[2], words[3], snap, comp, now)
|
||||
elif verb == "resurvey": # the settings page: look at the Dante switches now
|
||||
snap.dante_map_time = 0.0
|
||||
log.info("RESURVEY: Dante map survey requested")
|
||||
else:
|
||||
log.warning("unknown command %r", part)
|
||||
|
||||
|
||||
def run(cfg, data_dir, dry_run=False, stop=None):
|
||||
def run(cfg, data_dir, dry_run=False, stop=None, config_sha=None):
|
||||
stop = stop or threading.Event()
|
||||
names = cfg["variables"]
|
||||
comp = make_companion(cfg)
|
||||
snap = Snapshot()
|
||||
engine = Engine(cfg, os.path.join(data_dir, "service-mode.json"))
|
||||
engine = Engine(cfg, os.path.join(data_dir, "state.json"), data_dir)
|
||||
snap.dante_map = load_dante_map(dante_map_path(data_dir))
|
||||
site = cfg.get("site", {})
|
||||
prefix = site.get("alert_prefix") or site.get("ingest_name") or "Status panel"
|
||||
alert_after = cfg.get("timing", {}).get("alert_after_seconds", 30)
|
||||
|
||||
def on_term(signum, frame):
|
||||
log.info("stop requested: %d fix(es) running will finish first", len(engine.running))
|
||||
engine.stopping = True
|
||||
stop.set()
|
||||
try:
|
||||
signal.signal(signal.SIGTERM, on_term)
|
||||
except ValueError: # not the main thread (tests)
|
||||
pass
|
||||
threads = start_pollers(cfg, snap, stop, data_dir)
|
||||
if not dry_run:
|
||||
try:
|
||||
# A hold pressed while the watchdog was down must not fire on a config it was not meant for.
|
||||
queued = comp.get_custom(names["command"])
|
||||
if queued and queued.strip():
|
||||
comp.set_custom(names["command"], "")
|
||||
log.info("dropped commands queued before the start: %s", queued.strip())
|
||||
except (OSError, http.client.HTTPException) as e:
|
||||
log.warning("could not read the command variable at start: %s", e)
|
||||
stop.wait(2.5) # let the first polls land before judging anything
|
||||
last_json, last_write, last_eval = None, 0.0, 0.0
|
||||
down_since, alerted, warned_missing = None, False, set()
|
||||
log.info("status panel running (%s)", "dry run" if dry_run else "live")
|
||||
log.info("status panel running (code %s, %s)", VERSION, "dry run" if dry_run else "live")
|
||||
try:
|
||||
write_json(os.path.join(data_dir, "started.json"),
|
||||
{"pid": os.getpid(), "started_at": time.time(), "config_sha": config_sha, "version": VERSION})
|
||||
except OSError as e:
|
||||
log.warning("started.json: %s", e)
|
||||
while not stop.is_set():
|
||||
now = time.time()
|
||||
try:
|
||||
@@ -2428,18 +2756,27 @@ def run(cfg, data_dir, dry_run=False, stop=None):
|
||||
if down_since is not None:
|
||||
log.info("Companion reachable again")
|
||||
if alerted:
|
||||
post_webhook(cfg.get("alert_webhook"), "NA Ingest: Companion is answering again.")
|
||||
post_webhook(cfg.get("alert_webhook"), "%s: Companion is answering again." % prefix)
|
||||
down_since, alerted = None, False
|
||||
except (OSError, http.client.HTTPException) as e:
|
||||
if down_since is None:
|
||||
down_since = now
|
||||
log.warning("Companion unreachable: %s", e)
|
||||
elif not alerted and now - down_since > 30:
|
||||
post_webhook(cfg.get("alert_webhook"), "NA Ingest: Companion has not answered for 30 seconds.")
|
||||
elif not alerted and now - down_since > alert_after:
|
||||
post_webhook(cfg.get("alert_webhook"), "%s: Companion has not answered for %d seconds."
|
||||
% (prefix, alert_after))
|
||||
alerted = True
|
||||
stop.wait(0.25)
|
||||
# A fix that is running finishes before the process goes (launchd waits exit_timeout for us).
|
||||
budget = time.time() + max(5, exit_timeout(cfg) - 5)
|
||||
for t in engine.fix_threads:
|
||||
t.join(timeout=max(0.1, budget - time.time()))
|
||||
if any(t.is_alive() for t in engine.fix_threads):
|
||||
log.warning("stopping with a fix still running")
|
||||
else:
|
||||
log.info("stopped")
|
||||
for t in threads:
|
||||
t.join(timeout=10)
|
||||
t.join(timeout=2)
|
||||
|
||||
|
||||
def run_once(cfg):
|
||||
@@ -2483,8 +2820,16 @@ def main(argv=None):
|
||||
ap.add_argument("--once", action="store_true", help="poll once, print the tiles, write nothing")
|
||||
ap.add_argument("--dry-run", action="store_true", help="run but never write to Companion")
|
||||
ap.add_argument("--log", default=os.path.expanduser("~/Library/Logs/status-panel.log"))
|
||||
ap.add_argument("--check", action="store_true", help="load and validate the config, then exit")
|
||||
ap.add_argument("--exit-timeout", action="store_true", help="print the launchd exit timeout this config needs")
|
||||
args = ap.parse_args(argv)
|
||||
cfg = load_config(args.config)
|
||||
if args.check:
|
||||
print("config OK: %d keys, %d fixes" % (len(cfg["tiles"]), len(cfg.get("fixes", {}))))
|
||||
return
|
||||
if args.exit_timeout:
|
||||
print(exit_timeout(cfg))
|
||||
return
|
||||
if args.once:
|
||||
run_once(cfg)
|
||||
return
|
||||
@@ -2494,7 +2839,11 @@ def main(argv=None):
|
||||
if sys.stderr.isatty():
|
||||
log.addHandler(logging.StreamHandler())
|
||||
log.setLevel(logging.INFO)
|
||||
run(cfg, args.data_dir, dry_run=args.dry_run)
|
||||
if not (3, 9) <= sys.version_info[:2] <= (3, 12):
|
||||
log.warning("python %s: the Dante switch reader (telnetlib) needs 3.9-3.12", sys.version.split()[0])
|
||||
with open(args.config, "rb") as f:
|
||||
config_sha = hashlib.sha256(f.read()).hexdigest()
|
||||
run(cfg, args.data_dir, dry_run=args.dry_run, config_sha=config_sha)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user