christianandClaude Fable 5.1
01c18ba5ef
Overview (problems in the room with the hold action, the live deck, one line per computer), Keys (both pages, what each key will say and what holding does), Computers (map with the keys that turn red, cards with a light per thing that runs there from the watchdog's snapshot and a read-only probe, Gear, Setup), Logs, History (the installer's copies for now), Help (four cards, the generated key guide, cheat sheet and Where things are, glossary, Companion facts), Site setup and Fixes read-only. The editing code is checked (with a lockout) but nothing can change yet. Also: faults are attributed to the computers their check is about; the installer refuses while a HyperDeck is recording (--now overrides); the Lighting PC's MIDI session is listed; the probes read sudo's answer correctly and expand $HOME outside the quotes. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
732 lines
39 KiB
Python
732 lines
39 KiB
Python
#!/usr/bin/env python3
|
|
"""The settings page's server: a small stdlib HTTP server on the Ingest Mac, beside the watchdog.
|
|
|
|
Everything is readable by anyone on the church network (the alert web hook and the editing code
|
|
are masked); nothing changes without the editing code. The server never runs a command before an
|
|
Apply, never writes a file the watchdog writes, and reads the watchdog's own files for what it knows.
|
|
|
|
python3 -m settings.server --data-dir DIR [--port 8020] [--bind 127.0.0.1 ...]
|
|
"""
|
|
import argparse
|
|
import gzip
|
|
import glob
|
|
import hashlib
|
|
import http.server
|
|
import ipaddress
|
|
import json
|
|
import logging
|
|
import logging.handlers
|
|
import os
|
|
import re
|
|
import socket
|
|
import sys
|
|
import threading
|
|
import time
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
from settings import PARENT, HERE # noqa: F401
|
|
import schema
|
|
import status_panel as sp
|
|
from settings import keyguide, probes
|
|
|
|
log = logging.getLogger("status-settings")
|
|
UI = os.path.join(HERE, "ui")
|
|
PRIVATE = [ipaddress.ip_network(n) for n in ("10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16", "169.254.0.0/16", "127.0.0.0/8")]
|
|
LOGS = {"watchdog": "~/Library/Logs/status-panel.log", "errors": "~/Library/Logs/status-panel.err.log",
|
|
"settings": "~/Library/Logs/status-settings.log"}
|
|
LOG_FILTERS = {"holds": r"\b(HOLD|FORCE|TEST|FIX|ENABLE|RESTART|ACK)\b", "fixes": r"\bFIX\b|-> ",
|
|
"auto": r"\bAUTO\b|auto-fix", "errors": r"WARNING|ERROR|Traceback", "faults": r"\b(FAULT|CLEAR)\b"}
|
|
|
|
# What the Companion experiments found (the Help page shows this; filled in as they are run).
|
|
COMPANION_FACTS = [
|
|
("Which module a connection uses", "The HTTP API reports it (moduleId) - no need to read Companion's database for it."),
|
|
("Does OBS say when it is streaming?", "Yes: the OBS-Youtube connection's variable 'streaming' reads 'Off-Air' when it is not; "
|
|
"the on-air value is recorded the first time it streams."),
|
|
("Can the page create the custom variables?", "Not yet known; they are made by hand in Companion for now."),
|
|
("Can an old page's keys be blanked over the API?", "Not yet tried; after a page-number change the old keys are deleted by hand."),
|
|
]
|
|
|
|
|
|
def allowed_client(ip):
|
|
try:
|
|
addr = ipaddress.ip_address(ip.split("%")[0])
|
|
except ValueError:
|
|
return False
|
|
return any(addr in n for n in PRIVATE)
|
|
|
|
|
|
def bind_addresses(cfg, ifaces=None):
|
|
"""127.0.0.1 plus the IPv4 address of every LAN-use port named for this Mac (every private
|
|
address when none is named)."""
|
|
ifaces = probes.interfaces() if ifaces is None else ifaces
|
|
me = next((h for h in cfg.get("hosts", {}).values() if h.get("self")), {})
|
|
lan = [n["iface"] for n in me.get("networks", []) if n.get("use") == "LAN"]
|
|
out = ["127.0.0.1"]
|
|
for name, ip, active in ifaces:
|
|
if ip in out:
|
|
continue
|
|
if (name in lan) if lan else allowed_client(ip):
|
|
out.append(ip)
|
|
return out
|
|
|
|
|
|
def tail(path, lines=400, pattern=None):
|
|
path = os.path.expanduser(path)
|
|
try:
|
|
with open(path, "rb") as f:
|
|
f.seek(0, 2)
|
|
size = f.tell()
|
|
f.seek(max(0, size - 512 * 1024))
|
|
text = f.read().decode("utf-8", "replace")
|
|
except OSError:
|
|
return []
|
|
rows = text.split("\n")[1:] if len(text) >= 512 * 1024 else text.split("\n")
|
|
if pattern:
|
|
rx = re.compile(pattern)
|
|
rows = [r for r in rows if rx.search(r)]
|
|
return [r for r in rows if r][-lines:]
|
|
|
|
|
|
class Site:
|
|
"""Everything the handlers share: the config, the data dir, Companion, caches."""
|
|
|
|
def __init__(self, data_dir, config_path=None, companion=None):
|
|
self.data_dir = data_dir
|
|
self.config_path = config_path or os.path.join(data_dir, "config.json")
|
|
self.cfg, self.cfg_sha, self.cfg_mtime = None, None, 0
|
|
self.load()
|
|
self.comp = companion or sp.make_companion(self.cfg)
|
|
self.lock = threading.Lock()
|
|
self.probing = None
|
|
self.attempts = {} # client ip -> [count, first attempt time]
|
|
self.page_cache = {} # name -> (time, value)
|
|
|
|
def load(self):
|
|
st = os.stat(self.config_path)
|
|
if st.st_mtime != self.cfg_mtime:
|
|
with open(self.config_path, "rb") as f:
|
|
raw = f.read()
|
|
self.cfg = sp.load_config(self.config_path)
|
|
self.cfg_sha, self.cfg_mtime = hashlib.sha256(raw).hexdigest(), st.st_mtime
|
|
return self.cfg
|
|
|
|
def read_json(self, name, default=None):
|
|
try:
|
|
with open(os.path.join(self.data_dir, name)) as f:
|
|
return json.load(f)
|
|
except (OSError, ValueError):
|
|
return default
|
|
|
|
def companion_get(self, path, timeout=3):
|
|
comp = self.cfg["companion"]
|
|
url = "http://%s:%d%s" % (comp.get("host", "127.0.0.1"), comp.get("port", 8000), path)
|
|
with urllib.request.urlopen(url, timeout=timeout) as r:
|
|
return r.status, r.read()
|
|
|
|
def cached(self, name, seconds, fn):
|
|
now = time.time()
|
|
hit = self.page_cache.get(name)
|
|
if hit and now - hit[0] < seconds:
|
|
return hit[1]
|
|
value = fn()
|
|
self.page_cache[name] = (now, value)
|
|
return value
|
|
|
|
# --- what the screens show ---------------------------------------------
|
|
|
|
def overview(self):
|
|
cfg = self.load()
|
|
names = cfg["variables"]
|
|
deck, hb, companion_ok = None, None, True
|
|
try:
|
|
raw = self.comp.get_custom(names["state"])
|
|
deck = json.loads(raw) if raw else None
|
|
hb = self.comp.get_custom(names["heartbeat"])
|
|
except Exception as e: # Companion down: the page still works
|
|
companion_ok, deck = False, None
|
|
log.warning("Companion: %s", e)
|
|
now = time.time()
|
|
hb = float(hb) if hb and str(hb).strip().replace(".", "").isdigit() else None
|
|
snap = self.read_json("snapshot.json", {}) or {}
|
|
started = self.read_json("started.json", {}) or {}
|
|
live = self.read_json("live-now.json", {}) or {}
|
|
running = self.read_json("fixes-running.json", {}) or {}
|
|
tiles = {t["id"]: t for t in cfg["tiles"]}
|
|
problems, needs = [], []
|
|
for tid, faults in (snap.get("faults") or {}).items():
|
|
tile = tiles.get(tid)
|
|
if not tile:
|
|
continue
|
|
for f in faults:
|
|
if f["level"] not in ("err", "warn"):
|
|
continue
|
|
text = f["msg"] if f["msg"] and f["msg"] != tile["label"] else tile["label"]
|
|
hold = keyguide.plain(keyguide.hold_words(tile, cfg))
|
|
problems.append({"key": tid, "label": tile["label"], "text": "%s: %s" % (tile["label"], f["msg"]),
|
|
"level": f["level"], "acked": f.get("acked"), "since": f.get("since"),
|
|
"action": "hold the %s key (%s)" % (tile["label"], hold) if hold and not hold.startswith("nothing")
|
|
else "no automatic fix: check the device"})
|
|
fresh = snap.get("time") and now - snap["time"] < 30
|
|
if not problems and deck and not fresh:
|
|
# The watchdog has not written a snapshot (older code, or just started): read the deck itself.
|
|
for tid, s in deck.items():
|
|
tile = tiles.get(tid)
|
|
if not tile or tile.get("members") or tile.get("kind") == "master":
|
|
continue # groups and ALL repeat their members
|
|
if s.get("b") not in (0xFF0000, 0x8B0000, 0xFFA000, 0x3A0000):
|
|
continue
|
|
lines = (s.get("x") or "").split("\n")
|
|
text = " ".join(lines[1:]) if len(lines) > 1 else lines[0]
|
|
hold = keyguide.plain(keyguide.hold_words(tile, cfg))
|
|
problems.append({"key": tid, "label": tile["label"], "text": "%s: %s" % (tile["label"], text),
|
|
"level": "warn" if s.get("b") == 0xFFA000 else "err", "acked": s.get("b") == 0x8B0000, "since": None,
|
|
"action": "hold the %s key (%s)" % (tile["label"], hold) if hold and not hold.startswith("nothing")
|
|
else "no automatic fix: check the device"})
|
|
if started and started.get("config_sha") and started["config_sha"] != self.cfg_sha:
|
|
needs.append("The watchdog is running an older settings file than the one on disk (restart it to pick up the change).")
|
|
pages = self.pages_state()
|
|
if pages.get("status") and not pages["status"]["ok"]:
|
|
needs.append("Companion's page %d is not called %s." % (cfg["page"], pages["status"]["expected"]))
|
|
return {
|
|
"time": now, "site": cfg.get("site", {}), "deck": deck, "heartbeat_age": (now - hb) if hb else None,
|
|
"companion_ok": companion_ok, "watchdog": self.watchdog_state(started, hb, now),
|
|
"live": live, "running": running, "service_on": snap.get("service_on") if fresh else None,
|
|
"alarms_override": snap.get("alarms_override") if fresh else None,
|
|
"auto_fix": {"phrase": schema.phrases(cfg)["auto_fix"], "state": snap.get("auto_fix")},
|
|
"problems": sorted(problems, key=lambda p: (p["level"] != "err", p["since"] or 0)), "needs": needs,
|
|
"computers": self.computer_lines(snap), "pages": pages, "snapshot_age": (now - snap["time"]) if snap.get("time") else None,
|
|
"last_change": self.last_change(), "phrases": schema.phrases(cfg), "version": started.get("version"),
|
|
}
|
|
|
|
def watchdog_state(self, started, hb, now):
|
|
state = {"running": False, "since": started.get("started_at"), "version": started.get("version"), "pid": started.get("pid")}
|
|
try:
|
|
out = sp.subprocess.run(["/bin/launchctl", "print", "gui/%d/org.waterschurch.status-panel" % os.getuid()],
|
|
capture_output=True, text=True, timeout=5).stdout
|
|
m = re.search(r"state = (\w+)", out)
|
|
state["launchd"] = m.group(1) if m else "not loaded"
|
|
state["running"] = bool(m and m.group(1) == "running")
|
|
except (OSError, sp.subprocess.TimeoutExpired):
|
|
state["launchd"] = "unknown"
|
|
state["heartbeat_fresh"] = bool(hb and now - hb < cfg_stale(self.cfg))
|
|
return state
|
|
|
|
def check_hosts(self, cfg):
|
|
"""{(key id, check identity): {computer aliases}}: which computers a check is about - the Mac a
|
|
MIDI session or remote command runs on, the peer's computer when a computer is named like it,
|
|
the computer a check's fix runs on, this Mac for local things."""
|
|
hosts = cfg.get("hosts", {})
|
|
me = next((a for a, h in hosts.items() if h.get("self")), None)
|
|
by_session = {}
|
|
for alias, h in hosts.items():
|
|
for run in h.get("runs", []):
|
|
if run.get("kind") == "midi_session" and run.get("name"):
|
|
by_session[run["name"]] = alias
|
|
if h.get("name"):
|
|
by_session.setdefault(h["name"], alias)
|
|
out = {}
|
|
for t in cfg["tiles"]:
|
|
for c in t.get("checks", []):
|
|
who = set()
|
|
if c["type"] in ("remote", "midi_peer") and c.get("host"):
|
|
who.add(c["host"])
|
|
elif c["type"] == "midi_peer" and me:
|
|
who.add(me)
|
|
if c["type"] == "midi_peer" and c.get("peer") in by_session:
|
|
who.add(by_session[c["peer"]])
|
|
if c["type"] in ("launchd", "process", "disk", "pref", "spotify", "atem", "midi_rx") and me:
|
|
who.add(me)
|
|
fx = c.get("fix")
|
|
for f in (fx.values() if isinstance(fx, dict) else [fx]):
|
|
fix = cfg.get("fixes", {}).get(f) if f else None
|
|
if fix and fix.get("host"):
|
|
who.add(fix["host"])
|
|
elif fix and fix.get("kind") in ("local", "spotify") and me:
|
|
who.add(me)
|
|
if t.get("hold_always"):
|
|
fix = cfg["fixes"].get(t["hold_always"], {})
|
|
if fix.get("host"):
|
|
who.add(fix["host"])
|
|
out[(t["id"], sp.check_identity(c))] = who
|
|
return out
|
|
|
|
def computer_lines(self, snap):
|
|
"""One line per computer: a status word and the reason, from what the watchdog sees."""
|
|
cfg = self.cfg
|
|
live = self.read_json("machines-live.json", {}) or {}
|
|
owners = self.check_hosts(cfg)
|
|
tiles_by_host, labels = {}, {t["id"]: t["label"] for t in cfg["tiles"]}
|
|
for (tid, _), who in owners.items():
|
|
for alias in who:
|
|
tiles_by_host.setdefault(alias, set()).add(tid)
|
|
lines = []
|
|
for alias, h in cfg.get("hosts", {}).items():
|
|
probe = live.get(alias) or {}
|
|
word, reason = "OK", ""
|
|
faults = []
|
|
for tid, fs in (snap.get("faults") or {}).items():
|
|
for f in fs:
|
|
if f["level"] == "err" and alias in owners.get((tid, f.get("check")), set()):
|
|
faults.append("%s %s" % (labels.get(tid, tid), f["msg"]))
|
|
if probe and not probe.get("reachable") and not h.get("self"):
|
|
word, reason = "Problem", "cannot be reached (%s)" % probe.get("error", "no answer")
|
|
elif faults:
|
|
word, reason = "Problem", "; ".join(faults[:2])
|
|
elif probe and any(i.get("light") == "problem" and i.get("watched", True) for i in probe.get("items", [])):
|
|
bad = [i for i in probe.get("items", []) if i.get("light") == "problem" and i.get("watched", True)]
|
|
word, reason = "Problem", "; ".join("%s %s" % (i["name"], i.get("reason", "")) for i in bad[:2])
|
|
elif not probe and not h.get("self"):
|
|
word, reason = "Not looked yet", ""
|
|
lines.append({"alias": alias, "name": h.get("name", alias), "role": h.get("role", ""), "word": word, "reason": reason,
|
|
"keys": sorted(tiles_by_host.get(alias, [])), "self": bool(h.get("self")),
|
|
"looked": probe.get("time")})
|
|
return lines
|
|
|
|
def pages_state(self):
|
|
cfg = self.cfg
|
|
|
|
def look():
|
|
out = {}
|
|
for which, page, name in (("status", cfg["page"], cfg["page_name"]),) + (
|
|
(("restart", cfg["restart_page"]["page"], cfg["restart_page"]["page_name"]),) if cfg.get("restart_page") else ()):
|
|
try:
|
|
status, raw = self.companion_get("/api/variable/internal/page_number_%d_name/value" % page)
|
|
live = raw.decode().replace("\\n", "\n") if status == 200 else None
|
|
except (OSError, urllib.error.URLError):
|
|
live = None
|
|
out[which] = {"page": page, "expected": name.replace("\\n", " "), "live": live.replace("\n", " ") if live else None,
|
|
"ok": live is not None and live.replace("\n", " ") == name.replace("\\n", " ")}
|
|
return out
|
|
return self.cached("pages", 60, look)
|
|
|
|
def last_change(self):
|
|
baks = sorted(glob.glob(os.path.join(self.data_dir, "config.json.bak-*")))
|
|
if not baks:
|
|
return None
|
|
m = re.search(r"bak-(\d{8})-(\d{6})$", baks[-1])
|
|
return {"when": "%s-%s-%s %s:%s" % (m.group(1)[:4], m.group(1)[4:6], m.group(1)[6:], m.group(2)[:2], m.group(2)[2:4]) if m else None,
|
|
"by": "the installer", "note": "config.json replaced"}
|
|
|
|
def companion_view(self):
|
|
cfg = self.load()
|
|
watched = {}
|
|
for t in cfg["tiles"]:
|
|
for c in t.get("checks", []):
|
|
if c.get("conn"):
|
|
watched.setdefault(c["conn"], set()).add(t["id"])
|
|
for k in (cfg.get("restart_page") or {}).get("keys", []):
|
|
for w in k.get("watch", []):
|
|
if w.startswith("conn:"):
|
|
watched.setdefault(w[5:], set()).add(k["id"])
|
|
try:
|
|
conns = self.comp.connections()
|
|
ok = True
|
|
except Exception as e:
|
|
conns, ok = [], False
|
|
rows = []
|
|
for c in sorted(conns, key=lambda c: c["label"].lower()):
|
|
st = c.get("status") or {}
|
|
rows.append({"label": c["label"], "module": c.get("moduleId"), "enabled": c.get("enabled"), "category": st.get("category"),
|
|
"message": st.get("message"), "watched_by": sorted(watched.get(c["label"], []))})
|
|
labels = {c["label"] for c in conns}
|
|
missing = sorted(l for l in watched if ok and l not in labels)
|
|
names = cfg["variables"]
|
|
lights = {}
|
|
for key in ("state", "heartbeat", "command"):
|
|
try:
|
|
lights["variable " + names[key]] = self.comp.get_custom(names[key]) is not None
|
|
except Exception:
|
|
lights["variable " + names[key]] = None
|
|
try:
|
|
lights["blink clock " + names.get("flash", "flash_tick")] = self.comp.get_custom(names.get("flash", "flash_tick")) is not None
|
|
except Exception:
|
|
lights["blink clock " + names.get("flash", "flash_tick")] = None
|
|
modules = {c.get("moduleId") for c in conns}
|
|
lights["ATEM module"] = ("bmd-atem" in modules) if ok else None
|
|
lights["Tascam module"] = ("tascam-da-6400" in modules) if ok else None
|
|
pages = self.pages_state()
|
|
for which, p in pages.items():
|
|
lights["page %d is the %s page" % (p["page"], which.capitalize())] = p["ok"]
|
|
nav = self.read_json("nav_locations.json", {}) or {}
|
|
try:
|
|
here = json.load(open(os.path.join(PARENT, "nav_locations.json")))
|
|
except (OSError, ValueError):
|
|
here = {}
|
|
coloured = len({k.split("/")[0] for k in (nav or here)})
|
|
return {"ok": ok, "build": self.cached("build", 600, self.companion_build), "connections": rows, "missing": missing,
|
|
"lights": lights, "page_number_buttons": {"pages_coloured": coloured, "recorded_in": "data dir" if nav else "repo copy"},
|
|
"address": "%s:%d" % (cfg["companion"].get("host", "127.0.0.1"), cfg["companion"].get("port", 8000)),
|
|
"variables": names}
|
|
|
|
def companion_build(self):
|
|
try:
|
|
status, raw = self.companion_get("/int/export/page/%d?includeSecrets=false" % self.cfg["page"], timeout=8)
|
|
try:
|
|
raw = gzip.decompress(raw)
|
|
except OSError:
|
|
pass
|
|
return json.loads(raw).get("companionBuild")
|
|
except (OSError, ValueError, urllib.error.URLError):
|
|
return None
|
|
|
|
def machines_view(self):
|
|
cfg = self.load()
|
|
live = self.read_json("machines-live.json", {}) or {}
|
|
snap = self.read_json("snapshot.json", {}) or {}
|
|
out = []
|
|
for line in self.computer_lines(snap):
|
|
alias = line["alias"]
|
|
h = cfg["hosts"][alias]
|
|
probe = dict(live.get(alias) or {})
|
|
items = probe.get("items") or [{"name": r.get("name") or r.get("label") or r.get("kind"), "kind": r.get("kind"),
|
|
"light": "unknown", "reason": "not looked yet", "watched": r.get("watched", True)}
|
|
for r in h.get("runs", [])]
|
|
# what the watchdog already knows beats a stale probe
|
|
for item, run in zip(items, h.get("runs", [])):
|
|
k = run.get("kind")
|
|
if k == "midi_session":
|
|
sessions = snap.get("midi") if h.get("self") else (snap.get("remote_midi") or {}).get(alias)
|
|
if sessions is not None and run.get("name") in sessions:
|
|
peers = sessions[run["name"]].get("peers", [])
|
|
item.update({"light": "ok", "reason": "", "detail": "%d peer%s: %s" % (len(peers), "" if len(peers) == 1 else "s",
|
|
", ".join(p["name"] for p in peers))})
|
|
elif run.get("peer_of"):
|
|
for host_sessions in list((snap.get("remote_midi") or {}).values()) + [snap.get("midi") or {}]:
|
|
sess = (host_sessions or {}).get(run["peer_of"])
|
|
if sess and any(p["name"] == run["name"] for p in sess.get("peers", [])):
|
|
item.update({"light": "ok", "reason": "", "detail": "in %s" % run["peer_of"]})
|
|
break
|
|
else:
|
|
item.update({"light": "problem", "reason": "not in the %s session" % run.get("peer_of")})
|
|
elif sessions is not None:
|
|
item.update({"light": "problem", "reason": "no such MIDI session"})
|
|
elif k == "web" or (k == "app" and run.get("http")):
|
|
ok = (snap.get("http") or {}).get(run.get("url") or run.get("http"))
|
|
if ok is not None:
|
|
item.update({"light": "ok" if ok else "problem", "reason": "" if ok else "its web page does not answer"})
|
|
elif k == "artnet":
|
|
age = (snap.get("artnet") or {}).get(run.get("source"))
|
|
if age is not None:
|
|
item.update({"light": "ok" if age < 10 else "problem", "reason": "" if age < 10 else "silent for %d s" % age})
|
|
probe["items"] = items
|
|
probe.update({"alias": alias, "name": h.get("name", alias), "os": h.get("os", "mac"), "role": h.get("role", ""),
|
|
"notes": h.get("notes", ""), "ssh": h.get("ssh"), "mac": h.get("mac"), "pingable": h.get("pingable", True),
|
|
"self": bool(h.get("self")), "networks": h.get("networks", []), "manual_done": h.get("manual_done", {}),
|
|
"word": line["word"], "reason": line["reason"], "keys": line["keys"]})
|
|
if not h.get("self") and h.get("pingable", True):
|
|
ip = h.get("ssh", "").split("@")[-1]
|
|
fails = (snap.get("ping") or {}).get(ip)
|
|
probe["ping"] = None if fails is None else (fails == 0)
|
|
out.append(probe)
|
|
return {"computers": out, "probing": bool(self.probing and self.probing.is_alive()),
|
|
"setup_steps": self.setup_steps(cfg)}
|
|
|
|
def setup_steps(self, cfg):
|
|
"""The one-time checklist per computer kind, with the light each line gets from the probe."""
|
|
return {
|
|
"mac": ["Remote Login is on", "the panel's login key is accepted", "python3 is present (for MIDI reads)",
|
|
"the DVS helper and its sudo rule (Macs that run Dante Virtual Soundcard)", "Audio MIDI Setup sessions named as the keys expect",
|
|
"Studio B only: the Pro Tools helper runs and Accessibility is allowed"],
|
|
"windows": ["the SSH service runs and starts with Windows", "the firewall lets this Mac in", "the panel's login key is accepted",
|
|
"a \\StatusPanel\\ task per program", "rtpMIDI's service restarts by itself (Lighting PC)", "wake-on-LAN is on in Windows and in the BIOS",
|
|
"the network profile is Public where rtpMIDI runs", "Wi-Fi power saving is off (Bible TV)"],
|
|
"self": ["Command Line Tools / python 3.9-3.12", "the watchdog and the settings page are loaded", "Companion's custom variables and blink clock exist",
|
|
"the login key pair exists", "the Netgear login is in the keychain", "the Spotify permission was allowed once",
|
|
"the DVS helper and its sudo rule", "the Dante and switch-management ports are up", "Google Drive is signed in (saved switcher files)"],
|
|
}
|
|
|
|
def gear_view(self):
|
|
cfg = self.load()
|
|
snap = self.read_json("snapshot.json", {}) or {}
|
|
conns, pings, http = snap.get("conns") or {}, snap.get("ping") or {}, snap.get("http") or {}
|
|
hosts_ips = {h.get("ssh", "").split("@")[-1] for h in cfg.get("hosts", {}).values()}
|
|
gear = cfg.get("gear", {})
|
|
kinds = {"hyperdeck": "recorder", "tascam": "recorder", "shure": "wireless receiver", "psm": "in-ear transmitter",
|
|
"dante": "Dante", "atem": "switcher"}
|
|
rows = {}
|
|
for t in cfg["tiles"]:
|
|
for c in t.get("checks", []):
|
|
name, kind, address = None, None, None
|
|
if c.get("conn") and c["type"] != "dante":
|
|
name, address = c["conn"], (snap.get("conn_hosts") or {}).get(c["conn"])
|
|
mod = next((r.get("module") for r in [] if False), None)
|
|
kind = kinds.get(c["type"], "device")
|
|
elif c["type"] == "ping" and c.get("host") and c["host"] not in hosts_ips:
|
|
name, address, kind = c.get("name") or c["host"], c["host"], "device"
|
|
elif c["type"] == "http" and not any(ip in c["url"] for ip in hosts_ips) and "127.0.0.1" not in c["url"]:
|
|
name, address, kind = c.get("name") or c.get("msg") or c["url"], c["url"], "web address"
|
|
elif c["type"] == "artnet":
|
|
continue
|
|
if not name:
|
|
continue
|
|
row = rows.setdefault(name, {"name": name, "kind": kind, "address": address, "watched_by": set(), "light": "unknown",
|
|
"reason": "", "location": gear.get(name, {}).get("location", ""), "notes": gear.get(name, {}).get("notes", "")})
|
|
row["watched_by"].add(t["id"])
|
|
if address is None and row["address"] is None and c.get("host"):
|
|
row["address"] = c["host"]
|
|
if c.get("conn") and c["conn"] in conns:
|
|
st = conns[c["conn"]]
|
|
row["light"] = "ok" if st.get("category") == "good" else ("problem" if st.get("enabled") else "not_needed")
|
|
row["reason"] = "" if row["light"] == "ok" else (st.get("message") or st.get("category") or "")
|
|
if c["type"] == "ping" and c.get("host") in pings and row["light"] != "problem":
|
|
row["light"] = "ok" if pings[c["host"]] == 0 else "problem"
|
|
row["reason"] = "" if pings[c["host"]] == 0 else "does not answer ping"
|
|
if c["type"] == "http" and c["url"] in http and row["light"] == "unknown":
|
|
row["light"] = "ok" if http[c["url"]] else "problem"
|
|
for name in (snap.get("dante_map") or {}):
|
|
e = snap["dante_map"][name]
|
|
row = rows.setdefault(name, {"name": name, "kind": "Dante device", "address": e.get("ip"), "watched_by": {"dante"}, "light": "unknown",
|
|
"reason": "", "location": gear.get(name, {}).get("location", ""), "notes": gear.get(name, {}).get("notes", "")})
|
|
row["detail"] = "switch %s port %s%s" % (e.get("switch", "?"), e.get("port", "?"), ", PoE" if e.get("poe") else "")
|
|
for s in cfg.get("dante_switches", []):
|
|
rows.setdefault("Netgear " + s, {"name": "Netgear " + s, "kind": "Dante switch", "address": s, "watched_by": {"dante"}, "light": "unknown",
|
|
"reason": "", "location": gear.get("Netgear " + s, {}).get("location", ""), "notes": ""})
|
|
out = []
|
|
for r in rows.values():
|
|
r["watched_by"] = sorted(r["watched_by"])
|
|
out.append(r)
|
|
return sorted(out, key=lambda r: (r["light"] != "problem", r["kind"], r["name"].lower()))
|
|
|
|
def keys_view(self):
|
|
cfg = self.load()
|
|
tiles = []
|
|
for t in cfg["tiles"]:
|
|
words = []
|
|
for c in t.get("checks", []):
|
|
ws = [c["msg"]] if c.get("msg") else list(keyguide.FAULT_WORDS.get(c["type"], []))
|
|
if c.get("tag"):
|
|
ws = ["%s %s" % (c["tag"], w) for w in ws]
|
|
words += ws
|
|
tiles.append(dict(t, says=sorted(set(words)), holding=keyguide.plain(keyguide.hold_words(t, cfg)),
|
|
checks=[dict(c, sentence=keyguide.plain(keyguide.check_sentence(c, cfg)),
|
|
identity=sp.check_identity(c)) for c in t.get("checks", [])]))
|
|
rp = cfg.get("restart_page")
|
|
keys = []
|
|
if rp:
|
|
for k in rp["keys"]:
|
|
keys.append(dict(k, holding=keyguide.plain(keyguide.fix_words(k["fix"], cfg))))
|
|
return {"page": cfg["page"], "page_name": cfg["page_name"], "tiles": tiles, "restart_page": dict(rp, keys=keys) if rp else None,
|
|
"hold": schema.phrases(cfg)["hold"], "grid": cfg.get("page_grid", schema.F["page_grid"]["default"])}
|
|
|
|
def history_view(self):
|
|
rows = []
|
|
for path in sorted(glob.glob(os.path.join(self.data_dir, "config.json.bak-*")), reverse=True):
|
|
m = re.search(r"bak-(\d{8})-(\d{6})$", path)
|
|
rows.append({"file": os.path.basename(path), "when": "%s-%s-%s %s:%s:%s" % (m.group(1)[:4], m.group(1)[4:6], m.group(1)[6:],
|
|
m.group(2)[:2], m.group(2)[2:4], m.group(2)[4:]) if m else "?",
|
|
"size": os.path.getsize(path)})
|
|
return {"entries": rows, "note": "Full history (who, what, Go back) arrives with step 2; these are the installer's copies."}
|
|
|
|
# --- editing code (the gate; nothing to gate yet in step 1) --------------
|
|
|
|
def check_code(self, ip, code):
|
|
count, first = self.attempts.get(ip, (0, 0))
|
|
now = time.time()
|
|
if count >= 5 and now - first < 600:
|
|
return {"ok": False, "locked_for": int(600 - (now - first))}
|
|
wanted = (self.cfg.get("site") or {}).get("editing_code") or ""
|
|
if wanted and code and code.strip().lower() == wanted.strip().lower():
|
|
self.attempts.pop(ip, None)
|
|
return {"ok": True}
|
|
if now - first >= 600:
|
|
count, first = 0, now
|
|
self.attempts[ip] = (count + 1, first)
|
|
return {"ok": False, "tries_left": max(0, 5 - count - 1), "no_code": not wanted}
|
|
|
|
|
|
def cfg_stale(cfg):
|
|
return cfg.get("page_stale_seconds", 15)
|
|
|
|
|
|
class Handler(http.server.BaseHTTPRequestHandler):
|
|
site = None # set by serve()
|
|
server_version = "status-settings"
|
|
|
|
def log_message(self, fmt, *args):
|
|
log.info("%s %s", self.client_address[0], fmt % args)
|
|
|
|
def send(self, status, body, ctype="application/json; charset=utf-8"):
|
|
data = body if isinstance(body, bytes) else (json.dumps(body, default=str) if ctype.startswith("application/json") else body).encode()
|
|
self.send_response(status)
|
|
self.send_header("Content-Type", ctype)
|
|
self.send_header("Content-Length", str(len(data)))
|
|
self.send_header("Cache-Control", "no-store")
|
|
self.end_headers()
|
|
self.wfile.write(data)
|
|
|
|
def do_GET(self):
|
|
if not allowed_client(self.client_address[0]):
|
|
return self.send(403, {"error": "only from the church network"})
|
|
url = urllib.parse.urlparse(self.path)
|
|
q = urllib.parse.parse_qs(url.query)
|
|
path = url.path
|
|
site = self.site
|
|
try:
|
|
if path in ("/", "/index.html"):
|
|
return self.static("index.html", "text/html; charset=utf-8")
|
|
if path in ("/app.js", "/style.css"):
|
|
return self.static(path[1:], "text/javascript; charset=utf-8" if path.endswith(".js") else "text/css; charset=utf-8")
|
|
if path == "/key-guide.html":
|
|
return self.send(200, keyguide.key_guide(site.load()), "text/html; charset=utf-8")
|
|
if path == "/cheat.html":
|
|
return self.send(200, keyguide.cheat_sheet(site.load(), self.my_address()), "text/html; charset=utf-8")
|
|
if path == "/where.html":
|
|
return self.send(200, keyguide.where_things_are(site.load(), site.data_dir, self.my_address()), "text/html; charset=utf-8")
|
|
if path == "/api/config":
|
|
cfg = site.load()
|
|
return self.send(200, {"config": schema.mask_secrets(cfg), "sha": site.cfg_sha, "mtime": site.cfg_mtime})
|
|
if path == "/api/schema":
|
|
return self.send(200, {"fields": schema.F, "checks": schema.CHECKS, "fix_kinds": list(schema.FIX_KINDS),
|
|
"fix_kind_fields": schema.FIX_KIND_FIELDS, "glossary": schema.GLOSSARY})
|
|
if path == "/api/overview":
|
|
return self.send(200, site.overview())
|
|
if path == "/api/companion":
|
|
return self.send(200, site.companion_view())
|
|
if path == "/api/machines":
|
|
return self.send(200, site.machines_view())
|
|
if path == "/api/gear":
|
|
return self.send(200, site.gear_view())
|
|
if path == "/api/keys":
|
|
return self.send(200, site.keys_view())
|
|
if path == "/api/history":
|
|
return self.send(200, site.history_view())
|
|
if path == "/api/facts":
|
|
return self.send(200, {"facts": COMPANION_FACTS})
|
|
if path == "/api/snapshot":
|
|
return self.send(200, site.read_json("snapshot.json", {}) or {})
|
|
if path.startswith("/api/logs/"):
|
|
name = path[len("/api/logs/"):]
|
|
if name == "autofix":
|
|
folder = os.path.expanduser((site.cfg.get("auto_fix") or {}).get("log_dir", "~/Desktop/Auto-Fix Logs"))
|
|
files = sorted(f for f in (os.listdir(folder) if os.path.isdir(folder) else []) if sp.AutoLog.NAME.match(f))
|
|
want = q.get("file", [files[-1] if files else ""])[0]
|
|
if want and want in files:
|
|
return self.send(200, {"files": files, "file": want, "lines": tail(os.path.join(folder, want), 1000)})
|
|
return self.send(200, {"files": files, "file": None, "lines": []})
|
|
if name not in LOGS:
|
|
return self.send(404, {"error": "no such log"})
|
|
flt = q.get("filter", [""])[0]
|
|
return self.send(200, {"lines": tail(LOGS[name], int(q.get("lines", ["400"])[0]), LOG_FILTERS.get(flt))})
|
|
return self.send(404, {"error": "not found"})
|
|
except Exception as e:
|
|
log.exception("GET %s", path)
|
|
return self.send(500, {"error": str(e)})
|
|
|
|
def do_POST(self):
|
|
if not allowed_client(self.client_address[0]):
|
|
return self.send(403, {"error": "only from the church network"})
|
|
path = urllib.parse.urlparse(self.path).path
|
|
length = int(self.headers.get("Content-Length") or 0)
|
|
body = {}
|
|
if length:
|
|
try:
|
|
body = json.loads(self.rfile.read(length) or b"{}")
|
|
except ValueError:
|
|
return self.send(400, {"error": "bad JSON"})
|
|
host = (self.headers.get("Host") or "").split(":")[0]
|
|
origin = self.headers.get("Origin")
|
|
if origin and urllib.parse.urlparse(origin).hostname not in (host, "127.0.0.1", "localhost"):
|
|
return self.send(403, {"error": "wrong origin"})
|
|
site = self.site
|
|
try:
|
|
if path == "/api/machines/probe":
|
|
if site.read_json("fixes-running.json", {}):
|
|
return self.send(409, {"error": "a fix is running; try again in a minute"})
|
|
with site.lock:
|
|
if site.probing and site.probing.is_alive():
|
|
return self.send(200, {"started": False, "already": True})
|
|
only = body.get("only")
|
|
site.probing = threading.Thread(target=probes.probe_all, args=(site.load(), site.data_dir, only), daemon=True)
|
|
site.probing.start()
|
|
return self.send(200, {"started": True})
|
|
if path == "/api/editing/check":
|
|
return self.send(200, site.check_code(self.client_address[0], str(body.get("code", ""))))
|
|
return self.send(404, {"error": "not found"})
|
|
except Exception as e:
|
|
log.exception("POST %s", path)
|
|
return self.send(500, {"error": str(e)})
|
|
|
|
def static(self, name, ctype):
|
|
try:
|
|
with open(os.path.join(UI, name), "rb") as f:
|
|
return self.send(200, f.read(), ctype)
|
|
except OSError:
|
|
return self.send(404, {"error": "missing %s" % name})
|
|
|
|
def my_address(self):
|
|
cfg = self.site.cfg
|
|
host = cfg.get("site", {}).get("ingest_hostname") or (self.headers.get("Host") or "").split(":")[0] or "127.0.0.1"
|
|
return "http://%s:%d/" % (host, self.server.server_address[1])
|
|
|
|
|
|
class Server(http.server.ThreadingHTTPServer):
|
|
daemon_threads = True
|
|
allow_reuse_address = True
|
|
|
|
|
|
def serve(data_dir, port, binds=None, config_path=None):
|
|
site = Site(data_dir, config_path)
|
|
Handler.site = site
|
|
binds = binds or bind_addresses(site.cfg)
|
|
servers = []
|
|
for addr in binds:
|
|
try:
|
|
srv = Server((addr, port), Handler)
|
|
except OSError as e:
|
|
log.warning("cannot listen on %s:%d: %s", addr, port, e)
|
|
continue
|
|
threading.Thread(target=srv.serve_forever, name="http-" + addr, daemon=True).start()
|
|
servers.append(srv)
|
|
log.info("listening on http://%s:%d/", addr, port)
|
|
if not servers:
|
|
raise SystemExit("could not listen anywhere")
|
|
return site, servers
|
|
|
|
|
|
def rebind_loop(site, servers, port, stop):
|
|
"""Every minute: if a LAN port's address changed, listen there too (DHCP, a cable moved)."""
|
|
have = {s.server_address[0] for s in servers}
|
|
while not stop.wait(60):
|
|
try:
|
|
for addr in bind_addresses(site.load()):
|
|
if addr not in have:
|
|
srv = Server((addr, port), Handler)
|
|
threading.Thread(target=srv.serve_forever, name="http-" + addr, daemon=True).start()
|
|
servers.append(srv)
|
|
have.add(addr)
|
|
log.info("now also listening on http://%s:%d/", addr, port)
|
|
except Exception as e:
|
|
log.warning("rebind: %s", e)
|
|
|
|
|
|
def main(argv=None):
|
|
ap = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
ap.add_argument("--data-dir", default=PARENT)
|
|
ap.add_argument("--config")
|
|
ap.add_argument("--port", type=int)
|
|
ap.add_argument("--bind", nargs="*")
|
|
ap.add_argument("--log", default=os.path.expanduser("~/Library/Logs/status-settings.log"))
|
|
args = ap.parse_args(argv)
|
|
handler = logging.handlers.RotatingFileHandler(args.log, maxBytes=1_000_000, backupCount=2)
|
|
handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s"))
|
|
log.addHandler(handler)
|
|
if sys.stderr.isatty():
|
|
log.addHandler(logging.StreamHandler())
|
|
log.setLevel(logging.INFO)
|
|
cfg = sp.load_config(args.config or os.path.join(args.data_dir, "config.json"))
|
|
port = args.port or cfg.get("site", {}).get("settings_port", 8020)
|
|
site, servers = serve(args.data_dir, port, args.bind, args.config)
|
|
stop = threading.Event()
|
|
threading.Thread(target=rebind_loop, args=(site, servers, port, stop), daemon=True).start()
|
|
try:
|
|
while True:
|
|
time.sleep(3600)
|
|
except KeyboardInterrupt:
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|