From 3976667bba0632bc622a5897557abfa5b9a1f73d Mon Sep 17 00:00:00 2001 From: christian Date: Tue, 20 Oct 2026 14:31:32 -0400 Subject: [PATCH] Other campuses set themselves up: one install line, one join line per computer, keys made from what is found settings/discover.py reads Companion's connections (and their addresses and channel counts), the Dante devices, the network MIDI sessions here and on joined Macs, this Mac's wired network, and what runs on each joined computer. settings/autoconf.py turns that into keys, checks, start and restart fixes and a page layout the way North Attleboro's were made (group keys only when not everything fits; receivers fold first), all through the existing builders and rules. settings/campus.py prepares Companion through its internal API (the panel's custom variables and the blink trigger), picks free pages, applies the first config, serves the program kit, the campus install line and the Mac/PC join lines, and looks every ten minutes to add new gear (discover.auto; never during a service or a recording, never removes, never re-adds what someone took off). Settings > Auto-setup shows what was found and the lines to paste. install.sh keeps the kit and makes the panel's login key. Checked on North Attleboro read-only: discovery finds all 43 connections, 19 Dante devices, four Macs' MIDI sessions and seven computers, and a from-scratch build reproduces NA's layout. The Companion calls were tried with a throwaway variable and a disabled trigger, then removed. Co-Authored-By: Claude Opus 5.5 --- README.md | 33 +++ install.sh | 11 + schema.py | 3 + settings/autoconf.py | 517 +++++++++++++++++++++++++++++++++++++++++++ settings/builders.py | 5 + settings/campus.py | 429 +++++++++++++++++++++++++++++++++++ settings/discover.py | 312 ++++++++++++++++++++++++++ settings/server.py | 44 ++++ settings/ui/app.js | 28 ++- tests/test_campus.py | 247 +++++++++++++++++++++ tests/test_schema.py | 2 +- 11 files changed, 1629 insertions(+), 2 deletions(-) create mode 100644 settings/autoconf.py create mode 100644 settings/campus.py create mode 100644 settings/discover.py create mode 100644 tests/test_campus.py diff --git a/README.md b/README.md index f9d18a7..89d8e7c 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,39 @@ Fixes and the remote checks are built from fields (`settings/builders.py`): a bu The page reads what the watchdog writes next to its config (`snapshot.json` every 5 s, `started.json`, `live-now.json`, `fixes-running.json`) instead of asking the gear again. Its own look at the computers (`settings/probes.py`) runs fixed read-only commands over the same login key, fetches whole lists and matches them here, never while a fix runs. +## Other campuses: auto-setup + +A campus sets itself up. All a person does is let the panel in: one line on the campus's Mac that runs Companion, +one line on each other computer. + +1. **The Companion Mac.** Terminal, paste the campus's line from Settings › Auto-setup on any running campus (North + Attleboro's page shows the Apollo Beach, Southcoast and Woonsocket lines), for example + `curl -fsSL "http://10.1.1.1:8020/campus/install?id=ab" | zsh`. It downloads the program from that page (code only, + never that campus's settings or keys), installs it, makes the panel's login key, and runs + `python3 -m settings.campus setup`: Companion gets the panel's custom variables and the blink-clock trigger (through + Companion's own internal API, as its web UI does), the first two pages from 19 that hold nothing made by hand are + used, the gear is found and the config is applied like any Apply. It prints the editing PIN. North Attleboro's key is + added so it can help remotely (turn on Remote Login for that). Running the line again updates the program. +2. **Each other computer.** Mac: Terminal, `curl -fsSL http://:8020/join | sh` (adds the key, turns on + Remote Login when it can, says it is here). PC: PowerShell as administrator, `irm http://:8020/join.ps1 | iex` + (OpenSSH, the key, a StatusPanel start task for each known program running on it, Wake-on-LAN, then says it is here). + The lines are on that campus's Settings › Auto-setup. + +**What gets a key** (`settings/autoconf.py`): each Companion connection of a known kind (HyperDecks, Tascam, consoles, +Dante with every device it sees, Shure receivers and PSMs with their channel count, ATEM, Videohub, Resolume, NovaStar, +OBS, Resi, ProPresenter, Ableton/AbleSet), each known program running on a joined computer (ProPresenter, Ableton, +Arena, OBS, Church Translate, Pro Tools on a Mac; Vista, Arena, OBS, ProPresenter on a PC), and every network MIDI peer at +its wired address (with the wired-network rule). A program's key also carries its Companion connection, its start fix +(auto-fixable) and a Restart page key. rtpMIDI rides inside the MIDI key. When everything fits there are no group keys; +when it does not, REC AUDIO RF SHOW VIDEO STREAM group keys take the top row and wireless receivers fold into RF first. +Generic connections and unknown programs are listed under Settings › Auto-setup, not guessed at. + +**After that** (`discover.auto`, "add new gear by itself", on for a new campus, off at North Attleboro): the settings page +looks every ten minutes and soon after a computer joins, and adds what is new (a key in the first free cell, a Dante +device or MIDI link into its key). Never during the auto-fix hours or a recording; nothing is ever removed; a thing that +was there before and was taken off by someone is not added back (`discover-seen.json`). "Look now" adds what is new at +once. Everything it makes is ordinary settings, changed on the page like North Attleboro's. + ## One-time Companion setup 1. **Variables → Custom Variables:** add three variables. Leave the start values empty and "persist" off: diff --git a/install.sh b/install.sh index 933c2aa..f897d6a 100644 --- a/install.sh +++ b/install.sh @@ -212,6 +212,17 @@ fi if [[ -d "$SRC/companion-objects" ]]; then rm -rf "$DEST/companion-objects" && cp -R "$SRC/companion-objects" "$DEST/" fi +# The kit another campus installs from (the settings page serves it as /campus/program.tgz). +rm -rf "$DEST/kit" && mkdir -p "$DEST/kit/sites" +cp "$SRC/install.sh" "$DEST/kit/" +[[ -f "$SRC/README.md" ]] && cp "$SRC/README.md" "$DEST/kit/" +[[ -d "$SRC/sites/_template" ]] && cp -R "$SRC/sites/_template" "$DEST/kit/sites/" +# The panel's own login key for the other computers (made once, never replaced). +if [[ ! -f "$HOME/.ssh/id_status_panel" ]]; then + mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh" + /usr/bin/ssh-keygen -q -t ed25519 -N "" -C "status-panel@$(scutil --get LocalHostName 2>/dev/null || hostname -s)" -f "$HOME/.ssh/id_status_panel" + echo "made the panel's login key ~/.ssh/id_status_panel" +fi # 4. The LaunchAgent, with the exit timeout this config needs (a running fix finishes before a restart). T=$(date +%s) diff --git a/schema.py b/schema.py index b16d91f..1bee633 100644 --- a/schema.py +++ b/schema.py @@ -112,6 +112,9 @@ field("auto_fix.windows", "list", [], "When auto-fix may act.", "Schedules") field("auto_fix.windows[].days", "days", None, "", "Schedules", required=True) field("auto_fix.windows[].start", "hhmm", None, "", "Schedules", required=True) field("auto_fix.windows[].end", "hhmm", None, "", "Schedules", required=True) +field("discover", "dict", {}, "Auto-setup: this Mac looks for gear every ten minutes.", "Site", advanced=True) +field("discover.auto", "bool", False, "Add new gear by itself: a new Companion connection, Dante device, MIDI link, computer " + "or program gets its key (never during a service or a recording; nothing is ever removed).", "Site") field("auto_fix.disabled", "bool", False, "Auto-fix switched off as a whole: nothing is fixed by itself (holds " "still work).", "Schedules") field("auto_fix.after_seconds", "int", 20, "A fault must last this long before auto-fix acts.", "Schedules", diff --git a/settings/autoconf.py b/settings/autoconf.py new file mode 100644 index 0000000..5fd112c --- /dev/null +++ b/settings/autoconf.py @@ -0,0 +1,517 @@ +"""Auto-setup: what a campus has (settings/discover.py's inventory) becomes its keys, checks, fixes +and page layout - the same building blocks North Attleboro's were made of by hand. + +Pure functions, no I/O: + + build(base, inv, campus, pin) a whole config for a new campus, from the empty template + extend(cfg, inv, seen) a running campus: only what is new (never removes anything) + +Everything that comes out passes schema.validate and can be changed on the settings page like any +hand-made setting. Gear auto-setup does not know (a generic connection, an app it has no recipe +for) is reported, not guessed at. +""" +import copy +import ipaddress +import re + +from settings import PARENT # noqa: F401 (puts the project folder on sys.path) +from settings import builders as b +import status_panel as sp + +# --- what auto-setup knows ------------------------------------------------------------- + +# Group keys sum up their members on a big campus, in this order along the top row. +GROUPS = ["REC", "AUDIO", "RF", "SHOW", "VIDEO", "STREAM"] +# Who gets a cell of its own when not everything fits (the rest show inside their group key only): +# a row of wireless receivers folds first, as at North Attleboro. +PRIORITY = ["REC", "SHOW", "AUDIO", "STREAM", "VIDEO", "RF"] + +# Companion modules: which group, what kind of key, the key's text when there is only one. +COMPANION = { + "bmd-hyperdeck": {"group": "REC", "kind": "hyperdeck", "label": "HD"}, + "tascam-da-6400": {"group": "REC", "kind": "tascam", "label": "TASCAM"}, + "audinate-dantecontroller": {"group": "AUDIO", "kind": "dante", "label": "DANTE"}, + "behringer-x32": {"group": "AUDIO", "kind": "conn", "label": "X32"}, + "behringer-wing": {"group": "AUDIO", "kind": "conn", "label": "WING"}, + "midas-m32": {"group": "AUDIO", "kind": "conn", "label": "M32"}, + "allenheath-sq": {"group": "AUDIO", "kind": "conn", "label": "SQ"}, + "allenheath-dlive": {"group": "AUDIO", "kind": "conn", "label": "DLIVE"}, + "allenheath-avantis": {"group": "AUDIO", "kind": "conn", "label": "AVANTIS"}, + "yamaha-rcp": {"group": "AUDIO", "kind": "conn", "label": "YAMAHA"}, + "soundcraft-ui": {"group": "AUDIO", "kind": "conn", "label": "UI24"}, + "digico-sd": {"group": "AUDIO", "kind": "conn", "label": "DIGICO"}, + "shure-wireless": {"group": "RF", "kind": "shure", "label": "MICS"}, + "shure-psm1000": {"group": "RF", "kind": "psm", "label": "PSM"}, + "renewedvision-propresenter-api": {"group": "SHOW", "kind": "conn", "label": "PROP", "app": "propresenter"}, + "ableton-osc": {"group": "SHOW", "kind": "conn", "label": "ABLETON", "app": "ableton"}, + "leolabs-ableset": {"group": "SHOW", "kind": "conn", "label": "ABLESET", "app": "ableton"}, + "bmd-atem": {"group": "VIDEO", "kind": "conn", "label": "ATEM"}, + "bmd-videohub": {"group": "VIDEO", "kind": "conn", "label": "VIDEOHUB"}, + "resolume-arena": {"group": "VIDEO", "kind": "conn", "label": "ARENA", "app": "arena"}, + "novastar-controller": {"group": "VIDEO", "kind": "conn", "label": "LED", "ping": True}, + "obs-studio": {"group": "STREAM", "kind": "conn", "label": "OBS", "app": "obs"}, + "resi-decoders": {"group": "STREAM", "kind": "conn", "label": "RESI", "ping": True}, +} + +# Programs on the computers: by OS, the process (Mac, as pgrep sees it) or program file (Windows). +# "app" ties a program to a Companion connection of the same name; "restart" adds a Restart page key. +MAC_APPS = [ + {"app": "propresenter", "process": "ProPresenter", "label": "PROP", "open": "/Applications/ProPresenter*.app", "restart": True}, + {"app": "ableton", "process": "Live", "label": "ABLETON", "open": "/Applications/Ableton Live*.app", "ableton": True, "restart": True}, + {"app": "arena", "process": "Arena", "label": "ARENA", "open": "/Applications/Resolume Arena*/Arena.app", "restart": True}, + {"app": "obs", "process": "obs", "label": "OBS", "open": "/Applications/OBS.app", "restart": True}, + {"app": "translate", "process": "Church Translate", "label": "TRANSLATE", "open": "/Applications/Church Translate.app", "restart": True}, + {"app": "protools", "process": "Pro Tools", "label": "PRO TOOLS", "open": "/Applications/Pro Tools.app", "restart": True}, +] +WIN_APPS = [ + {"app": "vista", "exe": "Vista.exe", "label": "VISTA", "task": "Start Vista", "restart": True}, + {"app": "arena", "exe": "Arena.exe", "label": "ARENA", "task": "Start Arena", "restart": True}, + {"app": "obs", "exe": "obs64.exe", "label": "OBS", "task": "Start OBS", "restart": True}, + {"app": "propresenter", "exe": "ProPresenter.exe", "label": "PROP", "task": "Start ProPresenter", "restart": True}, + {"app": "rtpmidi", "exe": "rtpMIDI.exe", "label": "RTPMIDI", "task": "Start rtpMIDI", "restart": False, "fold": "midi_wired"}, +] +# The task names the Windows join script makes, by program file (it reads WIN_APPS through campus.py). +WIN_TASKS = {a["exe"]: a["task"] for a in WIN_APPS} + +# Fixes shared by every key of a kind (the same words North Attleboro's carry). +SHARED_FIXES = { + "hyperdeck": {"kind": "hyperdeck", "template": "hyperdeck", "params": {}, "short": "HYPERDECK", "verify_seconds": 15, + "hung_note": "STILL OFFLINE\nHOLD AGAIN", + "label": "turn Companion's connection to that HyperDeck off and on (what gets a restarted deck green); never reboots the deck"}, + "midi_reconnect": {"kind": "midi_connect", "template": "midi_connect", "params": {}, "short": "RECONNECT", + "label": "reconnect the peer to the network MIDI session it dropped out of (like Connect in Audio MIDI Setup), at its wired address"}, +} + +STATUS_ROWS, COLS = 4, 8 +DEFAULT_AUTO_FIX = {"after_seconds": 20, "max_tries": 2, "retry_seconds": 90, "pause_after_network_seconds": 60, + "restart_connections": True, "windows": [{"days": ["sun"], "start": "05:00", "end": "14:00"}]} + + +# --- small helpers --------------------------------------------------------------------- + +def slug(text, taken=()): + base = re.sub(r"[^a-z0-9]+", "_", str(text).lower()).strip("_") or "key" + out, n = base, 2 + while out in taken: + out, n = "%s_%d" % (base, n), n + 1 + return out + + +def key_text(label, limit=10): + t = re.sub(r"[_\s]+", " ", str(label)).strip().upper() + return t if len(t) <= limit else t[:limit].rstrip() + + +def tag_for(name): + return key_text(name, 12) + + +def ip_of(address): + ip = sp.peer_ip(address) + return str(ip) if ip else None + + +def in_nets(nets, address): + ip = sp.peer_ip(address) + return ip is not None and any(ip in ipaddress.ip_network(n, strict=False) for n in nets) + + +def computer_by_ip(inv, ip): + for alias, c in (inv.get("computers") or {}).items(): + if ip and ip in (c.get("ips") or []): + return alias + return None + + +def referenced(cfg): + """Everything a config already watches, by the same identities discovery uses.""" + out = set() + for t in sp.all_tiles(cfg): + for c in t.get("checks", []): + if c.get("conn"): + out.add("conn:" + c["conn"]) + if c.get("type") == "dante": + out.update("dante:" + d for d in (c.get("devices") or {})) + if c.get("type") == "midi_peer": + out.add("midi:%s|%s|%s" % (c.get("host", ""), c.get("session"), c.get("peer"))) + if c.get("type") == "midi_wired": + out.add("midi_wired") + p = c.get("params") or {} + if c.get("type") == "remote" and p.get("host") and (p.get("process") or p.get("exe")): + out.add("app:%s|%s" % (p["host"], p.get("process") or p.get("exe"))) + for alias, h in (cfg.get("hosts") or {}).items(): + out.add("host:" + alias) + ip = str(h.get("ssh", "")).split("@")[-1] + if ip: + out.add("hostip:" + ip) + return out + + +# --- from the inventory to items ------------------------------------------------------ + +def items(inv, nets): + """[{ident, group, label, checks, fixes, restart, also_idents}] for everything auto-setup can watch, + plus the computer records ({alias: host}). Each item is one key.""" + out, hosts = [], {} + comps = inv.get("computers") or {} + for alias, c in comps.items(): + if not c.get("reachable"): + continue + hosts[alias] = {"name": c.get("name") or alias, "os": c.get("os", "mac"), "ssh": "%s@%s" % (c["user"], c["ip"]), + "role": "found by auto-setup", "runs": [{"kind": "sshd"}]} + if c.get("macs"): + hosts[alias]["mac"] = c["macs"][0] + # Programs on the computers, by app, so a Companion connection to the same program joins its key. + apps_at = {} # (app, ip) -> item + for alias, c in comps.items(): + if not c.get("reachable"): + continue + procs = set(c.get("procs") or []) + catalog = MAC_APPS if c.get("os", "mac") == "mac" else WIN_APPS + for a in catalog: + if c.get("os", "mac") == "mac": + if a["process"] not in procs: + continue + app_path = newest_app(c.get("apps") or [], a["open"]) + if not app_path: + continue + check = {"type": "remote", "template": "mac_program", "params": {"host": alias, "process": a["process"]}, + "msg": "%s NOT RUNNING" % a["label"]} + if a.get("ableton"): + start = {"template": "ableton", "params": {"host": alias, "app": app_path.rsplit("/", 1)[-1][:-4], "restart": False}} + again = {"template": "ableton", "params": {"host": alias, "app": app_path.rsplit("/", 1)[-1][:-4], "restart": True}} + else: + start = {"template": "mac_start", "params": {"host": alias, "process": a["process"], "app": app_path}} + again = {"template": "mac_restart", "params": {"host": alias, "process": a["process"], "app": app_path, + "wait_seconds": 10, "force": True}} + ident = "app:%s|%s" % (alias, a["process"]) + else: + if a["exe"] not in procs or a["task"] not in (c.get("tasks") or []): + continue # no start task on that PC: a hold could not start it + check = {"type": "remote", "template": "win_program", "params": {"host": alias, "exe": a["exe"]}, + "msg": "%s NOT RUNNING" % a["label"]} + start = {"template": "win_start", "params": {"host": alias, "task": a["task"], "exe": a["exe"]}} + again = {"template": "win_restart", "params": {"host": alias, "task": a["task"], "exe": a["exe"]}} + ident = "app:%s|%s" % (alias, a["exe"]) + fid = slug("%s_%s" % (a["app"], alias)) + pretty = {"propresenter": "ProPresenter", "ableton": "Ableton", "arena": "Resolume Arena", "obs": "OBS", "translate": "Church Translate", + "protools": "Pro Tools", "vista": "Vista", "rtpmidi": "rtpMIDI"}.get(a["app"], a["label"].title()) + where = c.get("name") or alias + start = dict(start, label="start %s on %s if it is not running" % (pretty, where), short=a["label"]) + again = dict(again, label="restart %s on %s (closes it first)" % (pretty, where), short=a["label"]) + item = {"ident": ident, "group": "SHOW" if a["app"] not in ("obs",) else "STREAM", "label": a["label"], + "checks": [dict(check, fix=fid)], "fixes": {fid: start}, "restart": [], "also": [], "fold": a.get("fold")} + if a.get("restart"): + rid = "restart_" + fid + item["fixes"][rid] = again + item["restart"].append({"label": a["label"], "fix": rid}) + if a["app"] == "arena": + item["group"] = "VIDEO" + out.append(item) + apps_at[(a["app"], c.get("ip"))] = item + for ip in c.get("ips") or []: + apps_at.setdefault((a["app"], ip), item) + # Companion connections. + counts = {} + for conn in inv.get("connections") or []: + counts[conn.get("module")] = counts.get(conn.get("module"), 0) + 1 + hd_n = 0 + for conn in sorted(inv.get("connections") or [], key=lambda c: c["label"]): + spec = COMPANION.get(conn.get("module")) + if not spec or not conn.get("enabled", True): + continue + label = conn["label"] + ident = "conn:" + label + many = counts.get(conn["module"], 0) > 1 + text = spec["label"] if not many else key_text(label) + if spec["kind"] == "hyperdeck": + hd_n += 1 + text = "HD%d" % hd_n if many else "HD" + out.append({"ident": ident, "group": "REC", "label": text, "restart": [], "also": [], + "checks": [{"type": "hyperdeck", "conn": label, "fix": {"conn": "hyperdeck"}}], + "fixes": {"hyperdeck": dict(SHARED_FIXES["hyperdeck"])}}) + continue + if spec["kind"] == "tascam": + fid = slug("reconnect_" + label) + out.append({"ident": ident, "group": "REC", "label": text, "restart": [], "also": [], + "checks": [{"type": "tascam", "conn": label, "fix": fid}, {"type": "ping", "conn": label, "name": text}], + "fixes": {fid: {"kind": "reconnect", "template": "reconnect", "params": {}, "conn": label, "short": text, + "verify_seconds": 20, "label": "reconnect Companion to %s" % label}}}) + continue + if spec["kind"] == "dante": + devices = {d: key_text(d, 12) for d in (inv.get("dante") or {}).get("devices") or []} + out.append({"ident": ident, "group": "AUDIO", "label": "DANTE", "restart": [], "also": ["dante:" + d for d in devices], + "checks": [{"type": "dante", "conn": label, "devices": devices, "fix": {"*": sp.DANTE_AUTO}}], "fixes": {}}) + continue + if spec["kind"] in ("shure", "psm"): + n = int(conn.get("channels") or (4 if spec["kind"] == "shure" else 2)) + out.append({"ident": ident, "group": "RF", "label": text if many else ("MICS" if spec["kind"] == "shure" else "PSM"), + "restart": [], "also": [], + "checks": [{"type": spec["kind"], "conn": label, "channels": n}, {"type": "ping", "conn": label}], "fixes": {}}) + continue + checks = [{"type": "conn", "conn": label}] + if spec.get("ping") and conn.get("host"): + checks.append({"type": "ping", "conn": label}) + app_item = apps_at.get((spec.get("app"), ip_of(conn.get("host")))) if spec.get("app") else None + if app_item: # the program's own key carries its connection too + app_item["checks"] = checks + app_item["checks"] + app_item["also"].append(ident) + continue + out.append({"ident": ident, "group": spec["group"], "label": text, "checks": checks, "fixes": {}, "restart": [], "also": []}) + # Network MIDI: every peer in the sessions here and on the other Macs, at its wired address. + midi_checks = [] + for host, sessions in (inv.get("midi") or {}).items(): + for sname, sess in (sessions or {}).items(): + for p in sess.get("peers") or []: + addr = p.get("address") or "" + if not in_nets(nets, addr): + alias = computer_named(inv, p.get("name")) + ip = alias and next((i for i in comps[alias].get("ips") or [] if in_nets(nets, i)), None) + if not ip: + continue # no wired address known: the wired check will name it + addr = "%s:5004" % ip + chk = {"type": "midi_peer", "session": sname, "peer": p["name"], "address": addr, "fix": "midi_reconnect", + "tag": tag_for(p["name"])} + if host: + chk["host"] = host + midi_checks.append(("midi:%s|%s|%s" % (host, sname, p["name"]), chk)) + if midi_checks: + checks = [c for _, c in midi_checks] + [{"type": "midi_wired", "nets": list(nets), "fix": "midi_reconnect"}] + out.append({"ident": "midi_wired", "group": "SHOW", "label": "MIDI", "checks": checks, + "fixes": {"midi_reconnect": dict(SHARED_FIXES["midi_reconnect"])}, "restart": [], + "also": [i for i, _ in midi_checks]}) + # A helper program (rtpMIDI) rides inside the key it serves when that key exists. + for it in [x for x in out if x.get("fold")]: + home = next((x for x in out if x["ident"] == it["fold"]), None) + if home is not None: + home["checks"] += it["checks"] + home["fixes"].update(it["fixes"]) + home["also"].append(it["ident"]) + out.remove(it) + return out, hosts + + +def computer_named(inv, name): + n = str(name or "").lower() + for alias, c in (inv.get("computers") or {}).items(): + if n and n in (str(c.get("name", "")).lower(), str(c.get("hostname", "")).lower().replace(".local", "")): + return alias + return None + + +def newest_app(apps, pattern): + """The newest /Applications entry matching a glob-ish pattern ("/Applications/ProPresenter*.app").""" + rx = re.compile("^" + re.escape(pattern).replace(r"\*", ".*") + "$") + hits = sorted(a for a in apps if rx.match(a)) + return hits[-1] if hits else None + + +# --- layout ---------------------------------------------------------------------------- + +def free_cells(taken, rows=STATUS_ROWS): + return [(r, c) for r in range(rows) for c in range(1, COLS) if (r, c) not in taken] + + +def layout(cfg, new_tiles, restart_items): + """Put the keys on the Status page: everything fits, so no group keys; otherwise group keys along + the top row and the members after them (a member with no room left shows inside its group only).""" + fixed = {tuple(t["pos"]) for t in cfg["tiles"] if t.get("pos")} + if restart_items: + fixed.add(tuple(cfg["restart_page"]["jump_pos"])) + cells = free_cells(fixed) + groups = [g for g in GROUPS if any(t["_group"] == g for t in new_tiles)] + if len(new_tiles) > len(cells) and len(groups) > 1: + for g in groups: + members = [t["id"] for t in new_tiles if t["_group"] == g] + cfg["tiles"].append({"id": slug(g, {t["id"] for t in cfg["tiles"]} | {t["id"] for t in new_tiles}), + "label": g, "members": members, "_group": g}) + group_tiles = [t for t in cfg["tiles"] if t.get("_group")] + for t in group_tiles: + if cells: + t["pos"] = list(cells.pop(0)) + # Who gets a cell: by priority while cells last; then placed in group order, so a group reads as a block. + chosen = set() + for g in PRIORITY: + for t in new_tiles: + if t["_group"] == g and len(chosen) < len(cells): + chosen.add(t["id"]) + for t in new_tiles: + if t["id"] in chosen: + t["pos"] = list(cells.pop(0)) + cfg["tiles"].append(t) + for t in cfg["tiles"]: + t.pop("_group", None) + + +def place_new(cfg, tile): + """One more key on a running campus: the first free cell, else inside its group, else on ALL only.""" + taken = {tuple(t["pos"]) for t in cfg["tiles"] if t.get("pos")} + if cfg.get("restart_page"): + taken.add(tuple(cfg["restart_page"]["jump_pos"])) + cells = free_cells(taken) + group = next((t for t in cfg["tiles"] if t.get("label") == tile.get("_group") and t.get("members") is not None), None) + if cells: + tile["pos"] = list(cells[0]) + if group is not None: + group["members"].append(tile["id"]) + tile.pop("_group", None) + cfg["tiles"].append(tile) + + +def restart_layout(cfg, restart_items): + if not restart_items: + cfg.pop("restart_page", None) + return + rp = cfg["restart_page"] + taken = {tuple(rp["back_pos"])} | {tuple(k["pos"]) for k in rp.get("keys", [])} + cells = free_cells(taken) + ids = {k["id"] for k in rp.get("keys", [])} + for it in restart_items: + kid = slug("rs_" + it["label"], ids) + ids.add(kid) + if not cells: + break + rp.setdefault("keys", []).append({"id": kid, "label": key_text(it["label"]), "fix": it["fix"], "pos": list(cells.pop(0))}) + + +# --- the two entry points -------------------------------------------------------------- + +def nets_of(inv): + w = (inv.get("ingest") or {}).get("wired") or {} + return [w["net"]] if w.get("net") else [] + + +def build(base, inv, campus, pin, pages=(19, 20)): + """A whole config for a new campus: the template, this Mac, and a key for everything found.""" + cfg = copy.deepcopy(base) + ing = inv.get("ingest") or {} + wired = ing.get("wired") or {} + site = cfg["site"] + site.update({"id": campus["id"], "name": campus["name"], "ingest_name": ing.get("name") or "%s Ingest" % campus["name"], + "ingest_hostname": ing.get("hostname") or "", "ingest_ips": [ip for ip in [wired.get("ip")] + list(ing.get("ips") or []) if ip], + "editing_code": pin}) + site["ingest_ips"] = list(dict.fromkeys(site["ingest_ips"])) + cfg["discover"] = {"auto": True} + cfg["page"], cfg["page_name"] = pages[0], "%d\\nStatus" % pages[0] + cfg["restart_page"] = {"page": pages[1], "page_name": "%d\\nRestart" % pages[1], "jump_pos": [3, 6], "back_pos": [3, 0], "keys": []} + cfg["network"] = dict(cfg.get("network") or {}, interface=wired.get("interface") or "en0", gateway=wired.get("gateway") or "") + cfg["service_windows"] = [{"days": list(sp.DAYS), "start": "00:00", "end": "23:59"}] + cfg["hosts"]["ingest"]["name"] = site["ingest_name"] + for t in cfg["tiles"]: + if t.get("kind") == "network": + t["pos"] = [3, 7] + found, hosts = items(inv, nets_of(inv)) + cfg["hosts"].update(hosts) + tiles, restart, taken = [], [], {t["id"] for t in cfg["tiles"]} + for it in found: + tid = slug(it["label"], taken) + taken.add(tid) + tiles.append({"id": tid, "label": key_text(it["label"]), "checks": it["checks"], "_group": it["group"]}) + for fid, f in it["fixes"].items(): + cfg["fixes"].setdefault(fid, f) + restart += it["restart"] + tiles.sort(key=lambda t: GROUPS.index(t["_group"])) + layout(cfg, tiles, restart) + restart_layout(cfg, restart) + finish(cfg) + a = dict(DEFAULT_AUTO_FIX, **{k: v for k, v in (cfg.get("auto_fix") or {}).items() if k not in ("windows", "fixes")}) + a["fixes"] = gentle(cfg) + cfg["auto_fix"] = a + return cfg, [it["ident"] for it in found] + ["host:" + h for h in hosts] + [x for it in found for x in it["also"]] + + +def extend(cfg, inv, seen): + """What is new since the last look, added to a running campus. Never removes or changes what is + there; a thing seen before (and so deliberately taken off by someone) is not added again. + Returns (new config, [what was added, in words], [identities now seen]).""" + cfg = copy.deepcopy(cfg) + have = referenced(cfg) + skip = have | set(seen) + nets = nets_of(inv) or [n for t in sp.all_tiles(cfg) for c in t.get("checks", []) if c.get("type") == "midi_wired" for n in c.get("nets", [])] + found, hosts = items(inv, nets) + added, now_seen = [], [] + for alias, h in hosts.items(): + if "host:" + alias in skip or "hostip:" + h["ssh"].split("@")[-1] in have: + continue + cfg["hosts"][alias] = h + added.append("computer %s" % h["name"]) + now_seen.append("host:" + alias) + taken = {t["id"] for t in sp.all_tiles(cfg)} + for it in found: + # A key that already exists for this thing: only its new parts (a Dante device, a MIDI peer). + if it["ident"] in skip: + fresh = [x for x in it["also"] if x not in skip] + if fresh: + added += merge_parts(cfg, it, fresh) + now_seen += fresh + continue + if any(c.get("params", {}).get("host") not in cfg["hosts"] for c in it["checks"] if c.get("template")): + continue + tid = slug(it["label"], taken) + taken.add(tid) + for fid, f in it["fixes"].items(): + cfg["fixes"].setdefault(fid, f) + place_new(cfg, {"id": tid, "label": key_text(it["label"]), "checks": it["checks"], "_group": it["group"]}) + if it["restart"] and cfg.get("restart_page"): + restart_layout(cfg, it["restart"]) + added.append("%s key" % key_text(it["label"])) + now_seen += [it["ident"]] + it["also"] + if added: + finish(cfg) + auto = cfg.setdefault("auto_fix", dict(DEFAULT_AUTO_FIX, fixes=[])) + auto["fixes"] = list(dict.fromkeys(list(auto.get("fixes") or []) + [f for f in gentle(cfg) if f not in auto.get("fixes", []) + and any(f in it["fixes"] for it in found)])) + return cfg, added, now_seen + + +def merge_parts(cfg, it, fresh): + """New Dante devices and MIDI peers go into the key that already watches the rest.""" + words = [] + for t in sp.all_tiles(cfg): + for c in t.get("checks", []): + if c.get("type") == "dante" and it["ident"] == "conn:" + c.get("conn", ""): + for x in fresh: + if x.startswith("dante:"): + name = x[6:] + c.setdefault("devices", {})[name] = key_text(name, 12) + words.append("Dante device %s" % name) + if it["ident"] == "midi_wired": + midi = next((t for t in cfg["tiles"] if any(c.get("type") == "midi_wired" for c in t.get("checks", []))), None) + if midi is not None: + wired_at = next(i for i, c in enumerate(midi["checks"]) if c.get("type") == "midi_wired") + for c in it["checks"]: + if c["type"] == "midi_peer" and "midi:%s|%s|%s" % (c.get("host", ""), c["session"], c["peer"]) in fresh: + midi["checks"].insert(wired_at, c) + wired_at += 1 + words.append("MIDI link %s" % c["peer"]) + cfg["fixes"].setdefault("midi_reconnect", dict(SHARED_FIXES["midi_reconnect"])) + return words + + +def finish(cfg): + """Fixes and remote checks built by their builders, words filled in, as the page would.""" + for fid, f in list(cfg["fixes"].items()): + if f.get("template") and "run" not in f and b.TEMPLATES.get(f["template"], {}).get("render"): + built = b.build(cfg, f) + built.setdefault("label", b.suggest_label(cfg, built) or fid) + short = key_text(fid.replace("restart_", "").split("_")[0], 10) + built.setdefault("short", short) + cfg["fixes"][fid] = built + elif f.get("template") and not f.get("label"): + f["label"] = b.suggest_label(cfg, f) or fid + for t in cfg["tiles"]: + for i, c in enumerate(t.get("checks", [])): + if c.get("template"): + t["checks"][i] = b.build_check(cfg, c) + + +def gentle(cfg): + """The fixes auto-fix may run by itself: starting what is closed and reconnecting - never a restart.""" + out = [] + for fid, f in cfg["fixes"].items(): + if fid.startswith("restart_") or f.get("restart_ok") or f.get("never_auto"): + continue + if f.get("kind") in ("hyperdeck", "midi_connect", "reconnect") or (f.get("template") in ("mac_start", "win_start", "ableton") + and not (f.get("params") or {}).get("restart")): + out.append(fid) + return out diff --git a/settings/builders.py b/settings/builders.py index 4d69a68..d18fa3f 100644 --- a/settings/builders.py +++ b/settings/builders.py @@ -322,6 +322,11 @@ CHECK_TEMPLATES = { "params": [P_HOST, ("process", "Process name", "As pgrep sees it.", "str", None)], "render": lambda p, cfg: "pgrep -xq %s" % q(p["process"]), }, + "win_program": { + "words": "A program is running on a Windows PC", "type": "remote", + "params": [P_HOST, ("exe", "Program file", "As Task Manager shows it (Vista.exe).", "str", None)], + "render": lambda p, cfg: win_running(win(p["exe"], "program file")), + }, "helper_flag": { "words": "A helper file does not say something is wrong", "type": "remote", "params": [P_HOST, ("file", "File", "In the helper's folder (solo-mode.json).", "str", "solo-mode.json"), diff --git a/settings/campus.py b/settings/campus.py new file mode 100644 index 0000000..74ff1b5 --- /dev/null +++ b/settings/campus.py @@ -0,0 +1,429 @@ +"""A campus that sets itself up. + +On the campus's Mac that runs Companion, one line installs the panel from another campus's settings +page and runs `setup`: Companion gets the panel's custom variables and blink clock, two free pages +are picked, the gear is found (settings/discover.py) and turned into keys (settings/autoconf.py), +and that config is applied like any Apply on the page. Each other computer runs one join line +(Terminal on a Mac, an administrator PowerShell on a PC); from then on the settings page looks again +every ten minutes and adds what is new by itself (Settings > Site: "Add new gear by itself"). + + python3 -m settings.campus setup --id ab [--name "Apollo Beach"] + python3 -m settings.campus look what it finds (nothing changes) + python3 -m settings.campus add add what is new now +""" +import argparse +import base64 +import gzip +import hashlib +import io +import json +import os +import random +import re +import tarfile +import threading +import time + +from settings import PARENT # noqa: F401 (puts the project folder on sys.path) +from settings import autoconf, discover, scripts +import push_page +import schema +import status_panel as sp + +CAMPUSES = {"na": "North Attleboro", "ab": "Apollo Beach", "sc": "Southcoast", "ws": "Woonsocket"} +VARIABLES = ("sp_state", "sp_hb", "sp_cmd", "flash_tick") +TRIGGER_NAME = "flash_tick_500" +LOOK_EVERY = 600 +SEEN, LAST = "discover-seen.json", "discover-last.json" +# What the program kit carries (no settings, keys or history of the campus that serves it). +KIT_FILES = ("status_panel.py", "readiness.py", "coremidi.py", "netgear.py", "schema.py", "make_page.py", "push_page.py", + "nav_keys.py", "atem_watch.js", "atem_state.js", "nav_templates.json") +KIT_DIRS = ("settings", "remote-access", "companion-objects") + + +# --- Companion: the panel's variables, its blink clock, and two free pages --------------- + +def flash_trigger_export(build="5.0.7", name=TRIGGER_NAME, enabled=True): + """A Companion export holding one trigger: flash_tick 1 for half a second, every second.""" + def nid(): + return base64.urlsafe_b64encode(os.urandom(15)).decode().rstrip("=")[:21] + + def setvar(v): + return {"id": nid(), "definitionId": "custom_variable_set_value", "connectionId": "internal", + "options": {"name": {"isExpression": False, "value": "flash_tick"}, "value": {"isExpression": False, "value": v}}, + "type": "action", "children": {}} + tid = nid() + trig = {"type": "trigger", "options": {"name": name, "enabled": enabled, "sortOrder": 0}, + "actions": [setvar("1"), {"id": nid(), "definitionId": "wait", "connectionId": "internal", + "options": {"time": {"isExpression": False, "value": "500"}}, "type": "action", "children": {}}, setvar("0")], + "condition": [], "events": [{"id": nid(), "type": "interval", "enabled": True, "options": {"seconds": 1}}], "localVariables": []} + return tid, {"version": 12, "type": "full", "companionBuild": build, "triggers": {tid: trig}, "triggerCollections": [], + "custom_variables": {}, "customVariablesCollections": [], "instances": {}, "connectionCollections": []} + + +def companion_state(cfg): + raw = discover.companion_get(cfg, "/int/export/custom?buttons=false&connections=false&surfaces=&triggers=true&customVariables=true" + "&expressionVariables=false&includeSecrets=false&imageLibrary=false&format=json", 10) + d = json.loads(raw) + names = {(t.get("options") or {}).get("name") for t in (d.get("triggers") or {}).values()} + return set(d.get("custom_variables") or {}), names + + +def companion_prepare(cfg): + """Create what the pages need inside Companion and is otherwise made by hand in its UI. Returns words.""" + comp = cfg.get("companion") or {} + host, port = comp.get("host", "127.0.0.1"), int(comp.get("port", 8000)) + have_vars, triggers = companion_state(cfg) + done = [] + missing = [v for v in VARIABLES if v not in have_vars] + if not missing and TRIGGER_NAME in triggers: + return done + ws = push_page.WebSocket(host, port, "/trpc") + try: + api = push_page.Trpc(ws) + for v in missing: + api.mutation("customVariables.create", {"name": v, "defaultVal": "0" if v == "flash_tick" else ""}) + done.append("custom variable %s" % v) + if TRIGGER_NAME not in triggers: + tid, export = flash_trigger_export() + data = gzip.compress(json.dumps(export).encode()) + session = api.mutation("importExport.prepareImport.start", {"name": "status-panel-blink.companionconfig", "size": len(data)}) + api.mutation("importExport.prepareImport.uploadChunk", {"sessionId": session, "offset": 0, "data": base64.b64encode(data).decode()}) + summary = api.mutation("importExport.prepareImport.complete", {"sessionId": session, "expectedChecksum": hashlib.sha1(data).hexdigest(), "userData": None}) + if isinstance(summary, list) and summary and summary[0]: + raise RuntimeError("Companion could not read the blink trigger: %s" % summary[0]) + api.mutation("importExport.importTriggers", {"selectedTriggerIds": [tid], "connectionIdRemapping": {}, "replaceExisting": False}) + done.append("the %s trigger (the keys' blink clock)" % TRIGGER_NAME) + finally: + ws.close() + return done + + +def pick_pages(cfg, count=2, start=19): + """Pages that hold nothing a person made (or only this panel's keys): the first `count` from `start`.""" + comp = cfg.get("companion") or {} + host, port = comp.get("host", "127.0.0.1"), int(comp.get("port", 8000)) + out = [] + for page in list(range(start, 100)) + list(range(2, start)): + try: + ok, _ = push_page.safe_target(push_page.fetch_page(host, port, page)) + except (OSError, ValueError): + continue + if ok: + out.append(page) + if len(out) == count: + return out + raise RuntimeError("Companion has no %d empty pages for the status panel" % count) + + +# --- the first setup -------------------------------------------------------------------- + +def template(data_dir): + for path in (os.path.join(data_dir, "kit", "sites", "_template", "config.json"), + os.path.join(PARENT, "sites", "_template", "config.json")): + if os.path.exists(path): + with open(path) as f: + return json.load(f) + raise RuntimeError("no sites/_template/config.json to start from") + + +def is_fresh(cfg): + return (cfg.get("site") or {}).get("id") in ("new", "", None) and len(cfg.get("tiles") or []) <= 3 + + +def load_seen(data_dir): + try: + with open(os.path.join(data_dir, SEEN)) as f: + return set(json.load(f)) + except (OSError, ValueError): + return set() + + +def save_seen(data_dir, seen): + sp.write_json(os.path.join(data_dir, SEEN), sorted(seen)) + + +def wait_applied(applier, limit=240): + t0 = time.time() + while time.time() - t0 < limit: + st = applier.status or {} + if st.get("state") not in ("running",): + return st + time.sleep(1) + return applier.status or {} + + +def setup(data_dir, cid, name=None, say=print): + """The first setup of a campus (or, when it is set up already, a look for what is new).""" + from settings import server + site = server.Site(data_dir) + cfg = site.load() + if not is_fresh(cfg): + say("%s is set up already: looking for what is new." % (cfg.get("site") or {}).get("name", "This campus")) + return look_and_add(site, say=say, force=True) + name = name or CAMPUSES.get(cid) or cid.upper() + for line in companion_prepare(cfg): + say("Companion: added %s" % line) + pages = pick_pages(cfg) + say("Companion pages %d (status) and %d (restart) are free: using them." % tuple(pages)) + inv = discover.look(cfg, data_dir) + if inv.get("companion_error"): + raise RuntimeError("Companion's HTTP API does not answer (%s): turn it on in Companion's settings, then run this again" % inv["companion_error"]) + pin = "%04d" % random.SystemRandom().randrange(1000, 10000) + new, seen = autoconf.build(template(data_dir), inv, {"id": cid, "name": name}, pin, pages) + errors, _ = schema.validate(new) + if errors: + raise RuntimeError("the config auto-setup made does not pass the rules: %s" % "; ".join(errors[:4])) + r = site.applier.start(new, "auto-setup: first setup of %s" % name, "auto-setup", "127.0.0.1") + if not r.get("ok"): + raise RuntimeError(r.get("error") or "the Apply did not start") + st = wait_applied(site.applier) + save_seen(data_dir, seen) + keys = [t["label"] for t in new["tiles"] if not t.get("kind")] + record(data_dir, {"at": int(time.time()), "added": ["%d keys: %s" % (len(keys), ", ".join(keys))], "inventory": summary(inv)}) + say("%s: %s" % (st.get("state"), st.get("outcome", ""))) + say("Keys: %s" % ", ".join(keys)) + say("Editing code (PIN) for the settings page: %s" % pin) + return new + + +def summary(inv): + known = set(autoconf.COMPANION) + return {"connections": len(inv.get("connections") or []), + "not_known": sorted("%s (%s)" % (c["label"], c["module"]) for c in inv.get("connections") or [] if c["module"] not in known and c["module"] != "internal"), + "dante": len((inv.get("dante") or {}).get("devices") or []), + "computers": {a: {"name": c.get("name"), "os": c.get("os"), "ip": c.get("ip"), "reachable": bool(c.get("reachable")), + "error": c.get("error", "")} for a, c in (inv.get("computers") or {}).items()}, + "midi_sessions": sum(len(s or {}) for s in (inv.get("midi") or {}).values()), + "wired": ((inv.get("ingest") or {}).get("wired") or {}).get("net")} + + +def record(data_dir, result): + sp.write_json(os.path.join(data_dir, LAST), result) + + +# --- every ten minutes: what is new ------------------------------------------------------ + +def blocked(cfg, applier, now=None): + """Why auto-setup may not apply right now, or None: never during a service or a recording.""" + now = now or time.time() + live, why = applier.live() + if live: + return "live now (%s)" % why + if sp.in_windows((cfg.get("auto_fix") or {}).get("windows") or [], now): + return "inside the auto-fix hours (a service day): waits until they end" + if applier.busy(): + return "an Apply is running" + return None + + +def look_and_add(site, say=lambda s: None, force=False): + """Look, and add what is new (when the campus allows it, and not during a service).""" + cfg = site.load() + inv = discover.look(cfg, site.data_dir) + seen = load_seen(site.data_dir) + new, added, now_seen = autoconf.extend(cfg, inv, seen) + result = {"at": int(time.time()), "added": [], "waiting": [], "inventory": summary(inv)} + if added: + auto = (cfg.get("discover") or {}).get("auto") + why = blocked(cfg, site.applier) + if not (auto or force): + result["waiting"] = added + result["why"] = "adding by itself is off (Settings > Site)" + elif why: + result["waiting"] = added + result["why"] = why + else: + errors, _ = schema.validate(new) + if errors: + result["waiting"], result["why"] = added, "would not pass the rules: %s" % "; ".join(errors[:3]) + else: + r = site.applier.start(new, "auto-setup found: %s" % ", ".join(added)[:180], "auto-setup", "127.0.0.1") + if r.get("ok"): + st = wait_applied(site.applier) + if st.get("state") == "done": + save_seen(site.data_dir, seen | set(now_seen)) + result["added"] = added + else: + result["waiting"], result["why"] = added, st.get("outcome") or "the Apply did not finish" + else: + result["waiting"], result["why"] = added, r.get("error", "the Apply did not start") + record(site.data_dir, result) + say("added: %s" % (", ".join(result["added"]) or "nothing")) + if result["waiting"]: + say("waiting: %s (%s)" % (", ".join(result["waiting"]), result.get("why"))) + return result + + +def loop(site, stop, poke): + """The settings page's own background look: every ten minutes, and soon after a computer joins.""" + stop.wait(60) + while not stop.is_set(): + try: + if (site.load().get("discover") or {}).get("auto") and not is_fresh(site.cfg): + look_and_add(site) + except Exception as e: # noqa: BLE001 - the page must keep running + record(site.data_dir, dict(load_last(site.data_dir), error=str(e), at=int(time.time()))) + poke.wait(LOOK_EVERY) + if poke.is_set(): + poke.clear() + stop.wait(20) # a computer just joined: give its SSH a moment + + +def load_last(data_dir): + try: + with open(os.path.join(data_dir, LAST)) as f: + return json.load(f) + except (OSError, ValueError): + return {} + + +# --- the lines people run ---------------------------------------------------------------- + +def program_kit(data_dir): + """The program as a .tgz: the installed code, the settings page, the templates, install.sh.""" + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tar: + for f in KIT_FILES: + p = os.path.join(data_dir, f) + if os.path.exists(p): + tar.add(p, f) + for d in KIT_DIRS: + p = os.path.join(data_dir, d) + if os.path.isdir(p): + tar.add(p, d, filter=lambda ti: None if "__pycache__" in ti.name or ti.name.endswith(".pyc") else ti) + kit = os.path.join(data_dir, "kit") + for name in ("install.sh", "README.md"): + p = os.path.join(kit, name) + if os.path.exists(p): + tar.add(p, name) + p = os.path.join(kit, "sites", "_template") + if os.path.isdir(p): + tar.add(p, "sites/_template") + return buf.getvalue() + + +def shq(s): + return "'" + str(s).replace("'", "'\"'\"'") + "'" + + +def install_script(base, cid, support_key=None): + """What a new campus's Companion Mac runs: install from `base` (this page), then set itself up.""" + m = re.match(r"[a-z0-9]+", str(cid).lower()) + cid = m.group(0)[:8] if m else "new" + name = CAMPUSES.get(cid, cid.upper()) + key = support_key or "" + return """#!/bin/zsh +# Status panel for %(name)s: installs on this Mac (the one that runs Companion) and sets itself up. +# Safe to run again: it then updates the program and looks for new gear. +set -e +SRC=%(base)s +if ! /usr/bin/xcode-select -p >/dev/null 2>&1; then + echo "First the Command Line Tools: a window opens - click Install, wait for it, then run this line again." + /usr/bin/xcode-select --install 2>/dev/null || true + exit 1 +fi +DIR="$HOME/status-panel-install" +mkdir -p "$DIR" && cd "$DIR" +echo "Downloading the status panel from $SRC ..." +/usr/bin/curl -fsSL "$SRC/campus/program.tgz" | /usr/bin/tar -xz +[[ -f "$HOME/.ssh/id_status_panel" ]] || { mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh" && /usr/bin/ssh-keygen -q -t ed25519 -N "" -C "status-panel@$(scutil --get LocalHostName)" -f "$HOME/.ssh/id_status_panel"; } +SUPPORT=%(key)s +if [[ -n "$SUPPORT" ]]; then + touch "$HOME/.ssh/authorized_keys" && chmod 600 "$HOME/.ssh/authorized_keys" + grep -qF "$SUPPORT" "$HOME/.ssh/authorized_keys" || echo "$SUPPORT" >> "$HOME/.ssh/authorized_keys" +fi +zsh install.sh --site %(cid)s +cd "$HOME/Library/Application Support/status-panel" +/usr/bin/python3 -m settings.campus setup --id %(cid)s --name %(qname)s +if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then + echo + echo "One more thing: turn on Remote Login (System Settings > General > Sharing), so North Attleboro can help remotely." +fi +echo +echo "Next, on each other computer here (ProPresenter, Tracks, Lighting...):" +echo " Mac: open Terminal and paste curl -fsSL http://$(ipconfig getifaddr en0 2>/dev/null || hostname):8020/join | sh" +echo " PC: open PowerShell as administrator and paste irm http://$(ipconfig getifaddr en0 2>/dev/null || hostname):8020/join.ps1 | iex" +""" % {"name": name, "base": shq(base), "key": shq(key), "cid": cid, "qname": shq(name)} + + +def join_mac(cfg, base): + key, _ = scripts.public_key(cfg) + if not key: + raise RuntimeError("this Mac has no login key yet (run the install line again)") + ingest = (cfg.get("site") or {}).get("ingest_name") or "the Companion Mac" + return """#!/bin/sh +# Joins this Mac to %(ingest)s's status panel: lets it log in with its key (no password) and says this Mac is here. +KEY=%(key)s +mkdir -p "$HOME/.ssh" && touch "$HOME/.ssh/authorized_keys" && chmod 700 "$HOME/.ssh" && chmod 600 "$HOME/.ssh/authorized_keys" +grep -qF "$KEY" "$HOME/.ssh/authorized_keys" || echo "$KEY" >> "$HOME/.ssh/authorized_keys" +if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then + echo "Remote Login is off. Turning it on (it asks for this Mac's administrator password):" + if ! sudo /usr/sbin/systemsetup -setremotelogin on >/dev/null 2>&1; then + echo "Please turn it on in System Settings > General > Sharing > Remote Login, then paste this line again." + open "x-apple.systempreferences:com.apple.Sharing-Settings.extension" 2>/dev/null + exit 1 + fi +fi +esc() { printf '%%s' "$1" | sed 's/[\\\\"]//g'; } +IPS=$(/sbin/ifconfig | awk '/inet /{print $2}' | grep -v '^127\\.' | sed 's/.*/"&"/' | paste -sd, -) +BODY="{\\"user\\":\\"$(esc "$(id -un)")\\",\\"name\\":\\"$(esc "$(scutil --get ComputerName)")\\",\\"hostname\\":\\"$(esc "$(scutil --get LocalHostName).local")\\",\\"os\\":\\"mac\\",\\"ips\\":[$IPS]}" +if /usr/bin/curl -fsS -X POST -H 'Content-Type: application/json' -d "$BODY" %(hello)s >/dev/null; then + echo "Joined. %(ingest)s sets this Mac up in about a minute: watch its status page." +else + echo "Could not reach %(ingest)s at %(base)s - is this Mac on the wired network?" + exit 1 +fi +""" % {"ingest": ingest, "key": shq(key), "hello": shq(base + "/api/hello"), "base": base} + + +def join_windows(cfg, base): + """The Windows set-up script, with the programs found running on that PC instead of a list.""" + text = scripts.fill(scripts.template("windows-setup.ps1.template"), dict( + scripts.site_values(cfg), PC_NAME="this PC", APPS=" # filled in below", PROGRAMS="the ones running now", + SERVICES="'rtpMIDIService'")) + known = "; ".join("'%s' = '%s'" % (a["exe"], a["task"].replace("Start ", "", 1)) for a in autoconf.WIN_APPS) + detect = ("$known = @{ %s }\n" + "$apps = @(Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.Path -and $known.ContainsKey((Split-Path $_.Path -Leaf)) } |\n" + " Sort-Object Path -Unique | ForEach-Object { @{ Name = $known[(Split-Path $_.Path -Leaf)]; Exe = $_.Path } })" % known) + text = re.sub(r"\$apps = @\(\n # filled in below\n\)", lambda m: detect, text) + if "$known = @{" not in text: + raise RuntimeError("the Windows template changed: cannot put the program finder in") + text = text.replace("# Status panel remote access", "# Run in PowerShell as administrator: irm %s/join.ps1 | iex\n# Status panel remote access" % base, 1) + text += """ +# 5. Tell %(ingest)s this PC is here: it sets it up within a minute. +$hello = @{ user = $env:USERNAME; name = $env:COMPUTERNAME; os = 'windows' + ips = @(Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue | Where-Object { $_.IPAddress -notlike '127.*' } | ForEach-Object IPAddress) + macs = @($nics | ForEach-Object MacAddress); tasks = @($apps | ForEach-Object { 'Start ' + $_.Name }) } +try { + Invoke-RestMethod -Method Post -Uri '%(hello)s' -ContentType 'application/json' -Body ($hello | ConvertTo-Json) | Out-Null + Write-Host 'Joined: %(ingest)s sets this PC up in about a minute.' +} catch { Write-Host ('Could not reach %(ingest)s: ' + $_.Exception.Message) } +""" % {"ingest": (cfg.get("site") or {}).get("ingest_name") or "the Companion Mac", "hello": base + "/api/hello"} + return text + + +# --- command line ------------------------------------------------------------------------ + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("what", choices=("setup", "look", "add")) + ap.add_argument("--id", default="new") + ap.add_argument("--name") + ap.add_argument("--data-dir", default=PARENT) + a = ap.parse_args(argv) + if a.what == "setup": + setup(a.data_dir, a.id, a.name) + elif a.what == "look": + cfg = sp.load_config(os.path.join(a.data_dir, "config.json")) + inv = discover.look(cfg, a.data_dir) + print(json.dumps(summary(inv), indent=1)) + new, added, _ = autoconf.extend(cfg, inv, load_seen(a.data_dir)) + print("would add:", ", ".join(added) or "nothing") + else: + from settings import server + look_and_add(server.Site(a.data_dir), say=print, force=True) + + +if __name__ == "__main__": + main() diff --git a/settings/discover.py b/settings/discover.py new file mode 100644 index 0000000..d4a3234 --- /dev/null +++ b/settings/discover.py @@ -0,0 +1,312 @@ +"""Auto-setup's eyes: what this campus has, read from Companion, Dante, the network MIDI sessions, +this Mac's own network, and the computers that joined (each one ran the join line once). + + look(cfg, data_dir) the inventory autoconf.py turns into keys + register(data_dir, hello) a computer's join report, kept in computers.json + +Read-only everywhere: nothing here changes Companion, a session or a computer. Every remote +command is a fixed string; names from the network only ever come back as data. +""" +import ipaddress +import json +import os +import re +import subprocess +import time +import urllib.parse +import urllib.request + +from settings import PARENT # noqa: F401 (puts the project folder on sys.path) +import status_panel as sp + +TIMEOUT = 8 +MAC_LOOK = "echo SP_PROCS; ps -axo comm= 2>/dev/null; echo SP_APPS; ls -d /Applications/*.app /Applications/*/*.app 2>/dev/null; echo SP_END" +WIN_LOOK = "echo SP_PROCS & tasklist /FO CSV /NH 2>nul & echo SP_TASKS & schtasks /Query /FO CSV /NH 2>nul & echo SP_END" +HOST_FIELDS = ("host", "bonjourHost", "serverHost", "ip", "address") + + +# --- the computers that joined --------------------------------------------------------- + +def registry_path(data_dir): + return os.path.join(data_dir, "computers.json") + + +def registered(data_dir): + try: + with open(registry_path(data_dir)) as f: + return json.load(f) + except (OSError, ValueError): + return {} + + +def clean_hello(hello, peer_ip): + """A join report, kept to the fields auto-setup uses and checked: it comes from the network.""" + def s(v, n=80): + return re.sub(r"[^\w .@:\-()'+]", "", str(v or ""))[:n] + ips = [str(ipaddress.ip_address(i)) for i in (hello.get("ips") or []) if _is_ip(i)][:12] + if peer_ip and _is_ip(peer_ip) and peer_ip not in ips: + ips.insert(0, peer_ip) + user = str(hello.get("user") or "") + if not re.fullmatch(r"[A-Za-z0-9._\-][A-Za-z0-9._\- ]{0,39}", user): # as given: a name to log in with is never "cleaned up" + raise ValueError("no usable user name") + return {"name": s(hello.get("name")) or user, "hostname": s(hello.get("hostname"), 60), "user": user, + "os": "windows" if str(hello.get("os", "")).lower().startswith("win") else "mac", "ips": ips, + "macs": [m for m in (str(x).lower().replace("-", ":") for x in hello.get("macs") or []) if re.fullmatch(r"([0-9a-f]{2}:){5}[0-9a-f]{2}", m)][:4], + "tasks": [s(t, 60) for t in hello.get("tasks") or []][:20], "at": int(time.time())} + + +def _is_ip(v): + try: + ipaddress.ip_address(str(v)) + return True + except ValueError: + return False + + +def register(data_dir, hello, peer_ip=None): + entry = clean_hello(hello, peer_ip) + reg = registered(data_dir) + key = entry["name"] + reg[key] = entry + sp.write_json(registry_path(data_dir), reg) + return entry + + +# --- this Mac -------------------------------------------------------------------------- + +def run(argv, timeout=TIMEOUT): + try: + return subprocess.run(argv, capture_output=True, text=True, timeout=timeout).stdout + except (OSError, subprocess.TimeoutExpired): + return "" + + +def wifi_ports(): + out = run(["/usr/sbin/networksetup", "-listallhardwareports"]) + ports, last = set(), "" + for line in out.splitlines(): + if line.startswith("Hardware Port:"): + last = line.split(":", 1)[1].strip() + elif line.startswith("Device:") and last in ("Wi-Fi", "AirPort"): + ports.add(line.split(":", 1)[1].strip()) + return ports + + +def interfaces(): + """{interface: (ip, network)} for every IPv4 address on this Mac, from ifconfig.""" + out, cur = {}, None + for line in run(["/sbin/ifconfig"]).splitlines(): + m = re.match(r"^(\S+?):", line) + if m: + cur = m.group(1) + continue + m = re.search(r"inet (\d+\.\d+\.\d+\.\d+) netmask 0x([0-9a-f]{8})", line) + if m and cur: + mask = ipaddress.ip_address(int(m.group(2), 16)) + net = ipaddress.ip_network("%s/%s" % (m.group(1), mask), strict=False) + out.setdefault(cur, (m.group(1), str(net))) + return out + + +def gateway_of(interface): + out = run(["/usr/sbin/netstat", "-rn", "-f", "inet"]) + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 4 and parts[0] == "default" and parts[-1] == interface: + return parts[1] + return "" + + +def this_mac(): + """Its names and addresses, and which network is the wired one: the interface with a gateway that + is neither Wi-Fi nor a link-local (Dante) address, preferring the one the default route uses.""" + ifs = interfaces() + wifi = wifi_ports() + route = run(["/sbin/route", "-n", "get", "default"]) + m = re.search(r"interface: (\S+)", route) + default_if = m.group(1) if m else "" + wired = None + for name in [default_if] + sorted(ifs): + if name in ifs and name not in wifi and not ifs[name][0].startswith(("127.", "169.254.")) and gateway_of(name): + wired = name + break + if wired is None: + wired = next((n for n in sorted(ifs) if n not in wifi and not ifs[n][0].startswith(("127.", "169.254."))), None) + info = {"name": run(["/usr/sbin/scutil", "--get", "ComputerName"]).strip(), + "hostname": (run(["/usr/sbin/scutil", "--get", "LocalHostName"]).strip() + ".local").lstrip("."), + "ips": [ip for n, (ip, _) in sorted(ifs.items()) if not ip.startswith("127.")]} + if wired: + ip, net = ifs[wired] + # A /24 or smaller on a campus usually sits inside a /16 the rest of the gear shares: use what is configured. + info["wired"] = {"interface": wired, "ip": ip, "net": net, "gateway": gateway_of(wired)} + return info + + +# --- Companion ------------------------------------------------------------------------- + +def companion_get(cfg, path, timeout=5): + c = cfg.get("companion") or {} + url = "http://%s:%s%s" % (c.get("host", "127.0.0.1"), c.get("port", 8000), path) + with urllib.request.urlopen(url, timeout=timeout) as r: + return r.read().decode("utf-8", "replace") + + +def companion_var(cfg, label, name): + try: + return companion_get(cfg, "/api/variable/%s/%s/value" % (urllib.parse.quote(label, safe=""), urllib.parse.quote(name, safe="")), 3) + except OSError: + return "" + + +def connection_hosts(cfg): + """{label: address} from Companion's own database, opened read-only (the host field differs by module).""" + import sqlite3 + path = sp.companion_db_path(cfg) + if not path: + return {} + db = sqlite3.connect("file:%s?mode=ro" % urllib.parse.quote(path), uri=True, timeout=2) + try: + rows = db.execute("SELECT value FROM instances").fetchall() + finally: + db.close() + out = {} + for (value,) in rows: + try: + inst = json.loads(value) + except (TypeError, ValueError): + continue + if not isinstance(inst, dict) or not inst.get("label"): + continue + conf = inst.get("config") or {} + host = next((str(conf[k]).strip() for k in HOST_FIELDS if isinstance(conf.get(k), str) and conf[k].strip()), "") + out[inst["label"]] = host.split("|")[-1].rsplit(":", 1)[0] if host.count(":") == 1 else host + return out + + +def connections(cfg): + raw = json.loads(companion_get(cfg, "/api/connections")) + items = raw if isinstance(raw, list) else list(raw.values()) if isinstance(raw, dict) else [] + hosts = connection_hosts(cfg) + out = [] + for c in items: + if not isinstance(c, dict) or not c.get("label"): + continue + entry = {"label": c["label"], "module": c.get("moduleId") or c.get("instance_type") or "", "enabled": c.get("enabled", True) is not False, + "host": hosts.get(c["label"], ""), "status": ((c.get("status") or {}).get("category") if isinstance(c.get("status"), dict) else "")} + if entry["module"] in ("shure-wireless", "shure-psm1000"): + probe = "ch_%d_battery_bars" if entry["module"] == "shure-wireless" else "ch_%d_rf_mute" + n = 0 + for ch in range(1, 9): + if companion_var(cfg, c["label"], probe % ch).strip() in ("", "undefined"): + break + n = ch + entry["channels"] = n or None + out.append(entry) + return out + + +def dante(cfg, conns): + label = next((c["label"] for c in conns if c["module"] == "audinate-dantecontroller" and c["enabled"]), None) + if not label: + return {} + try: + devices = json.loads(companion_var(cfg, label, "devices") or "[]") + except ValueError: + devices = [] + return {"conn": label, "devices": sorted(d for d in devices if isinstance(d, str))} + + +# --- the computers --------------------------------------------------------------------- + +def sections(out): + parts, cur = {}, None + for line in out.splitlines(): + line = line.strip() + if line.startswith("SP_") and line[3:].isupper(): + cur = line[3:] + parts[cur] = [] + elif cur: + parts[cur].append(line) + return parts + + +def look_at(cfg, alias, host): + """What runs on one computer: its programs (and, on a Mac, its network MIDI sessions).""" + os_name = host.get("os", "mac") + try: + r = subprocess.run(sp.ssh_cmd(cfg, alias) + [MAC_LOOK if os_name == "mac" else WIN_LOOK], capture_output=True, text=True, timeout=TIMEOUT) + except subprocess.TimeoutExpired: + return {"reachable": False} + if r.returncode != 0 or "SP_END" not in r.stdout: + return {"reachable": False, "error": (r.stderr or r.stdout).strip()[-160:]} + got = sections(r.stdout) + out = {"reachable": True} + if os_name == "mac": + out["procs"] = sorted({os.path.basename(p.strip()) for p in got.get("PROCS", []) if p.strip()}) + out["apps"] = sorted(a.strip() for a in got.get("APPS", []) if a.strip().endswith(".app")) + try: + m = subprocess.run(sp.ssh_cmd(cfg, alias) + ["/usr/bin/python3", "-"], input=sp.remote_midi_script(), capture_output=True, text=True, timeout=15) + out["midi"] = sp.parse_remote_midi(m.stdout) or {} + except subprocess.TimeoutExpired: + out["midi"] = {} + else: + out["procs"] = sorted({row.split('","')[0].strip('"') for row in got.get("PROCS", []) if row.startswith('"')}) + out["tasks"] = sorted({row.split('","')[0].strip('"').split("\\")[-1] for row in got.get("TASKS", []) + if row.startswith('"\\StatusPanel\\')}) + return out + + +def computers(cfg, data_dir): + """The joined computers and the ones already in the settings, each looked at over SSH.""" + found = {} + hosts = dict(cfg.get("hosts") or {}) + for alias, h in hosts.items(): + if h.get("self") or h.get("disabled") or not h.get("ssh"): + continue + ip = str(h["ssh"]).split("@")[-1] + found[alias] = {"name": h.get("name", alias), "os": h.get("os", "mac"), "user": str(h["ssh"]).split("@")[0], "ip": ip, + "ips": [ip], "macs": [h["mac"]] if h.get("mac") else []} + known_ips = {c["ip"] for c in found.values()} + taken = set(found) | set(hosts) + nets = [(this_mac().get("wired") or {}).get("net")] + for name, j in registered(data_dir).items(): + ip = next((i for i in j["ips"] if any(n and ipaddress.ip_address(i) in ipaddress.ip_network(n) for n in nets)), j["ips"][0] if j["ips"] else None) + if not ip or ip in known_ips: + continue + from settings.autoconf import slug + alias = slug(j["name"].replace(" ", "_"), taken) + taken.add(alias) + found[alias] = {"name": j["name"], "hostname": j.get("hostname", ""), "os": j["os"], "user": j["user"], "ip": ip, "ips": j["ips"], + "macs": j.get("macs") or [], "tasks": j.get("tasks") or []} + probe_cfg = dict(cfg, hosts=dict(hosts, **{a: {"ssh": "%s@%s" % (c["user"], c["ip"]), "os": c["os"]} for a, c in found.items()})) + for alias, c in found.items(): + seen = look_at(probe_cfg, alias, probe_cfg["hosts"][alias]) + if c.get("tasks") and "tasks" in seen: + seen["tasks"] = sorted(set(seen["tasks"]) | set(c["tasks"])) + c.update(seen) + return found + + +# --- everything ------------------------------------------------------------------------ + +def look(cfg, data_dir): + inv = {"ingest": this_mac(), "at": int(time.time())} + # A campus whose MIDI rule names its wired network keeps it (it may be wider than this Mac's subnet). + rule = [n for t in sp.all_tiles(cfg) for c in t.get("checks", []) if c.get("type") == "midi_wired" for n in c.get("nets", [])] + if rule and inv["ingest"].get("wired"): + inv["ingest"]["wired"]["net"] = rule[0] + try: + inv["connections"] = connections(cfg) + except (OSError, ValueError) as e: + inv["connections"], inv["companion_error"] = [], str(e) + inv["dante"] = dante(cfg, inv["connections"]) + try: + import coremidi + inv["midi"] = {"": coremidi.sessions()} + except Exception as e: # noqa: BLE001 - no CoreMIDI (tests, Linux): no sessions here + inv["midi"], inv["midi_error"] = {}, str(e) + inv["computers"] = computers(cfg, data_dir) + for alias, c in inv["computers"].items(): + if c.get("midi"): + inv["midi"][alias] = c.pop("midi") + return inv diff --git a/settings/server.py b/settings/server.py index 6b76f9f..3eb96c5 100644 --- a/settings/server.py +++ b/settings/server.py @@ -40,6 +40,7 @@ from settings import builders from settings import scripts from settings import outlet as outlets from settings import keyguide, probes +from settings import campus, discover log = logging.getLogger("status-settings") UI = os.path.join(HERE, "ui") @@ -115,6 +116,7 @@ class Site: self.page_cache = {} # name -> (time, value) self.draft = applying.Draft(data_dir) self.applier = applying.Applier(self) + self.discover_poke = threading.Event() # a computer joined: look again soon outlets.DATA_DIR, outlets.FILE = self.data_dir, os.path.join(self.data_dir, "outlets.json") def load(self): @@ -684,6 +686,23 @@ class Handler(http.server.BaseHTTPRequestHandler): return self.send(200, keyguide.cheat_sheet(site.load(), self.my_address()), "text/html; charset=utf-8") if path == "/where.html": return self.send(200, keyguide.where_things_are(site.load(), site.data_dir, self.my_address()), "text/html; charset=utf-8") + # ---- auto-setup: the install line for a new campus, the join lines for its computers ---- + if path == "/campus/install": + key, _ = scripts.public_key(site.load()) + return self.send(200, campus.install_script(self.base_url(), q.get("id", ["new"])[0], key), "text/plain; charset=utf-8") + if path == "/campus/program.tgz": + return self.send(200, campus.program_kit(site.data_dir), "application/gzip") + if path == "/join": + return self.send(200, campus.join_mac(site.load(), self.base_url()), "text/plain; charset=utf-8") + if path == "/join.ps1": + return self.send(200, campus.join_windows(site.load(), self.base_url()), "text/plain; charset=utf-8") + if path == "/api/discover": + cfg = site.load() + base = self.base_url() + return self.send(200, {"auto": bool((cfg.get("discover") or {}).get("auto")), "last": campus.load_last(site.data_dir), + "joined": discover.registered(site.data_dir), + "join": {"mac": "curl -fsSL %s/join | sh" % base, "pc": "irm %s/join.ps1 | iex" % base}, + "base": base}) if path == "/api/config": cfg = site.load() return self.send(200, {"config": schema.mask_secrets(cfg), "sha": site.cfg_sha, "mtime": site.cfg_mtime}) @@ -845,6 +864,15 @@ class Handler(http.server.BaseHTTPRequestHandler): return self.send(403, {"error": "wrong origin"}) site = self.site try: + if path == "/api/hello": + # A computer ran the join line: keep what it says (checked), and look again soon. + try: + entry = discover.register(site.data_dir, body, self.client_address[0]) + except (ValueError, TypeError) as e: + return self.send(400, {"error": str(e)}) + log.info("joined: %s (%s, %s) from %s", entry["name"], entry["os"], entry["user"], self.client_address[0]) + site.discover_poke.set() + return self.send(200, {"ok": True}) if path == "/api/editing/check": return self.send(200, site.check_code(self.client_address[0], str(body.get("code", "")))) if path == "/api/machines/probe": @@ -861,6 +889,14 @@ class Handler(http.server.BaseHTTPRequestHandler): ok, who = self.gate() if not ok: return + if path == "/api/discover/now": + # Look now and add what is new (also when adding by itself is off); never during a service. + if getattr(site, "looking", None) and site.looking.is_alive(): + return self.send(200, {"started": False, "already": True}) + site.looking = threading.Thread(target=campus.look_and_add, args=(site,), kwargs={"force": True}, daemon=True) + site.looking.start() + log.info("auto-setup: look now, asked by %s", who) + return self.send(200, {"started": True}) if path == "/api/draft" and self.command == "PUT": cfg = schema.unmask(body.get("config") or {}, site.load()) # A page that loaded the settings before they changed here (an Apply from another tab, a @@ -1289,6 +1325,13 @@ class Handler(http.server.BaseHTTPRequestHandler): except OSError: return self.send(404, {"error": "missing %s" % name}) + def base_url(self): + """This page as the caller reached it (an address, across campuses: names do not cross subnets).""" + host = (self.headers.get("Host") or "").strip() or "%s:%d" % (self.server.server_address[0], self.server.server_address[1]) + if not re.fullmatch(r"[A-Za-z0-9.\-]+(:\d+)?", host): + host = "%s:%d" % (self.server.server_address[0], self.server.server_address[1]) + return "http://" + host + def my_address(self): cfg = self.site.cfg host = cfg.get("site", {}).get("ingest_hostname") or (self.headers.get("Host") or "").split(":")[0] or "127.0.0.1" @@ -1400,6 +1443,7 @@ def main(argv=None): site, servers = serve(args.data_dir, port, args.bind, args.config) stop = threading.Event() threading.Thread(target=rebind_loop, args=(site, servers, port, stop), daemon=True).start() + threading.Thread(target=campus.loop, args=(site, stop, site.discover_poke), name="auto-setup", daemon=True).start() try: while True: time.sleep(3600) diff --git a/settings/ui/app.js b/settings/ui/app.js index 762542b..1f650c3 100644 --- a/settings/ui/app.js +++ b/settings/ui/app.js @@ -151,7 +151,7 @@ if (f.type === "dict") { try { return raw.trim() ? JSON.parse(raw) : undefined; } catch (e) { return undefined; } } return raw === "" ? undefined : raw; } - const LABELS = { after_seconds: "wait before acting (seconds)", note_seconds: "keep showing ON … after (seconds)", interval_seconds: "look every (seconds)", failures: "misses before it counts", debounce_seconds: "a fault must last (seconds)", editing_code: "PIN", ssh: "log-in address", mac: "hardware address", ingest_ips: "this Mac's addresses", ingest_name: "this Mac's name", ingest_hostname: "network name", hold_always: "hold also when green", page_hold_ms: "hold time (ms)", optional: "may have no disk" }; + const LABELS = { auto: "add new gear by itself", after_seconds: "wait before acting (seconds)", note_seconds: "keep showing ON … after (seconds)", interval_seconds: "look every (seconds)", failures: "misses before it counts", debounce_seconds: "a fault must last (seconds)", editing_code: "PIN", ssh: "log-in address", mac: "hardware address", ingest_ips: "this Mac's addresses", ingest_name: "this Mac's name", ingest_hostname: "network name", hold_always: "hold also when green", page_hold_ms: "hold time (ms)", optional: "may have no disk" }; function inputHtml(path, f, value, extra) { if (!f) return ""; const id = "f-" + path.replace(/[^A-Za-z0-9]/g, "-"); @@ -877,6 +877,7 @@ ["outlets", "Outlets", "Smart plugs the panel may switch, for a device that only a power cycle brings back"], ["companion", "Companion", "Connections, and names the keys use that are not here"], ["network", "Network", "This Mac's network port and the Dante switches"], + ["setup", "Auto-setup", "What this Mac found, the line each computer runs to join, and the line that sets up another campus"], ["site", "Site & PIN", "The campus, contacts, this Mac, the PIN"], ["program", "Program & campus", "Update the code, export or start a campus"], ["advanced", "Advanced", "Every number and name, by screen"], @@ -902,6 +903,7 @@ else if (which === "outlets") body = await outletsHtml(); else if (which === "companion") body = (await namesHtml()) + (await companionHtml()); else if (which === "network") body = fieldsHtml("Network & Dante"); + else if (which === "setup") body = await setupHtml(c); else if (which === "site") body = siteHtml(c); else if (which === "program") body = await programHtml(); else if (which === "advanced") body = advancedHtml(); @@ -928,6 +930,28 @@ const screens = [...new Set(Object.values(state.schema.fields).map((f) => f.screen))]; return screens.map((s) => { const h = fieldsHtml(s, true); return h ? `

${esc(s)}

${h}` : ""; }).join("") || "

Nothing.

"; } + // Auto-setup: what was found and added, the join lines, and the install line for another campus. + async function setupHtml(c) { + const d = await api("/api/discover").catch(() => null); + if (!d) return "

Auto-setup is not answering.

"; + const last = d.last || {}, inv = last.inventory || {}; + const line = (text, h) => `
${h}
`; + const comps = Object.entries(inv.computers || {}).map(([a, x]) => `${lightDot(x.reachable ? "ok" : "problem", x.reachable ? "logs in" : x.error || "does not let this Mac in yet")}${esc(x.name || a)}${esc(x.os || "")}${esc(x.ip || "")}${x.reachable ? "" : esc(x.error || "run the join line on it")}`).join(""); + const joined = Object.entries(d.joined || {}).filter(([n]) => !Object.values(inv.computers || {}).some((x) => x.name === n)).map(([n, j]) => `${lightDot("warn", "joined; the next look sets it up")}${esc(n)}${esc(j.os)}${esc((j.ips || [])[0] || "")}joined ${esc(when(j.at))}`).join(""); + const list = (xs) => xs && xs.length ? `
    ${xs.map((x) => `
  • ${esc(x)}
    • `).join("")}
` : "

nothing

"; + const campuses = [["ab", "Apollo Beach"], ["sc", "Southcoast"], ["ws", "Woonsocket"], ["na", "North Attleboro"]].filter(([k]) => k !== (c.site || {}).id); + return `

${inputHtml("discover.auto", F("discover.auto"), (c.discover || {}).auto)}

+
last look
${last.at ? esc(when(last.at)) : "not yet"}${last.error ? ` · ${esc(last.error)}` : ""}
+
found
${inv.connections != null ? `${inv.connections} Companion connections · ${inv.dante || 0} Dante devices · ${inv.midi_sessions || 0} MIDI sessions · wired network ${esc(inv.wired || "?")}` : "–"}
+
+

Added last time

${list(last.added)}${last.waiting && last.waiting.length ? `

Waiting${hint(last.why || "")}

${list(last.waiting)}

${esc(last.why || "")}

` : ""} + ${inv.not_known && inv.not_known.length ? `
Companion connections auto-setup has no recipe for (${inv.not_known.length})${list(inv.not_known)}

Add a key for one by hand (Keys › + Key) if it matters.

` : ""}
+

Computers${hint("Each computer runs the join line once: it lets this Mac log in with its key and says it is here. The next look gives its programs keys.")}

+
${comps}${joined}${comps || joined ? "" : ""}
none yet
+ ${line(d.join.mac, "Mac: Terminal, paste")}${line(d.join.pc, "PC: PowerShell as administrator, paste")}
+

Set up another campus${hint("On that campus's Mac that runs Companion: Terminal, paste. It installs the panel from here, finds the gear and makes the pages. Then each computer there runs that Mac's join line.")}

+ ${campuses.map(([k, n]) => line(`curl -fsSL "${d.base}/campus/install?id=${k}" | zsh`, n)).join("")}
`; + } function siteHtml(c) { const f = F("site.editing_code"); return `
${inputHtml("site.editing_code", Object.assign({}, f, { help: "The PIN that switches Editing on. Digits are easiest. It lives in the settings file, so everyone with the repo has it." }), (c.site || {}).editing_code)}
` + fieldsHtml("Site"); @@ -1347,6 +1371,8 @@ function wireSettings(which) { const q = (s) => main.querySelector(s); wireCalendars(); + for (const el of main.querySelectorAll(".copyline")) el.onclick = () => el.select(); + if (q("#looknow")) q("#looknow").onclick = async () => { try { await post("/api/discover/now", {}); alertBox("Looking now. This page fills in within a minute."); setTimeout(() => route(), 30000); } catch (e) { alertBox(e.message); } }; for (const b of main.querySelectorAll("[data-changepress]")) b.onclick = () => { const id = b.dataset.changepress, f0 = cfg().fixes[id]; pickButtonDialog(`Which button is ${(f0.params || {}).button || id}?`, { page: f0.page, row: f0.row, col: f0.col }, async (p) => { diff --git a/tests/test_campus.py b/tests/test_campus.py new file mode 100644 index 0000000..82b10ba --- /dev/null +++ b/tests/test_campus.py @@ -0,0 +1,247 @@ +"""Auto-setup for the other campuses: an inventory becomes keys that pass every rule, laid out like +North Attleboro's; a running campus only gains what is new (never loses anything, never gets back +what someone took off); a computer's join report is checked; the lines people paste are right.""" +import io +import json +import os +import socket +import sys +import tarfile +import tempfile +import threading +import time +import unittest +from http.server import ThreadingHTTPServer + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +sys.path.insert(0, os.path.join(ROOT, "tests")) +import schema # noqa: E402 +import make_page # noqa: E402 +from settings import autoconf as ac, campus, discover, server # noqa: E402 +from test_settings import get # noqa: E402 +from test_status_panel import FakeCompanion, healthy_snapshot # noqa: E402 + +BASE = json.load(open(os.path.join(ROOT, "sites", "_template", "config.json"))) + + +def small(): + return { + "ingest": {"name": "SC Ingest", "hostname": "SC-Ingest.local", "ips": ["10.4.1.1", "10.6.20.5"], + "wired": {"interface": "en0", "ip": "10.4.1.1", "net": "10.4.0.0/16", "gateway": "10.4.0.1"}}, + "connections": [ + {"label": "HyperDeck_1", "module": "bmd-hyperdeck", "host": "10.4.10.11", "enabled": True}, + {"label": "HyperDeck_2", "module": "bmd-hyperdeck", "host": "10.4.10.12", "enabled": True}, + {"label": "x32", "module": "behringer-x32", "host": "10.4.1.112", "enabled": True}, + {"label": "dante", "module": "audinate-dantecontroller", "host": "", "enabled": True}, + {"label": "ProP", "module": "renewedvision-propresenter-api", "host": "10.4.14.20", "enabled": True}, + {"label": "Govee", "module": "govee-lights", "host": "", "enabled": True}, + {"label": "old_obs", "module": "obs-studio", "host": "10.4.9.9", "enabled": False}], + "dante": {"conn": "dante", "devices": ["SC-X32", "SC-ProP-Mac"]}, + "midi": {"": {"SC Ingest": {"port": 5004, "peers": [ + {"name": "Lighting PC", "address": "10.4.10.206:5004", "state": "running"}, + {"name": "Phone", "address": "10.6.20.77:5004", "state": "running"}]}}}, # Wi-Fi: not made a key + "computers": { + "prop": {"name": "ProPresenter Mac", "os": "mac", "user": "media", "ip": "10.4.14.20", "ips": ["10.4.14.20"], "reachable": True, + "procs": ["ProPresenter", "Finder"], "apps": ["/Applications/ProPresenter.app", "/Applications/Safari.app"]}, + "lighting_pc": {"name": "Lighting PC", "os": "windows", "user": "media", "ip": "10.4.10.206", "ips": ["10.4.10.206"], + "reachable": True, "procs": ["Vista.exe", "rtpMIDI.exe", "Arena.exe"], "tasks": ["Start Vista", "Start rtpMIDI"], + "macs": ["aa:bb:cc:dd:ee:ff"]}, + "off_pc": {"name": "Office", "os": "windows", "user": "x", "ip": "10.4.3.3", "ips": ["10.4.3.3"], "reachable": False}}} + + +def big(): + inv = small() + inv["connections"] += [{"label": "HyperDeck_%d" % n, "module": "bmd-hyperdeck", "host": "10.4.10.%d" % (10 + n), "enabled": True} for n in range(3, 7)] + inv["connections"] += [{"label": "Axient_%d" % n, "module": "shure-wireless", "host": "10.4.1.%d" % (100 + n), "enabled": True, "channels": 4} for n in range(1, 4)] + inv["connections"] += [{"label": "PSM_%d" % n, "module": "shure-psm1000", "host": "10.4.1.%d" % (110 + n), "enabled": True, "channels": 2} for n in range(1, 9)] + inv["connections"] += [{"label": m, "module": mod, "host": "10.4.5.%d" % i, "enabled": True} + for i, (m, mod) in enumerate([("ATEM", "bmd-atem"), ("VX1000", "novastar-controller"), ("OBS", "obs-studio"), + ("RESI", "resi-decoders"), ("tascam", "tascam-da-6400"), ("Arena", "resolume-arena")])] + return inv + + +def layout_of(cfg): + return {tuple(t["pos"]): t["label"] for t in cfg["tiles"] if t.get("pos")} + + +class TestBuild(unittest.TestCase): + def test_a_small_campus_gets_one_key_per_thing_in_order_and_passes_the_rules(self): + cfg, seen = ac.build(BASE, small(), {"id": "sc", "name": "Southcoast"}, "4821", (21, 22)) + self.assertEqual(schema.validate(cfg)[0], []) + labels = [t["label"] for t in cfg["tiles"] if not t.get("kind")] + self.assertEqual(labels, ["HD1", "HD2", "DANTE", "X32", "PROP", "VISTA", "MIDI"]) # REC, AUDIO, SHOW; no group keys + self.assertNotIn("RTPMIDI", labels) # rides in the MIDI key + midi = next(t for t in cfg["tiles"] if t["label"] == "MIDI") + self.assertEqual([c["peer"] for c in midi["checks"] if c["type"] == "midi_peer"], ["Lighting PC"]) # the Wi-Fi phone is not + self.assertEqual(midi["checks"][-2]["type"], "midi_wired") + self.assertEqual(midi["checks"][-1]["params"]["exe"], "rtpMIDI.exe") + prop = next(t for t in cfg["tiles"] if t["label"] == "PROP") + self.assertEqual([c["type"] for c in prop["checks"]], ["conn", "remote"]) # Companion connection + the program + self.assertEqual(prop["checks"][1]["cmd"], "pgrep -xq ProPresenter") + self.assertNotIn("ARENA", labels) # running, but no start task on that PC + self.assertNotIn("off_pc", cfg["hosts"]) # not reachable: not added + self.assertEqual(cfg["hosts"]["lighting_pc"]["ssh"], "media@10.4.10.206") + self.assertEqual((cfg["page"], cfg["restart_page"]["page"]), (21, 22)) + self.assertEqual([k["label"] for k in cfg["restart_page"]["keys"]], ["PROP", "VISTA"]) + self.assertEqual(cfg["site"]["editing_code"], "4821") + self.assertEqual(cfg["site"]["ingest_ips"][0], "10.4.1.1") + self.assertTrue(cfg["discover"]["auto"]) + self.assertIn("hyperdeck", cfg["auto_fix"]["fixes"]) + self.assertFalse(any(f.startswith("restart_") for f in cfg["auto_fix"]["fixes"])) # restarts stay a hold + for t in cfg["tiles"]: # every key's hold does something + if not t.get("kind") and not t.get("members"): + fixable = any(c.get("fix") or c["type"] in ("conn", "hyperdeck", "tascam", "shure", "psm") for c in t["checks"]) + self.assertTrue(fixable, t["label"]) + make_page.build(cfg) + make_page.build_restart(cfg) + self.assertIn("conn:x32", seen) + self.assertIn("dante:SC-X32", seen) + + def test_a_big_campus_gets_group_keys_and_the_receivers_fold_first(self): + cfg, _ = ac.build(BASE, big(), {"id": "ab", "name": "Apollo Beach"}, "1111") + self.assertEqual(schema.validate(cfg)[0], []) + top = [layout_of(cfg).get((0, c)) for c in range(2, 8)] + self.assertEqual(top, ["REC", "AUDIO", "RF", "SHOW", "VIDEO", "STREAM"]) + unplaced = [t["label"] for t in cfg["tiles"] if not t.get("pos")] + self.assertTrue(unplaced and all(u.startswith(("AXIENT", "PSM")) for u in unplaced), unplaced) + rf = next(t for t in cfg["tiles"] if t["label"] == "RF") + self.assertEqual(len(rf["members"]), 11) + self.assertEqual(layout_of(cfg)[(3, 7)], "NET") + self.assertEqual(layout_of(cfg)[(3, 0)], "ALARMS") + make_page.build(cfg) + + +class TestExtend(unittest.TestCase): + def setUp(self): + self.cfg, self.seen = ac.build(BASE, small(), {"id": "sc", "name": "Southcoast"}, "4821") + + def test_nothing_new_adds_nothing(self): + new, added, _ = ac.extend(self.cfg, small(), self.seen) + self.assertEqual(added, []) + self.assertEqual(new, self.cfg) + + def test_new_gear_is_added_into_free_cells_and_merged_into_its_key(self): + inv = small() + inv["connections"].append({"label": "HyperDeck_3", "module": "bmd-hyperdeck", "host": "10.4.10.13", "enabled": True}) + inv["dante"]["devices"].append("SC-Tracks") + inv["midi"][""]["SC Ingest"]["peers"].append({"name": "SuperRack", "address": "10.4.13.140:5004", "state": "running"}) + new, added, now_seen = ac.extend(self.cfg, inv, self.seen) + self.assertEqual(sorted(added), ["Dante device SC-Tracks", "HD3 key", "MIDI link SuperRack"]) + self.assertEqual(schema.validate(new)[0], []) + hd = [t for t in new["tiles"] if t["label"] == "HD3"][0] + self.assertTrue(hd.get("pos")) + self.assertIn("SC-Tracks", next(c for t in new["tiles"] for c in t.get("checks", []) if c["type"] == "dante")["devices"]) + # Nothing that was there changed place. + self.assertTrue(set(layout_of(self.cfg).items()) <= set(layout_of(new).items())) + again, added2, _ = ac.extend(new, inv, set(self.seen) | set(now_seen)) + self.assertEqual(added2, []) + + def test_a_key_someone_took_off_is_not_added_back(self): + cfg = json.loads(json.dumps(self.cfg)) + cfg["tiles"] = [t for t in cfg["tiles"] if t["label"] != "X32"] + _, added, _ = ac.extend(cfg, small(), self.seen) + self.assertNotIn("X32 key", added) + _, added, _ = ac.extend(cfg, small(), []) # never seen: it would be + self.assertIn("X32 key", added) + + def test_north_attleboro_like_config_only_gets_what_it_does_not_watch(self): + cfg = json.load(open(os.path.join(ROOT, "config.json"))) + inv = {"ingest": {"wired": {"net": "10.1.0.0/16"}}, + "connections": [{"label": "HyperDeck_1", "module": "bmd-hyperdeck", "host": "10.1.10.11", "enabled": True}, + {"label": "x32", "module": "behringer-x32", "host": "10.1.1.112", "enabled": True}], + "dante": {}, "midi": {}, "computers": {}} + _, added, _ = ac.extend(cfg, inv, []) + self.assertEqual(added, []) + + +class TestJoin(unittest.TestCase): + def test_a_join_report_is_checked(self): + e = discover.clean_hello({"user": "media", "name": "Lighting PC", "os": "Windows", "ips": ["10.4.10.206", "bad"], + "macs": ["AA-BB-CC-DD-EE-FF", "nope"], "tasks": ["Start Vista"]}, "10.4.10.206") + self.assertEqual((e["os"], e["ips"], e["macs"], e["tasks"]), ("windows", ["10.4.10.206"], ["aa:bb:cc:dd:ee:ff"], ["Start Vista"])) + for bad in ({"name": "x"}, {"user": "a;rm -rf /"}, {"user": ""}): + with self.assertRaises(ValueError): + discover.clean_hello(bad, "10.4.1.2") + self.assertEqual(discover.clean_hello({"user": "u", "name": "Mac"}, "10.4.1.2")["name"], "bMacb") + + def test_the_lines_people_paste(self): + line = campus.install_script("http://10.1.1.1:8020", "ab; rm", "ssh-ed25519 AAAA na") + self.assertIn("zsh install.sh --site ab", line) # the id is cleaned + self.assertIn("--name 'Apollo Beach'", line) + self.assertIn("SUPPORT='ssh-ed25519 AAAA na'", line) + cfg = {"site": {"ingest_name": "SC Ingest", "ingest_ips": ["10.4.1.1"]}} + ps = campus.join_windows(cfg, "http://10.4.1.1:8020") + self.assertIn("'Vista.exe' = 'Vista'", ps) + self.assertIn("Invoke-RestMethod -Method Post -Uri 'http://10.4.1.1:8020/api/hello'", ps) + self.assertNotIn("{{", ps) + + +class TestRoutes(unittest.TestCase): + @classmethod + def setUpClass(cls): + snap = healthy_snapshot(time.time()) + FakeCompanion.store = {"conns": snap.conns, "vars": dict(snap.vars), "custom": {}, "restarts": []} + cls.fake = ThreadingHTTPServer(("127.0.0.1", 0), FakeCompanion) + threading.Thread(target=cls.fake.serve_forever, daemon=True).start() + cls.data = tempfile.mkdtemp(prefix="sp-campus-") + key = os.path.join(cls.data, "id_test") + open(key + ".pub", "w").write("ssh-ed25519 AAAAC3NzaTESTKEY status-panel@sc\n") + cfg, _ = ac.build(BASE, small(), {"id": "sc", "name": "Southcoast"}, "4821") + cfg["companion"] = {"host": "127.0.0.1", "port": cls.fake.server_address[1]} + cfg["ssh_key"] = key + json.dump(cfg, open(os.path.join(cls.data, "config.json"), "w")) + json.dump({"pid": 1, "started_at": time.time(), "config_sha": "x", "version": "test"}, open(os.path.join(cls.data, "started.json"), "w")) + open(os.path.join(cls.data, "status_panel.py"), "w").write("# code\n") + os.makedirs(os.path.join(cls.data, "kit", "sites", "_template")) + open(os.path.join(cls.data, "kit", "install.sh"), "w").write("# installer\n") + open(os.path.join(cls.data, "kit", "sites", "_template", "config.json"), "w").write("{}") + open(os.path.join(cls.data, "outlets.json"), "w").write('{"tuya": {"access_secret": "never-shipped"}}') + sock = socket.socket(); sock.bind(("127.0.0.1", 0)); cls.port = sock.getsockname()[1]; sock.close() + cls.site, cls.servers = server.serve(cls.data, cls.port, ["127.0.0.1"]) + + @classmethod + def tearDownClass(cls): + for s in cls.servers: + s.shutdown() + cls.fake.shutdown() + + def test_hello_registers_a_computer_without_the_code(self): + status, body = get(self.port, "/api/hello", {"user": "media", "name": "Tracks Mac", "os": "mac", "ips": ["10.4.3.227"]}) + self.assertEqual(status, 200) + self.assertIn("Tracks Mac", discover.registered(self.data)) + self.assertTrue(self.site.discover_poke.is_set()) + self.assertEqual(get(self.port, "/api/hello", {"name": "no user"})[0], 400) + status, body = get(self.port, "/api/discover") + d = json.loads(body) + self.assertEqual(d["join"]["mac"], "curl -fsSL http://127.0.0.1:%d/join | sh" % self.port) + self.assertIn("Tracks Mac", d["joined"]) + + def test_join_lines_carry_this_macs_key_and_address(self): + status, body = get(self.port, "/join") + self.assertEqual(status, 200) + self.assertIn("KEY='ssh-ed25519 AAAAC3NzaTESTKEY status-panel@sc'", body.decode()) + self.assertIn("http://127.0.0.1:%d/api/hello" % self.port, body.decode()) + status, body = get(self.port, "/join.ps1") + self.assertEqual(status, 200) + self.assertIn("$IngestIPs = @('10.4.1.1', '10.6.20.5')", body.decode()) + + def test_the_program_kit_has_the_code_and_never_the_settings(self): + status, body = get(self.port, "/campus/program.tgz") + self.assertEqual(status, 200) + names = tarfile.open(fileobj=io.BytesIO(body), mode="r:gz").getnames() + self.assertIn("status_panel.py", names) + self.assertIn("install.sh", names) + self.assertIn("sites/_template/config.json", names) + self.assertFalse([n for n in names if n.endswith(("config.json",)) and not n.startswith("sites/_template")]) + self.assertNotIn("outlets.json", names) + status, body = get(self.port, "/campus/install?id=ws") + self.assertIn("--site ws", body.decode()) + self.assertIn("SUPPORT='ssh-ed25519 AAAAC3NzaTESTKEY", body.decode()) # this campus can help the new one + + def test_look_now_needs_the_code(self): + self.assertEqual(get(self.port, "/api/discover/now", {})[0], 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_schema.py b/tests/test_schema.py index c98a689..0bff32c 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -185,7 +185,7 @@ class TestSchemaMatchesCode(unittest.TestCase): "fixes", "hosts", "tiles", "flap", "auto_fix", "network", "variables", "companion", "atem", "spotify", "restart_page", "service_windows", "restart_page.keys", "tiles[].checks", "auto_fix.windows", "site.live_services", "tiles[].checks[].type", "fixes.*.playlist", "tiles[].checks[].playlist", - "ready_page", "ready_page.keys", "ready"} + "ready_page", "ready_page.keys", "ready", "discover", "discover.auto"} # discover: settings/campus.py read = {path_of(kind, key) for _, kind, key, _, _ in all_reads()} read |= {"service_windows[].days", "service_windows[].start", "service_windows[].end", "site.live_services[].days", "site.live_services[].start", "site.live_services[].end"}