diff --git a/README.md b/README.md index f9d18a7..89d8e7c 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,39 @@ Fixes and the remote checks are built from fields (`settings/builders.py`): a bu The page reads what the watchdog writes next to its config (`snapshot.json` every 5 s, `started.json`, `live-now.json`, `fixes-running.json`) instead of asking the gear again. Its own look at the computers (`settings/probes.py`) runs fixed read-only commands over the same login key, fetches whole lists and matches them here, never while a fix runs. +## Other campuses: auto-setup + +A campus sets itself up. All a person does is let the panel in: one line on the campus's Mac that runs Companion, +one line on each other computer. + +1. **The Companion Mac.** Terminal, paste the campus's line from Settings › Auto-setup on any running campus (North + Attleboro's page shows the Apollo Beach, Southcoast and Woonsocket lines), for example + `curl -fsSL "http://10.1.1.1:8020/campus/install?id=ab" | zsh`. It downloads the program from that page (code only, + never that campus's settings or keys), installs it, makes the panel's login key, and runs + `python3 -m settings.campus setup`: Companion gets the panel's custom variables and the blink-clock trigger (through + Companion's own internal API, as its web UI does), the first two pages from 19 that hold nothing made by hand are + used, the gear is found and the config is applied like any Apply. It prints the editing PIN. North Attleboro's key is + added so it can help remotely (turn on Remote Login for that). Running the line again updates the program. +2. **Each other computer.** Mac: Terminal, `curl -fsSL http://:8020/join | sh` (adds the key, turns on + Remote Login when it can, says it is here). PC: PowerShell as administrator, `irm http://:8020/join.ps1 | iex` + (OpenSSH, the key, a StatusPanel start task for each known program running on it, Wake-on-LAN, then says it is here). + The lines are on that campus's Settings › Auto-setup. + +**What gets a key** (`settings/autoconf.py`): each Companion connection of a known kind (HyperDecks, Tascam, consoles, +Dante with every device it sees, Shure receivers and PSMs with their channel count, ATEM, Videohub, Resolume, NovaStar, +OBS, Resi, ProPresenter, Ableton/AbleSet), each known program running on a joined computer (ProPresenter, Ableton, +Arena, OBS, Church Translate, Pro Tools on a Mac; Vista, Arena, OBS, ProPresenter on a PC), and every network MIDI peer at +its wired address (with the wired-network rule). A program's key also carries its Companion connection, its start fix +(auto-fixable) and a Restart page key. rtpMIDI rides inside the MIDI key. When everything fits there are no group keys; +when it does not, REC AUDIO RF SHOW VIDEO STREAM group keys take the top row and wireless receivers fold into RF first. +Generic connections and unknown programs are listed under Settings › Auto-setup, not guessed at. + +**After that** (`discover.auto`, "add new gear by itself", on for a new campus, off at North Attleboro): the settings page +looks every ten minutes and soon after a computer joins, and adds what is new (a key in the first free cell, a Dante +device or MIDI link into its key). Never during the auto-fix hours or a recording; nothing is ever removed; a thing that +was there before and was taken off by someone is not added back (`discover-seen.json`). "Look now" adds what is new at +once. Everything it makes is ordinary settings, changed on the page like North Attleboro's. + ## One-time Companion setup 1. **Variables → Custom Variables:** add three variables. Leave the start values empty and "persist" off: diff --git a/install.sh b/install.sh index 933c2aa..f897d6a 100644 --- a/install.sh +++ b/install.sh @@ -212,6 +212,17 @@ fi if [[ -d "$SRC/companion-objects" ]]; then rm -rf "$DEST/companion-objects" && cp -R "$SRC/companion-objects" "$DEST/" fi +# The kit another campus installs from (the settings page serves it as /campus/program.tgz). +rm -rf "$DEST/kit" && mkdir -p "$DEST/kit/sites" +cp "$SRC/install.sh" "$DEST/kit/" +[[ -f "$SRC/README.md" ]] && cp "$SRC/README.md" "$DEST/kit/" +[[ -d "$SRC/sites/_template" ]] && cp -R "$SRC/sites/_template" "$DEST/kit/sites/" +# The panel's own login key for the other computers (made once, never replaced). +if [[ ! -f "$HOME/.ssh/id_status_panel" ]]; then + mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh" + /usr/bin/ssh-keygen -q -t ed25519 -N "" -C "status-panel@$(scutil --get LocalHostName 2>/dev/null || hostname -s)" -f "$HOME/.ssh/id_status_panel" + echo "made the panel's login key ~/.ssh/id_status_panel" +fi # 4. The LaunchAgent, with the exit timeout this config needs (a running fix finishes before a restart). T=$(date +%s) diff --git a/schema.py b/schema.py index b16d91f..1bee633 100644 --- a/schema.py +++ b/schema.py @@ -112,6 +112,9 @@ field("auto_fix.windows", "list", [], "When auto-fix may act.", "Schedules") field("auto_fix.windows[].days", "days", None, "", "Schedules", required=True) field("auto_fix.windows[].start", "hhmm", None, "", "Schedules", required=True) field("auto_fix.windows[].end", "hhmm", None, "", "Schedules", required=True) +field("discover", "dict", {}, "Auto-setup: this Mac looks for gear every ten minutes.", "Site", advanced=True) +field("discover.auto", "bool", False, "Add new gear by itself: a new Companion connection, Dante device, MIDI link, computer " + "or program gets its key (never during a service or a recording; nothing is ever removed).", "Site") field("auto_fix.disabled", "bool", False, "Auto-fix switched off as a whole: nothing is fixed by itself (holds " "still work).", "Schedules") field("auto_fix.after_seconds", "int", 20, "A fault must last this long before auto-fix acts.", "Schedules", diff --git a/settings/autoconf.py b/settings/autoconf.py new file mode 100644 index 0000000..5fd112c --- /dev/null +++ b/settings/autoconf.py @@ -0,0 +1,517 @@ +"""Auto-setup: what a campus has (settings/discover.py's inventory) becomes its keys, checks, fixes +and page layout - the same building blocks North Attleboro's were made of by hand. + +Pure functions, no I/O: + + build(base, inv, campus, pin) a whole config for a new campus, from the empty template + extend(cfg, inv, seen) a running campus: only what is new (never removes anything) + +Everything that comes out passes schema.validate and can be changed on the settings page like any +hand-made setting. Gear auto-setup does not know (a generic connection, an app it has no recipe +for) is reported, not guessed at. +""" +import copy +import ipaddress +import re + +from settings import PARENT # noqa: F401 (puts the project folder on sys.path) +from settings import builders as b +import status_panel as sp + +# --- what auto-setup knows ------------------------------------------------------------- + +# Group keys sum up their members on a big campus, in this order along the top row. +GROUPS = ["REC", "AUDIO", "RF", "SHOW", "VIDEO", "STREAM"] +# Who gets a cell of its own when not everything fits (the rest show inside their group key only): +# a row of wireless receivers folds first, as at North Attleboro. +PRIORITY = ["REC", "SHOW", "AUDIO", "STREAM", "VIDEO", "RF"] + +# Companion modules: which group, what kind of key, the key's text when there is only one. +COMPANION = { + "bmd-hyperdeck": {"group": "REC", "kind": "hyperdeck", "label": "HD"}, + "tascam-da-6400": {"group": "REC", "kind": "tascam", "label": "TASCAM"}, + "audinate-dantecontroller": {"group": "AUDIO", "kind": "dante", "label": "DANTE"}, + "behringer-x32": {"group": "AUDIO", "kind": "conn", "label": "X32"}, + "behringer-wing": {"group": "AUDIO", "kind": "conn", "label": "WING"}, + "midas-m32": {"group": "AUDIO", "kind": "conn", "label": "M32"}, + "allenheath-sq": {"group": "AUDIO", "kind": "conn", "label": "SQ"}, + "allenheath-dlive": {"group": "AUDIO", "kind": "conn", "label": "DLIVE"}, + "allenheath-avantis": {"group": "AUDIO", "kind": "conn", "label": "AVANTIS"}, + "yamaha-rcp": {"group": "AUDIO", "kind": "conn", "label": "YAMAHA"}, + "soundcraft-ui": {"group": "AUDIO", "kind": "conn", "label": "UI24"}, + "digico-sd": {"group": "AUDIO", "kind": "conn", "label": "DIGICO"}, + "shure-wireless": {"group": "RF", "kind": "shure", "label": "MICS"}, + "shure-psm1000": {"group": "RF", "kind": "psm", "label": "PSM"}, + "renewedvision-propresenter-api": {"group": "SHOW", "kind": "conn", "label": "PROP", "app": "propresenter"}, + "ableton-osc": {"group": "SHOW", "kind": "conn", "label": "ABLETON", "app": "ableton"}, + "leolabs-ableset": {"group": "SHOW", "kind": "conn", "label": "ABLESET", "app": "ableton"}, + "bmd-atem": {"group": "VIDEO", "kind": "conn", "label": "ATEM"}, + "bmd-videohub": {"group": "VIDEO", "kind": "conn", "label": "VIDEOHUB"}, + "resolume-arena": {"group": "VIDEO", "kind": "conn", "label": "ARENA", "app": "arena"}, + "novastar-controller": {"group": "VIDEO", "kind": "conn", "label": "LED", "ping": True}, + "obs-studio": {"group": "STREAM", "kind": "conn", "label": "OBS", "app": "obs"}, + "resi-decoders": {"group": "STREAM", "kind": "conn", "label": "RESI", "ping": True}, +} + +# Programs on the computers: by OS, the process (Mac, as pgrep sees it) or program file (Windows). +# "app" ties a program to a Companion connection of the same name; "restart" adds a Restart page key. +MAC_APPS = [ + {"app": "propresenter", "process": "ProPresenter", "label": "PROP", "open": "/Applications/ProPresenter*.app", "restart": True}, + {"app": "ableton", "process": "Live", "label": "ABLETON", "open": "/Applications/Ableton Live*.app", "ableton": True, "restart": True}, + {"app": "arena", "process": "Arena", "label": "ARENA", "open": "/Applications/Resolume Arena*/Arena.app", "restart": True}, + {"app": "obs", "process": "obs", "label": "OBS", "open": "/Applications/OBS.app", "restart": True}, + {"app": "translate", "process": "Church Translate", "label": "TRANSLATE", "open": "/Applications/Church Translate.app", "restart": True}, + {"app": "protools", "process": "Pro Tools", "label": "PRO TOOLS", "open": "/Applications/Pro Tools.app", "restart": True}, +] +WIN_APPS = [ + {"app": "vista", "exe": "Vista.exe", "label": "VISTA", "task": "Start Vista", "restart": True}, + {"app": "arena", "exe": "Arena.exe", "label": "ARENA", "task": "Start Arena", "restart": True}, + {"app": "obs", "exe": "obs64.exe", "label": "OBS", "task": "Start OBS", "restart": True}, + {"app": "propresenter", "exe": "ProPresenter.exe", "label": "PROP", "task": "Start ProPresenter", "restart": True}, + {"app": "rtpmidi", "exe": "rtpMIDI.exe", "label": "RTPMIDI", "task": "Start rtpMIDI", "restart": False, "fold": "midi_wired"}, +] +# The task names the Windows join script makes, by program file (it reads WIN_APPS through campus.py). +WIN_TASKS = {a["exe"]: a["task"] for a in WIN_APPS} + +# Fixes shared by every key of a kind (the same words North Attleboro's carry). +SHARED_FIXES = { + "hyperdeck": {"kind": "hyperdeck", "template": "hyperdeck", "params": {}, "short": "HYPERDECK", "verify_seconds": 15, + "hung_note": "STILL OFFLINE\nHOLD AGAIN", + "label": "turn Companion's connection to that HyperDeck off and on (what gets a restarted deck green); never reboots the deck"}, + "midi_reconnect": {"kind": "midi_connect", "template": "midi_connect", "params": {}, "short": "RECONNECT", + "label": "reconnect the peer to the network MIDI session it dropped out of (like Connect in Audio MIDI Setup), at its wired address"}, +} + +STATUS_ROWS, COLS = 4, 8 +DEFAULT_AUTO_FIX = {"after_seconds": 20, "max_tries": 2, "retry_seconds": 90, "pause_after_network_seconds": 60, + "restart_connections": True, "windows": [{"days": ["sun"], "start": "05:00", "end": "14:00"}]} + + +# --- small helpers --------------------------------------------------------------------- + +def slug(text, taken=()): + base = re.sub(r"[^a-z0-9]+", "_", str(text).lower()).strip("_") or "key" + out, n = base, 2 + while out in taken: + out, n = "%s_%d" % (base, n), n + 1 + return out + + +def key_text(label, limit=10): + t = re.sub(r"[_\s]+", " ", str(label)).strip().upper() + return t if len(t) <= limit else t[:limit].rstrip() + + +def tag_for(name): + return key_text(name, 12) + + +def ip_of(address): + ip = sp.peer_ip(address) + return str(ip) if ip else None + + +def in_nets(nets, address): + ip = sp.peer_ip(address) + return ip is not None and any(ip in ipaddress.ip_network(n, strict=False) for n in nets) + + +def computer_by_ip(inv, ip): + for alias, c in (inv.get("computers") or {}).items(): + if ip and ip in (c.get("ips") or []): + return alias + return None + + +def referenced(cfg): + """Everything a config already watches, by the same identities discovery uses.""" + out = set() + for t in sp.all_tiles(cfg): + for c in t.get("checks", []): + if c.get("conn"): + out.add("conn:" + c["conn"]) + if c.get("type") == "dante": + out.update("dante:" + d for d in (c.get("devices") or {})) + if c.get("type") == "midi_peer": + out.add("midi:%s|%s|%s" % (c.get("host", ""), c.get("session"), c.get("peer"))) + if c.get("type") == "midi_wired": + out.add("midi_wired") + p = c.get("params") or {} + if c.get("type") == "remote" and p.get("host") and (p.get("process") or p.get("exe")): + out.add("app:%s|%s" % (p["host"], p.get("process") or p.get("exe"))) + for alias, h in (cfg.get("hosts") or {}).items(): + out.add("host:" + alias) + ip = str(h.get("ssh", "")).split("@")[-1] + if ip: + out.add("hostip:" + ip) + return out + + +# --- from the inventory to items ------------------------------------------------------ + +def items(inv, nets): + """[{ident, group, label, checks, fixes, restart, also_idents}] for everything auto-setup can watch, + plus the computer records ({alias: host}). Each item is one key.""" + out, hosts = [], {} + comps = inv.get("computers") or {} + for alias, c in comps.items(): + if not c.get("reachable"): + continue + hosts[alias] = {"name": c.get("name") or alias, "os": c.get("os", "mac"), "ssh": "%s@%s" % (c["user"], c["ip"]), + "role": "found by auto-setup", "runs": [{"kind": "sshd"}]} + if c.get("macs"): + hosts[alias]["mac"] = c["macs"][0] + # Programs on the computers, by app, so a Companion connection to the same program joins its key. + apps_at = {} # (app, ip) -> item + for alias, c in comps.items(): + if not c.get("reachable"): + continue + procs = set(c.get("procs") or []) + catalog = MAC_APPS if c.get("os", "mac") == "mac" else WIN_APPS + for a in catalog: + if c.get("os", "mac") == "mac": + if a["process"] not in procs: + continue + app_path = newest_app(c.get("apps") or [], a["open"]) + if not app_path: + continue + check = {"type": "remote", "template": "mac_program", "params": {"host": alias, "process": a["process"]}, + "msg": "%s NOT RUNNING" % a["label"]} + if a.get("ableton"): + start = {"template": "ableton", "params": {"host": alias, "app": app_path.rsplit("/", 1)[-1][:-4], "restart": False}} + again = {"template": "ableton", "params": {"host": alias, "app": app_path.rsplit("/", 1)[-1][:-4], "restart": True}} + else: + start = {"template": "mac_start", "params": {"host": alias, "process": a["process"], "app": app_path}} + again = {"template": "mac_restart", "params": {"host": alias, "process": a["process"], "app": app_path, + "wait_seconds": 10, "force": True}} + ident = "app:%s|%s" % (alias, a["process"]) + else: + if a["exe"] not in procs or a["task"] not in (c.get("tasks") or []): + continue # no start task on that PC: a hold could not start it + check = {"type": "remote", "template": "win_program", "params": {"host": alias, "exe": a["exe"]}, + "msg": "%s NOT RUNNING" % a["label"]} + start = {"template": "win_start", "params": {"host": alias, "task": a["task"], "exe": a["exe"]}} + again = {"template": "win_restart", "params": {"host": alias, "task": a["task"], "exe": a["exe"]}} + ident = "app:%s|%s" % (alias, a["exe"]) + fid = slug("%s_%s" % (a["app"], alias)) + pretty = {"propresenter": "ProPresenter", "ableton": "Ableton", "arena": "Resolume Arena", "obs": "OBS", "translate": "Church Translate", + "protools": "Pro Tools", "vista": "Vista", "rtpmidi": "rtpMIDI"}.get(a["app"], a["label"].title()) + where = c.get("name") or alias + start = dict(start, label="start %s on %s if it is not running" % (pretty, where), short=a["label"]) + again = dict(again, label="restart %s on %s (closes it first)" % (pretty, where), short=a["label"]) + item = {"ident": ident, "group": "SHOW" if a["app"] not in ("obs",) else "STREAM", "label": a["label"], + "checks": [dict(check, fix=fid)], "fixes": {fid: start}, "restart": [], "also": [], "fold": a.get("fold")} + if a.get("restart"): + rid = "restart_" + fid + item["fixes"][rid] = again + item["restart"].append({"label": a["label"], "fix": rid}) + if a["app"] == "arena": + item["group"] = "VIDEO" + out.append(item) + apps_at[(a["app"], c.get("ip"))] = item + for ip in c.get("ips") or []: + apps_at.setdefault((a["app"], ip), item) + # Companion connections. + counts = {} + for conn in inv.get("connections") or []: + counts[conn.get("module")] = counts.get(conn.get("module"), 0) + 1 + hd_n = 0 + for conn in sorted(inv.get("connections") or [], key=lambda c: c["label"]): + spec = COMPANION.get(conn.get("module")) + if not spec or not conn.get("enabled", True): + continue + label = conn["label"] + ident = "conn:" + label + many = counts.get(conn["module"], 0) > 1 + text = spec["label"] if not many else key_text(label) + if spec["kind"] == "hyperdeck": + hd_n += 1 + text = "HD%d" % hd_n if many else "HD" + out.append({"ident": ident, "group": "REC", "label": text, "restart": [], "also": [], + "checks": [{"type": "hyperdeck", "conn": label, "fix": {"conn": "hyperdeck"}}], + "fixes": {"hyperdeck": dict(SHARED_FIXES["hyperdeck"])}}) + continue + if spec["kind"] == "tascam": + fid = slug("reconnect_" + label) + out.append({"ident": ident, "group": "REC", "label": text, "restart": [], "also": [], + "checks": [{"type": "tascam", "conn": label, "fix": fid}, {"type": "ping", "conn": label, "name": text}], + "fixes": {fid: {"kind": "reconnect", "template": "reconnect", "params": {}, "conn": label, "short": text, + "verify_seconds": 20, "label": "reconnect Companion to %s" % label}}}) + continue + if spec["kind"] == "dante": + devices = {d: key_text(d, 12) for d in (inv.get("dante") or {}).get("devices") or []} + out.append({"ident": ident, "group": "AUDIO", "label": "DANTE", "restart": [], "also": ["dante:" + d for d in devices], + "checks": [{"type": "dante", "conn": label, "devices": devices, "fix": {"*": sp.DANTE_AUTO}}], "fixes": {}}) + continue + if spec["kind"] in ("shure", "psm"): + n = int(conn.get("channels") or (4 if spec["kind"] == "shure" else 2)) + out.append({"ident": ident, "group": "RF", "label": text if many else ("MICS" if spec["kind"] == "shure" else "PSM"), + "restart": [], "also": [], + "checks": [{"type": spec["kind"], "conn": label, "channels": n}, {"type": "ping", "conn": label}], "fixes": {}}) + continue + checks = [{"type": "conn", "conn": label}] + if spec.get("ping") and conn.get("host"): + checks.append({"type": "ping", "conn": label}) + app_item = apps_at.get((spec.get("app"), ip_of(conn.get("host")))) if spec.get("app") else None + if app_item: # the program's own key carries its connection too + app_item["checks"] = checks + app_item["checks"] + app_item["also"].append(ident) + continue + out.append({"ident": ident, "group": spec["group"], "label": text, "checks": checks, "fixes": {}, "restart": [], "also": []}) + # Network MIDI: every peer in the sessions here and on the other Macs, at its wired address. + midi_checks = [] + for host, sessions in (inv.get("midi") or {}).items(): + for sname, sess in (sessions or {}).items(): + for p in sess.get("peers") or []: + addr = p.get("address") or "" + if not in_nets(nets, addr): + alias = computer_named(inv, p.get("name")) + ip = alias and next((i for i in comps[alias].get("ips") or [] if in_nets(nets, i)), None) + if not ip: + continue # no wired address known: the wired check will name it + addr = "%s:5004" % ip + chk = {"type": "midi_peer", "session": sname, "peer": p["name"], "address": addr, "fix": "midi_reconnect", + "tag": tag_for(p["name"])} + if host: + chk["host"] = host + midi_checks.append(("midi:%s|%s|%s" % (host, sname, p["name"]), chk)) + if midi_checks: + checks = [c for _, c in midi_checks] + [{"type": "midi_wired", "nets": list(nets), "fix": "midi_reconnect"}] + out.append({"ident": "midi_wired", "group": "SHOW", "label": "MIDI", "checks": checks, + "fixes": {"midi_reconnect": dict(SHARED_FIXES["midi_reconnect"])}, "restart": [], + "also": [i for i, _ in midi_checks]}) + # A helper program (rtpMIDI) rides inside the key it serves when that key exists. + for it in [x for x in out if x.get("fold")]: + home = next((x for x in out if x["ident"] == it["fold"]), None) + if home is not None: + home["checks"] += it["checks"] + home["fixes"].update(it["fixes"]) + home["also"].append(it["ident"]) + out.remove(it) + return out, hosts + + +def computer_named(inv, name): + n = str(name or "").lower() + for alias, c in (inv.get("computers") or {}).items(): + if n and n in (str(c.get("name", "")).lower(), str(c.get("hostname", "")).lower().replace(".local", "")): + return alias + return None + + +def newest_app(apps, pattern): + """The newest /Applications entry matching a glob-ish pattern ("/Applications/ProPresenter*.app").""" + rx = re.compile("^" + re.escape(pattern).replace(r"\*", ".*") + "$") + hits = sorted(a for a in apps if rx.match(a)) + return hits[-1] if hits else None + + +# --- layout ---------------------------------------------------------------------------- + +def free_cells(taken, rows=STATUS_ROWS): + return [(r, c) for r in range(rows) for c in range(1, COLS) if (r, c) not in taken] + + +def layout(cfg, new_tiles, restart_items): + """Put the keys on the Status page: everything fits, so no group keys; otherwise group keys along + the top row and the members after them (a member with no room left shows inside its group only).""" + fixed = {tuple(t["pos"]) for t in cfg["tiles"] if t.get("pos")} + if restart_items: + fixed.add(tuple(cfg["restart_page"]["jump_pos"])) + cells = free_cells(fixed) + groups = [g for g in GROUPS if any(t["_group"] == g for t in new_tiles)] + if len(new_tiles) > len(cells) and len(groups) > 1: + for g in groups: + members = [t["id"] for t in new_tiles if t["_group"] == g] + cfg["tiles"].append({"id": slug(g, {t["id"] for t in cfg["tiles"]} | {t["id"] for t in new_tiles}), + "label": g, "members": members, "_group": g}) + group_tiles = [t for t in cfg["tiles"] if t.get("_group")] + for t in group_tiles: + if cells: + t["pos"] = list(cells.pop(0)) + # Who gets a cell: by priority while cells last; then placed in group order, so a group reads as a block. + chosen = set() + for g in PRIORITY: + for t in new_tiles: + if t["_group"] == g and len(chosen) < len(cells): + chosen.add(t["id"]) + for t in new_tiles: + if t["id"] in chosen: + t["pos"] = list(cells.pop(0)) + cfg["tiles"].append(t) + for t in cfg["tiles"]: + t.pop("_group", None) + + +def place_new(cfg, tile): + """One more key on a running campus: the first free cell, else inside its group, else on ALL only.""" + taken = {tuple(t["pos"]) for t in cfg["tiles"] if t.get("pos")} + if cfg.get("restart_page"): + taken.add(tuple(cfg["restart_page"]["jump_pos"])) + cells = free_cells(taken) + group = next((t for t in cfg["tiles"] if t.get("label") == tile.get("_group") and t.get("members") is not None), None) + if cells: + tile["pos"] = list(cells[0]) + if group is not None: + group["members"].append(tile["id"]) + tile.pop("_group", None) + cfg["tiles"].append(tile) + + +def restart_layout(cfg, restart_items): + if not restart_items: + cfg.pop("restart_page", None) + return + rp = cfg["restart_page"] + taken = {tuple(rp["back_pos"])} | {tuple(k["pos"]) for k in rp.get("keys", [])} + cells = free_cells(taken) + ids = {k["id"] for k in rp.get("keys", [])} + for it in restart_items: + kid = slug("rs_" + it["label"], ids) + ids.add(kid) + if not cells: + break + rp.setdefault("keys", []).append({"id": kid, "label": key_text(it["label"]), "fix": it["fix"], "pos": list(cells.pop(0))}) + + +# --- the two entry points -------------------------------------------------------------- + +def nets_of(inv): + w = (inv.get("ingest") or {}).get("wired") or {} + return [w["net"]] if w.get("net") else [] + + +def build(base, inv, campus, pin, pages=(19, 20)): + """A whole config for a new campus: the template, this Mac, and a key for everything found.""" + cfg = copy.deepcopy(base) + ing = inv.get("ingest") or {} + wired = ing.get("wired") or {} + site = cfg["site"] + site.update({"id": campus["id"], "name": campus["name"], "ingest_name": ing.get("name") or "%s Ingest" % campus["name"], + "ingest_hostname": ing.get("hostname") or "", "ingest_ips": [ip for ip in [wired.get("ip")] + list(ing.get("ips") or []) if ip], + "editing_code": pin}) + site["ingest_ips"] = list(dict.fromkeys(site["ingest_ips"])) + cfg["discover"] = {"auto": True} + cfg["page"], cfg["page_name"] = pages[0], "%d\\nStatus" % pages[0] + cfg["restart_page"] = {"page": pages[1], "page_name": "%d\\nRestart" % pages[1], "jump_pos": [3, 6], "back_pos": [3, 0], "keys": []} + cfg["network"] = dict(cfg.get("network") or {}, interface=wired.get("interface") or "en0", gateway=wired.get("gateway") or "") + cfg["service_windows"] = [{"days": list(sp.DAYS), "start": "00:00", "end": "23:59"}] + cfg["hosts"]["ingest"]["name"] = site["ingest_name"] + for t in cfg["tiles"]: + if t.get("kind") == "network": + t["pos"] = [3, 7] + found, hosts = items(inv, nets_of(inv)) + cfg["hosts"].update(hosts) + tiles, restart, taken = [], [], {t["id"] for t in cfg["tiles"]} + for it in found: + tid = slug(it["label"], taken) + taken.add(tid) + tiles.append({"id": tid, "label": key_text(it["label"]), "checks": it["checks"], "_group": it["group"]}) + for fid, f in it["fixes"].items(): + cfg["fixes"].setdefault(fid, f) + restart += it["restart"] + tiles.sort(key=lambda t: GROUPS.index(t["_group"])) + layout(cfg, tiles, restart) + restart_layout(cfg, restart) + finish(cfg) + a = dict(DEFAULT_AUTO_FIX, **{k: v for k, v in (cfg.get("auto_fix") or {}).items() if k not in ("windows", "fixes")}) + a["fixes"] = gentle(cfg) + cfg["auto_fix"] = a + return cfg, [it["ident"] for it in found] + ["host:" + h for h in hosts] + [x for it in found for x in it["also"]] + + +def extend(cfg, inv, seen): + """What is new since the last look, added to a running campus. Never removes or changes what is + there; a thing seen before (and so deliberately taken off by someone) is not added again. + Returns (new config, [what was added, in words], [identities now seen]).""" + cfg = copy.deepcopy(cfg) + have = referenced(cfg) + skip = have | set(seen) + nets = nets_of(inv) or [n for t in sp.all_tiles(cfg) for c in t.get("checks", []) if c.get("type") == "midi_wired" for n in c.get("nets", [])] + found, hosts = items(inv, nets) + added, now_seen = [], [] + for alias, h in hosts.items(): + if "host:" + alias in skip or "hostip:" + h["ssh"].split("@")[-1] in have: + continue + cfg["hosts"][alias] = h + added.append("computer %s" % h["name"]) + now_seen.append("host:" + alias) + taken = {t["id"] for t in sp.all_tiles(cfg)} + for it in found: + # A key that already exists for this thing: only its new parts (a Dante device, a MIDI peer). + if it["ident"] in skip: + fresh = [x for x in it["also"] if x not in skip] + if fresh: + added += merge_parts(cfg, it, fresh) + now_seen += fresh + continue + if any(c.get("params", {}).get("host") not in cfg["hosts"] for c in it["checks"] if c.get("template")): + continue + tid = slug(it["label"], taken) + taken.add(tid) + for fid, f in it["fixes"].items(): + cfg["fixes"].setdefault(fid, f) + place_new(cfg, {"id": tid, "label": key_text(it["label"]), "checks": it["checks"], "_group": it["group"]}) + if it["restart"] and cfg.get("restart_page"): + restart_layout(cfg, it["restart"]) + added.append("%s key" % key_text(it["label"])) + now_seen += [it["ident"]] + it["also"] + if added: + finish(cfg) + auto = cfg.setdefault("auto_fix", dict(DEFAULT_AUTO_FIX, fixes=[])) + auto["fixes"] = list(dict.fromkeys(list(auto.get("fixes") or []) + [f for f in gentle(cfg) if f not in auto.get("fixes", []) + and any(f in it["fixes"] for it in found)])) + return cfg, added, now_seen + + +def merge_parts(cfg, it, fresh): + """New Dante devices and MIDI peers go into the key that already watches the rest.""" + words = [] + for t in sp.all_tiles(cfg): + for c in t.get("checks", []): + if c.get("type") == "dante" and it["ident"] == "conn:" + c.get("conn", ""): + for x in fresh: + if x.startswith("dante:"): + name = x[6:] + c.setdefault("devices", {})[name] = key_text(name, 12) + words.append("Dante device %s" % name) + if it["ident"] == "midi_wired": + midi = next((t for t in cfg["tiles"] if any(c.get("type") == "midi_wired" for c in t.get("checks", []))), None) + if midi is not None: + wired_at = next(i for i, c in enumerate(midi["checks"]) if c.get("type") == "midi_wired") + for c in it["checks"]: + if c["type"] == "midi_peer" and "midi:%s|%s|%s" % (c.get("host", ""), c["session"], c["peer"]) in fresh: + midi["checks"].insert(wired_at, c) + wired_at += 1 + words.append("MIDI link %s" % c["peer"]) + cfg["fixes"].setdefault("midi_reconnect", dict(SHARED_FIXES["midi_reconnect"])) + return words + + +def finish(cfg): + """Fixes and remote checks built by their builders, words filled in, as the page would.""" + for fid, f in list(cfg["fixes"].items()): + if f.get("template") and "run" not in f and b.TEMPLATES.get(f["template"], {}).get("render"): + built = b.build(cfg, f) + built.setdefault("label", b.suggest_label(cfg, built) or fid) + short = key_text(fid.replace("restart_", "").split("_")[0], 10) + built.setdefault("short", short) + cfg["fixes"][fid] = built + elif f.get("template") and not f.get("label"): + f["label"] = b.suggest_label(cfg, f) or fid + for t in cfg["tiles"]: + for i, c in enumerate(t.get("checks", [])): + if c.get("template"): + t["checks"][i] = b.build_check(cfg, c) + + +def gentle(cfg): + """The fixes auto-fix may run by itself: starting what is closed and reconnecting - never a restart.""" + out = [] + for fid, f in cfg["fixes"].items(): + if fid.startswith("restart_") or f.get("restart_ok") or f.get("never_auto"): + continue + if f.get("kind") in ("hyperdeck", "midi_connect", "reconnect") or (f.get("template") in ("mac_start", "win_start", "ableton") + and not (f.get("params") or {}).get("restart")): + out.append(fid) + return out diff --git a/settings/builders.py b/settings/builders.py index 4d69a68..d18fa3f 100644 --- a/settings/builders.py +++ b/settings/builders.py @@ -322,6 +322,11 @@ CHECK_TEMPLATES = { "params": [P_HOST, ("process", "Process name", "As pgrep sees it.", "str", None)], "render": lambda p, cfg: "pgrep -xq %s" % q(p["process"]), }, + "win_program": { + "words": "A program is running on a Windows PC", "type": "remote", + "params": [P_HOST, ("exe", "Program file", "As Task Manager shows it (Vista.exe).", "str", None)], + "render": lambda p, cfg: win_running(win(p["exe"], "program file")), + }, "helper_flag": { "words": "A helper file does not say something is wrong", "type": "remote", "params": [P_HOST, ("file", "File", "In the helper's folder (solo-mode.json).", "str", "solo-mode.json"), diff --git a/settings/campus.py b/settings/campus.py new file mode 100644 index 0000000..74ff1b5 --- /dev/null +++ b/settings/campus.py @@ -0,0 +1,429 @@ +"""A campus that sets itself up. + +On the campus's Mac that runs Companion, one line installs the panel from another campus's settings +page and runs `setup`: Companion gets the panel's custom variables and blink clock, two free pages +are picked, the gear is found (settings/discover.py) and turned into keys (settings/autoconf.py), +and that config is applied like any Apply on the page. Each other computer runs one join line +(Terminal on a Mac, an administrator PowerShell on a PC); from then on the settings page looks again +every ten minutes and adds what is new by itself (Settings > Site: "Add new gear by itself"). + + python3 -m settings.campus setup --id ab [--name "Apollo Beach"] + python3 -m settings.campus look what it finds (nothing changes) + python3 -m settings.campus add add what is new now +""" +import argparse +import base64 +import gzip +import hashlib +import io +import json +import os +import random +import re +import tarfile +import threading +import time + +from settings import PARENT # noqa: F401 (puts the project folder on sys.path) +from settings import autoconf, discover, scripts +import push_page +import schema +import status_panel as sp + +CAMPUSES = {"na": "North Attleboro", "ab": "Apollo Beach", "sc": "Southcoast", "ws": "Woonsocket"} +VARIABLES = ("sp_state", "sp_hb", "sp_cmd", "flash_tick") +TRIGGER_NAME = "flash_tick_500" +LOOK_EVERY = 600 +SEEN, LAST = "discover-seen.json", "discover-last.json" +# What the program kit carries (no settings, keys or history of the campus that serves it). +KIT_FILES = ("status_panel.py", "readiness.py", "coremidi.py", "netgear.py", "schema.py", "make_page.py", "push_page.py", + "nav_keys.py", "atem_watch.js", "atem_state.js", "nav_templates.json") +KIT_DIRS = ("settings", "remote-access", "companion-objects") + + +# --- Companion: the panel's variables, its blink clock, and two free pages --------------- + +def flash_trigger_export(build="5.0.7", name=TRIGGER_NAME, enabled=True): + """A Companion export holding one trigger: flash_tick 1 for half a second, every second.""" + def nid(): + return base64.urlsafe_b64encode(os.urandom(15)).decode().rstrip("=")[:21] + + def setvar(v): + return {"id": nid(), "definitionId": "custom_variable_set_value", "connectionId": "internal", + "options": {"name": {"isExpression": False, "value": "flash_tick"}, "value": {"isExpression": False, "value": v}}, + "type": "action", "children": {}} + tid = nid() + trig = {"type": "trigger", "options": {"name": name, "enabled": enabled, "sortOrder": 0}, + "actions": [setvar("1"), {"id": nid(), "definitionId": "wait", "connectionId": "internal", + "options": {"time": {"isExpression": False, "value": "500"}}, "type": "action", "children": {}}, setvar("0")], + "condition": [], "events": [{"id": nid(), "type": "interval", "enabled": True, "options": {"seconds": 1}}], "localVariables": []} + return tid, {"version": 12, "type": "full", "companionBuild": build, "triggers": {tid: trig}, "triggerCollections": [], + "custom_variables": {}, "customVariablesCollections": [], "instances": {}, "connectionCollections": []} + + +def companion_state(cfg): + raw = discover.companion_get(cfg, "/int/export/custom?buttons=false&connections=false&surfaces=&triggers=true&customVariables=true" + "&expressionVariables=false&includeSecrets=false&imageLibrary=false&format=json", 10) + d = json.loads(raw) + names = {(t.get("options") or {}).get("name") for t in (d.get("triggers") or {}).values()} + return set(d.get("custom_variables") or {}), names + + +def companion_prepare(cfg): + """Create what the pages need inside Companion and is otherwise made by hand in its UI. Returns words.""" + comp = cfg.get("companion") or {} + host, port = comp.get("host", "127.0.0.1"), int(comp.get("port", 8000)) + have_vars, triggers = companion_state(cfg) + done = [] + missing = [v for v in VARIABLES if v not in have_vars] + if not missing and TRIGGER_NAME in triggers: + return done + ws = push_page.WebSocket(host, port, "/trpc") + try: + api = push_page.Trpc(ws) + for v in missing: + api.mutation("customVariables.create", {"name": v, "defaultVal": "0" if v == "flash_tick" else ""}) + done.append("custom variable %s" % v) + if TRIGGER_NAME not in triggers: + tid, export = flash_trigger_export() + data = gzip.compress(json.dumps(export).encode()) + session = api.mutation("importExport.prepareImport.start", {"name": "status-panel-blink.companionconfig", "size": len(data)}) + api.mutation("importExport.prepareImport.uploadChunk", {"sessionId": session, "offset": 0, "data": base64.b64encode(data).decode()}) + summary = api.mutation("importExport.prepareImport.complete", {"sessionId": session, "expectedChecksum": hashlib.sha1(data).hexdigest(), "userData": None}) + if isinstance(summary, list) and summary and summary[0]: + raise RuntimeError("Companion could not read the blink trigger: %s" % summary[0]) + api.mutation("importExport.importTriggers", {"selectedTriggerIds": [tid], "connectionIdRemapping": {}, "replaceExisting": False}) + done.append("the %s trigger (the keys' blink clock)" % TRIGGER_NAME) + finally: + ws.close() + return done + + +def pick_pages(cfg, count=2, start=19): + """Pages that hold nothing a person made (or only this panel's keys): the first `count` from `start`.""" + comp = cfg.get("companion") or {} + host, port = comp.get("host", "127.0.0.1"), int(comp.get("port", 8000)) + out = [] + for page in list(range(start, 100)) + list(range(2, start)): + try: + ok, _ = push_page.safe_target(push_page.fetch_page(host, port, page)) + except (OSError, ValueError): + continue + if ok: + out.append(page) + if len(out) == count: + return out + raise RuntimeError("Companion has no %d empty pages for the status panel" % count) + + +# --- the first setup -------------------------------------------------------------------- + +def template(data_dir): + for path in (os.path.join(data_dir, "kit", "sites", "_template", "config.json"), + os.path.join(PARENT, "sites", "_template", "config.json")): + if os.path.exists(path): + with open(path) as f: + return json.load(f) + raise RuntimeError("no sites/_template/config.json to start from") + + +def is_fresh(cfg): + return (cfg.get("site") or {}).get("id") in ("new", "", None) and len(cfg.get("tiles") or []) <= 3 + + +def load_seen(data_dir): + try: + with open(os.path.join(data_dir, SEEN)) as f: + return set(json.load(f)) + except (OSError, ValueError): + return set() + + +def save_seen(data_dir, seen): + sp.write_json(os.path.join(data_dir, SEEN), sorted(seen)) + + +def wait_applied(applier, limit=240): + t0 = time.time() + while time.time() - t0 < limit: + st = applier.status or {} + if st.get("state") not in ("running",): + return st + time.sleep(1) + return applier.status or {} + + +def setup(data_dir, cid, name=None, say=print): + """The first setup of a campus (or, when it is set up already, a look for what is new).""" + from settings import server + site = server.Site(data_dir) + cfg = site.load() + if not is_fresh(cfg): + say("%s is set up already: looking for what is new." % (cfg.get("site") or {}).get("name", "This campus")) + return look_and_add(site, say=say, force=True) + name = name or CAMPUSES.get(cid) or cid.upper() + for line in companion_prepare(cfg): + say("Companion: added %s" % line) + pages = pick_pages(cfg) + say("Companion pages %d (status) and %d (restart) are free: using them." % tuple(pages)) + inv = discover.look(cfg, data_dir) + if inv.get("companion_error"): + raise RuntimeError("Companion's HTTP API does not answer (%s): turn it on in Companion's settings, then run this again" % inv["companion_error"]) + pin = "%04d" % random.SystemRandom().randrange(1000, 10000) + new, seen = autoconf.build(template(data_dir), inv, {"id": cid, "name": name}, pin, pages) + errors, _ = schema.validate(new) + if errors: + raise RuntimeError("the config auto-setup made does not pass the rules: %s" % "; ".join(errors[:4])) + r = site.applier.start(new, "auto-setup: first setup of %s" % name, "auto-setup", "127.0.0.1") + if not r.get("ok"): + raise RuntimeError(r.get("error") or "the Apply did not start") + st = wait_applied(site.applier) + save_seen(data_dir, seen) + keys = [t["label"] for t in new["tiles"] if not t.get("kind")] + record(data_dir, {"at": int(time.time()), "added": ["%d keys: %s" % (len(keys), ", ".join(keys))], "inventory": summary(inv)}) + say("%s: %s" % (st.get("state"), st.get("outcome", ""))) + say("Keys: %s" % ", ".join(keys)) + say("Editing code (PIN) for the settings page: %s" % pin) + return new + + +def summary(inv): + known = set(autoconf.COMPANION) + return {"connections": len(inv.get("connections") or []), + "not_known": sorted("%s (%s)" % (c["label"], c["module"]) for c in inv.get("connections") or [] if c["module"] not in known and c["module"] != "internal"), + "dante": len((inv.get("dante") or {}).get("devices") or []), + "computers": {a: {"name": c.get("name"), "os": c.get("os"), "ip": c.get("ip"), "reachable": bool(c.get("reachable")), + "error": c.get("error", "")} for a, c in (inv.get("computers") or {}).items()}, + "midi_sessions": sum(len(s or {}) for s in (inv.get("midi") or {}).values()), + "wired": ((inv.get("ingest") or {}).get("wired") or {}).get("net")} + + +def record(data_dir, result): + sp.write_json(os.path.join(data_dir, LAST), result) + + +# --- every ten minutes: what is new ------------------------------------------------------ + +def blocked(cfg, applier, now=None): + """Why auto-setup may not apply right now, or None: never during a service or a recording.""" + now = now or time.time() + live, why = applier.live() + if live: + return "live now (%s)" % why + if sp.in_windows((cfg.get("auto_fix") or {}).get("windows") or [], now): + return "inside the auto-fix hours (a service day): waits until they end" + if applier.busy(): + return "an Apply is running" + return None + + +def look_and_add(site, say=lambda s: None, force=False): + """Look, and add what is new (when the campus allows it, and not during a service).""" + cfg = site.load() + inv = discover.look(cfg, site.data_dir) + seen = load_seen(site.data_dir) + new, added, now_seen = autoconf.extend(cfg, inv, seen) + result = {"at": int(time.time()), "added": [], "waiting": [], "inventory": summary(inv)} + if added: + auto = (cfg.get("discover") or {}).get("auto") + why = blocked(cfg, site.applier) + if not (auto or force): + result["waiting"] = added + result["why"] = "adding by itself is off (Settings > Site)" + elif why: + result["waiting"] = added + result["why"] = why + else: + errors, _ = schema.validate(new) + if errors: + result["waiting"], result["why"] = added, "would not pass the rules: %s" % "; ".join(errors[:3]) + else: + r = site.applier.start(new, "auto-setup found: %s" % ", ".join(added)[:180], "auto-setup", "127.0.0.1") + if r.get("ok"): + st = wait_applied(site.applier) + if st.get("state") == "done": + save_seen(site.data_dir, seen | set(now_seen)) + result["added"] = added + else: + result["waiting"], result["why"] = added, st.get("outcome") or "the Apply did not finish" + else: + result["waiting"], result["why"] = added, r.get("error", "the Apply did not start") + record(site.data_dir, result) + say("added: %s" % (", ".join(result["added"]) or "nothing")) + if result["waiting"]: + say("waiting: %s (%s)" % (", ".join(result["waiting"]), result.get("why"))) + return result + + +def loop(site, stop, poke): + """The settings page's own background look: every ten minutes, and soon after a computer joins.""" + stop.wait(60) + while not stop.is_set(): + try: + if (site.load().get("discover") or {}).get("auto") and not is_fresh(site.cfg): + look_and_add(site) + except Exception as e: # noqa: BLE001 - the page must keep running + record(site.data_dir, dict(load_last(site.data_dir), error=str(e), at=int(time.time()))) + poke.wait(LOOK_EVERY) + if poke.is_set(): + poke.clear() + stop.wait(20) # a computer just joined: give its SSH a moment + + +def load_last(data_dir): + try: + with open(os.path.join(data_dir, LAST)) as f: + return json.load(f) + except (OSError, ValueError): + return {} + + +# --- the lines people run ---------------------------------------------------------------- + +def program_kit(data_dir): + """The program as a .tgz: the installed code, the settings page, the templates, install.sh.""" + buf = io.BytesIO() + with tarfile.open(fileobj=buf, mode="w:gz") as tar: + for f in KIT_FILES: + p = os.path.join(data_dir, f) + if os.path.exists(p): + tar.add(p, f) + for d in KIT_DIRS: + p = os.path.join(data_dir, d) + if os.path.isdir(p): + tar.add(p, d, filter=lambda ti: None if "__pycache__" in ti.name or ti.name.endswith(".pyc") else ti) + kit = os.path.join(data_dir, "kit") + for name in ("install.sh", "README.md"): + p = os.path.join(kit, name) + if os.path.exists(p): + tar.add(p, name) + p = os.path.join(kit, "sites", "_template") + if os.path.isdir(p): + tar.add(p, "sites/_template") + return buf.getvalue() + + +def shq(s): + return "'" + str(s).replace("'", "'\"'\"'") + "'" + + +def install_script(base, cid, support_key=None): + """What a new campus's Companion Mac runs: install from `base` (this page), then set itself up.""" + m = re.match(r"[a-z0-9]+", str(cid).lower()) + cid = m.group(0)[:8] if m else "new" + name = CAMPUSES.get(cid, cid.upper()) + key = support_key or "" + return """#!/bin/zsh +# Status panel for %(name)s: installs on this Mac (the one that runs Companion) and sets itself up. +# Safe to run again: it then updates the program and looks for new gear. +set -e +SRC=%(base)s +if ! /usr/bin/xcode-select -p >/dev/null 2>&1; then + echo "First the Command Line Tools: a window opens - click Install, wait for it, then run this line again." + /usr/bin/xcode-select --install 2>/dev/null || true + exit 1 +fi +DIR="$HOME/status-panel-install" +mkdir -p "$DIR" && cd "$DIR" +echo "Downloading the status panel from $SRC ..." +/usr/bin/curl -fsSL "$SRC/campus/program.tgz" | /usr/bin/tar -xz +[[ -f "$HOME/.ssh/id_status_panel" ]] || { mkdir -p "$HOME/.ssh" && chmod 700 "$HOME/.ssh" && /usr/bin/ssh-keygen -q -t ed25519 -N "" -C "status-panel@$(scutil --get LocalHostName)" -f "$HOME/.ssh/id_status_panel"; } +SUPPORT=%(key)s +if [[ -n "$SUPPORT" ]]; then + touch "$HOME/.ssh/authorized_keys" && chmod 600 "$HOME/.ssh/authorized_keys" + grep -qF "$SUPPORT" "$HOME/.ssh/authorized_keys" || echo "$SUPPORT" >> "$HOME/.ssh/authorized_keys" +fi +zsh install.sh --site %(cid)s +cd "$HOME/Library/Application Support/status-panel" +/usr/bin/python3 -m settings.campus setup --id %(cid)s --name %(qname)s +if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then + echo + echo "One more thing: turn on Remote Login (System Settings > General > Sharing), so North Attleboro can help remotely." +fi +echo +echo "Next, on each other computer here (ProPresenter, Tracks, Lighting...):" +echo " Mac: open Terminal and paste curl -fsSL http://$(ipconfig getifaddr en0 2>/dev/null || hostname):8020/join | sh" +echo " PC: open PowerShell as administrator and paste irm http://$(ipconfig getifaddr en0 2>/dev/null || hostname):8020/join.ps1 | iex" +""" % {"name": name, "base": shq(base), "key": shq(key), "cid": cid, "qname": shq(name)} + + +def join_mac(cfg, base): + key, _ = scripts.public_key(cfg) + if not key: + raise RuntimeError("this Mac has no login key yet (run the install line again)") + ingest = (cfg.get("site") or {}).get("ingest_name") or "the Companion Mac" + return """#!/bin/sh +# Joins this Mac to %(ingest)s's status panel: lets it log in with its key (no password) and says this Mac is here. +KEY=%(key)s +mkdir -p "$HOME/.ssh" && touch "$HOME/.ssh/authorized_keys" && chmod 700 "$HOME/.ssh" && chmod 600 "$HOME/.ssh/authorized_keys" +grep -qF "$KEY" "$HOME/.ssh/authorized_keys" || echo "$KEY" >> "$HOME/.ssh/authorized_keys" +if ! /usr/bin/nc -z -G 2 127.0.0.1 22 >/dev/null 2>&1; then + echo "Remote Login is off. Turning it on (it asks for this Mac's administrator password):" + if ! sudo /usr/sbin/systemsetup -setremotelogin on >/dev/null 2>&1; then + echo "Please turn it on in System Settings > General > Sharing > Remote Login, then paste this line again." + open "x-apple.systempreferences:com.apple.Sharing-Settings.extension" 2>/dev/null + exit 1 + fi +fi +esc() { printf '%%s' "$1" | sed 's/[\\\\"]//g'; } +IPS=$(/sbin/ifconfig | awk '/inet /{print $2}' | grep -v '^127\\.' | sed 's/.*/"&"/' | paste -sd, -) +BODY="{\\"user\\":\\"$(esc "$(id -un)")\\",\\"name\\":\\"$(esc "$(scutil --get ComputerName)")\\",\\"hostname\\":\\"$(esc "$(scutil --get LocalHostName).local")\\",\\"os\\":\\"mac\\",\\"ips\\":[$IPS]}" +if /usr/bin/curl -fsS -X POST -H 'Content-Type: application/json' -d "$BODY" %(hello)s >/dev/null; then + echo "Joined. %(ingest)s sets this Mac up in about a minute: watch its status page." +else + echo "Could not reach %(ingest)s at %(base)s - is this Mac on the wired network?" + exit 1 +fi +""" % {"ingest": ingest, "key": shq(key), "hello": shq(base + "/api/hello"), "base": base} + + +def join_windows(cfg, base): + """The Windows set-up script, with the programs found running on that PC instead of a list.""" + text = scripts.fill(scripts.template("windows-setup.ps1.template"), dict( + scripts.site_values(cfg), PC_NAME="this PC", APPS=" # filled in below", PROGRAMS="the ones running now", + SERVICES="'rtpMIDIService'")) + known = "; ".join("'%s' = '%s'" % (a["exe"], a["task"].replace("Start ", "", 1)) for a in autoconf.WIN_APPS) + detect = ("$known = @{ %s }\n" + "$apps = @(Get-Process -ErrorAction SilentlyContinue | Where-Object { $_.Path -and $known.ContainsKey((Split-Path $_.Path -Leaf)) } |\n" + " Sort-Object Path -Unique | ForEach-Object { @{ Name = $known[(Split-Path $_.Path -Leaf)]; Exe = $_.Path } })" % known) + text = re.sub(r"\$apps = @\(\n # filled in below\n\)", lambda m: detect, text) + if "$known = @{" not in text: + raise RuntimeError("the Windows template changed: cannot put the program finder in") + text = text.replace("# Status panel remote access", "# Run in PowerShell as administrator: irm %s/join.ps1 | iex\n# Status panel remote access" % base, 1) + text += """ +# 5. Tell %(ingest)s this PC is here: it sets it up within a minute. +$hello = @{ user = $env:USERNAME; name = $env:COMPUTERNAME; os = 'windows' + ips = @(Get-NetIPAddress -AddressFamily IPv4 -ErrorAction SilentlyContinue | Where-Object { $_.IPAddress -notlike '127.*' } | ForEach-Object IPAddress) + macs = @($nics | ForEach-Object MacAddress); tasks = @($apps | ForEach-Object { 'Start ' + $_.Name }) } +try { + Invoke-RestMethod -Method Post -Uri '%(hello)s' -ContentType 'application/json' -Body ($hello | ConvertTo-Json) | Out-Null + Write-Host 'Joined: %(ingest)s sets this PC up in about a minute.' +} catch { Write-Host ('Could not reach %(ingest)s: ' + $_.Exception.Message) } +""" % {"ingest": (cfg.get("site") or {}).get("ingest_name") or "the Companion Mac", "hello": base + "/api/hello"} + return text + + +# --- command line ------------------------------------------------------------------------ + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("what", choices=("setup", "look", "add")) + ap.add_argument("--id", default="new") + ap.add_argument("--name") + ap.add_argument("--data-dir", default=PARENT) + a = ap.parse_args(argv) + if a.what == "setup": + setup(a.data_dir, a.id, a.name) + elif a.what == "look": + cfg = sp.load_config(os.path.join(a.data_dir, "config.json")) + inv = discover.look(cfg, a.data_dir) + print(json.dumps(summary(inv), indent=1)) + new, added, _ = autoconf.extend(cfg, inv, load_seen(a.data_dir)) + print("would add:", ", ".join(added) or "nothing") + else: + from settings import server + look_and_add(server.Site(a.data_dir), say=print, force=True) + + +if __name__ == "__main__": + main() diff --git a/settings/discover.py b/settings/discover.py new file mode 100644 index 0000000..d4a3234 --- /dev/null +++ b/settings/discover.py @@ -0,0 +1,312 @@ +"""Auto-setup's eyes: what this campus has, read from Companion, Dante, the network MIDI sessions, +this Mac's own network, and the computers that joined (each one ran the join line once). + + look(cfg, data_dir) the inventory autoconf.py turns into keys + register(data_dir, hello) a computer's join report, kept in computers.json + +Read-only everywhere: nothing here changes Companion, a session or a computer. Every remote +command is a fixed string; names from the network only ever come back as data. +""" +import ipaddress +import json +import os +import re +import subprocess +import time +import urllib.parse +import urllib.request + +from settings import PARENT # noqa: F401 (puts the project folder on sys.path) +import status_panel as sp + +TIMEOUT = 8 +MAC_LOOK = "echo SP_PROCS; ps -axo comm= 2>/dev/null; echo SP_APPS; ls -d /Applications/*.app /Applications/*/*.app 2>/dev/null; echo SP_END" +WIN_LOOK = "echo SP_PROCS & tasklist /FO CSV /NH 2>nul & echo SP_TASKS & schtasks /Query /FO CSV /NH 2>nul & echo SP_END" +HOST_FIELDS = ("host", "bonjourHost", "serverHost", "ip", "address") + + +# --- the computers that joined --------------------------------------------------------- + +def registry_path(data_dir): + return os.path.join(data_dir, "computers.json") + + +def registered(data_dir): + try: + with open(registry_path(data_dir)) as f: + return json.load(f) + except (OSError, ValueError): + return {} + + +def clean_hello(hello, peer_ip): + """A join report, kept to the fields auto-setup uses and checked: it comes from the network.""" + def s(v, n=80): + return re.sub(r"[^\w .@:\-()'+]", "", str(v or ""))[:n] + ips = [str(ipaddress.ip_address(i)) for i in (hello.get("ips") or []) if _is_ip(i)][:12] + if peer_ip and _is_ip(peer_ip) and peer_ip not in ips: + ips.insert(0, peer_ip) + user = str(hello.get("user") or "") + if not re.fullmatch(r"[A-Za-z0-9._\-][A-Za-z0-9._\- ]{0,39}", user): # as given: a name to log in with is never "cleaned up" + raise ValueError("no usable user name") + return {"name": s(hello.get("name")) or user, "hostname": s(hello.get("hostname"), 60), "user": user, + "os": "windows" if str(hello.get("os", "")).lower().startswith("win") else "mac", "ips": ips, + "macs": [m for m in (str(x).lower().replace("-", ":") for x in hello.get("macs") or []) if re.fullmatch(r"([0-9a-f]{2}:){5}[0-9a-f]{2}", m)][:4], + "tasks": [s(t, 60) for t in hello.get("tasks") or []][:20], "at": int(time.time())} + + +def _is_ip(v): + try: + ipaddress.ip_address(str(v)) + return True + except ValueError: + return False + + +def register(data_dir, hello, peer_ip=None): + entry = clean_hello(hello, peer_ip) + reg = registered(data_dir) + key = entry["name"] + reg[key] = entry + sp.write_json(registry_path(data_dir), reg) + return entry + + +# --- this Mac -------------------------------------------------------------------------- + +def run(argv, timeout=TIMEOUT): + try: + return subprocess.run(argv, capture_output=True, text=True, timeout=timeout).stdout + except (OSError, subprocess.TimeoutExpired): + return "" + + +def wifi_ports(): + out = run(["/usr/sbin/networksetup", "-listallhardwareports"]) + ports, last = set(), "" + for line in out.splitlines(): + if line.startswith("Hardware Port:"): + last = line.split(":", 1)[1].strip() + elif line.startswith("Device:") and last in ("Wi-Fi", "AirPort"): + ports.add(line.split(":", 1)[1].strip()) + return ports + + +def interfaces(): + """{interface: (ip, network)} for every IPv4 address on this Mac, from ifconfig.""" + out, cur = {}, None + for line in run(["/sbin/ifconfig"]).splitlines(): + m = re.match(r"^(\S+?):", line) + if m: + cur = m.group(1) + continue + m = re.search(r"inet (\d+\.\d+\.\d+\.\d+) netmask 0x([0-9a-f]{8})", line) + if m and cur: + mask = ipaddress.ip_address(int(m.group(2), 16)) + net = ipaddress.ip_network("%s/%s" % (m.group(1), mask), strict=False) + out.setdefault(cur, (m.group(1), str(net))) + return out + + +def gateway_of(interface): + out = run(["/usr/sbin/netstat", "-rn", "-f", "inet"]) + for line in out.splitlines(): + parts = line.split() + if len(parts) >= 4 and parts[0] == "default" and parts[-1] == interface: + return parts[1] + return "" + + +def this_mac(): + """Its names and addresses, and which network is the wired one: the interface with a gateway that + is neither Wi-Fi nor a link-local (Dante) address, preferring the one the default route uses.""" + ifs = interfaces() + wifi = wifi_ports() + route = run(["/sbin/route", "-n", "get", "default"]) + m = re.search(r"interface: (\S+)", route) + default_if = m.group(1) if m else "" + wired = None + for name in [default_if] + sorted(ifs): + if name in ifs and name not in wifi and not ifs[name][0].startswith(("127.", "169.254.")) and gateway_of(name): + wired = name + break + if wired is None: + wired = next((n for n in sorted(ifs) if n not in wifi and not ifs[n][0].startswith(("127.", "169.254."))), None) + info = {"name": run(["/usr/sbin/scutil", "--get", "ComputerName"]).strip(), + "hostname": (run(["/usr/sbin/scutil", "--get", "LocalHostName"]).strip() + ".local").lstrip("."), + "ips": [ip for n, (ip, _) in sorted(ifs.items()) if not ip.startswith("127.")]} + if wired: + ip, net = ifs[wired] + # A /24 or smaller on a campus usually sits inside a /16 the rest of the gear shares: use what is configured. + info["wired"] = {"interface": wired, "ip": ip, "net": net, "gateway": gateway_of(wired)} + return info + + +# --- Companion ------------------------------------------------------------------------- + +def companion_get(cfg, path, timeout=5): + c = cfg.get("companion") or {} + url = "http://%s:%s%s" % (c.get("host", "127.0.0.1"), c.get("port", 8000), path) + with urllib.request.urlopen(url, timeout=timeout) as r: + return r.read().decode("utf-8", "replace") + + +def companion_var(cfg, label, name): + try: + return companion_get(cfg, "/api/variable/%s/%s/value" % (urllib.parse.quote(label, safe=""), urllib.parse.quote(name, safe="")), 3) + except OSError: + return "" + + +def connection_hosts(cfg): + """{label: address} from Companion's own database, opened read-only (the host field differs by module).""" + import sqlite3 + path = sp.companion_db_path(cfg) + if not path: + return {} + db = sqlite3.connect("file:%s?mode=ro" % urllib.parse.quote(path), uri=True, timeout=2) + try: + rows = db.execute("SELECT value FROM instances").fetchall() + finally: + db.close() + out = {} + for (value,) in rows: + try: + inst = json.loads(value) + except (TypeError, ValueError): + continue + if not isinstance(inst, dict) or not inst.get("label"): + continue + conf = inst.get("config") or {} + host = next((str(conf[k]).strip() for k in HOST_FIELDS if isinstance(conf.get(k), str) and conf[k].strip()), "") + out[inst["label"]] = host.split("|")[-1].rsplit(":", 1)[0] if host.count(":") == 1 else host + return out + + +def connections(cfg): + raw = json.loads(companion_get(cfg, "/api/connections")) + items = raw if isinstance(raw, list) else list(raw.values()) if isinstance(raw, dict) else [] + hosts = connection_hosts(cfg) + out = [] + for c in items: + if not isinstance(c, dict) or not c.get("label"): + continue + entry = {"label": c["label"], "module": c.get("moduleId") or c.get("instance_type") or "", "enabled": c.get("enabled", True) is not False, + "host": hosts.get(c["label"], ""), "status": ((c.get("status") or {}).get("category") if isinstance(c.get("status"), dict) else "")} + if entry["module"] in ("shure-wireless", "shure-psm1000"): + probe = "ch_%d_battery_bars" if entry["module"] == "shure-wireless" else "ch_%d_rf_mute" + n = 0 + for ch in range(1, 9): + if companion_var(cfg, c["label"], probe % ch).strip() in ("", "undefined"): + break + n = ch + entry["channels"] = n or None + out.append(entry) + return out + + +def dante(cfg, conns): + label = next((c["label"] for c in conns if c["module"] == "audinate-dantecontroller" and c["enabled"]), None) + if not label: + return {} + try: + devices = json.loads(companion_var(cfg, label, "devices") or "[]") + except ValueError: + devices = [] + return {"conn": label, "devices": sorted(d for d in devices if isinstance(d, str))} + + +# --- the computers --------------------------------------------------------------------- + +def sections(out): + parts, cur = {}, None + for line in out.splitlines(): + line = line.strip() + if line.startswith("SP_") and line[3:].isupper(): + cur = line[3:] + parts[cur] = [] + elif cur: + parts[cur].append(line) + return parts + + +def look_at(cfg, alias, host): + """What runs on one computer: its programs (and, on a Mac, its network MIDI sessions).""" + os_name = host.get("os", "mac") + try: + r = subprocess.run(sp.ssh_cmd(cfg, alias) + [MAC_LOOK if os_name == "mac" else WIN_LOOK], capture_output=True, text=True, timeout=TIMEOUT) + except subprocess.TimeoutExpired: + return {"reachable": False} + if r.returncode != 0 or "SP_END" not in r.stdout: + return {"reachable": False, "error": (r.stderr or r.stdout).strip()[-160:]} + got = sections(r.stdout) + out = {"reachable": True} + if os_name == "mac": + out["procs"] = sorted({os.path.basename(p.strip()) for p in got.get("PROCS", []) if p.strip()}) + out["apps"] = sorted(a.strip() for a in got.get("APPS", []) if a.strip().endswith(".app")) + try: + m = subprocess.run(sp.ssh_cmd(cfg, alias) + ["/usr/bin/python3", "-"], input=sp.remote_midi_script(), capture_output=True, text=True, timeout=15) + out["midi"] = sp.parse_remote_midi(m.stdout) or {} + except subprocess.TimeoutExpired: + out["midi"] = {} + else: + out["procs"] = sorted({row.split('","')[0].strip('"') for row in got.get("PROCS", []) if row.startswith('"')}) + out["tasks"] = sorted({row.split('","')[0].strip('"').split("\\")[-1] for row in got.get("TASKS", []) + if row.startswith('"\\StatusPanel\\')}) + return out + + +def computers(cfg, data_dir): + """The joined computers and the ones already in the settings, each looked at over SSH.""" + found = {} + hosts = dict(cfg.get("hosts") or {}) + for alias, h in hosts.items(): + if h.get("self") or h.get("disabled") or not h.get("ssh"): + continue + ip = str(h["ssh"]).split("@")[-1] + found[alias] = {"name": h.get("name", alias), "os": h.get("os", "mac"), "user": str(h["ssh"]).split("@")[0], "ip": ip, + "ips": [ip], "macs": [h["mac"]] if h.get("mac") else []} + known_ips = {c["ip"] for c in found.values()} + taken = set(found) | set(hosts) + nets = [(this_mac().get("wired") or {}).get("net")] + for name, j in registered(data_dir).items(): + ip = next((i for i in j["ips"] if any(n and ipaddress.ip_address(i) in ipaddress.ip_network(n) for n in nets)), j["ips"][0] if j["ips"] else None) + if not ip or ip in known_ips: + continue + from settings.autoconf import slug + alias = slug(j["name"].replace(" ", "_"), taken) + taken.add(alias) + found[alias] = {"name": j["name"], "hostname": j.get("hostname", ""), "os": j["os"], "user": j["user"], "ip": ip, "ips": j["ips"], + "macs": j.get("macs") or [], "tasks": j.get("tasks") or []} + probe_cfg = dict(cfg, hosts=dict(hosts, **{a: {"ssh": "%s@%s" % (c["user"], c["ip"]), "os": c["os"]} for a, c in found.items()})) + for alias, c in found.items(): + seen = look_at(probe_cfg, alias, probe_cfg["hosts"][alias]) + if c.get("tasks") and "tasks" in seen: + seen["tasks"] = sorted(set(seen["tasks"]) | set(c["tasks"])) + c.update(seen) + return found + + +# --- everything ------------------------------------------------------------------------ + +def look(cfg, data_dir): + inv = {"ingest": this_mac(), "at": int(time.time())} + # A campus whose MIDI rule names its wired network keeps it (it may be wider than this Mac's subnet). + rule = [n for t in sp.all_tiles(cfg) for c in t.get("checks", []) if c.get("type") == "midi_wired" for n in c.get("nets", [])] + if rule and inv["ingest"].get("wired"): + inv["ingest"]["wired"]["net"] = rule[0] + try: + inv["connections"] = connections(cfg) + except (OSError, ValueError) as e: + inv["connections"], inv["companion_error"] = [], str(e) + inv["dante"] = dante(cfg, inv["connections"]) + try: + import coremidi + inv["midi"] = {"": coremidi.sessions()} + except Exception as e: # noqa: BLE001 - no CoreMIDI (tests, Linux): no sessions here + inv["midi"], inv["midi_error"] = {}, str(e) + inv["computers"] = computers(cfg, data_dir) + for alias, c in inv["computers"].items(): + if c.get("midi"): + inv["midi"][alias] = c.pop("midi") + return inv diff --git a/settings/server.py b/settings/server.py index 6b76f9f..3eb96c5 100644 --- a/settings/server.py +++ b/settings/server.py @@ -40,6 +40,7 @@ from settings import builders from settings import scripts from settings import outlet as outlets from settings import keyguide, probes +from settings import campus, discover log = logging.getLogger("status-settings") UI = os.path.join(HERE, "ui") @@ -115,6 +116,7 @@ class Site: self.page_cache = {} # name -> (time, value) self.draft = applying.Draft(data_dir) self.applier = applying.Applier(self) + self.discover_poke = threading.Event() # a computer joined: look again soon outlets.DATA_DIR, outlets.FILE = self.data_dir, os.path.join(self.data_dir, "outlets.json") def load(self): @@ -684,6 +686,23 @@ class Handler(http.server.BaseHTTPRequestHandler): return self.send(200, keyguide.cheat_sheet(site.load(), self.my_address()), "text/html; charset=utf-8") if path == "/where.html": return self.send(200, keyguide.where_things_are(site.load(), site.data_dir, self.my_address()), "text/html; charset=utf-8") + # ---- auto-setup: the install line for a new campus, the join lines for its computers ---- + if path == "/campus/install": + key, _ = scripts.public_key(site.load()) + return self.send(200, campus.install_script(self.base_url(), q.get("id", ["new"])[0], key), "text/plain; charset=utf-8") + if path == "/campus/program.tgz": + return self.send(200, campus.program_kit(site.data_dir), "application/gzip") + if path == "/join": + return self.send(200, campus.join_mac(site.load(), self.base_url()), "text/plain; charset=utf-8") + if path == "/join.ps1": + return self.send(200, campus.join_windows(site.load(), self.base_url()), "text/plain; charset=utf-8") + if path == "/api/discover": + cfg = site.load() + base = self.base_url() + return self.send(200, {"auto": bool((cfg.get("discover") or {}).get("auto")), "last": campus.load_last(site.data_dir), + "joined": discover.registered(site.data_dir), + "join": {"mac": "curl -fsSL %s/join | sh" % base, "pc": "irm %s/join.ps1 | iex" % base}, + "base": base}) if path == "/api/config": cfg = site.load() return self.send(200, {"config": schema.mask_secrets(cfg), "sha": site.cfg_sha, "mtime": site.cfg_mtime}) @@ -845,6 +864,15 @@ class Handler(http.server.BaseHTTPRequestHandler): return self.send(403, {"error": "wrong origin"}) site = self.site try: + if path == "/api/hello": + # A computer ran the join line: keep what it says (checked), and look again soon. + try: + entry = discover.register(site.data_dir, body, self.client_address[0]) + except (ValueError, TypeError) as e: + return self.send(400, {"error": str(e)}) + log.info("joined: %s (%s, %s) from %s", entry["name"], entry["os"], entry["user"], self.client_address[0]) + site.discover_poke.set() + return self.send(200, {"ok": True}) if path == "/api/editing/check": return self.send(200, site.check_code(self.client_address[0], str(body.get("code", "")))) if path == "/api/machines/probe": @@ -861,6 +889,14 @@ class Handler(http.server.BaseHTTPRequestHandler): ok, who = self.gate() if not ok: return + if path == "/api/discover/now": + # Look now and add what is new (also when adding by itself is off); never during a service. + if getattr(site, "looking", None) and site.looking.is_alive(): + return self.send(200, {"started": False, "already": True}) + site.looking = threading.Thread(target=campus.look_and_add, args=(site,), kwargs={"force": True}, daemon=True) + site.looking.start() + log.info("auto-setup: look now, asked by %s", who) + return self.send(200, {"started": True}) if path == "/api/draft" and self.command == "PUT": cfg = schema.unmask(body.get("config") or {}, site.load()) # A page that loaded the settings before they changed here (an Apply from another tab, a @@ -1289,6 +1325,13 @@ class Handler(http.server.BaseHTTPRequestHandler): except OSError: return self.send(404, {"error": "missing %s" % name}) + def base_url(self): + """This page as the caller reached it (an address, across campuses: names do not cross subnets).""" + host = (self.headers.get("Host") or "").strip() or "%s:%d" % (self.server.server_address[0], self.server.server_address[1]) + if not re.fullmatch(r"[A-Za-z0-9.\-]+(:\d+)?", host): + host = "%s:%d" % (self.server.server_address[0], self.server.server_address[1]) + return "http://" + host + def my_address(self): cfg = self.site.cfg host = cfg.get("site", {}).get("ingest_hostname") or (self.headers.get("Host") or "").split(":")[0] or "127.0.0.1" @@ -1400,6 +1443,7 @@ def main(argv=None): site, servers = serve(args.data_dir, port, args.bind, args.config) stop = threading.Event() threading.Thread(target=rebind_loop, args=(site, servers, port, stop), daemon=True).start() + threading.Thread(target=campus.loop, args=(site, stop, site.discover_poke), name="auto-setup", daemon=True).start() try: while True: time.sleep(3600) diff --git a/settings/ui/app.js b/settings/ui/app.js index 762542b..1f650c3 100644 --- a/settings/ui/app.js +++ b/settings/ui/app.js @@ -151,7 +151,7 @@ if (f.type === "dict") { try { return raw.trim() ? JSON.parse(raw) : undefined; } catch (e) { return undefined; } } return raw === "" ? undefined : raw; } - const LABELS = { after_seconds: "wait before acting (seconds)", note_seconds: "keep showing ON … after (seconds)", interval_seconds: "look every (seconds)", failures: "misses before it counts", debounce_seconds: "a fault must last (seconds)", editing_code: "PIN", ssh: "log-in address", mac: "hardware address", ingest_ips: "this Mac's addresses", ingest_name: "this Mac's name", ingest_hostname: "network name", hold_always: "hold also when green", page_hold_ms: "hold time (ms)", optional: "may have no disk" }; + const LABELS = { auto: "add new gear by itself", after_seconds: "wait before acting (seconds)", note_seconds: "keep showing ON … after (seconds)", interval_seconds: "look every (seconds)", failures: "misses before it counts", debounce_seconds: "a fault must last (seconds)", editing_code: "PIN", ssh: "log-in address", mac: "hardware address", ingest_ips: "this Mac's addresses", ingest_name: "this Mac's name", ingest_hostname: "network name", hold_always: "hold also when green", page_hold_ms: "hold time (ms)", optional: "may have no disk" }; function inputHtml(path, f, value, extra) { if (!f) return ""; const id = "f-" + path.replace(/[^A-Za-z0-9]/g, "-"); @@ -877,6 +877,7 @@ ["outlets", "Outlets", "Smart plugs the panel may switch, for a device that only a power cycle brings back"], ["companion", "Companion", "Connections, and names the keys use that are not here"], ["network", "Network", "This Mac's network port and the Dante switches"], + ["setup", "Auto-setup", "What this Mac found, the line each computer runs to join, and the line that sets up another campus"], ["site", "Site & PIN", "The campus, contacts, this Mac, the PIN"], ["program", "Program & campus", "Update the code, export or start a campus"], ["advanced", "Advanced", "Every number and name, by screen"], @@ -902,6 +903,7 @@ else if (which === "outlets") body = await outletsHtml(); else if (which === "companion") body = (await namesHtml()) + (await companionHtml()); else if (which === "network") body = fieldsHtml("Network & Dante"); + else if (which === "setup") body = await setupHtml(c); else if (which === "site") body = siteHtml(c); else if (which === "program") body = await programHtml(); else if (which === "advanced") body = advancedHtml(); @@ -928,6 +930,28 @@ const screens = [...new Set(Object.values(state.schema.fields).map((f) => f.screen))]; return screens.map((s) => { const h = fieldsHtml(s, true); return h ? `

${esc(s)}

${h}` : ""; }).join("") || "

Nothing.

"; } + // Auto-setup: what was found and added, the join lines, and the install line for another campus. + async function setupHtml(c) { + const d = await api("/api/discover").catch(() => null); + if (!d) return "

Auto-setup is not answering.

"; + const last = d.last || {}, inv = last.inventory || {}; + const line = (text, h) => `
${h}
`; + const comps = Object.entries(inv.computers || {}).map(([a, x]) => `${lightDot(x.reachable ? "ok" : "problem", x.reachable ? "logs in" : x.error || "does not let this Mac in yet")}${esc(x.name || a)}${esc(x.os || "")}${esc(x.ip || "")}${x.reachable ? "" : esc(x.error || "run the join line on it")}`).join(""); + const joined = Object.entries(d.joined || {}).filter(([n]) => !Object.values(inv.computers || {}).some((x) => x.name === n)).map(([n, j]) => `${lightDot("warn", "joined; the next look sets it up")}${esc(n)}${esc(j.os)}${esc((j.ips || [])[0] || "")}joined ${esc(when(j.at))}`).join(""); + const list = (xs) => xs && xs.length ? `` : "

nothing

"; + const campuses = [["ab", "Apollo Beach"], ["sc", "Southcoast"], ["ws", "Woonsocket"], ["na", "North Attleboro"]].filter(([k]) => k !== (c.site || {}).id); + return `

${inputHtml("discover.auto", F("discover.auto"), (c.discover || {}).auto)}

+
last look
${last.at ? esc(when(last.at)) : "not yet"}${last.error ? ` · ${esc(last.error)}` : ""}
+
found
${inv.connections != null ? `${inv.connections} Companion connections · ${inv.dante || 0} Dante devices · ${inv.midi_sessions || 0} MIDI sessions · wired network ${esc(inv.wired || "?")}` : "–"}
+
+

Added last time

${list(last.added)}${last.waiting && last.waiting.length ? `

Waiting${hint(last.why || "")}

${list(last.waiting)}

${esc(last.why || "")}

` : ""} + ${inv.not_known && inv.not_known.length ? `
Companion connections auto-setup has no recipe for (${inv.not_known.length})${list(inv.not_known)}

Add a key for one by hand (Keys › + Key) if it matters.

` : ""}
+

Computers${hint("Each computer runs the join line once: it lets this Mac log in with its key and says it is here. The next look gives its programs keys.")}

+
${comps}${joined}${comps || joined ? "" : ""}
none yet
+ ${line(d.join.mac, "Mac: Terminal, paste")}${line(d.join.pc, "PC: PowerShell as administrator, paste")}
+

Set up another campus${hint("On that campus's Mac that runs Companion: Terminal, paste. It installs the panel from here, finds the gear and makes the pages. Then each computer there runs that Mac's join line.")}

+ ${campuses.map(([k, n]) => line(`curl -fsSL "${d.base}/campus/install?id=${k}" | zsh`, n)).join("")}
`; + } function siteHtml(c) { const f = F("site.editing_code"); return `
${inputHtml("site.editing_code", Object.assign({}, f, { help: "The PIN that switches Editing on. Digits are easiest. It lives in the settings file, so everyone with the repo has it." }), (c.site || {}).editing_code)}
` + fieldsHtml("Site"); @@ -1347,6 +1371,8 @@ function wireSettings(which) { const q = (s) => main.querySelector(s); wireCalendars(); + for (const el of main.querySelectorAll(".copyline")) el.onclick = () => el.select(); + if (q("#looknow")) q("#looknow").onclick = async () => { try { await post("/api/discover/now", {}); alertBox("Looking now. This page fills in within a minute."); setTimeout(() => route(), 30000); } catch (e) { alertBox(e.message); } }; for (const b of main.querySelectorAll("[data-changepress]")) b.onclick = () => { const id = b.dataset.changepress, f0 = cfg().fixes[id]; pickButtonDialog(`Which button is ${(f0.params || {}).button || id}?`, { page: f0.page, row: f0.row, col: f0.col }, async (p) => { diff --git a/tests/test_campus.py b/tests/test_campus.py new file mode 100644 index 0000000..82b10ba --- /dev/null +++ b/tests/test_campus.py @@ -0,0 +1,247 @@ +"""Auto-setup for the other campuses: an inventory becomes keys that pass every rule, laid out like +North Attleboro's; a running campus only gains what is new (never loses anything, never gets back +what someone took off); a computer's join report is checked; the lines people paste are right.""" +import io +import json +import os +import socket +import sys +import tarfile +import tempfile +import threading +import time +import unittest +from http.server import ThreadingHTTPServer + +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, ROOT) +sys.path.insert(0, os.path.join(ROOT, "tests")) +import schema # noqa: E402 +import make_page # noqa: E402 +from settings import autoconf as ac, campus, discover, server # noqa: E402 +from test_settings import get # noqa: E402 +from test_status_panel import FakeCompanion, healthy_snapshot # noqa: E402 + +BASE = json.load(open(os.path.join(ROOT, "sites", "_template", "config.json"))) + + +def small(): + return { + "ingest": {"name": "SC Ingest", "hostname": "SC-Ingest.local", "ips": ["10.4.1.1", "10.6.20.5"], + "wired": {"interface": "en0", "ip": "10.4.1.1", "net": "10.4.0.0/16", "gateway": "10.4.0.1"}}, + "connections": [ + {"label": "HyperDeck_1", "module": "bmd-hyperdeck", "host": "10.4.10.11", "enabled": True}, + {"label": "HyperDeck_2", "module": "bmd-hyperdeck", "host": "10.4.10.12", "enabled": True}, + {"label": "x32", "module": "behringer-x32", "host": "10.4.1.112", "enabled": True}, + {"label": "dante", "module": "audinate-dantecontroller", "host": "", "enabled": True}, + {"label": "ProP", "module": "renewedvision-propresenter-api", "host": "10.4.14.20", "enabled": True}, + {"label": "Govee", "module": "govee-lights", "host": "", "enabled": True}, + {"label": "old_obs", "module": "obs-studio", "host": "10.4.9.9", "enabled": False}], + "dante": {"conn": "dante", "devices": ["SC-X32", "SC-ProP-Mac"]}, + "midi": {"": {"SC Ingest": {"port": 5004, "peers": [ + {"name": "Lighting PC", "address": "10.4.10.206:5004", "state": "running"}, + {"name": "Phone", "address": "10.6.20.77:5004", "state": "running"}]}}}, # Wi-Fi: not made a key + "computers": { + "prop": {"name": "ProPresenter Mac", "os": "mac", "user": "media", "ip": "10.4.14.20", "ips": ["10.4.14.20"], "reachable": True, + "procs": ["ProPresenter", "Finder"], "apps": ["/Applications/ProPresenter.app", "/Applications/Safari.app"]}, + "lighting_pc": {"name": "Lighting PC", "os": "windows", "user": "media", "ip": "10.4.10.206", "ips": ["10.4.10.206"], + "reachable": True, "procs": ["Vista.exe", "rtpMIDI.exe", "Arena.exe"], "tasks": ["Start Vista", "Start rtpMIDI"], + "macs": ["aa:bb:cc:dd:ee:ff"]}, + "off_pc": {"name": "Office", "os": "windows", "user": "x", "ip": "10.4.3.3", "ips": ["10.4.3.3"], "reachable": False}}} + + +def big(): + inv = small() + inv["connections"] += [{"label": "HyperDeck_%d" % n, "module": "bmd-hyperdeck", "host": "10.4.10.%d" % (10 + n), "enabled": True} for n in range(3, 7)] + inv["connections"] += [{"label": "Axient_%d" % n, "module": "shure-wireless", "host": "10.4.1.%d" % (100 + n), "enabled": True, "channels": 4} for n in range(1, 4)] + inv["connections"] += [{"label": "PSM_%d" % n, "module": "shure-psm1000", "host": "10.4.1.%d" % (110 + n), "enabled": True, "channels": 2} for n in range(1, 9)] + inv["connections"] += [{"label": m, "module": mod, "host": "10.4.5.%d" % i, "enabled": True} + for i, (m, mod) in enumerate([("ATEM", "bmd-atem"), ("VX1000", "novastar-controller"), ("OBS", "obs-studio"), + ("RESI", "resi-decoders"), ("tascam", "tascam-da-6400"), ("Arena", "resolume-arena")])] + return inv + + +def layout_of(cfg): + return {tuple(t["pos"]): t["label"] for t in cfg["tiles"] if t.get("pos")} + + +class TestBuild(unittest.TestCase): + def test_a_small_campus_gets_one_key_per_thing_in_order_and_passes_the_rules(self): + cfg, seen = ac.build(BASE, small(), {"id": "sc", "name": "Southcoast"}, "4821", (21, 22)) + self.assertEqual(schema.validate(cfg)[0], []) + labels = [t["label"] for t in cfg["tiles"] if not t.get("kind")] + self.assertEqual(labels, ["HD1", "HD2", "DANTE", "X32", "PROP", "VISTA", "MIDI"]) # REC, AUDIO, SHOW; no group keys + self.assertNotIn("RTPMIDI", labels) # rides in the MIDI key + midi = next(t for t in cfg["tiles"] if t["label"] == "MIDI") + self.assertEqual([c["peer"] for c in midi["checks"] if c["type"] == "midi_peer"], ["Lighting PC"]) # the Wi-Fi phone is not + self.assertEqual(midi["checks"][-2]["type"], "midi_wired") + self.assertEqual(midi["checks"][-1]["params"]["exe"], "rtpMIDI.exe") + prop = next(t for t in cfg["tiles"] if t["label"] == "PROP") + self.assertEqual([c["type"] for c in prop["checks"]], ["conn", "remote"]) # Companion connection + the program + self.assertEqual(prop["checks"][1]["cmd"], "pgrep -xq ProPresenter") + self.assertNotIn("ARENA", labels) # running, but no start task on that PC + self.assertNotIn("off_pc", cfg["hosts"]) # not reachable: not added + self.assertEqual(cfg["hosts"]["lighting_pc"]["ssh"], "media@10.4.10.206") + self.assertEqual((cfg["page"], cfg["restart_page"]["page"]), (21, 22)) + self.assertEqual([k["label"] for k in cfg["restart_page"]["keys"]], ["PROP", "VISTA"]) + self.assertEqual(cfg["site"]["editing_code"], "4821") + self.assertEqual(cfg["site"]["ingest_ips"][0], "10.4.1.1") + self.assertTrue(cfg["discover"]["auto"]) + self.assertIn("hyperdeck", cfg["auto_fix"]["fixes"]) + self.assertFalse(any(f.startswith("restart_") for f in cfg["auto_fix"]["fixes"])) # restarts stay a hold + for t in cfg["tiles"]: # every key's hold does something + if not t.get("kind") and not t.get("members"): + fixable = any(c.get("fix") or c["type"] in ("conn", "hyperdeck", "tascam", "shure", "psm") for c in t["checks"]) + self.assertTrue(fixable, t["label"]) + make_page.build(cfg) + make_page.build_restart(cfg) + self.assertIn("conn:x32", seen) + self.assertIn("dante:SC-X32", seen) + + def test_a_big_campus_gets_group_keys_and_the_receivers_fold_first(self): + cfg, _ = ac.build(BASE, big(), {"id": "ab", "name": "Apollo Beach"}, "1111") + self.assertEqual(schema.validate(cfg)[0], []) + top = [layout_of(cfg).get((0, c)) for c in range(2, 8)] + self.assertEqual(top, ["REC", "AUDIO", "RF", "SHOW", "VIDEO", "STREAM"]) + unplaced = [t["label"] for t in cfg["tiles"] if not t.get("pos")] + self.assertTrue(unplaced and all(u.startswith(("AXIENT", "PSM")) for u in unplaced), unplaced) + rf = next(t for t in cfg["tiles"] if t["label"] == "RF") + self.assertEqual(len(rf["members"]), 11) + self.assertEqual(layout_of(cfg)[(3, 7)], "NET") + self.assertEqual(layout_of(cfg)[(3, 0)], "ALARMS") + make_page.build(cfg) + + +class TestExtend(unittest.TestCase): + def setUp(self): + self.cfg, self.seen = ac.build(BASE, small(), {"id": "sc", "name": "Southcoast"}, "4821") + + def test_nothing_new_adds_nothing(self): + new, added, _ = ac.extend(self.cfg, small(), self.seen) + self.assertEqual(added, []) + self.assertEqual(new, self.cfg) + + def test_new_gear_is_added_into_free_cells_and_merged_into_its_key(self): + inv = small() + inv["connections"].append({"label": "HyperDeck_3", "module": "bmd-hyperdeck", "host": "10.4.10.13", "enabled": True}) + inv["dante"]["devices"].append("SC-Tracks") + inv["midi"][""]["SC Ingest"]["peers"].append({"name": "SuperRack", "address": "10.4.13.140:5004", "state": "running"}) + new, added, now_seen = ac.extend(self.cfg, inv, self.seen) + self.assertEqual(sorted(added), ["Dante device SC-Tracks", "HD3 key", "MIDI link SuperRack"]) + self.assertEqual(schema.validate(new)[0], []) + hd = [t for t in new["tiles"] if t["label"] == "HD3"][0] + self.assertTrue(hd.get("pos")) + self.assertIn("SC-Tracks", next(c for t in new["tiles"] for c in t.get("checks", []) if c["type"] == "dante")["devices"]) + # Nothing that was there changed place. + self.assertTrue(set(layout_of(self.cfg).items()) <= set(layout_of(new).items())) + again, added2, _ = ac.extend(new, inv, set(self.seen) | set(now_seen)) + self.assertEqual(added2, []) + + def test_a_key_someone_took_off_is_not_added_back(self): + cfg = json.loads(json.dumps(self.cfg)) + cfg["tiles"] = [t for t in cfg["tiles"] if t["label"] != "X32"] + _, added, _ = ac.extend(cfg, small(), self.seen) + self.assertNotIn("X32 key", added) + _, added, _ = ac.extend(cfg, small(), []) # never seen: it would be + self.assertIn("X32 key", added) + + def test_north_attleboro_like_config_only_gets_what_it_does_not_watch(self): + cfg = json.load(open(os.path.join(ROOT, "config.json"))) + inv = {"ingest": {"wired": {"net": "10.1.0.0/16"}}, + "connections": [{"label": "HyperDeck_1", "module": "bmd-hyperdeck", "host": "10.1.10.11", "enabled": True}, + {"label": "x32", "module": "behringer-x32", "host": "10.1.1.112", "enabled": True}], + "dante": {}, "midi": {}, "computers": {}} + _, added, _ = ac.extend(cfg, inv, []) + self.assertEqual(added, []) + + +class TestJoin(unittest.TestCase): + def test_a_join_report_is_checked(self): + e = discover.clean_hello({"user": "media", "name": "Lighting PC", "os": "Windows", "ips": ["10.4.10.206", "bad"], + "macs": ["AA-BB-CC-DD-EE-FF", "nope"], "tasks": ["Start Vista"]}, "10.4.10.206") + self.assertEqual((e["os"], e["ips"], e["macs"], e["tasks"]), ("windows", ["10.4.10.206"], ["aa:bb:cc:dd:ee:ff"], ["Start Vista"])) + for bad in ({"name": "x"}, {"user": "a;rm -rf /"}, {"user": ""}): + with self.assertRaises(ValueError): + discover.clean_hello(bad, "10.4.1.2") + self.assertEqual(discover.clean_hello({"user": "u", "name": "Mac"}, "10.4.1.2")["name"], "bMacb") + + def test_the_lines_people_paste(self): + line = campus.install_script("http://10.1.1.1:8020", "ab; rm", "ssh-ed25519 AAAA na") + self.assertIn("zsh install.sh --site ab", line) # the id is cleaned + self.assertIn("--name 'Apollo Beach'", line) + self.assertIn("SUPPORT='ssh-ed25519 AAAA na'", line) + cfg = {"site": {"ingest_name": "SC Ingest", "ingest_ips": ["10.4.1.1"]}} + ps = campus.join_windows(cfg, "http://10.4.1.1:8020") + self.assertIn("'Vista.exe' = 'Vista'", ps) + self.assertIn("Invoke-RestMethod -Method Post -Uri 'http://10.4.1.1:8020/api/hello'", ps) + self.assertNotIn("{{", ps) + + +class TestRoutes(unittest.TestCase): + @classmethod + def setUpClass(cls): + snap = healthy_snapshot(time.time()) + FakeCompanion.store = {"conns": snap.conns, "vars": dict(snap.vars), "custom": {}, "restarts": []} + cls.fake = ThreadingHTTPServer(("127.0.0.1", 0), FakeCompanion) + threading.Thread(target=cls.fake.serve_forever, daemon=True).start() + cls.data = tempfile.mkdtemp(prefix="sp-campus-") + key = os.path.join(cls.data, "id_test") + open(key + ".pub", "w").write("ssh-ed25519 AAAAC3NzaTESTKEY status-panel@sc\n") + cfg, _ = ac.build(BASE, small(), {"id": "sc", "name": "Southcoast"}, "4821") + cfg["companion"] = {"host": "127.0.0.1", "port": cls.fake.server_address[1]} + cfg["ssh_key"] = key + json.dump(cfg, open(os.path.join(cls.data, "config.json"), "w")) + json.dump({"pid": 1, "started_at": time.time(), "config_sha": "x", "version": "test"}, open(os.path.join(cls.data, "started.json"), "w")) + open(os.path.join(cls.data, "status_panel.py"), "w").write("# code\n") + os.makedirs(os.path.join(cls.data, "kit", "sites", "_template")) + open(os.path.join(cls.data, "kit", "install.sh"), "w").write("# installer\n") + open(os.path.join(cls.data, "kit", "sites", "_template", "config.json"), "w").write("{}") + open(os.path.join(cls.data, "outlets.json"), "w").write('{"tuya": {"access_secret": "never-shipped"}}') + sock = socket.socket(); sock.bind(("127.0.0.1", 0)); cls.port = sock.getsockname()[1]; sock.close() + cls.site, cls.servers = server.serve(cls.data, cls.port, ["127.0.0.1"]) + + @classmethod + def tearDownClass(cls): + for s in cls.servers: + s.shutdown() + cls.fake.shutdown() + + def test_hello_registers_a_computer_without_the_code(self): + status, body = get(self.port, "/api/hello", {"user": "media", "name": "Tracks Mac", "os": "mac", "ips": ["10.4.3.227"]}) + self.assertEqual(status, 200) + self.assertIn("Tracks Mac", discover.registered(self.data)) + self.assertTrue(self.site.discover_poke.is_set()) + self.assertEqual(get(self.port, "/api/hello", {"name": "no user"})[0], 400) + status, body = get(self.port, "/api/discover") + d = json.loads(body) + self.assertEqual(d["join"]["mac"], "curl -fsSL http://127.0.0.1:%d/join | sh" % self.port) + self.assertIn("Tracks Mac", d["joined"]) + + def test_join_lines_carry_this_macs_key_and_address(self): + status, body = get(self.port, "/join") + self.assertEqual(status, 200) + self.assertIn("KEY='ssh-ed25519 AAAAC3NzaTESTKEY status-panel@sc'", body.decode()) + self.assertIn("http://127.0.0.1:%d/api/hello" % self.port, body.decode()) + status, body = get(self.port, "/join.ps1") + self.assertEqual(status, 200) + self.assertIn("$IngestIPs = @('10.4.1.1', '10.6.20.5')", body.decode()) + + def test_the_program_kit_has_the_code_and_never_the_settings(self): + status, body = get(self.port, "/campus/program.tgz") + self.assertEqual(status, 200) + names = tarfile.open(fileobj=io.BytesIO(body), mode="r:gz").getnames() + self.assertIn("status_panel.py", names) + self.assertIn("install.sh", names) + self.assertIn("sites/_template/config.json", names) + self.assertFalse([n for n in names if n.endswith(("config.json",)) and not n.startswith("sites/_template")]) + self.assertNotIn("outlets.json", names) + status, body = get(self.port, "/campus/install?id=ws") + self.assertIn("--site ws", body.decode()) + self.assertIn("SUPPORT='ssh-ed25519 AAAAC3NzaTESTKEY", body.decode()) # this campus can help the new one + + def test_look_now_needs_the_code(self): + self.assertEqual(get(self.port, "/api/discover/now", {})[0], 401) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_schema.py b/tests/test_schema.py index c98a689..0bff32c 100644 --- a/tests/test_schema.py +++ b/tests/test_schema.py @@ -185,7 +185,7 @@ class TestSchemaMatchesCode(unittest.TestCase): "fixes", "hosts", "tiles", "flap", "auto_fix", "network", "variables", "companion", "atem", "spotify", "restart_page", "service_windows", "restart_page.keys", "tiles[].checks", "auto_fix.windows", "site.live_services", "tiles[].checks[].type", "fixes.*.playlist", "tiles[].checks[].playlist", - "ready_page", "ready_page.keys", "ready"} + "ready_page", "ready_page.keys", "ready", "discover", "discover.auto"} # discover: settings/campus.py read = {path_of(kind, key) for _, kind, key, _, _ in all_reads()} read |= {"service_windows[].days", "service_windows[].start", "service_windows[].end", "site.live_services[].days", "site.live_services[].start", "site.live_services[].end"}