diff --git a/README.md b/README.md index 6d9f937..fddc7c9 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ There are two parts: |--------|-------|-------|-------|-------|-------|-------|-------| | row 0 | **ALL** | REC | AUDIO | RF | SHOW | VIDEO | STREAM | | row 1 | HD1 | HD2 | HD3 | HD4 | HD5 | HD6 | TASCAM | -| row 2 | X32 | DANTE | TRACKS | LIGHT PC | RACK (SuperRack) | BIBLE TV | PROP (ProPresenter) | +| row 2 | X32 | DANTE | MIDI | APPS | | | | | row 3 | ATEM | OBS | RESI | NET | INGEST | LED | ARENA | Column 0 holds page up / page number / page down, plus **SERVICE** at row 3. @@ -46,7 +46,42 @@ Row 0 holds the group tiles. Each one shows the first problem in its group, plus | | HD2 and HD4 are `"optional": true` in `config.json`: with no disk they sit grey with no alarm, and once a disk is in they are watched like the others. Add or remove `"optional": true` on any deck to change that. | | TASCAM | connection; the recorder's own Error / Caution | | X32, ATEM, ARENA, LED, OBS | Companion connection | -| DANTE | Shows all 19 watched Dante devices as "19/19" (matches Dante Controller). Alarms for 16 of them: DiGiCo DMIs, WING, DQ rack, AD4Qs, Studio A X32 / Resi AJA / Ingest, Translation, and Studio B (RedNet TNX, TB, TB MON, MON, SOLO MON, Tascam). NA-Tracks, SuperRack and the ProPresenter Mac are on their own buttons. | +| DANTE | All 19 Dante devices ("19/19"), named when one drops: "WING GONE", "TRACKS GONE", "PROP MAC GONE". | +| MIDI | Lighting PC and SuperRack still joined to NA Ingest's network MIDI session; the Lighting PC and NA Tracks MIDI sessions still on the network. A clean disconnect shows within seconds; a crash or pulled cable takes about a minute (macOS's own timeout, measured). | +| APPS | ProPresenter API; Ableton OSC; tracks Mac pings; Bible TV app answering on :8080 (every 5 s); Bible TV OSC. Messages name the app ("PROPRES OFFLINE", "BIBLE APP DOWN"). Hold 2 s restarts the ProPresenter, Ableton and Bible TV Companion connections. | +| RESI | NET | INGEST | LED | ARENA | + +Column 0 holds page up / page number / page down, plus **SERVICE** at row 3. + +Row 0 holds the group tiles. Each one shows the first problem in its group, plus "+N" if there are more. RF covers all six Shure units: AD4Q ×2, ULXD4, and PSM1000 ×3. + +## Colours + +| Tile | Meaning | +|---|---| +| Green | OK. The second line shows something useful, like `28h` of record time or `10/10` Dante devices. | +| **Blinking red** | Show-stopping, during service mode. | +| Solid dark red | Acknowledged. It stays red until the problem clears, then goes green. A new failure blinks again. | +| Amber | Warning: low disk, battery at 2 bars, RF interference, a device that keeps dropping out ("WING GONE x3"), a network blip in the last 3 minutes ("BLIP 10:21"). | +| Dim red, not blinking | A fault outside service mode. | +| Grey | Expected off outside service: OBS, a HyperDeck with no disk, Bible TV. | +| **Purple** | The watchdog has stopped. The master shows `WATCHDOG DOWN` and Companion's own error count. | + +## Buttons + +- **Press any tile:** acknowledge its alarms (stops the blinking). Press **ALL** to acknowledge everything. +- **Hold a device tile for 2 seconds:** restart that tile's *Companion connection*. This is the fix for a connection stuck after a network blip. It only restarts Companion's side and never reboots the device. +- **SERVICE:** service mode turns on automatically **Sunday 07:00–12:30** (`service_windows` in `config.json`). Press SERVICE to switch it on or off by hand, for example for a Wednesday event. The manual setting lasts until the next scheduled start or end. + +## What each tile checks + +| Tile | Checks | +|---|---| +| HD1–HD6 | Companion connection; **NO DISK**; record time **LOW** (<30 min, amber) / **FULL** (<5 min); **NOT REC** when another deck is recording and this one isn't | +| | HD2 and HD4 are `"optional": true` in `config.json`: with no disk they sit grey with no alarm, and once a disk is in they are watched like the others. Add or remove `"optional": true` on any deck to change that. | +| TASCAM | connection; the recorder's own Error / Caution | +| X32, ATEM, ARENA, LED, OBS | Companion connection | +| DANTE | All 19 Dante devices ("19/19"), named when one drops: "WING GONE", "TRACKS GONE", "PROP MAC GONE". | | TRACKS | NA-Tracks Dante device; tracks Mac pings; "NA Tracks" MIDI session still on the network; Ableton OSC connection | | LIGHT PC | Lighting PC still joined to NA Ingest's network MIDI session; its MIDI session still on the network | | RACK | SuperRack-LiveBox still joined to the MIDI session; its Dante device | diff --git a/config.json b/config.json index 8017403..e1fcd3b 100644 --- a/config.json +++ b/config.json @@ -25,9 +25,9 @@ "tiles": [ { "id": "all", "label": "ALL", "pos": [0, 1], "kind": "master" }, { "id": "rec", "label": "REC", "pos": [0, 2], "members": ["hd1", "hd2", "hd3", "hd4", "hd5", "hd6", "tascam"] }, - { "id": "audio", "label": "AUDIO", "pos": [0, 3], "members": ["x32", "dante", "superrack"] }, + { "id": "audio", "label": "AUDIO", "pos": [0, 3], "members": ["x32", "dante"] }, { "id": "rf", "label": "RF", "pos": [0, 4], "members": ["axient1", "axient2", "ulxd", "psm1", "psm2", "psm3"] }, - { "id": "show", "label": "SHOW", "pos": [0, 5], "members": ["propres", "lightpc", "tracks", "bibletv"] }, + { "id": "show", "label": "SHOW", "pos": [0, 5], "members": ["midi", "apps"] }, { "id": "video", "label": "VIDEO", "pos": [0, 6], "members": ["atem", "led", "arena"] }, { "id": "stream", "label": "STREAM", "pos": [0, 7], "members": ["obs", "resi"] }, @@ -41,36 +41,27 @@ { "id": "x32", "label": "X32", "pos": [2, 1], "checks": [{ "type": "conn", "conn": "x32" }] }, { "id": "dante", "label": "DANTE", "pos": [2, 2], "checks": [ - { "type": "dante", "conn": "audinate-dantecontroller", "count_all": true, "devices": { + { "type": "dante", "conn": "audinate-dantecontroller", "devices": { "FOH-DiGiCo-DMI-1": "DMI-1", "FOH-DiGiCo-DMI-2": "DMI-2", "BOH-WING": "WING", "BOH-DQ-Rack": "DQ RACK", "BOH-AD4Q-A": "AD4Q-A", "BOH-AD4Q-B": "AD4Q-B", "STUDIO-A-X32": "STA X32", "STUDIO-A-Resi-AJA-SDI": "RESI AJA", "STUDIO-A-Ingest": "INGEST", "NA-Translation": "TRANSL", "STUDIO-B-RedNet-TNX": "STB TNX", "STUDIO-B-TB": "STB TB", "STUDIO-B-TB-MON": "STB TB MON", - "STUDIO-B-MON": "STB MON", "STUDIO-B-SOLO-MON": "STB SOLO", "STUDIO-B-Tascam": "STB TASCAM" } } + "STUDIO-B-MON": "STB MON", "STUDIO-B-SOLO-MON": "STB SOLO", "STUDIO-B-Tascam": "STB TASCAM", + "NA-Tracks": "TRACKS", "FOH-SuperRack-LiveBox": "SUPERRACK", "NA-ProPresenter-Mac": "PROP MAC" } } ] }, - { "id": "tracks", "label": "TRACKS", "pos": [2, 3], "checks": [ - { "type": "dante", "conn": "audinate-dantecontroller", "devices": { "NA-Tracks": "DANTE" } }, - { "type": "ping", "host": "10.1.3.227", "name": "MAC" }, - { "type": "mdns", "instance": "NA Tracks", "name": "MIDI SESS" }, - { "type": "conn", "conn": "ableton" }, - { "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRK MIDI" } + { "id": "midi", "label": "MIDI", "pos": [2, 3], "checks": [ + { "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC", "tag": "LIGHT PC" }, + { "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox", "tag": "SUPERRACK" }, + { "type": "mdns", "instance": "Lighting PC", "name": "LIGHT PC MIDI" }, + { "type": "mdns", "instance": "NA Tracks", "name": "TRACKS MIDI" }, + { "type": "midi_rx", "source": "Network NA Ingest Listen", "name": "TRACKS IN" } ] }, - { "id": "lightpc", "label": "LIGHT PC", "pos": [2, 4], "checks": [ - { "type": "midi_peer", "session": "NA Ingest", "peer": "Lighting PC" }, - { "type": "mdns", "instance": "Lighting PC", "name": "MIDI SESS" }, - { "type": "midi_rx", "source": "Network NA Ingest", "name": "MIDI IN" } - ] }, - { "id": "superrack", "label": "RACK", "pos": [2, 5], "checks": [ - { "type": "midi_peer", "session": "NA Ingest", "peer": "SuperRack-LiveBox" }, - { "type": "dante", "conn": "audinate-dantecontroller", "devices": { "FOH-SuperRack-LiveBox": "DANTE" } } - ] }, - { "id": "bibletv", "label": "BIBLE TV", "pos": [2, 6], "checks": [ - { "type": "http", "url": "http://10.6.7.13:8080/", "msg": "APP DOWN", "interval_seconds": 5 }, - { "type": "conn", "conn": "BibleTV" } - ] }, - { "id": "propres", "label": "PROP", "pos": [2, 7], "checks": [ - { "type": "conn", "conn": "Propresenter-API" }, - { "type": "dante", "conn": "audinate-dantecontroller", "devices": { "NA-ProPresenter-Mac": "DANTE" } } + { "id": "apps", "label": "APPS", "pos": [2, 4], "checks": [ + { "type": "conn", "conn": "Propresenter-API", "tag": "PROPRES" }, + { "type": "conn", "conn": "ableton", "tag": "ABLETON" }, + { "type": "ping", "host": "10.1.3.227", "name": "TRACKS MAC" }, + { "type": "http", "url": "http://10.6.7.13:8080/", "msg": "BIBLE APP DOWN", "interval_seconds": 5 }, + { "type": "conn", "conn": "BibleTV", "tag": "BIBLE OSC" } ] }, { "id": "service", "label": "ALARMS", "pos": [3, 0], "kind": "service" }, diff --git a/status-page.companionconfig b/status-page.companionconfig index ca61b4b..b9ea3e9 100644 Binary files a/status-page.companionconfig and b/status-page.companionconfig differ diff --git a/status_panel.py b/status_panel.py index d2f3fbf..87f474b 100644 --- a/status_panel.py +++ b/status_panel.py @@ -346,7 +346,7 @@ def check_ping(chk, snap, ctx): fails = snap.ping.get(chk["host"]) if fails is not None and fails >= chk.get("failures", 2): msg = chk.get("msg") or chk.get("name", chk["host"]) + " OFFLINE" - return [("ping", Result(ERR, msg, service_only=chk.get("service_only", False)))] + return [("ping", Result(ERR, msg, service_only=chk.get("service_only", False), named=True))] return [("ping", Result(OK))] @@ -370,7 +370,7 @@ def check_process(chk, snap, ctx): def check_http(chk, snap, ctx): if snap.http.get(chk["url"]) is False: - return [("http", Result(ERR, chk.get("msg") or chk.get("name", "WEB") + " UNREACH"))] + return [("http", Result(ERR, chk.get("msg") or chk.get("name", "WEB") + " UNREACH", named=True))] return [("http", Result(OK))] @@ -398,14 +398,14 @@ def check_midi_peer(chk, snap, ctx): # clock sync, then "running". A peer that dies is removed from the list: # at once if it says goodbye, after ~60 s if it crashes or its cable is pulled. if p["state"] in ("running", "connected"): - return [("peer", Result(OK, info="MIDI OK"))] + return [("peer", Result(OK))] return [("peer", Result(ERR, "MIDI " + (p["state"] or "?").upper()[:8]))] return [("peer", Result(ERR, "MIDI LOST"))] def check_mdns(chk, snap, ctx): if snap.mdns_time and chk["instance"] not in snap.mdns: - return [("mdns", Result(ERR, chk.get("name", chk["instance"]) + " GONE"))] + return [("mdns", Result(ERR, chk.get("name", chk["instance"]) + " GONE", named=True))] return [("mdns", Result(OK))] @@ -595,6 +595,9 @@ class Engine: log.exception("check %s/%s failed", t["id"], chk["type"]) results = [("error", Result(WARN, "CHECK BUG"))] for sub, res in results: + # "tag" names the device when one button covers several (APPS, MIDI). + if chk.get("tag") and res.level in (WARN, ERR) and not res.named: + res.msg, res.named = "%s %s" % (chk["tag"], res.msg), True raw["%s:%d:%s" % (t["id"], i, sub)] = (t["id"], res, chk) for sub, res in self.network_results(snap, now): raw["net:%s" % sub] = ("net", res, {"type": "network"}) @@ -877,6 +880,16 @@ def poll_system(snap, cfg): snap.procs = procs +def parse_dns_sd(out): + """Instance names still present in `dns-sd -B` output.""" + names = set() + for line in out.splitlines(): + m = re.match(r"^\s*\S+\s+(Add|Rmv)\s+\d+\s+\d+\s+\S+\s+_apple-midi\._udp\.\s+(.+?)\s*$", line) + if m: + (names.add if m.group(1) == "Add" else names.discard)(m.group(2)) + return names + + def poll_mdns(snap, seconds=3): proc = subprocess.Popen(["/usr/bin/dns-sd", "-B", "_apple-midi._udp", "local."], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) @@ -885,12 +898,7 @@ def poll_mdns(snap, seconds=3): except subprocess.TimeoutExpired: proc.kill() out, _ = proc.communicate() - names = set() - for line in out.splitlines(): - m = re.match(r"^\S+\s+(Add|Rmv)\s+\d+\s+\d+\s+\S+\s+_apple-midi\._udp\.\s+(.+?)\s*$", line) - if m: - (names.add if m.group(1) == "Add" else names.discard)(m.group(2)) - snap.mdns, snap.mdns_time = names, time.time() + snap.mdns, snap.mdns_time = parse_dns_sd(out), time.time() _UNVERIFIED = ssl._create_unverified_context() # reachability only; nothing is sent diff --git a/tests/test_status_panel.py b/tests/test_status_panel.py index a1be75e..99ae8d1 100644 --- a/tests/test_status_panel.py +++ b/tests/test_status_panel.py @@ -167,14 +167,14 @@ class TestChecks(unittest.TestCase): out = dict(sp.check_dante({"conn": "audinate-dantecontroller", "devices": ["BOH-WING"]}, self.snap, self.ctx)) self.assertEqual(out["conn"].msg, "NO DATA") - def test_dante_count_includes_other_buttons(self): + def test_dante_button_names_the_missing_device(self): devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")]) devices.remove("NA-Tracks") self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices) tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, self.now, 4) - self.assertEqual(tiles["dante"]["x"], "DANTE\n18/19") - self.assertEqual(level(tiles["dante"]), "ok") # the alarm is on TRACKS, not twice - self.assertEqual(tiles["tracks"]["x"], "TRACKS\nDANTE GONE") + self.assertEqual(tiles["dante"]["x"], "DANTE\nTRACKS GONE") + self.assertEqual(level(tiles["dante"]), "err") + self.assertEqual(tiles["all"]["x"], "1 DOWN\nDANTE TRACKS GONE") def test_shure_psm_tascam(self): s = self.snap @@ -204,24 +204,42 @@ class TestChecks(unittest.TestCase): tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, self.now, 4) self.assertIn("PSM 1 OFFLINE", tiles["rf"]["x"]) + def test_bonjour_parsing_morning_and_afternoon(self): + out = ("Browsing for _apple-midi._udp.local.\n" + " 9:20:51.321 Add 3 13 local. _apple-midi._udp. NA Tracks\n" + "19:20:51.321 Add 3 27 local. _apple-midi._udp. Lighting PC\n" + "19:21:02.001 Add 3 27 local. _apple-midi._udp. Gone Soon\n" + "19:21:09.001 Rmv 1 27 local. _apple-midi._udp. Gone Soon\n") + self.assertEqual(sp.parse_dns_sd(out), {"NA Tracks", "Lighting PC"}) + def test_midi_peer_joining_is_not_an_error(self): for p in self.snap.midi["NA Ingest"]["peers"]: p["state"] = "connected" # joined, clock sync not finished yet tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, self.now, 8) - self.assertEqual(tiles["lightpc"]["x"], "LIGHT PC\nMIDI OK") + self.assertEqual(tiles["midi"]["x"], "MIDI\nOK") def test_midi_peer_and_bonjour(self): self.snap.midi["NA Ingest"]["peers"] = [p for p in self.snap.midi["NA Ingest"]["peers"] if p["name"] != "Lighting PC"] self.snap.mdns.discard("NA Tracks") tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, self.now, 4) - self.assertEqual(tiles["lightpc"]["x"], "LIGHT PC\nMIDI LOST") - self.assertEqual(tiles["tracks"]["x"], "TRACKS\nMIDI SESS GONE") + self.assertEqual(tiles["midi"]["x"], "MIDI\nLIGHT PC MIDI LOST\n+1") + self.assertEqual(tiles["show"]["x"], "SHOW\nLIGHT PC MIDI LOST\n+1") # named, no "MIDI" prefix + self.assertNotIn("MIDI TRACKS", tiles["show"]["x"]) + self.snap.midi["NA Ingest"]["peers"].append({"name": "Lighting PC", "address": "", "state": "running"}) + tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, self.now, 4) + self.assertEqual(tiles["midi"]["x"], "MIDI\nTRACKS MIDI GONE") def test_bible_tv_app_down(self): self.snap.http["http://10.6.7.13:8080/"] = False tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, self.now, 4) - self.assertEqual(tiles["bibletv"]["x"], "BIBLE TV\nAPP DOWN") + self.assertEqual(tiles["apps"]["x"], "APPS\nBIBLE APP DOWN") + + def test_apps_names_the_app(self): + self.snap.conns["Propresenter-API"]["category"] = "error" + tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, self.now, 4) + self.assertEqual(tiles["apps"]["x"], "APPS\nPROPRES OFFLINE") + self.assertEqual(tiles["all"]["x"], "1 DOWN\nPROPRES OFFLINE") def test_http_intervals(self): checks = dict(sp.http_checks(CONFIG)) @@ -390,7 +408,8 @@ class TestBehaviour(unittest.TestCase): engine = sp.Engine(CONFIG) self.assertEqual(engine.restart_targets("hd3"), ["HyperDeck_3"]) self.assertEqual(engine.restart_targets("led"), ["VX1000", "UHDJr"]) - self.assertEqual(engine.restart_targets("tracks"), ["ableton"]) # not the shared Dante one + self.assertEqual(engine.restart_targets("apps"), ["Propresenter-API", "ableton", "BibleTV"]) + self.assertEqual(engine.restart_targets("dante"), []) # shared connection: never self.assertEqual(engine.restart_targets("ingest"), [])