Restart keys flash when their thing is down; Ableton reopens its last set

- A Restart page key flashes red "HOLD TO FIX" while any check that its
  fix (or a fix/connection in its "watch") covers is down, e.g. DVS
  TRACKS when DANTE misses NA-Tracks.
- Ableton's start and restart reopen the set Live last opened (read from
  its Log.txt before quitting; templates skipped) instead of the default
  template. Tested live: ABLETON and TRANSLATE restart keys.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

README.md

@@ -68,13 +68,13 @@ Row 0 holds the group buttons. Each one shows the first problem in its group, pl
 
 ## Restart page (page 20)
 
-Tap **RESTART PAGE** on the status page (row 2). Each key restarts one thing **even when the status page says it is fine**; hold it 2 s. It shows RESTARTING, then DONE (or FAILED / what to do). The same thing will not run twice within a minute. **STATUS PAGE** (bottom left) goes back.
+Tap **RESTART PAGE** on the status page (row 2). Each key restarts one thing **even when the status page says it is fine**; hold it 2 s. A key **flashes red (HOLD TO FIX)** while the thing it restarts is down, so you can see which one to hold. It shows RESTARTING, then DONE (or FAILED / what to do). The same thing will not run twice within a minute. **STATUS PAGE** (bottom left) goes back.
 
 | Key | Restarts |
 |---|---|
 | DVS PROP / TRACKS / TRANSL / INGEST | Dante Virtual Soundcard's engine on that Mac (audio to and from it drops 5–10 s). DVS INGEST also restarts ProdCom afterwards, which otherwise keeps a dead input. |
 | PROPRES | ProPresenter (quits and reopens the newest version) |
-| ABLETON | Ableton on the Tracks Mac (quits it: unsaved set changes are lost, and Live may ask to recover - click it on the Tracks Mac) |
+| ABLETON | Ableton on the Tracks Mac, reopening the set it had open (from Live's own log; templates skipped). Unsaved changes are lost, and Live may ask to recover - click it on the Tracks Mac. APPS's hold, which starts a closed Ableton, opens that set too. |
 | TRANSLATE | Church Translate on NA Translation |
 | PRODCOM | ProdCom on NA Ingest (lobby captions blank for 5–15 s) |
 | PRODDASH / CAPTIONS / BRIDGE | that service on NA Ingest |

config.json

@@ -40,8 +40,8 @@
   "fixes": {
     "propresenter":    { "host": "prop", "short": "PROPRES", "label": "restart ProPresenter (newest installed)",
                          "run": "APP=$(ls -d /Applications/ProPresenter*.app | sort -V | tail -1); if pgrep -xq ProPresenter; then pkill -TERM -x ProPresenter; for i in 1 2 3 4 5 6 7 8 9 10; do pgrep -xq ProPresenter || break; sleep 1; done; pkill -KILL -x ProPresenter; sleep 1; fi; open \"$APP\"" },
-    "ableton":         { "host": "tracks", "short": "ABLETON", "label": "start Ableton if it is not running",
+    "ableton":         { "host": "tracks", "short": "ABLETON", "label": "start Ableton if it is not running, with the set it last had open",
-                         "run": "pgrep -xq Live || open -a 'Ableton Live 11 Suite'" },
+                         "run": "pgrep -xq Live || { L=$(ls -td ~/Library/Preferences/Ableton/Live*/ | head -1); S=$(grep -a 'info: Loading document \"' \"${L}Log.txt\" | grep -v '/Templates/' | tail -1 | sed -E 's/^.*Loading document \"(.*)\"[^\"]*$/\\1/'); if [ -n \"$S\" ] && [ -f \"$S\" ]; then open -a 'Ableton Live 11 Suite' \"$S\"; else open -a 'Ableton Live 11 Suite'; fi; }" },
     "translate":       { "host": "translation", "short": "TRANSLATE", "label": "start Church Translate if it is not running",
                          "run": "pgrep -xq 'Church Translate' || open -a 'Church Translate'" },
     "dvs_prop":        { "host": "prop", "short": "DVS", "label": "restart Dante Virtual Soundcard on the ProPresenter Mac (its engine restarts; audio to and from that Mac drops for 5-10 s)", "run": "sudo -n /usr/local/sbin/status-panel-dvs-start && open -g -a 'Dante Virtual Soundcard'" },
@@ -68,8 +68,8 @@
                          "run": "pkill -TERM -x ProdDash; i=0; while pgrep -xq ProdDash && [ $i -lt 20 ]; do sleep 0.5; i=$((i+1)); done; open -g /Applications/ProdDash.app" },
     "prodcom":         { "kind": "local", "short": "PRODCOM", "label": "restart ProdCom (transcription; lobby captions blank for 5-15 s)",
                          "run": "pkill -TERM -x ProdCom; i=0; while pgrep -xq ProdCom && [ $i -lt 20 ]; do sleep 0.5; i=$((i+1)); done; pgrep -xq ProdCom && pkill -KILL -x ProdCom; sleep 1; open -g /Applications/ProdCom.app" },
-    "ableton_restart": { "host": "tracks", "short": "ABLETON", "label": "restart Ableton on the Tracks Mac (quits it; unsaved set changes are lost, and Live may ask to recover on the Tracks Mac)", "conn": "ableton", "reconnect": true, "timeout": 60,
+    "ableton_restart": { "host": "tracks", "short": "ABLETON", "label": "restart Ableton on the Tracks Mac and reopen the set it had open (unsaved changes are lost; Live may ask to recover on the Tracks Mac)", "conn": "ableton", "reconnect": true, "timeout": 60,
-                         "run": "pkill -TERM -x 'Live'; i=0; while pgrep -xq 'Live' && [ $i -lt 30 ]; do sleep 0.5; i=$((i+1)); done; pgrep -xq 'Live' && pkill -KILL -x 'Live'; sleep 1; open -a 'Ableton Live 11 Suite'" },
+                         "run": "L=$(ls -td ~/Library/Preferences/Ableton/Live*/ | head -1); S=$(grep -a 'info: Loading document \"' \"${L}Log.txt\" | grep -v '/Templates/' | tail -1 | sed -E 's/^.*Loading document \"(.*)\"[^\"]*$/\\1/'); pkill -TERM -x 'Live'; i=0; while pgrep -xq 'Live' && [ $i -lt 30 ]; do sleep 0.5; i=$((i+1)); done; pgrep -xq 'Live' && pkill -KILL -x 'Live'; sleep 1; if [ -n \"$S\" ] && [ -f \"$S\" ]; then open -a 'Ableton Live 11 Suite' \"$S\"; else open -a 'Ableton Live 11 Suite'; fi" },
     "translate_restart": { "host": "translation", "short": "TRANSLATE", "label": "restart Church Translate on NA Translation (and its translation server)", "timeout": 60,
                          "run": "pg=$(ps -o pgid= -p \"$(pgrep -x 'Church Translate' | head -1)\" 2>/dev/null | tr -d ' '); pkill -TERM -x 'Church Translate'; i=0; while pgrep -xq 'Church Translate' && [ $i -lt 30 ]; do sleep 0.5; i=$((i+1)); done; pgrep -xq 'Church Translate' && pkill -KILL -x 'Church Translate'; [ -n \"$pg\" ] && [ \"$pg\" != \"$(ps -o pgid= -p $$ | tr -d ' ')\" ] && kill -KILL -- -\"$pg\" 2>/dev/null; sleep 1; open -a 'Church Translate'" },
     "tascam_reconnect": { "kind": "reconnect", "conn": "tascam", "short": "TASCAM", "verify_seconds": 20, "hung_note": "HUNG - POWER\nCYCLE IT",
@@ -85,13 +85,13 @@
       { "id": "rs_dvs_transl", "label": "DVS TRANSL", "pos": [0, 3], "fix": "dvs_translation" },
       { "id": "rs_dvs_ingest", "label": "DVS INGEST", "pos": [0, 4], "fix": "dvs_ingest" },
       { "id": "rs_propres",    "label": "PROPRES",    "pos": [1, 1], "fix": "propresenter" },
-      { "id": "rs_ableton",    "label": "ABLETON",    "pos": [1, 2], "fix": "ableton_restart" },
+      { "id": "rs_ableton",    "label": "ABLETON",    "pos": [1, 2], "fix": "ableton_restart", "watch": ["ableton", "conn:ableton"] },
-      { "id": "rs_translate",  "label": "TRANSLATE",  "pos": [1, 3], "fix": "translate_restart" },
+      { "id": "rs_translate",  "label": "TRANSLATE",  "pos": [1, 3], "fix": "translate_restart", "watch": ["translate"] },
       { "id": "rs_prodcom",    "label": "PRODCOM",    "pos": [2, 1], "fix": "prodcom" },
       { "id": "rs_proddash",   "label": "PRODDASH",   "pos": [2, 2], "fix": "proddash" },
-      { "id": "rs_captions",   "label": "CAPTIONS",   "pos": [2, 3], "fix": "captions" },
+      { "id": "rs_captions",   "label": "CAPTIONS",   "pos": [2, 3], "fix": "captions", "watch": ["caption_screen"] },
       { "id": "rs_bridge",     "label": "BRIDGE",     "pos": [2, 4], "fix": "bridge" },
-      { "id": "rs_tascam",     "label": "TASCAM",     "pos": [3, 1], "fix": "tascam_reconnect" }
+      { "id": "rs_tascam",     "label": "TASCAM",     "pos": [3, 1], "fix": "tascam_reconnect", "watch": ["conn:tascam"] }
     ]
   },
 

status_panel.py

@@ -863,6 +863,23 @@ class Engine:
                     plan.append((ident, "restart", chk["conn"], chk))
         return plan
 
+    def failing_targets(self):
+        """(fix ids, Companion connections) of every check that is down right now: a Restart page
+        key whose fix (or a fix/connection in its "watch") is among them flashes."""
+        fixes, conns = set(), set()
+        for key, tr in self.trackers.items():
+            if not tr.confirmed or tr.result.level != ERR or key not in self.meta:
+                continue
+            _, chk, sub = self.meta[key]
+            fix_id = chk.get("fix")
+            if isinstance(fix_id, dict):
+                fix_id = fix_id.get(sub, fix_id.get("*"))
+            if fix_id:
+                fixes.add(fix_id)
+            if chk.get("conn"):
+                conns.add(chk["conn"])
+        return fixes, conns
+
     def failing(self, tile_id):
         """Is anything on this button (or its members) in a confirmed fault right now?"""
         for key in self.keys_for(tile_id):
@@ -1062,10 +1079,15 @@ def colored(text, level):
 
 def render(engine, service_on):
     out = {}
+    down_fixes, down_conns = engine.failing_targets()
     for kid, key in engine.restart_keys.items():
         note = engine.notes.get(kid)
+        watch = [key["fix"]] + key.get("watch", [])
+        down = any(w[5:] in down_conns if w.startswith("conn:") else w in down_fixes for w in watch)
         if note and engine.now < note["until"]:
             out[kid] = colored(key["label"] + "\n" + note["text"], note["level"])
+        elif down:                                    # flashes: this is the one to hold
+            out[kid] = colored(key["label"] + "\nHOLD TO FIX", "err")
         else:
             out[kid] = colored(key["label"] + "\nHOLD 2s", "idle")
     for tile in engine.cfg["tiles"]:
@@ -1697,6 +1719,9 @@ def load_config(path):
             raise ValueError("restart key %s has a tile's id" % k["id"])
         if k["fix"] not in cfg.get("fixes", {}):
             raise ValueError("restart key %s: unknown fix %s" % (k["id"], k["fix"]))
+        for w in k.get("watch", []):
+            if not w.startswith("conn:") and w not in cfg.get("fixes", {}):
+                raise ValueError("restart key %s: unknown fix %s in watch" % (k["id"], w))
     return cfg
 
 

tests/test_status_panel.py

@@ -660,6 +660,29 @@ class TestFixes(unittest.TestCase):
                       open(os.path.join(os.path.dirname(os.path.abspath(sp.__file__)), "remote-access", "mac-dvs-helper.sh")).read())
         self.assertIn("open -g /Applications/ProdCom.app", CONFIG["fixes"]["dvs_ingest"]["run"])  # ProdCom after DVS
 
+    def test_restart_keys_flash_when_their_thing_is_down(self):
+        devices = json.loads(self.snap.vars[("audinate-dantecontroller", "devices")])
+        devices.remove("NA-Tracks")                                          # DVS on the Tracks Mac gone
+        self.snap.vars[("audinate-dantecontroller", "devices")] = json.dumps(devices)
+        self.snap.remote["tracks\0pgrep -xq Live"] = False                   # Ableton not running
+        self.snap.conns["tascam"]["category"] = "error"
+        tiles, _ = run_steps(self.engine, self.snap, SUNDAY, 4)
+        flashing = {k for k, v in tiles.items() if k.startswith("rs_") and level(v) == "err"}
+        self.assertEqual(flashing, {"rs_dvs_tracks", "rs_ableton", "rs_tascam"})
+        self.assertEqual(tiles["rs_dvs_tracks"]["x"], "DVS TRACKS\nHOLD TO FIX")
+        self.assertEqual(level(tiles["rs_prodcom"]), "idle")
+        self.engine.ack("all")                                               # acked on the status page:
+        tiles = self.engine.step(self.snap, SUNDAY + 5)                      # still shows what to hold
+        self.assertEqual(level(tiles["rs_ableton"]), "err")
+
+    def test_ableton_reopens_the_set_it_had_open(self):
+        for fix_id in ("ableton", "ableton_restart"):
+            run = CONFIG["fixes"][fix_id]["run"]
+            self.assertIn("Log.txt", run)
+            self.assertIn("grep -v '/Templates/'", run)
+            self.assertIn('open -a \'Ableton Live 11 Suite\' "$S"', run)
+        self.assertTrue(CONFIG["fixes"]["ableton"]["run"].startswith("pgrep -xq Live || {"))
+
     def test_tascam_hold_always_reconnects(self):
         ran = []
         self.engine.executor = lambda cfg, fix_id, chk: ran.append(fix_id) or "HUNG - POWER\nCYCLE IT"