ATEM: read the saved-file folder through this Mac's own Remote Login when macOS keeps the page out of Google Drive

A launchd-run python3 gets 'Operation not permitted' on ~/Library/CloudStorage; sshd does not. The
page now lists and reads the folder over ssh to localhost with the panel's own key when the local
read is blocked (the Ingest Mac's Setup card gains that authorise step). Pick the newest verified
on NA Ingest.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

settings/scripts.py

@@ -110,7 +110,9 @@ def setup_for(cfg, alias):
             ("the Spotify permission was allowed once", "Hold the SPOTIFY key once while sitting at this Mac and click Allow on the Automation prompt."),
             ("the DVS helper and its sudo rule", "Run the DVS helper script below on this Mac too (as root, through ARD or sudo)."),
             ("Google Drive is signed in (saved switcher files)", "Finder: Google Drive is mounted and the switcher folder on Site setup > ATEM exists."),
+            ("the panel's key is accepted on this Mac too", "macOS keeps background programs out of cloud folders; the page reads the saved switcher files through this Mac's own Remote Login instead. Run the authorise line below here (Terminal, or ARD as root)."),
         ]
+        scripts.append({"name": "Authorise the panel's login on this Mac (run as root)", "how": "Terminal on this Mac with sudo, or ARD > Send UNIX Command as root.", "text": mac_authorize(cfg)})
         if "helper_dvs" in kinds:
             scripts.append({"name": "DVS helper (run as root)", "how": "Apple Remote Desktop > Send UNIX Command as root, or sudo zsh in Terminal here.", "text": mac_dvs_helper(cfg)})
     elif host.get("os") == "windows":

settings/server.py

@@ -19,6 +19,7 @@ import logging
 import logging.handlers
 import os
 import re
+import shlex
 import socket
 import subprocess
 import sys
@@ -645,15 +646,9 @@ class Handler(http.server.BaseHTTPRequestHandler):
             if path == "/api/atem-reference/folder":
                 folder = os.path.expanduser((site.cfg.get("atem") or {}).get("source_folder") or "")
                 mounted = bool(folder and os.path.isdir(folder))
-                files = sorted(glob.glob(os.path.join(folder, "*.xml")), key=os.path.getmtime, reverse=True) if mounted else []
-                blocked = False
-                if mounted and not files:
-                    try:
-                        blocked = not os.listdir(folder) and "/Library/CloudStorage/" in folder   # a cloud folder that looks empty to a background process
-                    except OSError:
-                        blocked = True
-                return self.send(200, {"folder": folder, "mounted": mounted, "blocked": blocked,
-                                       "files": [{"name": os.path.basename(f), "when": os.path.getmtime(f)} for f in files[:10]]})
+                files, via, blocked = cloud_files(site.cfg, folder) if mounted else ([], "", False)
+                return self.send(200, {"folder": folder, "mounted": mounted, "blocked": blocked, "via": via,
+                                       "files": [{"name": n, "when": t} for n, t in files[:10]]})
             if path == "/api/live/midi":
                 snap = site.read_json("snapshot.json", {}) or {}
                 return self.send(200, {"local": snap.get("midi"), "remote": snap.get("remote_midi")})
@@ -865,11 +860,15 @@ class Handler(http.server.BaseHTTPRequestHandler):
                 return self.send(200, {"draft": self.masked_draft(draft), "from": src, "errors": errors, "warnings": warnings})
             if path == "/api/atem-reference/pick-newest":
                 folder = os.path.expanduser((site.cfg.get("atem") or {}).get("source_folder") or "")
-                files = sorted(glob.glob(os.path.join(folder, "*.xml")), key=os.path.getmtime) if folder and os.path.isdir(folder) else []
+                files, via, blocked = cloud_files(site.cfg, folder) if folder and os.path.isdir(folder) else ([], "", False)
                 if not files:
-                    return self.send(404, {"error": "no .xml files in the folder (is Google Drive signed in?)"})
-                with open(files[-1], "rb") as f:
-                    return self.send(200, dict(self.take_atem(f.read()), file=os.path.basename(files[-1])))
+                    return self.send(404, {"error": "no .xml files in the folder (is Google Drive signed in?)" if not blocked else
+                                           "this page is not allowed to read the cloud folder; use Upload a saved file"})
+                newest = os.path.join(folder, files[0][0])
+                raw = cloud_read(site.cfg, newest, via)
+                if raw is None:
+                    return self.send(502, {"error": "could not read %s" % files[0][0]})
+                return self.send(200, dict(self.take_atem(raw), file=files[0][0]))
             if path == "/api/atem-reference/discard":
                 self.discard_atem()
                 return self.send(200, {"ok": True})
@@ -1109,6 +1108,52 @@ class Handler(http.server.BaseHTTPRequestHandler):
         return "http://%s:%d/" % (host, self.server.server_address[1])
 
 
+def self_ssh(cfg):
+    """ssh to this very Mac with the panel's own key: sshd may read the cloud folders a background
+    program is kept out of (macOS privacy), and the key is already on this Mac."""
+    key = os.path.expanduser(cfg.get("ssh_key", "~/.ssh/id_status_panel"))
+    return ["/usr/bin/ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=4", "-o", "StrictHostKeyChecking=accept-new", "-i", key, "localhost"]
+
+
+def cloud_files(cfg, folder):
+    """(files [(name, mtime)] newest first, how, blocked). Reads the folder here; when macOS keeps this process
+    out of a cloud folder, asks this Mac over ssh instead."""
+    try:
+        names = [n for n in os.listdir(folder) if n.lower().endswith(".xml")]
+        files = sorted(((n, os.path.getmtime(os.path.join(folder, n))) for n in names), key=lambda x: x[1], reverse=True)
+        if files or "/Library/CloudStorage/" not in folder:
+            return files, "local", False
+    except OSError:
+        pass
+    try:
+        r = subprocess.run(self_ssh(cfg) + ["cd %s && stat -f '%%m %%N' *.xml" % shlex.quote(folder)], capture_output=True, text=True, timeout=15)
+        if r.returncode == 0:
+            files = []
+            for line in r.stdout.splitlines():
+                m, _, n = line.partition(" ")
+                if n.lower().endswith(".xml"):
+                    files.append((n, float(m)))
+            return sorted(files, key=lambda x: x[1], reverse=True), "ssh", False
+        log.warning("cloud folder over ssh: %s", (r.stderr or r.stdout).strip()[-200:])
+    except (OSError, subprocess.TimeoutExpired, ValueError) as e:
+        log.warning("cloud folder over ssh: %s", e)
+    return [], "", True
+
+
+def cloud_read(cfg, path, via):
+    if via != "ssh":
+        try:
+            with open(path, "rb") as f:
+                return f.read()
+        except OSError:
+            pass
+    try:
+        r = subprocess.run(self_ssh(cfg) + ["cat %s" % shlex.quote(path)], capture_output=True, timeout=30)
+        return r.stdout if r.returncode == 0 else None
+    except (OSError, subprocess.TimeoutExpired):
+        return None
+
+
 class Server(http.server.ThreadingHTTPServer):
     daemon_threads = True
     allow_reuse_address = True

settings/ui/app.js

@@ -913,7 +913,7 @@
     return `<div class="card"><h3>The saved switcher file</h3><p class="small muted">The panel keeps the switcher's audio and camera settings in line with this file (atem-reference.xml). Save a new one from ATEM Software Control when the show changes.</p>${pending}
       ${editing() ? `<div class="row"><label class="btn">Upload a saved file <input type="file" id="atemfile" accept=".xml" hidden></label><button id="atemnewest" ${folder.mounted && (folder.files || []).length ? "" : "disabled"}>Pick the newest from the folder</button></div>` : ""}
       <p class="small">Folder: <span class="mono">${esc(folder.folder || "(not set)")}</span> ${folder.mounted ? `· newest: ${(folder.files || []).length ? esc(folder.files[0].name) + " (" + esc(when(folder.files[0].when)) + ")" : "none"}` : "· <b>not mounted</b>"}</p>
-      ${folder.blocked ? `<div class="notice grey">The folder is there but this page is not allowed to read Google Drive (macOS keeps background programs out of cloud folders). Once, on the Ingest Mac: System Settings › Privacy & Security › Full Disk Access › add <span class="mono">/usr/bin/python3</span>, then restart the settings page from the Overview. Until then, use Upload a saved file.</div>` : ""}<p id="ateminfo" class="help"></p></div>`;
+      ${folder.blocked ? `<div class="notice grey">The folder is there but this page cannot read it (macOS keeps background programs out of cloud folders, and this Mac's own login key is not accepted on this Mac). Use Upload a saved file, or let the panel's key log in to this Mac: Computers › Setup › ${esc((cfg().site || {}).ingest_name || "the Ingest Mac")}.</div>` : ""}<p id="ateminfo" class="help"></p></div>`;
   }
   function wireSite() {
     const q = (s) => main.querySelector(s);

tests/test_setup.py

@@ -72,7 +72,7 @@ class TestScripts(unittest.TestCase):
                 self.assertTrue(s["steps"], alias)
                 self.assertTrue(all(x["how"] for x in s["steps"]), alias)
             self.assertEqual([x["name"] for x in scripts.setup_for(CFG, "studiob")["scripts"]][-1], "Pro Tools solo-mode helper")
-            self.assertEqual(len(scripts.setup_for(CFG, "ingest")["scripts"]), 1)
+            self.assertEqual([x["name"][:9] for x in scripts.setup_for(CFG, "ingest")["scripts"]], ["Authorise", "DVS helpe"])
 
     def test_no_key_yet_is_said_not_crashed(self):
         with mock.patch.object(scripts, "public_key", lambda cfg: (None, None)):