A cloud account throttled (429 / rate-limit) is amber RATE LIMITED, not red OFFLINE

2026-10-25: RESI showed OFFLINE 15 times during services while the stream was fine: Resi's cloud was
rate-limiting the account Companion's resi-studio-cues (player every 3 s, cues every 5 s) shares with
ProdDash's Resi cues.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-25 11:25:29 -04:00
co-authored by Claude Opus 5.5
parent c471ac2c92
commit 0da1c20a17
3 changed files with 13 additions and 1 deletions
+1 -1
View File
@@ -548,7 +548,7 @@ GLOSSARY = {
"check the device or the cable": [ "check the device or the cable": [
"NO PING", "NO ANSWER", "UNREACHABLE", "LINK DOWN", "NO GATEWAY", "NO DISK", "FULL", "LOW", "BATT", "INTERF", "NO PING", "NO ANSWER", "UNREACHABLE", "LINK DOWN", "NO GATEWAY", "NO DISK", "FULL", "LOW", "BATT", "INTERF",
"RF MUTE", "HUNG - POWER CYCLE IT", "NO NETWORK CHECK POWER", "STILL DOWN", "REFUSED", "LOGIN", "BLIP", "RF MUTE", "HUNG - POWER CYCLE IT", "NO NETWORK CHECK POWER", "STILL DOWN", "REFUSED", "LOGIN", "BLIP",
"NOT REACHABLE", "DISK", "NOT REACHABLE", "DISK", "RATE LIMITED",
], ],
"the READY page: what is not ready yet (hold the key to fix it or look again)": [ "the READY page: what is not ready yet (hold the key to fix it or look again)": [
"TO DO", "ALL DONE", "TAP WHEN DONE", "TAP WHEN SET", "NOT DONE", "DONE", "NOT SET UP", "CHECKING", "TO DO", "ALL DONE", "TAP WHEN DONE", "TAP WHEN SET", "NOT DONE", "DONE", "NOT SET UP", "CHECKING",
+4
View File
@@ -230,6 +230,10 @@ def conn_result(snap, label, now, service_only=False):
return Result(OK) return Result(OK)
if cat == "warning": if cat == "warning":
return Result(WARN, short_reason(c.get("message"), "WARNING"), service_only=service_only) return Result(WARN, short_reason(c.get("message"), "WARNING"), service_only=service_only)
if cat == "error" and re.search(r"\b429\b|rate.?limit", c.get("message") or "", re.I):
# A cloud service throttling the account (Resi, 2026-10-25: "rate-limiting this account (429)" every few
# minutes, Companion and ProdDash share it): the device itself is fine, so amber, not OFFLINE.
return Result(WARN, "RATE LIMITED", service_only=service_only)
if cat == "error": if cat == "error":
return Result(ERR, short_reason(c.get("message"), "OFFLINE"), service_only=service_only) return Result(ERR, short_reason(c.get("message"), "OFFLINE"), service_only=service_only)
return Result(WARN, "STARTING", service_only=service_only) return Result(WARN, "STARTING", service_only=service_only)
+8
View File
@@ -348,6 +348,14 @@ class TestChecks(unittest.TestCase):
self.assertIn(f, CONFIG["auto_fix"]["fixes"]) self.assertIn(f, CONFIG["auto_fix"]["fixes"])
self.assertIn("midiEnabled -bool false", CONFIG["fixes"]["atem_midi"]["run"]) self.assertIn("midiEnabled -bool false", CONFIG["fixes"]["atem_midi"]["run"])
def test_a_rate_limited_cloud_account_is_amber_not_offline(self):
"""2026-10-25: Resi throttled the shared account (429) every few minutes; the stream was fine."""
self.snap.conns["resi-studio-cues"] = {"id": "r", "enabled": True, "category": "error",
"message": "Resi is rate-limiting this account (429)"}
tiles, _ = run_steps(sp.Engine(CONFIG), self.snap, self.now, 4)
self.assertEqual(tiles["resi"]["x"], "RESI\nRATE LIMITED")
self.assertNotEqual(level(tiles["resi"]), "err")
def test_resi_audio_silent(self): def test_resi_audio_silent(self):
"""An input stuck at the meter floor (2026-10-25: the Resi AJA passed no Dante audio and nothing said so): """An input stuck at the meter floor (2026-10-25: the Resi AJA passed no Dante audio and nothing said so):
red while a deck records (a service is on); between services the Resi send is quiet on purpose.""" red while a deck records (a service is on); between services the Resi send is quiet on purpose."""