modules/resi-cues/module.json

@@ -1,7 +1,7 @@
 {
   "id": "resi-cues",
   "name": "Resi Cues",
-  "version": "1.0.1",
+  "version": "1.0.2",
   "proddash": ">=1.4.0",
   "description": "Time until the next Resi cue point, on the board and as a timer",
   "provides": ["timers"],

modules/resi-cues/resi.js

@@ -39,6 +39,8 @@ function msToPosition(total) {
 
 const WINDOW_MS = 120000;   // how long a reading stays useful
 const JUMP_FLOOR_MS = 5000; // the smallest move ever treated as a seek
+const LOGIN_BACKOFF_MS = 15000;      // first wait after a failed Resi login
+const LOGIN_BACKOFF_MAX_MS = 300000; // longest wait between login attempts
 
 /**
  * Resi republishes a decoder's `position` only every few seconds, on an irregular
@@ -175,13 +177,43 @@ function createResiClient(options) {
     return h;
   }
 
+  /* A failed login used to be retried on the very next poll -- once a second, for
+     ever. Resi answers that with 429s, and since the retries never let up, the
+     account never got out of them. So a failure now holds off further attempts,
+     doubling from 15s to 5min (or as long as Resi's Retry-After asks), and every
+     call in between fails at once with the same reason instead of signing in. */
+  const now = typeof opts.now === 'function' ? opts.now : () => Date.now();
+  let loginFailures = 0;
+  let loginBlockedUntil = 0;
+  let loginError = '';
+
   async function login() {
-    const res = await doFetch(`${RESI}/api/v3/login?newToken=true`, {
+    if (now() < loginBlockedUntil) {
-      method: 'POST',
+      const wait = Math.ceil((loginBlockedUntil - now()) / 1000);
-      headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
+      throw new Error(`${loginError} — retrying in ${wait}s`);
-      body: JSON.stringify({ userName: opts.username, password: opts.password }),
+    }
-    });
+    let res;
-    if (!res.ok) throw new Error(`Resi login failed (${res.status})`);
+    try {
+      res = await doFetch(`${RESI}/api/v3/login?newToken=true`, {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json', Accept: 'application/json' },
+        body: JSON.stringify({ userName: opts.username, password: opts.password }),
+      });
+    } catch (err) {
+      holdOff(`Resi unreachable (${(err && err.message) || err})`, 0);
+      throw new Error(loginError);
+    }
+    if (!res.ok) {
+      const reason = res.status === 429 ? 'Resi is rate-limiting this login (429)'
+        : res.status === 400 || res.status === 401 ? `Resi rejected the login (${res.status}) — check the password`
+          : `Resi login failed (${res.status})`;
+      const retryAfter = Number(res.headers && res.headers.get && res.headers.get('retry-after')) || 0;
+      holdOff(reason, retryAfter * 1000);
+      throw new Error(reason);
+    }
+    loginFailures = 0;
+    loginBlockedUntil = 0;
+    loginError = '';
     const setCookies = (res.headers.getSetCookie && res.headers.getSetCookie())
       || (res.headers.get('set-cookie') ? [res.headers.get('set-cookie')] : []);
     const jar = {};
@@ -200,6 +232,13 @@ function createResiClient(options) {
     return me;
   }
 
+  function holdOff(reason, askedMs) {
+    loginFailures++;
+    const backoff = Math.min(LOGIN_BACKOFF_MAX_MS, LOGIN_BACKOFF_MS * 2 ** (loginFailures - 1));
+    loginBlockedUntil = now() + Math.max(backoff, askedMs || 0);
+    loginError = reason;
+  }
+
   async function ensureSession() {
     if (!token || Date.now() - loggedInAt > 50 * 60 * 1000) await login();
   }

modules/resi-cues/server.js

@@ -70,6 +70,7 @@ function emptyTimers(now) {
 }
 
 let current = null; // the live handle, shared with routes()
+let pickerClient = null; // { key, client } for the decoder picker when the module isn't running those credentials
 
 module.exports = {
   init({ config, log }) {
@@ -273,6 +274,7 @@ module.exports = {
       stateStreams,
       timerStreams,
       resi,
+      credentialsKey: `${username}\n${password}`,
       configured: Boolean(username && password),
       stop() {
         stopped = true;
@@ -334,9 +336,21 @@ module.exports = {
           return json(res, 200, { options: [], note: 'Enter the Resi login first' });
         }
         try {
-          const client = createResiClient({
+          // The running module's client when it has these credentials, else one kept
-            username: String(config.username), password: String(config.password),
+          // per login: a fresh client per request signed in on every admin page load,
-          });
+          // and its own hold-off never applied -- which kept Resi rate-limiting.
+          const username = String(config.username).trim();
+          const key = `${username}\n${config.password}`;
+          let client;
+          if (current && current.resi && current.credentialsKey === key) client = current.resi;
+          else {
+            if (!pickerClient || pickerClient.key !== key) {
+              pickerClient = { key, client: createResiClient({
+                username, password: String(config.password),
+              }) };
+            }
+            client = pickerClient.client;
+          }
           const options = [];
           for (const v of await client.venues()) {
             for (const p of await client.players(v.uuid)) {

modules/resi-cues/server.test.js

@@ -196,6 +196,53 @@ function collect(route, req) {
     assert.strictEqual(JSON.parse(res.body).status, 'error');
   });
 
+  // Seen live at North Attleboro: a failed login was retried on every 1s poll, so
+  // Resi answered 429 and kept answering it, and the decoder picker stayed empty.
+  await test('a failed login holds off instead of retrying every poll', async () => {
+    const { createResiClient } = require('./resi');
+    let t = 1700000000000;
+    let logins = 0;
+    let limited = true;
+    const client = createResiClient({
+      username: 'a@b.c',
+      password: 'secret',
+      now: () => t,
+      fetch: async (url) => {
+        const path = String(url).replace('https://central.resi.io', '').split('?')[0];
+        if (path === '/api/v3/login') {
+          logins++;
+          if (limited) return reply({}, { ok: false, status: 429 });
+          return reply({}, { headers: { get: () => null, getSetCookie: () => ['livingas1=T; Path=/'] } });
+        }
+        if (path === '/api_v2.svc/users/me') return reply({ customerId: 'CUST' });
+        if (path === '/api/v3/customers/CUST/venues') return reply([{ uuid: 'V1' }]);
+        return reply({}, { ok: false, status: 404 });
+      },
+    });
+    const poll = async () => { try { await client.venues(); return 'ok'; } catch (e) { return e.message; } };
+
+    assert.match(await poll(), /rate-limiting/);
+    for (let i = 0; i < 60; i++) { t += 1000; await poll(); }   // a minute of 1s polls
+    assert.ok(logins <= 3, `signed in ${logins} times in a minute while rate-limited`);
+    assert.match(await poll(), /retrying in \d+s/, 'the wait is shown, not hidden');
+
+    limited = false;
+    t += 5 * 60000;                                             // past the longest hold-off
+    assert.strictEqual(await poll(), 'ok', 'recovers once Resi lets it in');
+    const after = logins;
+    for (let i = 0; i < 30; i++) { t += 1000; await poll(); }
+    assert.strictEqual(logins, after, 'a good session is not signed in again');
+  });
+
+  await test('a rejected password says so', async () => {
+    const { createResiClient } = require('./resi');
+    const client = createResiClient({
+      username: 'a@b.c', password: 'wrong', now: () => 0,
+      fetch: async () => reply({}, { ok: false, status: 400 }),
+    });
+    await assert.rejects(client.venues(), /check the password/);
+  });
+
   globalThis.setInterval = realSetInterval;
   globalThis.clearInterval = realClearInterval;
   console.log(`\n${passed} tests passed`);