diff --git a/modules/resi-cues/module.json b/modules/resi-cues/module.json index 32d8348..6b507ab 100644 --- a/modules/resi-cues/module.json +++ b/modules/resi-cues/module.json @@ -1,7 +1,7 @@ { "id": "resi-cues", "name": "Resi Cues", - "version": "1.0.1", + "version": "1.0.2", "proddash": ">=1.4.0", "description": "Time until the next Resi cue point, on the board and as a timer", "provides": ["timers"], diff --git a/modules/resi-cues/resi.js b/modules/resi-cues/resi.js index d47e7be..4b5e861 100644 --- a/modules/resi-cues/resi.js +++ b/modules/resi-cues/resi.js @@ -39,6 +39,8 @@ function msToPosition(total) { const WINDOW_MS = 120000; // how long a reading stays useful const JUMP_FLOOR_MS = 5000; // the smallest move ever treated as a seek +const LOGIN_BACKOFF_MS = 15000; // first wait after a failed Resi login +const LOGIN_BACKOFF_MAX_MS = 300000; // longest wait between login attempts /** * Resi republishes a decoder's `position` only every few seconds, on an irregular @@ -175,13 +177,43 @@ function createResiClient(options) { return h; } + /* A failed login used to be retried on the very next poll -- once a second, for + ever. Resi answers that with 429s, and since the retries never let up, the + account never got out of them. So a failure now holds off further attempts, + doubling from 15s to 5min (or as long as Resi's Retry-After asks), and every + call in between fails at once with the same reason instead of signing in. */ + const now = typeof opts.now === 'function' ? opts.now : () => Date.now(); + let loginFailures = 0; + let loginBlockedUntil = 0; + let loginError = ''; + async function login() { - const res = await doFetch(`${RESI}/api/v3/login?newToken=true`, { - method: 'POST', - headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, - body: JSON.stringify({ userName: opts.username, password: opts.password }), - }); - if (!res.ok) throw new Error(`Resi login failed (${res.status})`); + if (now() < loginBlockedUntil) { + const wait = Math.ceil((loginBlockedUntil - now()) / 1000); + throw new Error(`${loginError} — retrying in ${wait}s`); + } + let res; + try { + res = await doFetch(`${RESI}/api/v3/login?newToken=true`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', Accept: 'application/json' }, + body: JSON.stringify({ userName: opts.username, password: opts.password }), + }); + } catch (err) { + holdOff(`Resi unreachable (${(err && err.message) || err})`, 0); + throw new Error(loginError); + } + if (!res.ok) { + const reason = res.status === 429 ? 'Resi is rate-limiting this login (429)' + : res.status === 400 || res.status === 401 ? `Resi rejected the login (${res.status}) — check the password` + : `Resi login failed (${res.status})`; + const retryAfter = Number(res.headers && res.headers.get && res.headers.get('retry-after')) || 0; + holdOff(reason, retryAfter * 1000); + throw new Error(reason); + } + loginFailures = 0; + loginBlockedUntil = 0; + loginError = ''; const setCookies = (res.headers.getSetCookie && res.headers.getSetCookie()) || (res.headers.get('set-cookie') ? [res.headers.get('set-cookie')] : []); const jar = {}; @@ -200,6 +232,13 @@ function createResiClient(options) { return me; } + function holdOff(reason, askedMs) { + loginFailures++; + const backoff = Math.min(LOGIN_BACKOFF_MAX_MS, LOGIN_BACKOFF_MS * 2 ** (loginFailures - 1)); + loginBlockedUntil = now() + Math.max(backoff, askedMs || 0); + loginError = reason; + } + async function ensureSession() { if (!token || Date.now() - loggedInAt > 50 * 60 * 1000) await login(); } diff --git a/modules/resi-cues/server.js b/modules/resi-cues/server.js index 3fe385b..e981b52 100644 --- a/modules/resi-cues/server.js +++ b/modules/resi-cues/server.js @@ -70,6 +70,7 @@ function emptyTimers(now) { } let current = null; // the live handle, shared with routes() +let pickerClient = null; // { key, client } for the decoder picker when the module isn't running those credentials module.exports = { init({ config, log }) { @@ -273,6 +274,7 @@ module.exports = { stateStreams, timerStreams, resi, + credentialsKey: `${username}\n${password}`, configured: Boolean(username && password), stop() { stopped = true; @@ -334,9 +336,21 @@ module.exports = { return json(res, 200, { options: [], note: 'Enter the Resi login first' }); } try { - const client = createResiClient({ - username: String(config.username), password: String(config.password), - }); + // The running module's client when it has these credentials, else one kept + // per login: a fresh client per request signed in on every admin page load, + // and its own hold-off never applied -- which kept Resi rate-limiting. + const username = String(config.username).trim(); + const key = `${username}\n${config.password}`; + let client; + if (current && current.resi && current.credentialsKey === key) client = current.resi; + else { + if (!pickerClient || pickerClient.key !== key) { + pickerClient = { key, client: createResiClient({ + username, password: String(config.password), + }) }; + } + client = pickerClient.client; + } const options = []; for (const v of await client.venues()) { for (const p of await client.players(v.uuid)) { diff --git a/modules/resi-cues/server.test.js b/modules/resi-cues/server.test.js index ddb43a3..9d50d6c 100644 --- a/modules/resi-cues/server.test.js +++ b/modules/resi-cues/server.test.js @@ -196,6 +196,53 @@ function collect(route, req) { assert.strictEqual(JSON.parse(res.body).status, 'error'); }); + // Seen live at North Attleboro: a failed login was retried on every 1s poll, so + // Resi answered 429 and kept answering it, and the decoder picker stayed empty. + await test('a failed login holds off instead of retrying every poll', async () => { + const { createResiClient } = require('./resi'); + let t = 1700000000000; + let logins = 0; + let limited = true; + const client = createResiClient({ + username: 'a@b.c', + password: 'secret', + now: () => t, + fetch: async (url) => { + const path = String(url).replace('https://central.resi.io', '').split('?')[0]; + if (path === '/api/v3/login') { + logins++; + if (limited) return reply({}, { ok: false, status: 429 }); + return reply({}, { headers: { get: () => null, getSetCookie: () => ['livingas1=T; Path=/'] } }); + } + if (path === '/api_v2.svc/users/me') return reply({ customerId: 'CUST' }); + if (path === '/api/v3/customers/CUST/venues') return reply([{ uuid: 'V1' }]); + return reply({}, { ok: false, status: 404 }); + }, + }); + const poll = async () => { try { await client.venues(); return 'ok'; } catch (e) { return e.message; } }; + + assert.match(await poll(), /rate-limiting/); + for (let i = 0; i < 60; i++) { t += 1000; await poll(); } // a minute of 1s polls + assert.ok(logins <= 3, `signed in ${logins} times in a minute while rate-limited`); + assert.match(await poll(), /retrying in \d+s/, 'the wait is shown, not hidden'); + + limited = false; + t += 5 * 60000; // past the longest hold-off + assert.strictEqual(await poll(), 'ok', 'recovers once Resi lets it in'); + const after = logins; + for (let i = 0; i < 30; i++) { t += 1000; await poll(); } + assert.strictEqual(logins, after, 'a good session is not signed in again'); + }); + + await test('a rejected password says so', async () => { + const { createResiClient } = require('./resi'); + const client = createResiClient({ + username: 'a@b.c', password: 'wrong', now: () => 0, + fetch: async () => reply({}, { ok: false, status: 400 }), + }); + await assert.rejects(client.venues(), /check the password/); + }); + globalThis.setInterval = realSetInterval; globalThis.clearInterval = realClearInterval; console.log(`\n${passed} tests passed`);