From 455322a7a446f3388722d8a6036bbf3500e09c2e Mon Sep 17 00:00:00 2001 From: Waters Church Date: Thu, 15 Oct 2026 16:29:29 -0400 Subject: [PATCH] Keep the monday import going when an author or a board fails The importer only creates accounts for active monday users. An update or reply written by a deactivated staff member or an outside guest looked up an account that didn't exist, bound undefined to SQLite, and stopped the whole import ("Provided value cannot be bound to SQLite parameter 4"). Those posts now import with the author's name and no linked account. A board that fails partway is now deleted and the import moves on, so the next run retries it instead of skipping a half-imported board. A rejected token or persistent rate limiting still stops the run. The import page says "finished with errors" when some boards made it. Co-Authored-By: Claude Opus 5.5 --- server/public/js/pages.js | 2 +- server/src/importMonday.js | 40 +++++++++++++++++++++++++++++--------- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/server/public/js/pages.js b/server/public/js/pages.js index c94f6a1..092bdb4 100644 --- a/server/public/js/pages.js +++ b/server/public/js/pages.js @@ -263,7 +263,7 @@ function MondayImport() { ` : null} ${status && (status.running || status.log.length) ? html`
-
${status.running ? 'Importing…' : status.error ? 'Import failed' : 'Import finished'}${status.running ? html`<${Spinner} />` : null} +
${status.running ? 'Importing…' : status.error ? (status.counts.boards ? 'Import finished with errors' : 'Import failed') : 'Import finished'}${status.running ? html`<${Spinner} />` : null} ${status.counts.boards || 0} boards · ${status.counts.items || 0} items · ${status.counts.updates || 0} updates · ${status.counts.files || 0} files
${status.log.slice(-200).join('\n')}
` : null}
`; diff --git a/server/src/importMonday.js b/server/src/importMonday.js index e57b4f4..0472f28 100644 --- a/server/src/importMonday.js +++ b/server/src/importMonday.js @@ -16,6 +16,8 @@ const UPLOADS = path.join(DATA_DIR, 'uploads'); const status = { running: false, log: [], startedAt: null, finishedAt: null, error: null, counts: {} }; const log = (msg) => { status.log.push(`${new Date().toLocaleTimeString()} ${msg}`); if (status.log.length > 500) status.log.shift(); console.log('[import]', msg); }; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +// Errors that would fail every remaining board the same way, so the import stops instead of moving on. +const fatal = (msg) => Object.assign(new Error(msg), { fatal: true }); async function gql(token, query, variables = {}, attempt = 0) { const res = await fetch(API, { @@ -26,13 +28,13 @@ async function gql(token, query, variables = {}, attempt = 0) { const body = await res.json().catch(() => ({})); const errs = body.errors || (body.error_message ? [{ message: body.error_message }] : null); if (res.status === 429 || (errs && /complexity|rate limit|budget/i.test(JSON.stringify(errs)))) { - if (attempt > 8) throw new Error('monday.com kept rate-limiting the import — try again later'); + if (attempt > 8) throw fatal('monday.com kept rate-limiting the import — try again later'); const wait = Math.min(60, 5 * (attempt + 1)); log(`Rate limited by monday.com, waiting ${wait}s…`); await sleep(wait * 1000); return gql(token, query, variables, attempt + 1); } - if (res.status === 401 || res.status === 403) throw new Error('monday.com rejected the API token'); + if (res.status === 401 || res.status === 403) throw fatal('monday.com rejected the API token'); if (errs) throw new Error(errs.map((e) => e.message).join('; ')); return body.data; } @@ -60,6 +62,11 @@ function htmlToText(html) { const parseJSON = (s) => { try { return s ? JSON.parse(s) : null; } catch { return null; } }; +// Only active monday users get an account here, so deactivated staff and outside guests have none. +// Their updates keep the author's name with no linked account. Must return null, never undefined: +// node:sqlite refuses to bind undefined and the whole import fails. +const userIdByEmail = (email) => (email ? get('SELECT id FROM users WHERE email = ?', String(email).toLowerCase())?.id ?? null : null); + // monday column type → { type, settings, convert(columnValue) } function mapColumn(col, ctx) { const st = parseJSON(col.settings_str) || {}; @@ -226,17 +233,31 @@ async function importAll(token, opts = {}) { const wanted = opts.boardIds?.length ? boards.filter((b) => opts.boardIds.includes(String(b.id))) : boards; log(`${wanted.length} boards to import`); + const failed = []; for (const [bi, b] of wanted.entries()) { if (get('SELECT 1 FROM boards WHERE external_id = ?', 'monday:' + b.id)) { log(`Skipping "${b.name}" (already imported)`); continue; } log(`(${bi + 1}/${wanted.length}) ${b.name}`); - await importBoard(token, b, ctx, { - workspaceId: b.workspace_id ? wsMap[b.workspace_id] || mainWs() : mainWs(), - folderId: b.board_folder_id ? folderMap[b.board_folder_id] || null : null, - position: bi + 1, - }); + const countsBefore = { ...status.counts }; + try { + await importBoard(token, b, ctx, { + workspaceId: b.workspace_id ? wsMap[b.workspace_id] || mainWs() : mainWs(), + folderId: b.board_folder_id ? folderMap[b.board_folder_id] || null : null, + position: bi + 1, + }); + } catch (e) { + // Remove the half-imported board so the next run retries it instead of skipping it as + // already imported. Deleting a board cascades to its columns, groups, items and updates. + run('DELETE FROM boards WHERE external_id = ?', 'monday:' + b.id); + status.counts = countsBefore; + if (e.fatal) throw e; + failed.push(b.name); + log(` "${b.name}" failed and was removed, so the next import retries it: ${e.message}`); + continue; + } status.counts.boards = (status.counts.boards || 0) + 1; M.emitTree(null); } + if (failed.length) status.error = `${failed.length} board${failed.length === 1 ? '' : 's'} failed: ${failed.join(', ')}`; // Tidy: drop imported folders left empty (e.g. only held boards you chose not to import). for (let changed = true; changed;) { @@ -249,6 +270,7 @@ async function importAll(token, opts = {}) { } setSetting('mondayImport', { at: now(), counts: status.counts }); log(`Done — ${status.counts.boards || 0} boards, ${status.counts.items || 0} items, ${status.counts.updates || 0} updates, ${status.counts.files || 0} files`); + if (status.error) log(`${status.error}. Run the import again to retry.`); } catch (e) { status.error = e.message; log(`FAILED: ${e.message}`); @@ -367,13 +389,13 @@ async function importItem(it, { bid, groupMap, colInfo, ctx, position }) { // Updates and replies, oldest thread first is irrelevant — they're sorted by date on display. for (const u of it.updates || []) { const uid = id(); - const author = u.creator?.email ? get('SELECT id FROM users WHERE email = ?', u.creator.email.toLowerCase())?.id : null; + const author = userIdByEmail(u.creator?.email); run('INSERT INTO updates(id, item_id, user_id, author_name, body, created_at) VALUES(?,?,?,?,?,?)', uid, iid, author, u.creator?.name || null, htmlToText(u.body), Date.parse(u.created_at) || now()); status.counts.updates = (status.counts.updates || 0) + 1; for (const a of u.assets || []) { used.add(String(a.id)); await downloadAsset(a, iid, null, uid, author); } for (const rep of u.replies || []) { - const ra = rep.creator?.email ? get('SELECT id FROM users WHERE email = ?', rep.creator.email.toLowerCase())?.id : null; + const ra = userIdByEmail(rep.creator?.email); run('INSERT INTO updates(id, item_id, parent_id, user_id, author_name, body, created_at) VALUES(?,?,?,?,?,?,?)', id(), iid, uid, ra, rep.creator?.name || null, htmlToText(rep.body), Date.parse(rep.created_at) || now()); }