companion-module-digico-s31/tools/probe-catalog.js

#!/usr/bin/env node
'use strict';

/**
 * probe-catalog.js - build a catalog of what a DiGiCo S31 answers over its iPad
 * OSC extension, using READ-ONLY queries only.
 *
 * SAFETY (live console):
 *   - Every message sent is an OSC address ending in "/?" with NO arguments.
 *     assertSafeQuery() refuses anything else, and the encoded packet is decoded
 *     again and checked to carry zero arguments before it is sent.
 *   - Lowercase S-Series addresses (/console/...) are refused.
 *   - One socket, sends at least 60 ms apart (INTERVAL_MS = 65), no retry bursts.
 *   - Write-only action nodes from the 2014 list (Save session, Fire snapshot,
 *     Macro press) are NOT queried unless --include-actions is passed.
 *   - Every HEARTBEAT_EVERY queries a /Console/Name/? heartbeat is sent; three
 *     missed heartbeats in a row stop the run (results so far are still written).
 *
 * Network path: queries go to the Companion relay UDP 10.3.11.152:8000, which
 * forwards to the console; replies are fanned out to this machine on UDP 9000.
 * The console (and so the relay) is shared with other controllers: everything it
 * sends to them arrives here too (meters, other clients' polling, ...). Such
 * traffic is recorded as "unsolicited". Because it can coincide with a query's
 * 400 ms window, addresses that also show up as unsolicited traffic are
 * "contested": they are queried again in a verification pass and only count as
 * answered when that pass answers them.
 *
 * Phases: connectivity check -> extras + 2014 list (probe) -> live name tables
 * -> verification of contested addresses -> discovery (addresses the console
 * was seen sending to other controllers, queried twice) -> 15 s listen.
 *
 * Usage:
 *   node tools/probe-catalog.js [--dry-run] [--include-actions] [--test-local]
 *                               [--raw-log FILE] [--tail-seconds 15]
 *
 * --test-local sends to a mock-s31.js on 127.0.0.1:8000 instead (tool testing only).
 *
 * Outputs (relative to this file): catalog/s31-catalog.json, catalog/s31-catalog.md
 */

const dgram = require('node:dgram');
const fs = require('node:fs');
const path = require('node:path');
const { encode, decode } = require('./osc-lite');

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

const argv = process.argv.slice(2);
const flag = (name) => argv.includes(name);
const opt = (name, dflt) => {
  const i = argv.indexOf(name);
  return i >= 0 && argv[i + 1] !== undefined ? argv[i + 1] : dflt;
};
const TEST_LOCAL = flag('--test-local');
const RELAY_HOST = TEST_LOCAL ? '127.0.0.1' : '10.3.11.152';
const RELAY_PORT = 8000;
const LISTEN_PORT = 9000;
const DRY_RUN = flag('--dry-run');
const INCLUDE_ACTIONS = flag('--include-actions');
const RAW_LOG = opt('--raw-log', null);
const TAIL_SECONDS = Number(opt('--tail-seconds', '15'));

const INTERVAL_MS = 65; // spec: >= 60 ms between sends
const WINDOW_MS = 400; // a reply must arrive within this long after its query
const CONNECT_TIMEOUT_MS = 2000;
const HEARTBEAT_EVERY = 120;
const HEARTBEAT_MAX_MISSES = 3;
const DISCOVERY_GAP_MS = 2300; // between the two discovery rounds
const MAX_DISCOVERY = 40;
const MAX_STORED_REPLIES = 300;

const HERE = __dirname;
const LIST_FILE = path.join(HERE, 'sd-osc-list-2014.txt');
const OUT_DIR = path.join(HERE, 'catalog');

/** Max index per entity on this console, and which instances to probe. */
const ENTITIES = {
  Input_Channels: { max: 60, probe: [1, 2] },
  Aux_Outputs: { max: 25, probe: [1, 2] },
  Group_Outputs: { max: 25, probe: [1, 2] },
  Matrix_Outputs: { max: 8, probe: [1, 2] },
  Matrix_Inputs: { max: 10, probe: [1, 2] },
  Control_Groups: { max: 10, probe: [1, 2] },
  Graphic_EQ: { max: 16, probe: [1, 2] },
  Multis: { max: 0, probe: [1] },
};

/** Sub-index probes for the second wildcard after these segments. */
const SUB_PROBE = { Aux_Send: [1, 24], Group_Send: [1, 24], Matrix_Send: [1] };

/** Write-only trigger nodes in the 2014 list: skipped unless --include-actions. */
const ACTION_NODES = new Set([
  '/Filing/Save_current_Session',
  '/Snapshots/Fire_Snapshot_number',
  '/Snapshots/Fire_Prev_Snapshot',
  '/Snapshots/Fire_Next_Snapshot',
  '/Macros/Buttons/press',
]);

/** Candidate name addresses per entity, in preference order ('#' = index). */
const NAME_CANDIDATES = {
  Input_Channels: ['/Input_Channels/#/Channel_Input/name'],
  Aux_Outputs: ['/Aux_Outputs/#/Buss_Trim/name'],
  Group_Outputs: ['/Group_Outputs/#/Buss_Trim/name'],
  Matrix_Outputs: ['/Matrix_Outputs/#/Buss_Trim/name', '/Matrix_Outputs/#/name'],
  Matrix_Inputs: ['/Matrix_Inputs/#/Buss_Trim/name', '/Matrix_Inputs/#/name'],
  Control_Groups: ['/Control_Groups/#/name', '/Control_Groups/#/Buss_Trim/name'],
  Graphic_EQ: ['/Graphic_EQ/#/name', '/Graphic_EQ/#/Buss_Trim/name'],
};

/** Extra candidates not in the 2014 list (base addresses; "/?" is appended). */
const EXTRA_ADDRESSES = [
  '/Console/Name', '/Console/Channels', '/Console/Input_Channels', '/Console/Aux_Outputs',
  '/Console/Group_Outputs', '/Console/Matrix_Outputs', '/Console/Matrix_Inputs',
  '/Console/Control_Groups', '/Console/Graphic_EQ', '/Console/Multis', '/Console/Talkback_Outputs',
  '/Console/Aux_Outputs/modes', '/Console/Group_Outputs/modes', '/Console/Matrix_Outputs/modes',
  '/Console/Version', '/Console/Session', '/Console/Session/name', '/Console/Sample_Rate',
  '/Console/Meters', '/Console/Meter', '/Console/Metering', '/Console/Solo', '/Console/Solo_Busses',
  '/Console/Talkback', '/Console/Oscillator', '/Console/FX',
  '/Snapshots/Current_Snapshot', '/Snapshots/names', '/Snapshots/count',
  '/Snapshots/Total',
  '/Macros/Buttons', '/Macros/names', '/Macros/count', '/Macros/1/name', '/Macros/Buttons/1/name',
  '/Macros/Buttons/state',
  '/Talkback_Outputs/1/name', '/Solo/1/level', '/Solo_Busses/1/fader',
  '/Input_Channels/1/meter', '/Input_Channels/1/Meter', '/Input_Channels/1/meters',
  '/Input_Channels/1/Channel_Input/meter', '/Input_Channels/1/Channel_Input/input_meter',
  '/Input_Channels/1/Channel_Output/meter', '/Input_Channels/1/Channel_Input/signal',
  '/Input_Channels/1/Channel_Input/source', '/Input_Channels/1/Channel_Input/route',
  '/Input_Channels/1/Channel_Input/input_route', '/Input_Channels/1/Channel_Input/socket',
  '/Input_Channels/1/Channel_Input/mode', '/Input_Channels/1/Channel_Input/stereo',
  '/Input_Channels/1/stereo', '/Input_Channels/1/colour', '/Input_Channels/1/color',
  '/Input_Channels/1/Channel_Input/colour', '/Input_Channels/1/Aux_Send/24/send_pre-post',
  '/Input_Channels/1/Group_Send/24/group', '/Input_Channels/1/Direct_Output/send_level',
  '/Input_Channels/1/Direct_Output/send_on', '/Input_Channels/1/Channel_Output/direct_out',
  '/Aux_Outputs/1/Buss_Trim/name', '/Aux_Outputs/1/Buss_Trim/colour', '/Aux_Outputs/1/mode',
  '/Aux_Outputs/1/stereo', '/Aux_Outputs/1/meter',
  '/Group_Outputs/1/Buss_Trim/name', '/Matrix_Outputs/1/Buss_Trim/name', '/Matrix_Outputs/1/name',
  '/Matrix_Inputs/1/name', '/Matrix_Inputs/1/Buss_Trim/name',
  '/Control_Groups/1/name', '/Control_Groups/1/Buss_Trim/name', '/Control_Groups/1/fader',
  '/Control_Groups/1/mute', '/Control_Groups/1/solo',
  '/Graphic_EQ/1/name', '/Graphic_EQ/1/Buss_Trim/name', '/Graphic_EQ/1/on',
  '/Graphic_EQ/1/Bands/1/gain', '/Graphic_EQ/1/gain/1',
  '/FX/1/name', '/Multis/1/name',
  '/Talkback/level', '/Talkback/on', '/Oscillator/on', '/Oscillator/level', '/Oscillator/freq',
  // Deliberately last: /Snapshots/names answers on /Snapshots/name, so the two queries must not share a reply window.
  '/Snapshots/name',
];

/**
 * Addresses seen in other controllers' traffic during an earlier run of this tool
 * (so not in the lists above). They join the discovery pass: each is queried twice.
 */
const PRE_DISCOVERED = [
  '/Console/Input_Channels/modes',
  '/Input_Channels/1/Channel_Input/stereo_mode', '/Input_Channels/2/Channel_Input/stereo_mode', '/Input_Channels/19/Channel_Input/stereo_mode',
  '/Input_Channels/1/Insert/send_sockets', '/Input_Channels/2/Insert/send_sockets', '/Input_Channels/19/Insert/send_sockets',
  '/Input_Channels/1/Aux_Send/0/send_pan', '/Input_Channels/2/Aux_Send/0/send_pan',
];

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

const T0 = process.hrtime.bigint();
const now = () => Number(process.hrtime.bigint() - T0) / 1e6;
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

/** Replace the first all-digit path segment with N and the second with M. */
function patternOf(address) {
  let count = 0;
  return address
    .split('/')
    .map((s) => (/^\d+$/.test(s) ? (count++ === 0 ? 'N' : 'M') : s))
    .join('/');
}

/** Numeric path segments of an address as { n, m } (null when absent). */
function instanceOf(address) {
  const nums = address.split('/').filter((s) => /^\d+$/.test(s)).map(Number);
  return { n: nums.length > 0 ? nums[0] : null, m: nums.length > 1 ? nums[1] : null };
}

const treeOf = (address) => address.split('/')[1] || '';
const round6 = (v) => (typeof v === 'number' && Number.isFinite(v) ? Number(v.toPrecision(6)) : v);
/** Alias key: the console drops/changes underscores and hyphens in some leaf names (eq_thresh_1 -> eq_thresh1). */
const normAddr = (a) => a.toLowerCase().replace(/[_-]/g, '');
const median = (arr) => {
  if (!arr.length) return null;
  const s = [...arr].sort((x, y) => x - y);
  return s[Math.floor(s.length / 2)];
};
const percentile = (arr, p) => {
  if (!arr.length) return null;
  const s = [...arr].sort((x, y) => x - y);
  return s[Math.min(s.length - 1, Math.floor(s.length * p))];
};

/** Refuse anything that is not a plain read-only query. Throws on violation. */
function assertSafeQuery(address) {
  if (typeof address !== 'string') throw new Error('query address must be a string');
  if (!address.endsWith('/?')) throw new Error(`refusing non-query address (no trailing /?): ${address}`);
  if (!/^\/[A-Z][A-Za-z0-9_\-/]*\/\?$/.test(address)) {
    throw new Error(`refusing address (must start with /<Capital>, only [A-Za-z0-9_-/]): ${address}`);
  }
  if (address.includes('//') || address.includes('*')) throw new Error(`refusing malformed address: ${address}`);
  if (address.startsWith('/console/')) throw new Error(`refusing S-Series address: ${address}`);
  const base = address.slice(0, -2);
  if (ACTION_NODES.has(base) && !INCLUDE_ACTIONS) throw new Error(`refusing action node: ${address}`);
  // The encoded packet must decode back to the same address with zero arguments.
  const buf = encode(address, []);
  const back = decode(buf);
  if (back.address !== address || !Array.isArray(back.args) || back.args.length !== 0 || back.types !== '') {
    throw new Error(`encoded packet is not an argument-less query: ${address}`);
  }
  return buf;
}

// ---------------------------------------------------------------------------
// Parse the 2014 command list
// ---------------------------------------------------------------------------

function parseDocList(text) {
  const rows = [];
  const skipped = [];
  for (const raw of text.split(/\r?\n/)) {
    const line = raw.trim();
    if (!line.startsWith('/sd/')) continue;
    // Example lines carry arguments ("/sd/Input_Channels/1/mute, i,1 - ..."): not table rows.
    if (/^\/sd\/\S*,\s/.test(line)) {
      skipped.push(line);
      continue;
    }
    const tokens = line.split(/\s+/);
    const ti = tokens.findIndex((t) => /^(Int|Float|String)$/i.test(t));
    if (ti < 0) {
      skipped.push(line);
      continue;
    }
    const addr = tokens.slice(0, ti).join('').replace(/\/+$/, '').replace(/^\/sd/, '');
    const type = tokens[ti][0].toUpperCase() + tokens[ti].slice(1).toLowerCase();
    const min = tokens[ti + 1] !== undefined ? Number(tokens[ti + 1]) : null;
    const max = tokens[ti + 2] !== undefined ? Number(tokens[ti + 2]) : null;
    rows.push({
      template: addr,
      documentedType: type,
      documentedRange: min !== null && max !== null ? { min, max } : null,
    });
  }
  return { rows, skipped };
}

/** Expand "*" wildcards of one row into concrete addresses (without "/?"). */
function expandRow(row) {
  const segs = row.template.split('/'); // ['', 'Input_Channels', '*', ...]
  const entity = segs[1];
  const star = segs.reduce((acc, s, i) => (s === '*' ? [...acc, i] : acc), []);
  if (star.length === 0) return [row.template];
  const entIdx = ENTITIES[entity] ? ENTITIES[entity].probe : [1];
  const out = [];
  for (const n of entIdx) {
    const segsN = segs.slice();
    segsN[star[0]] = String(n);
    if (star.length > 1) {
      const subs = SUB_PROBE[segs[star[1] - 1]] || [1];
      for (const m of subs) {
        const segsM = segsN.slice();
        segsM[star[1]] = String(m);
        out.push(segsM.join('/'));
      }
    } else {
      out.push(segsN.join('/'));
    }
  }
  return out;
}

// ---------------------------------------------------------------------------
// Build the query list
// ---------------------------------------------------------------------------

function newEntry(base, source) {
  return {
    address: base,
    query: `${base}/?`,
    pattern: patternOf(base),
    tree: treeOf(base),
    inst: instanceOf(base),
    sources: [source],
    documentedType: null,
    documentedRange: null,
    skipped: false,
    sends: [], // every query record sent for this address
  };
}

function buildEntries() {
  const entries = new Map(); // base address -> entry, in send order
  const actionSkipped = [];
  const ensure = (base, source) => {
    let e = entries.get(base);
    if (!e) {
      e = newEntry(base, source);
      entries.set(base, e);
    } else if (!e.sources.includes(source)) {
      e.sources.push(source);
    }
    return e;
  };
  for (const base of EXTRA_ADDRESSES) ensure(base, 'extra');

  const { rows, skipped } = parseDocList(fs.readFileSync(LIST_FILE, 'utf8'));
  for (const row of rows) {
    for (const base of expandRow(row)) {
      const e = ensure(base, 'doc');
      e.documentedType = row.documentedType;
      e.documentedRange = row.documentedRange;
      if (ACTION_NODES.has(base) && !INCLUDE_ACTIONS) {
        e.skipped = true;
        e.skipReason = 'write-only action node in the 2014 list (trigger); not queried by default';
        actionSkipped.push(base);
      }
    }
  }
  return { entries, rows, skippedLines: skipped, actionSkipped };
}

// ---------------------------------------------------------------------------
// Probe run
// ---------------------------------------------------------------------------

async function main() {
  const { entries, rows, skippedLines, actionSkipped } = buildEntries();
  const toSend = [...entries.values()].filter((e) => !e.skipped);
  for (const e of toSend) assertSafeQuery(e.query); // throws before any packet leaves
  assertSafeQuery('/Console/Name/?');

  console.log(
    `list: ${rows.length} doc rows (${skippedLines.length} example lines ignored), ` +
      `${new Set(rows.map((r) => patternOf(r.template))).size} distinct patterns, ${EXTRA_ADDRESSES.length} extra candidates`
  );
  console.log(`planned phase-1 queries: ${toSend.length} (+ names, verification, discovery, heartbeats)`);
  if (actionSkipped.length) console.log(`skipped action nodes: ${[...new Set(actionSkipped)].join(', ')}`);
  if (DRY_RUN) {
    const perTree = {};
    for (const e of toSend) perTree[e.tree] = (perTree[e.tree] || 0) + 1;
    console.log('queries per tree:', JSON.stringify(perTree));
    console.log('dry run only; nothing sent');
    return;
  }

  const sock = dgram.createSocket({ type: 'udp4', reuseAddr: false });
  const sentLog = []; // every query record sent, in order
  const packets = []; // every packet received
  const unsolicited = [];
  const occurrences = new Map(); // normalised address -> times of unsolicited arrivals
  const sendErrors = [];
  const parseFailures = [];
  const sourcesSeen = new Map();
  let lastSend = -1e9;
  let minGap = Infinity;
  let sentCount = 0;
  let currentPhase = 'connect';

  sock.on('error', (err) => {
    console.error(`socket error: ${err.message}`);
    process.exit(3);
  });

  sock.on('message', (msg, rinfo) => {
    const t = now();
    const key = `${rinfo.address}:${rinfo.port}`;
    sourcesSeen.set(key, (sourcesSeen.get(key) || 0) + 1);
    const d = decode(msg);
    const pkt = { tMs: round6(t), from: key, bytes: msg.length, phase: currentPhase };
    if (d.raw !== undefined) {
      pkt.raw = d.raw;
      packets.push(pkt);
      parseFailures.push(pkt);
      unsolicited.push({ ...pkt, reason: 'unparsed' });
      return;
    }
    pkt.address = d.address;
    pkt.types = d.types;
    pkt.args = d.args.map(round6);
    pkt.unpadded = d.unpadded;

    // Match against queries sent inside the window, newest first: exact address,
    // then alias (underscores/hyphens/case ignored), then plural ("names" -> "name").
    let match = null;
    let kind = null;
    const dn = normAddr(d.address);
    const tryKind = (name, test) => {
      for (let i = sentLog.length - 1; i >= 0; i--) {
        const q = sentLog[i];
        if (t - q.sentAt > WINDOW_MS) break;
        if (test(q)) {
          match = q;
          kind = name;
          return true;
        }
      }
      return false;
    };
    tryKind('exact', (q) => q.entry.address === d.address) ||
      tryKind('alias', (q) => normAddr(q.entry.address) === dn) ||
      tryKind('plural', (q) => q.entry.address === `${d.address}s`);

    if (match) {
      pkt.matched = match.query;
      pkt.matchKind = kind;
      pkt.pass = match.pass;
      pkt.latencyMs = round6(t - match.sentAt);
      match.replied = true;
      if (match.replies.length < MAX_STORED_REPLIES) {
        match.replies.push({ address: d.address, types: d.types, args: pkt.args, latencyMs: pkt.latencyMs, match: kind, unpadded: d.unpadded });
      }
      if (match.firstLatencyMs === null) match.firstLatencyMs = pkt.latencyMs;
    } else {
      // Unsolicited: note how long after the newest same-address query it arrived.
      let after = null;
      for (let i = sentLog.length - 1; i >= 0; i--) {
        if (normAddr(sentLog[i].entry.address) === dn) {
          after = round6(t - sentLog[i].sentAt);
          break;
        }
      }
      unsolicited.push({ ...pkt, reason: after === null ? 'no-query-for-address' : 'outside-window', afterQueryMs: after });
      if (!occurrences.has(dn)) occurrences.set(dn, []);
      occurrences.get(dn).push(t);
    }
    packets.push(pkt);
  });

  await new Promise((resolve, reject) => {
    sock.once('error', reject);
    sock.bind({ port: LISTEN_PORT, address: '0.0.0.0', exclusive: true }, () => {
      sock.removeListener('error', reject);
      resolve();
    });
  });
  console.log(`listening on udp4 0.0.0.0:${LISTEN_PORT}; queries go to ${RELAY_HOST}:${RELAY_PORT}`);

  /** Send one query (paced) and register it. */
  async function send(entry, pass, aligned) {
    const buf = assertSafeQuery(entry.query);
    const wait = lastSend + INTERVAL_MS - now();
    if (wait > 0) await sleep(wait);
    const t = now();
    if (sentCount > 0) minGap = Math.min(minGap, t - lastSend);
    lastSend = t;
    const rec = { query: entry.query, entry, pass, aligned: !!aligned, sentAt: t, replied: false, replies: [], firstLatencyMs: null };
    sentLog.push(rec);
    entry.sends.push(rec);
    sentCount++;
    sock.send(buf, RELAY_PORT, RELAY_HOST, (err) => {
      if (err) sendErrors.push({ query: entry.query, error: err.message });
    });
    return rec;
  }

  // ---- connectivity check ---------------------------------------------------
  const nameEntry = entries.get('/Console/Name');
  const connRec = await send(nameEntry, 'connectivity');
  const connDeadline = connRec.sentAt + CONNECT_TIMEOUT_MS;
  while (now() < connDeadline && !connRec.replied) await sleep(10);
  const lateConn = unsolicited.find((u) => u.address === '/Console/Name' && u.tMs >= connRec.sentAt);
  if (!connRec.replied && !lateConn) {
    console.error('CONNECTIVITY CHECK FAILED: no /Console/Name reply within 2 s from the relay. Stopping (nothing else sent).');
    console.error(`packets received during the wait: ${packets.length}`);
    sock.close();
    process.exitCode = 2;
    return;
  }
  const connReply = connRec.replies[0] || lateConn;
  const connName = connReply.args[0];
  console.log(`connectivity OK: /Console/Name -> ${JSON.stringify(connName)} (types ${connReply.types})`);

  // ---- helpers for the later phases ------------------------------------------
  const heartbeatEntry = newEntry('/Console/Name', 'heartbeat');
  const heartbeats = [];
  let misses = 0;
  let aborted = null;
  let sinceHeartbeat = 0;

  async function maybeHeartbeat() {
    if (sinceHeartbeat < HEARTBEAT_EVERY) return true;
    sinceHeartbeat = 0;
    const rec = await send(heartbeatEntry, 'heartbeat');
    await sleep(WINDOW_MS + 20);
    heartbeats.push({ tMs: round6(rec.sentAt), ok: rec.replied, latencyMs: rec.firstLatencyMs });
    misses = rec.replied ? 0 : misses + 1;
    if (!rec.replied) console.log(`  heartbeat MISSED at ${Math.round(rec.sentAt)} ms (${misses}/${HEARTBEAT_MAX_MISSES})`);
    if (misses >= HEARTBEAT_MAX_MISSES) {
      aborted = `console stopped answering the heartbeat ${misses} times in a row at ${Math.round(rec.sentAt)} ms`;
      return false;
    }
    return true;
  }

  /** ms to wait so a query lands ~150 ms after the next predicted push of a periodic unsolicited address, else null. */
  function alignDelay(address) {
    const ts = occurrences.get(normAddr(address)) || [];
    if (ts.length < 4) return null;
    const gaps = ts.slice(1).map((t, i) => t - ts[i]).filter((g) => g > 300);
    if (gaps.length < 3) return null;
    const med = median(gaps);
    const mad = median(gaps.map((g) => Math.abs(g - med)));
    if (med < 1000 || mad > 0.15 * med) return null;
    const last = ts[ts.length - 1];
    const k = Math.ceil((now() + 50 - last - 150) / med);
    return last + k * med + 150 - now();
  }

  /** Send a batch: plain ones back to back, periodic ones phase-aligned and one at a time. */
  async function sendBatch(list, pass) {
    const plain = [];
    const aligned = [];
    for (const e of list) (alignDelay(e.address) === null ? plain : aligned).push(e);
    for (const e of plain) await send(e, pass);
    for (const e of aligned) {
      const d = alignDelay(e.address);
      if (d !== null && d > 0) await sleep(d);
      await send(e, pass, true);
      await sleep(WINDOW_MS + 20);
    }
    await sleep(WINDOW_MS + 20);
  }

  // ---- phase 1: extras + 2014 list --------------------------------------------
  currentPhase = 'probe';
  const startProbe = now();
  const phase1 = toSend.filter((e) => e !== nameEntry);
  let done = 0;
  for (const e of phase1) {
    if (!(await maybeHeartbeat())) break;
    await send(e, 'probe');
    sinceHeartbeat++;
    done++;
    if (done % 100 === 0) {
      const answered = phase1.slice(0, done).filter((x) => x.sends[0] && x.sends[0].replied).length;
      console.log(`  ${done}/${phase1.length} sent, ${answered} answered so far (${Math.round((now() - startProbe) / 1000)} s)`);
    }
  }

  // ---- phase 2: live name tables ------------------------------------------------
  const nameTables = {};
  const nameEntries = {};
  if (!aborted) {
    await sleep(WINDOW_MS + 20);
    currentPhase = 'names';
    const nameQueries = [];
    for (const [entity, cands] of Object.entries(NAME_CANDIDATES)) {
      let chosen = null;
      for (const c of cands) {
        const probes = ENTITIES[entity].probe.map((n) => entries.get(c.replace('#', String(n)))).filter(Boolean);
        if (probes.some((p) => p.sends[0] && p.sends[0].replied)) {
          chosen = c;
          break;
        }
      }
      nameTables[entity] = { addressPattern: chosen ? patternOf(chosen.replace('#', '1')) : null, max: ENTITIES[entity].max, values: {}, unanswered: [] };
      nameEntries[entity] = [];
      if (!chosen) continue;
      for (let n = 1; n <= ENTITIES[entity].max; n++) {
        const base = chosen.replace('#', String(n));
        let e = entries.get(base);
        if (!e) {
          e = newEntry(base, 'names');
          const doc = [...entries.values()].find((x) => x.pattern === e.pattern && x.documentedType);
          if (doc) {
            e.documentedType = doc.documentedType;
            e.documentedRange = doc.documentedRange;
          }
          entries.set(base, e);
          nameQueries.push(e);
        } else if (!e.sources.includes('names')) {
          e.sources.push('names');
        }
        nameEntries[entity].push(e);
      }
    }
    console.log(`names phase: ${nameQueries.length} new queries`);
    for (const e of nameQueries) {
      if (!(await maybeHeartbeat())) break;
      await send(e, 'names');
      sinceHeartbeat++;
    }
    await sleep(WINDOW_MS + 100);
  }

  // ---- phase 3: verify contested addresses -----------------------------------------
  const firstReplied = (e) => !!(e.sends[0] && e.sends[0].replied);
  if (!aborted) {
    currentPhase = 'verify';
    const trio = ['/Console/Input_Channels', '/Console/Aux_Outputs', '/Console/Group_Outputs'];
    const contested = [...entries.values()].filter(
      (e) => e.sends.length > 0 && !e.skipped && ((firstReplied(e) && occurrences.has(normAddr(e.address))) || trio.includes(e.address))
    );
    for (const e of contested) e.verifyReason = occurrences.has(normAddr(e.address)) ? 'seen in other controllers\' traffic' : 'listed for verification';
    console.log(`verification pass: ${contested.length} contested addresses`);
    await sendBatch(contested, 'verify');
  }

  // ---- phase 4: discovery of addresses seen in other controllers' traffic ------------
  const discovered = [];
  const streams = [];
  if (!aborted) {
    currentPhase = 'discover';
    const known = new Set([...entries.keys()].map(normAddr));
    const seen = new Map(); // exact address -> { count, times }
    for (const u of unsolicited) {
      if (!u.address) continue;
      const s = seen.get(u.address) || { count: 0, times: [] };
      s.count++;
      s.times.push(u.tMs);
      seen.set(u.address, s);
    }
    const perPattern = new Map();
    for (const [addr, s] of seen) {
      if (known.has(normAddr(addr))) continue;
      const gaps = s.times.slice(1).map((t, i) => t - s.times[i]);
      if (s.count >= 40 && median(gaps) < 500) {
        streams.push({ address: addr, count: s.count, medianPeriodMs: round6(median(gaps)) });
        continue; // a fast stream cannot be verified by timing
      }
      const pat = patternOf(addr);
      const list = perPattern.get(pat) || [];
      if (list.length < 2) {
        list.push(addr);
        perPattern.set(pat, list);
      }
    }
    // New patterns first (not probed above), then other indexes of patterns we already probed.
    const probedPatterns = new Set([...entries.values()].map((e) => e.pattern));
    const ordered = [...perPattern.entries()].sort((a, b) => Number(probedPatterns.has(a[0])) - Number(probedPatterns.has(b[0])));
    for (const [, list] of ordered) {
      for (const addr of list) {
        if (discovered.length >= MAX_DISCOVERY) break;
        try {
          assertSafeQuery(`${addr}/?`);
        } catch {
          continue;
        }
        const e = newEntry(addr, 'discovered');
        entries.set(addr, e);
        discovered.push(e);
      }
    }
    for (const addr of PRE_DISCOVERED) {
      if (known.has(normAddr(addr)) || discovered.some((d) => d.address === addr)) continue;
      assertSafeQuery(`${addr}/?`);
      const e = newEntry(addr, 'discovered');
      e.sources.push('seen-in-earlier-run');
      entries.set(addr, e);
      discovered.push(e);
    }
    console.log(`discovery: ${discovered.length} addresses (each queried twice, ${DISCOVERY_GAP_MS} ms apart); streams not queried: ${streams.map((s) => s.address).join(', ') || 'none'}`);
    if (discovered.length) {
      await sendBatch(discovered, 'discover');
      await sleep(DISCOVERY_GAP_MS);
      await sendBatch(discovered, 'discover');
    }
  }

  // ---- tail: listen for unsolicited traffic -----------------------------------------
  await sleep(WINDOW_MS + 100);
  const probeEnd = now();
  currentPhase = 'tail';
  const tailStart = now();
  console.log(`probing finished at ${Math.round(probeEnd / 1000)} s; listening ${TAIL_SECONDS} s for unsolicited traffic`);
  await sleep(TAIL_SECONDS * 1000);
  const end = now();
  sock.close();

  // ---- finalize name tables ---------------------------------------------------------
  for (const [entity, tbl] of Object.entries(nameTables)) {
    for (const e of nameEntries[entity] || []) {
      const a = answerOf(e);
      const r = a.answered && a.rec && (a.rec.replies.find((x) => x.match === 'exact') || a.rec.replies[0]);
      if (r && r.args.length > 0) tbl.values[e.inst.n] = r.args[0];
      else tbl.unanswered.push(e.inst.n);
    }
    tbl.answered = Object.keys(tbl.values).length;
  }

  // ---- build catalog -------------------------------------------------------------------
  const unsolicitedKeys = new Set(unsolicited.filter((u) => u.address).map((u) => normAddr(u.address)));
  const catalog = [];
  for (const e of entries.values()) {
    const a = answerOf(e);
    const rec = a.rec;
    const first = rec ? rec.replies.find((r) => r.match === 'exact') || rec.replies[0] || null : null;
    catalog.push({
      address: e.address,
      query: e.query,
      pattern: e.pattern,
      tree: e.tree,
      instance: e.inst,
      sources: e.sources,
      documentedType: e.documentedType,
      documentedRange: e.documentedRange,
      queried: !e.skipped && e.sends.some((s) => s.pass !== 'heartbeat'),
      skipped: e.skipped || undefined,
      skipReason: e.skipReason,
      answered: a.answered,
      replyAddress: first ? first.address : null,
      replyTypes: first ? first.types : null,
      exampleValue: first ? (first.args.length === 1 ? first.args[0] : first.args) : null,
      replyCount: rec ? rec.replies.length : 0,
      replyMatch: first ? first.match : null,
      latencyMs: rec ? rec.firstLatencyMs : null,
      contested: a.contested || undefined,
      verified: a.verified === null ? undefined : a.verified,
      note: a.note || undefined,
      unsolicited: unsolicitedKeys.has(normAddr(e.address)),
      replies: rec && (rec.replies.length > 1 || (first && first.address !== e.address)) ? rec.replies : undefined,
    });
  }
  const queriedKeys = new Set(catalog.map((c) => normAddr(c.address)));
  for (const u of unsolicited) {
    if (!u.address || queriedKeys.has(normAddr(u.address))) continue;
    queriedKeys.add(normAddr(u.address));
    catalog.push({
      address: u.address,
      query: null,
      pattern: patternOf(u.address),
      tree: treeOf(u.address),
      instance: instanceOf(u.address),
      sources: ['unsolicited'],
      documentedType: null,
      documentedRange: null,
      queried: false,
      answered: false,
      replyAddress: u.address,
      replyTypes: u.types,
      exampleValue: u.args.length === 1 ? u.args[0] : u.args,
      unsolicited: true,
    });
  }

  // ---- summary --------------------------------------------------------------------------
  const perTree = {};
  for (const c of catalog) {
    if (!c.queried && !c.skipped) continue;
    const t = (perTree[c.tree] = perTree[c.tree] || { queried: 0, answered: 0, unanswered: 0, skipped: 0 });
    if (c.skipped) t.skipped++;
    else {
      t.queried++;
      if (c.answered) t.answered++;
      else t.unanswered++;
    }
  }
  const probeLat = catalog.filter((c) => c.answered && !c.contested && c.latencyMs !== null).map((c) => c.latencyMs);
  const summary = {
    relay: `${RELAY_HOST}:${RELAY_PORT}`,
    listen: `udp4 0.0.0.0:${LISTEN_PORT}`,
    consoleName: connName,
    startedAt: new Date(Date.now() - end).toISOString(),
    durationSeconds: round6(end / 1000),
    probeSeconds: round6((probeEnd - startProbe) / 1000),
    tailSeconds: round6((end - tailStart) / 1000),
    intervalMs: INTERVAL_MS,
    minObservedSendGapMs: round6(minGap),
    windowMs: WINDOW_MS,
    queriesSent: sentCount,
    distinctAddressesQueried: catalog.filter((c) => c.queried).length,
    addressesAnswered: catalog.filter((c) => c.answered).length,
    addressesUnanswered: catalog.filter((c) => c.queried && !c.answered).length,
    contestedAddresses: catalog.filter((c) => c.contested).length,
    contestedConfirmed: catalog.filter((c) => c.contested && c.answered).length,
    patternsQueried: new Set(catalog.filter((c) => c.queried).map((c) => c.pattern)).size,
    patternsAnswered: new Set(catalog.filter((c) => c.answered).map((c) => c.pattern)).size,
    packetsReceived: packets.length,
    packetsMatched: packets.filter((p) => p.matched).length,
    packetsUnsolicited: unsolicited.length,
    unsolicitedDuringProbe: unsolicited.filter((u) => u.phase !== 'tail').length,
    unsolicitedInTail: unsolicited.filter((u) => u.phase === 'tail').length,
    unparsedPackets: parseFailures.length,
    unpaddedPackets: packets.filter((p) => p.unpadded).length,
    replyLatencyMs: {
      n: probeLat.length,
      min: round6(Math.min(...probeLat)),
      median: round6(median(probeLat)),
      p95: round6(percentile(probeLat, 0.95)),
      max: round6(Math.max(...probeLat)),
    },
    replySources: Object.fromEntries(sourcesSeen),
    heartbeats: { total: heartbeats.length, missed: heartbeats.filter((h) => !h.ok).length, detail: heartbeats },
    aborted,
    sendErrors,
    skippedActionNodes: [...new Set(actionSkipped)],
    exampleLinesIgnored: skippedLines.length,
    streamsNotQueried: streams,
    perTree,
  };

  const out = { summary, names: nameTables, catalog, unsolicited };
  fs.mkdirSync(OUT_DIR, { recursive: true });
  const jsonPath = path.join(OUT_DIR, 's31-catalog.json');
  const oneLine = (list) => list.map((x) => '    ' + JSON.stringify(x)).join(',\n');
  const jsonText =
    '{\n' +
    `  "summary": ${JSON.stringify(out.summary, null, 2).replace(/\n/g, '\n  ')},\n` +
    `  "names": ${JSON.stringify(out.names, null, 2).replace(/\n/g, '\n  ')},\n` +
    `  "catalog": [\n${oneLine(out.catalog)}\n  ],\n` +
    `  "unsolicited": [\n${oneLine(out.unsolicited)}\n  ]\n}\n`;
  JSON.parse(jsonText); // sanity: must be valid JSON
  fs.writeFileSync(jsonPath, jsonText);
  fs.writeFileSync(path.join(OUT_DIR, 's31-catalog.md'), renderMarkdown(out));
  if (RAW_LOG) fs.writeFileSync(RAW_LOG, packets.map((p) => JSON.stringify(p)).join('\n') + '\n');

  console.log(
    `done in ${Math.round(end / 1000)} s: ${sentCount} queries, ${summary.addressesAnswered}/${summary.distinctAddressesQueried} addresses answered, ` +
      `${packets.length} packets (${unsolicited.length} unsolicited), min send gap ${summary.minObservedSendGapMs} ms`
  );
  if (aborted) console.log(`ABORTED EARLY: ${aborted}`);
  console.log(`wrote ${jsonPath}`);
}

/**
 * Final verdict for one entry.
 *  - not contested: answered when its first query was answered.
 *  - contested (its first answer could belong to another controller): answered only
 *    when a verification query was answered.
 *  - discovered: answered only when both discovery queries were answered.
 */
function answerOf(e) {
  const verifyRecs = e.sends.filter((s) => s.pass === 'verify');
  const discoverRecs = e.sends.filter((s) => s.pass === 'discover');
  const firstRec = e.sends.find((s) => s.pass !== 'verify' && s.pass !== 'heartbeat') || null;
  if (discoverRecs.length) {
    const replied = discoverRecs.filter((s) => s.replied).length;
    const answered = replied === discoverRecs.length;
    let note;
    if (replied === 0) note = "discovered in other controllers' traffic; not answered to our queries";
    else if (answered) note = "discovered in other controllers' traffic; answered both queries";
    else note = `discovered in other controllers' traffic; inconsistent (${replied}/${discoverRecs.length} queries answered)`;
    return { answered, rec: discoverRecs.find((s) => s.replied) || discoverRecs[0], contested: true, verified: answered, note };
  }
  if (verifyRecs.length) {
    const ok = verifyRecs.some((s) => s.replied);
    const first = !!(firstRec && firstRec.replied);
    let note;
    const why = e.verifyReason || "seen in other controllers' traffic";
    if (ok && first) note = `${why}; answered in the first pass and again in verification`;
    else if (ok) note = `${why}; no reply in the first pass, answered in verification`;
    else if (first) note = `${why}; first-pass reply not reproduced in verification: it came from other traffic (first-pass false positive)`;
    else note = `${why}; no reply in either pass`;
    return { answered: ok, rec: verifyRecs.find((s) => s.replied) || firstRec, contested: true, verified: ok, note };
  }
  return { answered: !!(firstRec && firstRec.replied), rec: firstRec, contested: false, verified: null, note: null };
}

// ---------------------------------------------------------------------------
// Markdown report
// ---------------------------------------------------------------------------

const EXPECT = { Int: 'i', Float: 'f', String: 's' };

function fmtValue(v) {
  if (Array.isArray(v)) return `[${v.map(fmtValue).join(', ')}]`;
  if (typeof v === 'string') return JSON.stringify(v);
  return String(v);
}
const esc = (s) => String(s).replace(/\|/g, '\\|');
const fmtRange = (r) => (r ? `${r.min}..${r.max}` : '');

/** [1,2,3,5] -> "1-3,5" */
function compressList(nums) {
  const a = [...new Set(nums)].sort((x, y) => x - y);
  const parts = [];
  for (let i = 0; i < a.length; ) {
    let j = i;
    while (j + 1 < a.length && a[j + 1] === a[j] + 1) j++;
    parts.push(j - i >= 2 ? `${a[i]}-${a[j]}` : a.slice(i, j + 1).join(','));
    i = j + 1;
  }
  return parts.join(',');
}

function renderMarkdown({ summary, names, catalog, unsolicited }) {
  const L = [];
  const queried = catalog.filter((c) => c.queried);
  const answered = queried.filter((c) => c.answered);

  const byPattern = new Map();
  for (const c of queried) {
    let g = byPattern.get(c.pattern);
    if (!g) {
      g = { pattern: c.pattern, tree: c.tree, documentedType: null, documentedRange: null, items: [] };
      byPattern.set(c.pattern, g);
    }
    if (c.documentedType) {
      g.documentedType = c.documentedType;
      g.documentedRange = c.documentedRange;
    }
    g.items.push(c);
  }
  const groups = [...byPattern.values()];
  const answeredGroups = groups.filter((g) => g.items.some((i) => i.answered));
  const unansweredGroups = groups.filter((g) => !g.items.some((i) => i.answered));

  L.push('# DiGiCo S31 OSC catalog (iPad extension, read-only probe)');
  L.push('');
  L.push(`Generated by \`tools/probe-catalog.js\` on ${summary.startedAt}. Console: ${fmtValue(summary.consoleName)}. Queries sent to the relay ${summary.relay}; replies on ${summary.listen}.`);
  L.push('');
  L.push(`- ${summary.queriesSent} queries sent (every message is \`<address>/?\` with no arguments) in ${summary.probeSeconds} s: interval ${summary.intervalMs} ms, smallest observed gap ${summary.minObservedSendGapMs} ms; then ${summary.tailSeconds} s of listening.`);
  L.push(`- ${summary.addressesAnswered} of ${summary.distinctAddressesQueried} distinct addresses answered (${summary.patternsAnswered} of ${summary.patternsQueried} patterns).`);
  L.push(`- ${summary.packetsReceived} packets received: ${summary.packetsMatched} matched to a query, ${summary.packetsUnsolicited} unsolicited (${summary.unsolicitedDuringProbe} while probing, ${summary.unsolicitedInTail} in the final listen). All came from ${Object.keys(summary.replySources).join(', ')}.`);
  if (summary.replyLatencyMs && summary.replyLatencyMs.n) {
    const l = summary.replyLatencyMs;
    L.push(`- Reply latency (uncontested answers): min ${l.min} ms, median ${l.median} ms, p95 ${l.p95} ms, max ${l.max} ms. ${l.max > 100 ? 'The console works through incoming queries in roughly 150 ms ticks, so most replies arrive in bursts at tick boundaries.' : ''}`);
  }
  L.push(`- Contested addresses (the same address also appeared in other controllers' traffic, or was discovered there): ${summary.contestedAddresses}; ${summary.contestedConfirmed} confirmed by a second query, ${summary.contestedAddresses - summary.contestedConfirmed} not.`);
  L.push(`- Heartbeats (/Console/Name/? every ${HEARTBEAT_EVERY} queries): ${summary.heartbeats.total} sent, ${summary.heartbeats.missed} missed.${summary.aborted ? ` **Run aborted early: ${summary.aborted}.**` : ''}`);
  if (summary.skippedActionNodes.length) {
    L.push(`- Not queried (write-only trigger nodes in the 2014 list): ${summary.skippedActionNodes.map((a) => `\`${a}\``).join(', ')}.`);
  }
  L.push('');
  L.push('Pattern notation: N = first numeric index (channel / buss / entity), M = second numeric index (send buss, band). "Documented" is the 2014 DiGiCo SD OSC list type and OSC min/max. A reply address is the query address without `/?` unless a different one is shown after `@`.');
  L.push('');

  L.push('## Counts per tree');
  L.push('');
  L.push('| Tree | Addresses queried | Answered | Unanswered | Skipped |');
  L.push('| --- | ---: | ---: | ---: | ---: |');
  for (const [tree, t] of Object.entries(summary.perTree)) {
    L.push(`| ${tree} | ${t.queried} | ${t.answered} | ${t.unanswered} | ${t.skipped || ''} |`);
  }
  L.push('');

  // ---- surprises ----
  L.push('## Surprising / noteworthy');
  L.push('');
  const surprises = [];
  const matrix = {};
  const mismatchByKey = {};
  for (const c of answered) {
    if (!c.documentedType || !c.replyTypes) continue;
    const key = `${c.documentedType} -> ${c.replyTypes}`;
    matrix[key] = (matrix[key] || 0) + 1;
    if (EXPECT[c.documentedType] !== c.replyTypes) (mismatchByKey[key] = mismatchByKey[key] || new Set()).add(c.pattern);
  }
  if (Object.keys(matrix).length) {
    surprises.push('Documented type -> type tag actually received (answered addresses that have a documented type):');
    for (const [k, n] of Object.entries(matrix).sort()) {
      const [doc, tag] = k.split(' -> ');
      surprises.push(`  - ${k}: ${n} addresses${EXPECT[doc] !== tag ? ' (MISMATCH)' : ''}`);
    }
  }
  for (const [key, set] of Object.entries(mismatchByKey)) {
    const list = [...set].sort();
    surprises.push(`Mismatch ${key}: ${list.length} pattern(s)${list.length <= 30 ? ': ' + list.map((p) => '`' + p + '`').join(', ') : ' (see the "type differs from doc" notes in the tables below; full list in the JSON)'}`);
  }
  const outOfRange = [];
  for (const g of answeredGroups) {
    if (!g.documentedRange) continue;
    const vals = g.items.filter((i) => i.answered && typeof i.exampleValue === 'number').map((i) => i.exampleValue);
    if (vals.some((v) => v < g.documentedRange.min || v > g.documentedRange.max)) outOfRange.push(`\`${g.pattern}\` (documented ${fmtRange(g.documentedRange)}, saw ${[...new Set(vals)].join(', ')})`);
  }
  if (outOfRange.length) surprises.push(`Values outside the documented OSC range (${outOfRange.length} patterns): ${outOfRange.slice(0, 30).join('; ')}${outOfRange.length > 30 ? '; ...' : ''}`);
  const partial = answeredGroups.filter((g) => g.items.some((i) => !i.answered)).map((g) => `\`${g.pattern}\` (${g.items.filter((i) => i.answered).length}/${g.items.length})`);
  if (partial.length) surprises.push(`Patterns answered for some probed instances but not all (${partial.length}): ${partial.slice(0, 60).join(', ')}${partial.length > 60 ? ', ...' : ''}`);
  const multi = answered.filter((c) => c.replyCount > 1);
  if (multi.length) surprises.push(`Queries that produced more than one reply message (${multi.length}): ${multi.slice(0, 20).map((c) => `\`${c.query}\` x${c.replyCount}`).join(', ')}`);
  const diffAddr = answered.filter((c) => c.replyAddress !== c.address);
  if (diffAddr.length) surprises.push(`Replies on a different address than the query minus "/?" (${diffAddr.length}): ${diffAddr.slice(0, 30).map((c) => `\`${c.query}\` -> \`${c.replyAddress}\``).join(', ')}${diffAddr.length > 30 ? ', ...' : ''}`);
  const rejected = catalog.filter((c) => c.contested && !c.answered && c.queried && /first-pass false positive/.test(c.note || ''));
  if (rejected.length) surprises.push(`First-pass "answers" rejected after verification (they were other traffic, not replies): ${rejected.map((c) => `\`${c.address}\``).join(', ')}`);
  const lateAnswered = catalog.filter((c) => c.contested && c.answered && /no reply in the first pass, answered in verification/.test(c.note || ''));
  if (lateAnswered.length) surprises.push(`No reply in the first pass but answered in verification: ${lateAnswered.map((c) => `\`${c.address}\``).join(', ')}`);
  const burstU = unsolicited.filter((u) => u.address && u.address.startsWith('/Console/') && u.types === 'i' && u.tMs < 2000);
  if (burstU.length + rejected.length >= 3) {
    const items = [
      ...rejected.map((c) => `\`${c.address.replace('/Console/', '')}\` ${c.exampleValue}`),
      ...burstU.map((u) => `\`${u.address.replace('/Console/', '')}\` ${u.args[0]}`),
    ];
    surprises.push(`Connect-time burst: about ${Math.round(Math.min(...burstU.map((u) => u.tMs)))} ms after our first packet the console sent ${items.length} /Console/<entity> count messages we had not asked for yet: ${items.join(', ')}. This is how /Console/Input_Channels, /Console/Aux_Outputs and /Console/Group_Outputs show up; they never answered an explicit query (see the notes above).`);
  }
  const inconsistent = catalog.filter((c) => /inconsistent/.test(c.note || ''));
  if (inconsistent.length) surprises.push(`Inconsistent discovery results (answered only some of the queries): ${inconsistent.map((c) => `\`${c.address}\``).join(', ')}`);
  if (summary.unpaddedPackets) surprises.push(`${summary.unpaddedPackets} received packets were not 4-byte padded OSC (osc-lite decoded them leniently).`);
  if (summary.unparsedPackets) surprises.push(`${summary.unparsedPackets} received packets could not be decoded (raw hex in the JSON "unsolicited" entries).`);
  if (summary.heartbeats.missed) surprises.push(`${summary.heartbeats.missed} of ${summary.heartbeats.total} heartbeats were missed: the console (or relay) went quiet during the run.`);
  if (surprises.length === 0) surprises.push('Nothing unexpected detected by the automatic checks.');
  for (const s of surprises) L.push(s.startsWith('  ') ? s : `- ${s}`);
  L.push('');

  // ---- answered tables ----
  L.push('## Answered addresses, by tree');
  L.push('');
  L.push('"(k/n)" after the instances means only k of n probed instances answered. "contested" means the address also appeared in other controllers\' traffic and was re-queried in a verification pass.');
  L.push('');
  const trees = [...new Set(answeredGroups.map((g) => g.tree))];
  for (const tree of trees) {
    const gs = answeredGroups.filter((g) => g.tree === tree);
    L.push(`### /${tree} (${gs.length} patterns answered)`);
    L.push('');
    L.push('| Pattern | Answered N / M | Reply tags | Example value(s) | Documented type, range | Notes |');
    L.push('| --- | --- | --- | --- | --- | --- |');
    for (const g of gs) {
      const ans = g.items.filter((i) => i.answered);
      const ns = ans.map((i) => i.instance.n).filter((v) => v !== null);
      const ms = ans.map((i) => i.instance.m).filter((v) => v !== null);
      const inst = [ns.length ? `N=${compressList(ns)}` : '', ms.length ? `M=${compressList(ms)}` : ''].filter(Boolean).join(' ') || '-';
      const partialMark = ans.length < g.items.length ? ` (${ans.length}/${g.items.length})` : '';
      const tags = [...new Set(ans.map((i) => (i.replyAddress !== i.address ? `${i.replyTypes} @${i.replyAddress.replace(/\/\d+/g, '/N')}` : i.replyTypes)))].join(', ');
      const ex = [...new Set(ans.map((i) => fmtValue(i.exampleValue)))].slice(0, 4).join(' / ');
      const doc = g.documentedType ? `${g.documentedType}${g.documentedRange ? ' ' + fmtRange(g.documentedRange) : ''}` : '(not in 2014 list)';
      const notes = [];
      if (ans.some((i) => i.contested && !i.sources.includes('discovered'))) notes.push('contested, verified');
      if (ans.some((i) => i.sources.includes('discovered'))) notes.push("discovered in other controllers' traffic, answered both queries");
      if (ans.some((i) => i.replyCount > 1)) notes.push(`multi-reply x${Math.max(...ans.map((i) => i.replyCount))}`);
      if (g.documentedType && ans.some((i) => EXPECT[g.documentedType] !== i.replyTypes)) notes.push('type differs from doc');
      L.push(`| \`${g.pattern}\` | ${inst}${partialMark} | ${esc(tags)} | ${esc(ex)} | ${doc} | ${notes.join('; ')} |`);
    }
    L.push('');
  }
  if (trees.length === 0) {
    L.push('_No address answered._');
    L.push('');
  }

  // ---- unanswered ----
  const docUnanswered = unansweredGroups.filter((g) => g.items.some((i) => i.sources.includes('doc')));
  const extraUnanswered = unansweredGroups.filter((g) => !g.items.some((i) => i.sources.includes('doc')) && !g.items.some((i) => i.sources.includes('discovered')));
  const discUnanswered = unansweredGroups.filter((g) => g.items.some((i) => i.sources.includes('discovered')));
  L.push(`## Unanswered documented patterns (${docUnanswered.length})`);
  L.push('');
  L.push('Patterns from the 2014 list for which none of the probed instances answered.');
  L.push('');
  for (const tree of [...new Set(docUnanswered.map((g) => g.tree))]) {
    const gs = docUnanswered.filter((g) => g.tree === tree);
    L.push(`### /${tree} (${gs.length})`);
    L.push('');
    for (const g of gs) L.push(`- \`${g.pattern}\``);
    L.push('');
  }
  L.push(`## Unanswered extra candidates (${extraUnanswered.length})`);
  L.push('');
  for (const g of extraUnanswered) L.push(`- \`${g.items.map((i) => i.address).join('`, `')}\``);
  L.push('');
  if (discUnanswered.length) {
    L.push(`## Discovered addresses that did not answer both queries (${discUnanswered.length})`);
    L.push('');
    for (const g of discUnanswered) L.push(`- \`${g.items.map((i) => i.address).join('`, `')}\` - ${g.items[0].note}`);
    L.push('');
  }
  if (summary.skippedActionNodes.length) {
    L.push('## Not queried');
    L.push('');
    for (const a of summary.skippedActionNodes) L.push(`- \`${a}\` (write-only trigger node; skipped for safety, pass --include-actions to query it)`);
    L.push('');
  }

  // ---- streams & unsolicited ----
  L.push(`## Unsolicited messages (${unsolicited.length})`);
  L.push('');
  L.push('Everything that arrived on UDP 9000 without matching one of our queries inside its 400 ms window. The console and relay are shared with other controllers, so this is mostly their traffic (queries from other clients, meters, pushed state). The full list with timestamps is in the JSON.');
  L.push('');
  const meters = unsolicited.filter((u) => u.address === '/Meters/values');
  if (meters.length) {
    const ids = new Map();
    for (const m of meters) {
      for (let i = 0; i + 1 < m.args.length; i += 2) {
        const s = ids.get(m.args[i]) || { n: 0, min: Infinity, max: -Infinity };
        s.n++;
        s.min = Math.min(s.min, m.args[i + 1]);
        s.max = Math.max(s.max, m.args[i + 1]);
        ids.set(m.args[i], s);
      }
    }
    const gaps = meters.slice(1).map((m, i) => m.tMs - meters[i].tMs);
    L.push('### /Meters/values (unsolicited stream)');
    L.push('');
    L.push(`- ${meters.length} messages; variable length (e.g. \`,${meters[0].types}\`), args are \`[id, value]\` int pairs; median period ${round6(median(gaps))} ms.`);
    L.push(`- ${ids.size} distinct meter ids seen: ${[...ids.keys()].sort((a, b) => a - b).join(', ')}.`);
    L.push(`- Value range per id: ${[...ids.entries()].sort((a, b) => a[0] - b[0]).map(([id, s]) => `${id}: ${s.min}..${s.max} (${s.n})`).join('; ')}.`);
    L.push(`- Example: \`${meters[0].address} ,${meters[0].types} ${fmtValue(meters[0].args)}\`. No query we sent produced this address; the console pushes it to its controller (here the relay), so it was not queried.`);
    L.push('');
  }
  const byPat = new Map();
  for (const u of unsolicited) {
    const p = u.address ? patternOf(u.address) : '(unparsed)';
    const g = byPat.get(p) || { pattern: p, n: 0, addrs: new Map(), types: new Set(), ex: u, inTail: 0 };
    g.n++;
    g.addrs.set(u.address, (g.addrs.get(u.address) || []).concat(u.tMs));
    if (u.types !== undefined) g.types.add(u.types.length > 12 ? u.types.slice(0, 12) + '...' : u.types);
    if (u.phase === 'tail') g.inTail++;
    byPat.set(p, g);
  }
  L.push('### Unsolicited traffic by pattern');
  L.push('');
  L.push('| Pattern | Messages | Distinct addresses | Tags | Typical period of the busiest address | In final listen | Example |');
  L.push('| --- | ---: | ---: | --- | --- | ---: | --- |');
  for (const g of [...byPat.values()].sort((a, b) => b.n - a.n)) {
    const busiest = [...g.addrs.entries()].sort((a, b) => b[1].length - a[1].length)[0];
    const ts = busiest[1];
    const gaps = ts.slice(1).map((t, i) => t - ts[i]);
    const period = ts.length >= 3 ? `${Math.round(median(gaps))} ms (x${ts.length})` : '';
    L.push(`| \`${g.pattern}\` | ${g.n} | ${g.addrs.size} | ${[...g.types].join(', ')} | ${period} | ${g.inTail} | \`${esc(g.ex.address || '')} ${esc(g.ex.args ? fmtValue(g.ex.args).slice(0, 60) : '')}\` |`);
  }
  L.push('');

  // ---- names ----
  L.push('## Live name tables');
  L.push('');
  for (const [entity, tbl] of Object.entries(names)) {
    L.push(`### ${entity}`);
    L.push('');
    if (!tbl.addressPattern) {
      L.push('_No name address answered for this entity (tried: ' + (NAME_CANDIDATES[entity] || []).map((c) => '`' + c.replace('#', 'N') + '`').join(', ') + ')._');
      L.push('');
      continue;
    }
    L.push(`Address: \`${tbl.addressPattern}\` (${tbl.answered} of ${tbl.max} answered${tbl.unanswered.length ? `, no reply for ${tbl.unanswered.join(', ')}` : ''}).`);
    const sample = catalog.find((c) => c.pattern === tbl.addressPattern && c.answered);
    if (sample) L.push(`Reply format: \`${sample.replyAddress} ,${sample.replyTypes} ${fmtValue(Array.isArray(sample.exampleValue) ? sample.exampleValue : [sample.exampleValue])}\`.`);
    L.push('');
    L.push('| # | Name |');
    L.push('| ---: | --- |');
    for (const [idx, nm] of Object.entries(tbl.values)) L.push(`| ${idx} | ${esc(fmtValue(nm))} |`);
    L.push('');
  }
  return L.join('\n') + '\n';
}

main().catch((err) => {
  console.error(`fatal: ${err.stack || err.message}`);
  process.exit(1);
});