#!/usr/bin/env node
'use strict';

/**
 * probe-catalog.js - build a catalog of what a DiGiCo S31 answers over its iPad
 * OSC extension, using READ-ONLY queries only.
 *
 * SAFETY (live console):
 *   - Every message sent is an OSC address ending in "/?" with NO arguments.
 *     assertSafeQuery() refuses anything else, and the encoded packet is decoded
 *     again and checked to carry zero arguments before it is sent.
 *   - Lowercase S-Series addresses (/console/...) are refused.
 *   - One socket, sends at least 60 ms apart (INTERVAL_MS = 65), no retries.
 *   - Write-only action nodes from the 2014 list (Save session, Fire snapshot,
 *     Macro press) are NOT queried unless --include-actions is passed.
 *   - Every HEARTBEAT_EVERY queries a /Console/Name/? heartbeat is sent; three
 *     missed heartbeats in a row stop the run (results so far are still written).
 *
 * Network path: queries go to the Companion relay UDP 10.3.11.152:8000, which
 * forwards to the console; replies are fanned out to this machine on UDP 9000.
 *
 * Usage:
 *   node tools/probe-catalog.js [--dry-run] [--include-actions] [--test-local]
 *                               [--raw-log FILE] [--tail-seconds 15]
 *
 * --test-local sends to a mock-s31.js on 127.0.0.1:8000 instead (tool testing only).
 *
 * Outputs (relative to this file): catalog/s31-catalog.json, catalog/s31-catalog.md
 */

const dgram = require('node:dgram');
const fs = require('node:fs');
const path = require('node:path');
const { encode, decode } = require('./osc-lite');

// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------

const INTERVAL_MS = 65; // spec: >= 60 ms between sends
const MIN_GAP_MS = 60;
const WINDOW_MS = 400; // reply must arrive within this long after its query
const CONNECT_TIMEOUT_MS = 2000;
const HEARTBEAT_EVERY = 120;
const HEARTBEAT_MAX_MISSES = 3;
const MAX_STORED_REPLIES = 300;

const HERE = __dirname;
const LIST_FILE = path.join(HERE, 'sd-osc-list-2014.txt');
const OUT_DIR = path.join(HERE, 'catalog');

const argv = process.argv.slice(2);
const flag = (name) => argv.includes(name);
const opt = (name, dflt) => {
  const i = argv.indexOf(name);
  return i >= 0 && argv[i + 1] !== undefined ? argv[i + 1] : dflt;
};
// --test-local points the probe at a mock-s31.js on 127.0.0.1 (for testing the tool only).
const TEST_LOCAL = flag('--test-local');
const RELAY_HOST = TEST_LOCAL ? '127.0.0.1' : '10.3.11.152';
const RELAY_PORT = 8000;
const LISTEN_PORT = 9000;
const DRY_RUN = flag('--dry-run');
const INCLUDE_ACTIONS = flag('--include-actions');
const RAW_LOG = opt('--raw-log', null);
const TAIL_SECONDS = Number(opt('--tail-seconds', '15'));

/** Max index per entity on this console, and which instances to probe. */
const ENTITIES = {
  Input_Channels: { max: 60, probe: [1, 2] },
  Aux_Outputs: { max: 25, probe: [1, 2] },
  Group_Outputs: { max: 25, probe: [1, 2] },
  Matrix_Outputs: { max: 8, probe: [1, 2] },
  Matrix_Inputs: { max: 10, probe: [1, 2] },
  Control_Groups: { max: 10, probe: [1, 2] },
  Graphic_EQ: { max: 16, probe: [1, 2] },
  Multis: { max: 0, probe: [1] },
};

/** Sub-index probes for the second wildcard after these segments. */
const SUB_PROBE = { Aux_Send: [1, 24], Group_Send: [1, 24], Matrix_Send: [1] };

/** Write-only trigger nodes in the 2014 list: skipped unless --include-actions. */
const ACTION_NODES = new Set([
  '/Filing/Save_current_Session',
  '/Snapshots/Fire_Snapshot_number',
  '/Snapshots/Fire_Prev_Snapshot',
  '/Snapshots/Fire_Next_Snapshot',
  '/Macros/Buttons/press',
]);

/** Candidate name addresses per entity, in preference order ('#' = index). */
const NAME_CANDIDATES = {
  Input_Channels: ['/Input_Channels/#/Channel_Input/name'],
  Aux_Outputs: ['/Aux_Outputs/#/Buss_Trim/name'],
  Group_Outputs: ['/Group_Outputs/#/Buss_Trim/name'],
  Matrix_Outputs: ['/Matrix_Outputs/#/Buss_Trim/name', '/Matrix_Outputs/#/name'],
  Matrix_Inputs: ['/Matrix_Inputs/#/Buss_Trim/name', '/Matrix_Inputs/#/name'],
  Control_Groups: ['/Control_Groups/#/name', '/Control_Groups/#/Buss_Trim/name'],
  Graphic_EQ: ['/Graphic_EQ/#/name', '/Graphic_EQ/#/Buss_Trim/name'],
};

/** Extra candidates not in the 2014 list (base addresses; "/?" is appended). */
const EXTRA_ADDRESSES = [
  '/Console/Name', '/Console/Channels', '/Console/Input_Channels', '/Console/Aux_Outputs',
  '/Console/Group_Outputs', '/Console/Matrix_Outputs', '/Console/Matrix_Inputs',
  '/Console/Control_Groups', '/Console/Graphic_EQ', '/Console/Multis', '/Console/Talkback_Outputs',
  '/Console/Aux_Outputs/modes', '/Console/Group_Outputs/modes', '/Console/Matrix_Outputs/modes',
  '/Console/Version', '/Console/Session', '/Console/Session/name', '/Console/Sample_Rate',
  '/Console/Meters', '/Console/Meter', '/Console/Metering', '/Console/Solo', '/Console/Solo_Busses',
  '/Console/Talkback', '/Console/Oscillator', '/Console/FX',
  '/Snapshots/Current_Snapshot', '/Snapshots/names', '/Snapshots/name', '/Snapshots/count',
  '/Snapshots/Total',
  '/Macros/Buttons', '/Macros/names', '/Macros/count', '/Macros/1/name', '/Macros/Buttons/1/name',
  '/Macros/Buttons/state',
  '/Talkback_Outputs/1/name', '/Solo/1/level', '/Solo_Busses/1/fader',
  '/Input_Channels/1/meter', '/Input_Channels/1/Meter', '/Input_Channels/1/meters',
  '/Input_Channels/1/Channel_Input/meter', '/Input_Channels/1/Channel_Input/input_meter',
  '/Input_Channels/1/Channel_Output/meter', '/Input_Channels/1/Channel_Input/signal',
  '/Input_Channels/1/Channel_Input/source', '/Input_Channels/1/Channel_Input/route',
  '/Input_Channels/1/Channel_Input/input_route', '/Input_Channels/1/Channel_Input/socket',
  '/Input_Channels/1/Channel_Input/mode', '/Input_Channels/1/Channel_Input/stereo',
  '/Input_Channels/1/stereo', '/Input_Channels/1/colour', '/Input_Channels/1/color',
  '/Input_Channels/1/Channel_Input/colour', '/Input_Channels/1/Aux_Send/24/send_pre-post',
  '/Input_Channels/1/Group_Send/24/group', '/Input_Channels/1/Direct_Output/send_level',
  '/Input_Channels/1/Direct_Output/send_on', '/Input_Channels/1/Channel_Output/direct_out',
  '/Aux_Outputs/1/Buss_Trim/name', '/Aux_Outputs/1/Buss_Trim/colour', '/Aux_Outputs/1/mode',
  '/Aux_Outputs/1/stereo', '/Aux_Outputs/1/meter',
  '/Group_Outputs/1/Buss_Trim/name', '/Matrix_Outputs/1/Buss_Trim/name', '/Matrix_Outputs/1/name',
  '/Matrix_Inputs/1/name', '/Matrix_Inputs/1/Buss_Trim/name',
  '/Control_Groups/1/name', '/Control_Groups/1/Buss_Trim/name', '/Control_Groups/1/fader',
  '/Control_Groups/1/mute', '/Control_Groups/1/solo',
  '/Graphic_EQ/1/name', '/Graphic_EQ/1/Buss_Trim/name', '/Graphic_EQ/1/on',
  '/Graphic_EQ/1/Bands/1/gain', '/Graphic_EQ/1/gain/1',
  '/FX/1/name', '/Multis/1/name',
  '/Talkback/level', '/Talkback/on', '/Oscillator/on', '/Oscillator/level', '/Oscillator/freq',
];

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

const T0 = process.hrtime.bigint();
const now = () => Number(process.hrtime.bigint() - T0) / 1e6;
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

/** Replace the first all-digit path segment with N and the second with M. */
function patternOf(address) {
  let count = 0;
  return address
    .split('/')
    .map((s) => (/^\d+$/.test(s) ? (count++ === 0 ? 'N' : 'M') : s))
    .join('/');
}

/** Numeric path segments of an address as { n, m } (null when absent). */
function instanceOf(address) {
  const nums = address.split('/').filter((s) => /^\d+$/.test(s)).map(Number);
  return { n: nums.length > 0 ? nums[0] : null, m: nums.length > 1 ? nums[1] : null };
}

const treeOf = (address) => address.split('/')[1] || '';

const round6 = (v) => (typeof v === 'number' && Number.isFinite(v) ? Number(v.toPrecision(6)) : v);

/** Refuse anything that is not a plain read-only query. Throws on violation. */
function assertSafeQuery(address) {
  if (typeof address !== 'string') throw new Error('query address must be a string');
  if (!address.endsWith('/?')) throw new Error(`refusing non-query address (no trailing /?): ${address}`);
  if (!/^\/[A-Z][A-Za-z0-9_\-/]*\/\?$/.test(address)) {
    throw new Error(`refusing address (must start with /<Capital>, only [A-Za-z0-9_-/]): ${address}`);
  }
  if (address.includes('//') || address.includes('*')) throw new Error(`refusing malformed address: ${address}`);
  if (address.startsWith('/console/')) throw new Error(`refusing S-Series address: ${address}`);
  const base = address.slice(0, -2);
  if (ACTION_NODES.has(base) && !INCLUDE_ACTIONS) throw new Error(`refusing action node: ${address}`);
  // The encoded packet must decode back to the same address with zero arguments.
  const buf = encode(address, []);
  const back = decode(buf);
  if (back.address !== address || !Array.isArray(back.args) || back.args.length !== 0 || back.types !== '') {
    throw new Error(`encoded packet is not an argument-less query: ${address}`);
  }
  return buf;
}

// ---------------------------------------------------------------------------
// Parse the 2014 command list
// ---------------------------------------------------------------------------

function parseDocList(text) {
  const rows = [];
  const skipped = [];
  for (const raw of text.split(/\r?\n/)) {
    const line = raw.trim();
    if (!line.startsWith('/sd/')) continue;
    // Example lines carry arguments ("/sd/Input_Channels/1/mute, i,1 - ..."): not table rows.
    if (/^\/sd\/\S*,\s/.test(line)) {
      skipped.push(line);
      continue;
    }
    const tokens = line.split(/\s+/);
    const ti = tokens.findIndex((t) => /^(Int|Float|String)$/i.test(t));
    if (ti < 0) {
      skipped.push(line);
      continue;
    }
    const docPath = tokens.slice(0, ti).join('').replace(/\/+$/, ''); // "/sd/Entity/*/Sub/param"
    const addr = docPath.replace(/^\/sd/, '');
    const type = tokens[ti][0].toUpperCase() + tokens[ti].slice(1).toLowerCase();
    const min = tokens[ti + 1] !== undefined ? Number(tokens[ti + 1]) : null;
    const max = tokens[ti + 2] !== undefined ? Number(tokens[ti + 2]) : null;
    rows.push({
      template: addr,
      documentedType: type,
      documentedRange: min !== null && max !== null ? { min, max } : null,
    });
  }
  return { rows, skipped };
}

/** Expand "*" wildcards of one row into concrete addresses (without "/?"). */
function expandRow(row) {
  const segs = row.template.split('/'); // ['', 'Input_Channels', '*', ...]
  const entity = segs[1];
  const star = segs.reduce((acc, s, i) => (s === '*' ? [...acc, i] : acc), []);
  if (star.length === 0) return [row.template];
  const ent = ENTITIES[entity];
  const entIdx = ent ? ent.probe : [1];
  let out = [];
  for (const n of entIdx) {
    const segsN = segs.slice();
    segsN[star[0]] = String(n);
    if (star.length > 1) {
      const subName = segs[star[1] - 1];
      const subs = SUB_PROBE[subName] || [1];
      for (const m of subs) {
        const segsM = segsN.slice();
        segsM[star[1]] = String(m);
        out.push(segsM.join('/'));
      }
    } else {
      out.push(segsN.join('/'));
    }
  }
  return out;
}

// ---------------------------------------------------------------------------
// Build the query list
// ---------------------------------------------------------------------------

/** entries: Map base-address -> entry (insertion order = send order). */
function buildEntries() {
  const entries = new Map();
  const sourceCounts = { extra: 0, doc: 0, actionSkipped: [] };

  const ensure = (base, source) => {
    let e = entries.get(base);
    if (!e) {
      e = {
        address: base,
        query: `${base}/?`,
        pattern: patternOf(base),
        tree: treeOf(base),
        inst: instanceOf(base),
        sources: [],
        documentedType: null,
        documentedRange: null,
        skipped: false,
      };
      entries.set(base, e);
    }
    if (!e.sources.includes(source)) e.sources.push(source);
    return e;
  };

  for (const base of EXTRA_ADDRESSES) {
    ensure(base, 'extra');
    sourceCounts.extra++;
  }

  const text = fs.readFileSync(LIST_FILE, 'utf8');
  const { rows, skipped } = parseDocList(text);
  for (const row of rows) {
    for (const base of expandRow(row)) {
      const e = ensure(base, 'doc');
      e.documentedType = row.documentedType;
      e.documentedRange = row.documentedRange;
      if (ACTION_NODES.has(base) && !INCLUDE_ACTIONS) {
        e.skipped = true;
        e.skipReason = 'write-only action node in the 2014 list (trigger); not queried by default';
        sourceCounts.actionSkipped.push(base);
      }
      sourceCounts.doc++;
    }
  }
  return { entries, rows, skippedLines: skipped, sourceCounts };
}

// ---------------------------------------------------------------------------
// Probe run
// ---------------------------------------------------------------------------

async function main() {
  const { entries, rows, skippedLines, sourceCounts } = buildEntries();
  const toSend = [...entries.values()].filter((e) => !e.skipped);

  // Validate every query up front (throws before any packet leaves the machine).
  for (const e of toSend) assertSafeQuery(e.query);
  assertSafeQuery('/Console/Name/?');

  const docPatterns = new Set(rows.map((r) => patternOf(r.template)));
  console.log(
    `list: ${rows.length} doc rows (${skippedLines.length} example lines ignored), ` +
      `${docPatterns.size} distinct patterns, ${sourceCounts.extra} extra candidates`
  );
  console.log(`planned phase-1 queries: ${toSend.length} (+ names phase, + heartbeats)`);
  if (sourceCounts.actionSkipped.length) {
    console.log(`skipped action nodes: ${[...new Set(sourceCounts.actionSkipped)].join(', ')}`);
  }
  if (DRY_RUN) {
    const perTree = {};
    for (const e of toSend) perTree[e.tree] = (perTree[e.tree] || 0) + 1;
    console.log('queries per tree:', JSON.stringify(perTree));
    console.log('dry run only; nothing sent');
    return;
  }

  const sock = dgram.createSocket({ type: 'udp4', reuseAddr: false });
  const sentLog = []; // every query sent, in order
  const packets = []; // every packet received
  const unsolicited = [];
  const sendErrors = [];
  let lastSend = -1e9;
  let minGap = Infinity;
  let sentCount = 0;
  let currentPhase = 'connect';
  const sourcesSeen = new Map();

  const parseFailures = [];

  sock.on('error', (err) => {
    console.error(`socket error: ${err.message}`);
    process.exit(3);
  });

  sock.on('message', (msg, rinfo) => {
    const t = now();
    const key = `${rinfo.address}:${rinfo.port}`;
    sourcesSeen.set(key, (sourcesSeen.get(key) || 0) + 1);
    const d = decode(msg);
    const pkt = { tMs: round6(t), from: key, bytes: msg.length, phase: currentPhase };
    if (d.raw !== undefined) {
      pkt.raw = d.raw;
      packets.push(pkt);
      parseFailures.push(pkt);
      unsolicited.push({ ...pkt, reason: 'unparsed' });
      return;
    }
    pkt.address = d.address;
    pkt.types = d.types;
    pkt.args = d.args.map(round6);
    pkt.unpadded = d.unpadded;
    // Match: newest query with the same reply address inside the window.
    let match = null;
    let kind = null;
    for (let i = sentLog.length - 1; i >= 0; i--) {
      const q = sentLog[i];
      if (t - q.sentAt > WINDOW_MS) break;
      if (q.replyAddress === d.address) {
        match = q;
        kind = 'exact';
        break;
      }
    }
    if (!match) {
      for (let i = sentLog.length - 1; i >= 0; i--) {
        const q = sentLog[i];
        if (t - q.sentAt > WINDOW_MS) break;
        if (q.replyAddress.startsWith(d.address) || d.address.startsWith(q.replyAddress)) {
          match = q;
          kind = 'related';
          break;
        }
      }
    }
    if (match) {
      pkt.matched = match.query;
      pkt.matchKind = kind;
      pkt.latencyMs = round6(t - match.sentAt);
      const e = match.entry;
      if (e.replies.length < MAX_STORED_REPLIES) {
        e.replies.push({
          address: d.address,
          types: d.types,
          args: d.args.map(round6),
          latencyMs: pkt.latencyMs,
          match: kind,
          unpadded: d.unpadded,
        });
      }
      e.replyCount++;
      if (e.firstLatencyMs === null) e.firstLatencyMs = pkt.latencyMs;
      match.replied = true;
    } else {
      // Unsolicited: tag how long after the newest same-address query it arrived (late reply?).
      let after = null;
      for (let i = sentLog.length - 1; i >= 0; i--) {
        if (sentLog[i].replyAddress === d.address) {
          after = round6(t - sentLog[i].sentAt);
          break;
        }
      }
      unsolicited.push({ ...pkt, reason: after === null ? 'no-query-for-address' : 'outside-window', afterQueryMs: after });
    }
    packets.push(pkt);
  });

  await new Promise((resolve, reject) => {
    sock.once('error', reject);
    sock.bind({ port: LISTEN_PORT, address: '0.0.0.0', exclusive: true }, () => {
      sock.removeListener('error', reject);
      resolve();
    });
  });
  console.log(`listening on udp4 0.0.0.0:${LISTEN_PORT}; queries go to ${RELAY_HOST}:${RELAY_PORT}`);

  /** Send one query (paced) and register it. */
  async function send(entry, tag) {
    const buf = assertSafeQuery(entry.query);
    const wait = lastSend + INTERVAL_MS - now();
    if (wait > 0) await sleep(wait);
    const t = now();
    if (sentCount > 0) minGap = Math.min(minGap, t - lastSend);
    lastSend = t;
    const rec = {
      query: entry.query,
      replyAddress: entry.address,
      sentAt: t,
      entry,
      tag: tag || 'probe',
      replied: false,
    };
    sentLog.push(rec);
    entry.sentCount = (entry.sentCount || 0) + 1;
    sentCount++;
    sock.send(buf, RELAY_PORT, RELAY_HOST, (err) => {
      if (err) sendErrors.push({ query: entry.query, error: err.message });
    });
    return rec;
  }

  const newEntryState = (e) => {
    e.replies = [];
    e.replyCount = 0;
    e.firstLatencyMs = null;
    return e;
  };
  for (const e of entries.values()) newEntryState(e);

  // ---- connectivity check -------------------------------------------------
  const nameEntry = entries.get('/Console/Name');
  const connRec = await send(nameEntry, 'connectivity');
  const connDeadline = connRec.sentAt + CONNECT_TIMEOUT_MS;
  while (now() < connDeadline && !(nameEntry.replyCount > 0)) await sleep(10);
  // The match window is 400 ms; a reply between 400 ms and 2 s is still accepted for this check.
  const connUnsolicited = unsolicited.find((u) => u.address === '/Console/Name');
  const connectivityOk = nameEntry.replyCount > 0 || !!connUnsolicited;
  const connReply = nameEntry.replies[0] || connUnsolicited || null;
  if (!connectivityOk) {
    console.error('CONNECTIVITY CHECK FAILED: no /Console/Name reply within 2 s from the relay. Stopping (nothing else sent).');
    console.error(`packets received during the wait: ${packets.length}`);
    sock.close();
    process.exitCode = 2;
    return;
  }
  const connName = connReply && (connReply.args || [])[0];
  console.log(`connectivity OK: /Console/Name -> ${JSON.stringify(connName)} (types ${connReply && connReply.types})`);
  if (!nameEntry.replyCount && connUnsolicited) {
    // Late (>400 ms) but within 2 s: keep as an unsolicited packet and record that it was slow.
    console.log(`note: the first reply was slow (${connUnsolicited.afterQueryMs} ms), outside the 400 ms window`);
  }

  // ---- phase 1 + names ----------------------------------------------------
  const heartbeats = [];
  let misses = 0;
  let aborted = null;
  let sinceHeartbeat = 0;

  async function maybeHeartbeat() {
    if (sinceHeartbeat < HEARTBEAT_EVERY) return true;
    sinceHeartbeat = 0;
    const hbEntry = { address: '/Console/Name', query: '/Console/Name/?', replies: [], replyCount: 0, firstLatencyMs: null };
    const rec = await send(hbEntry, 'heartbeat');
    await sleep(WINDOW_MS + 20);
    const ok = rec.replied;
    heartbeats.push({ tMs: round6(rec.sentAt), ok, latencyMs: hbEntry.firstLatencyMs });
    misses = ok ? 0 : misses + 1;
    if (!ok) console.log(`  heartbeat MISSED at ${Math.round(rec.sentAt)} ms (${misses}/${HEARTBEAT_MAX_MISSES})`);
    if (misses >= HEARTBEAT_MAX_MISSES) {
      aborted = `console stopped answering the heartbeat ${misses} times in a row at ${Math.round(rec.sentAt)} ms`;
      return false;
    }
    return true;
  }

  currentPhase = 'probe';
  const startProbe = now();
  let done = 0;
  const phase1 = toSend.filter((e) => e !== nameEntry);
  for (const e of phase1) {
    if (!(await maybeHeartbeat())) break;
    await send(e);
    sinceHeartbeat++;
    done++;
    if (done % 100 === 0) {
      const answered = phase1.slice(0, done).filter((x) => x.replyCount > 0).length;
      console.log(`  ${done}/${phase1.length} sent, ${answered} answered so far (${Math.round((now() - startProbe) / 1000)} s)`);
    }
  }

  // ---- names phase ----------------------------------------------------------
  const nameTables = {};
  if (!aborted) {
    await sleep(WINDOW_MS + 20);
    currentPhase = 'names';
    const nameQueries = [];
    for (const [entity, cands] of Object.entries(NAME_CANDIDATES)) {
      let chosen = null;
      for (const c of cands) {
        const probes = ENTITIES[entity].probe.map((n) => entries.get(c.replace('#', String(n)))).filter(Boolean);
        if (probes.some((p) => p.replyCount > 0)) {
          chosen = c;
          break;
        }
      }
      nameTables[entity] = { addressPattern: chosen ? patternOf(chosen.replace('#', '1')) : null, max: ENTITIES[entity].max, values: {}, unanswered: [] };
      if (!chosen) continue;
      for (let n = 1; n <= ENTITIES[entity].max; n++) {
        const base = chosen.replace('#', String(n));
        let e = entries.get(base);
        if (!e) {
          e = newEntryState({
            address: base,
            query: `${base}/?`,
            pattern: patternOf(base),
            tree: treeOf(base),
            inst: instanceOf(base),
            sources: ['names'],
            documentedType: null,
            documentedRange: null,
            skipped: false,
          });
          for (const doc of [...entries.values()]) {
            if (doc.pattern === e.pattern && doc.documentedType) {
              e.documentedType = doc.documentedType;
              e.documentedRange = doc.documentedRange;
              break;
            }
          }
          entries.set(base, e);
          nameQueries.push(e);
        } else if (!e.sources.includes('names')) {
          e.sources.push('names');
        }
        nameTables[entity].entryFor = nameTables[entity].entryFor || [];
        nameTables[entity].entryFor.push(e);
      }
    }
    console.log(`names phase: ${nameQueries.length} new queries`);
    for (const e of nameQueries) {
      if (!(await maybeHeartbeat())) break;
      await send(e);
      sinceHeartbeat++;
    }
  }

  // Let the last window close, then listen for unsolicited traffic.
  await sleep(WINDOW_MS + 100);
  const probeEnd = now();
  currentPhase = 'tail';
  const tailStart = now();
  console.log(`probe finished at ${Math.round(probeEnd / 1000)} s; listening ${TAIL_SECONDS} s for unsolicited traffic`);
  await sleep(TAIL_SECONDS * 1000);
  const end = now();
  sock.close();

  // ---- finalize name tables ---------------------------------------------------
  for (const [entity, tbl] of Object.entries(nameTables)) {
    for (const e of tbl.entryFor || []) {
      const r = e.replies.find((x) => x.match === 'exact') || e.replies[0];
      if (r && r.args.length > 0) tbl.values[e.inst.n] = r.args[0];
      else tbl.unanswered.push(e.inst.n);
    }
    delete tbl.entryFor;
    tbl.answered = Object.keys(tbl.values).length;
  }

  // ---- build catalog ------------------------------------------------------------
  const unsolicitedAddresses = new Set(unsolicited.filter((u) => u.address).map((u) => u.address));
  const catalog = [];
  for (const e of entries.values()) {
    const first = e.replies.find((r) => r.match === 'exact') || e.replies[0] || null;
    catalog.push({
      address: e.address,
      query: e.query,
      pattern: e.pattern,
      tree: e.tree,
      instance: e.inst,
      sources: e.sources,
      documentedType: e.documentedType,
      documentedRange: e.documentedRange,
      queried: !e.skipped && (e.sentCount || 0) > 0,
      skipped: e.skipped || undefined,
      skipReason: e.skipReason,
      answered: e.replyCount > 0,
      replyAddress: first ? first.address : null,
      replyTypes: first ? first.types : null,
      exampleValue: first ? (first.args.length === 1 ? first.args[0] : first.args) : null,
      replyCount: e.replyCount,
      replyMatch: first ? first.match : null,
      latencyMs: e.firstLatencyMs,
      unsolicited: unsolicitedAddresses.has(e.address),
      replies: e.replyCount > 1 || (first && first.address !== e.address) ? e.replies : undefined,
    });
  }
  const queriedAddresses = new Set(catalog.map((c) => c.address));
  for (const addr of unsolicitedAddresses) {
    if (queriedAddresses.has(addr)) continue;
    const u = unsolicited.find((x) => x.address === addr);
    catalog.push({
      address: addr,
      query: null,
      pattern: patternOf(addr),
      tree: treeOf(addr),
      instance: instanceOf(addr),
      sources: ['unsolicited'],
      documentedType: null,
      documentedRange: null,
      queried: false,
      answered: false,
      replyAddress: addr,
      replyTypes: u.types,
      exampleValue: u.args.length === 1 ? u.args[0] : u.args,
      unsolicited: true,
    });
  }

  // ---- summary ----------------------------------------------------------------------
  const perTree = {};
  for (const c of catalog) {
    if (!c.queried && !c.skipped) continue;
    const t = (perTree[c.tree] = perTree[c.tree] || { queried: 0, answered: 0, unanswered: 0, skipped: 0 });
    if (c.skipped) t.skipped++;
    else {
      t.queried++;
      if (c.answered) t.answered++;
      else t.unanswered++;
    }
  }
  const answeredPatterns = new Set(catalog.filter((c) => c.answered).map((c) => c.pattern));
  const queriedPatterns = new Set(catalog.filter((c) => c.queried).map((c) => c.pattern));
  const summary = {
    relay: `${RELAY_HOST}:${RELAY_PORT}`,
    listen: `udp4 0.0.0.0:${LISTEN_PORT}`,
    consoleName: connName,
    startedAt: new Date(Date.now() - end).toISOString(),
    durationSeconds: round6(end / 1000),
    probeSeconds: round6((probeEnd - startProbe) / 1000),
    tailSeconds: round6((end - tailStart) / 1000),
    intervalMs: INTERVAL_MS,
    minObservedSendGapMs: round6(minGap),
    windowMs: WINDOW_MS,
    queriesSent: sentCount,
    distinctAddressesQueried: catalog.filter((c) => c.queried).length,
    addressesAnswered: catalog.filter((c) => c.answered).length,
    addressesUnanswered: catalog.filter((c) => c.queried && !c.answered).length,
    patternsQueried: queriedPatterns.size,
    patternsAnswered: answeredPatterns.size,
    packetsReceived: packets.length,
    packetsMatched: packets.filter((p) => p.matched).length,
    packetsUnsolicited: unsolicited.length,
    unsolicitedDuringProbe: unsolicited.filter((u) => u.phase !== 'tail').length,
    unsolicitedInTail: unsolicited.filter((u) => u.phase === 'tail').length,
    unparsedPackets: parseFailures.length,
    unpaddedPackets: packets.filter((p) => p.unpadded).length,
    replySources: Object.fromEntries(sourcesSeen),
    heartbeats: { total: heartbeats.length, missed: heartbeats.filter((h) => !h.ok).length, detail: heartbeats },
    aborted,
    sendErrors,
    skippedActionNodes: [...new Set(sourceCounts.actionSkipped)],
    exampleLinesIgnored: skippedLines.length,
    perTree,
  };

  const out = { summary, names: nameTables, catalog, unsolicited };
  fs.mkdirSync(OUT_DIR, { recursive: true });
  const jsonPath = path.join(OUT_DIR, 's31-catalog.json');
  const oneLine = (list) => list.map((x) => '    ' + JSON.stringify(x)).join(',\n');
  const jsonText =
    '{\n' +
    `  "summary": ${JSON.stringify(out.summary, null, 2).replace(/\n/g, '\n  ')},\n` +
    `  "names": ${JSON.stringify(out.names, null, 2).replace(/\n/g, '\n  ')},\n` +
    `  "catalog": [\n${oneLine(out.catalog)}\n  ],\n` +
    `  "unsolicited": [\n${oneLine(out.unsolicited)}\n  ]\n}\n`;
  JSON.parse(jsonText); // sanity: must be valid JSON
  fs.writeFileSync(jsonPath, jsonText);
  fs.writeFileSync(path.join(OUT_DIR, 's31-catalog.md'), renderMarkdown(out));
  if (RAW_LOG) fs.writeFileSync(RAW_LOG, packets.map((p) => JSON.stringify(p)).join('\n') + '\n');

  console.log(
    `done in ${Math.round(end / 1000)} s: ${sentCount} queries, ${summary.addressesAnswered}/${summary.distinctAddressesQueried} addresses answered, ` +
      `${packets.length} packets (${unsolicited.length} unsolicited), min send gap ${summary.minObservedSendGapMs} ms`
  );
  if (aborted) console.log(`ABORTED EARLY: ${aborted}`);
  console.log(`wrote ${jsonPath}`);
}

// ---------------------------------------------------------------------------
// Markdown report
// ---------------------------------------------------------------------------

const EXPECT = { Int: 'i', Float: 'f', String: 's' };

function fmtValue(v) {
  if (Array.isArray(v)) return `[${v.map(fmtValue).join(', ')}]`;
  if (typeof v === 'string') return JSON.stringify(v);
  return String(v);
}
const esc = (s) => String(s).replace(/\|/g, '\\|');

/** [1,2,3,5] -> "1-3,5" */
function compressList(nums) {
  const a = [...new Set(nums)].sort((x, y) => x - y);
  const parts = [];
  for (let i = 0; i < a.length; ) {
    let j = i;
    while (j + 1 < a.length && a[j + 1] === a[j] + 1) j++;
    parts.push(j - i >= 2 ? `${a[i]}-${a[j]}` : a.slice(i, j + 1).join(','));
    i = j + 1;
  }
  return parts.join(',');
}
const fmtRange = (r) => (r ? `${r.min}..${r.max}` : '');

function renderMarkdown({ summary, names, catalog, unsolicited }) {
  const L = [];
  const queried = catalog.filter((c) => c.queried);
  const answered = queried.filter((c) => c.answered);

  // Group by pattern.
  const byPattern = new Map();
  for (const c of queried) {
    let g = byPattern.get(c.pattern);
    if (!g) {
      g = { pattern: c.pattern, tree: c.tree, documentedType: null, documentedRange: null, items: [] };
      byPattern.set(c.pattern, g);
    }
    if (c.documentedType) {
      g.documentedType = c.documentedType;
      g.documentedRange = c.documentedRange;
    }
    g.items.push(c);
  }
  const groups = [...byPattern.values()];
  const answeredGroups = groups.filter((g) => g.items.some((i) => i.answered));
  const unansweredGroups = groups.filter((g) => !g.items.some((i) => i.answered));

  L.push('# DiGiCo S31 OSC catalog (iPad extension, read-only probe)');
  L.push('');
  L.push(`Generated by \`tools/probe-catalog.js\` on ${summary.startedAt}. Console: ${fmtValue(summary.consoleName)}. Queries sent to the relay ${summary.relay}; replies on ${summary.listen}.`);
  L.push('');
  L.push(`- ${summary.queriesSent} queries sent in ${summary.probeSeconds} s (interval ${summary.intervalMs} ms, minimum observed gap ${summary.minObservedSendGapMs} ms), then ${summary.tailSeconds} s of listening.`);
  L.push(`- ${summary.addressesAnswered} of ${summary.distinctAddressesQueried} distinct addresses answered (${summary.patternsAnswered} of ${summary.patternsQueried} patterns).`);
  L.push(`- ${summary.packetsReceived} packets received: ${summary.packetsMatched} matched to a query, ${summary.packetsUnsolicited} unsolicited (${summary.unsolicitedDuringProbe} during probing, ${summary.unsolicitedInTail} in the final listen).`);
  L.push(`- Heartbeats (/Console/Name/? every ${HEARTBEAT_EVERY} queries): ${summary.heartbeats.total} sent, ${summary.heartbeats.missed} missed.${summary.aborted ? ` **Run aborted early: ${summary.aborted}.**` : ''}`);
  if (summary.skippedActionNodes.length) {
    L.push(`- Not queried (write-only trigger nodes in the 2014 list): ${summary.skippedActionNodes.map((a) => `\`${a}\``).join(', ')}.`);
  }
  L.push('');
  L.push('Pattern notation: N = first numeric index (channel / buss / entity), M = second numeric index (send buss, band). "Documented" is the 2014 DiGiCo SD OSC list type and OSC min/max.');
  L.push('');

  // Per-tree counts.
  L.push('## Counts per tree');
  L.push('');
  L.push('| Tree | Addresses queried | Answered | Unanswered | Skipped |');
  L.push('| --- | ---: | ---: | ---: | ---: |');
  for (const [tree, t] of Object.entries(summary.perTree)) {
    L.push(`| ${tree} | ${t.queried} | ${t.answered} | ${t.unanswered} | ${t.skipped || ''} |`);
  }
  L.push('');

  // Surprises.
  L.push('## Surprising / noteworthy');
  L.push('');
  const surprises = [];
  const matrix = {};
  const mismatchByLeaf = {};
  for (const c of answered) {
    if (!c.documentedType || !c.replyTypes) continue;
    const key = `${c.documentedType} -> ${c.replyTypes || '(none)'}`;
    matrix[key] = (matrix[key] || 0) + 1;
    if (EXPECT[c.documentedType] !== c.replyTypes) {
      const leaf = c.pattern;
      (mismatchByLeaf[key] = mismatchByLeaf[key] || new Set()).add(leaf);
    }
  }
  if (Object.keys(matrix).length) {
    surprises.push('Documented type -> type tag actually received (answered addresses with a documented type):');
    for (const [k, n] of Object.entries(matrix).sort()) surprises.push(`  - ${k}: ${n} addresses${k.split(' -> ')[0] && EXPECT[k.split(' -> ')[0]] !== k.split(' -> ')[1] ? ' (MISMATCH)' : ''}`);
  }
  for (const [key, set] of Object.entries(mismatchByLeaf)) {
    const list = [...set].sort();
    surprises.push(`Mismatch ${key}: ${list.length} pattern(s)${list.length <= 40 ? ': ' + list.map((p) => '`' + p + '`').join(', ') : ' (see JSON)'}`);
  }
  // Observed numeric values outside the documented range.
  const outOfRange = [];
  for (const g of answeredGroups) {
    if (!g.documentedRange) continue;
    const vals = g.items.filter((i) => i.answered && typeof i.exampleValue === 'number').map((i) => i.exampleValue);
    if (vals.some((v) => v < g.documentedRange.min || v > g.documentedRange.max)) outOfRange.push(`\`${g.pattern}\` (documented ${fmtRange(g.documentedRange)}, saw ${[...new Set(vals)].join(', ')})`);
  }
  if (outOfRange.length) surprises.push(`Values outside the documented OSC range (${outOfRange.length}): ${outOfRange.slice(0, 40).join('; ')}${outOfRange.length > 40 ? '; ...' : ''}`);
  // Partial answers.
  const partial = answeredGroups.filter((g) => g.items.some((i) => !i.answered)).map((g) => `\`${g.pattern}\` (${g.items.filter((i) => i.answered).length}/${g.items.length})`);
  if (partial.length) surprises.push(`Patterns answered for some instances but not all probed instances (${partial.length}): ${partial.slice(0, 60).join(', ')}${partial.length > 60 ? ', ...' : ''}`);
  // Multi-reply / different reply address.
  const multi = answered.filter((c) => c.replyCount > 1);
  if (multi.length) surprises.push(`Queries that produced more than one reply message (${multi.length}): ${multi.slice(0, 20).map((c) => `\`${c.query}\` x${c.replyCount}`).join(', ')}`);
  const diffAddr = answered.filter((c) => c.replyAddress !== c.address);
  if (diffAddr.length) surprises.push(`Replies on a different address than the query (${diffAddr.length}): ${diffAddr.slice(0, 20).map((c) => `\`${c.query}\` -> \`${c.replyAddress}\``).join(', ')}`);
  if (summary.unpaddedPackets) surprises.push(`${summary.unpaddedPackets} received packets were not 4-byte padded OSC (osc-lite decoded them leniently).`);
  if (summary.unparsedPackets) surprises.push(`${summary.unparsedPackets} received packets could not be decoded (raw hex in JSON "unsolicited" entries).`);
  const lateish = unsolicited.filter((u) => u.reason === 'outside-window' && u.afterQueryMs !== null && u.afterQueryMs < 2000);
  if (lateish.length) surprises.push(`${lateish.length} unsolicited replies arrived 0.4-2 s after a query for the same address (possibly slow replies, not human input); see the unsolicited section.`);
  if (summary.heartbeats.missed) surprises.push(`${summary.heartbeats.missed} of ${summary.heartbeats.total} heartbeats were missed: the console (or relay) went quiet during the run.`);
  if (surprises.length === 0) surprises.push('Nothing unexpected detected by the automatic checks.');
  for (const s of surprises) L.push(s.startsWith('  ') ? s : `- ${s}`);
  L.push('');

  // Answered table by tree.
  L.push('## Answered addresses, by tree');
  L.push('');
  const trees = [...new Set(answeredGroups.map((g) => g.tree))];
  for (const tree of trees) {
    const gs = answeredGroups.filter((g) => g.tree === tree);
    L.push(`### /${tree} (${gs.length} patterns answered)`);
    L.push('');
    L.push('| Pattern | Answered N / M | Reply tags | Example value(s) | Documented type, range |');
    L.push('| --- | --- | --- | --- | --- |');
    for (const g of gs) {
      const ans = g.items.filter((i) => i.answered);
      const ns = [...new Set(ans.map((i) => i.instance.n).filter((v) => v !== null))];
      const ms = [...new Set(ans.map((i) => i.instance.m).filter((v) => v !== null))];
      const inst = [ns.length ? `N=${compressList(ns)}` : '', ms.length ? `M=${compressList(ms)}` : ''].filter(Boolean).join(' ') || '-';
      const partialMark = ans.length < g.items.length ? ` (${ans.length}/${g.items.length})` : '';
      const tags = [...new Set(ans.map((i) => (i.replyAddress !== i.address ? `${i.replyTypes} @${i.replyAddress}` : i.replyTypes)))].join(', ');
      const ex = [...new Set(ans.map((i) => fmtValue(i.exampleValue)))].slice(0, 4).join(' / ');
      const doc = g.documentedType ? `${g.documentedType}${g.documentedRange ? ' ' + fmtRange(g.documentedRange) : ''}` : '(not in 2014 list)';
      L.push(`| \`${g.pattern}\` | ${inst}${partialMark} | ${esc(tags)} | ${esc(ex)} | ${doc} |`);
    }
    L.push('');
  }
  if (trees.length === 0) {
    L.push('_No address answered._');
    L.push('');
  }

  // Unanswered.
  const docUnanswered = unansweredGroups.filter((g) => g.items.some((i) => i.sources.includes('doc')));
  const extraUnanswered = unansweredGroups.filter((g) => !g.items.some((i) => i.sources.includes('doc')));
  L.push(`## Unanswered documented patterns (${docUnanswered.length})`);
  L.push('');
  L.push('Patterns from the 2014 list for which none of the probed instances answered.');
  L.push('');
  const unTrees = [...new Set(docUnanswered.map((g) => g.tree))];
  for (const tree of unTrees) {
    const gs = docUnanswered.filter((g) => g.tree === tree);
    L.push(`### /${tree} (${gs.length})`);
    L.push('');
    for (const g of gs) L.push(`- \`${g.pattern}\``);
    L.push('');
  }
  L.push(`## Unanswered extra candidates (${extraUnanswered.length})`);
  L.push('');
  for (const g of extraUnanswered) L.push(`- \`${g.items.map((i) => i.address).join('`, `')}\``);
  L.push('');
  if (summary.skippedActionNodes.length) {
    L.push('## Not queried');
    L.push('');
    for (const a of summary.skippedActionNodes) L.push(`- \`${a}\` (write-only trigger node; skipped for safety, pass --include-actions to query it)`);
    L.push('');
  }

  // Unsolicited.
  L.push(`## Unsolicited messages (${unsolicited.length})`);
  L.push('');
  if (unsolicited.length === 0) {
    L.push('None seen, including during the final listen.');
  } else {
    L.push('| t (s) | Phase | Address | Tags | Value | Note |');
    L.push('| ---: | --- | --- | --- | --- | --- |');
    for (const u of unsolicited.slice(0, 500)) {
      const note = u.reason === 'outside-window' ? `${u.afterQueryMs} ms after a query for this address` : u.reason;
      L.push(`| ${round6(u.tMs / 1000)} | ${u.phase} | \`${u.address || '(unparsed)'}\` | ${u.types ?? ''} | ${esc(u.args ? fmtValue(u.args) : u.raw || '')} | ${note} |`);
    }
    if (unsolicited.length > 500) L.push(`| | | ... ${unsolicited.length - 500} more (see JSON) | | | |`);
  }
  L.push('');

  // Names.
  L.push('## Live name tables');
  L.push('');
  for (const [entity, tbl] of Object.entries(names)) {
    L.push(`### ${entity}`);
    L.push('');
    if (!tbl.addressPattern) {
      L.push('_No name address answered for this entity._');
      L.push('');
      continue;
    }
    L.push(`Address: \`${tbl.addressPattern}\` (${tbl.answered} of ${tbl.max} answered${tbl.unanswered.length ? `, no reply for ${tbl.unanswered.join(', ')}` : ''}).`);
    const sample = catalog.find((c) => c.pattern === tbl.addressPattern && c.answered);
    if (sample) L.push(`Reply format: \`${sample.replyAddress} ,${sample.replyTypes} ${fmtValue(Array.isArray(sample.exampleValue) ? sample.exampleValue : [sample.exampleValue])}\`.`);
    L.push('');
    L.push('| # | Name |');
    L.push('| ---: | --- |');
    for (const [idx, nm] of Object.entries(tbl.values)) L.push(`| ${idx} | ${esc(fmtValue(nm))} |`);
    L.push('');
  }
  return L.join('\n') + '\n';
}

main().catch((err) => {
  console.error(`fatal: ${err.stack || err.message}`);
  process.exit(1);
});