arah2/search/test/corpus.test.mjs

#!/usr/bin/env node
/*
 * corpus.test.mjs — the D-14 Nyquist gate over the pinned golden corpus.
 * ----------------------------------------------------------------------
 * The suite re-runs the corpus runner's measurements IN-PROCESS (it imports
 * the shared module and the corpus directly — it never spawns the script),
 * so `npm run search:test` fails whenever the corpus regresses: the corpus
 * must be >= threshold at every wave merge, not just at the end.
 *
 * Cases:
 *   CO1 top-3 hit rate meets the D-14 threshold (corpus.threshold.top3_hit_rate)
 *   CO2 every negative is refused (zero leaks — never a fabricated pin)
 *   CO3 the corpus is not tautological (>= 15 non-negative cases whose
 *       normalized query is NOT string-equal to any alias or name — Pitfall 7)
 *   CO4 every positive's expectId exists in the gazetteer (no dangling ids)
 *   CO5 every expectId matches the start_param-safe charset
 *   CO6 the corpus meets its size and per-category floors
 *   CO7 every category's top-3 rate is reported (printed, so the suite
 *       output and the runner output agree)
 *   CO8 the refusal margin is pinned — every negative's best fuzzy
 *       similarity sits strictly below FUZZY_ADMIT_SIMILARITY, and at
 *       least one near-miss negative sits in [0.55, FUZZY_ADMIT_SIMILARITY)
 *       so lowering the admission line would leak a real negative and fail
 *       BOTH CO2 and CO8. The floor is pinned by test, not by hope.
 *
 * Run:  npm run search:test     (exit 0 = all green)
 */
import { strict as assert } from "node:assert";
import { readFileSync } from "node:fs";
import { test } from "./harness.mjs";

// Lazy shared-module load — a missing module => per-case ERR_MODULE_NOT_FOUND
// (the RED state), one FAIL per law instead of a lost suite load.
let searchModPromise = null;
function loadSearch() {
  if (!searchModPromise) {
    searchModPromise = import("../src/index.ts");
  }
  return searchModPromise;
}

// The corpus runner's measurement core (the script exports measureCorpus
// precisely so this suite measures the SAME thing the CLI reports).
let runnerPromise = null;
function loadRunner() {
  if (!runnerPromise) {
    runnerPromise = import("../scripts/gazetteer-corpus.mjs");
  }
  return runnerPromise;
}

function loadCorpus() {
  return JSON.parse(
    readFileSync(new URL("./fixtures/golden-corpus.json", import.meta.url), "utf8"),
  );
}

/** The scorer's similarity shape: 1 - damerau/maxLen. */
function similarity(a, b, damerau) {
  const maxLen = Math.max(a.length, b.length);
  if (maxLen === 0) return 1;
  return 1 - damerau(a, b) / maxLen;
}

/**
 * The SAME two-level (whole-string + token) fuzzy shape scoreOne uses —
 * CO8 replicates it here (it is not exported from the module) so the
 * refusal margin is asserted against exactly what the admission floor
 * admits on: max(wholeStringSim, tokenLevelSim) where the token level is
 * the mean of per-query-token bests scaled by the found fraction, and
 * sub-3-char tokens are skipped (contribute 0, not found, still counted
 * in the denominator).
 */
function fuzzySimOf(queryNorm, candidateNorm, damerau) {
  const whole = similarity(queryNorm, candidateNorm, damerau);
  const queryTokens = queryNorm.split(" ");
  const candidateTokens = candidateNorm.split(" ");
  let sum = 0;
  let found = 0;
  for (const queryToken of queryTokens) {
    if (queryToken.length < 3) continue;
    let best = 0;
    for (const candidateToken of candidateTokens) {
      if (candidateToken.length < 3) continue;
      const sim = similarity(queryToken, candidateToken, damerau);
      if (sim > best) best = sim;
    }
    if (best > 0) found++;
    sum += best;
  }
  const tokenLevel =
    queryTokens.length > 0 ? (sum / queryTokens.length) * (found / queryTokens.length) : 0;
  return Math.max(whole, tokenLevel);
}

/* ————— CO1 — the D-14 threshold ————— */

test("CO1 top-3 hit rate meets the D-14 threshold", async () => {
  const corpus = loadCorpus();
  const { measureCorpus } = await loadRunner();
  const results = measureCorpus(corpus);
  const rate = results.top3 / results.positiveCount;
  assert.ok(
    rate >= corpus.threshold.top3_hit_rate,
    `D-14 threshold not met: top-3 ${results.top3}/${results.positiveCount} ` +
      `(${(rate * 100).toFixed(1)}%) < ${(corpus.threshold.top3_hit_rate * 100).toFixed(1)}%. ` +
      `Escalation lever 1 is alias/normalizer table edits in search/ — ` +
      `record any lever pulled in docs/SEARCH-QUALITY.md. MISSED:\n${results.missed.join("\n")}`,
  );
});

/* ————— CO2 — honest refusal ————— */

test("CO2 every negative is refused", async () => {
  const corpus = loadCorpus();
  const { measureCorpus } = await loadRunner();
  const results = measureCorpus(corpus);
  assert.equal(
    results.leaked.length,
    0,
    `negatives MUST be refused with an empty shortlist (D-13), never a ` +
      `fabricated pin. LEAKED:\n${results.leaked.join("\n")}`,
  );
});

/* ————— CO3 — the tautology floor (Pitfall 7) ————— */

test("CO3 the corpus is not tautological — >= 15 queries are not string-equal to any alias", async () => {
  const corpus = loadCorpus();
  const { normalize, getEntries } = await loadSearch();
  const aliases = new Set();
  for (const entry of getEntries()) {
    aliases.add(normalize(entry.name));
    for (const alias of entry.aliases) aliases.add(normalize(alias));
  }
  const nonTautological = corpus.cases.filter(
    (c) => c.kind !== "negative" && !aliases.has(normalize(c.q)),
  );
  assert.ok(
    nonTautological.length >= 15,
    `tautology floor: only ${nonTautological.length} non-negative cases are not ` +
      `string-equal to any gazetteer alias/name (need >= 15) — a corpus that only ` +
      `repeats aliases measures nothing (RESEARCH Pitfall 7)`,
  );
});

/* ————— CO4 — no dangling ids ————— */

test("CO4 every positive's expectId exists in the gazetteer", async () => {
  const corpus = loadCorpus();
  const { getEntries } = await loadSearch();
  const ids = new Set(getEntries().map((e) => e.id));
  const dangling = corpus.cases
    .filter((c) => c.expectId !== null && !ids.has(c.expectId))
    .map((c) => `${c.q} -> ${c.expectId}`);
  assert.equal(
    dangling.length,
    0,
    `corpus bug (not a search bug): expectId has no entry — fix the corpus:\n${dangling.join("\n")}`,
  );
});

/* ————— CO5 — start_param-safe charset ————— */

test("CO5 every expectId matches the start_param-safe charset", () => {
  const corpus = loadCorpus();
  const bad = corpus.cases
    .filter((c) => c.expectId !== null && !/^[A-Za-z0-9_-]{1,64}$/.test(c.expectId))
    .map((c) => c.expectId);
  assert.equal(
    bad.length,
    0,
    `expectId must match /^[A-Za-z0-9_-]{1,64}$/ (the start_param charset): ${bad.join(", ")}`,
  );
});

/* ————— CO6 — size + per-category floors ————— */

test("CO6 the corpus meets its size and category floors", () => {
  const corpus = loadCorpus();
  assert.ok(corpus.cases.length >= 60, `corpus must have >= 60 cases, has ${corpus.cases.length}`);
  const floors = { landmark: 12, abbreviation: 15, glued: 6, typo: 12, alias: 12, negative: 6 };
  const kinds = {};
  for (const c of corpus.cases) kinds[c.kind] = (kinds[c.kind] ?? 0) + 1;
  for (const [kind, floor] of Object.entries(floors)) {
    assert.ok(
      (kinds[kind] ?? 0) >= floor,
      `category "${kind}" has ${kinds[kind] ?? 0} cases, floor is ${floor}`,
    );
  }
});

/* ————— CO7 — per-category reporting (printed, not asserted) ————— */

test("CO7 every category's top-3 rate is reported", async () => {
  const corpus = loadCorpus();
  const { measureCorpus } = await loadRunner();
  const results = measureCorpus(corpus);
  console.log("[corpus] per-category top-3 rates:");
  for (const cat of results.perCategory) {
    if (cat.kind === "negative") {
      console.log(
        `[corpus]   ${cat.kind.padEnd(12)} refused ${cat.refused}/${cat.total} (${((cat.refused / cat.total) * 100).toFixed(1)}%)`,
      );
    } else {
      console.log(
        `[corpus]   ${cat.kind.padEnd(12)} top-3 ${cat.top3}/${cat.total} (${((cat.top3 / cat.total) * 100).toFixed(1)}%)`,
      );
    }
  }
});

/* ————— CO8 — the refusal margin, pinned ————— */

test("CO8 the refusal margin is pinned — every negative below the admission line, one near-miss above 0.55", async () => {
  const corpus = loadCorpus();
  const { damerau, normalize, getEntries, FUZZY_ADMIT_SIMILARITY } = await loadSearch();
  const negatives = corpus.cases.filter((c) => c.expectId === null);
  const margins = negatives.map((c) => {
    const queryNorm = normalize(c.q);
    let best = 0;
    for (const entry of getEntries()) {
      for (const candidate of [entry.name, ...entry.aliases]) {
        const sim = fuzzySimOf(queryNorm, normalize(candidate), damerau);
        if (sim > best) best = sim;
      }
    }
    return { q: c.q, best };
  });
  const above = margins.filter((m) => m.best >= FUZZY_ADMIT_SIMILARITY);
  assert.equal(
    above.length,
    0,
    `a negative reaches the fuzzy admission line — lowering the threshold or ` +
      `admitting it would fabricate a pin (D-13). Offenders:\n` +
      above.map((m) => `  ${m.q} sim ${m.best.toFixed(3)} >= ${FUZZY_ADMIT_SIMILARITY}`).join("\n"),
  );
  const nearMisses = margins.filter((m) => m.best >= 0.55 && m.best < FUZZY_ADMIT_SIMILARITY);
  assert.ok(
    nearMisses.length >= 1,
    `the corpus must contain at least one NEAR-MISS negative whose best fuzzy ` +
      `similarity lands in [0.55, ${FUZZY_ADMIT_SIMILARITY}) — a query a LOWERED ` +
      `admission threshold would wrongly admit. None found; margins:\n` +
      margins.map((m) => `  ${m.q} ${m.best.toFixed(3)}`).join("\n"),
  );
});