arah2/search/scripts/gazetteer-corpus.mjs

#!/usr/bin/env node
/*
 * gazetteer-corpus.mjs — the D-14 bake-off close (golden corpus runner).
 * ----------------------------------------------------------------------
 * Header contract:
 *   Purpose: measure the PINNED golden corpus (D-13) against the SAME
 *   shared search module the bot and the Mini App use (D-03) — the
 *   measured quality IS the shipped quality.
 *   Inputs:  search/test/fixtures/golden-corpus.json (readFileSync + JSON.parse
 *            — a script MAY use fs; only the WORKER may not).
 *   Echoes:  the query text and result ids of each MISS/LEAK to stdout.
 *            This is a LOCAL DEV TOOL over a committed file — the Worker
 *            NEVER prints query text (D-15 category-only logging law).
 *   Exit:    0 ONLY when top-3 >= corpus.threshold.top3_hit_rate AND every
 *            negative is refused (zero leaks); 1 otherwise, with the fixed
 *            D-14 escalation ladder printed. Nothing in the repo — no
 *            deploy, no test gate — depends on this script succeeding
 *            beyond its own suite (corpus.test.mjs asserts the same
 *            measurements in-process).
 *
 * Reproduce the bake-off with:  npm run search:corpus
 */
import { readFileSync } from "node:fs";
import { resolve } from "node:path";
import { pathToFileURL } from "node:url";
import {
  FUZZY_ADMIT_SIMILARITY,
  damerau,
  getEntries,
  normalize,
  search,
} from "../src/index.ts";

/** The scorer's similarity shape: 1 - damerau/maxLen. */
function similarity(a, b) {
  const maxLen = Math.max(a.length, b.length);
  if (maxLen === 0) return 1;
  return 1 - damerau(a, b) / maxLen;
}

/**
 * The SAME two-level (whole-string + token) fuzzy shape scoreOne uses —
 * the refusal margin must be measured against exactly what the admission
 * floor admits on. Sub-3-char tokens contribute 0 and are not "found",
 * but stay in the denominator (the scorer's documented shape).
 */
function fuzzySimOf(queryNorm, candidateNorm) {
  const whole = similarity(queryNorm, candidateNorm);
  const queryTokens = queryNorm.split(" ");
  const candidateTokens = candidateNorm.split(" ");
  let sum = 0;
  let found = 0;
  for (const queryToken of queryTokens) {
    if (queryToken.length < 3) continue;
    let best = 0;
    for (const candidateToken of candidateTokens) {
      if (candidateToken.length < 3) continue;
      const sim = similarity(queryToken, candidateToken);
      if (sim > best) best = sim;
    }
    if (best > 0) found++;
    sum += best;
  }
  const tokenLevel =
    queryTokens.length > 0 ? (sum / queryTokens.length) * (found / queryTokens.length) : 0;
  return Math.max(whole, tokenLevel);
}

/**
 * Run every corpus case through the shared search module and return the
 * measurements. Exported so search/test/corpus.test.mjs asserts the SAME
 * numbers this CLI prints — one measurement, two reporters, no drift.
 */
export function measureCorpus(corpus) {
  const positives = corpus.cases.filter((c) => c.expectId !== null);
  const negatives = corpus.cases.filter((c) => c.expectId === null);
  const perCategory = new Map();
  const bucket = (kind) => {
    let b = perCategory.get(kind);
    if (!b) {
      b = { kind, total: 0, top3: 0, refused: 0 };
      perCategory.set(kind, b);
    }
    return b;
  };

  let top1 = 0;
  let top3 = 0;
  let totalMs = 0;
  const missed = [];
  for (const c of positives) {
    const t0 = performance.now();
    const hits = search(c.q, 3);
    totalMs += performance.now() - t0;
    const ids = hits.map((h) => h.entry.id);
    const hitTop1 = ids[0] === c.expectId;
    const hitTop3 = ids.includes(c.expectId);
    if (hitTop1) top1++;
    if (hitTop3) top3++;
    else missed.push(`${c.q} -> [${ids.join(", ") || "—"}] (expected ${c.expectId})`);
    const b = bucket(c.kind);
    b.total++;
    if (hitTop3) b.top3++;
  }

  let refused = 0;
  const leaked = [];
  for (const c of negatives) {
    const t0 = performance.now();
    const hits = search(c.q, 3);
    totalMs += performance.now() - t0;
    const b = bucket(c.kind);
    b.total++;
    if (hits.length === 0) {
      refused++;
      b.refused++;
    } else {
      leaked.push(`${c.q} -> ${hits[0].entry.id}`);
    }
  }

  // The refusal margin: the highest fuzzy similarity any negative reaches
  // against any alias — the gap to FUZZY_ADMIT_SIMILARITY is what makes
  // the refusals honest and VISIBLE (CO8 pins it by test as well).
  let bestNegativeSim = 0;
  for (const c of negatives) {
    const queryNorm = normalize(c.q);
    for (const entry of getEntries()) {
      for (const candidate of [entry.name, ...entry.aliases]) {
        const sim = fuzzySimOf(queryNorm, normalize(candidate));
        if (sim > bestNegativeSim) bestNegativeSim = sim;
      }
    }
  }

  const queryCount = corpus.cases.length;
  return {
    version: corpus.version,
    positiveCount: positives.length,
    negativeCount: negatives.length,
    top1,
    top3,
    missed,
    refused,
    leaked,
    perCategory: [...perCategory.values()],
    avgMs: queryCount > 0 ? totalMs / queryCount : 0,
    bestNegativeSim,
  };
}

/** The fixed D-14 escalation ladder — printed VERBATIM on failure. */
function printEscalationLadder() {
  console.log("D-14 threshold not met. Escalation order is FIXED:");
  console.log("  (1) alias/normalizer table edits in search/ -> re-run this runner");
  console.log("  (2) SearchSource backend #2 — the Malaysia-only WASM geocoder (D-05, post-v1)");
  console.log("  (3) Photon-on-VPS (D-05, Phase 5)");
  console.log("Record which lever was pulled in docs/SEARCH-QUALITY.md.");
}

function report(corpus, results) {
  const pct = (n, d) => (d > 0 ? ((n / d) * 100).toFixed(1) : "—");
  console.log(
    `golden corpus v${results.version} — ${corpus.cases.length} cases ` +
      `(${results.positiveCount} positive / ${results.negativeCount} negative)`,
  );
  console.log(
    `top-1 ${results.top1}/${results.positiveCount}  top-3 ${results.top3}/${results.positiveCount} ` +
      `(${pct(results.top3, results.positiveCount)}%)  ${results.avgMs.toFixed(2)} ms/query`,
  );
  console.log("per-category:");
  for (const b of results.perCategory) {
    if (b.kind === "negative") {
      console.log(`  ${b.kind.padEnd(12)} refused ${b.refused}/${b.total} (${pct(b.refused, b.total)}%)`);
    } else {
      console.log(`  ${b.kind.padEnd(12)} top-3 ${b.top3}/${b.total} (${pct(b.top3, b.total)}%)`);
    }
  }
  console.log(`negatives refused ${results.refused}/${results.negativeCount}`);
  console.log(
    `refusal margin ${results.bestNegativeSim.toFixed(3)} < ${FUZZY_ADMIT_SIMILARITY}`,
  );
  if (results.missed.length > 0) {
    console.log("MISSED:");
    for (const line of results.missed) console.log(`  ${line}`);
  }
  if (results.leaked.length > 0) {
    console.log("LEAKED (must refuse):");
    for (const line of results.leaked) console.log(`  ${line}`);
  }
  const thresholdPct = (corpus.threshold.top3_hit_rate * 100).toFixed(1);
  const pass =
    results.top3 / results.positiveCount >= corpus.threshold.top3_hit_rate &&
    results.leaked.length === 0;
  console.log(`threshold top3 >= ${thresholdPct}% — ${pass ? "PASS" : "FAIL"}`);
  return pass;
}

function main() {
  const corpus = JSON.parse(
    readFileSync(new URL("../test/fixtures/golden-corpus.json", import.meta.url), "utf8"),
  );
  const results = measureCorpus(corpus);
  const pass = report(corpus, results);
  if (!pass) {
    printEscalationLadder();
    process.exit(1);
  }
  process.exit(0);
}

// Run only when invoked directly — importing this module (the corpus suite
// does, for in-process measurement) must never execute the CLI.
const invoked = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : "";
if (invoked === import.meta.url) {
  main();
}