#!/usr/bin/env node /* * gazetteer-corpus.mjs — the D-14 bake-off close (golden corpus runner). * ---------------------------------------------------------------------- * Header contract: * Purpose: measure the PINNED golden corpus (D-13) against the SAME * shared search module the bot and the Mini App use (D-03) — the * measured quality IS the shipped quality. * Inputs: search/test/fixtures/golden-corpus.json (readFileSync + JSON.parse * — a script MAY use fs; only the WORKER may not). * Echoes: the query text and result ids of each MISS/LEAK to stdout. * This is a LOCAL DEV TOOL over a committed file — the Worker * NEVER prints query text (D-15 category-only logging law). * Exit: 0 ONLY when top-3 >= corpus.threshold.top3_hit_rate AND every * negative is refused (zero leaks); 1 otherwise, with the fixed * D-14 escalation ladder printed. Nothing in the repo — no * deploy, no test gate — depends on this script succeeding * beyond its own suite (corpus.test.mjs asserts the same * measurements in-process). * * Reproduce the bake-off with: npm run search:corpus */ import { readFileSync } from "node:fs"; import { resolve } from "node:path"; import { pathToFileURL } from "node:url"; import { FUZZY_ADMIT_SIMILARITY, damerau, getEntries, normalize, search, } from "../src/index.ts"; /** The scorer's similarity shape: 1 - damerau/maxLen. */ function similarity(a, b) { const maxLen = Math.max(a.length, b.length); if (maxLen === 0) return 1; return 1 - damerau(a, b) / maxLen; } /** * The SAME two-level (whole-string + token) fuzzy shape scoreOne uses — * the refusal margin must be measured against exactly what the admission * floor admits on. Sub-3-char tokens contribute 0 and are not "found", * but stay in the denominator (the scorer's documented shape). */ function fuzzySimOf(queryNorm, candidateNorm) { const whole = similarity(queryNorm, candidateNorm); const queryTokens = queryNorm.split(" "); const candidateTokens = candidateNorm.split(" "); let sum = 0; let found = 0; for (const queryToken of queryTokens) { if (queryToken.length < 3) continue; let best = 0; for (const candidateToken of candidateTokens) { if (candidateToken.length < 3) continue; const sim = similarity(queryToken, candidateToken); if (sim > best) best = sim; } if (best > 0) found++; sum += best; } const tokenLevel = queryTokens.length > 0 ? (sum / queryTokens.length) * (found / queryTokens.length) : 0; return Math.max(whole, tokenLevel); } /** * Run every corpus case through the shared search module and return the * measurements. Exported so search/test/corpus.test.mjs asserts the SAME * numbers this CLI prints — one measurement, two reporters, no drift. */ export function measureCorpus(corpus) { const positives = corpus.cases.filter((c) => c.expectId !== null); const negatives = corpus.cases.filter((c) => c.expectId === null); const perCategory = new Map(); const bucket = (kind) => { let b = perCategory.get(kind); if (!b) { b = { kind, total: 0, top3: 0, refused: 0 }; perCategory.set(kind, b); } return b; }; let top1 = 0; let top3 = 0; let totalMs = 0; const missed = []; for (const c of positives) { const t0 = performance.now(); const hits = search(c.q, 3); totalMs += performance.now() - t0; const ids = hits.map((h) => h.entry.id); const hitTop1 = ids[0] === c.expectId; const hitTop3 = ids.includes(c.expectId); if (hitTop1) top1++; if (hitTop3) top3++; else missed.push(`${c.q} -> [${ids.join(", ") || "—"}] (expected ${c.expectId})`); const b = bucket(c.kind); b.total++; if (hitTop3) b.top3++; } let refused = 0; const leaked = []; for (const c of negatives) { const t0 = performance.now(); const hits = search(c.q, 3); totalMs += performance.now() - t0; const b = bucket(c.kind); b.total++; if (hits.length === 0) { refused++; b.refused++; } else { leaked.push(`${c.q} -> ${hits[0].entry.id}`); } } // The refusal margin: the highest fuzzy similarity any negative reaches // against any alias — the gap to FUZZY_ADMIT_SIMILARITY is what makes // the refusals honest and VISIBLE (CO8 pins it by test as well). let bestNegativeSim = 0; for (const c of negatives) { const queryNorm = normalize(c.q); for (const entry of getEntries()) { for (const candidate of [entry.name, ...entry.aliases]) { const sim = fuzzySimOf(queryNorm, normalize(candidate)); if (sim > bestNegativeSim) bestNegativeSim = sim; } } } const queryCount = corpus.cases.length; return { version: corpus.version, positiveCount: positives.length, negativeCount: negatives.length, top1, top3, missed, refused, leaked, perCategory: [...perCategory.values()], avgMs: queryCount > 0 ? totalMs / queryCount : 0, bestNegativeSim, }; } /** The fixed D-14 escalation ladder — printed VERBATIM on failure. */ function printEscalationLadder() { console.log("D-14 threshold not met. Escalation order is FIXED:"); console.log(" (1) alias/normalizer table edits in search/ -> re-run this runner"); console.log(" (2) SearchSource backend #2 — the Malaysia-only WASM geocoder (D-05, post-v1)"); console.log(" (3) Photon-on-VPS (D-05, Phase 5)"); console.log("Record which lever was pulled in docs/SEARCH-QUALITY.md."); } function report(corpus, results) { const pct = (n, d) => (d > 0 ? ((n / d) * 100).toFixed(1) : "—"); console.log( `golden corpus v${results.version} — ${corpus.cases.length} cases ` + `(${results.positiveCount} positive / ${results.negativeCount} negative)`, ); console.log( `top-1 ${results.top1}/${results.positiveCount} top-3 ${results.top3}/${results.positiveCount} ` + `(${pct(results.top3, results.positiveCount)}%) ${results.avgMs.toFixed(2)} ms/query`, ); console.log("per-category:"); for (const b of results.perCategory) { if (b.kind === "negative") { console.log(` ${b.kind.padEnd(12)} refused ${b.refused}/${b.total} (${pct(b.refused, b.total)}%)`); } else { console.log(` ${b.kind.padEnd(12)} top-3 ${b.top3}/${b.total} (${pct(b.top3, b.total)}%)`); } } console.log(`negatives refused ${results.refused}/${results.negativeCount}`); console.log( `refusal margin ${results.bestNegativeSim.toFixed(3)} < ${FUZZY_ADMIT_SIMILARITY}`, ); if (results.missed.length > 0) { console.log("MISSED:"); for (const line of results.missed) console.log(` ${line}`); } if (results.leaked.length > 0) { console.log("LEAKED (must refuse):"); for (const line of results.leaked) console.log(` ${line}`); } const thresholdPct = (corpus.threshold.top3_hit_rate * 100).toFixed(1); const pass = results.top3 / results.positiveCount >= corpus.threshold.top3_hit_rate && results.leaked.length === 0; console.log(`threshold top3 >= ${thresholdPct}% — ${pass ? "PASS" : "FAIL"}`); return pass; } function main() { const corpus = JSON.parse( readFileSync(new URL("../test/fixtures/golden-corpus.json", import.meta.url), "utf8"), ); const results = measureCorpus(corpus); const pass = report(corpus, results); if (!pass) { printEscalationLadder(); process.exit(1); } process.exit(0); } // Run only when invoked directly — importing this module (the corpus suite // does, for in-process measurement) must never execute the CLI. const invoked = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : ""; if (invoked === import.meta.url) { main(); }