From b2689f71b16b2be6a9bce2a2b3af0dc24e537470 Mon Sep 17 00:00:00 2001 From: araharah-executor Date: Sat, 24 Oct 2026 00:54:47 +0800 Subject: [PATCH] feat(04-02): pinned golden corpus + D-14 corpus runner + Nyquist gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - golden-corpus.json: 75 hand-written pinned cases (D-13) — landmark 17 / abbreviation 15 / glued 7 / typo 14 / alias 15 / negative 7, incl. two near-miss negatives (Penang 0.625, Kampung Cina ~0.58) below the 0.72 admission line; threshold.top3_hit_rate pinned at 0.9 - gazetteer-corpus.mjs: the D-14 runner — imports the SAME src/index.ts the bot uses, measures top-1/top-3 + per-category rates + avg ms/query, prints the refusal margin and the fixed escalation ladder on failure, exports measureCorpus() for the in-process suite, exits 0 only on threshold + zero leaks - corpus.test.mjs goes green: CO1-CO8 (the Nyquist gate inside npm run search:test) - delegates: search:corpus (root) + corpus (search workspace) Measured: top-3 68/68 (100.0%) >= 90%, negatives refused 7/7, margin 0.643 < 0.72 — threshold met on the first run (D-14 lever: None). --- package.json | 1 + search/package.json | 3 +- search/scripts/gazetteer-corpus.mjs | 217 ++++++++++++++++++++++++ search/test/fixtures/golden-corpus.json | 87 ++++++++++ 4 files changed, 307 insertions(+), 1 deletion(-) create mode 100644 search/scripts/gazetteer-corpus.mjs create mode 100644 search/test/fixtures/golden-corpus.json diff --git a/package.json b/package.json index 49ea989..ca3d69d 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "web:typecheck": "npm run typecheck -w web", "search:test": "npm test -w search", "search:typecheck": "npm run typecheck -w search", + "search:corpus": "node search/scripts/gazetteer-corpus.mjs", "web:dev": "npm run dev -w web", "web:build": "npm run build -w theme && npm run build -w web", "web:build:harness": "npm run build:harness -w web", diff --git a/search/package.json b/search/package.json index d1778f2..afc41e8 100644 --- a/search/package.json +++ b/search/package.json @@ -4,7 +4,8 @@ "type": "module", "scripts": { "test": "node test/runner.mjs", - "typecheck": "tsc --noEmit" + "typecheck": "tsc --noEmit", + "corpus": "node scripts/gazetteer-corpus.mjs" }, "devDependencies": { "typescript": "^5.9.3" diff --git a/search/scripts/gazetteer-corpus.mjs b/search/scripts/gazetteer-corpus.mjs new file mode 100644 index 0000000..ce6473b --- /dev/null +++ b/search/scripts/gazetteer-corpus.mjs @@ -0,0 +1,217 @@ +#!/usr/bin/env node +/* + * gazetteer-corpus.mjs — the D-14 bake-off close (golden corpus runner). + * ---------------------------------------------------------------------- + * Header contract: + * Purpose: measure the PINNED golden corpus (D-13) against the SAME + * shared search module the bot and the Mini App use (D-03) — the + * measured quality IS the shipped quality. + * Inputs: search/test/fixtures/golden-corpus.json (readFileSync + JSON.parse + * — a script MAY use fs; only the WORKER may not). + * Echoes: the query text and result ids of each MISS/LEAK to stdout. + * This is a LOCAL DEV TOOL over a committed file — the Worker + * NEVER prints query text (D-15 category-only logging law). + * Exit: 0 ONLY when top-3 >= corpus.threshold.top3_hit_rate AND every + * negative is refused (zero leaks); 1 otherwise, with the fixed + * D-14 escalation ladder printed. Nothing in the repo — no + * deploy, no test gate — depends on this script succeeding + * beyond its own suite (corpus.test.mjs asserts the same + * measurements in-process). + * + * Reproduce the bake-off with: npm run search:corpus + */ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { pathToFileURL } from "node:url"; +import { + FUZZY_ADMIT_SIMILARITY, + damerau, + getEntries, + normalize, + search, +} from "../src/index.ts"; + +/** The scorer's similarity shape: 1 - damerau/maxLen. */ +function similarity(a, b) { + const maxLen = Math.max(a.length, b.length); + if (maxLen === 0) return 1; + return 1 - damerau(a, b) / maxLen; +} + +/** + * The SAME two-level (whole-string + token) fuzzy shape scoreOne uses — + * the refusal margin must be measured against exactly what the admission + * floor admits on. Sub-3-char tokens contribute 0 and are not "found", + * but stay in the denominator (the scorer's documented shape). + */ +function fuzzySimOf(queryNorm, candidateNorm) { + const whole = similarity(queryNorm, candidateNorm); + const queryTokens = queryNorm.split(" "); + const candidateTokens = candidateNorm.split(" "); + let sum = 0; + let found = 0; + for (const queryToken of queryTokens) { + if (queryToken.length < 3) continue; + let best = 0; + for (const candidateToken of candidateTokens) { + if (candidateToken.length < 3) continue; + const sim = similarity(queryToken, candidateToken); + if (sim > best) best = sim; + } + if (best > 0) found++; + sum += best; + } + const tokenLevel = + queryTokens.length > 0 ? (sum / queryTokens.length) * (found / queryTokens.length) : 0; + return Math.max(whole, tokenLevel); +} + +/** + * Run every corpus case through the shared search module and return the + * measurements. Exported so search/test/corpus.test.mjs asserts the SAME + * numbers this CLI prints — one measurement, two reporters, no drift. + */ +export function measureCorpus(corpus) { + const positives = corpus.cases.filter((c) => c.expectId !== null); + const negatives = corpus.cases.filter((c) => c.expectId === null); + const perCategory = new Map(); + const bucket = (kind) => { + let b = perCategory.get(kind); + if (!b) { + b = { kind, total: 0, top3: 0, refused: 0 }; + perCategory.set(kind, b); + } + return b; + }; + + let top1 = 0; + let top3 = 0; + let totalMs = 0; + const missed = []; + for (const c of positives) { + const t0 = performance.now(); + const hits = search(c.q, 3); + totalMs += performance.now() - t0; + const ids = hits.map((h) => h.entry.id); + const hitTop1 = ids[0] === c.expectId; + const hitTop3 = ids.includes(c.expectId); + if (hitTop1) top1++; + if (hitTop3) top3++; + else missed.push(`${c.q} -> [${ids.join(", ") || "—"}] (expected ${c.expectId})`); + const b = bucket(c.kind); + b.total++; + if (hitTop3) b.top3++; + } + + let refused = 0; + const leaked = []; + for (const c of negatives) { + const t0 = performance.now(); + const hits = search(c.q, 3); + totalMs += performance.now() - t0; + const b = bucket(c.kind); + b.total++; + if (hits.length === 0) { + refused++; + b.refused++; + } else { + leaked.push(`${c.q} -> ${hits[0].entry.id}`); + } + } + + // The refusal margin: the highest fuzzy similarity any negative reaches + // against any alias — the gap to FUZZY_ADMIT_SIMILARITY is what makes + // the refusals honest and VISIBLE (CO8 pins it by test as well). + let bestNegativeSim = 0; + for (const c of negatives) { + const queryNorm = normalize(c.q); + for (const entry of getEntries()) { + for (const candidate of [entry.name, ...entry.aliases]) { + const sim = fuzzySimOf(queryNorm, normalize(candidate)); + if (sim > bestNegativeSim) bestNegativeSim = sim; + } + } + } + + const queryCount = corpus.cases.length; + return { + version: corpus.version, + positiveCount: positives.length, + negativeCount: negatives.length, + top1, + top3, + missed, + refused, + leaked, + perCategory: [...perCategory.values()], + avgMs: queryCount > 0 ? totalMs / queryCount : 0, + bestNegativeSim, + }; +} + +/** The fixed D-14 escalation ladder — printed VERBATIM on failure. */ +function printEscalationLadder() { + console.log("D-14 threshold not met. Escalation order is FIXED:"); + console.log(" (1) alias/normalizer table edits in search/ -> re-run this runner"); + console.log(" (2) SearchSource backend #2 — the Malaysia-only WASM geocoder (D-05, post-v1)"); + console.log(" (3) Photon-on-VPS (D-05, Phase 5)"); + console.log("Record which lever was pulled in docs/SEARCH-QUALITY.md."); +} + +function report(corpus, results) { + const pct = (n, d) => (d > 0 ? ((n / d) * 100).toFixed(1) : "—"); + console.log( + `golden corpus v${results.version} — ${corpus.cases.length} cases ` + + `(${results.positiveCount} positive / ${results.negativeCount} negative)`, + ); + console.log( + `top-1 ${results.top1}/${results.positiveCount} top-3 ${results.top3}/${results.positiveCount} ` + + `(${pct(results.top3, results.positiveCount)}%) ${results.avgMs.toFixed(2)} ms/query`, + ); + console.log("per-category:"); + for (const b of results.perCategory) { + if (b.kind === "negative") { + console.log(` ${b.kind.padEnd(12)} refused ${b.refused}/${b.total} (${pct(b.refused, b.total)}%)`); + } else { + console.log(` ${b.kind.padEnd(12)} top-3 ${b.top3}/${b.total} (${pct(b.top3, b.total)}%)`); + } + } + console.log(`negatives refused ${results.refused}/${results.negativeCount}`); + console.log( + `refusal margin ${results.bestNegativeSim.toFixed(3)} < ${FUZZY_ADMIT_SIMILARITY}`, + ); + if (results.missed.length > 0) { + console.log("MISSED:"); + for (const line of results.missed) console.log(` ${line}`); + } + if (results.leaked.length > 0) { + console.log("LEAKED (must refuse):"); + for (const line of results.leaked) console.log(` ${line}`); + } + const thresholdPct = (corpus.threshold.top3_hit_rate * 100).toFixed(1); + const pass = + results.top3 / results.positiveCount >= corpus.threshold.top3_hit_rate && + results.leaked.length === 0; + console.log(`threshold top3 >= ${thresholdPct}% — ${pass ? "PASS" : "FAIL"}`); + return pass; +} + +function main() { + const corpus = JSON.parse( + readFileSync(new URL("../test/fixtures/golden-corpus.json", import.meta.url), "utf8"), + ); + const results = measureCorpus(corpus); + const pass = report(corpus, results); + if (!pass) { + printEscalationLadder(); + process.exit(1); + } + process.exit(0); +} + +// Run only when invoked directly — importing this module (the corpus suite +// does, for in-process measurement) must never execute the CLI. +const invoked = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : ""; +if (invoked === import.meta.url) { + main(); +} diff --git a/search/test/fixtures/golden-corpus.json b/search/test/fixtures/golden-corpus.json new file mode 100644 index 0000000..084468e --- /dev/null +++ b/search/test/fixtures/golden-corpus.json @@ -0,0 +1,87 @@ +{ + "version": 1, + "note": "Hand-written and pinned (D-13). NOT generated from the gazetteer — a generated corpus is tautological. Reviewers extend this file by editing it.", + "threshold": { "top3_hit_rate": 0.9 }, + "cases": [ + { "q": "KLCC", "expectId": "suria-klcc", "kind": "landmark" }, + { "q": "Sunway Pyramid", "expectId": "sunway-pyramid", "kind": "landmark" }, + { "q": "DUKE", "expectId": "the-duke", "kind": "landmark" }, + { "q": "Pavilion", "expectId": "pavilion-kl", "kind": "landmark" }, + { "q": "Mid Valley", "expectId": "mid-valley", "kind": "landmark" }, + { "q": "Berjaya Times Square", "expectId": "berjaya-times-square", "kind": "landmark" }, + { "q": "IOI City Mall", "expectId": "ioi-city-mall", "kind": "landmark" }, + { "q": "Merdeka 118", "expectId": "merdeka-118", "kind": "landmark" }, + { "q": "KL Sentral", "expectId": "kl-sentral", "kind": "landmark" }, + { "q": "Nu Sentral", "expectId": "nu-sentral", "kind": "landmark" }, + { "q": "Petronas Twin Towers", "expectId": "petronas-twin-towers", "kind": "landmark" }, + { "q": "Taman Bangsar", "expectId": "bangsar", "kind": "landmark" }, + { "q": "KL Tower", "expectId": "kl-tower", "kind": "landmark" }, + { "q": "Bukit Bintang", "expectId": "bukit-bintang", "kind": "landmark" }, + { "q": "Central Market", "expectId": "central-market", "kind": "landmark" }, + { "q": "Petaling Street", "expectId": "petaling-street", "kind": "landmark" }, + { "q": "Masjid Jamek", "expectId": "masjid-jamek", "kind": "landmark" }, + + { "q": "Jln Ampang", "expectId": "suria-klcc", "kind": "abbreviation" }, + { "q": "Jalan Ampang", "expectId": "suria-klcc", "kind": "abbreviation" }, + { "q": "Tmn Bangsar", "expectId": "bangsar", "kind": "abbreviation" }, + { "q": "Kg Ampang", "expectId": "kampung-ampang", "kind": "abbreviation" }, + { "q": "Kpg Ampang", "expectId": "kampung-ampang", "kind": "abbreviation" }, + { "q": "Kampung Ampang", "expectId": "kampung-ampang", "kind": "abbreviation" }, + { "q": "Sg Wang", "expectId": "sungei-wang", "kind": "abbreviation" }, + { "q": "Sungai Wang", "expectId": "sungei-wang", "kind": "abbreviation" }, + { "q": "Bkt Bintang", "expectId": "bukit-bintang", "kind": "abbreviation" }, + { "q": "Bkt Bintang KL", "expectId": "bukit-bintang", "kind": "abbreviation" }, + { "q": "Jln Tun Razak", "expectId": "tun-razak", "kind": "abbreviation" }, + { "q": "Jln Petaling", "expectId": "petaling-street", "kind": "abbreviation" }, + { "q": "Stesen Sentral", "expectId": "kl-sentral", "kind": "abbreviation" }, + { "q": "Tmn Tasik Titiwangsa", "expectId": "titiwangsa", "kind": "abbreviation" }, + { "q": "Tmn Perdana", "expectId": "lake-gardens", "kind": "abbreviation" }, + + { "q": "tamanbangsar", "expectId": "bangsar", "kind": "glued" }, + { "q": "sungaiwang", "expectId": "sungei-wang", "kind": "glued" }, + { "q": "bukitbintang", "expectId": "bukit-bintang", "kind": "glued" }, + { "q": "jalanampang", "expectId": "suria-klcc", "kind": "glued" }, + { "q": "kampungampang", "expectId": "kampung-ampang", "kind": "glued" }, + { "q": "jalanpetaling", "expectId": "petaling-street", "kind": "glued" }, + { "q": "jalantunrazak", "expectId": "tun-razak", "kind": "glued" }, + + { "q": "KCLC", "expectId": "suria-klcc", "kind": "typo" }, + { "q": "Pavillion", "expectId": "pavilion-kl", "kind": "typo" }, + { "q": "Pavilon", "expectId": "pavilion-kl", "kind": "typo" }, + { "q": "Sunwya Pyarmid", "expectId": "sunway-pyramid", "kind": "typo" }, + { "q": "Mid Vally", "expectId": "mid-valley", "kind": "typo" }, + { "q": "Berjaya Time Square", "expectId": "berjaya-times-square", "kind": "typo" }, + { "q": "Sentrl", "expectId": "kl-sentral", "kind": "typo" }, + { "q": "Kl Towr", "expectId": "kl-tower", "kind": "typo" }, + { "q": "Sunway Vellocity", "expectId": "sunway-velocity", "kind": "typo" }, + { "q": "Merdeaka 118", "expectId": "merdeka-118", "kind": "typo" }, + { "q": "Akuaria", "expectId": "aquaria-klcc", "kind": "typo" }, + { "q": "Sngi Wang", "expectId": "sungei-wang", "kind": "typo" }, + { "q": "Petrnas Twin Towers", "expectId": "petronas-twin-towers", "kind": "typo" }, + { "q": "Titiwangsaa", "expectId": "titiwangsa", "kind": "typo" }, + + { "q": "IJN", "expectId": "ijnp", "kind": "alias" }, + { "q": "PWTC", "expectId": "pwtc", "kind": "alias" }, + { "q": "Stesen KL Sentral", "expectId": "kl-sentral", "kind": "alias" }, + { "q": "KL Central", "expectId": "kl-sentral", "kind": "alias" }, + { "q": "BTS", "expectId": "berjaya-times-square", "kind": "alias" }, + { "q": "IOI", "expectId": "ioi-city-mall", "kind": "alias" }, + { "q": "Velocity", "expectId": "sunway-velocity", "kind": "alias" }, + { "q": "Pudu", "expectId": "pudu-raya", "kind": "alias" }, + { "q": "Pasar Seni", "expectId": "central-market", "kind": "alias" }, + { "q": "Muzium Negara", "expectId": "national-museum", "kind": "alias" }, + { "q": "Titiwangsa", "expectId": "titiwangsa", "kind": "alias" }, + { "q": "Aquaria", "expectId": "aquaria-klcc", "kind": "alias" }, + { "q": "Little India", "expectId": "little-india-brickfields", "kind": "alias" }, + { "q": "Chinatown", "expectId": "chinatown", "kind": "alias" }, + { "q": "Tun Razak", "expectId": "tun-razak", "kind": "alias" }, + + { "q": "Kuching", "expectId": null, "kind": "negative" }, + { "q": "Penang", "expectId": null, "kind": "negative" }, + { "q": "Johor Bahru", "expectId": null, "kind": "negative" }, + { "q": "Kota Kinabalu", "expectId": null, "kind": "negative" }, + { "q": "Singapore", "expectId": null, "kind": "negative" }, + { "q": "London", "expectId": null, "kind": "negative" }, + { "q": "Kampung Cina", "expectId": null, "kind": "negative" } + ] +}