feat(04-02): pinned golden corpus + D-14 corpus runner + Nyquist gate
- golden-corpus.json: 75 hand-written pinned cases (D-13) — landmark 17 / abbreviation 15 / glued 7 / typo 14 / alias 15 / negative 7, incl. two near-miss negatives (Penang 0.625, Kampung Cina ~0.58) below the 0.72 admission line; threshold.top3_hit_rate pinned at 0.9 - gazetteer-corpus.mjs: the D-14 runner — imports the SAME src/index.ts the bot uses, measures top-1/top-3 + per-category rates + avg ms/query, prints the refusal margin and the fixed escalation ladder on failure, exports measureCorpus() for the in-process suite, exits 0 only on threshold + zero leaks - corpus.test.mjs goes green: CO1-CO8 (the Nyquist gate inside npm run search:test) - delegates: search:corpus (root) + corpus (search workspace) Measured: top-3 68/68 (100.0%) >= 90%, negatives refused 7/7, margin 0.643 < 0.72 — threshold met on the first run (D-14 lever: None).
This commit is contained in:
araharah-executor
@@ -25,6 +25,7 @@
|
||||
"web:typecheck": "npm run typecheck -w web",
|
||||
"search:test": "npm test -w search",
|
||||
"search:typecheck": "npm run typecheck -w search",
|
||||
"search:corpus": "node search/scripts/gazetteer-corpus.mjs",
|
||||
"web:dev": "npm run dev -w web",
|
||||
"web:build": "npm run build -w theme && npm run build -w web",
|
||||
"web:build:harness": "npm run build:harness -w web",
|
||||
|
||||
+2
-1
@@ -4,7 +4,8 @@
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
"test": "node test/runner.mjs",
|
||||
"typecheck": "tsc --noEmit"
|
||||
"typecheck": "tsc --noEmit",
|
||||
"corpus": "node scripts/gazetteer-corpus.mjs"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.9.3"
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* gazetteer-corpus.mjs — the D-14 bake-off close (golden corpus runner).
|
||||
* ----------------------------------------------------------------------
|
||||
* Header contract:
|
||||
* Purpose: measure the PINNED golden corpus (D-13) against the SAME
|
||||
* shared search module the bot and the Mini App use (D-03) — the
|
||||
* measured quality IS the shipped quality.
|
||||
* Inputs: search/test/fixtures/golden-corpus.json (readFileSync + JSON.parse
|
||||
* — a script MAY use fs; only the WORKER may not).
|
||||
* Echoes: the query text and result ids of each MISS/LEAK to stdout.
|
||||
* This is a LOCAL DEV TOOL over a committed file — the Worker
|
||||
* NEVER prints query text (D-15 category-only logging law).
|
||||
* Exit: 0 ONLY when top-3 >= corpus.threshold.top3_hit_rate AND every
|
||||
* negative is refused (zero leaks); 1 otherwise, with the fixed
|
||||
* D-14 escalation ladder printed. Nothing in the repo — no
|
||||
* deploy, no test gate — depends on this script succeeding
|
||||
* beyond its own suite (corpus.test.mjs asserts the same
|
||||
* measurements in-process).
|
||||
*
|
||||
* Reproduce the bake-off with: npm run search:corpus
|
||||
*/
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import {
|
||||
FUZZY_ADMIT_SIMILARITY,
|
||||
damerau,
|
||||
getEntries,
|
||||
normalize,
|
||||
search,
|
||||
} from "../src/index.ts";
|
||||
|
||||
/** The scorer's similarity shape: 1 - damerau/maxLen. */
|
||||
function similarity(a, b) {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 1;
|
||||
return 1 - damerau(a, b) / maxLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SAME two-level (whole-string + token) fuzzy shape scoreOne uses —
|
||||
* the refusal margin must be measured against exactly what the admission
|
||||
* floor admits on. Sub-3-char tokens contribute 0 and are not "found",
|
||||
* but stay in the denominator (the scorer's documented shape).
|
||||
*/
|
||||
function fuzzySimOf(queryNorm, candidateNorm) {
|
||||
const whole = similarity(queryNorm, candidateNorm);
|
||||
const queryTokens = queryNorm.split(" ");
|
||||
const candidateTokens = candidateNorm.split(" ");
|
||||
let sum = 0;
|
||||
let found = 0;
|
||||
for (const queryToken of queryTokens) {
|
||||
if (queryToken.length < 3) continue;
|
||||
let best = 0;
|
||||
for (const candidateToken of candidateTokens) {
|
||||
if (candidateToken.length < 3) continue;
|
||||
const sim = similarity(queryToken, candidateToken);
|
||||
if (sim > best) best = sim;
|
||||
}
|
||||
if (best > 0) found++;
|
||||
sum += best;
|
||||
}
|
||||
const tokenLevel =
|
||||
queryTokens.length > 0 ? (sum / queryTokens.length) * (found / queryTokens.length) : 0;
|
||||
return Math.max(whole, tokenLevel);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run every corpus case through the shared search module and return the
|
||||
* measurements. Exported so search/test/corpus.test.mjs asserts the SAME
|
||||
* numbers this CLI prints — one measurement, two reporters, no drift.
|
||||
*/
|
||||
export function measureCorpus(corpus) {
|
||||
const positives = corpus.cases.filter((c) => c.expectId !== null);
|
||||
const negatives = corpus.cases.filter((c) => c.expectId === null);
|
||||
const perCategory = new Map();
|
||||
const bucket = (kind) => {
|
||||
let b = perCategory.get(kind);
|
||||
if (!b) {
|
||||
b = { kind, total: 0, top3: 0, refused: 0 };
|
||||
perCategory.set(kind, b);
|
||||
}
|
||||
return b;
|
||||
};
|
||||
|
||||
let top1 = 0;
|
||||
let top3 = 0;
|
||||
let totalMs = 0;
|
||||
const missed = [];
|
||||
for (const c of positives) {
|
||||
const t0 = performance.now();
|
||||
const hits = search(c.q, 3);
|
||||
totalMs += performance.now() - t0;
|
||||
const ids = hits.map((h) => h.entry.id);
|
||||
const hitTop1 = ids[0] === c.expectId;
|
||||
const hitTop3 = ids.includes(c.expectId);
|
||||
if (hitTop1) top1++;
|
||||
if (hitTop3) top3++;
|
||||
else missed.push(`${c.q} -> [${ids.join(", ") || "—"}] (expected ${c.expectId})`);
|
||||
const b = bucket(c.kind);
|
||||
b.total++;
|
||||
if (hitTop3) b.top3++;
|
||||
}
|
||||
|
||||
let refused = 0;
|
||||
const leaked = [];
|
||||
for (const c of negatives) {
|
||||
const t0 = performance.now();
|
||||
const hits = search(c.q, 3);
|
||||
totalMs += performance.now() - t0;
|
||||
const b = bucket(c.kind);
|
||||
b.total++;
|
||||
if (hits.length === 0) {
|
||||
refused++;
|
||||
b.refused++;
|
||||
} else {
|
||||
leaked.push(`${c.q} -> ${hits[0].entry.id}`);
|
||||
}
|
||||
}
|
||||
|
||||
// The refusal margin: the highest fuzzy similarity any negative reaches
|
||||
// against any alias — the gap to FUZZY_ADMIT_SIMILARITY is what makes
|
||||
// the refusals honest and VISIBLE (CO8 pins it by test as well).
|
||||
let bestNegativeSim = 0;
|
||||
for (const c of negatives) {
|
||||
const queryNorm = normalize(c.q);
|
||||
for (const entry of getEntries()) {
|
||||
for (const candidate of [entry.name, ...entry.aliases]) {
|
||||
const sim = fuzzySimOf(queryNorm, normalize(candidate));
|
||||
if (sim > bestNegativeSim) bestNegativeSim = sim;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const queryCount = corpus.cases.length;
|
||||
return {
|
||||
version: corpus.version,
|
||||
positiveCount: positives.length,
|
||||
negativeCount: negatives.length,
|
||||
top1,
|
||||
top3,
|
||||
missed,
|
||||
refused,
|
||||
leaked,
|
||||
perCategory: [...perCategory.values()],
|
||||
avgMs: queryCount > 0 ? totalMs / queryCount : 0,
|
||||
bestNegativeSim,
|
||||
};
|
||||
}
|
||||
|
||||
/** The fixed D-14 escalation ladder — printed VERBATIM on failure. */
|
||||
function printEscalationLadder() {
|
||||
console.log("D-14 threshold not met. Escalation order is FIXED:");
|
||||
console.log(" (1) alias/normalizer table edits in search/ -> re-run this runner");
|
||||
console.log(" (2) SearchSource backend #2 — the Malaysia-only WASM geocoder (D-05, post-v1)");
|
||||
console.log(" (3) Photon-on-VPS (D-05, Phase 5)");
|
||||
console.log("Record which lever was pulled in docs/SEARCH-QUALITY.md.");
|
||||
}
|
||||
|
||||
function report(corpus, results) {
|
||||
const pct = (n, d) => (d > 0 ? ((n / d) * 100).toFixed(1) : "—");
|
||||
console.log(
|
||||
`golden corpus v${results.version} — ${corpus.cases.length} cases ` +
|
||||
`(${results.positiveCount} positive / ${results.negativeCount} negative)`,
|
||||
);
|
||||
console.log(
|
||||
`top-1 ${results.top1}/${results.positiveCount} top-3 ${results.top3}/${results.positiveCount} ` +
|
||||
`(${pct(results.top3, results.positiveCount)}%) ${results.avgMs.toFixed(2)} ms/query`,
|
||||
);
|
||||
console.log("per-category:");
|
||||
for (const b of results.perCategory) {
|
||||
if (b.kind === "negative") {
|
||||
console.log(` ${b.kind.padEnd(12)} refused ${b.refused}/${b.total} (${pct(b.refused, b.total)}%)`);
|
||||
} else {
|
||||
console.log(` ${b.kind.padEnd(12)} top-3 ${b.top3}/${b.total} (${pct(b.top3, b.total)}%)`);
|
||||
}
|
||||
}
|
||||
console.log(`negatives refused ${results.refused}/${results.negativeCount}`);
|
||||
console.log(
|
||||
`refusal margin ${results.bestNegativeSim.toFixed(3)} < ${FUZZY_ADMIT_SIMILARITY}`,
|
||||
);
|
||||
if (results.missed.length > 0) {
|
||||
console.log("MISSED:");
|
||||
for (const line of results.missed) console.log(` ${line}`);
|
||||
}
|
||||
if (results.leaked.length > 0) {
|
||||
console.log("LEAKED (must refuse):");
|
||||
for (const line of results.leaked) console.log(` ${line}`);
|
||||
}
|
||||
const thresholdPct = (corpus.threshold.top3_hit_rate * 100).toFixed(1);
|
||||
const pass =
|
||||
results.top3 / results.positiveCount >= corpus.threshold.top3_hit_rate &&
|
||||
results.leaked.length === 0;
|
||||
console.log(`threshold top3 >= ${thresholdPct}% — ${pass ? "PASS" : "FAIL"}`);
|
||||
return pass;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const corpus = JSON.parse(
|
||||
readFileSync(new URL("../test/fixtures/golden-corpus.json", import.meta.url), "utf8"),
|
||||
);
|
||||
const results = measureCorpus(corpus);
|
||||
const pass = report(corpus, results);
|
||||
if (!pass) {
|
||||
printEscalationLadder();
|
||||
process.exit(1);
|
||||
}
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
// Run only when invoked directly — importing this module (the corpus suite
|
||||
// does, for in-process measurement) must never execute the CLI.
|
||||
const invoked = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : "";
|
||||
if (invoked === import.meta.url) {
|
||||
main();
|
||||
}
|
||||
+87
@@ -0,0 +1,87 @@
|
||||
{
|
||||
"version": 1,
|
||||
"note": "Hand-written and pinned (D-13). NOT generated from the gazetteer — a generated corpus is tautological. Reviewers extend this file by editing it.",
|
||||
"threshold": { "top3_hit_rate": 0.9 },
|
||||
"cases": [
|
||||
{ "q": "KLCC", "expectId": "suria-klcc", "kind": "landmark" },
|
||||
{ "q": "Sunway Pyramid", "expectId": "sunway-pyramid", "kind": "landmark" },
|
||||
{ "q": "DUKE", "expectId": "the-duke", "kind": "landmark" },
|
||||
{ "q": "Pavilion", "expectId": "pavilion-kl", "kind": "landmark" },
|
||||
{ "q": "Mid Valley", "expectId": "mid-valley", "kind": "landmark" },
|
||||
{ "q": "Berjaya Times Square", "expectId": "berjaya-times-square", "kind": "landmark" },
|
||||
{ "q": "IOI City Mall", "expectId": "ioi-city-mall", "kind": "landmark" },
|
||||
{ "q": "Merdeka 118", "expectId": "merdeka-118", "kind": "landmark" },
|
||||
{ "q": "KL Sentral", "expectId": "kl-sentral", "kind": "landmark" },
|
||||
{ "q": "Nu Sentral", "expectId": "nu-sentral", "kind": "landmark" },
|
||||
{ "q": "Petronas Twin Towers", "expectId": "petronas-twin-towers", "kind": "landmark" },
|
||||
{ "q": "Taman Bangsar", "expectId": "bangsar", "kind": "landmark" },
|
||||
{ "q": "KL Tower", "expectId": "kl-tower", "kind": "landmark" },
|
||||
{ "q": "Bukit Bintang", "expectId": "bukit-bintang", "kind": "landmark" },
|
||||
{ "q": "Central Market", "expectId": "central-market", "kind": "landmark" },
|
||||
{ "q": "Petaling Street", "expectId": "petaling-street", "kind": "landmark" },
|
||||
{ "q": "Masjid Jamek", "expectId": "masjid-jamek", "kind": "landmark" },
|
||||
|
||||
{ "q": "Jln Ampang", "expectId": "suria-klcc", "kind": "abbreviation" },
|
||||
{ "q": "Jalan Ampang", "expectId": "suria-klcc", "kind": "abbreviation" },
|
||||
{ "q": "Tmn Bangsar", "expectId": "bangsar", "kind": "abbreviation" },
|
||||
{ "q": "Kg Ampang", "expectId": "kampung-ampang", "kind": "abbreviation" },
|
||||
{ "q": "Kpg Ampang", "expectId": "kampung-ampang", "kind": "abbreviation" },
|
||||
{ "q": "Kampung Ampang", "expectId": "kampung-ampang", "kind": "abbreviation" },
|
||||
{ "q": "Sg Wang", "expectId": "sungei-wang", "kind": "abbreviation" },
|
||||
{ "q": "Sungai Wang", "expectId": "sungei-wang", "kind": "abbreviation" },
|
||||
{ "q": "Bkt Bintang", "expectId": "bukit-bintang", "kind": "abbreviation" },
|
||||
{ "q": "Bkt Bintang KL", "expectId": "bukit-bintang", "kind": "abbreviation" },
|
||||
{ "q": "Jln Tun Razak", "expectId": "tun-razak", "kind": "abbreviation" },
|
||||
{ "q": "Jln Petaling", "expectId": "petaling-street", "kind": "abbreviation" },
|
||||
{ "q": "Stesen Sentral", "expectId": "kl-sentral", "kind": "abbreviation" },
|
||||
{ "q": "Tmn Tasik Titiwangsa", "expectId": "titiwangsa", "kind": "abbreviation" },
|
||||
{ "q": "Tmn Perdana", "expectId": "lake-gardens", "kind": "abbreviation" },
|
||||
|
||||
{ "q": "tamanbangsar", "expectId": "bangsar", "kind": "glued" },
|
||||
{ "q": "sungaiwang", "expectId": "sungei-wang", "kind": "glued" },
|
||||
{ "q": "bukitbintang", "expectId": "bukit-bintang", "kind": "glued" },
|
||||
{ "q": "jalanampang", "expectId": "suria-klcc", "kind": "glued" },
|
||||
{ "q": "kampungampang", "expectId": "kampung-ampang", "kind": "glued" },
|
||||
{ "q": "jalanpetaling", "expectId": "petaling-street", "kind": "glued" },
|
||||
{ "q": "jalantunrazak", "expectId": "tun-razak", "kind": "glued" },
|
||||
|
||||
{ "q": "KCLC", "expectId": "suria-klcc", "kind": "typo" },
|
||||
{ "q": "Pavillion", "expectId": "pavilion-kl", "kind": "typo" },
|
||||
{ "q": "Pavilon", "expectId": "pavilion-kl", "kind": "typo" },
|
||||
{ "q": "Sunwya Pyarmid", "expectId": "sunway-pyramid", "kind": "typo" },
|
||||
{ "q": "Mid Vally", "expectId": "mid-valley", "kind": "typo" },
|
||||
{ "q": "Berjaya Time Square", "expectId": "berjaya-times-square", "kind": "typo" },
|
||||
{ "q": "Sentrl", "expectId": "kl-sentral", "kind": "typo" },
|
||||
{ "q": "Kl Towr", "expectId": "kl-tower", "kind": "typo" },
|
||||
{ "q": "Sunway Vellocity", "expectId": "sunway-velocity", "kind": "typo" },
|
||||
{ "q": "Merdeaka 118", "expectId": "merdeka-118", "kind": "typo" },
|
||||
{ "q": "Akuaria", "expectId": "aquaria-klcc", "kind": "typo" },
|
||||
{ "q": "Sngi Wang", "expectId": "sungei-wang", "kind": "typo" },
|
||||
{ "q": "Petrnas Twin Towers", "expectId": "petronas-twin-towers", "kind": "typo" },
|
||||
{ "q": "Titiwangsaa", "expectId": "titiwangsa", "kind": "typo" },
|
||||
|
||||
{ "q": "IJN", "expectId": "ijnp", "kind": "alias" },
|
||||
{ "q": "PWTC", "expectId": "pwtc", "kind": "alias" },
|
||||
{ "q": "Stesen KL Sentral", "expectId": "kl-sentral", "kind": "alias" },
|
||||
{ "q": "KL Central", "expectId": "kl-sentral", "kind": "alias" },
|
||||
{ "q": "BTS", "expectId": "berjaya-times-square", "kind": "alias" },
|
||||
{ "q": "IOI", "expectId": "ioi-city-mall", "kind": "alias" },
|
||||
{ "q": "Velocity", "expectId": "sunway-velocity", "kind": "alias" },
|
||||
{ "q": "Pudu", "expectId": "pudu-raya", "kind": "alias" },
|
||||
{ "q": "Pasar Seni", "expectId": "central-market", "kind": "alias" },
|
||||
{ "q": "Muzium Negara", "expectId": "national-museum", "kind": "alias" },
|
||||
{ "q": "Titiwangsa", "expectId": "titiwangsa", "kind": "alias" },
|
||||
{ "q": "Aquaria", "expectId": "aquaria-klcc", "kind": "alias" },
|
||||
{ "q": "Little India", "expectId": "little-india-brickfields", "kind": "alias" },
|
||||
{ "q": "Chinatown", "expectId": "chinatown", "kind": "alias" },
|
||||
{ "q": "Tun Razak", "expectId": "tun-razak", "kind": "alias" },
|
||||
|
||||
{ "q": "Kuching", "expectId": null, "kind": "negative" },
|
||||
{ "q": "Penang", "expectId": null, "kind": "negative" },
|
||||
{ "q": "Johor Bahru", "expectId": null, "kind": "negative" },
|
||||
{ "q": "Kota Kinabalu", "expectId": null, "kind": "negative" },
|
||||
{ "q": "Singapore", "expectId": null, "kind": "negative" },
|
||||
{ "q": "London", "expectId": null, "kind": "negative" },
|
||||
{ "q": "Kampung Cina", "expectId": null, "kind": "negative" }
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user