test(04-02): RED — golden-corpus D-14 gate suite (CO1-CO8)
- CO1 top-3 >= corpus threshold (D-14), measured in-process via the runner's measureCorpus - CO2 zero negative leaks (honest refusal, D-13) - CO3 tautology floor: >= 15 non-negative queries not string-equal to any alias/name - CO4/CO5 every expectId exists in the gazetteer + start_param-safe charset - CO6 size >= 60 + per-category floors (landmark 12 / abbreviation 15 / glued 6 / typo 12 / alias 12 / negative 6) - CO7 per-category rates printed so suite and runner output agree - CO8 refusal margin pinned: every negative's best two-level fuzzy sim < 0.72, one near-miss in [0.55, 0.72) All 8 cases verified failing (ENOENT corpus fixture) before GREEN.
This commit is contained in:
araharah-executor
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* corpus.test.mjs — the D-14 Nyquist gate over the pinned golden corpus.
|
||||
* ----------------------------------------------------------------------
|
||||
* The suite re-runs the corpus runner's measurements IN-PROCESS (it imports
|
||||
* the shared module and the corpus directly — it never spawns the script),
|
||||
* so `npm run search:test` fails whenever the corpus regresses: the corpus
|
||||
* must be >= threshold at every wave merge, not just at the end.
|
||||
*
|
||||
* Cases:
|
||||
* CO1 top-3 hit rate meets the D-14 threshold (corpus.threshold.top3_hit_rate)
|
||||
* CO2 every negative is refused (zero leaks — never a fabricated pin)
|
||||
* CO3 the corpus is not tautological (>= 15 non-negative cases whose
|
||||
* normalized query is NOT string-equal to any alias or name — Pitfall 7)
|
||||
* CO4 every positive's expectId exists in the gazetteer (no dangling ids)
|
||||
* CO5 every expectId matches the start_param-safe charset
|
||||
* CO6 the corpus meets its size and per-category floors
|
||||
* CO7 every category's top-3 rate is reported (printed, so the suite
|
||||
* output and the runner output agree)
|
||||
* CO8 the refusal margin is pinned — every negative's best fuzzy
|
||||
* similarity sits strictly below FUZZY_ADMIT_SIMILARITY, and at
|
||||
* least one near-miss negative sits in [0.55, FUZZY_ADMIT_SIMILARITY)
|
||||
* so lowering the admission line would leak a real negative and fail
|
||||
* BOTH CO2 and CO8. The floor is pinned by test, not by hope.
|
||||
*
|
||||
* Run: npm run search:test (exit 0 = all green)
|
||||
*/
|
||||
import { strict as assert } from "node:assert";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { test } from "./harness.mjs";
|
||||
|
||||
// Lazy shared-module load — a missing module => per-case ERR_MODULE_NOT_FOUND
|
||||
// (the RED state), one FAIL per law instead of a lost suite load.
|
||||
let searchModPromise = null;
|
||||
function loadSearch() {
|
||||
if (!searchModPromise) {
|
||||
searchModPromise = import("../src/index.ts");
|
||||
}
|
||||
return searchModPromise;
|
||||
}
|
||||
|
||||
// The corpus runner's measurement core (the script exports measureCorpus
|
||||
// precisely so this suite measures the SAME thing the CLI reports).
|
||||
let runnerPromise = null;
|
||||
function loadRunner() {
|
||||
if (!runnerPromise) {
|
||||
runnerPromise = import("../scripts/gazetteer-corpus.mjs");
|
||||
}
|
||||
return runnerPromise;
|
||||
}
|
||||
|
||||
function loadCorpus() {
|
||||
return JSON.parse(
|
||||
readFileSync(new URL("./fixtures/golden-corpus.json", import.meta.url), "utf8"),
|
||||
);
|
||||
}
|
||||
|
||||
/** The scorer's similarity shape: 1 - damerau/maxLen. */
|
||||
function similarity(a, b, damerau) {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 1;
|
||||
return 1 - damerau(a, b) / maxLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* The SAME two-level (whole-string + token) fuzzy shape scoreOne uses —
|
||||
* CO8 replicates it here (it is not exported from the module) so the
|
||||
* refusal margin is asserted against exactly what the admission floor
|
||||
* admits on: max(wholeStringSim, tokenLevelSim) where the token level is
|
||||
* the mean of per-query-token bests scaled by the found fraction, and
|
||||
* sub-3-char tokens are skipped (contribute 0, not found, still counted
|
||||
* in the denominator).
|
||||
*/
|
||||
function fuzzySimOf(queryNorm, candidateNorm, damerau) {
|
||||
const whole = similarity(queryNorm, candidateNorm, damerau);
|
||||
const queryTokens = queryNorm.split(" ");
|
||||
const candidateTokens = candidateNorm.split(" ");
|
||||
let sum = 0;
|
||||
let found = 0;
|
||||
for (const queryToken of queryTokens) {
|
||||
if (queryToken.length < 3) continue;
|
||||
let best = 0;
|
||||
for (const candidateToken of candidateTokens) {
|
||||
if (candidateToken.length < 3) continue;
|
||||
const sim = similarity(queryToken, candidateToken, damerau);
|
||||
if (sim > best) best = sim;
|
||||
}
|
||||
if (best > 0) found++;
|
||||
sum += best;
|
||||
}
|
||||
const tokenLevel =
|
||||
queryTokens.length > 0 ? (sum / queryTokens.length) * (found / queryTokens.length) : 0;
|
||||
return Math.max(whole, tokenLevel);
|
||||
}
|
||||
|
||||
/* ————— CO1 — the D-14 threshold ————— */
|
||||
|
||||
test("CO1 top-3 hit rate meets the D-14 threshold", async () => {
|
||||
const corpus = loadCorpus();
|
||||
const { measureCorpus } = await loadRunner();
|
||||
const results = measureCorpus(corpus);
|
||||
const rate = results.top3 / results.positiveCount;
|
||||
assert.ok(
|
||||
rate >= corpus.threshold.top3_hit_rate,
|
||||
`D-14 threshold not met: top-3 ${results.top3}/${results.positiveCount} ` +
|
||||
`(${(rate * 100).toFixed(1)}%) < ${(corpus.threshold.top3_hit_rate * 100).toFixed(1)}%. ` +
|
||||
`Escalation lever 1 is alias/normalizer table edits in search/ — ` +
|
||||
`record any lever pulled in docs/SEARCH-QUALITY.md. MISSED:\n${results.missed.join("\n")}`,
|
||||
);
|
||||
});
|
||||
|
||||
/* ————— CO2 — honest refusal ————— */
|
||||
|
||||
test("CO2 every negative is refused", async () => {
|
||||
const corpus = loadCorpus();
|
||||
const { measureCorpus } = await loadRunner();
|
||||
const results = measureCorpus(corpus);
|
||||
assert.equal(
|
||||
results.leaked.length,
|
||||
0,
|
||||
`negatives MUST be refused with an empty shortlist (D-13), never a ` +
|
||||
`fabricated pin. LEAKED:\n${results.leaked.join("\n")}`,
|
||||
);
|
||||
});
|
||||
|
||||
/* ————— CO3 — the tautology floor (Pitfall 7) ————— */
|
||||
|
||||
test("CO3 the corpus is not tautological — >= 15 queries are not string-equal to any alias", async () => {
|
||||
const corpus = loadCorpus();
|
||||
const { normalize, getEntries } = await loadSearch();
|
||||
const aliases = new Set();
|
||||
for (const entry of getEntries()) {
|
||||
aliases.add(normalize(entry.name));
|
||||
for (const alias of entry.aliases) aliases.add(normalize(alias));
|
||||
}
|
||||
const nonTautological = corpus.cases.filter(
|
||||
(c) => c.kind !== "negative" && !aliases.has(normalize(c.q)),
|
||||
);
|
||||
assert.ok(
|
||||
nonTautological.length >= 15,
|
||||
`tautology floor: only ${nonTautological.length} non-negative cases are not ` +
|
||||
`string-equal to any gazetteer alias/name (need >= 15) — a corpus that only ` +
|
||||
`repeats aliases measures nothing (RESEARCH Pitfall 7)`,
|
||||
);
|
||||
});
|
||||
|
||||
/* ————— CO4 — no dangling ids ————— */
|
||||
|
||||
test("CO4 every positive's expectId exists in the gazetteer", async () => {
|
||||
const corpus = loadCorpus();
|
||||
const { getEntries } = await loadSearch();
|
||||
const ids = new Set(getEntries().map((e) => e.id));
|
||||
const dangling = corpus.cases
|
||||
.filter((c) => c.expectId !== null && !ids.has(c.expectId))
|
||||
.map((c) => `${c.q} -> ${c.expectId}`);
|
||||
assert.equal(
|
||||
dangling.length,
|
||||
0,
|
||||
`corpus bug (not a search bug): expectId has no entry — fix the corpus:\n${dangling.join("\n")}`,
|
||||
);
|
||||
});
|
||||
|
||||
/* ————— CO5 — start_param-safe charset ————— */
|
||||
|
||||
test("CO5 every expectId matches the start_param-safe charset", () => {
|
||||
const corpus = loadCorpus();
|
||||
const bad = corpus.cases
|
||||
.filter((c) => c.expectId !== null && !/^[A-Za-z0-9_-]{1,64}$/.test(c.expectId))
|
||||
.map((c) => c.expectId);
|
||||
assert.equal(
|
||||
bad.length,
|
||||
0,
|
||||
`expectId must match /^[A-Za-z0-9_-]{1,64}$/ (the start_param charset): ${bad.join(", ")}`,
|
||||
);
|
||||
});
|
||||
|
||||
/* ————— CO6 — size + per-category floors ————— */
|
||||
|
||||
test("CO6 the corpus meets its size and category floors", () => {
|
||||
const corpus = loadCorpus();
|
||||
assert.ok(corpus.cases.length >= 60, `corpus must have >= 60 cases, has ${corpus.cases.length}`);
|
||||
const floors = { landmark: 12, abbreviation: 15, glued: 6, typo: 12, alias: 12, negative: 6 };
|
||||
const kinds = {};
|
||||
for (const c of corpus.cases) kinds[c.kind] = (kinds[c.kind] ?? 0) + 1;
|
||||
for (const [kind, floor] of Object.entries(floors)) {
|
||||
assert.ok(
|
||||
(kinds[kind] ?? 0) >= floor,
|
||||
`category "${kind}" has ${kinds[kind] ?? 0} cases, floor is ${floor}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
/* ————— CO7 — per-category reporting (printed, not asserted) ————— */
|
||||
|
||||
test("CO7 every category's top-3 rate is reported", async () => {
|
||||
const corpus = loadCorpus();
|
||||
const { measureCorpus } = await loadRunner();
|
||||
const results = measureCorpus(corpus);
|
||||
console.log("[corpus] per-category top-3 rates:");
|
||||
for (const cat of results.perCategory) {
|
||||
if (cat.kind === "negative") {
|
||||
console.log(
|
||||
`[corpus] ${cat.kind.padEnd(12)} refused ${cat.refused}/${cat.total} (${((cat.refused / cat.total) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
} else {
|
||||
console.log(
|
||||
`[corpus] ${cat.kind.padEnd(12)} top-3 ${cat.top3}/${cat.total} (${((cat.top3 / cat.total) * 100).toFixed(1)}%)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/* ————— CO8 — the refusal margin, pinned ————— */
|
||||
|
||||
test("CO8 the refusal margin is pinned — every negative below the admission line, one near-miss above 0.55", async () => {
|
||||
const corpus = loadCorpus();
|
||||
const { damerau, normalize, getEntries, FUZZY_ADMIT_SIMILARITY } = await loadSearch();
|
||||
const negatives = corpus.cases.filter((c) => c.expectId === null);
|
||||
const margins = negatives.map((c) => {
|
||||
const queryNorm = normalize(c.q);
|
||||
let best = 0;
|
||||
for (const entry of getEntries()) {
|
||||
for (const candidate of [entry.name, ...entry.aliases]) {
|
||||
const sim = fuzzySimOf(queryNorm, normalize(candidate), damerau);
|
||||
if (sim > best) best = sim;
|
||||
}
|
||||
}
|
||||
return { q: c.q, best };
|
||||
});
|
||||
const above = margins.filter((m) => m.best >= FUZZY_ADMIT_SIMILARITY);
|
||||
assert.equal(
|
||||
above.length,
|
||||
0,
|
||||
`a negative reaches the fuzzy admission line — lowering the threshold or ` +
|
||||
`admitting it would fabricate a pin (D-13). Offenders:\n` +
|
||||
above.map((m) => ` ${m.q} sim ${m.best.toFixed(3)} >= ${FUZZY_ADMIT_SIMILARITY}`).join("\n"),
|
||||
);
|
||||
const nearMisses = margins.filter((m) => m.best >= 0.55 && m.best < FUZZY_ADMIT_SIMILARITY);
|
||||
assert.ok(
|
||||
nearMisses.length >= 1,
|
||||
`the corpus must contain at least one NEAR-MISS negative whose best fuzzy ` +
|
||||
`similarity lands in [0.55, ${FUZZY_ADMIT_SIMILARITY}) — a query a LOWERED ` +
|
||||
`admission threshold would wrongly admit. None found; margins:\n` +
|
||||
margins.map((m) => ` ${m.q} ${m.best.toFixed(3)}`).join("\n"),
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user