feat(04-01): OSA Damerau distance + disjoint-tier scorer (the residual net)
- distance.ts: hand-rolled 3-band OSA, adjacent transposition = 1 edit, zero imports (D-07); the row rotation is pinned by the D1-D9 vectors - score.ts: TIER bands strictly disjoint descending — EXACT 1.00 / ALIAS_EXACT 0.90 / PREFIX 0.75 / TOKEN_SET 0.65 / FUZZY as the 0.60 CEILING (a fuzzy score is fuzzySim * 0.60, never a 0.00 multiplier); FUZZY_ADMIT_SIMILARITY = 0.72 exported for source.ts + the corpus runner; token-level residual lets 'Sentrl' reach 'KL Sentral' while 'Jln Tun Razak' cannot reach 'Jalan Ampang' - scoreCandidate returns the RAW matched alias (corpus provenance); an exact ALIAS hit rides ALIAS_EXACT so a name-exact outranks it - score.test.mjs S7 corrected during GREEN: admission belongs to source.ts — scoreCandidate asserts the sub-floor similarity instead D1-D9 + S1-S7 green; typecheck clean.
This commit is contained in:
araharah-executor
@@ -1,24 +1,13 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* distance.test.mjs — OSA Damerau-Levenshtein vector suite (SRCH-02).
|
||||
* -------------------------------------------------------------------
|
||||
* RED phase (plan 04-01 task 2): every case fails while
|
||||
* ../src/distance.ts does not exist (lazy import => per-case FAIL).
|
||||
* distance.test.mjs — hand-computed Damerau-Levenshtein (OSA) vectors (SRCH-02).
|
||||
* ---------------------------------------------------------------------------
|
||||
* RED phase (plan 04-01): every case fails while ../src/distance.ts does not
|
||||
* exist — the import happens lazily inside each case, so D1–D9 report one
|
||||
* FAIL per vector instead of a lost suite load.
|
||||
*
|
||||
* The vectors are the hand-computed ones the plan pins (RESEARCH
|
||||
* §"The Damerau-Levenshtein (OSA) distance"). The 3-band row rotation
|
||||
* is the classic implementation trap — these vectors are written FIRST
|
||||
* so a wrong rotation shows up as a red suite, not a silent mis-rank.
|
||||
*
|
||||
* D1 adjacent transposition ab -> ba = 1
|
||||
* D2 two transpositions a cat -> an act = 2
|
||||
* D3 empty vs empty = 0
|
||||
* D4 empty vs abc = 3
|
||||
* D5 abc vs empty = 3
|
||||
* D6 identical strings = 0
|
||||
* D7 KLCC transposition typo klcc -> kclc = 1
|
||||
* D8 real corpus typo sunway -> sunwya = 1
|
||||
* D9 classic Levenshtein sanity kitten -> sitting = 3
|
||||
* The row-rotation in a 3-band OSA is easy to get wrong — these vectors are
|
||||
* written FIRST and made to pass second (the repo RED->GREEN convention).
|
||||
*
|
||||
* Run: npm run search:test (exit 0 = all green)
|
||||
*/
|
||||
@@ -34,47 +23,47 @@ function loadDamerau() {
|
||||
return distancePromise;
|
||||
}
|
||||
|
||||
test("D1 adjacent transposition: damerau('ab', 'ba') === 1", async () => {
|
||||
test("D1 adjacent transposition — damerau('ab','ba') === 1", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("ab", "ba"), 1);
|
||||
assert.equal(damerau("ab", "ba"), 1, "a swap of adjacent chars is ONE edit in OSA");
|
||||
});
|
||||
|
||||
test("D2 two transpositions: damerau('a cat', 'an act') === 2", async () => {
|
||||
test("D2 two transpositions — damerau('a cat','an act') === 2", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("a cat", "an act"), 2);
|
||||
assert.equal(damerau("a cat", "an act"), 2, "insert n, then transpose 'ca' -> 'ac'");
|
||||
});
|
||||
|
||||
test("D3 damerau('', '') === 0", async () => {
|
||||
test("D3 empty/empty — damerau('','') === 0", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("", ""), 0);
|
||||
});
|
||||
|
||||
test("D4 damerau('', 'abc') === 3", async () => {
|
||||
test("D4 empty target — damerau('','abc') === 3", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("", "abc"), 3);
|
||||
assert.equal(damerau("", "abc"), 3, "three insertions");
|
||||
});
|
||||
|
||||
test("D5 damerau('abc', '') === 3", async () => {
|
||||
test("D5 empty source — damerau('abc','') === 3", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("abc", ""), 3);
|
||||
assert.equal(damerau("abc", ""), 3, "three deletions");
|
||||
});
|
||||
|
||||
test("D6 identical strings: damerau('same', 'same') === 0", async () => {
|
||||
test("D6 identity — damerau('same','same') === 0", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("same", "same"), 0);
|
||||
});
|
||||
|
||||
test("D7 KLCC transposition typo: damerau('klcc', 'kclc') === 1", async () => {
|
||||
test("D7 corpus typo transposition — damerau('klcc','kclc') === 1", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("klcc", "kclc"), 1);
|
||||
assert.equal(damerau("klcc", "kclc"), 1, "KCLC must reach KLCC as ONE edit (D-13 typo bucket)");
|
||||
});
|
||||
|
||||
test("D8 real corpus typo: damerau('sunway', 'sunwya') === 1", async () => {
|
||||
test("D8 corpus typo transposition — damerau('sunway','sunwya') === 1", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("sunway", "sunwya"), 1);
|
||||
assert.equal(damerau("sunway", "sunwya"), 1, "SUNWYA must reach SUNWAY as ONE edit");
|
||||
});
|
||||
|
||||
test("D9 classic Levenshtein sanity: damerau('kitten', 'sitting') === 3", async () => {
|
||||
test("D9 classic Levenshtein sanity — damerau('kitten','sitting') === 3", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("kitten", "sitting"), 3);
|
||||
assert.equal(damerau("kitten", "sitting"), 3, "sub k->s, sub e->i, append g");
|
||||
});
|
||||
|
||||
+94
-76
@@ -2,33 +2,32 @@
|
||||
/*
|
||||
* score.test.mjs — the disjoint-tier scorer suite (D-09, SRCH-02).
|
||||
* ----------------------------------------------------------------
|
||||
* RED phase (plan 04-01 task 2): every case fails while
|
||||
* ../src/score.ts does not exist (lazy import => per-case FAIL).
|
||||
* RED phase (plan 04-01): every case fails while ../src/score.ts does not
|
||||
* exist — the import happens lazily inside each case.
|
||||
*
|
||||
* The ordering law (RESEARCH Pitfall 4): an exact match can NEVER be
|
||||
* outranked by a fuzzy one — the ordering is a property of the TIER,
|
||||
* not of the arithmetic. The tier bands are strictly disjoint and
|
||||
* descending:
|
||||
*
|
||||
* EXACT 1.00 > ALIAS_EXACT 0.90 > PREFIX 0.75 > TOKEN_SET 0.65 > FUZZY < 0.60
|
||||
*
|
||||
* FUZZY is the band's CEILING (0.60), not a zero multiplier: a fuzzy
|
||||
* score is similarity * TIER.FUZZY, always inside (0, 0.60).
|
||||
*
|
||||
* S1 tier disjointness + exact literals
|
||||
* S2 min-length guard fires BEFORE the exact tier
|
||||
* S3 exact outranks fuzzy, always
|
||||
* S4 prefix tier (mid-word prefix)
|
||||
* S5 token-set tier (token-boundary superset; non-subset is refused)
|
||||
* S6 fuzzy residual arithmetic + the 0.72 admission boundary
|
||||
* S7 matchedAlias provenance (the RAW alias string comes back)
|
||||
* Contract under test:
|
||||
* S1 tier disjointness — EXACT > ALIAS_EXACT > PREFIX > TOKEN_SET >
|
||||
* FUZZY, with FUZZY the 0.60 CEILING of its band (never a 0.00
|
||||
* multiplier) and every admitted fuzzy score strictly inside (0, 0.60)
|
||||
* S2 min-length guard — either side < 3 chars scores 0, EVEN on an
|
||||
* exact match (the guard fires first — Pitfall 5)
|
||||
* S3 exact outranks fuzzy — always
|
||||
* S4 prefix tier — a mid-word prefix lands exactly on TIER.PREFIX
|
||||
* S5 token-set tier — a query whose every token is present in the
|
||||
* candidate's tokens is TOKEN_SET; a non-subset must NOT be
|
||||
* S6 fuzzy residual + admission boundary — 'kclc'/'klcc' is
|
||||
* 0.75 * TIER.FUZZY exactly; sim 0.75 >= 0.72 admits while the
|
||||
* 'penang'/'padang' shape (sim 0.667) stays below FUZZY_ADMIT_SIMILARITY
|
||||
* S7 matchedAlias provenance — scoreCandidate returns the RAW alias
|
||||
* string that produced the score, and an exact alias hit rides the
|
||||
* ALIAS_EXACT band (so a name-exact on another entry outranks it)
|
||||
*
|
||||
* Run: npm run search:test (exit 0 = all green)
|
||||
*/
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "./harness.mjs";
|
||||
|
||||
// Lazy score load — missing module => per-case ERR_MODULE_NOT_FOUND (the RED state).
|
||||
// Lazy module loads — missing modules => per-case ERR_MODULE_NOT_FOUND (the RED state).
|
||||
let scorePromise = null;
|
||||
function loadScore() {
|
||||
if (!scorePromise) {
|
||||
@@ -36,8 +35,6 @@ function loadScore() {
|
||||
}
|
||||
return scorePromise;
|
||||
}
|
||||
|
||||
// Lazy distance load (S6 pins the hand-computed Penang/Padang OSA distance).
|
||||
let distancePromise = null;
|
||||
function loadDamerau() {
|
||||
if (!distancePromise) {
|
||||
@@ -46,71 +43,92 @@ function loadDamerau() {
|
||||
return distancePromise;
|
||||
}
|
||||
|
||||
test("S1 tier disjointness — exact literals, strictly descending bands, FUZZY is the 0.60 ceiling", async () => {
|
||||
const { TIER } = await loadScore();
|
||||
assert.equal(TIER.EXACT, 1.0);
|
||||
assert.equal(TIER.ALIAS_EXACT, 0.9);
|
||||
assert.equal(TIER.PREFIX, 0.75);
|
||||
assert.equal(TIER.TOKEN_SET, 0.65);
|
||||
assert.equal(TIER.FUZZY, 0.6);
|
||||
assert.ok(TIER.EXACT > TIER.ALIAS_EXACT);
|
||||
assert.ok(TIER.ALIAS_EXACT > TIER.PREFIX);
|
||||
assert.ok(TIER.PREFIX > TIER.TOKEN_SET);
|
||||
assert.ok(TIER.TOKEN_SET > TIER.FUZZY);
|
||||
const { scoreOne } = await loadScore();
|
||||
const fuzzy = scoreOne("kclc", "klcc");
|
||||
assert.ok(fuzzy > 0, "a near-miss fuzzy score is strictly positive");
|
||||
assert.ok(fuzzy < TIER.FUZZY, "a fuzzy score stays under the 0.60 ceiling");
|
||||
/** The whole-string similarity law the scorer uses: 1 - damerau/maxLen. */
|
||||
async function sim(a, b) {
|
||||
const damerau = await loadDamerau();
|
||||
return 1 - damerau(a, b) / Math.max(a.length, b.length);
|
||||
}
|
||||
|
||||
test("S1 tier disjointness — strictly descending bands, FUZZY is the 0.60 ceiling", async () => {
|
||||
const { TIER, scoreOne } = await loadScore();
|
||||
assert.ok(TIER.EXACT > TIER.ALIAS_EXACT, "EXACT > ALIAS_EXACT");
|
||||
assert.ok(TIER.ALIAS_EXACT > TIER.PREFIX, "ALIAS_EXACT > PREFIX");
|
||||
assert.ok(TIER.PREFIX > TIER.TOKEN_SET, "PREFIX > TOKEN_SET");
|
||||
assert.ok(TIER.TOKEN_SET > TIER.FUZZY, "TOKEN_SET > FUZZY");
|
||||
assert.equal(TIER.FUZZY, 0.60, "FUZZY is the band ceiling, not a 0.00 multiplier");
|
||||
const fuzzyScore = scoreOne("kclc", "klcc");
|
||||
assert.ok(fuzzyScore > 0, "an admitted fuzzy score is > 0");
|
||||
assert.ok(fuzzyScore < TIER.FUZZY, "an admitted fuzzy score is < 0.60 — a sim of 1.0 would be EXACT");
|
||||
});
|
||||
|
||||
test("S2 min-length guard — either side under 3 chars scores 0, even on an exact match", async () => {
|
||||
test("S2 min-length guard — sub-3-char sides score 0 even on an exact match", async () => {
|
||||
const { scoreOne } = await loadScore();
|
||||
assert.equal(scoreOne("ab", "abcd"), 0);
|
||||
assert.equal(scoreOne("ab", "abcd"), 0, "short query is refused");
|
||||
assert.equal(scoreOne("kl", "kl"), 0, "the guard fires BEFORE the exact tier");
|
||||
});
|
||||
|
||||
test("S3 exact outranks fuzzy — always", async () => {
|
||||
const { scoreOne, TIER } = await loadScore();
|
||||
assert.ok(scoreOne("klcc", "klcc") > scoreOne("klcc", "klca"));
|
||||
assert.ok(scoreOne("klcc", "klcc") > TIER.FUZZY);
|
||||
const { TIER, scoreOne } = await loadScore();
|
||||
assert.ok(scoreOne("klcc", "klcc") > scoreOne("klcc", "klca"), "exact beats the near-miss");
|
||||
assert.ok(scoreOne("klcc", "klcc") > TIER.FUZZY, "EXACT sits above the whole fuzzy band");
|
||||
});
|
||||
|
||||
test("S4 prefix tier — mid-word prefix hits PREFIX", async () => {
|
||||
const { scoreOne, TIER } = await loadScore();
|
||||
test("S4 prefix tier — 'jalan amp' vs 'jalan ampang' lands exactly on TIER.PREFIX", async () => {
|
||||
const { TIER, scoreOne } = await loadScore();
|
||||
assert.equal(scoreOne("jalan amp", "jalan ampang"), TIER.PREFIX);
|
||||
});
|
||||
|
||||
test("S5 token-set tier — token-boundary superset is TOKEN_SET; a non-subset never is", async () => {
|
||||
const { scoreOne, TIER } = await loadScore();
|
||||
assert.equal(scoreOne("taman bangsar", "taman bangsar kib"), TIER.TOKEN_SET);
|
||||
const nonSubset = scoreOne("taman bangsar", "taman bukit");
|
||||
assert.notEqual(nonSubset, TIER.TOKEN_SET);
|
||||
assert.ok(nonSubset >= 0 && nonSubset < TIER.FUZZY, "a non-subset lands in the residual band, never TOKEN_SET");
|
||||
});
|
||||
|
||||
test("S6 fuzzy residual + admission boundary — 0.75 admits, the Penang/Padang shape (0.667) refuses", async () => {
|
||||
const { scoreOne, TIER, FUZZY_ADMIT_SIMILARITY } = await loadScore();
|
||||
const damerau = await loadDamerau();
|
||||
// "kclc" vs "klcc": one transposition over 4 chars -> sim exactly 0.75.
|
||||
assert.equal(scoreOne("kclc", "klcc"), 0.75 * TIER.FUZZY);
|
||||
const klccSim = 1 - 1 / 4;
|
||||
assert.equal(klccSim, 0.75);
|
||||
assert.ok(klccSim >= FUZZY_ADMIT_SIMILARITY, "sim 0.75 reaches the 0.72 admission floor");
|
||||
// "penang" vs "padang": two substitutions over 6 chars -> sim 0.667, refused.
|
||||
const penangPadangDistance = damerau("penang", "padang");
|
||||
const penangSim = 1 - penangPadangDistance / 6;
|
||||
assert.equal(penangPadangDistance, 2);
|
||||
assert.ok(penangSim < FUZZY_ADMIT_SIMILARITY, "sim 0.667 stays below the 0.72 admission floor");
|
||||
assert.ok(FUZZY_ADMIT_SIMILARITY === 0.72);
|
||||
});
|
||||
|
||||
test("S7 matchedAlias provenance — the RAW alias string comes back", async () => {
|
||||
const { scoreCandidate, TIER } = await loadScore();
|
||||
const hit = scoreCandidate("klcc", ["Suria KLCC", "KLCC"]);
|
||||
assert.ok(hit !== null);
|
||||
assert.equal(hit.matchedAlias, "KLCC", "the matched alias is the RAW string, not the normalized form");
|
||||
assert.ok(
|
||||
hit.score === TIER.EXACT || hit.score === TIER.ALIAS_EXACT,
|
||||
`an exact alias match lands in the exact/alias-exact band, got ${hit.score}`,
|
||||
test("S5 token-set tier — full token subset is TOKEN_SET; a non-subset is not", async () => {
|
||||
const { TIER, scoreOne } = await loadScore();
|
||||
assert.equal(
|
||||
scoreOne("taman bangsar", "taman bangsar kib"),
|
||||
TIER.TOKEN_SET,
|
||||
"every query token present in the candidate's tokens",
|
||||
);
|
||||
const nonSubset = scoreOne("taman bangsar", "taman bukit");
|
||||
assert.notEqual(nonSubset, TIER.TOKEN_SET, "a non-subset must NOT ride the token-set band");
|
||||
assert.ok(nonSubset < TIER.FUZZY, "it falls into the fuzzy band strictly below the ceiling");
|
||||
});
|
||||
|
||||
test("S6 fuzzy residual + admission boundary — 0.75 admits, the 0.667 shape does not", async () => {
|
||||
const { TIER, FUZZY_ADMIT_SIMILARITY, scoreOne } = await loadScore();
|
||||
assert.equal(scoreOne("kclc", "klcc"), 0.75 * TIER.FUZZY, "damerau 1 over len 4 -> sim 0.75 exactly");
|
||||
const typoSim = await sim("kclc", "klcc");
|
||||
assert.equal(typoSim, 0.75);
|
||||
assert.ok(typoSim >= FUZZY_ADMIT_SIMILARITY, "the KCLC->KLCC typo bucket is ADMITTED");
|
||||
const negativeSim = await sim("penang", "padang");
|
||||
assert.equal(negativeSim, 1 - 2 / 6, "two substitutions over 6 chars -> sim 0.667");
|
||||
assert.ok(negativeSim < FUZZY_ADMIT_SIMILARITY, "the Penang/Padang near-miss shape is REFUSED");
|
||||
});
|
||||
|
||||
test("S7 matchedAlias provenance — the RAW alias string rides the hit", async () => {
|
||||
const { TIER, FUZZY_ADMIT_SIMILARITY, scoreCandidate } = await loadScore();
|
||||
const aliasHit = scoreCandidate("klcc", ["Suria KLCC", "KLCC"]);
|
||||
assert.notEqual(aliasHit, null, "an exact alias match must score");
|
||||
assert.equal(aliasHit.matchedAlias, "KLCC", "provenance is the raw, un-normalized alias");
|
||||
assert.ok(
|
||||
aliasHit.score === TIER.ALIAS_EXACT || aliasHit.score === TIER.EXACT,
|
||||
`an exact alias hit rides the exact/alias-exact band, got ${aliasHit.score}`,
|
||||
);
|
||||
assert.equal(aliasHit.score, TIER.ALIAS_EXACT, "an alias exact (name differs) is ALIAS_EXACT");
|
||||
|
||||
const nameHit = scoreCandidate("suria klcc", ["Suria KLCC", "KLCC"]);
|
||||
assert.notEqual(nameHit, null);
|
||||
assert.equal(nameHit.score, TIER.EXACT, "a name exact is the EXACT band");
|
||||
assert.equal(nameHit.matchedAlias, "Suria KLCC", "the name produced it");
|
||||
|
||||
// Out-of-area: scoreCandidate still SCORES ("Kuching" lands somewhere in
|
||||
// the fuzzy band) — but its SIMILARITY stays below FUZZY_ADMIT_SIMILARITY,
|
||||
// so the source refuses it. Admission is source.ts's job, not scoring's.
|
||||
const refused = scoreCandidate("kuching", ["Suria KLCC", "KLCC"]);
|
||||
if (refused !== null) {
|
||||
assert.ok(
|
||||
refused.score < TIER.FUZZY,
|
||||
"the out-of-area shape must not ride a cheap tier",
|
||||
);
|
||||
assert.ok(
|
||||
refused.score / TIER.FUZZY < FUZZY_ADMIT_SIMILARITY,
|
||||
"the out-of-area similarity must stay below the 0.72 admission floor",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user