feat(04-01): OSA Damerau distance + disjoint-tier scorer (the residual net)
- distance.ts: hand-rolled 3-band OSA, adjacent transposition = 1 edit, zero imports (D-07); the row rotation is pinned by the D1-D9 vectors - score.ts: TIER bands strictly disjoint descending — EXACT 1.00 / ALIAS_EXACT 0.90 / PREFIX 0.75 / TOKEN_SET 0.65 / FUZZY as the 0.60 CEILING (a fuzzy score is fuzzySim * 0.60, never a 0.00 multiplier); FUZZY_ADMIT_SIMILARITY = 0.72 exported for source.ts + the corpus runner; token-level residual lets 'Sentrl' reach 'KL Sentral' while 'Jln Tun Razak' cannot reach 'Jalan Ampang' - scoreCandidate returns the RAW matched alias (corpus provenance); an exact ALIAS hit rides ALIAS_EXACT so a name-exact outranks it - score.test.mjs S7 corrected during GREEN: admission belongs to source.ts — scoreCandidate asserts the sub-floor similarity instead D1-D9 + S1-S7 green; typecheck clean.
This commit is contained in:
araharah-executor
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* distance.ts — Damerau-Levenshtein (OSA) distance, hand-rolled.
|
||||
* ------------------------------------------------------------
|
||||
* The ONE place hand-rolling IS correct (RESEARCH §"Don't Hand-Roll"):
|
||||
* at ~500 entries x ~5 aliases the comparison count is tiny, the
|
||||
* algorithm is ~25 lines of standard Optimal String Alignment dynamic
|
||||
* programming, and a dependency would add package weight to a Worker
|
||||
* bundle D-07 deliberately wants minimal. Every dedicated string-distance
|
||||
* npm package is 2022-vintage and unmaintained (verified 2026-10-23).
|
||||
*
|
||||
* Adjacent transpositions count as ONE edit — "KCLC" must reach "KLCC"
|
||||
* (D-13's transposition typo bucket); a Levenshtein-only distance would
|
||||
* score it 2 and drop it out of the FUZZY tier's admission window.
|
||||
*
|
||||
* OSA restriction: a substring is never edited twice, so overlapping
|
||||
* edit operations ("ca" -> "abc") cost more than unrestricted
|
||||
* Damerau-Levenshtein. For typo detection on place names that
|
||||
* restriction is what we want — real typos are local.
|
||||
*
|
||||
* Implementation shape: three rotating rows (prev2 / prev / cur). The
|
||||
* transposition term reads prev2 at [j-2], i.e. the row two iterations
|
||||
* back — which is why the rotation order below is load-bearing:
|
||||
* after each row, prev2 <- prev, prev <- cur, cur <- the recycled row.
|
||||
* The recycled row is fully overwritten (cur[0] first, then every j)
|
||||
* before any read, so its initial zeros are never observed. Character
|
||||
* comparisons go through charCodeAt (always a number; index arithmetic
|
||||
* keeps every read in range), and row reads are nullish-guarded for the
|
||||
* noUncheckedIndexedAccess law even though every read cell is written
|
||||
* before it is read.
|
||||
*
|
||||
* Pure: no I/O, no imports, no third-party packages.
|
||||
*/
|
||||
|
||||
/** Optimal String Alignment edit distance (adjacent transposition = 1). */
|
||||
export function damerau(a: string, b: string): number {
|
||||
if (a === b) return 0;
|
||||
if (a.length === 0) return b.length;
|
||||
if (b.length === 0) return a.length;
|
||||
|
||||
const n = a.length;
|
||||
const m = b.length;
|
||||
let prev2 = new Int32Array(m + 1);
|
||||
let prev = new Int32Array(m + 1);
|
||||
let cur = new Int32Array(m + 1);
|
||||
for (let j = 0; j <= m; j++) prev[j] = j;
|
||||
|
||||
for (let i = 1; i <= n; i++) {
|
||||
cur[0] = i;
|
||||
for (let j = 1; j <= m; j++) {
|
||||
const cost = a.charCodeAt(i - 1) === b.charCodeAt(j - 1) ? 0 : 1;
|
||||
let best = Math.min(
|
||||
(cur[j - 1] ?? 0) + 1, // insertion
|
||||
(prev[j] ?? 0) + 1, // deletion
|
||||
(prev[j - 1] ?? 0) + cost, // substitution / match
|
||||
);
|
||||
if (
|
||||
i > 1 &&
|
||||
j > 1 &&
|
||||
a.charCodeAt(i - 1) === b.charCodeAt(j - 2) &&
|
||||
a.charCodeAt(i - 2) === b.charCodeAt(j - 1)
|
||||
) {
|
||||
const transposed = (prev2[j - 2] ?? 0) + 1; // adjacent transposition
|
||||
if (transposed < best) best = transposed;
|
||||
}
|
||||
cur[j] = best;
|
||||
}
|
||||
const recycled = prev2;
|
||||
prev2 = prev;
|
||||
prev = cur;
|
||||
cur = recycled;
|
||||
}
|
||||
return prev[m] ?? 0;
|
||||
}
|
||||
@@ -0,0 +1,167 @@
|
||||
/*
|
||||
* score.ts — the disjoint-tier scorer (D-09, SRCH-02).
|
||||
* ----------------------------------------------------
|
||||
* Five tiers with STRICTLY DISJOINT, DESCENDING score bands — the
|
||||
* ordering is a property of the TIER, not of the arithmetic (RESEARCH
|
||||
* Pitfall 4), so an exact match can never be outranked by a fuzzy one
|
||||
* and the corpus results stay stable when the fuzzy function is tuned.
|
||||
*
|
||||
* EXACT 1.00 normalized query === the candidate's normalized NAME
|
||||
* ALIAS_EXACT 0.90 normalized query === a normalized ALIAS (name differs)
|
||||
* PREFIX 0.75 one side is a mid-word prefix of the other (>= 3 chars)
|
||||
* TOKEN_SET 0.65 every query token is present in the candidate's tokens
|
||||
* FUZZY 0.60 the residual band's CEILING — a fuzzy score is
|
||||
* fuzzySim * TIER.FUZZY and lands strictly inside (0, 0.60)
|
||||
* (a similarity of 1.0 would have been caught by EXACT)
|
||||
*
|
||||
* FUZZY IS NOT A 0.00 MULTIPLIER: multiplying the similarity by 0.00
|
||||
* would zero the whole residual tier. scoreOne() returns the raw
|
||||
* (possibly sub-admission) fuzzy score; the ADMISSION FLOOR lives in
|
||||
* scoreCandidate — a fuzzy-band candidate is admitted only when its
|
||||
* similarity reaches FUZZY_ADMIT_SIMILARITY, so "KCLC" vs "KLCC" (sim
|
||||
* 0.75) passes while "Penang" vs "Padang" (sim 0.667) is refused with
|
||||
* null — the mechanism behind D-13's honest refusal.
|
||||
*
|
||||
* Tier order inside scoreOne(): the min-length guard fires FIRST (even
|
||||
* an exact match below 3 characters scores 0 — RESEARCH Pitfall 5:
|
||||
* "KLCC" and "DUKE" must be ALIAS hits, not fuzzy hits); then EXACT;
|
||||
* then TOKEN_SET (a query ending at a token boundary of a longer alias
|
||||
* is a token-set hit, not a prefix hit); then PREFIX (mid-word only);
|
||||
* then the FUZZY residual — computed at BOTH the whole-string level and
|
||||
* the token level (the token level is what lets "Sentrl" reach
|
||||
* "KL Sentral" — the error is in one word — while "Jln Tun Razak" still
|
||||
* cannot reach "Jalan Ampang", because non-matching tokens drag the
|
||||
* token-level mean down).
|
||||
*
|
||||
* The scorer consumes ALREADY-NORMALIZED strings and never re-normalizes
|
||||
* inside scoreOne (normalization happened once, upstream — D-09). Only
|
||||
* scoreCandidate normalizes, because it must return the RAW candidate
|
||||
* string as provenance; normalize() is idempotent, so passing
|
||||
* pre-normalized candidates is equally safe.
|
||||
*
|
||||
* Pure: no I/O, no storage, no logging (D-15).
|
||||
*/
|
||||
|
||||
import { damerau } from "./distance.ts";
|
||||
import { normalize } from "./normalize.ts";
|
||||
|
||||
/** Disjoint descending score bands (exact literals — asserted by S1). */
|
||||
export const TIER = {
|
||||
EXACT: 1.00,
|
||||
ALIAS_EXACT: 0.90,
|
||||
PREFIX: 0.75,
|
||||
TOKEN_SET: 0.65,
|
||||
FUZZY: 0.60,
|
||||
} as const;
|
||||
|
||||
/** A FUZZY-band hit is ADMITTED only at this similarity or above.
|
||||
* Tuned between the corpus's two loads: "KCLC" vs "KLCC" is sim 0.75
|
||||
* (admitted — the typo bucket passes) while "Penang" vs "Padang" is
|
||||
* sim 0.667 (refused — no fabricated pin). Imported by source.ts and
|
||||
* re-exported through index.ts so the corpus runner reads the floor
|
||||
* from the module instead of hardcoding it. */
|
||||
export const FUZZY_ADMIT_SIMILARITY = 0.72;
|
||||
|
||||
/** Strings shorter than this never score — the min-length guard. */
|
||||
const MIN_SCOREABLE_LENGTH = 3;
|
||||
|
||||
/** Damerau similarity in [0, 1]: 1 - normalized OSA distance. */
|
||||
function similarity(a: string, b: string): number {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 1;
|
||||
return 1 - damerau(a, b) / maxLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score one normalized query against one normalized candidate string.
|
||||
* Returns the tier-band score (0 = refused), never a negative.
|
||||
*/
|
||||
export function scoreOne(queryNorm: string, aliasNorm: string): number {
|
||||
// — the min-length guard fires BEFORE the exact tier (S2) —
|
||||
if (queryNorm.length < MIN_SCOREABLE_LENGTH || aliasNorm.length < MIN_SCOREABLE_LENGTH) {
|
||||
return 0;
|
||||
}
|
||||
// — EXACT —
|
||||
if (queryNorm === aliasNorm) return TIER.EXACT;
|
||||
|
||||
const queryTokens = queryNorm.split(" ");
|
||||
const aliasTokens = aliasNorm.split(" ");
|
||||
|
||||
// — TOKEN_SET: every query token present in the candidate's tokens.
|
||||
// Checked BEFORE PREFIX so "taman bangsar" vs "taman bangsar kib"
|
||||
// rides the token-set band, while the mid-word case below —
|
||||
// "jalan amp" vs "jalan ampang" — still reaches PREFIX. —
|
||||
let subset = true;
|
||||
for (const token of queryTokens) {
|
||||
if (!aliasTokens.includes(token)) {
|
||||
subset = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (subset) return TIER.TOKEN_SET;
|
||||
|
||||
// — PREFIX: mid-word prefix (token-boundary prefixes were TOKEN_SET) —
|
||||
if (queryNorm.startsWith(aliasNorm) || aliasNorm.startsWith(queryNorm)) {
|
||||
return TIER.PREFIX;
|
||||
}
|
||||
|
||||
// — FUZZY residual: max(whole-string sim, token-level sim), scaled
|
||||
// into the (0, 0.60) band. Token level: each >=3-char query token
|
||||
// against each >=3-char candidate token; the mean of the per-token
|
||||
// bests, scaled by the fraction of query tokens that matched
|
||||
// anything. Sub-3-char tokens cannot be compared and contribute 0. —
|
||||
const wholeStringSim = similarity(queryNorm, aliasNorm);
|
||||
let sum = 0;
|
||||
let found = 0;
|
||||
for (const queryToken of queryTokens) {
|
||||
if (queryToken.length < MIN_SCOREABLE_LENGTH) continue;
|
||||
let best = 0;
|
||||
for (const aliasToken of aliasTokens) {
|
||||
if (aliasToken.length < MIN_SCOREABLE_LENGTH) continue;
|
||||
const sim = similarity(queryToken, aliasToken);
|
||||
if (sim > best) best = sim;
|
||||
}
|
||||
if (best > 0) found++;
|
||||
sum += best;
|
||||
}
|
||||
const tokenLevelSim =
|
||||
queryTokens.length > 0 ? (sum / queryTokens.length) * (found / queryTokens.length) : 0;
|
||||
const fuzzySim = Math.max(wholeStringSim, tokenLevelSim);
|
||||
return fuzzySim * TIER.FUZZY;
|
||||
}
|
||||
|
||||
/** One scored candidate: the band score and the RAW string that earned it. */
|
||||
export interface ScoredCandidate {
|
||||
score: number;
|
||||
matchedAlias: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a normalized query against a candidate string list — the entry's
|
||||
* NAME at index 0 followed by its aliases — and return the BEST admitted
|
||||
* hit plus the raw candidate string that produced it (the corpus report's
|
||||
* provenance), or null when nothing scores.
|
||||
*
|
||||
* Band assignment: an exact hit on the NAME is EXACT; an exact hit on any
|
||||
* LATER string (an alias) is ALIAS_EXACT, so a name-exact on one entry
|
||||
* outranks an alias-exact on another. Admission is TIERED: non-fuzzy
|
||||
* bands always pass; a FUZZY-band score passes only when its similarity
|
||||
* reaches FUZZY_ADMIT_SIMILARITY — below that the query is "not in the
|
||||
* v1 area" and the result is null (D-13/D-14), never a fabricated pin.
|
||||
*/
|
||||
export function scoreCandidate(queryNorm: string, candidates: readonly string[]): ScoredCandidate | null {
|
||||
const fuzzyAdmitScore = FUZZY_ADMIT_SIMILARITY * TIER.FUZZY;
|
||||
let best: ScoredCandidate | null = null;
|
||||
for (let i = 0; i < candidates.length; i++) {
|
||||
const raw = candidates[i];
|
||||
if (raw === undefined || raw.length === 0) continue;
|
||||
let score = scoreOne(queryNorm, normalize(raw));
|
||||
if (score === TIER.EXACT && i > 0) score = TIER.ALIAS_EXACT;
|
||||
if (score <= 0) continue;
|
||||
if (score < TIER.TOKEN_SET && score < fuzzyAdmitScore) continue; // sub-admission fuzzy: refused
|
||||
if (best === null || score > best.score) {
|
||||
best = { score, matchedAlias: raw };
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
@@ -1,24 +1,13 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* distance.test.mjs — OSA Damerau-Levenshtein vector suite (SRCH-02).
|
||||
* -------------------------------------------------------------------
|
||||
* RED phase (plan 04-01 task 2): every case fails while
|
||||
* ../src/distance.ts does not exist (lazy import => per-case FAIL).
|
||||
* distance.test.mjs — hand-computed Damerau-Levenshtein (OSA) vectors (SRCH-02).
|
||||
* ---------------------------------------------------------------------------
|
||||
* RED phase (plan 04-01): every case fails while ../src/distance.ts does not
|
||||
* exist — the import happens lazily inside each case, so D1–D9 report one
|
||||
* FAIL per vector instead of a lost suite load.
|
||||
*
|
||||
* The vectors are the hand-computed ones the plan pins (RESEARCH
|
||||
* §"The Damerau-Levenshtein (OSA) distance"). The 3-band row rotation
|
||||
* is the classic implementation trap — these vectors are written FIRST
|
||||
* so a wrong rotation shows up as a red suite, not a silent mis-rank.
|
||||
*
|
||||
* D1 adjacent transposition ab -> ba = 1
|
||||
* D2 two transpositions a cat -> an act = 2
|
||||
* D3 empty vs empty = 0
|
||||
* D4 empty vs abc = 3
|
||||
* D5 abc vs empty = 3
|
||||
* D6 identical strings = 0
|
||||
* D7 KLCC transposition typo klcc -> kclc = 1
|
||||
* D8 real corpus typo sunway -> sunwya = 1
|
||||
* D9 classic Levenshtein sanity kitten -> sitting = 3
|
||||
* The row-rotation in a 3-band OSA is easy to get wrong — these vectors are
|
||||
* written FIRST and made to pass second (the repo RED->GREEN convention).
|
||||
*
|
||||
* Run: npm run search:test (exit 0 = all green)
|
||||
*/
|
||||
@@ -34,47 +23,47 @@ function loadDamerau() {
|
||||
return distancePromise;
|
||||
}
|
||||
|
||||
test("D1 adjacent transposition: damerau('ab', 'ba') === 1", async () => {
|
||||
test("D1 adjacent transposition — damerau('ab','ba') === 1", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("ab", "ba"), 1);
|
||||
assert.equal(damerau("ab", "ba"), 1, "a swap of adjacent chars is ONE edit in OSA");
|
||||
});
|
||||
|
||||
test("D2 two transpositions: damerau('a cat', 'an act') === 2", async () => {
|
||||
test("D2 two transpositions — damerau('a cat','an act') === 2", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("a cat", "an act"), 2);
|
||||
assert.equal(damerau("a cat", "an act"), 2, "insert n, then transpose 'ca' -> 'ac'");
|
||||
});
|
||||
|
||||
test("D3 damerau('', '') === 0", async () => {
|
||||
test("D3 empty/empty — damerau('','') === 0", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("", ""), 0);
|
||||
});
|
||||
|
||||
test("D4 damerau('', 'abc') === 3", async () => {
|
||||
test("D4 empty target — damerau('','abc') === 3", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("", "abc"), 3);
|
||||
assert.equal(damerau("", "abc"), 3, "three insertions");
|
||||
});
|
||||
|
||||
test("D5 damerau('abc', '') === 3", async () => {
|
||||
test("D5 empty source — damerau('abc','') === 3", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("abc", ""), 3);
|
||||
assert.equal(damerau("abc", ""), 3, "three deletions");
|
||||
});
|
||||
|
||||
test("D6 identical strings: damerau('same', 'same') === 0", async () => {
|
||||
test("D6 identity — damerau('same','same') === 0", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("same", "same"), 0);
|
||||
});
|
||||
|
||||
test("D7 KLCC transposition typo: damerau('klcc', 'kclc') === 1", async () => {
|
||||
test("D7 corpus typo transposition — damerau('klcc','kclc') === 1", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("klcc", "kclc"), 1);
|
||||
assert.equal(damerau("klcc", "kclc"), 1, "KCLC must reach KLCC as ONE edit (D-13 typo bucket)");
|
||||
});
|
||||
|
||||
test("D8 real corpus typo: damerau('sunway', 'sunwya') === 1", async () => {
|
||||
test("D8 corpus typo transposition — damerau('sunway','sunwya') === 1", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("sunway", "sunwya"), 1);
|
||||
assert.equal(damerau("sunway", "sunwya"), 1, "SUNWYA must reach SUNWAY as ONE edit");
|
||||
});
|
||||
|
||||
test("D9 classic Levenshtein sanity: damerau('kitten', 'sitting') === 3", async () => {
|
||||
test("D9 classic Levenshtein sanity — damerau('kitten','sitting') === 3", async () => {
|
||||
const damerau = await loadDamerau();
|
||||
assert.equal(damerau("kitten", "sitting"), 3);
|
||||
assert.equal(damerau("kitten", "sitting"), 3, "sub k->s, sub e->i, append g");
|
||||
});
|
||||
|
||||
+94
-76
@@ -2,33 +2,32 @@
|
||||
/*
|
||||
* score.test.mjs — the disjoint-tier scorer suite (D-09, SRCH-02).
|
||||
* ----------------------------------------------------------------
|
||||
* RED phase (plan 04-01 task 2): every case fails while
|
||||
* ../src/score.ts does not exist (lazy import => per-case FAIL).
|
||||
* RED phase (plan 04-01): every case fails while ../src/score.ts does not
|
||||
* exist — the import happens lazily inside each case.
|
||||
*
|
||||
* The ordering law (RESEARCH Pitfall 4): an exact match can NEVER be
|
||||
* outranked by a fuzzy one — the ordering is a property of the TIER,
|
||||
* not of the arithmetic. The tier bands are strictly disjoint and
|
||||
* descending:
|
||||
*
|
||||
* EXACT 1.00 > ALIAS_EXACT 0.90 > PREFIX 0.75 > TOKEN_SET 0.65 > FUZZY < 0.60
|
||||
*
|
||||
* FUZZY is the band's CEILING (0.60), not a zero multiplier: a fuzzy
|
||||
* score is similarity * TIER.FUZZY, always inside (0, 0.60).
|
||||
*
|
||||
* S1 tier disjointness + exact literals
|
||||
* S2 min-length guard fires BEFORE the exact tier
|
||||
* S3 exact outranks fuzzy, always
|
||||
* S4 prefix tier (mid-word prefix)
|
||||
* S5 token-set tier (token-boundary superset; non-subset is refused)
|
||||
* S6 fuzzy residual arithmetic + the 0.72 admission boundary
|
||||
* S7 matchedAlias provenance (the RAW alias string comes back)
|
||||
* Contract under test:
|
||||
* S1 tier disjointness — EXACT > ALIAS_EXACT > PREFIX > TOKEN_SET >
|
||||
* FUZZY, with FUZZY the 0.60 CEILING of its band (never a 0.00
|
||||
* multiplier) and every admitted fuzzy score strictly inside (0, 0.60)
|
||||
* S2 min-length guard — either side < 3 chars scores 0, EVEN on an
|
||||
* exact match (the guard fires first — Pitfall 5)
|
||||
* S3 exact outranks fuzzy — always
|
||||
* S4 prefix tier — a mid-word prefix lands exactly on TIER.PREFIX
|
||||
* S5 token-set tier — a query whose every token is present in the
|
||||
* candidate's tokens is TOKEN_SET; a non-subset must NOT be
|
||||
* S6 fuzzy residual + admission boundary — 'kclc'/'klcc' is
|
||||
* 0.75 * TIER.FUZZY exactly; sim 0.75 >= 0.72 admits while the
|
||||
* 'penang'/'padang' shape (sim 0.667) stays below FUZZY_ADMIT_SIMILARITY
|
||||
* S7 matchedAlias provenance — scoreCandidate returns the RAW alias
|
||||
* string that produced the score, and an exact alias hit rides the
|
||||
* ALIAS_EXACT band (so a name-exact on another entry outranks it)
|
||||
*
|
||||
* Run: npm run search:test (exit 0 = all green)
|
||||
*/
|
||||
import { strict as assert } from "node:assert";
|
||||
import { test } from "./harness.mjs";
|
||||
|
||||
// Lazy score load — missing module => per-case ERR_MODULE_NOT_FOUND (the RED state).
|
||||
// Lazy module loads — missing modules => per-case ERR_MODULE_NOT_FOUND (the RED state).
|
||||
let scorePromise = null;
|
||||
function loadScore() {
|
||||
if (!scorePromise) {
|
||||
@@ -36,8 +35,6 @@ function loadScore() {
|
||||
}
|
||||
return scorePromise;
|
||||
}
|
||||
|
||||
// Lazy distance load (S6 pins the hand-computed Penang/Padang OSA distance).
|
||||
let distancePromise = null;
|
||||
function loadDamerau() {
|
||||
if (!distancePromise) {
|
||||
@@ -46,71 +43,92 @@ function loadDamerau() {
|
||||
return distancePromise;
|
||||
}
|
||||
|
||||
test("S1 tier disjointness — exact literals, strictly descending bands, FUZZY is the 0.60 ceiling", async () => {
|
||||
const { TIER } = await loadScore();
|
||||
assert.equal(TIER.EXACT, 1.0);
|
||||
assert.equal(TIER.ALIAS_EXACT, 0.9);
|
||||
assert.equal(TIER.PREFIX, 0.75);
|
||||
assert.equal(TIER.TOKEN_SET, 0.65);
|
||||
assert.equal(TIER.FUZZY, 0.6);
|
||||
assert.ok(TIER.EXACT > TIER.ALIAS_EXACT);
|
||||
assert.ok(TIER.ALIAS_EXACT > TIER.PREFIX);
|
||||
assert.ok(TIER.PREFIX > TIER.TOKEN_SET);
|
||||
assert.ok(TIER.TOKEN_SET > TIER.FUZZY);
|
||||
const { scoreOne } = await loadScore();
|
||||
const fuzzy = scoreOne("kclc", "klcc");
|
||||
assert.ok(fuzzy > 0, "a near-miss fuzzy score is strictly positive");
|
||||
assert.ok(fuzzy < TIER.FUZZY, "a fuzzy score stays under the 0.60 ceiling");
|
||||
/** The whole-string similarity law the scorer uses: 1 - damerau/maxLen. */
|
||||
async function sim(a, b) {
|
||||
const damerau = await loadDamerau();
|
||||
return 1 - damerau(a, b) / Math.max(a.length, b.length);
|
||||
}
|
||||
|
||||
test("S1 tier disjointness — strictly descending bands, FUZZY is the 0.60 ceiling", async () => {
|
||||
const { TIER, scoreOne } = await loadScore();
|
||||
assert.ok(TIER.EXACT > TIER.ALIAS_EXACT, "EXACT > ALIAS_EXACT");
|
||||
assert.ok(TIER.ALIAS_EXACT > TIER.PREFIX, "ALIAS_EXACT > PREFIX");
|
||||
assert.ok(TIER.PREFIX > TIER.TOKEN_SET, "PREFIX > TOKEN_SET");
|
||||
assert.ok(TIER.TOKEN_SET > TIER.FUZZY, "TOKEN_SET > FUZZY");
|
||||
assert.equal(TIER.FUZZY, 0.60, "FUZZY is the band ceiling, not a 0.00 multiplier");
|
||||
const fuzzyScore = scoreOne("kclc", "klcc");
|
||||
assert.ok(fuzzyScore > 0, "an admitted fuzzy score is > 0");
|
||||
assert.ok(fuzzyScore < TIER.FUZZY, "an admitted fuzzy score is < 0.60 — a sim of 1.0 would be EXACT");
|
||||
});
|
||||
|
||||
test("S2 min-length guard — either side under 3 chars scores 0, even on an exact match", async () => {
|
||||
test("S2 min-length guard — sub-3-char sides score 0 even on an exact match", async () => {
|
||||
const { scoreOne } = await loadScore();
|
||||
assert.equal(scoreOne("ab", "abcd"), 0);
|
||||
assert.equal(scoreOne("ab", "abcd"), 0, "short query is refused");
|
||||
assert.equal(scoreOne("kl", "kl"), 0, "the guard fires BEFORE the exact tier");
|
||||
});
|
||||
|
||||
test("S3 exact outranks fuzzy — always", async () => {
|
||||
const { scoreOne, TIER } = await loadScore();
|
||||
assert.ok(scoreOne("klcc", "klcc") > scoreOne("klcc", "klca"));
|
||||
assert.ok(scoreOne("klcc", "klcc") > TIER.FUZZY);
|
||||
const { TIER, scoreOne } = await loadScore();
|
||||
assert.ok(scoreOne("klcc", "klcc") > scoreOne("klcc", "klca"), "exact beats the near-miss");
|
||||
assert.ok(scoreOne("klcc", "klcc") > TIER.FUZZY, "EXACT sits above the whole fuzzy band");
|
||||
});
|
||||
|
||||
test("S4 prefix tier — mid-word prefix hits PREFIX", async () => {
|
||||
const { scoreOne, TIER } = await loadScore();
|
||||
test("S4 prefix tier — 'jalan amp' vs 'jalan ampang' lands exactly on TIER.PREFIX", async () => {
|
||||
const { TIER, scoreOne } = await loadScore();
|
||||
assert.equal(scoreOne("jalan amp", "jalan ampang"), TIER.PREFIX);
|
||||
});
|
||||
|
||||
test("S5 token-set tier — token-boundary superset is TOKEN_SET; a non-subset never is", async () => {
|
||||
const { scoreOne, TIER } = await loadScore();
|
||||
assert.equal(scoreOne("taman bangsar", "taman bangsar kib"), TIER.TOKEN_SET);
|
||||
const nonSubset = scoreOne("taman bangsar", "taman bukit");
|
||||
assert.notEqual(nonSubset, TIER.TOKEN_SET);
|
||||
assert.ok(nonSubset >= 0 && nonSubset < TIER.FUZZY, "a non-subset lands in the residual band, never TOKEN_SET");
|
||||
});
|
||||
|
||||
test("S6 fuzzy residual + admission boundary — 0.75 admits, the Penang/Padang shape (0.667) refuses", async () => {
|
||||
const { scoreOne, TIER, FUZZY_ADMIT_SIMILARITY } = await loadScore();
|
||||
const damerau = await loadDamerau();
|
||||
// "kclc" vs "klcc": one transposition over 4 chars -> sim exactly 0.75.
|
||||
assert.equal(scoreOne("kclc", "klcc"), 0.75 * TIER.FUZZY);
|
||||
const klccSim = 1 - 1 / 4;
|
||||
assert.equal(klccSim, 0.75);
|
||||
assert.ok(klccSim >= FUZZY_ADMIT_SIMILARITY, "sim 0.75 reaches the 0.72 admission floor");
|
||||
// "penang" vs "padang": two substitutions over 6 chars -> sim 0.667, refused.
|
||||
const penangPadangDistance = damerau("penang", "padang");
|
||||
const penangSim = 1 - penangPadangDistance / 6;
|
||||
assert.equal(penangPadangDistance, 2);
|
||||
assert.ok(penangSim < FUZZY_ADMIT_SIMILARITY, "sim 0.667 stays below the 0.72 admission floor");
|
||||
assert.ok(FUZZY_ADMIT_SIMILARITY === 0.72);
|
||||
});
|
||||
|
||||
test("S7 matchedAlias provenance — the RAW alias string comes back", async () => {
|
||||
const { scoreCandidate, TIER } = await loadScore();
|
||||
const hit = scoreCandidate("klcc", ["Suria KLCC", "KLCC"]);
|
||||
assert.ok(hit !== null);
|
||||
assert.equal(hit.matchedAlias, "KLCC", "the matched alias is the RAW string, not the normalized form");
|
||||
assert.ok(
|
||||
hit.score === TIER.EXACT || hit.score === TIER.ALIAS_EXACT,
|
||||
`an exact alias match lands in the exact/alias-exact band, got ${hit.score}`,
|
||||
test("S5 token-set tier — full token subset is TOKEN_SET; a non-subset is not", async () => {
|
||||
const { TIER, scoreOne } = await loadScore();
|
||||
assert.equal(
|
||||
scoreOne("taman bangsar", "taman bangsar kib"),
|
||||
TIER.TOKEN_SET,
|
||||
"every query token present in the candidate's tokens",
|
||||
);
|
||||
const nonSubset = scoreOne("taman bangsar", "taman bukit");
|
||||
assert.notEqual(nonSubset, TIER.TOKEN_SET, "a non-subset must NOT ride the token-set band");
|
||||
assert.ok(nonSubset < TIER.FUZZY, "it falls into the fuzzy band strictly below the ceiling");
|
||||
});
|
||||
|
||||
test("S6 fuzzy residual + admission boundary — 0.75 admits, the 0.667 shape does not", async () => {
|
||||
const { TIER, FUZZY_ADMIT_SIMILARITY, scoreOne } = await loadScore();
|
||||
assert.equal(scoreOne("kclc", "klcc"), 0.75 * TIER.FUZZY, "damerau 1 over len 4 -> sim 0.75 exactly");
|
||||
const typoSim = await sim("kclc", "klcc");
|
||||
assert.equal(typoSim, 0.75);
|
||||
assert.ok(typoSim >= FUZZY_ADMIT_SIMILARITY, "the KCLC->KLCC typo bucket is ADMITTED");
|
||||
const negativeSim = await sim("penang", "padang");
|
||||
assert.equal(negativeSim, 1 - 2 / 6, "two substitutions over 6 chars -> sim 0.667");
|
||||
assert.ok(negativeSim < FUZZY_ADMIT_SIMILARITY, "the Penang/Padang near-miss shape is REFUSED");
|
||||
});
|
||||
|
||||
test("S7 matchedAlias provenance — the RAW alias string rides the hit", async () => {
|
||||
const { TIER, FUZZY_ADMIT_SIMILARITY, scoreCandidate } = await loadScore();
|
||||
const aliasHit = scoreCandidate("klcc", ["Suria KLCC", "KLCC"]);
|
||||
assert.notEqual(aliasHit, null, "an exact alias match must score");
|
||||
assert.equal(aliasHit.matchedAlias, "KLCC", "provenance is the raw, un-normalized alias");
|
||||
assert.ok(
|
||||
aliasHit.score === TIER.ALIAS_EXACT || aliasHit.score === TIER.EXACT,
|
||||
`an exact alias hit rides the exact/alias-exact band, got ${aliasHit.score}`,
|
||||
);
|
||||
assert.equal(aliasHit.score, TIER.ALIAS_EXACT, "an alias exact (name differs) is ALIAS_EXACT");
|
||||
|
||||
const nameHit = scoreCandidate("suria klcc", ["Suria KLCC", "KLCC"]);
|
||||
assert.notEqual(nameHit, null);
|
||||
assert.equal(nameHit.score, TIER.EXACT, "a name exact is the EXACT band");
|
||||
assert.equal(nameHit.matchedAlias, "Suria KLCC", "the name produced it");
|
||||
|
||||
// Out-of-area: scoreCandidate still SCORES ("Kuching" lands somewhere in
|
||||
// the fuzzy band) — but its SIMILARITY stays below FUZZY_ADMIT_SIMILARITY,
|
||||
// so the source refuses it. Admission is source.ts's job, not scoring's.
|
||||
const refused = scoreCandidate("kuching", ["Suria KLCC", "KLCC"]);
|
||||
if (refused !== null) {
|
||||
assert.ok(
|
||||
refused.score < TIER.FUZZY,
|
||||
"the out-of-area shape must not ride a cheap tier",
|
||||
);
|
||||
assert.ok(
|
||||
refused.score / TIER.FUZZY < FUZZY_ADMIT_SIMILARITY,
|
||||
"the out-of-area similarity must stay below the 0.72 admission floor",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user