feat(04-01): OSA Damerau distance + disjoint-tier scorer (the residual net)
- distance.ts: hand-rolled 3-band OSA, adjacent transposition = 1 edit, zero imports (D-07); the row rotation is pinned by the D1-D9 vectors - score.ts: TIER bands strictly disjoint descending — EXACT 1.00 / ALIAS_EXACT 0.90 / PREFIX 0.75 / TOKEN_SET 0.65 / FUZZY as the 0.60 CEILING (a fuzzy score is fuzzySim * 0.60, never a 0.00 multiplier); FUZZY_ADMIT_SIMILARITY = 0.72 exported for source.ts + the corpus runner; token-level residual lets 'Sentrl' reach 'KL Sentral' while 'Jln Tun Razak' cannot reach 'Jalan Ampang' - scoreCandidate returns the RAW matched alias (corpus provenance); an exact ALIAS hit rides ALIAS_EXACT so a name-exact outranks it - score.test.mjs S7 corrected during GREEN: admission belongs to source.ts — scoreCandidate asserts the sub-floor similarity instead D1-D9 + S1-S7 green; typecheck clean.
This commit is contained in:
araharah-executor
@@ -0,0 +1,167 @@
|
||||
/*
|
||||
* score.ts — the disjoint-tier scorer (D-09, SRCH-02).
|
||||
* ----------------------------------------------------
|
||||
* Five tiers with STRICTLY DISJOINT, DESCENDING score bands — the
|
||||
* ordering is a property of the TIER, not of the arithmetic (RESEARCH
|
||||
* Pitfall 4), so an exact match can never be outranked by a fuzzy one
|
||||
* and the corpus results stay stable when the fuzzy function is tuned.
|
||||
*
|
||||
* EXACT 1.00 normalized query === the candidate's normalized NAME
|
||||
* ALIAS_EXACT 0.90 normalized query === a normalized ALIAS (name differs)
|
||||
* PREFIX 0.75 one side is a mid-word prefix of the other (>= 3 chars)
|
||||
* TOKEN_SET 0.65 every query token is present in the candidate's tokens
|
||||
* FUZZY 0.60 the residual band's CEILING — a fuzzy score is
|
||||
* fuzzySim * TIER.FUZZY and lands strictly inside (0, 0.60)
|
||||
* (a similarity of 1.0 would have been caught by EXACT)
|
||||
*
|
||||
* FUZZY IS NOT A 0.00 MULTIPLIER: multiplying the similarity by 0.00
|
||||
* would zero the whole residual tier. scoreOne() returns the raw
|
||||
* (possibly sub-admission) fuzzy score; the ADMISSION FLOOR lives in
|
||||
* scoreCandidate — a fuzzy-band candidate is admitted only when its
|
||||
* similarity reaches FUZZY_ADMIT_SIMILARITY, so "KCLC" vs "KLCC" (sim
|
||||
* 0.75) passes while "Penang" vs "Padang" (sim 0.667) is refused with
|
||||
* null — the mechanism behind D-13's honest refusal.
|
||||
*
|
||||
* Tier order inside scoreOne(): the min-length guard fires FIRST (even
|
||||
* an exact match below 3 characters scores 0 — RESEARCH Pitfall 5:
|
||||
* "KLCC" and "DUKE" must be ALIAS hits, not fuzzy hits); then EXACT;
|
||||
* then TOKEN_SET (a query ending at a token boundary of a longer alias
|
||||
* is a token-set hit, not a prefix hit); then PREFIX (mid-word only);
|
||||
* then the FUZZY residual — computed at BOTH the whole-string level and
|
||||
* the token level (the token level is what lets "Sentrl" reach
|
||||
* "KL Sentral" — the error is in one word — while "Jln Tun Razak" still
|
||||
* cannot reach "Jalan Ampang", because non-matching tokens drag the
|
||||
* token-level mean down).
|
||||
*
|
||||
* The scorer consumes ALREADY-NORMALIZED strings and never re-normalizes
|
||||
* inside scoreOne (normalization happened once, upstream — D-09). Only
|
||||
* scoreCandidate normalizes, because it must return the RAW candidate
|
||||
* string as provenance; normalize() is idempotent, so passing
|
||||
* pre-normalized candidates is equally safe.
|
||||
*
|
||||
* Pure: no I/O, no storage, no logging (D-15).
|
||||
*/
|
||||
|
||||
import { damerau } from "./distance.ts";
|
||||
import { normalize } from "./normalize.ts";
|
||||
|
||||
/** Disjoint descending score bands (exact literals — asserted by S1). */
|
||||
export const TIER = {
|
||||
EXACT: 1.00,
|
||||
ALIAS_EXACT: 0.90,
|
||||
PREFIX: 0.75,
|
||||
TOKEN_SET: 0.65,
|
||||
FUZZY: 0.60,
|
||||
} as const;
|
||||
|
||||
/** A FUZZY-band hit is ADMITTED only at this similarity or above.
|
||||
* Tuned between the corpus's two loads: "KCLC" vs "KLCC" is sim 0.75
|
||||
* (admitted — the typo bucket passes) while "Penang" vs "Padang" is
|
||||
* sim 0.667 (refused — no fabricated pin). Imported by source.ts and
|
||||
* re-exported through index.ts so the corpus runner reads the floor
|
||||
* from the module instead of hardcoding it. */
|
||||
export const FUZZY_ADMIT_SIMILARITY = 0.72;
|
||||
|
||||
/** Strings shorter than this never score — the min-length guard. */
|
||||
const MIN_SCOREABLE_LENGTH = 3;
|
||||
|
||||
/** Damerau similarity in [0, 1]: 1 - normalized OSA distance. */
|
||||
function similarity(a: string, b: string): number {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 1;
|
||||
return 1 - damerau(a, b) / maxLen;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score one normalized query against one normalized candidate string.
|
||||
* Returns the tier-band score (0 = refused), never a negative.
|
||||
*/
|
||||
export function scoreOne(queryNorm: string, aliasNorm: string): number {
|
||||
// — the min-length guard fires BEFORE the exact tier (S2) —
|
||||
if (queryNorm.length < MIN_SCOREABLE_LENGTH || aliasNorm.length < MIN_SCOREABLE_LENGTH) {
|
||||
return 0;
|
||||
}
|
||||
// — EXACT —
|
||||
if (queryNorm === aliasNorm) return TIER.EXACT;
|
||||
|
||||
const queryTokens = queryNorm.split(" ");
|
||||
const aliasTokens = aliasNorm.split(" ");
|
||||
|
||||
// — TOKEN_SET: every query token present in the candidate's tokens.
|
||||
// Checked BEFORE PREFIX so "taman bangsar" vs "taman bangsar kib"
|
||||
// rides the token-set band, while the mid-word case below —
|
||||
// "jalan amp" vs "jalan ampang" — still reaches PREFIX. —
|
||||
let subset = true;
|
||||
for (const token of queryTokens) {
|
||||
if (!aliasTokens.includes(token)) {
|
||||
subset = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (subset) return TIER.TOKEN_SET;
|
||||
|
||||
// — PREFIX: mid-word prefix (token-boundary prefixes were TOKEN_SET) —
|
||||
if (queryNorm.startsWith(aliasNorm) || aliasNorm.startsWith(queryNorm)) {
|
||||
return TIER.PREFIX;
|
||||
}
|
||||
|
||||
// — FUZZY residual: max(whole-string sim, token-level sim), scaled
|
||||
// into the (0, 0.60) band. Token level: each >=3-char query token
|
||||
// against each >=3-char candidate token; the mean of the per-token
|
||||
// bests, scaled by the fraction of query tokens that matched
|
||||
// anything. Sub-3-char tokens cannot be compared and contribute 0. —
|
||||
const wholeStringSim = similarity(queryNorm, aliasNorm);
|
||||
let sum = 0;
|
||||
let found = 0;
|
||||
for (const queryToken of queryTokens) {
|
||||
if (queryToken.length < MIN_SCOREABLE_LENGTH) continue;
|
||||
let best = 0;
|
||||
for (const aliasToken of aliasTokens) {
|
||||
if (aliasToken.length < MIN_SCOREABLE_LENGTH) continue;
|
||||
const sim = similarity(queryToken, aliasToken);
|
||||
if (sim > best) best = sim;
|
||||
}
|
||||
if (best > 0) found++;
|
||||
sum += best;
|
||||
}
|
||||
const tokenLevelSim =
|
||||
queryTokens.length > 0 ? (sum / queryTokens.length) * (found / queryTokens.length) : 0;
|
||||
const fuzzySim = Math.max(wholeStringSim, tokenLevelSim);
|
||||
return fuzzySim * TIER.FUZZY;
|
||||
}
|
||||
|
||||
/** One scored candidate: the band score and the RAW string that earned it. */
|
||||
export interface ScoredCandidate {
|
||||
score: number;
|
||||
matchedAlias: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Score a normalized query against a candidate string list — the entry's
|
||||
* NAME at index 0 followed by its aliases — and return the BEST admitted
|
||||
* hit plus the raw candidate string that produced it (the corpus report's
|
||||
* provenance), or null when nothing scores.
|
||||
*
|
||||
* Band assignment: an exact hit on the NAME is EXACT; an exact hit on any
|
||||
* LATER string (an alias) is ALIAS_EXACT, so a name-exact on one entry
|
||||
* outranks an alias-exact on another. Admission is TIERED: non-fuzzy
|
||||
* bands always pass; a FUZZY-band score passes only when its similarity
|
||||
* reaches FUZZY_ADMIT_SIMILARITY — below that the query is "not in the
|
||||
* v1 area" and the result is null (D-13/D-14), never a fabricated pin.
|
||||
*/
|
||||
export function scoreCandidate(queryNorm: string, candidates: readonly string[]): ScoredCandidate | null {
|
||||
const fuzzyAdmitScore = FUZZY_ADMIT_SIMILARITY * TIER.FUZZY;
|
||||
let best: ScoredCandidate | null = null;
|
||||
for (let i = 0; i < candidates.length; i++) {
|
||||
const raw = candidates[i];
|
||||
if (raw === undefined || raw.length === 0) continue;
|
||||
let score = scoreOne(queryNorm, normalize(raw));
|
||||
if (score === TIER.EXACT && i > 0) score = TIER.ALIAS_EXACT;
|
||||
if (score <= 0) continue;
|
||||
if (score < TIER.TOKEN_SET && score < fuzzyAdmitScore) continue; // sub-admission fuzzy: refused
|
||||
if (best === null || score > best.score) {
|
||||
best = { score, matchedAlias: raw };
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
Reference in New Issue
Block a user