feat(04-01): search contracts + JUPEM abbreviation table as DATA + symmetric normalizer
- types.ts: GazetteerEntry / SearchQuery / SearchHit / SearchSource (D-03/D-07/D-08) — byId() is separate and synchronous; empty search() array = honest refusal - rules/abbreviations.ts: the vendored JUPEM table (49 rows incl. Presint→Prsn vowel-drop and MS 2039 A3 Kpg forms) as a ReadonlyArray of pairs, inverted once by a single generic builder — extends by file edit, never a code change (D-09) - normalize.ts: NFKD fold → punctuation collapse (keeps '/') → whole-token expansion (canonical = long form) → longest-first glued split (remainder >= 3) T1-T9 green; typecheck clean.
This commit is contained in:
araharah-executor
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
* normalize.ts — the Malay place-name normalizer (D-09, SRCH-02).
|
||||
* ---------------------------------------------------------------
|
||||
* Pure: no I/O, no storage, no logging (D-15). Its only import is the
|
||||
* rules table — the abbreviation table is DATA, so extending the
|
||||
* normalizer is a file edit in rules/abbreviations.ts plus a vector in
|
||||
* test/normalizer.test.mjs, never a code change here.
|
||||
*
|
||||
* Pipeline (each step separately testable):
|
||||
* 1. NFKD normalize + strip combining marks
|
||||
* 2. lowercase
|
||||
* 3. punctuation -> space; keep letters, digits, whitespace and "/"
|
||||
* (Malaysian plot shorthand "Jln 3/1", "Presint 8/1" needs it)
|
||||
* 4. collapse whitespace + trim
|
||||
* 5. whole-token abbreviation expansion (canonical form = the long
|
||||
* form, so "Jln Ampang" and "Jalan Ampang" normalize IDENTICALLY —
|
||||
* the D-09 symmetry law). Applied symmetrically to query AND entry
|
||||
* aliases (the loader normalizes both sides at index-build time).
|
||||
* 6. glued-word split — longest-starter-first against the starter
|
||||
* list, prefix position only, requiring the remainder to be >= 3
|
||||
* characters (a naive "split before any known word" corrupts words
|
||||
* that legitimately contain a starter as a substring).
|
||||
* 7. collapse whitespace + trim
|
||||
*
|
||||
* IDEMPOTENCE LAW: normalize(normalize(x)) === normalize(x) — asserted
|
||||
* by T7 of the normalizer suite. A token is expanded only when it is a
|
||||
* WHOLE token (RESEARCH Pitfall 5: "Pdang" is never expanded, only
|
||||
* "pdg"), and the ambiguity policy for multi-full-form abbreviations
|
||||
* ("kg" -> kampung/kampong) is: keep the list in table order, expand to
|
||||
* the FIRST form, and let the fuzzy-residual tier resolve the rest.
|
||||
*/
|
||||
|
||||
import { buildAbbreviationMap } from "./rules/abbreviations.ts";
|
||||
|
||||
/** Built ONCE at module scope — ~60 entries of plain data, cheap.
|
||||
* The lazy-initialization law applies to the gazetteer INDEX, not to
|
||||
* this table. */
|
||||
const ABBREVIATIONS = buildAbbreviationMap();
|
||||
|
||||
/** Glued-word starters, LONGEST FIRST, so "perkampungan" is considered
|
||||
* before "kampung" and a maximal starter always wins the split. */
|
||||
const GLUED_STARTERS = [
|
||||
"perkampungan",
|
||||
"persiaran",
|
||||
"kampung",
|
||||
"kampong",
|
||||
"presint",
|
||||
"bandar",
|
||||
"lorong",
|
||||
"sungai",
|
||||
"bukit",
|
||||
"taman",
|
||||
"jalan",
|
||||
"lebuh",
|
||||
] as const;
|
||||
|
||||
/** Minimum length of the remainder after a split point (RESEARCH: the
|
||||
* remainder guard is what keeps legitimate words intact). */
|
||||
const SPLIT_MIN_REMAINDER = 3;
|
||||
|
||||
/**
|
||||
* Normalize a raw query or alias into the canonical matching form.
|
||||
* Symmetric by construction: both sides of every comparison are run
|
||||
* through this exact function (D-09).
|
||||
*/
|
||||
export function normalize(input: string): string {
|
||||
// 1 + 2. NFKD fold, lowercase, strip combining marks.
|
||||
let s = input.normalize("NFKD").toLowerCase();
|
||||
s = s.replace(/[\u0300-\u036f]/g, "");
|
||||
// 3. Punctuation -> space; keep letters, digits, whitespace, "/".
|
||||
s = s.replace(/[^\p{L}\p{N}\s/]/gu, " ");
|
||||
// 4. Collapse whitespace + trim.
|
||||
s = s.replace(/\s+/g, " ").trim();
|
||||
if (s === "") {
|
||||
return "";
|
||||
}
|
||||
// 5. Whole-token abbreviation expansion (first listed full form wins).
|
||||
const expanded: string[] = [];
|
||||
for (const token of s.split(" ")) {
|
||||
if (token === "") {
|
||||
continue;
|
||||
}
|
||||
const fullForms = ABBREVIATIONS.get(token);
|
||||
expanded.push(fullForms === undefined ? token : (fullForms[0] ?? token));
|
||||
}
|
||||
// 6. Glued-word split, then 7. final collapse + trim.
|
||||
const parts: string[] = [];
|
||||
for (const token of expanded) {
|
||||
splitGluedToken(token, parts);
|
||||
}
|
||||
return parts.join(" ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
/** Recursively split one token at starter boundaries (prefix position
|
||||
* only, longest starter first, remainder >= SPLIT_MIN_REMAINDER chars)
|
||||
* and append the resulting pieces to `out`. */
|
||||
function splitGluedToken(token: string, out: string[]): void {
|
||||
for (const starter of GLUED_STARTERS) {
|
||||
if (token.length >= starter.length + SPLIT_MIN_REMAINDER && token.startsWith(starter)) {
|
||||
out.push(starter);
|
||||
splitGluedToken(token.slice(starter.length), out);
|
||||
return;
|
||||
}
|
||||
}
|
||||
out.push(token);
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
/*
|
||||
* abbreviations.ts — the Malay abbreviation table, as DATA (D-09).
|
||||
* --------------------------------------------------------------
|
||||
* PRIMARY SOURCE: JUPEM Toponymic Guidelines — "Abbreviations of
|
||||
* Geographical Terms and Other Words Used in Maps"
|
||||
* (Malaysia, 2017 ed.) — the normative national list.
|
||||
* TRANSCRIPTION: https://github.com/osm-search/Nominatim/pull/4056
|
||||
* (settings/icu-rules/variants-ms.yaml, merged 2026-04-02)
|
||||
* SECONDARY: https://wiki.openstreetmap.org/wiki/Malaysia/Abbreviations
|
||||
*
|
||||
* D-09's law: this table extends by FILE EDIT, never by code change.
|
||||
* Add a row, add a vector to test/normalizer.test.mjs, done.
|
||||
*
|
||||
* "Presint → Prsn" is an explicit vowel-drop expansion (Putrajaya
|
||||
* precincts are written both ways in live addresses: "JALAN P8G
|
||||
* PRESINT 8", "PRSN RESIDEN DESA PARKCI"). "Prsn" is NOT a prefix
|
||||
* abbreviation — it drops an interior "e". Do NOT generalize this into
|
||||
* a generic vowel-drop rule (it over-expands legitimate names).
|
||||
*
|
||||
* "Kpg" is added to the Kampung row beyond the Nominatim transcription:
|
||||
* MS 2039:2006 Annex A3 lists "Kampung/Kampong → Kg, Kpg", and the plan's
|
||||
* normalizer contract pins the vector "Kpg Ampang" -> "kampung ampang".
|
||||
*
|
||||
* City codes (PJ, JB, KB, KK) are deliberately ABSENT: they are place
|
||||
* identities, not address-word forms, so they live in the gazetteer's
|
||||
* alias table (D-08), not here.
|
||||
*
|
||||
* Format: [fullForm, "comma,separated,abbreviations"]. The canonical form
|
||||
* is ALWAYS the long form — buildAbbreviationMap inverts the table so
|
||||
* every abbreviation reaches its full form and every full form reaches
|
||||
* itself ("Jln Ampang" and "Jalan Ampang" both normalize to
|
||||
* "jalan ampang" — the D-09 symmetry law).
|
||||
*/
|
||||
|
||||
export const ABBREVIATION_ROWS: ReadonlyArray<readonly [string, string]> = [
|
||||
// — shared section (the JUPEM core, via Nominatim variants-ms.yaml) —
|
||||
["Jalan", "Jln"],
|
||||
["Lorong", "Lrg,Lor"],
|
||||
["Lebuh", "Lbh"],
|
||||
["Simpang", "Spg"],
|
||||
["Kampung", "Kg,Kpg"],
|
||||
["Kampong", "Kg"],
|
||||
["Taman", "Tmn"],
|
||||
["Bangunan", "Bgn"],
|
||||
["Bukit", "Bkt"],
|
||||
["Batu", "Bt"],
|
||||
["Sungai", "Sg"],
|
||||
["Pulau", "P,Pl"],
|
||||
["Tanjung", "Tg"],
|
||||
["Tanjong", "Tg"],
|
||||
["Teluk", "Tlk"],
|
||||
["Telok", "Tlk"],
|
||||
["Gunung", "G,Gn"],
|
||||
["Gunong", "G,Gn"],
|
||||
["Haji", "Hj"],
|
||||
["Hajjah", "Hjh"],
|
||||
["Abdul", "Abd"],
|
||||
// — country: my section —
|
||||
["Persiaran", "Psrn"],
|
||||
["Lebuhraya", "Lbh.Raya"],
|
||||
["Bandar", "Bdr"],
|
||||
["Kondominium", "Kondo"],
|
||||
["Bahagian", "Bhg"],
|
||||
["Blok", "Blk"],
|
||||
["Kuala", "K,Kl"],
|
||||
["Alur", "Alr"],
|
||||
["Bakau", "Bak"],
|
||||
["Hulu", "H"],
|
||||
["Hilir", "Hr"],
|
||||
["Jeram", "J,Jr"],
|
||||
["Lembah", "Lmbh"],
|
||||
["Padang", "Pdg"],
|
||||
["Parit", "Pt"],
|
||||
["Permatang", "Pmtg"],
|
||||
["Pelabuhan", "Plbh"],
|
||||
["Kepulauan", "Kep"],
|
||||
["Ladang", "Ldg"],
|
||||
["Lapangan", "Lpg"],
|
||||
["Gosong", "Gsg"],
|
||||
["Sekolah kebangsaan", "SK"],
|
||||
["Sekolah menengah", "SM"],
|
||||
["Sekolah menengah kebangsaan", "SMK"],
|
||||
["Sekolah rendah", "SR"],
|
||||
["Hospital", "Hosp"],
|
||||
["Dispensari", "Disp"],
|
||||
// — the explicit D-09 vowel-drop row (Putrajaya precincts) —
|
||||
["Presint", "Prsn"],
|
||||
];
|
||||
|
||||
/**
|
||||
* The BIDIRECTIONAL map the normalizer consumes: every lowercased
|
||||
* abbreviation maps to its full form(s) in table order, and every full
|
||||
* form maps to itself — so the canonical form is always the long form.
|
||||
* Where one abbreviation covers several historical spellings ("kg" ->
|
||||
* kampung/kampong), the list keeps them in table order and consumers
|
||||
* take the FIRST form; the fuzzy-residual tier resolves what the table
|
||||
* cannot. Built once per process; the rows themselves stay pure DATA.
|
||||
*/
|
||||
export function buildAbbreviationMap(): Map<string, string[]> {
|
||||
const map = new Map<string, string[]>();
|
||||
for (const row of ABBREVIATION_ROWS) {
|
||||
const fullForm = row[0];
|
||||
const abbreviations = row[1];
|
||||
const full = fullForm.toLowerCase();
|
||||
for (const piece of abbreviations.split(",")) {
|
||||
const abbr = piece.trim().toLowerCase();
|
||||
if (abbr === "") {
|
||||
continue;
|
||||
}
|
||||
const existing = map.get(abbr);
|
||||
if (existing === undefined) {
|
||||
map.set(abbr, [full]);
|
||||
} else if (!existing.includes(full)) {
|
||||
existing.push(full);
|
||||
}
|
||||
}
|
||||
const selfList = map.get(full);
|
||||
if (selfList === undefined) {
|
||||
map.set(full, [full]);
|
||||
} else if (!selfList.includes(full)) {
|
||||
selfList.push(full);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/*
|
||||
* types.ts — the search contracts (D-03, D-07, D-08, D-11).
|
||||
* --------------------------------------------
|
||||
* GazetteerEntry / SearchQuery / SearchHit / SearchSource are the seam
|
||||
* every backend implements: the in-Worker gazetteer (backend #1, this
|
||||
* phase), the Malaysia-only WASM geocoder (backend #2, D-05, post-v1),
|
||||
* and Photon-on-VPS (backend #3, D-05, Phase 5). Backend #2/#3 do NOT
|
||||
* ship in this phase — the interface exists so they can be added without
|
||||
* a rewrite.
|
||||
*
|
||||
* SearchSource.search() returns an EMPTY array as a legitimate,
|
||||
* load-bearing value: "no match in the KL gazetteer" (D-13's
|
||||
* out-of-area negatives must be refused honestly, never answered with
|
||||
* the nearest KL pin).
|
||||
*
|
||||
* byId() is separate and SYNCHRONOUS: the alternates callback handler
|
||||
* and the webview's deep-link reader both need an id -> entry lookup and
|
||||
* neither should await a network.
|
||||
*
|
||||
* Type-only module: no runtime values, no imports, no storage (GEO-03).
|
||||
*/
|
||||
|
||||
/** One curated gazetteer landmark (D-07 typed JSON, D-08 native aliases). */
|
||||
export interface GazetteerEntry {
|
||||
/** Stable, slug-shaped: "suria-klcc", "sunway-pyramid". Never renamed once shipped. */
|
||||
id: string;
|
||||
/** Canonical display name (the OSM landmark name, e.g. "Suria KLCC"). */
|
||||
name: string;
|
||||
/** Locals' shorthand (D-08) — never empty; always includes the canonical name. */
|
||||
aliases: string[];
|
||||
lat: number;
|
||||
lng: number;
|
||||
/** "mall" | "office-tower" | "park" | "station" | ... (small closed set). */
|
||||
category: string;
|
||||
/** Human address for sendVenue's `address` field — shown VERBATIM, never normalized. */
|
||||
address?: string;
|
||||
}
|
||||
|
||||
/** A search request. */
|
||||
export interface SearchQuery {
|
||||
/** Raw user query. */
|
||||
text: string;
|
||||
/** Default 5 (1 best + 2-4 alternates, D-02). */
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/** One ranked result. */
|
||||
export interface SearchHit {
|
||||
entry: GazetteerEntry;
|
||||
/** 0..1, higher is better. */
|
||||
score: number;
|
||||
/** Which alias matched — the corpus report needs this provenance. */
|
||||
matchedAlias: string;
|
||||
}
|
||||
|
||||
/** The pluggable search seam (D-03). Backends: "gazetteer" | "wasm-geocoder" | "photon". */
|
||||
export interface SearchSource {
|
||||
readonly name: string;
|
||||
/** EMPTY array = honest refusal (D-13) — never a nearest-match pin. */
|
||||
search(q: SearchQuery): Promise<SearchHit[]>;
|
||||
/** Synchronous id -> entry, honest null when unknown. */
|
||||
byId(id: string): GazetteerEntry | null;
|
||||
}
|
||||
Reference in New Issue
Block a user