From a9f9ab380cdac954fc6131ee24da5165632675df Mon Sep 17 00:00:00 2001 From: araharah-executor Date: Sat, 24 Oct 2026 00:23:37 +0800 Subject: [PATCH] feat(04-01): search contracts + JUPEM abbreviation table as DATA + symmetric normalizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - types.ts: GazetteerEntry / SearchQuery / SearchHit / SearchSource (D-03/D-07/D-08) — byId() is separate and synchronous; empty search() array = honest refusal - rules/abbreviations.ts: the vendored JUPEM table (49 rows incl. Presint→Prsn vowel-drop and MS 2039 A3 Kpg forms) as a ReadonlyArray of pairs, inverted once by a single generic builder — extends by file edit, never a code change (D-09) - normalize.ts: NFKD fold → punctuation collapse (keeps '/') → whole-token expansion (canonical = long form) → longest-first glued split (remainder >= 3) T1-T9 green; typecheck clean. --- search/src/normalize.ts | 106 +++++++++++++++++++++++++ search/src/rules/abbreviations.ts | 125 ++++++++++++++++++++++++++++++ search/src/types.ts | 63 +++++++++++++++ 3 files changed, 294 insertions(+) create mode 100644 search/src/normalize.ts create mode 100644 search/src/rules/abbreviations.ts create mode 100644 search/src/types.ts diff --git a/search/src/normalize.ts b/search/src/normalize.ts new file mode 100644 index 0000000..340642a --- /dev/null +++ b/search/src/normalize.ts @@ -0,0 +1,106 @@ +/* + * normalize.ts — the Malay place-name normalizer (D-09, SRCH-02). + * --------------------------------------------------------------- + * Pure: no I/O, no storage, no logging (D-15). Its only import is the + * rules table — the abbreviation table is DATA, so extending the + * normalizer is a file edit in rules/abbreviations.ts plus a vector in + * test/normalizer.test.mjs, never a code change here. + * + * Pipeline (each step separately testable): + * 1. NFKD normalize + strip combining marks + * 2. lowercase + * 3. punctuation -> space; keep letters, digits, whitespace and "/" + * (Malaysian plot shorthand "Jln 3/1", "Presint 8/1" needs it) + * 4. collapse whitespace + trim + * 5. whole-token abbreviation expansion (canonical form = the long + * form, so "Jln Ampang" and "Jalan Ampang" normalize IDENTICALLY — + * the D-09 symmetry law). Applied symmetrically to query AND entry + * aliases (the loader normalizes both sides at index-build time). + * 6. glued-word split — longest-starter-first against the starter + * list, prefix position only, requiring the remainder to be >= 3 + * characters (a naive "split before any known word" corrupts words + * that legitimately contain a starter as a substring). + * 7. collapse whitespace + trim + * + * IDEMPOTENCE LAW: normalize(normalize(x)) === normalize(x) — asserted + * by T7 of the normalizer suite. A token is expanded only when it is a + * WHOLE token (RESEARCH Pitfall 5: "Pdang" is never expanded, only + * "pdg"), and the ambiguity policy for multi-full-form abbreviations + * ("kg" -> kampung/kampong) is: keep the list in table order, expand to + * the FIRST form, and let the fuzzy-residual tier resolve the rest. + */ + +import { buildAbbreviationMap } from "./rules/abbreviations.ts"; + +/** Built ONCE at module scope — ~60 entries of plain data, cheap. + * The lazy-initialization law applies to the gazetteer INDEX, not to + * this table. */ +const ABBREVIATIONS = buildAbbreviationMap(); + +/** Glued-word starters, LONGEST FIRST, so "perkampungan" is considered + * before "kampung" and a maximal starter always wins the split. */ +const GLUED_STARTERS = [ + "perkampungan", + "persiaran", + "kampung", + "kampong", + "presint", + "bandar", + "lorong", + "sungai", + "bukit", + "taman", + "jalan", + "lebuh", +] as const; + +/** Minimum length of the remainder after a split point (RESEARCH: the + * remainder guard is what keeps legitimate words intact). */ +const SPLIT_MIN_REMAINDER = 3; + +/** + * Normalize a raw query or alias into the canonical matching form. + * Symmetric by construction: both sides of every comparison are run + * through this exact function (D-09). + */ +export function normalize(input: string): string { + // 1 + 2. NFKD fold, lowercase, strip combining marks. + let s = input.normalize("NFKD").toLowerCase(); + s = s.replace(/[\u0300-\u036f]/g, ""); + // 3. Punctuation -> space; keep letters, digits, whitespace, "/". + s = s.replace(/[^\p{L}\p{N}\s/]/gu, " "); + // 4. Collapse whitespace + trim. + s = s.replace(/\s+/g, " ").trim(); + if (s === "") { + return ""; + } + // 5. Whole-token abbreviation expansion (first listed full form wins). + const expanded: string[] = []; + for (const token of s.split(" ")) { + if (token === "") { + continue; + } + const fullForms = ABBREVIATIONS.get(token); + expanded.push(fullForms === undefined ? token : (fullForms[0] ?? token)); + } + // 6. Glued-word split, then 7. final collapse + trim. + const parts: string[] = []; + for (const token of expanded) { + splitGluedToken(token, parts); + } + return parts.join(" ").replace(/\s+/g, " ").trim(); +} + +/** Recursively split one token at starter boundaries (prefix position + * only, longest starter first, remainder >= SPLIT_MIN_REMAINDER chars) + * and append the resulting pieces to `out`. */ +function splitGluedToken(token: string, out: string[]): void { + for (const starter of GLUED_STARTERS) { + if (token.length >= starter.length + SPLIT_MIN_REMAINDER && token.startsWith(starter)) { + out.push(starter); + splitGluedToken(token.slice(starter.length), out); + return; + } + } + out.push(token); +} diff --git a/search/src/rules/abbreviations.ts b/search/src/rules/abbreviations.ts new file mode 100644 index 0000000..81d665c --- /dev/null +++ b/search/src/rules/abbreviations.ts @@ -0,0 +1,125 @@ +/* + * abbreviations.ts — the Malay abbreviation table, as DATA (D-09). + * -------------------------------------------------------------- + * PRIMARY SOURCE: JUPEM Toponymic Guidelines — "Abbreviations of + * Geographical Terms and Other Words Used in Maps" + * (Malaysia, 2017 ed.) — the normative national list. + * TRANSCRIPTION: https://github.com/osm-search/Nominatim/pull/4056 + * (settings/icu-rules/variants-ms.yaml, merged 2026-04-02) + * SECONDARY: https://wiki.openstreetmap.org/wiki/Malaysia/Abbreviations + * + * D-09's law: this table extends by FILE EDIT, never by code change. + * Add a row, add a vector to test/normalizer.test.mjs, done. + * + * "Presint → Prsn" is an explicit vowel-drop expansion (Putrajaya + * precincts are written both ways in live addresses: "JALAN P8G + * PRESINT 8", "PRSN RESIDEN DESA PARKCI"). "Prsn" is NOT a prefix + * abbreviation — it drops an interior "e". Do NOT generalize this into + * a generic vowel-drop rule (it over-expands legitimate names). + * + * "Kpg" is added to the Kampung row beyond the Nominatim transcription: + * MS 2039:2006 Annex A3 lists "Kampung/Kampong → Kg, Kpg", and the plan's + * normalizer contract pins the vector "Kpg Ampang" -> "kampung ampang". + * + * City codes (PJ, JB, KB, KK) are deliberately ABSENT: they are place + * identities, not address-word forms, so they live in the gazetteer's + * alias table (D-08), not here. + * + * Format: [fullForm, "comma,separated,abbreviations"]. The canonical form + * is ALWAYS the long form — buildAbbreviationMap inverts the table so + * every abbreviation reaches its full form and every full form reaches + * itself ("Jln Ampang" and "Jalan Ampang" both normalize to + * "jalan ampang" — the D-09 symmetry law). + */ + +export const ABBREVIATION_ROWS: ReadonlyArray = [ + // — shared section (the JUPEM core, via Nominatim variants-ms.yaml) — + ["Jalan", "Jln"], + ["Lorong", "Lrg,Lor"], + ["Lebuh", "Lbh"], + ["Simpang", "Spg"], + ["Kampung", "Kg,Kpg"], + ["Kampong", "Kg"], + ["Taman", "Tmn"], + ["Bangunan", "Bgn"], + ["Bukit", "Bkt"], + ["Batu", "Bt"], + ["Sungai", "Sg"], + ["Pulau", "P,Pl"], + ["Tanjung", "Tg"], + ["Tanjong", "Tg"], + ["Teluk", "Tlk"], + ["Telok", "Tlk"], + ["Gunung", "G,Gn"], + ["Gunong", "G,Gn"], + ["Haji", "Hj"], + ["Hajjah", "Hjh"], + ["Abdul", "Abd"], + // — country: my section — + ["Persiaran", "Psrn"], + ["Lebuhraya", "Lbh.Raya"], + ["Bandar", "Bdr"], + ["Kondominium", "Kondo"], + ["Bahagian", "Bhg"], + ["Blok", "Blk"], + ["Kuala", "K,Kl"], + ["Alur", "Alr"], + ["Bakau", "Bak"], + ["Hulu", "H"], + ["Hilir", "Hr"], + ["Jeram", "J,Jr"], + ["Lembah", "Lmbh"], + ["Padang", "Pdg"], + ["Parit", "Pt"], + ["Permatang", "Pmtg"], + ["Pelabuhan", "Plbh"], + ["Kepulauan", "Kep"], + ["Ladang", "Ldg"], + ["Lapangan", "Lpg"], + ["Gosong", "Gsg"], + ["Sekolah kebangsaan", "SK"], + ["Sekolah menengah", "SM"], + ["Sekolah menengah kebangsaan", "SMK"], + ["Sekolah rendah", "SR"], + ["Hospital", "Hosp"], + ["Dispensari", "Disp"], + // — the explicit D-09 vowel-drop row (Putrajaya precincts) — + ["Presint", "Prsn"], +]; + +/** + * The BIDIRECTIONAL map the normalizer consumes: every lowercased + * abbreviation maps to its full form(s) in table order, and every full + * form maps to itself — so the canonical form is always the long form. + * Where one abbreviation covers several historical spellings ("kg" -> + * kampung/kampong), the list keeps them in table order and consumers + * take the FIRST form; the fuzzy-residual tier resolves what the table + * cannot. Built once per process; the rows themselves stay pure DATA. + */ +export function buildAbbreviationMap(): Map { + const map = new Map(); + for (const row of ABBREVIATION_ROWS) { + const fullForm = row[0]; + const abbreviations = row[1]; + const full = fullForm.toLowerCase(); + for (const piece of abbreviations.split(",")) { + const abbr = piece.trim().toLowerCase(); + if (abbr === "") { + continue; + } + const existing = map.get(abbr); + if (existing === undefined) { + map.set(abbr, [full]); + } else if (!existing.includes(full)) { + existing.push(full); + } + } + const selfList = map.get(full); + if (selfList === undefined) { + map.set(full, [full]); + } else if (!selfList.includes(full)) { + selfList.push(full); + } + } + return map; +} diff --git a/search/src/types.ts b/search/src/types.ts new file mode 100644 index 0000000..ce0d0ad --- /dev/null +++ b/search/src/types.ts @@ -0,0 +1,63 @@ +/* + * types.ts — the search contracts (D-03, D-07, D-08, D-11). + * -------------------------------------------- + * GazetteerEntry / SearchQuery / SearchHit / SearchSource are the seam + * every backend implements: the in-Worker gazetteer (backend #1, this + * phase), the Malaysia-only WASM geocoder (backend #2, D-05, post-v1), + * and Photon-on-VPS (backend #3, D-05, Phase 5). Backend #2/#3 do NOT + * ship in this phase — the interface exists so they can be added without + * a rewrite. + * + * SearchSource.search() returns an EMPTY array as a legitimate, + * load-bearing value: "no match in the KL gazetteer" (D-13's + * out-of-area negatives must be refused honestly, never answered with + * the nearest KL pin). + * + * byId() is separate and SYNCHRONOUS: the alternates callback handler + * and the webview's deep-link reader both need an id -> entry lookup and + * neither should await a network. + * + * Type-only module: no runtime values, no imports, no storage (GEO-03). + */ + +/** One curated gazetteer landmark (D-07 typed JSON, D-08 native aliases). */ +export interface GazetteerEntry { + /** Stable, slug-shaped: "suria-klcc", "sunway-pyramid". Never renamed once shipped. */ + id: string; + /** Canonical display name (the OSM landmark name, e.g. "Suria KLCC"). */ + name: string; + /** Locals' shorthand (D-08) — never empty; always includes the canonical name. */ + aliases: string[]; + lat: number; + lng: number; + /** "mall" | "office-tower" | "park" | "station" | ... (small closed set). */ + category: string; + /** Human address for sendVenue's `address` field — shown VERBATIM, never normalized. */ + address?: string; +} + +/** A search request. */ +export interface SearchQuery { + /** Raw user query. */ + text: string; + /** Default 5 (1 best + 2-4 alternates, D-02). */ + limit?: number; +} + +/** One ranked result. */ +export interface SearchHit { + entry: GazetteerEntry; + /** 0..1, higher is better. */ + score: number; + /** Which alias matched — the corpus report needs this provenance. */ + matchedAlias: string; +} + +/** The pluggable search seam (D-03). Backends: "gazetteer" | "wasm-geocoder" | "photon". */ +export interface SearchSource { + readonly name: string; + /** EMPTY array = honest refusal (D-13) — never a nearest-match pin. */ + search(q: SearchQuery): Promise; + /** Synchronous id -> entry, honest null when unknown. */ + byId(id: string): GazetteerEntry | null; +}