feat(04-01): search contracts + JUPEM abbreviation table as DATA + symmetric normalizer
- types.ts: GazetteerEntry / SearchQuery / SearchHit / SearchSource (D-03/D-07/D-08) — byId() is separate and synchronous; empty search() array = honest refusal - rules/abbreviations.ts: the vendored JUPEM table (49 rows incl. Presint→Prsn vowel-drop and MS 2039 A3 Kpg forms) as a ReadonlyArray of pairs, inverted once by a single generic builder — extends by file edit, never a code change (D-09) - normalize.ts: NFKD fold → punctuation collapse (keeps '/') → whole-token expansion (canonical = long form) → longest-first glued split (remainder >= 3) T1-T9 green; typecheck clean.
This commit is contained in:
@@ -0,0 +1,106 @@
|
|||||||
|
/*
|
||||||
|
* normalize.ts — the Malay place-name normalizer (D-09, SRCH-02).
|
||||||
|
* ---------------------------------------------------------------
|
||||||
|
* Pure: no I/O, no storage, no logging (D-15). Its only import is the
|
||||||
|
* rules table — the abbreviation table is DATA, so extending the
|
||||||
|
* normalizer is a file edit in rules/abbreviations.ts plus a vector in
|
||||||
|
* test/normalizer.test.mjs, never a code change here.
|
||||||
|
*
|
||||||
|
* Pipeline (each step separately testable):
|
||||||
|
* 1. NFKD normalize + strip combining marks
|
||||||
|
* 2. lowercase
|
||||||
|
* 3. punctuation -> space; keep letters, digits, whitespace and "/"
|
||||||
|
* (Malaysian plot shorthand "Jln 3/1", "Presint 8/1" needs it)
|
||||||
|
* 4. collapse whitespace + trim
|
||||||
|
* 5. whole-token abbreviation expansion (canonical form = the long
|
||||||
|
* form, so "Jln Ampang" and "Jalan Ampang" normalize IDENTICALLY —
|
||||||
|
* the D-09 symmetry law). Applied symmetrically to query AND entry
|
||||||
|
* aliases (the loader normalizes both sides at index-build time).
|
||||||
|
* 6. glued-word split — longest-starter-first against the starter
|
||||||
|
* list, prefix position only, requiring the remainder to be >= 3
|
||||||
|
* characters (a naive "split before any known word" corrupts words
|
||||||
|
* that legitimately contain a starter as a substring).
|
||||||
|
* 7. collapse whitespace + trim
|
||||||
|
*
|
||||||
|
* IDEMPOTENCE LAW: normalize(normalize(x)) === normalize(x) — asserted
|
||||||
|
* by T7 of the normalizer suite. A token is expanded only when it is a
|
||||||
|
* WHOLE token (RESEARCH Pitfall 5: "Pdang" is never expanded, only
|
||||||
|
* "pdg"), and the ambiguity policy for multi-full-form abbreviations
|
||||||
|
* ("kg" -> kampung/kampong) is: keep the list in table order, expand to
|
||||||
|
* the FIRST form, and let the fuzzy-residual tier resolve the rest.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { buildAbbreviationMap } from "./rules/abbreviations.ts";
|
||||||
|
|
||||||
|
/** Built ONCE at module scope — ~60 entries of plain data, cheap.
|
||||||
|
* The lazy-initialization law applies to the gazetteer INDEX, not to
|
||||||
|
* this table. */
|
||||||
|
const ABBREVIATIONS = buildAbbreviationMap();
|
||||||
|
|
||||||
|
/** Glued-word starters, LONGEST FIRST, so "perkampungan" is considered
|
||||||
|
* before "kampung" and a maximal starter always wins the split. */
|
||||||
|
const GLUED_STARTERS = [
|
||||||
|
"perkampungan",
|
||||||
|
"persiaran",
|
||||||
|
"kampung",
|
||||||
|
"kampong",
|
||||||
|
"presint",
|
||||||
|
"bandar",
|
||||||
|
"lorong",
|
||||||
|
"sungai",
|
||||||
|
"bukit",
|
||||||
|
"taman",
|
||||||
|
"jalan",
|
||||||
|
"lebuh",
|
||||||
|
] as const;
|
||||||
|
|
||||||
|
/** Minimum length of the remainder after a split point (RESEARCH: the
|
||||||
|
* remainder guard is what keeps legitimate words intact). */
|
||||||
|
const SPLIT_MIN_REMAINDER = 3;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Normalize a raw query or alias into the canonical matching form.
|
||||||
|
* Symmetric by construction: both sides of every comparison are run
|
||||||
|
* through this exact function (D-09).
|
||||||
|
*/
|
||||||
|
export function normalize(input: string): string {
|
||||||
|
// 1 + 2. NFKD fold, lowercase, strip combining marks.
|
||||||
|
let s = input.normalize("NFKD").toLowerCase();
|
||||||
|
s = s.replace(/[\u0300-\u036f]/g, "");
|
||||||
|
// 3. Punctuation -> space; keep letters, digits, whitespace, "/".
|
||||||
|
s = s.replace(/[^\p{L}\p{N}\s/]/gu, " ");
|
||||||
|
// 4. Collapse whitespace + trim.
|
||||||
|
s = s.replace(/\s+/g, " ").trim();
|
||||||
|
if (s === "") {
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
// 5. Whole-token abbreviation expansion (first listed full form wins).
|
||||||
|
const expanded: string[] = [];
|
||||||
|
for (const token of s.split(" ")) {
|
||||||
|
if (token === "") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const fullForms = ABBREVIATIONS.get(token);
|
||||||
|
expanded.push(fullForms === undefined ? token : (fullForms[0] ?? token));
|
||||||
|
}
|
||||||
|
// 6. Glued-word split, then 7. final collapse + trim.
|
||||||
|
const parts: string[] = [];
|
||||||
|
for (const token of expanded) {
|
||||||
|
splitGluedToken(token, parts);
|
||||||
|
}
|
||||||
|
return parts.join(" ").replace(/\s+/g, " ").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Recursively split one token at starter boundaries (prefix position
|
||||||
|
* only, longest starter first, remainder >= SPLIT_MIN_REMAINDER chars)
|
||||||
|
* and append the resulting pieces to `out`. */
|
||||||
|
function splitGluedToken(token: string, out: string[]): void {
|
||||||
|
for (const starter of GLUED_STARTERS) {
|
||||||
|
if (token.length >= starter.length + SPLIT_MIN_REMAINDER && token.startsWith(starter)) {
|
||||||
|
out.push(starter);
|
||||||
|
splitGluedToken(token.slice(starter.length), out);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
out.push(token);
|
||||||
|
}
|
||||||
@@ -0,0 +1,125 @@
|
|||||||
|
/*
|
||||||
|
* abbreviations.ts — the Malay abbreviation table, as DATA (D-09).
|
||||||
|
* --------------------------------------------------------------
|
||||||
|
* PRIMARY SOURCE: JUPEM Toponymic Guidelines — "Abbreviations of
|
||||||
|
* Geographical Terms and Other Words Used in Maps"
|
||||||
|
* (Malaysia, 2017 ed.) — the normative national list.
|
||||||
|
* TRANSCRIPTION: https://github.com/osm-search/Nominatim/pull/4056
|
||||||
|
* (settings/icu-rules/variants-ms.yaml, merged 2026-04-02)
|
||||||
|
* SECONDARY: https://wiki.openstreetmap.org/wiki/Malaysia/Abbreviations
|
||||||
|
*
|
||||||
|
* D-09's law: this table extends by FILE EDIT, never by code change.
|
||||||
|
* Add a row, add a vector to test/normalizer.test.mjs, done.
|
||||||
|
*
|
||||||
|
* "Presint → Prsn" is an explicit vowel-drop expansion (Putrajaya
|
||||||
|
* precincts are written both ways in live addresses: "JALAN P8G
|
||||||
|
* PRESINT 8", "PRSN RESIDEN DESA PARKCI"). "Prsn" is NOT a prefix
|
||||||
|
* abbreviation — it drops an interior "e". Do NOT generalize this into
|
||||||
|
* a generic vowel-drop rule (it over-expands legitimate names).
|
||||||
|
*
|
||||||
|
* "Kpg" is added to the Kampung row beyond the Nominatim transcription:
|
||||||
|
* MS 2039:2006 Annex A3 lists "Kampung/Kampong → Kg, Kpg", and the plan's
|
||||||
|
* normalizer contract pins the vector "Kpg Ampang" -> "kampung ampang".
|
||||||
|
*
|
||||||
|
* City codes (PJ, JB, KB, KK) are deliberately ABSENT: they are place
|
||||||
|
* identities, not address-word forms, so they live in the gazetteer's
|
||||||
|
* alias table (D-08), not here.
|
||||||
|
*
|
||||||
|
* Format: [fullForm, "comma,separated,abbreviations"]. The canonical form
|
||||||
|
* is ALWAYS the long form — buildAbbreviationMap inverts the table so
|
||||||
|
* every abbreviation reaches its full form and every full form reaches
|
||||||
|
* itself ("Jln Ampang" and "Jalan Ampang" both normalize to
|
||||||
|
* "jalan ampang" — the D-09 symmetry law).
|
||||||
|
*/
|
||||||
|
|
||||||
|
export const ABBREVIATION_ROWS: ReadonlyArray<readonly [string, string]> = [
|
||||||
|
// — shared section (the JUPEM core, via Nominatim variants-ms.yaml) —
|
||||||
|
["Jalan", "Jln"],
|
||||||
|
["Lorong", "Lrg,Lor"],
|
||||||
|
["Lebuh", "Lbh"],
|
||||||
|
["Simpang", "Spg"],
|
||||||
|
["Kampung", "Kg,Kpg"],
|
||||||
|
["Kampong", "Kg"],
|
||||||
|
["Taman", "Tmn"],
|
||||||
|
["Bangunan", "Bgn"],
|
||||||
|
["Bukit", "Bkt"],
|
||||||
|
["Batu", "Bt"],
|
||||||
|
["Sungai", "Sg"],
|
||||||
|
["Pulau", "P,Pl"],
|
||||||
|
["Tanjung", "Tg"],
|
||||||
|
["Tanjong", "Tg"],
|
||||||
|
["Teluk", "Tlk"],
|
||||||
|
["Telok", "Tlk"],
|
||||||
|
["Gunung", "G,Gn"],
|
||||||
|
["Gunong", "G,Gn"],
|
||||||
|
["Haji", "Hj"],
|
||||||
|
["Hajjah", "Hjh"],
|
||||||
|
["Abdul", "Abd"],
|
||||||
|
// — country: my section —
|
||||||
|
["Persiaran", "Psrn"],
|
||||||
|
["Lebuhraya", "Lbh.Raya"],
|
||||||
|
["Bandar", "Bdr"],
|
||||||
|
["Kondominium", "Kondo"],
|
||||||
|
["Bahagian", "Bhg"],
|
||||||
|
["Blok", "Blk"],
|
||||||
|
["Kuala", "K,Kl"],
|
||||||
|
["Alur", "Alr"],
|
||||||
|
["Bakau", "Bak"],
|
||||||
|
["Hulu", "H"],
|
||||||
|
["Hilir", "Hr"],
|
||||||
|
["Jeram", "J,Jr"],
|
||||||
|
["Lembah", "Lmbh"],
|
||||||
|
["Padang", "Pdg"],
|
||||||
|
["Parit", "Pt"],
|
||||||
|
["Permatang", "Pmtg"],
|
||||||
|
["Pelabuhan", "Plbh"],
|
||||||
|
["Kepulauan", "Kep"],
|
||||||
|
["Ladang", "Ldg"],
|
||||||
|
["Lapangan", "Lpg"],
|
||||||
|
["Gosong", "Gsg"],
|
||||||
|
["Sekolah kebangsaan", "SK"],
|
||||||
|
["Sekolah menengah", "SM"],
|
||||||
|
["Sekolah menengah kebangsaan", "SMK"],
|
||||||
|
["Sekolah rendah", "SR"],
|
||||||
|
["Hospital", "Hosp"],
|
||||||
|
["Dispensari", "Disp"],
|
||||||
|
// — the explicit D-09 vowel-drop row (Putrajaya precincts) —
|
||||||
|
["Presint", "Prsn"],
|
||||||
|
];
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The BIDIRECTIONAL map the normalizer consumes: every lowercased
|
||||||
|
* abbreviation maps to its full form(s) in table order, and every full
|
||||||
|
* form maps to itself — so the canonical form is always the long form.
|
||||||
|
* Where one abbreviation covers several historical spellings ("kg" ->
|
||||||
|
* kampung/kampong), the list keeps them in table order and consumers
|
||||||
|
* take the FIRST form; the fuzzy-residual tier resolves what the table
|
||||||
|
* cannot. Built once per process; the rows themselves stay pure DATA.
|
||||||
|
*/
|
||||||
|
export function buildAbbreviationMap(): Map<string, string[]> {
|
||||||
|
const map = new Map<string, string[]>();
|
||||||
|
for (const row of ABBREVIATION_ROWS) {
|
||||||
|
const fullForm = row[0];
|
||||||
|
const abbreviations = row[1];
|
||||||
|
const full = fullForm.toLowerCase();
|
||||||
|
for (const piece of abbreviations.split(",")) {
|
||||||
|
const abbr = piece.trim().toLowerCase();
|
||||||
|
if (abbr === "") {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const existing = map.get(abbr);
|
||||||
|
if (existing === undefined) {
|
||||||
|
map.set(abbr, [full]);
|
||||||
|
} else if (!existing.includes(full)) {
|
||||||
|
existing.push(full);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const selfList = map.get(full);
|
||||||
|
if (selfList === undefined) {
|
||||||
|
map.set(full, [full]);
|
||||||
|
} else if (!selfList.includes(full)) {
|
||||||
|
selfList.push(full);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return map;
|
||||||
|
}
|
||||||
@@ -0,0 +1,63 @@
|
|||||||
|
/*
|
||||||
|
* types.ts — the search contracts (D-03, D-07, D-08, D-11).
|
||||||
|
* --------------------------------------------
|
||||||
|
* GazetteerEntry / SearchQuery / SearchHit / SearchSource are the seam
|
||||||
|
* every backend implements: the in-Worker gazetteer (backend #1, this
|
||||||
|
* phase), the Malaysia-only WASM geocoder (backend #2, D-05, post-v1),
|
||||||
|
* and Photon-on-VPS (backend #3, D-05, Phase 5). Backend #2/#3 do NOT
|
||||||
|
* ship in this phase — the interface exists so they can be added without
|
||||||
|
* a rewrite.
|
||||||
|
*
|
||||||
|
* SearchSource.search() returns an EMPTY array as a legitimate,
|
||||||
|
* load-bearing value: "no match in the KL gazetteer" (D-13's
|
||||||
|
* out-of-area negatives must be refused honestly, never answered with
|
||||||
|
* the nearest KL pin).
|
||||||
|
*
|
||||||
|
* byId() is separate and SYNCHRONOUS: the alternates callback handler
|
||||||
|
* and the webview's deep-link reader both need an id -> entry lookup and
|
||||||
|
* neither should await a network.
|
||||||
|
*
|
||||||
|
* Type-only module: no runtime values, no imports, no storage (GEO-03).
|
||||||
|
*/
|
||||||
|
|
||||||
|
/** One curated gazetteer landmark (D-07 typed JSON, D-08 native aliases). */
|
||||||
|
export interface GazetteerEntry {
|
||||||
|
/** Stable, slug-shaped: "suria-klcc", "sunway-pyramid". Never renamed once shipped. */
|
||||||
|
id: string;
|
||||||
|
/** Canonical display name (the OSM landmark name, e.g. "Suria KLCC"). */
|
||||||
|
name: string;
|
||||||
|
/** Locals' shorthand (D-08) — never empty; always includes the canonical name. */
|
||||||
|
aliases: string[];
|
||||||
|
lat: number;
|
||||||
|
lng: number;
|
||||||
|
/** "mall" | "office-tower" | "park" | "station" | ... (small closed set). */
|
||||||
|
category: string;
|
||||||
|
/** Human address for sendVenue's `address` field — shown VERBATIM, never normalized. */
|
||||||
|
address?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A search request. */
|
||||||
|
export interface SearchQuery {
|
||||||
|
/** Raw user query. */
|
||||||
|
text: string;
|
||||||
|
/** Default 5 (1 best + 2-4 alternates, D-02). */
|
||||||
|
limit?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** One ranked result. */
|
||||||
|
export interface SearchHit {
|
||||||
|
entry: GazetteerEntry;
|
||||||
|
/** 0..1, higher is better. */
|
||||||
|
score: number;
|
||||||
|
/** Which alias matched — the corpus report needs this provenance. */
|
||||||
|
matchedAlias: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** The pluggable search seam (D-03). Backends: "gazetteer" | "wasm-geocoder" | "photon". */
|
||||||
|
export interface SearchSource {
|
||||||
|
readonly name: string;
|
||||||
|
/** EMPTY array = honest refusal (D-13) — never a nearest-match pin. */
|
||||||
|
search(q: SearchQuery): Promise<SearchHit[]>;
|
||||||
|
/** Synchronous id -> entry, honest null when unknown. */
|
||||||
|
byId(id: string): GazetteerEntry | null;
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user