#!/usr/bin/env node /* * normalizer.test.mjs — the Malay normalizer vector suite (D-09, SRCH-02). * ---------------------------------------------------------------------- * RED phase (plan 04-01): every case below fails while ../src/normalize.ts * does not exist — the module import happens lazily inside each case, so * T1–T9 report one FAIL per law instead of a lost suite load. * * Contract under test (D-09 — rule table + folding + fuzzy-residual): * T1 rule vectors — every expansion/folding vector from the plan's * normalizer contract, asserted one by one * T2 symmetry law — "Jln Ampang" and "Jalan Ampang" normalize to the * IDENTICAL string (the canonical form is always the long form) * T3 whole-token-only expansion — "Pdang" is never expanded; "pdg" is * T4 slash preservation — "Jln 3/1" keeps its "3/1" * T5 glued-word split — "tamanbangsar" splits; "jalankanpar" yields a * standalone "jalan" token * T6 vowel-drop — "Prsn 15" and "Presint 15" both hit "presint 15" * T7 idempotence — normalize(normalize(v)) === normalize(v) for every * vector in this suite * T8 case/whitespace folding * T9 the table is DATA — abbreviations.ts is data plus one generic * builder (no algorithm code), and the vendored table is >= 47 rows * * Run: npm run search:test (exit 0 = all green) */ import { strict as assert } from "node:assert"; import { readFileSync } from "node:fs"; import { test } from "./harness.mjs"; // Lazy normalizer load — missing module => per-case ERR_MODULE_NOT_FOUND (the RED state). let normalizePromise = null; function loadNormalizer() { if (!normalizePromise) { normalizePromise = import("../src/normalize.ts").then((m) => m.normalize); } return normalizePromise; } // Lazy rules-table load (T9 asserts on the module itself, not just its consumer). let rulesPromise = null; function loadRules() { if (!rulesPromise) { rulesPromise = import("../src/rules/abbreviations.ts"); } return rulesPromise; } /** The plan's minimum vector set — every row is [input, expectedNormalForm]. */ const VECTORS = [ ["Jln Ampang", "jalan ampang"], ["Jalan Ampang", "jalan ampang"], ["Jln Tmn Desa", "jalan taman desa"], ["Tmn Bangsar", "taman bangsar"], ["Kg Ampang", "kampung ampang"], ["Kpg Ampang", "kampung ampang"], ["Prsn 15", "presint 15"], ["Presint 15", "presint 15"], ["tamanbangsar", "taman bangsar"], ["Jln 3/1", "jalan 3/1"], ["Bdr Baru", "bandar baru"], ["Sunway Pyramid", "sunway pyramid"], [" SUNWAY PYRAMID ", "sunway pyramid"], ["Kuala Lumpur", "kuala lumpur"], ["PJ", "pj"], ]; test("T1 rule vectors — the plan's minimum expansion/folding set", async () => { const normalize = await loadNormalizer(); for (const [input, expected] of VECTORS) { assert.equal(normalize(input), expected, `normalize(${JSON.stringify(input)})`); } }); test("T2 symmetry law — abbreviated and full forms normalize identically", async () => { const normalize = await loadNormalizer(); assert.equal(normalize("Jln Ampang"), normalize("Jalan Ampang"), "Jln/Jalan must fold to one form"); assert.equal(normalize("Tmn Bangsar"), normalize("Taman Bangsar"), "Tmn/Taman must fold to one form"); }); test("T3 whole-token-only expansion — 'Pdang' is never expanded, 'pdg' is", async () => { const normalize = await loadNormalizer(); assert.equal(normalize("Pdang"), "pdang", "a non-token substring must never expand"); assert.equal(normalize("pdg"), "padang", "the standalone abbreviation must expand"); assert.ok(!normalize("Pdang").includes("padang"), "Pdang must not contain the expansion"); }); test("T4 slash preservation — Malaysian plot shorthand 'Jln 3/1' keeps its slash", async () => { const normalize = await loadNormalizer(); assert.equal(normalize("Jln 3/1"), "jalan 3/1"); }); test("T5 glued-word split — 'tamanbangsar' splits; 'jalankanpar' yields a standalone 'jalan'", async () => { const normalize = await loadNormalizer(); assert.equal(normalize("tamanbangsar"), "taman bangsar"); const tokens = normalize("jalankanpar").split(" "); assert.ok(tokens.includes("jalan"), `"jalan" must be its own token, got: ${tokens.join("|")}`); }); test("T6 vowel-drop — 'Prsn 15' and 'Presint 15' both reach 'presint 15'", async () => { const normalize = await loadNormalizer(); assert.equal(normalize("Prsn 15"), "presint 15"); assert.equal(normalize("Presint 15"), "presint 15"); }); test("T7 idempotence — normalize(normalize(v)) === normalize(v) for every vector", async () => { const normalize = await loadNormalizer(); for (const [input] of VECTORS) { const once = normalize(input); assert.equal(normalize(once), once, `idempotence broken for ${JSON.stringify(input)}`); } }); test("T8 case/whitespace folding — ' SUNWAY PYRAMID ' folds to 'sunway pyramid'", async () => { const normalize = await loadNormalizer(); assert.equal(normalize(" SUNWAY PYRAMID "), "sunway pyramid"); }); test("T9 the table is DATA — data plus one generic builder, >= 47 rows", async () => { const path = new URL("../src/rules/abbreviations.ts", import.meta.url); const raw = readFileSync(path, "utf8"); // Strip block comments and full-line // comments before asserting code shape. const stripped = raw .replace(/\/\*[\s\S]*?\*\//g, "") .replace(/^\s*\/\/.*$/gm, ""); const builderMarker = "export function buildAbbreviationMap"; const builderAt = stripped.indexOf(builderMarker); assert.ok(builderAt !== -1, "buildAbbreviationMap must exist"); const dataSection = stripped.slice(0, builderAt); for (const banned of [/\bfunction\b/, /\bif\b/, /\bfor\b/, /\bwhile\b/, /=>/]) { assert.doesNotMatch(dataSection, banned, `the DATA section must not contain ${banned}`); } const functionCount = (stripped.match(/\bfunction\b/g) ?? []).length; assert.equal(functionCount, 1, "the only function in the file must be buildAbbreviationMap"); assert.doesNotMatch(stripped, /=>/, "no arrow functions anywhere — data plus one loop-based builder"); const rules = await loadRules(); assert.ok(Array.isArray(rules.ABBREVIATION_ROWS), "ABBREVIATION_ROWS must be an array"); assert.ok(rules.ABBREVIATION_ROWS.length >= 47, `table must be >= 47 rows, got ${rules.ABBREVIATION_ROWS.length}`); for (const row of rules.ABBREVIATION_ROWS) { assert.ok(Array.isArray(row) && row.length === 2, "every row is a [fullForm, abbreviations] pair"); assert.equal(typeof row[0], "string"); assert.equal(typeof row[1], "string"); assert.ok(row[0].length > 0 && row[1].length > 0, "neither side of a row may be empty"); } const map = rules.buildAbbreviationMap(); assert.equal(map.get("jln")?.[0], "jalan", "jln must map to jalan"); assert.deepEqual(map.get("presint"), ["presint"], "the canonical long form maps to itself"); assert.deepEqual(map.get("prsn"), ["presint"], "Prsn -> Presint (the D-09 vowel-drop row)"); assert.ok(map.get("kampung")?.includes("kampung"), "kampung maps to itself"); assert.ok(map.get("kg")?.includes("kampung"), "kg must reach kampung"); });