arah2/search/test/normalizer.test.mjs

#!/usr/bin/env node
/*
 * normalizer.test.mjs — the Malay normalizer vector suite (D-09, SRCH-02).
 * ----------------------------------------------------------------------
 * RED phase (plan 04-01): every case below fails while ../src/normalize.ts
 * does not exist — the module import happens lazily inside each case, so
 * T1–T9 report one FAIL per law instead of a lost suite load.
 *
 * Contract under test (D-09 — rule table + folding + fuzzy-residual):
 *   T1  rule vectors — every expansion/folding vector from the plan's
 *       normalizer contract, asserted one by one
 *   T2  symmetry law — "Jln Ampang" and "Jalan Ampang" normalize to the
 *       IDENTICAL string (the canonical form is always the long form)
 *   T3  whole-token-only expansion — "Pdang" is never expanded; "pdg" is
 *   T4  slash preservation — "Jln 3/1" keeps its "3/1"
 *   T5  glued-word split — "tamanbangsar" splits; "jalankanpar" yields a
 *       standalone "jalan" token
 *   T6  vowel-drop — "Prsn 15" and "Presint 15" both hit "presint 15"
 *   T7  idempotence — normalize(normalize(v)) === normalize(v) for every
 *       vector in this suite
 *   T8  case/whitespace folding
 *   T9  the table is DATA — abbreviations.ts is data plus one generic
 *       builder (no algorithm code), and the vendored table is >= 47 rows
 *
 * Run:  npm run search:test     (exit 0 = all green)
 */
import { strict as assert } from "node:assert";
import { readFileSync } from "node:fs";
import { test } from "./harness.mjs";

// Lazy normalizer load — missing module => per-case ERR_MODULE_NOT_FOUND (the RED state).
let normalizePromise = null;
function loadNormalizer() {
  if (!normalizePromise) {
    normalizePromise = import("../src/normalize.ts").then((m) => m.normalize);
  }
  return normalizePromise;
}

// Lazy rules-table load (T9 asserts on the module itself, not just its consumer).
let rulesPromise = null;
function loadRules() {
  if (!rulesPromise) {
    rulesPromise = import("../src/rules/abbreviations.ts");
  }
  return rulesPromise;
}

/** The plan's minimum vector set — every row is [input, expectedNormalForm]. */
const VECTORS = [
  ["Jln Ampang", "jalan ampang"],
  ["Jalan Ampang", "jalan ampang"],
  ["Jln Tmn Desa", "jalan taman desa"],
  ["Tmn Bangsar", "taman bangsar"],
  ["Kg Ampang", "kampung ampang"],
  ["Kpg Ampang", "kampung ampang"],
  ["Prsn 15", "presint 15"],
  ["Presint 15", "presint 15"],
  ["tamanbangsar", "taman bangsar"],
  ["Jln 3/1", "jalan 3/1"],
  ["Bdr Baru", "bandar baru"],
  ["Sunway Pyramid", "sunway pyramid"],
  ["  SUNWAY   PYRAMID  ", "sunway pyramid"],
  ["Kuala Lumpur", "kuala lumpur"],
  ["PJ", "pj"],
];

test("T1 rule vectors — the plan's minimum expansion/folding set", async () => {
  const normalize = await loadNormalizer();
  for (const [input, expected] of VECTORS) {
    assert.equal(normalize(input), expected, `normalize(${JSON.stringify(input)})`);
  }
});

test("T2 symmetry law — abbreviated and full forms normalize identically", async () => {
  const normalize = await loadNormalizer();
  assert.equal(normalize("Jln Ampang"), normalize("Jalan Ampang"), "Jln/Jalan must fold to one form");
  assert.equal(normalize("Tmn Bangsar"), normalize("Taman Bangsar"), "Tmn/Taman must fold to one form");
});

test("T3 whole-token-only expansion — 'Pdang' is never expanded, 'pdg' is", async () => {
  const normalize = await loadNormalizer();
  assert.equal(normalize("Pdang"), "pdang", "a non-token substring must never expand");
  assert.equal(normalize("pdg"), "padang", "the standalone abbreviation must expand");
  assert.ok(!normalize("Pdang").includes("padang"), "Pdang must not contain the expansion");
});

test("T4 slash preservation — Malaysian plot shorthand 'Jln 3/1' keeps its slash", async () => {
  const normalize = await loadNormalizer();
  assert.equal(normalize("Jln 3/1"), "jalan 3/1");
});

test("T5 glued-word split — 'tamanbangsar' splits; 'jalankanpar' yields a standalone 'jalan'", async () => {
  const normalize = await loadNormalizer();
  assert.equal(normalize("tamanbangsar"), "taman bangsar");
  const tokens = normalize("jalankanpar").split(" ");
  assert.ok(tokens.includes("jalan"), `"jalan" must be its own token, got: ${tokens.join("|")}`);
});

test("T6 vowel-drop — 'Prsn 15' and 'Presint 15' both reach 'presint 15'", async () => {
  const normalize = await loadNormalizer();
  assert.equal(normalize("Prsn 15"), "presint 15");
  assert.equal(normalize("Presint 15"), "presint 15");
});

test("T7 idempotence — normalize(normalize(v)) === normalize(v) for every vector", async () => {
  const normalize = await loadNormalizer();
  for (const [input] of VECTORS) {
    const once = normalize(input);
    assert.equal(normalize(once), once, `idempotence broken for ${JSON.stringify(input)}`);
  }
});

test("T8 case/whitespace folding — '  SUNWAY   PYRAMID  ' folds to 'sunway pyramid'", async () => {
  const normalize = await loadNormalizer();
  assert.equal(normalize("  SUNWAY   PYRAMID  "), "sunway pyramid");
});

test("T9 the table is DATA — data plus one generic builder, >= 47 rows", async () => {
  const path = new URL("../src/rules/abbreviations.ts", import.meta.url);
  const raw = readFileSync(path, "utf8");
  // Strip block comments and full-line // comments before asserting code shape.
  const stripped = raw
    .replace(/\/\*[\s\S]*?\*\//g, "")
    .replace(/^\s*\/\/.*$/gm, "");
  const builderMarker = "export function buildAbbreviationMap";
  const builderAt = stripped.indexOf(builderMarker);
  assert.ok(builderAt !== -1, "buildAbbreviationMap must exist");
  const dataSection = stripped.slice(0, builderAt);
  for (const banned of [/\bfunction\b/, /\bif\b/, /\bfor\b/, /\bwhile\b/, /=>/]) {
    assert.doesNotMatch(dataSection, banned, `the DATA section must not contain ${banned}`);
  }
  const functionCount = (stripped.match(/\bfunction\b/g) ?? []).length;
  assert.equal(functionCount, 1, "the only function in the file must be buildAbbreviationMap");
  assert.doesNotMatch(stripped, /=>/, "no arrow functions anywhere — data plus one loop-based builder");

  const rules = await loadRules();
  assert.ok(Array.isArray(rules.ABBREVIATION_ROWS), "ABBREVIATION_ROWS must be an array");
  assert.ok(rules.ABBREVIATION_ROWS.length >= 47, `table must be >= 47 rows, got ${rules.ABBREVIATION_ROWS.length}`);
  for (const row of rules.ABBREVIATION_ROWS) {
    assert.ok(Array.isArray(row) && row.length === 2, "every row is a [fullForm, abbreviations] pair");
    assert.equal(typeof row[0], "string");
    assert.equal(typeof row[1], "string");
    assert.ok(row[0].length > 0 && row[1].length > 0, "neither side of a row may be empty");
  }
  const map = rules.buildAbbreviationMap();
  assert.equal(map.get("jln")?.[0], "jalan", "jln must map to jalan");
  assert.deepEqual(map.get("presint"), ["presint"], "the canonical long form maps to itself");
  assert.deepEqual(map.get("prsn"), ["presint"], "Prsn -> Presint (the D-09 vowel-drop row)");
  assert.ok(map.get("kampung")?.includes("kampung"), "kampung maps to itself");
  assert.ok(map.get("kg")?.includes("kampung"), "kg must reach kampung");
});