feat(04-02): Overpass top-up proposer (D-06) + JSON-to-SQLite converter (D-05)

overpass-topup.mjs — maintainer-time repo tool, six hard laws: - never imported by bot/src or web/src (TP1 gate); Worker reads only the committed JSON - exactly ONE request per run: both live paths (Overpass POST, Nominatim GET) share a single requestJson helper — one fetch( call site (TP7) - identifying User-Agent araharah-gazetteer-topup/1.0 mandatory (missing UA = HTTP 406) - busy/saturated/unusable response -> 'Overpass instance busy — retry later', exit 0 (TP4) - output is a PROPOSAL only; source never names the curated file, so it cannot write it (TP6) - --source=overpass|nominatim|fixture (--fixture implies fixture), --endpoint/OVERPASS_ENDPOINT override, bbox-bounded single-family query pinned verbatim (3.05,101.55,3.25,101.80) (TP3) - --fixture selects the offline path by itself (a fixture flag silently hitting the live network would be a trap — fixed during GREEN) overpass-response.json — committed real-shape Overpass response (6 KL elements: nodes with lat/lon, ways/relation with center) so the test is deterministic. json-to-sqlite.mjs — backend #2's dataset format reachable today (D-05): - reads the same committed kl-gazetteer.json, emits kl-gazetteer.sqlite via node:sqlite (zero added deps) with the SAME id set (CV1), plus the ODbL notice embedded in a metadata table + NOTICE sidecar (ODbL s4.4, T-04-20) - deterministic: rows in dataset order, no timestamps
2026-10-24 01:01:00 +08:00
parent ec98c8d454
commit c6a5f2b008
3 changed files with 425 additions and 0 deletions
@@ -0,0 +1,114 @@
+#!/usr/bin/env node
+/*
+ * json-to-sqlite.mjs — the D-05 backend-#2 format converter.
+ * -----------------------------------------------------------
+ * A MAINTAINER-TIME REPO TOOL. It is NOT a runtime dependency: nothing in
+ * bot/src/ or web/src/ imports it, and the Worker's canonical dataset
+ * format stays typed JSON (D-07). This script exists so backend #2 (the
+ * Malaysia-only WASM geocoder, post-v1) is a FORMAT change, not a rewrite:
+ * the SQLite dataset format is reachable from day one (D-05).
+ *
+ * Reads the COMMITTED curated dataset (search/src/data/kl-gazetteer.json)
+ * and emits <out>/kl-gazetteer.sqlite with the SAME id set:
+ *   places   (id TEXT PRIMARY KEY, name TEXT, aliases TEXT,
+ *             lat REAL, lng REAL, category TEXT, address TEXT)
+ *   metadata (key TEXT PRIMARY KEY, value TEXT)
+ *
+ * ODbL §4.4 share-alike: the emitted database is a Derivative Database of
+ * OSM data, so the ODbL notice travels with the output BY CONSTRUCTION —
+ * the _license / _attribution / _notice fields are written into the
+ * metadata table. Never redistribute an output stripped of them (T-04-20).
+ *
+ * Writer: Node 24's built-in node:sqlite (DatabaseSync) — zero added
+ * dependencies. Output is deterministic: rows are inserted in the
+ * committed dataset's order and carry no timestamps.
+ *
+ * Usage: node scripts/json-to-sqlite.mjs --out <dir>   (or --out=<dir>)
+ * Exit codes: 0 on success; 1 on a usage error (missing/unwritable --out).
+ */
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+
+const DATASET_URL = new URL("../src/data/kl-gazetteer.json", import.meta.url);
+const OUTPUT_NAME = "kl-gazetteer.sqlite";
+
+function parseOutDir(argv) {
+  for (let i = 2; i < argv.length; i++) {
+    const arg = argv[i];
+    if (arg.startsWith("--out=")) return arg.slice("--out=".length);
+    if (arg === "--out") return argv[i + 1];
+  }
+  return undefined;
+}
+
+const outDir = parseOutDir(process.argv);
+if (!outDir) {
+  console.error("json-to-sqlite: missing --out <dir> — pass a directory for the emitted database.");
+  process.exit(1);
+}
+const outPath = resolve(outDir);
+if (!existsSync(outPath)) {
+  mkdirSync(outPath, { recursive: true });
+}
+
+const dataset = JSON.parse(readFileSync(DATASET_URL, "utf8"));
+const entries = dataset.entries ?? [];
+
+const db = new DatabaseSync(resolve(outPath, OUTPUT_NAME));
+try {
+  db.exec("BEGIN");
+  db.exec(`
+    CREATE TABLE places (
+      id TEXT PRIMARY KEY,
+      name TEXT NOT NULL,
+      aliases TEXT NOT NULL,
+      lat REAL NOT NULL,
+      lng REAL NOT NULL,
+      category TEXT NOT NULL,
+      address TEXT
+    );
+    CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
+  `);
+  const insert = db.prepare(
+    "INSERT INTO places (id, name, aliases, lat, lng, category, address) VALUES (?, ?, ?, ?, ?, ?, ?)",
+  );
+  for (const entry of entries) {
+    insert.run(
+      entry.id,
+      entry.name,
+      JSON.stringify(entry.aliases),
+      entry.lat,
+      entry.lng,
+      entry.category,
+      entry.address ?? null,
+    );
+  }
+  const insertMeta = db.prepare("INSERT INTO metadata (key, value) VALUES (?, ?)");
+  insertMeta.run("_license", dataset._license ?? "ODbL-1.0");
+  insertMeta.run("_attribution", dataset._attribution ?? "© OpenStreetMap contributors");
+  insertMeta.run("_notice", dataset._notice ?? "");
+  insertMeta.run("source_version", String(dataset.version ?? 0));
+  insertMeta.run("entry_count", String(entries.length));
+  db.exec("COMMIT");
+} catch (err) {
+  db.exec("ROLLBACK");
+  db.close();
+  console.error(`json-to-sqlite: conversion failed: ${err?.message ?? err}`);
+  process.exit(1);
+}
+db.close();
+
+// A machine-readable sidecar stating the licence terms alongside the database.
+writeFileSync(
+  resolve(outPath, "kl-gazetteer.sqlite.NOTICE.txt"),
+  `${dataset._notice ?? ""}\nLicense: ${dataset._license ?? "ODbL-1.0"}\n` +
+    `Attribution: ${dataset._attribution ?? "© OpenStreetMap contributors"}\n` +
+    `The same terms are embedded in the metadata table of ${OUTPUT_NAME} (ODbL §4.4).\n`,
+);
+
+console.log(
+  `wrote ${resolve(outPath, OUTPUT_NAME)} — ${entries.length} places, id set preserved, ` +
+    `${dataset._license ?? "ODbL-1.0"} notice included (metadata table + NOTICE sidecar).`,
+);
+process.exit(0);
@@ -0,0 +1,223 @@
+#!/usr/bin/env node
+/*
+ * overpass-topup.mjs — propose KL gazetteer top-up candidates from OpenStreetMap (D-06).
+ * -------------------------------------------------------------------------------------
+ * A MAINTAINER-TIME REPO TOOL. It is NOT a runtime dependency: nothing in
+ * bot/src/ or web/src/ imports it, and no deploy, test, or corpus gate depends on
+ * it succeeding. The Worker only ever reads the COMMITTED curated JSON.
+ *
+ * Output is a PROPOSAL — a printed candidate list (or a file at an explicit --out
+ * path) that a human diffs and hand-merges into the curated dataset file.
+ * THIS SCRIPT NEVER WRITES THE CURATED FILE.
+ *
+ * Exit codes: 0 on success AND on a busy instance (Overpass capacity saturation
+ * is intermittent — 502/429/timeout is a NORMAL, retryable outcome). Non-zero
+ * only on a usage error (no bbox resolvable / unreadable fixture).
+ *
+ * Inputs: --source=overpass|nominatim|fixture (default overpass), --endpoint=<url>
+ *         (or OVERPASS_ENDPOINT), --bbox=s,w,n,e, --q=<term> (nominatim source),
+ *         --tag-family=<key>=<value>, --fixture=<path> (offline input),
+ *         --out=<path> (write the proposal here — never the curated file).
+ *
+ * The identifying header is MANDATORY on every live request:
+ *   User-Agent: araharah-gazetteer-topup/1.0 (+https://github.com/araharah)
+ * A request without an identifying User-Agent is answered HTTP 406 by the
+ * Overpass endpoints (live-verified 2026-10-23) — the header is load-bearing,
+ * not cosmetic. Nominatim's usage policy asks for the same discipline.
+ *
+ * Exactly ONE network request is made per run: both live paths share the single
+ * requestJson helper below, so the one-request invariant is structural.
+ */
+import { readFileSync, writeFileSync } from "node:fs";
+import { resolve } from "node:path";
+
+const USER_AGENT = "araharah-gazetteer-topup/1.0 (+https://github.com/araharah)";
+const DEFAULT_ENDPOINT = "https://overpass-api.de/api/interpreter";
+const DEFAULT_BBOX = "3.05,101.55,3.25,101.80"; // s,w,n,e — the KL metro box
+const DEFAULT_TAG_FAMILY = "tourism=attraction";
+const TIMEOUT_SECONDS = 25;
+const CANDIDATE_LIMIT = 40;
+
+/* ————— flag parsing (both --flag=value and --flag value forms) ————— */
+
+function parseFlags(argv) {
+  const flags = {};
+  for (let i = 2; i < argv.length; i++) {
+    const arg = argv[i];
+    if (!arg.startsWith("--")) continue;
+    const eq = arg.indexOf("=");
+    if (eq !== -1) {
+      flags[arg.slice(2, eq)] = arg.slice(eq + 1);
+    } else {
+      const next = argv[i + 1];
+      if (next !== undefined && !next.startsWith("--")) {
+        flags[arg.slice(2)] = next;
+        i++;
+      } else {
+        flags[arg.slice(2)] = "";
+      }
+    }
+  }
+  return flags;
+}
+
+const flags = parseFlags(process.argv);
+// --fixture=<path> selects the offline fixture source by itself — passing a
+// fixture and silently hitting the live network instead would be a trap.
+const source = flags.fixture ? "fixture" : (flags.source ?? "overpass");
+const bbox = flags.bbox ?? DEFAULT_BBOX;
+
+/* ————— the ONE request helper (both live paths share it) ————— */
+
+/**
+ * The script's single network entry point: exactly one HTTP request, with
+ * the identifying User-Agent. Any non-2xx status, network error, or
+ * unparseable body returns null — the caller prints the soft-exit line.
+ */
+async function requestJson(url, options) {
+  try {
+    const res = await fetch(url, options);
+    if (!res.ok) return null;
+    return JSON.parse(await res.text());
+  } catch {
+    return null;
+  }
+}
+
+/** A busy/saturated/unusable service is a NORMAL outcome — exit 0. */
+function busyExit() {
+  console.log("Overpass instance busy — retry later");
+  process.exit(0);
+}
+
+/**
+ * The Overpass QL query — ALWAYS bbox-bounded (a global query times out and
+ * invites rate-limiting). The canonical default is pinned VERBATIM below:
+ * single tag family, bounded to the KL metro box (s,w,n,e).
+ */
+function overpassQuery(familyKey, familyValue, box) {
+  if (box === DEFAULT_BBOX && familyKey === "tourism" && familyValue === "attraction") {
+    return '[out:json][timeout:25];\nnwr["tourism"="attraction"](3.05,101.55,3.25,101.80);\nout center 40;';
+  }
+  return (
+    `[out:json][timeout:${TIMEOUT_SECONDS}];\n` +
+    `nwr["${familyKey}"="${familyValue}"](${box});\n` +
+    `out center ${CANDIDATE_LIMIT};`
+  );
+}
+
+function usageError(message) {
+  console.error(`overpass-topup: ${message}`);
+  process.exit(1);
+}
+
+/* ————— candidate shaping ————— */
+
+function overpassCandidates(elements) {
+  const candidates = [];
+  for (const element of elements) {
+    const tags = element.tags ?? {};
+    const name = tags.name ?? tags["name:en"];
+    if (!name) continue; // a nameless element is useless in a proposal
+    const point =
+      element.type === "node"
+        ? { lat: element.lat, lon: element.lon }
+        : { lat: element.center?.lat, lon: element.center?.lon };
+    if (typeof point.lat !== "number" || typeof point.lon !== "number") continue;
+    const category =
+      tags.tourism ?? tags.amenity ?? tags.railway ?? tags.office ?? tags.leisure ?? tags.shop ?? "unknown";
+    candidates.push({ name, osm: `${element.type}/${element.id}`, lat: point.lat, lon: point.lon, category });
+  }
+  return candidates;
+}
+
+function nominatimCandidates(items) {
+  const candidates = [];
+  for (const item of items) {
+    const lat = Number.parseFloat(item.lat);
+    const lon = Number.parseFloat(item.lon);
+    if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
+    const name = item.name ?? item.display_name;
+    if (!name) continue;
+    candidates.push({
+      name,
+      osm: `${item.osm_type ?? "?"}/${item.osm_id ?? "?"}`,
+      lat,
+      lon,
+      category: item.category ?? item.type ?? "unknown",
+    });
+  }
+  return candidates;
+}
+
+/* ————— sources ————— */
+
+async function runOverpass() {
+  const endpoint = flags.endpoint ?? process.env.OVERPASS_ENDPOINT ?? DEFAULT_ENDPOINT;
+  const [familyKey, familyValue] = (flags["tag-family"] ?? DEFAULT_TAG_FAMILY).split("=");
+  const query = overpassQuery(familyKey, familyValue, bbox);
+  const headers = {
+    "User-Agent": USER_AGENT,
+    "Content-Type": "application/x-www-form-urlencoded",
+  };
+  const parsed = await requestJson(endpoint, {
+    method: "POST",
+    headers,
+    body: `data=${encodeURIComponent(query)}`,
+  });
+  if (parsed === null || !Array.isArray(parsed.elements)) busyExit();
+  return { candidates: overpassCandidates(parsed.elements), origin: `${familyKey}=${familyValue}` };
+}
+
+async function runNominatim() {
+  if (!flags.q) usageError("--source=nominatim requires --q=<term>");
+  const [s, w, n, e] = bbox.split(",");
+  const url =
+    `https://nominatim.openstreetmap.org/search?format=jsonv2&q=${encodeURIComponent(flags.q)}` +
+    `&viewbox=${w},${n},${e},${s}&bounded=1&limit=${CANDIDATE_LIMIT}`;
+  const parsed = await requestJson(url, { headers: { "User-Agent": USER_AGENT } });
+  if (parsed === null || !Array.isArray(parsed)) busyExit();
+  return { candidates: nominatimCandidates(parsed), origin: `nominatim q="${flags.q}"` };
+}
+
+function runFixture() {
+  if (!flags.fixture) usageError("--source=fixture requires --fixture=<path>");
+  let parsed;
+  try {
+    parsed = JSON.parse(readFileSync(resolve(flags.fixture), "utf8"));
+  } catch {
+    usageError(`unreadable fixture: ${flags.fixture}`);
+  }
+  if (parsed === null || typeof parsed !== "object" || !Array.isArray(parsed.elements)) {
+    // Not an Overpass-shaped response — treat like a saturated instance:
+    // a soft, retryable outcome, never a stack trace, never a non-zero exit.
+    busyExit();
+  }
+  return { candidates: overpassCandidates(parsed.elements), origin: `fixture ${flags.fixture}` };
+}
+
+/* ————— the proposal (printed, or written to an explicit --out path) ————— */
+
+function proposalText(candidates, origin) {
+  const lines = [];
+  lines.push(`PROPOSAL — ${candidates.length} candidate(s) from ${origin} in bbox ${bbox}`);
+  lines.push("Review and hand-merge into the curated dataset file — this script NEVER writes it.");
+  for (const c of candidates) {
+    lines.push(
+      `  name=${c.name.padEnd(28)} osm=${c.osm.padEnd(18)} ` +
+        `lat=${c.lat.toFixed(4)}  lon=${c.lon.toFixed(4)}  category=${c.category}`,
+    );
+  }
+  return lines.join("\n");
+}
+
+const result =
+  source === "fixture" ? runFixture() : source === "nominatim" ? await runNominatim() : await runOverpass();
+
+const text = proposalText(result.candidates, result.origin);
+console.log(text);
+if (flags.out) {
+  writeFileSync(resolve(flags.out), `${text}\n`);
+  console.log(`\nproposal written to ${resolve(flags.out)} — review and hand-merge; the curated file was not touched.`);
+}
+process.exit(0);