feat(04-02): Overpass top-up proposer (D-06) + JSON-to-SQLite converter (D-05)

overpass-topup.mjs — maintainer-time repo tool, six hard laws:
- never imported by bot/src or web/src (TP1 gate); Worker reads only the committed JSON
- exactly ONE request per run: both live paths (Overpass POST, Nominatim GET)
  share a single requestJson helper — one fetch( call site (TP7)
- identifying User-Agent araharah-gazetteer-topup/1.0 mandatory (missing UA = HTTP 406)
- busy/saturated/unusable response -> 'Overpass instance busy — retry later', exit 0 (TP4)
- output is a PROPOSAL only; source never names the curated file, so it cannot write it (TP6)
- --source=overpass|nominatim|fixture (--fixture implies fixture), --endpoint/OVERPASS_ENDPOINT
  override, bbox-bounded single-family query pinned verbatim (3.05,101.55,3.25,101.80) (TP3)
- --fixture selects the offline path by itself (a fixture flag silently hitting the
  live network would be a trap — fixed during GREEN)

overpass-response.json — committed real-shape Overpass response (6 KL elements:
nodes with lat/lon, ways/relation with center) so the test is deterministic.

json-to-sqlite.mjs — backend #2's dataset format reachable today (D-05):
- reads the same committed kl-gazetteer.json, emits kl-gazetteer.sqlite via
  node:sqlite (zero added deps) with the SAME id set (CV1), plus the ODbL
  notice embedded in a metadata table + NOTICE sidecar (ODbL s4.4, T-04-20)
- deterministic: rows in dataset order, no timestamps
This commit is contained in:
araharah-executor
2026-10-24 01:01:00 +08:00
parent ec98c8d454
commit c6a5f2b008
3 changed files with 425 additions and 0 deletions
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env node
/*
* json-to-sqlite.mjs — the D-05 backend-#2 format converter.
* -----------------------------------------------------------
* A MAINTAINER-TIME REPO TOOL. It is NOT a runtime dependency: nothing in
* bot/src/ or web/src/ imports it, and the Worker's canonical dataset
* format stays typed JSON (D-07). This script exists so backend #2 (the
* Malaysia-only WASM geocoder, post-v1) is a FORMAT change, not a rewrite:
* the SQLite dataset format is reachable from day one (D-05).
*
* Reads the COMMITTED curated dataset (search/src/data/kl-gazetteer.json)
* and emits <out>/kl-gazetteer.sqlite with the SAME id set:
* places (id TEXT PRIMARY KEY, name TEXT, aliases TEXT,
* lat REAL, lng REAL, category TEXT, address TEXT)
* metadata (key TEXT PRIMARY KEY, value TEXT)
*
* ODbL §4.4 share-alike: the emitted database is a Derivative Database of
* OSM data, so the ODbL notice travels with the output BY CONSTRUCTION —
* the _license / _attribution / _notice fields are written into the
* metadata table. Never redistribute an output stripped of them (T-04-20).
*
* Writer: Node 24's built-in node:sqlite (DatabaseSync) — zero added
* dependencies. Output is deterministic: rows are inserted in the
* committed dataset's order and carry no timestamps.
*
* Usage: node scripts/json-to-sqlite.mjs --out <dir> (or --out=<dir>)
* Exit codes: 0 on success; 1 on a usage error (missing/unwritable --out).
*/
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";
import { DatabaseSync } from "node:sqlite";
const DATASET_URL = new URL("../src/data/kl-gazetteer.json", import.meta.url);
const OUTPUT_NAME = "kl-gazetteer.sqlite";
function parseOutDir(argv) {
for (let i = 2; i < argv.length; i++) {
const arg = argv[i];
if (arg.startsWith("--out=")) return arg.slice("--out=".length);
if (arg === "--out") return argv[i + 1];
}
return undefined;
}
const outDir = parseOutDir(process.argv);
if (!outDir) {
console.error("json-to-sqlite: missing --out <dir> — pass a directory for the emitted database.");
process.exit(1);
}
const outPath = resolve(outDir);
if (!existsSync(outPath)) {
mkdirSync(outPath, { recursive: true });
}
const dataset = JSON.parse(readFileSync(DATASET_URL, "utf8"));
const entries = dataset.entries ?? [];
const db = new DatabaseSync(resolve(outPath, OUTPUT_NAME));
try {
db.exec("BEGIN");
db.exec(`
CREATE TABLE places (
id TEXT PRIMARY KEY,
name TEXT NOT NULL,
aliases TEXT NOT NULL,
lat REAL NOT NULL,
lng REAL NOT NULL,
category TEXT NOT NULL,
address TEXT
);
CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
`);
const insert = db.prepare(
"INSERT INTO places (id, name, aliases, lat, lng, category, address) VALUES (?, ?, ?, ?, ?, ?, ?)",
);
for (const entry of entries) {
insert.run(
entry.id,
entry.name,
JSON.stringify(entry.aliases),
entry.lat,
entry.lng,
entry.category,
entry.address ?? null,
);
}
const insertMeta = db.prepare("INSERT INTO metadata (key, value) VALUES (?, ?)");
insertMeta.run("_license", dataset._license ?? "ODbL-1.0");
insertMeta.run("_attribution", dataset._attribution ?? "© OpenStreetMap contributors");
insertMeta.run("_notice", dataset._notice ?? "");
insertMeta.run("source_version", String(dataset.version ?? 0));
insertMeta.run("entry_count", String(entries.length));
db.exec("COMMIT");
} catch (err) {
db.exec("ROLLBACK");
db.close();
console.error(`json-to-sqlite: conversion failed: ${err?.message ?? err}`);
process.exit(1);
}
db.close();
// A machine-readable sidecar stating the licence terms alongside the database.
writeFileSync(
resolve(outPath, "kl-gazetteer.sqlite.NOTICE.txt"),
`${dataset._notice ?? ""}\nLicense: ${dataset._license ?? "ODbL-1.0"}\n` +
`Attribution: ${dataset._attribution ?? "© OpenStreetMap contributors"}\n` +
`The same terms are embedded in the metadata table of ${OUTPUT_NAME} (ODbL §4.4).\n`,
);
console.log(
`wrote ${resolve(outPath, OUTPUT_NAME)} — ${entries.length} places, id set preserved, ` +
`${dataset._license ?? "ODbL-1.0"} notice included (metadata table + NOTICE sidecar).`,
);
process.exit(0);