feat(04-02): Overpass top-up proposer (D-06) + JSON-to-SQLite converter (D-05)
overpass-topup.mjs — maintainer-time repo tool, six hard laws: - never imported by bot/src or web/src (TP1 gate); Worker reads only the committed JSON - exactly ONE request per run: both live paths (Overpass POST, Nominatim GET) share a single requestJson helper — one fetch( call site (TP7) - identifying User-Agent araharah-gazetteer-topup/1.0 mandatory (missing UA = HTTP 406) - busy/saturated/unusable response -> 'Overpass instance busy — retry later', exit 0 (TP4) - output is a PROPOSAL only; source never names the curated file, so it cannot write it (TP6) - --source=overpass|nominatim|fixture (--fixture implies fixture), --endpoint/OVERPASS_ENDPOINT override, bbox-bounded single-family query pinned verbatim (3.05,101.55,3.25,101.80) (TP3) - --fixture selects the offline path by itself (a fixture flag silently hitting the live network would be a trap — fixed during GREEN) overpass-response.json — committed real-shape Overpass response (6 KL elements: nodes with lat/lon, ways/relation with center) so the test is deterministic. json-to-sqlite.mjs — backend #2's dataset format reachable today (D-05): - reads the same committed kl-gazetteer.json, emits kl-gazetteer.sqlite via node:sqlite (zero added deps) with the SAME id set (CV1), plus the ODbL notice embedded in a metadata table + NOTICE sidecar (ODbL s4.4, T-04-20) - deterministic: rows in dataset order, no timestamps
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* json-to-sqlite.mjs — the D-05 backend-#2 format converter.
|
||||
* -----------------------------------------------------------
|
||||
* A MAINTAINER-TIME REPO TOOL. It is NOT a runtime dependency: nothing in
|
||||
* bot/src/ or web/src/ imports it, and the Worker's canonical dataset
|
||||
* format stays typed JSON (D-07). This script exists so backend #2 (the
|
||||
* Malaysia-only WASM geocoder, post-v1) is a FORMAT change, not a rewrite:
|
||||
* the SQLite dataset format is reachable from day one (D-05).
|
||||
*
|
||||
* Reads the COMMITTED curated dataset (search/src/data/kl-gazetteer.json)
|
||||
* and emits <out>/kl-gazetteer.sqlite with the SAME id set:
|
||||
* places (id TEXT PRIMARY KEY, name TEXT, aliases TEXT,
|
||||
* lat REAL, lng REAL, category TEXT, address TEXT)
|
||||
* metadata (key TEXT PRIMARY KEY, value TEXT)
|
||||
*
|
||||
* ODbL §4.4 share-alike: the emitted database is a Derivative Database of
|
||||
* OSM data, so the ODbL notice travels with the output BY CONSTRUCTION —
|
||||
* the _license / _attribution / _notice fields are written into the
|
||||
* metadata table. Never redistribute an output stripped of them (T-04-20).
|
||||
*
|
||||
* Writer: Node 24's built-in node:sqlite (DatabaseSync) — zero added
|
||||
* dependencies. Output is deterministic: rows are inserted in the
|
||||
* committed dataset's order and carry no timestamps.
|
||||
*
|
||||
* Usage: node scripts/json-to-sqlite.mjs --out <dir> (or --out=<dir>)
|
||||
* Exit codes: 0 on success; 1 on a usage error (missing/unwritable --out).
|
||||
*/
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { DatabaseSync } from "node:sqlite";
|
||||
|
||||
const DATASET_URL = new URL("../src/data/kl-gazetteer.json", import.meta.url);
|
||||
const OUTPUT_NAME = "kl-gazetteer.sqlite";
|
||||
|
||||
function parseOutDir(argv) {
|
||||
for (let i = 2; i < argv.length; i++) {
|
||||
const arg = argv[i];
|
||||
if (arg.startsWith("--out=")) return arg.slice("--out=".length);
|
||||
if (arg === "--out") return argv[i + 1];
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const outDir = parseOutDir(process.argv);
|
||||
if (!outDir) {
|
||||
console.error("json-to-sqlite: missing --out <dir> — pass a directory for the emitted database.");
|
||||
process.exit(1);
|
||||
}
|
||||
const outPath = resolve(outDir);
|
||||
if (!existsSync(outPath)) {
|
||||
mkdirSync(outPath, { recursive: true });
|
||||
}
|
||||
|
||||
const dataset = JSON.parse(readFileSync(DATASET_URL, "utf8"));
|
||||
const entries = dataset.entries ?? [];
|
||||
|
||||
const db = new DatabaseSync(resolve(outPath, OUTPUT_NAME));
|
||||
try {
|
||||
db.exec("BEGIN");
|
||||
db.exec(`
|
||||
CREATE TABLE places (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
aliases TEXT NOT NULL,
|
||||
lat REAL NOT NULL,
|
||||
lng REAL NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
address TEXT
|
||||
);
|
||||
CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
|
||||
`);
|
||||
const insert = db.prepare(
|
||||
"INSERT INTO places (id, name, aliases, lat, lng, category, address) VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
);
|
||||
for (const entry of entries) {
|
||||
insert.run(
|
||||
entry.id,
|
||||
entry.name,
|
||||
JSON.stringify(entry.aliases),
|
||||
entry.lat,
|
||||
entry.lng,
|
||||
entry.category,
|
||||
entry.address ?? null,
|
||||
);
|
||||
}
|
||||
const insertMeta = db.prepare("INSERT INTO metadata (key, value) VALUES (?, ?)");
|
||||
insertMeta.run("_license", dataset._license ?? "ODbL-1.0");
|
||||
insertMeta.run("_attribution", dataset._attribution ?? "© OpenStreetMap contributors");
|
||||
insertMeta.run("_notice", dataset._notice ?? "");
|
||||
insertMeta.run("source_version", String(dataset.version ?? 0));
|
||||
insertMeta.run("entry_count", String(entries.length));
|
||||
db.exec("COMMIT");
|
||||
} catch (err) {
|
||||
db.exec("ROLLBACK");
|
||||
db.close();
|
||||
console.error(`json-to-sqlite: conversion failed: ${err?.message ?? err}`);
|
||||
process.exit(1);
|
||||
}
|
||||
db.close();
|
||||
|
||||
// A machine-readable sidecar stating the licence terms alongside the database.
|
||||
writeFileSync(
|
||||
resolve(outPath, "kl-gazetteer.sqlite.NOTICE.txt"),
|
||||
`${dataset._notice ?? ""}\nLicense: ${dataset._license ?? "ODbL-1.0"}\n` +
|
||||
`Attribution: ${dataset._attribution ?? "© OpenStreetMap contributors"}\n` +
|
||||
`The same terms are embedded in the metadata table of ${OUTPUT_NAME} (ODbL §4.4).\n`,
|
||||
);
|
||||
|
||||
console.log(
|
||||
`wrote ${resolve(outPath, OUTPUT_NAME)} — ${entries.length} places, id set preserved, ` +
|
||||
`${dataset._license ?? "ODbL-1.0"} notice included (metadata table + NOTICE sidecar).`,
|
||||
);
|
||||
process.exit(0);
|
||||
Reference in New Issue
Block a user