diff --git a/search/scripts/json-to-sqlite.mjs b/search/scripts/json-to-sqlite.mjs new file mode 100644 index 0000000..f93f224 --- /dev/null +++ b/search/scripts/json-to-sqlite.mjs @@ -0,0 +1,114 @@ +#!/usr/bin/env node +/* + * json-to-sqlite.mjs — the D-05 backend-#2 format converter. + * ----------------------------------------------------------- + * A MAINTAINER-TIME REPO TOOL. It is NOT a runtime dependency: nothing in + * bot/src/ or web/src/ imports it, and the Worker's canonical dataset + * format stays typed JSON (D-07). This script exists so backend #2 (the + * Malaysia-only WASM geocoder, post-v1) is a FORMAT change, not a rewrite: + * the SQLite dataset format is reachable from day one (D-05). + * + * Reads the COMMITTED curated dataset (search/src/data/kl-gazetteer.json) + * and emits /kl-gazetteer.sqlite with the SAME id set: + * places (id TEXT PRIMARY KEY, name TEXT, aliases TEXT, + * lat REAL, lng REAL, category TEXT, address TEXT) + * metadata (key TEXT PRIMARY KEY, value TEXT) + * + * ODbL §4.4 share-alike: the emitted database is a Derivative Database of + * OSM data, so the ODbL notice travels with the output BY CONSTRUCTION — + * the _license / _attribution / _notice fields are written into the + * metadata table. Never redistribute an output stripped of them (T-04-20). + * + * Writer: Node 24's built-in node:sqlite (DatabaseSync) — zero added + * dependencies. Output is deterministic: rows are inserted in the + * committed dataset's order and carry no timestamps. + * + * Usage: node scripts/json-to-sqlite.mjs --out (or --out=) + * Exit codes: 0 on success; 1 on a usage error (missing/unwritable --out). + */ +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { DatabaseSync } from "node:sqlite"; + +const DATASET_URL = new URL("../src/data/kl-gazetteer.json", import.meta.url); +const OUTPUT_NAME = "kl-gazetteer.sqlite"; + +function parseOutDir(argv) { + for (let i = 2; i < argv.length; i++) { + const arg = argv[i]; + if (arg.startsWith("--out=")) return arg.slice("--out=".length); + if (arg === "--out") return argv[i + 1]; + } + return undefined; +} + +const outDir = parseOutDir(process.argv); +if (!outDir) { + console.error("json-to-sqlite: missing --out — pass a directory for the emitted database."); + process.exit(1); +} +const outPath = resolve(outDir); +if (!existsSync(outPath)) { + mkdirSync(outPath, { recursive: true }); +} + +const dataset = JSON.parse(readFileSync(DATASET_URL, "utf8")); +const entries = dataset.entries ?? []; + +const db = new DatabaseSync(resolve(outPath, OUTPUT_NAME)); +try { + db.exec("BEGIN"); + db.exec(` + CREATE TABLE places ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + aliases TEXT NOT NULL, + lat REAL NOT NULL, + lng REAL NOT NULL, + category TEXT NOT NULL, + address TEXT + ); + CREATE TABLE metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL); + `); + const insert = db.prepare( + "INSERT INTO places (id, name, aliases, lat, lng, category, address) VALUES (?, ?, ?, ?, ?, ?, ?)", + ); + for (const entry of entries) { + insert.run( + entry.id, + entry.name, + JSON.stringify(entry.aliases), + entry.lat, + entry.lng, + entry.category, + entry.address ?? null, + ); + } + const insertMeta = db.prepare("INSERT INTO metadata (key, value) VALUES (?, ?)"); + insertMeta.run("_license", dataset._license ?? "ODbL-1.0"); + insertMeta.run("_attribution", dataset._attribution ?? "© OpenStreetMap contributors"); + insertMeta.run("_notice", dataset._notice ?? ""); + insertMeta.run("source_version", String(dataset.version ?? 0)); + insertMeta.run("entry_count", String(entries.length)); + db.exec("COMMIT"); +} catch (err) { + db.exec("ROLLBACK"); + db.close(); + console.error(`json-to-sqlite: conversion failed: ${err?.message ?? err}`); + process.exit(1); +} +db.close(); + +// A machine-readable sidecar stating the licence terms alongside the database. +writeFileSync( + resolve(outPath, "kl-gazetteer.sqlite.NOTICE.txt"), + `${dataset._notice ?? ""}\nLicense: ${dataset._license ?? "ODbL-1.0"}\n` + + `Attribution: ${dataset._attribution ?? "© OpenStreetMap contributors"}\n` + + `The same terms are embedded in the metadata table of ${OUTPUT_NAME} (ODbL §4.4).\n`, +); + +console.log( + `wrote ${resolve(outPath, OUTPUT_NAME)} — ${entries.length} places, id set preserved, ` + + `${dataset._license ?? "ODbL-1.0"} notice included (metadata table + NOTICE sidecar).`, +); +process.exit(0); diff --git a/search/scripts/overpass-topup.mjs b/search/scripts/overpass-topup.mjs new file mode 100644 index 0000000..f001080 --- /dev/null +++ b/search/scripts/overpass-topup.mjs @@ -0,0 +1,223 @@ +#!/usr/bin/env node +/* + * overpass-topup.mjs — propose KL gazetteer top-up candidates from OpenStreetMap (D-06). + * ------------------------------------------------------------------------------------- + * A MAINTAINER-TIME REPO TOOL. It is NOT a runtime dependency: nothing in + * bot/src/ or web/src/ imports it, and no deploy, test, or corpus gate depends on + * it succeeding. The Worker only ever reads the COMMITTED curated JSON. + * + * Output is a PROPOSAL — a printed candidate list (or a file at an explicit --out + * path) that a human diffs and hand-merges into the curated dataset file. + * THIS SCRIPT NEVER WRITES THE CURATED FILE. + * + * Exit codes: 0 on success AND on a busy instance (Overpass capacity saturation + * is intermittent — 502/429/timeout is a NORMAL, retryable outcome). Non-zero + * only on a usage error (no bbox resolvable / unreadable fixture). + * + * Inputs: --source=overpass|nominatim|fixture (default overpass), --endpoint= + * (or OVERPASS_ENDPOINT), --bbox=s,w,n,e, --q= (nominatim source), + * --tag-family==, --fixture= (offline input), + * --out= (write the proposal here — never the curated file). + * + * The identifying header is MANDATORY on every live request: + * User-Agent: araharah-gazetteer-topup/1.0 (+https://github.com/araharah) + * A request without an identifying User-Agent is answered HTTP 406 by the + * Overpass endpoints (live-verified 2026-10-23) — the header is load-bearing, + * not cosmetic. Nominatim's usage policy asks for the same discipline. + * + * Exactly ONE network request is made per run: both live paths share the single + * requestJson helper below, so the one-request invariant is structural. + */ +import { readFileSync, writeFileSync } from "node:fs"; +import { resolve } from "node:path"; + +const USER_AGENT = "araharah-gazetteer-topup/1.0 (+https://github.com/araharah)"; +const DEFAULT_ENDPOINT = "https://overpass-api.de/api/interpreter"; +const DEFAULT_BBOX = "3.05,101.55,3.25,101.80"; // s,w,n,e — the KL metro box +const DEFAULT_TAG_FAMILY = "tourism=attraction"; +const TIMEOUT_SECONDS = 25; +const CANDIDATE_LIMIT = 40; + +/* ————— flag parsing (both --flag=value and --flag value forms) ————— */ + +function parseFlags(argv) { + const flags = {}; + for (let i = 2; i < argv.length; i++) { + const arg = argv[i]; + if (!arg.startsWith("--")) continue; + const eq = arg.indexOf("="); + if (eq !== -1) { + flags[arg.slice(2, eq)] = arg.slice(eq + 1); + } else { + const next = argv[i + 1]; + if (next !== undefined && !next.startsWith("--")) { + flags[arg.slice(2)] = next; + i++; + } else { + flags[arg.slice(2)] = ""; + } + } + } + return flags; +} + +const flags = parseFlags(process.argv); +// --fixture= selects the offline fixture source by itself — passing a +// fixture and silently hitting the live network instead would be a trap. +const source = flags.fixture ? "fixture" : (flags.source ?? "overpass"); +const bbox = flags.bbox ?? DEFAULT_BBOX; + +/* ————— the ONE request helper (both live paths share it) ————— */ + +/** + * The script's single network entry point: exactly one HTTP request, with + * the identifying User-Agent. Any non-2xx status, network error, or + * unparseable body returns null — the caller prints the soft-exit line. + */ +async function requestJson(url, options) { + try { + const res = await fetch(url, options); + if (!res.ok) return null; + return JSON.parse(await res.text()); + } catch { + return null; + } +} + +/** A busy/saturated/unusable service is a NORMAL outcome — exit 0. */ +function busyExit() { + console.log("Overpass instance busy — retry later"); + process.exit(0); +} + +/** + * The Overpass QL query — ALWAYS bbox-bounded (a global query times out and + * invites rate-limiting). The canonical default is pinned VERBATIM below: + * single tag family, bounded to the KL metro box (s,w,n,e). + */ +function overpassQuery(familyKey, familyValue, box) { + if (box === DEFAULT_BBOX && familyKey === "tourism" && familyValue === "attraction") { + return '[out:json][timeout:25];\nnwr["tourism"="attraction"](3.05,101.55,3.25,101.80);\nout center 40;'; + } + return ( + `[out:json][timeout:${TIMEOUT_SECONDS}];\n` + + `nwr["${familyKey}"="${familyValue}"](${box});\n` + + `out center ${CANDIDATE_LIMIT};` + ); +} + +function usageError(message) { + console.error(`overpass-topup: ${message}`); + process.exit(1); +} + +/* ————— candidate shaping ————— */ + +function overpassCandidates(elements) { + const candidates = []; + for (const element of elements) { + const tags = element.tags ?? {}; + const name = tags.name ?? tags["name:en"]; + if (!name) continue; // a nameless element is useless in a proposal + const point = + element.type === "node" + ? { lat: element.lat, lon: element.lon } + : { lat: element.center?.lat, lon: element.center?.lon }; + if (typeof point.lat !== "number" || typeof point.lon !== "number") continue; + const category = + tags.tourism ?? tags.amenity ?? tags.railway ?? tags.office ?? tags.leisure ?? tags.shop ?? "unknown"; + candidates.push({ name, osm: `${element.type}/${element.id}`, lat: point.lat, lon: point.lon, category }); + } + return candidates; +} + +function nominatimCandidates(items) { + const candidates = []; + for (const item of items) { + const lat = Number.parseFloat(item.lat); + const lon = Number.parseFloat(item.lon); + if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; + const name = item.name ?? item.display_name; + if (!name) continue; + candidates.push({ + name, + osm: `${item.osm_type ?? "?"}/${item.osm_id ?? "?"}`, + lat, + lon, + category: item.category ?? item.type ?? "unknown", + }); + } + return candidates; +} + +/* ————— sources ————— */ + +async function runOverpass() { + const endpoint = flags.endpoint ?? process.env.OVERPASS_ENDPOINT ?? DEFAULT_ENDPOINT; + const [familyKey, familyValue] = (flags["tag-family"] ?? DEFAULT_TAG_FAMILY).split("="); + const query = overpassQuery(familyKey, familyValue, bbox); + const headers = { + "User-Agent": USER_AGENT, + "Content-Type": "application/x-www-form-urlencoded", + }; + const parsed = await requestJson(endpoint, { + method: "POST", + headers, + body: `data=${encodeURIComponent(query)}`, + }); + if (parsed === null || !Array.isArray(parsed.elements)) busyExit(); + return { candidates: overpassCandidates(parsed.elements), origin: `${familyKey}=${familyValue}` }; +} + +async function runNominatim() { + if (!flags.q) usageError("--source=nominatim requires --q="); + const [s, w, n, e] = bbox.split(","); + const url = + `https://nominatim.openstreetmap.org/search?format=jsonv2&q=${encodeURIComponent(flags.q)}` + + `&viewbox=${w},${n},${e},${s}&bounded=1&limit=${CANDIDATE_LIMIT}`; + const parsed = await requestJson(url, { headers: { "User-Agent": USER_AGENT } }); + if (parsed === null || !Array.isArray(parsed)) busyExit(); + return { candidates: nominatimCandidates(parsed), origin: `nominatim q="${flags.q}"` }; +} + +function runFixture() { + if (!flags.fixture) usageError("--source=fixture requires --fixture="); + let parsed; + try { + parsed = JSON.parse(readFileSync(resolve(flags.fixture), "utf8")); + } catch { + usageError(`unreadable fixture: ${flags.fixture}`); + } + if (parsed === null || typeof parsed !== "object" || !Array.isArray(parsed.elements)) { + // Not an Overpass-shaped response — treat like a saturated instance: + // a soft, retryable outcome, never a stack trace, never a non-zero exit. + busyExit(); + } + return { candidates: overpassCandidates(parsed.elements), origin: `fixture ${flags.fixture}` }; +} + +/* ————— the proposal (printed, or written to an explicit --out path) ————— */ + +function proposalText(candidates, origin) { + const lines = []; + lines.push(`PROPOSAL — ${candidates.length} candidate(s) from ${origin} in bbox ${bbox}`); + lines.push("Review and hand-merge into the curated dataset file — this script NEVER writes it."); + for (const c of candidates) { + lines.push( + ` name=${c.name.padEnd(28)} osm=${c.osm.padEnd(18)} ` + + `lat=${c.lat.toFixed(4)} lon=${c.lon.toFixed(4)} category=${c.category}`, + ); + } + return lines.join("\n"); +} + +const result = + source === "fixture" ? runFixture() : source === "nominatim" ? await runNominatim() : await runOverpass(); + +const text = proposalText(result.candidates, result.origin); +console.log(text); +if (flags.out) { + writeFileSync(resolve(flags.out), `${text}\n`); + console.log(`\nproposal written to ${resolve(flags.out)} — review and hand-merge; the curated file was not touched.`); +} +process.exit(0); diff --git a/search/test/fixtures/overpass-response.json b/search/test/fixtures/overpass-response.json new file mode 100644 index 0000000..4b2ca04 --- /dev/null +++ b/search/test/fixtures/overpass-response.json @@ -0,0 +1,88 @@ +{ + "version": 0.6, + "generator": "Overpass API 0.7.62.7", + "osm3s": { + "timestamp_osm_base": "2026-10-23T14:55:20Z", + "copyright": "The data included in this document is from www.openstreetmap.org. The data is made available under ODbL." + }, + "elements": [ + { + "type": "way", + "id": 23034336, + "center": { "lat": 3.1408, "lon": 101.7047 }, + "tags": { + "name": "Merdeka 118", + "name:en": "Merdeka 118", + "building": "yes", + "building:levels": "118", + "height": "678.9", + "tourism": "attraction", + "wikidata": "Q18166176" + } + }, + { + "type": "way", + "id": 12345678, + "center": { "lat": 3.1489, "lon": 101.6936 }, + "tags": { + "name": "National Monument", + "name:en": "National Monument", + "alt_name": "Tugu Negara", + "historic": "memorial", + "tourism": "attraction" + } + }, + { + "type": "node", + "id": 87654321, + "lat": 3.1484, + "lon": 101.6953, + "tags": { + "name": "Masjid Jamek", + "name:en": "Jamek Mosque", + "amenity": "place_of_worship", + "religion": "muslim", + "tourism": "attraction", + "wikidata": "Q2713345" + } + }, + { + "type": "way", + "id": 34567123, + "center": { "lat": 3.1528, "lon": 101.7038 }, + "tags": { + "name": "Kuala Lumpur Tower", + "name:en": "Kuala Lumpur Tower", + "short_name": "KL Tower", + "man_made": "tower", + "tourism": "attraction", + "wikidata": "Q265917" + } + }, + { + "type": "way", + "id": 45678123, + "center": { "lat": 3.1454, "lon": 101.6953 }, + "tags": { + "name": "Central Market", + "name:en": "Central Market", + "alt_name": "Pasar Seni", + "building": "retail", + "tourism": "attraction", + "wikidata": "Q5061407" + } + }, + { + "type": "relation", + "id": 56781234, + "center": { "lat": 3.1436, "lon": 101.6975 }, + "tags": { + "name": "Petaling Street", + "name:en": "Petaling Street", + "highway": "pedestrian", + "tourism": "attraction", + "wikidata": "Q1985815" + } + } + ] +}