#!/usr/bin/env node /* * overpass-topup.mjs — propose KL gazetteer top-up candidates from OpenStreetMap (D-06). * ------------------------------------------------------------------------------------- * A MAINTAINER-TIME REPO TOOL. It is NOT a runtime dependency: nothing in * bot/src/ or web/src/ imports it, and no deploy, test, or corpus gate depends on * it succeeding. The Worker only ever reads the COMMITTED curated JSON. * * Output is a PROPOSAL — a printed candidate list (or a file at an explicit --out * path) that a human diffs and hand-merges into the curated dataset file. * THIS SCRIPT NEVER WRITES THE CURATED FILE. * * Exit codes: 0 on success AND on a busy instance (Overpass capacity saturation * is intermittent — 502/429/timeout is a NORMAL, retryable outcome). Non-zero * only on a usage error (no bbox resolvable / unreadable fixture). * * Inputs: --source=overpass|nominatim|fixture (default overpass), --endpoint= * (or OVERPASS_ENDPOINT), --bbox=s,w,n,e, --q= (nominatim source), * --tag-family==, --fixture= (offline input), * --out= (write the proposal here — never the curated file). * * The identifying header is MANDATORY on every live request: * User-Agent: araharah-gazetteer-topup/1.0 (+https://github.com/araharah) * A request without an identifying User-Agent is answered HTTP 406 by the * Overpass endpoints (live-verified 2026-10-23) — the header is load-bearing, * not cosmetic. Nominatim's usage policy asks for the same discipline. * * Exactly ONE network request is made per run: both live paths share the single * requestJson helper below, so the one-request invariant is structural. */ import { readFileSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; const USER_AGENT = "araharah-gazetteer-topup/1.0 (+https://github.com/araharah)"; const DEFAULT_ENDPOINT = "https://overpass-api.de/api/interpreter"; const DEFAULT_BBOX = "3.05,101.55,3.25,101.80"; // s,w,n,e — the KL metro box const DEFAULT_TAG_FAMILY = "tourism=attraction"; const TIMEOUT_SECONDS = 25; const CANDIDATE_LIMIT = 40; /* ————— flag parsing (both --flag=value and --flag value forms) ————— */ function parseFlags(argv) { const flags = {}; for (let i = 2; i < argv.length; i++) { const arg = argv[i]; if (!arg.startsWith("--")) continue; const eq = arg.indexOf("="); if (eq !== -1) { flags[arg.slice(2, eq)] = arg.slice(eq + 1); } else { const next = argv[i + 1]; if (next !== undefined && !next.startsWith("--")) { flags[arg.slice(2)] = next; i++; } else { flags[arg.slice(2)] = ""; } } } return flags; } const flags = parseFlags(process.argv); // --fixture= selects the offline fixture source by itself — passing a // fixture and silently hitting the live network instead would be a trap. const source = flags.fixture ? "fixture" : (flags.source ?? "overpass"); const bbox = flags.bbox ?? DEFAULT_BBOX; /* ————— the ONE request helper (both live paths share it) ————— */ /** * The script's single network entry point: exactly one HTTP request, with * the identifying User-Agent. Any non-2xx status, network error, or * unparseable body returns null — the caller prints the soft-exit line. */ async function requestJson(url, options) { try { const res = await fetch(url, options); if (!res.ok) return null; return JSON.parse(await res.text()); } catch { return null; } } /** A busy/saturated/unusable service is a NORMAL outcome — exit 0. */ function busyExit() { console.log("Overpass instance busy — retry later"); process.exit(0); } /** * The Overpass QL query — ALWAYS bbox-bounded (a global query times out and * invites rate-limiting). The canonical default is pinned VERBATIM below: * single tag family, bounded to the KL metro box (s,w,n,e). */ function overpassQuery(familyKey, familyValue, box) { if (box === DEFAULT_BBOX && familyKey === "tourism" && familyValue === "attraction") { return '[out:json][timeout:25];\nnwr["tourism"="attraction"](3.05,101.55,3.25,101.80);\nout center 40;'; } return ( `[out:json][timeout:${TIMEOUT_SECONDS}];\n` + `nwr["${familyKey}"="${familyValue}"](${box});\n` + `out center ${CANDIDATE_LIMIT};` ); } function usageError(message) { console.error(`overpass-topup: ${message}`); process.exit(1); } /* ————— candidate shaping ————— */ function overpassCandidates(elements) { const candidates = []; for (const element of elements) { const tags = element.tags ?? {}; const name = tags.name ?? tags["name:en"]; if (!name) continue; // a nameless element is useless in a proposal const point = element.type === "node" ? { lat: element.lat, lon: element.lon } : { lat: element.center?.lat, lon: element.center?.lon }; if (typeof point.lat !== "number" || typeof point.lon !== "number") continue; const category = tags.tourism ?? tags.amenity ?? tags.railway ?? tags.office ?? tags.leisure ?? tags.shop ?? "unknown"; candidates.push({ name, osm: `${element.type}/${element.id}`, lat: point.lat, lon: point.lon, category }); } return candidates; } function nominatimCandidates(items) { const candidates = []; for (const item of items) { const lat = Number.parseFloat(item.lat); const lon = Number.parseFloat(item.lon); if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue; const name = item.name ?? item.display_name; if (!name) continue; candidates.push({ name, osm: `${item.osm_type ?? "?"}/${item.osm_id ?? "?"}`, lat, lon, category: item.category ?? item.type ?? "unknown", }); } return candidates; } /* ————— sources ————— */ async function runOverpass() { const endpoint = flags.endpoint ?? process.env.OVERPASS_ENDPOINT ?? DEFAULT_ENDPOINT; const [familyKey, familyValue] = (flags["tag-family"] ?? DEFAULT_TAG_FAMILY).split("="); const query = overpassQuery(familyKey, familyValue, bbox); const headers = { "User-Agent": USER_AGENT, "Content-Type": "application/x-www-form-urlencoded", }; const parsed = await requestJson(endpoint, { method: "POST", headers, body: `data=${encodeURIComponent(query)}`, }); if (parsed === null || !Array.isArray(parsed.elements)) busyExit(); return { candidates: overpassCandidates(parsed.elements), origin: `${familyKey}=${familyValue}` }; } async function runNominatim() { if (!flags.q) usageError("--source=nominatim requires --q="); const [s, w, n, e] = bbox.split(","); const url = `https://nominatim.openstreetmap.org/search?format=jsonv2&q=${encodeURIComponent(flags.q)}` + `&viewbox=${w},${n},${e},${s}&bounded=1&limit=${CANDIDATE_LIMIT}`; const parsed = await requestJson(url, { headers: { "User-Agent": USER_AGENT } }); if (parsed === null || !Array.isArray(parsed)) busyExit(); return { candidates: nominatimCandidates(parsed), origin: `nominatim q="${flags.q}"` }; } function runFixture() { if (!flags.fixture) usageError("--source=fixture requires --fixture="); let parsed; try { parsed = JSON.parse(readFileSync(resolve(flags.fixture), "utf8")); } catch { usageError(`unreadable fixture: ${flags.fixture}`); } if (parsed === null || typeof parsed !== "object" || !Array.isArray(parsed.elements)) { // Not an Overpass-shaped response — treat like a saturated instance: // a soft, retryable outcome, never a stack trace, never a non-zero exit. busyExit(); } return { candidates: overpassCandidates(parsed.elements), origin: `fixture ${flags.fixture}` }; } /* ————— the proposal (printed, or written to an explicit --out path) ————— */ function proposalText(candidates, origin) { const lines = []; lines.push(`PROPOSAL — ${candidates.length} candidate(s) from ${origin} in bbox ${bbox}`); lines.push("Review and hand-merge into the curated dataset file — this script NEVER writes it."); for (const c of candidates) { lines.push( ` name=${c.name.padEnd(28)} osm=${c.osm.padEnd(18)} ` + `lat=${c.lat.toFixed(4)} lon=${c.lon.toFixed(4)} category=${c.category}`, ); } return lines.join("\n"); } const result = source === "fixture" ? runFixture() : source === "nominatim" ? await runNominatim() : await runOverpass(); const text = proposalText(result.candidates, result.origin); console.log(text); if (flags.out) { writeFileSync(resolve(flags.out), `${text}\n`); console.log(`\nproposal written to ${resolve(flags.out)} — review and hand-merge; the curated file was not touched.`); } process.exit(0);