arah2/search/scripts/overpass-topup.mjs

#!/usr/bin/env node
/*
 * overpass-topup.mjs — propose KL gazetteer top-up candidates from OpenStreetMap (D-06).
 * -------------------------------------------------------------------------------------
 * A MAINTAINER-TIME REPO TOOL. It is NOT a runtime dependency: nothing in
 * bot/src/ or web/src/ imports it, and no deploy, test, or corpus gate depends on
 * it succeeding. The Worker only ever reads the COMMITTED curated JSON.
 *
 * Output is a PROPOSAL — a printed candidate list (or a file at an explicit --out
 * path) that a human diffs and hand-merges into the curated dataset file.
 * THIS SCRIPT NEVER WRITES THE CURATED FILE.
 *
 * Exit codes: 0 on success AND on a busy instance (Overpass capacity saturation
 * is intermittent — 502/429/timeout is a NORMAL, retryable outcome). Non-zero
 * only on a usage error (no bbox resolvable / unreadable fixture).
 *
 * Inputs: --source=overpass|nominatim|fixture (default overpass), --endpoint=<url>
 *         (or OVERPASS_ENDPOINT), --bbox=s,w,n,e, --q=<term> (nominatim source),
 *         --tag-family=<key>=<value>, --fixture=<path> (offline input),
 *         --out=<path> (write the proposal here — never the curated file).
 *
 * The identifying header is MANDATORY on every live request:
 *   User-Agent: araharah-gazetteer-topup/1.0 (+https://github.com/araharah)
 * A request without an identifying User-Agent is answered HTTP 406 by the
 * Overpass endpoints (live-verified 2026-10-23) — the header is load-bearing,
 * not cosmetic. Nominatim's usage policy asks for the same discipline.
 *
 * Exactly ONE network request is made per run: both live paths share the single
 * requestJson helper below, so the one-request invariant is structural.
 */
import { readFileSync, writeFileSync } from "node:fs";
import { resolve } from "node:path";

const USER_AGENT = "araharah-gazetteer-topup/1.0 (+https://github.com/araharah)";
const DEFAULT_ENDPOINT = "https://overpass-api.de/api/interpreter";
const DEFAULT_BBOX = "3.05,101.55,3.25,101.80"; // s,w,n,e — the KL metro box
const DEFAULT_TAG_FAMILY = "tourism=attraction";
const TIMEOUT_SECONDS = 25;
const CANDIDATE_LIMIT = 40;

/* ————— flag parsing (both --flag=value and --flag value forms) ————— */

function parseFlags(argv) {
  const flags = {};
  for (let i = 2; i < argv.length; i++) {
    const arg = argv[i];
    if (!arg.startsWith("--")) continue;
    const eq = arg.indexOf("=");
    if (eq !== -1) {
      flags[arg.slice(2, eq)] = arg.slice(eq + 1);
    } else {
      const next = argv[i + 1];
      if (next !== undefined && !next.startsWith("--")) {
        flags[arg.slice(2)] = next;
        i++;
      } else {
        flags[arg.slice(2)] = "";
      }
    }
  }
  return flags;
}

const flags = parseFlags(process.argv);
// --fixture=<path> selects the offline fixture source by itself — passing a
// fixture and silently hitting the live network instead would be a trap.
const source = flags.fixture ? "fixture" : (flags.source ?? "overpass");
const bbox = flags.bbox ?? DEFAULT_BBOX;

/* ————— the ONE request helper (both live paths share it) ————— */

/**
 * The script's single network entry point: exactly one HTTP request, with
 * the identifying User-Agent. Any non-2xx status, network error, or
 * unparseable body returns null — the caller prints the soft-exit line.
 */
async function requestJson(url, options) {
  try {
    const res = await fetch(url, options);
    if (!res.ok) return null;
    return JSON.parse(await res.text());
  } catch {
    return null;
  }
}

/** A busy/saturated/unusable service is a NORMAL outcome — exit 0. */
function busyExit() {
  console.log("Overpass instance busy — retry later");
  process.exit(0);
}

/**
 * The Overpass QL query — ALWAYS bbox-bounded (a global query times out and
 * invites rate-limiting). The canonical default is pinned VERBATIM below:
 * single tag family, bounded to the KL metro box (s,w,n,e).
 */
function overpassQuery(familyKey, familyValue, box) {
  if (box === DEFAULT_BBOX && familyKey === "tourism" && familyValue === "attraction") {
    return '[out:json][timeout:25];\nnwr["tourism"="attraction"](3.05,101.55,3.25,101.80);\nout center 40;';
  }
  return (
    `[out:json][timeout:${TIMEOUT_SECONDS}];\n` +
    `nwr["${familyKey}"="${familyValue}"](${box});\n` +
    `out center ${CANDIDATE_LIMIT};`
  );
}

function usageError(message) {
  console.error(`overpass-topup: ${message}`);
  process.exit(1);
}

/* ————— candidate shaping ————— */

function overpassCandidates(elements) {
  const candidates = [];
  for (const element of elements) {
    const tags = element.tags ?? {};
    const name = tags.name ?? tags["name:en"];
    if (!name) continue; // a nameless element is useless in a proposal
    const point =
      element.type === "node"
        ? { lat: element.lat, lon: element.lon }
        : { lat: element.center?.lat, lon: element.center?.lon };
    if (typeof point.lat !== "number" || typeof point.lon !== "number") continue;
    const category =
      tags.tourism ?? tags.amenity ?? tags.railway ?? tags.office ?? tags.leisure ?? tags.shop ?? "unknown";
    candidates.push({ name, osm: `${element.type}/${element.id}`, lat: point.lat, lon: point.lon, category });
  }
  return candidates;
}

function nominatimCandidates(items) {
  const candidates = [];
  for (const item of items) {
    const lat = Number.parseFloat(item.lat);
    const lon = Number.parseFloat(item.lon);
    if (!Number.isFinite(lat) || !Number.isFinite(lon)) continue;
    const name = item.name ?? item.display_name;
    if (!name) continue;
    candidates.push({
      name,
      osm: `${item.osm_type ?? "?"}/${item.osm_id ?? "?"}`,
      lat,
      lon,
      category: item.category ?? item.type ?? "unknown",
    });
  }
  return candidates;
}

/* ————— sources ————— */

async function runOverpass() {
  const endpoint = flags.endpoint ?? process.env.OVERPASS_ENDPOINT ?? DEFAULT_ENDPOINT;
  const [familyKey, familyValue] = (flags["tag-family"] ?? DEFAULT_TAG_FAMILY).split("=");
  const query = overpassQuery(familyKey, familyValue, bbox);
  const headers = {
    "User-Agent": USER_AGENT,
    "Content-Type": "application/x-www-form-urlencoded",
  };
  const parsed = await requestJson(endpoint, {
    method: "POST",
    headers,
    body: `data=${encodeURIComponent(query)}`,
  });
  if (parsed === null || !Array.isArray(parsed.elements)) busyExit();
  return { candidates: overpassCandidates(parsed.elements), origin: `${familyKey}=${familyValue}` };
}

async function runNominatim() {
  if (!flags.q) usageError("--source=nominatim requires --q=<term>");
  const [s, w, n, e] = bbox.split(",");
  const url =
    `https://nominatim.openstreetmap.org/search?format=jsonv2&q=${encodeURIComponent(flags.q)}` +
    `&viewbox=${w},${n},${e},${s}&bounded=1&limit=${CANDIDATE_LIMIT}`;
  const parsed = await requestJson(url, { headers: { "User-Agent": USER_AGENT } });
  if (parsed === null || !Array.isArray(parsed)) busyExit();
  return { candidates: nominatimCandidates(parsed), origin: `nominatim q="${flags.q}"` };
}

function runFixture() {
  if (!flags.fixture) usageError("--source=fixture requires --fixture=<path>");
  let parsed;
  try {
    parsed = JSON.parse(readFileSync(resolve(flags.fixture), "utf8"));
  } catch {
    usageError(`unreadable fixture: ${flags.fixture}`);
  }
  if (parsed === null || typeof parsed !== "object" || !Array.isArray(parsed.elements)) {
    // Not an Overpass-shaped response — treat like a saturated instance:
    // a soft, retryable outcome, never a stack trace, never a non-zero exit.
    busyExit();
  }
  return { candidates: overpassCandidates(parsed.elements), origin: `fixture ${flags.fixture}` };
}

/* ————— the proposal (printed, or written to an explicit --out path) ————— */

function proposalText(candidates, origin) {
  const lines = [];
  lines.push(`PROPOSAL — ${candidates.length} candidate(s) from ${origin} in bbox ${bbox}`);
  lines.push("Review and hand-merge into the curated dataset file — this script NEVER writes it.");
  for (const c of candidates) {
    lines.push(
      `  name=${c.name.padEnd(28)} osm=${c.osm.padEnd(18)} ` +
        `lat=${c.lat.toFixed(4)}  lon=${c.lon.toFixed(4)}  category=${c.category}`,
    );
  }
  return lines.join("\n");
}

const result =
  source === "fixture" ? runFixture() : source === "nominatim" ? await runNominatim() : await runOverpass();

const text = proposalText(result.candidates, result.origin);
console.log(text);
if (flags.out) {
  writeFileSync(resolve(flags.out), `${text}\n`);
  console.log(`\nproposal written to ${resolve(flags.out)} — review and hand-merge; the curated file was not touched.`);
}
process.exit(0);