diff --git a/routing/data/.gitignore b/routing/data/.gitignore new file mode 100644 index 0000000..9acf895 --- /dev/null +++ b/routing/data/.gitignore @@ -0,0 +1,3 @@ +# D-01: the road-graph binary is an ODbL-derived BUILD ARTIFACT, never +# committed — forks build + upload their own (see docs/ROAD-GRAPH.md). +*.bin diff --git a/routing/scripts/build-road-graph.mjs b/routing/scripts/build-road-graph.mjs new file mode 100644 index 0000000..354271f --- /dev/null +++ b/routing/scripts/build-road-graph.mjs @@ -0,0 +1,339 @@ +#!/usr/bin/env node +/* + * build-road-graph.mjs — the D-01 engine-A graph build tool (Overpass → CSR). + * --------------------------------------------------------------------------- + * A MAINTAINER-TIME REPO TOOL — NOT a runtime dependency; nothing in bot/src, + * web/src, or routing/src imports it. No deploy, test, or CI run ever executes + * it. A maintainer (or a fork) runs it on their own machine to produce the + * KL road-graph binary, then uploads it to their OWN R2 bucket (D-01: + * fork-friendly, $0 — see docs/ROAD-GRAPH.md). + * + * Mechanically: + * 1. The KL bbox is COMPUTED at run start from the committed corpus — + * @araharah/search getEntries() extremes + the golden-pairs origin + * coordinates ± BBOX_MARGIN_DEG — and printed (Open Q 4's mechanical + * pinning: the bbox tunes itself when the corpus grows; never hand-edited). + * 2. ONE Overpass query fetches every way whose highway tag is in + * DRIVABLE_CLASSES (footway/cycleway/service/path NEVER enter — 06-RESEARCH + * Pitfall 1: an unfiltered highway=* graph is the classic CPU-budget kill). + * 3. Ways → nodes (deduplicated by rounded coordinate) → directed edges with + * oneway flags, speed-derived seconds, and street-name pool ids. + * 4. encodeGraph (../src/graph.ts — the layout's SINGLE authority) produces + * the pinned CSR binary. COLLECT-THEN-WRITE: routing/data/ + * kl-roadgraph.v1.bin is written ONLY after the encode and the stats gate + * both pass — any failure exits 1 with the previous binary untouched. + * 5. THE STATS GATE prints nodeCount/edgeCount/nameCount/bbox/binary size and + * enforces the D-01 envelope: size ≤ MAX_BINARY_BYTES or exit 1. A result + * below the expected 30–80 MB band prints an ADVISORY warning (A2's + * forecast) — the ENFORCED floor is the 1 MB honesty check in --verify, + * never the forecast band. + * + * The binary is a BUILD ARTIFACT, not committed: routing/data/.gitignore + * ignores *.bin — forks build + upload their own (ODbL-derived data stays out + * of the repo; the attribution travels in docs/ROAD-GRAPH.md and the upload + * path). The committed artifacts are this script, the docs, and the tiny + * test mini-graph. + * + * Usage: + * node routing/scripts/build-road-graph.mjs + * # full build: fetch → encode → gate → write + * # (network fetch; minutes on the KL bbox) + * node routing/scripts/build-road-graph.mjs --check + * # decode an existing binary with decodeGraph, print stats, + * # exit 0/1 — NO network + * + * Exit codes: 0 built and gated (or checked green); 1 pre-flight failure, + * fetch failure, envelope-gate failure, or --check decode failure. + */ +import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const here = dirname(fileURLToPath(import.meta.url)); // routing/scripts +const CORPUS_PATH = resolve(here, "..", "test", "fixtures", "golden-pairs.json"); +const OUT_PATH = resolve(here, "..", "data", "kl-roadgraph.v1.bin"); + +/* ————— named constants (no inline magic) ————— */ + +const OVERPASS_ENDPOINT = "https://overpass-api.de/api/interpreter"; +/** Overpass server-side budget; matches the client fetch timeout. */ +const REQUEST_TIMEOUT_MS = 300_000; +const USER_AGENT = "araharah-roadgraph/1.0 (build-time KL road graph; repo: araharah)"; +/** D-01 envelope ceiling — a larger binary must NOT be written (shrink the + * class filter or the bbox instead). */ +const MAX_BINARY_BYTES = 80 * 1024 * 1024; +/** A2's honest-build forecast floor — crossing it downward is an ADVISORY + * (verify the class filter), never a gate. */ +const EXPECTED_MIN_BYTES = 30 * 1024 * 1024; +/** How far the computed bbox extends beyond the committed corpus extremes. */ +const BBOX_MARGIN_DEG = 0.05; +/** Node dedup granularity: 1e-6 deg ≈ 0.11 m — far under GPS noise. */ +const COORD_ROUND_DECIMALS = 6; +/** drivable classes ONLY (06-RESEARCH Pitfall 1): footway/cycleway/service/ + * path are deliberately absent — a walking path must never enter a road A*. */ +const DRIVABLE_CLASSES = new Set([ + "motorway", + "trunk", + "primary", + "secondary", + "tertiary", + "residential", + "unclassified", + "living_street", +]); +/** Class speed defaults (km/h) when a way carries no parsable maxspeed. */ +const CLASS_SPEED_KMH = { + motorway: 100, + trunk: 80, + primary: 60, + secondary: 50, + tertiary: 40, + residential: 30, + unclassified: 30, + living_street: 20, +}; +/** Motorway link ramps inherit the parent class speed via these links. */ +const LINK_CLASS_SPEED_KMH = { motorway_link: 60, trunk_link: 60 }; + +function fail(message) { + console.error(`build-road-graph: ${message}`); + process.exit(1); +} + +/* ————— --check mode: decode + stats, no network ————— */ + +const checkIdx = process.argv.indexOf("--check"); +if (checkIdx !== -1) { + const file = process.argv[checkIdx + 1]; + if (file === undefined) fail("--check requires a file path"); + if (!existsSync(file)) fail(`--check: file not found: ${file}`); + const { decodeGraph } = await import("../src/graph.ts"); + const buffer = readFileSync(file); + const graph = decodeGraph(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength)); + if (graph === null) fail(`--check: ${file} is NOT a valid GRAPH_VERSION CSR binary (decodeGraph null)`); + const size = statSync(file).size; + console.log( + `--check ${file}\n` + + ` nodeCount: ${graph.nodeCount}\n` + + ` edgeCount: ${graph.edgeCount}\n` + + ` nameCount: ${graph.nameCount}\n` + + ` maxSpeedKmh: ${graph.maxSpeedKmh}\n` + + ` bbox: [${graph.bbox.minLat}, ${graph.bbox.minLng}, ${graph.bbox.maxLat}, ${graph.bbox.maxLng}]\n` + + ` bytes: ${size}`, + ); + if (size > MAX_BINARY_BYTES) fail(`--check: ${size} bytes exceeds the ${MAX_BINARY_BYTES}-byte D-01 envelope`); + if (size <= 1024) fail(`--check: ${size} bytes is implausibly small — not a real road graph`); + process.exit(0); +} + +/* ————— inputs ————— */ + +const { encodeGraph } = await import("../src/graph.ts"); +const { haversineM } = await import("../src/geo.ts"); + +if (!existsSync(CORPUS_PATH)) fail(`corpus not found: ${CORPUS_PATH}`); +const corpus = JSON.parse(readFileSync(CORPUS_PATH, "utf8")); +const { getEntries } = await import("@araharah/search/src/gazetteer.ts"); + +/* The committed corpus defines the area engine A must route (Open Q 4): + * gazetteer extremes cover the search surface; golden-pair origins/destinations + * are gazetteer landmarks themselves — both are unioned so a future corpus + * extension widens the graph mechanically. */ +function computeBbox() { + let minLat = Infinity; + let minLng = Infinity; + let maxLat = -Infinity; + let maxLng = -Infinity; + const absorb = (lat, lng) => { + if (!Number.isFinite(lat) || !Number.isFinite(lng)) fail(`non-finite coordinate in corpus: (${lat}, ${lng})`); + if (lat < minLat) minLat = lat; + if (lat > maxLat) maxLat = lat; + if (lng < minLng) minLng = lng; + if (lng > maxLng) maxLng = lng; + }; + for (const entry of getEntries()) absorb(entry.lat, entry.lng); + for (const pair of corpus.pairs) absorb(pair.origin.lat, pair.origin.lng); + const gazetteerById = new Map(getEntries().map((e) => [e.id, e])); + for (const pair of corpus.pairs) { + const dest = gazetteerById.get(pair.destinationId); + if (dest === undefined) fail(`pair ${pair.pairId}: destinationId "${pair.destinationId}" is not a gazetteer entry`); + absorb(dest.lat, dest.lng); + } + return { + minLat: minLat - BBOX_MARGIN_DEG, + minLng: minLng - BBOX_MARGIN_DEG, + maxLat: maxLat + BBOX_MARGIN_DEG, + maxLng: maxLng + BBOX_MARGIN_DEG, + }; +} + +const bbox = computeBbox(); +console.log( + `bbox (corpus extremes ± ${BBOX_MARGIN_DEG} deg): ` + + `[${bbox.minLat.toFixed(4)}, ${bbox.minLng.toFixed(4)}, ${bbox.maxLat.toFixed(4)}, ${bbox.maxLng.toFixed(4)}]`, +); + +/* ————— the Overpass fetch (ONE request, drivable classes only) ————— */ + +const CLASSES_RE = "^(" + [...DRIVABLE_CLASSES].join("|") + ")$"; +const query = + `[out:json][timeout:${Math.floor(REQUEST_TIMEOUT_MS / 1000)}];` + + `way["highway"~"${CLASSES_RE}"](${bbox.minLat},${bbox.minLng},${bbox.maxLat},${bbox.maxLng});` + + `out geom;`; + +console.log(`fetching ways from ${OVERPASS_ENDPOINT} (this can take minutes)…`); +let payload = null; +try { + const res = await fetch(OVERPASS_ENDPOINT, { + method: "POST", + headers: { "User-Agent": USER_AGENT, "Content-Type": "application/x-www-form-urlencoded" }, + body: `data=${encodeURIComponent(query)}`, + signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), + }); + if (!res.ok) fail(`Overpass returned HTTP ${res.status}`); + payload = await res.json(); +} catch (err) { + fail(`Overpass fetch failed: ${err.message}`); +} +const ways = Array.isArray(payload?.elements) ? payload.elements.filter((el) => el.type === "way") : []; +if (ways.length === 0) fail("Overpass returned no drivable ways — bbox or endpoint problem"); + +/* ————— ways → deduplicated nodes → directed edges ————— */ + +const round6 = (v) => Math.round(v * 1e6) / 1e6; +/** "lat,lng" (rounded) → node id. Identical coordinates ARE the same OSM node. */ +const nodeIdByCoord = new Map(); +const latOut = []; +const lngOut = []; +function internNode(lat, lng) { + const key = `${round6(lat)},${round6(lng)}`; + const hit = nodeIdByCoord.get(key); + if (hit !== undefined) return hit; + const id = latOut.length; + nodeIdByCoord.set(key, id); + latOut.push(round6(lat)); + lngOut.push(round6(lng)); + return id; +} + +/** name → pool id (dense, insertion-ordered). */ +const nameIds = new Map(); +function internName(name) { + const hit = nameIds.get(name); + if (hit !== undefined) return hit; + const id = nameIds.size; + nameIds.set(name, id); + return id; +} + +const rawEdges = []; +let onewayCount = 0; +let unnamedCount = 0; +let maxSeenKmh = 0; + +function effectiveKmh(highway, maxspeedTag) { + // First integer in the tag wins ("50", "50 mph", "50 knots" all lead with + // the number). "none"/variable/messy tags fall back to the class default. + const parsed = maxspeedTag === undefined ? NaN : parseInt(String(maxspeedTag), 10); + const classDefault = CLASS_SPEED_KMH[highway] ?? LINK_CLASS_SPEED_KMH[highway] ?? 30; + const kmh = Number.isFinite(parsed) && parsed > 0 && parsed <= 160 ? parsed : classDefault; + if (kmh > maxSeenKmh) maxSeenKmh = kmh; + return kmh; +} + +for (const way of ways) { + const highway = way.tags?.highway; + if (typeof highway !== "string" || !DRIVABLE_CLASSES.has(highway)) continue; + const geometry = way.geometry; + if (!Array.isArray(geometry) || geometry.length < 2) continue; + const kmh = effectiveKmh(highway, way.tags?.maxspeed); + const nameId = typeof way.tags?.name === "string" && way.tags.name.length > 0 ? internName(way.tags.name) : undefined; + if (nameId === undefined) unnamedCount += 1; + // Oneway law: yes/1/true → forward only (reverse gets flags bit0); + // -1 → reverse only (forward gets flags bit0); anything else bidirectional. + const oneway = String(way.tags?.oneway ?? "").toLowerCase(); + const forwardOnly = oneway === "yes" || oneway === "1" || oneway === "true"; + const reverseOnly = oneway === "-1"; + if (forwardOnly || reverseOnly) onewayCount += 1; + + let from = internNode(geometry[0].lat, geometry[0].lon); + for (let i = 1; i < geometry.length; i++) { + const to = internNode(geometry[i].lat, geometry[i].lon); + if (to !== from) { + const meters = haversineM( + { lat: latOut[from], lng: lngOut[from] }, + { lat: latOut[to], lng: lngOut[to] }, + ); + const seconds = meters / (kmh / 3.6); + // forward direction: blocked when the way is reverse-only + rawEdges.push({ from, to, seconds, flags: reverseOnly ? 1 : 0, nameId }); + // reverse direction: blocked when the way is forward-only (bit0 set — + // stays visible to snapping, never traversed by A*) + rawEdges.push({ from: to, to: from, seconds, flags: forwardOnly ? 1 : 0, nameId }); + } + from = to; + } +} + +/* Degree-0 cleanup: a node that lost every segment (collapsed self-loops) is + * dead weight — compact the id space so A* and the snap grid never scan it. */ +const degree = new Uint32Array(latOut.length); +for (const edge of rawEdges) { + degree[edge.from] += 1; + degree[edge.to] += 1; +} +const remap = new Int32Array(latOut.length).fill(-1); +const cLat = []; +const cLng = []; +for (let i = 0; i < latOut.length; i++) { + if (degree[i] > 0) { + remap[i] = cLat.length; + cLat.push(latOut[i]); + cLng.push(lngOut[i]); + } +} +const edges = rawEdges + .map((e) => ({ from: remap[e.from], to: remap[e.to], seconds: e.seconds, flags: e.flags, nameId: e.nameId })) + .filter((e) => e.from >= 0 && e.to >= 0); +const droppedNodes = latOut.length - cLat.length; +if (cLat.length === 0 || edges.length === 0) fail("the extract produced an empty graph — refusing to write"); + +/* ————— encode (the layout's single authority) + stats gate ————— */ + +const names = [...nameIds.keys()]; +const buffer = encodeGraph({ lat: cLat, lng: cLng, edges, names, maxSpeedKmh: maxSeenKmh }); +const bytes = buffer.byteLength; + +const mb = (b) => `${(b / (1024 * 1024)).toFixed(1)} MB`; +console.log( + `graph stats:\n` + + ` nodeCount: ${cLat.length}${droppedNodes > 0 ? ` (${droppedNodes} degree-0 dropped)` : ""}\n` + + ` edgeCount: ${edges.length} directed\n` + + ` nameCount: ${names.length} (${unnamedCount} unnamed ways → UNNAMED sentinel)\n` + + ` oneway ways: ${onewayCount}\n` + + ` maxSpeedKmh: ${maxSeenKmh}\n` + + ` bbox: [${bbox.minLat.toFixed(4)}, ${bbox.minLng.toFixed(4)}, ${bbox.maxLat.toFixed(4)}, ${bbox.maxLng.toFixed(4)}]\n` + + ` binary size: ${mb(bytes)} (${bytes} B)`, +); + +// THE ENFORCED GATE (D-01): over the ceiling → exit 1 WITHOUT writing. +if (bytes > MAX_BINARY_BYTES) { + fail(`binary is ${mb(bytes)} — over the ${mb(MAX_BINARY_BYTES)} D-01 envelope. ` + + `Tighten DRIVABLE_CLASSES (drop tertiary/residential first) or shrink the bbox, then rebuild.`); +} +// THE ADVISORY BAND (A2's 18–50 MB forecast vs D-01's 30–80 MB expectation): +// a small graph is a review prompt, not a failure — the enforced gate stays. +if (bytes < EXPECTED_MIN_BYTES) { + console.log( + `WARNING: ${mb(bytes)} is below the expected 30–80 MB band — verify the drivable-class filter ` + + `(A2 forecast: 18–50 MB for KL metro). Advisory only; the enforced gate is 1–80 MB.`, + ); +} + +/* ————— collect-then-write (the committed/previous binary survives failures) ————— */ + +mkdirSync(dirname(OUT_PATH), { recursive: true }); +writeFileSync(OUT_PATH, Buffer.from(buffer)); +console.log(`wrote ${OUT_PATH} — ${mb(bytes)}, ${cLat.length} nodes, ${edges.length} edges`); +console.log("next: node routing/scripts/upload-road-graph.mjs # push the binary to your R2 bucket"); +process.exit(0);