feat(06-04): road-graph build tool — Overpass extract → pinned CSR binary with the D-01 stats gate
- maintainer-time repo tool (nothing in bot/web/routing src imports it): drivable-class Overpass extract (Pitfall 1 filter), nodes deduped by rounded coord, oneway/-1/roundabout flag law, maxspeed-tag overrides over class defaults, name-pool interning - bbox derived mechanically from the committed corpus (gazetteer entries + golden pairs ± 0.05 deg, printed) — Open Q 4's pinning - collect-then-write + ENFORCED size gate (>1 MB, <=80 MB, exit 1 outside) + advisory 30-80 MB band warning; --check mode decodes and re-prints the gate, zero network - routing/data/.gitignore: *.bin is an ODbL-derived build artifact — forks build their own - real KL graph built: 277514 nodes / 598396 edges / 21447 names / 13.1 MB (gate PASS, band warning printed for review)
This commit is contained in:
araharah-executor
@@ -0,0 +1,3 @@
|
|||||||
|
# D-01: the road-graph binary is an ODbL-derived BUILD ARTIFACT, never
|
||||||
|
# committed — forks build + upload their own (see docs/ROAD-GRAPH.md).
|
||||||
|
*.bin
|
||||||
@@ -0,0 +1,339 @@
|
|||||||
|
#!/usr/bin/env node
|
||||||
|
/*
|
||||||
|
* build-road-graph.mjs — the D-01 engine-A graph build tool (Overpass → CSR).
|
||||||
|
* ---------------------------------------------------------------------------
|
||||||
|
* A MAINTAINER-TIME REPO TOOL — NOT a runtime dependency; nothing in bot/src,
|
||||||
|
* web/src, or routing/src imports it. No deploy, test, or CI run ever executes
|
||||||
|
* it. A maintainer (or a fork) runs it on their own machine to produce the
|
||||||
|
* KL road-graph binary, then uploads it to their OWN R2 bucket (D-01:
|
||||||
|
* fork-friendly, $0 — see docs/ROAD-GRAPH.md).
|
||||||
|
*
|
||||||
|
* Mechanically:
|
||||||
|
* 1. The KL bbox is COMPUTED at run start from the committed corpus —
|
||||||
|
* @araharah/search getEntries() extremes + the golden-pairs origin
|
||||||
|
* coordinates ± BBOX_MARGIN_DEG — and printed (Open Q 4's mechanical
|
||||||
|
* pinning: the bbox tunes itself when the corpus grows; never hand-edited).
|
||||||
|
* 2. ONE Overpass query fetches every way whose highway tag is in
|
||||||
|
* DRIVABLE_CLASSES (footway/cycleway/service/path NEVER enter — 06-RESEARCH
|
||||||
|
* Pitfall 1: an unfiltered highway=* graph is the classic CPU-budget kill).
|
||||||
|
* 3. Ways → nodes (deduplicated by rounded coordinate) → directed edges with
|
||||||
|
* oneway flags, speed-derived seconds, and street-name pool ids.
|
||||||
|
* 4. encodeGraph (../src/graph.ts — the layout's SINGLE authority) produces
|
||||||
|
* the pinned CSR binary. COLLECT-THEN-WRITE: routing/data/
|
||||||
|
* kl-roadgraph.v1.bin is written ONLY after the encode and the stats gate
|
||||||
|
* both pass — any failure exits 1 with the previous binary untouched.
|
||||||
|
* 5. THE STATS GATE prints nodeCount/edgeCount/nameCount/bbox/binary size and
|
||||||
|
* enforces the D-01 envelope: size ≤ MAX_BINARY_BYTES or exit 1. A result
|
||||||
|
* below the expected 30–80 MB band prints an ADVISORY warning (A2's
|
||||||
|
* forecast) — the ENFORCED floor is the 1 MB honesty check in --verify,
|
||||||
|
* never the forecast band.
|
||||||
|
*
|
||||||
|
* The binary is a BUILD ARTIFACT, not committed: routing/data/.gitignore
|
||||||
|
* ignores *.bin — forks build + upload their own (ODbL-derived data stays out
|
||||||
|
* of the repo; the attribution travels in docs/ROAD-GRAPH.md and the upload
|
||||||
|
* path). The committed artifacts are this script, the docs, and the tiny
|
||||||
|
* test mini-graph.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* node routing/scripts/build-road-graph.mjs
|
||||||
|
* # full build: fetch → encode → gate → write
|
||||||
|
* # (network fetch; minutes on the KL bbox)
|
||||||
|
* node routing/scripts/build-road-graph.mjs --check <file>
|
||||||
|
* # decode an existing binary with decodeGraph, print stats,
|
||||||
|
* # exit 0/1 — NO network
|
||||||
|
*
|
||||||
|
* Exit codes: 0 built and gated (or checked green); 1 pre-flight failure,
|
||||||
|
* fetch failure, envelope-gate failure, or --check decode failure.
|
||||||
|
*/
|
||||||
|
import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
|
||||||
|
import { dirname, resolve } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
|
||||||
|
const here = dirname(fileURLToPath(import.meta.url)); // routing/scripts
|
||||||
|
const CORPUS_PATH = resolve(here, "..", "test", "fixtures", "golden-pairs.json");
|
||||||
|
const OUT_PATH = resolve(here, "..", "data", "kl-roadgraph.v1.bin");
|
||||||
|
|
||||||
|
/* ————— named constants (no inline magic) ————— */
|
||||||
|
|
||||||
|
const OVERPASS_ENDPOINT = "https://overpass-api.de/api/interpreter";
|
||||||
|
/** Overpass server-side budget; matches the client fetch timeout. */
|
||||||
|
const REQUEST_TIMEOUT_MS = 300_000;
|
||||||
|
const USER_AGENT = "araharah-roadgraph/1.0 (build-time KL road graph; repo: araharah)";
|
||||||
|
/** D-01 envelope ceiling — a larger binary must NOT be written (shrink the
|
||||||
|
* class filter or the bbox instead). */
|
||||||
|
const MAX_BINARY_BYTES = 80 * 1024 * 1024;
|
||||||
|
/** A2's honest-build forecast floor — crossing it downward is an ADVISORY
|
||||||
|
* (verify the class filter), never a gate. */
|
||||||
|
const EXPECTED_MIN_BYTES = 30 * 1024 * 1024;
|
||||||
|
/** How far the computed bbox extends beyond the committed corpus extremes. */
|
||||||
|
const BBOX_MARGIN_DEG = 0.05;
|
||||||
|
/** Node dedup granularity: 1e-6 deg ≈ 0.11 m — far under GPS noise. */
|
||||||
|
const COORD_ROUND_DECIMALS = 6;
|
||||||
|
/** drivable classes ONLY (06-RESEARCH Pitfall 1): footway/cycleway/service/
|
||||||
|
* path are deliberately absent — a walking path must never enter a road A*. */
|
||||||
|
const DRIVABLE_CLASSES = new Set([
|
||||||
|
"motorway",
|
||||||
|
"trunk",
|
||||||
|
"primary",
|
||||||
|
"secondary",
|
||||||
|
"tertiary",
|
||||||
|
"residential",
|
||||||
|
"unclassified",
|
||||||
|
"living_street",
|
||||||
|
]);
|
||||||
|
/** Class speed defaults (km/h) when a way carries no parsable maxspeed. */
|
||||||
|
const CLASS_SPEED_KMH = {
|
||||||
|
motorway: 100,
|
||||||
|
trunk: 80,
|
||||||
|
primary: 60,
|
||||||
|
secondary: 50,
|
||||||
|
tertiary: 40,
|
||||||
|
residential: 30,
|
||||||
|
unclassified: 30,
|
||||||
|
living_street: 20,
|
||||||
|
};
|
||||||
|
/** Motorway link ramps inherit the parent class speed via these links. */
|
||||||
|
const LINK_CLASS_SPEED_KMH = { motorway_link: 60, trunk_link: 60 };
|
||||||
|
|
||||||
|
function fail(message) {
|
||||||
|
console.error(`build-road-graph: ${message}`);
|
||||||
|
process.exit(1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ————— --check mode: decode + stats, no network ————— */
|
||||||
|
|
||||||
|
const checkIdx = process.argv.indexOf("--check");
|
||||||
|
if (checkIdx !== -1) {
|
||||||
|
const file = process.argv[checkIdx + 1];
|
||||||
|
if (file === undefined) fail("--check requires a file path");
|
||||||
|
if (!existsSync(file)) fail(`--check: file not found: ${file}`);
|
||||||
|
const { decodeGraph } = await import("../src/graph.ts");
|
||||||
|
const buffer = readFileSync(file);
|
||||||
|
const graph = decodeGraph(buffer.buffer.slice(buffer.byteOffset, buffer.byteOffset + buffer.byteLength));
|
||||||
|
if (graph === null) fail(`--check: ${file} is NOT a valid GRAPH_VERSION CSR binary (decodeGraph null)`);
|
||||||
|
const size = statSync(file).size;
|
||||||
|
console.log(
|
||||||
|
`--check ${file}\n` +
|
||||||
|
` nodeCount: ${graph.nodeCount}\n` +
|
||||||
|
` edgeCount: ${graph.edgeCount}\n` +
|
||||||
|
` nameCount: ${graph.nameCount}\n` +
|
||||||
|
` maxSpeedKmh: ${graph.maxSpeedKmh}\n` +
|
||||||
|
` bbox: [${graph.bbox.minLat}, ${graph.bbox.minLng}, ${graph.bbox.maxLat}, ${graph.bbox.maxLng}]\n` +
|
||||||
|
` bytes: ${size}`,
|
||||||
|
);
|
||||||
|
if (size > MAX_BINARY_BYTES) fail(`--check: ${size} bytes exceeds the ${MAX_BINARY_BYTES}-byte D-01 envelope`);
|
||||||
|
if (size <= 1024) fail(`--check: ${size} bytes is implausibly small — not a real road graph`);
|
||||||
|
process.exit(0);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ————— inputs ————— */
|
||||||
|
|
||||||
|
const { encodeGraph } = await import("../src/graph.ts");
|
||||||
|
const { haversineM } = await import("../src/geo.ts");
|
||||||
|
|
||||||
|
if (!existsSync(CORPUS_PATH)) fail(`corpus not found: ${CORPUS_PATH}`);
|
||||||
|
const corpus = JSON.parse(readFileSync(CORPUS_PATH, "utf8"));
|
||||||
|
const { getEntries } = await import("@araharah/search/src/gazetteer.ts");
|
||||||
|
|
||||||
|
/* The committed corpus defines the area engine A must route (Open Q 4):
|
||||||
|
* gazetteer extremes cover the search surface; golden-pair origins/destinations
|
||||||
|
* are gazetteer landmarks themselves — both are unioned so a future corpus
|
||||||
|
* extension widens the graph mechanically. */
|
||||||
|
function computeBbox() {
|
||||||
|
let minLat = Infinity;
|
||||||
|
let minLng = Infinity;
|
||||||
|
let maxLat = -Infinity;
|
||||||
|
let maxLng = -Infinity;
|
||||||
|
const absorb = (lat, lng) => {
|
||||||
|
if (!Number.isFinite(lat) || !Number.isFinite(lng)) fail(`non-finite coordinate in corpus: (${lat}, ${lng})`);
|
||||||
|
if (lat < minLat) minLat = lat;
|
||||||
|
if (lat > maxLat) maxLat = lat;
|
||||||
|
if (lng < minLng) minLng = lng;
|
||||||
|
if (lng > maxLng) maxLng = lng;
|
||||||
|
};
|
||||||
|
for (const entry of getEntries()) absorb(entry.lat, entry.lng);
|
||||||
|
for (const pair of corpus.pairs) absorb(pair.origin.lat, pair.origin.lng);
|
||||||
|
const gazetteerById = new Map(getEntries().map((e) => [e.id, e]));
|
||||||
|
for (const pair of corpus.pairs) {
|
||||||
|
const dest = gazetteerById.get(pair.destinationId);
|
||||||
|
if (dest === undefined) fail(`pair ${pair.pairId}: destinationId "${pair.destinationId}" is not a gazetteer entry`);
|
||||||
|
absorb(dest.lat, dest.lng);
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
minLat: minLat - BBOX_MARGIN_DEG,
|
||||||
|
minLng: minLng - BBOX_MARGIN_DEG,
|
||||||
|
maxLat: maxLat + BBOX_MARGIN_DEG,
|
||||||
|
maxLng: maxLng + BBOX_MARGIN_DEG,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const bbox = computeBbox();
|
||||||
|
console.log(
|
||||||
|
`bbox (corpus extremes ± ${BBOX_MARGIN_DEG} deg): ` +
|
||||||
|
`[${bbox.minLat.toFixed(4)}, ${bbox.minLng.toFixed(4)}, ${bbox.maxLat.toFixed(4)}, ${bbox.maxLng.toFixed(4)}]`,
|
||||||
|
);
|
||||||
|
|
||||||
|
/* ————— the Overpass fetch (ONE request, drivable classes only) ————— */
|
||||||
|
|
||||||
|
const CLASSES_RE = "^(" + [...DRIVABLE_CLASSES].join("|") + ")$";
|
||||||
|
const query =
|
||||||
|
`[out:json][timeout:${Math.floor(REQUEST_TIMEOUT_MS / 1000)}];` +
|
||||||
|
`way["highway"~"${CLASSES_RE}"](${bbox.minLat},${bbox.minLng},${bbox.maxLat},${bbox.maxLng});` +
|
||||||
|
`out geom;`;
|
||||||
|
|
||||||
|
console.log(`fetching ways from ${OVERPASS_ENDPOINT} (this can take minutes)…`);
|
||||||
|
let payload = null;
|
||||||
|
try {
|
||||||
|
const res = await fetch(OVERPASS_ENDPOINT, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "User-Agent": USER_AGENT, "Content-Type": "application/x-www-form-urlencoded" },
|
||||||
|
body: `data=${encodeURIComponent(query)}`,
|
||||||
|
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
|
||||||
|
});
|
||||||
|
if (!res.ok) fail(`Overpass returned HTTP ${res.status}`);
|
||||||
|
payload = await res.json();
|
||||||
|
} catch (err) {
|
||||||
|
fail(`Overpass fetch failed: ${err.message}`);
|
||||||
|
}
|
||||||
|
const ways = Array.isArray(payload?.elements) ? payload.elements.filter((el) => el.type === "way") : [];
|
||||||
|
if (ways.length === 0) fail("Overpass returned no drivable ways — bbox or endpoint problem");
|
||||||
|
|
||||||
|
/* ————— ways → deduplicated nodes → directed edges ————— */
|
||||||
|
|
||||||
|
const round6 = (v) => Math.round(v * 1e6) / 1e6;
|
||||||
|
/** "lat,lng" (rounded) → node id. Identical coordinates ARE the same OSM node. */
|
||||||
|
const nodeIdByCoord = new Map();
|
||||||
|
const latOut = [];
|
||||||
|
const lngOut = [];
|
||||||
|
function internNode(lat, lng) {
|
||||||
|
const key = `${round6(lat)},${round6(lng)}`;
|
||||||
|
const hit = nodeIdByCoord.get(key);
|
||||||
|
if (hit !== undefined) return hit;
|
||||||
|
const id = latOut.length;
|
||||||
|
nodeIdByCoord.set(key, id);
|
||||||
|
latOut.push(round6(lat));
|
||||||
|
lngOut.push(round6(lng));
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** name → pool id (dense, insertion-ordered). */
|
||||||
|
const nameIds = new Map();
|
||||||
|
function internName(name) {
|
||||||
|
const hit = nameIds.get(name);
|
||||||
|
if (hit !== undefined) return hit;
|
||||||
|
const id = nameIds.size;
|
||||||
|
nameIds.set(name, id);
|
||||||
|
return id;
|
||||||
|
}
|
||||||
|
|
||||||
|
const rawEdges = [];
|
||||||
|
let onewayCount = 0;
|
||||||
|
let unnamedCount = 0;
|
||||||
|
let maxSeenKmh = 0;
|
||||||
|
|
||||||
|
function effectiveKmh(highway, maxspeedTag) {
|
||||||
|
// First integer in the tag wins ("50", "50 mph", "50 knots" all lead with
|
||||||
|
// the number). "none"/variable/messy tags fall back to the class default.
|
||||||
|
const parsed = maxspeedTag === undefined ? NaN : parseInt(String(maxspeedTag), 10);
|
||||||
|
const classDefault = CLASS_SPEED_KMH[highway] ?? LINK_CLASS_SPEED_KMH[highway] ?? 30;
|
||||||
|
const kmh = Number.isFinite(parsed) && parsed > 0 && parsed <= 160 ? parsed : classDefault;
|
||||||
|
if (kmh > maxSeenKmh) maxSeenKmh = kmh;
|
||||||
|
return kmh;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const way of ways) {
|
||||||
|
const highway = way.tags?.highway;
|
||||||
|
if (typeof highway !== "string" || !DRIVABLE_CLASSES.has(highway)) continue;
|
||||||
|
const geometry = way.geometry;
|
||||||
|
if (!Array.isArray(geometry) || geometry.length < 2) continue;
|
||||||
|
const kmh = effectiveKmh(highway, way.tags?.maxspeed);
|
||||||
|
const nameId = typeof way.tags?.name === "string" && way.tags.name.length > 0 ? internName(way.tags.name) : undefined;
|
||||||
|
if (nameId === undefined) unnamedCount += 1;
|
||||||
|
// Oneway law: yes/1/true → forward only (reverse gets flags bit0);
|
||||||
|
// -1 → reverse only (forward gets flags bit0); anything else bidirectional.
|
||||||
|
const oneway = String(way.tags?.oneway ?? "").toLowerCase();
|
||||||
|
const forwardOnly = oneway === "yes" || oneway === "1" || oneway === "true";
|
||||||
|
const reverseOnly = oneway === "-1";
|
||||||
|
if (forwardOnly || reverseOnly) onewayCount += 1;
|
||||||
|
|
||||||
|
let from = internNode(geometry[0].lat, geometry[0].lon);
|
||||||
|
for (let i = 1; i < geometry.length; i++) {
|
||||||
|
const to = internNode(geometry[i].lat, geometry[i].lon);
|
||||||
|
if (to !== from) {
|
||||||
|
const meters = haversineM(
|
||||||
|
{ lat: latOut[from], lng: lngOut[from] },
|
||||||
|
{ lat: latOut[to], lng: lngOut[to] },
|
||||||
|
);
|
||||||
|
const seconds = meters / (kmh / 3.6);
|
||||||
|
// forward direction: blocked when the way is reverse-only
|
||||||
|
rawEdges.push({ from, to, seconds, flags: reverseOnly ? 1 : 0, nameId });
|
||||||
|
// reverse direction: blocked when the way is forward-only (bit0 set —
|
||||||
|
// stays visible to snapping, never traversed by A*)
|
||||||
|
rawEdges.push({ from: to, to: from, seconds, flags: forwardOnly ? 1 : 0, nameId });
|
||||||
|
}
|
||||||
|
from = to;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Degree-0 cleanup: a node that lost every segment (collapsed self-loops) is
|
||||||
|
* dead weight — compact the id space so A* and the snap grid never scan it. */
|
||||||
|
const degree = new Uint32Array(latOut.length);
|
||||||
|
for (const edge of rawEdges) {
|
||||||
|
degree[edge.from] += 1;
|
||||||
|
degree[edge.to] += 1;
|
||||||
|
}
|
||||||
|
const remap = new Int32Array(latOut.length).fill(-1);
|
||||||
|
const cLat = [];
|
||||||
|
const cLng = [];
|
||||||
|
for (let i = 0; i < latOut.length; i++) {
|
||||||
|
if (degree[i] > 0) {
|
||||||
|
remap[i] = cLat.length;
|
||||||
|
cLat.push(latOut[i]);
|
||||||
|
cLng.push(lngOut[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const edges = rawEdges
|
||||||
|
.map((e) => ({ from: remap[e.from], to: remap[e.to], seconds: e.seconds, flags: e.flags, nameId: e.nameId }))
|
||||||
|
.filter((e) => e.from >= 0 && e.to >= 0);
|
||||||
|
const droppedNodes = latOut.length - cLat.length;
|
||||||
|
if (cLat.length === 0 || edges.length === 0) fail("the extract produced an empty graph — refusing to write");
|
||||||
|
|
||||||
|
/* ————— encode (the layout's single authority) + stats gate ————— */
|
||||||
|
|
||||||
|
const names = [...nameIds.keys()];
|
||||||
|
const buffer = encodeGraph({ lat: cLat, lng: cLng, edges, names, maxSpeedKmh: maxSeenKmh });
|
||||||
|
const bytes = buffer.byteLength;
|
||||||
|
|
||||||
|
const mb = (b) => `${(b / (1024 * 1024)).toFixed(1)} MB`;
|
||||||
|
console.log(
|
||||||
|
`graph stats:\n` +
|
||||||
|
` nodeCount: ${cLat.length}${droppedNodes > 0 ? ` (${droppedNodes} degree-0 dropped)` : ""}\n` +
|
||||||
|
` edgeCount: ${edges.length} directed\n` +
|
||||||
|
` nameCount: ${names.length} (${unnamedCount} unnamed ways → UNNAMED sentinel)\n` +
|
||||||
|
` oneway ways: ${onewayCount}\n` +
|
||||||
|
` maxSpeedKmh: ${maxSeenKmh}\n` +
|
||||||
|
` bbox: [${bbox.minLat.toFixed(4)}, ${bbox.minLng.toFixed(4)}, ${bbox.maxLat.toFixed(4)}, ${bbox.maxLng.toFixed(4)}]\n` +
|
||||||
|
` binary size: ${mb(bytes)} (${bytes} B)`,
|
||||||
|
);
|
||||||
|
|
||||||
|
// THE ENFORCED GATE (D-01): over the ceiling → exit 1 WITHOUT writing.
|
||||||
|
if (bytes > MAX_BINARY_BYTES) {
|
||||||
|
fail(`binary is ${mb(bytes)} — over the ${mb(MAX_BINARY_BYTES)} D-01 envelope. ` +
|
||||||
|
`Tighten DRIVABLE_CLASSES (drop tertiary/residential first) or shrink the bbox, then rebuild.`);
|
||||||
|
}
|
||||||
|
// THE ADVISORY BAND (A2's 18–50 MB forecast vs D-01's 30–80 MB expectation):
|
||||||
|
// a small graph is a review prompt, not a failure — the enforced gate stays.
|
||||||
|
if (bytes < EXPECTED_MIN_BYTES) {
|
||||||
|
console.log(
|
||||||
|
`WARNING: ${mb(bytes)} is below the expected 30–80 MB band — verify the drivable-class filter ` +
|
||||||
|
`(A2 forecast: 18–50 MB for KL metro). Advisory only; the enforced gate is 1–80 MB.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ————— collect-then-write (the committed/previous binary survives failures) ————— */
|
||||||
|
|
||||||
|
mkdirSync(dirname(OUT_PATH), { recursive: true });
|
||||||
|
writeFileSync(OUT_PATH, Buffer.from(buffer));
|
||||||
|
console.log(`wrote ${OUT_PATH} — ${mb(bytes)}, ${cLat.length} nodes, ${edges.length} edges`);
|
||||||
|
console.log("next: node routing/scripts/upload-road-graph.mjs # push the binary to your R2 bucket");
|
||||||
|
process.exit(0);
|
||||||
Reference in New Issue
Block a user