#!/usr/bin/env node /* * budget-audit.test.mjs — the TDD suite for the consolidated budget audit * (07-04, D-06). Named vectors in the repo's gate-suite style. * --------------------------------------------------------------------------- * A MAINTAINER-TIME REPO TOOL — NOT a runtime dependency; nothing in any src * tree imports it. GPL-3.0: forks rerun it before their first driver. * * What the vectors pin: * * CPU* the route-CPU gate — the ladder is RE-RUN as a child process * (never re-implemented), the p95 is PRINTED against the 8 ms * product budget and the 10 ms free ceiling, the child's own * GATE FAIL line is carried verbatim, a missing graph is * notRun (never a pass), and the drift guard fails uncharted * regression beyond the documented accepted c3 band * BA* the bundle gate — BA9's gzip measurement + the whole-app * bundle ceiling + the missing-dist notRun branch * KV* the KV write-class gate — the cron zero-write law is * source-proven (a .put( inside runGuidanceCronTick FAILS the * gate), every write references a named TTL constant, and the * TTL table is READ from the constants (never hard-coded) * SQ* the subrequest gate — the per-request-class inventory, * source-derived honesty sentence, uninventoried route classes * FAIL, and never a fabricated zero * CH* the cache-header gate — reuses cloudflare-probes' live probe * (never forks it) and degrades to notRun without an origin * GC* the golden-corpus gate — RE-RUNS `routing:corpus` + * `search:corpus` as child processes; a failing corpus FAILS * the gate with that child's output * * Hand-rolled runner (the bot/web harness precedent — zero dependencies): * every test is async, failures are collected, and the process exits 1 when * any vector fails. */ import { strict as assert } from "node:assert"; import { spawnSync } from "node:child_process"; import { mkdtempSync, mkdirSync, rmSync, writeFileSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; const HERE = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(HERE, "..", ".."); /* ————— minimal harness (the hand-rolled runner convention) ————— */ const failures = []; let ran = 0; async function test(name, fn) { ran += 1; try { await fn(); console.log(`ok — ${name}`); } catch (err) { failures.push({ name, err }); console.log(`FAIL — ${name}\n ${err?.message ?? err}`); } } /* ————— fixture helper: a synthetic tree the gates scan via opts ————— */ function makeFixtureRoot(name, files) { const dir = mkdtempSync(path.join(tmpdir(), "budget-audit-")); for (const [rel, body] of Object.entries(files)) { const full = path.join(dir, rel); mkdirSync(path.dirname(full), { recursive: true }); writeFileSync(full, body); } return { name, dir, cleanup: () => rmSync(dir, { recursive: true, force: true }) }; } /* ————— CPU — the route-CPU gate ————— */ const gatesUrl = path.join(ROOT, "scripts", "lib", "budget-gates.mjs"); await test("CPU1 budget-gates exports the six gates; each carries its contract id", async () => { assert.ok(existsSync(gatesUrl), "scripts/lib/budget-gates.mjs must exist"); const m = await import(gatesUrl); const expected = { runCpuLadderGate: "route-cpu", runBundleBudgetGate: "bundle-budget", runKvWriteClassGate: "kv-writes", runSubrequestClassGate: "subrequests", runCacheHeaderGate: "cache-headers", runGoldenCorpusGate: "golden-corpus", }; for (const [fn, id] of Object.entries(expected)) { assert.equal(typeof m[fn], "function", `${fn} must be exported`); const r = await m[fn](); assert.equal(r.id, id, `${fn}() must carry id "${id}"`); assert.equal(typeof r.pass, "boolean", `${fn}() must return a boolean pass`); assert.equal(typeof r.evidence, "string", `${fn}() must return evidence`); } }); await test("CPU2 the ladder is RE-RUN (the gate references measure-route-cpu) and P95_BUDGET_MS = 8 is the same constant the script uses", async () => { const src = (await import("node:fs")).readFileSync(gatesUrl, "utf8"); assert.ok(src.includes("measure-route-cpu"), "the gate must spawn the existing ladder script"); assert.ok(src.includes("P95_BUDGET_MS = 8"), "the gate must carry the 8 ms product budget constant"); }); await test("CPU3 when the local graph binary is present the ladder runs, the measured p95 is printed against the 8 ms budget and the 10 ms free ceiling, and the child's own GATE line is carried", async () => { const m = await import(gatesUrl); // The binary is a GITIGNORED FORK-REBUILDABLE ARTIFACT, not a committed // file (routing/data/.gitignore ignores *.bin; docs/ROAD-GRAPH.md — forks // build + upload their own). On a fresh fork its absence is the legal // notRun class — CPU4's law — so the real-graph leg SKIPs honestly and is // NEVER a failure; the ladder is demanded only when the binary is present. const graphPresent = existsSync(path.join(ROOT, "routing", "data", "kl-roadgraph.v1.bin")); if (!graphPresent) { console.log(" SKIP — CPU3 needs the gitignored local binary (build it first: node routing/scripts/build-road-graph.mjs); not a failure"); return; } const r = await m.runCpuLadderGate(); assert.equal(r.notRun, undefined, "the binary is present — the ladder must run, not skip"); assert.equal(r.pass, true, `the measured state must sit inside the documented accepted c3 band: ${r.evidence}`); assert.match(r.evidence, /p95 \d+(\.\d+)? ms/, "the measured p95 must be printed"); assert.match(r.evidence, /8 ms/, "the 8 ms product budget must be printed beside the measurement"); assert.match(r.evidence, /10 ms free-plan ceiling/, "the free-plan ceiling must be printed"); assert.match(r.evidence, /GATE FAIL/, "the child's own GATE FAIL line (cross-metro class) is part of the evidence"); assert.match(r.evidence, /c3/, "the accepted c3 escalation class must be named"); // 08-05 pin: the record carries the POST-ALT truth — the ALT lever was // executed and measured over budget (never "a next-milestone item" again). assert.match(r.evidence, /EXECUTED in Phase 8/, "the executed-and-measured ALT verdict must be named"); assert.doesNotMatch(r.evidence, /next-milestone/, "the stale next-milestone wording must never return"); }); await test("CPU4 a missing graph binary is notRun — NEVER a pass (T-07-22)", async () => { const m = await import(gatesUrl); const r = await m.runCpuLadderGate({ graphPath: path.join(tmpdir(), "no-such-graph.bin") }); assert.equal(r.pass, false, "a skip is never a pass"); assert.equal(r.notRun, true, "a missing graph is the notRun class"); assert.match(r.evidence, /absent/, "the evidence must name the absent binary and the fix"); }); await test("CPU5 an uncharted regression (p95 beyond the documented accepted band) FAILS the gate", async () => { const m = await import(gatesUrl); const r = await m.runCpuLadderGate({ ladderOutput: "p95: 999.000 ms (max 999.000 ms, mean 900.000 ms, n=22)\n" }); assert.equal(r.pass, false, "a p95 far beyond the documented c3 band is uncharted drift"); assert.equal(r.notRun, undefined, "this is a measured FAIL, not a skip"); assert.match(r.evidence, /999\.000 ms/, "the measured p95 must appear in the evidence"); }); /* ————— BA — the bundle gate ————— */ await test("BA1 the bundle gate PASSES on the current tree; both measurements are printed", async () => { const m = await import(gatesUrl); const r = await m.runBundleBudgetGate(); assert.equal(r.pass, true, `the shipped bundle must sit inside its recorded budgets: ${r.evidence}`); assert.match(r.evidence, /route bundle budget: \d+ \/ 204800 gz bytes/, "BA9's route-payload measurement must be printed"); assert.match(r.evidence, /app bundle/, "the whole-app bundle measurement must be printed"); }); await test("BA2 a missing web/dist is notRun — NEVER a pass (exit-2 class)", async () => { const m = await import(gatesUrl); const r = await m.runBundleBudgetGate({ distDir: path.join(tmpdir(), "no-such-dist") }); assert.equal(r.pass, false); assert.equal(r.notRun, true); assert.match(r.evidence, /web\/dist/, "the evidence must name the missing build"); }); await test("BA3 a fixture dist over the app-bundle ceiling FAILS the gate with the overage", async () => { const { randomBytes } = await import("node:crypto"); const fix = makeFixtureRoot("ba3-dist", { "assets/huge.js": randomBytes(600000).toString("base64") }); try { const m = await import(gatesUrl); const r = await m.runBundleBudgetGate({ distDir: fix.dir }); assert.equal(r.pass, false, "600 KB gz of a single asset is over the 500000-byte app ceiling"); assert.match(r.evidence, /over/, "the evidence must carry the overage"); } finally { fix.cleanup(); } }); /* ————— KV — the KV write-class gate ————— */ const CRON_WITH_PUT = `async function runGuidanceCronTick(controller: ScheduledController, env: Env, opts: { baseFetch?: typeof fetch }): Promise { await env.chat_sessions.put("guidance:x", "{}", { expirationTtl: 60 }); } `; const KV_SESSIONS_OK = `export const SESSION_TTL_SECONDS = 900; export async function mintSession(kv, claim) { await kv.put("session:t", "{}", { expirationTtl: SESSION_TTL_SECONDS }); } `; await test("KV1 the KV gate PASSES on the current tree: write classes inventoried, cron write count 0, TTLs read from the constants", async () => { const m = await import(gatesUrl); const r = await m.runKvWriteClassGate(); assert.equal(r.pass, true, `the tree's write classes must all be TTL-named: ${r.evidence}`); assert.match(r.evidence, /cron write count: 0/, "the cron's zero-write law must print the count explicitly"); assert.match(r.evidence, /SESSION_TTL_SECONDS = 900/, "the TTL table must be read from the source constants"); assert.match(r.evidence, /PIN_TTL_SECONDS = 300/, "the pin TTL must be read from the source"); assert.match(r.evidence, /GUIDANCE_SESSION_TTL_SECONDS = 7200/, "the guidance TTL must be read from the source"); assert.match(r.evidence, /session:/, "the session write class must be inventoried"); assert.match(r.evidence, /loc:/, "the location-pin write class must be inventoried"); assert.match(r.evidence, /guidance:/, "the guidance write class must be inventoried"); }); await test("KV2 a cron tick that WRITES fails the gate — the zero-write law is source-proven (T-07-18)", async () => { const fix = makeFixtureRoot("kv2-bot", { "src/webhook.ts": CRON_WITH_PUT, "src/kv/sessions.ts": KV_SESSIONS_OK, "src/kv/location-pin.ts": "export const PIN_TTL_SECONDS = 300;\n", "src/kv/guidance-sessions.ts": "export const GUIDANCE_SESSION_TTL_SECONDS = 7200;\n", }); try { const m = await import(gatesUrl); const r = await m.runKvWriteClassGate({ botRoot: fix.dir }); assert.equal(r.pass, false, "a .put( inside the cron body burns the 1,000/day write budget"); assert.match(r.evidence, /\.put\(/, "the evidence must name the violating call"); } finally { fix.cleanup(); } }); /* ————— SQ — the subrequest gate ————— */ await test("SQ1 the subrequest gate PASSES on the current tree with the per-class inventory and the honesty sentence", async () => { const m = await import(gatesUrl); const r = await m.runSubrequestClassGate(); assert.equal(r.pass, true, `the inventory must cover every route class: ${r.evidence}`); assert.match(r.evidence, /source-derived inventory, not a live Workers Logs metric/, "the honesty sentence is mandatory, verbatim"); assert.match(r.evidence, /\/api\/route.*0 external subrequests/, "the route endpoint's 0 must be printed WITH its source"); assert.match(r.evidence, /not measured outside --live/, "the live metric absence must read not measured outside --live"); }); await test("SQ2 an uninventoried route class FAILS the gate — the inventory must follow the route table", async () => { const fix = makeFixtureRoot("sq2-bot", { "src/webhook.ts": `if (url.pathname === "/api/new-thing") { return handleNewThing(req, env); }\nif (url.pathname === "/api/route") { return handleRoute(req, env); }\nasync function runGuidanceCronTick(controller, env, opts): Promise {}\n`, }); try { const m = await import(gatesUrl); const r = await m.runSubrequestClassGate({ webhookPath: path.join(fix.dir, "src", "webhook.ts") }); assert.equal(r.pass, false, "a route class with no inventory row is a silent budget hole"); assert.match(r.evidence, /\/api\/new-thing/, "the evidence must name the uninventoried class"); } finally { fix.cleanup(); } }); /* ————— CH — the cache-header gate ————— */ await test("CH1 the cache-header gate REUSES cloudflare-probes' runAssetHeaderProbe (imported, never forked)", async () => { const src = (await import("node:fs")).readFileSync(gatesUrl, "utf8"); assert.match(src, /from ["']\.\/cloudflare-probes\.mjs["']/, "the live probe must be imported from the shared lib"); assert.match(src, /runAssetHeaderProbe/, "the probe must be the shared one"); }); await test("CH2 without a deployed origin the cache-header gate is notRun — never a pass", async () => { const m = await import(gatesUrl); const r = await m.runCacheHeaderGate({ baseUrl: null }); assert.equal(r.pass, false); assert.equal(r.notRun, true); assert.match(r.evidence, /not run/i, "the evidence must say the probe did not run"); }); /* ————— GC — the golden-corpus gate ————— */ await test("GC1 the golden-corpus gate RE-RUNS both corpora as child processes and PASSES on the current tree", async () => { const m = await import(gatesUrl); const r = await m.runGoldenCorpusGate(); assert.equal(r.pass, true, `both corpora must be green: ${r.evidence}`); assert.match(r.evidence, /routing:corpus/, "the route corpus must be named by id"); assert.match(r.evidence, /search:corpus/, "the search corpus must be named by id"); }); await test("GC2 a failing corpus FAILS the gate carrying that child's own output verbatim", async () => { const m = await import(gatesUrl); const r = await m.runGoldenCorpusGate({ commands: [["routing:corpus", ["node", "-e", "console.error('CORPUS GATE FAIL: synthetic'); process.exit(1)"]]] }); assert.equal(r.pass, false); assert.match(r.evidence, /CORPUS GATE FAIL: synthetic/, "the child's own failure output must be carried verbatim"); }); /* ————— AU — the CLI ————— */ const cliPath = path.join(ROOT, "scripts", "budget-audit.mjs"); function runCli(args, env = {}) { return spawnSync(process.execPath, [cliPath, ...args], { cwd: ROOT, encoding: "utf8", timeout: 300000, env: { ...process.env, ...env }, maxBuffer: 1 << 26, }); } await test("AU1 without --live: every required gate prints its evidence line, the free-tier comparison table is printed, and the audit is CLEAN with exit 0", () => { assert.ok(existsSync(cliPath), "scripts/budget-audit.mjs must exist"); const r = runCli([]); assert.equal(r.status, 0, `exit 0 expected on the current tree (stdout: ${r.stdout?.slice(-400)})`); const out = r.stdout ?? ""; for (const id of ["route-cpu", "bundle-budget", "kv-writes", "subrequests", "golden-corpus"]) { assert.match(out, new RegExp(`${id}: (PASS|FAIL|NOT RUN) — `), `${id} must print its one-line evidence`); } assert.match(out, /BUDGET AUDIT CLEAN/, "the CLEAN summary must print"); assert.match(out, /50\/request/, "the subrequest ceiling must appear in the comparison table"); assert.match(out, /10 ms\/invocation/, "the CPU ceiling must appear in the comparison table"); assert.match(out, /not measured/, "unmeasured ceilings must read `not measured` — never 0"); assert.match(out, /cache-headers: NOT RUN.*\(pass --live\)/, "the flag-skipped live gate is visible and honest"); }); await test("AU2 with --live: the live header probe runs and the exit contract holds — a NOT RUN cache-header gate NEVER coexists with exit 0", () => { const r = runCli(["--live"]); const out = r.stdout ?? ""; assert.ok([0, 1, 2].includes(r.status), `exit must be 0/1/2 (got ${r.status})`); assert.match(out, /cache-headers: /, "the cache-header gate must print with --live"); if (out.includes("cache-headers: NOT RUN")) { assert.equal(r.status, 2, "a NOT RUN gate must map to exit 2 (environment drift) — never a silent pass"); assert.match(out, /BUDGET AUDIT NOT RUN/, "the NOT RUN summary must print"); } }); await test("AU3 environment drift (a root with no builds/graph) maps to exit 2 — BUDGET AUDIT NOT RUN, never a pass (T-07-22)", () => { const empty = mkdtempSync(path.join(tmpdir(), "budget-empty-")); try { const r = runCli([], { PRIVACY_AUDIT_ROOT: empty }); assert.equal(r.status, 2, `a root without the measured surfaces must be exit 2 (stdout: ${r.stdout?.slice(-300)})`); assert.match(r.stdout ?? "", /BUDGET AUDIT NOT RUN/, "the NOT RUN summary must print"); } finally { rmSync(empty, { recursive: true, force: true }); } }); /* ————— summary ————— */ if (failures.length > 0) { console.error(`\n${failures.length}/${ran} vector(s) FAILED:`); for (const f of failures) console.error(` - ${f.name}`); process.exit(1); } console.log(`\nBUDGET-AUDIT SUITE: ${ran}/${ran} vectors ok`); process.exit(0);