arah2/scripts/test/budget-audit.test.mjs

#!/usr/bin/env node
/*
 * budget-audit.test.mjs — the TDD suite for the consolidated budget audit
 * (07-04, D-06). Named vectors in the repo's gate-suite style.
 * ---------------------------------------------------------------------------
 * A MAINTAINER-TIME REPO TOOL — NOT a runtime dependency; nothing in any src
 * tree imports it. GPL-3.0: forks rerun it before their first driver.
 *
 * What the vectors pin:
 *
 *   CPU*  the route-CPU gate — the ladder is RE-RUN as a child process
 *         (never re-implemented), the p95 is PRINTED against the 8 ms
 *         product budget and the 10 ms free ceiling, the child's own
 *         GATE FAIL line is carried verbatim, a missing graph is
 *         notRun (never a pass), and the drift guard fails uncharted
 *         regression beyond the documented accepted c3 band
 *   BA*   the bundle gate — BA9's gzip measurement + the whole-app
 *         bundle ceiling + the missing-dist notRun branch
 *   KV*   the KV write-class gate — the cron zero-write law is
 *         source-proven (a .put( inside runGuidanceCronTick FAILS the
 *         gate), every write references a named TTL constant, and the
 *         TTL table is READ from the constants (never hard-coded)
 *   SQ*   the subrequest gate — the per-request-class inventory,
 *         source-derived honesty sentence, uninventoried route classes
 *         FAIL, and never a fabricated zero
 *   CH*   the cache-header gate — reuses cloudflare-probes' live probe
 *         (never forks it) and degrades to notRun without an origin
 *   GC*   the golden-corpus gate — RE-RUNS `routing:corpus` +
 *         `search:corpus` as child processes; a failing corpus FAILS
 *         the gate with that child's output
 *
 * Hand-rolled runner (the bot/web harness precedent — zero dependencies):
 * every test is async, failures are collected, and the process exits 1 when
 * any vector fails.
 */
import { strict as assert } from "node:assert";
import { spawnSync } from "node:child_process";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync, existsSync } from "node:fs";
import { tmpdir } from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";

const HERE = path.dirname(fileURLToPath(import.meta.url));
const ROOT = path.resolve(HERE, "..", "..");

/* ————— minimal harness (the hand-rolled runner convention) ————— */

const failures = [];
let ran = 0;

async function test(name, fn) {
  ran += 1;
  try {
    await fn();
    console.log(`ok — ${name}`);
  } catch (err) {
    failures.push({ name, err });
    console.log(`FAIL — ${name}\n    ${err?.message ?? err}`);
  }
}

/* ————— fixture helper: a synthetic tree the gates scan via opts ————— */

function makeFixtureRoot(name, files) {
  const dir = mkdtempSync(path.join(tmpdir(), "budget-audit-"));
  for (const [rel, body] of Object.entries(files)) {
    const full = path.join(dir, rel);
    mkdirSync(path.dirname(full), { recursive: true });
    writeFileSync(full, body);
  }
  return { name, dir, cleanup: () => rmSync(dir, { recursive: true, force: true }) };
}

/* ————— CPU — the route-CPU gate ————— */

const gatesUrl = path.join(ROOT, "scripts", "lib", "budget-gates.mjs");

await test("CPU1 budget-gates exports the six gates; each carries its contract id", async () => {
  assert.ok(existsSync(gatesUrl), "scripts/lib/budget-gates.mjs must exist");
  const m = await import(gatesUrl);
  const expected = {
    runCpuLadderGate: "route-cpu",
    runBundleBudgetGate: "bundle-budget",
    runKvWriteClassGate: "kv-writes",
    runSubrequestClassGate: "subrequests",
    runCacheHeaderGate: "cache-headers",
    runGoldenCorpusGate: "golden-corpus",
  };
  for (const [fn, id] of Object.entries(expected)) {
    assert.equal(typeof m[fn], "function", `${fn} must be exported`);
    const r = await m[fn]();
    assert.equal(r.id, id, `${fn}() must carry id "${id}"`);
    assert.equal(typeof r.pass, "boolean", `${fn}() must return a boolean pass`);
    assert.equal(typeof r.evidence, "string", `${fn}() must return evidence`);
  }
});

await test("CPU2 the ladder is RE-RUN (the gate references measure-route-cpu) and P95_BUDGET_MS = 8 is the same constant the script uses", async () => {
  const src = (await import("node:fs")).readFileSync(gatesUrl, "utf8");
  assert.ok(src.includes("measure-route-cpu"), "the gate must spawn the existing ladder script");
  assert.ok(src.includes("P95_BUDGET_MS = 8"), "the gate must carry the 8 ms product budget constant");
});

await test("CPU3 when the local graph binary is present the ladder runs, the measured p95 is printed against the 8 ms budget and the 10 ms free ceiling, and the child's own GATE line is carried", async () => {
  const m = await import(gatesUrl);
  // The binary is a GITIGNORED FORK-REBUILDABLE ARTIFACT, not a committed
  // file (routing/data/.gitignore ignores *.bin; docs/ROAD-GRAPH.md — forks
  // build + upload their own). On a fresh fork its absence is the legal
  // notRun class — CPU4's law — so the real-graph leg SKIPs honestly and is
  // NEVER a failure; the ladder is demanded only when the binary is present.
  const graphPresent = existsSync(path.join(ROOT, "routing", "data", "kl-roadgraph.v1.bin"));
  if (!graphPresent) {
    console.log("    SKIP — CPU3 needs the gitignored local binary (build it first: node routing/scripts/build-road-graph.mjs); not a failure");
    return;
  }
  const r = await m.runCpuLadderGate();
  assert.equal(r.notRun, undefined, "the binary is present — the ladder must run, not skip");
  assert.equal(r.pass, true, `the measured state must sit inside the documented accepted c3 band: ${r.evidence}`);
  assert.match(r.evidence, /p95 \d+(\.\d+)? ms/, "the measured p95 must be printed");
  assert.match(r.evidence, /8 ms/, "the 8 ms product budget must be printed beside the measurement");
  assert.match(r.evidence, /10 ms free-plan ceiling/, "the free-plan ceiling must be printed");
  assert.match(r.evidence, /GATE FAIL/, "the child's own GATE FAIL line (cross-metro class) is part of the evidence");
  assert.match(r.evidence, /c3/, "the accepted c3 escalation class must be named");
  // 08-05 pin: the record carries the POST-ALT truth — the ALT lever was
  // executed and measured over budget (never "a next-milestone item" again).
  assert.match(r.evidence, /EXECUTED in Phase 8/, "the executed-and-measured ALT verdict must be named");
  assert.doesNotMatch(r.evidence, /next-milestone/, "the stale next-milestone wording must never return");
});

await test("CPU4 a missing graph binary is notRun — NEVER a pass (T-07-22)", async () => {
  const m = await import(gatesUrl);
  const r = await m.runCpuLadderGate({ graphPath: path.join(tmpdir(), "no-such-graph.bin") });
  assert.equal(r.pass, false, "a skip is never a pass");
  assert.equal(r.notRun, true, "a missing graph is the notRun class");
  assert.match(r.evidence, /absent/, "the evidence must name the absent binary and the fix");
});

await test("CPU5 an uncharted regression (p95 beyond the documented accepted band) FAILS the gate", async () => {
  const m = await import(gatesUrl);
  const r = await m.runCpuLadderGate({ ladderOutput: "p95: 999.000 ms (max 999.000 ms, mean 900.000 ms, n=22)\n" });
  assert.equal(r.pass, false, "a p95 far beyond the documented c3 band is uncharted drift");
  assert.equal(r.notRun, undefined, "this is a measured FAIL, not a skip");
  assert.match(r.evidence, /999\.000 ms/, "the measured p95 must appear in the evidence");
});

/* ————— BA — the bundle gate ————— */

await test("BA1 the bundle gate PASSES on the current tree; both measurements are printed", async () => {
  const m = await import(gatesUrl);
  const r = await m.runBundleBudgetGate();
  assert.equal(r.pass, true, `the shipped bundle must sit inside its recorded budgets: ${r.evidence}`);
  assert.match(r.evidence, /route bundle budget: \d+ \/ 204800 gz bytes/, "BA9's route-payload measurement must be printed");
  assert.match(r.evidence, /app bundle/, "the whole-app bundle measurement must be printed");
});

await test("BA2 a missing web/dist is notRun — NEVER a pass (exit-2 class)", async () => {
  const m = await import(gatesUrl);
  const r = await m.runBundleBudgetGate({ distDir: path.join(tmpdir(), "no-such-dist") });
  assert.equal(r.pass, false);
  assert.equal(r.notRun, true);
  assert.match(r.evidence, /web\/dist/, "the evidence must name the missing build");
});

await test("BA3 a fixture dist over the app-bundle ceiling FAILS the gate with the overage", async () => {
  const { randomBytes } = await import("node:crypto");
  const fix = makeFixtureRoot("ba3-dist", { "assets/huge.js": randomBytes(600000).toString("base64") });
  try {
    const m = await import(gatesUrl);
    const r = await m.runBundleBudgetGate({ distDir: fix.dir });
    assert.equal(r.pass, false, "600 KB gz of a single asset is over the 500000-byte app ceiling");
    assert.match(r.evidence, /over/, "the evidence must carry the overage");
  } finally {
    fix.cleanup();
  }
});

/* ————— KV — the KV write-class gate ————— */

const CRON_WITH_PUT = `async function runGuidanceCronTick(controller: ScheduledController, env: Env, opts: { baseFetch?: typeof fetch }): Promise<void> {
  await env.chat_sessions.put("guidance:x", "{}", { expirationTtl: 60 });
}
`;

const KV_SESSIONS_OK = `export const SESSION_TTL_SECONDS = 900;
export async function mintSession(kv, claim) {
  await kv.put("session:t", "{}", { expirationTtl: SESSION_TTL_SECONDS });
}
`;

await test("KV1 the KV gate PASSES on the current tree: write classes inventoried, cron write count 0, TTLs read from the constants", async () => {
  const m = await import(gatesUrl);
  const r = await m.runKvWriteClassGate();
  assert.equal(r.pass, true, `the tree's write classes must all be TTL-named: ${r.evidence}`);
  assert.match(r.evidence, /cron write count: 0/, "the cron's zero-write law must print the count explicitly");
  assert.match(r.evidence, /SESSION_TTL_SECONDS = 900/, "the TTL table must be read from the source constants");
  assert.match(r.evidence, /PIN_TTL_SECONDS = 300/, "the pin TTL must be read from the source");
  assert.match(r.evidence, /GUIDANCE_SESSION_TTL_SECONDS = 7200/, "the guidance TTL must be read from the source");
  assert.match(r.evidence, /session:/, "the session write class must be inventoried");
  assert.match(r.evidence, /loc:/, "the location-pin write class must be inventoried");
  assert.match(r.evidence, /guidance:/, "the guidance write class must be inventoried");
});

await test("KV2 a cron tick that WRITES fails the gate — the zero-write law is source-proven (T-07-18)", async () => {
  const fix = makeFixtureRoot("kv2-bot", {
    "src/webhook.ts": CRON_WITH_PUT,
    "src/kv/sessions.ts": KV_SESSIONS_OK,
    "src/kv/location-pin.ts": "export const PIN_TTL_SECONDS = 300;\n",
    "src/kv/guidance-sessions.ts": "export const GUIDANCE_SESSION_TTL_SECONDS = 7200;\n",
  });
  try {
    const m = await import(gatesUrl);
    const r = await m.runKvWriteClassGate({ botRoot: fix.dir });
    assert.equal(r.pass, false, "a .put( inside the cron body burns the 1,000/day write budget");
    assert.match(r.evidence, /\.put\(/, "the evidence must name the violating call");
  } finally {
    fix.cleanup();
  }
});

/* ————— SQ — the subrequest gate ————— */

await test("SQ1 the subrequest gate PASSES on the current tree with the per-class inventory and the honesty sentence", async () => {
  const m = await import(gatesUrl);
  const r = await m.runSubrequestClassGate();
  assert.equal(r.pass, true, `the inventory must cover every route class: ${r.evidence}`);
  assert.match(r.evidence, /source-derived inventory, not a live Workers Logs metric/, "the honesty sentence is mandatory, verbatim");
  assert.match(r.evidence, /\/api\/route.*0 external subrequests/, "the route endpoint's 0 must be printed WITH its source");
  assert.match(r.evidence, /not measured outside --live/, "the live metric absence must read not measured outside --live");
});

await test("SQ2 an uninventoried route class FAILS the gate — the inventory must follow the route table", async () => {
  const fix = makeFixtureRoot("sq2-bot", {
    "src/webhook.ts": `if (url.pathname === "/api/new-thing") { return handleNewThing(req, env); }\nif (url.pathname === "/api/route") { return handleRoute(req, env); }\nasync function runGuidanceCronTick(controller, env, opts): Promise<void> {}\n`,
  });
  try {
    const m = await import(gatesUrl);
    const r = await m.runSubrequestClassGate({ webhookPath: path.join(fix.dir, "src", "webhook.ts") });
    assert.equal(r.pass, false, "a route class with no inventory row is a silent budget hole");
    assert.match(r.evidence, /\/api\/new-thing/, "the evidence must name the uninventoried class");
  } finally {
    fix.cleanup();
  }
});

/* ————— CH — the cache-header gate ————— */

await test("CH1 the cache-header gate REUSES cloudflare-probes' runAssetHeaderProbe (imported, never forked)", async () => {
  const src = (await import("node:fs")).readFileSync(gatesUrl, "utf8");
  assert.match(src, /from ["']\.\/cloudflare-probes\.mjs["']/, "the live probe must be imported from the shared lib");
  assert.match(src, /runAssetHeaderProbe/, "the probe must be the shared one");
});

await test("CH2 without a deployed origin the cache-header gate is notRun — never a pass", async () => {
  const m = await import(gatesUrl);
  const r = await m.runCacheHeaderGate({ baseUrl: null });
  assert.equal(r.pass, false);
  assert.equal(r.notRun, true);
  assert.match(r.evidence, /not run/i, "the evidence must say the probe did not run");
});

/* ————— GC — the golden-corpus gate ————— */

await test("GC1 the golden-corpus gate RE-RUNS both corpora as child processes and PASSES on the current tree", async () => {
  const m = await import(gatesUrl);
  const r = await m.runGoldenCorpusGate();
  assert.equal(r.pass, true, `both corpora must be green: ${r.evidence}`);
  assert.match(r.evidence, /routing:corpus/, "the route corpus must be named by id");
  assert.match(r.evidence, /search:corpus/, "the search corpus must be named by id");
});

await test("GC2 a failing corpus FAILS the gate carrying that child's own output verbatim", async () => {
  const m = await import(gatesUrl);
  const r = await m.runGoldenCorpusGate({ commands: [["routing:corpus", ["node", "-e", "console.error('CORPUS GATE FAIL: synthetic'); process.exit(1)"]]] });
  assert.equal(r.pass, false);
  assert.match(r.evidence, /CORPUS GATE FAIL: synthetic/, "the child's own failure output must be carried verbatim");
});

/* ————— AU — the CLI ————— */

const cliPath = path.join(ROOT, "scripts", "budget-audit.mjs");

function runCli(args, env = {}) {
  return spawnSync(process.execPath, [cliPath, ...args], {
    cwd: ROOT,
    encoding: "utf8",
    timeout: 300000,
    env: { ...process.env, ...env },
    maxBuffer: 1 << 26,
  });
}

await test("AU1 without --live: every required gate prints its evidence line, the free-tier comparison table is printed, and the audit is CLEAN with exit 0", () => {
  assert.ok(existsSync(cliPath), "scripts/budget-audit.mjs must exist");
  const r = runCli([]);
  assert.equal(r.status, 0, `exit 0 expected on the current tree (stdout: ${r.stdout?.slice(-400)})`);
  const out = r.stdout ?? "";
  for (const id of ["route-cpu", "bundle-budget", "kv-writes", "subrequests", "golden-corpus"]) {
    assert.match(out, new RegExp(`${id}: (PASS|FAIL|NOT RUN) — `), `${id} must print its one-line evidence`);
  }
  assert.match(out, /BUDGET AUDIT CLEAN/, "the CLEAN summary must print");
  assert.match(out, /50\/request/, "the subrequest ceiling must appear in the comparison table");
  assert.match(out, /10 ms\/invocation/, "the CPU ceiling must appear in the comparison table");
  assert.match(out, /not measured/, "unmeasured ceilings must read `not measured` — never 0");
  assert.match(out, /cache-headers: NOT RUN.*\(pass --live\)/, "the flag-skipped live gate is visible and honest");
});

await test("AU2 with --live: the live header probe runs and the exit contract holds — a NOT RUN cache-header gate NEVER coexists with exit 0", () => {
  const r = runCli(["--live"]);
  const out = r.stdout ?? "";
  assert.ok([0, 1, 2].includes(r.status), `exit must be 0/1/2 (got ${r.status})`);
  assert.match(out, /cache-headers: /, "the cache-header gate must print with --live");
  if (out.includes("cache-headers: NOT RUN")) {
    assert.equal(r.status, 2, "a NOT RUN gate must map to exit 2 (environment drift) — never a silent pass");
    assert.match(out, /BUDGET AUDIT NOT RUN/, "the NOT RUN summary must print");
  }
});

await test("AU3 environment drift (a root with no builds/graph) maps to exit 2 — BUDGET AUDIT NOT RUN, never a pass (T-07-22)", () => {
  const empty = mkdtempSync(path.join(tmpdir(), "budget-empty-"));
  try {
    const r = runCli([], { PRIVACY_AUDIT_ROOT: empty });
    assert.equal(r.status, 2, `a root without the measured surfaces must be exit 2 (stdout: ${r.stdout?.slice(-300)})`);
    assert.match(r.stdout ?? "", /BUDGET AUDIT NOT RUN/, "the NOT RUN summary must print");
  } finally {
    rmSync(empty, { recursive: true, force: true });
  }
});

/* ————— summary ————— */
if (failures.length > 0) {
  console.error(`\n${failures.length}/${ran} vector(s) FAILED:`);
  for (const f of failures) console.error(`  - ${f.name}`);
  process.exit(1);
}
console.log(`\nBUDGET-AUDIT SUITE: ${ran}/${ran} vectors ok`);
process.exit(0);