test(07-04): RED — the budget-gates TDD suite (15 vectors: CPU ladder re-run, bundle bytes, KV cron zero-write law, subrequest inventory, cache-header reuse, golden corpora)
- every gate must print its measurement and carry its contract id - every gate must be able to FAIL on a synthetic violation (fixture injection) - a skip is never a pass (notRun class; T-07-22)
This commit is contained in:
araharah-executor
@@ -0,0 +1,271 @@
|
||||
#!/usr/bin/env node
|
||||
/*
|
||||
* budget-audit.test.mjs — the TDD suite for the consolidated budget audit
|
||||
* (07-04, D-06). Named vectors in the repo's gate-suite style.
|
||||
* ---------------------------------------------------------------------------
|
||||
* A MAINTAINER-TIME REPO TOOL — NOT a runtime dependency; nothing in any src
|
||||
* tree imports it. GPL-3.0: forks rerun it before their first driver.
|
||||
*
|
||||
* What the vectors pin:
|
||||
*
|
||||
* CPU* the route-CPU gate — the ladder is RE-RUN as a child process
|
||||
* (never re-implemented), the p95 is PRINTED against the 8 ms
|
||||
* product budget and the 10 ms free ceiling, the child's own
|
||||
* GATE FAIL line is carried verbatim, a missing graph is
|
||||
* notRun (never a pass), and the drift guard fails uncharted
|
||||
* regression beyond the documented accepted c3 band
|
||||
* BA* the bundle gate — BA9's gzip measurement + the whole-app
|
||||
* bundle ceiling + the missing-dist notRun branch
|
||||
* KV* the KV write-class gate — the cron zero-write law is
|
||||
* source-proven (a .put( inside runGuidanceCronTick FAILS the
|
||||
* gate), every write references a named TTL constant, and the
|
||||
* TTL table is READ from the constants (never hard-coded)
|
||||
* SQ* the subrequest gate — the per-request-class inventory,
|
||||
* source-derived honesty sentence, uninventoried route classes
|
||||
* FAIL, and never a fabricated zero
|
||||
* CH* the cache-header gate — reuses cloudflare-probes' live probe
|
||||
* (never forks it) and degrades to notRun without an origin
|
||||
* GC* the golden-corpus gate — RE-RUNS `routing:corpus` +
|
||||
* `search:corpus` as child processes; a failing corpus FAILS
|
||||
* the gate with that child's output
|
||||
*
|
||||
* Hand-rolled runner (the bot/web harness precedent — zero dependencies):
|
||||
* every test is async, failures are collected, and the process exits 1 when
|
||||
* any vector fails.
|
||||
*/
|
||||
import { strict as assert } from "node:assert";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { mkdtempSync, mkdirSync, rmSync, writeFileSync, existsSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const HERE = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(HERE, "..", "..");
|
||||
|
||||
/* ————— minimal harness (the hand-rolled runner convention) ————— */
|
||||
|
||||
const failures = [];
|
||||
let ran = 0;
|
||||
|
||||
async function test(name, fn) {
|
||||
ran += 1;
|
||||
try {
|
||||
await fn();
|
||||
console.log(`ok — ${name}`);
|
||||
} catch (err) {
|
||||
failures.push({ name, err });
|
||||
console.log(`FAIL — ${name}\n ${err?.message ?? err}`);
|
||||
}
|
||||
}
|
||||
|
||||
/* ————— fixture helper: a synthetic tree the gates scan via opts ————— */
|
||||
|
||||
function makeFixtureRoot(name, files) {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), "budget-audit-"));
|
||||
for (const [rel, body] of Object.entries(files)) {
|
||||
const full = path.join(dir, rel);
|
||||
mkdirSync(path.dirname(full), { recursive: true });
|
||||
writeFileSync(full, body);
|
||||
}
|
||||
return { name, dir, cleanup: () => rmSync(dir, { recursive: true, force: true }) };
|
||||
}
|
||||
|
||||
/* ————— CPU — the route-CPU gate ————— */
|
||||
|
||||
const gatesUrl = path.join(ROOT, "scripts", "lib", "budget-gates.mjs");
|
||||
|
||||
await test("CPU1 budget-gates exports the six gates; each carries its contract id", async () => {
|
||||
assert.ok(existsSync(gatesUrl), "scripts/lib/budget-gates.mjs must exist");
|
||||
const m = await import(gatesUrl);
|
||||
const expected = {
|
||||
runCpuLadderGate: "route-cpu",
|
||||
runBundleBudgetGate: "bundle-budget",
|
||||
runKvWriteClassGate: "kv-writes",
|
||||
runSubrequestClassGate: "subrequests",
|
||||
runCacheHeaderGate: "cache-headers",
|
||||
runGoldenCorpusGate: "golden-corpus",
|
||||
};
|
||||
for (const [fn, id] of Object.entries(expected)) {
|
||||
assert.equal(typeof m[fn], "function", `${fn} must be exported`);
|
||||
const r = await m[fn]();
|
||||
assert.equal(r.id, id, `${fn}() must carry id "${id}"`);
|
||||
assert.equal(typeof r.pass, "boolean", `${fn}() must return a boolean pass`);
|
||||
assert.equal(typeof r.evidence, "string", `${fn}() must return evidence`);
|
||||
}
|
||||
});
|
||||
|
||||
await test("CPU2 the ladder is RE-RUN (the gate references measure-route-cpu) and P95_BUDGET_MS = 8 is the same constant the script uses", async () => {
|
||||
const src = (await import("node:fs")).readFileSync(gatesUrl, "utf8");
|
||||
assert.ok(src.includes("measure-route-cpu"), "the gate must spawn the existing ladder script");
|
||||
assert.ok(src.includes("P95_BUDGET_MS = 8"), "the gate must carry the 8 ms product budget constant");
|
||||
});
|
||||
|
||||
await test("CPU3 on the current tree the ladder runs, the measured p95 is printed against the 8 ms budget and the 10 ms free ceiling, and the child's own GATE line is carried", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runCpuLadderGate();
|
||||
assert.equal(r.notRun, undefined, "the graph binary is committed — the ladder must run, not skip");
|
||||
assert.equal(r.pass, true, `the measured state must sit inside the documented accepted c3 band: ${r.evidence}`);
|
||||
assert.match(r.evidence, /p95 \d+(\.\d+)? ms/, "the measured p95 must be printed");
|
||||
assert.match(r.evidence, /8 ms/, "the 8 ms product budget must be printed beside the measurement");
|
||||
assert.match(r.evidence, /10 ms free-plan ceiling/, "the free-plan ceiling must be printed");
|
||||
assert.match(r.evidence, /GATE FAIL/, "the child's own GATE FAIL line (cross-metro class) is part of the evidence");
|
||||
assert.match(r.evidence, /c3/, "the accepted c3 escalation class must be named");
|
||||
});
|
||||
|
||||
await test("CPU4 a missing graph binary is notRun — NEVER a pass (T-07-22)", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runCpuLadderGate({ graphPath: path.join(tmpdir(), "no-such-graph.bin") });
|
||||
assert.equal(r.pass, false, "a skip is never a pass");
|
||||
assert.equal(r.notRun, true, "a missing graph is the notRun class");
|
||||
assert.match(r.evidence, /absent/, "the evidence must name the absent binary and the fix");
|
||||
});
|
||||
|
||||
await test("CPU5 an uncharted regression (p95 beyond the documented accepted band) FAILS the gate", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runCpuLadderGate({ ladderOutput: "p95: 999.000 ms (max 999.000 ms, mean 900.000 ms, n=22)\n" });
|
||||
assert.equal(r.pass, false, "a p95 far beyond the documented c3 band is uncharted drift");
|
||||
assert.equal(r.notRun, undefined, "this is a measured FAIL, not a skip");
|
||||
assert.match(r.evidence, /999\.000 ms/, "the measured p95 must appear in the evidence");
|
||||
});
|
||||
|
||||
/* ————— BA — the bundle gate ————— */
|
||||
|
||||
await test("BA1 the bundle gate PASSES on the current tree; both measurements are printed", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runBundleBudgetGate();
|
||||
assert.equal(r.pass, true, `the shipped bundle must sit inside its recorded budgets: ${r.evidence}`);
|
||||
assert.match(r.evidence, /route bundle budget: \d+ \/ 204800 gz bytes/, "BA9's route-payload measurement must be printed");
|
||||
assert.match(r.evidence, /app bundle/, "the whole-app bundle measurement must be printed");
|
||||
});
|
||||
|
||||
await test("BA2 a missing web/dist is notRun — NEVER a pass (exit-2 class)", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runBundleBudgetGate({ distDir: path.join(tmpdir(), "no-such-dist") });
|
||||
assert.equal(r.pass, false);
|
||||
assert.equal(r.notRun, true);
|
||||
assert.match(r.evidence, /web\/dist/, "the evidence must name the missing build");
|
||||
});
|
||||
|
||||
await test("BA3 a fixture dist over the app-bundle ceiling FAILS the gate with the overage", async () => {
|
||||
const fix = makeFixtureRoot("ba3-dist", { "assets/huge.js": "x".repeat(600000) });
|
||||
try {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runBundleBudgetGate({ distDir: fix.dir });
|
||||
assert.equal(r.pass, false, "600 KB gz of a single asset is over the 500000-byte app ceiling");
|
||||
assert.match(r.evidence, /over/, "the evidence must carry the overage");
|
||||
} finally {
|
||||
fix.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
/* ————— KV — the KV write-class gate ————— */
|
||||
|
||||
const CRON_WITH_PUT = `async function runGuidanceCronTick(controller: ScheduledController, env: Env, opts: { baseFetch?: typeof fetch }): Promise<void> {
|
||||
await env.chat_sessions.put("guidance:x", "{}", { expirationTtl: 60 });
|
||||
}
|
||||
`;
|
||||
|
||||
const KV_SESSIONS_OK = `export const SESSION_TTL_SECONDS = 900;
|
||||
export async function mintSession(kv, claim) {
|
||||
await kv.put("session:t", "{}", { expirationTtl: SESSION_TTL_SECONDS });
|
||||
}
|
||||
`;
|
||||
|
||||
await test("KV1 the KV gate PASSES on the current tree: write classes inventoried, cron write count 0, TTLs read from the constants", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runKvWriteClassGate();
|
||||
assert.equal(r.pass, true, `the tree's write classes must all be TTL-named: ${r.evidence}`);
|
||||
assert.match(r.evidence, /cron write count: 0/, "the cron's zero-write law must print the count explicitly");
|
||||
assert.match(r.evidence, /SESSION_TTL_SECONDS = 900/, "the TTL table must be read from the source constants");
|
||||
assert.match(r.evidence, /PIN_TTL_SECONDS = 300/, "the pin TTL must be read from the source");
|
||||
assert.match(r.evidence, /GUIDANCE_SESSION_TTL_SECONDS = 7200/, "the guidance TTL must be read from the source");
|
||||
assert.match(r.evidence, /session:/, "the session write class must be inventoried");
|
||||
assert.match(r.evidence, /loc:/, "the location-pin write class must be inventoried");
|
||||
assert.match(r.evidence, /guidance:/, "the guidance write class must be inventoried");
|
||||
});
|
||||
|
||||
await test("KV2 a cron tick that WRITES fails the gate — the zero-write law is source-proven (T-07-18)", async () => {
|
||||
const fix = makeFixtureRoot("kv2-bot", {
|
||||
"src/webhook.ts": CRON_WITH_PUT,
|
||||
"src/kv/sessions.ts": KV_SESSIONS_OK,
|
||||
"src/kv/location-pin.ts": "export const PIN_TTL_SECONDS = 300;\n",
|
||||
"src/kv/guidance-sessions.ts": "export const GUIDANCE_SESSION_TTL_SECONDS = 7200;\n",
|
||||
});
|
||||
try {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runKvWriteClassGate({ botRoot: fix.dir });
|
||||
assert.equal(r.pass, false, "a .put( inside the cron body burns the 1,000/day write budget");
|
||||
assert.match(r.evidence, /\.put\(/, "the evidence must name the violating call");
|
||||
} finally {
|
||||
fix.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
/* ————— SQ — the subrequest gate ————— */
|
||||
|
||||
await test("SQ1 the subrequest gate PASSES on the current tree with the per-class inventory and the honesty sentence", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runSubrequestClassGate();
|
||||
assert.equal(r.pass, true, `the inventory must cover every route class: ${r.evidence}`);
|
||||
assert.match(r.evidence, /source-derived inventory, not a live Workers Logs metric/, "the honesty sentence is mandatory, verbatim");
|
||||
assert.match(r.evidence, /\/api\/route.*0 external subrequests/, "the route endpoint's 0 must be printed WITH its source");
|
||||
assert.match(r.evidence, /not measured outside --live/, "the live metric absence must read not measured outside --live");
|
||||
});
|
||||
|
||||
await test("SQ2 an uninventoried route class FAILS the gate — the inventory must follow the route table", async () => {
|
||||
const fix = makeFixtureRoot("sq2-bot", {
|
||||
"src/webhook.ts": `if (url.pathname === "/api/new-thing") { return handleNewThing(req, env); }\nif (url.pathname === "/api/route") { return handleRoute(req, env); }\nasync function runGuidanceCronTick(controller, env, opts): Promise<void> {}\n`,
|
||||
});
|
||||
try {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runSubrequestClassGate({ webhookPath: path.join(fix.dir, "src", "webhook.ts") });
|
||||
assert.equal(r.pass, false, "a route class with no inventory row is a silent budget hole");
|
||||
assert.match(r.evidence, /\/api\/new-thing/, "the evidence must name the uninventoried class");
|
||||
} finally {
|
||||
fix.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
/* ————— CH — the cache-header gate ————— */
|
||||
|
||||
await test("CH1 the cache-header gate REUSES cloudflare-probes' runAssetHeaderProbe (imported, never forked)", async () => {
|
||||
const src = (await import("node:fs")).readFileSync(gatesUrl, "utf8");
|
||||
assert.match(src, /from ["']\.\/cloudflare-probes\.mjs["']/, "the live probe must be imported from the shared lib");
|
||||
assert.match(src, /runAssetHeaderProbe/, "the probe must be the shared one");
|
||||
});
|
||||
|
||||
await test("CH2 without a deployed origin the cache-header gate is notRun — never a pass", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runCacheHeaderGate({ baseUrl: null });
|
||||
assert.equal(r.pass, false);
|
||||
assert.equal(r.notRun, true);
|
||||
assert.match(r.evidence, /not run/i, "the evidence must say the probe did not run");
|
||||
});
|
||||
|
||||
/* ————— GC — the golden-corpus gate ————— */
|
||||
|
||||
await test("GC1 the golden-corpus gate RE-RUNS both corpora as child processes and PASSES on the current tree", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runGoldenCorpusGate();
|
||||
assert.equal(r.pass, true, `both corpora must be green: ${r.evidence}`);
|
||||
assert.match(r.evidence, /routing:corpus/, "the route corpus must be named by id");
|
||||
assert.match(r.evidence, /search:corpus/, "the search corpus must be named by id");
|
||||
});
|
||||
|
||||
await test("GC2 a failing corpus FAILS the gate carrying that child's own output verbatim", async () => {
|
||||
const m = await import(gatesUrl);
|
||||
const r = await m.runGoldenCorpusGate({ commands: [["routing:corpus", ["node", "-e", "console.error('CORPUS GATE FAIL: synthetic'); process.exit(1)"]]] });
|
||||
assert.equal(r.pass, false);
|
||||
assert.match(r.evidence, /CORPUS GATE FAIL: synthetic/, "the child's own failure output must be carried verbatim");
|
||||
});
|
||||
|
||||
/* ————— summary ————— */
|
||||
|
||||
if (failures.length > 0) {
|
||||
console.error(`\n${failures.length}/${ran} vector(s) FAILED:`);
|
||||
for (const f of failures) console.error(` - ${f.name}`);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log(`\nBUDGET-AUDIT SUITE: ${ran}/${ran} vectors ok`);
|
||||
process.exit(0);
|
||||
Reference in New Issue
Block a user