diff --git a/scripts/lib/source-gates.mjs b/scripts/lib/source-gates.mjs new file mode 100644 index 0000000..2424b9e --- /dev/null +++ b/scripts/lib/source-gates.mjs @@ -0,0 +1,323 @@ +#!/usr/bin/env node +/* + * source-gates.mjs — the repo-level source privacy gates (07-03, D-01). + * --------------------------------------------------------------------------- + * A MAINTAINER-TIME REPO TOOL — NOT a runtime dependency; nothing in bot/src, + * web/src, routing/src, or search/src imports it (the law from + * routing/scripts/measure-route-cpu.mjs lines 3–9). GPL-3.0: a fork reruns + * these gates over ITS OWN trees before its first driver. + * + * The gate lineage this consolidates (extended repo-wide, not rewritten): + * storage-api ← web/test/no-storage.test.mjs GEO3 (the memory-only + * client law) + the service-worker ban + * kv-ttl ← bot/test/privacy.test.mjs PG1 (same-line + * expirationTtl referencing a NAMED constant) + * log-vocabulary ← bot/src/webhook.ts D-14 (category-only logs; the + * web-side ARAHARAH-* state tokens) + * runtime-hosts ← D-01's POSITIVE fetch-host allowlist (the host + * inventory is PRINTED, never assumed — A4) + * + * Every gate returns { id, pass, evidence } (07-RESEARCH Pattern 4) and is + * safe to call with no arguments in production. `opts` exists ONLY for the + * test suite (fixture-root injection); production callers pass nothing. + * + * THE COMMENT STRIPPER IS COPIED VERBATIM from web/test/no-storage.test.mjs + * lines 29–42 (the canonical twin also lives in bot/test/route-privacy.test.mjs + * lines 42–59). Its over-stripping is DELIBERATE — "prefer-fail reading": a + * prose mention that survives stripping fails the gate LOUDLY; do NOT write a + * smarter stripper (07-RESEARCH Pitfall 6). The line-numbered gates + * (kv-ttl, log-vocabulary) scan raw lines with only the `// …` tail removed + * per line, so evidence line numbers stay honest; prose inside a block + * comment that mentions a scanned token fails prefer-fail too. + * + * PRIVACY_AUDIT_ROOT: the repo root is derived from THIS module's URL — the + * gates are cwd-INDEPENDENT (a stray subdirectory run must not scan the wrong + * tree and false-pass). The env override exists for the test suite and for a + * fork auditing a checkout elsewhere. + */ +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const MODULE_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..", ".."); +export const REPO_ROOT = process.env.PRIVACY_AUDIT_ROOT ?? MODULE_ROOT; + +/* ————— the canonical helpers (copied verbatim — see header) ————— */ + +export function listFiles(dir, exts, acc = []) { + for (const entry of readdirSync(dir)) { + const full = path.join(dir, entry); + if (statSync(full).isDirectory()) listFiles(full, exts, acc); + else if (exts.some((e) => entry.endsWith(e))) acc.push(full); + } + return acc; +} + +export function stripComments(src) { + return src + .replace(/\/\*[\s\S]*?\*\//g, " ") // block comments + .replace(/(^|[^:\\])\/\/[^\n]*/g, "$1"); // line comments (not URLs inside strings — benign over-stripping, gate is prefer-fail reading) +} + +/** Line-preserving variant for line-numbered evidence: removes ONLY the + * `// …` tail of a single raw line (the same guard as the stripper, so + * `https://` URLs inside strings stay alive). Block-comment prose mentioning + * a scanned token fails prefer-fail — documented in the header. */ +function stripLineComment(line) { + return line.replace(/(^|[^:\\])\/\/[^\n]*/g, "$1"); +} + +/* ————— the scan roots: EVERY production source tree, repo-wide ————— */ + +const ALL_TREES = ["web/src", "bot/src", "routing/src", "search/src", "theme/src"].map((name) => ({ + name, + dir: path.join(REPO_ROOT, ...name.split("/")), +})); + +const CODE_EXTS = [".ts", ".tsx", ".js", ".mjs", ".html"]; + +function scanRoots(roots, exts) { + const files = []; + const skipped = []; + for (const { name, dir } of roots) { + if (!existsSync(dir)) { + skipped.push(name); + continue; + } + for (const file of listFiles(dir, exts)) files.push({ name, file }); + } + return { files, skipped }; +} + +/* ————— storage-api — the memory-only client law, extended repo-wide ————— */ + +/** The BANNED persistence vocabulary (web/test/no-storage.test.mjs lines 22–25) + * plus the service-worker family (the GEO3 "and none was added" half). */ +const BANNED_PERSISTENCE = [ + "localStorage", "sessionStorage", "indexedDB", "caches", "CacheStorage", + "document.cookie", "WebApp.DeviceStorage", "WebApp.SecureStorage", "WebApp.CloudStorage", +]; +const BANNED_SERVICE_WORKER = ["serviceWorker.register", "navigator.serviceWorker", "workbox"]; + +export function runStorageGate(opts = {}) { + const id = "storage-api"; + const roots = opts.roots ?? ALL_TREES; + const { files, skipped } = scanRoots(roots, CODE_EXTS); + const violations = []; + for (const { file } of files) { + const code = stripComments(readFileSync(file, "utf8")); + for (const banned of [...BANNED_PERSISTENCE, ...BANNED_SERVICE_WORKER]) { + if (code.includes(banned)) violations.push(`${path.relative(REPO_ROOT, file)}: ${banned}`); + } + } + const pass = violations.length === 0; + const skippedNote = skipped.length > 0 ? ` (skipped, absent: ${skipped.join(", ")})` : ""; + const evidence = pass + ? `${files.length} file(s) across ${roots.length - skipped.length} src tree(s) — no persistence API outside the platform allowlist${skippedNote}` + : `${files.length} file(s) scanned${skippedNote}; ${violations.length} violation(s): ${violations.join("; ")}`; + return { id, pass, evidence, violations: violations.length }; +} + +/* ————— kv-ttl — the PG1 same-line law, extended repo-wide (T-07-01) ————— */ + +export function runKvTtlGate(opts = {}) { + const id = "kv-ttl"; + const roots = opts.roots ?? ALL_TREES; + const { files, skipped } = scanRoots(roots, CODE_EXTS); + const violations = []; + let writes = 0; + for (const { file } of files) { + const lines = readFileSync(file, "utf8").split("\n"); + lines.forEach((line, i) => { + if (line.includes(".put(")) { + writes += 1; + const ok = + line.includes("expirationTtl") && + /^[A-Z_]{2,}$/.test(line.match(/expirationTtl:\s*([A-Za-z0-9_]+)/)?.[1] ?? ""); + if (!ok) violations.push(`${path.relative(REPO_ROOT, file)}:${i + 1} — KV write without a named expirationTtl: ${line.trim()}`); + } + }); + } + const pass = violations.length === 0; + const skippedNote = skipped.length > 0 ? ` (skipped, absent: ${skipped.join(", ")})` : ""; + const evidence = pass + ? `${writes} KV write(s) across ${files.length} file(s) — every .put( line carries expirationTtl referencing a named ^[A-Z_]{2,}$ constant${skippedNote}` + : `${writes} KV write(s) scanned${skippedNote}; ${violations.length} violation(s): ${violations.join("; ")}`; + return { id, pass, evidence, violations: violations.length }; +} + +/* ————— log-vocabulary — the category-only law, machine-verified (T-07-03) ————— */ + +/** Coordinate/identifier carriers FORBIDDEN on any bot/src log line. `lat`, + * `lng`, `token`, `initData`, `geometry`, `polyline`, `payload` are checked + * as case-sensitive substrings; `chat_id`/`user_id` as WHOLE WORDS so the + * `[location]` surface token never trips the `chat` substring. Comments are + * stripped first (the "never the token" prose on the call lines is not code). */ +const BANNED_CARRIERS = ["lat", "lng", "token", "initData", "geometry", "polyline", "payload"]; +const BANNED_CARRIER_WORDS = ["chat_id", "user_id"]; + +/** bot/src shape: `console.log(` + a quote + a bracketed surface token. */ +const BOT_LOG_SHAPE = /console\.log\(\s*["'`]\[([a-z-]+)\]/; + +/** web/src vocabulary: the ARAHARAH-* state tokens / the basemap signal. */ +const WEB_LOG_ALLOWED = [ + /ARAHARAH-(DEST|LIVE|ROUTE|GUIDE|DRIVE)\s+state=/, + /ARAHARAH-LIVE\s+fix=/, + /RAMS-BASEMAP-READY/, +]; + +function violationsInTree(dir, isBot, violations) { + if (!existsSync(dir)) return 0; + let logs = 0; + for (const file of listFiles(dir, [".ts", ".tsx", ".js", ".mjs"])) { + const rel = path.relative(REPO_ROOT, file); + const lines = readFileSync(file, "utf8").split("\n"); + lines.forEach((rawLine, i) => { + const line = stripLineComment(rawLine); + if (!/console\.log\(/.test(line)) return; + logs += 1; + const from = line.indexOf("console.log("); + const args = line.slice(from); + if (args.includes("JSON.stringify(")) { + violations.push(`${rel}:${i + 1} — a serialized payload inside a log argument: ${line.trim()}`); + return; + } + if (isBot) { + if (!BOT_LOG_SHAPE.test(line)) { + violations.push(`${rel}:${i + 1} — bot log without a bracketed surface token: ${line.trim()}`); + return; + } + for (const carrier of BANNED_CARRIERS) { + if (line.includes(carrier)) { + violations.push(`${rel}:${i + 1} — bot log carries the "${carrier}" carrier: ${line.trim()}`); + return; + } + } + for (const word of BANNED_CARRIER_WORDS) { + if (new RegExp(`\\b${word}\\b`).test(line)) { + violations.push(`${rel}:${i + 1} — bot log carries the "${word}" identifier: ${line.trim()}`); + return; + } + } + } else { + if (!WEB_LOG_ALLOWED.some((rx) => rx.test(line))) { + violations.push(`${rel}:${i + 1} — web log outside the state-token vocabulary: ${line.trim()}`); + } + } + }); + } + return logs; +} + +export function runLogVocabularyGate(opts = {}) { + const id = "log-vocabulary"; + const botRoot = opts.botRoot ?? path.join(REPO_ROOT, "bot", "src"); + const webRoot = opts.webRoot ?? path.join(REPO_ROOT, "web", "src"); + const violations = []; + const botLogs = violationsInTree(botRoot, true, violations); + const webLogs = violationsInTree(webRoot, false, violations); + const pass = violations.length === 0; + const evidence = pass + ? `${botLogs} bot + ${webLogs} web console.log call site(s) — every bot log opens with a bracketed surface token and carries NO lat/lng/chat_id/user_id/token/initData/geometry/polyline/payload carrier, every web log is an ARAHARAH-* state token or the RAMS-BASEMAP-READY signal, and no log serializes a payload` + : `${botLogs + webLogs} log call site(s) scanned; ${violations.length} violation(s): ${violations.join("; ")}`; + return { id, pass, evidence, violations: violations.length }; +} + +/* ————— runtime-hosts — the POSITIVE allowlist, inventory printed (A4) ————— */ + +/** The ONLY hosts a fetch target may name (D-01). A relative path like + * `/api/guidance` is the app's own Worker origin — not a host — and is + * allowed. */ +const ALLOWED_FETCH_HOSTS = ["tiles.openfreemap.org", "telegram.org"]; + +/** Documented, deliberate scope adjudication (the theme-audit precedent): + * the SVG namespace URIs (`http://www.w3.org/2000/svg`) are spec + * IDENTIFIERS, never fetched. Any other http:// host fails prefer-fail. */ +const SVG_NAMESPACE = "http://www.w3.org/2000/svg"; + +/** Extract the hostname of an absolute URL string, or null. */ +function hostOf(urlText) { + try { + return new URL(urlText).hostname; + } catch { + return null; + } +} + +export function runRuntimeHostGate(opts = {}) { + const id = "runtime-hosts"; + const roots = opts.roots ?? [ + ...ALL_TREES.filter((t) => t.name !== "theme/src"), + { name: "theme/src", dir: path.join(REPO_ROOT, "theme", "src") }, + { name: "web/index.html", dir: path.join(REPO_ROOT, "web", "index.html"), singleFile: true }, + ]; + const violations = []; + const hostInventory = new Map(); // host → count + const fetchTargets = []; + let fetchDeclarations = 0; + + function noteHost(urlText) { + const host = hostOf(urlText); + if (host === null) return; + hostInventory.set(host, (hostInventory.get(host) ?? 0) + 1); + } + + function checkUrl(rel, urlText, where) { + noteHost(urlText); + const host = hostOf(urlText); + if (host === null) return; // not a parseable URL — leave it to the bundle scan / live probe + if (urlText === SVG_NAMESPACE) return; // documented adjudication — a spec identifier, never fetched + if (ALLOWED_FETCH_HOSTS.includes(host)) return; + violations.push(`${rel} — absolute URL to an unapproved host (${where}): ${host}`); + } + + const { files, skipped } = scanRoots(roots.filter((r) => !r.singleFile), CODE_EXTS); + for (const { file } of files) { + const rel = path.relative(REPO_ROOT, file); + const code = stripComments(readFileSync(file, "utf8")); + for (const urlText of code.match(/https?:\/\/[a-zA-Z0-9._~:/?#[\]@!$&'()*+,;=%-]+/g) ?? []) { + checkUrl(rel, urlText, "URL literal"); + } + // fetch( call sites: the FIRST argument must be relative (or an approved + // host) — never an absolute URL to an unapproved host. TS method + // DECLARATIONS (`fetch(req: Request, …)`) are skipped — the annotation + // colon identifies them. + for (const m of code.matchAll(/\bfetch\s*\(/g)) { + const rest = code.slice(m.index + m[0].length, code.indexOf("\n", m.index)); + const argText = rest.split(/(? `${h}×${n}`).join(", ") || "none"; + const skippedNote = skipped.length > 0 ? ` (skipped, absent: ${skipped.join(", ")})` : ""; + const evidence = pass + ? `host inventory (printed, not assumed): ${inventory}; ${fetchTargets.length} relative fetch target(s) to the app's own Worker, ${fetchDeclarations} fetch declaration(s) skipped; every absolute URL host is in the positive allowlist [${ALLOWED_FETCH_HOSTS.join(", ")}]${skippedNote}` + : `host inventory: ${inventory}; ${violations.length} violation(s): ${violations.join("; ")}`; + return { id, pass, evidence, violations: violations.length }; +}