Sylvain Duchesne
9e62bdea53
Fondation « l'agent valide tout d'un coup » (audit couverture, levier C1). Enchaîne et agrège : polyfill unit + polyfill e2e réel + Festipod @data + @multibrowser (profils frais/rotation), matrice finale + exit non-zéro si rouge, passe @wip informative. Baseline établi (premier run réel) : polyfill unit 120 ✅, polyfill e2e 42/42 ✅ (broker réel), @multibrowser 1 rouge (limite wallet-partagé A/B), @data 4 rouges dont 2 régressions phase B confirmées. → a trouvé les problèmes tout seul. À TUNER (suivi) : le budget @data (~14min) est trop court → la suite complète est tuée par timeout ; augmenter le budget OU exécuter un sous-ensemble clé rapide. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
358 lines
13 KiB
TypeScript
358 lines
13 KiB
TypeScript
#!/usr/bin/env bun
|
||
/**
|
||
* validate.ts — Full validation matrix (broker-level tests only, no @ui/@e2e).
|
||
*
|
||
* Runs in sequence:
|
||
* (a) Polyfill unit tests (@ng-eventually/client — bun test)
|
||
* (b) Polyfill e2e real-broker (@ng-eventually/client — bun run test:e2e)
|
||
* (c) Festipod @data (cucumber --tags @data)
|
||
* (d) Festipod @multibrowser (cucumber --tags @multibrowser)
|
||
* (e) Festipod @wip [informational only, non-blocking]
|
||
*
|
||
* Each step runs even if the previous one failed (--bail mode is OFF).
|
||
* Exit code is non-zero if any non-informational step has failures.
|
||
*
|
||
* Profile rotation: both Playwright profiles are rotated before the run
|
||
* when their size exceeds BLOAT_THRESHOLD_MB (default 50 MB), to avoid
|
||
* the sparql_query hang described in caveat_wallet-bloat-hang.
|
||
*/
|
||
|
||
import { spawnSync, execSync } from "child_process";
|
||
import * as fs from "fs";
|
||
import * as path from "path";
|
||
|
||
// ─── Config ────────────────────────────────────────────────────────────────
|
||
|
||
const FESTIPOD_DIR = "/home/sylvain/projects/festipod/festipod";
|
||
const POLYFILL_DIR =
|
||
"/home/sylvain/projects/nextgraph/ng-eventually-js/packages/client";
|
||
|
||
const FESTIPOD_PROFILE = path.join(FESTIPOD_DIR, ".playwright-profile");
|
||
const POLYFILL_PROFILE = path.join(
|
||
POLYFILL_DIR,
|
||
"e2e",
|
||
".playwright-profile-lib",
|
||
);
|
||
|
||
/** Rotate profile when it exceeds this many MB (caveat_wallet-bloat-hang). */
|
||
const BLOAT_THRESHOLD_MB = 50;
|
||
|
||
/** Per-step timeout in ms — generous for broker + headless wallet creation. */
|
||
const STEP_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
|
||
|
||
// ─── Helpers ───────────────────────────────────────────────────────────────
|
||
|
||
function dirSizeMB(dir: string): number {
|
||
if (!fs.existsSync(dir)) return 0;
|
||
try {
|
||
const result = spawnSync("du", ["-sm", dir], { encoding: "utf-8" });
|
||
const line = result.stdout.trim().split("\n")[0] ?? "";
|
||
return parseInt(line.split("\t")[0] ?? "0", 10);
|
||
} catch {
|
||
return 0;
|
||
}
|
||
}
|
||
|
||
function rotateProfile(profilePath: string, label: string): void {
|
||
const sizeMB = dirSizeMB(profilePath);
|
||
if (sizeMB > BLOAT_THRESHOLD_MB) {
|
||
console.log(
|
||
`[rotate] ${label}: ${sizeMB}MB > ${BLOAT_THRESHOLD_MB}MB — rotating profile...`,
|
||
);
|
||
try {
|
||
fs.rmSync(profilePath, { recursive: true, force: true });
|
||
console.log(`[rotate] ${label}: profile removed. Will be recreated.`);
|
||
} catch (e) {
|
||
console.warn(`[rotate] ${label}: failed to remove profile: ${e}`);
|
||
}
|
||
} else {
|
||
// Even if we keep the profile, remove any stale SingletonLock left by a
|
||
// previous crashed run — Chromium refuses to launch if the lock exists.
|
||
const lockPath = path.join(profilePath, "SingletonLock");
|
||
if (fs.existsSync(lockPath)) {
|
||
try {
|
||
fs.rmSync(lockPath, { force: true });
|
||
console.log(`[rotate] ${label}: removed stale SingletonLock.`);
|
||
} catch {
|
||
// Non-fatal: if we can't remove it, launch will fail with a clear error
|
||
}
|
||
}
|
||
console.log(
|
||
`[rotate] ${label}: ${sizeMB}MB — below threshold, keeping profile.`,
|
||
);
|
||
}
|
||
}
|
||
|
||
interface StepResult {
|
||
label: string;
|
||
status: "passed" | "failed" | "error";
|
||
/** Lines to show in the summary (failed scenario names, FAIL lines, etc.) */
|
||
failures: string[];
|
||
/** Raw exit code */
|
||
exitCode: number;
|
||
durationMs: number;
|
||
}
|
||
|
||
/**
|
||
* Run a command and capture its output. Returns the result with parsed
|
||
* pass/fail summary. Never throws — all errors are captured in StepResult.
|
||
*/
|
||
function runStep(
|
||
label: string,
|
||
cmd: string,
|
||
args: string[],
|
||
cwd: string,
|
||
extraEnv: Record<string, string> = {},
|
||
): StepResult {
|
||
const t0 = Date.now();
|
||
console.log(`\n${"═".repeat(60)}`);
|
||
console.log(`▶ ${label}`);
|
||
console.log(` ${cmd} ${args.join(" ")} (cwd: ${cwd})`);
|
||
console.log(`${"═".repeat(60)}`);
|
||
|
||
const env = { ...process.env, ...extraEnv };
|
||
|
||
const result = spawnSync(cmd, args, {
|
||
cwd,
|
||
env,
|
||
encoding: "utf-8",
|
||
timeout: STEP_TIMEOUT_MS,
|
||
maxBuffer: 20 * 1024 * 1024, // 20MB
|
||
});
|
||
|
||
const durationMs = Date.now() - t0;
|
||
const stdout = result.stdout ?? "";
|
||
const stderr = result.stderr ?? "";
|
||
const combined = stdout + "\n" + stderr;
|
||
|
||
// Print output in real-time equivalent (post-hoc since spawnSync)
|
||
if (stdout) process.stdout.write(stdout);
|
||
if (stderr) process.stderr.write(stderr);
|
||
|
||
if (result.error) {
|
||
console.error(`[${label}] process error:`, result.error.message);
|
||
return {
|
||
label,
|
||
status: "error",
|
||
failures: [`Process error: ${result.error.message}`],
|
||
exitCode: result.status ?? 1,
|
||
durationMs,
|
||
};
|
||
}
|
||
|
||
const exitCode = result.status ?? 1;
|
||
const failures = extractFailures(combined, label);
|
||
const status = exitCode === 0 ? "passed" : "failed";
|
||
|
||
return { label, status, failures, exitCode, durationMs };
|
||
}
|
||
|
||
/**
|
||
* Extract meaningful failure lines from combined stdout+stderr.
|
||
* Heuristics per step type (cucumber scenario names, FAIL lines, etc.).
|
||
*/
|
||
function extractFailures(output: string, label: string): string[] {
|
||
const lines = output.split("\n");
|
||
const failures: string[] = [];
|
||
|
||
if (label.includes("polyfill:unit")) {
|
||
// bun test output: lines starting with "✗" or "FAIL" or "fail"
|
||
for (const line of lines) {
|
||
const l = line.trim();
|
||
if (/^(✗|✕|FAIL|fail)\s/.test(l) || l.includes("tests failed")) {
|
||
failures.push(l);
|
||
}
|
||
}
|
||
// Also capture summary line "N passed, M failed"
|
||
const summary = lines.find(
|
||
(l) => l.includes("passed") && l.includes("failed"),
|
||
);
|
||
if (summary) failures.push(summary.trim());
|
||
} else if (label.includes("polyfill:e2e")) {
|
||
// e2e/run.ts output: lines starting with " [FAIL]"
|
||
for (const line of lines) {
|
||
const l = line.trim();
|
||
if (l.startsWith("[FAIL]")) failures.push(l);
|
||
}
|
||
// Summary: "N passed / M failed" style
|
||
const summary = lines.find(
|
||
(l) => l.includes("passed") || l.includes("failed"),
|
||
);
|
||
if (summary && !failures.includes(summary.trim()))
|
||
failures.push(summary.trim());
|
||
} else {
|
||
// Cucumber steps: look for "✗" scenario lines, "FAILED" scenario names,
|
||
// or lines beginning with "✖" / "×" / "Scenario:" after a failure tag
|
||
for (const line of lines) {
|
||
const l = line.trim();
|
||
if (
|
||
/^(✗|✕|×|✖)\s/.test(l) ||
|
||
l.startsWith("✘") ||
|
||
l.includes("# Scénario:") ||
|
||
l.includes("# Scenario:") ||
|
||
(l.startsWith("F") && l.length === 1) // progress-bar failure tick
|
||
) {
|
||
if (l.length > 1) failures.push(l);
|
||
}
|
||
}
|
||
// Cucumber "N scenarios (M failed)" summary
|
||
const summary = lines.find((l) =>
|
||
/\d+ scenarios?.*(failed|undefined)/.test(l),
|
||
);
|
||
if (summary) failures.push(summary.trim());
|
||
// Individual scenario fail lines: "✗ Scenario name (features/...)"
|
||
for (const line of lines) {
|
||
const l = line.trim();
|
||
if (l.startsWith("✗") || l.startsWith("✕")) {
|
||
if (!failures.includes(l)) failures.push(l);
|
||
}
|
||
}
|
||
}
|
||
|
||
return failures.filter(Boolean);
|
||
}
|
||
|
||
function fmtDuration(ms: number): string {
|
||
if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`;
|
||
const m = Math.floor(ms / 60_000);
|
||
const s = ((ms % 60_000) / 1000).toFixed(0);
|
||
return `${m}m${s}s`;
|
||
}
|
||
|
||
function printMatrix(
|
||
steps: StepResult[],
|
||
wipResult: StepResult | null,
|
||
): void {
|
||
console.log("\n");
|
||
console.log("╔══════════════════════════════════════════════════════════╗");
|
||
console.log("║ VALIDATION MATRIX ║");
|
||
console.log("╚══════════════════════════════════════════════════════════╝");
|
||
console.log("");
|
||
|
||
const maxLabel = Math.max(...steps.map((s) => s.label.length));
|
||
|
||
for (const step of steps) {
|
||
const icon = step.status === "passed" ? "✅" : step.status === "failed" ? "❌" : "⚠️ ";
|
||
const pad = step.label.padEnd(maxLabel + 2);
|
||
console.log(` ${icon} ${pad} [${fmtDuration(step.durationMs)}]`);
|
||
for (const f of step.failures) {
|
||
console.log(` ↳ ${f}`);
|
||
}
|
||
}
|
||
|
||
if (wipResult) {
|
||
console.log("");
|
||
console.log(" ── @wip (informational, non-blocking) ──────────────────");
|
||
const icon =
|
||
wipResult.status === "passed"
|
||
? "✅"
|
||
: wipResult.status === "failed"
|
||
? "❌"
|
||
: "⚠️ ";
|
||
const pad = wipResult.label.padEnd(maxLabel + 2);
|
||
console.log(` ${icon} ${pad} [${fmtDuration(wipResult.durationMs)}]`);
|
||
for (const f of wipResult.failures) {
|
||
console.log(` ↳ ${f}`);
|
||
}
|
||
}
|
||
|
||
console.log("");
|
||
const allPassed = steps.every((s) => s.status === "passed");
|
||
const totalMs = steps.reduce((sum, s) => sum + s.durationMs, 0) +
|
||
(wipResult?.durationMs ?? 0);
|
||
if (allPassed) {
|
||
console.log(" 🟢 ALL STEPS PASSED");
|
||
} else {
|
||
const failed = steps.filter((s) => s.status !== "passed");
|
||
console.log(` 🔴 ${failed.length} STEP(S) FAILED: ${failed.map((s) => s.label).join(", ")}`);
|
||
}
|
||
console.log(` ⏱ Total: ${fmtDuration(totalMs)}`);
|
||
console.log("");
|
||
}
|
||
|
||
// ─── Cucumber command builder ───────────────────────────────────────────────
|
||
|
||
function cucumberArgs(tags: string): string[] {
|
||
return [
|
||
"--import",
|
||
"tsx/esm",
|
||
"node_modules/.bin/cucumber-js",
|
||
"--config",
|
||
"cucumber.json",
|
||
"--tags",
|
||
tags,
|
||
];
|
||
}
|
||
|
||
// ─── Main ──────────────────────────────────────────────────────────────────
|
||
|
||
async function main(): Promise<void> {
|
||
console.log("🔍 Festipod — Full Validation Run");
|
||
console.log(` Festipod: ${FESTIPOD_DIR}`);
|
||
console.log(` Polyfill: ${POLYFILL_DIR}`);
|
||
console.log("");
|
||
|
||
// ── Profile rotation ─────────────────────────────────────────────────────
|
||
console.log("── Profile rotation check ──────────────────────────────────");
|
||
rotateProfile(FESTIPOD_PROFILE, "festipod");
|
||
rotateProfile(POLYFILL_PROFILE, "polyfill-lib");
|
||
|
||
const steps: StepResult[] = [];
|
||
|
||
// ── (a) Polyfill unit tests ───────────────────────────────────────────────
|
||
steps.push(
|
||
runStep("polyfill:unit", "bun", ["test"], POLYFILL_DIR),
|
||
);
|
||
|
||
// ── (b) Polyfill e2e real broker ──────────────────────────────────────────
|
||
steps.push(
|
||
runStep(
|
||
"polyfill:e2e",
|
||
"bun",
|
||
["run", "e2e/run.ts"],
|
||
POLYFILL_DIR,
|
||
),
|
||
);
|
||
|
||
// ── (c) Festipod @data ────────────────────────────────────────────────────
|
||
steps.push(
|
||
runStep(
|
||
"festipod:@data",
|
||
"node",
|
||
cucumberArgs("@data"),
|
||
FESTIPOD_DIR,
|
||
),
|
||
);
|
||
|
||
// ── (d) Festipod @multibrowser ────────────────────────────────────────────
|
||
steps.push(
|
||
runStep(
|
||
"festipod:@multibrowser",
|
||
"node",
|
||
cucumberArgs("@multibrowser"),
|
||
FESTIPOD_DIR,
|
||
),
|
||
);
|
||
|
||
// ── (e) Festipod @wip [informational] ────────────────────────────────────
|
||
console.log("\n── @wip informational pass (non-blocking) ──────────────────");
|
||
const wipResult = runStep(
|
||
"festipod:@wip",
|
||
"node",
|
||
cucumberArgs("@wip"),
|
||
FESTIPOD_DIR,
|
||
);
|
||
|
||
// ── Matrix ────────────────────────────────────────────────────────────────
|
||
printMatrix(steps, wipResult);
|
||
|
||
// ── Exit code ─────────────────────────────────────────────────────────────
|
||
const anyFailed = steps.some((s) => s.status !== "passed");
|
||
process.exit(anyFailed ? 1 : 0);
|
||
}
|
||
|
||
main().catch((e) => {
|
||
console.error("validate.ts: unhandled error:", e);
|
||
process.exit(1);
|
||
});
|