tooling: commande bun run validate — validation complète 2 niveaux
Fondation « l'agent valide tout d'un coup » (audit couverture, levier C1). Enchaîne et agrège : polyfill unit + polyfill e2e réel + Festipod @data + @multibrowser (profils frais/rotation), matrice finale + exit non-zéro si rouge, passe @wip informative. Baseline établi (premier run réel) : polyfill unit 120 ✅, polyfill e2e 42/42 ✅ (broker réel), @multibrowser 1 rouge (limite wallet-partagé A/B), @data 4 rouges dont 2 régressions phase B confirmées. → a trouvé les problèmes tout seul. À TUNER (suivi) : le budget @data (~14min) est trop court → la suite complète est tuée par timeout ; augmenter le budget OU exécuter un sous-ensemble clé rapide. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sylvain Duchesne
@@ -0,0 +1,357 @@
|
||||
#!/usr/bin/env bun
|
||||
/**
|
||||
* validate.ts — Full validation matrix (broker-level tests only, no @ui/@e2e).
|
||||
*
|
||||
* Runs in sequence:
|
||||
* (a) Polyfill unit tests (@ng-eventually/client — bun test)
|
||||
* (b) Polyfill e2e real-broker (@ng-eventually/client — bun run test:e2e)
|
||||
* (c) Festipod @data (cucumber --tags @data)
|
||||
* (d) Festipod @multibrowser (cucumber --tags @multibrowser)
|
||||
* (e) Festipod @wip [informational only, non-blocking]
|
||||
*
|
||||
* Each step runs even if the previous one failed (--bail mode is OFF).
|
||||
* Exit code is non-zero if any non-informational step has failures.
|
||||
*
|
||||
* Profile rotation: both Playwright profiles are rotated before the run
|
||||
* when their size exceeds BLOAT_THRESHOLD_MB (default 50 MB), to avoid
|
||||
* the sparql_query hang described in caveat_wallet-bloat-hang.
|
||||
*/
|
||||
|
||||
import { spawnSync, execSync } from "child_process";
|
||||
import * as fs from "fs";
|
||||
import * as path from "path";
|
||||
|
||||
// ─── Config ────────────────────────────────────────────────────────────────
|
||||
|
||||
const FESTIPOD_DIR = "/home/sylvain/projects/festipod/festipod";
|
||||
const POLYFILL_DIR =
|
||||
"/home/sylvain/projects/nextgraph/ng-eventually-js/packages/client";
|
||||
|
||||
const FESTIPOD_PROFILE = path.join(FESTIPOD_DIR, ".playwright-profile");
|
||||
const POLYFILL_PROFILE = path.join(
|
||||
POLYFILL_DIR,
|
||||
"e2e",
|
||||
".playwright-profile-lib",
|
||||
);
|
||||
|
||||
/** Rotate profile when it exceeds this many MB (caveat_wallet-bloat-hang). */
|
||||
const BLOAT_THRESHOLD_MB = 50;
|
||||
|
||||
/** Per-step timeout in ms — generous for broker + headless wallet creation. */
|
||||
const STEP_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
|
||||
|
||||
// ─── Helpers ───────────────────────────────────────────────────────────────
|
||||
|
||||
function dirSizeMB(dir: string): number {
|
||||
if (!fs.existsSync(dir)) return 0;
|
||||
try {
|
||||
const result = spawnSync("du", ["-sm", dir], { encoding: "utf-8" });
|
||||
const line = result.stdout.trim().split("\n")[0] ?? "";
|
||||
return parseInt(line.split("\t")[0] ?? "0", 10);
|
||||
} catch {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function rotateProfile(profilePath: string, label: string): void {
|
||||
const sizeMB = dirSizeMB(profilePath);
|
||||
if (sizeMB > BLOAT_THRESHOLD_MB) {
|
||||
console.log(
|
||||
`[rotate] ${label}: ${sizeMB}MB > ${BLOAT_THRESHOLD_MB}MB — rotating profile...`,
|
||||
);
|
||||
try {
|
||||
fs.rmSync(profilePath, { recursive: true, force: true });
|
||||
console.log(`[rotate] ${label}: profile removed. Will be recreated.`);
|
||||
} catch (e) {
|
||||
console.warn(`[rotate] ${label}: failed to remove profile: ${e}`);
|
||||
}
|
||||
} else {
|
||||
// Even if we keep the profile, remove any stale SingletonLock left by a
|
||||
// previous crashed run — Chromium refuses to launch if the lock exists.
|
||||
const lockPath = path.join(profilePath, "SingletonLock");
|
||||
if (fs.existsSync(lockPath)) {
|
||||
try {
|
||||
fs.rmSync(lockPath, { force: true });
|
||||
console.log(`[rotate] ${label}: removed stale SingletonLock.`);
|
||||
} catch {
|
||||
// Non-fatal: if we can't remove it, launch will fail with a clear error
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
`[rotate] ${label}: ${sizeMB}MB — below threshold, keeping profile.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
interface StepResult {
|
||||
label: string;
|
||||
status: "passed" | "failed" | "error";
|
||||
/** Lines to show in the summary (failed scenario names, FAIL lines, etc.) */
|
||||
failures: string[];
|
||||
/** Raw exit code */
|
||||
exitCode: number;
|
||||
durationMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a command and capture its output. Returns the result with parsed
|
||||
* pass/fail summary. Never throws — all errors are captured in StepResult.
|
||||
*/
|
||||
function runStep(
|
||||
label: string,
|
||||
cmd: string,
|
||||
args: string[],
|
||||
cwd: string,
|
||||
extraEnv: Record<string, string> = {},
|
||||
): StepResult {
|
||||
const t0 = Date.now();
|
||||
console.log(`\n${"═".repeat(60)}`);
|
||||
console.log(`▶ ${label}`);
|
||||
console.log(` ${cmd} ${args.join(" ")} (cwd: ${cwd})`);
|
||||
console.log(`${"═".repeat(60)}`);
|
||||
|
||||
const env = { ...process.env, ...extraEnv };
|
||||
|
||||
const result = spawnSync(cmd, args, {
|
||||
cwd,
|
||||
env,
|
||||
encoding: "utf-8",
|
||||
timeout: STEP_TIMEOUT_MS,
|
||||
maxBuffer: 20 * 1024 * 1024, // 20MB
|
||||
});
|
||||
|
||||
const durationMs = Date.now() - t0;
|
||||
const stdout = result.stdout ?? "";
|
||||
const stderr = result.stderr ?? "";
|
||||
const combined = stdout + "\n" + stderr;
|
||||
|
||||
// Print output in real-time equivalent (post-hoc since spawnSync)
|
||||
if (stdout) process.stdout.write(stdout);
|
||||
if (stderr) process.stderr.write(stderr);
|
||||
|
||||
if (result.error) {
|
||||
console.error(`[${label}] process error:`, result.error.message);
|
||||
return {
|
||||
label,
|
||||
status: "error",
|
||||
failures: [`Process error: ${result.error.message}`],
|
||||
exitCode: result.status ?? 1,
|
||||
durationMs,
|
||||
};
|
||||
}
|
||||
|
||||
const exitCode = result.status ?? 1;
|
||||
const failures = extractFailures(combined, label);
|
||||
const status = exitCode === 0 ? "passed" : "failed";
|
||||
|
||||
return { label, status, failures, exitCode, durationMs };
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract meaningful failure lines from combined stdout+stderr.
|
||||
* Heuristics per step type (cucumber scenario names, FAIL lines, etc.).
|
||||
*/
|
||||
function extractFailures(output: string, label: string): string[] {
|
||||
const lines = output.split("\n");
|
||||
const failures: string[] = [];
|
||||
|
||||
if (label.includes("polyfill:unit")) {
|
||||
// bun test output: lines starting with "✗" or "FAIL" or "fail"
|
||||
for (const line of lines) {
|
||||
const l = line.trim();
|
||||
if (/^(✗|✕|FAIL|fail)\s/.test(l) || l.includes("tests failed")) {
|
||||
failures.push(l);
|
||||
}
|
||||
}
|
||||
// Also capture summary line "N passed, M failed"
|
||||
const summary = lines.find(
|
||||
(l) => l.includes("passed") && l.includes("failed"),
|
||||
);
|
||||
if (summary) failures.push(summary.trim());
|
||||
} else if (label.includes("polyfill:e2e")) {
|
||||
// e2e/run.ts output: lines starting with " [FAIL]"
|
||||
for (const line of lines) {
|
||||
const l = line.trim();
|
||||
if (l.startsWith("[FAIL]")) failures.push(l);
|
||||
}
|
||||
// Summary: "N passed / M failed" style
|
||||
const summary = lines.find(
|
||||
(l) => l.includes("passed") || l.includes("failed"),
|
||||
);
|
||||
if (summary && !failures.includes(summary.trim()))
|
||||
failures.push(summary.trim());
|
||||
} else {
|
||||
// Cucumber steps: look for "✗" scenario lines, "FAILED" scenario names,
|
||||
// or lines beginning with "✖" / "×" / "Scenario:" after a failure tag
|
||||
for (const line of lines) {
|
||||
const l = line.trim();
|
||||
if (
|
||||
/^(✗|✕|×|✖)\s/.test(l) ||
|
||||
l.startsWith("✘") ||
|
||||
l.includes("# Scénario:") ||
|
||||
l.includes("# Scenario:") ||
|
||||
(l.startsWith("F") && l.length === 1) // progress-bar failure tick
|
||||
) {
|
||||
if (l.length > 1) failures.push(l);
|
||||
}
|
||||
}
|
||||
// Cucumber "N scenarios (M failed)" summary
|
||||
const summary = lines.find((l) =>
|
||||
/\d+ scenarios?.*(failed|undefined)/.test(l),
|
||||
);
|
||||
if (summary) failures.push(summary.trim());
|
||||
// Individual scenario fail lines: "✗ Scenario name (features/...)"
|
||||
for (const line of lines) {
|
||||
const l = line.trim();
|
||||
if (l.startsWith("✗") || l.startsWith("✕")) {
|
||||
if (!failures.includes(l)) failures.push(l);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return failures.filter(Boolean);
|
||||
}
|
||||
|
||||
function fmtDuration(ms: number): string {
|
||||
if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`;
|
||||
const m = Math.floor(ms / 60_000);
|
||||
const s = ((ms % 60_000) / 1000).toFixed(0);
|
||||
return `${m}m${s}s`;
|
||||
}
|
||||
|
||||
function printMatrix(
|
||||
steps: StepResult[],
|
||||
wipResult: StepResult | null,
|
||||
): void {
|
||||
console.log("\n");
|
||||
console.log("╔══════════════════════════════════════════════════════════╗");
|
||||
console.log("║ VALIDATION MATRIX ║");
|
||||
console.log("╚══════════════════════════════════════════════════════════╝");
|
||||
console.log("");
|
||||
|
||||
const maxLabel = Math.max(...steps.map((s) => s.label.length));
|
||||
|
||||
for (const step of steps) {
|
||||
const icon = step.status === "passed" ? "✅" : step.status === "failed" ? "❌" : "⚠️ ";
|
||||
const pad = step.label.padEnd(maxLabel + 2);
|
||||
console.log(` ${icon} ${pad} [${fmtDuration(step.durationMs)}]`);
|
||||
for (const f of step.failures) {
|
||||
console.log(` ↳ ${f}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (wipResult) {
|
||||
console.log("");
|
||||
console.log(" ── @wip (informational, non-blocking) ──────────────────");
|
||||
const icon =
|
||||
wipResult.status === "passed"
|
||||
? "✅"
|
||||
: wipResult.status === "failed"
|
||||
? "❌"
|
||||
: "⚠️ ";
|
||||
const pad = wipResult.label.padEnd(maxLabel + 2);
|
||||
console.log(` ${icon} ${pad} [${fmtDuration(wipResult.durationMs)}]`);
|
||||
for (const f of wipResult.failures) {
|
||||
console.log(` ↳ ${f}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log("");
|
||||
const allPassed = steps.every((s) => s.status === "passed");
|
||||
const totalMs = steps.reduce((sum, s) => sum + s.durationMs, 0) +
|
||||
(wipResult?.durationMs ?? 0);
|
||||
if (allPassed) {
|
||||
console.log(" 🟢 ALL STEPS PASSED");
|
||||
} else {
|
||||
const failed = steps.filter((s) => s.status !== "passed");
|
||||
console.log(` 🔴 ${failed.length} STEP(S) FAILED: ${failed.map((s) => s.label).join(", ")}`);
|
||||
}
|
||||
console.log(` ⏱ Total: ${fmtDuration(totalMs)}`);
|
||||
console.log("");
|
||||
}
|
||||
|
||||
// ─── Cucumber command builder ───────────────────────────────────────────────
|
||||
|
||||
function cucumberArgs(tags: string): string[] {
|
||||
return [
|
||||
"--import",
|
||||
"tsx/esm",
|
||||
"node_modules/.bin/cucumber-js",
|
||||
"--config",
|
||||
"cucumber.json",
|
||||
"--tags",
|
||||
tags,
|
||||
];
|
||||
}
|
||||
|
||||
// ─── Main ──────────────────────────────────────────────────────────────────
|
||||
|
||||
async function main(): Promise<void> {
|
||||
console.log("🔍 Festipod — Full Validation Run");
|
||||
console.log(` Festipod: ${FESTIPOD_DIR}`);
|
||||
console.log(` Polyfill: ${POLYFILL_DIR}`);
|
||||
console.log("");
|
||||
|
||||
// ── Profile rotation ─────────────────────────────────────────────────────
|
||||
console.log("── Profile rotation check ──────────────────────────────────");
|
||||
rotateProfile(FESTIPOD_PROFILE, "festipod");
|
||||
rotateProfile(POLYFILL_PROFILE, "polyfill-lib");
|
||||
|
||||
const steps: StepResult[] = [];
|
||||
|
||||
// ── (a) Polyfill unit tests ───────────────────────────────────────────────
|
||||
steps.push(
|
||||
runStep("polyfill:unit", "bun", ["test"], POLYFILL_DIR),
|
||||
);
|
||||
|
||||
// ── (b) Polyfill e2e real broker ──────────────────────────────────────────
|
||||
steps.push(
|
||||
runStep(
|
||||
"polyfill:e2e",
|
||||
"bun",
|
||||
["run", "e2e/run.ts"],
|
||||
POLYFILL_DIR,
|
||||
),
|
||||
);
|
||||
|
||||
// ── (c) Festipod @data ────────────────────────────────────────────────────
|
||||
steps.push(
|
||||
runStep(
|
||||
"festipod:@data",
|
||||
"node",
|
||||
cucumberArgs("@data"),
|
||||
FESTIPOD_DIR,
|
||||
),
|
||||
);
|
||||
|
||||
// ── (d) Festipod @multibrowser ────────────────────────────────────────────
|
||||
steps.push(
|
||||
runStep(
|
||||
"festipod:@multibrowser",
|
||||
"node",
|
||||
cucumberArgs("@multibrowser"),
|
||||
FESTIPOD_DIR,
|
||||
),
|
||||
);
|
||||
|
||||
// ── (e) Festipod @wip [informational] ────────────────────────────────────
|
||||
console.log("\n── @wip informational pass (non-blocking) ──────────────────");
|
||||
const wipResult = runStep(
|
||||
"festipod:@wip",
|
||||
"node",
|
||||
cucumberArgs("@wip"),
|
||||
FESTIPOD_DIR,
|
||||
);
|
||||
|
||||
// ── Matrix ────────────────────────────────────────────────────────────────
|
||||
printMatrix(steps, wipResult);
|
||||
|
||||
// ── Exit code ─────────────────────────────────────────────────────────────
|
||||
const anyFailed = steps.some((s) => s.status !== "passed");
|
||||
process.exit(anyFailed ? 1 : 0);
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error("validate.ts: unhandled error:", e);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user