/** * What a run REPORTS — a constant number of rows, whatever fails. * * ── The arithmetic is the point ────────────────────────────────────────────── * A suite whose check TOTAL is a function of how far it got cannot be compared with itself. A * journey that dies halfway takes its unreported checks with it and simply never mentions * them: three runs of the same suite reported 24, 26 and 27 checks (VERIFIED 2026-08-16), and * a moving total compares nothing. Worse, the checks that vanish are the ones nobody looks * for — silence reads as absence, not as failure. A shrinking total even reads like a * SMALLER problem instead of a bigger one. * * So every check is DECLARED before anything can fail. Read off the declaration, the * arithmetic survives any death: every journey contributes exactly `checks.length + 1` rows * whatever happens to it, including journeys that never ran because the setup died first. A * difference between two runs is then always a real difference. * * The declaration doubles as the suite's table of contents, which is the other reason to keep * it whole and in execution order. * * The related trap that made this self-perpetuating once: the checks were declared inside each * journey, so a run that died in the SETUP — before any journey — printed `fatal:` and left, * with no summary and nothing a previous run could be compared to. */ import { firstLine, within } from "./deadline"; import { printTimings, timingsWanted } from "./measure"; /** One journey and every check it reports. Declared up front; never assembled at run time. */ export interface JourneyDeclaration { readonly name: string; readonly checks: readonly string[]; } /** Why a journey cannot start, or `null` when it can. */ export type Prerequisite = () => Promise | (string | null); export interface JourneySpec { /** Must name a declared journey, which is where its checks are declared. */ readonly name: string; /** * What this journey needs from the ones before it. A prerequisite that is provably dead is * reported as such INSTEAD of being driven — not to spare the journey, but because driving * a closed page answers with "Target page, context or browser has been closed", a verdict * that names the innocent operation and hides the journey that actually broke. */ readonly needs?: readonly Prerequisite[]; readonly run: () => Promise; } export interface SuiteOptions { /** Names the suite in its summary line, e.g. "Application e2e". */ readonly label: string; /** Every journey, in execution order, with its checks. */ readonly journeys: readonly JourneyDeclaration[]; /** The bound on ONE journey — what catches a journey that never returns. */ readonly journeyBound: number; /** * Asked on a journey's failure: is there a KNOWN failure mode to name instead of the * operation that happened to be in flight? Typically `() => browserTrouble(label, ctx)`. * Its answer is put in front of the journey's reason, never in place of it. */ readonly diagnose?: () => Promise; } export interface SuiteReport { /** Report a declared check. Throws if the name is not one the journey declared. */ check(name: string, ok: boolean, detail?: string): void; /** Run one journey: bounded, isolated, unable to change the shape of the report. */ journey(spec: JourneySpec): Promise; /** Report everything this run did not get to, print the summary, and leave. */ finish(fatal: string | null): never; } interface Check { name: string; ok: boolean; detail?: string; } /** * Build the reporting for a suite from its declaration. * * The returned functions do not use `this`, so a caller may destructure them * (`const { check, journey, finish } = declareSuite(...)`) and read like a test file. */ export function declareSuite(options: SuiteOptions): SuiteReport { const results: Check[] = []; /** The journeys already reported, so `finish` knows what is missing. */ const reported = new Set(); /** The checks the journey in flight has DECLARED and not yet reported — `null` between * journeys, which is what makes a stray report detectable. */ let outstanding: Set | null = null; const startedAt = Date.now(); const record = (name: string, ok: boolean, detail?: string): void => { results.push({ name, ok, detail }); console.log(` [${ok ? "PASS" : "FAIL"}] ${name}${detail === undefined ? "" : " — " + detail}`); }; /** * Both rules — the name must be declared, and each may be reported once — are enforced by * throwing rather than by tolerating, because either violation silently breaks the * arithmetic the declaration exists to fix. An undeclared name adds a row no other run has; * a repeated one consumes a row that then reads as "not reached". A throw here fails the * journey it happens in and says exactly what is wrong with it, which is a harness bug * reported the same way as any other failure. */ const check = (name: string, ok: boolean, detail?: string): void => { if (outstanding === null) { throw new Error(`[${options.label}] the check ${JSON.stringify(name)} was reported outside any journey`); } if (!outstanding.delete(name)) { throw new Error( `[${options.label}] the check ${JSON.stringify(name)} was reported but its journey does not ` + "declare it (or declares it once and reports it twice) — fix the journey's `checks` list", ); } record(name, ok, detail); }; /** * ── What "isolated" buys, and what it does not ────────────────────────────── * It does NOT mean a failure is absorbed — a contained failure is still a failure and is * still counted, here as every one of the journey's declared checks plus the "ran to the * end" row. What it means is that the journey's failure cannot take the FOLLOWING journeys' * checks off the report, cannot leave them reporting a timeout that names the wrong suspect, * and cannot end the run before its summary. * * The bound is what makes the catch honest: catching everything and recording a FAIL is * right for a journey that fails, but a journey that never RETURNS is caught by nothing — * and that is what three killed runs looked like from the outside. * * The last row, `ran to the end`, is not decoration either. Without it a journey that throws * AFTER reporting its last check would report no failure at all, since there would be no * unreached check left to carry the reason. */ const journey = async (spec: JourneySpec): Promise => { console.log(`\n── ${spec.name} ──`); const planned = options.journeys.find((j) => j.name === spec.name); if (planned === undefined) { throw new Error( `[${options.label}] the journey ${JSON.stringify(spec.name)} is not declared — add it, or fix the name`, ); } const declared = new Set(planned.checks); if (declared.size !== planned.checks.length) { throw new Error(`[${options.label}] the same check is declared twice under "${spec.name}"`); } reported.add(spec.name); const journeyStartedAt = Date.now(); let why: string | null = null; const blocked = (await Promise.all((spec.needs ?? []).map(async (needed) => needed()))).filter( (r): r is string => r !== null, ); if (blocked.length > 0) { why = `it could not start: ${blocked.join("; ")}`; console.error(` [blocked] ${why}`); } else { outstanding = declared; try { await within(`the journey "${spec.name}"`, options.journeyBound, spec.run); } catch (e) { why = firstLine(e); // In full, and to stderr: the one-liner above is what the report carries, and it is // never the whole of a driver's call log or a broker crossing's trail. console.error(` [threw] ${String((e as Error)?.stack ?? e)}`); // A known failure mode goes IN FRONT of the reason, never in place of it: the // operation in flight is still worth having, it is just not the cause. if (options.diagnose !== undefined) { const known = await options.diagnose().catch(() => null); if (known !== null) why = `${known} — the operation it died on: ${why}`; } } finally { outstanding = null; } } for (const name of declared) { record(name, false, why === null ? "the journey ended without reporting it" : `not reached — ${why}`); } record( `the journey "${spec.name}" ran to the end`, why === null, why ?? `${((Date.now() - journeyStartedAt) / 1000).toFixed(1)}s`, ); }; /** * The journeys that never ran are read off the declaration, so a run that died in its setup * reports exactly the same number of checks as one that finished — all of them failed, and * each saying why. "24 checks" and "27 checks" are not two results of the same suite; they * are two different suites, and comparing them quietly compares nothing. */ const finish = (fatal: string | null): never => { for (const planned of options.journeys) { if (reported.has(planned.name)) continue; const why = fatal === null ? "the suite ended before this journey ran" : `the suite died first: ${fatal}`; for (const name of planned.checks) record(name, false, `not reached — ${why}`); record(`the journey "${planned.name}" ran to the end`, false, why); } // The measurement every bound is sized from, on request. Printed BEFORE the summary so the // summary stays the last line — which is what a reader and a `tail` look at. if (timingsWanted()) printTimings(); const failed = results.filter((r) => !r.ok); if (failed.length > 0) { console.log("\n── what failed ──"); for (const r of failed) console.log(` ${r.name}${r.detail === undefined ? "" : " — " + r.detail}`); } const minutes = ((Date.now() - startedAt) / 60000).toFixed(1); console.log( `\n══ ${options.label} summary: ${results.length - failed.length} passed, ${failed.length} failed, ` + `${results.length} total — ${minutes} min ══`, ); process.exit(failed.length === 0 ? 0 : 1); }; return { check, journey, finish }; }