ng-eventually/packages/ng-e2e-helpers/src/report.ts

/**
 * What a run REPORTS — a constant number of rows, whatever fails.
 *
 * ── The arithmetic is the point ──────────────────────────────────────────────
 * A suite whose check TOTAL is a function of how far it got cannot be compared with itself. A
 * journey that dies halfway takes its unreported checks with it and simply never mentions
 * them: three runs of the same suite reported 24, 26 and 27 checks (VERIFIED 2026-08-16), and
 * a moving total compares nothing. Worse, the checks that vanish are the ones nobody looks
 * for — silence reads as absence, not as failure. A shrinking total even reads like a
 * SMALLER problem instead of a bigger one.
 *
 * So every check is DECLARED before anything can fail. Read off the declaration, the
 * arithmetic survives any death: every journey contributes exactly `checks.length + 1` rows
 * whatever happens to it, including journeys that never ran because the setup died first. A
 * difference between two runs is then always a real difference.
 *
 * The declaration doubles as the suite's table of contents, which is the other reason to keep
 * it whole and in execution order.
 *
 * The related trap that made this self-perpetuating once: the checks were declared inside each
 * journey, so a run that died in the SETUP — before any journey — printed `fatal:` and left,
 * with no summary and nothing a previous run could be compared to.
 */

import { firstLine, within } from "./deadline";
import { printTimings, timingsWanted } from "./measure";

/** One journey and every check it reports. Declared up front; never assembled at run time. */
export interface JourneyDeclaration {
  readonly name: string;
  readonly checks: readonly string[];
}

/** Why a journey cannot start, or `null` when it can. */
export type Prerequisite = () => Promise<string | null> | (string | null);

export interface JourneySpec {
  /** Must name a declared journey, which is where its checks are declared. */
  readonly name: string;
  /**
   * What this journey needs from the ones before it. A prerequisite that is provably dead is
   * reported as such INSTEAD of being driven — not to spare the journey, but because driving
   * a closed page answers with "Target page, context or browser has been closed", a verdict
   * that names the innocent operation and hides the journey that actually broke.
   */
  readonly needs?: readonly Prerequisite[];
  readonly run: () => Promise<void>;
}

export interface SuiteOptions {
  /** Names the suite in its summary line, e.g. "Application e2e". */
  readonly label: string;
  /** Every journey, in execution order, with its checks. */
  readonly journeys: readonly JourneyDeclaration[];
  /** The bound on ONE journey — what catches a journey that never returns. */
  readonly journeyBound: number;
  /**
   * Asked on a journey's failure: is there a KNOWN failure mode to name instead of the
   * operation that happened to be in flight? Typically `() => browserTrouble(label, ctx)`.
   * Its answer is put in front of the journey's reason, never in place of it.
   */
  readonly diagnose?: () => Promise<string | null>;
}

export interface SuiteReport {
  /** Report a declared check. Throws if the name is not one the journey declared. */
  check(name: string, ok: boolean, detail?: string): void;
  /** Run one journey: bounded, isolated, unable to change the shape of the report. */
  journey(spec: JourneySpec): Promise<void>;
  /** Report everything this run did not get to, print the summary, and leave. */
  finish(fatal: string | null): never;
}

interface Check {
  name: string;
  ok: boolean;
  detail?: string;
}

/**
 * Build the reporting for a suite from its declaration.
 *
 * The returned functions do not use `this`, so a caller may destructure them
 * (`const { check, journey, finish } = declareSuite(...)`) and read like a test file.
 */
export function declareSuite(options: SuiteOptions): SuiteReport {
  const results: Check[] = [];
  /** The journeys already reported, so `finish` knows what is missing. */
  const reported = new Set<string>();
  /** The checks the journey in flight has DECLARED and not yet reported — `null` between
   *  journeys, which is what makes a stray report detectable. */
  let outstanding: Set<string> | null = null;
  const startedAt = Date.now();

  const record = (name: string, ok: boolean, detail?: string): void => {
    results.push({ name, ok, detail });
    console.log(`  [${ok ? "PASS" : "FAIL"}] ${name}${detail === undefined ? "" : " — " + detail}`);
  };

  /**
   * Both rules — the name must be declared, and each may be reported once — are enforced by
   * throwing rather than by tolerating, because either violation silently breaks the
   * arithmetic the declaration exists to fix. An undeclared name adds a row no other run has;
   * a repeated one consumes a row that then reads as "not reached". A throw here fails the
   * journey it happens in and says exactly what is wrong with it, which is a harness bug
   * reported the same way as any other failure.
   */
  const check = (name: string, ok: boolean, detail?: string): void => {
    if (outstanding === null) {
      throw new Error(`[${options.label}] the check ${JSON.stringify(name)} was reported outside any journey`);
    }
    if (!outstanding.delete(name)) {
      throw new Error(
        `[${options.label}] the check ${JSON.stringify(name)} was reported but its journey does not ` +
          "declare it (or declares it once and reports it twice) — fix the journey's `checks` list",
      );
    }
    record(name, ok, detail);
  };

  /**
   * ── What "isolated" buys, and what it does not ──────────────────────────────
   * It does NOT mean a failure is absorbed — a contained failure is still a failure and is
   * still counted, here as every one of the journey's declared checks plus the "ran to the
   * end" row. What it means is that the journey's failure cannot take the FOLLOWING journeys'
   * checks off the report, cannot leave them reporting a timeout that names the wrong suspect,
   * and cannot end the run before its summary.
   *
   * The bound is what makes the catch honest: catching everything and recording a FAIL is
   * right for a journey that fails, but a journey that never RETURNS is caught by nothing —
   * and that is what three killed runs looked like from the outside.
   *
   * The last row, `ran to the end`, is not decoration either. Without it a journey that throws
   * AFTER reporting its last check would report no failure at all, since there would be no
   * unreached check left to carry the reason.
   */
  const journey = async (spec: JourneySpec): Promise<void> => {
    console.log(`\n── ${spec.name} ──`);
    const planned = options.journeys.find((j) => j.name === spec.name);
    if (planned === undefined) {
      throw new Error(
        `[${options.label}] the journey ${JSON.stringify(spec.name)} is not declared — add it, or fix the name`,
      );
    }
    const declared = new Set(planned.checks);
    if (declared.size !== planned.checks.length) {
      throw new Error(`[${options.label}] the same check is declared twice under "${spec.name}"`);
    }
    reported.add(spec.name);
    const journeyStartedAt = Date.now();
    let why: string | null = null;

    const blocked = (await Promise.all((spec.needs ?? []).map(async (needed) => needed()))).filter(
      (r): r is string => r !== null,
    );
    if (blocked.length > 0) {
      why = `it could not start: ${blocked.join("; ")}`;
      console.error(`  [blocked] ${why}`);
    } else {
      outstanding = declared;
      try {
        await within(`the journey "${spec.name}"`, options.journeyBound, spec.run);
      } catch (e) {
        why = firstLine(e);
        // In full, and to stderr: the one-liner above is what the report carries, and it is
        // never the whole of a driver's call log or a broker crossing's trail.
        console.error(`  [threw] ${String((e as Error)?.stack ?? e)}`);
        // A known failure mode goes IN FRONT of the reason, never in place of it: the
        // operation in flight is still worth having, it is just not the cause.
        if (options.diagnose !== undefined) {
          const known = await options.diagnose().catch(() => null);
          if (known !== null) why = `${known} — the operation it died on: ${why}`;
        }
      } finally {
        outstanding = null;
      }
    }

    for (const name of declared) {
      record(name, false, why === null ? "the journey ended without reporting it" : `not reached — ${why}`);
    }
    record(
      `the journey "${spec.name}" ran to the end`,
      why === null,
      why ?? `${((Date.now() - journeyStartedAt) / 1000).toFixed(1)}s`,
    );
  };

  /**
   * The journeys that never ran are read off the declaration, so a run that died in its setup
   * reports exactly the same number of checks as one that finished — all of them failed, and
   * each saying why. "24 checks" and "27 checks" are not two results of the same suite; they
   * are two different suites, and comparing them quietly compares nothing.
   */
  const finish = (fatal: string | null): never => {
    for (const planned of options.journeys) {
      if (reported.has(planned.name)) continue;
      const why = fatal === null ? "the suite ended before this journey ran" : `the suite died first: ${fatal}`;
      for (const name of planned.checks) record(name, false, `not reached — ${why}`);
      record(`the journey "${planned.name}" ran to the end`, false, why);
    }

    // The measurement every bound is sized from, on request. Printed BEFORE the summary so the
    // summary stays the last line — which is what a reader and a `tail` look at.
    if (timingsWanted()) printTimings();

    const failed = results.filter((r) => !r.ok);
    if (failed.length > 0) {
      console.log("\n── what failed ──");
      for (const r of failed) console.log(`  ${r.name}${r.detail === undefined ? "" : " — " + r.detail}`);
    }
    const minutes = ((Date.now() - startedAt) / 60000).toFixed(1);
    console.log(
      `\n══ ${options.label} summary: ${results.length - failed.length} passed, ${failed.length} failed, ` +
        `${results.length} total — ${minutes} min ══`,
    );
    process.exit(failed.length === 0 ? 0 : 1);
  };

  return { check, journey, finish };
}