ng-helpers/e2e/indexing-app.ts

/**
 * The application the end-to-end suite drives — written the way a consumer of
 * `@ng-helpers/indexing` writes one, and nothing more.
 *
 * Why an application and not a bag of library calls
 * The 80 unit tests in `test/` run against a fake this repository wrote. They prove the
 * indexing RULES are consistent; they cannot prove that NextGraph does what the fake
 * pretends, because the fake is the thing being asked. This page closes that gap by
 * putting the real broker underneath: it imports `@ng-eventually/polyfill` for real,
 * crosses the real broker, and calls `indexing(sessionId)` exactly as an application
 * would.
 *
 * What this application takes from `@ng-helpers/indexing` is its WHOLE published
 * surface: `indexing(sessionId)`, the two acts on what it hands back, and the two IRIs.
 * Nothing here makes an entry of a deposit, and nothing here can ask for one: that is
 * the business of the layer below, and this page is where that shows or does not.
 * Everything else here is the polyfill (`configure`, `ensureIdentity`, `init`, `initNg`,
 * `ng`, `docs`, `useShapeQuery`) -- including reading the index, which is an ordinary
 * anchored `docs.sparqlQuery` of an ordinary document. If something here is awkward, it
 * is awkward for every consumer, which is the second reason to write it this way.
 *
 * This page WIRES THE ORM, because one thing here has no other door: the store enumeration it
 * used to discover a leaked document left the engagement, and what replaces it is a shape
 * query, which reads through NextGraph's own and therefore needs `ormSubscription`,
 * `initNg` and a hand-written shape. An application that hands `configure` a stub gets
 * none of it.
 *
 * The two things here no application does, and why they reach past the surface
 * `createIndexWithBrokenInbox` injects a failure into the inbox step of `create`: the
 * question it answers — does a failed `openInbox` leave a document behind? — cannot be
 * reached from outside, because nothing a caller controls makes a real
 * `openDocumentInbox` fail on demand. And `publishObject` writes through the very
 * primitive `create` declares an index's field with, which is what makes it the CONTROL
 * for the write-form question. Both therefore build the package's INTERNAL port, and
 * both are named for what they are. Everything around them is real, including the
 * broker.
 */

import {
  configure,
  docs,
  ensureIdentity,
  init,
  initNg,
  ng,
  useShapeQuery,
  type Scope,
  type ShapeObservable,
  type ShapeQuery,
  type ShapeType,
} from "@ng-eventually/polyfill";
import { ng as realNg, init as realInit } from "@ng-org/web";
import { RdfOrmSubscription, initNg as ormInitNg } from "@ng-org/orm";

// The whole published surface of the package under test.
import { ENTRY_VALUE, INDEX_FIELD, indexing, type Indexing } from "../src/index";
// NOT published, and reached only by the two probes above — see the header.
import { indexingOn } from "../src/indexing";
import type { NextGraphPort } from "../src/port";
import { polyfillPort } from "../src/polyfill-adapter";
import type {
  BrokenInboxOutcome,
  DocumentSubject,
  IndexRow,
  IndexingBridge,
  SelectOutcome,
} from "./bridge";

// bootstrap: the one polyfill-era call, then the SDK-shaped ones
//
// `sharedWallet` is declared because the access gate wants somewhere to point when it
// has to render, and never used: this suite always enters through the broker's redirect,
// where the wallet is already open in the run's profile. Nothing is served at that path.
//
// THE ORM IS WIRED HERE, and it is not optional dressing: `useShapeQuery` reads through
// NextGraph's own shape query, which arrives through `ormSubscription` and is started by
// `initNg`. Without them that read publishes `isError` naming the field it never got.
// `RdfOrmSubscription.getOrCreate` is passed as it stands -- no wrapper, no cast.
configure({
  ng: realNg,
  // Required by `configure`, and `@ng-org/orm` publishes `useShape` only through its
  // framework adapters (react / svelte / vue). This page has no framework, so there is
  // nothing honest to hand over: it REFUSES instead of answering `undefined`, which a
  // caller could not tell from an empty read-filtered view. Nothing here calls it, and
  // the gap is declared in `usage_ng-helpers`'s `## Frictions`.
  useShape: () => {
    throw new Error(
      "[e2e] useShape was never wired: this page has no framework adapter to give it, " +
        "and it reads through docs.sparqlQuery and useShapeQuery instead",
    );
  },
  ormSubscription: RdfOrmSubscription.getOrCreate,
  init: realInit,
  initNg: ormInitNg,
  sharedWallet: { fileUrl: "/wallet-never-served.ngw", password: "" },
});

// The library's `init`, not the injected one: it settles the identity BEFORE handing the
// page to the broker, so the round-trip leaves with `?ng-id=` in the address it carries.
// The callback is this application's own business — it keeps the session because
// `indexing(sessionId)` takes one, exactly as the real SDK's primitives do.
const sessionReady = new Promise<{ session_id: string }>((resolve) => {
  init(
    (event: { status: string; session?: { session_id: string } }) => {
      if (event.status === "loggedin" && event.session) resolve(event.session);
    },
    true,
    [],
  );
});

// this application's state

const state: { status: string; error: string | null; who: string } = {
  status: "connecting",
  error: null,
  who: "",
};

let api: Indexing | null = null;

/** The index this deployment contributes to, read off its own configuration. */
function configuredIndex(): string | null {
  return new URLSearchParams(window.location.search).get("index");
}

async function boot(): Promise<void> {
  // One await, and it covers everything: the identity settles, the connection work runs,
  // and the identity comes back. The application keeps it only to show it.
  state.who = await ensureIdentity();
  const session = await sessionReady;
  // The ORM is started once the session exists and not before: `useShapeQuery` reads
  // through it, and this package's own `initNg` is the one to CALL -- not the SDK's,
  // which was handed to `configure` and is reached from there. The `ng` it is given is
  // this package's too, the one the surface says to call instead of the one `configure`
  // was passed. The engagement states the argument is discarded either way: the call
  // substitutes that same published `ng`.
  initNg(ng, session);
  // The session id, and nothing else — this application never names a port, and never
  // awaits anything here: a handle reaches nothing. A reference this application hands
  // an index becomes an entry because the layer below processes that index's inbox,
  // and there is nothing to call, schedule or configure for it.
  api = indexing(session.session_id);
  state.status = "ready";
}

void boot().catch((e: unknown) => {
  state.status = "failed";
  state.error = String((e as Error)?.message ?? e);
});

/** The library, once the page is up. Throws with the boot's own reason if it is not. */
function ready(): Indexing {
  if (api === null) {
    throw new Error(`[e2e] the application is not ready (${state.status}): ${state.error ?? "still connecting"}`);
  }
  return api;
}

/**
 * The package's INTERNAL port, for the two probes that need one. Never used by an act
 * this application performs as an application — see the header for why each needs it.
 */
async function probePort(): Promise<NextGraphPort> {
  const session = await sessionReady;
  return polyfillPort({ sessionId: session.session_id });
}

/**
 * THE SHAPE AN INDEX DOCUMENT MATCHES, hand-written here because there is no other way
 * to ask this question any more.
 *
 * `storeRegistry.listMyEntityDocs` left the engagement, and what replaces it is the read
 * itself: a shape over `{ graphs: "did:ng:i" }` brings back what this identity HOLDS of
 * that shape. So the descriptor `create` writes -- one triple, on the index document's
 * own subject -- is what makes an index findable, and it is declared here as the one
 * predicate this shape names. A predicate a shape does not name is not projected, so
 * `field` is both the match and the whole of what comes back.
 *
 * NOT A STORE LISTING, and the difference is recorded rather than glossed: `Scope` has
 * two axes, neither of them a store. This finds an index document wherever it sits, and
 * says nothing about which of the three stores holds it.
 */
type IndexDescriptor = {
  "@id": string;
  "@graph": string;
  field: string;
};

const INDEX_SHAPE = "urn:ng-helpers-e2e:shape:index";

const IndexShape: ShapeType<IndexDescriptor> = {
  shape: INDEX_SHAPE,
  schema: {
    [INDEX_SHAPE]: {
      iri: INDEX_SHAPE,
      predicates: [
        {
          iri: INDEX_FIELD,
          readablePredicate: "field",
          dataTypes: [{ valType: "string" }],
          minCardinality: 1,
          maxCardinality: 1,
        },
      ],
    },
  },
};

/** Everything this identity holds, which is as close to "my documents" as `Scope` goes. */
const EVERYTHING_HELD: Scope = { graphs: "did:ng:i" };

/**
 * Which of the three states a reactive read is in.
 *
 * Read BEFORE the data, always: an empty `data` under `isSuccess` means this scope holds
 * nothing of that shape, and an empty `data` under anything else means nobody answered.
 * The two must never be reported as one, which is the whole reason this is carried out
 * to the report instead of being collapsed into an empty list.
 */
type LoadState = "pending" | "success" | "error";

function loadStateOf(snapshot: ShapeQuery<IndexDescriptor>): LoadState {
  if (snapshot.isError) return "error";
  if (snapshot.isSuccess) return "success";
  return "pending";
}

/** The documents the snapshot's objects came out of -- one entry per index document. */
function documentsIn(snapshot: ShapeQuery<IndexDescriptor>): string[] {
  return snapshot.data.map((descriptor) => descriptor["@graph"]);
}

/**
 * Wait for the read to ANSWER -- to reach `isSuccess` or `isError`.
 *
 * A shape query the engine refuses stays `isPending` for ever: NextGraph catches that
 * failure, logs it, and its readiness signal is only ever resolved. So a bound here is
 * not impatience, it is the only way that failure becomes a named observation instead of
 * a hang, and what it returns is the state it gave up in.
 */
async function answered(
  held: ShapeObservable<IndexDescriptor>,
  budgetMs: number,
): Promise<LoadState> {
  const deadline = Date.now() + budgetMs;
  for (;;) {
    const state = loadStateOf(held.getSnapshot());
    if (state !== "pending" || Date.now() >= deadline) return state;
    await new Promise((r) => setTimeout(r, 500));
  }
}

/**
 * Wait for an index document this identity did not hold before to turn up.
 *
 * A document written a moment ago is not owed to be readable instantly, and this read is
 * push-driven rather than polled -- so the loop asks the OBSERVABLE again (`refetch`)
 * rather than issuing a fresh read, and it is bounded: an empty answer under `success` at
 * the end is evidence, not a hang. An empty answer under any other state is neither, and
 * that is why the state travels back with it.
 */
async function indexDocsAfter(
  held: ShapeObservable<IndexDescriptor>,
  before: ReadonlySet<string>,
  budgetMs: number,
): Promise<{ appeared: readonly string[]; observed: LoadState }> {
  const deadline = Date.now() + budgetMs;
  for (;;) {
    const snapshot = held.getSnapshot();
    const observed = loadStateOf(snapshot);
    const appeared =
      observed === "success" ? documentsIn(snapshot).filter((d) => !before.has(d)) : [];
    if (appeared.length > 0 || Date.now() >= deadline) return { appeared, observed };
    await new Promise((r) => setTimeout(r, 500));
    held.refetch();
  }
}

/**
 * How long the reactive read has, both to answer at all and to notice what turned up.
 *
 * It replaces the bound the withdrawn store enumeration was polled under, and it carries one
 * more job than that one did: a shape query the engine refuses never finishes loading, so
 * this is also what turns that failure into a reported state.
 */
const OBSERVE_MS = 15_000;

/**
 * How much of an answer a report carries. Bounded so a report stays one, generous enough
 * that the answer is readable rather than merely counted.
 */
const RAW_LIMIT = 2000;

/** Whatever came back, as JSON — `undefined` and a value that will not render included,
 *  because both of those are answers too and a report that hides them is worth nothing. */
function render(result: unknown): string {
  let text: string;
  try {
    text = JSON.stringify(result) ?? String(result);
  } catch (e: unknown) {
    text = `(did not render: ${String((e as Error)?.message ?? e)})`;
  }
  return text.length <= RAW_LIMIT ? text : `${text.slice(0, RAW_LIMIT)}…(${text.length} chars)`;
}

/**
 * The SELECT's bindings, in the shape the SPARQL results JSON specifies — the same
 * `results.bindings` the polyfill itself reads out of this very call (`surface/inbox.ts`,
 * `surface/read-model.ts`). A binding whose term carries no string `value` is dropped
 * rather than guessed at; `raw` beside it is what keeps that honest.
 */
function compare(a: string, b: string): number {
  return a < b ? -1 : a > b ? 1 : 0;
}

function rowsOf(result: unknown): Array<Record<string, string>> {
  if (result === null || typeof result !== "object") return [];
  const answered = result as {
    results?: { bindings?: ReadonlyArray<Record<string, { value?: unknown } | undefined>> };
  };
  const bindings = answered.results?.bindings ?? [];
  return bindings.map((binding) => {
    const row: Record<string, string> = {};
    for (const [variable, term] of Object.entries(binding)) {
      if (term !== undefined && typeof term.value === "string") row[variable] = term.value;
    }
    return row;
  });
}

/**
 * Everything a document holds, grouped by subject -- what an application writes in place
 * of the `readUnion` the polyfill withdrew in `1.0.0-dev.8`.
 *
 * One anchored `SELECT ?s ?p ?o`: the anchor makes the document the query's default graph,
 * so the body names no graph and reads that document and nothing else. A document this
 * identity cannot read REJECTS, and the rejection is left to reach the caller -- an
 * unreadable document is "unknown", never "empty". The polyfill's own registers live
 * under `urn:ng-eventually:` in the same graph and come back with the rest; they are the
 * polyfill's, not the document's, so they are dropped here.
 */
async function readDocument(doc: string): Promise<DocumentSubject[]> {
  const session = await sessionReady;
  const result = await docs.sparqlQuery(
    session.session_id,
    "SELECT ?s ?p ?o WHERE { ?s ?p ?o }",
    undefined,
    doc,
  );
  const bySubject = new Map<string, Record<string, string[]>>();
  for (const row of rowsOf(result)) {
    const { s, p, o } = row;
    if (s === undefined || p === undefined || o === undefined) continue;
    if (s.startsWith(RESERVED_PREFIX)) continue;
    const props = bySubject.get(s) ?? {};
    (props[p] ??= []).push(o);
    bySubject.set(s, props);
  }
  return [...bySubject].map(([subject, props]) => ({ subject, props }));
}

/** The polyfill's reserved prefix: what it writes into a document under it is its own. */
const RESERVED_PREFIX = "urn:ng-eventually:";

// the acts

const bridge: IndexingBridge = {
  status: () => state.status,
  error: () => state.error,
  whoami: () => state.who,
  configuredIndex,

  async createIndex(field: string): Promise<string> {
    return ready().create(field);
  },

  /**
   * Publish a public document carrying one value for one predicate.
   *
   * It goes through the SAME primitive `create` declares an index's field with
   * (`addLiteralProperty`), with the document as its own subject. That makes it the
   * CONTROL for the write-form question: if this round-trips and an index entry does
   * not, the difference is the foreign subject and nothing else.
   */
  async publishObject(predicate: string, value: string): Promise<string> {
    const p = await probePort();
    const doc = await p.createPublicDocument();
    await p.addLiteralProperty(doc, doc, predicate, value);
    return doc;
  },

  async addToConfigured(object: string): Promise<void> {
    const index = configuredIndex();
    if (index === null) {
      throw new Error("[e2e] this application was not configured with an index reference");
    }
    await ready().add(index, object);
  },

  async addTo(index: string, object: string): Promise<void> {
    await ready().add(index, object);
  },

  async rebuildHandle(): Promise<void> {
    const session = await sessionReady;
    api = indexing(session.session_id);
  },

  /**
   * The index read BY THIS APPLICATION, with nothing the package publishes but
   * `ENTRY_VALUE` — the claim "an index is an ordinary document" performed rather than
   * repeated. `readDocument(index)` is the same read this page makes of any other
   * document; the index's own subject is the one that is not an entry, told apart by
   * being the document itself.
   */
  async read(index: string): Promise<IndexRow[]> {
    const subjects = await readDocument(index);
    const rows: IndexRow[] = [];
    for (const subject of subjects) {
      if (subject.subject === index) continue;
      for (const value of subject.props[ENTRY_VALUE] ?? []) {
        rows.push({ object: subject.subject, value });
      }
    }
    // Ordered by value, ties broken on the object, so two readers of the same document
    // see the same order. The package used to do this; an application does it in four
    // lines, and gets to choose differently.
    rows.sort((a, b) =>
      a.value === b.value ? compare(a.object, b.object) : compare(a.value, b.value),
    );
    return rows;
  },

  async readRaw(doc: string): Promise<DocumentSubject[]> {
    return readDocument(doc);
  },

  /**
   * A SPARQL SELECT anchored on a document, through the polyfill's published `docs`.
   *
   * The session id is the one this application already holds — the same one every write
   * of this layer is made with. Nothing is caught and rethrown: a rejection is REPORTED,
   * because "the query failed" is a different answer from "the query found nothing" and
   * the whole point of this probe is to tell them apart.
   */
  async select(anchor: string, query: string): Promise<SelectOutcome> {
    const session = await sessionReady;
    try {
      const result = await docs.sparqlQuery(session.session_id, query, undefined, anchor);
      return { failed: null, raw: render(result), rows: rowsOf(result) };
    } catch (e: unknown) {
      return { failed: String((e as Error)?.message ?? e), raw: "(the query rejected)", rows: [] };
    }
  },

  async createIndexWithBrokenInbox(field: string): Promise<BrokenInboxOutcome> {
    const p = await probePort();

    // The read that will notice the leak, opened BEFORE anything is created so that
    // "what was already held" is a reading and not an assumption. It is subscribed
    // because an observable with nobody listening has nowhere to push what arrives,
    // and it is released at the end: the probe is its only caller.
    const held = useShapeQuery(IndexShape, EVERYTHING_HELD);
    const release = held.subscribe(() => {});
    try {
      // The opening read has to ANSWER first. A `before` taken from a read that never
      // happened is empty, and an empty `before` would make the first document this
      // identity has ever held look like something the failure left behind.
      const opening = await answered(held, OBSERVE_MS);
      const before = new Set<string>(
        opening === "success" ? documentsIn(held.getSnapshot()) : [],
      );

      // Everything real except the inbox step. The failure is injected at the exact moment
      // the question is about: after the document exists and carries its descriptor, before
      // anyone can deposit into it.
      const broken = indexingOn({
        ...p,
        openInbox: async (): Promise<void> => {
          throw new Error("[e2e] injected: the inbox could not be opened");
        },
      });

      let rejected: string | null = null;
      let returned: string | null = null;
      try {
        returned = await broken.create(field);
      } catch (e: unknown) {
        rejected = String((e as Error)?.message ?? e);
      }

      const after = await indexDocsAfter(held, before, OBSERVE_MS);
      return {
        rejected,
        returned,
        appeared: after.appeared,
        observed: opening === "success" ? after.observed : `${opening} before the probe ran`,
      };
    } finally {
      release();
    }
  },
};

window.__indexing = bridge;