From 9951cd5223da098c03b73c740df4861768f94c01 Mon Sep 17 00:00:00 2001 From: Sylvain Duchesne Date: Sat, 4 Jul 2026 09:33:42 +0200 Subject: [PATCH] feat(client): discovery via a global index (special @index account) Add a generic discovery-index surface: submitToIndex(ref) deposits a reference into the index document's inbox; readIndex() returns the materialized entries. A reserved special account (@index) owns the index document; deposits flow through the emulated inbox and are materialized by the emulated curator (the dedup/ moderation point). This replaces cross-account fan-out as the discovery path and is more faithful to the target (a single owned index fed via its inbox). Generic (the consumer supplies the reference to index). 79 tests pass; tsc rc=0. Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/decisions/discovery-model.md | 26 ++- docs/simulation.md | 45 +++++ packages/client/src/discovery.ts | 163 +++++++++++++++++ packages/client/src/index.ts | 2 + packages/client/test/discovery.test.ts | 233 +++++++++++++++++++++++++ 5 files changed, 461 insertions(+), 8 deletions(-) create mode 100644 packages/client/src/discovery.ts create mode 100644 packages/client/test/discovery.test.ts diff --git a/docs/decisions/discovery-model.md b/docs/decisions/discovery-model.md index 980d552..4f3ed52 100644 --- a/docs/decisions/discovery-model.md +++ b/docs/decisions/discovery-model.md @@ -60,16 +60,26 @@ bound to the developer-user — **not implemented, uncertain**, to explore later This is why a global-index curator is a **deferred separate package** in this lib (see the top-level README). -## Polyfill reality (fan-out) vs target (global index) +## Polyfill reality — the fan-out drift is now RESOLVED (special-account index) -What ships in the shared-wallet polyfill today is the **cross-account fan-out over +The shared-wallet polyfill originally shipped a **cross-account fan-out over every account's public documents** (`store-registry.ts` `listEntityDocs('public')` -/ `resolveReadGraphs`) — one account sees another's public entity **without a -connection**. This ADR classified per-account fan-out as a **drift** to be -replaced by the single global index; the target (inbox-fed global index) remains -valid but the fan-out is the mechanism the shared-wallet staging actually runs on -until the global-index owner is decided. Recorded here as mechanism history — the -resolution belongs to [`../migration-guide.md`](../migration-guide.md). +/ `resolveReadGraphs`) — one account saw another's public entity **without a +connection**. This ADR classified that per-account fan-out as a **drift** to be +replaced by the single global index. + +**That drift is now resolved in the polyfill.** The inbox-fed global index of +this ADR is implemented on top of a **RESERVED SPECIAL ACCOUNT** in the shim +(`discovery.ts`, `INDEX_ACCOUNT = "@index"`) that owns the index document while +the target owner stays undecided: `submitToIndex(ref)` deposits into the index +document's inbox; `readIndex()` materializes (dedup) the entries. The app-facing +discovery path is now **read the index**, exactly as this ADR prescribes — NOT +the fan-out. The cross-account fan-out survives only as an **internal lib +fallback** (it still powers per-scope listing like `resolveReadGraphs`), never +the discovery route. The special account is the provisional owner; at migration +it disappears and ownership moves to the decided global-index owner (see +[`../migration-guide.md`](../migration-guide.md)) with the consumer surface +(`submitToIndex` / `readIndex`) unchanged. ## Alternatives rejected (mechanism) diff --git a/docs/simulation.md b/docs/simulation.md index 7de90ea..3e6c287 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -274,6 +274,51 @@ the deferred global-index note in the top-level README and watcher is the ONE deposit/materialization mechanism reused for BOTH meeting-point registration AND submission to a discovery index — same `post` API, same watcher. +## Emulated discovery index + special account (`discovery.ts`) + +Discovery is a **surface on top of the inbox**, not a new primitive. **Access ≠ +discovery**: a public entity is world-readable *with its NURI*; the discovery +index is how a client learns that NURI **exists** without holding a connection +to its creator (see [`decisions/discovery-model.md`](./decisions/discovery-model.md)). +The model is: ONE global index = an **owned document** (public read), **fed via +ITS inbox**, **materialized by a curator**. Nobody writes the index directly — a +creator DEPOSITS a reference into the index's inbox; the curator ingests deposits +into entries. Materialization is the natural **dedup / moderation point**. + +- **The special account (polyfill owner).** "Who owns the global index" is + undecided in the target (NextGraph is mono-user with no global data — a + singleton app is the only glimpsed path). So the polyfill parks ownership on a + **RESERVED SPECIAL ACCOUNT** in the shim — `INDEX_ACCOUNT = "@index"`. It is a + normal shim account (so its 3 scope documents are created on first sight like + any other), but never a real user; it only HOSTS the index document. Its + `public` scope document IS the index document, and its inbox receives the + deposits — a **stable NURI**: every client opening the same shared wallet + resolves the same account → same document, so all clients read/write ONE + shared index. +- **`submitToIndex(ref, opts?)`** — the SDK act "make this discoverable". + Deposits `ref` into the index document's inbox via `inbox.post`. `from` follows + the inbox convention (anonymous when `null`). `ref` is **opaque** here — the + consumer serializes whatever locates the entity (e.g. an entity document NURI + + discovery metadata). +- **`readIndex()`** — the EMULATED CURATOR. Reads every submission, **dedups by + serialized `ref`** (the moderation point: a duplicate submission surfaces + once), returns entries sorted by `ts`. `watchIndex(onEntries, opts?)` is the + emulated watcher (polls `readIndex`). + +**This REPLACES the cross-account fan-out** (`store-registry.ts` +`listEntityDocs('public')` / `resolveReadGraphs`) as the app-facing discovery +path: the app submits public entities to the index and reads the index, instead +of fanning out over every account's public documents. The fan-out survives only +as an **internal lib fallback** — kept for the per-scope listing it also powers +(e.g. `resolveReadGraphs`), never the app's discovery route. + +GENERIC: `discovery.ts` knows no application domain — the consumer defines the +`ref` shape and its meaning. At migration the special account disappears: +ownership moves to the decided global-index owner, `submitToIndex` becomes the +native `inbox_post_link` on the index's inbox, and `readIndex` queries the real +materialized index document. The consumer surface (`submitToIndex` / `readIndex`) +is designed to survive that swap unchanged. + ## Emulated write guard (`ng-proxy.ts`) The public `ng` proxy overrides `sparql_update` to enforce an emulated **write diff --git a/packages/client/src/discovery.ts b/packages/client/src/discovery.ts new file mode 100644 index 0000000..1215cdb --- /dev/null +++ b/packages/client/src/discovery.ts @@ -0,0 +1,163 @@ +/** + * discovery — a GENERIC discovery-index surface, reusing the ONE deposit + + * materialization mechanism (`inbox.ts`). GENERIC by construction: this module + * knows no application domain (no event, no meeting-point). The consumer submits + * an opaque reference and interprets the entries it reads back. + * + * ── The mechanism (see docs/decisions/discovery-model.md) ───────────────── + * Access ≠ discovery. A public entity is world-readable *with its NURI*; the + * discovery index is how a client learns that NURI EXISTS without holding a + * connection to its creator. There is ONE global index — an OWNED document + * (public read), fed via ITS OWN inbox, and MATERIALIZED by a curator. Nobody + * writes the index directly: a creator DEPOSITS a reference into the index's + * inbox; the (emulated) curator ingests deposits into entries. Materialization + * is the natural dedup / moderation point. + * + * ── The special account (polyfill owner) ────────────────────────────────── + * "Who owns the global index" is undecided in the target (NextGraph is + * mono-user with no global data — see docs/nextgraph-current-state.md § Apps & + * services; a singleton app is the only glimpsed path). So the polyfill parks + * ownership on a RESERVED SPECIAL ACCOUNT in the shim ({@link INDEX_ACCOUNT}). + * Its `public` scope document is the index document; deposits land in that + * document's inbox (a stable NURI: every client opening the same shared wallet + * resolves the same account → same document). This replaces the cross-account + * fan-out (`store-registry.ts` `listEntityDocs`) as the app-facing discovery + * path — the fan-out survives only as an internal fallback (see {@link readIndex}). + * + * ── TARGET vs POLYFILL ──────────────────────────────────────────────────── + * Target: `submitToIndex` seals a reference into the index's native inbox + * (`inbox_post_link`) and a SEPARATE curator package materializes deposits into + * the owned index document; `readIndex` is a query on the materialized index. + * Here, everything is emulated in-lib on the shared wallet (deposit via + * `inbox.post`, materialize via `inbox.read`). At migration the special account + * disappears; ownership moves to the decided global-index owner and this module + * points `readIndex` at the real materialized document. The consumer surface + * (`submitToIndex` / `readIndex`) is designed to survive that swap unchanged. + * + * All NextGraph I/O routes through `inbox.ts` (which routes through the T01.a + * `docs` primitives, the REAL injected `ng`), so this module imports NO + * `@ng-org` package. + */ + +import * as inbox from "./inbox"; +import { ensureAccount } from "./store-registry"; +import type { Nuri, PrincipalId } from "./types"; + +/** + * The reserved SPECIAL ACCOUNT that owns the global discovery index in the + * polyfill. A normal shim account (username), so its scope documents are created + * on first sight like any other account — but it is never a real user; it only + * hosts the index document. Disappears at migration (see file header). + */ +export const INDEX_ACCOUNT = "@index"; + +/** One entry as materialized from the discovery index. */ +export interface IndexEntry { + /** The reference submitted by a creator (opaque — the consumer interprets it). */ + ref: unknown; + /** The submitter, if identified; `null` when the submission was anonymous. */ + from: PrincipalId | null; + /** Submission timestamp (ms epoch). */ + ts: number; +} + +/** Options for {@link submitToIndex}. */ +export interface SubmitOptions { + /** + * Who is submitting. Omit (or pass `null`) for an ANONYMOUS submission; pass a + * principal id to identify the submitter. Defaults to the current polyfill user + * when the property is entirely absent (mirrors {@link inbox.post}). + */ + from?: PrincipalId | null; + /** Optional deposit timestamp (ms epoch). Omitted → `Date.now()`. Passing it + * keeps tests deterministic. */ + ts?: number; +} + +/** + * Resolve the NURI of the index document — the stable inbox where discovery + * submissions land. The special account owns this document (its `public` scope + * document, a real repo NURI from `docCreate`); deposits go into that document's + * inbox exactly as host-registration deposits go into a host inbox. Because the + * special account lives in the shim (persisted in the shared wallet's private + * store), EVERY client opening the same wallet resolves the same account → the + * same document NURI → ONE shared index for all clients. Distinct from + * host-registration inboxes because it is a distinct document NURI. + */ +async function indexInboxNuri(): Promise { + // Ensure the special account exists (idempotent) so its scope documents are + // created and stably resolvable across clients. + const record = await ensureAccount(INDEX_ACCOUNT); + return record.docPublic; +} + +/** + * Submit a reference to the global discovery index — the SDK act "make this + * discoverable". Deposits `ref` into the index document's inbox via + * {@link inbox.post}; the curator ({@link readIndex}) materializes it into an + * entry. GENERIC: `ref` is opaque here (the consumer serializes whatever a + * client needs to later locate the entity — e.g. an entity document NURI plus + * discovery metadata). `from` follows the inbox convention (anonymous if `null`). + */ +export async function submitToIndex(ref: unknown, opts?: SubmitOptions): Promise { + const target = await indexInboxNuri(); + await inbox.post(target, { + payload: ref, + ...(opts && "from" in opts ? { from: opts.from } : {}), + ...(opts?.ts !== undefined ? { ts: opts.ts } : {}), + }); +} + +/** + * Read (materialize) the global discovery index — the EMULATED CURATOR. Reads + * every submission from the index inbox, DEDUPLICATES by serialized `ref` (the + * materialization dedup / moderation point of the discovery model: a duplicate + * submission surfaces once), and returns the entries sorted by `ts` ascending. + * At migration this becomes a query on the real materialized index document. + */ +export async function readIndex(): Promise { + const target = await indexInboxNuri(); + const deposits = await inbox.read(target); + const seen = new Set(); + const entries: IndexEntry[] = []; + for (const d of deposits) { + // Dedup on the serialized reference — the materialization moderation point. + const key = JSON.stringify(d.payload ?? null); + if (seen.has(key)) continue; + seen.add(key); + entries.push({ ref: d.payload, from: d.from, ts: d.ts }); + } + return entries; +} + +/** + * Watch the discovery index — the emulated curator's watcher. Polls + * {@link readIndex} and fires `onEntries` whenever the index changes. Returns an + * unsubscribe. Fires once immediately. (Deduplication is applied on each read.) + */ +export function watchIndex( + onEntries: (entries: IndexEntry[]) => void, + opts?: { intervalMs?: number }, +): () => void { + let stopped = false; + let lastCount = -1; + const intervalMs = opts?.intervalMs ?? 1000; + const tick = async (): Promise => { + if (stopped) return; + try { + const entries = await readIndex(); + if (!stopped && entries.length !== lastCount) { + lastCount = entries.length; + onEntries(entries); + } + } catch (error) { + console.error("[discovery] watchIndex read failed:", error); + } + }; + void tick(); + const handle = setInterval(() => void tick(), intervalMs); + return () => { + stopped = true; + clearInterval(handle); + }; +} diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 4e5db12..d4f9eed 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -15,6 +15,8 @@ export * from "./types"; export { useShape } from "./use-shape"; export { init, initNg } from "./lifecycle"; export * as inbox from "./inbox"; +export * as discovery from "./discovery"; +export type { IndexEntry, SubmitOptions } from "./discovery"; export * as docs from "./docs"; export * as storeRegistry from "./store-registry"; export type { AccountRecord, RegistrySession } from "./store-registry"; diff --git a/packages/client/test/discovery.test.ts b/packages/client/test/discovery.test.ts new file mode 100644 index 0000000..234a8a5 --- /dev/null +++ b/packages/client/test/discovery.test.ts @@ -0,0 +1,233 @@ +import { test, expect, mock, beforeEach, afterAll } from "bun:test"; +import { submitToIndex, readIndex, watchIndex, INDEX_ACCOUNT } from "../src/discovery"; +import type { IndexEntry } from "../src/discovery"; +import { + configure, + configureStoreRegistry, + resetStoreRegistry, + resetConfig, + setCurrentUser, +} from "../src/polyfill"; +import { resetRegistryCache } from "../src/store-registry"; +import type { RegistrySession } from "../src/store-registry"; + +// discovery.ts submits to / reads from a global index owned by a RESERVED +// SPECIAL ACCOUNT (@index) in the shim. This suite injects one fake `ng` that +// emulates BOTH the shim SPARQL (ensureAccount('@index') → doc_create ×3 + +// shim INSERT/SELECT) AND the inbox SPARQL (deposit INSERT + read SELECT), over +// a single in-memory quad store. Restore un-configured state at the end. +afterAll(() => { + resetConfig(); + resetStoreRegistry(); + setCurrentUser(null); +}); + +test("throws a clear error when configureStoreRegistry() was not called", async () => { + resetStoreRegistry(); + resetRegistryCache(); + await expect(submitToIndex({ ref: 1 })).rejects.toThrow( + /configureStoreRegistry\(\) must be called before use/, + ); +}); + +interface Quad { g: string; s: string; p: string; o: string } + +const SHIM = "urn:ng-eventually:shim"; +const INBOX = "urn:ng-eventually:inbox"; + +/** Reverse of the lib's escapeLiteral: single left-to-right pass over `\x`. */ +function unescapeLiteral(s: string): string { + let out = ""; + for (let i = 0; i < s.length; i++) { + if (s[i] === "\\" && i + 1 < s.length) { + const next = s[++i]; + out += next === "n" ? "\n" : next === "r" ? "\r" : next === "t" ? "\t" : next; + } else { + out += s[i]; + } + } + return out; +} + +// A stateful fake `ng` serving BOTH the shim and the inbox SPARQL. +function makeFakeNg() { + const quads: Quad[] = []; + let docCounter = 0; + + const doc_create = mock(async (..._a: unknown[]) => `did:ng:o:doc${++docCounter}`); + + const sparql_update = mock(async (...a: unknown[]) => { + const query = a[1] as string; + const gm = query.match(/GRAPH <([^>]+)>\s*\{([\s\S]*)\}/); + if (!gm) return undefined; + const g = gm[1]!; + const body = gm[2]!; + const sm = body.match(/<([^>]+)>/); + if (!sm) return undefined; + const s = sm[1]!; + const after = body.slice(body.indexOf(sm[0]) + sm[0].length); + const pairRe = /(?:a|<([^>]+)>)\s+(?:"((?:[^"\\]|\\.)*)"|<([^>]+)>)/g; + let m: RegExpExecArray | null; + while ((m = pairRe.exec(after)) !== null) { + // `a` → an rdf:type marker; the two type IRIs the modules use differ, so + // pick by which body we're in (deposit vs account) — harmless if wrong, + // the SELECT filters by the real predicates below. + const isDeposit = query.includes(`${INBOX}:Deposit`); + const p = m[1] ?? (isDeposit ? `${INBOX}:Deposit` : `${SHIM}:Account`); + const o = m[2] !== undefined ? unescapeLiteral(m[2]) : (m[3] ?? ""); + quads.push({ g, s, p, o }); + } + return undefined; + }); + + const sparql_query = mock(async (...a: unknown[]) => { + const query = a[1] as string; + const anchor = a[3] as string | undefined; + // Shim account SELECT. + if (query.includes(`<${SHIM}:username>`)) { + const bySubject = new Map>(); + for (const q of quads) { + if (q.g !== anchor) continue; + const rec = bySubject.get(q.s) ?? {}; + if (q.p === `${SHIM}:username`) rec.username = q.o; + if (q.p === `${SHIM}:docPublic`) rec.docPublic = q.o; + if (q.p === `${SHIM}:docProtected`) rec.docProtected = q.o; + if (q.p === `${SHIM}:docPrivate`) rec.docPrivate = q.o; + bySubject.set(q.s, rec); + } + const bindings = [...bySubject.values()] + .filter((r) => r.username) + .map((r) => ({ + username: { value: r.username! }, + docPublic: { value: r.docPublic ?? "" }, + docProtected: { value: r.docProtected ?? "" }, + docPrivate: { value: r.docPrivate ?? "" }, + })); + return { results: { bindings } }; + } + // Inbox deposit SELECT (?payload ?ts ?from). + if (query.includes(`<${INBOX}:payload>`)) { + const bySubject = new Map>(); + for (const q of quads) { + if (q.g !== anchor) continue; + if (q.p === `${INBOX}:Deposit`) { + if (!bySubject.has(q.s)) bySubject.set(q.s, {}); + continue; + } + const rec = bySubject.get(q.s) ?? {}; + if (q.p === `${INBOX}:payload`) rec.payload = q.o; + if (q.p === `${INBOX}:ts`) rec.ts = q.o; + if (q.p === `${INBOX}:from`) rec.from = q.o; + bySubject.set(q.s, rec); + } + const bindings = [...bySubject.values()] + .filter((r) => r.payload !== undefined && r.ts !== undefined) + .map((r) => { + const row: Record = { + payload: { value: r.payload! }, + ts: { value: r.ts! }, + }; + if (r.from !== undefined) row.from = { value: r.from }; + return row; + }); + return { results: { bindings } }; + } + // Entity-index SELECT (shim contains) — unused here. + return { results: { bindings: [] } }; + }); + + return { doc_create, sparql_update, sparql_query, _quads: quads }; +} + +const SESSION: RegistrySession = { sessionId: "sid-1", privateStoreId: "PRIV" }; + +function inject() { + const ng = makeFakeNg(); + configure({ ng: ng as any, useShape: (() => {}) as any }); + configureStoreRegistry({ + getSession: async () => SESSION, + normalizeUser: (u) => u.trim().replace(/^@+/, "").toLowerCase(), + }); + resetRegistryCache(); + setCurrentUser(null); + return ng; +} + +let fake: ReturnType; +beforeEach(() => { + fake = inject(); +}); + +test("submitToIndex creates the @index special account on first sight (3 docs)", async () => { + await submitToIndex({ nuri: "did:ng:o:event1", title: "Concert" }); + // ensureAccount('@index') created its 3 scope docs. + expect(fake.doc_create).toHaveBeenCalledTimes(3); + // The deposit landed in the @index public document (its inbox). + const depositCall = fake.sparql_update.mock.calls.find((c) => + (c[1] as string).includes(`${INBOX}:Deposit`), + )!; + expect(depositCall, "a deposit INSERT was issued").not.toBeUndefined(); + expect(depositCall[2]).toMatch(/^did:ng:o:doc/); // the index document NURI +}); + +test("submit → read round-trips the reference as an index entry", async () => { + const ref = { nuri: "did:ng:o:event1", title: "Concert au parc" }; + await submitToIndex(ref, { from: "alice", ts: 100 }); + const entries = await readIndex(); + expect(entries).toHaveLength(1); + expect(entries[0]).toEqual({ ref, from: "alice", ts: 100 } as IndexEntry); +}); + +test("a reference submitted by A is discovered by a NON-connected reader via the index", async () => { + // A submits (identified). No connection is ever declared. A separate reader + // materializes the SAME index (same special account → same document) and sees + // the reference — discovery is via the index, not any direct fan-out/link. + setCurrentUser("alice"); + const ref = { nuri: "did:ng:o:evA", title: "Public event by A" }; + await submitToIndex(ref, { ts: 100 }); + + // Reader B: a fresh cache, never connected to A, reads the index. + resetRegistryCache(); + setCurrentUser("bob"); + const entries = await readIndex(); + const refs = entries.map((e) => e.ref); + expect(refs).toContainEqual(ref); + expect(entries.find((e) => JSON.stringify(e.ref) === JSON.stringify(ref))!.from).toBe("alice"); +}); + +test("readIndex deduplicates identical references (materialization moderation point)", async () => { + const ref = { nuri: "did:ng:o:dup", title: "Twice" }; + await submitToIndex(ref, { from: "alice", ts: 100 }); + await submitToIndex(ref, { from: "bob", ts: 200 }); // duplicate reference + const entries = await readIndex(); + expect(entries).toHaveLength(1); // surfaced once +}); + +test("from: null makes an anonymous submission", async () => { + await submitToIndex({ nuri: "did:ng:o:anon" }, { from: null, ts: 100 }); + const entries = await readIndex(); + expect(entries[0]!.from).toBeNull(); +}); + +test("INDEX_ACCOUNT is the reserved special account", () => { + expect(INDEX_ACCOUNT).toBe("@index"); +}); + +test("watchIndex fires immediately then when a submission arrives", async () => { + const seen: IndexEntry[][] = []; + const stop = watchIndex((e) => seen.push(e), { intervalMs: 5 }); + await new Promise((r) => setTimeout(r, 20)); + expect(seen.length).toBeGreaterThanOrEqual(1); + expect(seen[seen.length - 1]).toEqual([]); + + await submitToIndex({ nuri: "did:ng:o:watched" }, { from: null, ts: 1 }); + await new Promise((r) => setTimeout(r, 20)); + const last = seen[seen.length - 1]!; + expect(last.map((e) => (e.ref as any).nuri)).toContain("did:ng:o:watched"); + + stop(); + const countAfterStop = seen.length; + await submitToIndex({ nuri: "did:ng:o:after" }, { from: null, ts: 2 }); + await new Promise((r) => setTimeout(r, 20)); + expect(seen.length).toBe(countAfterStop); +});