diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index ec53c437b3..55e8d5261a 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -48,9 +48,9 @@ classDiagram <> +writeWorkingMemory() +liftWorkingMemoryToSharedMemory() - +sharedMemoryWrite() - +publishFromSharedMemory() - +publisherEnqueue() + +createKnowledgeAsset() + +publishAssertion() + +knowledgeAssetPublishAsync() +publisherJob() +query() } @@ -81,7 +81,7 @@ classDiagram class PublisherRuntime { <> - +publishFromSharedMemory() + +knowledgeAssetPublishAsync() +enqueue() +processNext() +finalize() @@ -165,18 +165,18 @@ sequenceDiagram loop warmups and measured iterations Script->>Script: create unique sync payload - Script->>Client: sharedMemoryWrite(sync quads) - Client->>Daemon: POST shared memory write - Daemon->>Agent: write benchmark triples - Agent->>Store: persist SWM payload + Script->>Client: createKnowledgeAsset(sync quads, alsoShareSwm) + Client->>Daemon: POST /api/knowledge-assets + Daemon->>Agent: create + write + finalize + share + Agent->>Store: persist named KA and SWM share Store-->>Agent: share operation recorded - Agent-->>Daemon: write accepted - Daemon-->>Client: share operation id + Agent-->>Daemon: named KA shared + Daemon-->>Client: assertion + share metadata - Script->>Client: publishFromSharedMemory(sync root) - Client->>Daemon: POST shared memory publish - Daemon->>Publisher: publish synchronously - Publisher->>Store: read staged triples + Script->>Client: publishAssertion(sync name) + Client->>Daemon: POST /api/knowledge-assets/:name/vm/publish + Daemon->>Publisher: publish finalized named KA + Publisher->>Store: read sealed SWM share Publisher->>Chain: anchor knowledge asset Chain-->>Publisher: commitment finalized Publisher-->>Daemon: kc id @@ -192,17 +192,17 @@ sequenceDiagram Script->>Script: validate returned marker Script->>Script: create unique async payload - Script->>Client: sharedMemoryWrite(async quads) - Client->>Daemon: POST shared memory write - Daemon->>Agent: write benchmark triples - Agent->>Store: persist SWM payload + Script->>Client: createKnowledgeAsset(async quads, alsoShareSwm) + Client->>Daemon: POST /api/knowledge-assets + Daemon->>Agent: create + write + finalize + share + Agent->>Store: persist named KA and SWM share Store-->>Agent: share operation recorded - Agent-->>Daemon: write accepted - Daemon-->>Client: share operation id + Agent-->>Daemon: named KA shared + Daemon-->>Client: assertion + share metadata - Script->>Client: publisherEnqueue(share operation id) - Client->>Daemon: POST publisher enqueue - Daemon->>Publisher: enqueue job + Script->>Client: knowledgeAssetPublishAsync(name) + Client->>Daemon: POST /api/knowledge-assets/:name/vm/publish-async + Daemon->>Publisher: enqueue named KA VM publish job Publisher-->>Daemon: job id Daemon-->>Client: enqueue result @@ -273,13 +273,13 @@ sequenceDiagram else synchronous publish with finalization Client->>WM: write payload Client->>SWM: lift payload - Suite->>Client: publishFromSharedMemory(root) + Suite->>Client: publishAssertion(name) Client->>VM: promote root with kc id Client-->>Suite: finalized publish result else asynchronous publish enqueue and finalization Client->>WM: write payload Client->>SWM: lift payload - Suite->>Client: publisherEnqueue(share operation) + Suite->>Client: knowledgeAssetPublishAsync(name) Client->>Jobs: create queued job Suite->>Client: publisherJob(job id) Client->>VM: promote queued roots @@ -715,10 +715,11 @@ decision to keep private-store RDF plaintext after message decryption. daemon-reserved metadata, partitions root entities, inserts the promoted quads into `_shared_memory`, removes the promoted rows from the assertion graph, and updates lifecycle metadata in `_meta`. -- **SWM writes** from `/api/shared-memory/write`, - `/api/shared-memory/conditional-write`, and assertion promotion persist - normalized quads directly in `did:dkg:context-graph:/_shared_memory` or - the sub-graph equivalent. SWM operation metadata and ownership live in +- **SWM substrate writes** are produced by named KA sharing, gossip receive, + catch-up, and host-mode replication. Product callers share through + `POST /api/knowledge-assets/:name/swm/share` or `/swm/share-async`; those + paths persist normalized quads in `did:dkg:context-graph:/_shared_memory` + or the sub-graph equivalent. SWM operation metadata and ownership live in `_shared_memory_meta`; the public snapshot store records the same public quads for replay and catch-up. The store rows are not `enc:gcm:v1` envelopes. - **Private content** lives in `_private` graphs through `PrivateContentStore`. @@ -814,9 +815,9 @@ loop each configured source Runner-->>Runner: skip processSource else source content changed or retry is needed Runner->>Handler: processSource(source, fingerprint, priorState) - Handler->>SWM: POST /api/shared-memory/write - SWM-->>Handler: shareOperationId - Handler->>Publisher: POST /api/publisher/enqueue + Handler->>KA: POST /api/knowledge-assets + KA-->>Handler: name + shareOperationId + Handler->>Publisher: POST /api/knowledge-assets/:name/vm/publish-async Publisher-->>Handler: jobId Handler-->>Runner: nextState end diff --git a/README.md b/README.md index 5874e24aed..5585f961b4 100644 --- a/README.md +++ b/README.md @@ -250,8 +250,7 @@ dkg assertion query -c # read assertion quads from dkg assertion promote -c # WM → SWM # Shared memory (team-visible) and publishing -dkg shared-memory write ... # write triples directly to SWM -dkg shared-memory publish # SWM → Verifiable Memory (costs TRAC) +dkg assertion promote -c # share a named KA from WM to SWM dkg publish -f # one-shot RDF publish to a context graph dkg verify --context-graph --verified-graph # propose M-of-N verification dkg endorse --context-graph --agent # endorse a published KA @@ -264,7 +263,7 @@ dkg subscribe # subscribe to a CG's gossip topics # Async publisher (optional, for batching) dkg publisher enable # enable the async publisher -dkg publisher enqueue ... # enqueue a publish job +dkg publisher publish-async # enqueue a named KA VM publish job dkg publisher jobs # list publisher jobs dkg publisher stats # publisher throughput stats diff --git a/bench/analyze-publish-async-get.ts b/bench/analyze-publish-async-get.ts index 7b56ff344e..06acbb8745 100644 --- a/bench/analyze-publish-async-get.ts +++ b/bench/analyze-publish-async-get.ts @@ -52,7 +52,7 @@ const DEFAULT_OUTPUT_DIR = 'bench/results/profiles'; const PUBLISH_ASYNC_GET_PAGES: Array<[string, string]> = [ ['get/read retrieval', 'bench/results/publish-async-get/get-read-retrieval.html'], ['synchronous publish with finalization', 'bench/results/publish-async-get/sync-publish-finalization.html'], - ['asynchronous publish enqueue and finalization', 'bench/results/publish-async-get/async-publish-finalization.html'], + ['asynchronous VM publish request and finalization', 'bench/results/publish-async-get/async-publish-finalization.html'], ['upload payload to local working memory', 'bench/results/publish-async-get/working-memory-upload.html'], ['lift local working memory to shared working memory', 'bench/results/publish-async-get/working-to-shared-memory.html'], ]; @@ -197,25 +197,15 @@ async function analyzeSyncPublishFlow(config: BenchmarkConfig, payloadSize: Payl async function analyzeAsyncPublishFlow(config: BenchmarkConfig, payloadSize: PayloadSizeLabel): Promise { const client = new LayeredDkgBenchmarkClient(); - return analyzeFlow('asynchronous publish enqueue and finalization', payloadSize, async (trace) => { + return analyzeFlow('asynchronous VM publish request and finalization', payloadSize, async (trace) => { const payload = traceSync(trace, 'setup', 'createPayload', [], 'Generate the payload for async publish.', () => ( createPayload(config, `analysis-async-${payloadSize}`, 1, 'async', false) )); - const prepared = await traceSharedMemoryWrite(trace, client, config, payload); - const shareOperationId = prepared.shareOperationId ?? ''; - const queued = await traceAsync(trace, 'measured', 'publisherEnqueue', ['publisherJobs.set'], 'Enqueue the publish request through the publisher runtime path.', () => ( - client.publisherEnqueue({ - contextGraphId: config.contextGraphId, - shareOperationId, - roots: [payload.rootEntity], - namespace: config.namespace, - scope: config.scope, - authorityProofRef: config.authorityProofRef, - swmId: 'swm-main', - transitionType: 'CREATE', - authorityType: 'owner', - }) - ), { rootEntity: payload.rootEntity, shareOperationId }); + const name = `analysis-async-${payloadSize}`; + await traceCreateSharedKnowledgeAsset(trace, client, config, name, payload); + const queued = await traceAsync(trace, 'measured', 'knowledgeAssetPublishAsync', ['publisherJobs.set'], 'Queue VM publish for the named knowledge asset.', () => ( + client.knowledgeAssetPublishAsync(config.contextGraphId, name) + ), { rootEntity: payload.rootEntity, name }); await traceAsync(trace, 'measured', 'publisherJob', ['promoteSharedRoot'], 'Poll the publisher job and finalize queued content.', () => ( client.publisherJob(queued.jobId ?? '') ), { jobId: queued.jobId }); @@ -268,15 +258,20 @@ async function analyzeFlow( return { flow, payloadSize, totalMs, measuredMs, traces: normalizedTraces }; } -async function traceSharedMemoryWrite( +async function traceCreateSharedKnowledgeAsset( traces: MethodTrace[], client: LayeredDkgBenchmarkClient, config: BenchmarkConfig, + name: string, payload: BenchmarkPayload, ) { - return traceAsync(traces, 'setup', 'sharedMemoryWrite', ['writeWorkingMemory', 'liftWorkingMemoryToSharedMemory'], 'Stage generated quads in local memory and lift them into shared working memory.', () => ( - client.sharedMemoryWrite(config.contextGraphId, payload.quads) - ), payloadContext(payload)); + return traceAsync(traces, 'setup', 'createKnowledgeAsset', ['writeWorkingMemory', 'liftWorkingMemoryToSharedMemory'], 'Create, seal, and share the named knowledge asset into shared working memory.', () => ( + client.createKnowledgeAsset(config.contextGraphId, name, { + quads: payload.quads, + finalize: true, + alsoShareSwm: true, + }) + ), { ...payloadContext(payload), name }); } async function traceAsync( diff --git a/bench/publish-async-get.bench.ts b/bench/publish-async-get.bench.ts index cde28e206e..7b0ab7d8d8 100644 --- a/bench/publish-async-get.bench.ts +++ b/bench/publish-async-get.bench.ts @@ -94,11 +94,9 @@ export default defineSuite({ }, { beforeIteration: async () => { - // The measured publishAssertion stages the quads internally — no separate + // The measured publishAssertion stages the quads internally; no separate // sharedMemoryWrite here, or the sync flow would double-write SWM and stop - // matching the canonical create/write/share/publish path (cf. analyze-* and - // the get/read flow). The async flow below keeps its write — it needs the - // returned shareOperationId for the publisher enqueue/lift path. + // matching the canonical create/write/share/publish path. syncName = `esbench-sync-${sequence++}`; syncPayload = createPayload(config, syncName, 1, 'sync', false); }, @@ -110,26 +108,16 @@ export default defineSuite({ ); let asyncPayload: BenchmarkPayload | undefined; - let asyncShareOperationId: string | undefined; + let asyncName: string | undefined; const asyncClient = new LayeredDkgBenchmarkClient(); benchAsyncWithHooks( scene, - 'asynchronous publish enqueue and finalization', + 'asynchronous VM publish request and finalization', async () => { - const payload = requirePayload(asyncPayload, 'asynchronous publish enqueue and finalization'); - if (!asyncShareOperationId) throw new Error('async setup did not produce a share operation id'); - - const queued = await asyncClient.publisherEnqueue({ - contextGraphId: config.contextGraphId, - shareOperationId: asyncShareOperationId, - roots: [payload.rootEntity], - namespace: config.namespace, - scope: config.scope, - authorityProofRef: config.authorityProofRef, - swmId: 'swm-main', - transitionType: 'CREATE', - authorityType: 'owner', - }); + const payload = requirePayload(asyncPayload, 'asynchronous VM publish request and finalization'); + if (!asyncName) throw new Error('async setup did not produce a knowledge asset name'); + + const queued = await asyncClient.knowledgeAssetPublishAsync(config.contextGraphId, asyncName); if (!queued.jobId) throw new Error('async publisher did not return a job id'); const completed = await asyncClient.publisherJob(queued.jobId); @@ -139,13 +127,17 @@ export default defineSuite({ }, { beforeIteration: async () => { - asyncPayload = createPayload(config, `esbench-async-${sequence++}`, 1, 'async', false); - const prepared = await asyncClient.sharedMemoryWrite(config.contextGraphId, asyncPayload.quads); - asyncShareOperationId = prepared.shareOperationId; + asyncName = `esbench-async-${sequence++}`; + asyncPayload = createPayload(config, asyncName, 1, 'async', false); + await asyncClient.createKnowledgeAsset(config.contextGraphId, asyncName, { + quads: asyncPayload.quads, + finalize: true, + alsoShareSwm: true, + }); }, afterIteration: () => { asyncPayload = undefined; - asyncShareOperationId = undefined; + asyncName = undefined; asyncClient.clear(); }, }, diff --git a/bench/support/cpu-profile-report.mjs b/bench/support/cpu-profile-report.mjs index 8b650fabc4..17cabe836c 100644 --- a/bench/support/cpu-profile-report.mjs +++ b/bench/support/cpu-profile-report.mjs @@ -314,7 +314,7 @@ function labelForWidth(label, width) { } function formatValue(value, unit) { - if (unit === 'ms') return `${value.toLocaleString(undefined, { maximumFractionDigits: 2, minimumFractionDigits: value < 10 ? 2 : 0 })} ms`; + if (unit === 'ms') return `${value.toLocaleString('en-US', { maximumFractionDigits: 2, minimumFractionDigits: value < 10 ? 2 : 0 })} ms`; return `${Math.round(value).toLocaleString()} samples`; } diff --git a/bench/support/layered-dkg-client.ts b/bench/support/layered-dkg-client.ts index 4bd661aa9f..4128171f66 100644 --- a/bench/support/layered-dkg-client.ts +++ b/bench/support/layered-dkg-client.ts @@ -12,11 +12,13 @@ interface MemoryRecord { marker: string; quads: Quad[]; shareOperationId?: string; + assertionName?: string; kaId?: string; } interface PublisherJob { contextGraphId: string; + assertionName: string; jobId: string; roots: string[]; status: 'queued' | 'finalized' | 'failed'; @@ -27,6 +29,7 @@ export class LayeredDkgBenchmarkClient implements BenchmarkClient { readonly workingMemory = new Map(); readonly sharedWorkingMemory = new Map(); readonly verifiableMemory = new Map(); + readonly knowledgeAssets = new Map(); readonly publisherJobs = new Map(); private shareSequence = 0; @@ -50,21 +53,31 @@ export class LayeredDkgBenchmarkClient implements BenchmarkClient { this.workingMemory.clear(); this.sharedWorkingMemory.clear(); this.verifiableMemory.clear(); + this.knowledgeAssets.clear(); this.publisherJobs.clear(); } - async sharedMemoryWrite(contextGraphId: string, quads: Quad[]) { - const working = await this.writeWorkingMemory(contextGraphId, quads); - const shared = await this.liftWorkingMemoryToSharedMemory(contextGraphId, uniqueSubjects(quads)); + async createKnowledgeAsset( + contextGraphId: string, + name: string, + options: { quads: Quad[]; finalize: true; alsoShareSwm: true; subGraphName?: string }, + ) { + const roots = uniqueSubjects(options.quads); + await this.writeWorkingMemory(contextGraphId, options.quads, { assertionName: name }); + const shared = await this.liftWorkingMemoryToSharedMemory(contextGraphId, roots); + this.knowledgeAssets.set(assetKey(contextGraphId, name), { contextGraphId, name, roots }); return { + assertionUri: `urn:benchmark:assertion:${encodeURIComponent(contextGraphId)}:${encodeURIComponent(name)}`, + promotedCount: roots.length, + publishReady: true, shareOperationId: shared.shareOperationId, }; } - async writeWorkingMemory(contextGraphId: string, quads: Quad[]) { + async writeWorkingMemory(contextGraphId: string, quads: Quad[], ids: Pick = {}) { const shareOperationId = `draft-${++this.draftSequence}`; for (const rootEntity of uniqueSubjects(quads)) { - const record = createMemoryRecord(contextGraphId, rootEntity, quads, { shareOperationId }); + const record = createMemoryRecord(contextGraphId, rootEntity, quads, { shareOperationId, ...ids }); this.workingMemory.set(rootEntity, record); } return { shareOperationId }; @@ -92,7 +105,11 @@ export class LayeredDkgBenchmarkClient implements BenchmarkClient { // Named-KA one-shot: stage the quads (create → write → share) then publish // its roots to verifiable memory. Mirrors the create → /vm/publish flow the // real benchmark client (ApiClient.publishAssertion) drives. - await this.sharedMemoryWrite(contextGraphId, quads); + await this.createKnowledgeAsset(contextGraphId, _name, { + quads, + finalize: true, + alsoShareSwm: true, + }); const roots = uniqueSubjects(quads); const kaId = `kc-${++this.kcSequence}`; for (const rootEntity of roots) { @@ -106,8 +123,12 @@ export class LayeredDkgBenchmarkClient implements BenchmarkClient { }; } - async publisherEnqueue(request: Parameters[0]) { - for (const rootEntity of request.roots) { + async knowledgeAssetPublishAsync(contextGraphId: string, name: string) { + const asset = this.knowledgeAssets.get(assetKey(contextGraphId, name)); + if (!asset) { + throw new Error(`Knowledge asset ${name} is missing from context graph ${contextGraphId}`); + } + for (const rootEntity of asset.roots) { if (!this.sharedWorkingMemory.has(rootEntity)) { throw new Error(`Root ${rootEntity} is missing from shared working memory`); } @@ -115,9 +136,10 @@ export class LayeredDkgBenchmarkClient implements BenchmarkClient { const jobId = `job-${++this.jobSequence}`; this.publisherJobs.set(jobId, { - contextGraphId: request.contextGraphId, + contextGraphId, + assertionName: name, jobId, - roots: [...request.roots], + roots: [...asset.roots], status: 'queued', }); return { jobId }; @@ -196,6 +218,10 @@ function uniqueSubjects(quads: Quad[]): string[] { return [...new Set(quads.map((quad) => quad.subject))]; } +function assetKey(contextGraphId: string, name: string): string { + return `${contextGraphId}\0${name}`; +} + function markerFromQuads(quads: Quad[]): string { const markerQuad = quads.find((quad) => quad.predicate === 'http://schema.org/identifier'); if (!markerQuad) throw new Error('Benchmark payload is missing a marker quad'); diff --git a/devnet/agent-provenance/README.md b/devnet/agent-provenance/README.md index 46c4d2853a..8d4b3df5d9 100644 --- a/devnet/agent-provenance/README.md +++ b/devnet/agent-provenance/README.md @@ -18,8 +18,8 @@ prefer running them before falling back to the manual recipes below. Phase 4 author override (RFC §4(b)) is now wired end-to-end via the agent-keystore: end-user agents register on a daemon (`POST -/api/agent/register`) and publish through that daemon's -`/api/shared-memory/publish` route. The route resolves the bearer +/api/agent/register`) and publish through that daemon's named +knowledge-asset lifecycle routes. The publish route resolves the bearer token to an `AgentKeyRecord`, looks up the custodial private key, and threads it into the publisher as `authorPrivateKey` so the on-chain `AuthorAttestation` is signed by the *agent*, not the daemon. See @@ -172,23 +172,26 @@ AGENT_TOKEN=$(curl -s -X POST \ # Action — end-user agent writes quads + publishes through core 2, # authenticating with its OWN bearer token. The daemon resolves the # token → agent → custodial privateKey → AuthorAttestation signer. -curl -s http://127.0.0.1:9202/api/shared-memory/write \ +curl -s http://127.0.0.1:9202/api/knowledge-assets \ -H "Authorization: Bearer $AGENT_TOKEN" \ -H "Content-Type: application/json" \ -d '{ "contextGraphId": "", + "name": "mode-b-agent-ka", "quads": [{ "subject": "urn:test:mode-b:1", "predicate": "https://schema.org/name", "object": "\"mode-b\"", "graph": "did:dkg:context-graph:" - }] + }], + "finalize": true, + "alsoShareSwm": true }' -curl -s http://127.0.0.1:9202/api/shared-memory/publish \ +curl -s http://127.0.0.1:9202/api/knowledge-assets/mode-b-agent-ka/vm/publish \ -H "Authorization: Bearer $AGENT_TOKEN" \ -H "Content-Type: application/json" \ - -d '{ "contextGraphId": "", "selection": "all", "clearAfter": true }' + -d '{ "contextGraphId": "", "options": { "clearAfter": true } }' ``` Optional: register on core 2's PCA so the publish is discounted (this diff --git a/devnet/v10-core-flows/FINDINGS.md b/devnet/v10-core-flows/FINDINGS.md index e8859f399e..bbe424f1a1 100644 --- a/devnet/v10-core-flows/FINDINGS.md +++ b/devnet/v10-core-flows/FINDINGS.md @@ -20,7 +20,7 @@ Curated snapshot of bugs and observations surfaced by `pnpm test:devnet:v10-core ### Edge-node publish is "tentative" by design -When an edge node (`nodeRole: "edge"`, no on-chain identity registered) calls `/api/shared-memory/publish`, the daemon: +When an edge node (`nodeRole: "edge"`, no on-chain identity registered) calls `/api/knowledge-assets/:name/vm/publish`, the daemon: 1. Walks the full assertion lifecycle (create/write/finalize/promote) successfully. 2. Computes the merkle root over the SWM selection and prepares the publish payload. diff --git a/devnet/v10-stress/FINDINGS.md b/devnet/v10-stress/FINDINGS.md index a4ea7cf694..e439cbf15a 100644 --- a/devnet/v10-stress/FINDINGS.md +++ b/devnet/v10-stress/FINDINGS.md @@ -20,7 +20,7 @@ Originally: `publishFromFinalizedAssertion` called `publishFromSharedMemory(cont **Where:** publisher / chain-adapter nonce management. The publisher's operational-wallet pool reads the on-chain nonce as `latest` rather than `pending`, and a second publish that lands on the same wallet inside the same block window can pick a stale value. -**Reproduction (probabilistic, ~1 in 25 fresh-cluster publishes):** issue rapid-fire `POST /api/shared-memory/publish { assertionName }` calls into the same daemon. Some `token.approve` or KC-create transactions revert with `Nonce too low. Expected nonce to be N+1 but got N`, and the publish flips to `tentative kaId: "0"`. +**Reproduction (probabilistic, ~1 in 25 fresh-cluster publishes):** issue rapid-fire `POST /api/knowledge-assets/:name/vm/publish` calls into the same daemon. Some `token.approve` or KC-create transactions revert with `Nonce too low. Expected nonce to be N+1 but got N`, and the publish flips to `tentative kaId: "0"`. **Why it doesn't always fire:** the pool rotates through op-wallets, so adjacent publishes usually hit different wallets. Hitting the same wallet within the stale window is what triggers it. Background work (gossip-publish reactions, worker sweeps) can also occupy a wallet behind the scenes, which is what we suspect occasionally beats the foreground request to a nonce. diff --git a/docs/references/cli.md b/docs/references/cli.md index 7919912129..2241394f4f 100644 --- a/docs/references/cli.md +++ b/docs/references/cli.md @@ -43,8 +43,7 @@ dkg assertion query -c # read the WM draft's quads dkg assertion promote -c # WM → SWM (the share operation; CLI verb retained) # Shared memory (team-visible) and publishing -dkg shared-memory write [--name ] ... # stage triples into a named WM Knowledge Asset draft (write-first; share + publish later) -dkg shared-memory publish --name # finalize + share + publish a staged WM Knowledge Asset → Verifiable Memory (costs TRAC) +dkg publisher publish-async # enqueue VM publish for a named KA already shared to SWM dkg publish -f # one-shot RDF publish to a context graph dkg verify --context-graph --verified-graph # propose M-of-N verification dkg endorse --context-graph [--agent ] # endorse a published KA as the authenticated agent (--agent only asserts the token's agent matches) @@ -57,7 +56,7 @@ dkg subscribe # subscribe to a CG's gossip topics # Async publisher (optional, for batching) dkg publisher enable # enable the async publisher -dkg publisher enqueue --root --namespace --scope --authority-proof-ref --share-operation-id # enqueue a publish job (flags required) +dkg publisher publish-async # enqueue VM publish for a named KA already shared to SWM dkg publisher jobs # list publisher jobs dkg publisher stats # publisher throughput stats diff --git a/docs/rfcs/ka-metadata-trim.md b/docs/rfcs/ka-metadata-trim.md index 85e0cf7233..be7f46c93f 100644 --- a/docs/rfcs/ka-metadata-trim.md +++ b/docs/rfcs/ka-metadata-trim.md @@ -5,7 +5,7 @@ ## Ground truth (live dump, KA #122, `megagiga` CG) -One `dkg shared-memory publish` of 1 user triple writes, resident: +One named KA VM publish of 1 user triple writes, resident: | Graph | Quads | Content | |---|---|---| diff --git a/docs/use-dkg/publish-and-query.md b/docs/use-dkg/publish-and-query.md index 4cd349571b..4a50bb89cb 100644 --- a/docs/use-dkg/publish-and-query.md +++ b/docs/use-dkg/publish-and-query.md @@ -23,14 +23,15 @@ CG="/my-project" dkg assertion import-file notes -f ./notes.md -c "$CG" dkg assertion query notes -c "$CG" dkg context-graph register "$CG" -dkg shared-memory publish "$CG" --name notes +dkg assertion promote notes -c "$CG" +dkg publisher publish-async "$CG" notes ``` Bare context-graph IDs are scoped by the daemon before use. After `create`, use the `ID:` printed by the CLI for later commands; it has the form `/my-project`. Use `dkg assertion promote notes -c "$CG"` when you want to stop at Shared Working Memory without publishing to Verifiable Memory. -Use `dkg shared-memory publish "$CG" --name notes` for the high-level finalize → share → publish path. Publishing to Verifiable Memory requires an on-chain context graph, so run `dkg context-graph register "$CG"` before the publish step unless the context graph was already registered. +Use `dkg publisher publish-async "$CG" notes` to enqueue the Verifiable Memory publish after the KA has been promoted to SWM. Publishing to Verifiable Memory requires an on-chain context graph, so run `dkg context-graph register "$CG"` before the publish step unless the context graph was already registered. ## Agent shape diff --git a/esbench.config.mjs b/esbench.config.mjs index 16dc9b91f2..2e31264b82 100644 --- a/esbench.config.mjs +++ b/esbench.config.mjs @@ -11,7 +11,7 @@ export const publishAsyncGetSuite = 'bench/publish-async-get.bench.ts'; export const publishAsyncGetPages = [ ['get/read retrieval', 'bench/results/publish-async-get/get-read-retrieval.html'], ['synchronous publish with finalization', 'bench/results/publish-async-get/sync-publish-finalization.html'], - ['asynchronous publish enqueue and finalization', 'bench/results/publish-async-get/async-publish-finalization.html'], + ['asynchronous VM publish request and finalization', 'bench/results/publish-async-get/async-publish-finalization.html'], ['upload payload to local working memory', 'bench/results/publish-async-get/working-memory-upload.html'], ['lift local working memory to shared working memory', 'bench/results/publish-async-get/working-to-shared-memory.html'], ]; diff --git a/packages/adapter-openclaw/src/dkg-client.ts b/packages/adapter-openclaw/src/dkg-client.ts index e7e723cb62..5a7c1caad3 100644 --- a/packages/adapter-openclaw/src/dkg-client.ts +++ b/packages/adapter-openclaw/src/dkg-client.ts @@ -32,7 +32,7 @@ export class DkgDaemonHttpError extends Error { * Build a {@link DkgDaemonHttpError} from a non-2xx daemon response. Attempts to * parse `text` as JSON for the typed `body`; a non-JSON body leaves `body` * undefined (the raw text still appears in the message). The message format is - * fixed — do NOT change it without auditing the `responded ` substring checks. + * fixed -- do NOT change it without auditing the `responded ` substring checks. */ function daemonHttpError(message: string, status: number, text: string): DkgDaemonHttpError { let body: unknown; @@ -48,7 +48,7 @@ export interface DkgClientOptions { /** Bearer token for daemon API auth. If omitted, tries `/auth.token`. */ apiToken?: string; /** - * T70 — DKG home directory used to read `auth.token` when `apiToken` is + * T70 -- DKG home directory used to read `auth.token` when `apiToken` is * not supplied. Caller (typically `DkgNodePlugin.register`) passes the * runtime-resolved home (`resolveDkgHome({daemonUrl})`) so the constructor * fallback reads from the right place when the active daemon is in @@ -171,7 +171,7 @@ export interface LocalAgentIntegrationRecord extends LocalAgentIntegrationPayloa } /** - * T63 — Shape of `/api/agent/identity` response. + * T63 -- Shape of `/api/agent/identity` response. * * Mirrors the daemon route handler at * `packages/cli/src/daemon/routes/agent-chat.ts:391`. `agentAddress` is the @@ -273,7 +273,7 @@ function isChatTurnStoreNotFoundError(err: unknown): boolean { export interface PreSignedAuthorAttestationPayload { address: string; /** - * OT-RFC-43 §F2 — the packed reservedKaId the author signed the + * OT-RFC-43 Section F2 -- the packed reservedKaId the author signed the * AuthorAttestation over, as a decimal string (uint256-safe over JSON). * Required: the daemon binds it into the digest and honours the reserved slot. */ @@ -324,7 +324,7 @@ function assertCreateFinalizeFieldsHaveQuads(args: { args.preSignedAuthorAttestation != null || args.schemeVersion !== undefined; // These fields only take effect at finalize, so they require both non-empty - // quads AND finalize !== false — mirrors the daemon create-route guard. + // quads AND finalize !== false -- mirrors the daemon create-route guard. const willFinalize = Array.isArray(args.quads) && args.quads.length > 0 && args.finalize !== false; if (hasFinalizeOnlyField && !willFinalize) { throw new Error('authorAgentAddress, preSignedAuthorAttestation, and schemeVersion require non-empty quads and finalize !== false'); @@ -440,7 +440,7 @@ export class DkgDaemonClient { /** * Run a SPARQL query against the daemon. Forwards the full V10 field set - * the `/api/query` route accepts — `view` (`'working-memory' | 'shared-working-memory' | 'verifiable-memory'`), + * the `/api/query` route accepts -- `view` (`'working-memory' | 'shared-working-memory' | 'verifiable-memory'`), * `agentAddress` (required for WM reads), `assertionName` (scopes WM reads * to a single per-agent assertion), `subGraphName`, `verifiedGraph`, * `graphSuffix`, `includeSharedMemory`. @@ -460,7 +460,7 @@ export class DkgDaemonClient { * P-13: minimum trust level. Only meaningful for * `view: "verifiable-memory"`; ignored (silently) on WM/SWM views. * - * The daemon implements only `SelfAttested` / `Endorsed` today — + * The daemon implements only `SelfAttested` / `Endorsed` today -- * higher tiers (Q-1 follow-up) are rejected with HTTP 400, so the * public client surface only advertises the implementable values. * See `packages/query/src/query-engine.ts QueryOptions.minTrust`. @@ -512,40 +512,7 @@ export class DkgDaemonClient { } // --------------------------------------------------------------------------- - // Shared memory write (SWM layer — NOT used by v1 chat-turn / memory paths) - // --------------------------------------------------------------------------- - - /** - * Write quads to a context graph's Shared Working Memory graph. Retained - * as a general primitive for callers that deliberately want SWM semantics - * (e.g. user-initiated promotion). v1 chat-turn and per-project memory - * writes use `writeAssertion` instead — SWM is the wrong layer for private - * per-agent memory per `21_TRI_MODAL_MEMORY.md §5`. - */ - async share( - contextGraphId: string, - quads: Array<{ subject: string; predicate: string; object: string; graph?: string }>, - opts?: { localOnly?: boolean; subGraphName?: string }, - ): Promise<{ shareOperationId: string }> { - const cgId = normalizeContextGraphId(contextGraphId); - // SWM data gossips to peers in the context graph's allowlist by default. - // Privacy is governed by the CG's curation policy (curated CGs gate - // gossip to allowed peers; public CGs gossip to all subscribers). - // Callers that explicitly want a local-only write can still pass - // `localOnly: true`. Aligned with Hermes's default in - // `packages/adapter-hermes/hermes-plugin/client.py` which omits the - // field and relies on the daemon's `false` default at - // `packages/cli/src/daemon/routes/memory.ts:490`. - return this.post('/api/shared-memory/write', { - contextGraphId: cgId, - quads, - localOnly: opts?.localOnly ?? false, - subGraphName: opts?.subGraphName, - }); - } - - // --------------------------------------------------------------------------- - // Working Memory — assertion lifecycle + // Working Memory -- assertion lifecycle // --------------------------------------------------------------------------- /** @@ -577,7 +544,7 @@ export class DkgDaemonClient { /** * Append quads into an existing Working Memory assertion. The assertion - * must have been created first — callers that create-then-write in a + * must have been created first -- callers that create-then-write in a * single call should use `ensureAssertion` + `writeAssertion` together, * with `createAssertion` swallowing duplicates. */ @@ -629,7 +596,7 @@ export class DkgDaemonClient { /** * Dump all quads from a single Working Memory assertion's graph. This is - * not a SPARQL endpoint — the daemon returns every quad in the assertion + * not a SPARQL endpoint -- the daemon returns every quad in the assertion * as `{ quads, count }`. For ad-hoc SPARQL use `query()` with * `view: 'working-memory'` + `assertionName` instead. */ @@ -1188,11 +1155,11 @@ export class DkgDaemonClient { return res.json() as Promise; } - // ── OT-RFC-43 §10.5 — GitHub-shaped Knowledge Asset client ────────────── + // -- OT-RFC-43 Section 10.5 -- GitHub-shaped Knowledge Asset client -------------- // Layer-explicit wrappers over the clean /api/knowledge-assets/... surface. // One file = one Knowledge Asset (Design B / OT-RFC-44), carrying any number - // of member entities. WM → SWM → VM via the git-shaped verbs write → - // finalize → share → publish. + // of member entities. WM -> SWM -> VM via the git-shaped verbs write -> + // finalize -> share -> publish. /** * Create a KA + open its WM draft. Pass `quads` to atomically write+seal. @@ -1200,7 +1167,7 @@ export class DkgDaemonClient { * the draft will seal (quads present and `finalize !== false`), so the * one-shot seals AND shares to SWM. Pass `alsoShareSwm: false` to stop at a * sealed WM draft, or `finalize: false` to keep an unsealed editable WM draft. - * (The bare daemon route is a primitive — seal-only — and never auto-shares; + * (The bare daemon route is a primitive -- seal-only -- and never auto-shares; * the default-share lives here in the client.) */ async createKnowledgeAsset( @@ -1211,7 +1178,7 @@ export class DkgDaemonClient { quads?: Array<{ subject: string; predicate: string; object: string; graph: string }>; /** * Seal the draft after writing `quads` (default true). `false` keeps an - * editable WM draft that never touches the chain — the only lifecycle + * editable WM draft that never touches the chain -- the only lifecycle * available to local-only / on-chain-unregistered CGs. Cannot be combined * with `alsoShareSwm`/`alsoPublishVm` (those require a sealed assertion). */ @@ -1270,10 +1237,10 @@ export class DkgDaemonClient { quads: Array<{ subject: string; predicate: string; object: string; graph?: string }>, opts?: { subGraphName?: string }, ): Promise<{ written: number }> { - // Strip any per-quad `graph` at the client (CONTRACT §A): the write wire + // Strip any per-quad `graph` at the client (CONTRACT Section A): the write wire // shape is `{subject, predicate, object}` only and the daemon pins every - // quad to the per-KA WM graph (§0 invariant 2), so a caller-supplied or - // normalizer-emitted `graph` is dropped here — not just in the tool schema. + // quad to the per-KA WM graph (Section 0 invariant 2), so a caller-supplied or + // normalizer-emitted `graph` is dropped here -- not just in the tool schema. const wireQuads = quads.map((q) => ({ subject: q.subject, predicate: q.predicate, @@ -1286,7 +1253,7 @@ export class DkgDaemonClient { }); } - /** Seal the WM draft — computes the merkle root + signs the seal (git commit). */ + /** Seal the WM draft -- computes the merkle root + signs the seal (git commit). */ async knowledgeAssetFinalize( contextGraphId: string, name: string, @@ -1333,13 +1300,13 @@ export class DkgDaemonClient { }); } - /** Advance the SWM pointer (WM → SWM; git push origin ). */ + /** Advance the SWM pointer (WM -> SWM; git push origin ). */ async knowledgeAssetShare( contextGraphId: string, name: string, opts?: { subGraphName?: string; entities?: string[] | 'all'; skipSeal?: boolean }, ): Promise<{ swmShared: boolean; promotedCount: number; sealed: boolean; publishReady: boolean }> { - // Only include the optional fields when actually set — don't spread the whole + // Only include the optional fields when actually set -- don't spread the whole // opts object (which would carry `entities: undefined` / `subGraphName: // undefined` keys). Parity with MCP's knowledgeAssetShare body construction. const body: Record = { @@ -1352,7 +1319,7 @@ export class DkgDaemonClient { return this.post(`/api/knowledge-assets/${encodeURIComponent(name)}/swm/share`, body); } - /** Publish to VM — mint or update on chain (git push origin main). */ + /** Publish to VM -- mint or update on chain (git push origin main). */ async knowledgeAssetPublish( contextGraphId: string, name: string, diff --git a/packages/adapter-openclaw/src/index.ts b/packages/adapter-openclaw/src/index.ts index 74f9cc7e35..6d8b4f8324 100644 --- a/packages/adapter-openclaw/src/index.ts +++ b/packages/adapter-openclaw/src/index.ts @@ -20,7 +20,6 @@ export type { LocalWorkspaceDiscardRequest, LocalWorkspacePromoteRequest, LocalWorkspaceWriteRequest, - SharedMemoryWriteRequest, VerifiableMemoryPublishRequest, } from '@origintrail-official/dkg-core'; export { DkgChannelPlugin, CHANNEL_NAME } from './DkgChannelPlugin.js'; diff --git a/packages/adapter-openclaw/test/dkg-client.test.ts b/packages/adapter-openclaw/test/dkg-client.test.ts index 7c1a211aaa..813812904d 100644 --- a/packages/adapter-openclaw/test/dkg-client.test.ts +++ b/packages/adapter-openclaw/test/dkg-client.test.ts @@ -318,58 +318,6 @@ describe('DkgDaemonClient', () => { .toThrow('responded 404'); }); - // --------------------------------------------------------------------------- - // Workspace write - // --------------------------------------------------------------------------- - - it('share should POST quads to /api/shared-memory/write with localOnly defaulted to false', async () => { - // SWM gossips to peers in the context graph's allowlist by default — - // privacy is governed by the CG's curation policy, not by suppressing - // gossip. Aligned with Hermes adapter's default and the daemon's own - // default at packages/cli/src/daemon/routes/memory.ts:490 - // (`const localOnly = parsed.localOnly === true`). - fetchResponses.push( - new Response(JSON.stringify({ shareOperationId: 'op-1' }), { status: 200 }), - ); - - const quads = [{ subject: 'urn:a', predicate: 'urn:b', object: '"hello"' }]; - await client.share('research-x', quads); - - const body = JSON.parse(fetchCalls[0][1]?.body as string); - expect(body.contextGraphId).toBe('research-x'); - expect(body.quads).toHaveLength(1); - expect(body.localOnly).toBe(false); - }); - - it('share should pass through localOnly:true when the caller explicitly opts in', async () => { - fetchResponses.push( - new Response(JSON.stringify({ shareOperationId: 'op-2' }), { status: 200 }), - ); - - const quads = [{ subject: 'urn:a', predicate: 'urn:b', object: '"hello"' }]; - await client.share('research-x', quads, { localOnly: true }); - - const body = JSON.parse(fetchCalls[0][1]?.body as string); - expect(body.localOnly).toBe(true); - }); - - it('share should forward explicit team-visible sub-graph writes', async () => { - fetchResponses.push( - new Response(JSON.stringify({ shareOperationId: 'op-2' }), { status: 200 }), - ); - - const quads = [{ subject: 'urn:a', predicate: 'urn:b', object: '"hello"' }]; - await client.share('research-x', quads, { localOnly: false, subGraphName: 'protocols' }); - - const body = JSON.parse(fetchCalls[0][1]?.body as string); - expect(body).toEqual({ - contextGraphId: 'research-x', - quads, - localOnly: false, - subGraphName: 'protocols', - }); - }); - // --------------------------------------------------------------------------- // Working Memory assertion lifecycle // --------------------------------------------------------------------------- @@ -617,7 +565,7 @@ describe('DkgDaemonClient', () => { const [url, opts] = fetchCalls[0]; expect(url).toBe('http://localhost:9200/api/knowledge-assets/notes/wm/import-file'); expect(opts?.method).toBe('POST'); - // `body` must be a FormData — Node's fetch sets the multipart boundary automatically. + // `body` must be a FormData -- Node's fetch sets the multipart boundary automatically. expect(opts?.body).toBeInstanceOf(FormData); const form = opts?.body as FormData; expect(form.get('contextGraphId')).toBe('ctx'); @@ -848,7 +796,7 @@ describe('DkgDaemonClient', () => { expect(body.id).toBe('my-research'); expect(body.name).toBe('My Research'); expect(body.description).toBe('A research context graph'); - // No accessPolicy/allowedAgents passed when caller omits opts — the + // No accessPolicy/allowedAgents passed when caller omits opts -- the // tool handler decides the default privacy-mode. The client itself // is parameter-passing only. expect(body.accessPolicy).toBeUndefined(); @@ -894,7 +842,7 @@ describe('DkgDaemonClient', () => { const body = JSON.parse(fetchCalls[0][1]?.body as string); expect(body.accessPolicy).toBe(1); - // Empty array is dropped — daemon distinguishes "no allowlist" from + // Empty array is dropped -- daemon distinguishes "no allowlist" from // "empty allowlist". Sending [] would unhelpfully pin an empty // allowlist when the agent's creator-auto-include logic should // populate it. @@ -1010,7 +958,7 @@ describe('DkgDaemonClient', () => { expect(token === undefined || typeof token === 'string').toBe(true); }); - // ── OT-RFC-43 §10.5 — GitHub-shaped Knowledge Asset client ────────────── + // -- OT-RFC-43 Section 10.5 -- GitHub-shaped Knowledge Asset client -------------- describe('knowledge-assets surface', () => { const ok = (body: unknown = {}) => fetchResponses.push(new Response(JSON.stringify(body), { status: 200 })); @@ -1039,7 +987,7 @@ describe('DkgDaemonClient', () => { }); it('createKnowledgeAsset omits finalize when unspecified, but defaults alsoShareSwm:true (seal+share)', async () => { - // #1116 D5: quads present + finalize unspecified ⇒ the draft seals (server + // #1116 D5: quads present + finalize unspecified => the draft seals (server // default), so the combined CLIENT function also defaults alsoShareSwm to // true. `finalize` is still omitted (the server defaults it to seal). ok({ name: 'f', status: 'swm-shared' }); @@ -1050,9 +998,9 @@ describe('DkgDaemonClient', () => { expect(body().alsoShareSwm).toBe(true); }); - it('createKnowledgeAsset does NOT default alsoShareSwm when finalize:false (no seal ⇒ no share)', async () => { + it('createKnowledgeAsset does NOT default alsoShareSwm when finalize:false (no seal => no share)', async () => { // #1116 D5: an unsealed draft can't be shared, so the client must NOT - // default-on alsoShareSwm — the route guard would otherwise reject it. + // default-on alsoShareSwm -- the route guard would otherwise reject it. ok({ name: 'f', status: 'draft-open' }); await client.createKnowledgeAsset('cg-1', 'f', { finalize: false, @@ -1062,14 +1010,14 @@ describe('DkgDaemonClient', () => { }); it('createKnowledgeAsset does NOT default alsoShareSwm without quads', async () => { - // No quads ⇒ nothing to seal ⇒ no auto-share default. + // No quads => nothing to seal => no auto-share default. ok({ name: 'f', status: 'draft-open' }); await client.createKnowledgeAsset('cg-1', 'f'); expect(body()).not.toHaveProperty('alsoShareSwm'); }); it('createKnowledgeAsset honors an explicit alsoShareSwm:false over the seal-default', async () => { - // An explicit false must win — stop at a sealed WM draft. + // An explicit false must win -- stop at a sealed WM draft. ok({ name: 'f', status: 'wm-sealed' }); await client.createKnowledgeAsset('cg-1', 'f', { quads: [{ subject: 's', predicate: 'p', object: 'o', graph: '' }], @@ -1096,9 +1044,9 @@ describe('DkgDaemonClient', () => { expect(body().quads).toHaveLength(1); }); - it('knowledgeAssetWrite strips any per-quad `graph` at the client (CONTRACT §A)', async () => { + it('knowledgeAssetWrite strips any per-quad `graph` at the client (CONTRACT Section A)', async () => { ok({ written: 1 }); - // Even a NON-EMPTY graph must be dropped before the POST — the daemon pins + // Even a NON-EMPTY graph must be dropped before the POST -- the daemon pins // every quad to the per-KA WM graph, so the write wire shape is // {subject,predicate,object} only. Stripping at the client (not just the // tool schema) defends a hand-built or normalizer-emitted `graph`. diff --git a/packages/agent/src/dkg-agent-cg-registry.ts b/packages/agent/src/dkg-agent-cg-registry.ts index d52e49895d..c29c1e32a6 100644 --- a/packages/agent/src/dkg-agent-cg-registry.ts +++ b/packages/agent/src/dkg-agent-cg-registry.ts @@ -1098,7 +1098,7 @@ export class ContextGraphRegistryMethods extends DKGAgentBase { return [...subjects]; } - // ── ENDORSE ─���──────────────────────────────────────────────────────── + // ENDORSE } diff --git a/packages/agent/src/dkg-agent-publish.ts b/packages/agent/src/dkg-agent-publish.ts index 969d777596..323000c77f 100644 --- a/packages/agent/src/dkg-agent-publish.ts +++ b/packages/agent/src/dkg-agent-publish.ts @@ -110,6 +110,7 @@ import { computeTripleHashV10 as computeTripleHash, computeFlatKCRootV10 as computeFlatKCRoot, skolemizeByEntity, isReservedSubject, canonicalPublishPayload, generatedPrivateCatalogTripleKeys, + createKnowledgeAssetVmPublishLiftRequest, resolveLiftWorkspaceSlice, validateLiftPublishPayload, subtractFinalizedExactQuads, @@ -125,7 +126,7 @@ import { KA_ID_PRED, RESERVED_UAL_PRED, WM_CURRENT_ASSERTION_PRED, SWM_CURRENT_ASSERTION_PRED, VM_CURRENT_ASSERTION_PRED, type CollectedACK, type LiftAuthorityProof, type LiftTransitionType, - type LiftRequest, type LiftRequestAuthorSeal, + type LiftRequest, type LiftRequestAuthorSeal, type KnowledgeAssetVmPublishRequest, type WorkspaceAgentRecipient, type WorkspaceAgentRecipientResolution, type WorkspaceAgentRecipientResolverInput, @@ -1524,6 +1525,8 @@ export class PublishMethods extends DKGAgentBase { onPhase?: PhaseCallback; operationCtx?: OperationContext; precomputedUpdateAttestation?: PublishOptions['precomputedUpdateAttestation']; + publisherOverride?: DKGPublisher; + subGraphName?: string; }, ): Promise { const ctx = opts?.operationCtx ?? createOperationContext('update'); @@ -1573,7 +1576,7 @@ export class PublishMethods extends DKGAgentBase { // same-CG update as an explicit remap. const updateEncryptInlinePayload = await this._resolveEncryptInlinePayload( contextGraphId, - undefined, + opts?.subGraphName, undefined, undefined, updateOnChainId @@ -1594,7 +1597,7 @@ export class PublishMethods extends DKGAgentBase { const updateEncryptInlineChunked = isCuratedUpdate ? await this._resolveEncryptInlineChunked( contextGraphId, - undefined, + opts?.subGraphName, undefined, undefined, updateOnChainId @@ -1637,7 +1640,8 @@ export class PublishMethods extends DKGAgentBase { updateQuads = [...nonCatalogQuads, ...catalogFloor]; } - const result = await this.publisher.update(kaId, { + const publisher = opts?.publisherOverride ?? this.publisher; + const result = await publisher.update(kaId, { contextGraphId, quads: updateQuads, privateQuads, @@ -1645,6 +1649,7 @@ export class PublishMethods extends DKGAgentBase { publishContextGraphId: updateOnChainId ?? undefined, operationCtx: ctx, onPhase, + subGraphName: opts?.subGraphName, precomputedUpdateAttestation: opts?.precomputedUpdateAttestation, trustedNonManifestCatalogTriples: shouldInjectCuratedCatalogFloor ? generatedPrivateCatalogTripleKeys(contextGraphId) @@ -3374,6 +3379,575 @@ export class PublishMethods extends DKGAgentBase { * hasn't been promoted yet, publish will see an empty/wrong quad * set and the merkleRoot sanity check inside `publish()` will fire. */ + async resolveFinalizedAssertionVmPublishIntent(this: DKGAgent, + contextGraphId: string, + name: string, + opts?: { + subGraphName?: string; + publishEpochs?: number; + clearSharedMemoryAfter?: boolean; + publisherNodeIdentityIdOverride?: bigint | `${bigint}`; + publisherOverride?: DKGPublisher; + }, + ): Promise { + const agentAddress = this.defaultAgentAddress ?? this.peerId; + const publisher = opts?.publisherOverride ?? this.publisher; + const history = await this.assertion.history(contextGraphId, name, { + agentAddress, + ...(opts?.subGraphName ? { subGraphName: opts.subGraphName } : {}), + }); + if (!history) { + throw new Error( + `publishFromFinalizedAssertion: assertion "${name}" in context graph "${contextGraphId}" is not finalized or does not exist.`, + ); + } + if (!(await publisher.hasSwmShareComplete(contextGraphId, name, agentAddress, opts?.subGraphName))) { + throw Object.assign( + new Error( + `Cannot publish "${name}" in context graph "${contextGraphId}": it is not a complete full share ` + + `resident in Shared Memory. Seal and share the full asset before publishing.`, + ), + { code: 'PUBLISH_NOT_FULL_SHARE' }, + ); + } + + const metaGraph = contextGraphMetaUri(contextGraphId); + const assertionUri = contextGraphAssertionUri(contextGraphId, agentAddress, name, opts?.subGraphName); + const metaResult = await this.store.query( + `CONSTRUCT { <${assertionUri}> ?p ?o } WHERE { GRAPH <${metaGraph}> { <${assertionUri}> ?p ?o } }`, + ); + const metaQuads = metaResult.type === 'quads' ? metaResult.quads : []; + const seal = parseAssertionSealQuads(metaQuads, assertionUri); + if (!seal) { + throw Object.assign( + new Error( + `Cannot publish "${name}" asynchronously: the current SWM share is not sealed. ` + + `Finalize and share the full asset before publishing.`, + ), + { code: 'PUBLISH_INTENT_STALE' }, + ); + } + + const latestPromote = [...history.events] + .reverse() + .find((event) => event.type === 'promoted' && event.shareOperationId); + const shareOperationId = latestPromote?.shareOperationId?.trim() ?? history.currentShareOperationId?.trim(); + if (!shareOperationId) { + throw Object.assign( + new Error( + `Cannot publish "${name}" asynchronously: the current SWM share is missing a shareOperationId. ` + + `Re-share the asset through /api/knowledge-assets/${encodeURIComponent(name)}/swm/share before publishing.`, + ), + { code: 'PUBLISH_INTENT_STALE' }, + ); + } + + const roots = [...new Set([ + ...(latestPromote?.rootEntities ?? []), + ...(latestPromote?.rootEntities?.length ? [] : seal.rootEntities), + ])].sort(); + if (roots.length === 0) { + throw Object.assign( + new Error( + `Cannot publish "${name}" asynchronously: the current SWM share has no recorded root entities. ` + + `Re-share the full asset before publishing.`, + ), + { code: 'PUBLISH_INTENT_STALE' }, + ); + } + + const merkleBare = ethers.hexlify(seal.merkleRoot).slice(2); + if (!merkleBare) { + throw Object.assign( + new Error( + `Cannot publish "${name}" asynchronously: the current SWM share has no sealed assertion pointer. ` + + `Finalize and share the full asset before publishing.`, + ), + { code: 'PUBLISH_INTENT_STALE' }, + ); + } + const sealMerkleRoot = (merkleBare.startsWith('0x') ? merkleBare : `0x${merkleBare}`) as `0x${string}`; + const queuedSeal: LiftRequestAuthorSeal = { + merkleRoot: ethers.hexlify(seal.merkleRoot) as `0x${string}`, + authorAddress: ethers.getAddress(seal.authorAddress) as `0x${string}`, + signature: { + r: ethers.hexlify(seal.authorAttestationR) as `0x${string}`, + vs: ethers.hexlify(seal.authorAttestationVS) as `0x${string}`, + }, + schemeVersion: seal.authorSchemeVersion, + ...(seal.reservedKaId !== undefined ? { reservedKaId: seal.reservedKaId.toString() as `${bigint}` } : {}), + }; + const publisherOverrideString = opts?.publisherNodeIdentityIdOverride !== undefined + ? opts.publisherNodeIdentityIdOverride.toString() as `${bigint}` + : undefined; + const canonicalIntent = { + contextGraphId, + name, + subGraphName: opts?.subGraphName ?? null, + shareOperationId, + roots, + sealMerkleRoot: sealMerkleRoot.toLowerCase(), + seal: queuedSeal, + sealChainId: seal.chainId.toString(), + sealKav10Address: ethers.getAddress(seal.kav10Address), + sealFinalizedAtIso: seal.finalizedAtIso, + wmCurrentAssertion: history.wmCurrentAssertion ?? null, + swmCurrentAssertion: history.swmCurrentAssertion ?? null, + vmCurrentAssertion: history.vmCurrentAssertion ?? null, + kaNumber: history.kaNumber ?? null, + reservedUal: history.reservedUal ?? null, + publishEpochs: opts?.publishEpochs ?? null, + clearSharedMemoryAfter: opts?.clearSharedMemoryAfter ?? null, + publisherNodeIdentityIdOverride: publisherOverrideString ?? null, + }; + const intentKey = `sha256:${createHash('sha256').update(JSON.stringify(canonicalIntent)).digest('hex')}`; + + return { + contextGraphId, + name, + ...(opts?.subGraphName ? { subGraphName: opts.subGraphName } : {}), + shareOperationId, + roots, + seal: queuedSeal, + sealChainId: seal.chainId.toString() as `${bigint}`, + sealKav10Address: ethers.getAddress(seal.kav10Address) as `0x${string}`, + sealFinalizedAtIso: seal.finalizedAtIso, + sealMerkleRoot, + intentKey, + ...(history.wmCurrentAssertion ? { wmCurrentAssertion: history.wmCurrentAssertion } : {}), + ...(history.swmCurrentAssertion ? { swmCurrentAssertion: history.swmCurrentAssertion } : {}), + ...(history.vmCurrentAssertion ? { vmCurrentAssertion: history.vmCurrentAssertion } : {}), + ...(history.kaNumber ? { kaNumber: history.kaNumber } : {}), + ...(history.reservedUal ? { reservedUal: history.reservedUal } : {}), + ...(opts?.publishEpochs !== undefined ? { publishEpochs: opts.publishEpochs } : {}), + ...(opts?.clearSharedMemoryAfter !== undefined ? { clearSharedMemoryAfter: opts.clearSharedMemoryAfter } : {}), + ...(publisherOverrideString !== undefined ? { publisherNodeIdentityIdOverride: publisherOverrideString } : {}), + }; + } + + async preflightKnowledgeAssetVmPublishSnapshot( + this: DKGAgent, + request: KnowledgeAssetVmPublishRequest, + ): Promise { + const liftRequest: LiftRequest = createKnowledgeAssetVmPublishLiftRequest(request); + try { + const resolved = await resolveLiftWorkspaceSlice({ + store: this.store, + graphManager: new GraphManager(this.store), + request: liftRequest, + publicSnapshotStore: this.publicSnapshotStore, + }); + validateLiftPublishPayload({ + request: liftRequest, + resolved, + }); + if (resolved.quads.length === 0 && (resolved.privateQuads ?? []).length === 0) { + throw new Error( + `No queued shared-memory snapshot quads for context graph ${request.contextGraphId} ` + + `share operation ${request.shareOperationId}`, + ); + } + } catch (err) { + const wrapped = new Error( + `Cannot enqueue VM publish for "${request.name}" because share snapshot ` + + `${request.shareOperationId} is unavailable or stale. Re-share the knowledge asset before enqueueing: ` + + (err instanceof Error ? err.message : String(err)), + ); + (wrapped as Error & { code?: string }).code = 'PUBLISH_INTENT_STALE'; + throw wrapped; + } + } + + async publishQueuedKnowledgeAssetVmPublish( + this: DKGAgent, + request: KnowledgeAssetVmPublishRequest, + publishOptions: PublishOptions, + opts?: { + operationCtx?: OperationContext; + onPhase?: PhaseCallback; + publisherOverride?: DKGPublisher; + }, + ): Promise { + const ctx = opts?.operationCtx ?? publishOptions.operationCtx ?? createOperationContext('publishFromSWM'); + const publisher = opts?.publisherOverride ?? this.publisher; + const agentAddress = this.defaultAgentAddress ?? this.peerId; + const assertionUri = contextGraphAssertionUri( + request.contextGraphId, + agentAddress, + request.name, + request.subGraphName, + ); + const lifecycleUri = assertionLifecycleUri( + request.contextGraphId, + agentAddress, + request.name, + request.subGraphName, + ); + const metaGraph = contextGraphMetaUri(request.contextGraphId); + + const seal: AssertionSeal = { + merkleRoot: ethers.getBytes(request.seal.merkleRoot), + authorAddress: ethers.getAddress(request.seal.authorAddress), + authorAttestationR: ethers.getBytes(request.seal.signature.r), + authorAttestationVS: ethers.getBytes(request.seal.signature.vs), + authorSchemeVersion: request.seal.schemeVersion, + chainId: BigInt(request.sealChainId), + kav10Address: ethers.getAddress(request.sealKav10Address), + finalizedAtIso: request.sealFinalizedAtIso, + rootEntities: [...request.roots], + ...(request.seal.reservedKaId !== undefined ? { reservedKaId: BigInt(request.seal.reservedKaId) } : {}), + }; + const queuedMerkleRoot = ethers.hexlify(seal.merkleRoot).toLowerCase(); + if (queuedMerkleRoot !== request.sealMerkleRoot.toLowerCase()) { + throw Object.assign( + new Error( + `Queued VM publish for "${request.name}" has inconsistent seal roots: ` + + `${request.sealMerkleRoot} != ${queuedMerkleRoot}.`, + ), + { code: 'PUBLISH_INTENT_STALE' }, + ); + } + + if ( + typeof this.chain.getEvmChainId === 'function' && + typeof this.chain.getKnowledgeAssetsLifecycleAddress === 'function' + ) { + const liveChainId = await this.chain.getEvmChainId(); + const liveKav10 = await this.chain.getKnowledgeAssetsLifecycleAddress(); + if (liveChainId !== seal.chainId) { + throw new Error( + `publishQueuedKnowledgeAssetVmPublish: seal binds chainId=${seal.chainId.toString()} but daemon ` + + `is configured for chainId=${liveChainId.toString()}. Re-finalize the assertion against the target chain.`, + ); + } + if (liveKav10.toLowerCase() !== seal.kav10Address.toLowerCase()) { + throw new Error( + `publishQueuedKnowledgeAssetVmPublish: seal binds KAv10=${seal.kav10Address} but daemon ` + + `is configured for KAv10=${liveKav10}.`, + ); + } + } + + const snapshotQuads = publishOptions.quads.map((q) => ({ ...q, graph: '' })); + const snapshotPrivateQuads = (publishOptions.privateQuads ?? []).map((q) => ({ ...q, graph: '' })); + if (snapshotQuads.length === 0 && snapshotPrivateQuads.length === 0) { + throw new Error( + `No queued shared-memory snapshot quads for context graph ${request.contextGraphId} ` + + `share operation ${request.shareOperationId}`, + ); + } + + const pointerRes = await this.store.query( + `SELECT ?vm ?kaNum WHERE { GRAPH <${metaGraph}> { + OPTIONAL { <${lifecycleUri}> <${VM_CURRENT_ASSERTION_PRED}> ?vm } + OPTIONAL { <${lifecycleUri}> <${KA_ID_PRED}> ?kaNum } + } } LIMIT 1`, + ); + const stripLit = (v?: string) => v?.replace(/^"/, '').replace(/"(\^\^<[^>]+>)?$/, ''); + const pointerRow = pointerRes.type === 'bindings' ? pointerRes.bindings[0] : undefined; + const vmCurrent = request.vmCurrentAssertion ?? stripLit(pointerRow?.['vm']); + const stampedNumberStr = request.kaNumber ?? stripLit(pointerRow?.['kaNum']); + + let packedKaId: bigint | undefined; + if (stampedNumberStr != null && stampedNumberStr !== '') { + try { + const authorBits = BigInt(ethers.getAddress(seal.authorAddress)); + packedKaId = (authorBits << 96n) | BigInt(stampedNumberStr); + } catch (err) { + this.log.warn( + ctx, + `Failed to re-pack queued kaId number "${stampedNumberStr}" for <${lifecycleUri}>: ` + + (err instanceof Error ? err.message : String(err)), + ); + } + } + + const newMerkleHexBare = ethers.hexlify(seal.merkleRoot).slice(2); + let result: PublishResult; + const clearPublishedRoots = async (label: string): Promise => { + try { + await publisher.clearPublishedSwmRoots( + request.contextGraphId, + [...request.roots], + request.subGraphName, + ctx, + ); + } catch (err) { + this.log.warn( + ctx, + `Failed to clear published SWM roots after confirmed queued ${label} of <${lifecycleUri}>: ` + + (err instanceof Error ? err.message : String(err)), + ); + } + }; + const clearRemainingSharedMemory = async (): Promise => { + try { + await publisher.clearRemainingSharedMemory(request.contextGraphId, request.subGraphName, ctx); + } catch (err) { + this.log.warn( + ctx, + `Failed to clear remaining SWM after confirmed queued publish of <${lifecycleUri}>: ` + + (err instanceof Error ? err.message : String(err)), + ); + } + }; + + if (vmCurrent && packedKaId !== undefined) { + const updateAttestation = await this._buildPrecomputedUpdateAttestationForSeal( + packedKaId, + seal, + publisher, + ); + result = await this.update( + packedKaId, + request.contextGraphId, + snapshotQuads, + snapshotPrivateQuads, + { + operationCtx: ctx, + onPhase: opts?.onPhase ?? publishOptions.onPhase, + precomputedUpdateAttestation: updateAttestation, + publisherOverride: publisher, + subGraphName: request.subGraphName, + }, + ); + + if (result.status === 'confirmed') { + await clearPublishedRoots('update'); + if (request.clearSharedMemoryAfter === true) { + await clearRemainingSharedMemory(); + } + } + + if (result.status === 'confirmed' || result.status === 'tentative') { + try { + const priorBare = vmCurrent.startsWith('0x') ? vmCurrent.slice(2) : vmCurrent; + const priorUri = `${lifecycleUri}#assertion-${priorBare}`; + await this._stampPointer(lifecycleUri, VM_CURRENT_ASSERTION_PRED, newMerkleHexBare, metaGraph); + await this._stampPointerIfDivergedFromVm(lifecycleUri, WM_CURRENT_ASSERTION_PRED, newMerkleHexBare, metaGraph); + await this.store.insert([ + { subject: lifecycleUri, predicate: 'http://www.w3.org/ns/prov#wasRevisionOf', object: priorUri, graph: metaGraph }, + { subject: priorUri, predicate: VM_CURRENT_ASSERTION_PRED, object: `"${priorBare}"`, graph: metaGraph }, + ]); + } catch (err) { + this.log.warn( + ctx, + `Failed to stamp queued update provenance for <${lifecycleUri}>: ` + + (err instanceof Error ? err.message : String(err)), + ); + } + } + } else { + const recoveredReservedKaId = seal.reservedKaId ?? packedKaId; + const onChainCapable = + typeof this.chain.getEvmChainId === 'function' && + typeof this.chain.getKnowledgeAssetsLifecycleAddress === 'function'; + if (recoveredReservedKaId === undefined && onChainCapable) { + throw new Error( + `publishQueuedKnowledgeAssetVmPublish: cannot recover the reservedKaId for <${assertionUri}>. ` + + `Re-finalize the assertion before publishing asynchronously.`, + ); + } + const resolvedEncryptInlinePayload = await this._resolveEncryptInlinePayload( + request.contextGraphId, + request.subGraphName, + undefined, + publishOptions.publishContextGraphId, + ); + const resolvedEncryptInlineChunked = await this._resolveEncryptInlineChunked( + request.contextGraphId, + request.subGraphName, + undefined, + publishOptions.publishContextGraphId, + ); + const encryptInlinePayload = resolvedEncryptInlinePayload ?? publishOptions.encryptInlinePayload; + const encryptInlineChunked = resolvedEncryptInlineChunked ?? publishOptions.encryptInlineChunked; + result = await publisher.publish({ + ...publishOptions, + contextGraphId: request.contextGraphId, + quads: snapshotQuads, + privateQuads: snapshotPrivateQuads.length > 0 ? snapshotPrivateQuads : undefined, + publisherPeerId: publishOptions.publisherPeerId ?? this.peerId, + subGraphName: request.subGraphName, + operationCtx: ctx, + onPhase: opts?.onPhase ?? publishOptions.onPhase, + skipContextGraphEnsure: true, + v10ACKProvider: publishOptions.v10ACKProvider ?? this.createV10ACKProvider(request.contextGraphId), + publishEpochs: request.publishEpochs ?? publishOptions.publishEpochs, + publisherNodeIdentityIdOverride: request.publisherNodeIdentityIdOverride !== undefined + ? BigInt(request.publisherNodeIdentityIdOverride) + : publishOptions.publisherNodeIdentityIdOverride, + precomputedAttestation: { + expectedMerkleRoot: seal.merkleRoot, + authorAddress: seal.authorAddress, + signature: { r: seal.authorAttestationR, vs: seal.authorAttestationVS }, + schemeVersion: seal.authorSchemeVersion, + reservedKaId: recoveredReservedKaId ?? 0n, + }, + encryptInlinePayload, + encryptInlineChunked, + }); + + if (result.status === 'confirmed' && result.onChainResult) { + try { + const receiptQuads = buildAssertionPublishReceiptQuads({ + assertionUri, + metaGraph, + txHash: result.onChainResult.txHash ?? '', + blockNumber: BigInt(result.onChainResult.blockNumber ?? 0), + kaId: result.onChainResult.batchId ?? 0n, + }); + await this.store.insert(receiptQuads); + } catch (err) { + this.log.warn( + ctx, + `Failed to write publish receipt for <${assertionUri}>: ` + + (err instanceof Error ? err.message : String(err)), + ); + } + } + + if (result.status === 'confirmed') { + await clearPublishedRoots('publish'); + if (request.clearSharedMemoryAfter === true) { + await clearRemainingSharedMemory(); + } + } + } + + if (result.status === 'confirmed' || result.status === 'tentative') { + try { + await this._stampPointer(lifecycleUri, VM_CURRENT_ASSERTION_PRED, newMerkleHexBare, metaGraph); + const MEMORY_LAYER_PRED = 'http://dkg.io/ontology/memoryLayer'; + const STATE_PRED = 'http://dkg.io/ontology/state'; + for (const subj of [lifecycleUri, assertionUri]) { + await this.store.deleteByPattern({ subject: subj, predicate: MEMORY_LAYER_PRED, graph: metaGraph }); + await this.store.insert([ + { subject: subj, predicate: MEMORY_LAYER_PRED, object: `"${MemoryLayer.VerifiableMemory}"`, graph: metaGraph }, + ]); + } + await this.store.deleteByPattern({ subject: lifecycleUri, predicate: STATE_PRED, graph: metaGraph }); + await this.store.insert([ + { subject: lifecycleUri, predicate: STATE_PRED, object: '"published"', graph: metaGraph }, + ]); + if (result.ual) { + const PUBLISHED_UAL_PRED = 'http://dkg.io/ontology/publishedUal'; + await this.store.deleteByPattern({ subject: lifecycleUri, predicate: PUBLISHED_UAL_PRED, graph: metaGraph }); + await this.store.insert([ + { subject: lifecycleUri, predicate: PUBLISHED_UAL_PRED, object: `"${result.ual}"`, graph: metaGraph }, + ]); + } + if (result.status === 'confirmed' && result.onChainResult) { + const ASSERTION_GRAPH_PRED = 'http://dkg.io/ontology/assertionGraph'; + const vmKaId = packedKaId ?? seal.reservedKaId ?? result.onChainResult.kaId ?? result.kaId; + if (vmKaId !== undefined && vmKaId !== null) { + const vmKaIdBig = BigInt(vmKaId); + const vmAuthor = '0x' + (vmKaIdBig >> 96n).toString(16).padStart(40, '0'); + const vmNumber = vmKaIdBig & ((1n << 96n) - 1n); + const vmGraph = contextGraphLayerUri( + request.contextGraphId, + MemoryLayer.VerifiableMemory, + vmAuthor, + vmNumber, + request.subGraphName, + ); + await this.store.deleteByPattern({ subject: lifecycleUri, predicate: ASSERTION_GRAPH_PRED, graph: metaGraph }); + await this.store.insert([ + { subject: lifecycleUri, predicate: ASSERTION_GRAPH_PRED, object: vmGraph, graph: metaGraph }, + ]); + const wmGraph = contextGraphLayerUri( + request.contextGraphId, + MemoryLayer.WorkingMemory, + vmAuthor, + vmNumber, + request.subGraphName, + ); + await this.store.deleteByPattern({ subject: wmGraph, predicate: MEMORY_LAYER_PRED, graph: metaGraph }); + await this.store.insert([ + { subject: wmGraph, predicate: MEMORY_LAYER_PRED, object: `"${MemoryLayer.VerifiableMemory}"`, graph: metaGraph }, + ]); + } + } + } catch (err) { + this.log.warn( + ctx, + `Failed to stamp queued VM lifecycle marker for <${lifecycleUri}>: ` + + (err instanceof Error ? err.message : String(err)), + ); + } + } + + if (result.status === 'confirmed' && result.onChainResult) { + const rootEntities = result.kaManifest.length > 0 + ? result.kaManifest.map((ka) => ka.rootEntity) + : [...request.roots]; + const broadcastCgId = publishOptions.publishContextGraphId; + const keepRootCopyOnLabel = true; + const msg: FinalizationMessageMsg = { + ual: result.ual, + contextGraphId: request.contextGraphId, + kcMerkleRoot: result.merkleRoot, + txHash: result.onChainResult.txHash ?? '', + blockNumber: result.onChainResult.blockNumber ?? 0, + txIndex: result.onChainResult.txIndex ?? 0, + batchId: result.onChainResult.batchId ?? 0n, + startKAId: result.onChainResult.startKAId ?? 0n, + endKAId: result.onChainResult.endKAId ?? 0n, + publisherAddress: result.onChainResult.publisherAddress ?? '', + rootEntities, + timestampMs: Date.now(), + operationId: ctx.operationId, + targetContextGraphId: result.contextGraphError ? undefined : broadcastCgId, + subGraphName: request.subGraphName, + keepRootCopyOnLabel, + }; + const topic = contextGraphFinalizationTopic(request.contextGraphId); + try { + await this.gossip.publish(topic, encodeFinalizationMessage(msg)); + this.log.info(ctx, `Broadcast queued finalization for ${result.ual} to ${topic}${broadcastCgId ? ` (contextGraph=${broadcastCgId})` : ''}`); + } catch { + this.log.warn(ctx, `No peers subscribed to ${topic} yet`); + } + + try { + const gm = new GraphManager(this.store); + const wsMetaGraph = request.subGraphName + ? gm.sharedMemoryMetaUri(request.contextGraphId, request.subGraphName) + : contextGraphWorkspaceMetaGraphUri(request.contextGraphId); + const keepLiteral = `"${keepRootCopyOnLabel}"`; + for (const root of rootEntities.filter(isSafeIri)) { + await this.store.deleteByPattern({ + subject: root, + predicate: KEEP_ROOT_COPY_PREDICATE, + graph: wsMetaGraph, + }); + await this.store.insert([{ + subject: root, + predicate: KEEP_ROOT_COPY_PREDICATE, + object: keepLiteral, + graph: wsMetaGraph, + }]); + } + } catch (err) { + this.log.warn(ctx, `Failed to persist keepRootCopyOnLabel signal for ${result.ual}: ${err instanceof Error ? err.message : String(err)}`); + } + } + + if (result.status === 'confirmed') { + try { + await publisher.clearSwmShareComplete(request.contextGraphId, request.name, agentAddress, request.subGraphName); + } catch (err) { + this.log.warn( + ctx, + `Failed to clear swmShareComplete after queued publish of <${lifecycleUri}>: ` + + (err instanceof Error ? err.message : String(err)), + ); + } + } + + return { ...result, assertionUri, seal }; + } + async publishFromFinalizedAssertion(this: DKGAgent, contextGraphId: string, name: string, @@ -3384,9 +3958,11 @@ export class PublishMethods extends DKGAgentBase { publisherNodeIdentityIdOverride?: bigint; publishEpochs?: number; clearSharedMemoryAfter?: boolean; + publisherOverride?: DKGPublisher; }, ): Promise { const agentAddress = this.defaultAgentAddress ?? this.peerId; + const publisher = opts?.publisherOverride ?? this.publisher; const assertionUri = contextGraphAssertionUri( contextGraphId, agentAddress, @@ -3447,7 +4023,7 @@ export class PublishMethods extends DKGAgentBase { // paths. A legitimate full-share publish has the marker (assertionPromote set // it on the full share); an UPDATE re-publish re-sets it via the required // re-promote (a confirmed publish drained SWM, so a re-publish MUST re-share). - if (!(await this.publisher.hasSwmShareComplete(contextGraphId, name, agentAddress, opts?.subGraphName))) { + if (!(await publisher.hasSwmShareComplete(contextGraphId, name, agentAddress, opts?.subGraphName))) { throw Object.assign( new Error( `Cannot publish "${name}" in context graph "${contextGraphId}": it is not a complete full share ` + @@ -3527,6 +4103,7 @@ export class PublishMethods extends DKGAgentBase { const updateAttestation = await this._buildPrecomputedUpdateAttestationForSeal( packedKaId, seal, + publisher, ); result = await this.update( packedKaId, @@ -3537,6 +4114,8 @@ export class PublishMethods extends DKGAgentBase { operationCtx: opts?.operationCtx, onPhase: opts?.onPhase, precomputedUpdateAttestation: updateAttestation, + publisherOverride: publisher, + subGraphName: opts?.subGraphName, }, ); @@ -3547,7 +4126,7 @@ export class PublishMethods extends DKGAgentBase { // that mirrored the share), so SWM and VM permanently disagreed. if (result.status === 'confirmed') { try { - await this.publisher.clearPublishedSwmRoots( + await publisher.clearPublishedSwmRoots( contextGraphId, seal.rootEntities, opts?.subGraphName, @@ -3669,6 +4248,7 @@ export class PublishMethods extends DKGAgentBase { // where it is never submitted. reservedKaId: recoveredReservedKaId ?? 0n, }, + publisherOverride: publisher, }, ); @@ -3826,7 +4406,7 @@ export class PublishMethods extends DKGAgentBase { // re-sets the marker via assertionPromote). Covers BOTH MINT and UPDATE. if (result.status === 'confirmed') { try { - await this.publisher.clearSwmShareComplete(contextGraphId, name, agentAddress, opts?.subGraphName); + await publisher.clearSwmShareComplete(contextGraphId, name, agentAddress, opts?.subGraphName); } catch (err) { this.log.warn( opts?.operationCtx ?? createOperationContext('publishFromSWM'), @@ -3852,6 +4432,7 @@ export class PublishMethods extends DKGAgentBase { this: DKGAgent, kaId: bigint, seal: AssertionSeal, + publisherOverride?: DKGPublisher, ): Promise> { const typedData = buildUpdateAuthorAttestationTypedData({ chainId: seal.chainId, @@ -3871,7 +4452,8 @@ export class PublishMethods extends DKGAgentBase { r = ethers.getBytes(sig.r); vs = ethers.getBytes(sig.yParityAndS); } else { - const fallbackAddress = await this.publisher.publisherFallbackAuthorAddress(); + const publisher = publisherOverride ?? this.publisher; + const fallbackAddress = await publisher.publisherFallbackAuthorAddress(); if (!fallbackAddress || fallbackAddress.toLowerCase() !== seal.authorAddress.toLowerCase()) { throw new Error( `publishFromFinalizedAssertion (update path): cannot re-sign UpdateAuthorAttestation for author ` + @@ -3879,7 +4461,7 @@ export class PublishMethods extends DKGAgentBase { `Use the /api/update route with a pre-signed UpdateAuthorAttestation instead.`, ); } - const compact = await this.publisher.signAuthorAttestationAsPublisher(typedData); + const compact = await publisher.signAuthorAttestationAsPublisher(typedData); r = compact.r; vs = compact.vs; } @@ -4035,8 +4617,8 @@ export class PublishMethods extends DKGAgentBase { * SWM state can promote it to canonical without re-downloading the full payload. * * #1116 (round 9) — INTENTIONALLY NOT marker-gated. This is the - * "publish an arbitrary caller-selected SWM slice" escape hatch (the legacy - * /api/shared-memory/publish path, #1087): it mints a FRESH inline seal over the + * "publish an arbitrary caller-selected SWM slice" internal escape hatch + * retained for substrate mechanics (#1087): it mints a FRESH inline seal over the * selected slice rather than consuming a finalized named lifecycle, so the * swmShareComplete full-share invariant does not apply. The marker gate lives on * `publishFromFinalizedAssertion` (the named-lifecycle /vm/publish path) only. @@ -4114,6 +4696,7 @@ export class PublishMethods extends DKGAgentBase { preSignedAuthorAttestation?: PreSignedAuthorAttestation; /** Author scheme version override (defaults to AUTHOR_SCHEME_VERSION_V1). */ schemeVersion?: number; + publisherOverride?: DKGPublisher; }, ): Promise { const ctx = options?.operationCtx ?? createOperationContext('publishFromSWM'); @@ -4129,8 +4712,8 @@ export class PublishMethods extends DKGAgentBase { const v10ACKProvider = this.createV10ACKProvider(contextGraphId); // OT-RFC-49 — inject the public `_catalog` projection for a curated CG - // publishing from raw SWM (the `/api/shared-memory/write` + `/publish` - // shortcut) that did NOT go through `assertionFinalize` — which is what + // publishing from raw SWM through the internal substrate shortcut that did + // NOT go through `assertionFinalize` — which is what // normally injects the catalog. Without it the reloaded payload carries no // catalog, the on-chain catalog commitment stays zero, and the core ACK // falls back to SWM-lookup and DECLINEs `NO_DATA_IN_SWM`. Run BEFORE the @@ -4228,7 +4811,8 @@ export class PublishMethods extends DKGAgentBase { this.log.info(ctx, `LU-11: curated CG ${contextGraphId} — chunked path active (per-chunk SWM gossip + V2 ACK)`); } - const result = await this.publisher.publishFromSharedMemory(contextGraphId, selection, { + const publisher = options?.publisherOverride ?? this.publisher; + const result = await publisher.publishFromSharedMemory(contextGraphId, selection, { operationCtx: ctx, clearSharedMemoryAfter: options?.clearSharedMemoryAfter, onPhase: options?.onPhase, diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index 778eaac007..290a619cae 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -442,6 +442,8 @@ export interface AssertionHistoryDescriptor extends AssertionDescriptor { swmCurrentAssertion?: string; /** Merkle hex of the assertion confirmed on-chain (VM). */ vmCurrentAssertion?: string; + /** Latest SWM share operation id stamped on the lifecycle subject. */ + currentShareOperationId?: string; /** OT-RFC-43 §10.5.4 derived overall status. */ status: KaStatus; /** The per-author KA NUMBER (low 96 bits) stamped at finalize, as a string. */ @@ -1908,7 +1910,7 @@ export class DKGAgent extends DKGAgentBase { ): Promise<{ seeded: number; fromLayer: 'swm' | 'vm'; entities: number }> { return agent.publisher.assertionPullFrom(contextGraphId, name, agentAddress, sourceLayer, opts); }, - async promote(contextGraphId: string, name: string, opts?: { entities?: string[] | 'all'; subGraphName?: string; authorAgentAddress?: string; preSignedAuthorAttestation?: PreSignedAuthorAttestation; awaitCuratorAck?: boolean; curatorAckTimeoutMs?: number; skipSeal?: boolean }): Promise<{ promotedCount: number; sealed: boolean; publishReady: boolean }> { + async promote(contextGraphId: string, name: string, opts?: { entities?: string[] | 'all'; subGraphName?: string; authorAgentAddress?: string; preSignedAuthorAttestation?: PreSignedAuthorAttestation; awaitCuratorAck?: boolean; curatorAckTimeoutMs?: number; skipSeal?: boolean }): Promise<{ promotedCount: number; sealed: boolean; publishReady: boolean; shareOperationId?: string }> { // Seal-before-share: the on-chain publish path // (`publishFromFinalizedAssertion`) requires a FINALIZED assertion, and // the seal must be computed over the Working-Memory content BEFORE @@ -2029,7 +2031,7 @@ export class DKGAgent extends DKGAgentBase { const trustedNonManifestCatalogTriples = isPrivateContextGraph ? generatedPrivateCatalogTripleKeys(contextGraphId) : undefined; - const { promotedCount, gossipMessage, promotedAllRoots } = await agent.publisher.assertionPromote( + const { promotedCount, gossipMessage, promotedAllRoots, shareOperationId } = await agent.publisher.assertionPromote( contextGraphId, name, agentAddress, { ...opts, @@ -2077,8 +2079,8 @@ export class DKGAgent extends DKGAgentBase { // different merkleRoot and fail the seal guard. The seal still EXISTS // (`sealed:true`), but the asset is NOT publish-ready. The single-author // happy path skips no roots ⇒ promotedAllRoots:true ⇒ publishReady:true. - const publishReady = promotingAllEntities && sealed && promotedAllRoots; - return { promotedCount, sealed, publishReady }; + const publishReady = promotingAllEntities && sealed && promotedAllRoots && promotedCount > 0 && !!shareOperationId; + return { promotedCount, sealed, publishReady, ...(shareOperationId ? { shareOperationId } : {}) }; }, async discard(contextGraphId: string, name: string, opts?: { subGraphName?: string }): Promise { return agent.publisher.assertionDiscard(contextGraphId, name, agentAddress, opts?.subGraphName); @@ -2231,7 +2233,7 @@ export class DKGAgent extends DKGAgentBase { // Query assertion entity (current state + layer + OT-RFC-43 A2 pointers). const entityResult = await agent.store.query( - `SELECT ?state ?memoryLayer ?assertionGraph ?wm ?swm ?vm ?kaNum ?reservedUal ?publishedUal WHERE { + `SELECT ?state ?memoryLayer ?assertionGraph ?wm ?swm ?vm ?currentShareOpId ?kaNum ?reservedUal ?publishedUal WHERE { GRAPH <${metaGraph}> { <${lifecycleUri}> <${DKG_NS}state> ?state . OPTIONAL { <${lifecycleUri}> <${DKG_NS}memoryLayer> ?memoryLayer } @@ -2239,6 +2241,7 @@ export class DKGAgent extends DKGAgentBase { OPTIONAL { <${lifecycleUri}> <${WM_CURRENT_ASSERTION_PRED}> ?wm } OPTIONAL { <${lifecycleUri}> <${SWM_CURRENT_ASSERTION_PRED}> ?swm } OPTIONAL { <${lifecycleUri}> <${VM_CURRENT_ASSERTION_PRED}> ?vm } + OPTIONAL { <${lifecycleUri}> <${DKG_NS}shareOperationId> ?currentShareOpId } OPTIONAL { <${lifecycleUri}> <${KA_ID_PRED}> ?kaNum } OPTIONAL { <${lifecycleUri}> <${RESERVED_UAL_PRED}> ?reservedUal } OPTIONAL { <${lifecycleUri}> <${DKG_NS}publishedUal> ?publishedUal } @@ -2258,6 +2261,7 @@ export class DKGAgent extends DKGAgentBase { const vmCurrentAssertion = strip(row['vm']); const wmCurrentAssertion = strip(row['wm']) ?? vmCurrentAssertion; const swmCurrentAssertion = strip(row['swm']) ?? vmCurrentAssertion; + const currentShareOperationId = strip(row['currentShareOpId']); const kaNumberStr = strip(row['kaNum']); const reservedUal = strip(row['reservedUal']); const publishedUal = strip(row['publishedUal']); @@ -2352,6 +2356,9 @@ export class DKGAgent extends DKGAgentBase { if (ev.type === 'promoted' && !ev.rootEntities && subjectRoots.length > 0) { ev.rootEntities = [...subjectRoots]; } + if (ev.type === 'promoted' && !ev.shareOperationId && currentShareOperationId) { + ev.shareOperationId = currentShareOperationId; + } } } @@ -2370,6 +2377,7 @@ export class DKGAgent extends DKGAgentBase { wmCurrentAssertion, swmCurrentAssertion, vmCurrentAssertion, + currentShareOperationId, status: deriveStatus(pointers), kaNumber: kaNumberStr, reservedUal, diff --git a/packages/agent/src/source-worker.ts b/packages/agent/src/source-worker.ts index 6087bf83ef..56532bfbf7 100644 --- a/packages/agent/src/source-worker.ts +++ b/packages/agent/src/source-worker.ts @@ -5,6 +5,9 @@ import { basename, dirname, join } from 'node:path'; export interface SourceWorkerJobState { fingerprint?: string; lastRunAt?: string; + assertionName?: string; + shareOperationId?: string; + intentKey?: string; lastJobIds?: string[]; lastJobStatuses?: Record; lastStatus?: string; diff --git a/packages/agent/test/_helpers/v10-acks.ts b/packages/agent/test/_helpers/v10-acks.ts index 7261b21362..89de118f46 100644 --- a/packages/agent/test/_helpers/v10-acks.ts +++ b/packages/agent/test/_helpers/v10-acks.ts @@ -3,9 +3,10 @@ import type { DKGAgent } from '../../src/index.js'; import type { V10ACKProvider } from '@origintrail-official/dkg-publisher'; import { createProvider, + getSharedContext, HARDHAT_KEYS, } from '../../../chain/test/evm-test-context.js'; -import { hardhatACKProvider } from '../../../publisher/test/_helpers/acks.js'; +import { hardhatACKProvider, makeHardhatUpdateACKProvider } from '../../../publisher/test/_helpers/acks.js'; import { wrapPublisherForTest } from '../../../publisher/test/_helpers/seal.js'; type Kav10Chain = { @@ -31,11 +32,20 @@ export async function installHardhatACKProvider( const effectiveChain = chain ?? internals.chain; const kav10Address = await effectiveChain.getKnowledgeAssetsLifecycleAddress(); const v10ACKProvider = hardhatACKProvider(kav10Address); + const v10UpdateACKProvider = makeHardhatUpdateACKProvider( + getSharedContext(), + effectiveChain as any, + [HARDHAT_KEYS.REC1_OP, HARDHAT_KEYS.REC2_OP, HARDHAT_KEYS.REC3_OP], + ); Object.defineProperty(agent, 'createV10ACKProvider', { value: () => v10ACKProvider, configurable: true, }); + Object.defineProperty(agent, 'createV10UpdateACKProvider', { + value: () => v10UpdateACKProvider, + configurable: true, + }); const wrappedPublisher = wrapPublisherForTest(internals.publisher, { author: new ethers.Wallet(HARDHAT_KEYS.CORE_OP), diff --git a/packages/agent/test/e2e-memory-layers.test.ts b/packages/agent/test/e2e-memory-layers.test.ts index 216c2e1fad..138c18d1cd 100644 --- a/packages/agent/test/e2e-memory-layers.test.ts +++ b/packages/agent/test/e2e-memory-layers.test.ts @@ -11,11 +11,12 @@ */ import { describe, it, expect, afterEach, beforeAll, afterAll, vi } from 'vitest'; import { makeTestKaNumberAllocator } from "./_helpers/ka-allocator.js"; -import { DKGAgent } from '../src/index.js'; +import { DKGAgent, type DKGAgentConfig } from '../src/index.js'; import { SEAL_CAPABILITY_GAP_CODE } from '../src/dkg-agent-publish.js'; import { createEVMAdapter, getSharedContext, createProvider, takeSnapshot, revertSnapshot, HARDHAT_KEYS } from '../../chain/test/evm-test-context.js'; import { mintTokens } from '../../chain/test/hardhat-harness.js'; import { ethers } from 'ethers'; +import { TripleStoreAsyncLiftPublisher } from '@origintrail-official/dkg-publisher'; import { installHardhatACKProvider } from './_helpers/v10-acks.js'; import { assertionLifecycleUri, @@ -53,10 +54,11 @@ function sleep(ms: number) { return new Promise(r => setTimeout(r, ms)); } const CG_ID = 'memory-layers-e2e'; const ENTITY_BASE = 'urn:mem:entity'; -async function createAgent(name: string) { +async function createAgent(name: string, overrides: Partial = {}) { const chain = createEVMAdapter(HARDHAT_KEYS.CORE_OP); const agent = await DKGAgent.create({ - kaNumberAllocator: makeTestKaNumberAllocator(), + ...overrides, + kaNumberAllocator: makeTestKaNumberAllocator(), name, listenPort: 0, chainAdapter: chain, @@ -479,13 +481,220 @@ describe('#1116 seal decoupled from CG — full vs skipSeal share, seal-in-SWM', await expect( agent.publishFromFinalizedAssertion(CG_ID, 'unsealed-share'), ).rejects.toThrow(/not finalized/i); + await expect( + agent.resolveFinalizedAssertionVmPublishIntent(CG_ID, 'unsealed-share'), + ).rejects.toMatchObject({ code: 'PUBLISH_INTENT_STALE' }); + }, 30_000); + + it('async publish intent resolves roots when provenance events are disabled', async () => { + const agent = await createAgent('LiteProvenanceAsyncIntentBot', { metadataProvenanceEvents: false }); + await agent.createContextGraph({ id: CG_ID, name: 'Lite Provenance Async Intent E2E' }); + await agent.registerContextGraph(CG_ID); + + const name = 'lite-provenance-async'; + const root = `${ENTITY_BASE}:lite-provenance`; + await agent.assertion.create(CG_ID, name); + await agent.assertion.write(CG_ID, name, [ + { subject: root, predicate: 'http://schema.org/name', object: '"Lite Provenance Async"' }, + ]); + const share = await agent.assertion.promote(CG_ID, name); + expect(share.sealed).toBe(true); + expect(share.publishReady).toBe(true); + + const history = await agent.assertion.history(CG_ID, name); + expect(history?.events).toEqual([]); + expect(history?.currentShareOperationId).toBe(share.shareOperationId); + + const intent = await agent.resolveFinalizedAssertionVmPublishIntent(CG_ID, name); + expect(intent.shareOperationId).toBe(share.shareOperationId); + expect(intent.roots).toEqual([root]); + expect(intent.sealMerkleRoot).toMatch(/^0x[0-9a-f]+$/); }, 30_000); - // B2: SEAL-IN-SWM round trip — the key new capability. An asset shared - // UNSEALED (stuck, unpublishable) is made publishable by finalize(layer:'swm') - // WITHOUT recreating it: pull-from reconstructs a transient WM draft, finalize - // seals it, then the transient WM draft is dropped so the asset is left PURELY - // in SWM (now sealed) and publishes. + it('async VM publish executes the queued share snapshot after live SWM is drained', async () => { + const agent = await createAgent('QueuedAsyncVmPublishBot'); + await agent.createContextGraph({ id: CG_ID, name: 'Queued Async VM Publish E2E' }); + await agent.registerContextGraph(CG_ID); + + const name = 'queued-async-vm'; + const root = `${ENTITY_BASE}:queued-async`; + await agent.assertion.create(CG_ID, name); + await agent.assertion.write(CG_ID, name, [ + { subject: root, predicate: 'http://schema.org/name', object: '"Queued Async VM"' }, + ]); + const share = await agent.assertion.promote(CG_ID, name); + expect(share.publishReady).toBe(true); + + const intent = await agent.resolveFinalizedAssertionVmPublishIntent(CG_ID, name); + expect(intent.shareOperationId).toBe(share.shareOperationId); + + await (agent as any).publisher.clearPublishedSwmRoots( + CG_ID, + [...intent.roots], + undefined, + createOperationContext('publishFromSWM'), + ); + + const asyncPublisher = new TripleStoreAsyncLiftPublisher((agent as any).store, { + knowledgeAssetVmPublishExecutor: async ({ request, publishOptions }) => + agent.publishQueuedKnowledgeAssetVmPublish(request, publishOptions), + }); + const jobId = await asyncPublisher.enqueueKnowledgeAssetVmPublish(intent); + const processed = await asyncPublisher.processNext('wallet-1'); + expect(processed?.jobId).toBe(jobId); + expect(processed?.status).toBe('finalized'); + + const history = await agent.assertion.history(CG_ID, name); + expect(history?.vmCurrentAssertion).toBe(intent.sealMerkleRoot.slice(2)); + expect(history?.state).toBe('published'); + expect(history?.memoryLayer).toBe(MemoryLayer.VerifiableMemory); + }, 60_000); + + it('no-op SWM share after VM publish does not re-arm async publish intent', async () => { + const agent = await createAgent('NoopShareDoesNotRearmAsyncIntentBot'); + await agent.createContextGraph({ id: CG_ID, name: 'No-op Share Async Intent E2E' }); + await agent.registerContextGraph(CG_ID); + + const name = 'noop-share-after-publish'; + const root = `${ENTITY_BASE}:noop-share-after-publish`; + await agent.assertion.create(CG_ID, name); + await agent.assertion.write(CG_ID, name, [ + { subject: root, predicate: 'http://schema.org/name', object: '"No-op Share After Publish"' }, + ]); + const share = await agent.assertion.promote(CG_ID, name); + expect(share.publishReady).toBe(true); + + const publish = await agent.publishFromFinalizedAssertion(CG_ID, name); + expect(publish.status).toBe('confirmed'); + + const author = agent.defaultAgentAddress ?? agent.peerId; + const noopShare = await (agent as any).publisher.assertionPromote(CG_ID, name, author); + expect(noopShare.promotedCount).toBe(0); + expect(noopShare.promotedAllRoots).toBe(false); + expect(noopShare.shareOperationId).toBeUndefined(); + + const history = await agent.assertion.history(CG_ID, name); + expect(history?.currentShareOperationId).not.toBe(share.shareOperationId); + await expect( + agent.resolveFinalizedAssertionVmPublishIntent(CG_ID, name), + ).rejects.toMatchObject({ code: 'PUBLISH_NOT_FULL_SHARE' }); + }, 60_000); + + it('async VM publish with clearAfter false clears published roots but leaves unrelated SWM content', async () => { + const agent = await createAgent('QueuedAsyncVmPublishCleanupBot'); + await agent.createContextGraph({ id: CG_ID, name: 'Queued Async VM Cleanup E2E' }); + await agent.registerContextGraph(CG_ID); + + const publishedName = 'queued-async-cleanup-published'; + const retainedName = 'queued-async-cleanup-retained'; + const publishedRoot = `${ENTITY_BASE}:queued-cleanup-published`; + const retainedRoot = `${ENTITY_BASE}:queued-cleanup-retained`; + await agent.assertion.create(CG_ID, publishedName); + await agent.assertion.write(CG_ID, publishedName, [ + { subject: publishedRoot, predicate: 'http://schema.org/name', object: '"Published root"' }, + ]); + const publishedShare = await agent.assertion.promote(CG_ID, publishedName); + expect(publishedShare.publishReady).toBe(true); + + await agent.assertion.create(CG_ID, retainedName); + await agent.assertion.write(CG_ID, retainedName, [ + { subject: retainedRoot, predicate: 'http://schema.org/name', object: '"Retained root"' }, + ]); + const retainedShare = await agent.assertion.promote(CG_ID, retainedName); + expect(retainedShare.publishReady).toBe(true); + + const intent = await agent.resolveFinalizedAssertionVmPublishIntent(CG_ID, publishedName, { + clearSharedMemoryAfter: false, + }); + const asyncPublisher = new TripleStoreAsyncLiftPublisher((agent as any).store, { + knowledgeAssetVmPublishExecutor: async ({ request, publishOptions }) => + agent.publishQueuedKnowledgeAssetVmPublish(request, publishOptions), + }); + const jobId = await asyncPublisher.enqueueKnowledgeAssetVmPublish(intent); + const processed = await asyncPublisher.processNext('wallet-1'); + expect(processed?.jobId).toBe(jobId); + expect(processed?.status).toBe('finalized'); + + const publishedSwm = await agent.query( + `SELECT ?name WHERE { <${publishedRoot}> ?name }`, + { contextGraphId: CG_ID, graphSuffix: '_shared_memory' }, + ); + expect(publishedSwm.bindings).toHaveLength(0); + + const retainedSwm = await agent.query( + `SELECT ?name WHERE { <${retainedRoot}> ?name }`, + { contextGraphId: CG_ID, graphSuffix: '_shared_memory' }, + ); + expect(retainedSwm.bindings).toHaveLength(1); + }, 60_000); + + it('async VM publish updates a sub-graph KA in the sub-graph VM graph', async () => { + const agent = await createAgent('QueuedAsyncVmSubgraphUpdateBot'); + await agent.createContextGraph({ id: CG_ID, name: 'Queued Async VM Subgraph Update E2E' }); + await agent.registerContextGraph(CG_ID); + const subGraphName = 'async-update'; + await agent.createSubGraph(CG_ID, subGraphName); + + const name = 'queued-async-subgraph-update'; + const root = `${ENTITY_BASE}:queued-subgraph-update`; + await agent.assertion.create(CG_ID, name, { subGraphName }); + await agent.assertion.write(CG_ID, name, [ + { subject: root, predicate: 'http://schema.org/name', object: '"Subgraph v1"' }, + ], { subGraphName }); + const firstShare = await agent.assertion.promote(CG_ID, name, { subGraphName }); + expect(firstShare.publishReady).toBe(true); + const firstPublish = await agent.publishFromFinalizedAssertion(CG_ID, name, { subGraphName }); + expect(firstPublish.status).toBe('confirmed'); + + const reopened = await agent.assertion.pullFrom(CG_ID, name, 'vm', { + subGraphName, + onConflict: 'replace', + }); + expect(reopened.seeded).toBe(1); + await agent.assertion.write(CG_ID, name, [ + { subject: root, predicate: 'http://schema.org/name', object: '"Subgraph v2"' }, + ], { subGraphName }); + await agent.assertion.finalize(CG_ID, name, { subGraphName }); + const updateShare = await agent.assertion.promote(CG_ID, name, { subGraphName }); + expect(updateShare.publishReady).toBe(true); + + const intent = await agent.resolveFinalizedAssertionVmPublishIntent(CG_ID, name, { subGraphName }); + expect(intent.vmCurrentAssertion).toBeDefined(); + const asyncPublisher = new TripleStoreAsyncLiftPublisher((agent as any).store, { + knowledgeAssetVmPublishExecutor: async ({ request, publishOptions }) => + agent.publishQueuedKnowledgeAssetVmPublish(request, publishOptions), + }); + const jobId = await asyncPublisher.enqueueKnowledgeAssetVmPublish(intent); + const processed = await asyncPublisher.processNext('wallet-1'); + expect(processed?.jobId).toBe(jobId); + + if (processed?.status !== 'finalized') { + throw new Error(`Expected queued sub-graph update to finalize: ${JSON.stringify((processed as any)?.failure)}`); + } + + const subgraphVm = await agent.query( + `SELECT ?name WHERE { <${root}> ?name }`, + { contextGraphId: CG_ID, subGraphName }, + ); + expect(subgraphVm.bindings.map((row) => row['name'])).toContain('"Subgraph v2"'); + + const rootVm = await agent.query( + `SELECT ?name WHERE { <${root}> ?name }`, + { contextGraphId: CG_ID }, + ); + expect(rootVm.bindings.map((row) => row['name'])).not.toContain('"Subgraph v2"'); + + const history = await agent.assertion.history(CG_ID, name, { subGraphName }); + expect(history?.state).toBe('published'); + expect(history?.memoryLayer).toBe(MemoryLayer.VerifiableMemory); + expect(history?.vmCurrentAssertion).toBe(intent.sealMerkleRoot.slice(2)); + }, 120_000); + + // B2: SEAL-IN-SWM round trip. An asset shared UNSEALED (stuck, unpublishable) + // is made publishable by finalize(layer:'swm') WITHOUT recreating it: + // pull-from reconstructs a transient WM draft, finalize seals it, then the + // transient WM draft is dropped so the asset is left PURELY in SWM (now + // sealed) and publishes. it('finalize(layer:swm) seals a stuck unsealed SWM asset, empties the WM draft, and makes it publishable', async () => { const agent = await createAgent('SealInSwmBot'); await agent.createContextGraph({ id: CG_ID, name: 'Seal-in-SWM E2E' }); @@ -502,17 +711,17 @@ describe('#1116 seal decoupled from CG — full vs skipSeal share, seal-in-SWM', expect(share.sealed).toBe(false); const wmAfterShare = await agent.assertion.query(CG_ID, name); expect(wmAfterShare.length).toBe(0); // WM emptied by promote - // Unsealed ⇒ not yet publishable. + // Unsealed => not yet publishable. await expect( agent.publishFromFinalizedAssertion(CG_ID, name), ).rejects.toThrow(/not finalized/i); - // Seal in SWM — reconstruct from SWM, finalize, drop the transient WM draft. + // Seal in SWM: reconstruct from SWM, finalize, drop the transient WM draft. const seal = await agent.assertion.finalize(CG_ID, name, { layer: 'swm' }); expect(seal.merkleRoot).toBeDefined(); expect(seal.authorAddress).toBeDefined(); - // Post-condition #1: the asset is left PURELY in SWM — the WM draft is empty + // Post-condition #1: the asset is left PURELY in SWM: the WM draft is empty // again (the transient reconstruction draft was dropped). const wmAfterSeal = await agent.assertion.query(CG_ID, name); expect(wmAfterSeal.length).toBe(0); diff --git a/packages/agent/test/encrypt-inline-policy.test.ts b/packages/agent/test/encrypt-inline-policy.test.ts index 14bfd4879b..9eb30cca2e 100644 --- a/packages/agent/test/encrypt-inline-policy.test.ts +++ b/packages/agent/test/encrypt-inline-policy.test.ts @@ -18,7 +18,7 @@ import { encodePublishIntent, isStorageACKDecline, } from '@origintrail-official/dkg-core'; -import { StorageACKHandler } from '@origintrail-official/dkg-publisher'; +import { StorageACKHandler, type KnowledgeAssetVmPublishRequest } from '@origintrail-official/dkg-publisher'; import { DKGAgent } from '../src/dkg-agent.js'; // Hand-rolled call recorder (replaces vitest spy factories): wraps an @@ -636,6 +636,103 @@ describe('DKGAgent.publishFromSharedMemory inline encryption routing', () => { }); }); +describe('DKGAgent.publishQueuedKnowledgeAssetVmPublish inline encryption routing', () => { + it('lets resolved real encryption callbacks override queued fail-closed placeholders', async () => { + const realInline = recorder(async (plaintext: Uint8Array) => new Uint8Array([...plaintext, 0xaa])); + const realChunked = recorder(async () => ({ + ciphertextChunksRoot: ethers.getBytes(ethers.id('queued-real-chunk-root')), + ciphertextChunkCount: 1, + totalCiphertextBytes: 1, + })); + const failClosedInline = recorder(async () => { + throw new Error('fail-closed placeholder should not be used'); + }); + const failClosedChunked = recorder(async () => { + throw new Error('fail-closed chunk placeholder should not be used'); + }); + const publisherPublish = recorder(async (_opts: any) => ({ + status: 'tentative', + ual: 'did:dkg:local/queued-encryption', + })); + const store = { + query: recorder(async () => ({ type: 'bindings', bindings: [] })), + insert: recorder(async () => undefined), + deleteByPattern: recorder(async () => undefined), + }; + const agentLike = { + peerId: 'did:dkg:agent:queued-encryption', + defaultAgentAddress: '0x1111111111111111111111111111111111111111', + chain: {}, + store, + log: { + info: recorder(() => undefined), + warn: recorder(() => undefined), + error: recorder(() => undefined), + debug: recorder(() => undefined), + }, + publisher: { + publish: publisherPublish, + clearSwmShareComplete: recorder(async () => undefined), + }, + createV10ACKProvider: recorder(() => undefined), + _resolveEncryptInlinePayload: recorder(async () => realInline), + _resolveEncryptInlineChunked: recorder(async () => realChunked), + _stampPointer: recorder(async () => undefined), + } as any; + const request: KnowledgeAssetVmPublishRequest = { + contextGraphId: 'private-cg', + name: 'queued-private-ka', + shareOperationId: 'share-op-1', + roots: ['urn:test:queued-private'], + seal: { + merkleRoot: `0x${'12'.repeat(32)}`, + authorAddress: '0x1111111111111111111111111111111111111111', + signature: { + r: `0x${'34'.repeat(32)}`, + vs: `0x${'56'.repeat(32)}`, + }, + schemeVersion: 1, + }, + sealChainId: '31337', + sealKav10Address: '0x2222222222222222222222222222222222222222', + sealFinalizedAtIso: '2026-01-01T00:00:00.000Z', + sealMerkleRoot: `0x${'12'.repeat(32)}`, + intentKey: `sha256:${'ab'.repeat(32)}`, + }; + + await (DKGAgent.prototype as any).publishQueuedKnowledgeAssetVmPublish.call(agentLike, request, { + contextGraphId: request.contextGraphId, + quads: [{ + subject: 'urn:test:queued-private', + predicate: 'http://schema.org/name', + object: '"Queued Private"', + graph: '', + }], + encryptInlinePayload: failClosedInline, + encryptInlineChunked: failClosedChunked, + }); + + expect(agentLike._resolveEncryptInlinePayload.calls.at(-1)).toEqual([ + 'private-cg', + undefined, + undefined, + undefined, + ]); + expect(agentLike._resolveEncryptInlineChunked.calls.at(-1)).toEqual([ + 'private-cg', + undefined, + undefined, + undefined, + ]); + expect(publisherPublish.calls.at(-1)?.[0]).toMatchObject({ + encryptInlinePayload: realInline, + encryptInlineChunked: realChunked, + }); + expect(publisherPublish.calls.at(-1)?.[0].encryptInlinePayload).not.toBe(failClosedInline); + expect(publisherPublish.calls.at(-1)?.[0].encryptInlineChunked).not.toBe(failClosedChunked); + }); +}); + describe('DKGAgent._resolveEncryptInlineChunked nonce domain', () => { it('uses publishOperationId, not batchId, as the chunked AEAD nonce domain', async () => { const signer = ethers.Wallet.createRandom(); diff --git a/packages/agent/test/swm-curator-recovery-plan.test.ts b/packages/agent/test/swm-curator-recovery-plan.test.ts new file mode 100644 index 0000000000..d9948ab97e --- /dev/null +++ b/packages/agent/test/swm-curator-recovery-plan.test.ts @@ -0,0 +1,120 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { ethers } from 'ethers'; +import { createOperationContext } from '@origintrail-official/dkg-core'; +import { MockChainAdapter } from '@origintrail-official/dkg-chain'; +import { DKGAgent } from '../src/index.js'; + +describe('private SWM curator recovery planning', () => { + const agents: DKGAgent[] = []; + + afterEach(async () => { + await Promise.all(agents.splice(0).map(async (agent) => { + await agent.stop().catch(() => {}); + await agent.store.close().catch(() => {}); + })); + }); + + it('recovers wallet-scoped private CGs from the structural curator even when local meta self-stamps curator rows', async () => { + const curator = ethers.Wallet.createRandom().address.toLowerCase(); + const member = ethers.Wallet.createRandom().address.toLowerCase(); + const curatorPeer = '12D3KooWStructuralCuratorPeer'; + const contextGraphId = `${curator}/curator-converge-plan`; + const agent = await createAgent('CuratorRecoveryMemberPlan'); + + // This simulates the rfc38 pre-create pattern from the retired devnet + // script: the member has local curator/creator-looking meta, but the + // wallet-scoped CG id still names a different structural curator. + installPlanningStubs(agent, { + localAgent: member, + curator, + curatorPeers: [curatorPeer], + isPrivate: true, + isCuratorOf: async () => { + throw new Error('legacy triple-based curatorship must not gate wallet-scoped CGs'); + }, + resolveCuratorPeerId: async () => { + throw new Error('legacy curator peer lookup must not gate wallet-scoped CGs'); + }, + }); + + const plan = await agent.planSharedMemorySyncContextGraphs( + curatorPeer, + [contextGraphId], + createOperationContext('sync'), + ); + + expect(plan.publicContextGraphIds).toEqual([]); + expect(plan.privateRecoverFromCurator).toEqual([contextGraphId]); + expect(plan.eligibleContextGraphIds).toEqual([contextGraphId]); + }); + + it('skips private SWM recovery when the local node owns the structural curator agent', async () => { + const curator = ethers.Wallet.createRandom().address.toLowerCase(); + const memberPeer = '12D3KooWMemberReconnectPeer'; + const contextGraphId = `${curator}/curator-owned-plan`; + const agent = await createAgent('CuratorRecoveryLocalCuratorPlan'); + + installPlanningStubs(agent, { + localAgent: curator, + curator, + curatorPeers: ['12D3KooWStructuralCuratorPeer'], + isPrivate: true, + }); + + const plan = await agent.planSharedMemorySyncContextGraphs( + memberPeer, + [contextGraphId], + createOperationContext('sync'), + ); + + expect(plan.publicContextGraphIds).toEqual([]); + expect(plan.privateRecoverFromCurator).toEqual([]); + expect(plan.eligibleContextGraphIds).toEqual([]); + }); + + async function createAgent(name: string): Promise { + const agent = await DKGAgent.create({ + name, + listenHost: '127.0.0.1', + chainAdapter: new MockChainAdapter(), + }); + agents.push(agent); + return agent; + } +}); + +function installPlanningStubs( + agent: DKGAgent, + options: { + localAgent: string; + curator: string; + curatorPeers: string[]; + isPrivate: boolean; + isCuratorOf?: (contextGraphId: string) => Promise; + resolveCuratorPeerId?: (contextGraphId: string) => Promise; + }, +): void { + const anyAgent = agent as unknown as { + localAgents: Map; + canUseSharedMemoryForContextGraph: (contextGraphId: string) => Promise; + isPrivateContextGraph: (contextGraphId: string) => Promise; + refreshMetaFromCurator: (contextGraphId: string) => Promise; + isCuratorOf: (contextGraphId: string) => Promise; + resolveCuratorPeerId: (contextGraphId: string) => Promise; + discovery: { findAgents: () => Promise> }; + }; + + anyAgent.localAgents.clear(); + anyAgent.localAgents.set(options.localAgent.toLowerCase(), {}); + anyAgent.canUseSharedMemoryForContextGraph = async () => true; + anyAgent.isPrivateContextGraph = async () => options.isPrivate; + anyAgent.refreshMetaFromCurator = async () => {}; + anyAgent.isCuratorOf = options.isCuratorOf ?? (async () => false); + anyAgent.resolveCuratorPeerId = options.resolveCuratorPeerId ?? (async () => null); + anyAgent.discovery = { + findAgents: async () => options.curatorPeers.map((peerId) => ({ + agentAddress: options.curator, + peerId, + })), + }; +} diff --git a/packages/cli/README.md b/packages/cli/README.md index 7423668bee..2545e8a9b1 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -40,7 +40,7 @@ dkg status dkg context-graph create my-project dkg assertion import-file notes -f data.md -c my-project dkg assertion promote notes -c my-project -dkg shared-memory publish my-project +dkg publisher publish-async my-project notes # Query the knowledge graph dkg query my-project -q "SELECT ?s ?p ?o WHERE { ?s ?p ?o } LIMIT 10" @@ -252,8 +252,7 @@ modes auto-renewal can't recover from: | `dkg assertion import-file -f -c ` | Import a document into Working Memory | | `dkg assertion promote -c ` | Promote a WM assertion to Shared Working Memory | | `dkg assertion query -c ` | Read assertion quads from WM | -| `dkg shared-memory write ` | Write triples directly to Shared Working Memory | -| `dkg shared-memory publish ` | Publish from SWM to Verifiable Memory (costs TRAC) | +| `dkg publisher publish-async ` | Queue a named Knowledge Asset publish from SWM to Verifiable Memory | | `dkg publish ` | One-shot RDF publish to a context graph | | `dkg verify ` | Propose M-of-N verification for a published batch | | `dkg endorse ` | Endorse a published Knowledge Asset | @@ -308,17 +307,25 @@ The handler module is loaded from the config file's directory: ```js export const sourceWorker = { - createSourceWorkerDeps({ sharedMemory, asyncLift }) { + createSourceWorkerDeps({ knowledgeAssets }) { return { async getFingerprint(source) { return source.contentHash; }, async processSource(source, fingerprint) { - const share = await sharedMemory.share(source.contextGraphId, source.quads); - const jobId = await asyncLift.lift({ - ...source.liftRequest, - shareOperationId: share.shareOperationId - }); + const name = `source-${source.id}`; + const share = await knowledgeAssets.createAndShare( + source.contextGraphId, + name, + source.quads, + { subGraphName: source.subGraphName } + ); + const publish = await knowledgeAssets.publishAsync( + source.contextGraphId, + name, + { subGraphName: source.subGraphName } + ); + const jobId = publish.jobId; return { sourceId: source.id, skipped: false, @@ -327,6 +334,9 @@ export const sourceWorker = { status: "queued", nextState: { fingerprint, + assertionName: name, + shareOperationId: share.shareOperationId, + intentKey: publish.intentKey, lastStatus: "queued", lastJobIds: [jobId], lastJobStatuses: { [jobId]: "queued" } @@ -349,8 +359,7 @@ When the daemon is running, it exposes a local HTTP API (default: `http://localh - `GET /api/status`, `GET /api/info` — node status and health - `POST /api/agent/register`, `GET /api/agent/identity` — agent identity - `POST /api/context-graph/create`, `/register`, `/invite`, `GET /api/context-graph/list` — context graph management -- `POST /api/knowledge-assets`, `/{name}/wm/write`, `/{name}/swm/share`, `/{name}/wm/discard`, `/{name}/wm/import-file`, `GET /api/knowledge-assets/{name}` — Working Memory assertions -- `POST /api/shared-memory/write`, `/publish` — Shared Working Memory and publishing to Verifiable Memory +- `POST /api/knowledge-assets`, `/{name}/wm/write`, `/{name}/swm/share`, `/{name}/vm/publish`, `/{name}/vm/publish-async`, `/{name}/wm/discard`, `/{name}/wm/import-file`, `GET /api/knowledge-assets/{name}` — named knowledge asset lifecycle - `POST /api/query`, `POST /api/query-remote` — SPARQL querying - `POST /api/endorse`, `POST /api/verify`, `POST /api/update` — Verifiable Memory trust operations - `GET /api/peers`, `GET /api/connections`, `GET /api/agents` — network introspection @@ -368,9 +377,9 @@ The full API surface — including request bodies, response shapes, and error co ## Local Benchmarks The live publish/get benchmark measures four operation timings against a running -DKG daemon: synchronous publish end-to-end latency, async publisher enqueue -latency, async job completion/finalization latency, and SPARQL get latency for -the published benchmark content. +DKG daemon: synchronous publish end-to-end latency, async lifecycle publish +request latency, async job completion/finalization latency, and SPARQL get +latency for the published benchmark content. Prerequisites: diff --git a/packages/cli/scripts/split-handle-request.mjs b/packages/cli/scripts/split-handle-request.mjs index 5568d5e859..006ebf98d7 100644 --- a/packages/cli/scripts/split-handle-request.mjs +++ b/packages/cli/scripts/split-handle-request.mjs @@ -104,9 +104,6 @@ const BLOCKS = [ [1223, 'openclaw', '/api/openclaw-channel/health'], // GET /api/openclaw-channel/health [1228, 'agent-chat', '/api/connect'], // POST /api/connect [1243, 'agent-chat', '/api/update'], // POST /api/update - [1318, 'memory', '/api/shared-memory/write'], // POST /api/shared-memory/write - [1378, 'memory', '/api/shared-memory/publish'], // POST /api/shared-memory/publish - [1463, 'publisher', '/api/publisher/enqueue'], // POST /api/publisher/enqueue [1515, 'publisher', '/api/publisher/jobs'], // GET /api/publisher/jobs [1527, 'publisher', '"/api/publisher/job"'], // GET /api/publisher/job (quote-anchored to disambiguate from /jobs and /job-payload) [1539, 'publisher', '/api/publisher/job-payload'], // GET /api/publisher/job-payload @@ -129,7 +126,6 @@ const BLOCKS = [ [2207, 'assertion', '/api/assertion/|/import-file'], // multi-line if — assertion import-file [3116, 'assertion', '/api/assertion/|/extraction-status'], // late fallthrough — extraction-status [3169, 'assertion', '/api/file/'], // GET /api/file/* - [3209, 'memory', '/api/shared-memory/conditional-write'], // multi-line if [3254, 'query', '/api/query'], // POST /api/query [3477, 'query', '/api/genui/render'], // POST /api/genui/render [3659, 'query', '/api/query-remote'], // POST /api/query-remote @@ -410,8 +406,8 @@ const GROUP_DOC = { 'status': 'status, info, connections, host, wallet, chain, identity, integrations, shutdown', 'agent-chat': 'agent registration/identity/listing, skills, chat, messages, connect, update', 'openclaw': 'OpenClaw agent listing, chat, channel send/stream/persist-turn/health', - 'memory': 'shared-memory / workspace write + publish + conditional-write, memory turn/search', - 'publisher': 'publisher enqueue / jobs / stats / cancel / retry / clear', + 'memory': 'workspace write, memory turn/search', + 'publisher': 'publisher jobs / stats / cancel / retry / clear', 'context-graph': 'context-graph (+ contextGraph, sub-graph) CRUD, participants, join flow, manifest publish/install', 'assertion': 'assertion CRUD + import + file download', 'query': 'SPARQL query, GenUI render, catchup-status, verify, endorse, CCL policy + eval', diff --git a/packages/cli/scripts/swm-large-payload-benchmark.cjs b/packages/cli/scripts/swm-large-payload-benchmark.cjs index bb536f5fc3..c7956cda1d 100644 --- a/packages/cli/scripts/swm-large-payload-benchmark.cjs +++ b/packages/cli/scripts/swm-large-payload-benchmark.cjs @@ -21,8 +21,9 @@ const FAILURE_PATTERNS = [ function usage() { return `Usage: pnpm --filter @origintrail-official/dkg benchmark:swm-large-payload -- [options] -Writes large public SWM payloads to a running multi-node devnet and verifies that -the replicated payload is not duplicated into dkg:publicStagedQuads metadata. +Writes large public SWM payloads through named Knowledge Asset lifecycle +create+seal+share calls to a running multi-node devnet and verifies that the +replicated payload is not duplicated into dkg:publicStagedQuads metadata. Options: --ports Comma-separated daemon API ports. Default derives from --api-port-base and --nodes. @@ -30,7 +31,7 @@ Options: --nodes Node count when --ports is omitted. Default: 5. --context-graph-id Context graph to write/query. Default: devnet-test. --payload-mib-per-node Payload size written through each node. Default: 100. - --chunk-mib Payload literal size per SWM write. Default: 0.5. + --chunk-mib Payload literal size per named KA create+share. Default: 0.5. --write-concurrency Concurrent write requests. Default: 1. --replication-timeout-ms Time to wait for every node to see every payload. Default: 900000. --poll-interval-ms Replication polling interval. Default: 5000. @@ -298,6 +299,23 @@ function resolveInputPath(value) { return isAbsolute(value) ? value : resolve(INVOCATION_CWD, value); } +function assertionNamePart(value) { + return String(value) + .trim() + .replace(/[<>"{}|^`\\\s/]+/g, '-') + .replace(/^-+|-+$/g, '') + || 'unknown'; +} + +function lifecycleAssetName(config, nodeNumber, chunkNumber) { + return [ + assertionNamePart(config.namespace), + assertionNamePart(config.runId), + `n${nodeNumber}`, + `c${chunkNumber}`, + ].join('-').slice(0, 256); +} + function bytesFromMiB(value) { return Math.round(value * 1024 * 1024); } @@ -432,25 +450,35 @@ async function writeChunk(config, plan, nodeIndex, chunkNumber) { const sizeBytes = payloadBytesForChunk(plan, chunkNumber); const subject = `${plan.rootPrefix}node:${nodeNumber}:chunk:${chunkNumber}`; const payload = makePayload(config.runId, nodeNumber, chunkNumber, sizeBytes); + const name = lifecycleAssetName(config, nodeNumber, chunkNumber); const startedAt = performance.now(); - const response = await postJson(port, '/api/shared-memory/write', { + const response = await postJson(port, '/api/knowledge-assets', { contextGraphId: config.contextGraphId, + name, quads: [{ subject, predicate: config.predicate, object: JSON.stringify(payload), graph: '', }], + finalize: true, + alsoShareSwm: true, }, config, config.requestTimeoutMs); + if (Array.isArray(response.errors) && response.errors.length > 0) { + throw new Error(`Lifecycle create+share for ${name} returned errors: ${JSON.stringify(response.errors)}`); + } const durationMs = performance.now() - startedAt; return { port, nodeNumber, chunkNumber, + assetName: name, subject, payloadBytes: sizeBytes, durationMs: Number(durationMs.toFixed(2)), - operationId: response.operationId, + operationId: response.assertionUri ?? name, + swmShared: response.swmShared === true, + promotedCount: response.promotedCount, }; } diff --git a/packages/cli/scripts/swm-triple-volume-benchmark.cjs b/packages/cli/scripts/swm-triple-volume-benchmark.cjs index 31040149aa..541d981646 100644 --- a/packages/cli/scripts/swm-triple-volume-benchmark.cjs +++ b/packages/cli/scripts/swm-triple-volume-benchmark.cjs @@ -40,9 +40,10 @@ const DIAGNOSTIC_LOG_PATTERNS = [ function usage() { return `Usage: pnpm --filter @origintrail-official/dkg benchmark:swm-triple-volume -- [options] -Writes many small public SWM triples to a running multi-node devnet and verifies -that every node can count the replicated triple volume. This stresses Oxigraph -triple/index volume rather than large literal byte externalization. +Writes many small public SWM triples through named Knowledge Asset lifecycle +create+seal+share calls to a running multi-node devnet and verifies that every +node can count the replicated triple volume. This stresses Oxigraph triple/index +volume rather than large literal byte externalization. Options: --ports Comma-separated daemon API ports. Default derives from --api-port-base and --nodes. @@ -51,7 +52,7 @@ Options: --context-graph-id Context graph to write/query. Default: devnet-test. --target-mib-per-node Approximate serialized N-Quad bytes to write per node. Default: 1024. --target-gib-per-node Same target in GiB; overrides --target-mib-per-node. - --triples-per-write Small triples per /api/shared-memory/write request. Default: 1000. + --triples-per-write Small triples per named KA create+share request. Default: 1000. --object-bytes Lexical bytes for generated literal objects. Default: 64. --predicate-count Number of predicates to rotate through. Default: 8. --write-concurrency Concurrent write requests. Default: 1. @@ -363,6 +364,23 @@ function resolveInputPath(value) { return isAbsolute(value) ? value : resolve(INVOCATION_CWD, value); } +function assertionNamePart(value) { + return String(value) + .trim() + .replace(/[<>"{}|^`\\\s/]+/g, '-') + .replace(/^-+|-+$/g, '') + || 'unknown'; +} + +function lifecycleAssetName(config, nodeNumber, writeNumber) { + return [ + assertionNamePart(config.namespace), + assertionNamePart(config.runId), + `n${nodeNumber}`, + `w${writeNumber}`, + ].join('-').slice(0, 256); +} + function bytesFromMiB(value) { return Math.round(value * 1024 * 1024); } @@ -516,21 +534,31 @@ async function writeBatch(config, plan, nodeIndex, writeNumber) { const nodeNumber = nodeIndex + 1; const quads = makeQuads(config, plan, nodeNumber, writeNumber); const estimatedNQuadBytes = estimateQuadsNQuadBytes(quads); + const name = lifecycleAssetName(config, nodeNumber, writeNumber); const startedAt = performance.now(); - const response = await postJson(port, '/api/shared-memory/write', { + const response = await postJson(port, '/api/knowledge-assets', { contextGraphId: config.contextGraphId, + name, quads, + finalize: true, + alsoShareSwm: true, }, config, config.requestTimeoutMs); + if (Array.isArray(response.errors) && response.errors.length > 0) { + throw new Error(`Lifecycle create+share for ${name} returned errors: ${JSON.stringify(response.errors)}`); + } const durationMs = performance.now() - startedAt; return { port, nodeNumber, writeNumber, + assetName: name, root: quads[0].subject, triples: quads.length, estimatedNQuadBytes, durationMs: Number(durationMs.toFixed(2)), - operationId: response.operationId ?? response.shareOperationId, + operationId: response.assertionUri ?? name, + swmShared: response.swmShared === true, + promotedCount: response.promotedCount, }; } diff --git a/packages/cli/skills/dkg-node/SKILL.md b/packages/cli/skills/dkg-node/SKILL.md index 13c121f386..0f83eab3bd 100644 --- a/packages/cli/skills/dkg-node/SKILL.md +++ b/packages/cli/skills/dkg-node/SKILL.md @@ -272,14 +272,13 @@ SWM is for knowledge you've shared from WM and want peers to see. Agents put dat > Working Memory is per-agent regardless of CG visibility. - `POST /api/knowledge-assets/{name}/swm/share` — the canonical way an agent puts a named assertion into SWM (after `create` → `write`); a full share seals by default and is publish-ready. **This is the agent path** — see §5 WM and the lifecycle in §3. +- `POST /api/knowledge-assets/{name}/swm/share-async` — enqueue the same WM→SWM share for the in-daemon worker. Use this for bulk importers that should not block on the synchronous share round. - `POST /api/knowledge-assets/publish` — direct explicit-quads one-shot mint, for **CLI / programmatic** callers that already hold the exact quads to publish (the ACK path carries the inline payload, no SWM stage). Body: `{ contextGraphId, quads, privateQuads?, accessPolicy?, allowedPeers?, subGraphName? }`. **Not the agent path** — agents use the named-KA lifecycle (`…/swm/share` → `…/vm/publish`). -> **Note — daemon loose-write primitives.** `POST /api/shared-memory/write` and -> `POST /api/shared-memory/conditional-write` write un-named triples directly to SWM. They are retained for -> **non-agent producers** (bulk source-worker ingest, programmatic clients); they are **not an agent tool or -> path**, and content written this way is un-named loose SWM that is **not publishable through the canonical -> lifecycle** (no assertion name for `/vm/publish` to key on). Consolidating these onto the named-KA model is -> tracked in OriginTrail/dkg#1260. Agents: always author a named knowledge asset. +> **No loose SWM writes.** Agent-facing producers must author a named knowledge +> asset and move it through WM→SWM→VM. The legacy loose-write SWM routes were +> retired so shared/published data always has a lifecycle name, seal, and audit +> record. ### Verifiable Memory (VM) — Permanent, on-chain @@ -760,10 +759,10 @@ Use the job queue for bulk or long-running publishes, publishes that must surviv | Method | Route | Purpose | |---|---|---| -| `POST` | `/api/publisher/enqueue` | Enqueue a publish job. Body: `{ contextGraphId, ... }`. Returns `{ jobId }`. | +| `POST` | `/api/knowledge-assets/{name}/vm/publish-async` | Enqueue VM publish for a named KA already shared to SWM. Body: `{ contextGraphId, options? }`. Returns `202 { jobId, status: "accepted" }`. | | `GET` | `/api/publisher/jobs?status=...` | List jobs, optionally filtered by status. | | `GET` | `/api/publisher/job?id=...` | Fetch one job's status. | -| `GET` | `/api/publisher/job-payload?id=...` | Fetch a job's payload. | +| `GET` | `/api/publisher/job-payload?id=...` | Fetch the prepared payload for internal raw LIFT jobs. Named lifecycle publish jobs return no raw payload. | | `GET` | `/api/publisher/stats` | Queue statistics (running / pending / completed / failed). | | `POST` | `/api/publisher/cancel` | Cancel a job. Body: `{ jobId }`. | | `POST` | `/api/publisher/retry` | Retry a failed job. Body: `{ jobId }`. | diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index 6021ce376d..39d8ca897d 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -11,7 +11,7 @@ export type QueryResult = export interface PreSignedAuthorAttestationPayload { address: string; /** - * OT-RFC-43 §F2 — the packed reservedKaId the author signed the + * OT-RFC-43 Section F2 -- the packed reservedKaId the author signed the * AuthorAttestation over, as a decimal string (uint256-safe over JSON). * Required: the digest binds it, so the daemon honours the author's * reserved slot rather than re-allocating. @@ -91,7 +91,7 @@ function assertCreateFinalizeFieldsHaveQuads(args: { args.preSignedAuthorAttestation != null || args.schemeVersion !== undefined; // These fields only take effect at finalize, so they require both non-empty - // quads AND finalize !== false — mirrors the daemon create-route guard. + // quads AND finalize !== false -- mirrors the daemon create-route guard. const willFinalize = Array.isArray(args.quads) && args.quads.length > 0 && args.finalize !== false; if (hasFinalizeOnlyField && !willFinalize) { throw new Error('authorAgentAddress, preSignedAuthorAttestation, and schemeVersion require non-empty quads and finalize !== false'); @@ -103,7 +103,7 @@ function assertCreateFinalizeFieldsHaveQuads(args: { * `RandomSamplingStatus` from `@origintrail-official/dkg-agent` but * lives here so the CLI doesn't take a runtime dep on the agent * package (only types). The `loop.lastOutcome` is intentionally - * `unknown` — the CLI prints it as JSON; the structured discrimination + * `unknown` -- the CLI prints it as JSON; the structured discrimination * is the prover's concern, not the CLI's. */ export interface RandomSamplingStatusResponse { @@ -486,51 +486,21 @@ export class ApiClient { }); } - /** - * Direct SWM write — appends loose triples to shared memory without - * creating a named WM assertion. Triples land ungrouped and are NOT - * publishable through the named-KA lifecycle. - * - * Non-agent primitive: used by the source-worker bulk-ingest pipeline, - * benchmarks, and network-sim. There is no longer an agent tool for - * this (`dkg_share` was removed); consolidating the remaining - * producers onto the named-KA model is tracked in OriginTrail/dkg#1260. - * For sealed-from-creation provenance, use `createAssertion` / - * `appendToAssertion` and publish the named assertion via - * `knowledgeAssetPublish` instead — the seal binds to the named - * assertion at finalize time. - */ - async sharedMemoryWrite(contextGraphId: string, quads: Array<{ - subject: string; predicate: string; object: string; graph: string; - }>): Promise<{ - shareOperationId: string; - contextGraphId: string; - graph: string; - triplesWritten: number; - skolemizedBlankNodes?: number; - }> { - return this.post('/api/shared-memory/write', { contextGraphId, quads }); - } - /** * Create an assertion in WM, optionally writing quads + finalizing + * promoting in the same call. Maps directly to the extended * `POST /api/knowledge-assets` body. * - * RFC-001 §9.x — the assertion lifecycle is the canonical entry - * point for staging content for VM publish. Callers that previously - * went through the legacy `/api/shared-memory/write` (now removed) - * use this method instead. + * RFC-001 Section 9.x -- the assertion lifecycle is the canonical entry + * point for staging content for VM publish. */ - // ── OT-RFC-43 §10.5 — GitHub-shaped Knowledge Asset SDK ────────────────── - // Layer-explicit wrappers over /api/knowledge-assets/... (the new clean - // surface). The legacy assertion/* + shared-memory/* methods below remain - // for back-compat during the migration window. + // -- OT-RFC-43 Section 10.5 -- GitHub-shaped Knowledge Asset SDK ------------------ + // Layer-explicit wrappers over /api/knowledge-assets/... (the clean product surface). /** * Create a KA + open a WM draft. Pass `quads` to write them atomically; by * default the draft is also sealed (finalized). Pass `finalize: false` to - * write a draft WITHOUT sealing — an editable WM-only assertion that never + * write a draft WITHOUT sealing -- an editable WM-only assertion that never * touches the chain (the only lifecycle available to local-only / * on-chain-unregistered CGs). */ @@ -611,12 +581,12 @@ export class ApiClient { return this.post(`/api/knowledge-assets/${encodeURIComponent(name)}/wm/pull-from`, { contextGraphId, layer, ...(options ?? {}) }); } - /** Advance the SWM pointer (WM → SWM; git push origin ). */ + /** Advance the SWM pointer (WM -> SWM; git push origin ). */ async knowledgeAssetShare( contextGraphId: string, name: string, options?: { subGraphName?: string; entities?: string[] | 'all' }, - ): Promise<{ swmShared: boolean; promotedCount: number }> { + ): Promise<{ swmShared: boolean; promotedCount: number; shareOperationId?: string }> { return this.post(`/api/knowledge-assets/${encodeURIComponent(name)}/swm/share`, { contextGraphId, ...(options ?? {}) }); } @@ -634,6 +604,29 @@ export class ApiClient { }); } + async knowledgeAssetPublishAsync( + contextGraphId: string, + name: string, + options?: { subGraphName?: string } & KnowledgeAssetFinalizedPublishOptions, + ): Promise<{ + jobId: string; + status: string; + contextGraphId: string; + name: string; + subGraphName?: string; + shareOperationId?: string; + rootsCount?: number; + sealMerkleRoot?: string; + intentKey?: string; + }> { + const publishOptions = finalizedPublishOptionsPayload(options, ['subGraphName']); + return this.post(`/api/knowledge-assets/${encodeURIComponent(name)}/vm/publish-async`, { + contextGraphId, + ...(options?.subGraphName ? { subGraphName: options.subGraphName } : {}), + ...(publishOptions ? { options: publishOptions } : {}), + }); + } + async createAssertion( contextGraphId: string, name: string, @@ -641,7 +634,7 @@ export class ApiClient { subGraphName?: string; quads?: Array<{ subject: string; predicate: string; object: string; graph: string }>; finalize?: boolean; - promote?: boolean; + alsoShareSwm?: boolean; authorAgentAddress?: string; preSignedAuthorAttestation?: PreSignedAuthorAttestationPayload; schemeVersion?: number; @@ -658,6 +651,7 @@ export class ApiClient { eip712Digest: string; }; promotedCount?: number; + shareOperationId?: string; }> { return this.post('/api/knowledge-assets', { contextGraphId, @@ -665,7 +659,7 @@ export class ApiClient { ...(options?.subGraphName ? { subGraphName: options.subGraphName } : {}), ...(options?.quads ? { quads: options.quads } : {}), ...(options?.finalize !== undefined ? { finalize: options.finalize } : {}), - ...(options?.promote !== undefined ? { promote: options.promote } : {}), + ...(options?.alsoShareSwm !== undefined ? { alsoShareSwm: options.alsoShareSwm } : {}), ...(options?.authorAgentAddress ? { authorAgentAddress: options.authorAgentAddress } : {}), @@ -698,7 +692,7 @@ export class ApiClient { } /** - * Finalize a previously-created assertion. RFC-001 §9.x — computes + * Finalize a previously-created assertion. RFC-001 Section 9.x -- computes * the canonical merkleRoot, builds the EIP-712 AuthorAttestation, * signs (custodial / pre-signed / publisher fallback), and stamps * the seal triples to `_meta`. @@ -752,8 +746,8 @@ export class ApiClient { * Routes to the canonical per-KA publish `POST * /api/knowledge-assets/:name/vm/publish` (the URL name selects the * assertion). Kept as a thin wrapper over `knowledgeAssetPublish` so - * existing callers (`dkg shared-memory publish`, `dkg index`, - * `publishAssertion`) keep their signature + typed return. + * lifecycle callers (`dkg publisher publish-async`, `dkg index`, + * `publishAssertion`) keep a narrow typed return. */ async publishFromFinalizedAssertion( contextGraphId: string, @@ -789,9 +783,9 @@ export class ApiClient { } /** - * High-level convenience: create → write → finalize → promote → + * High-level convenience: create -> write -> finalize -> promote -> * publish, all in two HTTP round-trips. The composite mirrors what - * a typical OpenClaw/Hermes client does — stage content, commit it, + * a typical OpenClaw/Hermes client does -- stage content, commit it, * push it on-chain. Use this unless you need fine-grained control * over the individual steps. */ @@ -822,7 +816,7 @@ export class ApiClient { ...(options?.subGraphName ? { subGraphName: options.subGraphName } : {}), quads, finalize: true, - promote: true, + alsoShareSwm: true, ...(options?.authorAgentAddress ? { authorAgentAddress: options.authorAgentAddress } : {}), @@ -863,12 +857,12 @@ export class ApiClient { }; } - // ─── Publishing Conviction Account (PCA) ──────────────────────────── + // --- Publishing Conviction Account (PCA) ---------------------------- async createPca(request: { tokens: string; // OT-RFC-51: the node identityId this PCA's committed TRAC funds. Required - // — a PCA created with no node seeds publishing allocation to nobody. + // -- a PCA created with no node seeds publishing allocation to nobody. primaryNode: string; }): Promise<{ accountId: string; @@ -941,35 +935,6 @@ export class ApiClient { return this.get(`/api/pca/${encodeURIComponent(accountId)}${qs}`); } - async publisherEnqueue(request: { - contextGraphId: string; - shareOperationId: string; - roots: string[]; - namespace: string; - scope: string; - authorityProofRef: string; - swmId?: string; - transitionType?: 'CREATE' | 'MUTATE' | 'REVOKE'; - authorityType?: 'owner' | 'multisig' | 'quorum' | 'capability'; - priorVersion?: string; - subGraphName?: string; - accessPolicy?: 'public' | 'ownerOnly' | 'allowList'; - allowedPeers?: string[]; - // V10 sign-at-enqueue. Absent `seal` → tentative; supply for on-chain attestation. - entityProofs?: boolean; - publishEpochs?: number; - /** Stringified bigint; `'0'` = mode d (no attribution) per RFC-001 §4. */ - publisherNodeIdentityIdOverride?: string; - seal?: { - merkleRoot: `0x${string}`; - authorAddress: `0x${string}`; - signature: { r: `0x${string}`; vs: `0x${string}` }; - schemeVersion: number; - }; - }): Promise<{ jobId: string; contextGraphId: string; shareOperationId: string; rootsCount: number }> { - return this.post('/api/publisher/enqueue', request); - } - async publisherJobs(status?: string): Promise<{ jobs: any[] }> { const qs = status ? `?status=${encodeURIComponent(status)}` : ''; return this.get(`/api/publisher/jobs${qs}`); @@ -999,7 +964,7 @@ export class ApiClient { return this.post('/api/publisher/clear', { status }); } - // ───────────────────────── EPCIS ───────────────────────────────────── + // ------------------------- EPCIS ------------------------------------- async captureEpcis(request: { epcisDocument: unknown; @@ -1082,7 +1047,7 @@ export class ApiClient { } /** - * Run SPARQL via the daemon. `opts` covers the full /api/query surface — + * Run SPARQL via the daemon. `opts` covers the full /api/query surface -- * memory-layer routing (`view`, `graphSuffix`, `verifiedGraph`, * `subGraphName`, `includeSharedMemory`, `includeContextGraphPartitions`, * `agentAddress`, `assertionName`), and P-13's `minTrust` (only meaningful @@ -1359,7 +1324,7 @@ export class ApiClient { * Atomic combined-flow flag. When `true`, the daemon registers the * CG on-chain in the same call after the local create step * succeeds. Required when `pcaAccountId` is supplied (a standalone - * `createContextGraph` does NOT persist PCA ids — Codex PR #502 + * `createContextGraph` does NOT persist PCA ids -- Codex PR #502 * round-3). */ register?: boolean; @@ -1368,7 +1333,7 @@ export class ApiClient { * the combined-flow path. Only meaningful together with * `register: true`. The agent otherwise defaults * `publishPolicy = curated (0)` for curated/private CGs and - * `publishPolicy = open (1)` for public CGs — which makes the + * `publishPolicy = open (1)` for public CGs -- which makes the * valid `{ accessPolicy: 0 (public), publishPolicy: 0 (curated), * pcaAccountId }` combo unreachable unless the caller can pin * `publishPolicy` explicitly. Codex PR #502 round-10 (raised by @@ -1470,7 +1435,7 @@ export class ApiClient { * the local agent produced; does NOT forward over P2P. To deliver it * to the curator, follow up with `requestJoin(...)` and the * `curatorPeerId` from the V10 invite. PR #448 split sign vs forward - * to fix a duplicate-forward bug — see daemon route comment. + * to fix a duplicate-forward bug -- see daemon route comment. * * The `delegation` shape mirrors `SignedAgentDelegation` from * `@dkg/agent`: `version` is part of the digest grammar (see @@ -1597,7 +1562,7 @@ export class ApiClient { /** * Optional. If supplied it MUST match the address resolved from * the bearer token; the daemon rejects any mismatch with 403. - * Prefer omitting and relying on the token — see A-12 review on + * Prefer omitting and relying on the token -- see A-12 review on * /api/endorse for the provenance-forgery rationale. */ agentAddress?: string; diff --git a/packages/cli/src/benchmark/publish-get/runner.ts b/packages/cli/src/benchmark/publish-get/runner.ts index 5ffbe701d2..0d418ce90d 100644 --- a/packages/cli/src/benchmark/publish-get/runner.ts +++ b/packages/cli/src/benchmark/publish-get/runner.ts @@ -57,9 +57,9 @@ async function runIteration(args: { const syncContext = reproductionContext(config, iteration, syncPayload, { flow: 'sync' }); const syncPublish = await measureOperation('syncPublish', iteration, warmup, syncContext, config.timeoutMs, now, async () => { - // Canonical named-KA publish (create → write → finalize → promote → - // publish in one composite). Replaces the removed loose-SWM-write + - // selection-fork publish path (the `/api/shared-memory/publish` bridge). + // Canonical named-KA publish (create -> write -> finalize -> promote -> + // publish in one composite). Replaces the removed loose-SWM selection + // publish bridge. // The assertion name must be unique per PUBLISHED payload — `runIteration` // runs with the same numeric `iteration` for BOTH the warmup and measured // phases, so the name includes the phase (mirrors the payload's unique @@ -103,57 +103,50 @@ async function runIteration(args: { } const asyncPayload = createPayload(config, runId, iteration, 'async', warmup); + const asyncName = `benchmark-async-${runId}-${warmup ? 'warmup' : 'measured'}-${iteration}`; const asyncBaseContext = reproductionContext(config, iteration, asyncPayload, { flow: 'async' }); - let shareOperationId: string; try { - const prepared = await withTimeout( - client.sharedMemoryWrite(config.contextGraphId, asyncPayload.quads), + await withTimeout( + client.createKnowledgeAsset(config.contextGraphId, asyncName, { + quads: asyncPayload.quads, + finalize: true, + alsoShareSwm: true, + }), config.timeoutMs, - 'async shared-memory write', + 'async knowledge-asset create/share', ); - shareOperationId = prepared.shareOperationId ?? ''; - if (!shareOperationId) throw new Error('Shared-memory write response did not include shareOperationId'); } catch (error) { - operations.push(createFailureRecord('asyncEnqueue', iteration, error, { ...asyncBaseContext, phase: 'sharedMemoryWrite' }, warmup)); - operations.push(createFailureRecord('asyncCompletion', iteration, new Error('Skipped because async enqueue preparation failed'), { + operations.push(createFailureRecord('asyncPublishRequest', iteration, error, { ...asyncBaseContext, phase: 'createKnowledgeAsset', name: asyncName }, warmup)); + operations.push(createFailureRecord('asyncCompletion', iteration, new Error('Skipped because async publish preparation failed'), { ...asyncBaseContext, - skippedAfter: 'sharedMemoryWrite', + name: asyncName, + skippedAfter: 'createKnowledgeAsset', }, warmup)); return; } - const enqueue = await measureOperation('asyncEnqueue', iteration, warmup, { + const enqueue = await measureOperation('asyncPublishRequest', iteration, warmup, { ...asyncBaseContext, - shareOperationId, + name: asyncName, }, config.timeoutMs, now, async () => { - const result = await client.publisherEnqueue({ - contextGraphId: config.contextGraphId, - shareOperationId, - roots: [asyncPayload.rootEntity], - namespace: config.namespace, - scope: config.scope, - authorityProofRef: config.authorityProofRef, - swmId: 'swm-main', - transitionType: 'CREATE', - authorityType: 'owner', - }); - if (!result.jobId) throw new Error('Async enqueue response did not include jobId'); + const result = await client.knowledgeAssetPublishAsync(config.contextGraphId, asyncName); + if (!result.jobId) throw new Error('Async publish response did not include jobId'); return result; }); operations.push(enqueue.timing); if (!enqueue.ok || !enqueue.value?.jobId) { - operations.push(createFailureRecord('asyncCompletion', iteration, new Error('Skipped because asyncEnqueue failed'), { + operations.push(createFailureRecord('asyncCompletion', iteration, new Error('Skipped because async publish request failed'), { ...asyncBaseContext, - shareOperationId, - skippedAfter: 'asyncEnqueue', + name: asyncName, + skippedAfter: 'asyncPublishRequest', }, warmup)); return; } const completion = await measureOperation('asyncCompletion', iteration, warmup, { ...asyncBaseContext, - shareOperationId, + name: asyncName, jobId: enqueue.value.jobId, }, config.timeoutMs, now, () => waitForAsyncCompletion(config, client, enqueue.value!.jobId!)); operations.push(completion.timing); diff --git a/packages/cli/src/benchmark/publish-get/types.ts b/packages/cli/src/benchmark/publish-get/types.ts index 86bcb711ab..b2ac18e981 100644 --- a/packages/cli/src/benchmark/publish-get/types.ts +++ b/packages/cli/src/benchmark/publish-get/types.ts @@ -6,7 +6,7 @@ export const DEFAULT_TIMEOUT_MS = 120_000; export const DEFAULT_PAYLOAD_SIZE_BYTES = 1024; export const DEFAULT_POLL_INTERVAL_MS = 1000; -export const OPERATIONS = ['syncPublish', 'asyncEnqueue', 'asyncCompletion', 'get'] as const; +export const OPERATIONS = ['syncPublish', 'asyncPublishRequest', 'asyncCompletion', 'get'] as const; export type BenchmarkOperation = (typeof OPERATIONS)[number]; export type OutputFormat = 'json' | 'ndjson'; @@ -79,27 +79,30 @@ export interface BenchmarkResult { export interface BenchmarkClient { status(): Promise; - sharedMemoryWrite( + createKnowledgeAsset( contextGraphId: string, - quads: BenchmarkPayload['quads'], - ): Promise<{ shareOperationId?: string }>; + name: string, + options: { + quads: BenchmarkPayload['quads']; + finalize: true; + alsoShareSwm: true; + subGraphName?: string; + }, + ): Promise>; + knowledgeAssetPublishAsync( + contextGraphId: string, + name: string, + options?: { + subGraphName?: string; + publishEpochs?: number; + }, + ): Promise<{ jobId?: string; status?: string }>; publishAssertion( contextGraphId: string, name: string, quads: BenchmarkPayload['quads'], options?: { clearAfter?: boolean }, ): Promise<{ kaId?: string; status?: string; kas?: Array<{ tokenId: string; rootEntity: string }> }>; - publisherEnqueue(request: { - contextGraphId: string; - shareOperationId: string; - roots: string[]; - namespace: string; - scope: string; - authorityProofRef: string; - swmId?: string; - transitionType?: 'CREATE' | 'MUTATE' | 'REVOKE'; - authorityType?: 'owner' | 'multisig' | 'quorum' | 'capability'; - }): Promise<{ jobId?: string }>; publisherJob(jobId: string): Promise<{ job: { status?: string; error?: string; lastError?: string } | null }>; query( sparql: string, diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index f6d5930c31..2253826829 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -17,7 +17,6 @@ import { registerMcpCommand } from './commands/mcp.js'; import { registerHermesCommand } from './commands/hermes.js'; import { registerCclCommand } from './commands/ccl.js'; import { registerIndexCommand } from './commands/index-command.js'; -import { registerSharedMemoryCommand } from './commands/shared-memory.js'; import { registerSourceWorkerCommand } from './commands/source-worker.js'; import { registerPcaCommand } from './commands/pca.js'; import { registerPublisherCommand } from './commands/publisher.js'; @@ -47,7 +46,6 @@ registerMcpCommand(program); registerHermesCommand(program); registerCclCommand(program); registerIndexCommand(program); -registerSharedMemoryCommand(program); registerSourceWorkerCommand(program); registerPcaCommand(program); registerPublisherCommand(program); diff --git a/packages/cli/src/commands/assertion.ts b/packages/cli/src/commands/assertion.ts index 9cc39e3694..27f12dca04 100644 --- a/packages/cli/src/commands/assertion.ts +++ b/packages/cli/src/commands/assertion.ts @@ -221,7 +221,7 @@ assertionCmd if (Array.isArray(result.rootEntities) && result.rootEntities.length > 0) { console.log(` Root entities: ${result.rootEntities.join(', ')}`); } - console.log(` Next: dkg shared-memory publish ${opts.contextGraph} --name ${name}${opts.subGraphName ? ` --sub-graph-name ${opts.subGraphName}` : ''}`); + console.log(` Next: dkg publisher publish-async ${opts.contextGraph} ${name}${opts.subGraphName ? ` --sub-graph ${opts.subGraphName}` : ''}`); } catch (err) { console.error(toErrorMessage(err)); process.exit(1); diff --git a/packages/cli/src/commands/index-command.ts b/packages/cli/src/commands/index-command.ts index 76b9f425e7..44128beb19 100644 --- a/packages/cli/src/commands/index-command.ts +++ b/packages/cli/src/commands/index-command.ts @@ -184,7 +184,7 @@ program if (useSharedMemory) { console.log(`\n\n Staged ${result.quads.length} quads into WM assertion "${indexAssertionName}" for context graph "${targetContextGraph}".`); - console.log(` Next: dkg shared-memory publish ${targetContextGraph} --name ${indexAssertionName}`); + console.log(` Next: dkg assertion promote ${indexAssertionName} -c ${targetContextGraph}`); } else { console.log(`\n\n Published ${result.quads.length} quads to context graph "${targetContextGraph}".`); } diff --git a/packages/cli/src/commands/knowledge.ts b/packages/cli/src/commands/knowledge.ts index e6b449373e..2cfeb97c7c 100644 --- a/packages/cli/src/commands/knowledge.ts +++ b/packages/cli/src/commands/knowledge.ts @@ -269,7 +269,7 @@ program .description('Run a SPARQL query against a context graph (or all)') .option('-q, --sparql ', 'SPARQL query string') .option('-f, --file ', 'File containing SPARQL query') - .option('--include-shared-memory', 'Include shared (unpublished) memory in the query — use for data written via `dkg index` / `dkg shared-memory write` before on-chain registration') + .option('--include-shared-memory', 'Include shared (unpublished) memory in the query - use for data promoted through the KA lifecycle before on-chain registration') .action(async (contextGraph: string | undefined, opts: ActionOpts) => { try { const client = await ApiClient.connect(); diff --git a/packages/cli/src/commands/publisher.ts b/packages/cli/src/commands/publisher.ts index 801a322277..fdac7546ab 100644 --- a/packages/cli/src/commands/publisher.ts +++ b/packages/cli/src/commands/publisher.ts @@ -205,84 +205,27 @@ publisherCmd }); publisherCmd - .command('enqueue ') - .description('Enqueue an async lift/publish job from shared memory') - .requiredOption('--root ', 'Root entities to include in the lift request') - .requiredOption('--namespace ', 'Namespace for the lifted publish') - .requiredOption('--scope ', 'Scope for the lifted publish') - .requiredOption('--authority-proof-ref ', 'Authority proof reference') - .option('--swm-id ', 'Shared memory id', 'swm-main') - .option('--workspace-id ', 'Legacy alias for --swm-id') - .option('--share-operation-id ', 'Share operation id') - .option('--transition-type ', 'Transition type (CREATE|MUTATE|REVOKE)', 'CREATE') - .option('--authority-type ', 'Authority type (owner|multisig|quorum|capability)', 'owner') - .option('--prior-version ', 'Prior version reference for MUTATE/REVOKE flows') + .command('publish-async ') + .description('Enqueue a named knowledge asset VM publish job') + .option('--sub-graph ', 'Target sub-graph within the context graph') .option('--publish-epochs ', 'On-chain publish lifetime in epochs (default: 12; PCA-funded publishes may coerce to PCA lock duration)') - .action(async (contextGraph: string, opts: ActionOpts) => { + .action(async (contextGraph: string, name: string, opts: ActionOpts) => { try { - const shareOperationId = opts.shareOperationId; - if (!shareOperationId) { - console.error('Provide --share-operation-id.'); - process.exit(1); - } - const roots = (opts.root as string[] | undefined)?.map((v) => v.trim()).filter(Boolean) ?? []; - if (roots.length === 0) { - console.error('Provide at least one --root.'); - process.exit(1); - } - const transitionType = String(opts.transitionType ?? 'CREATE').toUpperCase(); - if (!['CREATE', 'MUTATE', 'REVOKE'].includes(transitionType)) { - console.error('Invalid --transition-type. Use CREATE, MUTATE, or REVOKE.'); - process.exit(1); - } - const authorityType = String(opts.authorityType ?? 'owner'); - if (!['owner', 'multisig', 'quorum', 'capability'].includes(authorityType)) { - console.error('Invalid --authority-type. Use owner, multisig, quorum, or capability.'); - process.exit(1); - } const publishEpochs = opts.publishEpochs !== undefined ? parsePositiveIntegerOption(String(opts.publishEpochs), '--publish-epochs') : undefined; - const enqueueFields = { - swmId: opts.swmId ?? opts.workspaceId ?? 'swm-main', - shareOperationId, - roots, - contextGraphId: contextGraph, - namespace: String(opts.namespace), - scope: String(opts.scope), - transitionType: transitionType as 'CREATE' | 'MUTATE' | 'REVOKE', - authorityType: authorityType as 'owner' | 'multisig' | 'quorum' | 'capability', - authorityProofRef: String(opts.authorityProofRef), - priorVersion: opts.priorVersion ? String(opts.priorVersion) : undefined, - publishEpochs, - }; - - let jobId: string; - try { - const client = await ApiClient.connect(); - const result = await client.publisherEnqueue(enqueueFields); - jobId = result.jobId; - } catch (err) { - if (!isDaemonUnreachable(err)) throw err; - const config = await loadConfig(); - const { createPublisherInspector } = await import('../publisher-runner.js'); - const inspector = await createPublisherInspector({ dataDir: dkgDir(), config }); - try { - jobId = await inspector.publisher.lift({ - ...enqueueFields, - authority: { type: enqueueFields.authorityType, proofRef: enqueueFields.authorityProofRef }, - } as any); - } finally { - await inspector.stop(); - } - } + const client = await ApiClient.connect(); + const result = await client.knowledgeAssetPublishAsync(contextGraph, name, { + ...(opts.subGraph ? { subGraphName: String(opts.subGraph) } : {}), + ...(publishEpochs !== undefined ? { publishEpochs } : {}), + }); - console.log('Async publisher job enqueued:'); - console.log(` Job ID: ${jobId}`); + console.log('Knowledge asset publish job accepted:'); + console.log(` Job ID: ${result.jobId}`); console.log(` Context: ${contextGraph}`); - console.log(` Share op: ${shareOperationId}`); - console.log(` Roots: ${roots.length}`); + console.log(` Name: ${name}`); + console.log(` Status: ${result.status}`); } catch (err) { console.error(toErrorMessage(err)); process.exit(1); diff --git a/packages/cli/src/commands/shared-memory.ts b/packages/cli/src/commands/shared-memory.ts deleted file mode 100644 index 5a9511803a..0000000000 --- a/packages/cli/src/commands/shared-memory.ts +++ /dev/null @@ -1,244 +0,0 @@ -import { Command } from 'commander'; -import { readFileSync, existsSync } from 'node:fs'; -import { createInterface } from 'node:readline'; -import { spawn, execSync } from 'node:child_process'; -import { createReadStream } from 'node:fs'; -import { fileURLToPath } from 'node:url'; -import { dirname, join } from 'node:path'; -import { readFile, writeFile, unlink, appendFile } from 'node:fs/promises'; -import { ethers } from 'ethers'; -import { resolveRpcUrls } from '@origintrail-official/dkg-chain'; -import { - dkgAuthTokenPath, - FAUCET_WALLETS_PER_REQUEST, - getFundableWalletAddresses, - requestFaucetFunding, - resolveDkgConfigHome, - toErrorMessage, - hasErrorCode, -} from '@origintrail-official/dkg-core'; -import yaml from 'js-yaml'; -import { - loadConfig, saveConfig, configExists, configPath, - readPid, readApiPort, isProcessRunning, dkgDir, logPath, ensureDkgDir, removeApiPort, - apiPortPath, - loadNetworkConfig, loadProjectConfig, resolveAutoUpdateConfig, resolveAutoUpdateSource, resolveChainConfig, - releasesDir, activeSlot, swapSlot, - slotEntryPoint, isStandaloneInstall, repoDir, isDkgMonorepo, - resolveContextGraphs, resolveNetworkDefaultContextGraphs, - readNodeRoleFromConfigSync, - type AutoUpdateConfig, -} from '../config.js'; -import { ApiClient } from '../api-client.js'; -import { parsePositiveIntegerOption, parsePositiveMsOption } from '../publisher-runner.js'; -import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; -import { runConfiguredSourceWorker } from '../source-worker-runner.js'; -import { batchEntityQuads } from '../batching.js'; -import { - runDaemon, - checkForNpmVersionUpdate, - performNpmUpdate, - performNpmUpdateEdge, - getCurrentCliVersion, - DAEMON_EXIT_CODE_RESTART, - resolveStandaloneInstall, - decodeForcedExitCode, -} from '../daemon.js'; -import { - isLivenessProbeEnabled, - startLivenessWatcher, - LIVENESS_CONSECUTIVE_FAILURES_TO_KILL, -} from '../daemon/supervisor-liveness.js'; -import { migrateToBlueGreen, noteEdgeLegacyReleases } from '../migration.js'; -import { ensureRollbackNodeUiBundle } from '../rollback-node-ui.js'; -import { - isDaemonUnreachable, - cliSleep, - cliErrorMessage, - STARTUP_BANNER, - normalizeVersionTagRef, - getCliVersion, - parseOptionalVerifyTimeoutOption, - loadStructuredFile, - loadQuadsFromInput, - resolveDaemonEntryPoint, - probeHostForApiHost, - selectedDkgHomeForEnv, - withSelectedDkgHome, - VERIFY_COLLECTION_TIMEOUT_MIN_MS, - VERIFY_COLLECTION_TIMEOUT_MAX_MS, - printCatchupStatus, - runCatchupStatusCommand, - printMessage, - shortId, - formatUptime, - publishEntityBatches, - formatPublisherJobOutput, - formatPublisherJobValue, - stripQuotes, - formatQuadObject, - sleep, - stopDaemonIfRunning, -} from '../cli-helpers.js'; -import type { ActionOpts, CatchupStatusCommandOptions } from '../cli-helpers.js'; -import { - cliWithTimeout, - isCliKnownTransactionError, - isCliRetryableRpcError, - createCliEvmProviders, - getCliReceiptWithFailover, - assertCliSuccessfulReceipt, - sendCliRawTransactionWithFailover, - CLI_RPC_READ_STALL_TIMEOUT_MS, - CLI_RPC_BROADCAST_TIMEOUT_MS, - CLI_RPC_RECEIPT_ATTEMPT_TIMEOUT_MS, - CLI_RPC_RECEIPT_POLL_INTERVAL_MS, - CLI_RPC_RECEIPT_TIMEOUT_MS, -} from '../cli-rpc.js'; -import { - appendSupervisorLog, - supervisorWarn, - maybeStartSupervisorLivenessWatcher, - runDaemonSupervisor, - runForegroundSupervisor, -} from '../cli-supervisor.js'; - -export function registerSharedMemoryCommand(program: Command): void { -// ─── dkg shared-memory (alias: workspace) ─────────────────────────── - -const sharedMemoryCmd = program - .command('shared-memory') - .alias('workspace') - .description('Shared memory operations (write-first workflow)'); - -sharedMemoryCmd - .command('write [context-graph]') - .description('Stage triples into a named WM assertion (the new home of "shared-memory write")') - .option('-f, --file ', 'RDF file (.nq, .nt, .ttl, .trig, .jsonld, .json)') - .option('--format ', 'Explicit RDF format (nquads|ntriples|turtle|trig|json|jsonld)') - .option('-t, --triples ', 'Inline JSON array of {subject, predicate, object} triples') - .option('-s, --subject ', 'Subject URI for simple write') - .option('-p, --predicate ', 'Predicate URI for simple write') - .option('-o, --object ', 'Object value for simple write') - .option('--name ', 'Assertion name (auto-generated if omitted)') - .option('--sub-graph-name ', 'Optional sub-graph name') - .action(async (contextGraph: string | undefined, opts: ActionOpts) => { - try { - const targetContextGraph = contextGraph ?? 'dev-coordination'; - const client = await ApiClient.connect(); - const defaultGraph = `did:dkg:context-graph:${targetContextGraph}`; - const quads = await loadQuadsFromInput(opts, defaultGraph); - const assertionName = - typeof opts.name === 'string' && opts.name.length > 0 - ? (opts.name as string) - : `cli-write-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; - const subGraphOption = (opts.subGraphName as string | undefined) ?? undefined; - - // RFC-001 §9.x — the legacy `shared-memory write` was a - // free-form append into SWM. The new lifecycle requires a named - // assertion. We create lazily on the first batch (no quads), - // then append batches via /api/knowledge-assets/:name/wm/write. - await client.createAssertion(targetContextGraph, assertionName, { - ...(subGraphOption ? { subGraphName: subGraphOption } : {}), - }); - let totalWritten = 0; - await publishEntityBatches( - quads, - async (batch) => { - const result = await client.appendToAssertion( - targetContextGraph, - assertionName, - batch, - subGraphOption ? { subGraphName: subGraphOption } : undefined, - ); - totalWritten += result.written; - return result; - }, - (sent) => { - process.stdout.write(`\r Writing to WM assertion: ${sent}/${quads.length} quads`); - }, - { - maxBatchBytes: 240 * 1024, - estimateBatchBytes: (batch) => new TextEncoder().encode(JSON.stringify({ contextGraphId: targetContextGraph, quads: batch })).length, - splitOversizedEntities: true, - }, - ); - console.log(); - console.log(`Staged WM assertion for "${targetContextGraph}":`); - console.log(` Assertion name: ${assertionName}`); - console.log(` Triples written: ${totalWritten}`); - if (subGraphOption) { - console.log(` Sub-graph: ${subGraphOption}`); - } - console.log(` Next: dkg shared-memory publish ${targetContextGraph} --name ${assertionName}`); - } catch (err) { - console.error(toErrorMessage(err)); - process.exit(1); - } - }); - -sharedMemoryCmd - .command('publish [context-graph]') - .description('Finalize, promote, and publish a previously-staged WM assertion') - .requiredOption('--name ', 'Assertion name (from `dkg shared-memory write --name ...`)') - .option('--sub-graph-name ', 'Optional sub-graph name') - .option('--author-agent-address
', 'Override author EOA (must be a registered local agent)') - .option('--publish-epochs ', 'On-chain publish lifetime in epochs (default: 12; PCA-funded publishes may coerce to PCA lock duration)') - .action(async (contextGraph: string | undefined, opts: ActionOpts) => { - try { - const targetContextGraph = contextGraph ?? 'dev-coordination'; - const assertionName = String(opts.name); - const subGraphOption = (opts.subGraphName as string | undefined) ?? undefined; - const authorAgentAddress = (opts.authorAgentAddress as string | undefined) ?? undefined; - const publishEpochs = opts.publishEpochs !== undefined - ? parsePositiveIntegerOption(String(opts.publishEpochs), '--publish-epochs') - : undefined; - const client = await ApiClient.connect(); - - // RFC-001 §9.x assertion lifecycle: - // finalize → seal in _meta (idempotent on matching merkleRoot) - // promote → SWM gossip - // publish → VM via /api/knowledge-assets/:name/vm/publish - const seal = await client.finalizeAssertion( - targetContextGraph, - assertionName, - { - ...(subGraphOption ? { subGraphName: subGraphOption } : {}), - ...(authorAgentAddress ? { authorAgentAddress } : {}), - }, - ); - const promoted = await client.promoteAssertion(assertionName, { - contextGraphId: targetContextGraph, - ...(subGraphOption ? { subGraphName: subGraphOption } : {}), - }); - const result = await client.publishFromFinalizedAssertion( - targetContextGraph, - assertionName, - (subGraphOption || publishEpochs !== undefined) - ? { - ...(subGraphOption ? { subGraphName: subGraphOption } : {}), - ...(publishEpochs !== undefined ? { publishEpochs } : {}), - } - : undefined, - ); - console.log(`Published WM assertion to "${targetContextGraph}":`); - console.log(` Assertion: ${seal.assertionUri}`); - console.log(` Author: ${seal.authorAddress}`); - console.log(` Merkle root: ${seal.merkleRoot}`); - console.log(` Promoted: ${promoted.promotedCount ?? promoted.count ?? 0} quads`); - console.log(` Status: ${result.status}`); - console.log(` KC ID: ${result.kaId}`); - console.log(` KAs: ${result.kas.length}`); - if (subGraphOption) { - console.log(` Sub-graph: ${subGraphOption}`); - } - if (result.txHash) { - console.log(` TX: ${result.txHash}`); - } - } catch (err) { - console.error(toErrorMessage(err)); - process.exit(1); - } - }); - -} diff --git a/packages/cli/src/daemon/http-utils.ts b/packages/cli/src/daemon/http-utils.ts index 88a35f0a37..a51e7aff12 100644 --- a/packages/cli/src/daemon/http-utils.ts +++ b/packages/cli/src/daemon/http-utils.ts @@ -1003,7 +1003,7 @@ export function validateConditions(conditions: unknown, res: ServerResponse): bo if (!Array.isArray(conditions) || conditions.length === 0) { jsonResponse(res, 400, { error: - '"conditions" must be a non-empty array (use /api/shared-memory/write for unconditional writes)', + '"conditions" must be a non-empty array (use the knowledge asset lifecycle routes for unconditional writes)', }); return false; } diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index b2a03fc711..46eaefeb9c 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -1791,6 +1791,34 @@ export async function runDaemonInner( log, }), publishEncryptionFactory: (publishOptions) => resolveDaemonPublishEncryption(agent, publishOptions), + knowledgeAssetVmPublishExecutor: async ({ request, publishOptions, publisher }) => { + const publishOpts = { + ...(publisher ? { publisherOverride: publisher } : {}), + }; + try { + return await agent.publishQueuedKnowledgeAssetVmPublish( + request, + publishOptions, + publishOpts, + ); + } catch (firstErr: any) { + if ( + firstErr?.code !== "CG_NOT_REGISTERED" && + !/not registered on-chain/i.test(firstErr?.message ?? String(firstErr)) + ) { + throw firstErr; + } + const defaultAgentAddress = agent.getDefaultAgentAddress(); + await agent.ensureRegisteredForPublish(request.contextGraphId, { + ...(defaultAgentAddress ? { callerAgentAddress: defaultAgentAddress } : {}), + }); + return await agent.publishQueuedKnowledgeAssetVmPublish( + request, + publishOptions, + publishOpts, + ); + } + }, log, }); publisherRuntime = runtime; @@ -2505,21 +2533,6 @@ export async function runDaemonInner( subGraphName?: string; }, ) => agent.query(sparql, opts), - share: ( - contextGraphId: string, - quads: any[], - opts?: { localOnly?: boolean; subGraphName?: string }, - ) => agent.share(contextGraphId, quads, opts).then((result: any) => { - emitMemoryGraphChanged({ - contextGraphId, - layers: ["swm"], - subGraphName: opts?.subGraphName, - operation: "shared_memory_written", - source: opts?.localOnly ? "agent_tool_local" : "agent_tool", - counts: { triples: quads.length }, - }); - return result; - }), createAssertion: async ( contextGraphId: string, name: string, @@ -2561,40 +2574,6 @@ export async function runDaemonInner( }); return { written: quads.length }; }, - publishFromSharedMemory: ( - contextGraphId: string, - selection: "all" | { rootEntities: string[] }, - opts?: { clearSharedMemoryAfter?: boolean; subGraphName?: string }, - ) => { - const publishOpts = { - ...opts, - clearSharedMemoryAfter: opts?.clearSharedMemoryAfter ?? false, - }; - return agent.publishFromSharedMemory(contextGraphId, selection, publishOpts).then((result: any) => { - const clearAfter = publishOpts.clearSharedMemoryAfter; - const publishedSwmCleaned = result?.status === "confirmed"; - const rootCount = Array.isArray(result?.kaManifest) - ? result.kaManifest.length - : undefined; - const publicTripleCount = Array.isArray(result?.publicQuads) - ? result.publicQuads.length - : undefined; - emitMemoryGraphChanged({ - contextGraphId, - layers: publishedSwmCleaned ? ["swm", "vm"] : ["vm"], - subGraphName: opts?.subGraphName, - operation: "shared_memory_published", - source: "agent_tool", - clearSharedMemoryAfter: clearAfter, - status: typeof result?.status === "string" ? result.status : undefined, - counts: { - roots: rootCount, - triples: publicTripleCount, - }, - }); - return result; - }); - }, createContextGraph: (opts: { id: string; name: string; diff --git a/packages/cli/src/daemon/routes/knowledge-assets.ts b/packages/cli/src/daemon/routes/knowledge-assets.ts index ccea125f61..a84e3642fb 100644 --- a/packages/cli/src/daemon/routes/knowledge-assets.ts +++ b/packages/cli/src/daemon/routes/knowledge-assets.ts @@ -14,6 +14,7 @@ // POST /api/knowledge-assets/:name/wm/pull-from seed draft from SWM/VM [TODO] // POST /api/knowledge-assets/:name/swm/share advance the SWM pointer // POST /api/knowledge-assets/:name/vm/publish mint/update on chain +// POST /api/knowledge-assets/:name/vm/publish-async enqueue mint/update on chain // // These delegate to the SAME agent lifecycle methods the legacy // `/api/assertion/*` + `/api/shared-memory/*` routes use, so behavior is @@ -63,7 +64,7 @@ import { decodePromoteJobId, asyncPromoteUnavailable, } from "./shared-assertion-helpers.js"; -import { PromoteJobConflictError } from "@origintrail-official/dkg-publisher"; +import { AsyncLiftJobConflictError, PromoteJobConflictError } from "@origintrail-official/dkg-publisher"; import { deriveStatus } from "@origintrail-official/dkg-publisher"; import { validateAssertionName, contextGraphAssertionUri } from "@origintrail-official/dkg-core"; @@ -505,7 +506,7 @@ async function verifyDirectPublishOnChainContextGraphId( } export async function handleKnowledgeAssetsRoutes(ctx: RequestContext): Promise { - const { req, res, agent, path, url, requestToken, requestAgentAddress, emitMemoryGraphChanged } = ctx; + const { req, res, agent, publisherControl, path, url, requestToken, requestAgentAddress, emitMemoryGraphChanged } = ctx; if (path !== PREFIX && !path.startsWith(`${PREFIX}/`)) return; const method = req.method ?? "GET"; @@ -684,14 +685,16 @@ export async function handleKnowledgeAssetsRoutes(ctx: RequestContext): Promise< preSignedAuthorAttestation, schemeVersion, alsoPublishVm, + awaitCuratorAck, } = parsed; - // OT-RFC-43 migration alias: the legacy one-shot publish shape posts - // `promote: true` (ApiClient.publishAssertion and network-sim still send - // { quads, finalize: true, promote: true }). - // Honor it as `alsoShareSwm` so those calls still promote WM→SWM — otherwise - // they seal WM but never promote, and a follow-up VM publish runs against an - // empty SWM and fails. An explicit `alsoShareSwm` wins when both are supplied. - const alsoShareSwm = parsed.alsoShareSwm ?? parsed.promote; + // No legacy alias: product callers must name the lifecycle transition. + // `alsoShareSwm: true` is the only accepted create-route flag for advancing + // a sealed WM assertion into SWM. The async/sync publish preflight relies on + // this explicit share step. + const alsoShareSwm = parsed.alsoShareSwm; + if (parsed.promote !== undefined) { + return jsonResponse(res, 400, { error: '"promote" is retired; use "alsoShareSwm" for the WM to SWM lifecycle transition' }); + } if (!name) { return jsonResponse(res, 400, { error: 'Missing "name"' }); } @@ -717,6 +720,9 @@ export async function handleKnowledgeAssetsRoutes(ctx: RequestContext): Promise< if (alsoShareSwm !== undefined && typeof alsoShareSwm !== "boolean") { return jsonResponse(res, 400, { error: '"alsoShareSwm" must be a boolean when supplied' }); } + if (awaitCuratorAck !== undefined && typeof awaitCuratorAck !== "boolean") { + return jsonResponse(res, 400, { error: '"awaitCuratorAck" must be a boolean when supplied' }); + } if (alsoPublishVm !== undefined && typeof alsoPublishVm !== "boolean" && (typeof alsoPublishVm !== "object" || alsoPublishVm === null || Array.isArray(alsoPublishVm))) { return jsonResponse(res, 400, { error: '"alsoPublishVm" must be a boolean or an options object when supplied' }); } @@ -825,16 +831,17 @@ export async function handleKnowledgeAssetsRoutes(ctx: RequestContext): Promise< try { // Carry the same resolved author into the share. The asset is already // sealed (finalize above), so promote shares the existing seal verbatim - // — passing the author keeps the whole atomic flow in one namespace and - // covers the seal-on-share path too. + // and passing the author keeps the whole atomic flow in one namespace. const share = await agent.assertion.promote(resolvedContextGraphId, name, { subGraphName, + awaitCuratorAck, ...(resolvedAuthorAgentAddress ? { authorAgentAddress: resolvedAuthorAgentAddress } : {}), }); result.swmShared = true; result.promotedCount = share.promotedCount; result.sealed = share.sealed; result.publishReady = share.publishReady; + if (share.shareOperationId) result.shareOperationId = share.shareOperationId; // #1116: the one-shot finalizes BEFORE sharing, so a shared asset is // normally sealed ("swm-shared"). The unsealed status is only reachable // if a future path shares without sealing. @@ -1157,7 +1164,13 @@ export async function handleKnowledgeAssetsRoutes(ctx: RequestContext): Promise< } // #1116: surface the seal outcome. `sealed`/`publishReady` describe THIS // share (subset or skipSeal → false by design, not a failure). - return jsonResponse(res, 200, { swmShared: true, promotedCount: share.promotedCount, sealed: share.sealed, publishReady: share.publishReady }); + return jsonResponse(res, 200, { + swmShared: true, + promotedCount: share.promotedCount, + sealed: share.sealed, + publishReady: share.publishReady, + ...(share.shareOperationId ? { shareOperationId: share.shareOperationId } : {}), + }); } catch (e: any) { // #1116 D1: a default full share that can't seal (a residual capability // gap, no skipSeal) fails CLOSED with WM preserved — map to a 409 that @@ -1226,6 +1239,60 @@ export async function handleKnowledgeAssetsRoutes(ctx: RequestContext): Promise< // Publish keeps its own generic-500 catch: on-chain/storage/publisher // failures can carry "Invalid"/"Unsafe" text and must NOT be down-classified // to 400 (parity with the legacy publish path). + if (layer === "vm" && verb === "publish-async") { + try { + if (!validateFinalizedAssertionPublishRequest(parsed, res)) return; + const opts = resolveFinalizedPublishOptions(ctx, parsed.options); + if (opts === null) return; + const publishOptions = opts as { + publishEpochs?: number; + clearSharedMemoryAfter?: boolean; + publisherNodeIdentityIdOverride?: bigint; + }; + const intent = await agent.resolveFinalizedAssertionVmPublishIntent(contextGraphId, name, { + ...(subGraphName ? { subGraphName } : {}), + ...(publishOptions.publishEpochs !== undefined ? { publishEpochs: publishOptions.publishEpochs } : {}), + ...(publishOptions.clearSharedMemoryAfter !== undefined + ? { clearSharedMemoryAfter: publishOptions.clearSharedMemoryAfter } + : {}), + ...(publishOptions.publisherNodeIdentityIdOverride !== undefined + ? { publisherNodeIdentityIdOverride: publishOptions.publisherNodeIdentityIdOverride } + : {}), + }); + await agent.preflightKnowledgeAssetVmPublishSnapshot(intent); + const jobId = await publisherControl.enqueueKnowledgeAssetVmPublish(intent); + return jsonResponse(res, 202, { + jobId, + status: "accepted", + contextGraphId, + name, + shareOperationId: intent.shareOperationId, + rootsCount: intent.roots.length, + sealMerkleRoot: intent.sealMerkleRoot, + intentKey: intent.intentKey, + ...(subGraphName ? { subGraphName } : {}), + }); + } catch (err: any) { + if (err instanceof AsyncLiftJobConflictError) { + return jsonResponse(res, 409, { + error: err.message, + existingJobId: err.existingJobId, + }); + } + if (err?.code === "PUBLISH_NOT_FULL_SHARE" || err?.code === "PUBLISH_INTENT_STALE") { + return jsonResponse(res, 409, { code: err.code, error: err.message ?? String(err) }); + } + if ( + err.message?.includes("required") || + err.message?.includes("Invalid") || + err.message?.includes("must be") + ) { + return jsonResponse(res, 400, { error: err.message }); + } + throw err; + } + } + if (layer === "vm" && verb === "publish") { // #988: publish keeps its OWN generic-500 catch (NOT the outer // respondAssertionError) so on-chain/storage "Invalid"/"Unsafe" text isn't diff --git a/packages/cli/src/daemon/routes/memory.ts b/packages/cli/src/daemon/routes/memory.ts index 6aefc3f3e1..80d39ea540 100644 --- a/packages/cli/src/daemon/routes/memory.ts +++ b/packages/cli/src/daemon/routes/memory.ts @@ -2,7 +2,7 @@ // // Route handlers for shared-memory / workspace write + publish + conditional-write, memory turn/search. // -// Extracted verbatim from the legacy monolithic `handleRequest` — +// Extracted verbatim from the legacy monolithic `handleRequest` -- // every block is a contiguous slice of the original source with zero // edits to route bodies. Dispatch is driven by the surviving // `handle-request.ts` shell, which awaits each group handler in @@ -37,7 +37,7 @@ import { existsSync, readdirSync, readFileSync, openSync, closeSync, writeFileSy // Namespace import: our Phase-8 install-context builder (~line 290) calls // `osModule.homedir()`, and the later agent-identity probe (~line 6851) // uses `osModule.hostname()` + `osModule.userInfo()`. v10-rc's new -// OpenClaw config helper (~line 2535) uses a bare `homedir()` — aliased +// OpenClaw config helper (~line 2535) uses a bare `homedir()` -- aliased // below so both sites coexist without a duplicate-module import. import * as osModule from 'node:os'; const { homedir } = osModule; @@ -62,7 +62,7 @@ import { createSwmCatchupPeerSelector, loadOpWallets, } from '@origintrail-official/dkg-agent'; -import { computeNetworkId, createOperationContext, DKGEvent, Logger, PayloadTooLargeError, GET_VIEWS, TrustLevel, validateSubGraphName, validateContextGraphId, isSafeIri, assertSafeIri, assertSafeRdfTerm, contextGraphSharedMemoryUri, contextGraphAssertionUri, contextGraphMetaUri, escapeDkgRdfLiteral, escapeSparqlLiteral, PROTOCOL_SYNC } from '@origintrail-official/dkg-core'; +import { computeNetworkId, createOperationContext, DKGEvent, Logger, PayloadTooLargeError, GET_VIEWS, TrustLevel, validateSubGraphName, validateContextGraphId, isSafeIri, assertSafeIri, assertSafeRdfTerm, contextGraphSharedMemoryUri, contextGraphMetaUri, escapeDkgRdfLiteral, escapeSparqlLiteral, PROTOCOL_SYNC } from '@origintrail-official/dkg-core'; import type { Quad } from '@origintrail-official/dkg-storage'; import { DashboardDB, @@ -129,7 +129,7 @@ import { type ExtractionStatusRecord, getExtractionStatusRecord, setExtractionSt import { FileStore } from '../../file-store.js'; import { VectorStore, OpenAIEmbeddingProvider, type EmbeddingProvider } from '../../vector-store.js'; import { parseBoundary, parseMultipart, MultipartParseError } from '../../http/multipart.js'; -// Phase 8 — project-manifest publish + install (UI-driven onboarding flow). +// Phase 8 -- project-manifest publish + install (UI-driven onboarding flow). // Daemon constructs a self-pointing DkgClient (localhost:listenPort) and // reuses the same publish/fetch/plan/write helpers the CLI uses, so wire // format stays identical between curator/joiner/CLI paths. @@ -146,7 +146,7 @@ import { } from '@origintrail-official/dkg-mcp/manifest/install'; import { DkgClient } from '@origintrail-official/dkg-mcp/client'; -// Daemon sub-module imports — every public symbol from sibling +// Daemon sub-module imports -- every public symbol from sibling // modules is pulled in here because the legacy monolithic file used // them all without explicit imports. Unused ones are tolerated by // the project's tsconfig (`noUnusedLocals` is off). @@ -348,13 +348,13 @@ import type { RequestContext } from './context.js'; * path (it recovers the address from the EIP-712 digest and fails closed * if the recovered signer doesn't match the claimed address). * - * RFC-001 §9.x — Phase C — pre-signed attestations are a finalize-time + * RFC-001 Section 9.x -- Phase C -- pre-signed attestations are a finalize-time * concern. The publish layer no longer accepts them; they're consumed * here and stamped into the seal. */ type PreSignedAuthorAttestation = { address: string; - // OT-RFC-43 §F2 — the packed reservedKaId the author signed over. Required so + // OT-RFC-43 Section F2 -- the packed reservedKaId the author signed over. Required so // the daemon honours the author's reserved slot (the digest binds it) rather // than re-allocating; threaded into agent.assertion.finalize. reservedKaId: bigint; @@ -403,13 +403,13 @@ export function validatePreSignedAuthorAttestation( }); return undefined; } - // OT-RFC-43 §F2 — the AuthorAttestation digest binds the packed reservedKaId, + // OT-RFC-43 Section F2 -- the AuthorAttestation digest binds the packed reservedKaId, // so the caller MUST forward the exact id they signed over. It travels as a // decimal string (uint256-safe over JSON); accept an integer number too. const reservedKaId = decodeReservedKaId(obj.reservedKaId); if (reservedKaId === undefined) { jsonResponse(res, 400, { - error: '"preSignedAuthorAttestation.reservedKaId" must be the packed KA id the author signed over, as a non-negative decimal string (OT-RFC-43 §F2)', + error: '"preSignedAuthorAttestation.reservedKaId" must be the packed KA id the author signed over, as a non-negative decimal string (OT-RFC-43 Section F2)', }); return undefined; } @@ -638,19 +638,19 @@ WHERE { // POST /api/shared-memory/catchup // - // OT-RFC-38 LU-7 — explicit SWMCatchupRequest endpoint. Pulls the + // OT-RFC-38 LU-7 -- explicit SWMCatchupRequest endpoint. Pulls the // remote SWM state for one or more context graphs from connected // peers, applying everything authorized into the local triple store. // // Body: { contextGraphId: string | string[], peerId?: string } // - peerId: optional. When set, sync only from this specific peer. // When omitted, iterate ALL currently-connected libp2p peers and - // try each — first peer that authorises serves the request, + // try each -- first peer that authorises serves the request, // subsequent peers' decisions are independent. // // Returns: per-peer outcome with inserted/fetched counters. // - // Auth model (per SPEC_CG_HOSTING_MEMBERSHIP §5.6.4): + // Auth model (per SPEC_CG_HOSTING_MEMBERSHIP Section 5.6.4): // - Public CGs (accessPolicy == 0): the responder's sync handler // accepts anonymous catchup (no `authorizePrivateSyncRequest` // gate). Any reachable peer can backfill SWM. @@ -679,13 +679,13 @@ WHERE { if (cgIds.length === 0) { return jsonResponse(res, 400, { error: - 'Missing "contextGraphId" — pass a single context graph id string or an array of ids', + 'Missing "contextGraphId" -- pass a single context graph id string or an array of ids', }); } // OT-RFC-38 LU-7: SWMCatchupRequest is SWM-only. The durable // (knowledge-collection) layer has its own publish-time - // commit→fanout→ACK protocol and a separate sync substrate; it's + // commit->fanout->ACK protocol and a separate sync substrate; it's // out of scope for the catchup endpoint and would otherwise compound // the request budget (240s vs 120s). Opt-in via includeDurable=true // for callers that want the full data leg in the same call. @@ -794,7 +794,7 @@ WHERE { }); } - // Per-CG × per-peer sync. The previous shape called + // Per-CG × per-peer sync. The previous shape called // `syncSharedMemoryFromPeer(peer, cgIds)` ONCE per peer with the // full CG list, which only returned an aggregate count and made // a per-CG LU-6 fallback decision impossible (Codex PR #610 R1 @@ -802,9 +802,9 @@ WHERE { // for the others got skipped on the aggregate gate). // // Now: iterate CGs serially (keeps wire load bounded across many - // peers × many CGs), select a narrowed per-CG peer set, and parallelize + // peers × many CGs), select a narrowed per-CG peer set, and parallelize // only that set. Per-peer dial+request is 5-20s on devnet; serialising - // the selected peers would compound to N×20s. + // the selected peers would compound to N×20s. type PerPeerLeg = { peerId: string; insertedTriples: number; @@ -917,7 +917,7 @@ WHERE { }); } - // OT-RFC-38 LU-6 — per-CG host-catchup fallback. For each CG + // OT-RFC-38 LU-6 -- per-CG host-catchup fallback. For each CG // whose standard sync inserted 0 triples, fall back to fetching // opaque ciphertext envelopes from connected core hosts and // re-applying them through the local sender-key decryptor. @@ -1053,7 +1053,7 @@ WHERE { }); } - // OT-RFC-38 LU-6 — dedicated host-catchup endpoint. + // OT-RFC-38 LU-6 -- dedicated host-catchup endpoint. // // POST /api/shared-memory/host-catchup // Body: { contextGraphId: string, peerId?: string, sinceSeqno?: number, maxRounds?: number } @@ -1062,7 +1062,7 @@ WHERE { // hosting the curated CG's SWM substrate and re-applies each // through the local agent so the existing Sender-Key decrypt // path runs verbatim. Distinct from the "fallback" leg embedded - // in /catchup above — exposed so operators can debug host + // in /catchup above -- exposed so operators can debug host // hosting independently (e.g. to confirm a specific core has // stored ciphertext for a CG). if (req.method === 'POST' && path === '/api/shared-memory/host-catchup') { @@ -1104,7 +1104,7 @@ WHERE { } } - // OT-RFC-38 LU-6 — host-mode store diagnostics. + // OT-RFC-38 LU-6 -- host-mode store diagnostics. // GET /api/shared-memory/host-mode/stats // Returns { enabled, cgCount, totalBytes, totalEntries, subscribedCgIds }. if (req.method === 'GET' && path === '/api/shared-memory/host-mode/stats') { @@ -1119,7 +1119,7 @@ WHERE { } } - // OT-RFC-38 LU-6 — explicit host-mode subscribe. + // OT-RFC-38 LU-6 -- explicit host-mode subscribe. // POST /api/shared-memory/host-mode/subscribe { contextGraphId } // Tells a core to start hosting the curated CG's encrypted SWM // substrate WITHOUT requiring the core to become a CG member. @@ -1144,10 +1144,10 @@ WHERE { } } - // Tiny local helper — kept inline to avoid adding a new import for + // Tiny local helper -- kept inline to avoid adding a new import for // a single use; the existing route module already has utilities // for hex/bytes interop scattered across the file but none are - // strictly typed `bytes32`. 64-char hex (no 0x) → 32-byte buffer. + // strictly typed `bytes32`. 64-char hex (no 0x) -> 32-byte buffer. function hexToBytes32(h: string): Uint8Array { const clean = h.startsWith('0x') ? h.slice(2) : h; if (clean.length !== 64) throw new Error('expected 32-byte hex'); @@ -1158,9 +1158,9 @@ WHERE { // POST /api/shared-memory/verify-batch // - // OT-RFC-38 LU-8 — Member post-decrypt batch verification. + // OT-RFC-38 LU-8 -- Member post-decrypt batch verification. // - // SPEC_CG_HOSTING_MEMBERSHIP §5.3.1: members re-derive the plaintext + // SPEC_CG_HOSTING_MEMBERSHIP Section 5.3.1: members re-derive the plaintext // merkle root from a reconstructed batch and compare to the on-chain // anchor. This endpoint exposes the recompute step. // @@ -1231,11 +1231,12 @@ WHERE { }); } - // POST /api/shared-memory/report-batch-rejection + // POST /api/knowledge-assets/batch-rejections/report // - // OT-RFC-38 LU-8 — when verifyBatch returns ok=false, the member - // gossips a structured BatchRejection record so other members can - // sanity-check and re-pull from a different host. + // OT-RFC-38 LU-8 - when verifyBatch returns ok=false, the member + // creates and shares a named BatchRejection KA so other members can + // sanity-check and re-pull from a different host without using a loose + // shared-memory write route. // // Body: { // contextGraphId: string, @@ -1243,7 +1244,7 @@ WHERE { // verifyResult: { ok: false, expectedRoot, actualRoot, leafCount, reason }, // rejectedBy?: { agentAddress, peerId }, // defaults to local agent // } - if (req.method === "POST" && path === "/api/shared-memory/report-batch-rejection") { + if (req.method === "POST" && path === "/api/knowledge-assets/batch-rejections/report") { const body = await readBody(req, SMALL_BODY_BYTES); const parsed = safeParseJson(body, res); if (!parsed) return; @@ -1287,9 +1288,9 @@ WHERE { return jsonResponse(res, 400, { error: err?.message ?? String(err) }); } - // Persist the record as SWM triples so it gossips via the - // standard SWM substrate to other members. Reuses agent.share() - // for the write — no new transport. + // Persist the record as a named KA and share it to SWM so it gossips via + // the standard SWM substrate to other members. This intentionally avoids + // the retired loose shared-memory write route. // // Codex PR #609: every value that originates from HTTP body // (contextGraphId, batchId, peerId, reason, agentAddress) is @@ -1298,11 +1299,12 @@ WHERE { // store insert outright or lets the caller smuggle malformed / // attacker-controlled triples through this endpoint. We pipe // every interpolated literal body through `escapeDkgRdfLiteral` - // (defense in depth — even fields like rootHashes that are + // (defense in depth - even fields like rootHashes that are // structurally constrained to 0x-hex still get escaped, so a // future input-validation regression doesn't reopen the hole). const lit = (s: string) => `"${escapeDkgRdfLiteral(s)}"`; const subject = `did:dkg:batch-rejection:${record.digest}`; + const assertionName = `batch-rejection-${String(record.digest).toLowerCase().replace(/^0x/, '').slice(0, 48)}`; const NS = 'http://dkg.io/ontology/'; const quads = [ { subject, predicate: `${NS}rejectedContextGraphId`, object: lit(record.contextGraphId), graph: '' }, @@ -1318,9 +1320,16 @@ WHERE { ]; try { - await agent.share(resolvedContextGraphId, quads, { - operationCtx: createOperationContext('share'), - callerAgentAddress: requestAgentAddress, + await agent.assertion.create(resolvedContextGraphId, assertionName); + await agent.assertion.write(resolvedContextGraphId, assertionName, quads); + await agent.assertion.finalize(resolvedContextGraphId, assertionName); + const share = await agent.assertion.promote(resolvedContextGraphId, assertionName); + return jsonResponse(res, 200, { + record, + gossiped: true, + assertionName, + shareOperationId: share.shareOperationId, + promotedCount: share.promotedCount, }); } catch (err: any) { // The record itself is the deliverable; gossip is best-effort. @@ -1329,16 +1338,15 @@ WHERE { return jsonResponse(res, 200, { record, gossiped: false, + assertionName, gossipError: err?.message ?? String(err), }); } - - return jsonResponse(res, 200, { record, gossiped: true }); } // POST /api/attestation/mint // - // OT-RFC-38 LU-9 — Member-attested verification token. + // OT-RFC-38 LU-9 -- Member-attested verification token. // // Body: { // contextGraphId: string, // local CG id (numeric on-chain id resolved server-side) @@ -1373,7 +1381,7 @@ WHERE { const chainId = chain?.chainId ?? parsed.chainId ?? '31337'; if (!kavAddress || !/^0x[0-9a-fA-F]{40}$/.test(String(kavAddress))) { return jsonResponse(res, 400, { - error: 'cannot determine KAV10 address — pass `kavAddress` explicitly', + error: 'cannot determine KAV10 address -- pass `kavAddress` explicitly', }); } @@ -1383,15 +1391,15 @@ WHERE { // subscription metadata couldn't resolve the on-chain id. That // silently minted an attestation token bound to ContextGraphId=0 // (the sentinel for "no on-chain CG") even though a real KC for - // this batch already exists on-chain — outsiders verifying the + // this batch already exists on-chain -- outsiders verifying the // token would see it pass cryptographic checks but reject as // wrong-domain, with no diagnostic linking back to the actual CG. // Three resolution layers, all fail-closed: // 1. Caller-supplied `onChainContextGraphId` (explicit override). - // 2. Chain-truth via `chain.getKAContextGraphId(batchId)` — - // authoritative because the KC ↔ CG binding is on-chain. + // 2. Chain-truth via `chain.getKAContextGraphId(batchId)` -- + // authoritative because the KC <-> CG binding is on-chain. // 3. Local CG listing (last-resort, may be stale post-event-replay). - // If none resolve, reject with 400 — minting against id=0 is never + // If none resolve, reject with 400 -- minting against id=0 is never // correct. let onChainCgId: string | undefined; if (typeof parsed.onChainContextGraphId === 'string' && /^\d+$/.test(parsed.onChainContextGraphId)) { @@ -1448,7 +1456,7 @@ WHERE { attestedAt: Math.floor(Date.now() / 1000), }, sign: async (digest) => { - // Convert (r, vs) → compact 65-byte hex via ethers.Signature. + // Convert (r, vs) -> compact 65-byte hex via ethers.Signature. const sigParts = await chain.signMessage(digest); const r = '0x' + Array.from(sigParts.r as Uint8Array).map((b: number) => b.toString(16).padStart(2, '0')).join(''); const vs = '0x' + Array.from(sigParts.vs as Uint8Array).map((b: number) => b.toString(16).padStart(2, '0')).join(''); @@ -1465,14 +1473,14 @@ WHERE { // POST /api/attestation/verify // - // OT-RFC-38 LU-9 — outsider-side verification. + // OT-RFC-38 LU-9 -- outsider-side verification. // // Body: { // attestation: MemberAttestation, // candidateLeafHex?: string, // optional 0x-prefixed bytes for leaf check // chainCheckMembership?: boolean // if true, the daemon attempts a chain-side // // membership lookup (Phase B); currently - // // always returns "unknown" — surfaces the + // // always returns "unknown" -- surfaces the // // gap honestly. // } if (req.method === "POST" && path === "/api/attestation/verify") { @@ -1495,7 +1503,7 @@ WHERE { } const { verifyMemberAttestation } = await import('@origintrail-official/dkg-agent'); - // Codex PR #609 R2 #3 — only supply a membership resolver when + // Codex PR #609 R2 #3 -- only supply a membership resolver when // the caller explicitly opted into `chainCheckMembership`. // Previously we always passed a stub, which made every response // carry `membership: "unknown"` and erased the distinction @@ -1515,213 +1523,20 @@ WHERE { return jsonResponse(res, 200, result); } - // POST /api/shared-memory/write - // - // Direct SWM write entry point. Writes loose triples to shared memory - // without minting a named-assertion seal. Triples land in SWM as - // ungrouped content. This is a non-agent primitive (source-worker - // bulk-ingest, programmatic clients); the loose content it produces is - // not publishable through the named-KA lifecycle. Consolidating these - // producers onto the named-KA model is tracked in OriginTrail/dkg#1260. - // - // For seal-from-creation provenance, use the named-assertion lifecycle - // instead: POST /api/knowledge-assets with `quads, finalize: true`, - // then POST /api/knowledge-assets//swm/share, then - // POST /api/knowledge-assets//vm/publish. - if (req.method === "POST" && path === "/api/shared-memory/write") { - const body = await readBody(req); - const parsed = safeParseJson(body, res); - if (!parsed) return; - const { quads, subGraphName } = parsed; - const localOnly = parsed.localOnly === true; - if ( - parsed.localOnly !== undefined && - typeof parsed.localOnly !== "boolean" - ) { - return jsonResponse(res, 400, { error: '"localOnly" must be a boolean' }); - } - // Per-request override of the strict curator-ack gate (OT-RFC-49 - // curator-leader). Omitted → the agent uses its config default - // (`swmAwaitCuratorAck`). Only meaningful for private, non-localOnly writes. - if ( - parsed.awaitCuratorAck !== undefined && - typeof parsed.awaitCuratorAck !== "boolean" - ) { - return jsonResponse(res, 400, { error: '"awaitCuratorAck" must be a boolean' }); - } - const awaitCuratorAck: boolean | undefined = parsed.awaitCuratorAck; - const contextGraphId = parsed.contextGraphId; - if (!quads?.length) - return jsonResponse(res, 400, { error: 'Missing "quads"' }); - // GH #787 / #306 — reject string-shaped / malformed quads here (4xx) instead - // of crashing the SWM write path with a TypeError (HTTP 500). - if (!Array.isArray(quads) || !quads.every(isWritableQuad)) - return jsonResponse(res, 400, { error: '"quads" must be an array of { subject, predicate, object } objects (graph optional); string-shaped quads are not accepted' }); - // GH #306/#787 (follow-up) — also reject objects that are neither a quoted - // literal nor an absolute IRI; otherwise they slip past the shape guard and - // crash the RDF parser ("No scheme found in an absolute IRI") with HTTP 500. - { - const objErr = validateQuadObjectTerms("quads", quads); - if (objErr) return jsonResponse(res, 400, { error: objErr }); - } - const literalSize = validateWritableQuadLiteralSizes("quads", quads); - if (!literalSize.ok) return jsonResponse(res, 400, literalSize.body); - const resolvedContextGraphId = await resolveRequiredWriteContextGraphId( - agent, - contextGraphId, - res, - writePreflightContextGraphOpts, - ); - if (!resolvedContextGraphId) return; - if (!validateOptionalSubGraphName(subGraphName, res)) return; - const ctx = createOperationContext("share"); - tracker.start(ctx, { - contextGraphId: resolvedContextGraphId, - details: { tripleCount: quads.length, source: "api", subGraphName }, - }); - try { - await tracker.trackPhase(ctx, "validate", async () => { - // validation happens inside share - }); - const result = await tracker.trackPhase(ctx, "store", () => - agent.share(resolvedContextGraphId, quads, { - subGraphName, - localOnly, - operationCtx: ctx, - callerAgentAddress: requestAgentAddress, - awaitCuratorAck, - }), - ); - tracker.complete(ctx, { tripleCount: quads.length }); - emitMemoryGraphChanged?.({ - contextGraphId: resolvedContextGraphId, - layers: ["swm"], - subGraphName, - operation: "shared_memory_written", - source: localOnly ? "api-local" : "api", - counts: { triples: quads.length }, - }); - return jsonResponse(res, 200, { - shareOperationId: result?.shareOperationId, - contextGraphId: resolvedContextGraphId, - graph: contextGraphSharedMemoryUri(resolvedContextGraphId, subGraphName), - triplesWritten: quads.length, - }); - } catch (err: any) { - tracker.fail(ctx, err); - if ( - typeof err?.message === "string" && - err.message.includes("has not been registered") - ) { - return jsonResponse(res, 400, { error: err.message }); - } - // Strict curator-ack gate (OT-RFC-49 curator-leader): the write was NOT - // persisted because the curator (the authoritative replica) did not - // confirm it. Surface a distinct, actionable status instead of a generic - // 500 — the client is TOLD, never silently led to believe it succeeded. - // Duck-type on `.code` (the publisher's wire contract). - if (err?.code === "CURATOR_UNCONFIRMED") { - return jsonResponse(res, 503, { - error: err.message, - code: "CURATOR_UNCONFIRMED", - curatorDelivery: "unconfirmed", - }); - } - if (err?.code === "CURATOR_REJECTED") { - return jsonResponse(res, 409, { - error: err.message, - code: "CURATOR_REJECTED", - curatorDelivery: "rejected", - }); - } - throw err; - } - } - - // POST /api/shared-memory/conditional-write { contextGraphId, quads, conditions, subGraphName? } - if ( - req.method === "POST" && - path === "/api/shared-memory/conditional-write" - ) { - const body = await readBody(req); - const parsed = safeParseJson(body, res); - if (!parsed) return; - const { quads, conditions, subGraphName } = parsed; - const contextGraphId = parsed.contextGraphId; - if (!quads?.length) - return jsonResponse(res, 400, { error: 'Missing "quads"' }); - // GH #787 / #306 — reject string-shaped / malformed quads (4xx, not a 500 crash). - if (!Array.isArray(quads) || !quads.every(isWritableQuad)) - return jsonResponse(res, 400, { error: '"quads" must be an array of { subject, predicate, object } objects (graph optional); string-shaped quads are not accepted' }); - // GH #306/#787 (follow-up) — also reject objects that are neither a quoted - // literal nor an absolute IRI; otherwise they slip past the shape guard and - // crash the RDF parser ("No scheme found in an absolute IRI") with HTTP 500. - { - const objErr = validateQuadObjectTerms("quads", quads); - if (objErr) return jsonResponse(res, 400, { error: objErr }); - } - const literalSize = validateWritableQuadLiteralSizes("quads", quads); - if (!literalSize.ok) return jsonResponse(res, 400, literalSize.body); - const resolvedContextGraphId = await resolveRequiredWriteContextGraphId( - agent, - contextGraphId, - res, - writePreflightContextGraphOpts, - ); - if (!resolvedContextGraphId) return; - if (!validateConditions(conditions, res)) return; - if (!validateOptionalSubGraphName(subGraphName, res)) return; - const ctx = createOperationContext("share"); - tracker.start(ctx, { - contextGraphId: resolvedContextGraphId, - details: { tripleCount: quads.length, source: "api-cas", subGraphName }, - }); - try { - const result = await agent.conditionalShare( - resolvedContextGraphId, - quads, - conditions, - { subGraphName, operationCtx: ctx, callerAgentAddress: requestAgentAddress }, - ); - tracker.complete(ctx, { tripleCount: quads.length }); - emitMemoryGraphChanged?.({ - contextGraphId: resolvedContextGraphId, - layers: ["swm"], - subGraphName, - operation: "shared_memory_conditional_written", - source: "api-cas", - counts: { triples: quads.length }, - }); - return jsonResponse(res, 200, { - ok: true, - shareOperationId: result?.shareOperationId, - }); - } catch (err: any) { - tracker.fail(ctx, err); - if ( - err.name === "StaleWriteError" || - err.message?.includes("stale") || - err.message?.includes("CAS condition failed") - ) { - return jsonResponse(res, 409, { error: err.message }); - } - throw err; - } - } - - // POST /api/memory/turn — ingest a conversation turn as a tri-modal Knowledge Asset. + // POST /api/memory/turn -- ingest a conversation turn as a tri-modal Knowledge Asset. // // Streamlined path for agent memory: accepts a markdown conversation turn, // stores it in the file store, runs structural + optional semantic extraction, - // and writes the resulting triples to SWM (or WM if layer=wm). + // and writes the resulting triples to WM. Use the knowledge asset lifecycle + // routes to share or publish turns. // - // Spec: 21_TRI_MODAL_MEMORY.md §8 + // Spec: 21_TRI_MODAL_MEMORY.md Section 8 if (req.method === 'POST' && path === '/api/memory/turn') { const body = await readBody(req); const parsed = safeParseJson(body, res); if (!parsed) return; - const { markdown, contextGraphId, sessionUri, layer, subGraphName } = parsed; + const { markdown, contextGraphId, sessionUri, layer, subGraphName, turnId } = parsed; if (!markdown || typeof markdown !== 'string') { return jsonResponse(res, 400, { error: 'Missing or invalid "markdown" field (string)' }); } @@ -1739,9 +1554,19 @@ WHERE { } } - const targetLayer = layer === 'wm' ? 'wm' : 'swm'; + if (layer !== undefined && layer !== 'wm') { + return jsonResponse(res, 400, { + error: '/api/memory/turn only supports layer:"wm"; use the knowledge asset lifecycle to share or publish turns.', + }); + } + const targetLayer = 'wm' as const; const agentDid = `did:dkg:agent:${agent.peerId}`; const now = new Date().toISOString(); + if (turnId !== undefined && (typeof turnId !== 'string' || turnId.trim().length === 0)) { + return jsonResponse(res, 400, { error: 'Invalid "turnId": must be a non-empty string when supplied' }); + } + const normalizedTurnId = typeof turnId === 'string' ? turnId.trim() : undefined; + const effectiveTurnId = normalizedTurnId ?? randomUUID(); // 1. Store markdown in the file store const mdBytes = Buffer.from(markdown, 'utf-8'); @@ -1753,8 +1578,17 @@ WHERE { } const fileUri = `urn:dkg:file:${fileEntry.keccak256}`; - // Derive turn URI from agent address + timestamp for collision avoidance - const turnUri = `did:dkg:context-graph:${resolvedContextGraphId}/turn/${agent.peerId}-${now}`; + const turnIdentity = { + contextGraphId: resolvedContextGraphId, + subGraphName: subGraphName ?? null, + sessionUri: sessionUri ?? null, + turnId: effectiveTurnId, + fileHash: fileEntry.keccak256, + agent: requestAgentAddress, + }; + const turnDigest = createHash('sha256').update(JSON.stringify(turnIdentity)).digest('hex'); + const assertionName = `turn-${turnDigest.slice(0, 32)}`; + const turnUri = `did:dkg:context-graph:${resolvedContextGraphId}/turn/${assertionName}`; // 2. Run structural extraction let extractResult; @@ -1779,28 +1613,25 @@ WHERE { ); semanticTriples = llmResult.triples; } catch { - // Semantic extraction is best-effort — structural extraction alone is sufficient + // Semantic extraction is best-effort -- structural extraction alone is sufficient } } - // 4. Build quads for the target graph - const targetGraph = targetLayer === 'swm' - ? contextGraphSharedMemoryUri(resolvedContextGraphId, subGraphName) - : contextGraphAssertionUri(resolvedContextGraphId, requestAgentAddress, `turn-${now}`, subGraphName); - + // 4. Build assertion quads. assertion.write stamps the lifecycle WM graph. + const assertionGraphPlaceholder = ''; const quads: Array<{ subject: string; predicate: string; object: string; graph: string }> = []; // Content triples from structural extraction for (const t of extractResult.triples) { - quads.push({ ...t, graph: targetGraph }); + quads.push({ ...t, graph: assertionGraphPlaceholder }); } // Source-file linkage from extractor (rows 1 + 3) for (const t of extractResult.sourceFileLinkage) { - quads.push({ ...t, graph: targetGraph }); + quads.push({ ...t, graph: assertionGraphPlaceholder }); } // Semantic triples (if any) for (const t of semanticTriples) { - quads.push({ ...t, graph: targetGraph }); + quads.push({ ...t, graph: assertionGraphPlaceholder }); } // Ensure the turn is typed as a ConversationTurn @@ -1808,37 +1639,43 @@ WHERE { subject: turnUri, predicate: 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type', object: 'http://schema.org/ConversationTurn', - graph: targetGraph, + graph: assertionGraphPlaceholder, + }); + quads.push({ + subject: turnUri, + predicate: 'http://schema.org/name', + object: JSON.stringify(`Conversation turn ${effectiveTurnId}`), + graph: assertionGraphPlaceholder, }); // Persist the markdown body so the UI can display turn content // without fetching the source file separately - const truncatedBody = markdown.length > 2000 ? markdown.slice(0, 2000) + '…' : markdown; + const truncatedBody = markdown.length > 2000 ? markdown.slice(0, 2000) + '...' : markdown; quads.push({ subject: turnUri, predicate: 'http://schema.org/description', object: JSON.stringify(truncatedBody), - graph: targetGraph, + graph: assertionGraphPlaceholder, }); // Source content type quads.push({ subject: turnUri, predicate: 'http://dkg.io/ontology/sourceContentType', object: JSON.stringify('text/markdown'), - graph: targetGraph, + graph: assertionGraphPlaceholder, }); // Agent attribution quads.push({ subject: turnUri, predicate: 'http://schema.org/agent', object: agentDid, - graph: targetGraph, + graph: assertionGraphPlaceholder, }); // Timestamp quads.push({ subject: turnUri, predicate: 'http://schema.org/dateCreated', object: `"${now}"^^`, - graph: targetGraph, + graph: assertionGraphPlaceholder, }); // Session linking (if session URI provided) @@ -1847,43 +1684,33 @@ WHERE { subject: turnUri, predicate: 'http://schema.org/isPartOf', object: sessionUri, - graph: targetGraph, + graph: assertionGraphPlaceholder, }); quads.push({ subject: sessionUri, predicate: 'http://schema.org/hasPart', object: turnUri, - graph: targetGraph, + graph: assertionGraphPlaceholder, }); } const literalSize = validateWritableQuadLiteralSizes("quads", quads); if (!literalSize.ok) return jsonResponse(res, 400, literalSize.body); - // 5. Write to target layer + // 5. Write to WM through the named knowledge asset lifecycle. + let targetGraph: string; try { - if (targetLayer === 'swm') { - // agent.share sets the graph field itself — pass quads with empty graph - const shareQuads = quads.map(({ subject, predicate, object }) => ({ subject, predicate, object, graph: '' })); - const ctx = createOperationContext('share'); - tracker.start(ctx, { contextGraphId: resolvedContextGraphId, details: { tripleCount: shareQuads.length, source: 'memory-turn', subGraphName } }); - try { - await tracker.trackPhase(ctx, 'store', () => - agent.share(resolvedContextGraphId, shareQuads, { - subGraphName, - localOnly: false, - operationCtx: ctx, - callerAgentAddress: requestAgentAddress, - }), - ); - tracker.complete(ctx, { tripleCount: shareQuads.length }); - } catch (err: any) { - tracker.fail(ctx, err); - throw err; - } - } else { - await agent.store.insert(quads); - } + targetGraph = await agent.assertion.create( + resolvedContextGraphId, + assertionName, + subGraphName ? { subGraphName } : undefined, + ); + await agent.assertion.write( + resolvedContextGraphId, + assertionName, + quads, + subGraphName ? { subGraphName } : undefined, + ); } catch (err: any) { if (err?.code === "OVERSIZED_RDF_LITERAL") { return jsonResponse(res, 400, oversizedRdfLiteralResponseBody(err)); @@ -1922,6 +1749,7 @@ WHERE { return jsonResponse(res, 200, { turnUri, + assertionName, fileHash: fileEntry.keccak256, layer: targetLayer, graph: targetGraph, @@ -1930,15 +1758,16 @@ WHERE { totalQuads: quads.length, embeddingId, sessionUri: sessionUri ?? null, + turnId: effectiveTurnId, }); } - // POST /api/memory/search — tri-modal search across text, graph, and vector stores. + // POST /api/memory/search -- tri-modal search across text, graph, and vector stores. // // Fans out the query to SPARQL (triple store), text search (file store), // and vector similarity (vector store), then merges and deduplicates results. // - // Spec: 21_TRI_MODAL_MEMORY.md §7 + // Spec: 21_TRI_MODAL_MEMORY.md Section 7 if (req.method === 'POST' && path === '/api/memory/search') { const body = await readBody(req); const parsed = safeParseJson(body, res); @@ -1951,7 +1780,14 @@ WHERE { if (!validateRequiredContextGraphId(contextGraphId, res)) return; const resultLimit = typeof rawLimit === 'number' && rawLimit > 0 ? Math.min(rawLimit, 100) : 20; - const memoryLayers: Array<'swm' | 'vm'> = parsed.memoryLayers ?? ['swm', 'vm']; + const requestedLayers = Array.isArray(parsed.memoryLayers) + ? parsed.memoryLayers + : ['wm', 'swm', 'vm']; + const invalidLayers = requestedLayers.filter((layer: unknown) => layer !== 'wm' && layer !== 'swm' && layer !== 'vm'); + if (invalidLayers.length > 0) { + return jsonResponse(res, 400, { error: 'memoryLayers must contain only "wm", "swm", or "vm"' }); + } + const memoryLayers = [...new Set(requestedLayers)] as Array<'wm' | 'swm' | 'vm'>; const results: Array<{ entityUri: string; @@ -1994,21 +1830,23 @@ WHERE { // Fan-out 2: SPARQL text search (scoped to the requested CG + layers). // escapeSparqlLiteral escapes backslashes, quotes, and CR/LF/TAB per the - // SPARQL STRING_LITERAL2 grammar — a simple `replace(/"/g, '\\"')` would + // SPARQL STRING_LITERAL2 grammar -- a simple `replace(/"/g, '\\"')` would // still allow `\` to escape the closing quote and break out of the literal. const escapedQuery = escapeSparqlLiteral(query.toLowerCase()); const cgUri = `did:dkg:context-graph:${contextGraphId}`; - const graphFilters = memoryLayers.map((l: string) => { + const graphFilters = memoryLayers.map((l) => { + if (l === 'wm') { + return `(STRSTARTS(STR(?g), "${cgUri}/_working_memory") || STRSTARTS(STR(?g), "${cgUri}/assertion/"))`; + } if (l === 'swm') return `STRSTARTS(STR(?g), "${cgUri}/_shared_memory")`; // #1096: VM graphs live under `/_verifiable_memory/` (see // contextGraphVerifiableMemoryUri in dkg-core). The pre-rc.16 // "_verified" prefix matched nothing, so memory layer "vm" could // never return SPARQL hits. - if (l === 'vm') return `STRSTARTS(STR(?g), "${cgUri}/_verifiable_memory")`; - return `STRSTARTS(STR(?g), "${cgUri}/")`; - }).join(' || '); + return `STRSTARTS(STR(?g), "${cgUri}/_verifiable_memory")`; + }).join(' || ') || 'false'; try { - // #1096: accept both http:// and https:// schema.org forms — real + // #1096: accept both http:// and https:// schema.org forms -- real // payloads overwhelmingly use https://schema.org, which the previous // http-only property path silently excluded. const sparqlResult = await agent.store.query(` diff --git a/packages/cli/src/daemon/routes/publisher.ts b/packages/cli/src/daemon/routes/publisher.ts index 72165323c3..2cb8520781 100644 --- a/packages/cli/src/daemon/routes/publisher.ts +++ b/packages/cli/src/daemon/routes/publisher.ts @@ -1,8 +1,8 @@ // daemon/routes/publisher.ts // -// Route handlers for publisher enqueue / jobs / stats / cancel / retry / clear. +// Route handlers for publisher jobs / stats / cancel / retry / clear. // -// Extracted verbatim from the legacy monolithic `handleRequest` — +// Extracted verbatim from the legacy monolithic `handleRequest` -- // every block is a contiguous slice of the original source with zero // edits to route bodies. Dispatch is driven by the surviving // `handle-request.ts` shell, which awaits each group handler in @@ -37,7 +37,7 @@ import { existsSync, readdirSync, readFileSync, openSync, closeSync, writeFileSy // Namespace import: our Phase-8 install-context builder (~line 290) calls // `osModule.homedir()`, and the later agent-identity probe (~line 6851) // uses `osModule.hostname()` + `osModule.userInfo()`. v10-rc's new -// OpenClaw config helper (~line 2535) uses a bare `homedir()` — aliased +// OpenClaw config helper (~line 2535) uses a bare `homedir()` -- aliased // below so both sites coexist without a duplicate-module import. import * as osModule from 'node:os'; const { homedir } = osModule; @@ -123,7 +123,7 @@ import { type ExtractionStatusRecord, getExtractionStatusRecord, setExtractionSt import { FileStore } from '../../file-store.js'; import { VectorStore, OpenAIEmbeddingProvider, type EmbeddingProvider } from '../../vector-store.js'; import { parseBoundary, parseMultipart, MultipartParseError } from '../../http/multipart.js'; -// Phase 8 — project-manifest publish + install (UI-driven onboarding flow). +// Phase 8 -- project-manifest publish + install (UI-driven onboarding flow). // Daemon constructs a self-pointing DkgClient (localhost:listenPort) and // reuses the same publish/fetch/plan/write helpers the CLI uses, so wire // format stays identical between curator/joiner/CLI paths. @@ -140,7 +140,7 @@ import { } from '@origintrail-official/dkg-mcp/manifest/install'; import { DkgClient } from '@origintrail-official/dkg-mcp/client'; -// Daemon sub-module imports — every public symbol from sibling +// Daemon sub-module imports -- every public symbol from sibling // modules is pulled in here because the legacy monolithic file used // them all without explicit imports. Unused ones are tolerated by // the project's tsconfig (`noUnusedLocals` is off). @@ -357,105 +357,6 @@ export async function handlePublisherRoutes(ctx: RequestContext): Promise } = ctx; - // POST /api/publisher/enqueue - // Accepts both the old wrapped shape { request: LiftRequest } and the new flat shape. - if (req.method === "POST" && path === "/api/publisher/enqueue") { - const body = await readBody(req, SMALL_BODY_BYTES); - let raw: any; - try { - raw = JSON.parse(body); - } catch { - return jsonResponse(res, 400, { error: "Invalid JSON body" }); - } - const parsed = - raw.request && typeof raw.request === "object" ? raw.request : raw; - const { - roots, - namespace, - scope, - authorityProofRef, - priorVersion, - subGraphName, - accessPolicy, - allowedPeers, - entityProofs, - publishEpochs, - publisherNodeIdentityIdOverride, - seal, - } = parsed; - const rawPublishEpochs = publishEpochs ?? parsed.epochs; - const publishEpochsField = publishEpochs !== undefined ? "publishEpochs" : "epochs"; - const contextGraphId = parsed.contextGraphId; - const shareOperationId = parsed.shareOperationId; - const swmId = parsed.swmId ?? parsed.workspaceId ?? "swm-main"; - const transitionType = parsed.transitionType ?? "CREATE"; - const authorityType = - parsed.authorityType ?? parsed.authority?.type ?? "owner"; - const proofRef = authorityProofRef ?? parsed.authority?.proofRef; - if ( - !contextGraphId || - !shareOperationId || - !Array.isArray(roots) || - roots.length === 0 || - !namespace || - !scope || - !proofRef - ) { - return jsonResponse(res, 400, { - error: "Missing required enqueue fields", - }); - } - let resolvedPublishEpochs: number | undefined; - if (rawPublishEpochs !== undefined && rawPublishEpochs !== null) { - const raw = String(rawPublishEpochs).trim(); - if (!/^[1-9]\d*$/.test(raw)) { - return jsonResponse(res, 400, { - error: `"${publishEpochsField}" must be a positive integer (string or number)`, - }); - } - const parsedEpochs = Number(raw); - if (!Number.isSafeInteger(parsedEpochs)) { - return jsonResponse(res, 400, { - error: `"${publishEpochsField}" is too large to safely represent as a JavaScript integer`, - }); - } - if (parsedEpochs > MAX_PUBLISH_EPOCHS) { - return jsonResponse(res, 400, { - error: `"${publishEpochsField}" must be less than or equal to ${MAX_PUBLISH_EPOCHS}`, - }); - } - resolvedPublishEpochs = parsedEpochs; - } - // V10 sign-at-enqueue: callers build the EIP-712 AuthorAttestation themselves and pass it as `seal`. Sealless enqueues fall back to tentative. - const jobId = await publisherControl.lift({ - swmId, - shareOperationId, - roots, - contextGraphId, - namespace, - scope, - transitionType, - authority: { type: authorityType, proofRef }, - ...(priorVersion ? { priorVersion } : {}), - ...(subGraphName ? { subGraphName } : {}), - ...(accessPolicy ? { accessPolicy } : {}), - ...(Array.isArray(allowedPeers) && allowedPeers.length > 0 ? { allowedPeers } : {}), - // Strict boolean — `!!"false"` is `true`, silently inverting intent. - ...(typeof entityProofs === 'boolean' ? { entityProofs } : {}), - ...(resolvedPublishEpochs !== undefined ? { publishEpochs: resolvedPublishEpochs } : {}), - ...(publisherNodeIdentityIdOverride !== undefined - ? { publisherNodeIdentityIdOverride: String(publisherNodeIdentityIdOverride) } - : {}), - ...(seal !== undefined ? { seal } : {}), - } as any); - return jsonResponse(res, 200, { - jobId, - contextGraphId, - shareOperationId, - rootsCount: roots.length, - }); - } - // GET /api/publisher/jobs?status=... if (req.method === "GET" && path === "/api/publisher/jobs") { const status = @@ -510,7 +411,7 @@ export async function handlePublisherRoutes(ctx: RequestContext): Promise return jsonResponse(res, 200, job); } - // GET /api/publisher/stats — returns the raw status map directly for backward compat + // GET /api/publisher/stats -- returns the raw status map directly for backward compat if (req.method === "GET" && path === "/api/publisher/stats") { const stats = await publisherControl.getStats(); return jsonResponse(res, 200, stats); diff --git a/packages/cli/src/publisher-runner.ts b/packages/cli/src/publisher-runner.ts index f68b70834d..edcbd6020e 100644 --- a/packages/cli/src/publisher-runner.ts +++ b/packages/cli/src/publisher-runner.ts @@ -2,7 +2,7 @@ import { join } from 'node:path'; import { DKGAgentWallet } from '@origintrail-official/dkg-agent'; import { EVMChainAdapter, NoChainAdapter } from '@origintrail-official/dkg-chain'; import { TypedEventBus, type Ed25519Keypair } from '@origintrail-official/dkg-core'; -import { ACKCollector, AsyncLiftRunner, DKGPublisher, FileWorkspacePublicSnapshotStore, TripleStoreAsyncLiftPublisher, wrapAsRpcPreconditionIfApplicable, type AsyncLiftPublishExecutionInput, type AsyncLiftPublisher, type AsyncLiftPublisherRecoveryResult, type LiftJobBroadcast, type LiftJobIncluded, type PublishOptions, type WorkspacePublicSnapshotStore } from '@origintrail-official/dkg-publisher'; +import { ACKCollector, AsyncLiftRunner, DKGPublisher, FileWorkspacePublicSnapshotStore, TripleStoreAsyncLiftPublisher, wrapAsRpcPreconditionIfApplicable, type AsyncLiftPublishExecutionInput, type AsyncLiftPublisher, type AsyncLiftPublisherConfig, type AsyncLiftPublisherRecoveryResult, type LiftJobBroadcast, type LiftJobIncluded, type PublishOptions, type WorkspacePublicSnapshotStore } from '@origintrail-official/dkg-publisher'; import { createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; import { loadNetworkConfig, resolveReadyChainConfig, type DkgConfig } from './config.js'; import { loadPublisherWallets } from './publisher-wallets.js'; @@ -47,6 +47,7 @@ export async function startPublisherRuntimeIfEnabled(args: { log: (message: string) => void; ackTransportFactory?: () => ACKTransportFactory; publishEncryptionFactory?: PublishEncryptionFactory; + knowledgeAssetVmPublishExecutor?: AsyncLiftPublisherConfig['knowledgeAssetVmPublishExecutor']; }): Promise { if (!args.config.publisher?.enabled) { return null; @@ -64,6 +65,7 @@ export async function startPublisherRuntimeIfEnabled(args: { config: args.config, ackTransportFactory: args.ackTransportFactory, publishEncryptionFactory: args.publishEncryptionFactory, + knowledgeAssetVmPublishExecutor: args.knowledgeAssetVmPublishExecutor, }); await runtime.runner.start(); args.log(`Async publisher runner started (${runtime.walletIds.length} wallet${runtime.walletIds.length === 1 ? '' : 's'})`); @@ -96,6 +98,7 @@ interface PublisherRuntimeBaseArgs { ackTransportFactory?: () => ACKTransportFactory; v10ACKProviderFactory?: () => PublishOptions['v10ACKProvider']; publishEncryptionFactory?: PublishEncryptionFactory; + knowledgeAssetVmPublishExecutor?: AsyncLiftPublisherConfig['knowledgeAssetVmPublishExecutor']; publicSnapshotStore?: WorkspacePublicSnapshotStore; closeStoreOnStop: boolean; } @@ -186,6 +189,7 @@ export async function createPublisherRuntimeFromAgent(args: { ackTransportFactory?: () => ACKTransportFactory; v10ACKProviderFactory?: () => PublishOptions['v10ACKProvider']; publishEncryptionFactory?: PublishEncryptionFactory; + knowledgeAssetVmPublishExecutor?: AsyncLiftPublisherConfig['knowledgeAssetVmPublishExecutor']; }): Promise { return createPublisherRuntimeFromBase({ dataDir: args.dataDir, @@ -198,6 +202,7 @@ export async function createPublisherRuntimeFromAgent(args: { ackTransportFactory: args.ackTransportFactory, v10ACKProviderFactory: args.v10ACKProviderFactory, publishEncryptionFactory: args.publishEncryptionFactory, + knowledgeAssetVmPublishExecutor: args.knowledgeAssetVmPublishExecutor, publicSnapshotStore: createPublicSnapshotStore(args.dataDir, args.config), closeStoreOnStop: false, }); @@ -268,6 +273,15 @@ async function createPublisherRuntimeFromBase(args: PublisherRuntimeBaseArgs): P chainRecoveryResolver: hasChainRecovery ? createChainRecoveryResolver(publishers) : undefined, maxRetries: args.maxRetries, publicSnapshotStore: args.publicSnapshotStore, + knowledgeAssetVmPublishExecutor: args.knowledgeAssetVmPublishExecutor + ? async (input) => { + const publisher = publishers.get(input.walletId); + if (!publisher) { + throw new Error(`No publisher configured for wallet ${input.walletId}`); + } + return args.knowledgeAssetVmPublishExecutor!({ ...input, publisher }); + } + : undefined, publishExecutor: async ({ walletId, publishOptions }: AsyncLiftPublishExecutionInput) => { const publisher = publishers.get(walletId); if (!publisher) { diff --git a/packages/cli/src/source-worker-daemon-client.ts b/packages/cli/src/source-worker-daemon-client.ts index a1d80c9f68..171f349d6f 100644 --- a/packages/cli/src/source-worker-daemon-client.ts +++ b/packages/cli/src/source-worker-daemon-client.ts @@ -1,32 +1,42 @@ -import type { LiftRequest } from '@origintrail-official/dkg-publisher'; import type { AssetPartitionQuad } from '@origintrail-official/dkg-core'; import type { SourceWorkerJobFailureDetails, SourceWorkerJobStatusResult } from '@origintrail-official/dkg-agent'; -export interface SharedMemoryWriteResult { - shareOperationId: string; -} - -export interface SharedMemoryWriteClient { - share(contextGraphId: string, quads: AssetPartitionQuad[], options?: { subGraphName?: string }): Promise; -} - -export interface AsyncLiftJobClient { - lift(request: LiftRequest): Promise; +export interface KnowledgeAssetLifecycleClient { + createAndShare( + contextGraphId: string, + name: string, + quads: AssetPartitionQuad[], + options?: { subGraphName?: string }, + ): Promise<{ promotedCount: number; publishReady?: boolean; shareOperationId?: string }>; + publishAsync(contextGraphId: string, name: string, options?: { subGraphName?: string }): Promise<{ + jobId: string; + shareOperationId?: string; + rootsCount?: number; + intentKey?: string; + }>; getJobStatus(jobId: string): Promise; } -export function createDaemonSharedMemoryWriteClient(daemonUrl: string, token: string): SharedMemoryWriteClient { +export function createDaemonKnowledgeAssetLifecycleClient( + daemonUrl: string, + token: string, +): KnowledgeAssetLifecycleClient { return { - async share(contextGraphId: string, quads: AssetPartitionQuad[], options: { subGraphName?: string } = {}): Promise { - const response = await fetch(`${daemonUrl}/api/shared-memory/write`, { + async createAndShare( + contextGraphId: string, + name: string, + quads: AssetPartitionQuad[], + options: { subGraphName?: string } = {}, + ): Promise<{ promotedCount: number; publishReady?: boolean; shareOperationId?: string }> { + const response = await fetch(`${daemonUrl}/api/knowledge-assets`, { method: 'POST', - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - }, + headers: jsonHeaders(token), body: JSON.stringify({ contextGraphId, + name, quads, + finalize: true, + alsoShareSwm: true, subGraphName: options.subGraphName, }), }); @@ -34,30 +44,57 @@ export function createDaemonSharedMemoryWriteClient(daemonUrl: string, token: st if (!response.ok) { throw new Error((payload as { error?: string }).error ?? `HTTP ${response.status}`); } - return { shareOperationId: (payload as { shareOperationId?: string }).shareOperationId ?? '' }; + const errors = Array.isArray((payload as { errors?: unknown }).errors) + ? (payload as { errors: unknown[] }).errors + : []; + if (errors.length > 0) { + throw new Error(`Knowledge asset create/share returned partial lifecycle errors: ${JSON.stringify(errors)}`); + } + const promotedCount = numberField((payload as { promotedCount?: unknown }).promotedCount) ?? 0; + const publishReady = booleanField((payload as { publishReady?: unknown }).publishReady); + const shareOperationId = stringField((payload as { shareOperationId?: unknown }).shareOperationId); + const swmShared = booleanField((payload as { swmShared?: unknown }).swmShared); + if (swmShared !== true || publishReady !== true || promotedCount <= 0 || !shareOperationId) { + throw new Error( + `Knowledge asset create/share did not produce a publish-ready SWM share ` + + `(swmShared=${String(swmShared)}, publishReady=${String(publishReady)}, ` + + `promotedCount=${promotedCount}, shareOperationId=${shareOperationId ?? 'missing'})`, + ); + } + return { + promotedCount, + publishReady, + shareOperationId, + }; }, - }; -} -export function createDaemonAsyncLiftJobClient(daemonUrl: string, token: string): AsyncLiftJobClient { - return { - async lift(request: LiftRequest): Promise { - const response = await fetch(`${daemonUrl}/api/publisher/enqueue`, { + async publishAsync( + contextGraphId: string, + name: string, + options: { subGraphName?: string } = {}, + ): Promise<{ jobId: string; shareOperationId?: string; rootsCount?: number; intentKey?: string }> { + const response = await fetch(`${daemonUrl}/api/knowledge-assets/${encodeURIComponent(name)}/vm/publish-async`, { method: 'POST', - headers: { - Authorization: `Bearer ${token}`, - 'Content-Type': 'application/json', - }, - body: JSON.stringify(request), + headers: jsonHeaders(token), + body: JSON.stringify({ + contextGraphId, + subGraphName: options.subGraphName, + }), }); const payload = await response.json().catch(() => ({})); if (!response.ok) { throw new Error((payload as { error?: string }).error ?? `HTTP ${response.status}`); } const jobId = (payload as { jobId?: string }).jobId; - if (!jobId) throw new Error('Async publisher enqueue did not return a job id'); - return jobId; + if (!jobId) throw new Error('Knowledge asset async publish did not return a job id'); + return { + jobId, + shareOperationId: stringField((payload as { shareOperationId?: unknown }).shareOperationId), + rootsCount: numberField((payload as { rootsCount?: unknown }).rootsCount), + intentKey: stringField((payload as { intentKey?: unknown }).intentKey), + }; }, + async getJobStatus(jobId: string): Promise { const response = await fetch(`${daemonUrl}/api/publisher/job?id=${encodeURIComponent(jobId)}`, { headers: { @@ -79,10 +116,25 @@ export function createDaemonAsyncLiftJobClient(daemonUrl: string, token: string) }; } +function jsonHeaders(token: string): Record { + return { + Authorization: `Bearer ${token}`, + 'Content-Type': 'application/json', + }; +} + function stringField(value: unknown): string | undefined { return typeof value === 'string' && value.length > 0 ? value : undefined; } +function numberField(value: unknown): number | undefined { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function booleanField(value: unknown): boolean | undefined { + return typeof value === 'boolean' ? value : undefined; +} + function recordField(value: unknown): Record | undefined { return value !== null && typeof value === 'object' ? value as Record : undefined; } diff --git a/packages/cli/src/source-worker-runner.ts b/packages/cli/src/source-worker-runner.ts index e0b20c1f61..7eb3ba92bc 100644 --- a/packages/cli/src/source-worker-runner.ts +++ b/packages/cli/src/source-worker-runner.ts @@ -7,17 +7,14 @@ import { type SourceWorkerSource, } from '@origintrail-official/dkg-agent'; import { - createDaemonAsyncLiftJobClient, - createDaemonSharedMemoryWriteClient, - type AsyncLiftJobClient, - type SharedMemoryWriteClient, + createDaemonKnowledgeAssetLifecycleClient, + type KnowledgeAssetLifecycleClient, } from './source-worker-daemon-client.js'; import { loadSourceWorkerConfig, type SourceWorkerConfig } from './source-worker-config.js'; export interface SourceWorkerHandlerContext { config: SourceWorkerConfig; - sharedMemory: SharedMemoryWriteClient; - asyncLift: AsyncLiftJobClient; + knowledgeAssets: KnowledgeAssetLifecycleClient; } export interface SourceWorkerHandlerModule { @@ -33,13 +30,11 @@ export interface SourceWorkerHandlerModule { const config = await loadSourceWorkerConfig(configPath); - const sharedMemory = createDaemonSharedMemoryWriteClient(config.daemonUrl, config.daemonToken); - const asyncLift = createDaemonAsyncLiftJobClient(config.daemonUrl, config.daemonToken); + const knowledgeAssets = createDaemonKnowledgeAssetLifecycleClient(config.daemonUrl, config.daemonToken); const handlerModule = await loadHandlerModule(config); const workerDeps = await handlerModule.createSourceWorkerDeps({ config, - sharedMemory, - asyncLift, + knowledgeAssets, }); const deps: SourceWorkerDeps = { @@ -49,7 +44,7 @@ export async function runConfiguredSourceWorker(configPath: string, options: { o getFingerprint: workerDeps.getFingerprint, processSource: workerDeps.processSource, getJobStatus(jobId: string) { - return asyncLift.getJobStatus(jobId); + return knowledgeAssets.getJobStatus(jobId); }, }; diff --git a/packages/cli/test/api-client.test.ts b/packages/cli/test/api-client.test.ts index 67f031e842..4f80c09123 100644 --- a/packages/cli/test/api-client.test.ts +++ b/packages/cli/test/api-client.test.ts @@ -695,6 +695,24 @@ describe('ApiClient — GitHub-shaped knowledge-assets SDK (OT-RFC-43 §10.5)', publisherNodeIdentityIdOverride: '123', }, }); + + calls = track({ jobId: 'job-1', status: 'accepted' }); + await client.knowledgeAssetPublishAsync('cg', 'f', { + subGraphName: 'notes', + clearAfter: true, + publishEpochs: 12, + publisherNodeIdentityIdOverride: 123n, + }); + expect(calls[0].url).toBe(`${base}/api/knowledge-assets/f/vm/publish-async`); + expect(JSON.parse(calls[0].opts.body as string)).toMatchObject({ + contextGraphId: 'cg', + subGraphName: 'notes', + options: { + clearSharedMemoryAfter: true, + publishEpochs: 12, + publisherNodeIdentityIdOverride: '123', + }, + }); }); it('knowledgeAssetPublish rejects unsupported option keys before HTTP serialization', async () => { @@ -749,13 +767,13 @@ describe('ApiClient — GitHub-shaped knowledge-assets SDK (OT-RFC-43 §10.5)', [{ subject: 'urn:s', predicate: 'urn:p', object: '"o"', graph: '' }], { clearAfter: false, subGraphName: 'sg2' }, ); - // 1st call creates (finalize+promote); the sequence ENDS at the per-KA vm/publish route + // 1st call creates (finalize+share to SWM); the sequence ENDS at the per-KA vm/publish route expect(calls[0].url).toBe(`${base}/api/knowledge-assets`); expect(JSON.parse(calls[0].opts.body as string)).toMatchObject({ contextGraphId: 'cg', name: 'asset2', finalize: true, - promote: true, + alsoShareSwm: true, }); const last = calls[calls.length - 1]; expect(last.url).toBe(`${base}/api/knowledge-assets/asset2/vm/publish`); diff --git a/packages/cli/test/assertion-cli-smoke.test.ts b/packages/cli/test/assertion-cli-smoke.test.ts index d54ad66954..3c27d8b55f 100644 --- a/packages/cli/test/assertion-cli-smoke.test.ts +++ b/packages/cli/test/assertion-cli-smoke.test.ts @@ -206,7 +206,7 @@ describe.sequential('assertion CLI smoke', () => { expect(promoted.stdout).toContain('Assertion promoted to shared memory:'); expect(promoted.stdout).toContain('Triples: 14'); expect(promoted.stdout).toContain('urn:company:acme'); - expect(promoted.stdout).toContain('Next: dkg shared-memory publish research --name paper'); + expect(promoted.stdout).toContain('Next: dkg publisher publish-async research paper'); const promotedSubgraph = await execFileAsync('node', [ CLI_ENTRY, @@ -219,6 +219,26 @@ describe.sequential('assertion CLI smoke', () => { 'lab', ], { env }); - expect(promotedSubgraph.stdout).toContain('Next: dkg shared-memory publish research --name paper --sub-graph-name lab'); + expect(promotedSubgraph.stdout).toContain('Next: dkg publisher publish-async research paper --sub-graph lab'); + }, 15000); + + it('does not expose retired shared-memory or raw publisher enqueue commands', async () => { + const env = { ...process.env, DKG_HOME: dkgHome, DKG_API_PORT: smokeApiPort }; + + await expectCliFailure(['shared-memory', 'write', 'research'], env); + await expectCliFailure(['publisher', 'enqueue', 'research'], env); }, 15000); }); + +async function expectCliFailure(args: string[], env: NodeJS.ProcessEnv): Promise { + let failure: any; + try { + await execFileAsync('node', [CLI_ENTRY, ...args], { env }); + } catch (err) { + failure = err; + } + + expect(failure).toBeTruthy(); + expect(failure.code).not.toBe(0); + expect(`${failure.stdout ?? ''}${failure.stderr ?? ''}`).toMatch(/unknown command|error/i); +} diff --git a/packages/cli/test/auth.test.ts b/packages/cli/test/auth.test.ts index 6106b5a9fb..b01981c821 100644 --- a/packages/cli/test/auth.test.ts +++ b/packages/cli/test/auth.test.ts @@ -145,7 +145,7 @@ describe('httpAuthGuard', () => { }); it('allows OPTIONS without token (CORS preflight)', async () => { - const res = await fetch(`${baseUrl}/api/shared-memory/write`, { method: 'OPTIONS' }); + const res = await fetch(`${baseUrl}/api/knowledge-assets`, { method: 'OPTIONS' }); expect(res.status).toBe(200); }); @@ -226,7 +226,7 @@ describe('httpAuthGuard', () => { }); it('rejects protected endpoint without token', async () => { - const res = await fetch(`${baseUrl}/api/shared-memory/write`, { method: 'POST' }); + const res = await fetch(`${baseUrl}/api/knowledge-assets`, { method: 'POST' }); expect(res.status).toBe(401); const body = await res.json(); expect(body.error).toContain('Unauthorized'); @@ -256,7 +256,7 @@ describe('httpAuthGuard', () => { }); it('allows protected endpoint with raw token (no Bearer prefix)', async () => { - const res = await fetch(`${baseUrl}/api/shared-memory/write`, { + const res = await fetch(`${baseUrl}/api/knowledge-assets`, { method: 'POST', headers: { Authorization: VALID_TOKEN }, }); @@ -291,7 +291,7 @@ describe('httpAuthGuard (auth disabled)', () => { }); it('allows all requests when auth is disabled', async () => { - const res = await fetch(`${baseUrl}/api/shared-memory/write`, { method: 'POST' }); + const res = await fetch(`${baseUrl}/api/knowledge-assets`, { method: 'POST' }); expect(res.status).toBe(200); }); }); diff --git a/packages/cli/test/context-graph-write-path-validation.test.ts b/packages/cli/test/context-graph-write-path-validation.test.ts index 95a554b9ae..9cf76539bc 100644 --- a/packages/cli/test/context-graph-write-path-validation.test.ts +++ b/packages/cli/test/context-graph-write-path-validation.test.ts @@ -335,8 +335,8 @@ describe('context-graph write-path validation — real daemon route wiring', () expect(res.body).toMatchObject({ code: 'CONTEXT_GRAPH_NOT_FOUND' }); }); - it('rejects unknown shared-memory write targets with CONTEXT_GRAPH_NOT_FOUND', async () => { - const res = await postJson(daemon, '/api/shared-memory/write', { contextGraphId: 'missing-cg', quads: QUADS }); + it('rejects unknown knowledge-asset create targets with CONTEXT_GRAPH_NOT_FOUND', async () => { + const res = await postJson(daemon, '/api/knowledge-assets', { contextGraphId: 'missing-cg', name: 'draft', quads: QUADS }); expect(res.status).toBe(400); expect(res.body).toMatchObject({ code: 'CONTEXT_GRAPH_NOT_FOUND' }); }); diff --git a/packages/cli/test/daemon-http-behavior-extra.test.ts b/packages/cli/test/daemon-http-behavior-extra.test.ts index 76217485dd..d53412b940 100644 --- a/packages/cli/test/daemon-http-behavior-extra.test.ts +++ b/packages/cli/test/daemon-http-behavior-extra.test.ts @@ -1176,8 +1176,8 @@ describe('CLI-7 — SPARQL endpoint 4xx matrix', () => { }); }); -describe('context graph write-target validation', () => { - it('POST /api/shared-memory/write rejects unknown context graphs instead of creating them lazily', async () => { +describe('removed shared-memory write route', () => { + it('legacy shared-memory write and raw publisher enqueue routes are no longer served', async () => { const d = daemon!; const contextGraphId = 'lazy-swm-http-' + Math.random().toString(36).slice(2, 8); const write = await fetch(urlFor(d, '/api/shared-memory/write'), { @@ -1195,10 +1195,29 @@ describe('context graph write-target validation', () => { ], }), }); - expect(write.status).toBe(400); - const writeBody = await write.json() as { code?: string; error?: string }; - expect(writeBody.code).toBe('CONTEXT_GRAPH_NOT_FOUND'); - expect(writeBody.error).toMatch(/Unknown contextGraphId/); + expect(write.status).toBe(404); + + const conditionalWrite = await fetch(urlFor(d, '/api/shared-memory/conditional-write'), { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...authHeaders(d) }, + body: JSON.stringify({ contextGraphId, quads: [], conditions: [] }), + }); + expect(conditionalWrite.status).toBe(404); + + const enqueue = await fetch(urlFor(d, '/api/publisher/enqueue'), { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...authHeaders(d) }, + body: JSON.stringify({ + contextGraphId, + shareOperationId: 'legacy-op', + roots: ['urn:legacy-root'], + namespace: 'legacy', + scope: 'legacy', + transitionType: 'CREATE', + authority: { type: 'owner', proofRef: 'proof:legacy' }, + }), + }); + expect(enqueue.status).toBe(404); const list = await fetch(urlFor(d, '/api/context-graph/list'), { headers: authHeaders(d), @@ -1208,6 +1227,53 @@ describe('context graph write-target validation', () => { const entry = body.contextGraphs?.find((row) => row.id === contextGraphId); expect(entry).toBeUndefined(); }, 30_000); + + it('batch rejection reports use the named KA lifecycle route, not the old shared-memory URL', async () => { + const d = daemon!; + const contextGraphId = 'batch-rejection-' + Math.random().toString(36).slice(2, 8); + + const created = await fetch(urlFor(d, '/api/context-graph/create'), { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...authHeaders(d) }, + body: JSON.stringify({ id: contextGraphId, name: contextGraphId }), + }); + expect(created.status).toBeLessThan(300); + + const oldRoute = await fetch(urlFor(d, '/api/shared-memory/report-batch-rejection'), { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...authHeaders(d) }, + body: JSON.stringify({ contextGraphId, verifyResult: { ok: false } }), + }); + expect(oldRoute.status).toBe(404); + + const report = await fetch(urlFor(d, '/api/knowledge-assets/batch-rejections/report'), { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...authHeaders(d) }, + body: JSON.stringify({ + contextGraphId, + batchId: 'batch-1', + verifyResult: { + ok: false, + expectedRoot: `0x${'11'.repeat(32)}`, + actualRoot: `0x${'22'.repeat(32)}`, + leafCount: 1, + reason: 'test mismatch', + }, + }), + }); + expect(report.status).toBe(200); + const body = await report.json() as { + gossiped?: boolean; + assertionName?: string; + shareOperationId?: string; + record?: { digest?: string }; + gossipError?: string; + }; + expect(body.gossiped, JSON.stringify(body)).toBe(true); + expect(body.assertionName).toMatch(/^batch-rejection-/); + expect(body.shareOperationId).toMatch(/\S/); + expect(body.record?.digest).toBeTruthy(); + }, 30_000); }); // --------------------------------------------------------------------------- diff --git a/packages/cli/test/devnet-publish-helpers-smoke.test.ts b/packages/cli/test/devnet-publish-helpers-smoke.test.ts new file mode 100644 index 0000000000..4d639bc4f8 --- /dev/null +++ b/packages/cli/test/devnet-publish-helpers-smoke.test.ts @@ -0,0 +1,131 @@ +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { describe, expect, it } from 'vitest'; + +const execFileAsync = promisify(execFile); + +function toWslPath(path: string): string { + const normalized = resolve(path).replace(/\\/g, '/'); + return normalized.replace(/^([A-Za-z]):/, (_match, drive: string) => `/mnt/${drive.toLowerCase()}`); +} + +describe('devnet publish helper smoke', () => { + it('routes create/share and publish through named KA lifecycle helpers', async () => { + const repoRoot = resolve(process.cwd(), '../..'); + const script = String.raw` +set -euo pipefail +DEVNET_DIR="/tmp/devnet-publish-smoke-$$" +rm -rf "$DEVNET_DIR" +mkdir -p "$DEVNET_DIR" +CALLS_FILE="$DEVNET_DIR/calls.jsonl" +export DEVNET_DIR CALLS_FILE + +api_call() { + local node_id="$1" method="$2" path="$3" data="{}" + if [ "$#" -ge 4 ]; then + data="$4" + fi + CALL_NODE="$node_id" CALL_METHOD="$method" CALL_PATH="$path" CALL_DATA="$data" node -e ' + const fs = require("fs"); + fs.appendFileSync(process.env.CALLS_FILE, JSON.stringify({ + node: process.env.CALL_NODE, + method: process.env.CALL_METHOD, + path: process.env.CALL_PATH, + data: JSON.parse(process.env.CALL_DATA || "{}"), + }) + "\n"); + ' + if [ "$method" = "POST" ] && [ "$path" = "/api/knowledge-assets" ]; then + BODY="$data" node -e ' + const body = JSON.parse(process.env.BODY); + console.log(JSON.stringify({ + status: "swm-shared", + name: body.name, + written: Array.isArray(body.quads) ? body.quads.length : 0, + promotedCount: Array.isArray(body.quads) ? body.quads.length : 0, + swmShared: true, + sealed: true, + publishReady: true, + shareOperationId: "share-" + body.name, + })); + ' + return 0 + fi + if [ "$method" = "POST" ] && [[ "$path" == /api/knowledge-assets/*/vm/publish ]]; then + BODY="$data" node -e ' + const body = JSON.parse(process.env.BODY); + console.log(JSON.stringify({ + status: "confirmed", + ual: "did:dkg:31337/0x1111111111111111111111111111111111111111/1", + contextGraphId: body.contextGraphId, + })); + ' + return 0 + fi + printf '{"error":"unexpected call","path":%q}\n' "$path" + return 1 +} + +tr -d '\r' < scripts/devnet-publish-helpers.sh > "$DEVNET_DIR/devnet-publish-helpers.sh" +source "$DEVNET_DIR/devnet-publish-helpers.sh" + +payload='{"contextGraphId":"cg-smoke","quads":[{"subject":"urn:root:a","predicate":"http://schema.org/name","object":"\"A\""},{"subject":"urn:root:b","predicate":"http://schema.org/name","object":"\"B\""}]}' +create_resp="$(devnet_create_shared_ka node-a "$payload" smoke)" +devnet_publish_load_state +publish_resp="$(devnet_publish_swm_all_roots node-a cg-smoke true)" + +CREATE_RESP="$create_resp" PUBLISH_RESP="$publish_resp" node <<'NODE' +const fs = require('fs'); +const calls = fs.readFileSync(process.env.CALLS_FILE, 'utf8') + .trim() + .split(/\n+/) + .filter(Boolean) + .map((line) => JSON.parse(line)); +const createCalls = calls.filter((call) => call.method === 'POST' && call.path === '/api/knowledge-assets'); +const publishCalls = calls.filter((call) => + call.method === 'POST' && + call.path.startsWith('/api/knowledge-assets/') && + call.path.endsWith('/vm/publish') +); +if (createCalls.length !== 2) throw new Error('expected 2 KA create calls, got ' + createCalls.length); +if (publishCalls.length !== 2) throw new Error('expected 2 KA publish calls, got ' + publishCalls.length); +for (const call of createCalls) { + if (call.data.finalize !== true || call.data.alsoShareSwm !== true) { + throw new Error('create helper must request finalize+SWM share: ' + JSON.stringify(call)); + } +} +if (calls.some((call) => call.path.includes('/shared-memory') || call.path.includes('/api/publisher/enqueue'))) { + throw new Error('legacy route used: ' + JSON.stringify(calls)); +} +if (publishCalls[0].data.options.clearAfter !== false) { + throw new Error('first publish should keep SWM for later assets: ' + JSON.stringify(publishCalls[0])); +} +if (publishCalls[1].data.options.clearAfter !== true) { + throw new Error('last publish should honor caller clearAfter=true: ' + JSON.stringify(publishCalls[1])); +} +const create = JSON.parse(process.env.CREATE_RESP); +if (create.triplesWritten !== 2 || create.names.length !== 2 || create.rootEntities.length !== 2) { + throw new Error('unexpected create response: ' + process.env.CREATE_RESP); +} +const publish = JSON.parse(process.env.PUBLISH_RESP); +if (publish.status !== 'confirmed') throw new Error('unexpected publish response: ' + process.env.PUBLISH_RESP); +NODE +`; + + const tempDir = await mkdtemp(join(tmpdir(), 'dkg-devnet-helper-smoke-')); + const scriptPath = join(tempDir, 'smoke.sh'); + try { + await writeFile(scriptPath, script.replace(/\r\n/g, '\n'), 'utf8'); + const { stderr } = await execFileAsync('bash', [toWslPath(scriptPath)], { + cwd: repoRoot, + timeout: 30_000, + maxBuffer: 1024 * 1024, + }); + expect(stderr).toBe(''); + } finally { + await rm(tempDir, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/test/helpers/publish-async-get-benchmark.ts b/packages/cli/test/helpers/publish-async-get-benchmark.ts index 8b265ae7ca..1a97c9d9cb 100644 --- a/packages/cli/test/helpers/publish-async-get-benchmark.ts +++ b/packages/cli/test/helpers/publish-async-get-benchmark.ts @@ -59,11 +59,12 @@ export function trackingFetch(calls: Array<{ url: string; init?: RequestInit }>, export class MockBenchmarkClient implements BenchmarkClient { readonly publishCalls: Array<{ name: string; roots: string[]; clearAfter?: boolean }> = []; - readonly enqueueCalls: Array<{ roots: string[] }> = []; + readonly asyncPublishCalls: Array<{ name: string; roots: string[] }> = []; private readonly markersByRoot = new Map(); + private readonly assetsByName = new Map(); constructor(private readonly opts: { - enqueueError?: string; + asyncPublishError?: string; jobStatus?: string; jobError?: string; queryMarkerOverride?: string; @@ -75,13 +76,16 @@ export class MockBenchmarkClient implements BenchmarkClient { return { ok: true }; } - async sharedMemoryWrite( + async createKnowledgeAsset( _contextGraphId: string, - quads: Array<{ subject: string; predicate: string; object: string }>, + name: string, + options: { quads: Array<{ subject: string; predicate: string; object: string }> }, ) { - const markerQuad = quads.find((quad) => quad.predicate === 'http://schema.org/identifier'); + const markerQuad = options.quads.find((quad) => quad.predicate === 'http://schema.org/identifier'); if (markerQuad) this.markersByRoot.set(markerQuad.subject, markerQuad.object); - return { shareOperationId: `share-${this.markersByRoot.size}` }; + const roots = [...new Set(options.quads.map((quad) => quad.subject))]; + this.assetsByName.set(name, roots); + return { assertionUri: `urn:test:${name}`, promotedCount: roots.length, publishReady: true }; } async publishAssertion( @@ -91,10 +95,10 @@ export class MockBenchmarkClient implements BenchmarkClient { options?: { clearAfter?: boolean }, ) { // The named-KA composite stages the quads itself, so register the marker - // here (the sync leg no longer calls sharedMemoryWrite first) — the `get` + // here (the sync leg no longer calls sharedMemoryWrite first), and the `get` // validation looks up the marker by root. The KA `name` is recorded so tests can // assert warmup/measured name uniqueness: KA create is name-idempotent, so a reused - // name would silently collide with the warmup KA — the root alone wouldn't catch it. + // name would silently collide with the warmup KA; the root alone would not catch it. const markerQuad = quads.find((quad) => quad.predicate === 'http://schema.org/identifier'); if (markerQuad) this.markersByRoot.set(markerQuad.subject, markerQuad.object); const roots = [...new Set(quads.map((quad) => quad.subject))]; @@ -102,10 +106,12 @@ export class MockBenchmarkClient implements BenchmarkClient { return { kaId: `kc-${this.publishCalls.length}`, kas: roots.map((rootEntity) => ({ tokenId: '1', rootEntity })) }; } - async publisherEnqueue(request: { roots: string[] }) { - if (this.opts.enqueueError) throw new Error(this.opts.enqueueError); - this.enqueueCalls.push({ roots: request.roots }); - return { jobId: `job-${this.enqueueCalls.length}` }; + async knowledgeAssetPublishAsync(_contextGraphId: string, name: string) { + if (this.opts.asyncPublishError) throw new Error(this.opts.asyncPublishError); + const roots = this.assetsByName.get(name); + if (!roots) throw new Error(`Knowledge asset ${name} was not created`); + this.asyncPublishCalls.push({ name, roots }); + return { jobId: `job-${this.asyncPublishCalls.length}` }; } async publisherJob(_jobId: string) { diff --git a/packages/cli/test/issue-306-787-write-quad-validation.test.ts b/packages/cli/test/issue-306-787-write-quad-validation.test.ts index c590035322..7cbd599798 100644 --- a/packages/cli/test/issue-306-787-write-quad-validation.test.ts +++ b/packages/cli/test/issue-306-787-write-quad-validation.test.ts @@ -2,9 +2,7 @@ * GH #306 / #787 — write routes must reject malformed (string-shaped) quads with * an actionable 4xx instead of crashing with a TypeError → HTTP 500. * - * #787 — POST /api/shared-memory/write with N-Quad *string* quads → was 500 - * ("Cannot read properties of undefined (reading 'toLowerCase')"). - * https://github.com/OriginTrail/dkg/issues/787 + * #787 — the retired POST /api/shared-memory/write route must stay removed. * #306 — POST /api/knowledge-assets/{name}/wm/write with string quads → was 500 * ("Cannot use 'in' operator to search for 'graph' in

."). * https://github.com/OriginTrail/dkg/issues/306 @@ -37,21 +35,12 @@ afterAll(async () => { await stopLiveDaemon(daemon); }); -describe('GH #787 — POST /api/shared-memory/write quad-shape validation', () => { - it('returns 4xx (not 500) for N-Quad string-shaped quads', async () => { +describe('GH #787 — retired shared-memory write route', () => { + it('returns route-not-found (not 500) for N-Quad string-shaped quads', async () => { const { status } = await postJson(daemon!, '/api/shared-memory/write', { contextGraphId: CG, quads: [' "v" .'], }); - expect(status).not.toBe(500); - expect(status).toBeGreaterThanOrEqual(400); - expect(status).toBeLessThan(500); - }); - - it('accepts well-formed object quads (regression: valid SWM write still succeeds)', async () => { - const { status, body } = await postJson(daemon!, '/api/shared-memory/write', { - contextGraphId: CG, quads: [{ subject: 'urn:wq:s787', predicate: 'http://schema.org/name', object: '"ok787"' }], - }); - expect(status, JSON.stringify(body)).toBe(200); + expect(status).toBe(404); }); it('returns 400 for oversized RDF literals before SWM write', async () => { diff --git a/packages/cli/test/knowledge-assets-1116-share-errors.test.ts b/packages/cli/test/knowledge-assets-1116-share-errors.test.ts index f7bad35561..61c1ac9b4e 100644 --- a/packages/cli/test/knowledge-assets-1116-share-errors.test.ts +++ b/packages/cli/test/knowledge-assets-1116-share-errors.test.ts @@ -18,8 +18,18 @@ import { afterEach, describe, expect, it } from 'vitest'; import { createServer, type Server } from 'node:http'; +import { mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { ethers } from 'ethers'; +import { NoChainAdapter } from '@origintrail-official/dkg-chain'; +import { generateEd25519Keypair, TypedEventBus } from '@origintrail-official/dkg-core'; +import { DKGPublisher } from '@origintrail-official/dkg-publisher'; +import { createTripleStore } from '@origintrail-official/dkg-storage'; import { handleKnowledgeAssetsRoutes } from '../src/daemon/routes/knowledge-assets.js'; import { daemonState } from '../src/daemon/state.js'; +import { addPublisherWallet } from '../src/publisher-wallets.js'; +import { createPublisherRuntimeFromAgent } from '../src/publisher-runner.js'; const CG_ID = 'issue-1116-cg'; const ASSERTION_NAME = 'seal-asset'; @@ -37,7 +47,11 @@ describe('#1116 share/seal route error mapping (fake agent)', () => { } }); - async function startWith(assertion: Record, agentOverrides: Record = {}) { + async function startWith( + assertion: Record, + agentOverrides: Record = {}, + publisherControl: Record = {}, + ) { const agent = { async listContextGraphs() { return [{ @@ -64,7 +78,7 @@ describe('#1116 share/seal route error mapping (fake agent)', () => { req, res, agent, - publisherControl: {}, + publisherControl, publisherRuntime: null, config: {}, startedAt: Date.now(), @@ -179,6 +193,143 @@ describe('#1116 share/seal route error mapping (fake agent)', () => { expect(res.status).toBeGreaterThanOrEqual(500); }); + it('vm/publish-async preflights the immutable share snapshot before enqueue', async () => { + let enqueueCalls = 0; + await startWith({}, { + resolveFinalizedAssertionVmPublishIntent: async () => ({ + contextGraphId: CG_ID, + name: ASSERTION_NAME, + shareOperationId: 'missing-share-op', + roots: ['urn:test:root'], + seal: { + merkleRoot: `0x${'12'.repeat(32)}`, + authorAddress: '0x1111111111111111111111111111111111111111', + signature: { + r: `0x${'34'.repeat(32)}`, + vs: `0x${'56'.repeat(32)}`, + }, + schemeVersion: 1, + }, + sealChainId: '31337', + sealKav10Address: '0x2222222222222222222222222222222222222222', + sealFinalizedAtIso: '2026-01-01T00:00:00.000Z', + sealMerkleRoot: `0x${'12'.repeat(32)}`, + intentKey: `sha256:${'ab'.repeat(32)}`, + }), + preflightKnowledgeAssetVmPublishSnapshot: async () => { + throw Object.assign( + new Error('share snapshot missing; re-share before enqueue'), + { code: 'PUBLISH_INTENT_STALE' }, + ); + }, + }, { + enqueueKnowledgeAssetVmPublish: async () => { + enqueueCalls += 1; + return 'job-should-not-exist'; + }, + }); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(409); + expect(res.body.code).toBe('PUBLISH_INTENT_STALE'); + expect(String(res.body.error)).toContain('re-share'); + expect(enqueueCalls).toBe(0); + }); + + it('vm/publish-async enqueue is processable by createPublisherRuntimeFromAgent', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-ka-vm-runtime-')); + const wallet = ethers.Wallet.createRandom(); + const store = await createTripleStore({ backend: 'oxigraph' }); + const keypair = await generateEd25519Keypair(); + await addPublisherWallet(dataDir, wallet.privateKey); + + const writer = new DKGPublisher({ + store, + chain: new NoChainAdapter(), + eventBus: new TypedEventBus(), + keypair, + publisherPrivateKey: wallet.privateKey, + }); + const share = await writer.share(CG_ID, [ + { + subject: 'urn:test:runtime-root', + predicate: 'http://schema.org/name', + object: '"Runtime Root"', + graph: '', + }, + ], { publisherPeerId: 'peer-1' }); + + const intent = { + contextGraphId: CG_ID, + name: ASSERTION_NAME, + shareOperationId: share.shareOperationId, + roots: ['urn:test:runtime-root'], + seal: { + merkleRoot: `0x${'12'.repeat(32)}` as `0x${string}`, + authorAddress: '0x1111111111111111111111111111111111111111' as `0x${string}`, + signature: { + r: `0x${'34'.repeat(32)}` as `0x${string}`, + vs: `0x${'56'.repeat(32)}` as `0x${string}`, + }, + schemeVersion: 1, + }, + sealChainId: '31337' as `${bigint}`, + sealKav10Address: '0x2222222222222222222222222222222222222222' as `0x${string}`, + sealFinalizedAtIso: '2026-01-01T00:00:00.000Z', + sealMerkleRoot: `0x${'12'.repeat(32)}` as `0x${string}`, + intentKey: `sha256:${'cd'.repeat(32)}`, + }; + const executorCalls: any[] = []; + const preflighted: any[] = []; + const runtime = await createPublisherRuntimeFromAgent({ + dataDir, + store, + keypair, + chainBase: undefined, + pollIntervalMs: 10, + errorBackoffMs: 10, + knowledgeAssetVmPublishExecutor: async (input) => { + executorCalls.push(input); + return { + status: 'tentative', + ual: 'did:dkg:local/runtime-route', + merkleRoot: ethers.getBytes(input.request.sealMerkleRoot), + kaManifest: [], + } as any; + }, + }); + + try { + await startWith({}, { + resolveFinalizedAssertionVmPublishIntent: async () => intent, + preflightKnowledgeAssetVmPublishSnapshot: async (request: unknown) => { + preflighted.push(request); + }, + }, runtime.publisher as any); + + const res = await post('vm/publish-async', { contextGraphId: CG_ID }); + expect(res.status).toBe(202); + expect(res.body.status).toBe('accepted'); + expect(res.body.shareOperationId).toBe(share.shareOperationId); + expect(preflighted).toEqual([intent]); + + const processed = await runtime.publisher.processNext(wallet.address); + expect(processed?.jobId).toBe(res.body.jobId); + expect(processed?.status).toBe('finalized'); + expect(executorCalls).toHaveLength(1); + expect(executorCalls[0].request).toMatchObject({ + contextGraphId: CG_ID, + name: ASSERTION_NAME, + shareOperationId: share.shareOperationId, + roots: ['urn:test:runtime-root'], + }); + expect(typeof executorCalls[0].publisher.publish).toBe('function'); + } finally { + await runtime.stop(); + await store.close(); + } + }); + // #1116 (round 5, FIX 1): the seal-less SWM reconstruction is reachable via the // wm/pull-from route + a plain finalize, which bypasses the finalize(layer:"swm") // wrapper guard. The publisher now rejects a subset-only asset at the source with diff --git a/packages/cli/test/memory-graph-events.test.ts b/packages/cli/test/memory-graph-events.test.ts index d05be0cff2..4c78fabfd6 100644 --- a/packages/cli/test/memory-graph-events.test.ts +++ b/packages/cli/test/memory-graph-events.test.ts @@ -1,4 +1,4 @@ -// memory_graph_changed emissions — NO MOCKS, real end-to-end SSE pipeline. +// memory_graph_changed emissions -- NO MOCKS, real end-to-end SSE pipeline. // // The mutation routes call `ctx.emitMemoryGraphChanged(event)` after each // create / write / finalize / promote / shared-memory write so the node-ui @@ -8,10 +8,10 @@ // its captured calls, these tests SUBSCRIBE to the real `/api/events` SSE // stream of a real edge daemon (startLiveDaemon vs the shared Hardhat node), // drive the real routes over HTTP, and assert on the frames that actually -// arrive — the real emit pipeline, no fabricated daemon behaviour. +// arrive -- the real emit pipeline, no fabricated daemon behaviour. // // Real-daemon facts pinned while writing this (the mock hid all of them): -// - finalize (auto-finalize on create-with-quads, and POST …/wm/finalize) +// - finalize (auto-finalize on create-with-quads, and POST .../wm/finalize) // binds the author signature to the on-chain CG id, so it 500s unless the // context graph is REGISTERED on-chain first; create / wm/write / swm/share // / shared-memory write / finalize:false all work pre-registration. @@ -20,7 +20,7 @@ // - the `assertion_finalized` frame carries no `counts`; the write/promote // frames carry `counts.triples`. // -// DEVNET-TIER (documented, NOT faked here — needs real core peers): +// DEVNET-TIER (documented, NOT faked here -- needs real core peers): // - the confirmed selective-publish SWM+VM emission and the publish remap // paths: a confirmed publish mints on-chain + needs StorageACK quorum from // connected core peers, which a single edge daemon cannot reach. @@ -37,6 +37,7 @@ import { startLiveDaemon, stopLiveDaemon, postJson, + getJson, openEventStream, type LiveDaemon, type EventStream, @@ -44,7 +45,7 @@ import { const QUADS = [{ subject: 'urn:root', predicate: 'http://schema.org/name', object: '"v1"' }]; -describe('memory_graph_changed — real daemon SSE emissions', () => { +describe('memory_graph_changed -- real daemon SSE emissions', () => { let daemon: LiveDaemon; let stream: EventStream; let cgCounter = 0; @@ -98,7 +99,7 @@ describe('memory_graph_changed — real daemon SSE emissions', () => { expect(typeof frame.data.timestamp).toBe('string'); }); - it('emits an assertion_written refresh after POST …/wm/write', async () => { + it('emits an assertion_written refresh after POST .../wm/write', async () => { const cg = await freshCg(); const res = await postJson(daemon, '/api/knowledge-assets/draft/wm/write', { contextGraphId: cg, quads: QUADS }); expect(res.status).toBe(200); @@ -107,7 +108,7 @@ describe('memory_graph_changed — real daemon SSE emissions', () => { expect(frame.data).toMatchObject({ contextGraphId: cg, layers: ['wm'], operation: 'assertion_written', source: 'api', counts: { triples: 1 } }); }); - it('auto-finalizes a create-with-quads on a registered CG — writes AND seals, emits assertion_finalized', async () => { + it('auto-finalizes a create-with-quads on a registered CG -- writes AND seals, emits assertion_finalized', async () => { const cg = await freshCg(true); const res = await postJson(daemon, '/api/knowledge-assets', { contextGraphId: cg, name: 'sealed', quads: QUADS }); expect(res.status).toBe(201); @@ -117,7 +118,7 @@ describe('memory_graph_changed — real daemon SSE emissions', () => { expect(frame.data).toMatchObject({ contextGraphId: cg, layers: ['wm'], operation: 'assertion_finalized', source: 'api' }); }); - it('honors finalize:false — writes quads but does NOT seal (assertion_written, no assertion_finalized)', async () => { + it('honors finalize:false -- writes quads but does NOT seal (assertion_written, no assertion_finalized)', async () => { const cg = await freshCg(); const res = await postJson(daemon, '/api/knowledge-assets', { contextGraphId: cg, name: 'draft', quads: QUADS, finalize: false }); expect(res.status).toBe(201); @@ -127,7 +128,7 @@ describe('memory_graph_changed — real daemon SSE emissions', () => { expect(stream.events.some(isFrame(cg, 'assertion_finalized'))).toBe(false); }); - it('emits an assertion_finalized refresh on POST …/wm/finalize (registered CG)', async () => { + it('emits an assertion_finalized refresh on POST .../wm/finalize (registered CG)', async () => { const cg = await freshCg(true); await postJson(daemon, '/api/knowledge-assets/draft/wm/write', { contextGraphId: cg, quads: QUADS }); const res = await postJson(daemon, '/api/knowledge-assets/draft/wm/finalize', { contextGraphId: cg }); @@ -148,31 +149,96 @@ describe('memory_graph_changed — real daemon SSE emissions', () => { expect(frame.data.counts.triples).toBeGreaterThanOrEqual(1); }); - it('emits a metadata-only SWM refresh after a shared-memory write (no quads/content in the frame)', async () => { + it('does not emit for the removed shared-memory write route', async () => { const cg = await freshCg(); - const sg = await postJson(daemon, '/api/sub-graph/create', { contextGraphId: cg, subGraphName: 'notes' }); - expect(sg.status).toBe(200); - const res = await postJson(daemon, '/api/shared-memory/write', { contextGraphId: cg, subGraphName: 'notes', quads: QUADS }); + const res = await postJson(daemon, '/api/shared-memory/write', { contextGraphId: cg, quads: QUADS }); + expect(res.status).toBe(404); + await expectNoEmit(cg, 'shared_memory_written'); + }); + + it('writes /api/memory/turn through a named WM knowledge asset when layer is omitted', async () => { + const cg = await freshCg(); + const res = await postJson(daemon, '/api/memory/turn', { + contextGraphId: cg, + markdown: '# Turn\n\nUser likes lifecycle-backed memory.', + sessionUri: 'urn:test:session:memory-turn', + turnId: 'memory-turn-route-regression', + }); expect(res.status).toBe(200); - expect(res.body).toMatchObject({ triplesWritten: 1 }); - const frame = await stream.waitFor(isFrame(cg, 'shared_memory_written')); - expect(frame.data).toMatchObject({ contextGraphId: cg, layers: ['swm'], subGraphName: 'notes', operation: 'shared_memory_written', source: 'api', counts: { triples: 1 } }); - // metadata-only: the SWM payload must never be broadcast on the refresh bus. - expect(frame.data).not.toHaveProperty('quads'); - expect(frame.data).not.toHaveProperty('content'); + expect(res.body).toMatchObject({ + layer: 'wm', + sessionUri: 'urn:test:session:memory-turn', + }); + expect(res.body.assertionName).toMatch(/^turn-[0-9a-f]{32}$/); + expect(typeof res.body.graph).toBe('string'); + expect(res.body.graph.length).toBeGreaterThan(0); + + const frame = await stream.waitFor(isFrame(cg, 'memory_turn_written')); + expect(frame.data).toMatchObject({ + contextGraphId: cg, + layers: ['wm'], + operation: 'memory_turn_written', + source: 'memory-turn', + }); + + const quads = await getJson( + daemon, + `/api/knowledge-assets/${encodeURIComponent(res.body.assertionName)}/wm/quads?contextGraphId=${encodeURIComponent(cg)}`, + ); + expect(quads.status).toBe(200); + expect(JSON.stringify(quads.body)).toContain(res.body.turnUri); + expect(JSON.stringify(quads.body)).toContain('ConversationTurn'); + + const search = await postJson(daemon, '/api/memory/search', { + contextGraphId: cg, + query: 'lifecycle-backed memory', + }); + expect(search.status).toBe(200); + expect(search.body.results.some((hit: any) => hit.entityUri === res.body.turnUri)).toBe(true); }); - it('does not emit when a shared-memory write fails validation (empty quads → 400)', async () => { + it('creates distinct /api/memory/turn KAs for repeated identical markdown when turnId is omitted', async () => { const cg = await freshCg(); - const res = await postJson(daemon, '/api/shared-memory/write', { contextGraphId: cg, quads: [] }); - expect(res.status).toBe(400); - await expectNoEmit(cg, 'shared_memory_written'); + const payload = { + contextGraphId: cg, + markdown: '# Turn\n\nRepeated identical markdown.', + sessionUri: 'urn:test:session:memory-turn-repeat', + }; + + const first = await postJson(daemon, '/api/memory/turn', payload); + const second = await postJson(daemon, '/api/memory/turn', payload); + + expect(first.status).toBe(200); + expect(second.status).toBe(200); + expect(first.body.turnId).toMatch(/^[0-9a-f-]{36}$/); + expect(second.body.turnId).toMatch(/^[0-9a-f-]{36}$/); + expect(second.body.turnId).not.toBe(first.body.turnId); + expect(second.body.assertionName).not.toBe(first.body.assertionName); + expect(second.body.turnUri).not.toBe(first.body.turnUri); + + const firstQuads = await getJson( + daemon, + `/api/knowledge-assets/${encodeURIComponent(first.body.assertionName)}/wm/quads?contextGraphId=${encodeURIComponent(cg)}`, + ); + const secondQuads = await getJson( + daemon, + `/api/knowledge-assets/${encodeURIComponent(second.body.assertionName)}/wm/quads?contextGraphId=${encodeURIComponent(cg)}`, + ); + expect(firstQuads.status).toBe(200); + expect(secondQuads.status).toBe(200); + expect(JSON.stringify(firstQuads.body)).toContain(first.body.turnUri); + expect(JSON.stringify(secondQuads.body)).toContain(second.body.turnUri); }); - it('rejects an unsafe shared-memory contextGraphId before the agent acts (and does not emit)', async () => { - const res = await postJson(daemon, '/api/shared-memory/write', { contextGraphId: 'bad { + const cg = await freshCg(); + const res = await postJson(daemon, '/api/memory/turn', { + contextGraphId: cg, + markdown: 'Do not write this directly to SWM.', + layer: 'swm', + }); expect(res.status).toBe(400); - expect(res.body.error).toMatch(/Invalid "contextGraphId"/); + expect(res.body.error).toMatch(/only supports layer:"wm"/); }); it('rejects finalize:false combined with alsoShareSwm before any mutation', async () => { @@ -197,7 +263,7 @@ describe('memory_graph_changed — real daemon SSE emissions', () => { expect(res.body.error).toMatch(/requires explicit `quads`/); }); - it('accepts explicit verify-batch quads over the small request limit (≈270 KB, not 413)', async () => { + it('accepts explicit verify-batch quads over the small request limit (~270 KB, not 413)', async () => { const cg = await freshCg(); const largeLiteral = `"${'x'.repeat(270 * 1024)}"`; const res = await postJson(daemon, '/api/shared-memory/verify-batch', { diff --git a/packages/cli/test/publish-async-get-benchmark.test.ts b/packages/cli/test/publish-async-get-benchmark.test.ts index 91afe48630..48cfcc7fd6 100644 --- a/packages/cli/test/publish-async-get-benchmark.test.ts +++ b/packages/cli/test/publish-async-get-benchmark.test.ts @@ -183,7 +183,7 @@ describe('publish async get benchmark', () => { const roots = [ ...client.publishCalls.flatMap((call) => call.roots), - ...client.enqueueCalls.flatMap((call) => call.roots), + ...client.asyncPublishCalls.flatMap((call) => call.roots), ]; expect(new Set(roots).size).toBe(roots.length); expect(roots.some((root) => root.includes(':warmup-1:'))).toBe(true); @@ -232,11 +232,11 @@ describe('publish async get benchmark', () => { }); }); - it('reports async enqueue failures and skipped completion context', async () => { - const client = new MockBenchmarkClient({ enqueueError: 'publisher queue disabled' }); + it('reports async publish request failures and skipped completion context', async () => { + const client = new MockBenchmarkClient({ asyncPublishError: 'publisher queue disabled' }); const result = await runPublishAsyncGetBenchmark({ ...baseConfig(), repeat: 1, warmups: 0 }, client, monotonicClock()); - expect(result.failures.map((failure) => failure.operation)).toEqual(['asyncEnqueue', 'asyncCompletion']); + expect(result.failures.map((failure) => failure.operation)).toEqual(['asyncPublishRequest', 'asyncCompletion']); expect(result.failures[0]).toMatchObject({ iteration: 1, error: expect.stringContaining('publisher queue disabled'), @@ -244,11 +244,11 @@ describe('publish async get benchmark', () => { expect(result.failures[0].context).toMatchObject({ contextGraphId: 'bench-cg', flow: 'async', - shareOperationId: 'share-2', + name: expect.stringContaining('benchmark-async-'), }); expect(result.failures[1].context).toMatchObject({ - skippedAfter: 'asyncEnqueue', - shareOperationId: 'share-2', + skippedAfter: 'asyncPublishRequest', + name: expect.stringContaining('benchmark-async-'), }); }); @@ -339,7 +339,7 @@ describe('publish async get benchmark', () => { publishAsyncGetPages, publishAsyncGetSuite, } = await import('../../../esbench.config.mjs') as EsbenchConfigForTest; - const caseName = 'asynchronous publish enqueue and finalization'; + const caseName = 'asynchronous VM publish request and finalization'; const result = { [publishAsyncGetSuite]: [ { @@ -366,7 +366,7 @@ describe('publish async get benchmark', () => { expect(publishAsyncGetPages).toEqual([ ['get/read retrieval', 'bench/results/publish-async-get/get-read-retrieval.html'], ['synchronous publish with finalization', 'bench/results/publish-async-get/sync-publish-finalization.html'], - ['asynchronous publish enqueue and finalization', 'bench/results/publish-async-get/async-publish-finalization.html'], + ['asynchronous VM publish request and finalization', 'bench/results/publish-async-get/async-publish-finalization.html'], ['upload payload to local working memory', 'bench/results/publish-async-get/working-memory-upload.html'], ['lift local working memory to shared working memory', 'bench/results/publish-async-get/working-to-shared-memory.html'], ]); @@ -402,7 +402,7 @@ describe('publish async get benchmark', () => { expect(html).toContain('dkg-benchmark-report-nav'); expect(html).toContain('../latest.html'); expect(html).toContain('sync-publish-finalization.html'); - expect(html).toContain('asynchronous publish enqueue and finalization'); + expect(html).toContain('asynchronous VM publish request and finalization'); expect(html).toContain('aria-current=\\"page\\"'); expect(html).toContain('DOMContentLoaded'); expect(repeated.match(/dkg-benchmark-report-nav:start/g)).toHaveLength(1); @@ -459,24 +459,24 @@ describe('publish async get benchmark', () => { payloadSizes: ['200mb'], flows: [ { - flow: 'asynchronous publish enqueue and finalization', + flow: 'asynchronous VM publish request and finalization', payloadSize: '200mb', totalMs: 12, measuredMs: 7, traces: [ { - flow: 'asynchronous publish enqueue and finalization', + flow: 'asynchronous VM publish request and finalization', payloadSize: '200mb', phase: 'measured', - method: 'publisherEnqueue', + method: 'knowledgeAssetPublishAsync', invokes: ['publisherJobs.set'], - detail: 'Enqueue the publish request through the publisher runtime path.', + detail: 'Queue VM publish for the named knowledge asset.', durationMs: 2, success: true, context: { rootEntity: 'urn:test:root', quadCount: 1 }, }, { - flow: 'asynchronous publish enqueue and finalization', + flow: 'asynchronous VM publish request and finalization', payloadSize: '200mb', phase: 'measured', method: 'publisherJob', @@ -492,8 +492,8 @@ describe('publish async get benchmark', () => { }); expect(html).toContain('DKG Benchmark Method Analysis'); - expect(html).toContain('asynchronous publish enqueue and finalization'); - expect(html).toContain('publisherEnqueue'); + expect(html).toContain('asynchronous VM publish request and finalization'); + expect(html).toContain('knowledgeAssetPublishAsync'); expect(html).toContain('publisherJob'); expect(html).toContain('promoteSharedRoot'); expect(html).toContain('2.000 ms'); diff --git a/packages/cli/test/publisher-cli-smoke.test.ts b/packages/cli/test/publisher-cli-smoke.test.ts deleted file mode 100644 index f537a5ecd6..0000000000 --- a/packages/cli/test/publisher-cli-smoke.test.ts +++ /dev/null @@ -1,182 +0,0 @@ -import { beforeAll, afterAll, describe, expect, it } from 'vitest'; -import { execFile, spawn } from 'node:child_process'; -import { promisify } from 'node:util'; -import { mkdtemp, writeFile, rm } from 'node:fs/promises'; -import { existsSync } from 'node:fs'; -import { join, dirname } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import { tmpdir } from 'node:os'; -import { ethers } from 'ethers'; -import { publisherWalletsPath } from '../src/publisher-wallets.js'; - -const execFileAsync = promisify(execFile); -const __dirname = dirname(fileURLToPath(import.meta.url)); -const CLI_ENTRY = join(__dirname, '..', 'dist', 'cli.js'); -const SMOKE_API_PORT = '19291'; - -// TODO(RFC-001 §9.x follow-up): pre-existing breakage from Phase B-1 -// CLI migration. `dkg shared-memory write` now stages a named WM -// assertion (returning an assertion name) instead of writing directly -// to SWM (returning a shareOperationId). The publisher CLI's job queue -// keying still expects a shareOperationId. Either: -// (a) the CLI gains a `--legacy-direct-swm` flag that hits the -// still-alive `/api/shared-memory/write` route and surfaces a -// shareOperationId, or -// (b) the publisher job-queue is rekeyed off assertion names, or -// (c) the test is rewritten to exercise the assertion lifecycle and -// the publisher CLI supports `--assertion-name` enqueue input. -// Tracked separately from this PR's scope (sign-at-creation lifecycle). -describe.sequential.skip('publisher CLI smoke', () => { - let dkgHome: string; - let daemon: ReturnType | undefined; - - beforeAll(async () => { - dkgHome = await mkdtemp(join(tmpdir(), 'dkg-cli-smoke-')); - if (!existsSync(CLI_ENTRY)) { - await execFileAsync('pnpm', ['build'], { cwd: join(__dirname, '..') }); - } - await writeFile(join(dkgHome, 'smoke.nt'), ' "Rihana" .\n'); - await writeFile( - join(dkgHome, 'config.json'), - JSON.stringify({ - name: 'smoke-node', - apiPort: Number.parseInt(SMOKE_API_PORT, 10), - listenPort: 0, - nodeRole: 'edge', - contextGraphs: [], - auth: { enabled: false }, - store: { - backend: 'oxigraph-worker', - options: { path: join(dkgHome, 'store.nq') }, - }, - }), - ); - }); - - afterAll(async () => { - if (daemon) { - daemon.kill('SIGTERM'); - await Promise.race([ - new Promise((resolve) => daemon?.once('exit', resolve)), - new Promise((resolve) => setTimeout(resolve, 3000)).then(() => { - daemon?.kill('SIGKILL'); - }), - ]); - } - await rm(dkgHome, { recursive: true, force: true }); - }); - - it('supports wallet add, enable, jobs, and job payload inspection', async () => { - const wallet = ethers.Wallet.createRandom(); - const env = { ...process.env, DKG_HOME: dkgHome, DKG_API_PORT: SMOKE_API_PORT }; - - await execFileAsync('node', [CLI_ENTRY, 'publisher', 'wallet', 'add', wallet.privateKey], { env }); - const walletList = await execFileAsync('node', [CLI_ENTRY, 'publisher', 'wallet', 'list'], { env }); - expect(walletList.stdout).toContain(wallet.address); - expect(walletList.stdout).toContain(publisherWalletsPath(dkgHome)); - - const enabled = await execFileAsync('node', [CLI_ENTRY, 'publisher', 'enable', '--poll-interval', '1000', '--error-backoff', '1000'], { env }); - expect(enabled.stdout).toContain('Async publisher enabled'); - const disabled = await execFileAsync('node', [CLI_ENTRY, 'publisher', 'disable'], { env }); - expect(disabled.stdout).toContain('Async publisher disabled'); - - daemon = spawn('node', [CLI_ENTRY, 'daemon-worker'], { - env, - stdio: 'ignore', - }); - let ready = false; - for (let i = 0; i < 120; i += 1) { - if (daemon.exitCode !== null) { - throw new Error(`daemon-worker exited early with code ${daemon.exitCode}`); - } - try { - const response = await fetch(`http://127.0.0.1:${SMOKE_API_PORT}/api/status`); - if (response.ok) { - ready = true; - break; - } - } catch { - // wait for daemon readiness - } - await new Promise((resolve) => setTimeout(resolve, 250)); - } - expect(ready).toBe(true); - - const staged = await execFileAsync('node', [CLI_ENTRY, 'shared-memory', 'write', 'music-social', '--file', join(dkgHome, 'smoke.nt')], { env }); - expect(staged.stdout).toContain('Written to shared memory for "music-social":'); - const stagedMatch = staged.stdout.match(/Share operation:\s+(\S+)/); - expect(stagedMatch?.[1]).toBeDefined(); - const shareOperationId = stagedMatch![1]; - - // Stop the daemon before publisher file-based commands. The daemon's - // in-memory Oxigraph store can flush to the same .nq file and overwrite - // data written by the CLI's own store instances, causing flaky "not found". - // Use the /api/shutdown endpoint for an orderly exit, then wait for the - // process to terminate — this gives the store's 50ms debounced flush time - // to persist shared-memory data before the process exits. - const daemonExited = daemon.exitCode !== null - ? Promise.resolve() - : new Promise((resolve) => daemon?.once('exit', resolve)); - const killTimeout = setTimeout(() => { daemon?.kill('SIGKILL'); }, 5000); - await fetch(`http://127.0.0.1:${SMOKE_API_PORT}/api/shutdown`, { method: 'POST' }).catch(() => {}); - await daemonExited; - clearTimeout(killTimeout); - daemon = undefined; - - const enqueue = await execFileAsync('node', [ - CLI_ENTRY, - 'publisher', - 'enqueue', - 'music-social', - '--share-operation-id', - shareOperationId, - '--root', - 'urn:local:/rihana', - '--namespace', - 'aloha', - '--scope', - 'person-profile', - '--authority-proof-ref', - 'proof:owner:1', - ], { env }); - const match = enqueue.stdout.match(/Job ID:\s+(\S+)/); - expect(match?.[1]).toBeDefined(); - const jobId = match![1]; - - const jobs = await execFileAsync('node', [CLI_ENTRY, 'publisher', 'jobs'], { env }); - expect(jobs.stdout).toContain(jobId); - expect(jobs.stdout).toMatch(/accepted|claimed|validated|broadcast|included|finalized|failed/); - - await expect( - execFileAsync('node', [CLI_ENTRY, 'publisher', 'jobs', '--status', 'bogus'], { env }), - ).rejects.toMatchObject({ - stderr: expect.stringContaining('Invalid publisher job status: bogus'), - }); - - const job = await execFileAsync('node', [CLI_ENTRY, 'publisher', 'job', jobId], { env }); - expect(job.stdout).toContain(jobId); - expect(job.stdout).toContain('"status":'); - expect(job.stdout).toContain('jobSlug'); - - let payload: Awaited> | undefined; - for (let attempt = 0; attempt < 60; attempt += 1) { - try { - payload = await execFileAsync('node', [CLI_ENTRY, 'publisher', 'job', jobId, '--payload'], { env }); - break; - } catch (error: any) { - const stderr = String(error?.stderr ?? ''); - if (!stderr.includes('No shared-memory roots found')) { - throw error; - } - await new Promise((resolve) => setTimeout(resolve, 250)); - } - } - expect(payload).toBeDefined(); - if (!payload) { - throw new Error('publisher job --payload did not become available in time'); - } - expect(payload.stdout).toContain('"status": "accepted"'); - expect(payload.stdout).toContain('publishOptions'); - expect(payload.stdout).toContain('music-social'); - }, 45000); -}); diff --git a/packages/cli/test/publisher-route-snapshot.test.ts b/packages/cli/test/publisher-route-snapshot.test.ts index fcff62a2d8..7b9021f8b0 100644 --- a/packages/cli/test/publisher-route-snapshot.test.ts +++ b/packages/cli/test/publisher-route-snapshot.test.ts @@ -42,20 +42,17 @@ describe('publisher routes with disk public snapshot refs', () => { ], { publisherPeerId: 'peer-route' }); const publisherControl = createPublisherControlFromStore(store, publicSnapshotStore); - const enqueue = createContext('POST', '/api/publisher/enqueue', { + const jobId = await publisherControl.lift({ contextGraphId: CONTEXT_GRAPH, + swmId: write.shareOperationId, shareOperationId: write.shareOperationId, roots: [ENTITY], namespace: 'aloha', scope: 'person-profile', transitionType: 'CREATE', - authorityProofRef: 'proof:owner:route', - publishEpochs: '9', - }, publisherControl); - - await handlePublisherRoutes(enqueue); - expect(responseStatus(enqueue)).toBe(200); - const jobId = String(responseBody(enqueue).jobId); + authority: { type: 'owner', proofRef: 'proof:owner:route' }, + publishEpochs: 9, + }); const payloadCtx = createContext('GET', `/api/publisher/job-payload?id=${encodeURIComponent(jobId)}`, undefined, publisherControl); await handlePublisherRoutes(payloadCtx); diff --git a/packages/cli/test/source-worker-daemon-client-validation.test.ts b/packages/cli/test/source-worker-daemon-client-validation.test.ts new file mode 100644 index 0000000000..51a6159758 --- /dev/null +++ b/packages/cli/test/source-worker-daemon-client-validation.test.ts @@ -0,0 +1,50 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { createDaemonKnowledgeAssetLifecycleClient } from '../src/source-worker-daemon-client.js'; + +describe('source worker daemon lifecycle client response validation', () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it('rejects createAndShare partial lifecycle responses with tail errors', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify({ + created: true, + status: 'wm-sealed', + errors: [{ phase: 'swm-share', error: 'share failed' }], + promotedCount: 0, + publishReady: false, + }), { + status: 207, + headers: { 'Content-Type': 'application/json' }, + }))); + + const client = createDaemonKnowledgeAssetLifecycleClient('http://daemon.test', 'token'); + + await expect( + client.createAndShare('cg', 'ka', [ + { subject: 'urn:s', predicate: 'urn:p', object: '"v"', graph: '' }, + ]), + ).rejects.toThrow(/partial lifecycle errors/); + }); + + it('rejects createAndShare 2xx responses that are not publish-ready SWM shares', async () => { + vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify({ + created: true, + status: 'wm-sealed', + swmShared: true, + promotedCount: 1, + publishReady: false, + }), { + status: 201, + headers: { 'Content-Type': 'application/json' }, + }))); + + const client = createDaemonKnowledgeAssetLifecycleClient('http://daemon.test', 'token'); + + await expect( + client.createAndShare('cg', 'ka', [ + { subject: 'urn:s', predicate: 'urn:p', object: '"v"', graph: '' }, + ]), + ).rejects.toThrow(/did not produce a publish-ready SWM share/); + }); +}); diff --git a/packages/cli/test/source-worker-daemon-client.test.ts b/packages/cli/test/source-worker-daemon-client.test.ts index 71dc8ec6a8..16136bee5c 100644 --- a/packages/cli/test/source-worker-daemon-client.test.ts +++ b/packages/cli/test/source-worker-daemon-client.test.ts @@ -1,29 +1,19 @@ /** - * Source-worker daemon clients — REAL daemon round-trips, NO mocks. + * Source-worker daemon client - REAL daemon round-trips, NO mocks. * - * The retired version replaced `globalThis.fetch` with a stub that returned - * canned `{ shareOperationId: 'swm-1' }` / `{ jobId: 'job-1' }` / finalized - * job bodies for any matching URL — so a renamed route, a reshaped response, - * or a changed required field kept it green while the real round-trip broke - * (the exact drift class the no-mocks policy removes). - * - * This version drives both clients against a REAL edge daemon: - * - `share()` writes real quads into a real context graph's SWM and gets a - * REAL shareOperationId back, - * - `lift()` enqueues a real async-lift job (the daemon's persistent - * publisher queue accepts and stores it) and returns the REAL jobId, - * - `getJobStatus()` reads the REAL stored job back and deserializes the - * real lifecycle fields. - * Runs in the standard cli lane against the shared Hardhat node. + * This drives the lifecycle client against a REAL edge daemon: + * - createAndShare() creates, seals, and shares a real named KA into SWM, + * - publishAsync() enqueues a named KA VM publish job and returns the REAL jobId, + * - getJobStatus() reads the REAL stored job back and deserializes lifecycle fields. */ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; -import { createDaemonAsyncLiftJobClient, createDaemonSharedMemoryWriteClient } from '../src/source-worker-daemon-client.js'; +import { createDaemonKnowledgeAssetLifecycleClient } from '../src/source-worker-daemon-client.js'; import { startLiveDaemon, stopLiveDaemon, postJson, type LiveDaemon } from './helpers/live-daemon.js'; const CG = `swdc-${Date.now().toString(36)}`; const ROOT = `urn:swdc:${Date.now().toString(36)}:s`; -describe('source worker daemon clients (real daemon)', () => { +describe('source worker daemon client (real daemon)', () => { let daemon: LiveDaemon; beforeAll(async () => { @@ -36,46 +26,36 @@ describe('source worker daemon clients (real daemon)', () => { await stopLiveDaemon(daemon); }); - it('share() writes real SWM quads and returns the daemon-issued shareOperationId', async () => { - const share = createDaemonSharedMemoryWriteClient(daemon.base, daemon.token ?? ''); - const result = await share.share(CG, [{ subject: ROOT, predicate: 'urn:p', object: '"v"' }]); - // The daemon mints `swm--` ids — assert the real shape, - // not a canned constant. - expect(result.shareOperationId).toMatch(/^swm-/); + it('createAndShare() creates, seals, and shares a real named KA', async () => { + const client = createDaemonKnowledgeAssetLifecycleClient(daemon.base, daemon.token ?? ''); + const result = await client.createAndShare(CG, 'source-worker-create', [ + { subject: ROOT, predicate: 'urn:p', object: '"v"', graph: '' }, + ]); + expect(result.promotedCount).toBeGreaterThan(0); + expect(result.shareOperationId).toMatch(/^[0-9a-z]+-[0-9a-z]+$/); }); - it('lift() enqueues a real job and getJobStatus() reads the real stored job back', async () => { - const share = createDaemonSharedMemoryWriteClient(daemon.base, daemon.token ?? ''); - const jobs = createDaemonAsyncLiftJobClient(daemon.base, daemon.token ?? ''); - - const { shareOperationId } = await share.share(CG, [ - { subject: ROOT, predicate: 'urn:p:type', object: 'urn:Note' }, + it('publishAsync() enqueues a real job and getJobStatus() reads the real stored job back', async () => { + const client = createDaemonKnowledgeAssetLifecycleClient(daemon.base, daemon.token ?? ''); + const name = 'source-worker-async-publish'; + const share = await client.createAndShare(CG, name, [ + { subject: ROOT, predicate: 'urn:p:type', object: 'urn:Note', graph: '' }, ]); - const jobId = await jobs.lift({ - swmId: 'swm-live', - shareOperationId, - roots: [ROOT], - contextGraphId: CG, - namespace: 'ns', - scope: 'scope', - transitionType: 'CREATE', - authority: { type: 'owner', proofRef: 'proof' }, - } as never); - // Real daemon issues a UUID job id. + const publish = await client.publishAsync(CG, name); + const jobId = publish.jobId; expect(jobId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/); + expect(publish.shareOperationId).toBe(share.shareOperationId); + expect(publish.rootsCount).toBeGreaterThan(0); + expect(publish.intentKey).toMatch(/^sha256:[0-9a-f]{64}$/); - const status = await jobs.getJobStatus(jobId); - // The job was REALLY persisted: a real lifecycle status comes back (the - // edge daemon has no publisher runtime processing the queue, so the job - // sits in its initial state — any real status string proves the - // round-trip; a canned 'finalized' would prove nothing). + const status = await client.getJobStatus(jobId); expect(typeof status.status).toBe('string'); expect(status.status.length).toBeGreaterThan(0); }); it('getJobStatus() surfaces a real not-found for an unknown job id', async () => { - const jobs = createDaemonAsyncLiftJobClient(daemon.base, daemon.token ?? ''); - await expect(jobs.getJobStatus('00000000-0000-0000-0000-000000000000')).rejects.toThrow(); + const client = createDaemonKnowledgeAssetLifecycleClient(daemon.base, daemon.token ?? ''); + await expect(client.getJobStatus('00000000-0000-0000-0000-000000000000')).rejects.toThrow(); }); }); diff --git a/packages/cli/test/source-worker-runner.test.ts b/packages/cli/test/source-worker-runner.test.ts index a26fe577b8..0488bed292 100644 --- a/packages/cli/test/source-worker-runner.test.ts +++ b/packages/cli/test/source-worker-runner.test.ts @@ -1,19 +1,8 @@ /** - * Source-worker runner — REAL daemon round-trip, NO mocks. + * Source-worker runner - REAL daemon round-trip, NO mocks. * - * The retired version replaced `globalThis.fetch` with a stub returning - * canned `swm-1` / `job-1` bodies and asserted the OUTGOING request shapes. - * That pinned the wire format against a double — a daemon-side rename or a - * rejected body would keep it green. - * - * This version keeps everything that was already real (the runner's dynamic - * handler import, context wiring, state-file persistence) and points - * `daemonUrl` at a REAL edge daemon: the handler's `sharedMemory.share()` - * lands real quads in a real context graph and `asyncLift.lift()` enqueues a - * real job in the daemon's persistent publisher queue. The daemon ACCEPTING - * both calls (auth header included — auth is enabled, a wrong token would - * 401) and the state file recording the REAL daemon-issued ids is the proof - * the old request-capture only simulated. + * The dynamic handler receives the lifecycle client and drives a real named KA + * create/share plus named async VM publish through the daemon. */ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; @@ -31,9 +20,6 @@ declare global { const CG = `swr-${Date.now().toString(36)}`; const cleanup: string[] = []; - -// Silence the runner's progress logging with a hand-rolled save/restore (no -// vitest mock API — nothing under test is faked). const originalConsoleLog = console.log; describe('source worker runner (real daemon)', () => { @@ -59,7 +45,7 @@ describe('source worker runner (real daemon)', () => { await Promise.all(cleanup.splice(0).map((path) => rm(path, { recursive: true, force: true }))); }); - it('dynamically imports the handler, wires REAL daemon clients, and persists the real job state', async () => { + it('dynamically imports the handler, wires REAL lifecycle clients, and persists the real job state', async () => { const dir = await mkdtemp(join(tmpdir(), 'source-worker-runner-')); cleanup.push(dir); const configPath = join(dir, 'worker.json'); @@ -74,29 +60,24 @@ export const namedHandler = { daemonToken: context.config.daemonToken, stateFile: context.config.stateFile, sourceIds: context.config.sources.map((source) => source.id), - hasSharedMemory: typeof context.sharedMemory.share === 'function', - hasAsyncLift: typeof context.asyncLift.lift === 'function', + hasKnowledgeAssets: typeof context.knowledgeAssets.createAndShare === 'function' + && typeof context.knowledgeAssets.publishAsync === 'function', }; return { getFingerprint: async (source) => \`fp-\${source.version}\`, processSource: async (source, fingerprint) => { - const share = await context.sharedMemory.share('${CG}', [ - { subject: 'urn:src', predicate: 'urn:hasId', object: \`"\${source.id}"\` }, + const name = \`source-worker-\${source.id}\`; + const share = await context.knowledgeAssets.createAndShare('${CG}', name, [ + { subject: 'urn:src', predicate: 'urn:hasId', object: \`"\${source.id}"\`, graph: '' }, ], { subGraphName: 'sg-1' }); - const jobId = await context.asyncLift.lift({ - swmId: 'swm-live', - shareOperationId: share.shareOperationId, - roots: ['urn:src'], - contextGraphId: '${CG}', - namespace: 'ns', - scope: 'scope', - transitionType: 'CREATE', - authority: { type: 'owner', proofRef: 'proof' }, - }); + const publish = await context.knowledgeAssets.publishAsync('${CG}', name, { subGraphName: 'sg-1' }); + const jobId = publish.jobId; globalThis.__sourceWorkerRunnerProcessed = { sourceId: source.id, fingerprint, + name, shareOperationId: share.shareOperationId, + intentKey: publish.intentKey, jobId, }; return { @@ -107,6 +88,9 @@ export const namedHandler = { status: 'queued', nextState: { fingerprint, + assertionName: name, + shareOperationId: share.shareOperationId, + intentKey: publish.intentKey, lastStatus: 'queued', lastJobIds: [jobId], lastJobStatuses: { [jobId]: 'queued' }, @@ -134,30 +118,34 @@ export const namedHandler = { daemonToken: daemon.token, stateFile: statePath, sourceIds: ['src-1'], - hasSharedMemory: true, - hasAsyncLift: true, + hasKnowledgeAssets: true, }); - // The handler ran against the REAL daemon: real swm-* share id, real UUID - // job id (canned 'swm-1'/'job-1' constants would prove nothing). const processed = globalThis.__sourceWorkerRunnerProcessed; expect(processed.sourceId).toBe('src-1'); expect(processed.fingerprint).toBe('fp-v1'); - expect(processed.shareOperationId).toMatch(/^swm-/); + expect(processed.name).toBe('source-worker-src-1'); + expect(processed.shareOperationId).toMatch(/^[0-9a-z]+-[0-9a-z]+$/); + expect(processed.intentKey).toMatch(/^sha256:[0-9a-f]{64}$/); expect(processed.jobId).toMatch(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/); - // The job REALLY exists in the daemon's queue. const jobRes = await fetch(`${daemon.base}/api/publisher/job?id=${processed.jobId}`, { headers: { Authorization: `Bearer ${daemon.token}` }, }); expect(jobRes.status).toBe(200); const jobBody: any = await jobRes.json(); expect(jobBody.job.request.contextGraphId).toBe(CG); + expect(jobBody.job.request.jobType).toBe('knowledge-asset-vm-publish'); + expect(jobBody.job.request.knowledgeAssetVmPublish.name).toBe(processed.name); + expect(jobBody.job.request.knowledgeAssetVmPublish.shareOperationId).toBe(processed.shareOperationId); + expect(jobBody.job.request.knowledgeAssetVmPublish.intentKey).toBe(processed.intentKey); - // State persistence recorded the REAL ids. const state = JSON.parse(await readFile(statePath, 'utf8')); expect(state.sources['src-1']).toMatchObject({ fingerprint: 'fp-v1', + assertionName: 'source-worker-src-1', + shareOperationId: processed.shareOperationId, + intentKey: processed.intentKey, lastStatus: 'queued', lastJobIds: [processed.jobId], }); diff --git a/packages/core/src/assertion-seal.ts b/packages/core/src/assertion-seal.ts index ec92cb63da..2f0e0ebf06 100644 --- a/packages/core/src/assertion-seal.ts +++ b/packages/core/src/assertion-seal.ts @@ -78,9 +78,8 @@ export const ASSERTION_SEAL_PREDICATES = { } as const; /** - * Predicates written by `/api/shared-memory/publish` after a - * successful on-chain publish. These are receipts; they don't - * affect the seal's validity. + * Predicates written by named VM publish lifecycle routes after a successful + * on-chain publish. These are receipts; they don't affect the seal's validity. */ export const ASSERTION_PUBLISH_RECEIPT_PREDICATES = { /** Transaction hash of the KAv10 publish (string literal). */ diff --git a/packages/core/src/publisher-extension.ts b/packages/core/src/publisher-extension.ts index 5428741bb7..72cf59ec43 100644 --- a/packages/core/src/publisher-extension.ts +++ b/packages/core/src/publisher-extension.ts @@ -21,10 +21,6 @@ export interface DkgPublisherExtensionWriteResult { written: number; } -export interface DkgPublisherExtensionShareResult { - shareOperationId: string; -} - export interface DkgPublisherExtensionPublishResult { kaId?: string | number | bigint; kas?: unknown[]; @@ -57,12 +53,6 @@ export interface DkgPublisherExtensionTransport { opts?: { subGraphName?: string }, ): Promise>; - share( - contextGraphId: string, - quads: DkgPublisherExtensionQuad[], - opts?: { localOnly?: boolean; subGraphName?: string }, - ): Promise; - publish( contextGraphId: string, quads: DkgPublisherExtensionQuad[], @@ -103,13 +93,6 @@ export interface LocalWorkspaceDiscardRequest { subGraphName?: string; } -export interface SharedMemoryWriteRequest { - contextGraphId: string; - quads: DkgPublisherExtensionQuadInput[]; - localOnly?: boolean; - subGraphName?: string; -} - export interface VerifiableMemoryPublishRequest { contextGraphId: string; quads: DkgPublisherExtensionQuadInput[]; @@ -166,17 +149,6 @@ export class DkgPublisherExtension { }); } - async writeSharedMemory(request: SharedMemoryWriteRequest): Promise { - return this.transport.share( - request.contextGraphId, - normalizeDkgPublisherQuads(request.quads), - { - localOnly: request.localOnly, - subGraphName: request.subGraphName, - }, - ); - } - async publishVerifiableMemory( request: VerifiableMemoryPublishRequest, ): ReturnType { diff --git a/packages/core/test/publisher-extension.test.ts b/packages/core/test/publisher-extension.test.ts index 81bae11333..01850cb8c1 100644 --- a/packages/core/test/publisher-extension.test.ts +++ b/packages/core/test/publisher-extension.test.ts @@ -28,10 +28,6 @@ function createTransport(): DkgPublisherExtensionTransport & { calls: Array<[str calls.push(['discardAssertion', args]); return { discarded: true }; }, - async share(...args) { - calls.push(['share', args]); - return { shareOperationId: 'swm-1' }; - }, async publish(...args) { calls.push(['publish', args]); return { kaId: '1', kas: [{ tokenId: '1', rootEntity: 'did:dkg:entity:1' }] }; @@ -96,25 +92,6 @@ describe('DkgPublisherExtension', () => { ]); }); - it('can write to shared memory without creating a workspace assertion', async () => { - const transport = createTransport(); - const publisher = createDkgPublisherExtension(transport); - - await publisher.writeSharedMemory({ - contextGraphId: 'cg', - localOnly: false, - quads: [{ subject: 'did:dkg:e:1', predicate: 'http://schema.org/url', object: 'https://example.org/a' }], - }); - - expect(transport.calls).toEqual([ - ['share', [ - 'cg', - [{ subject: 'did:dkg:e:1', predicate: 'http://schema.org/url', object: 'https://example.org/a', graph: '' }], - { localOnly: false, subGraphName: undefined }, - ]], - ]); - }); - it('publishes fresh quads into verifiable memory', async () => { const transport = createTransport(); const publisher = createDkgPublisherExtension(transport); diff --git a/packages/network-sim/src/api.ts b/packages/network-sim/src/api.ts index af3779f2f9..969f61b1ac 100644 --- a/packages/network-sim/src/api.ts +++ b/packages/network-sim/src/api.ts @@ -93,7 +93,7 @@ export async function publishKA( name: assertionName, quads: quadsWithGraph, finalize: true, - promote: true, + alsoShareSwm: true, }, ); const published = await post<{ @@ -124,15 +124,20 @@ export async function queryNode( }); } -export async function share( +export async function createSharedKnowledgeAsset( nodeId: number, contextGraphId: string, quads: Array<{ subject: string; predicate: string; object: string; graph?: string }>, ) { - return post<{ shareOperationId: string }>(`${nodeBase(nodeId)}/api/shared-memory/write`, { + const name = `netsim-share-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`; + const result = await post>(`${nodeBase(nodeId)}/api/knowledge-assets`, { contextGraphId, + name, quads, + finalize: true, + alsoShareSwm: true, }); + return { ...result, name }; } export async function sendChat(nodeId: number, to: string, text: string) { diff --git a/packages/network-sim/src/components/ControlPanel.tsx b/packages/network-sim/src/components/ControlPanel.tsx index d96a06e453..c820093bcf 100644 --- a/packages/network-sim/src/components/ControlPanel.tsx +++ b/packages/network-sim/src/components/ControlPanel.tsx @@ -537,11 +537,11 @@ function SharedMemoryTab() { setResult(''); const graph = contextGraphId.startsWith('did:') ? contextGraphId : `did:dkg:context-graph:${contextGraphId}`; const quads = [{ subject, predicate, object: fmtObj(object), graph }]; - const opId = addBroadcast('workspace', state.selectedNode, 'shared memory write'); + const opId = addBroadcast('workspace', state.selectedNode, 'KA share'); try { - const res = await api.share(state.selectedNode, contextGraphId, quads); - completeOperation(opId, 'success', res.shareOperationId); - setResult(`Written to shared memory.\nOperation: ${res.shareOperationId}`); + const res = await api.createSharedKnowledgeAsset(state.selectedNode, contextGraphId, quads); + completeOperation(opId, 'success', res.name); + setResult(`Created and shared knowledge asset.\nName: ${res.name}`); } catch (e: any) { completeOperation(opId, 'error', e.message); setResult(`Error: ${e.message}`); @@ -569,8 +569,7 @@ function SharedMemoryTab() { return (

- Write triples to shared memory (free, no gas). Publishing to the chain is - done per knowledge asset from the Publish tab. + Create, seal, and share a named knowledge asset into shared working memory.
diff --git a/packages/network-sim/src/server/sim-engine.ts b/packages/network-sim/src/server/sim-engine.ts index 2de127f6e8..1e8440be59 100644 --- a/packages/network-sim/src/server/sim-engine.ts +++ b/packages/network-sim/src/server/sim-engine.ts @@ -256,8 +256,8 @@ async function execPublish( // RFC-001 §9.x — route through the knowledge-asset lifecycle (sign at // creation): the daemon's `/api/knowledge-assets` endpoint accepts a - // `quads + finalize: true + promote: true` shape that folds the - // create→write→finalize→promote chain into a single round-trip. + // `quads + finalize: true + alsoShareSwm: true` shape that folds the + // create→write→finalize→share chain into a single round-trip. // The publish call then forwards the seal verbatim. const assertionName = `netsim-publish-${Date.now()}-${rndId()}`; try { @@ -269,7 +269,7 @@ async function execPublish( name: assertionName, quads, finalize: true, - promote: true, + alsoShareSwm: true, }), signal: opSignal(signal, 'publish'), }); @@ -380,13 +380,20 @@ async function execWorkspace( ]; try { - const res = await fetch(`http://127.0.0.1:${node.port}/api/shared-memory/write`, { + const name = `netsim-workspace-${Date.now()}-${rndId()}`; + const res = await fetch(`http://127.0.0.1:${node.port}/api/knowledge-assets`, { method: 'POST', headers: { 'Content-Type': 'application/json', ...authHeaders(node) }, - body: JSON.stringify({ contextGraphId: config.contextGraph, quads }), + body: JSON.stringify({ + contextGraphId: config.contextGraph, + name, + quads, + finalize: true, + alsoShareSwm: true, + }), signal: opSignal(signal, 'sharedMemory'), }); - const body = (await res.json()) as { shareOperationId?: string; phases?: Record }; + const body = (await res.json()) as { phases?: Record }; const dur = Date.now() - t0; return { type: 'op', @@ -394,7 +401,7 @@ async function execWorkspace( nodeId: node.id, success: res.ok, durationMs: dur, - detail: `opId: ${body.shareOperationId?.slice(0, 8) ?? '?'}`, + detail: `ka: ${name}`, phases: body.phases ?? {}, }; } catch (err) { diff --git a/packages/node-ui/e2e/helpers/devnet-publish.ts b/packages/node-ui/e2e/helpers/devnet-publish.ts index 9d58ecde5e..f5468532b6 100644 --- a/packages/node-ui/e2e/helpers/devnet-publish.ts +++ b/packages/node-ui/e2e/helpers/devnet-publish.ts @@ -214,7 +214,7 @@ export async function createWmAssertion(opts: { contextGraphId: string; name: string; quads: PublishQuads[]; - promote?: boolean; + alsoShareSwm?: boolean; subGraphName?: string; nodeNum?: number; }): Promise<{ ok: boolean; status: number; body: string }> { @@ -228,7 +228,7 @@ export async function createWmAssertion(opts: { finalize: true, // rc.17 KA-routes-unification: the inline SWM-promote flag is `alsoShareSwm` // (the old `promote` is silently ignored → the assertion never reaches SWM). - alsoShareSwm: opts.promote ?? false, + alsoShareSwm: opts.alsoShareSwm ?? false, ...(opts.subGraphName ? { subGraphName: opts.subGraphName } : {}), }), }); @@ -332,7 +332,7 @@ export async function runWmSwmVmPipeline(opts: { contextGraphId: opts.contextGraphId, name: assertionName, quads, - promote: false, + alsoShareSwm: false, nodeNum: opts.nodeNum, }); if (!wm.ok) throw new Error(`WM create failed: ${wm.status} ${wm.body}`); diff --git a/packages/node-ui/e2e/helpers/real-node.ts b/packages/node-ui/e2e/helpers/real-node.ts index 90a2a8da2a..43600f59f6 100644 --- a/packages/node-ui/e2e/helpers/real-node.ts +++ b/packages/node-ui/e2e/helpers/real-node.ts @@ -104,7 +104,7 @@ export async function seedWmEntity(cgId: string, prefix = 'wm'): Promise { } }); - test('publish without PCA registration uses standard shared-memory path', async () => { + test('publish without PCA registration uses the KA lifecycle path', async () => { const { runWmSwmVmPipeline, listContextGraphs } = await import('../../helpers/devnet-publish.js'); const cgs = await listContextGraphs(1); requireDevnetPrecondition(test, cgs.length === 0, 'No CGs'); @@ -34,7 +34,7 @@ test.describe('Conviction NFT (PCA) API', () => { }); test.describe('Non-conviction publishing (baseline)', () => { - test('shared-memory publish endpoint responds for devnet CG', async () => { + test('named KA VM publish responds for devnet CG', async () => { const cgsRes = await devnetApiFetch('/api/context-graphs'); const { contextGraphs } = (await cgsRes.json()) as { contextGraphs: Array<{ id: string }> }; requireDevnetPrecondition(test, contextGraphs.length === 0, 'No CGs'); diff --git a/packages/node-ui/e2e/specs/devnet/publishing-lifecycle.devnet.spec.ts b/packages/node-ui/e2e/specs/devnet/publishing-lifecycle.devnet.spec.ts index c13302d1b1..ac71965c23 100644 --- a/packages/node-ui/e2e/specs/devnet/publishing-lifecycle.devnet.spec.ts +++ b/packages/node-ui/e2e/specs/devnet/publishing-lifecycle.devnet.spec.ts @@ -55,7 +55,7 @@ test.describe('Publishing lifecycle (devnet)', () => { }, ], finalize: true, - promote: false, + alsoShareSwm: false, }), }); expect(res.ok).toBe(true); diff --git a/packages/node-ui/e2e/specs/devnet/wm-swm-vm-lifecycle.devnet.spec.ts b/packages/node-ui/e2e/specs/devnet/wm-swm-vm-lifecycle.devnet.spec.ts index 13d708db95..bc4a60ccc8 100644 --- a/packages/node-ui/e2e/specs/devnet/wm-swm-vm-lifecycle.devnet.spec.ts +++ b/packages/node-ui/e2e/specs/devnet/wm-swm-vm-lifecycle.devnet.spec.ts @@ -39,7 +39,7 @@ test.describe('WM → SWM → VM API pipeline', () => { contextGraphId: run.cgId!, name, quads: buildTestQuads(run.cgId!, stamp, `WM Only ${stamp}`), - promote: false, + alsoShareSwm: false, }); expect(res.ok).toBe(true); }); @@ -51,7 +51,7 @@ test.describe('WM → SWM → VM API pipeline', () => { contextGraphId: run.cgId!, name, quads: buildTestQuads(run.cgId!, stamp, `SWM Test ${stamp}`), - promote: false, + alsoShareSwm: false, }); expect(create.ok).toBe(true); const promote = await promoteAssertion({ contextGraphId: run.cgId!, assertionName: name }); diff --git a/packages/node-ui/e2e/specs/known-bugs/typed-literal-datatype-leak.spec.ts b/packages/node-ui/e2e/specs/known-bugs/typed-literal-datatype-leak.spec.ts index 8a0be5a1be..27e5a468ee 100644 --- a/packages/node-ui/e2e/specs/known-bugs/typed-literal-datatype-leak.spec.ts +++ b/packages/node-ui/e2e/specs/known-bugs/typed-literal-datatype-leak.spec.ts @@ -61,7 +61,7 @@ test.describe('KNOWN BUG: typed RDF literals leak their datatype suffix in entit graph, }, ], - promote: false, + alsoShareSwm: false, }); expect(seed.ok, `WM seed failed: ${seed.status} ${seed.body}`).toBe(true); diff --git a/packages/node-ui/scripts/stage5-scale-benchmark.mts b/packages/node-ui/scripts/stage5-scale-benchmark.mts index c2c7b96b3f..76dc46eb4a 100644 --- a/packages/node-ui/scripts/stage5-scale-benchmark.mts +++ b/packages/node-ui/scripts/stage5-scale-benchmark.mts @@ -134,8 +134,8 @@ async function benchmarkSessionList(sessionCount: number): Promise ({}), - publishFromSharedMemory: async () => ({}), + createAssertion: async () => ({ assertionUri: 'urn:bench:assertion', alreadyExists: true }), + writeAssertion: async (_contextGraphId: string, _name: string, quads: any[]) => ({ written: quads.length }), createContextGraph: async () => undefined, listContextGraphs: async () => [{ id: MEMORY_CONTEXT_GRAPH, name: 'Agent Memory' }], }, diff --git a/packages/node-ui/scripts/stage6-delta-benchmark.mts b/packages/node-ui/scripts/stage6-delta-benchmark.mts index 0f56988426..186b8aeae9 100644 --- a/packages/node-ui/scripts/stage6-delta-benchmark.mts +++ b/packages/node-ui/scripts/stage6-delta-benchmark.mts @@ -267,8 +267,8 @@ function makeManager(store: OxigraphStore): ChatMemoryManager { if (opts?.graphSuffix === '_workspace') return executeOnGraph(store, sparql, WORKSPACE_GRAPH); return executeOnGraph(store, sparql, DATA_GRAPH); }, - writeToWorkspace: async () => ({ shareOperationId: 'noop' }), - publishFromSharedMemory: async () => ({}), + createAssertion: async () => ({ assertionUri: 'urn:bench:assertion', alreadyExists: true }), + writeAssertion: async (_contextGraphId: string, _name: string, quads: any[]) => ({ written: quads.length }), createContextGraph: async () => undefined, listContextGraphs: async () => [{ id: MEMORY_CONTEXT_GRAPH, name: 'Agent Memory' }], }, diff --git a/packages/node-ui/src/chat-memory.ts b/packages/node-ui/src/chat-memory.ts index a5b5e29621..f4ce1d8db2 100644 --- a/packages/node-ui/src/chat-memory.ts +++ b/packages/node-ui/src/chat-memory.ts @@ -17,13 +17,6 @@ export interface MemoryToolContext { subGraphName?: string; }, ) => Promise; - /** - * Direct SWM write primitive. Retained on the context so callers that - * legitimately want Shared Working Memory semantics (e.g. user-initiated - * promotion to a project's shared memory) have access to it, but chat-turn - * and per-project memory writes use `writeAssertion` instead. - */ - share: (contextGraphId: string, quads: any[], opts?: { localOnly?: boolean; subGraphName?: string }) => Promise<{ shareOperationId: string }>; /** * Create a per-agent Working Memory assertion graph. Idempotent: "already * exists" is resolved quietly, any other error surfaces. @@ -40,11 +33,6 @@ export interface MemoryToolContext { quads: any[], opts?: { subGraphName?: string }, ) => Promise<{ written: number }>; - publishFromSharedMemory: ( - contextGraphId: string, - selection: 'all' | { rootEntities: string[] }, - opts?: { clearSharedMemoryAfter?: boolean }, - ) => Promise; createContextGraph: (opts: { id: string; name: string; description?: string; private?: boolean }) => Promise; listContextGraphs: () => Promise; } @@ -1523,32 +1511,11 @@ export class ChatMemoryManager { sessionId: string, opts?: { rootEntities?: string[]; clearSharedMemoryAfter?: boolean }, ): Promise { - await this.ensureInitialized(); - const sessionRoots = await this.getSessionRootEntities(sessionId); - if (sessionRoots.length === 0) { - throw new Error(`No shared memory entities found for session ${sessionId}`); - } - const sessionRootSet = new Set(sessionRoots); - const requestedRoots = (opts?.rootEntities ?? []) - .map((r) => String(r).trim()) - .filter((r) => isSafeIri(r)); - const rootEntities = requestedRoots.length > 0 - ? [...new Set(requestedRoots.filter((r) => sessionRootSet.has(r)))] - : sessionRoots; - if (rootEntities.length === 0) { - throw new Error(`Selected root entities are not part of session ${sessionId}`); - } - const published = await this.publishFromSwm( - { rootEntities }, - { clearSharedMemoryAfter: opts?.clearSharedMemoryAfter ?? false }, + void opts; + throw new Error( + `Session publication is not implemented in v1 for session ${sessionId}. ` + + 'Chat turns live in Working Memory assertions and must be promoted through named knowledge-asset lifecycle routes.', ); - const publication = await this.getSessionPublicationStatus(sessionId); - return { - ...published, - sessionId, - rootEntityCount: rootEntities.length, - publication, - }; } // importMemories / parseMemoriesWithLlm / parseMemoriesHeuristic / @@ -1563,16 +1530,11 @@ export class ChatMemoryManager { selection: 'all' | { rootEntities: string[] } = 'all', opts?: { clearSharedMemoryAfter?: boolean }, ): Promise { - await this.ensureInitialized(); - const result = await this.tools.publishFromSharedMemory(this.agentContextGraph, selection, { - clearSharedMemoryAfter: opts?.clearSharedMemoryAfter ?? false, - }); - return { - kaId: result?.kaId, - ual: result?.ual, - status: result?.status ?? 'confirmed', - tripleCount: result?.publicQuads?.length ?? 0, - }; + void selection; + void opts; + throw new Error( + 'publishFromSwm is retired in v1; use named knowledge-asset lifecycle share/publish routes.', + ); } private parseNTriples(text: string): Array<{ subject: string; predicate: string; object: string }> { diff --git a/packages/node-ui/src/ui/hooks/useMemoryEntities.ts b/packages/node-ui/src/ui/hooks/useMemoryEntities.ts index 74df5cfbfd..09e78307c4 100644 --- a/packages/node-ui/src/ui/hooks/useMemoryEntities.ts +++ b/packages/node-ui/src/ui/hooks/useMemoryEntities.ts @@ -196,7 +196,7 @@ function deriveEntityLabel(entity: MemoryEntity): string { // three layers. // // The V10 named-graph layout we rely on (see `resolveViewGraphs` -// in `@origintrail-official/dkg-query` and `publishFromSharedMemory`): +// in `@origintrail-official/dkg-query` and named KA lifecycle publish): // // WM (drafts) : did:dkg:context-graph://assertion// // SWM (proposed) : did:dkg:context-graph://_shared_memory diff --git a/packages/node-ui/test/chat-memory-persistence-regression.test.ts b/packages/node-ui/test/chat-memory-persistence-regression.test.ts index 8f7c05f529..f7866e4212 100644 --- a/packages/node-ui/test/chat-memory-persistence-regression.test.ts +++ b/packages/node-ui/test/chat-memory-persistence-regression.test.ts @@ -66,7 +66,6 @@ function buildStoreBackedTools(writeAgentAddress: string) { const quads = store.get(graphUri) ?? []; return executeMiniSparql(sparql, quads); }, - share: async () => ({ shareOperationId: 'noop' }), createAssertion: async (contextGraphId: string, name: string) => { const graphUri = contextGraphAssertionUri(contextGraphId, writeAgentAddress, name); if (!store.has(graphUri)) store.set(graphUri, []); @@ -79,7 +78,6 @@ function buildStoreBackedTools(writeAgentAddress: string) { store.set(graphUri, bucket); return { written: quads.length }; }, - publishFromSharedMemory: async () => ({}), createContextGraph: async () => {}, listContextGraphs: async () => [{ id: 'agent-context', name: 'Agent Context' }], }; diff --git a/packages/node-ui/test/chat-memory.test.ts b/packages/node-ui/test/chat-memory.test.ts index 4a4e65374f..c8e6087c24 100644 --- a/packages/node-ui/test/chat-memory.test.ts +++ b/packages/node-ui/test/chat-memory.test.ts @@ -33,18 +33,14 @@ function createTools(overrides?: { const mockShare = overrides?.mockShare ?? trackFn({ shareOperationId: 'op-1' }); const mockCreateContextGraph = overrides?.mockCreateContextGraph ?? trackFn(undefined); const mockListContextGraphs = overrides?.mockListContextGraphs ?? trackFn([{ id: 'agent-memory', name: 'Agent Memory' }]); - const mockPublishFromSharedMemory = overrides?.mockPublishFromSharedMemory ?? trackFn({}); return { mockQuery, mockShare, mockCreateContextGraph, mockListContextGraphs, - mockPublishFromSharedMemory, tools: { query: mockQuery, - share: mockShare, - publishFromSharedMemory: mockPublishFromSharedMemory, createContextGraph: mockCreateContextGraph, listContextGraphs: mockListContextGraphs, }, @@ -147,10 +143,8 @@ describe('ChatMemoryManager', () => { manager = new ChatMemoryManager( { query: mockQuery, - share: mockShare, createAssertion: mockCreateAssertion, writeAssertion: mockWriteAssertion, - publishFromSharedMemory: trackFn({}), createContextGraph: mockCreateContextGraph, listContextGraphs: mockListContextGraphs, }, @@ -292,10 +286,8 @@ describe('ChatMemoryManager', () => { const m = new ChatMemoryManager( { query: mockQuery, - share: mockShare, createAssertion: mockCreateAssertion, writeAssertion: mockWriteAssertion, - publishFromSharedMemory: trackFn({}), createContextGraph: mockCreateContextGraph, listContextGraphs: mockListContextGraphs, }, @@ -680,10 +672,8 @@ describe('ChatMemoryManager', () => { const gpt5Manager = new ChatMemoryManager( { query: mockQuery, - share: mockShare, createAssertion: mockCreateAssertion, writeAssertion: mockWriteAssertion, - publishFromSharedMemory: trackFn({}), createContextGraph: mockCreateContextGraph, listContextGraphs: mockListContextGraphs, }, @@ -760,116 +750,12 @@ describe('ChatMemoryManager', () => { expect(query).not.toContain(''); }); - it('publishSession uses derived session root entities when none are provided', async () => { - const publishFromSharedMemory = trackFn({ - status: 'confirmed', - publicQuads: [{}, {}], - kaId: 10n, - ual: 'did:dkg:mock:123', - }); - const managerWithPublish = new ChatMemoryManager( - { - query: mockQuery, - share: mockShare, - createAssertion: mockCreateAssertion, - writeAssertion: mockWriteAssertion, - publishFromSharedMemory, - createContextGraph: mockCreateContextGraph, - listContextGraphs: mockListContextGraphs, - }, - { apiKey: 'test' }, - { agentAddress: 'did:dkg:agent:test' }, - ); - - mockQuery.returns.push( - { bindings: [] }, - { bindings: [{ s: 'urn:dkg:chat:session:s-2' }, { s: 'urn:dkg:chat:msg:m-2' }] }, - { bindings: [{ c: '"1"^^' }] }, - { bindings: [{ c: '"1"^^' }] }, - { bindings: [{ s: 'urn:dkg:chat:session:s-2' }] }, - ); - - const result = await managerWithPublish.publishSession('s-2'); - expect(publishFromSharedMemory.calls[0]).toEqual([ - 'agent-context', - { rootEntities: ['urn:dkg:chat:session:s-2', 'urn:dkg:chat:msg:m-2'] }, - { clearSharedMemoryAfter: false }, - ]); - expect(result.sessionId).toBe('s-2'); - expect(result.rootEntityCount).toBe(2); - expect(result.publication.scope).toBe('published'); + it('publishSession is retired until chat turns are promoted through named KA lifecycle routes', async () => { + await expect(manager.publishSession('s-2')).rejects.toThrow('Session publication is not implemented in v1'); }); - it('publishSession restricts requested roots to entities belonging to the target session', async () => { - const publishFromSharedMemory = trackFn({ - status: 'confirmed', - publicQuads: [{}, {}], - kaId: 11n, - ual: 'did:dkg:mock:124', - }); - const managerWithPublish = new ChatMemoryManager( - { - query: mockQuery, - share: mockShare, - createAssertion: mockCreateAssertion, - writeAssertion: mockWriteAssertion, - publishFromSharedMemory, - createContextGraph: mockCreateContextGraph, - listContextGraphs: mockListContextGraphs, - }, - { apiKey: 'test' }, - { agentAddress: 'did:dkg:agent:test' }, - ); - - mockQuery.returns.push( - { bindings: [] }, - { bindings: [{ s: 'urn:dkg:chat:session:s-3' }, { s: 'urn:dkg:chat:msg:m-3' }] }, - { bindings: [{ c: '"2"^^' }] }, - { bindings: [{ c: '"2"^^' }] }, - { bindings: [{ s: 'urn:dkg:chat:session:s-3' }, { s: 'urn:dkg:chat:msg:m-3' }] }, - ); - - await managerWithPublish.publishSession('s-3', { - rootEntities: ['urn:dkg:chat:msg:m-3', 'urn:dkg:chat:msg:not-in-session'], - }); - - expect(publishFromSharedMemory.calls[0]).toEqual([ - 'agent-context', - { rootEntities: ['urn:dkg:chat:msg:m-3'] }, - { clearSharedMemoryAfter: false }, - ]); - }); - - it('publishSession rejects requested roots that are not in session scope', async () => { - const publishFromSharedMemory = trackFn({ - status: 'confirmed', - publicQuads: [{}, {}], - }); - const managerWithPublish = new ChatMemoryManager( - { - query: mockQuery, - share: mockShare, - createAssertion: mockCreateAssertion, - writeAssertion: mockWriteAssertion, - publishFromSharedMemory, - createContextGraph: mockCreateContextGraph, - listContextGraphs: mockListContextGraphs, - }, - { apiKey: 'test' }, - { agentAddress: 'did:dkg:agent:test' }, - ); - - mockQuery.returns.push( - { bindings: [] }, - { bindings: [{ s: 'urn:dkg:chat:session:s-4' }] }, - ); - - await expect( - managerWithPublish.publishSession('s-4', { - rootEntities: ['urn:dkg:chat:msg:not-in-session'], - }), - ).rejects.toThrow('Selected root entities are not part of session s-4'); - expect(publishFromSharedMemory.calls).toHaveLength(0); + it('publishFromSwm is retired as a raw SWM publish helper', async () => { + await expect(manager.publishFromSwm('all')).rejects.toThrow('publishFromSwm is retired in v1'); }); it('getSessionGraphDelta returns turn-scoped triples when watermark matches', async () => { @@ -1043,10 +929,8 @@ describe('ChatMemoryManager WM write discipline', () => { return new ChatMemoryManager( { query: mockQuery, - share: mockShare, createAssertion: mockCreateAssertion, writeAssertion: mockWriteAssertion, - publishFromSharedMemory: trackFn({}), createContextGraph: mockCreateContextGraph, listContextGraphs: mockListContextGraphs, }, diff --git a/packages/publisher/src/async-lift-publisher-impl.ts b/packages/publisher/src/async-lift-publisher-impl.ts index f1706d856d..273ff19a91 100644 --- a/packages/publisher/src/async-lift-publisher-impl.ts +++ b/packages/publisher/src/async-lift-publisher-impl.ts @@ -14,6 +14,7 @@ import { type LiftJobFinalizationMetadata, type LiftJobRecoveryMetadata, type LiftJobState, + type KnowledgeAssetVmPublishRequest, type LiftRequest, } from './lift-job.js'; import type { @@ -21,6 +22,7 @@ import type { AsyncLiftPublisherConfig, AsyncLiftPublisherRecoveryResolver, } from './async-lift-publisher-types.js'; +import { AsyncLiftJobConflictError } from './async-lift-publisher-types.js'; import { mapPublishExceptionToLiftJobFailure, mapPublishResultToLiftJobSuccess, @@ -42,6 +44,7 @@ import { PAYLOAD_PREDICATE, STATUS_PREDICATE, compareAcceptedJobs, + createKnowledgeAssetVmPublishLiftRequest, createJobSlug, expectBindings, getRecoveryTxHash, @@ -71,6 +74,7 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { private readonly idGenerator: () => string; private readonly chainRecoveryResolver?: AsyncLiftPublisherRecoveryResolver; private readonly publishExecutor?: AsyncLiftPublisherConfig['publishExecutor']; + private readonly knowledgeAssetVmPublishExecutor?: AsyncLiftPublisherConfig['knowledgeAssetVmPublishExecutor']; private readonly resolvedSliceOverrides?: Partial; private readonly publicSnapshotStore?: AsyncLiftPublisherConfig['publicSnapshotStore']; private readonly graphManager: GraphManager; @@ -90,6 +94,7 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { this.idGenerator = config.idGenerator ?? (() => crypto.randomUUID()); this.chainRecoveryResolver = config.chainRecoveryResolver; this.publishExecutor = config.publishExecutor; + this.knowledgeAssetVmPublishExecutor = config.knowledgeAssetVmPublishExecutor; this.resolvedSliceOverrides = config.resolvedSliceOverrides; this.publicSnapshotStore = config.publicSnapshotStore; this.graphManager = new GraphManager(store); @@ -115,6 +120,42 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { return jobId; } + async enqueueKnowledgeAssetVmPublish(request: KnowledgeAssetVmPublishRequest): Promise { + return this.withClaimLock(async () => { + await this.ensureGraph(); + if (!request.shareOperationId.trim()) { + throw new Error('Knowledge asset VM publish requires a shareOperationId'); + } + if (request.roots.length === 0) { + throw new Error('Knowledge asset VM publish requires at least one shared root'); + } + const existing = await this.findActiveKnowledgeAssetVmPublishJob(request); + if (existing?.compatible) { + return existing.job.jobId; + } + if (existing?.job) { + throw new AsyncLiftJobConflictError( + `Knowledge asset VM publish is already queued for "${request.name}" in context graph "${request.contextGraphId}" with a different share intent`, + existing.job.jobId, + ); + } + const liftRequest = createKnowledgeAssetVmPublishLiftRequest(request); + const now = this.now(); + const jobId = this.idGenerator(); + const job: LiftJobAccepted = { + jobId, + jobSlug: createJobSlug(liftRequest), + request: liftRequest, + status: 'accepted', + timestamps: { acceptedAt: now, updatedAt: now }, + retries: { retryCount: 0, maxRetries: this.maxRetries }, + controlPlane: { jobRef: jobSubject(jobId) }, + }; + await this.writeJob(job); + return jobId; + }); + } + // Adapt the lift's canonicalization to the SWM partition: for every // request root that already has private staging from the share, insert // a ` dkg:privateDataAnchor "true"` triple into @@ -223,6 +264,9 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { if (!job) { return null; } + if (job.request.jobType === 'knowledge-asset-vm-publish') { + return null; + } const resolved = await resolveLiftWorkspaceSlice({ store: this.store, @@ -259,10 +303,6 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { } async processNext(walletId: string): Promise { - if (!this.publishExecutor) { - throw new Error('Async lift publisher processNext requires a configured publishExecutor'); - } - const claimed = await this.claimNext(walletId); if (!claimed) { return null; @@ -270,6 +310,12 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { let failureState: LiftJobState = claimed.status; try { + if (claimed.request.jobType === 'knowledge-asset-vm-publish') { + return await this.processKnowledgeAssetVmPublish(claimed, walletId); + } + if (!this.publishExecutor) { + throw new Error('Async lift publisher processNext requires a configured publishExecutor'); + } const resolved = await resolveLiftWorkspaceSlice({ store: this.store, graphManager: this.graphManager, @@ -320,6 +366,100 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { } } + private async processKnowledgeAssetVmPublish(claimed: LiftJob, walletId: string): Promise { + if (!this.knowledgeAssetVmPublishExecutor) { + throw new Error('Async knowledge asset VM publish requires a configured knowledgeAssetVmPublishExecutor'); + } + const request = claimed.request.knowledgeAssetVmPublish; + if (!request) { + throw new Error(`Knowledge asset VM publish job ${claimed.jobId} is missing request metadata`); + } + + let validated!: ReturnType; + let prepared!: AsyncPreparedPublishPayload; + try { + const resolved = await resolveLiftWorkspaceSlice({ + store: this.store, + graphManager: this.graphManager, + request: claimed.request, + publicSnapshotStore: this.publicSnapshotStore, + }); + validated = validateLiftPublishPayload({ + request: claimed.request, + resolved: { + ...resolved, + ...this.resolvedSliceOverrides, + }, + }); + await this.update(claimed.jobId, 'validated', { + validation: validated.validation, + }); + prepared = prepareAsyncPublishPayload({ + request: claimed.request, + validation: validated.validation, + resolved: validated.resolved, + }); + } catch (error) { + return await this.recordExecutionFailure(claimed.jobId, 'claimed', error); + } + + try { + const publishResult = await this.knowledgeAssetVmPublishExecutor({ + walletId, + request, + liftRequest: claimed.request, + validation: validated.validation, + resolved: validated.resolved, + publishOptions: prepared.publishOptions, + }); + return await this.recordPublishResult(claimed.jobId, publishResult, { + publicByteSize: this.computePublicByteSize(prepared.publishOptions.quads), + }); + } catch (error) { + const failedFromState: LiftJobState = this.isKnowledgeAssetPublishPreconditionFailure(error) + ? 'validated' + : 'broadcast'; + return await this.recordExecutionFailure(claimed.jobId, failedFromState, error); + } + } + + private async findActiveKnowledgeAssetVmPublishJob( + request: KnowledgeAssetVmPublishRequest, + ): Promise<{ job: LiftJob; compatible: boolean } | null> { + const jobs = await this.list(); + for (const job of jobs) { + if (job.status === 'finalized') continue; + if (job.status === 'failed' && job.failure?.retryable !== true) continue; + if (job.request.jobType !== 'knowledge-asset-vm-publish') continue; + const publish = job.request.knowledgeAssetVmPublish; + if (!publish) continue; + const sameName = publish.contextGraphId === request.contextGraphId + && publish.name === request.name + && (publish.subGraphName ?? '') === (request.subGraphName ?? ''); + if (!sameName) continue; + return { job, compatible: publish.intentKey === request.intentKey }; + } + return null; + } + + private isKnowledgeAssetPublishPreconditionFailure(error: unknown): boolean { + const anyError = error as { code?: unknown; message?: unknown }; + if ( + anyError?.code === 'PUBLISH_NOT_FULL_SHARE' || + anyError?.code === 'PUBLISH_INTENT_STALE' || + anyError?.code === 'CG_NOT_REGISTERED' + ) { + return true; + } + const message = String(anyError?.message ?? error); + return /is not finalized/i.test(message) + || /No quads in shared memory/i.test(message) + || /has no private payload/i.test(message) + || /not a complete full share/i.test(message) + || /cannot recover .*reservedKaId/i.test(message) + || /seal binds/i.test(message); + } + async recordPublishResult( jobId: string, publishResult: PublishResult, @@ -424,7 +564,11 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { continue; } - if ((job.status === 'broadcast' || job.status === 'included') && this.chainRecoveryResolver) { + if ( + (job.status === 'broadcast' || job.status === 'included') && + this.chainRecoveryResolver && + job.request.jobType !== 'knowledge-asset-vm-publish' + ) { const resolved = await this.chainRecoveryResolver(job); if (resolved) { await this.releaseWalletLockForJob(job); @@ -442,6 +586,15 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { continue; } + if (job.status === 'included' && job.request.jobType === 'knowledge-asset-vm-publish') { + if (this.hasInconclusiveRecoveryTimedOut(job)) { + await this.releaseWalletLockForJob(job); + await this.writeJob(this.failKnowledgeAssetInconclusiveRecovery(job)); + recovered += 1; + } + continue; + } + if (job.status === 'broadcast') { await this.releaseWalletLockForJob(job); await this.writeJob(this.resetJobToAccepted(job, 'reset_to_accepted', 'broadcast', getRecoveryTxHash(job))); @@ -454,6 +607,7 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { if (this.chainRecoveryResolver) { const retryRecoveryJobs = (await this.list({ status: 'failed' })) .filter(isFailedJob) + .filter((job) => job.request.jobType !== 'knowledge-asset-vm-publish') .filter((job) => job.failure.resolution === 'retry_recovery' && 'broadcast' in job && job.broadcast); for (const job of retryRecoveryJobs) { @@ -972,6 +1126,20 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { return this.mergeJob(job, 'failed', { failure: failure as any }); } + private failKnowledgeAssetInconclusiveRecovery(job: LiftJobIncluded): LiftJob { + const failure = createLiftJobFailureMetadata({ + failedFromState: job.status, + code: 'recovery_state_inconsistent', + message: + `Named knowledge asset VM publish job ${job.jobId} reached included state, but generic chain ` + + `recovery cannot safely perform lifecycle finalization for this job type. Inspect the on-chain ` + + `transaction and re-run the named lifecycle publish if needed.`, + errorPayloadRef: `urn:dkg:publisher:error:${job.jobId}:ka-recovery-inconclusive`, + }); + + return this.mergeJob(job, 'failed', { failure: failure as any }); + } + private async finalizeNoopPublish(jobId: string): Promise { const current = await this.getRequiredJob(jobId); await this.assertActiveClaimLock(current); @@ -989,6 +1157,7 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { private async promoteFinalizedPrivateStaging(job: LiftJob): Promise { if (job.status !== 'finalized' || !job.validation) return; + if (job.request.jobType === 'knowledge-asset-vm-publish') return; const privateStore = new PrivateContentStore(this.store, this.graphManager); for (const sourceRoot of job.request.roots) { diff --git a/packages/publisher/src/async-lift-publisher-types.ts b/packages/publisher/src/async-lift-publisher-types.ts index 1fcf184967..7cf1b124f4 100644 --- a/packages/publisher/src/async-lift-publisher-types.ts +++ b/packages/publisher/src/async-lift-publisher-types.ts @@ -1,11 +1,25 @@ -import type { LiftJob, LiftJobBroadcast, LiftJobFinalizationMetadata, LiftJobIncluded, LiftJobInclusionMetadata, LiftJobState, LiftRequest } from './lift-job.js'; +import type { KnowledgeAssetVmPublishRequest, LiftJob, LiftJobBroadcast, LiftJobFinalizationMetadata, LiftJobIncluded, LiftJobInclusionMetadata, LiftJobState, LiftJobValidationMetadata, LiftRequest } from './lift-job.js'; +import type { DKGPublisher } from './dkg-publisher.js'; import type { PublishOptions, PublishResult } from './publisher.js'; import type { AsyncLiftPublishFailureInput } from './async-lift-publish-result.js'; import type { AsyncPreparedPublishPayload, LiftResolvedPublishSlice } from './async-lift-publish-options.js'; import type { WorkspacePublicSnapshotStore } from './workspace-snapshot-store.js'; +export class AsyncLiftJobConflictError extends Error { + readonly code = 'ASYNC_LIFT_JOB_CONFLICT'; + + constructor( + message: string, + readonly existingJobId: string, + ) { + super(message); + this.name = 'AsyncLiftJobConflictError'; + } +} + export interface AsyncLiftPublisher { lift(request: LiftRequest): Promise; + enqueueKnowledgeAssetVmPublish(request: KnowledgeAssetVmPublishRequest): Promise; claimNext(walletId: string): Promise; update(jobId: string, status: LiftJobState, data?: Partial): Promise; getStatus(jobId: string): Promise; @@ -33,6 +47,16 @@ export interface AsyncLiftPublishExecutionInput { readonly publishOptions: PublishOptions; } +export interface AsyncKnowledgeAssetVmPublishExecutionInput { + readonly walletId: string; + readonly request: KnowledgeAssetVmPublishRequest; + readonly liftRequest: LiftRequest; + readonly validation: LiftJobValidationMetadata; + readonly resolved: LiftResolvedPublishSlice; + readonly publishOptions: PublishOptions; + readonly publisher?: DKGPublisher; +} + export type AsyncLiftPublisherRecoveryResolver = ( job: LiftJobBroadcast | LiftJobIncluded, ) => Promise; @@ -45,6 +69,7 @@ export interface AsyncLiftPublisherConfig { idGenerator?: () => string; chainRecoveryResolver?: AsyncLiftPublisherRecoveryResolver; publishExecutor?: (input: AsyncLiftPublishExecutionInput) => Promise; + knowledgeAssetVmPublishExecutor?: (input: AsyncKnowledgeAssetVmPublishExecutionInput) => Promise; resolvedSliceOverrides?: Partial; publicSnapshotStore?: WorkspacePublicSnapshotStore; } diff --git a/packages/publisher/src/async-lift-publisher-utils.ts b/packages/publisher/src/async-lift-publisher-utils.ts index 61e657c7a9..5b802d7a80 100644 --- a/packages/publisher/src/async-lift-publisher-utils.ts +++ b/packages/publisher/src/async-lift-publisher-utils.ts @@ -1,5 +1,10 @@ import type { QueryResult } from '@origintrail-official/dkg-storage'; -import type { LiftJob, LiftJobHex } from './lift-job.js'; +import type { + KnowledgeAssetVmPublishRequest, + LiftJob, + LiftJobHex, + LiftRequest, +} from './lift-job.js'; export { CONTROL_CLAIM_TOKEN, CONTROL_LOCKED_JOB, @@ -46,3 +51,31 @@ export function getRecoveryTxHash(job: LiftJob): LiftJobHex | undefined { export function isFailedJob(job: LiftJob): job is PersistedFailedJob { return job.status === 'failed' && 'failure' in job; } + +export function createKnowledgeAssetVmPublishLiftRequest( + request: KnowledgeAssetVmPublishRequest, +): LiftRequest { + const subGraphPart = request.subGraphName ? `:${request.subGraphName}` : ''; + const operationKey = `${request.contextGraphId}:${request.name}${subGraphPart}:${request.shareOperationId}`; + return { + jobType: 'knowledge-asset-vm-publish', + knowledgeAssetVmPublish: request, + swmId: request.shareOperationId, + shareOperationId: request.shareOperationId, + roots: request.roots, + contextGraphId: request.contextGraphId, + namespace: 'knowledge-assets', + scope: 'vm-publish', + transitionType: 'CREATE', + authority: { + type: 'owner', + proofRef: `urn:dkg:knowledge-assets:${operationKey}:vm-publish`, + }, + ...(request.subGraphName ? { subGraphName: request.subGraphName } : {}), + ...(request.publishEpochs !== undefined ? { publishEpochs: request.publishEpochs } : {}), + ...(request.publisherNodeIdentityIdOverride !== undefined + ? { publisherNodeIdentityIdOverride: request.publisherNodeIdentityIdOverride } + : {}), + seal: request.seal, + }; +} diff --git a/packages/publisher/src/async-lift-publisher.ts b/packages/publisher/src/async-lift-publisher.ts index 76fc0380ad..d66e29e2e2 100644 --- a/packages/publisher/src/async-lift-publisher.ts +++ b/packages/publisher/src/async-lift-publisher.ts @@ -1,8 +1,10 @@ export type { + AsyncKnowledgeAssetVmPublishExecutionInput, AsyncLiftPublisher, AsyncLiftPublisherConfig, AsyncLiftPublishExecutionInput, AsyncLiftPublisherRecoveryResolver, AsyncLiftPublisherRecoveryResult, } from './async-lift-publisher-types.js'; +export { AsyncLiftJobConflictError } from './async-lift-publisher-types.js'; export { TripleStoreAsyncLiftPublisher } from './async-lift-publisher-impl.js'; diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index c1701798c7..e068bb5cb1 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -84,6 +84,7 @@ const WORKSPACE_OWNER_PREDICATE = 'http://dkg.io/ontology/workspaceOwner'; // finalize(layer:"swm") so a subset share — which also stamps dkg:rootEntity // member rows — cannot be sealed-in-SWM and published as a partial asset. const SWM_SHARE_COMPLETE_PRED = 'http://dkg.io/ontology/swmShareComplete'; +const SHARE_OPERATION_ID_PRED = 'http://dkg.io/ontology/shareOperationId'; async function listGraphFamily(store: TripleStore, rootGraph: string): Promise { const graphs = await listGraphsByPrefix(store, `${rootGraph}/`); @@ -1842,25 +1843,15 @@ export class DKGPublisher implements Publisher { } // SWM cleanup: ALWAYS remove published triples from SWM after chain confirmation. - // Published triples must not linger in SWM — they live in LTM now. + // Published triples must not linger in SWM; they live in LTM now. // clearSharedMemoryAfter controls only whether the REMAINING unpublished triples are also cleared. if (publishResult.status === 'confirmed') { - const swmOwnershipKey = options?.subGraphName ? `${contextGraphId}\0${options.subGraphName}` : contextGraphId; const kaMap = skolemizeByEntity(quads); await this.clearPublishedSwmRoots(contextGraphId, [...kaMap.keys()], options?.subGraphName, ctx); // If clearSharedMemoryAfter is explicitly true, also clear any remaining unpublished content. // Default is false: unpublished entities stay in SWM for future publishes. if (options?.clearSharedMemoryAfter === true) { - const swmMetaGraph = this.graphManager.sharedMemoryMetaUri(contextGraphId, options?.subGraphName); - let remainingCount = 0; - for (const g of await this.swmGraphsUnder(swmGraph)) { - remainingCount += await this.store.deleteByPattern({ graph: g }); - } - const remainingMetaCount = await this.store.deleteByPattern({ graph: swmMetaGraph }); - if (remainingCount > 0 || remainingMetaCount > 0) { - this.log.info(ctx, `Cleared remaining SWM content: ${remainingCount} triples, ${remainingMetaCount} meta`); - } - this.sharedMemoryOwnedEntities.delete(swmOwnershipKey); + await this.clearRemainingSharedMemory(contextGraphId, options?.subGraphName, ctx); } } @@ -2864,8 +2855,8 @@ export class DKGPublisher implements Publisher { // expectedMerkleRoot, wrong-signer recovery) propagate up as // hard errors instead of being downgraded to a "tentative" // result with a `On-chain tx failed` log line. These are - // protocol-correctness violations, not transient chain issues — - // /api/shared-memory/publish callers must see a 4xx for a + // protocol-correctness violations, not transient chain issues. + // Named lifecycle publish callers must see a 4xx for a // broken seal, not a 200 OK with `status: tentative` and // `kaId: 0` (which the daemon previously had to special-case). // @@ -2985,7 +2976,7 @@ export class DKGPublisher implements Publisher { 'Publish rejected: on-chain publish requires precomputedAttestation. ' + 'RFC-001 §9.x — every published assertion must be sealed at finalize-time. ' + 'Call agent.assertion.finalize(...) first; the daemon\'s assertion-name-aware ' + - '/api/shared-memory/publish path resolves the seal automatically.', + '/api/knowledge-assets/:name/vm/publish route resolves the seal automatically.', ); } const effectiveAuthorAddress = options.precomputedAttestation.authorAddress; @@ -3446,12 +3437,6 @@ export class DKGPublisher implements Publisher { } async update(kaId: bigint, options: PublishOptions): Promise { - if (options.subGraphName) { - throw new Error( - 'Updating sub-graph KCs is not yet supported. The update path does not resolve sub-graph data/private graphs. ' + - 'Publish a new KC instead, or remove and recreate the sub-graph.', - ); - } const { contextGraphId, quads, privateQuads = [], operationCtx, onPhase } = options; // Round 12 Bug 34: `update()` is a Bucket A public write entry // point (accepts user-authored quads) that Round 9 missed. Apply @@ -3468,6 +3453,7 @@ export class DKGPublisher implements Publisher { rejectOversizedRdfLiterals(quads, 'update.quads'); if (privateQuads.length > 0) rejectOversizedRdfLiterals(privateQuads, 'update.privateQuads'); const ctx: OperationContext = operationCtx ?? createOperationContext('publish'); + await this.ensureSubGraphRegistered(contextGraphId, options.subGraphName); let publisherContextGraphId: bigint | undefined; try { const parsed = BigInt(options.publishContextGraphId ?? contextGraphId); @@ -3516,6 +3502,7 @@ export class DKGPublisher implements Publisher { MemoryLayer.VerifiableMemory, '0x' + (kaId >> 96n).toString(16).padStart(40, '0'), kaId & ((1n << 96n) - 1n), + options.subGraphName, ); onPhase?.('prepare', 'start'); @@ -3599,7 +3586,7 @@ export class DKGPublisher implements Publisher { const DKG_ONT = 'http://dkg.io/ontology/'; const priorRootEntities = new Set(); try { - const labelMetaForPriors = this.graphManager.metaGraphUri(contextGraphId); + const labelMetaForPriors = contextGraphMetaUri(contextGraphId, options.subGraphName); let ualForPriors = await resolveUalByBatchId(this.store, labelMetaForPriors, kaId); if (!ualForPriors) { // Same local-only deterministic-UAL fallback as the restate @@ -3673,7 +3660,7 @@ export class DKGPublisher implements Publisher { // when `resolveKaUal` would throw. For the on-chain path we still // let `resolveKaUal` throw (failing the update) — which is the // correct behavior when chain-truth is required but unavailable. - const labelMeta = this.graphManager.metaGraphUri(contextGraphId); + const labelMeta = contextGraphMetaUri(contextGraphId, options.subGraphName); let ualForRestate = await resolveUalByBatchId(this.store, labelMeta, kaId); if (!ualForRestate) { if (localOnlyUpdate && publisherAddress) { @@ -5212,6 +5199,25 @@ export class DKGPublisher implements Publisher { } } + async clearRemainingSharedMemory( + contextGraphId: string, + subGraphName: string | undefined, + ctx: OperationContext, + ): Promise { + const swmGraph = this.graphManager.sharedMemoryUri(contextGraphId, subGraphName); + const swmMetaGraph = this.graphManager.sharedMemoryMetaUri(contextGraphId, subGraphName); + const swmOwnershipKey = subGraphName ? `${contextGraphId}\0${subGraphName}` : contextGraphId; + let remainingCount = 0; + for (const graph of await this.swmGraphsUnder(swmGraph)) { + remainingCount += await this.store.deleteByPattern({ graph }); + } + const remainingMetaCount = await this.store.deleteByPattern({ graph: swmMetaGraph }); + if (remainingCount > 0 || remainingMetaCount > 0) { + this.log.info(ctx, `Cleared remaining SWM content: ${remainingCount} triples, ${remainingMetaCount} meta`); + } + this.sharedMemoryOwnedEntities.delete(swmOwnershipKey); + } + async assertionCreate( contextGraphId: string, name: string, @@ -5584,7 +5590,7 @@ export class DKGPublisher implements Publisher { */ confirmBeforeCommit?: (message: Uint8Array) => Promise<{ applied: boolean; rejected?: boolean }>; }, - ): Promise<{ promotedCount: number; gossipMessage?: Uint8Array; promotedAllRoots: boolean }> { + ): Promise<{ promotedCount: number; gossipMessage?: Uint8Array; promotedAllRoots: boolean; shareOperationId?: string }> { await this.ensureSubGraphRegistered(contextGraphId, opts?.subGraphName); const graphUri = await this.wmGraphUri(contextGraphId, agentAddress, name, opts?.subGraphName); const swmGraphUri = await this.swmGraphUri(contextGraphId, agentAddress, name, opts?.subGraphName); @@ -5600,11 +5606,21 @@ export class DKGPublisher implements Publisher { // (Member-row REPLACE stays at the success path — it only matters when quads // are actually promoted; the MARKER is the cross-cutting invariant.) const promotingAllEntities = !opts?.entities || opts.entities === 'all'; + const lifecycleSubject = assertionLifecycleUri(contextGraphId, agentAddress, name, opts?.subGraphName); + const promoteMetaGraph = contextGraphMetaUri(contextGraphId); + const clearCurrentShareOperationId = async (): Promise => { + await this.store.deleteByPattern({ + graph: promoteMetaGraph, + subject: lifecycleSubject, + predicate: SHARE_OPERATION_ID_PRED, + }); + }; const maintainMarker = async (isFullCompletePromote: boolean): Promise => { if (isFullCompletePromote) { await this.markSwmShareComplete(contextGraphId, name, agentAddress, opts?.subGraphName); } else { await this.clearSwmShareComplete(contextGraphId, name, agentAddress, opts?.subGraphName); + await clearCurrentShareOperationId(); } }; @@ -5634,12 +5650,11 @@ export class DKGPublisher implements Publisher { // `assertionCreate` are not consulted: they fire for empty-write // flows where promoting nothing is legitimate. await this.assertAssertionDataPersisted(contextGraphId, graphUri); - // No roots to promote ⇒ none were foreign-skipped. promotedAllRoots:true ⇒ - // isFull == promotingAllEntities (a subset of an empty draft still CLEARS a - // stale marker; a full empty share SETS it — harmless, finalize finds no - // members). Maintain the marker before returning (round 10). - await maintainMarker(promotingAllEntities); - return { promotedCount: 0, promotedAllRoots: true }; + // No roots to promote is a no-op, not a fresh full share. Clear any prior + // marker and current share intent so an empty WM retry cannot re-arm async + // VM publish with an old shareOperationId. + await maintainMarker(false); + return { promotedCount: 0, promotedAllRoots: false }; } let quadsToPromote = result.quads; @@ -5734,15 +5749,12 @@ export class DKGPublisher implements Publisher { ); } - // Nothing left after the reserved-subject / selective-entity filters ⇒ - // no roots were foreign-skipped. round 10 (reviewer 🔴): a SUBSET share whose - // selection matched ZERO current quads reaches here — it MUST clear a stale - // full-share marker (isFull == promotingAllEntities, which is false for a - // subset) before returning, else finalize(layer:"swm") passes its gate against - // the OLD SWM contents. + // Nothing left after reserved-subject or selective-entity filtering is also + // a no-op. It must clear stale full-share state rather than act like a fresh + // complete share. if (quadsToPromote.length === 0) { - await maintainMarker(promotingAllEntities); - return { promotedCount: 0, promotedAllRoots: true }; + await maintainMarker(false); + return { promotedCount: 0, promotedAllRoots: false }; } const operationId = `${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}`; @@ -5886,16 +5898,17 @@ export class DKGPublisher implements Publisher { // the SWM copy is missing part of the sealed set, so the seal exists but the // asset is NOT publish-ready (publishFromFinalizedAssertion would recompute a // different merkleRoot over the partial SWM slice and fail the seal guard). - // The agent's promote() threads this into `publishReady`. - const promotedAllRoots = skippedRoots.size === 0; + // The agent's promote() threads this into `publishReady`. A no-op with roots + // but no effective quads is not a complete share and must not stamp a fresh + // shareOperationId for async VM publish. + const promotedAllRoots = skippedRoots.size === 0 && effectiveQuads.length > 0; - if (effectiveRoots.length === 0) { - // Every requested root was foreign-skipped — nothing promoted, and the - // sealed set is definitively not fully present in SWM. promotedAllRoots is - // false here (skippedRoots.size > 0), so isFull is false ⇒ CLEAR the marker - // (round 10): a fully-foreign-skipped share is never a complete full share. - await maintainMarker(promotingAllEntities && promotedAllRoots); - return { promotedCount: 0, promotedAllRoots }; + if (effectiveRoots.length === 0 || effectiveQuads.length === 0) { + // Nothing is actually promoted, either because every root was skipped or + // because the remaining root set has no publishable quads. This is never a + // complete full share. + await maintainMarker(false); + return { promotedCount: 0, promotedAllRoots: false }; } // Delete-then-insert for existing SWM entities (upsert), matching @@ -5962,9 +5975,8 @@ export class DKGPublisher implements Publisher { // `promotingAllEntities` is hoisted to the top of the method (round 10) so the // early-return marker maintenance can use it; reuse it here. const isFullCompletePromote = promotingAllEntities && promotedAllRoots; + await clearCurrentShareOperationId(); if (isFullCompletePromote) { - const lifecycleSubject = assertionLifecycleUri(contextGraphId, agentAddress, name, opts?.subGraphName); - const promoteMetaGraph = contextGraphMetaUri(contextGraphId); await this.store.deleteByPattern({ graph: promoteMetaGraph, subject: lifecycleSubject, predicate: DKG_ROOT_ENTITY_LEGACY }); await this.store.deleteByPattern({ graph: promoteMetaGraph, subject: lifecycleSubject, predicate: DKG_ENTITY }); } @@ -6036,7 +6048,7 @@ export class DKGPublisher implements Publisher { } } - return { promotedCount: swmQuads.length, gossipMessage, promotedAllRoots }; + return { promotedCount: swmQuads.length, gossipMessage, promotedAllRoots, shareOperationId: operationId }; } async assertionDiscard(contextGraphId: string, name: string, agentAddress: string, subGraphName?: string): Promise { diff --git a/packages/publisher/src/errors.ts b/packages/publisher/src/errors.ts index 17360f3d7e..b9a2eb0c96 100644 --- a/packages/publisher/src/errors.ts +++ b/packages/publisher/src/errors.ts @@ -153,7 +153,7 @@ export class MultiRootPublishNotAtomicError extends Error { readonly rootEntities: string[]; constructor(contextGraphId: string, rootEntities: readonly string[]) { super( - `V10 shared-memory publish is single-root only for this operation. ` + + `V10 VM publish from SWM is single-root only for this operation. ` + `Resolved ${rootEntities.length} root entities; select exactly one root or use a durable multi-publish flow.`, ); this.name = 'MultiRootPublishNotAtomicError'; diff --git a/packages/publisher/src/index.ts b/packages/publisher/src/index.ts index 4fb56c93ef..c77d2ef48c 100644 --- a/packages/publisher/src/index.ts +++ b/packages/publisher/src/index.ts @@ -118,6 +118,7 @@ export { type LiftJobChainRecoverableState, type LiftJobHex, type LiftJobBigInt, + type KnowledgeAssetVmPublishRequest, type LiftJobTimeoutMetadata, type LiftJobFailurePolicy, type LiftAuthorityProof, @@ -166,9 +167,11 @@ export { isTimeoutLiftJobFailure, } from './lift-job.js'; export { + AsyncLiftJobConflictError, TripleStoreAsyncLiftPublisher, type AsyncLiftPublisher, type AsyncLiftPublisherConfig, + type AsyncKnowledgeAssetVmPublishExecutionInput, type AsyncLiftPublishExecutionInput, type AsyncLiftPublisherRecoveryResult, type AsyncLiftPublisherRecoveryResolver, @@ -223,6 +226,7 @@ export { type AsyncLiftPublishSuccess, type AsyncLiftPublishFailureInput, } from './async-lift-publish-result.js'; +export { createKnowledgeAssetVmPublishLiftRequest } from './async-lift-publisher-utils.js'; export { SharedMemoryHandler, WorkspaceHandler } from './workspace-handler.js'; export { FileWorkspacePublicSnapshotStore, diff --git a/packages/publisher/src/lift-job-types.ts b/packages/publisher/src/lift-job-types.ts index 8ef7ff91d5..9a7222c18c 100644 --- a/packages/publisher/src/lift-job-types.ts +++ b/packages/publisher/src/lift-job-types.ts @@ -39,7 +39,32 @@ export interface LiftRequestAuthorSeal { readonly reservedKaId?: LiftJobBigInt; } +export interface KnowledgeAssetVmPublishRequest { + readonly contextGraphId: string; + readonly name: string; + readonly subGraphName?: string; + readonly shareOperationId: string; + readonly roots: readonly string[]; + /** Author seal captured with the queued SWM share snapshot. */ + readonly seal: LiftRequestAuthorSeal; + readonly sealChainId: LiftJobBigInt; + readonly sealKav10Address: LiftJobHex; + readonly sealFinalizedAtIso: string; + readonly sealMerkleRoot: LiftJobHex; + readonly intentKey: string; + readonly wmCurrentAssertion?: string; + readonly swmCurrentAssertion?: string; + readonly vmCurrentAssertion?: string; + readonly kaNumber?: string; + readonly reservedUal?: string; + readonly publishEpochs?: number; + readonly clearSharedMemoryAfter?: boolean; + readonly publisherNodeIdentityIdOverride?: LiftJobBigInt; +} + export interface LiftRequest { + readonly jobType?: 'lift' | 'knowledge-asset-vm-publish'; + readonly knowledgeAssetVmPublish?: KnowledgeAssetVmPublishRequest; readonly swmId: string; readonly shareOperationId: string; readonly roots: readonly string[]; @@ -63,6 +88,8 @@ export interface LiftRequest { } export const LIFT_REQUEST_IMMUTABLE_FIELDS = [ + 'jobType', + 'knowledgeAssetVmPublish', 'swmId', 'shareOperationId', 'roots', diff --git a/packages/publisher/src/metadata.ts b/packages/publisher/src/metadata.ts index 56b9155e50..24ae6f135b 100644 --- a/packages/publisher/src/metadata.ts +++ b/packages/publisher/src/metadata.ts @@ -1465,6 +1465,7 @@ export function generateAssertionPromotedMetadata(meta: AssertionPromotedMeta, o mq(subject, `${DKG}state`, lit('promoted'), metaGraph), mq(subject, `${DKG}memoryLayer`, lit(MemoryLayer.SharedWorkingMemory), metaGraph), mq(subject, `${DKG}assertionGraph`, swmGraphUri, metaGraph), + mq(subject, `${DKG}shareOperationId`, lit(meta.shareOperationId), metaGraph), ]; if (provenanceEvents) { ins.push( diff --git a/packages/publisher/test/async-lift-publisher.test.ts b/packages/publisher/test/async-lift-publisher.test.ts index 227a905392..9ae27c09ea 100644 --- a/packages/publisher/test/async-lift-publisher.test.ts +++ b/packages/publisher/test/async-lift-publisher.test.ts @@ -95,6 +95,37 @@ describe('TripleStoreAsyncLiftPublisher', () => { authority: { type: 'owner', proofRef: 'proof:owner:1' }, }); + const kaVmPublishRequest = (overrides: Partial[0]> = {}) => { + const authorAddress = '0x1111111111111111111111111111111111111111'; + const kaNumber = 7n; + const base = { + contextGraphId: 'music-social', + name: 'albums', + shareOperationId: 'share-op-1', + roots: ['urn:album:one', 'urn:album:two'], + seal: { + merkleRoot: (`0x${'12'.repeat(32)}`) as `0x${string}`, + authorAddress: authorAddress as `0x${string}`, + signature: { + r: (`0x${'34'.repeat(32)}`) as `0x${string}`, + vs: (`0x${'56'.repeat(32)}`) as `0x${string}`, + }, + schemeVersion: 1, + reservedKaId: ((BigInt(authorAddress) << 96n) | kaNumber).toString() as `${bigint}`, + }, + sealChainId: '31337' as `${bigint}`, + sealKav10Address: '0x2222222222222222222222222222222222222222' as `0x${string}`, + sealFinalizedAtIso: '2026-01-01T00:00:00.000Z', + sealMerkleRoot: (`0x${'12'.repeat(32)}`) as `0x${string}`, + intentKey: 'sha256:' + 'ab'.repeat(32), + wmCurrentAssertion: '12'.repeat(32), + swmCurrentAssertion: '12'.repeat(32), + kaNumber: kaNumber.toString(), + reservedUal: 'did:dkg:31337/0x1111111111111111111111111111111111111111/7', + }; + return { ...base, ...overrides }; + }; + beforeEach(() => { store = new OxigraphStore(); now = 1_000; @@ -116,6 +147,25 @@ describe('TripleStoreAsyncLiftPublisher', () => { }); } + function confirmedPublishResult() { + return { + kaId: 11n, + ual: 'did:dkg:mock:31337/0xdef/11', + merkleRoot: new Uint8Array([0xde, 0xf0]), + kaManifest: [], + status: 'confirmed' as const, + onChainResult: { + batchId: 11n, + startKAId: 11n, + endKAId: 11n, + txHash: '0xdef', + blockNumber: 77, + blockTimestamp: 1700000077, + publisherAddress: '0x2222222222222222222222222222222222222222', + }, + }; + } + async function readLockExpiresAt(walletId: string): Promise { const result = await store.query(`SELECT ?expiresAt WHERE { GRAPH <${DEFAULT_WALLET_LOCK_GRAPH_URI}> { @@ -152,7 +202,159 @@ describe('TripleStoreAsyncLiftPublisher', () => { expect(job?.retries.maxRetries).toBe(10); }); - it('exposes the renamed shared-memory publisher contract', async () => { + it('enqueues named knowledge asset VM publish jobs', async () => { + const publisher = createPublisher(); + + const jobId = await publisher.enqueueKnowledgeAssetVmPublish(kaVmPublishRequest({ + subGraphName: 'research', + publishEpochs: 3, + clearSharedMemoryAfter: false, + publisherNodeIdentityIdOverride: '42', + })); + const job = await publisher.getStatus(jobId); + + expect(jobId).toBe('job-1'); + expect(job?.status).toBe('accepted'); + expect(job?.request.jobType).toBe('knowledge-asset-vm-publish'); + expect(job?.request.knowledgeAssetVmPublish).toEqual({ + contextGraphId: 'music-social', + name: 'albums', + subGraphName: 'research', + shareOperationId: 'share-op-1', + roots: ['urn:album:one', 'urn:album:two'], + seal: { + merkleRoot: `0x${'12'.repeat(32)}`, + authorAddress: '0x1111111111111111111111111111111111111111', + signature: { + r: `0x${'34'.repeat(32)}`, + vs: `0x${'56'.repeat(32)}`, + }, + schemeVersion: 1, + reservedKaId: ((BigInt('0x1111111111111111111111111111111111111111') << 96n) | 7n).toString(), + }, + sealChainId: '31337', + sealKav10Address: '0x2222222222222222222222222222222222222222', + sealFinalizedAtIso: '2026-01-01T00:00:00.000Z', + sealMerkleRoot: `0x${'12'.repeat(32)}`, + intentKey: 'sha256:' + 'ab'.repeat(32), + wmCurrentAssertion: '12'.repeat(32), + swmCurrentAssertion: '12'.repeat(32), + kaNumber: '7', + reservedUal: 'did:dkg:31337/0x1111111111111111111111111111111111111111/7', + publishEpochs: 3, + clearSharedMemoryAfter: false, + publisherNodeIdentityIdOverride: '42', + }); + expect(job?.request.scope).toBe('vm-publish'); + expect(job?.request.shareOperationId).toBe('share-op-1'); + expect(job?.request.roots).toEqual(['urn:album:one', 'urn:album:two']); + }); + + it('rejects duplicate active knowledge asset VM publish jobs', async () => { + const publisher = createPublisher(); + + await publisher.enqueueKnowledgeAssetVmPublish(kaVmPublishRequest({ + subGraphName: 'research', + })); + + await expect( + publisher.enqueueKnowledgeAssetVmPublish(kaVmPublishRequest({ + subGraphName: 'research', + })), + ).resolves.toBe('job-1'); + + await expect( + publisher.enqueueKnowledgeAssetVmPublish(kaVmPublishRequest({ + subGraphName: 'research', + shareOperationId: 'share-op-2', + intentKey: 'sha256:' + 'cd'.repeat(32), + })), + ).rejects.toMatchObject({ + name: 'AsyncLiftJobConflictError', + code: 'ASYNC_LIFT_JOB_CONFLICT', + existingJobId: 'job-1', + }); + }); + + it('processes knowledge asset VM publish jobs through the lifecycle executor', async () => { + const calls: unknown[] = []; + const publisher = createPublisher({ + config: { + knowledgeAssetVmPublishExecutor: async (input) => { + calls.push(input); + return confirmedPublishResult(); + }, + }, + }); + const publisherContract: Publisher = makeTestPublisher({ + store, + chain: createEVMAdapter(HARDHAT_KEYS.CORE_OP), + eventBus: new TypedEventBus(), + keypair: await generateEd25519Keypair(), + publisherPrivateKey: HARDHAT_KEYS.CORE_OP, + publisherNodeIdentityId: BigInt(getSharedContext().coreProfileId), + }); + const share = await publisherContract.share('music-social', [ + { subject: 'urn:album:one', predicate: 'http://schema.org/name', object: '"One"', graph: '' }, + { subject: 'urn:album:two', predicate: 'http://schema.org/name', object: '"Two"', graph: '' }, + ], { publisherPeerId: 'peer-1' }); + + const jobId = await publisher.enqueueKnowledgeAssetVmPublish(kaVmPublishRequest({ + shareOperationId: share.shareOperationId, + clearSharedMemoryAfter: true, + })); + const processed = await publisher.processNext('wallet-1'); + + expect(processed?.jobId).toBe(jobId); + expect(processed?.status).toBe('finalized'); + expect(processed?.finalization?.ual).toBe('did:dkg:mock:31337/0xdef/11'); + expect(calls).toHaveLength(1); + expect(calls[0]).toMatchObject({ + walletId: 'wallet-1', + request: { + contextGraphId: 'music-social', + name: 'albums', + shareOperationId: share.shareOperationId, + roots: ['urn:album:one', 'urn:album:two'], + seal: { + merkleRoot: `0x${'12'.repeat(32)}`, + authorAddress: '0x1111111111111111111111111111111111111111', + signature: { + r: `0x${'34'.repeat(32)}`, + vs: `0x${'56'.repeat(32)}`, + }, + schemeVersion: 1, + }, + sealChainId: '31337', + sealKav10Address: '0x2222222222222222222222222222222222222222', + sealFinalizedAtIso: '2026-01-01T00:00:00.000Z', + sealMerkleRoot: `0x${'12'.repeat(32)}`, + intentKey: 'sha256:' + 'ab'.repeat(32), + wmCurrentAssertion: '12'.repeat(32), + swmCurrentAssertion: '12'.repeat(32), + kaNumber: '7', + reservedUal: 'did:dkg:31337/0x1111111111111111111111111111111111111111/7', + clearSharedMemoryAfter: true, + }, + validation: { + canonicalRoots: ['urn:album:one', 'urn:album:two'], + swmQuadCount: 2, + transitionType: 'CREATE', + }, + resolved: { + publisherPeerId: 'peer-1', + }, + publishOptions: { + publisherPeerId: 'peer-1', + quads: [ + { subject: 'urn:album:one', predicate: 'http://schema.org/name', object: '"One"', graph: '' }, + { subject: 'urn:album:two', predicate: 'http://schema.org/name', object: '"Two"', graph: '' }, + ], + }, + }); + }); + + it('exposes the SWM share operation contract', async () => { const publisherContract: Publisher = makeTestPublisher({ store, chain: createEVMAdapter(HARDHAT_KEYS.CORE_OP), @@ -1242,6 +1444,44 @@ describe('TripleStoreAsyncLiftPublisher', () => { expect(job?.timestamps?.failedAt).toBeUndefined(); }); + it('fails included knowledge asset VM publish jobs as clearable lifecycle recovery failures', async () => { + const publisher = createPublisher({ + config: { + recoveryLookupTimeoutMs: 50, + }, + }); + + const jobId = await publisher.enqueueKnowledgeAssetVmPublish(kaVmPublishRequest()); + await publisher.claimNext('wallet-1'); + await publisher.update(jobId, 'validated', { + validation: { + canonicalRoots: ['urn:album:one', 'urn:album:two'], + canonicalRootMap: {}, + swmQuadCount: 2, + authorityProofRef: 'knowledge-asset-lifecycle', + transitionType: 'CREATE', + }, + }); + await publisher.update(jobId, 'broadcast', { + broadcast: { txHash: '0xkavm', walletId: 'wallet-1' }, + }); + await publisher.update(jobId, 'included', { + inclusion: { txHash: '0xkavm', blockNumber: 99 }, + }); + + now += 100; + const recovered = await publisher.recover(); + const job = await publisher.getStatus(jobId); + + expect(recovered).toBe(1); + expect(job?.status).toBe('failed'); + expect(job?.failure?.code).toBe('recovery_state_inconsistent'); + expect(job?.failure?.resolution).toBe('fail_job'); + + expect(await publisher.clear('failed')).toBe(1); + expect(await publisher.getStatus(jobId)).toBeNull(); + }); + it('supports pause, resume, cancel, retry, and clear', async () => { const publisher = createPublisher(); const cancelId = await publisher.lift(request()); diff --git a/packages/publisher/test/lift-job-types.test.ts b/packages/publisher/test/lift-job-types.test.ts index 68e6a87615..835d061969 100644 --- a/packages/publisher/test/lift-job-types.test.ts +++ b/packages/publisher/test/lift-job-types.test.ts @@ -24,6 +24,8 @@ describe('LiftJob request and record types', () => { expect(LIFT_TRANSITION_TYPES).toEqual(['CREATE', 'MUTATE', 'REVOKE']); expect(LIFT_AUTHORITY_TYPES).toEqual(['owner', 'multisig', 'quorum', 'capability']); expect(LIFT_REQUEST_IMMUTABLE_FIELDS).toEqual([ + 'jobType', + 'knowledgeAssetVmPublish', 'swmId', 'shareOperationId', 'roots', diff --git a/packages/publisher/test/metadata.test.ts b/packages/publisher/test/metadata.test.ts index d9fbee47dd..6e401b07f2 100644 --- a/packages/publisher/test/metadata.test.ts +++ b/packages/publisher/test/metadata.test.ts @@ -493,7 +493,7 @@ describe('RFC ka-metadata-trim P3.3 — provenanceEvents gate', () => { } }); - it('promoted: provenanceEvents=false keeps the subject re-stamp + member-entity rows, skips the event', () => { + it('promoted: provenanceEvents=false keeps the subject re-stamp + share operation id + member rows, skips the event', () => { const meta = { contextGraphId: CONTEXT_GRAPH, agentAddress: createdMeta.agentAddress, @@ -506,7 +506,11 @@ describe('RFC ka-metadata-trim P3.3 — provenanceEvents gate', () => { }; const { insert, delete: del } = generateAssertionPromotedMetadata(meta, { provenanceEvents: false }); expect(insert.find(q => q.subject.includes('/event/'))).toBeUndefined(); - expect(insert.find(q => q.predicate === `${DKG}shareOperationId`)).toBeUndefined(); + expect(insert).toContainEqual(expect.objectContaining({ + subject: assertionLifecycleUri(CONTEXT_GRAPH, createdMeta.agentAddress, 'lite-mode-ka'), + predicate: `${DKG}shareOperationId`, + object: '"op-1"', + })); const subjectPreds = insert.map(q => q.predicate); expect(subjectPreds).toContain(`${DKG}state`); expect(subjectPreds).toContain(`${DKG}memoryLayer`); diff --git a/packages/query/src/dkg-query-engine.ts b/packages/query/src/dkg-query-engine.ts index ca08b2462e..fba1b46b90 100644 --- a/packages/query/src/dkg-query-engine.ts +++ b/packages/query/src/dkg-query-engine.ts @@ -183,9 +183,9 @@ export function resolveViewGraphs( // already part of VM pre-PR2. // // Dropping the root graph here was a behavioural break for - // existing callers (memory-search flows, the daemon's - // `/api/query?view=verifiable-memory` route after - // `/api/shared-memory/publish`): a successful publish would + // existing callers (memory-search flows and the daemon's + // `/api/query?view=verifiable-memory` route after named VM publish): + // a successful publish would // silently disappear from VM until a separate `verify()` wrote // into `_verifiable_memory/{vmId}`. Restoring the root graph keeps // confirmed publisher-side data immediately queryable via VM diff --git a/scripts/_devnet-full-sweep.sh b/scripts/_devnet-full-sweep.sh index cec19d7b6f..bfbc7b8f99 100755 --- a/scripts/_devnet-full-sweep.sh +++ b/scripts/_devnet-full-sweep.sh @@ -20,7 +20,6 @@ mkdir -p "$RESULTS_DIR" SCRIPTS=( "rfc38-curator-offline-midbatch" "rfc38-revocation" - "rfc38-prereg-bytecap-stress" "rfc38-unclean-restart" "publish" "sharing" diff --git a/scripts/devnet-probe-ack-rejection-reasons.sh b/scripts/devnet-probe-ack-rejection-reasons.sh index dde66b01f8..36190cc719 100755 --- a/scripts/devnet-probe-ack-rejection-reasons.sh +++ b/scripts/devnet-probe-ack-rejection-reasons.sh @@ -105,7 +105,7 @@ done # --- 3. SWM-based publish trigger: confirm new ACK traffic + zero structured rejections --- # rc.12 deprecated the synchronous /api/publish endpoint (404 in -# this branch). The async publish path is /api/publisher/enqueue +# this branch). The async publish path is /api/knowledge-assets//vm/publish-async # which requires a pre-staged shareOperationId + authority proof. # Reproducing that surface for a probe is brittle; instead we let the # devnet-full-sweep / rfc38-all suites do the publishing and use the diff --git a/scripts/devnet-publish-helpers.sh b/scripts/devnet-publish-helpers.sh index 73e93f0428..96ca497c85 100755 --- a/scripts/devnet-publish-helpers.sh +++ b/scripts/devnet-publish-helpers.sh @@ -2,10 +2,10 @@ # # Shared helpers for devnet test scripts — rc.12+ single-root sync publish. # -# Since #925, POST /api/shared-memory/publish rejects `selection: "all"` -# when multiple root entities exist (MULTI_ROOT_PUBLISH_NOT_ATOMIC). Scripts -# that previously published whole SWM graphs in one call must publish one root -# at a time. +# Devnet scripts publish through named knowledge assets. Multi-root payloads are +# split into one named KA per root by default so each VM publish remains +# independently verifiable. Set DEVNET_PUBLISH_PRESERVE_BATCH=1 for tests that +# intentionally need one KC containing the full multi-root payload. # # Usage: source this file AFTER defining `api_call NODE METHOD PATH [DATA]`. # @@ -22,6 +22,8 @@ DEVNET_PUBLISH_STATE_FILE="${DEVNET_PUBLISH_STATE_FILE:-${DEVNET_DIR:-/tmp}/.devnet-publish-state-$$.json}" DEVNET_PUBLISH_ALL_RESPONSES='[]' DEVNET_PUBLISH_ROOT_ENTITIES='[]' +DEVNET_PUBLISH_ASSET_NAMES='[]' +DEVNET_PUBLISH_PENDING_ASSETS='[]' # Node that performed the last publish. KC metadata (merkleRoot, KCS records) # must be read from this node — late-joining/non-curator verifier peers may not # have materialized the batch yet, which would race verify-batch. @@ -37,6 +39,142 @@ devnet_json_field() { " } +_devnet_append_pending_assets() { + local new_assets="$1" + devnet_publish_load_state + local merged names + merged=$(OLD="$DEVNET_PUBLISH_PENDING_ASSETS" NEW="$new_assets" node -e ' + const oldAssets = JSON.parse(process.env.OLD || "[]"); + const newAssets = JSON.parse(process.env.NEW || "[]"); + console.log(JSON.stringify(oldAssets.concat(newAssets))); + ') + names=$(ASSETS="$merged" node -e ' + const assets = JSON.parse(process.env.ASSETS || "[]"); + console.log(JSON.stringify(assets.map(a => a.name))); + ') + _devnet_publish_persist_state "$DEVNET_PUBLISH_ALL_RESPONSES" "$DEVNET_PUBLISH_ROOT_ENTITIES" "$DEVNET_PUBLISH_NODE" "$names" "$merged" +} + +# Create one or more named knowledge assets, seal them, and share them to SWM. +# Echoes a legacy-shaped write response so older devnet scripts can keep their +# `triplesWritten` assertions while exercising the named KA lifecycle path. +devnet_create_shared_ka() { + local node_id="$1" payload="$2" name_prefix="${3:-devnet-ka}" extra_fields="${4:-}" + local nonce plan count i asset body resp responses new_assets triples names roots + + _devnet_publish_init_state_file + nonce="$(date +%s)-$$-${RANDOM:-0}" + plan=$(PAYLOAD="$payload" NAME_PREFIX="$name_prefix" EXTRA_FIELDS="$extra_fields" NONCE="$nonce" node -e ' + const payload = JSON.parse(process.env.PAYLOAD); + const extra = process.env.EXTRA_FIELDS ? JSON.parse("{" + process.env.EXTRA_FIELDS + "}") : {}; + const quads = Array.isArray(payload.quads) ? payload.quads : []; + const rootOf = (subject) => String(subject || "").split("/.well-known/genid/")[0]; + const slug = (value) => String(value || "ka") + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, "-") + .replace(/^-+|-+$/g, "") + .slice(0, 48) || "ka"; + const prefix = slug(process.env.NAME_PREFIX || "devnet-ka"); + const roots = [...new Set(quads.map(q => rootOf(q.subject)).filter(Boolean))]; + const effectiveRoots = roots.length > 0 ? roots : [prefix]; + const preserveBatch = process.env.DEVNET_PUBLISH_PRESERVE_BATCH === "1"; + const assetRoots = preserveBatch ? [effectiveRoots[0]] : effectiveRoots; + const assets = assetRoots.map((root, index) => { + const rootQuads = roots.length > 0 + ? (preserveBatch ? quads : quads.filter(q => rootOf(q.subject) === root)) + : quads; + const name = `${prefix}-${process.env.NONCE}-${index + 1}-${slug(root).slice(0, 24)}`; + return { + name, + root, + rootEntities: effectiveRoots, + preserveBatch, + ...(extra.subGraphName || payload.subGraphName ? { subGraphName: extra.subGraphName || payload.subGraphName } : {}), + quads: rootQuads, + body: { + contextGraphId: payload.contextGraphId, + name, + quads: rootQuads, + finalize: true, + alsoShareSwm: true, + ...(payload.subGraphName ? { subGraphName: payload.subGraphName } : {}), + ...extra, + }, + }; + }); + console.log(JSON.stringify({ + contextGraphId: payload.contextGraphId, + totalQuads: quads.length, + assets, + })); + ') + + count=$(printf '%s' "$plan" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>console.log(JSON.parse(d).assets.length));') + responses='[]' + new_assets='[]' + i=0 + while [ "$i" -lt "$count" ]; do + asset=$(printf '%s' "$plan" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>console.log(JSON.stringify(JSON.parse(d).assets[Number(process.argv[1])])));' "$i") + body=$(printf '%s' "$asset" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>console.log(JSON.stringify(JSON.parse(d).body)));') + resp=$(api_call "$node_id" POST /api/knowledge-assets "$body") + if ! printf '%s' "$resp" | node -e ' + let d=""; process.stdin.on("data", c => d += c); + process.stdin.on("end", () => { + try { + const j = JSON.parse(d); + if (j.error || (Array.isArray(j.errors) && j.errors.length > 0)) process.exit(1); + if (j.swmShared !== true) process.exit(1); + if (j.publishReady !== true) process.exit(1); + if (typeof j.shareOperationId !== "string" || j.shareOperationId.trim().length === 0) process.exit(1); + if (Number(j.promotedCount || 0) <= 0) process.exit(1); + } catch { + process.exit(1); + } + }); + '; then + printf 'devnet_create_shared_ka: /api/knowledge-assets did not return a publish-ready SWM share\n%s\n' "$resp" >&2 + return 1 + fi + responses=$(node -e 'const arr=JSON.parse(process.argv[1]); arr.push(JSON.parse(process.argv[2])); console.log(JSON.stringify(arr));' "$responses" "$resp") + new_assets=$(node -e ' + const arr = JSON.parse(process.argv[1]); + const asset = JSON.parse(process.argv[2]); + arr.push({ + name: asset.name, + root: asset.root, + rootEntities: asset.rootEntities || [asset.root], + preserveBatch: Boolean(asset.preserveBatch), + ...(asset.subGraphName ? { subGraphName: asset.subGraphName } : {}), + }); + console.log(JSON.stringify(arr)); + ' "$new_assets" "$asset") + i=$((i + 1)) + done + + _devnet_append_pending_assets "$new_assets" + triples=$(printf '%s' "$plan" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>console.log(JSON.parse(d).totalQuads));') + names=$(printf '%s' "$new_assets" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>console.log(JSON.stringify(JSON.parse(d).map(a=>a.name))));') + roots=$(printf '%s' "$new_assets" | node -e ' + let d=""; process.stdin.on("data",c=>d+=c); + process.stdin.on("end",()=>{ + const assets = JSON.parse(d); + console.log(JSON.stringify([...new Set(assets.flatMap(a => a.rootEntities || [a.root]))])); + }); + ') + node -e ' + const names = JSON.parse(process.argv[1]); + const roots = JSON.parse(process.argv[2]); + const responses = JSON.parse(process.argv[3]); + console.log(JSON.stringify({ + triplesWritten: Number(process.argv[4]), + shareOperationId: "knowledge-assets:" + names.join(","), + names, + rootEntities: roots, + responses, + })); + ' "$names" "$roots" "$responses" "$triples" +} + # Publish statuses the devnet harness treats as success (matches devnet-test.sh). _devnet_publish_status_ok() { case "$1" in @@ -52,7 +190,7 @@ _devnet_publish_init_state_file() { } _devnet_publish_persist_state() { - local all_responses="$1" root_entities="${2:-[]}" publish_node="${3:-}" + local all_responses="$1" root_entities="${2:-[]}" publish_node="${3:-}" asset_names="${4:-[]}" pending_assets="${5:-[]}" node -e " require('fs').writeFileSync( process.argv[1], @@ -60,9 +198,11 @@ _devnet_publish_persist_state() { allResponses: JSON.parse(process.argv[2]), rootEntities: JSON.parse(process.argv[3]), publishNode: process.argv[4], + assetNames: JSON.parse(process.argv[5]), + pendingAssets: JSON.parse(process.argv[6]), }), ); - " "$DEVNET_PUBLISH_STATE_FILE" "$all_responses" "$root_entities" "$publish_node" + " "$DEVNET_PUBLISH_STATE_FILE" "$all_responses" "$root_entities" "$publish_node" "$asset_names" "$pending_assets" } devnet_publish_load_state() { @@ -70,6 +210,8 @@ devnet_publish_load_state() { if [ ! -f "$DEVNET_PUBLISH_STATE_FILE" ]; then DEVNET_PUBLISH_ALL_RESPONSES='[]' DEVNET_PUBLISH_ROOT_ENTITIES='[]' + DEVNET_PUBLISH_ASSET_NAMES='[]' + DEVNET_PUBLISH_PENDING_ASSETS='[]' DEVNET_PUBLISH_NODE='' return 0 fi @@ -85,14 +227,22 @@ devnet_publish_load_state() { const s = JSON.parse(require('fs').readFileSync(process.argv[1], 'utf8')); console.log(s.publishNode || ''); " "$DEVNET_PUBLISH_STATE_FILE") + DEVNET_PUBLISH_ASSET_NAMES=$(node -e " + const s = JSON.parse(require('fs').readFileSync(process.argv[1], 'utf8')); + console.log(JSON.stringify(s.assetNames || [])); + " "$DEVNET_PUBLISH_STATE_FILE") + DEVNET_PUBLISH_PENDING_ASSETS=$(node -e " + const s = JSON.parse(require('fs').readFileSync(process.argv[1], 'utf8')); + console.log(JSON.stringify(s.pendingAssets || [])); + " "$DEVNET_PUBLISH_STATE_FILE") } # Echo the number of publishes recorded by the last devnet_publish_swm_all_roots # call. This counts `allResponses` (one entry per confirmed publish), NOT # `rootEntities`: the single-root path persists 1 response but `[]` root -# subjects (the `/api/shared-memory/publish` success body carries no -# `rootEntities`), so counting rootEntities would report 0 for a perfectly good -# one-root publish and make `devnet_verify_each_published_root` reject it. The +# subjects for older single-root runs may be absent, so counting rootEntities +# would report 0 for a perfectly good one-root publish and make +# `devnet_verify_each_published_root` reject it. The # verify loop indexes `allResponses[i]` via `devnet_publish_ka_id_at`, so this # count must track `allResponses`. rootEntities is supplementary (per-root quad # filtering, guarded by `roots.length > 0`). @@ -264,7 +414,9 @@ devnet_verify_each_published_root() { return 0 } -devnet_publish_swm_all_roots() { +_devnet_publish_swm_all_roots_retired_legacy() { + echo "retired legacy anonymous SWM publish helper; use devnet_create_shared_ka + devnet_publish_swm_all_roots" >&2 + return 1 local node="$1" cg="$2" clear_after="${3:-false}" local extra_fields="${4:-}" @@ -279,7 +431,7 @@ devnet_publish_swm_all_roots() { " "$cg" "$clear_after" "$extra_fields") local probe probe_code - probe=$(api_call "$node" POST /api/shared-memory/publish "$probe_body") + probe=$(api_call "$node" POST /api/retired-anonymous-swm-publish "$probe_body") probe_code=$(devnet_json_field "$probe" '.code') if [ "$probe_code" != "MULTI_ROOT_PUBLISH_NOT_ATOMIC" ]; then @@ -352,7 +504,7 @@ devnet_publish_swm_all_roots() { ...extra, })); " "$cg" "$root" "$ca" "$extra_fields") - last_resp=$(api_call "$node" POST /api/shared-memory/publish "$last_resp") + last_resp=$(api_call "$node" POST /api/retired-anonymous-swm-publish "$last_resp") st=$(devnet_json_field "$last_resp" '.status') if ! _devnet_publish_status_ok "$st"; then printf '%s' "$last_resp" >&2 @@ -371,6 +523,78 @@ devnet_publish_swm_all_roots() { return 0 } +devnet_publish_swm_all_roots() { + local node="$1" cg="$2" clear_after="${3:-false}" + local extra_fields="${4:-}" + + _devnet_publish_init_state_file + devnet_publish_load_state + + local pending_json count roots_json names_json i name pending_subgraph ca body last_resp st all_resps + pending_json="$DEVNET_PUBLISH_PENDING_ASSETS" + count=$(printf '%s' "$pending_json" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>console.log(JSON.parse(d || "[]").length))') + if [ "$count" -eq 0 ]; then + echo "devnet_publish_swm_all_roots: no pending named KAs; use devnet_create_shared_ka before publishing" >&2 + return 1 + fi + + roots_json=$(printf '%s' "$pending_json" | node -e ' + let d=""; process.stdin.on("data",c=>d+=c); + process.stdin.on("end",()=>{ + const assets = JSON.parse(d || "[]"); + if (assets.some(a => a.preserveBatch)) { + console.log("[]"); + return; + } + console.log(JSON.stringify(assets.map(a => a.root))); + }); + ') + names_json=$(printf '%s' "$pending_json" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>console.log(JSON.stringify(JSON.parse(d).map(a=>a.name))))') + all_resps="[]" + i=0 + last_resp="" + while [ "$i" -lt "$count" ]; do + name=$(printf '%s' "$pending_json" | node -e "let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>console.log(JSON.parse(d)[Number(process.argv[1])].name))" "$i") + pending_subgraph=$(printf '%s' "$pending_json" | node -e "let d='';process.stdin.on('data',c=>d+=c);process.stdin.on('end',()=>console.log(JSON.parse(d)[Number(process.argv[1])].subGraphName || ''))" "$i") + ca="$clear_after" + if [ "$clear_after" = "true" ] && [ "$i" -lt $((count - 1)) ]; then + ca="false" + fi + body=$(node -e " + const cg = process.argv[1]; + const clearAfter = process.argv[2] === 'true'; + const extra = process.argv[3] ? JSON.parse('{' + process.argv[3] + '}') : {}; + const pendingSubGraphName = process.argv[4] || ''; + const { subGraphName, ...publishOptions } = extra; + const resolvedSubGraphName = subGraphName || pendingSubGraphName; + console.log(JSON.stringify({ + contextGraphId: cg, + ...(resolvedSubGraphName ? { subGraphName: resolvedSubGraphName } : {}), + options: { + clearAfter, + ...publishOptions, + }, + })); + " "$cg" "$ca" "$extra_fields" "$pending_subgraph") + last_resp=$(api_call "$node" POST "/api/knowledge-assets/${name}/vm/publish" "$body") + st=$(devnet_json_field "$last_resp" '.status') + if ! _devnet_publish_status_ok "$st"; then + printf '%s' "$last_resp" >&2 + return 1 + fi + all_resps=$(node -e " + const arr = JSON.parse(process.argv[1]); + arr.push(JSON.parse(process.argv[2])); + console.log(JSON.stringify(arr)); + " "$all_resps" "$last_resp") + i=$((i + 1)) + sleep 1 + done + _devnet_publish_persist_state "$all_resps" "$roots_json" "$node" "$names_json" "[]" + printf '%s' "$last_resp" + return 0 +} + # Filter quads to those belonging to a published root entity (matches publisher selection). devnet_quads_for_root_json() { local quads_payload="$1" root="$2" diff --git a/scripts/devnet-rc12-release-validation.sh b/scripts/devnet-rc12-release-validation.sh index 95bb0c26f1..d40be9516e 100755 --- a/scripts/devnet-rc12-release-validation.sh +++ b/scripts/devnet-rc12-release-validation.sh @@ -639,21 +639,21 @@ print(json.dumps(q)) PY } -# Single publish: write -> publish; emits a metrics jsonl line. +# Single publish: create named KA -> share to SWM -> publish; emits a metrics jsonl line. publish_one() { # idx node cgid kind local idx=$1 node=$2 cgid=$3 kind=$4 port="${NODE_PORT[$((node-1))]}" - local E root quads w op sel p st kc + local E root name quads w p st kc E=$(( RANDOM % (MAX_ENTITIES - MIN_ENTITIES + 1) + MIN_ENTITIES )) root="urn:rc12:ka:${RUN_TAG}:${idx}:n${node}" + name="rc12-${RUN_TAG}-${idx}-n${node}" # Large bodies (up to ~1000 entities * 3 triples) go via a temp file to stay # well clear of ARG_MAX in this unattended run. local bodyf; bodyf=$(mktemp -t rc12pub.XXXXXX) - { printf '{"contextGraphId":"%s","quads":' "$cgid"; gen_quads "$root" "$E"; printf '}'; } > "$bodyf" + { printf '{"contextGraphId":"%s","name":"%s","quads":' "$cgid" "$name"; gen_quads "$root" "$E"; printf ',"finalize":true,"alsoShareSwm":true}'; } > "$bodyf" w=$(curl -s --max-time 60 -H "$H" -H "Content-Type: application/json" -X POST \ - "http://127.0.0.1:$port/api/shared-memory/write" --data @"$bodyf") + "http://127.0.0.1:$port/api/knowledge-assets" --data @"$bodyf") rm -f "$bodyf" - op=$(echo "$w" | pyf "d.get('shareOperationId','')") - if [ -z "$op" ]; then + if [ "$(echo "$w" | pyf "1 if d.get('swmShared') or d.get('status') in ('swm-shared','vm-confirmed') else 0")" != "1" ]; then printf '{"idx":"%s","node":%d,"cg":"%s","kind":"%s","entities":%d,"ok":false,"stage":"write","err":%s}\n' \ "$idx" "$node" "$cgid" "$kind" "$E" "$(python3 -c "import json,sys;print(json.dumps(sys.argv[1][:200]))" "$w")" >> "$METRICS_JSONL" return 1 @@ -670,8 +670,8 @@ publish_one() { # idx node cgid kind local attempt p st kc for attempt in 1 2 3 4 5; do p=$(curl -s --max-time 120 -H "$H" -H "Content-Type: application/json" -X POST \ - "http://127.0.0.1:$port/api/shared-memory/publish" \ - -d "{\"contextGraphId\":\"$cgid\",\"selection\":{\"rootEntities\":[\"$root\"]},\"clearAfter\":false}") + "http://127.0.0.1:$port/api/knowledge-assets/$name/vm/publish" \ + -d "{\"contextGraphId\":\"$cgid\",\"options\":{\"clearAfter\":false}}") st=$(echo "$p" | pyf "d.get('status','')") kc=$(echo "$p" | pyf "d.get('kaId','')") if [ "$st" = "confirmed" ] || [ "$st" = "finalized" ]; then @@ -864,12 +864,9 @@ else fi # ── Section A2: canonical assertion lifecycle smoke ───────────────────────── -# The bulk publish path above uses /api/shared-memory/write directly. The -# canonical RFC-001 §9.x path is /api/knowledge-assets (create) with finalize+promote -# (create → write → finalize → promote in one shot), then publish to VM. A -# regression in assertion finalize/promote could still let the SWM-write path -# pass, so we exercise the canonical path end-to-end with one KA and gate the -# release on it. +# The bulk publish path above uses the named KA lifecycle. Keep one compact +# lifecycle smoke here as a release gate for create -> finalize -> share -> +# publish on a single easy-to-debug KA. section "A2. ASSERTION LIFECYCLE — canonical create→write→finalize→promote→publish" A2_NODE=1 A2_PORT="${NODE_PORT[$((A2_NODE-1))]}" @@ -901,13 +898,13 @@ PY # forwards it verbatim, so this is what exercises create→finalize→publish # end-to-end. clearAfter is harmless for this one-shot path but kept # false-explicit for consistency with the bulk path. - # /api/shared-memory/publish drives an on-chain tx; the default `post` + # /api/knowledge-assets//vm/publish drives an on-chain tx; the default `post` # helper caps curl at 30s which is shorter than a busy devnet's mempool # round-trip. Use a dedicated 180s budget (the bulk loop already does # this for the same reason — keep the named-assertion fork in line). pp=$(curl -s --max-time 180 -H "$H" -H "Content-Type: application/json" -X POST \ - "http://127.0.0.1:$A2_PORT/api/shared-memory/publish" \ - -d "{\"contextGraphId\":\"$A2_CG\",\"assertionName\":\"$A2_NAME\",\"clearAfter\":false}") + "http://127.0.0.1:$A2_PORT/api/knowledge-assets/$A2_NAME/vm/publish" \ + -d "{\"contextGraphId\":\"$A2_CG\",\"options\":{\"clearAfter\":false}}") pps=$(echo "$pp" | pyf "d.get('status','')") case "$pps" in confirmed|finalized) pass A2 assertion-publish "VM publish via finalized-assertion fork landed (status=$pps)" ;; diff --git a/scripts/devnet-rs-validation.sh b/scripts/devnet-rs-validation.sh index bcf4344837..a0ed760b96 100755 --- a/scripts/devnet-rs-validation.sh +++ b/scripts/devnet-rs-validation.sh @@ -192,23 +192,23 @@ print(json.dumps(q)) PY } -# Single publish: write -> publish. Echoes the kaId on success, empty on failure. +# Single publish: create named KA -> share to SWM -> publish. Echoes the kaId on success, empty on failure. publish_one() { # idx node root local idx=$1 node=$2 root=$3 local port="${NODE_PORT[$((node-1))]}" - local E w op p st kc bodyf attempt + local E name w p st kc bodyf attempt E=$(( RANDOM % (MAX_ENTITIES - MIN_ENTITIES + 1) + MIN_ENTITIES )) + name="rs-${RUN_TAG}-${idx}-n${node}" bodyf=$(mktemp -t rspub.XXXXXX) - { printf '{"contextGraphId":"%s","quads":' "$CGID"; gen_quads "$root" "$E"; printf '}'; } > "$bodyf" + { printf '{"contextGraphId":"%s","name":"%s","quads":' "$CGID" "$name"; gen_quads "$root" "$E"; printf ',"finalize":true,"alsoShareSwm":true}'; } > "$bodyf" w=$(curl -s --max-time 60 -H "$H" -H "Content-Type: application/json" -X POST \ - "http://127.0.0.1:$port/api/shared-memory/write" --data @"$bodyf") + "http://127.0.0.1:$port/api/knowledge-assets" --data @"$bodyf") rm -f "$bodyf" - op=$(echo "$w" | pyf "d.get('shareOperationId','')") - [ -z "$op" ] && { echo ""; return 1; } + [ "$(echo "$w" | pyf "1 if d.get('swmShared') or d.get('status') in ('swm-shared','vm-confirmed') else 0")" = "1" ] || { echo ""; return 1; } for attempt in 1 2 3 4 5; do p=$(curl -s --max-time 120 -H "$H" -H "Content-Type: application/json" -X POST \ - "http://127.0.0.1:$port/api/shared-memory/publish" \ - -d "{\"contextGraphId\":\"$CGID\",\"selection\":{\"rootEntities\":[\"$root\"]},\"clearAfter\":false}") + "http://127.0.0.1:$port/api/knowledge-assets/$name/vm/publish" \ + -d "{\"contextGraphId\":\"$CGID\",\"options\":{\"clearAfter\":false}}") st=$(echo "$p" | pyf "d.get('status','')") kc=$(echo "$p" | pyf "d.get('kaId','')") if [ "$st" = "confirmed" ] || [ "$st" = "finalized" ]; then echo "$kc"; return 0; fi diff --git a/scripts/devnet-test-curator-ack-gate.sh b/scripts/devnet-test-curator-ack-gate.sh index 30d6bc20a2..fe78045abd 100755 --- a/scripts/devnet-test-curator-ack-gate.sh +++ b/scripts/devnet-test-curator-ack-gate.sh @@ -40,23 +40,6 @@ api_call() { # node method path [body] → body on stdout, fails on non-2xx identity_field() { local body; body="$(api_call "$1" GET /api/agent/identity)"; parse_json "$body" ".$2"; } -# Member write that CAPTURES the HTTP status + curatorDelivery field. -# Prints: " " -write_capture() { # node subj value - local node="$1" subj="$2" value="$3" port token tmp code body cd payload - port=$(node_port "$node"); token=$(node_token "$node"); tmp="$(mktemp "$TMPDIR/ackw-XXXXXX")" - # Opt INTO the strict gate per-request (the agent default stays OFF — phase-1 - # rollout). This isolates the gate test from the M2-a converge test, which - # deliberately relies on the legacy (gate-off) offline-write path. - payload=$(CG="$CG_ID" R="$subj" P="$PRED" V="$value" node -e 'console.log(JSON.stringify({contextGraphId:process.env.CG,awaitCuratorAck:true,quads:[{subject:process.env.R,predicate:process.env.P,object:JSON.stringify(process.env.V),graph:""}]}))') - code=$(curl -sS --max-time 30 --connect-timeout 5 -o "$tmp" -w "%{http_code}" -X POST \ - -H "Authorization: Bearer $token" -H "Content-Type: application/json" \ - --data "$payload" "http://127.0.0.1:${port}/api/shared-memory/write" || echo "000") - body="$(cat "$tmp")"; rm -f "$tmp" - cd="$(parse_json "$body" ".curatorDelivery")" - echo "$code ${cd:-none}" -} - post_capture() { # node path body → " " local node="$1" path="$2" body="$3" port token tmp code resp cd port=$(node_port "$node"); token=$(node_token "$node"); tmp="$(mktemp "$TMPDIR/ackp-XXXXXX")" @@ -105,35 +88,6 @@ api_call "$CURATOR_NODE" POST /api/context-graph/subscribe "{\"contextGraphId\": api_call "$MEMBER_NODE" POST /api/context-graph/subscribe "{\"contextGraphId\":\"$CG_ID\",\"includeSharedMemory\":true}" >/dev/null sleep 3 -act "A. curator UP: member-owned write must CONFIRM (200) and land on the curator" -RES="$(write_capture "$MEMBER_NODE" "$ROOT" "v1")" -log "member write (curator up) -> HTTP+delivery: $RES" -[ "${RES%% *}" = "200" ] || fail "expected 200 with curator up, got: $RES" -[ "$(swm_values "$MEMBER_NODE" "$ROOT")" = '["v1"]' ] || fail "member should hold v1 (got $(swm_values "$MEMBER_NODE" "$ROOT"))" -GOTC="$(await_values "$CURATOR_NODE" "$ROOT" '["v1"]' 30)" || fail "curator never received the confirmed write (got $GOTC) — gate did not actually deliver" -log "curator received v1 ✓ (the gate's reliable send delivered + applied)" - -act "B. curator DOWN: member-owned write must be REJECTED (503) and NOT persisted" -stop_node "$CURATOR_NODE" -log "curator stopped" -RES="$(write_capture "$MEMBER_NODE" "$ROOT" "v2")" -log "member write (curator down) -> HTTP+delivery: $RES" -CODE="${RES%% *}"; DELIV="${RES##* }" -[ "$CODE" = "503" ] || fail "expected HTTP 503 with curator down, got: $RES (the silent-200 bug is NOT fixed)" -[ "$DELIV" = "unconfirmed" ] || fail "expected curatorDelivery=unconfirmed, got: $RES" -AFTER="$(swm_values "$MEMBER_NODE" "$ROOT")" -[ "$AFTER" = '["v1"]' ] || fail "NO-PERSIST violated: member should still hold ONLY v1, got $AFTER (v2 leaked into the local store)" -log "member still holds [v1], v2 was NOT persisted ✓ (no silent state)" - -act "C. curator back UP: the same write now CONFIRMS again" -start_node "$CURATOR_NODE"; sleep 3 -RES="$(write_capture "$MEMBER_NODE" "$ROOT" "v2")" -log "member write (curator up again) -> HTTP+delivery: $RES" -[ "${RES%% *}" = "200" ] || fail "expected 200 after curator restart, got: $RES" -GOTC="$(await_values "$CURATOR_NODE" "$ROOT" '["v2"]' 30)" || fail "curator never received v2 after restart (got $GOTC)" -[ "$(swm_values "$MEMBER_NODE" "$ROOT")" = '["v2"]' ] || fail "member should hold v2" -log "curator + member both [v2] ✓" - act "D. PROMOTE path (WM→SWM via /knowledge-assets/:name/swm/share — the path the original silent-loss counterexample used)" KA1="kapromote-a-$STAMP"; KA2="kapromote-b-$STAMP"; KAROOT1="urn:ackgate:ka1"; KAROOT2="urn:ackgate:ka2" # D1. curator UP: create a WM draft, then promote → must CONFIRM (200) and land on curator @@ -154,7 +108,5 @@ log "promote aborted, KA2 NOT in SWM (WM intact) ✓" start_node "$CURATOR_NODE"; sleep 3 echo "" -log "GATE PASS — strict curator-ack on BOTH paths:" -log " • share() (/api/shared-memory/write): confirmed→200+durable, unconfirmed→503 no-persist" +log "GATE PASS - strict curator-ack on named KA share:" log " • promote (/knowledge-assets/:name/swm/share): confirmed→200+durable, unconfirmed→503 no-persist" -log " The silent-200 same-root loss (M2-b gap) is closed for the SWM write + promote paths." diff --git a/scripts/devnet-test-curator-converge.sh b/scripts/devnet-test-curator-converge.sh index 9b842d08b2..b4fd6e3689 100755 --- a/scripts/devnet-test-curator-converge.sh +++ b/scripts/devnet-test-curator-converge.sh @@ -8,8 +8,8 @@ # the curator's current private shared-memory state WITHOUT any manual call, and # the curator must NOT be polluted in reverse. # -# Writes model a real UPDATE of one fact via direct /api/shared-memory/write -# (per-SUBJECT REPLACE on the base graph, single-valued). This is the ONLY +# Writes used to model a real UPDATE of one fact via the retired direct loose +# SWM write path (per-SUBJECT REPLACE on the base graph, single-valued). This was the ONLY # working in-place update path — see share_value below for why the KA-share / # finalized-KA paths are not updates. The same-graph v1->v2->v3 update is exactly # what exercises the gossip-vs-sync union bug the gate fixes: live gossip REPLACEs @@ -41,6 +41,9 @@ set -euo pipefail +echo "[curator-converge] retired: this scenario depended on removed loose SWM in-place update routes; replace with named KA update lifecycle coverage before re-enabling." >&2 +exit 2 + REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}" API_PORT_BASE="${API_PORT_BASE:-9201}" @@ -102,7 +105,7 @@ swm_values() { share_value() { local node="$1" subj="$2" value="$3" payload payload=$(CG="$CG_ID" R="$subj" P="$PRED" V="$value" node -e 'console.log(JSON.stringify({contextGraphId:process.env.CG,quads:[{subject:process.env.R,predicate:process.env.P,object:JSON.stringify(process.env.V),graph:""}]}))') - api_call "$node" POST /api/shared-memory/write "$payload" >/dev/null + devnet_create_shared_ka "$node" "$payload" >/dev/null } wait_down() { local p; p=$(node_port "$1"); for _ in $(seq 1 60); do curl -sf --max-time 1 -o /dev/null "http://127.0.0.1:$p/api/status" 2>/dev/null || return 0; sleep 0.5; done; fail "node $1 still up"; } diff --git a/scripts/devnet-test-rc11-shutdown-mid-publish.sh b/scripts/devnet-test-rc11-shutdown-mid-publish.sh index 426d842a68..316dc43341 100755 --- a/scripts/devnet-test-rc11-shutdown-mid-publish.sh +++ b/scripts/devnet-test-rc11-shutdown-mid-publish.sh @@ -47,6 +47,9 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=devnet-publish-helpers.sh +source "$SCRIPT_DIR/devnet-publish-helpers.sh" DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}" API_PORT_BASE=9201 @@ -124,7 +127,7 @@ log "Target node $TARGET_NODE current [shutdown-timeout] hit count: $PRE_TIMEOUT # --------------------------------------------------------------------------- # Stage 1: Launch concurrent publishes from the load node. Each publish -# enqueues SWM triples, calls /api/shared-memory/publish (which drives +# enqueues SWM triples, calls /api/knowledge-assets//vm/publish (which drives # StorageACK reads against cores 1-4), and returns when ACK collection # settles or fails. # --------------------------------------------------------------------------- @@ -140,6 +143,7 @@ for i in $(seq 1 $CONCURRENCY); do # publish them. Run in the background, capture stdout for later # post-mortem (we don't gate the shutdown test on publish success). ( + export DEVNET_PUBLISH_STATE_FILE="$TMP_OUT_DIR/state-$i.json" QUADS=$(node -e " const triples = []; for (let j = 0; j < 8; j++) { @@ -155,9 +159,9 @@ for i in $(seq 1 $CONCURRENCY); do quads: triples, })); ") - api_call "$LOAD_NODE" POST /api/shared-memory/write "$QUADS" \ + devnet_create_shared_ka "$LOAD_NODE" "$QUADS" \ > "$TMP_OUT_DIR/write-$i.json" 2>&1 || true - # Codex (#673#discussion_r3302023873): `/api/shared-memory/publish` + # Codex (#673#discussion_r3302023873): `/api/knowledge-assets//vm/publish` # accepts `selection: "all"` or a root-entity string array — NOT a # SPARQL-shaped object. Pass the 8 generated root entities directly so # each background pipeline drives a real StorageACK round trip. @@ -171,7 +175,7 @@ for i in $(seq 1 $CONCURRENCY); do selection: roots, })); ") - PUBLISH_OUT=$(api_call "$LOAD_NODE" POST /api/shared-memory/publish "$ROOT_ENTITIES" 2>&1) || PUBLISH_RC=$? && PUBLISH_RC=${PUBLISH_RC:-0} + PUBLISH_OUT=$(devnet_publish_swm_all_roots "$LOAD_NODE" "$CG_ID" false 2>&1) || PUBLISH_RC=$? && PUBLISH_RC=${PUBLISH_RC:-0} echo "$PUBLISH_OUT" > "$TMP_OUT_DIR/publish-$i.json" if [ "$PUBLISH_RC" -ne 0 ]; then echo "[publish-$i] api_call exit=$PUBLISH_RC" >> "$TMP_OUT_DIR/publish-$i.json" diff --git a/scripts/devnet-test-rfc38-cross-cg.sh b/scripts/devnet-test-rfc38-cross-cg.sh index 535f8b45ce..f8b5952379 100755 --- a/scripts/devnet-test-rfc38-cross-cg.sh +++ b/scripts/devnet-test-rfc38-cross-cg.sh @@ -28,6 +28,9 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=devnet-publish-helpers.sh +source "$SCRIPT_DIR/devnet-publish-helpers.sh" DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}" API_PORT_BASE=9201 CURATOR_NODE=5 @@ -111,7 +114,7 @@ sleep 2 act "2. Curator writes private content to BOTH CGs" # =========================================================================== log "Writing to CG-A..." -WRITE_A=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$(cat </vm/publish to finish the path to VM without # the curator coming back online. # # Test phases: @@ -25,11 +25,11 @@ # [curator, M1=N6, M2=N4] in the allowlist; all three pre-create. # (curated access policy + open publish policy: only allowlisted # members can read/write, but ANY of them can publish to VM.) -# 2. Curator writes 4 triples. Both members + the core network -# catch up so the ciphertext is on their disks. +# 2. M1 writes 4 triples through the named KA lifecycle. Both members +# + the core network catch up so the ciphertext is on their disks. # 3. Curator's daemon is stopped (simulating offline). # 4. M1 + M2 confirm they still have the 4 triples (no regression). -# 5. M1 (non-curator) calls /api/shared-memory/publish. Must +# 5. M1 (non-curator) calls /api/knowledge-assets//vm/publish. Must # succeed: status=confirmed + a txHash. # 6. Outsider (N1) fetches the on-chain KC and verifies the # merkleRoot decodes; this proves the publish landed without @@ -217,7 +217,7 @@ if ! wait_for_m1_onchain_id; then fi # =========================================================================== -act "2. Curator writes 4 triples; members catch up" +act "2. M1 creates, seals, and shares 4 triples; members catch up" # =========================================================================== PAYLOAD=$(STAMP="$STAMP" CG_ID="$CG_ID" node -e ' const stamp = process.env.STAMP; @@ -229,9 +229,9 @@ PAYLOAD=$(STAMP="$STAMP" CG_ID="$CG_ID" node -e ' } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WRITE_RESP=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$PAYLOAD") +WRITE_RESP=$(devnet_create_shared_ka "$M1_NODE" "$PAYLOAD") [ "$(parse_json "$WRITE_RESP" '.triplesWritten')" = "4" ] || fail "write expected 4 triples: $WRITE_RESP" -log "✓ curator wrote 4 triples" +log "✓ M1 wrote 4 triples through the named KA lifecycle" sleep 5 # Codex PR #622 R-list: /api/shared-memory/list isn't a daemon route. @@ -398,7 +398,7 @@ log "================================================================" log " RFC-38 LU-6 C2 (curator-offline mid-batch): PASS" log "================================================================" log " Curated CG: $CG_ID (onChainId=$ON_CHAIN_ID, publishPolicy=open)" -log " Triples in: 4 (curator-written, gossiped to members)" +log " Triples in: 4 (M1-written named KA, gossiped to members)" log " Curator: SIGTERMed before publish" log " M1 published: kaId=$KC tx=$TX merkleRoot=$MERKLE_ROOT" log " Outsider: observed merkleRoot on chain" diff --git a/scripts/devnet-test-rfc38-e2e.sh b/scripts/devnet-test-rfc38-e2e.sh index b512da7703..4b9a3be688 100755 --- a/scripts/devnet-test-rfc38-e2e.sh +++ b/scripts/devnet-test-rfc38-e2e.sh @@ -34,7 +34,7 @@ # • Asserts: ok=true, actualRoot==expectedRoot. # • Forges a tampered quads array, calls verify-batch again, # asserts: ok=false with reason=root-mismatch. -# • Reports the failed batch via POST /api/shared-memory/report-batch-rejection +# • Reports the failed batch via POST /api/knowledge-assets/batch-rejections/report # — asserts: returns rejection record with a populated digest. # # ACT 4 (LU-9 member-attestation — outsider verification path) @@ -198,7 +198,7 @@ QUADS_PAYLOAD=$(node -e " ") log "Curator writes 12 SWM triples..." -WRITE_RESP=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$QUADS_PAYLOAD") +WRITE_RESP=$(devnet_create_shared_ka "$CURATOR_NODE" "$QUADS_PAYLOAD") WRITTEN=$(parse_json "$WRITE_RESP" '.triplesWritten') [ "$WRITTEN" = "12" ] || fail "expected 12 triples written, got '$WRITTEN' — response: $WRITE_RESP" log "✓ 12 triples written to curator's SWM (op=$(parse_json "$WRITE_RESP" '.shareOperationId'))" @@ -369,7 +369,7 @@ REPORT_BODY=$(VERIFY_BAD="$VERIFY_BAD" CG_ID="$CG_ID" PUBLISH_KC="$PUBLISH_KC" n verifyResult })); ") -REPORT_RESP=$(api_call "$CURATOR_NODE" POST /api/shared-memory/report-batch-rejection "$REPORT_BODY") +REPORT_RESP=$(api_call "$CURATOR_NODE" POST /api/knowledge-assets/batch-rejections/report "$REPORT_BODY") log "report-batch-rejection response: $REPORT_RESP" RR_DIGEST=$(parse_json "$REPORT_RESP" '.record.digest') [[ "$RR_DIGEST" =~ ^0x[0-9a-fA-F]{64}$ ]] || fail "rejection record digest missing/invalid: '$RR_DIGEST'" diff --git a/scripts/devnet-test-rfc38-late-joiner.sh b/scripts/devnet-test-rfc38-late-joiner.sh index 94c31ecc6f..b8d233c2db 100755 --- a/scripts/devnet-test-rfc38-late-joiner.sh +++ b/scripts/devnet-test-rfc38-late-joiner.sh @@ -79,6 +79,9 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=devnet-publish-helpers.sh +source "$SCRIPT_DIR/devnet-publish-helpers.sh" DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}" API_PORT_BASE=9201 CURATOR_NODE=5 @@ -276,7 +279,7 @@ A_PAYLOAD=$(CG_ID="$CG_A" N=5 LABEL="A" node -e ' } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WROTE_A=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$A_PAYLOAD") +WROTE_A=$(devnet_create_shared_ka "$CURATOR_NODE" "$A_PAYLOAD") TRIPLES_WROTE_A=$(parse_json "$WROTE_A" '.triplesWritten') [ "$TRIPLES_WROTE_A" = "5" ] || fail "expected 5 triplesWritten, got '$TRIPLES_WROTE_A' (response: $WROTE_A)" log "✓ curator wrote 5 triples" @@ -354,7 +357,7 @@ B_PAYLOAD=$(CG_ID="$CG_B" N=7 LABEL="B" node -e ' } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WROTE_B=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$B_PAYLOAD") +WROTE_B=$(devnet_create_shared_ka "$CURATOR_NODE" "$B_PAYLOAD") TRIPLES_WROTE_B=$(parse_json "$WROTE_B" '.triplesWritten') [ "$TRIPLES_WROTE_B" = "7" ] || fail "expected 7 triplesWritten, got '$TRIPLES_WROTE_B' (response: $WROTE_B)" log "✓ curator wrote 7 triples" @@ -475,7 +478,7 @@ C_PAYLOAD=$(CG_ID="$CG_C" N=4 LABEL="C" node -e ' } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WROTE_C=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$C_PAYLOAD") +WROTE_C=$(devnet_create_shared_ka "$CURATOR_NODE" "$C_PAYLOAD") TRIPLES_WROTE_C=$(parse_json "$WROTE_C" '.triplesWritten') [ "$TRIPLES_WROTE_C" = "4" ] || fail "expected 4 triplesWritten, got '$TRIPLES_WROTE_C' (response: $WROTE_C)" log "✓ curator wrote 4 triples" @@ -589,7 +592,7 @@ D0_PAYLOAD=$(CG_ID="$CG_D" node -e ' }], })); ') -WROTE_D0=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$D0_PAYLOAD") +WROTE_D0=$(devnet_create_shared_ka "$CURATOR_NODE" "$D0_PAYLOAD") TRIPLES_WROTE_D0=$(parse_json "$WROTE_D0" '.triplesWritten') [ "$TRIPLES_WROTE_D0" = "1" ] || fail "expected 1 triplesWritten for handshake, got '$TRIPLES_WROTE_D0' (response: $WROTE_D0)" log "✓ handshake write OK" @@ -630,7 +633,7 @@ D5_PAYLOAD=$(CG_ID="$CG_D" node -e ' } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WROTE_D5=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$D5_PAYLOAD") +WROTE_D5=$(devnet_create_shared_ka "$CURATOR_NODE" "$D5_PAYLOAD") TRIPLES_WROTE_D5=$(parse_json "$WROTE_D5" '.triplesWritten') [ "$TRIPLES_WROTE_D5" = "5" ] || fail "expected 5 triplesWritten, got '$TRIPLES_WROTE_D5' (response: $WROTE_D5)" log "✓ curator wrote 5 triples while member offline" @@ -791,7 +794,7 @@ E0_PAYLOAD=$(CG_ID="$CG_E" node -e ' }], })); ') -WROTE_E0=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$E0_PAYLOAD") +WROTE_E0=$(devnet_create_shared_ka "$CURATOR_NODE" "$E0_PAYLOAD") TRIPLES_WROTE_E0=$(parse_json "$WROTE_E0" '.triplesWritten') [ "$TRIPLES_WROTE_E0" = "1" ] || fail "expected 1 triplesWritten for E handshake, got '$TRIPLES_WROTE_E0' (response: $WROTE_E0)" log "✓ CG_E handshake write OK" @@ -829,7 +832,7 @@ E5_PAYLOAD=$(CG_ID="$CG_E" node -e ' } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WROTE_E5=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$E5_PAYLOAD") +WROTE_E5=$(devnet_create_shared_ka "$CURATOR_NODE" "$E5_PAYLOAD") TRIPLES_WROTE_E5=$(parse_json "$WROTE_E5" '.triplesWritten') [ "$TRIPLES_WROTE_E5" = "5" ] || fail "expected 5 triplesWritten on CG_E, got '$TRIPLES_WROTE_E5' (response: $WROTE_E5)" log "✓ curator wrote 5 triples to CG_E while member offline" diff --git a/scripts/devnet-test-rfc38-lu10.sh b/scripts/devnet-test-rfc38-lu10.sh index 3dacfb9b87..05e147af39 100755 --- a/scripts/devnet-test-rfc38-lu10.sh +++ b/scripts/devnet-test-rfc38-lu10.sh @@ -126,7 +126,7 @@ QUADS_PAYLOAD=$(STAMP="$STAMP" CG_ID="$CG_ID" node -e ' console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WRITE_RESP=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$QUADS_PAYLOAD") +WRITE_RESP=$(devnet_create_shared_ka "$CURATOR_NODE" "$QUADS_PAYLOAD") WRITTEN=$(parse_json "$WRITE_RESP" '.triplesWritten') [ "$WRITTEN" = "10" ] || fail "expected 10 triples written, got '$WRITTEN'" log "✓ 10 triples written to SWM" diff --git a/scripts/devnet-test-rfc38-lu5-public.sh b/scripts/devnet-test-rfc38-lu5-public.sh index 81ad54ddd1..d28094af46 100755 --- a/scripts/devnet-test-rfc38-lu5-public.sh +++ b/scripts/devnet-test-rfc38-lu5-public.sh @@ -65,7 +65,7 @@ ON_CHAIN_ID=$(printf '%s' "$CREATE_RESP" | node -e 'let d="";process.stdin.on("d log "Public CG onChainId=$ON_CHAIN_ID" log "Writing public quads to SWM..." -WRITE_RESP=$(api_call "$EDGE_CURATOR_NODE" POST /api/shared-memory/write "$(cat <] } # → curated CG registered on-chain by an edge identity-less node. -# 3. POST /api/shared-memory/write → encrypted-payload SWM share. -# 4. POST /api/shared-memory/publish → on-chain publishKnowledgeCollections. +# 3. POST /api/knowledge-assets → encrypted-payload SWM share. +# 4. POST /api/knowledge-assets//vm/publish → on-chain publishKnowledgeCollections. # 5. Assertions: # - response carries status=confirmed AND non-empty txHash # - KnowledgeCollectionStorage.getKnowledgeAssetMetadata(kaId) @@ -89,7 +89,7 @@ log "Member agent: $MEMBER_AGENT (node $EDGE_MEMBER_NODE)" STAMP=$(date +%s) # Mirror the UI pattern: prefix the CG id with the creator's agent address. -# Without the prefix, /api/shared-memory/publish looks up the on-chain id +# Without the prefix, /api/knowledge-assets//vm/publish looks up the on-chain id # under the prefixed key (the SWM URI builder includes the curator), so a # bare id registers fine but can't be published. CG_SLUG="lu5-curated-${STAMP}" @@ -134,7 +134,7 @@ CG_URI="${CG_LOCAL_ID}" # 3 entities × 2 triples = 6 quads = small enough that we get exactly 3 KAs # (one per root entity) when we publish. -WRITE_RESP=$(api_call "$EDGE_CURATOR_NODE" POST /api/shared-memory/write "$(cat <&2 +exit 2 + REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}" API_PORT_BASE=9201 @@ -168,7 +171,7 @@ for i in $(seq 1 "$WRITES_COUNT"); do }]; console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') - RESP=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$PAYLOAD") + RESP=$(devnet_create_shared_ka "$CURATOR_NODE" "$PAYLOAD") N=$(parse_json "$RESP" '.triplesWritten' 2>/dev/null || echo "?") printf '[cap] write %02d/%02d → triplesWritten=%s\n' "$i" "$WRITES_COUNT" "$N" if [ "$N" = "1" ]; then diff --git a/scripts/devnet-test-rfc38-revocation.sh b/scripts/devnet-test-rfc38-revocation.sh index 6a8c813d50..0d141746ce 100755 --- a/scripts/devnet-test-rfc38-revocation.sh +++ b/scripts/devnet-test-rfc38-revocation.sh @@ -69,6 +69,9 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=devnet-publish-helpers.sh +source "$SCRIPT_DIR/devnet-publish-helpers.sh" DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}" API_PORT_BASE=9201 CURATOR_NODE=5 @@ -169,7 +172,7 @@ PRE_QUADS=$(STAMP="$STAMP" CG_ID="$CG_ID" node -e ' } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WRITE_PRE=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$PRE_QUADS") +WRITE_PRE=$(devnet_create_shared_ka "$CURATOR_NODE" "$PRE_QUADS") [ "$(parse_json "$WRITE_PRE" '.triplesWritten')" = "3" ] || fail "pre-write expected 3 triples: $WRITE_PRE" log "✓ pre-revocation: 3 triples written by curator" sleep 3 @@ -271,7 +274,7 @@ POST_QUADS=$(STAMP="$STAMP" CG_ID="$CG_ID" node -e ' } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WRITE_POST=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$POST_QUADS") +WRITE_POST=$(devnet_create_shared_ka "$CURATOR_NODE" "$POST_QUADS") [ "$(parse_json "$WRITE_POST" '.triplesWritten')" = "3" ] || fail "post-write expected 3 triples: $WRITE_POST" log "✓ post-revocation: 3 NEW triples written by curator" sleep 8 diff --git a/scripts/devnet-test-rfc38-scale.sh b/scripts/devnet-test-rfc38-scale.sh index c61dff32f2..454b289dba 100755 --- a/scripts/devnet-test-rfc38-scale.sh +++ b/scripts/devnet-test-rfc38-scale.sh @@ -119,7 +119,7 @@ QUADS_PAYLOAD=$(STAMP="$STAMP" CG_ID="$CG_ID" TRIPLE_COUNT="$TRIPLE_COUNT" node } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') -WRITE_RESP=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$QUADS_PAYLOAD") +WRITE_RESP=$(devnet_create_shared_ka "$CURATOR_NODE" "$QUADS_PAYLOAD") WRITTEN=$(parse_json "$WRITE_RESP" '.triplesWritten') [ "$WRITTEN" = "$TRIPLE_COUNT" ] || fail "expected $TRIPLE_COUNT triples written, got '$WRITTEN' — $WRITE_RESP" log "✓ $WRITTEN triples written to SWM" diff --git a/scripts/devnet-test-rfc38-unclean-restart.sh b/scripts/devnet-test-rfc38-unclean-restart.sh index 7e8df1dfc2..c45ebfbdca 100755 --- a/scripts/devnet-test-rfc38-unclean-restart.sh +++ b/scripts/devnet-test-rfc38-unclean-restart.sh @@ -45,6 +45,9 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=devnet-publish-helpers.sh +source "$SCRIPT_DIR/devnet-publish-helpers.sh" DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}" API_PORT_BASE=9201 CURATOR_NODE=5 @@ -66,7 +69,7 @@ CORE_NODE=1 # above the 100 ms poll cadence below. Operators on slower boxes can # dial these back via the env vars. # -# Codex r2 RED on #777: a single `/api/shared-memory/write` POST is +# Codex r2 RED on #777: a single `/api/knowledge-assets` POST is # capped at `MAX_BODY_BYTES = 10 MiB` by the daemon — packing all # 1000 × 32 KiB quads into one request would 413 before catchup # starts. Split the writes across multiple `/write` calls of at most @@ -239,7 +242,7 @@ while [ "$BATCH_START" -lt "$WRITES_COUNT" ]; do } console.log(JSON.stringify({ contextGraphId: cgId, quads })); ') - W=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$PAYLOAD") + W=$(devnet_create_shared_ka "$CURATOR_NODE" "$PAYLOAD") GOT=$(parse_json "$W" '.triplesWritten') [ "$GOT" = "$BATCH_LEN" ] || fail "batch [$BATCH_START..$BATCH_END) expected $BATCH_LEN triples, got '$GOT': $W" TOTAL_WRITTEN=$(( TOTAL_WRITTEN + BATCH_LEN )) diff --git a/scripts/devnet-test-rfc39-comprehensive.sh b/scripts/devnet-test-rfc39-comprehensive.sh index 60d6047b8f..0a3254cfa1 100755 --- a/scripts/devnet-test-rfc39-comprehensive.sh +++ b/scripts/devnet-test-rfc39-comprehensive.sh @@ -346,7 +346,7 @@ EOF ;; *) fail "Scenario $tag: unknown payload mode '$payload_mode'";; esac - write_resp=$(api_call "$EDGE_CURATOR_NODE" POST /api/shared-memory/write "$write_body") + write_resp=$(DEVNET_PUBLISH_PRESERVE_BATCH=1 devnet_create_shared_ka "$EDGE_CURATOR_NODE" "$write_body") local triples_written triples_written=$(printf '%s' "$write_resp" | node -e ' let d=""; process.stdin.on("data",c=>d+=c); @@ -388,6 +388,7 @@ EOF log " ciphertextChunkCount: $ct_count" log " plain merkleRoot: $plain_root (== batchId)" log " plain merkleLeafCount: $plain_count" + [ "$plain_count" = "$triples_written" ] || fail "Scenario $tag: preserve-batch publish plain merkleLeafCount=$plain_count, expected triplesWritten=$triples_written" local zero_root="0x0000000000000000000000000000000000000000000000000000000000000000" if [ "$access_policy" = "1" ]; then @@ -573,7 +574,7 @@ EOF log "Writing multi-chunk SWM payload to skew picker weight..." local write_body write_resp write_body=$(build_swm_write_payload "$cg_uri" "$stamp" 98304) - write_resp=$(api_call "$EDGE_CURATOR_NODE" POST /api/shared-memory/write "$write_body") + write_resp=$(devnet_create_shared_ka "$EDGE_CURATOR_NODE" "$write_body") local triples_written triples_written=$(printf '%s' "$write_resp" | node -e 'let d="";process.stdin.on("data",c=>d+=c);process.stdin.on("end",()=>{try{console.log(JSON.parse(d).triplesWritten||0)}catch(e){console.log(0)}})' 2>/dev/null || echo 0) [ "$triples_written" -ge 1 ] || fail "Scenario D: SWM write reported zero triples: $write_resp" diff --git a/scripts/devnet-test-rfc39-curated-random-sampling.sh b/scripts/devnet-test-rfc39-curated-random-sampling.sh index b48fd30677..4d83cc87cb 100755 --- a/scripts/devnet-test-rfc39-curated-random-sampling.sh +++ b/scripts/devnet-test-rfc39-curated-random-sampling.sh @@ -114,7 +114,7 @@ CG_URI="${CG_LOCAL_ID}" # --- 4. Write triples to SWM ------------------------------------------------- # Use a fairly large set of triples to encourage multi-chunk splitting. log "Writing 12 triples into SWM..." -WRITE_RESP=$(api_call "$EDGE_CURATOR_NODE" POST /api/shared-memory/write "$(cat </dev/null @@ -475,54 +475,6 @@ done pass "member edge$EDGE_MEMBER converged to the UPDATED private payload — the curated update DISTRIBUTES to members (producer emit + curator-held + converge-on-reconnect)" # --------------------------------------------------------------------------- -# 10. FROM-SWM SHORTCUT (OT-RFC-49 catalog auto-injection): a curated publish via -# the RAW /api/shared-memory/write + /api/shared-memory/publish path (NO -# assertion finalize) must ALSO get the `_catalog` injected → confirmed ACK + -# on-chain catalog commitment + cores hold it. Sections 1-9 above (KA-lifecycle -# finalize path) double as the REGRESSION baseline: their catalogLeafCount=4 -# proves the idempotency guard left the finalize path intact. -# --------------------------------------------------------------------------- -log "── FROM-SWM SHORTCUT path (raw write+publish, no finalize) ──" -SC_CG="rfc49-shortcut-${STAMP}" -SC_URI="did:dkg:context-graph:${SC_CG}" -SC_SUBJ="urn:rfc49:secret-shortcut:${STAMP}/carol" -SC_CREATE=$(api_call "$EDGE_CURATOR" POST /api/context-graph/create "{\"id\":\"${SC_CG}\",\"name\":\"RFC-49 from-SWM shortcut ${STAMP}\",\"accessPolicy\":1,\"publishPolicy\":0,\"allowedAgents\":[\"${CURATOR_AGENT}\",\"${MEMBER_AGENT}\"],\"register\":true}") -SC_OID=$(printf '%s' "$SC_CREATE" | jq_field ".onChainId") -[ -n "$SC_OID" ] || fail "shortcut CG create did not return onChainId: $SC_CREATE" -log "shortcut CG on-chain id = $SC_OID" -sleep 4 -# mirror the main flow: a subscribed member establishes the curated sender-key -# recipient set so the curator's SWM write lands + is reloadable by the publish. -api_call "$EDGE_MEMBER" POST /api/subscribe "{\"contextGraphId\":\"${SC_CG}\",\"includeSharedMemory\":true}" >/dev/null 2>&1 -sleep 4 -SC_WRITE=$(api_call "$EDGE_CURATOR" POST /api/shared-memory/write "{\"contextGraphId\":\"${SC_URI}\",\"quads\":[{\"subject\":\"${SC_SUBJ}\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"Carol via the raw from-SWM shortcut ${STAMP}, padded to exercise the encrypted member payload end-to-end\\\"\",\"graph\":\"\"},{\"subject\":\"${SC_SUBJ}\",\"predicate\":\"http://schema.org/email\",\"object\":\"\\\"carol-${STAMP}@example.org\\\"\",\"graph\":\"\"}]}") -log "raw SWM write: $SC_WRITE" -log "publishing via /api/shared-memory/publish (no finalize)…" -sleep 5 -SC_PUB=$(devnet_publish_swm_all_roots "$EDGE_CURATOR" "${SC_URI}" false) -SC_STATUS=$(printf '%s' "$SC_PUB" | jq_field ".status") -SC_KA=$(printf '%s' "$SC_PUB" | jq_field ".kaId"); [ -z "$SC_KA" ] && SC_KA=$(printf '%s' "$SC_PUB" | jq_field ".knowledgeAssetId") -[ "$SC_STATUS" = "confirmed" ] || fail "FROM-SWM shortcut publish status=$SC_STATUS (catalog auto-injection did NOT unblock the curated ACK): $SC_PUB" -[ -n "$SC_KA" ] && [ "$SC_KA" != "0" ] || fail "FROM-SWM shortcut: no kaId: $SC_PUB" -pass "FROM-SWM shortcut publish confirmed (kaId=$SC_KA) — catalog auto-injection unblocked the curated ACK" -# on-chain catalog commitment for the shortcut KA -SC_CHAIN=$(cd "$REPO_ROOT/packages/evm-module" && RPC_URL="http://127.0.0.1:$HARDHAT_PORT" CONTRACTS_JSON="$CONTRACTS_JSON" ABI_DIR="$ABI_DIR" KA_ID="$SC_KA" node -e ' -const {ethers}=require("ethers");const fs=require("fs");const path=require("path"); -(async()=>{const p=new ethers.JsonRpcProvider(process.env.RPC_URL);const c=JSON.parse(fs.readFileSync(process.env.CONTRACTS_JSON,"utf8")).contracts;const a=c.DKGKnowledgeAssets?.evmAddress||c.DKGKnowledgeAssets?.address;const abi=JSON.parse(fs.readFileSync(path.join(process.env.ABI_DIR,"DKGKnowledgeAssets.json"),"utf8"));const k=new ethers.Contract(a,abi,p);console.log(JSON.stringify({root:await k.getCatalogRoot(BigInt(process.env.KA_ID)),count:(await k.getCatalogLeafCount(BigInt(process.env.KA_ID))).toString()}))})().catch(e=>{console.error(e?.message||e);process.exit(1)});') || fail "shortcut on-chain catalog read failed" -SC_ROOT=$(printf '%s' "$SC_CHAIN" | jq_field ".root"); SC_COUNT=$(printf '%s' "$SC_CHAIN" | jq_field ".count") -[ "$SC_ROOT" != "$ZERO" ] || fail "FROM-SWM shortcut: on-chain catalogRoot is ZERO — injection produced no catalog" -[ "${SC_COUNT:-0}" -ge 1 ] 2>/dev/null || fail "FROM-SWM shortcut: catalogLeafCount < 1" -pass "FROM-SWM shortcut: on-chain catalog commitment set (root non-zero, leafCount=$SC_COUNT) — same as the finalize path" -# a core holds the injected catalog -ONCHAIN_ID="$SC_OID" -SC_HELD=0 -for i in $(seq 1 60); do - for n in "${STRIPPED_CORES[@]}"; do c=$(catalog_count "$n"); [ "${c:-0}" -ge 1 ] 2>/dev/null && { SC_HELD=1; break; }; done - [ "$SC_HELD" -eq 1 ] && break; sleep 3 -done -[ "$SC_HELD" -eq 1 ] || fail "FROM-SWM shortcut: no stripped core holds the injected _catalog" -pass "FROM-SWM shortcut: stripped cores hold the injected public _catalog" - echo "" log "============================================================" pass "RFC-49 CATALOG-SAMPLING STRIP — DEVNET VALIDATION PASSED" @@ -531,5 +483,4 @@ log " • stripped cores ${STRIPPED_CORES[*]}: ZERO private ciphertext, hold th log " • ${BASELINE_SUMMARY:-baseline core: (not evaluated)}" log " • a core proved the public _catalog in random sampling" log " • curated UPDATE (POST /api/update): re-committed the stable catalog floor (root==baseline, leaves=${UPD_COUNT:-?}), cores re-host it, a core proved the updated catalog" -log " • FROM-SWM shortcut (raw write+publish, no finalize): catalog auto-injected (leafCount=$SC_COUNT), cores hold it — and the finalize path above stayed intact (leafCount=$CAT_COUNT)" log "============================================================" diff --git a/scripts/devnet-test-sharing.sh b/scripts/devnet-test-sharing.sh index d2eff7eb16..38e2ed0856 100755 --- a/scripts/devnet-test-sharing.sh +++ b/scripts/devnet-test-sharing.sh @@ -737,8 +737,8 @@ c -X POST "http://127.0.0.1:9201/api/knowledge-assets/wm-secret/wm/write" \ ok "WM data written to join-flow project" echo "--- 13c: Promote some data to SWM on Node 1 ---" -c -X POST "http://127.0.0.1:9201/api/shared-memory/write" \ - -d "{\"contextGraphId\":\"$CG2_ID\",\"quads\":[ +c -X POST "http://127.0.0.1:9201/api/knowledge-assets" \ + -d "{\"contextGraphId\":\"$CG2_ID\",\"name\":\"swm-shared\",\"finalize\":true,\"alsoShareSwm\":true,\"quads\":[ $(ql 'urn:join-flow:shared1' 'http://schema.org/name' 'Shared Data'), $(q 'urn:join-flow:shared1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Thing') ]}" > /dev/null @@ -987,26 +987,20 @@ else fi sleep 3 -echo "--- 15a: Publish SWM data to VM on Node 1 (promote-test project) ---" -PUBLISH=$(devnet_publish_swm_all_roots 1 "$CG3_ID" false) -PUB_STATUS=$(json_get "$PUBLISH" status) -PUB_KCID=$(json_get "$PUBLISH" kaId) -PUB_KAS=$(echo "$PUBLISH" | python3 -c ' -import sys,json -try: - d=json.load(sys.stdin) - kas=d.get("kas",[]) - print(len(kas)) -except: print("ERR") -' 2>/dev/null) -PUB_TX=$(json_get "$PUBLISH" txHash) -echo " Publish result: status=$PUB_STATUS kaId=$PUB_KCID kas=$PUB_KAS txHash=${PUB_TX:0:20}..." -if [[ "$PUB_STATUS" == "published" || "$PUB_STATUS" == "created" || "$PUB_STATUS" == "mined" ]]; then - ok "SWM published to VM (status=$PUB_STATUS, $PUB_KAS knowledge asset(s))" -elif [[ "$PUB_KCID" != "__NONE__" && "$PUB_KCID" != "__ERR__" ]]; then - ok "SWM publish completed (kaId=$PUB_KCID, status=$PUB_STATUS)" +echo "--- 15a: Publish named KAs to VM on Node 1 (promote-test project) ---" +PUBLISH_DOC=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets/promo-doc/vm/publish" \ + -d "{\"contextGraphId\":\"$CG3_ID\",\"options\":{\"clearAfter\":false}}") +PUBLISH_API=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets/api-draft/vm/publish" \ + -d "{\"contextGraphId\":\"$CG3_ID\",\"options\":{\"clearAfter\":false}}") +PUB_STATUS=$(json_get "$PUBLISH_API" status) +PUB_KCID=$(json_get "$PUBLISH_API" kaId) +PUB_TX=$(json_get "$PUBLISH_API" txHash) +DOC_STATUS=$(json_get "$PUBLISH_DOC" status) +echo " promo-doc status=$DOC_STATUS; api-draft status=$PUB_STATUS kaId=$PUB_KCID txHash=${PUB_TX:0:20}..." +if [[ ( "$DOC_STATUS" == "confirmed" || "$DOC_STATUS" == "finalized" ) && ( "$PUB_STATUS" == "confirmed" || "$PUB_STATUS" == "finalized" ) ]]; then + ok "Named KAs published to VM (promo-doc + api-draft)" else - fail "SWM publish failed: $(echo "$PUBLISH" | head -c 300)" + fail "Named KA publish failed: promo-doc=${PUBLISH_DOC:0:180} api-draft=${PUBLISH_API:0:180}" fi echo "--- 15b: Verify VM data on Node 1 ---" @@ -1081,25 +1075,20 @@ else fi sleep 3 -echo "--- 16a: Publish CG1 SWM → VM on Node 1 ---" -PUB_CG1=$(devnet_publish_swm_all_roots 1 "$CG_ID" false) -PUB_CG1_STATUS=$(json_get "$PUB_CG1" status) -PUB_CG1_KAS=$(echo "$PUB_CG1" | python3 -c ' -import sys,json -try: print(len(json.load(sys.stdin).get("kas",[]))) -except: print("ERR") -' 2>/dev/null) -if [[ "$PUB_CG1_STATUS" == "published" || "$PUB_CG1_STATUS" == "created" || "$PUB_CG1_STATUS" == "mined" ]]; then - ok "CG1 SWM published to VM ($PUB_CG1_KAS KAs, status=$PUB_CG1_STATUS)" +echo "--- 16a: Publish CG1 named KAs to VM on Node 1 ---" +PUB_CG1_BETA=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets/draft-beta/vm/publish" \ + -d "{\"contextGraphId\":\"$CG_ID\",\"options\":{\"clearAfter\":false}}") +PUB_CG1_DOC=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets/doc-alpha/vm/publish" \ + -d "{\"contextGraphId\":\"$CG_ID\",\"options\":{\"clearAfter\":false}}") +PUB_CG1_STATUS=$(json_get "$PUB_CG1_BETA" status) +PUB_CG1_DOC_STATUS=$(json_get "$PUB_CG1_DOC" status) +if [[ ( "$PUB_CG1_STATUS" == "confirmed" || "$PUB_CG1_STATUS" == "finalized" ) && ( "$PUB_CG1_DOC_STATUS" == "confirmed" || "$PUB_CG1_DOC_STATUS" == "finalized" ) ]]; then + ok "CG1 named KAs published to VM (draft-beta + doc-alpha)" else - PUB_CG1_KCID=$(json_get "$PUB_CG1" kaId) - if [[ "$PUB_CG1_KCID" != "__NONE__" && "$PUB_CG1_KCID" != "__ERR__" ]]; then - ok "CG1 SWM publish completed (kaId=$PUB_CG1_KCID, status=$PUB_CG1_STATUS)" - else - fail "CG1 SWM publish failed: $(echo "$PUB_CG1" | head -c 300)" - fi + fail "CG1 named KA publish failed: draft-beta=${PUB_CG1_BETA:0:180} doc-alpha=${PUB_CG1_DOC:0:180}" fi +echo "--- 16b: Verify VM data on Node 1 for private CG ---" echo "--- 16b: Node 2 (participant) sees VM data ---" N2_CG1_VM_OK=false for i in $(seq 1 20); do @@ -1156,7 +1145,13 @@ for port_label in "9202:Node2" "9204:Node4" "9203:Node3"; do done echo "--- 16f: Publish with clearAfter=true, then verify SWM is empty ---" -PUB_CLEAR=$(devnet_publish_swm_all_roots 1 "$CG3_ID" true) +CLEAR_NAME="clear-after-smoke" +c -X POST "http://127.0.0.1:9201/api/knowledge-assets" \ + -d "{\"contextGraphId\":\"$CG3_ID\",\"name\":\"$CLEAR_NAME\",\"quads\":[ + $(ql 'urn:promote-test:clear-after' 'http://schema.org/name' 'Clear After Smoke') + ],\"finalize\":true,\"alsoShareSwm\":true}" > /dev/null +PUB_CLEAR=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets/$CLEAR_NAME/vm/publish" \ + -d "{\"contextGraphId\":\"$CG3_ID\",\"options\":{\"clearAfter\":true}}") PUB_CLEAR_STATUS=$(json_get "$PUB_CLEAR" status) sleep 2 N1_SWM_CLEARED=$(c -X POST "http://127.0.0.1:9201/api/query" \ diff --git a/scripts/devnet-test-swm-late-joiner-subgraph.sh b/scripts/devnet-test-swm-late-joiner-subgraph.sh index 144e431fa3..7130d63e88 100755 --- a/scripts/devnet-test-swm-late-joiner-subgraph.sh +++ b/scripts/devnet-test-swm-late-joiner-subgraph.sh @@ -51,6 +51,9 @@ set -euo pipefail REPO_ROOT="$(cd "$(dirname "$0")/.." && pwd)" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=devnet-publish-helpers.sh +source "$SCRIPT_DIR/devnet-publish-helpers.sh" DEVNET_DIR="${DEVNET_DIR:-$REPO_ROOT/.devnet}" API_PORT_BASE="${API_PORT_BASE:-9201}" CURATOR_NODE="${CURATOR_NODE:-5}" @@ -172,14 +175,14 @@ log "✓ sub-graph \"$SUB_GRAPH_NAME\" created" act "STEP 3: curator publishes $SUB_GRAPH_TRIPLES SWM triples into sub-graph + $ROOT_TRIPLES into root" # =========================================================================== SUB_PAYLOAD=$(build_swm_payload "$CG_ID" "$SUB_GRAPH_TRIPLES" "sub" "$SUB_GRAPH_NAME") -WROTE_SUB=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$SUB_PAYLOAD") +WROTE_SUB=$(devnet_create_shared_ka "$CURATOR_NODE" "$SUB_PAYLOAD") TRIPLES_WROTE_SUB=$(parse_json "$WROTE_SUB" '.triplesWritten') [ "$TRIPLES_WROTE_SUB" = "$SUB_GRAPH_TRIPLES" ] \ || fail "expected $SUB_GRAPH_TRIPLES sub-graph triplesWritten, got '$TRIPLES_WROTE_SUB' (response: $WROTE_SUB)" log "✓ wrote $SUB_GRAPH_TRIPLES SWM triples into sub-graph \"$SUB_GRAPH_NAME\"" ROOT_PAYLOAD=$(build_swm_payload "$CG_ID" "$ROOT_TRIPLES" "root" "") -WROTE_ROOT=$(api_call "$CURATOR_NODE" POST /api/shared-memory/write "$ROOT_PAYLOAD") +WROTE_ROOT=$(devnet_create_shared_ka "$CURATOR_NODE" "$ROOT_PAYLOAD") TRIPLES_WROTE_ROOT=$(parse_json "$WROTE_ROOT" '.triplesWritten') [ "$TRIPLES_WROTE_ROOT" = "$ROOT_TRIPLES" ] \ || fail "expected $ROOT_TRIPLES root triplesWritten, got '$TRIPLES_WROTE_ROOT' (response: $WROTE_ROOT)" diff --git a/scripts/devnet-test.sh b/scripts/devnet-test.sh index ffd4f2f443..c78d5d8bd3 100755 --- a/scripts/devnet-test.sh +++ b/scripts/devnet-test.sh @@ -183,30 +183,46 @@ http_post_capture() { q() { echo "{\"subject\":\"$1\",\"predicate\":\"$2\",\"object\":\"$3\",\"graph\":\"\"}"; } ql() { echo "{\"subject\":\"$1\",\"predicate\":\"$2\",\"object\":\"\\\"$3\\\"\",\"graph\":\"\"}"; } -swm_publish() { - local port=$1 cg=$2 - shift 2 +ka_create_share() { + local port=$1 cg=$2 name=$3 sub_graph=$4 + shift 4 local quads="$*" - - local write_resp - write_resp=$(c -X POST "http://127.0.0.1:$port/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$cg\", - \"quads\":[$quads] - }") - local write_ok - write_ok=$(json_get "$write_resp" triplesWritten) - if [[ "$write_ok" == "__NONE__" || "$write_ok" == "0" ]]; then - echo "$write_resp" - return 1 + local sub_graph_json="" + if [[ -n "$sub_graph" ]]; then + sub_graph_json=",\"subGraphName\":\"$sub_graph\"" fi + c -X POST "http://127.0.0.1:$port/api/knowledge-assets" -d "{ + \"contextGraphId\":\"$cg\", + \"name\":\"$name\"$sub_graph_json, + \"quads\":[$quads], + \"finalize\":true, + \"alsoShareSwm\":true + }" +} - sleep 2 +ka_publish() { + local port=$1 cg=$2 name=$3 + c -X POST "http://127.0.0.1:$port/api/knowledge-assets/$name/vm/publish" -d "{ + \"contextGraphId\":\"$cg\" + }" +} - local pub_resp - pub_resp=$(c -X POST "http://127.0.0.1:$port/api/shared-memory/publish" -d "{ +ka_publish_async() { + local port=$1 cg=$2 name=$3 + c -X POST "http://127.0.0.1:$port/api/knowledge-assets/$name/vm/publish-async" -d "{ \"contextGraphId\":\"$cg\" - }") - echo "$pub_resp" + }" +} + +retired_post_assert() { + local desc="$1" port="$2" path="$3" body="$4" + local retired_body retired_code + http_post_capture "http://127.0.0.1:$port$path" "$body" retired_body retired_code + if [[ "$retired_code" == "404" || "$retired_code" == "410" ]]; then + ok "$desc retired (HTTP $retired_code)" + else + fail "$desc unexpectedly served (HTTP $retired_code): ${retired_body:0:200}" + fi } DEVNET_NODES="${DEVNET_NODES:-}" @@ -297,23 +313,21 @@ echo "=== SECTION 2: Shared Memory Writes (free operations) ===" echo "" TRAC_BEFORE=$(c "http://127.0.0.1:9202/api/wallets/balances" | python3 -c "import sys,json; print(json.load(sys.stdin)['balances'][0]['trac'])" 2>/dev/null) -echo " Node2 TRAC before SWM write: $TRAC_BEFORE" +echo " Node2 TRAC before KA create/share: $TRAC_BEFORE" -SWM_W=$(c -X POST "http://127.0.0.1:9202/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[ +ALICE_BOB_KA="alice-bob-$(date +%s%N)" +SWM_W=$(ka_create_share 9202 "$CONTEXT_GRAPH" "$ALICE_BOB_KA" "" \ $(ql 'http://example.org/entity/alice' 'http://schema.org/name' 'Alice'), $(ql 'http://example.org/entity/alice' 'http://schema.org/age' '30'), $(q 'http://example.org/entity/alice' 'http://schema.org/knows' 'http://example.org/entity/bob'), $(ql 'http://example.org/entity/bob' 'http://schema.org/name' 'Bob'), $(ql 'http://example.org/entity/bob' 'http://schema.org/age' '25') - ] -}") -swm_written=$(json_get "$SWM_W" triplesWritten) -[[ "$swm_written" != "__NONE__" && "$swm_written" != "0" ]] && ok "SWM write OK ($swm_written triples)" || fail "SWM write failed: $SWM_W" +) +swm_written=$(json_get "$SWM_W" promotedCount) +[[ "$swm_written" != "__NONE__" && "$swm_written" != "0" ]] && ok "KA create/share OK ($swm_written triples)" || fail "KA create/share failed: $SWM_W" TRAC_AFTER=$(c "http://127.0.0.1:9202/api/wallets/balances" | python3 -c "import sys,json; print(json.load(sys.stdin)['balances'][0]['trac'])" 2>/dev/null) -check "SWM write is FREE (TRAC unchanged)" "$TRAC_BEFORE" "$TRAC_AFTER" +check "KA create/share is FREE (TRAC unchanged)" "$TRAC_BEFORE" "$TRAC_AFTER" echo "" echo "--- 2b: Query SWM locally ---" @@ -350,33 +364,43 @@ echo "=== SECTION 3: PUBLISH via SWM-first flow (WM→SWM→VM) ===" echo "" echo "--- 3a: Write + Publish from Node1 (core) ---" -c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[ +CITY1_KA="city1-ka-$(date +%s%N)" +CITY2_KA="city2-ka-$(date +%s%N)" +CITY1_SHARE=$(ka_create_share 9201 "$CONTEXT_GRAPH" "$CITY1_KA" "" \ $(ql 'http://example.org/entity/city1' 'http://schema.org/name' 'Ljubljana'), $(q 'http://example.org/entity/city1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/City'), - $(ql 'http://example.org/entity/city1' 'http://schema.org/population' '290000'), + $(ql 'http://example.org/entity/city1' 'http://schema.org/population' '290000') +) +CITY1_SHARED=$(json_get "$CITY1_SHARE" swmShared) +[[ "$CITY1_SHARED" == "true" ]] && ok "$CITY1_KA shared to SWM" || fail "$CITY1_KA share failed: $CITY1_SHARE" +CITY2_SHARE=$(ka_create_share 9201 "$CONTEXT_GRAPH" "$CITY2_KA" "" \ $(ql 'http://example.org/entity/city2' 'http://schema.org/name' 'Maribor'), $(q 'http://example.org/entity/city2' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/City'), $(ql 'http://example.org/entity/city2' 'http://schema.org/population' '95000') - ] -}" > /dev/null +) +CITY2_SHARED=$(json_get "$CITY2_SHARE" swmShared) +[[ "$CITY2_SHARED" == "true" ]] && ok "$CITY2_KA shared to SWM" || fail "$CITY2_KA share failed: $CITY2_SHARE" sleep 2 -PUB1=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/publish" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"selection\":[\"http://example.org/entity/city1\",\"http://example.org/entity/city2\"] -}") +PUB1=$(ka_publish 9201 "$CONTEXT_GRAPH" "$CITY1_KA") +PUB2=$(ka_publish 9201 "$CONTEXT_GRAPH" "$CITY2_KA") PUB1_ST=$(json_get "$PUB1" status) PUB1_KC=$(json_get "$PUB1" kaId) PUB1_TX=$(json_get "$PUB1" txHash) PUB1_BN=$(json_get "$PUB1" blockNumber) -PUB1_KAS=$(echo "$PUB1" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('kas',[])))" 2>/dev/null) +PUB2_ST=$(json_get "$PUB2" status) +PUB2_KC=$(json_get "$PUB2" kaId) +PUB2_TX=$(json_get "$PUB2" txHash) +PUB1_KAS=0 +[[ "$PUB1_ST" == "confirmed" || "$PUB1_ST" == "finalized" ]] && PUB1_KAS=$((PUB1_KAS + 1)) +[[ "$PUB2_ST" == "confirmed" || "$PUB2_ST" == "finalized" ]] && PUB1_KAS=$((PUB1_KAS + 1)) -echo " status=$PUB1_ST kaId=$PUB1_KC tx=$PUB1_TX block=$PUB1_BN KAs=$PUB1_KAS" -[[ "$PUB1_ST" == "confirmed" || "$PUB1_ST" == "finalized" ]] && ok "Publish from SWM succeeded ($PUB1_ST)" || fail "Publish status=$PUB1_ST: $PUB1" -[[ "$PUB1_TX" != "__NONE__" ]] && ok "On-chain tx: $PUB1_TX" || fail "No txHash" -[[ "$PUB1_KAS" == "2" ]] && ok "Published 2 KAs (both selected roots)" || fail "Expected 2 KAs, got $PUB1_KAS" +echo " city1: status=$PUB1_ST kaId=$PUB1_KC tx=$PUB1_TX block=$PUB1_BN" +echo " city2: status=$PUB2_ST kaId=$PUB2_KC tx=$PUB2_TX" +[[ "$PUB1_ST" == "confirmed" || "$PUB1_ST" == "finalized" ]] && ok "$CITY1_KA VM publish succeeded ($PUB1_ST)" || fail "$CITY1_KA publish status=$PUB1_ST: $PUB1" +[[ "$PUB2_ST" == "confirmed" || "$PUB2_ST" == "finalized" ]] && ok "$CITY2_KA VM publish succeeded ($PUB2_ST)" || fail "$CITY2_KA publish status=$PUB2_ST: $PUB2" +[[ "$PUB1_TX" != "__NONE__" && "$PUB2_TX" != "__NONE__" ]] && ok "On-chain txs: $PUB1_TX / $PUB2_TX" || fail "Missing city publish txHash" +[[ "$PUB1_KAS" == "2" ]] && ok "Published 2 named KAs ($CITY1_KA, $CITY2_KA)" || fail "Expected 2 named KA publishes, got $PUB1_KAS" echo "" echo "--- 3b: Query Verifiable Memory for published city root entities on publisher ---" @@ -416,57 +440,58 @@ echo "" # publish-authority rejection is covered by private/curated sharing tests. echo "--- 4a: Node2 (core) shares a Product triple-set to SWM ---" -SWM2=$(c -X POST "http://127.0.0.1:9202/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[ +PRODUCT1_KA="product1-ka-$(date +%s%N)" +PRODUCT2_KA="product2-ka-$(date +%s%N)" +PERSON1_KA="person1-ka-$(date +%s%N)" +LAKE1_KA="lake1-ka-$(date +%s%N)" +LAKE1_PUBLISH_KA="lake1-publish-ka-$(date +%s%N)" +SWM2=$(ka_create_share 9202 "$CONTEXT_GRAPH" "$PRODUCT1_KA" "" \ $(ql 'http://example.org/entity/product1' 'http://schema.org/name' 'Potica'), $(q 'http://example.org/entity/product1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Product'), $(ql 'http://example.org/entity/product1' 'http://schema.org/description' 'Traditional Slovenian nut roll') - ] -}") -SWM2_W=$(json_get "$SWM2" triplesWritten) -[[ "$SWM2_W" == "3" ]] && ok "Node2 SWM contribution accepted ($SWM2_W triples)" || fail "Node2 SWM write: $SWM2" +) +SWM2_W=$(json_get "$SWM2" promotedCount) +[[ "$SWM2_W" == "3" ]] && ok "Node2 SWM contribution accepted ($SWM2_W triples)" || fail "Node2 KA create/share: $SWM2" echo "--- 4b: Node3 (core, oxigraph) shares a second Product triple-set ---" -SWM3=$(c -X POST "http://127.0.0.1:9203/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[ +SWM3=$(ka_create_share 9203 "$CONTEXT_GRAPH" "$PRODUCT2_KA" "" \ $(ql 'http://example.org/entity/product2' 'http://schema.org/name' 'Carniolan Sausage'), $(q 'http://example.org/entity/product2' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Product'), $(ql 'http://example.org/entity/product2' 'http://schema.org/description' 'PGI sausage') - ] -}") -SWM3_W=$(json_get "$SWM3" triplesWritten) -[[ "$SWM3_W" == "3" ]] && ok "Node3 SWM contribution accepted ($SWM3_W triples)" || fail "Node3 SWM write: $SWM3" +) +SWM3_W=$(json_get "$SWM3" promotedCount) +[[ "$SWM3_W" == "3" ]] && ok "Node3 SWM contribution accepted ($SWM3_W triples)" || fail "Node3 KA create/share: $SWM3" echo "--- 4c: Node4 (core) shares a Person triple-set to SWM ---" -SWM4=$(c -X POST "http://127.0.0.1:9204/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[ +SWM4=$(ka_create_share 9204 "$CONTEXT_GRAPH" "$PERSON1_KA" "" \ $(ql 'http://example.org/entity/person1' 'http://schema.org/name' 'France Prešeren'), $(q 'http://example.org/entity/person1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Person'), $(ql 'http://example.org/entity/person1' 'http://schema.org/birthDate' '1800-12-03') - ] -}") -SWM4_W=$(json_get "$SWM4" triplesWritten) -[[ "$SWM4_W" == "3" ]] && ok "Node4 SWM contribution accepted ($SWM4_W triples)" || fail "Node4 SWM write: $SWM4" +) +SWM4_W=$(json_get "$SWM4" promotedCount) +[[ "$SWM4_W" == "3" ]] && ok "Node4 SWM contribution accepted ($SWM4_W triples)" || fail "Node4 KA create/share: $SWM4" echo "--- 4d: Node5 (edge) shares a Lake triple-set to SWM ---" -SWM5=$(c -X POST "http://127.0.0.1:9205/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[ +SWM5=$(ka_create_share 9205 "$CONTEXT_GRAPH" "$LAKE1_KA" "" \ $(ql 'http://example.org/entity/lake1' 'http://schema.org/name' 'Lake Bled'), $(q 'http://example.org/entity/lake1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/LakeBodyOfWater'), $(ql 'http://example.org/entity/lake1' 'http://schema.org/description' 'Glacial lake in the Julian Alps') - ] -}") -SWM5_W=$(json_get "$SWM5" triplesWritten) -[[ "$SWM5_W" == "3" ]] && ok "Node5 (edge) SWM contribution accepted ($SWM5_W triples)" || fail "Node5 SWM write: $SWM5" +) +SWM5_W=$(json_get "$SWM5" promotedCount) +[[ "$SWM5_W" == "3" ]] && ok "Node5 (edge) SWM contribution accepted ($SWM5_W triples)" || fail "Node5 KA create/share: $SWM5" + +LAKE1_CORE_SHARE=$(ka_create_share 9201 "$CONTEXT_GRAPH" "$LAKE1_PUBLISH_KA" "" \ + $(ql 'http://example.org/entity/lake1' 'http://schema.org/name' 'Lake Bled'), + $(q 'http://example.org/entity/lake1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/LakeBodyOfWater'), + $(ql 'http://example.org/entity/lake1' 'http://schema.org/description' 'Glacial lake in the Julian Alps') +) +LAKE1_CORE_SHARED=$(json_get "$LAKE1_CORE_SHARE" swmShared) +[[ "$LAKE1_CORE_SHARED" == "true" ]] && ok "Curator-owned lake KA shared for VM publish" || fail "Curator lake KA share failed: ${LAKE1_CORE_SHARE:0:200}" echo "--- 4e: Open CG allows Node2 to publish its SWM contribution ---" sleep 2 -http_post_capture "http://127.0.0.1:9202/api/shared-memory/publish" \ - "{\"contextGraphId\":\"$CONTEXT_GRAPH\",\"selection\":[\"http://example.org/entity/product1\"]}" \ +http_post_capture "http://127.0.0.1:9202/api/knowledge-assets/$PRODUCT1_KA/vm/publish" \ + "{\"contextGraphId\":\"$CONTEXT_GRAPH\"}" \ NON_CURATOR_BODY NON_CURATOR_CODE NON_CURATOR_ST=$(json_get "$NON_CURATOR_BODY" status) if [[ "$NON_CURATOR_CODE" == "200" && ( "$NON_CURATOR_ST" == "confirmed" || "$NON_CURATOR_ST" == "finalized" ) ]]; then @@ -475,26 +500,22 @@ else fail "Open-CG publish from Node2 failed, HTTP $NON_CURATOR_CODE status=$NON_CURATOR_ST: ${NON_CURATOR_BODY:0:200}" fi -# Aggregated promote: Node1 picks up the remaining SWM contributions in a -# single on-chain tx. Each entity becomes its own KA (rootEntity), but they -# share one on-chain batch. -echo "--- 4f: Node1 publishes the remaining aggregated multi-node SWM batch ---" +# Named lifecycle publish: each chain-capable contributor promotes its KA name; +# the edge payload is published through a curator-owned KA with the same triples. +echo "--- 4f: Remaining named KA publishes succeed across contributors ---" sleep 2 -AGG_PUB=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/publish" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"selection\":[ - \"http://example.org/entity/product1\", - \"http://example.org/entity/product2\", - \"http://example.org/entity/person1\", - \"http://example.org/entity/lake1\" - ] -}") -AGG_ST=$(json_get "$AGG_PUB" status) -AGG_TX=$(json_get "$AGG_PUB" txHash) -if [[ "$AGG_ST" == "confirmed" || "$AGG_ST" == "finalized" ]]; then - ok "Curator aggregated publish OK (status=$AGG_ST, tx=${AGG_TX:0:18}…)" +AGG_PRODUCT2=$(ka_publish 9203 "$CONTEXT_GRAPH" "$PRODUCT2_KA") +AGG_PERSON1=$(ka_publish 9204 "$CONTEXT_GRAPH" "$PERSON1_KA") +AGG_LAKE1=$(ka_publish 9201 "$CONTEXT_GRAPH" "$LAKE1_PUBLISH_KA") +AGG_PRODUCT2_ST=$(json_get "$AGG_PRODUCT2" status) +AGG_PERSON1_ST=$(json_get "$AGG_PERSON1" status) +AGG_LAKE1_ST=$(json_get "$AGG_LAKE1" status) +if [[ ( "$AGG_PRODUCT2_ST" == "confirmed" || "$AGG_PRODUCT2_ST" == "finalized" ) \ + && ( "$AGG_PERSON1_ST" == "confirmed" || "$AGG_PERSON1_ST" == "finalized" ) \ + && ( "$AGG_LAKE1_ST" == "confirmed" || "$AGG_LAKE1_ST" == "finalized" ) ]]; then + ok "Remaining named KA publishes OK (product2=$AGG_PRODUCT2_ST person1=$AGG_PERSON1_ST lake1=$AGG_LAKE1_ST)" else - fail "Curator aggregated publish=$AGG_ST: $AGG_PUB" + fail "Remaining named KA publishes failed: product2=$AGG_PRODUCT2_ST person1=$AGG_PERSON1_ST lake1=$AGG_LAKE1_ST" fi echo "--- 4g: ALL published entities replicate to ALL nodes ---" @@ -520,22 +541,21 @@ echo "" # only node authorised to publish to VM per §2.2). Measuring against # Node5 as in the previous revision was invalid: Node5 can't publish # to `devnet-test` at all, so the balance delta would always be zero -# for the wrong reason. This section therefore writes SWM from Node5 -# (any node may share), then has Node1 promote it to VM and checks -# Node1's balance delta. +# for the wrong reason. This section therefore has Node1 create/share a +# publish-ready KA, promote it to VM, and checks Node1's balance delta. TRAC1_B=$(c "http://127.0.0.1:9201/api/wallets/balances" | python3 -c "import sys,json; print(json.load(sys.stdin)['balances'][0]['trac'])" 2>/dev/null) echo " Node1 (curator) TRAC before: $TRAC1_B" -c -X POST "http://127.0.0.1:9205/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[ +COST_KA="cost-test-ka-$(date +%s%N)" +COST_SHARE=$(ka_create_share 9201 "$CONTEXT_GRAPH" "$COST_KA" "" \ $(ql 'http://example.org/entity/cost-test' 'http://schema.org/name' 'CostTest'), $(q 'http://example.org/entity/cost-test' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Thing') - ] -}" > /dev/null +) +COST_SHARED=$(json_get "$COST_SHARE" swmShared) +[[ "$COST_SHARED" == "true" ]] && ok "Cost-test KA shared by Node1" || fail "Cost-test KA share failed: ${COST_SHARE:0:200}" sleep 1 -COST_PUB=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/publish" -d "{\"contextGraphId\":\"$CONTEXT_GRAPH\",\"selection\":[\"http://example.org/entity/cost-test\"]}") +COST_PUB=$(ka_publish 9201 "$CONTEXT_GRAPH" "$COST_KA") COST_ST=$(json_get "$COST_PUB" status) [[ "$COST_ST" == "confirmed" || "$COST_ST" == "finalized" ]] && ok "Cost-test publish OK ($COST_ST)" || fail "Cost-test publish failed: status=$COST_ST: ${COST_PUB:0:200}" @@ -559,10 +579,7 @@ UPD=$(c -X POST "http://127.0.0.1:9201/api/update" -d "{ \"quads\":[ $(ql 'http://example.org/entity/city1' 'http://schema.org/name' 'Ljubljana'), $(q 'http://example.org/entity/city1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/City'), - $(ql 'http://example.org/entity/city1' 'http://schema.org/population' '295000'), - $(ql 'http://example.org/entity/city2' 'http://schema.org/name' 'Maribor'), - $(q 'http://example.org/entity/city2' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/City'), - $(ql 'http://example.org/entity/city2' 'http://schema.org/population' '97000') + $(ql 'http://example.org/entity/city1' 'http://schema.org/population' '295000') ] }") UPD_ST=$(json_get "$UPD" status) @@ -609,18 +626,19 @@ echo "" echo "=== SECTION 8: Triple Deduplication ===" echo "" -c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[ +DEDUP_KA="dedup1-ka-$(date +%s%N)" +DEDUP_SHARE=$(ka_create_share 9201 "$CONTEXT_GRAPH" "$DEDUP_KA" "" \ $(ql 'http://example.org/entity/dedup1' 'http://schema.org/name' 'DedupTest'), $(ql 'http://example.org/entity/dedup1' 'http://schema.org/name' 'DedupTest'), $(ql 'http://example.org/entity/dedup1' 'http://schema.org/name' 'DedupTest') - ] -}" > /dev/null +) +DEDUP_SHARED=$(json_get "$DEDUP_SHARE" swmShared) +[[ "$DEDUP_SHARED" == "true" ]] && ok "Dedup KA shared to SWM" || fail "Dedup share failed: ${DEDUP_SHARE:0:200}" sleep 1 -DEDUP=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/publish" -d "{\"contextGraphId\":\"$CONTEXT_GRAPH\",\"selection\":[\"http://example.org/entity/dedup1\"]}") +DEDUP=$(ka_publish 9201 "$CONTEXT_GRAPH" "$DEDUP_KA") DD_ST=$(json_get "$DEDUP" status) -DD_KAS=$(echo "$DEDUP" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('kas',[])))" 2>/dev/null) +DD_KAS=0 +[[ "$DD_ST" == "confirmed" || "$DD_ST" == "finalized" ]] && DD_KAS=1 [[ "$DD_ST" == "confirmed" || "$DD_ST" == "finalized" ]] && ok "Dedup publish OK" || fail "Dedup status=$DD_ST" check "1 KA (dedup: 3 identical → 1 entity)" "$DD_KAS" "1" @@ -636,18 +654,17 @@ for i in $(seq 1 50); do done BATCH_QUADS="${BATCH_QUADS%,}" -c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d "{\"contextGraphId\":\"$CONTEXT_GRAPH\",\"quads\":[$BATCH_QUADS]}" > /dev/null +BATCH_KA="batch-50-ka-$(date +%s%N)" +BATCH_SHARE=$(ka_create_share 9201 "$CONTEXT_GRAPH" "$BATCH_KA" "" $BATCH_QUADS) +BATCH_SHARED=$(json_get "$BATCH_SHARE" swmShared) +[[ "$BATCH_SHARED" == "true" ]] && ok "Batch KA shared to SWM" || fail "Batch KA share failed: ${BATCH_SHARE:0:200}" sleep 2 -BATCH_SELECTION="" -for i in $(seq 1 50); do BATCH_SELECTION="$BATCH_SELECTION\"http://example.org/entity/batch_$i\","; done -BATCH_SELECTION="[${BATCH_SELECTION%,}]" -BATCH=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/publish" -d "{\"contextGraphId\":\"$CONTEXT_GRAPH\",\"selection\":$BATCH_SELECTION}") +BATCH=$(ka_publish 9201 "$CONTEXT_GRAPH" "$BATCH_KA") B_ST=$(json_get "$BATCH" status) B_TX=$(json_get "$BATCH" txHash) -B_KAS=$(echo "$BATCH" | python3 -c "import sys,json; print(len(json.load(sys.stdin).get('kas',[])))" 2>/dev/null) [[ "$B_ST" == "confirmed" || "$B_ST" == "finalized" ]] && ok "Batch(50) publish OK ($B_ST)" || fail "Batch publish=$B_ST: $BATCH" [[ "$B_TX" != "__NONE__" ]] && ok "Batch tx: $B_TX" || fail "No batch txHash" -[[ "$B_KAS" == "50" ]] && ok "Batch published 50 KAs" || fail "Expected 50 KAs, got $B_KAS" +echo " Batch named KA submitted 50 entities; replication checked below" echo "" echo "--- 9b: Batch entities replicate to ALL nodes ---" @@ -674,26 +691,23 @@ echo "" echo "=== SECTION 10: Concurrent SWM Writers from Multiple Nodes ===" echo "" -c -X POST "http://127.0.0.1:9202/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[$(ql 'http://example.org/entity/song1' 'http://schema.org/name' 'Zdravljica'),$(q 'http://example.org/entity/song1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/MusicComposition')] -}" > /dev/null 2>&1 & +ka_create_share 9202 "$CONTEXT_GRAPH" "song1-ka-$(date +%s%N)" "" \ + $(ql 'http://example.org/entity/song1' 'http://schema.org/name' 'Zdravljica'),$(q 'http://example.org/entity/song1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/MusicComposition') \ + > /dev/null 2>&1 & PID1=$! -c -X POST "http://127.0.0.1:9204/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[$(ql 'http://example.org/entity/mountain1' 'http://schema.org/name' 'Triglav'),$(q 'http://example.org/entity/mountain1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Mountain'),$(ql 'http://example.org/entity/mountain1' 'http://schema.org/elevation' '2864')] -}" > /dev/null 2>&1 & +ka_create_share 9204 "$CONTEXT_GRAPH" "mountain1-ka-$(date +%s%N)" "" \ + $(ql 'http://example.org/entity/mountain1' 'http://schema.org/name' 'Triglav'),$(q 'http://example.org/entity/mountain1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Mountain'),$(ql 'http://example.org/entity/mountain1' 'http://schema.org/elevation' '2864') \ + > /dev/null 2>&1 & PID2=$! -c -X POST "http://127.0.0.1:9203/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[$(ql 'http://example.org/entity/river1' 'http://schema.org/name' 'Sava'),$(q 'http://example.org/entity/river1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/RiverBodyOfWater')] -}" > /dev/null 2>&1 & +ka_create_share 9203 "$CONTEXT_GRAPH" "river1-ka-$(date +%s%N)" "" \ + $(ql 'http://example.org/entity/river1' 'http://schema.org/name' 'Sava'),$(q 'http://example.org/entity/river1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/RiverBodyOfWater') \ + > /dev/null 2>&1 & PID3=$! wait $PID1 $PID2 $PID3 -ok "3 concurrent SWM writes completed" +ok "3 concurrent KA create/share calls completed" sleep 6 for entity in song1 mountain1 river1; do @@ -758,10 +772,8 @@ echo "--- 13a: Removed /api/publish returns 404 ---" REMOVED=$(curl -s -o /dev/null -w "%{http_code}" -X POST "http://127.0.0.1:9201/api/publish" -H "Authorization: Bearer $AUTH" -H "Content-Type: application/json" -d '{"contextGraphId":"devnet-test","quads":[]}') [[ "$REMOVED" == "404" ]] && ok "/api/publish correctly removed (404)" || warn "/api/publish returns $REMOVED (expected 404)" -echo "--- 13b: Empty quads in SWM write ---" -EMPTY=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d '{"contextGraphId":"devnet-test","quads":[]}') -echo " Empty quads response: $(echo "$EMPTY" | head -c 200)" -echo "$EMPTY" | grep -qi "error\|missing\|invalid" && ok "Empty quads rejected with error" || fail "Empty quads not rejected: $EMPTY" +echo "--- 13b: Retired /api/shared-memory/write is not served ---" +retired_post_assert "/api/shared-memory/write" 9201 "/api/shared-memory/write" '{"contextGraphId":"devnet-test","quads":[]}' echo "--- 13c: Malformed SPARQL ---" BAD_SPARQL=$(c -X POST "http://127.0.0.1:9201/api/query" -d '{ @@ -770,16 +782,13 @@ BAD_SPARQL=$(c -X POST "http://127.0.0.1:9201/api/query" -d '{ }') echo "$BAD_SPARQL" | grep -qi "error" && ok "Malformed SPARQL returns error" || fail "Malformed SPARQL didn't error: $BAD_SPARQL" -echo "--- 13d: Missing contextGraphId ---" -NO_CG=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d '{"quads":[]}') -echo "$NO_CG" | grep -qi "error\|missing\|required" && ok "Missing contextGraphId rejected" || warn "Missing contextGraphId response: $NO_CG" +echo "--- 13d: Retired /api/shared-memory/conditional-write is not served ---" +retired_post_assert "/api/shared-memory/conditional-write" 9201 "/api/shared-memory/conditional-write" '{"contextGraphId":"devnet-test","quads":[],"conditions":[]}' -echo "--- 13e: Publish from empty SWM ---" +echo "--- 13e: Retired /api/shared-memory/publish is not served ---" EMPTY_CG="empty-swm-test-$$" c -X POST "http://127.0.0.1:9205/api/context-graph/create" -d "{\"id\":\"$EMPTY_CG\",\"name\":\"empty swm test\"}" >/dev/null 2>&1 -EMPTY_PUB=$(c -X POST "http://127.0.0.1:9205/api/shared-memory/publish" -d "{\"contextGraphId\":\"$EMPTY_CG\"}") -echo " Empty SWM publish: $(echo "$EMPTY_PUB" | head -c 200)" -echo "$EMPTY_PUB" | grep -qi "error\|empty\|nothing\|no.*triple" && ok "Empty SWM publish rejected with error" || fail "Empty SWM publish not rejected: $(echo "$EMPTY_PUB" | head -c 200)" +retired_post_assert "/api/shared-memory/publish" 9205 "/api/shared-memory/publish" "{\"contextGraphId\":\"$EMPTY_CG\"}" #------------------------------------------------------------ echo "" @@ -787,18 +796,19 @@ echo "=== SECTION 14: Assertion Lifecycle (Working Memory) ===" echo "" ASSERT_CG="devnet-test" +ASSERT_NAME="devnet-draft-$(date +%s%N)" echo "--- 14a: Create an assertion ---" ASSERT_CREATE=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets" -d "{ \"contextGraphId\":\"$ASSERT_CG\", - \"name\":\"devnet-draft\" + \"name\":\"$ASSERT_NAME\" }") ASSERT_URI=$(json_get "$ASSERT_CREATE" assertionUri) echo " Assertion URI: $ASSERT_URI" [[ "$ASSERT_URI" != "__NONE__" && "$ASSERT_URI" != "__ERR__" ]] && ok "Assertion created: $ASSERT_URI" || fail "Assertion create failed: $ASSERT_CREATE" echo "--- 14b: Write triples to the assertion ---" -ASSERT_WRITE=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets/devnet-draft/wm/write" -d "{ +ASSERT_WRITE=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets/$ASSERT_NAME/wm/write" -d "{ \"contextGraphId\":\"$ASSERT_CG\", \"quads\":[ $(ql 'urn:devnet:assert:entity1' 'http://schema.org/name' 'Assertion Entity'), @@ -809,7 +819,7 @@ echo " Write response: $(echo "$ASSERT_WRITE" | head -c 200)" echo "$ASSERT_WRITE" | grep -qi "error" && fail "Assertion write failed: $ASSERT_WRITE" || ok "Assertion write OK" echo "--- 14c: Query the assertion ---" -ASSERT_QUERY=$(c "http://127.0.0.1:9201/api/knowledge-assets/devnet-draft/wm/quads?contextGraphId=$ASSERT_CG") +ASSERT_QUERY=$(c "http://127.0.0.1:9201/api/knowledge-assets/$ASSERT_NAME/wm/quads?contextGraphId=$ASSERT_CG") ASSERT_Q_CT=$(safe_quads_count "$ASSERT_QUERY") if [[ "$ASSERT_Q_CT" == "PARSE_ERR" ]]; then fail "Assertion query returned unparseable response: ${ASSERT_QUERY:0:200}" @@ -819,7 +829,7 @@ else fi echo "--- 14d: Promote the assertion to SWM ---" -ASSERT_PROMOTE=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets/devnet-draft/swm/share" -d "{ +ASSERT_PROMOTE=$(c -X POST "http://127.0.0.1:9201/api/knowledge-assets/$ASSERT_NAME/swm/share" -d "{ \"contextGraphId\":\"$ASSERT_CG\" }") PROMOTED_CT=$(json_get "$ASSERT_PROMOTE" promotedCount) @@ -882,20 +892,12 @@ PUB_JOBS=$(c "http://127.0.0.1:9201/api/publisher/jobs") echo " Jobs: $(echo "$PUB_JOBS" | head -c 300)" echo "$PUB_JOBS" | python3 -c 'import sys,json;d=json.load(sys.stdin);print(len(d) if isinstance(d,list) else len(d.get("jobs",[])))' 2>/dev/null && ok "Publisher jobs endpoint works" || warn "Publisher jobs: $PUB_JOBS" -echo "--- 15c: Enqueue a publish job ---" -ENQUEUE_OP_ID="devnet-enqueue-test-$(date +%s)" -PUB_ENQUEUE=$(c -X POST "http://127.0.0.1:9201/api/publisher/enqueue" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"shareOperationId\":\"$ENQUEUE_OP_ID\", - \"roots\":[{\"rootEntity\":\"urn:devnet:assert:entity1\",\"privateMerkleRoot\":null,\"privateTripleCount\":0}], - \"namespace\":\"did:dkg:context-graph:$CONTEXT_GRAPH\", - \"scope\":\"full\", - \"authorityType\":\"owner\", - \"authorityProofRef\":\"urn:dkg:proof:devnet-test\" -}") +echo "--- 15c: Enqueue a named KA VM publish job ---" +PUB_ENQUEUE=$(ka_publish_async 9201 "$CONTEXT_GRAPH" "$ASSERT_NAME") echo " Enqueue: $(echo "$PUB_ENQUEUE" | head -c 300)" PUB_JOB_ID=$(json_get "$PUB_ENQUEUE" jobId) -[[ "$PUB_JOB_ID" != "__NONE__" && "$PUB_JOB_ID" != "__ERR__" ]] && ok "Publisher job enqueued: $PUB_JOB_ID" || warn "Enqueue response: $PUB_ENQUEUE" +PUB_ENQUEUE_ST=$(json_get "$PUB_ENQUEUE" status) +[[ "$PUB_JOB_ID" != "__NONE__" && "$PUB_JOB_ID" != "__ERR__" ]] && ok "Named KA VM publish job enqueued: $PUB_JOB_ID (status=$PUB_ENQUEUE_ST)" || warn "Enqueue response: $PUB_ENQUEUE" if [[ "$PUB_JOB_ID" != "__NONE__" && "$PUB_JOB_ID" != "__ERR__" && -n "$PUB_JOB_ID" ]]; then echo "--- 15d: Check job status ---" @@ -999,10 +1001,9 @@ $SYNC_COMPLETED && ok "Sync catch-up reported completion on Node5 (status=$SYNC_ echo "--- 18b: Write fresh post-subscribe SWM data on Node1 for sync verification ---" SYNC_ENTITY="urn:sync-verify:post-sub-$(date +%s)" -c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[$(ql "$SYNC_ENTITY" 'http://schema.org/name' 'Post-Subscribe Sync Test')] -}" > /dev/null +ka_create_share 9201 "$CONTEXT_GRAPH" "sync-verify-$(date +%s%N)" "" \ + $(ql "$SYNC_ENTITY" 'http://schema.org/name' 'Post-Subscribe Sync Test') \ + > /dev/null sleep "$LOCAL_SETTLE_S" echo "--- 18c: Verify post-subscribe SWM data synced to Node5 ---" @@ -1379,42 +1380,33 @@ echo "" echo "=== SECTION 22: Publisher Queue End-to-End ===" echo "" -echo "--- 22a: Write SWM data for publisher test ---" +echo "--- 22a: Create/share named KA data for publisher test ---" PQ_ENTITY="http://example.org/entity/pub-queue-$(date +%s)" -PQ_WRITE=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[ +PQ_NAME="pub-queue-ka-$(date +%s%N)" +PQ_WRITE=$(ka_create_share 9201 "$CONTEXT_GRAPH" "$PQ_NAME" "" \ $(q "$PQ_ENTITY" 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Thing'), $(ql "$PQ_ENTITY" 'http://schema.org/name' 'Publisher Queue Test') - ] -}") -PQ_OP_ID=$(json_get "$PQ_WRITE" shareOperationId) -echo " SWM write shareOperationId=$PQ_OP_ID" -[[ "$PQ_OP_ID" != "__NONE__" && "$PQ_OP_ID" != "__ERR__" ]] && ok "SWM write for publisher test" || fail "SWM write failed: ${PQ_WRITE:0:200}" +) +PQ_SHARED=$(json_get "$PQ_WRITE" swmShared) +PQ_PROMOTED=$(json_get "$PQ_WRITE" promotedCount) +echo " KA share swmShared=$PQ_SHARED promotedCount=$PQ_PROMOTED" +[[ "$PQ_SHARED" == "true" ]] && ok "Named KA shared for publisher test" || fail "KA share failed: ${PQ_WRITE:0:200}" -# P1-9: also assert triplesWritten >= 2. A silent zero-write pipeline would +# P1-9: also assert promotedCount >= 2. A silent zero-share pipeline would # let the publisher enqueue an empty payload and 22c would "pass" with no # actual data to publish. -PQ_TW=$(json_get "$PQ_WRITE" triplesWritten) +PQ_TW="$PQ_PROMOTED" if [[ "$PQ_TW" != "__NONE__" && "$PQ_TW" != "__ERR__" && "$PQ_TW" -ge 2 ]] 2>/dev/null; then - ok "SWM write persisted $PQ_TW triples (>= 2)" + ok "KA share persisted $PQ_TW triples (>= 2)" else - fail "SWM write triplesWritten=$PQ_TW (expected >= 2) — publisher queue test will be meaningless" + fail "KA share promotedCount=$PQ_TW (expected >= 2) — publisher queue test will be meaningless" fi -echo "--- 22b: Enqueue publish job ---" -PQ_ENQUEUE=$(c -X POST "http://127.0.0.1:9201/api/publisher/enqueue" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"shareOperationId\":\"$PQ_OP_ID\", - \"roots\":[\"$PQ_ENTITY\"], - \"namespace\":\"did:dkg:context-graph:$CONTEXT_GRAPH\", - \"scope\":\"full\", - \"authorityType\":\"owner\", - \"authorityProofRef\":\"urn:dkg:proof:devnet-pub-queue\" -}") +echo "--- 22b: Enqueue named KA VM publish job ---" +PQ_ENQUEUE=$(ka_publish_async 9201 "$CONTEXT_GRAPH" "$PQ_NAME") PQ_JOB_ID=$(json_get "$PQ_ENQUEUE" jobId) echo " Enqueue jobId=$PQ_JOB_ID" -[[ "$PQ_JOB_ID" != "__NONE__" && "$PQ_JOB_ID" != "__ERR__" ]] && ok "Publisher job enqueued: $PQ_JOB_ID" || warn "Enqueue response: ${PQ_ENQUEUE:0:200}" +[[ "$PQ_JOB_ID" != "__NONE__" && "$PQ_JOB_ID" != "__ERR__" && -n "$PQ_JOB_ID" ]] && ok "Named KA VM publish job enqueued: $PQ_JOB_ID" || fail "Named KA VM publish enqueue failed: ${PQ_ENQUEUE:0:200}" if [[ "$PQ_JOB_ID" != "__NONE__" && "$PQ_JOB_ID" != "__ERR__" && -n "$PQ_JOB_ID" ]]; then echo "--- 22c: Poll job status ---" @@ -1478,9 +1470,7 @@ except Exception: echo " Cleared: $PQ_CLEARED jobs" [[ "$PQ_CLEARED" != "__ERR__" ]] && ok "Publisher clear returned ($PQ_CLEARED)" || warn "Publisher clear: $PQ_CLEAR" else - # P2-2: silent no-op was confusing when 22a succeeds but the job id is - # missing. Emit an explicit [SKIP] so the test log carries the reason. - skip "22c-22f skipped: publisher enqueue did not return a usable jobId (PQ_JOB_ID=$PQ_JOB_ID)" + fail "22c-22f skipped: publisher enqueue did not return a usable jobId (PQ_JOB_ID=$PQ_JOB_ID)" fi #------------------------------------------------------------ @@ -1582,17 +1572,8 @@ else ok "Double discard is idempotent (${DD_SECOND:0:80})" fi -echo "--- 23g: Publisher enqueue missing fields ---" -# P0-3: same treatment as 23c — must return a real 4xx, not just a 500 -# with an "error" string in the body. -http_post_capture "http://127.0.0.1:9201/api/publisher/enqueue" \ - "{\"contextGraphId\":\"$CONTEXT_GRAPH\"}" \ - BAD_ENQ BAD_ENQ_CODE -if [[ "$BAD_ENQ_CODE" =~ ^4 ]] && echo "$BAD_ENQ" | grep -qiE 'error|missing|required'; then - ok "Publisher enqueue missing fields rejected (HTTP $BAD_ENQ_CODE)" -else - fail "Bad enqueue not cleanly rejected (HTTP $BAD_ENQ_CODE): ${BAD_ENQ:0:200}" -fi +echo "--- 23g: Retired /api/publisher/enqueue is not served ---" +retired_post_assert "/api/publisher/enqueue" 9201 "/api/publisher/enqueue" "{\"contextGraphId\":\"$CONTEXT_GRAPH\"}" #------------------------------------------------------------ echo "" @@ -1609,24 +1590,16 @@ SG_B_CREATE=$(c -X POST "http://127.0.0.1:9201/api/sub-graph/create" -d "{\"cont echo "$SG_B_CREATE" | grep -qi "error" && fail "Sub-graph B create failed: $SG_B_CREATE" || ok "Sub-graph '$SG_B' created" echo "--- 24b: Write distinct data to each sub-graph ---" -c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"subGraphName\":\"$SG_A\", - \"quads\":[ +ka_create_share 9201 "$CONTEXT_GRAPH" "isolation-alpha-ka" "$SG_A" \ $(ql 'urn:iso:alpha1' 'http://schema.org/name' 'Alpha Only Entity'), $(q 'urn:iso:alpha1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Thing') - ] -}" > /dev/null -c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"subGraphName\":\"$SG_B\", - \"quads\":[ + > /dev/null +ka_create_share 9201 "$CONTEXT_GRAPH" "isolation-beta-ka" "$SG_B" \ $(ql 'urn:iso:beta1' 'http://schema.org/name' 'Beta Only Entity'), $(q 'urn:iso:beta1' 'http://www.w3.org/1999/02/22-rdf-syntax-ns#type' 'http://schema.org/Thing') - ] -}" > /dev/null + > /dev/null -# P2-1: brief settle window for the local SWM write to hit the triple +# P2-1: brief settle window for the local KA share to hit the triple # store before we query it. Round 8 Bug 24: this is a LOCAL write→query # settle, NOT a cross-node gossip wait, so it uses its own env var. # Otherwise a dev running with `GOSSIP_WAIT_S=0` to speed up a local-only @@ -1747,13 +1720,13 @@ echo "--- 24g: Write to unregistered sub-graph rejected (negative test) ---" # timestamp to avoid collisions with anything a previous test run might # have created. UNREG_SG="never-created-$(date +%s%N)" -http_post_capture "http://127.0.0.1:9201/api/shared-memory/write" \ - "{\"contextGraphId\":\"$CONTEXT_GRAPH\",\"subGraphName\":\"$UNREG_SG\",\"quads\":[$(ql 'urn:unreg:x' 'http://schema.org/name' 'nope')]}" \ +http_post_capture "http://127.0.0.1:9201/api/knowledge-assets" \ + "{\"contextGraphId\":\"$CONTEXT_GRAPH\",\"name\":\"unreg-subgraph-ka\",\"subGraphName\":\"$UNREG_SG\",\"quads\":[$(ql 'urn:unreg:x' 'http://schema.org/name' 'nope')],\"finalize\":true,\"alsoShareSwm\":true}" \ UNREG_BODY UNREG_CODE if [[ "$UNREG_CODE" =~ ^4 ]]; then - ok "Write to unregistered sub-graph rejected (HTTP $UNREG_CODE)" + ok "KA create/share to unregistered sub-graph rejected (HTTP $UNREG_CODE)" else - fail "Write to unregistered sub-graph not rejected (HTTP $UNREG_CODE): ${UNREG_BODY:0:200}" + fail "KA create/share to unregistered sub-graph not rejected (HTTP $UNREG_CODE): ${UNREG_BODY:0:200}" fi #------------------------------------------------------------ @@ -1785,17 +1758,17 @@ UPDATE_ERR=$(c -X POST "http://127.0.0.1:9201/api/update" -d "{ echo " Update error response: $(echo "$UPDATE_ERR" | head -c 200)" echo "$UPDATE_ERR" | grep -qi "error\|BatchNotFound\|NotBatchPublisher\|does not exist" && ok "UPDATE to non-existent KC returned meaningful error" || warn "UPDATE error not decoded: ${UPDATE_ERR:0:200}" -echo "--- 25c: SWM write to unregistered sub-graph returns 400 ---" +echo "--- 25c: KA create/share to unregistered sub-graph returns 400 ---" UNREG_SG2="regression-unreg-$(date +%s%N)" -http_post_capture "http://127.0.0.1:9201/api/shared-memory/write" \ - "{\"contextGraphId\":\"$CONTEXT_GRAPH\",\"subGraphName\":\"$UNREG_SG2\",\"quads\":[{\"subject\":\"urn:unreg:x\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"nope\\\"\",\"graph\":\"\"}]}" \ +http_post_capture "http://127.0.0.1:9201/api/knowledge-assets" \ + "{\"contextGraphId\":\"$CONTEXT_GRAPH\",\"name\":\"regression-unreg-ka\",\"subGraphName\":\"$UNREG_SG2\",\"quads\":[{\"subject\":\"urn:unreg:x\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"nope\\\"\",\"graph\":\"\"}],\"finalize\":true,\"alsoShareSwm\":true}" \ UNREG2_BODY UNREG2_CODE if [[ "$UNREG2_CODE" == "400" ]]; then - ok "Unregistered sub-graph write returns HTTP 400" + ok "Unregistered sub-graph KA create/share returns HTTP 400" elif [[ "$UNREG2_CODE" =~ ^4 ]]; then - ok "Unregistered sub-graph write returns HTTP $UNREG2_CODE" + ok "Unregistered sub-graph KA create/share returns HTTP $UNREG2_CODE" else - fail "Unregistered sub-graph write not rejected properly (HTTP $UNREG2_CODE): ${UNREG2_BODY:0:200}" + fail "Unregistered sub-graph KA create/share not rejected properly (HTTP $UNREG2_CODE): ${UNREG2_BODY:0:200}" fi echo "--- 25d: Dynamic node count — NUM_NODES matches expected ---" @@ -2039,18 +2012,16 @@ except: print('false') [[ "$LIST_HAS_CG" == "true" ]] && ok "Free CG found in context-graph list" || fail "Free CG not in list" echo "--- 27c: Write to SWM on free CG (should work without chain) ---" -SWM_FREE_RESP=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$FREE_CG_ID\", - \"quads\":[ +FREE_CG_KA="free-test-ka" +SWM_FREE_RESP=$(ka_create_share 9201 "$FREE_CG_ID" "$FREE_CG_KA" "" \ $(ql "http://example.org/entity/free-test-1" "http://schema.org/name" "FreeCGEntity"), $(q "http://example.org/entity/free-test-1" "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" "http://schema.org/Thing") - ] -}") -SWM_FREE_OK=$(json_get "$SWM_FREE_RESP" triplesWritten) +) +SWM_FREE_OK=$(json_get "$SWM_FREE_RESP" promotedCount) if [[ "$SWM_FREE_OK" != "__NONE__" && "$SWM_FREE_OK" != "0" && "$SWM_FREE_OK" != "__ERR__" ]]; then - ok "SWM write to free CG succeeded ($SWM_FREE_OK triples)" + ok "KA create/share to free CG succeeded ($SWM_FREE_OK triples)" else - fail "SWM write to free CG failed: $SWM_FREE_RESP" + fail "KA create/share to free CG failed: $SWM_FREE_RESP" fi echo "--- 27d: Query SWM on free CG ---" @@ -2069,17 +2040,8 @@ else fail "SWM query on free CG returns 0 bindings" fi -echo "--- 27e: VM publish on unregistered CG should fail ---" -http_post_capture "http://127.0.0.1:9201/api/shared-memory/publish" \ - "{\"contextGraphId\":\"$FREE_CG_ID\"}" \ - VM_GUARD_BODY VM_GUARD_CODE -if [[ "$VM_GUARD_CODE" == "500" ]] && echo "$VM_GUARD_BODY" | grep -qi "not registered"; then - ok "VM publish blocked on unregistered CG (HTTP $VM_GUARD_CODE)" -elif [[ "$VM_GUARD_CODE" =~ ^[45] ]]; then - ok "VM publish blocked on unregistered CG (HTTP $VM_GUARD_CODE)" -else - fail "VM publish should be blocked on unregistered CG (HTTP $VM_GUARD_CODE): ${VM_GUARD_BODY:0:200}" -fi +echo "--- 27e: Retired /api/shared-memory/publish is not served on free CG ---" +retired_post_assert "/api/shared-memory/publish on free CG" 9201 "/api/shared-memory/publish" "{\"contextGraphId\":\"$FREE_CG_ID\"}" echo "--- 27f: Register CG on-chain ---" REG_RESP=$(c -X POST "http://127.0.0.1:9201/api/context-graph/register" -d "{ @@ -2104,9 +2066,7 @@ else fi echo "--- 27h: VM publish after registration should work ---" -PUB_RESP=$(c -X POST "http://127.0.0.1:9201/api/shared-memory/publish" -d "{ - \"contextGraphId\":\"$FREE_CG_ID\" -}") +PUB_RESP=$(ka_publish 9201 "$FREE_CG_ID" "$FREE_CG_KA") PUB_ST=$(json_get "$PUB_RESP" status) if [[ "$PUB_ST" == "confirmed" || "$PUB_ST" == "finalized" || "$PUB_ST" == "tentative" ]]; then ok "VM publish after registration succeeded (status=$PUB_ST)" @@ -3188,7 +3148,7 @@ elif [[ "$NUM_NODES" -lt 2 ]]; then else N1_PORT=${NODE_PORTS[0]} # `delivered` is keyed by contextGraphId (a literal cgId string, - # the same value passed to /api/shared-memory/write). Codex PR #588 + # the same value passed to KA create/share). Codex PR #588 # round 3: summing across ALL cgIds let unrelated background traffic # (retries on other CGs) make the canary pass; compare the delta # for $CONTEXT_GRAPH specifically so this section actually proves @@ -3231,13 +3191,11 @@ except Exception: FAILED_BEFORE=$(cg_substrate_counter "$SLO_BEFORE" "$CONTEXT_GRAPH" failed) TAG="urn:rc9-fanout:$(date +%s%N)" - WRITE=$(c -X POST "http://127.0.0.1:$N1_PORT/api/shared-memory/write" -d "{ - \"contextGraphId\":\"$CONTEXT_GRAPH\", - \"quads\":[$(ql "$TAG" "http://schema.org/name" "rc9-substrate-fanout-canary")] - }") - WRITE_OK=$(json_get "$WRITE" triplesWritten) + WRITE=$(ka_create_share "$N1_PORT" "$CONTEXT_GRAPH" "rc9-fanout-$(date +%s%N)" "" \ + $(ql "$TAG" "http://schema.org/name" "rc9-substrate-fanout-canary")) + WRITE_OK=$(json_get "$WRITE" promotedCount) if [[ "$WRITE_OK" != "1" ]]; then - fail "N1 SWM write for substrate fan-out canary failed (triplesWritten=$WRITE_OK)" + fail "N1 KA create/share for substrate fan-out canary failed (promotedCount=$WRITE_OK)" else ok "N1 wrote canary $TAG to '$CONTEXT_GRAPH'" sleep $((GOSSIP_WAIT_S + 3)) @@ -3284,7 +3242,7 @@ section_done section_start "SECTION 35: rc.9 — SWM ack-quorum reaches steady state (PR-D)" # rc.9 PR-D + PR-H added an ack-quorum watchdog that tracks per-share # delivery quorum and re-arms substrate top-ups on stalls. After the -# §34 write (or any earlier SWM write in this run), the tracker +# §34 KA share (or any earlier SWM share in this run), the tracker # should show `pending ≈ 0` on a healthy mesh — every started share # either completed or got dropped via deadline/watchdog. if [[ "$SKIP_RC9_SUBSTRATE" == "1" ]]; then diff --git a/scripts/dkg-v10-decision-time-accounting.mjs b/scripts/dkg-v10-decision-time-accounting.mjs index 35aa60aa5e..cb31114050 100644 --- a/scripts/dkg-v10-decision-time-accounting.mjs +++ b/scripts/dkg-v10-decision-time-accounting.mjs @@ -3,7 +3,7 @@ // Decision pointer on the V10 session. // // Pipe into: -// dkg shared-memory write --file - --format nquads +// POST /api/knowledge-assets + /wm/write const GRAPH = ''; diff --git a/scripts/dkg-v10-decisions.mjs b/scripts/dkg-v10-decisions.mjs index bbe2d7999d..3bda47be6b 100644 --- a/scripts/dkg-v10-decisions.mjs +++ b/scripts/dkg-v10-decisions.mjs @@ -1,7 +1,7 @@ #!/usr/bin/env node // Emits N-Quads for the V10 staking redesign session + decisions, scoped to // the dkg-v10-smart-contracts context graph. Writes to stdout; pipe into -// dkg shared-memory write --file - --format nquads +// POST /api/knowledge-assets + /wm/write // (or to a file). // // Hand-written rather than converted from Turtle so we control every byte diff --git a/scripts/dkg-v10-implementation-time-accounting.mjs b/scripts/dkg-v10-implementation-time-accounting.mjs index 17cd872e10..95cd12bdcd 100644 --- a/scripts/dkg-v10-implementation-time-accounting.mjs +++ b/scripts/dkg-v10-implementation-time-accounting.mjs @@ -5,7 +5,7 @@ // storages, and tests. // // Pipe into: -// dkg shared-memory write --file - --format nquads +// POST /api/knowledge-assets + /wm/write const GRAPH = ''; diff --git a/scripts/seed-demo.sh b/scripts/seed-demo.sh index 86faf24f93..d387e50f01 100755 --- a/scripts/seed-demo.sh +++ b/scripts/seed-demo.sh @@ -12,8 +12,9 @@ post /api/context-graph/create "{\"id\":\"$CG\",\"name\":\"V10 Tri-Modal Memory echo "" echo "=== Seeding knowledge entities ===" -post /api/shared-memory/write "{ +post /api/knowledge-assets "{ \"contextGraphId\":\"$CG\", + \"name\":\"seed-demo-entities\", \"quads\":[ {\"subject\":\"urn:entity:branimir\",\"predicate\":\"http://www.w3.org/1999/02/22-rdf-syntax-ns#type\",\"object\":\"http://schema.org/Person\",\"graph\":\"\"}, {\"subject\":\"urn:entity:branimir\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"Branimir\\\"\",\"graph\":\"\"}, @@ -67,8 +68,10 @@ post /api/shared-memory/write "{ {\"subject\":\"urn:concept:memory-explorer-ui\",\"predicate\":\"http://www.w3.org/1999/02/22-rdf-syntax-ns#type\",\"object\":\"http://schema.org/SoftwareApplication\",\"graph\":\"\"}, {\"subject\":\"urn:concept:memory-explorer-ui\",\"predicate\":\"http://schema.org/name\",\"object\":\"\\\"Memory Explorer UI\\\"\",\"graph\":\"\"}, {\"subject\":\"urn:concept:memory-explorer-ui\",\"predicate\":\"http://schema.org/description\",\"object\":\"\\\"React-based project view with Timeline (conversation turns + dated events), Knowledge Assets (concepts, code, people), and Graph visualization. Trust indicated by colored left borders and status badges.\\\"\",\"graph\":\"\"} - ] -}" | python3 -c "import sys,json; d=json.load(sys.stdin); print(f' {d.get(\"triplesWritten\",d.get(\"error\"))} triples')" + ], + \"finalize\":true, + \"alsoShareSwm\":true +}" | python3 -c "import sys,json; d=json.load(sys.stdin); print(f' {d.get(\"written\", d.get(\"error\"))} triples')" echo "" echo "=== Seeding conversation turns (real chat history) ===" @@ -76,7 +79,7 @@ echo "=== Seeding conversation turns (real chat history) ===" turn() { local n="$1"; shift local md="$1" - post /api/memory/turn "{\"contextGraphId\":\"$CG\",\"markdown\":$(python3 -c "import json; print(json.dumps('''$md'''))"),\"layer\":\"swm\"}" \ + post /api/memory/turn "{\"contextGraphId\":\"$CG\",\"markdown\":$(python3 -c "import json; print(json.dumps('''$md'''))"),\"layer\":\"wm\"}" \ | python3 -c "import sys,json; d=json.load(sys.stdin); print(f' Turn $n: {d.get(\"totalQuads\",0)} quads')" sleep 0.15 } diff --git a/scripts/swm-soak-test.sh b/scripts/swm-soak-test.sh index d8109052a0..4012c8baef 100755 --- a/scripts/swm-soak-test.sh +++ b/scripts/swm-soak-test.sh @@ -343,6 +343,7 @@ write_one_share() { import json print(json.dumps({ 'contextGraphId': '$cgId', + 'name': ('swm-soak-' + ''.join(c if c.isalnum() or c in '-_.' else '-' for c in '$RUN_ID-$seq'))[:120], 'quads': [{ 'subject': '$subject', 'predicate': 'urn:swm-soak:sentBy', @@ -374,9 +375,11 @@ print(json.dumps({ 'object': '\"$SOAK_COHORT_ID\"', 'graph': '', }], + 'finalize': True, + 'alsoShareSwm': True, })) ") - resp=$(curl -s --max-time 30 -X POST "$API/api/shared-memory/write" \ + resp=$(curl -s --max-time 30 -X POST "$API/api/knowledge-assets" \ -H "Authorization: Bearer $AUTH" \ -H "Content-Type: application/json" \ -d "$body") @@ -388,7 +391,7 @@ print(json.dumps({ import json, sys try: d = json.loads(sys.stdin.read()) - print('ok' if d.get('shareOperationId') else 'fail') + print('ok' if d.get('swmShared') or d.get('status') in ('swm-shared','vm-confirmed') else 'fail') except: print('fail') " 2>/dev/null) log " write cg=$cgId seq=$seq → $ok" @@ -603,7 +606,7 @@ try: continue if r.get('cgId') != '$cgId': continue resp = r.get('resp') - if isinstance(resp, dict) and resp.get('shareOperationId'): + if isinstance(resp, dict) and (resp.get('swmShared') or resp.get('status') in ('swm-shared', 'vm-confirmed')): n += 1 except FileNotFoundError: pass @@ -613,6 +616,7 @@ print(n) import json print(json.dumps({ 'contextGraphId': '$cgId', + 'name': ('swm-soak-summary-' + ''.join(c if c.isalnum() or c in '-_.' else '-' for c in '$RUN_ID'))[:120], 'quads': [{ 'subject': '$subject', 'predicate': 'urn:swm-soak:writesAccepted', @@ -639,9 +643,11 @@ print(json.dumps({ 'object': '\"$ts\"', 'graph': '', }], + 'finalize': True, + 'alsoShareSwm': True, })) ") - resp=$(curl -s --max-time 30 -X POST "$API/api/shared-memory/write" \ + resp=$(curl -s --max-time 30 -X POST "$API/api/knowledge-assets" \ -H "Authorization: Bearer $AUTH" \ -H "Content-Type: application/json" \ -d "$body") @@ -652,7 +658,7 @@ print(json.dumps({ import json, sys try: d = json.loads(sys.stdin.read()) - print('ok' if d.get('shareOperationId') else 'fail') + print('ok' if d.get('swmShared') or d.get('status') in ('swm-shared','vm-confirmed') else 'fail') except: print('fail') " 2>/dev/null) log " writes-accepted-summary cg=$cgId tag=$SENDER_TAG writesAccepted=$writes_accepted → $ok" @@ -786,7 +792,20 @@ log "" log "Summary:" writes_total=$(wc -l < "$LOG_DIR/writes.jsonl" | tr -d ' ') log " total writes attempted: $writes_total" -writes_ok=$(grep -c '"shareOperationId"' "$LOG_DIR/writes.jsonl" 2>/dev/null || echo 0) +writes_ok=$(node -e ' +const fs = require("fs"); +let count = 0; +try { + for (const line of fs.readFileSync(process.argv[1], "utf8").split(/\n/)) { + if (!line.trim()) continue; + try { + const resp = JSON.parse(line).resp; + if (resp && (resp.swmShared || resp.status === "swm-shared" || resp.status === "vm-confirmed")) count += 1; + } catch {} + } +} catch {} +console.log(count); +' "$LOG_DIR/writes.jsonl") log " writes accepted by local daemon: $writes_ok" log "" log "Per-CG final inbox (delivery validation — operators cross-reference):" @@ -924,7 +943,7 @@ try: cg = rec.get('cgId') resp = rec.get('resp') if not cg or not isinstance(resp, dict): continue - if resp.get('shareOperationId'): + if resp.get('swmShared') or resp.get('status') in ('swm-shared', 'vm-confirmed'): self_writes_accepted_by_cg[cg] = self_writes_accepted_by_cg.get(cg, 0) + 1 except FileNotFoundError: pass diff --git a/scripts/testnet-publish-stress/publish-loop.mjs b/scripts/testnet-publish-stress/publish-loop.mjs index ff401b25ca..572ab9f79e 100644 --- a/scripts/testnet-publish-stress/publish-loop.mjs +++ b/scripts/testnet-publish-stress/publish-loop.mjs @@ -7,7 +7,7 @@ * * Lifecycle per partition (2 HTTP calls + an unknown wait for chain confirm): * 1. POST /api/knowledge-assets { name, contextGraphId, quads, - * finalize: true, promote: true } + * finalize: true, alsoShareSwm: true } * Combined create+write+finalize+promote (one round-trip, agent does * the work in-process). * 2. POST /api/knowledge-assets/:name/vm/publish { contextGraphId } @@ -277,14 +277,14 @@ async function publishOnePartition(partition, partitionIdx, attempt = 0) { const { anchor, quads } = buildPartitionQuads(partition, CFG.cgId, CFG.stressRunId, partitionIdx); // 1. Combined create + write + finalize + promote. The route requires - // `finalize: true` to allow `promote: true`, and `quads` to be present + // `finalize: true` to allow `alsoShareSwm: true`, and `quads` to be present // to allow `finalize: true` — exactly the bundle we want. const createRes = await apiCall('POST', '/api/knowledge-assets', { name, contextGraphId: CFG.cgId, quads, finalize: true, - promote: true, + alsoShareSwm: true, }, { timeoutMs: 60_000 }); const merkleRoot = createRes.seal?.merkleRoot ?? createRes.merkleRoot; diff --git a/scripts/two-laptop-test.sh b/scripts/two-laptop-test.sh index 56fa32a4bf..1dfb4d485c 100755 --- a/scripts/two-laptop-test.sh +++ b/scripts/two-laptop-test.sh @@ -371,8 +371,8 @@ else fi sleep "$ONCHAIN_REGISTER_SLEEP" -publish_resp=$(apiA POST /api/shared-memory/publish \ - "{\"contextGraphId\":\"$CG_ID\",\"selection\":\"all\",\"clearAfter\":false}") +publish_resp=$(apiA POST /api/knowledge-assets/widget-info/vm/publish \ + "{\"contextGraphId\":\"$CG_ID\",\"options\":{\"clearAfter\":false}}") pub_status=$(echo "$publish_resp" | jq_field status) pub_kcid=$(echo "$publish_resp" | jq_field kaId) pub_tx=$(echo "$publish_resp" | jq_field txHash) diff --git a/scripts/v10-rc-validation.sh b/scripts/v10-rc-validation.sh index 91f4934726..67be1d7309 100755 --- a/scripts/v10-rc-validation.sh +++ b/scripts/v10-rc-validation.sh @@ -8,12 +8,12 @@ # # Exit code is non-zero when any sub-test fails — orchestrators key off this. # -# Wire shapes assumed (rc.12+): -# - publish: POST /api/shared-memory/write + POST /api/shared-memory/publish +# Wire shapes assumed: +# - publish: POST /api/knowledge-assets + POST /api/knowledge-assets//vm/publish # - update/private quads: POST /api/update (legacy is intentionally retained) -# - SWM: POST /api/shared-memory/{write,publish} +# - SWM: POST /api/knowledge-assets//swm/share # - knowledge-assets: POST /api/knowledge-assets (create), POST /api/knowledge-assets//wm/write, GET /api/knowledge-assets//wm/quads (query), POST /api/knowledge-assets//swm/share (promote) -# - CAS: POST /api/shared-memory/conditional-write (conditions REQUIRED non-empty) +# - CAS: retired with loose SWM writes # - chat: POST /api/chat { to, text } # - identity: GET /api/identity (replaces deprecated /api/profile) # - status: GET /api/status (carries peerId, name, nodeRole — covers profile cases) @@ -112,27 +112,22 @@ CG="${CG:-devnet-test}" # split on the first two `|` and treat the rest as the raw response. publish_swm() { local port=$1 cgid=$2 quads_json=$3 sgname=${4:-} root_entity=${5:-} + local name="rc-${RUN_TAG}-${port}-${RANDOM:-0}" local write_body=$(cat <