From 4138664a1b387e8d604e5eea5d89069a87ddac0c Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 16:40:15 +0200 Subject: [PATCH 01/14] fix(agent): deterministic per-root tokenId in chain-driven VM reconcile (#936) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit promoteSharedMemoryToCanonical assigned per-root tokenIds positionally over the oxigraph SPARQL binding order, which is store-history-dependent — so two replicas chain-reconciling the same KC minted divergent rootEntity→tokenId maps. Sort the roots lexicographically before the mint loop so the map is a pure function of the root SET (identical on every replica, both gossip and chain-reconcile paths), and emit explicit `/` dkg:tokenId / dkg:entity rows so the map is queryable. Kept in the handler (not generateKCMetadata, which metadata.test.ts pins). Verified: repro green; 72 finalization/reconcile/collapse tests + full agent suite (1614) green. Co-Authored-By: Claude Opus 4.8 --- packages/agent/src/finalization-handler.ts | 43 +++++- .../issue-936-tokenid-determinism.test.ts | 124 ++++++++++++++++++ 2 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 packages/agent/test/issue-936-tokenid-determinism.test.ts diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index effa8927a..ed217311d 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -1066,8 +1066,18 @@ export class FinalizationHandler { } const kaMetadata: KAMetadata[] = []; - for (let tokenIdx = 0; tokenIdx < rootEntities.length; tokenIdx++) { - const rootEntity = rootEntities[tokenIdx]; + // GH #936 — assign per-root tokenIds over a CANONICAL (lexicographic) root + // order, NOT the SPARQL/gossip binding order. oxigraph binding order is + // store-history-dependent, so two replicas reconciling the same KC from + // chain would otherwise mint divergent root→tokenId maps. These tokenIds are + // local compatibility labels (the on-chain KA count is 1, no on-chain + // dependency — see dkg-publisher.ts), so a content-derived sort makes the + // map a pure function of the root SET: identical on every replica and on + // both the gossip and chain-reconcile promotion paths. + const orderedRoots = [...rootEntities].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + + for (let tokenIdx = 0; tokenIdx < orderedRoots.length; tokenIdx++) { + const rootEntity = orderedRoots[tokenIdx]; const entityQuads = partitioned.get(rootEntity) ?? []; if (entityQuads.length === 0) continue; kaMetadata.push({ @@ -1159,6 +1169,35 @@ export class FinalizationHandler { } catch { /* tentative status may not exist */ } let metaQuads = generateConfirmedFullMetadata(kcMeta, kaMetadata, provenance); + + // GH #936 — emit an EXPLICIT, deterministic per-root token map + // (`/` dkg:tokenId / dkg:entity) for multi-root KCs so two + // replicas reconciling the same KC expose an IDENTICAL, queryable + // rootEntity→tokenId mapping (the tokenId now derives from the canonical + // root sort above). Emitted here rather than in generateKCMetadata, which + // metadata.test.ts pins to forbid these predicates. graph = the default + // `/_meta` so the ctxGraphId remap below routes them to the per-cgId + // `_meta` (and dual-writes a root copy when keepRootCopyOnLabel). + if (kaMetadata.length > 1) { + const defaultMetaGraph = `did:dkg:context-graph:${contextGraphId}/_meta`; + for (const ka of kaMetadata) { + const labelSubject = `${ual}/${ka.tokenId}`; + metaQuads.push( + { + subject: labelSubject, + predicate: `${DKG_NS}tokenId`, + object: `"${ka.tokenId}"^^`, + graph: defaultMetaGraph, + }, + { + subject: labelSubject, + predicate: DKG_ENTITY, + object: ka.rootEntity, + graph: defaultMetaGraph, + }, + ); + } + } if (ctxGraphId) { const defaultMeta = `did:dkg:context-graph:${contextGraphId}/_meta`; const targetMeta = contextGraphMetaUri(contextGraphId, ctxGraphId); diff --git a/packages/agent/test/issue-936-tokenid-determinism.test.ts b/packages/agent/test/issue-936-tokenid-determinism.test.ts new file mode 100644 index 000000000..2e45b88b5 --- /dev/null +++ b/packages/agent/test/issue-936-tokenid-determinism.test.ts @@ -0,0 +1,124 @@ +/** + * Issue-liveness repro for GH #936 — "Chain-driven VM reconcile: per-root + * tokenId order is non-deterministic." https://github.com/OriginTrail/dkg/issues/936 + * + * On the chain-driven reconcile path (`handleChainReconciledKC`, no gossip + * `rootEntities` on the wire) the recovered roots come straight from a SPARQL + * read of SWM workspace meta, and each KA's on-chain `tokenId` is then assigned + * POSITIONALLY from that array (`tokenId = index + 1`). The merkle root can't + * pin the order — `V10MerkleTree` sorts+dedupes its leaves, so any root + * permutation verifies. And the SPARQL binding order is store-history-dependent + * (oxigraph returns the SAME triples in DIFFERENT orders depending on insertion + * history — see the two replicas below). + * + * Consequence: two replicas reconciling the SAME knowledge collection from chain + * map the SAME rootEntity to DIFFERENT tokenIds → they disagree on which content + * lives at `/1` vs `/2`. + * + * This test asserts the CORRECT (post-fix) behaviour, so it is RED today + * (the bug is live) and turns GREEN once the fix lands; it stays red until #936 is fixed.. + * Hermetic — two in-memory oxigraph stores, a binding chain stub, no network. + */ +import { describe, it, expect } from 'vitest'; +import { OxigraphStore } from '@origintrail-official/dkg-storage'; +import { + createOperationContext, + contextGraphWorkspaceGraphUri, + contextGraphWorkspaceMetaGraphUri, +} from '@origintrail-official/dkg-core'; +import type { ChainAdapter } from '@origintrail-official/dkg-chain'; +import { computeFlatKCRootV10 } from '@origintrail-official/dkg-publisher'; +import { FinalizationHandler } from '../src/finalization-handler.js'; + +const CG = 'gh936-cg'; +const ON_CHAIN_CG = '42'; +const UAL = 'did:dkg:evm:31337/0xABC/7'; +const PUBLISHER = '0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266'; +const KA_ID = 7n; + +// Three roots whose names also feed the (order-insensitive) merkle. +const ROOTS = [ + { iri: 'urn:gh936:zzz', name: '"Zulu"' }, + { iri: 'urn:gh936:aaa', name: '"Alpha"' }, + { iri: 'urn:gh936:mmm', name: '"Mike"' }, +]; + +function makeBindingChain(boundCg: bigint): ChainAdapter { + return { chainId: 'evm:31337', getKAContextGraphId: async () => boundCg } as unknown as ChainAdapter; +} + +/** Seed the 3-root SWM snapshot with the data + op→root meta inserted in + * `insertOrder`. Returns the (order-insensitive) flat-KC merkle root. */ +async function seedSnapshot(store: OxigraphStore, insertOrder: typeof ROOTS): Promise { + const wsGraph = contextGraphWorkspaceGraphUri(CG); + const wsMetaGraph = contextGraphWorkspaceMetaGraphUri(CG); + for (const r of insertOrder) { + await store.insert([ + { subject: r.iri, predicate: 'http://schema.org/name', object: r.name, graph: wsGraph }, + { subject: 'urn:dkg:share:gh936:op-1', predicate: 'http://dkg.io/ontology/rootEntity', object: r.iri, graph: wsMetaGraph }, + ]); + } + // Merkle over all roots' triples (graph stripped), order-insensitive. + return computeFlatKCRootV10( + ROOTS.map((r) => ({ subject: r.iri, predicate: 'http://schema.org/name', object: r.name, graph: '' })), + [], + ); +} + +/** Read the confirmed rootEntity→tokenId mapping from a reconciled replica. + * Reads the PER-ROOT label rows (`/`) only — the aggregate + * `` row carries every member's tokenId AND entity, which would + * cross-product. */ +async function readRootTokenMap(store: OxigraphStore): Promise> { + const metaGraph = `did:dkg:context-graph:${CG}/context/${ON_CHAIN_CG}/_meta`; + const res: any = await store.query( + `SELECT ?ka ?root ?tid WHERE { + GRAPH <${metaGraph}> { + ?ka ?tid . + ?ka ?root . + FILTER(STRSTARTS(STR(?ka), "${UAL}/")) + } + }`, + ); + const map: Record = {}; + if (res.type === 'bindings') { + for (const b of res.bindings) { + const root = String(b.root).replace(/^<|>$/g, ''); + const tid = String(b.tid).replace(/^"/, '').replace(/"(\^\^.*)?$/, ''); + map[root] = tid; + } + } + return map; +} + +async function reconcile(insertOrder: typeof ROOTS): Promise> { + const store = new OxigraphStore(); + const merkleRoot = await seedSnapshot(store, insertOrder); + const handler = new FinalizationHandler(store, makeBindingChain(BigInt(ON_CHAIN_CG))); + const outcome = await handler.handleChainReconciledKC( + { contextGraphId: CG, onChainCgId: ON_CHAIN_CG, ual: UAL, merkleRoot, publisherAddress: PUBLISHER, kaId: KA_ID, versionBlock: 500 }, + createOperationContext('system'), + ); + expect(outcome).toBe('promoted'); + return readRootTokenMap(store); +} + +describe('GH #936 — chain-driven reconcile must map each root to a deterministic tokenId', () => { + it('two replicas reconciling the same KC agree on the rootEntity→tokenId mapping', async () => { + // Replica A and replica B received the same 3 roots in DIFFERENT orders + // (independent share-time histories). oxigraph's SPARQL binding order + // tracks insertion history, so each replica recovers a different root order. + const replicaA = await reconcile([ROOTS[0], ROOTS[1], ROOTS[2]]); // z, a, m + const replicaB = await reconcile([ROOTS[1], ROOTS[2], ROOTS[0]]); // a, m, z + + // Control: each replica produced a full 3-root mapping (so the equality + // assertion below is meaningful, not vacuously comparing empty maps). + expect(Object.keys(replicaA)).toHaveLength(3); + expect(Object.keys(replicaB)).toHaveLength(3); + + // CORRECT (post-fix): the rootEntity→tokenId mapping is content-derived and + // identical on every replica. Today it is positional over a store-dependent + // order, so the two replicas disagree on which root owns `/1`. + expect(replicaA).toEqual(replicaB); + }); +}); From 802f95ddb8821299b32d6b989369383f647f0e1b Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 16:40:34 +0200 Subject: [PATCH 02/14] fix(publisher): async-lift carries an inline-encryption path for private CGs (#1121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit mapLiftRequestToPublishOptions never set encryptInlinePayload/encryptInlineChunked, so a private (ownerOnly/allowList) async-lift publish would ship PLAINTEXT to cores. Thread the agent-resolved encryption callbacks through LiftResolvedPublishSlice and, for any non-public CG, attach a FAIL-CLOSED default that throws if invoked without a real factory — so plaintext can never silently ship. Fix publisher-runner.ts merge precedence so the agent-resolved chainKey-bound AEAD closure wins over the default (else every real curated async publish would throw). Verified: repro green; async-lift-publish-options + metadata + full publisher suite (1179) green; publisher-runner cli tests green. Co-Authored-By: Claude Opus 4.8 --- packages/cli/src/publisher-runner.ts | 11 ++- .../src/async-lift-publish-options.ts | 39 ++++++++++ ...ft-canonicalization-and-encryption.test.ts | 73 +++++++++++++++++++ 3 files changed, 121 insertions(+), 2 deletions(-) create mode 100644 packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts diff --git a/packages/cli/src/publisher-runner.ts b/packages/cli/src/publisher-runner.ts index 030d1b355..ac02c97ff 100644 --- a/packages/cli/src/publisher-runner.ts +++ b/packages/cli/src/publisher-runner.ts @@ -274,10 +274,17 @@ async function createPublisherRuntimeFromBase(args: PublisherRuntimeBaseArgs): P throw new Error(`No publisher configured for wallet ${walletId}`); } const encryption = await args.publishEncryptionFactory?.(publishOptions); + // GH #1121 — the agent-resolved, chainKey-bound AEAD closure MUST win over + // any callback already on publishOptions. The async-lift mapper now + // pre-populates a fail-closed default `encryptInlinePayload` for non-public + // CGs (so plaintext can never silently ship); that default must only apply + // when the real factory yields nothing — otherwise it would shadow the + // real curated-publish encryption and make every private async publish + // throw. Hence: real factory first, mapper default as the fallback. const publishOptionsWithEncryption: PublishOptions = { ...publishOptions, - encryptInlinePayload: publishOptions.encryptInlinePayload ?? encryption?.encryptInlinePayload, - encryptInlineChunked: publishOptions.encryptInlineChunked ?? encryption?.encryptInlineChunked, + encryptInlinePayload: encryption?.encryptInlinePayload ?? publishOptions.encryptInlinePayload, + encryptInlineChunked: encryption?.encryptInlineChunked ?? publishOptions.encryptInlineChunked, }; const v10ACKProvider = publishOptionsWithEncryption.v10ACKProvider ?? args.v10ACKProviderFactory?.() diff --git a/packages/publisher/src/async-lift-publish-options.ts b/packages/publisher/src/async-lift-publish-options.ts index c197f08f5..371544e15 100644 --- a/packages/publisher/src/async-lift-publish-options.ts +++ b/packages/publisher/src/async-lift-publish-options.ts @@ -37,8 +37,34 @@ export interface LiftResolvedPublishSlice { readonly onPhase?: PhaseCallback; readonly receiverSignatureProvider?: ReceiverSignatureProvider; readonly publishContextGraphId?: string; + /** + * GH #1121 — inline-encryption callbacks for private/curated CGs, threaded + * from the async worker (the agent-resolved, chainKey-bound AEAD closures) + * into the mapper exactly like `onPhase`/`receiverSignatureProvider`. When a + * non-public CG reaches the mapper WITHOUT these, the mapper substitutes a + * fail-closed default so cores can never silently receive plaintext. + */ + readonly encryptInlinePayload?: PublishOptions['encryptInlinePayload']; + readonly encryptInlineChunked?: PublishOptions['encryptInlineChunked']; } +/** + * GH #1121 fail-closed default. A private/curated async publish must never ship + * plaintext to cores. The real chainKey-bound AEAD closure is resolved live at + * execute time by the agent and threaded through `LiftResolvedPublishSlice` + * (and, in the CLI runtime, merged in `publisher-runner.ts` where it takes + * precedence over this default). If a non-public CG ever reaches the publisher + * with NO inline-encryption callback, this default makes the publish FAIL LOUD + * instead of leaking plaintext. + */ +const failClosedInlineEncrypt: NonNullable = () => { + throw new Error( + 'Async-lift private/curated CG publish requires an encryptInlinePayload factory threaded ' + + 'through LiftResolvedPublishSlice (a non-public context graph must not ship plaintext to cores). ' + + 'Wire publishEncryptionFactory in the async publisher runtime.', + ); +}; + export interface LiftPublishMappingInput { readonly request: LiftRequest; readonly validation: Pick; @@ -108,6 +134,17 @@ export function mapLiftRequestToPublishOptions(input: LiftPublishMappingInput): throw new Error('Lift publish mapping only allows allowedPeers when accessPolicy is allowList'); } + // GH #1121 — derive the inline-encryption path. Public CGs stay plaintext + // (undefined). Non-public CGs MUST carry an inline-encryption callback: use + // the agent-resolved one threaded through `resolved`, else the fail-closed + // default so a missing factory throws at publish time rather than leaking + // plaintext. The chunked callback is optional (the inline blob path is the + // floor) and only ever passes through the resolved value. + const encryptInlinePayload = accessPolicy === 'public' + ? input.resolved.encryptInlinePayload + : (input.resolved.encryptInlinePayload ?? failClosedInlineEncrypt); + const encryptInlineChunked = input.resolved.encryptInlineChunked; + // Request flags (enqueue-time caller intent) win over resolved hints (per-process defaults). const entityProofs = input.request.entityProofs ?? input.resolved.entityProofs; const publishEpochs = input.request.publishEpochs; @@ -133,6 +170,8 @@ export function mapLiftRequestToPublishOptions(input: LiftPublishMappingInput): onPhase: input.resolved.onPhase, receiverSignatureProvider: input.resolved.receiverSignatureProvider, publishContextGraphId: input.resolved.publishContextGraphId, + encryptInlinePayload, + encryptInlineChunked, ...(publishEpochs !== undefined ? { publishEpochs } : {}), diff --git a/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts b/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts new file mode 100644 index 000000000..1b066e541 --- /dev/null +++ b/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts @@ -0,0 +1,73 @@ +/** + * Liveness/regression tests for two async-lift publish gaps. + * + * GH #1122 — "Sync and async publish canonicalize root subjects differently". + * The async lift validator rewrites caller-provided root IRIs to + * generated `dkg:::/-` roots, so the same + * domain payload yields different RDF subjects than sync publish — + * breaking stable IRI linking for integrations. + * https://github.com/OriginTrail/dkg/issues/1122 + * + * GH #1121 — "publishAsync should support encrypted VM publishing for + * curated/private context graphs". The sync path resolves + * `encryptInlinePayload` / `encryptInlineChunked`; the async-lift + * PublishOptions mapping carries no encryption path, so a private-CG + * async publish would ship plaintext to cores. + * https://github.com/OriginTrail/dkg/issues/1121 + * + * Both are `it.fails` repros: the assertion of the correct (sync-parity) + * behaviour fails today. When fixed, drop `.fails` and close the issue. + * Hermetic — pure functions, no chain/network. + */ +import { describe, expect, it } from 'vitest'; +import { validateLiftPublishPayload } from '../src/async-lift-validation.js'; +import { mapLiftRequestToPublishOptions } from '../src/async-lift-publish-options.js'; +import type { LiftRequest } from '../src/lift-job-types.js'; + +const CALLER_ROOT = 'urn:dmaast:tenant:tenant-a'; + +const baseRequest: LiftRequest = { + swmId: 'swm-main', + shareOperationId: 'op-1122', + roots: [CALLER_ROOT], + contextGraphId: 'dmaast', + namespace: 'tenants', + scope: 'devnet', + transitionType: 'CREATE', + authority: { type: 'owner', proofRef: 'devnet-proof' }, +}; + +describe('GH #1122 — async lift preserves caller-provided root IRIs (sync parity)', () => { + it('does not rewrite a caller root IRI to a generated dkg:… subject', () => { + const out = validateLiftPublishPayload({ + request: baseRequest, + resolved: { + quads: [ + { subject: CALLER_ROOT, predicate: 'https://schema.org/name', object: '"Tenant A"', graph: 'g' }, + ], + }, + }); + // Sync publish keeps the caller IRI as the subject; async must match. + expect(out.resolved.quads[0].subject).toBe(CALLER_ROOT); + }); +}); + +describe('GH #1121 — async lift carries an inline-encryption path for private CGs', () => { + it('a private (ownerOnly) async publish maps to PublishOptions with an encryption callback', () => { + const opts = mapLiftRequestToPublishOptions({ + request: { ...baseRequest, accessPolicy: 'ownerOnly' }, + validation: { authorityProofRef: 'devnet-proof', transitionType: 'CREATE' }, + resolved: { + quads: [ + { subject: CALLER_ROOT, predicate: 'https://schema.org/name', object: '"Tenant A"', graph: 'g' }, + ], + accessPolicy: 'ownerOnly', + publisherPeerId: '12D3KooWGH1121PrivatePublisherPeerIdForTest', + }, + }); + expect(opts.accessPolicy).toBe('ownerOnly'); + // A private publish must not ship plaintext to cores — there has to be an + // inline-encryption path threaded through, like the sync publish() does. + expect(opts.encryptInlinePayload).toBeDefined(); + }); +}); From 0adc2306576e70b125afce8dffdf5038b740519d Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 16:40:34 +0200 Subject: [PATCH 03/14] fix(storage): scope private payload storage by verifiable commitment (#1078) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PrivateContentStore keyed the finalized private partition only by (contextGraphId[,subGraphName]) → one `_private` graph, so two distinct private commitments for the same root commingled and getPrivateTriples returned BOTH — a privateDataAnchor on a verifiable KA could hydrate a different commitment's slice. Add an optional commitmentId to storePrivateTriples: when it differs from the commitment recorded for a root, the new commitment supersedes the stale slice (a marker on a non-hydrated subject tracks the current commitment). With NO commitmentId the behaviour is unchanged (append + dedup), so every existing caller and the multi-value-predicate tests are untouched. Wire the production write sites (sync publish + async-lift finalize) to pass the per-root privateMerkleRoot. Verified: repro green; full storage suite (218) + full publisher suite (1179) green (zero existing-test changes). Co-Authored-By: Claude Opus 4.8 --- .../src/async-lift-publisher-impl.ts | 8 ++ packages/publisher/src/dkg-publisher.ts | 8 +- packages/storage/src/private-store.ts | 84 +++++++++++++++++++ .../issue-1078-private-layer-scope.test.ts | 65 ++++++++++++++ 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 packages/storage/test/issue-1078-private-layer-scope.test.ts diff --git a/packages/publisher/src/async-lift-publisher-impl.ts b/packages/publisher/src/async-lift-publisher-impl.ts index d286e0275..f1706d856 100644 --- a/packages/publisher/src/async-lift-publisher-impl.ts +++ b/packages/publisher/src/async-lift-publisher-impl.ts @@ -28,6 +28,7 @@ import { } from './async-lift-publish-result.js'; import { prepareAsyncPublishPayload, type AsyncPreparedPublishPayload, type LiftResolvedPublishSlice } from './async-lift-publish-options.js'; import { canonicalRootIri, validateLiftPublishPayload } from './async-lift-validation.js'; +import { computePrivateRootV10 } from './merkle.js'; import { subtractFinalizedExactQuads } from './async-lift-subtraction.js'; import { resolveLiftWorkspaceSlice } from './workspace-resolution.js'; import { @@ -1001,11 +1002,18 @@ export class TripleStoreAsyncLiftPublisher implements AsyncLiftPublisher { const canonicalRoot = job.validation.canonicalRootMap[sourceRoot] ?? sourceRoot; const canonicalQuads = canonicalizePrivateStagedQuads(staged, job.validation.canonicalRootMap); + // GH #1078 — derive the commitment from the finalized slice's + // privateMerkleRoot so a re-finalize that produces DIFFERENT content + // supersedes the stale slice (re-finalizing identical content is + // idempotent: same digest → append+dedup, no churn). + const commitmentRoot = computePrivateRootV10(canonicalQuads); + const commitmentId = commitmentRoot ? Buffer.from(commitmentRoot).toString('hex') : undefined; await privateStore.storePrivateTriples( job.request.contextGraphId, canonicalRoot, canonicalQuads, job.request.subGraphName, + commitmentId, ); await privateStore.deletePrivateTriplesForOperation( job.request.contextGraphId, diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index 49b76a52b..2f2e07173 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -1983,7 +1983,13 @@ export class DKGPublisher implements Publisher { (q) => q.subject === entry.rootEntity || q.subject.startsWith(entry.rootEntity + '/.well-known/genid/'), ); if (entityPrivateQuads.length > 0) { - await this.privateStore.storePrivateTriples(contextGraphId, entry.rootEntity, entityPrivateQuads, options.subGraphName); + // GH #1078 — tag the stored slice with the commitment this root just + // committed (its privateMerkleRoot) so a re-publish for the same root + // supersedes the stale slice instead of commingling it. + const commitmentId = entry.privateMerkleRoot + ? Buffer.from(entry.privateMerkleRoot).toString('hex') + : undefined; + await this.privateStore.storePrivateTriples(contextGraphId, entry.rootEntity, entityPrivateQuads, options.subGraphName, commitmentId); } } diff --git a/packages/storage/src/private-store.ts b/packages/storage/src/private-store.ts index f9877a0a8..37495e3f3 100644 --- a/packages/storage/src/private-store.ts +++ b/packages/storage/src/private-store.ts @@ -7,6 +7,16 @@ import { import type { TripleStore, Quad } from './triple-store.js'; import type { ContextGraphManager } from './graph-manager.js'; +/** + * GH #1078 — predicate used for the per-(cg,root[,sub]) "current verifiable + * commitment" marker. Stored on a dedicated `urn:dkg:private-commitment-marker:…` + * subject inside the private graph so it is (a) never returned by + * `getPrivateTriples` (which filters on the root subject / its skolem children), + * (b) never collected by the plaintext dedup scan (different predicate), and + * (c) dropped together with the private graph on `dropContextGraph`. + */ +const PRIVATE_COMMITMENT_PRED = 'http://dkg.io/ontology/privateCommitment'; + /** * Manages private triples stored on the local node. Peer-to-peer private * payloads are encrypted before they arrive here; after a node decrypts and @@ -171,6 +181,7 @@ export class PrivateContentStore { rootEntity: string, quads: Quad[], subGraphName?: string, + commitmentId?: string, ): Promise { if (quads.length === 0) return; @@ -178,9 +189,41 @@ export class PrivateContentStore { for (const q of quads) { assertSafePrivateQuad(q); } + if (commitmentId !== undefined && !/^[A-Za-z0-9_.:-]+$/.test(commitmentId)) { + throw new Error(`Unsafe private commitmentId: ${commitmentId.slice(0, 80)}`); + } const graphUri = this.privateGraph(contextGraphId, subGraphName); await this.withGraphWriteLock(graphUri, async () => { + // GH #1078 — verifiable-commitment scoping. When the caller supplies the + // commitment this root actually committed (its `privateMerkleRoot`) AND it + // DIFFERS from the commitment currently recorded for the root, a NEW + // commitment is superseding a stale one (a re-publish, or a draft slice + // replaced by the finalized one): drop the root's prior finalized private + // slice before inserting the new one, so a later hydration / privateData- + // Anchor never returns a different commitment's triples. With NO + // commitmentId the behaviour is unchanged (append + dedup), preserving + // every existing caller and multi-value private predicates. + if (commitmentId !== undefined) { + const markerSubject = privateCommitmentMarkerSubject(contextGraphId, rootEntity, subGraphName); + const current = await this.readPrivateCommitment(graphUri, markerSubject); + if (current !== commitmentId) { + // A different (or first) commitment takes over this root: drop the + // root's prior finalized private slice (a no-op on the very first + // write) so the new commitment fully replaces it, then stamp the new + // commitment marker. Repeated stores under the SAME commitment fall + // through to the append+dedup path below (chunked/retry-safe). + await this.store.deleteBySubjectPrefix(graphUri, rootEntity); + await this.store.deleteByPattern({ graph: graphUri, subject: markerSubject }); + await this.store.insert([{ + subject: markerSubject, + predicate: PRIVATE_COMMITMENT_PRED, + object: `"${commitmentId}"`, + graph: graphUri, + }]); + } + } + const existingPlainKeys = await this.collectExistingPlaintextKeys( graphUri, quads, @@ -328,6 +371,30 @@ export class PrivateContentStore { const entities = this.privateEntities.get(key); if (entities) entities.delete(rootEntity); } + + /** + * GH #1078 — read the commitment id currently recorded for a root's private + * slice (undefined when none has been recorded, i.e. legacy/append-only + * writers). Used to decide whether an incoming commitment supersedes the + * stored slice. + */ + private async readPrivateCommitment( + graphUri: string, + markerSubject: string, + ): Promise { + const result = await this.store.query( + `SELECT ?c WHERE { + GRAPH <${assertSafeIri(graphUri)}> { + <${assertSafeIri(markerSubject)}> <${PRIVATE_COMMITMENT_PRED}> ?c . + } + } LIMIT 1`, + ); + if (result.type !== 'bindings' || result.bindings.length === 0) return undefined; + const raw = result.bindings[0]?.['c']; + if (typeof raw !== 'string') return undefined; + const m = raw.match(/^"([\s\S]*)"(\^\^.*)?$/); + return m ? m[1] : raw; + } } function privateStageSubject( @@ -343,6 +410,23 @@ function privateStageSubject( return subject; } +/** + * GH #1078 — subject for the per-(cg,root[,sub]) commitment marker. Deliberately + * NOT prefixed by the root IRI so `getPrivateTriples`' root-subject filter and + * the `deleteBySubjectPrefix(root)` supersede sweep never touch it. + */ +function privateCommitmentMarkerSubject( + contextGraphId: string, + rootEntity: string, + subGraphName?: string, +): string { + const parts = [contextGraphId, subGraphName ?? '_', rootEntity] + .map((part) => encodeURIComponent(part)); + const subject = `urn:dkg:private-commitment-marker:${parts.join(':')}`; + assertSafeIri(subject); + return subject; +} + function parseLiteral(value: string | undefined): unknown { if (!value) return undefined; try { diff --git a/packages/storage/test/issue-1078-private-layer-scope.test.ts b/packages/storage/test/issue-1078-private-layer-scope.test.ts new file mode 100644 index 000000000..ac5b6f37d --- /dev/null +++ b/packages/storage/test/issue-1078-private-layer-scope.test.ts @@ -0,0 +1,65 @@ +/** + * Issue-liveness repro for GH #1078 — "Private payload storage is not scoped to + * memory layer or verifiable commitment." + * https://github.com/OriginTrail/dkg/issues/1078 + * + * `PrivateContentStore` keys the finalized private partition only by + * `(contextGraphId[, subGraphName])` → `…/_private`. It is NOT split by memory + * layer / KA version the way public state is (WM/SWM/VM each get their own + * graph). So two DISTINCT private commitments for the SAME root — e.g. a stale + * draft slice and the slice the verifiable KA actually committed — land in ONE + * graph and `getPrivateTriples(cg, root)` returns BOTH. A caller that follows a + * `dkg:privateDataAnchor` on a verifiable KA then hydrates triples that a + * different layer/version committed. + * + * This test asserts the CORRECT (post-fix) behaviour, so it is RED today + * (the bug is live) and turns GREEN once the fix lands; it stays red until #1078 is fixed.. Hermetic — in-memory oxigraph only. + */ +import { describe, expect, it } from 'vitest'; +import { OxigraphStore, GraphManager, PrivateContentStore, type Quad } from '../src/index.js'; + +const CG = 'gh1078-cg'; +const ROOT = 'urn:gh1078:device'; + +function priv(predicate: string, object: string): Quad { + return { subject: ROOT, predicate, object, graph: '' }; +} + +describe('GH #1078 — private payload storage must be scoped to the committing layer/commitment', () => { + it( + 'a root hydrates only the authoritative private slice, not a different commitment for the same root', + async () => { + const store = new OxigraphStore(); + const gm = new GraphManager(store); + const pcs = new PrivateContentStore(store, gm); + + // Commitment #1 — an EARLIER private slice for ROOT (e.g. a WM/SWM draft or + // a superseded KA version). The exact authority does not matter; what + // matters is that it is a DIFFERENT private payload committed under the + // same root, identified by its own verifiable-commitment id. + await pcs.storePrivateTriples(CG, ROOT, [priv('https://schema.org/serialNumber', '"OLD-0001"')], undefined, 'commitment-old'); + + // Commitment #2 — the slice the AUTHORITATIVE / verifiable KA actually + // committed for ROOT. This is what a `privateDataAnchor` on the verifiable + // KA should resolve to. A NEW commitment supersedes the stale one for the + // same root, so the stale slice must not survive into hydration. + await pcs.storePrivateTriples(CG, ROOT, [priv('https://schema.org/serialNumber', '"NEW-0002"')], undefined, 'commitment-new'); + + // Hydration for ROOT (the only main-API read path). + const hydrated = await pcs.getPrivateTriples(CG, ROOT); + const serials = hydrated + .filter((q) => q.predicate === 'https://schema.org/serialNumber') + .map((q) => q.object); + + // Control: the authoritative slice IS present (so the negative assertion + // below is meaningful and not vacuously true on an empty read). + expect(serials).toContain('"NEW-0002"'); + + // CORRECT (post-fix): a verifiable-commitment-scoped hydration returns ONLY + // the authoritative slice. Today private storage is layer-blind, so the + // superseded "OLD-0001" leaks back in — exactly the cross-commitment + // hydration #1078 describes. + expect(serials).not.toContain('"OLD-0001"'); + }, + ); +}); From d13ff9364900b26d1e0bb434680917dcb0221df0 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 16:40:34 +0200 Subject: [PATCH 04/14] fix(agent): encrypt operational wallet private keys at rest (#11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit saveOpWallets wrote raw secp256k1 private keys to wallets.json in plaintext (mode 0600 but cleartext) — on mainnet those hold real TRAC/ETH. Encrypt each key with AES-256-GCM under a machine-local 32-byte secret (wallets.key, zero operator interaction), optionally strengthened by DKG_WALLETS_PASSPHRASE via scrypt. Addresses stay plaintext so every address-only reader keeps working; loadOpWallets decrypts in memory so the chain config is unchanged. Legacy plaintext files load as-is and are NOT rewritten (so devnet provisioning + the live-daemon fixture, which read wallets[0].privateKey directly, keep working) — new nodes generate encrypted. Verified: repro green (no plaintext key anywhere on disk); op-wallets + chain-reset + publisher-wallets tests green; full agent (1614) + cli (2029) suites green; fresh 6-node devnet boots healthy. Co-Authored-By: Claude Opus 4.8 --- packages/agent/src/op-wallets.ts | 154 +++++++++++++++++- .../op-wallets-at-rest-encryption.test.ts | 65 ++++++++ 2 files changed, 211 insertions(+), 8 deletions(-) create mode 100644 packages/agent/test/op-wallets-at-rest-encryption.test.ts diff --git a/packages/agent/src/op-wallets.ts b/packages/agent/src/op-wallets.ts index d7995ebe9..cfb1f3368 100644 --- a/packages/agent/src/op-wallets.ts +++ b/packages/agent/src/op-wallets.ts @@ -1,6 +1,7 @@ import { ethers } from 'ethers'; import { chmod, readFile, writeFile, mkdir } from 'node:fs/promises'; import { join } from 'node:path'; +import { randomBytes, scryptSync, createCipheriv, createDecipheriv } from 'node:crypto'; export interface WalletEntry { address: string; @@ -16,6 +17,31 @@ export interface OpWalletsConfig { const DEFAULT_WALLET_COUNT = 3; +// GH #11 — operational wallet private keys are encrypted at rest (AES-256-GCM) +// so `wallets.json` never carries a plaintext key. The key is derived from a +// machine-local 32-byte secret in `wallets.key` (zero operator interaction); +// when `DKG_WALLETS_PASSPHRASE` is set it is mixed in via scrypt for an extra +// factor against host-FS compromise. `address` stays plaintext so every +// address-only reader (faucet, openclaw setup, status) keeps working, and +// `loadOpWallets` still returns decrypted keys in memory for the chain config. +const WALLET_SECRET_FILE = 'wallets.key'; +const PASSPHRASE_ENV = 'DKG_WALLETS_PASSPHRASE'; + +interface EncryptedKeystore { + v: number; + alg: 'aes-256-gcm'; + kdf: 'raw' | 'scrypt'; + iv: string; + ct: string; + tag: string; +} + +interface StoredWalletEntry { + address: string; + privateKey?: string; + keystore?: EncryptedKeystore; +} + /** * Load admin + operational wallets from `wallets.json` in the data directory. * Legacy files without `adminWallet` remain readable, but profile @@ -34,7 +60,7 @@ export async function loadOpWallets( try { const raw = await readFile(filePath, 'utf-8'); - const parsed = JSON.parse(raw) as Partial | WalletEntry[]; + const parsed = JSON.parse(raw) as { adminWallet?: StoredWalletEntry; wallets?: StoredWalletEntry[] } | StoredWalletEntry[]; const existingWallets = Array.isArray(parsed) ? parsed : parsed.wallets; if (!Array.isArray(existingWallets)) { throw new Error('wallets.json must contain a wallets array'); @@ -44,12 +70,37 @@ export async function loadOpWallets( } { - const wallets = existingWallets.map((w, index) => - validateWalletEntry(w, `wallets[${index}]`), - ); - const adminWallet = !Array.isArray(parsed) && parsed.adminWallet - ? validateWalletEntry(parsed.adminWallet, 'adminWallet') - : undefined; + const adminStored = !Array.isArray(parsed) ? parsed.adminWallet : undefined; + // Decrypt any keystore entries before validating addresses. The secret is + // only required when at least one entry is encrypted; legacy plaintext + // files load with no secret and are opportunistically re-encrypted below. + const hasEncrypted = existingWallets.some(isEncryptedEntry) + || (adminStored ? isEncryptedEntry(adminStored) : false); + const secret = hasEncrypted ? await loadWalletSecret(dataDir) : undefined; + if (hasEncrypted && !secret) { + throw new Error( + `wallets.json holds encrypted wallet keystores but ${WALLET_SECRET_FILE} is missing — cannot decrypt. ` + + `Restore ${WALLET_SECRET_FILE} (or set ${PASSPHRASE_ENV} if it was passphrase-protected) from backup.`, + ); + } + + const resolve = (stored: StoredWalletEntry, path: string): WalletEntry => { + if (isEncryptedEntry(stored)) { + const privateKey = decryptKey(stored.keystore!, secret!); + return validateWalletEntry({ address: stored.address, privateKey }, path); + } + // Legacy plaintext entry — load as-is. We deliberately do NOT rewrite + // the file: a hand-provisioned / tooling-written plaintext wallets.json + // (devnet provisioning, the live-daemon test fixture, operators who + // manage the file directly) must stay byte-compatible with external + // readers. New nodes generate encrypted wallets (see saveOpWallets), so + // a fresh install is encrypted at rest; existing plaintext files are + // honoured unchanged. + return validateWalletEntry({ address: stored.address, privateKey: stored.privateKey as string }, path); + }; + + const wallets = existingWallets.map((w, index) => resolve(w, `wallets[${index}]`)); + const adminWallet = adminStored ? resolve(adminStored, 'adminWallet') : undefined; if (adminWallet) { const adminKey = adminWallet.address.toLowerCase(); @@ -85,8 +136,18 @@ export function generateWallets(count: number): OpWalletsConfig { async function saveOpWallets(dataDir: string, config: OpWalletsConfig): Promise { await mkdir(dataDir, { recursive: true }); + const secret = await loadOrCreateWalletSecret(dataDir); + const passphrase = process.env[PASSPHRASE_ENV]; + const encEntry = (w: WalletEntry): StoredWalletEntry => ({ + address: w.address, + keystore: encryptKey(w.privateKey, secret, passphrase), + }); + const stored: { adminWallet?: StoredWalletEntry; wallets: StoredWalletEntry[] } = { + ...(config.adminWallet ? { adminWallet: encEntry(config.adminWallet) } : {}), + wallets: config.wallets.map(encEntry), + }; const filePath = join(dataDir, 'wallets.json'); - await writeFile(filePath, JSON.stringify(config, null, 2) + '\n', { mode: 0o600 }); + await writeFile(filePath, JSON.stringify(stored, null, 2) + '\n', { mode: 0o600 }); await chmod(filePath, 0o600); } @@ -104,3 +165,80 @@ function validateWalletEntry(entry: WalletEntry, path: string): WalletEntry { } return { address: derived.address, privateKey: derived.privateKey }; } + +// ── GH #11 — at-rest encryption helpers ──────────────────────────────────── + +function isEncryptedEntry(entry: StoredWalletEntry): boolean { + return !!entry && typeof entry === 'object' && !!entry.keystore; +} + +/** Derive the 32-byte AES key: the raw machine-local secret, optionally + * strengthened with a scrypt pass over an operator passphrase. */ +function deriveAtRestKey(secret: Buffer, kdf: 'raw' | 'scrypt', passphrase?: string): Buffer { + if (kdf === 'scrypt') { + if (!passphrase) { + throw new Error(`${PASSPHRASE_ENV} is required to derive the wallet key for a passphrase-protected keystore`); + } + return scryptSync(passphrase, secret, 32, { N: 16384, r: 8, p: 1 }); + } + return secret; +} + +function encryptKey(privateKey: string, secret: Buffer, passphrase?: string): EncryptedKeystore { + const kdf: 'raw' | 'scrypt' = passphrase ? 'scrypt' : 'raw'; + const key = deriveAtRestKey(secret, kdf, passphrase); + const iv = randomBytes(12); + const cipher = createCipheriv('aes-256-gcm', key, iv); + const ct = Buffer.concat([cipher.update(privateKey, 'utf8'), cipher.final()]); + const tag = cipher.getAuthTag(); + return { + v: 1, + alg: 'aes-256-gcm', + kdf, + iv: iv.toString('base64'), + ct: ct.toString('base64'), + tag: tag.toString('base64'), + }; +} + +function decryptKey(keystore: EncryptedKeystore, secret: Buffer): string { + const passphrase = process.env[PASSPHRASE_ENV]; + if (keystore.kdf === 'scrypt' && !passphrase) { + throw new Error( + `wallet keystore was encrypted with ${PASSPHRASE_ENV} which is not currently set — refusing to decrypt`, + ); + } + const key = deriveAtRestKey(secret, keystore.kdf, passphrase); + const decipher = createDecipheriv('aes-256-gcm', key, Buffer.from(keystore.iv, 'base64')); + decipher.setAuthTag(Buffer.from(keystore.tag, 'base64')); + const pt = Buffer.concat([decipher.update(Buffer.from(keystore.ct, 'base64')), decipher.final()]); + return pt.toString('utf8'); +} + +/** Read the machine-local wallet secret, or undefined if it doesn't exist. */ +async function loadWalletSecret(dataDir: string): Promise { + try { + const raw = await readFile(join(dataDir, WALLET_SECRET_FILE), 'utf-8'); + const buf = Buffer.from(raw.trim(), 'base64'); + if (buf.length !== 32) { + throw new Error(`${WALLET_SECRET_FILE} is malformed (expected a 32-byte base64 secret)`); + } + return buf; + } catch (err: any) { + if (err.code === 'ENOENT') return undefined; + throw err; + } +} + +/** Read or create the machine-local wallet secret (32 random bytes, mode 0600). + * The secret is independent of any wallet key, so it never leaks a private key. */ +async function loadOrCreateWalletSecret(dataDir: string): Promise { + const existing = await loadWalletSecret(dataDir); + if (existing) return existing; + await mkdir(dataDir, { recursive: true }); + const secret = randomBytes(32); + const secretPath = join(dataDir, WALLET_SECRET_FILE); + await writeFile(secretPath, secret.toString('base64') + '\n', { mode: 0o600 }); + await chmod(secretPath, 0o600); + return secret; +} diff --git a/packages/agent/test/op-wallets-at-rest-encryption.test.ts b/packages/agent/test/op-wallets-at-rest-encryption.test.ts new file mode 100644 index 000000000..c3563e984 --- /dev/null +++ b/packages/agent/test/op-wallets-at-rest-encryption.test.ts @@ -0,0 +1,65 @@ +/** + * Liveness/regression test for GH #11 — "Secrets partially unencrypted on disk". + * https://github.com/OriginTrail/dkg/issues/11 + * + * An AES-256-GCM keystore module exists (`packages/cli/src/keystore.ts`) but is + * not wired into the operational-wallet storage path: `loadOpWallets` generates + * wallets and persists them via `saveOpWallets`, which writes + * `JSON.stringify(config)` — including each wallet's raw `privateKey` — to + * `wallets.json` (mode 0o600 but unencrypted). On mainnet those wallets hold + * real TRAC/ETH, so plaintext-at-rest is a real exposure. + * + * This asserts the CORRECT (post-fix) behaviour — no wallet's raw private key + * appears in plaintext ANYWHERE under the data dir — so it is RED while the keys + * are stored plaintext and GREEN once the keystore is wired in. It scans EVERY + * persisted file (not just `wallets.json`) so a valid fix that moves secrets into + * an encrypted keystore, renames the artifact, or leaves only non-secret + * metadata still turns it green (Codex review on PR #1129). Hermetic — tmpdir. + */ +import { describe, expect, it, afterEach } from 'vitest'; +import { mkdtemp, readFile, rm, readdir } from 'node:fs/promises'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { loadOpWallets } from '../src/op-wallets.js'; + +/** Every regular file under `dir`, recursively. */ +async function walkFiles(dir: string): Promise { + const out: string[] = []; + for (const entry of await readdir(dir, { withFileTypes: true })) { + const p = join(dir, entry.name); + if (entry.isDirectory()) out.push(...(await walkFiles(p))); + else if (entry.isFile()) out.push(p); + } + return out; +} + +const dirs: string[] = []; +afterEach(async () => { + await Promise.all(dirs.map((d) => rm(d, { recursive: true, force: true }).catch(() => {}))); + dirs.length = 0; +}); + +describe('GH #11 — operational wallet private keys at rest', () => { + it('does not persist any wallet raw private key in plaintext anywhere on disk', async () => { + const dir = await mkdtemp(join(tmpdir(), 'gh11-opwallets-')); + dirs.push(dir); + + const config = await loadOpWallets(dir, 2); // generates + persists on first call + + // Control: the in-memory config really does carry private keys (so the + // negative assertion below is meaningful, not vacuously true). + expect(config.wallets[0].privateKey).toMatch(/^0x[0-9a-fA-F]{64}$/); + + // Scan EVERY persisted file (and the bare-hex form, in case a future writer + // strips the 0x prefix). No file may contain a wallet's raw private key. + const files = await walkFiles(dir); + const blobs = await Promise.all(files.map((f) => readFile(f, 'utf-8').catch(() => ''))); + const combined = blobs.join('\n'); + for (const w of config.wallets) { + const hex = w.privateKey; + const bare = hex.replace(/^0x/, ''); + expect(combined, `a persisted file under ${dir} contains a raw private key`).not.toContain(hex); + expect(combined).not.toContain(bare); + } + }); +}); From 2cd60a33205842673c6a92024962dd8cc7cbb9de Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 16:40:51 +0200 Subject: [PATCH 05/14] fix(agent): bind onChainId for pre-subscribed public CGs in VM reconcile (#1098, partial) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PARTIAL FIX — closes layer 1 of #1098. A peer that subscribed to a PUBLIC CG BEFORE its first publish never bound sub.onChainId (only curated CGs bind on the ContextGraphCreated event; ACK-signers bind via the storage-ACK hook), so the chain-driven VM-reconcile sweep skipped it and the peer was stranded on unreliable one-shot gossip. Self-prime onChainId in runVmReconcileSweep (resolve from the local ontology/_meta OnChainId quad) and trigger a self-priming sweep from the live KA-registration nudge when no local CG is bound. Regression-clean: gated on subscribed && !onChainId (binding from undefined never resets the cursor); full agent suite (1614) + reconcile tests green. KNOWN-INCOMPLETE (layer 2): live devnet shows materialization is still unreliable (~1/3). Root cause diagnosed: once bound, the chain-reconcile DOES promote the KC, but it lands ONLY in the per-cgId graph `/context/` without the root-label dual-write copy (keepRootCopyOnLabel isn't recovered on a peer that missed the SWM-meta gossip), and the `verifiable-memory` view query resolves via the root-label `_meta` → misses per-cgId-only data. The fix belongs in either the query-engine VM-view graph resolution (resolve `/context/` from the bound subscription) or publisher keepRootCopyOnLabel-signal replication — a core view-routing change deferred pending review. Co-Authored-By: Claude Opus 4.8 --- packages/agent/src/dkg-agent-lifecycle.ts | 19 +++++++++++++++++- packages/agent/src/dkg-agent-swm-host.ts | 24 +++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 2c913f380..01fd13692 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -1421,7 +1421,24 @@ export class LifecycleSyncMethods extends DKGAgentBase { onKARegisteredToContextGraph: this.vmReconcileEnabled() ? async ({ contextGraphId: onChainId, kaId }) => { const localCgId = this.resolveLocalCgIdByOnChainId(BigInt(onChainId)); - if (!localCgId) return; // chain replay hasn't resolved the cleartext CG yet; sweep heals it + if (!localCgId) { + // GH #1098 — a pre-subscribed PUBLIC member peer may not have + // bound this CG's on-chain id yet (only curated CGs bind on the + // ContextGraphCreated event; ACK-signers bind via the storage-ACK + // hook). Rather than wait up to a full sweep interval, run a + // self-priming sweep NOW so any subscribed-but-unbound CG resolves + // + binds its on-chain id from the local ontology/_meta and + // reconciles immediately. Gated on there actually being an unbound + // subscription so a KA registration for an unrelated CG is a no-op. + const hasUnbound = [...this.subscribedContextGraphs.values()].some( + (s) => s.subscribed && !s.onChainId, + ); + if (hasUnbound) { + this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> self-priming reconcile sweep (unbound pre-subscription)`); + await this.runVmReconcileSweep(); + } + return; // chain replay hasn't resolved the cleartext CG yet; periodic sweep is the safety net + } const sub = this.subscribedContextGraphs.get(localCgId); // Populate VM for CGs we member-subscribe to OR (Phase D) public // CGs this Core hosts — a hosted Core fills its own gaps too. diff --git a/packages/agent/src/dkg-agent-swm-host.ts b/packages/agent/src/dkg-agent-swm-host.ts index b754ad61f..e5d706931 100644 --- a/packages/agent/src/dkg-agent-swm-host.ts +++ b/packages/agent/src/dkg-agent-swm-host.ts @@ -2348,6 +2348,30 @@ export class SwmHostModeMethods extends DKGAgentBase { async runVmReconcileSweep(this: DKGAgent): Promise { if (!this.vmReconcileEnabled() || !this.reconcileCoalescer) return; for (const [localCgId, sub] of this.subscribedContextGraphs) { + // GH #1098 — self-prime the on-chain id for a pre-subscribed PUBLIC member + // CG. A peer that subscribed BEFORE the CG's first publish has + // `sub.onChainId` unset: the chain `ContextGraphCreated` handler only binds + // it for CURATED CGs, and the ACK-signer path (`recordCoreHostedPublicCg`) + // only fires for cores in that publish's storage-ACK set. Left unbound, the + // sweep below skips the CG and the peer is stranded on the unreliable + // one-shot finalization gossip (the ~1/3 materialization + kc-not-synced + // spam in #1098). As soon as the CG's OnChainId quad is local (publisher + // ontology-topic broadcast or durable _meta sync), resolve + bind it so the + // reliable chain-driven reconcile/backfill runs for this peer — the same + // path the late-subscriber (#886) and ACK-signer cores already converge on. + // Best-effort + gated on `!sub.onChainId` so it binds at most once (a later + // id CHANGE would reset the cursor; binding from undefined never does). + if (sub.subscribed && !sub.onChainId) { + try { + const resolved = await this.getContextGraphOnChainId(localCgId); + if (resolved && resolved !== localCgId) { + this.bindSubscriptionOnChainId(localCgId, sub, resolved); + this.persistContextGraphSubscription(localCgId); + } + } catch { + /* a store/RPC hiccup on one CG must not abort the whole sweep */ + } + } // Member subscriptions AND Phase D core-hosted public CGs get swept. if ((!sub.subscribed && !sub.coreHosted) || !sub.onChainId) continue; void this.reconcileCoalescer.trigger(localCgId); From c976b412408fcf32992b418cf149f7dd71eed477 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 16:54:25 +0200 Subject: [PATCH 06/14] fix(query): verifiable-memory view resolves per-cgId data graphs (#1098, completes fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes #1098 (layer 2). The chain-driven VM reconcile materialises confirmed data into the per-cgId graph `/context/`, but the `verifiable-memory` view only resolved the root content graph + `_verifiable_memory/*` — so a peer that subscribed BEFORE publish and recovered via the reconcile sweep (per-cgId only, no root-label dual-write copy) had the KA materialised yet INVISIBLE to a VM query. Union the per-cgId DATA graphs (resolved from the store by prefix, excluding `/_meta` + nested sub-partitions) into the unscoped VM allow-set. No subscription state needed; additive to the allow-set so dual-written publishes (already in the root graph) just dedup. Verified on a live 6-node devnet: a pre-subscribed peer now materialises a published KA reliably — 5/5 rapid-fire + 4/4 paced (was ~1/3). Combined with the onChainId-binding commit this closes #1098. Cross-checked #936 on the same devnet: node2 + node3 chain-reconcile the same multi-root KC to an identical deterministic root->tokenId map. Full query suite (277) green. Co-Authored-By: Claude Opus 4.8 --- packages/query/src/dkg-query-engine.ts | 34 ++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/packages/query/src/dkg-query-engine.ts b/packages/query/src/dkg-query-engine.ts index 936c8246f..ca08b2462 100644 --- a/packages/query/src/dkg-query-engine.ts +++ b/packages/query/src/dkg-query-engine.ts @@ -499,6 +499,21 @@ export class DKGQueryEngine implements QueryEngine { } } + // GH #1098 — include the per-cgId VM data graph(s) `/context/`. + // Chain-driven VM reconcile (and any per-cgId-only materialisation — e.g. a + // peer that subscribed BEFORE publish and recovered via the reconcile sweep, + // which writes confirmed data into the per-cgId graph WITHOUT the root-label + // dual-write copy) lands confirmed data ONLY in `/context/`. The base + // `verifiable-memory` resolution above only reads the root content graph + + // `_verifiable_memory/*`, so that data was invisible to a VM read (the + // observable symptom of #1098: a pre-subscribed peer never "saw" the + // published KA even though it had materialised it). Union the per-cgId DATA + // graphs (resolved from the store, so no subscription state is needed) into + // the allow-set for an unscoped VM read. + if (view === 'verifiable-memory' && !options.verifiedGraph && !options.subGraphName) { + allGraphs.push(...(await this.discoverContextGraphPerCgIdDataGraphs(contextGraphId))); + } + // De-dup so a sub-graph never gets unioned twice. const dedupedGraphs = [...new Set(allGraphs)]; allGraphs.length = 0; @@ -642,6 +657,25 @@ export class DKGQueryEngine implements QueryEngine { ); } + /** + * GH #1098 — discover the per-cgId VM DATA graphs `/context/` + * for a context graph. Chain-reconciled / per-cgId-only publishes materialise + * confirmed VM content here rather than in the root content graph. We keep + * ONLY the bare `/context/` data graphs: `discoverGraphsByPrefix` + * already drops `/_meta` + `/staging/`, and the extra `!rest.includes('/')` + * guard excludes any nested per-cgId sub-partition (`…/context//_private`, + * `…/context//_shared_memory`, …) so a VM content read never pulls + * private/SWM rows. + */ + private async discoverContextGraphPerCgIdDataGraphs(contextGraphId: string): Promise { + const base = `did:dkg:context-graph:${contextGraphId}/context/`; + const discovered = await this.discoverGraphsByPrefix(base); + return discovered.filter((g) => { + const rest = g.slice(base.length); + return rest.length > 0 && !rest.includes('/'); + }); + } + /** * Resolve a WM assertion's allocated KA number (`dkg:kaId`) off its lifecycle URN * in `_meta`, for single-graph by-name reads under the uniform layout. Returns From 5b40242e2aec2263e75a20327d1fbc142652ba11 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 17:47:39 +0200 Subject: [PATCH 07/14] fix(publisher): scope #1121 encrypted-inline requirement to on-chain publishes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The #1121 fail-closed encryption default broke chainless / local-only publishes: an ownerOnly KA on an UNREGISTERED context graph (e.g. the kafka-plugin's single-daemon private-stream registration, or a chain-not-ready node) ships nothing to other nodes, so there is no plaintext-to-cores leak — yet the publisher took the encrypted-inline path purely because a callback was defined, invoked the fail-closed default, and threw ("requires an encryptInlinePayload factory"). Gate `useEncryptedInline` on `canAttemptOnChainPublish` so the encryption hook is only required when the publish actually goes on-chain (the path that collects core StorageACKs / distributes to members — the only place private data could leak in the clear). On-chain private publishes still require encryption; chainless/local publishes finalize locally as before. Verified: full publisher suite (1179) green; kafka-plugin live-daemon E2E (11) green (was 4 failed on the PR's first CI run). Co-Authored-By: Claude Opus 4.8 --- packages/publisher/src/dkg-publisher.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index 2f2e07173..e1ed21eee 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -2108,7 +2108,19 @@ export class DKGPublisher implements Publisher { // with NO_DATA_IN_SWM — the exact bug §1.1 surfaces. Public CGs keep // the existing behaviour: `fromSharedMemory` → cores look up SWM // locally; otherwise plaintext inline. - const useEncryptedInline = typeof options.encryptInlinePayload === 'function'; + // GH #1121 — take the encrypted-inline path ONLY when this publish actually + // goes on-chain (collects core StorageACKs / distributes to CG members). A + // chainless / local-only publish (e.g. an ownerOnly KA on an unregistered + // CG, or a chain-not-ready node) ships nothing to other nodes, so there is + // no plaintext-to-cores leak to guard against — and forcing the encryption + // hook there would break a legitimate local publish (the curated chain-key + // the real hook needs cannot be resolved off-chain, so the async-lift + // mapper's fail-closed default would throw). When the publish IS on-chain + // the requirement stands: a defined hook (the agent-resolved real callback, + // or the fail-closed default) is used, so private data is never shipped to + // cores in the clear. + const useEncryptedInline = + canAttemptOnChainPublish && typeof options.encryptInlinePayload === 'function'; // OT-RFC-49 / WS-D — a curated publish is now identified by a non-zero // catalog commitment. The on-chain commitment + the core ACK verify the // PUBLIC `_catalog`; the PRIVATE data stays encrypted for MEMBERS only. From 28a1161081728e158239e29681f00b7bb1c5ab81 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 18:12:13 +0200 Subject: [PATCH 08/14] fix: resolve otReviewAgent findings on PR #1228 (2 bugs + coverage) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🔴 #936 cross-path consistency — the publisher (DKGPublisher.publish) minted compatibility tokenIds in input-quad order while replicas sort, so the originator and replicas could map a different root to the same `/` label. Sort canonical.manifestEntries lexicographically by rootEntity before the mint, matching promoteSharedMemoryToCanonical. Now all three mint paths (publisher, gossip, chain-reconcile) agree. 🔴 #1098 self-prime guard — the sweep skipped binding when the resolved on-chain id equalled the local CG id (a direct CG whose local id IS its numeric id, e.g. "42"), leaving it unbound forever. getContextGraphOnChainId never falls back to localCgId, so bind ANY non-null resolved id. 🟡 #1098 nudge locality — the KA-registration nudge ran a GLOBAL self-priming sweep whenever any subscription was unbound. Scope it: bind + reconcile only the subscribed-unbound CG whose resolved on-chain id matches THIS event. 🟡 test coverage: - op-wallets-at-rest: also scan adminWallet for plaintext + assert a reload decrypts back to the same admin + operational keys. - query/vm-percgid-resolution (new): unscoped verifiable-memory read returns data that lives ONLY in /context/, excluding meta/private. - agent/vm-reconcile-self-prime (new): a subscribed-but-unbound CG binds onChainId from the ontology quad + triggers reconcile in the sweep. Verified: query 278, agent 1621, publisher 1179 — all green. Co-Authored-By: Claude Opus 4.8 --- packages/agent/src/dkg-agent-lifecycle.ts | 32 +++++--- packages/agent/src/dkg-agent-swm-host.ts | 7 +- .../op-wallets-at-rest-encryption.test.ts | 21 ++++- .../test/vm-reconcile-self-prime.test.ts | 80 +++++++++++++++++++ packages/publisher/src/dkg-publisher.ts | 13 ++- .../query/test/vm-percgid-resolution.test.ts | 64 +++++++++++++++ 6 files changed, 201 insertions(+), 16 deletions(-) create mode 100644 packages/agent/test/vm-reconcile-self-prime.test.ts create mode 100644 packages/query/test/vm-percgid-resolution.test.ts diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index ab0d7bc92..6f4d00ae4 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -1515,17 +1515,27 @@ export class LifecycleSyncMethods extends DKGAgentBase { // GH #1098 — a pre-subscribed PUBLIC member peer may not have // bound this CG's on-chain id yet (only curated CGs bind on the // ContextGraphCreated event; ACK-signers bind via the storage-ACK - // hook). Rather than wait up to a full sweep interval, run a - // self-priming sweep NOW so any subscribed-but-unbound CG resolves - // + binds its on-chain id from the local ontology/_meta and - // reconciles immediately. Gated on there actually being an unbound - // subscription so a KA registration for an unrelated CG is a no-op. - const hasUnbound = [...this.subscribedContextGraphs.values()].some( - (s) => s.subscribed && !s.onChainId, - ); - if (hasUnbound) { - this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> self-priming reconcile sweep (unbound pre-subscription)`); - await this.runVmReconcileSweep(); + // hook). Find the subscribed-but-unbound CG whose locally-resolved + // on-chain id matches THIS event and bind + reconcile only it — + // targeted, not a global sweep, so an unrelated KA registration + // touches nothing. The periodic self-priming sweep remains the + // safety net for a CG whose OnChainId quad hasn't arrived yet. + let targetOnChain: bigint | null = null; + try { targetOnChain = BigInt(onChainId); } catch { targetOnChain = null; } + if (targetOnChain !== null) { + for (const [lcg, sub] of this.subscribedContextGraphs) { + if (!sub.subscribed || sub.onChainId) continue; + const resolved = await this.getContextGraphOnChainId(lcg).catch(() => null); + let resolvedNum: bigint | null = null; + try { resolvedNum = resolved ? BigInt(resolved) : null; } catch { resolvedNum = null; } + if (resolvedNum !== null && resolvedNum === targetOnChain) { + this.bindSubscriptionOnChainId(lcg, sub, resolved!); + this.persistContextGraphSubscription(lcg); + this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> bound + reconcile pre-subscribed "${lcg}"`); + if (this.reconcileCoalescer) void this.reconcileCoalescer.trigger(lcg); + break; + } + } } return; // chain replay hasn't resolved the cleartext CG yet; periodic sweep is the safety net } diff --git a/packages/agent/src/dkg-agent-swm-host.ts b/packages/agent/src/dkg-agent-swm-host.ts index e5d706931..f1be0eedc 100644 --- a/packages/agent/src/dkg-agent-swm-host.ts +++ b/packages/agent/src/dkg-agent-swm-host.ts @@ -2363,8 +2363,13 @@ export class SwmHostModeMethods extends DKGAgentBase { // id CHANGE would reset the cursor; binding from undefined never does). if (sub.subscribed && !sub.onChainId) { try { + // Bind ANY non-null resolved on-chain id. getContextGraphOnChainId + // returns the persisted OnChainId quad (or null) — it never falls back + // to localCgId, so a `resolved === localCgId` match is legitimate for a + // direct CG whose local id IS its numeric on-chain id (e.g. "42"); a + // `!== localCgId` guard would wrongly leave such a sub unbound forever. const resolved = await this.getContextGraphOnChainId(localCgId); - if (resolved && resolved !== localCgId) { + if (resolved) { this.bindSubscriptionOnChainId(localCgId, sub, resolved); this.persistContextGraphSubscription(localCgId); } diff --git a/packages/agent/test/op-wallets-at-rest-encryption.test.ts b/packages/agent/test/op-wallets-at-rest-encryption.test.ts index c3563e984..d23b21a36 100644 --- a/packages/agent/test/op-wallets-at-rest-encryption.test.ts +++ b/packages/agent/test/op-wallets-at-rest-encryption.test.ts @@ -49,17 +49,32 @@ describe('GH #11 — operational wallet private keys at rest', () => { // Control: the in-memory config really does carry private keys (so the // negative assertion below is meaningful, not vacuously true). expect(config.wallets[0].privateKey).toMatch(/^0x[0-9a-fA-F]{64}$/); + expect(config.adminWallet?.privateKey).toMatch(/^0x[0-9a-fA-F]{64}$/); // Scan EVERY persisted file (and the bare-hex form, in case a future writer - // strips the 0x prefix). No file may contain a wallet's raw private key. + // strips the 0x prefix). No file may contain a wallet's raw private key — + // including the ADMIN wallet (a regression that encrypts operational wallets + // but leaves the admin key plaintext must fail here too). + const allKeys = [ + ...config.wallets.map((w) => w.privateKey), + ...(config.adminWallet ? [config.adminWallet.privateKey] : []), + ]; const files = await walkFiles(dir); const blobs = await Promise.all(files.map((f) => readFile(f, 'utf-8').catch(() => ''))); const combined = blobs.join('\n'); - for (const w of config.wallets) { - const hex = w.privateKey; + for (const hex of allKeys) { const bare = hex.replace(/^0x/, ''); expect(combined, `a persisted file under ${dir} contains a raw private key`).not.toContain(hex); expect(combined).not.toContain(bare); } + + // Reload from the now-encrypted files (simulates a daemon restart): the + // keystore must decrypt back to the SAME admin + operational keys. Guards + // against writing an unreadable keystore or rotating keys on reload. + const reloaded = await loadOpWallets(dir, 2); + expect(reloaded.adminWallet?.privateKey).toBe(config.adminWallet?.privateKey); + expect(reloaded.adminWallet?.address).toBe(config.adminWallet?.address); + expect(reloaded.wallets.map((w) => w.privateKey)).toEqual(config.wallets.map((w) => w.privateKey)); + expect(reloaded.wallets.map((w) => w.address)).toEqual(config.wallets.map((w) => w.address)); }); }); diff --git a/packages/agent/test/vm-reconcile-self-prime.test.ts b/packages/agent/test/vm-reconcile-self-prime.test.ts new file mode 100644 index 000000000..b7c336ae2 --- /dev/null +++ b/packages/agent/test/vm-reconcile-self-prime.test.ts @@ -0,0 +1,80 @@ +/** + * Regression test for GH #1098 (layer 1) — the chain-driven VM reconcile sweep + * must SELF-PRIME `onChainId` for a peer that subscribed to a PUBLIC CG BEFORE + * its first publish. Such a peer has `subscribed: true` but no `onChainId` (only + * curated CGs bind it on the ContextGraphCreated event; ACK-signers bind via the + * storage-ACK hook), so the sweep would otherwise skip it forever and the peer + * never reconciles the published KA into VM. + * + * This pins the state transition: a `subscribed && !onChainId` entry whose + * ontology OnChainId quad is locally present gets bound + persisted, and the + * sweep then triggers its reconcile. Hermetic — MockChainAdapter, no network. + */ +import { afterEach, describe, it, expect } from 'vitest'; +import { MockChainAdapter } from '@origintrail-official/dkg-chain'; +import { + DKG_ONTOLOGY, + SYSTEM_CONTEXT_GRAPHS, + contextGraphDataGraphUri, +} from '@origintrail-official/dkg-core'; +import type { TripleStore } from '@origintrail-official/dkg-storage'; +import { DKGAgent } from '../src/index.js'; + +interface AgentInternals { + runVmReconcileSweep(): Promise; + subscribedContextGraphs: Map; + reconcileCoalescer: { trigger: (cg: string) => void } | null; + store: TripleStore; +} + +// DKGNode getter throws on peerId access without a real start(); stub it so the +// subscription bookkeeping path runs (mirrors core-fills-gap.test.ts). +function stubNode(agent: DKGAgent): void { + (agent as unknown as { node: unknown }).node = { + peerId: '12D3KooWSelfPrimeTestPeer', + libp2p: { getPeers: () => [] }, + }; +} + +describe('GH #1098 — VM reconcile sweep self-primes onChainId for a pre-subscribed CG', () => { + let agent: DKGAgent | null = null; + afterEach(async () => { + if (agent) { await agent.stop().catch(() => undefined); agent = null; } + }); + + it('binds onChainId from the ontology quad, persists, and triggers reconcile for a subscribed-but-unbound CG', async () => { + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'SelfPrimeSweep', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + + const LOCAL = 'gh1098-presub'; + const ONCHAIN = '4242'; + + // The publisher broadcasts the CG's OnChainId quad on the ontology topic at + // publish time (durable _meta sync also delivers it). Seed it — this is the + // exact source `getContextGraphOnChainId` reads. + await internals.store.insert([{ + subject: `did:dkg:context-graph:${LOCAL}`, + predicate: `${DKG_ONTOLOGY.DKG_CONTEXT_GRAPH}OnChainId`, + object: `"${ONCHAIN}"`, + graph: contextGraphDataGraphUri(SYSTEM_CONTEXT_GRAPHS.ONTOLOGY), + }]); + + // The #1098 state: a pre-subscribed member CG with NO onChainId bound. + internals.subscribedContextGraphs.set(LOCAL, { subscribed: true }); + + const triggered: string[] = []; + internals.reconcileCoalescer = { trigger: (cg: string) => { triggered.push(cg); } }; + + // Precondition: unbound before the sweep (so the assertion below is meaningful). + expect(internals.subscribedContextGraphs.get(LOCAL)?.onChainId).toBeUndefined(); + + await internals.runVmReconcileSweep(); + + // Post-fix: the sweep self-primed onChainId from the ontology quad and then + // — no longer skipped by the `!onChainId` guard — triggered its reconcile. + expect(internals.subscribedContextGraphs.get(LOCAL)?.onChainId).toBe(ONCHAIN); + expect(triggered).toContain(LOCAL); + }); +}); diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index e1ed21eee..67142fe45 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -1898,8 +1898,19 @@ export class DKGPublisher implements Publisher { // many entities it contains. The on-chain KA count and ACK digest stay at // one below, while these token IDs remain compatibility labels for // per-root response/meta subjects (`/1`, `/2`, ...). + // GH #936 — mint the compatibility tokenIds over a CANONICAL (lexicographic + // by rootEntity) order, the SAME order the replica reconcile/gossip path + // uses in `FinalizationHandler.promoteSharedMemoryToCanonical`. Without this, + // the ORIGINATOR would label `/` by input-quad order while + // replicas label by sorted order, so a multi-root KC could resolve a + // different root for the same token label depending on which node a client + // queries. These tokenIds are non-on-chain compatibility labels (the + // on-chain KA count is 1), so a content-derived sort is safe. + const orderedEntries = [...canonical.manifestEntries].sort((a, b) => + a.rootEntity < b.rootEntity ? -1 : a.rootEntity > b.rootEntity ? 1 : 0, + ); let compatibilityTokenId = 1n; - for (const entry of canonical.manifestEntries) { + for (const entry of orderedEntries) { const tokenId = compatibilityTokenId++; manifestEntries.push({ tokenId, diff --git a/packages/query/test/vm-percgid-resolution.test.ts b/packages/query/test/vm-percgid-resolution.test.ts new file mode 100644 index 000000000..df5968c8f --- /dev/null +++ b/packages/query/test/vm-percgid-resolution.test.ts @@ -0,0 +1,64 @@ +/** + * Regression test for GH #1098 (layer 2) — the `verifiable-memory` view must + * resolve the per-cgId VM data graph `did:dkg:context-graph:/context/`. + * + * Chain-driven VM reconcile (and any per-cgId-only materialisation, e.g. a peer + * that subscribed BEFORE publish and recovered via the reconcile sweep) writes + * confirmed data ONLY into the per-cgId graph, never the root content graph. The + * base VM resolution reads only the root content graph + `_verifiable_memory/*`, + * so that data was materialised yet INVISIBLE to a VM query (the observable half + * of #1098). This pins that an unscoped VM read returns per-cgId data while still + * excluding the per-cgId `_meta` and the CG's `_private` partition. + * + * Hermetic: in-memory OxigraphStore, zero mocks, zero chain. + */ +import { describe, expect, it } from 'vitest'; +import { OxigraphStore } from '@origintrail-official/dkg-storage'; +import { DKGQueryEngine } from '../src/dkg-query-engine.js'; + +const CG = 'gh1098-percgid-cg'; +const ONCHAIN = '7'; +const NAME = 'http://schema.org/name'; +const DATA_ENTITY = 'https://example.org/gh1098/data'; +const META_ENTITY = 'https://example.org/gh1098/meta'; +const PRIVATE_ENTITY = 'https://example.org/gh1098/private'; + +function q(subject: string, predicate: string, object: string, graph: string) { + return { subject, predicate, object, graph }; +} + +describe('GH #1098 layer 2 — verifiable-memory view resolves per-cgId data graphs', () => { + it('an unscoped VM read returns data that lives ONLY in /context/, excluding meta/private', async () => { + const store = new OxigraphStore(); + const engine = new DKGQueryEngine(store); + + const perCgIdData = `did:dkg:context-graph:${CG}/context/${ONCHAIN}`; + const perCgIdMeta = `${perCgIdData}/_meta`; + const privateGraph = `did:dkg:context-graph:${CG}/_private`; + + await store.insert([ + // Confirmed VM content — lives ONLY in the per-cgId data graph (where the + // chain-reconcile path promotes it; NOT dual-written to the root graph). + q(DATA_ENTITY, NAME, '"PerCgIdConfirmed"', perCgIdData), + // A schema:name row in the per-cgId `_meta` — a content read must NOT pull + // metadata-graph rows into the result set. + q(META_ENTITY, NAME, '"PerCgIdMetaRow"', perCgIdMeta), + // A schema:name row in the CG's private partition — never surfaced by a + // VM content read (no includePrivate opt-in). + q(PRIVATE_ENTITY, NAME, '"SecretPrivate"', privateGraph), + ]); + + const result = await engine.query( + `SELECT ?s ?o WHERE { ?s <${NAME}> ?o }`, + { contextGraphId: CG, view: 'verifiable-memory' }, + ); + const objects = result.bindings.map((b) => b['o']); + + // The per-cgId confirmed data IS visible (the #1098 layer-2 fix). + expect(objects).toContain('"PerCgIdConfirmed"'); + // The per-cgId `_meta` graph is NOT unioned into a content VM read. + expect(objects).not.toContain('"PerCgIdMetaRow"'); + // The `_private` partition is NOT unioned into a VM read. + expect(objects).not.toContain('"SecretPrivate"'); + }); +}); From bda3a9c0a09bc91d15bd1a971f3a01b1aeee5ae2 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 18:39:16 +0200 Subject: [PATCH 09/14] fix: resolve otReviewAgent round-2 findings on PR #1228 (2 bugs + consistency + coverage) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🔴 #1078 supersede atomicity — the commitment-superseding private write ran pre-chain, so a failed/rejected re-publish could delete the still-current KA's private slice while the chain still pointed at the old version. Defer the private-store write (and its supersede) to the terminal branches (post-chain-confirmation + intentional-local finalize), never on the chain-failure path — mirroring how the public data insert already defers. 🔴 #11 legacy migration — re-add the legacy-plaintext → encrypted-keystore migration on load (after successful validation) so UPGRADED nodes don't keep operational keys in plaintext indefinitely. Gated by a test-only opt-out `DKG_WALLETS_NO_MIGRATE=1` (set by the devnet staking harness, which reads the raw privateKey field directly and cannot decrypt). Production daemons migrate. 🟡 #936 cross-path token-row consistency — extract the per-root `/` dkg:tokenId/dkg:entity rows into a SHARED `buildDeterministicTokenRows` helper called by BOTH the publisher (originator) and the gossip/chain-reconcile path, so a multi-root KC exposes an identical queryable token map regardless of which node materialised it. (Kept out of generateKCMetadata, which metadata.test.ts pins.) 🟡 #1121 test coverage — assert the mapper PRESERVES a real agent-resolved encryption callback (not just that some callback exists) and that a PUBLIC async publish stays plaintext. Verified: publisher 1179, agent (936 repro + finalization/reconcile) green, op-wallets migration + at-rest green, async-lift 4 green. Co-Authored-By: Claude Opus 4.8 --- packages/agent/src/finalization-handler.ts | 35 +++--------- packages/agent/src/op-wallets.ts | 34 +++++++++--- .../op-wallets-at-rest-encryption.test.ts | 39 +++++++++++++- packages/publisher/src/dkg-publisher.ts | 54 ++++++++++++++----- packages/publisher/src/index.ts | 2 +- packages/publisher/src/metadata.ts | 30 +++++++++++ ...ft-canonicalization-and-encryption.test.ts | 43 +++++++++++++++ scripts/devnet.sh | 6 ++- 8 files changed, 192 insertions(+), 51 deletions(-) diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index ed217311d..4155f0150 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -16,7 +16,7 @@ import { GraphManager, type TripleStore, type Quad } from '@origintrail-official import { type ChainAdapter, type EventFilter } from '@origintrail-official/dkg-chain'; import { computeFlatKCRootV10 as computeFlatKCRoot, skolemizeByEntity, - generateConfirmedFullMetadata, getTentativeStatusQuad, + generateConfirmedFullMetadata, buildDeterministicTokenRows, getTentativeStatusQuad, generateSubGraphRegistration, shouldApplyMaterialization, writeMaterializedVersion, withMaterializationLock, type MaterializedVersion, @@ -1170,34 +1170,15 @@ export class FinalizationHandler { let metaQuads = generateConfirmedFullMetadata(kcMeta, kaMetadata, provenance); - // GH #936 — emit an EXPLICIT, deterministic per-root token map - // (`/` dkg:tokenId / dkg:entity) for multi-root KCs so two - // replicas reconciling the same KC expose an IDENTICAL, queryable - // rootEntity→tokenId mapping (the tokenId now derives from the canonical - // root sort above). Emitted here rather than in generateKCMetadata, which - // metadata.test.ts pins to forbid these predicates. graph = the default + // GH #936 — append the SHARED deterministic per-root token rows (no-op for + // single-root). This is the SAME helper the publisher uses on the originator + // path, so a locally-published and a chain-reconciled multi-root KC expose + // an identical, queryable rootEntity→tokenId map. graph = the default // `/_meta` so the ctxGraphId remap below routes them to the per-cgId // `_meta` (and dual-writes a root copy when keepRootCopyOnLabel). - if (kaMetadata.length > 1) { - const defaultMetaGraph = `did:dkg:context-graph:${contextGraphId}/_meta`; - for (const ka of kaMetadata) { - const labelSubject = `${ual}/${ka.tokenId}`; - metaQuads.push( - { - subject: labelSubject, - predicate: `${DKG_NS}tokenId`, - object: `"${ka.tokenId}"^^`, - graph: defaultMetaGraph, - }, - { - subject: labelSubject, - predicate: DKG_ENTITY, - object: ka.rootEntity, - graph: defaultMetaGraph, - }, - ); - } - } + metaQuads.push( + ...buildDeterministicTokenRows(ual, kaMetadata, `did:dkg:context-graph:${contextGraphId}/_meta`), + ); if (ctxGraphId) { const defaultMeta = `did:dkg:context-graph:${contextGraphId}/_meta`; const targetMeta = contextGraphMetaUri(contextGraphId, ctxGraphId); diff --git a/packages/agent/src/op-wallets.ts b/packages/agent/src/op-wallets.ts index cfb1f3368..3150ef43a 100644 --- a/packages/agent/src/op-wallets.ts +++ b/packages/agent/src/op-wallets.ts @@ -26,6 +26,10 @@ const DEFAULT_WALLET_COUNT = 3; // `loadOpWallets` still returns decrypted keys in memory for the chain config. const WALLET_SECRET_FILE = 'wallets.key'; const PASSPHRASE_ENV = 'DKG_WALLETS_PASSPHRASE'; +// Test-only escape hatch: skip the legacy-plaintext→encrypted migration on load. +// Set by harnesses/tooling that read the raw `privateKey` field out of +// wallets.json directly and cannot decrypt (e.g. the devnet staking script). +const NO_MIGRATE_ENV = 'DKG_WALLETS_NO_MIGRATE'; interface EncryptedKeystore { v: number; @@ -84,18 +88,15 @@ export async function loadOpWallets( ); } + let sawLegacyPlaintext = false; const resolve = (stored: StoredWalletEntry, path: string): WalletEntry => { if (isEncryptedEntry(stored)) { const privateKey = decryptKey(stored.keystore!, secret!); return validateWalletEntry({ address: stored.address, privateKey }, path); } - // Legacy plaintext entry — load as-is. We deliberately do NOT rewrite - // the file: a hand-provisioned / tooling-written plaintext wallets.json - // (devnet provisioning, the live-daemon test fixture, operators who - // manage the file directly) must stay byte-compatible with external - // readers. New nodes generate encrypted wallets (see saveOpWallets), so - // a fresh install is encrypted at rest; existing plaintext files are - // honoured unchanged. + // Legacy plaintext entry — accepted (back-compat), and flagged for + // opportunistic migration to an encrypted keystore below. + sawLegacyPlaintext = true; return validateWalletEntry({ address: stored.address, privateKey: stored.privateKey as string }, path); }; @@ -111,7 +112,24 @@ export async function loadOpWallets( } } - return { adminWallet, wallets }; + const config = { adminWallet, wallets }; + // GH #11 migration — an upgraded node that still has a LEGACY plaintext + // wallets.json (the deployed wallets most likely to hold real funds) gets + // its keys transparently re-saved as encrypted keystores after a + // successful load (same keys, same addresses — no rotation, no lockout). + // This closes the plaintext-at-rest exposure for existing operators, not + // just fresh installs. Opt OUT via `DKG_WALLETS_NO_MIGRATE=1` for test + // harnesses / provisioning tooling that reads the raw `privateKey` field + // directly (e.g. the devnet staking script) and cannot decrypt. The + // re-save is best-effort: a write failure must not block loading. + if (sawLegacyPlaintext && process.env[NO_MIGRATE_ENV] !== '1') { + try { + await saveOpWallets(dataDir, config); + } catch { + /* keep serving the loaded keys even if the migration re-save fails */ + } + } + return config; } } catch (err: any) { if (err.code !== 'ENOENT') throw err; diff --git a/packages/agent/test/op-wallets-at-rest-encryption.test.ts b/packages/agent/test/op-wallets-at-rest-encryption.test.ts index d23b21a36..87331e91e 100644 --- a/packages/agent/test/op-wallets-at-rest-encryption.test.ts +++ b/packages/agent/test/op-wallets-at-rest-encryption.test.ts @@ -17,9 +17,10 @@ * metadata still turns it green (Codex review on PR #1129). Hermetic — tmpdir. */ import { describe, expect, it, afterEach } from 'vitest'; -import { mkdtemp, readFile, rm, readdir } from 'node:fs/promises'; +import { mkdtemp, readFile, rm, readdir, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; import { tmpdir } from 'node:os'; +import { ethers } from 'ethers'; import { loadOpWallets } from '../src/op-wallets.js'; /** Every regular file under `dir`, recursively. */ @@ -77,4 +78,40 @@ describe('GH #11 — operational wallet private keys at rest', () => { expect(reloaded.wallets.map((w) => w.privateKey)).toEqual(config.wallets.map((w) => w.privateKey)); expect(reloaded.wallets.map((w) => w.address)).toEqual(config.wallets.map((w) => w.address)); }); + + it('migrates an existing LEGACY plaintext wallets.json to an encrypted keystore on load', async () => { + const dir = await mkdtemp(join(tmpdir(), 'gh11-migrate-')); + dirs.push(dir); + + // Simulate an upgraded node carrying a pre-encryption plaintext wallets.json + // (the deployed wallets most likely to hold real funds). + const admin = ethers.Wallet.createRandom(); + const op = ethers.Wallet.createRandom(); + await writeFile( + join(dir, 'wallets.json'), + JSON.stringify({ + adminWallet: { address: admin.address, privateKey: admin.privateKey }, + wallets: [{ address: op.address, privateKey: op.privateKey }], + }), + ); + + // Load: accepts the legacy keys AND transparently re-encrypts the file. + const loaded = await loadOpWallets(dir); + expect(loaded.adminWallet?.privateKey).toBe(admin.privateKey); + expect(loaded.wallets[0].privateKey).toBe(op.privateKey); + + // The on-disk file no longer contains either raw private key. + const files = await walkFiles(dir); + const combined = (await Promise.all(files.map((f) => readFile(f, 'utf-8').catch(() => '')))).join('\n'); + for (const hex of [admin.privateKey, op.privateKey]) { + expect(combined, 'legacy plaintext key still on disk after migration').not.toContain(hex); + expect(combined).not.toContain(hex.replace(/^0x/, '')); + } + + // Reload (simulated restart): decrypts back to the SAME keys — no rotation, + // no lockout. + const reloaded = await loadOpWallets(dir); + expect(reloaded.adminWallet?.privateKey).toBe(admin.privateKey); + expect(reloaded.wallets[0].privateKey).toBe(op.privateKey); + }); }); diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index 67142fe45..bd179d4cd 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -19,6 +19,7 @@ import { import { validatePublishRequest } from './validation.js'; import { generateConfirmedFullMetadata, + buildDeterministicTokenRows, generateOwnershipQuads, generateAssertionCreatedMetadata, generateAssertionPromotedMetadata, @@ -1989,20 +1990,31 @@ export class DKGPublisher implements Publisher { // to ACK / chain digests — moving it past the chain-success branch // would risk a race where the publisher returns 'confirmed' before // its own private store has the data. - for (const entry of canonical.manifestEntries) { - const entityPrivateQuads = privateQuads.filter( - (q) => q.subject === entry.rootEntity || q.subject.startsWith(entry.rootEntity + '/.well-known/genid/'), - ); - if (entityPrivateQuads.length > 0) { - // GH #1078 — tag the stored slice with the commitment this root just - // committed (its privateMerkleRoot) so a re-publish for the same root - // supersedes the stale slice instead of commingling it. - const commitmentId = entry.privateMerkleRoot - ? Buffer.from(entry.privateMerkleRoot).toString('hex') - : undefined; - await this.privateStore.storePrivateTriples(contextGraphId, entry.rootEntity, entityPrivateQuads, options.subGraphName, commitmentId); + // GH #1078 — persist the finalized private slices. DEFERRED to the terminal + // branches (post-chain-confirmation, or the intentional-local finalize) and + // NEVER run on the chain-failure path. Because `storePrivateTriples(…, + // commitmentId)` now SUPERSEDES a root's prior private slice when the + // commitment differs, running it pre-chain would let a failed/rejected + // re-publish delete the private data of the still-current KA while the chain + // still points at the old version. Gating it on confirmation keeps the + // private store consistent with the committed `privateMerkleRoot`, and + // invoking it BEFORE the publish returns 'confirmed' preserves the + // no-"confirmed-before-data" guarantee the pre-chain insert used to give. + const persistFinalizedPrivateSlices = async (): Promise => { + for (const entry of canonical.manifestEntries) { + const entityPrivateQuads = privateQuads.filter( + (q) => q.subject === entry.rootEntity || q.subject.startsWith(entry.rootEntity + '/.well-known/genid/'), + ); + if (entityPrivateQuads.length > 0) { + // Tag the stored slice with the commitment this root committed (its + // privateMerkleRoot) so a later re-publish supersedes the stale slice. + const commitmentId = entry.privateMerkleRoot + ? Buffer.from(entry.privateMerkleRoot).toString('hex') + : undefined; + await this.privateStore.storePrivateTriples(contextGraphId, entry.rootEntity, entityPrivateQuads, options.subGraphName, commitmentId); + } } - } + }; onPhase?.('store', 'end'); @@ -2571,6 +2583,9 @@ export class DKGPublisher implements Publisher { } this.log.info(ctx, `Storing ${normalizedQuads.length} triples in local store (${reasonLog})`); await this.store.insert(normalizedQuads); + // GH #1078 — persist private slices on this intentional-local terminal + // branch too (a chainless / ownerOnly publish still finalizes here). + await persistFinalizedPrivateSlices(); await this.store.insert(tentativeMeta); // B3: only now that the local publish has persisted do we refresh the // public catalog entry (CLEAR/REPLACE — see persistCatalogEntry). @@ -2958,6 +2973,15 @@ export class DKGPublisher implements Publisher { chainId: this.chain.chainId, }, ); + // GH #936 — append the SHARED deterministic per-root token rows (the + // same helper the gossip / chain-reconcile path uses) so the originator + // exposes the IDENTICAL `/` → root map as replicas. kaMetadata + // is already in canonical (sorted) tokenId order. graph = default + // `/_meta` so the remap below routes them to the per-cgId `_meta`. + confirmedQuads = [ + ...confirmedQuads, + ...buildDeterministicTokenRows(ual, kaMetadata, `did:dkg:context-graph:${contextGraphId}/_meta`), + ]; if (options.targetMetaGraphUri) { const defaultMeta = `did:dkg:context-graph:${contextGraphId}/_meta`; confirmedQuads = confirmedQuads.map((q) => @@ -2983,6 +3007,10 @@ export class DKGPublisher implements Publisher { this.log.info(ctx, `Storing ${vmQuads.length} triples in ${vmGraph} (post-confirmation)`); await this.store.insert(vmQuads); await this.store.insert(confirmedQuads); + // GH #1078 — supersede/persist private slices only now that the chain + // has confirmed (before returning 'confirmed', so no read sees the KA + // confirmed without its private data). + await persistFinalizedPrivateSlices(); await stampTrustLevel( this.store, vmGraph, diff --git a/packages/publisher/src/index.ts b/packages/publisher/src/index.ts index 11e17f684..a6415cbb7 100644 --- a/packages/publisher/src/index.ts +++ b/packages/publisher/src/index.ts @@ -25,7 +25,7 @@ export { computeKCRootV10, } from './merkle.js'; export { validatePublishRequest, type ValidationResult, type ValidationOptions } from './validation.js'; -export { generateKCMetadata, generateTentativeMetadata, generateConfirmedFullMetadata, getTentativeStatusQuad, getConfirmedStatusQuad, generateOwnershipQuads, generateShareMetadata, generateWorkspaceMetadata, generateSubGraphRegistration, subGraphDeregistrationSparql, subGraphDiscoverySparql, subGraphWritersSparql, toHex, resolveUalByBatchId, updateMetaMerkleRoot, promoteUpdatedKaToPerCgId, restateKaPartition, restateLabelGraphForUpdate, readMaterializedVersion, shouldApplyMaterialization, writeMaterializedVersion, withMaterializationLock, compareMaterializedVersion, type MaterializedVersion, generateAssertionCreatedMetadata, generateAssertionPromotedMetadata, generateAssertionUpdatedMetadata, generateAssertionDiscardedMetadata, assertionStateQuad, assertionLayerQuad, deriveStatus, assertionLayerPointerQuad, stampLayerPointerSparql, type LifecycleMetadataOptions, WM_CURRENT_ASSERTION_PRED, SWM_CURRENT_ASSERTION_PRED, VM_CURRENT_ASSERTION_PRED, KA_ID_PRED, RESERVED_UAL_PRED, PROV_WAS_REVISION_OF, type KaStatus, type StatusPointers, type KCMetadata, type KAMetadata, type OnChainProvenance, type ShareMetadata, type WorkspaceMetadata, type SubGraphRegistration, type AssertionCreatedMeta, type AssertionPromotedMeta, type AssertionUpdatedMeta, type AssertionDiscardedMeta } from './metadata.js'; +export { generateKCMetadata, generateTentativeMetadata, generateConfirmedFullMetadata, buildDeterministicTokenRows, getTentativeStatusQuad, getConfirmedStatusQuad, generateOwnershipQuads, generateShareMetadata, generateWorkspaceMetadata, generateSubGraphRegistration, subGraphDeregistrationSparql, subGraphDiscoverySparql, subGraphWritersSparql, toHex, resolveUalByBatchId, updateMetaMerkleRoot, promoteUpdatedKaToPerCgId, restateKaPartition, restateLabelGraphForUpdate, readMaterializedVersion, shouldApplyMaterialization, writeMaterializedVersion, withMaterializationLock, compareMaterializedVersion, type MaterializedVersion, generateAssertionCreatedMetadata, generateAssertionPromotedMetadata, generateAssertionUpdatedMetadata, generateAssertionDiscardedMetadata, assertionStateQuad, assertionLayerQuad, deriveStatus, assertionLayerPointerQuad, stampLayerPointerSparql, type LifecycleMetadataOptions, WM_CURRENT_ASSERTION_PRED, SWM_CURRENT_ASSERTION_PRED, VM_CURRENT_ASSERTION_PRED, KA_ID_PRED, RESERVED_UAL_PRED, PROV_WAS_REVISION_OF, type KaStatus, type StatusPointers, type KCMetadata, type KAMetadata, type OnChainProvenance, type ShareMetadata, type WorkspaceMetadata, type SubGraphRegistration, type AssertionCreatedMeta, type AssertionPromotedMeta, type AssertionUpdatedMeta, type AssertionDiscardedMeta } from './metadata.js'; export { DKGPublisher, StaleWriteError, diff --git a/packages/publisher/src/metadata.ts b/packages/publisher/src/metadata.ts index c54ee42fe..5e1d609de 100644 --- a/packages/publisher/src/metadata.ts +++ b/packages/publisher/src/metadata.ts @@ -298,6 +298,36 @@ export function generateConfirmedFullMetadata( ]; } +/** + * GH #936 — explicit, deterministic per-root token map + * (`/` dkg:tokenId / dkg:entity) for MULTI-root KCs. Emitted via a + * SHARED helper so the publisher (originator) and the gossip / chain-reconcile + * replica paths expose an IDENTICAL, queryable rootEntity→tokenId mapping — the + * same multi-root KC must not surface different token rows depending on which + * node materialised it. Kept OUT of generateKCMetadata (metadata.test.ts pins + * that output to the collapsed `dkg:rootEntity` shape and forbids these + * predicates); both call sites append these rows alongside the confirmed + * metadata. `kaEntries` MUST already be in the canonical + * (lexicographically-sorted-by-rootEntity) tokenId order — both call sites sort + * before minting. + */ +export function buildDeterministicTokenRows( + ual: string, + kaEntries: ReadonlyArray<{ tokenId: bigint; rootEntity: string }>, + metaGraph: string, +): Quad[] { + if (kaEntries.length <= 1) return []; + const rows: Quad[] = []; + for (const ka of kaEntries) { + const subject = `${ual}/${ka.tokenId}`; + rows.push( + mq(subject, `${DKG}tokenId`, intLit(ka.tokenId), metaGraph), + mq(subject, DKG_ENTITY, ka.rootEntity, metaGraph), + ); + } + return rows; +} + function mq(s: string, p: string, o: string, g: string): Quad { return { subject: s, predicate: p, object: o, graph: g }; } diff --git a/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts b/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts index 1b066e541..745bcb117 100644 --- a/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts +++ b/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts @@ -70,4 +70,47 @@ describe('GH #1121 — async lift carries an inline-encryption path for private // inline-encryption path threaded through, like the sync publish() does. expect(opts.encryptInlinePayload).toBeDefined(); }); + + it('PRESERVES the agent-resolved real encryption callbacks (does not shadow them with the fail-closed default)', () => { + // The fail-closed default exists only so a non-public CG can never ship + // plaintext when NO factory is wired. When the async worker DOES thread the + // agent-resolved chainKey-bound AEAD closures through `resolved`, the mapper + // must pass THOSE through verbatim — otherwise a private publish would invoke + // the throwing default and fail instead of encrypting. (Guards the + // publishEncryptionFactory wiring / precedence from silently regressing.) + const realInline = async (b: Uint8Array): Promise => new Uint8Array([...b, 0xff]); + const realChunked = async (): Promise => { /* member fan-out */ }; + const opts = mapLiftRequestToPublishOptions({ + request: { ...baseRequest, accessPolicy: 'ownerOnly' }, + validation: { authorityProofRef: 'devnet-proof', transitionType: 'CREATE' }, + resolved: { + quads: [ + { subject: CALLER_ROOT, predicate: 'https://schema.org/name', object: '"Tenant A"', graph: 'g' }, + ], + accessPolicy: 'ownerOnly', + publisherPeerId: '12D3KooWGH1121PrivatePublisherPeerIdForTest', + encryptInlinePayload: realInline, + encryptInlineChunked: realChunked, + }, + }); + expect(opts.encryptInlinePayload).toBe(realInline); + expect(opts.encryptInlineChunked).toBe(realChunked); + }); + + it('a PUBLIC async publish stays plaintext (no encryption callback forced)', () => { + const opts = mapLiftRequestToPublishOptions({ + request: { ...baseRequest, accessPolicy: 'public' }, + validation: { authorityProofRef: 'devnet-proof', transitionType: 'CREATE' }, + resolved: { + quads: [ + { subject: CALLER_ROOT, predicate: 'https://schema.org/name', object: '"Tenant A"', graph: 'g' }, + ], + accessPolicy: 'public', + }, + }); + expect(opts.accessPolicy).toBe('public'); + // A public CG must NOT get a forced encryption hook — that would push the + // publisher onto the encrypted-inline path and break public ACK verification. + expect(opts.encryptInlinePayload).toBeUndefined(); + }); }); diff --git a/scripts/devnet.sh b/scripts/devnet.sh index d4a457f7a..b873f1ba7 100755 --- a/scripts/devnet.sh +++ b/scripts/devnet.sh @@ -615,7 +615,11 @@ start_node() { rm -f "$node_dir/daemon.pid" log "Starting node $node_num..." - DKG_HOME="$node_dir" DKG_NO_BLUE_GREEN=1 \ + # DKG_WALLETS_NO_MIGRATE=1: this harness writes a plaintext wallets.json with + # RANDOM operational keys and the staking step (cmd_start) re-reads + # wallets[0].privateKey directly, so the daemon must NOT migrate it to an + # encrypted keystore (GH #11). Production daemons omit this and auto-migrate. + DKG_HOME="$node_dir" DKG_NO_BLUE_GREEN=1 DKG_WALLETS_NO_MIGRATE=1 \ node "$REPO_ROOT/packages/cli/dist/cli.js" start --foreground \ > "$node_dir/daemon.log" 2>&1 & local node_pid=$! From e560438ba81faab19b6ba019a01626e7d8b652ca Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 19:02:17 +0200 Subject: [PATCH 10/14] fix(publisher): only skip the FAIL-CLOSED encryption default on chainless publishes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous `useEncryptedInline = canAttemptOnChainPublish && …` gate (added to unbreak chainless ownerOnly publishes from the #1121 fail-closed default) was too broad: it also stopped a REAL injected encryption callback from running on a chainless publish, breaking publisher-runner-lu11 (which wires a runtime chunked callback over a NoChainAdapter and asserts it is invoked). Mark the mapper's fail-closed default and skip ONLY that default on chainless publishes; always honour a real agent-resolved callback, and still fire the default (fail-closed) for an actual on-chain publish with no real encryption. Fixes the kafka-plugin E2E AND publisher-runner-lu11. Verified: publisher 1181, full agent 1622, full cli 2029 — all green. Co-Authored-By: Claude Opus 4.8 --- .../src/async-lift-publish-options.ts | 30 ++++++++++++++----- packages/publisher/src/dkg-publisher.ts | 25 ++++++++-------- 2 files changed, 36 insertions(+), 19 deletions(-) diff --git a/packages/publisher/src/async-lift-publish-options.ts b/packages/publisher/src/async-lift-publish-options.ts index 371544e15..85d06dcab 100644 --- a/packages/publisher/src/async-lift-publish-options.ts +++ b/packages/publisher/src/async-lift-publish-options.ts @@ -57,13 +57,29 @@ export interface LiftResolvedPublishSlice { * with NO inline-encryption callback, this default makes the publish FAIL LOUD * instead of leaking plaintext. */ -const failClosedInlineEncrypt: NonNullable = () => { - throw new Error( - 'Async-lift private/curated CG publish requires an encryptInlinePayload factory threaded ' + - 'through LiftResolvedPublishSlice (a non-public context graph must not ship plaintext to cores). ' + - 'Wire publishEncryptionFactory in the async publisher runtime.', - ); -}; +const FAIL_CLOSED_MARKER = '__dkgFailClosedInlineEncrypt'; +const failClosedInlineEncrypt: NonNullable = Object.assign( + (() => { + throw new Error( + 'Async-lift private/curated CG publish requires an encryptInlinePayload factory threaded ' + + 'through LiftResolvedPublishSlice (a non-public context graph must not ship plaintext to cores). ' + + 'Wire publishEncryptionFactory in the async publisher runtime.', + ); + }) as NonNullable, + { [FAIL_CLOSED_MARKER]: true as const }, +); + +/** + * True iff `cb` is the mapper's fail-closed default (not a real agent-resolved + * encryption callback). The publisher uses this to skip the encrypted-inline + * path on a chainless/local publish that ships nothing to cores — there is no + * plaintext leak to guard, so the default must not throw — while still honouring + * any REAL callback the caller wired, and still firing the default (fail-closed) + * for an actual on-chain publish with no real encryption resolved. + */ +export function isFailClosedInlineEncrypt(cb: unknown): boolean { + return typeof cb === 'function' && (cb as unknown as Record)[FAIL_CLOSED_MARKER] === true; +} export interface LiftPublishMappingInput { readonly request: LiftRequest; diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index bd179d4cd..eded1615a 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -17,6 +17,7 @@ import { computeFlatKCMerkleLeafCountV10, } from './merkle.js'; import { validatePublishRequest } from './validation.js'; +import { isFailClosedInlineEncrypt } from './async-lift-publish-options.js'; import { generateConfirmedFullMetadata, buildDeterministicTokenRows, @@ -2131,19 +2132,19 @@ export class DKGPublisher implements Publisher { // with NO_DATA_IN_SWM — the exact bug §1.1 surfaces. Public CGs keep // the existing behaviour: `fromSharedMemory` → cores look up SWM // locally; otherwise plaintext inline. - // GH #1121 — take the encrypted-inline path ONLY when this publish actually - // goes on-chain (collects core StorageACKs / distributes to CG members). A - // chainless / local-only publish (e.g. an ownerOnly KA on an unregistered - // CG, or a chain-not-ready node) ships nothing to other nodes, so there is - // no plaintext-to-cores leak to guard against — and forcing the encryption - // hook there would break a legitimate local publish (the curated chain-key - // the real hook needs cannot be resolved off-chain, so the async-lift - // mapper's fail-closed default would throw). When the publish IS on-chain - // the requirement stands: a defined hook (the agent-resolved real callback, - // or the fail-closed default) is used, so private data is never shipped to - // cores in the clear. + // GH #1121 — take the encrypted-inline path whenever a REAL encryption + // callback is wired. The ONE exception: skip the async-lift mapper's + // fail-closed DEFAULT on a chainless / local-only publish (ownerOnly KA on + // an unregistered CG, chain-not-ready node, …). Such a publish ships nothing + // to other nodes, so there is no plaintext-to-cores leak to guard — and the + // default (which exists only to prevent that leak) cannot resolve a real + // chain-key off-chain, so invoking it would needlessly fail a legitimate + // local publish. For an actual on-chain publish the default still fires + // (fail-closed): a private payload is never shipped to cores in the clear. + const inlineEncryptCb = options.encryptInlinePayload; const useEncryptedInline = - canAttemptOnChainPublish && typeof options.encryptInlinePayload === 'function'; + typeof inlineEncryptCb === 'function' + && (canAttemptOnChainPublish || !isFailClosedInlineEncrypt(inlineEncryptCb)); // OT-RFC-49 / WS-D — a curated publish is now identified by a non-zero // catalog commitment. The on-chain commitment + the core ACK verify the // PUBLIC `_catalog`; the PRIVATE data stays encrypted for MEMBERS only. From 8a062d07350a907af8d80ff46bea768e8817eeba Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 19:18:01 +0200 Subject: [PATCH 11/14] fix: resolve otReviewAgent round-3 findings on PR #1228 (1 bug + dedup) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 🔴 #1078 cross-root delete — the supersede deleted by raw `deleteBySubjectPrefix( rootEntity)`, but RDF root IRIs are not prefix-delimited, so superseding `urn:device:1` would ALSO wipe `urn:device:10`'s private slice (STRSTARTS). Add `deleteRootPrivateSlice`: delete the EXACT root subject + its skolem children (`/.well-known/genid/…`), the same shape getPrivateTriples reads. Used by both the supersede path and deletePrivateTriples. + regression test (sibling root survives a prefix-sharing supersede). 🟡 #1098 dedup — extract `selfPrimeSubscriptionOnChainId` so the periodic sweep and the live KACG nudge share ONE bind/persist/cursor-reset path (optional target-id match for the nudge), instead of two hand-rolled flows. 🟡/🔵 #936 dedup — extract the shared `compareRootIris` canonical comparator so the publisher and finalization handler assign tokenIds over the SAME ordering by API, not by duplicated inline comparators + comments. Verified: storage 218 (+collision test), agent 1622, publisher 1181 — all green. Co-Authored-By: Claude Opus 4.8 --- packages/agent/src/dkg-agent-lifecycle.ts | 14 ++-- packages/agent/src/dkg-agent-swm-host.ts | 70 ++++++++++++------- packages/agent/src/finalization-handler.ts | 4 +- packages/publisher/src/dkg-publisher.ts | 3 +- packages/publisher/src/index.ts | 2 +- packages/publisher/src/metadata.ts | 11 +++ packages/storage/src/private-store.ts | 19 ++++- .../issue-1078-private-layer-scope.test.ts | 27 +++++++ 8 files changed, 108 insertions(+), 42 deletions(-) diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index 6f4d00ae4..b8f8bda01 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -1518,19 +1518,15 @@ export class LifecycleSyncMethods extends DKGAgentBase { // hook). Find the subscribed-but-unbound CG whose locally-resolved // on-chain id matches THIS event and bind + reconcile only it — // targeted, not a global sweep, so an unrelated KA registration - // touches nothing. The periodic self-priming sweep remains the - // safety net for a CG whose OnChainId quad hasn't arrived yet. + // touches nothing. Uses the SAME self-prime helper as the + // periodic sweep (single bind/persist/cursor-reset path); the + // sweep remains the safety net for a CG whose quad hasn't arrived. let targetOnChain: bigint | null = null; try { targetOnChain = BigInt(onChainId); } catch { targetOnChain = null; } if (targetOnChain !== null) { for (const [lcg, sub] of this.subscribedContextGraphs) { - if (!sub.subscribed || sub.onChainId) continue; - const resolved = await this.getContextGraphOnChainId(lcg).catch(() => null); - let resolvedNum: bigint | null = null; - try { resolvedNum = resolved ? BigInt(resolved) : null; } catch { resolvedNum = null; } - if (resolvedNum !== null && resolvedNum === targetOnChain) { - this.bindSubscriptionOnChainId(lcg, sub, resolved!); - this.persistContextGraphSubscription(lcg); + const bound = await this.selfPrimeSubscriptionOnChainId(lcg, sub, targetOnChain); + if (bound) { this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> bound + reconcile pre-subscribed "${lcg}"`); if (this.reconcileCoalescer) void this.reconcileCoalescer.trigger(lcg); break; diff --git a/packages/agent/src/dkg-agent-swm-host.ts b/packages/agent/src/dkg-agent-swm-host.ts index f1be0eedc..5fcbdfb4a 100644 --- a/packages/agent/src/dkg-agent-swm-host.ts +++ b/packages/agent/src/dkg-agent-swm-host.ts @@ -2348,34 +2348,11 @@ export class SwmHostModeMethods extends DKGAgentBase { async runVmReconcileSweep(this: DKGAgent): Promise { if (!this.vmReconcileEnabled() || !this.reconcileCoalescer) return; for (const [localCgId, sub] of this.subscribedContextGraphs) { - // GH #1098 — self-prime the on-chain id for a pre-subscribed PUBLIC member - // CG. A peer that subscribed BEFORE the CG's first publish has - // `sub.onChainId` unset: the chain `ContextGraphCreated` handler only binds - // it for CURATED CGs, and the ACK-signer path (`recordCoreHostedPublicCg`) - // only fires for cores in that publish's storage-ACK set. Left unbound, the - // sweep below skips the CG and the peer is stranded on the unreliable - // one-shot finalization gossip (the ~1/3 materialization + kc-not-synced - // spam in #1098). As soon as the CG's OnChainId quad is local (publisher - // ontology-topic broadcast or durable _meta sync), resolve + bind it so the - // reliable chain-driven reconcile/backfill runs for this peer — the same - // path the late-subscriber (#886) and ACK-signer cores already converge on. - // Best-effort + gated on `!sub.onChainId` so it binds at most once (a later - // id CHANGE would reset the cursor; binding from undefined never does). + // GH #1098 — self-prime onChainId for a pre-subscribed PUBLIC member CG + // (subscribed BEFORE its first publish, so unbound) before the skip-gate + // below would pass it over. Shared with the live KACG nudge. if (sub.subscribed && !sub.onChainId) { - try { - // Bind ANY non-null resolved on-chain id. getContextGraphOnChainId - // returns the persisted OnChainId quad (or null) — it never falls back - // to localCgId, so a `resolved === localCgId` match is legitimate for a - // direct CG whose local id IS its numeric on-chain id (e.g. "42"); a - // `!== localCgId` guard would wrongly leave such a sub unbound forever. - const resolved = await this.getContextGraphOnChainId(localCgId); - if (resolved) { - this.bindSubscriptionOnChainId(localCgId, sub, resolved); - this.persistContextGraphSubscription(localCgId); - } - } catch { - /* a store/RPC hiccup on one CG must not abort the whole sweep */ - } + await this.selfPrimeSubscriptionOnChainId(localCgId, sub); } // Member subscriptions AND Phase D core-hosted public CGs get swept. if ((!sub.subscribed && !sub.coreHosted) || !sub.onChainId) continue; @@ -2383,6 +2360,45 @@ export class SwmHostModeMethods extends DKGAgentBase { } } + /** + * GH #1098 — bind `sub.onChainId` for a subscribed-but-unbound CG from the + * locally-resolvable OnChainId quad (publisher ontology broadcast / durable + * _meta sync), then persist. The chain `ContextGraphCreated` handler only + * binds CURATED CGs and the ACK-signer hook only fires for cores in a + * publish's storage-ACK set, so a pre-subscribed PUBLIC member would otherwise + * stay unbound — stranded on the unreliable one-shot finalization gossip. + * SHARED by the periodic sweep and the live KACG nudge so the bind / persist / + * cursor-reset semantics (in {@link bindSubscriptionOnChainId}) live in ONE + * place. `targetOnChainId`: when set (the nudge), bind only if the resolved id + * matches THIS event; when omitted (the sweep), bind any non-null id — + * `getContextGraphOnChainId` never falls back to `localCgId`, so a + * `resolved === localCgId` match is legitimate for a direct CG. Best-effort: + * a store/RPC hiccup yields null instead of throwing. Returns the bound id. + */ + async selfPrimeSubscriptionOnChainId( + this: DKGAgent, + localCgId: string, + sub: ContextGraphSub, + targetOnChainId?: bigint, + ): Promise { + if (!sub.subscribed || sub.onChainId) return null; + let resolved: string | null = null; + try { + resolved = await this.getContextGraphOnChainId(localCgId); + } catch { + return null; + } + if (!resolved) return null; + if (targetOnChainId !== undefined) { + let resolvedNum: bigint | null = null; + try { resolvedNum = BigInt(resolved); } catch { return null; } + if (resolvedNum !== targetOnChainId) return null; + } + this.bindSubscriptionOnChainId(localCgId, sub, resolved); + this.persistContextGraphSubscription(localCgId); + return resolved; + } + /** * One reconcile pass for a single CG: build the injected deps and hand off to * the pure {@link reconcileContextGraph} orchestrator (which owns the cursor diff --git a/packages/agent/src/finalization-handler.ts b/packages/agent/src/finalization-handler.ts index 4155f0150..2c6998795 100644 --- a/packages/agent/src/finalization-handler.ts +++ b/packages/agent/src/finalization-handler.ts @@ -16,7 +16,7 @@ import { GraphManager, type TripleStore, type Quad } from '@origintrail-official import { type ChainAdapter, type EventFilter } from '@origintrail-official/dkg-chain'; import { computeFlatKCRootV10 as computeFlatKCRoot, skolemizeByEntity, - generateConfirmedFullMetadata, buildDeterministicTokenRows, getTentativeStatusQuad, + generateConfirmedFullMetadata, buildDeterministicTokenRows, compareRootIris, getTentativeStatusQuad, generateSubGraphRegistration, shouldApplyMaterialization, writeMaterializedVersion, withMaterializationLock, type MaterializedVersion, @@ -1074,7 +1074,7 @@ export class FinalizationHandler { // dependency — see dkg-publisher.ts), so a content-derived sort makes the // map a pure function of the root SET: identical on every replica and on // both the gossip and chain-reconcile promotion paths. - const orderedRoots = [...rootEntities].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); + const orderedRoots = [...rootEntities].sort(compareRootIris); for (let tokenIdx = 0; tokenIdx < orderedRoots.length; tokenIdx++) { const rootEntity = orderedRoots[tokenIdx]; diff --git a/packages/publisher/src/dkg-publisher.ts b/packages/publisher/src/dkg-publisher.ts index eded1615a..f73756945 100644 --- a/packages/publisher/src/dkg-publisher.ts +++ b/packages/publisher/src/dkg-publisher.ts @@ -21,6 +21,7 @@ import { isFailClosedInlineEncrypt } from './async-lift-publish-options.js'; import { generateConfirmedFullMetadata, buildDeterministicTokenRows, + compareRootIris, generateOwnershipQuads, generateAssertionCreatedMetadata, generateAssertionPromotedMetadata, @@ -1909,7 +1910,7 @@ export class DKGPublisher implements Publisher { // queries. These tokenIds are non-on-chain compatibility labels (the // on-chain KA count is 1), so a content-derived sort is safe. const orderedEntries = [...canonical.manifestEntries].sort((a, b) => - a.rootEntity < b.rootEntity ? -1 : a.rootEntity > b.rootEntity ? 1 : 0, + compareRootIris(a.rootEntity, b.rootEntity), ); let compatibilityTokenId = 1n; for (const entry of orderedEntries) { diff --git a/packages/publisher/src/index.ts b/packages/publisher/src/index.ts index a6415cbb7..7b6cf3226 100644 --- a/packages/publisher/src/index.ts +++ b/packages/publisher/src/index.ts @@ -25,7 +25,7 @@ export { computeKCRootV10, } from './merkle.js'; export { validatePublishRequest, type ValidationResult, type ValidationOptions } from './validation.js'; -export { generateKCMetadata, generateTentativeMetadata, generateConfirmedFullMetadata, buildDeterministicTokenRows, getTentativeStatusQuad, getConfirmedStatusQuad, generateOwnershipQuads, generateShareMetadata, generateWorkspaceMetadata, generateSubGraphRegistration, subGraphDeregistrationSparql, subGraphDiscoverySparql, subGraphWritersSparql, toHex, resolveUalByBatchId, updateMetaMerkleRoot, promoteUpdatedKaToPerCgId, restateKaPartition, restateLabelGraphForUpdate, readMaterializedVersion, shouldApplyMaterialization, writeMaterializedVersion, withMaterializationLock, compareMaterializedVersion, type MaterializedVersion, generateAssertionCreatedMetadata, generateAssertionPromotedMetadata, generateAssertionUpdatedMetadata, generateAssertionDiscardedMetadata, assertionStateQuad, assertionLayerQuad, deriveStatus, assertionLayerPointerQuad, stampLayerPointerSparql, type LifecycleMetadataOptions, WM_CURRENT_ASSERTION_PRED, SWM_CURRENT_ASSERTION_PRED, VM_CURRENT_ASSERTION_PRED, KA_ID_PRED, RESERVED_UAL_PRED, PROV_WAS_REVISION_OF, type KaStatus, type StatusPointers, type KCMetadata, type KAMetadata, type OnChainProvenance, type ShareMetadata, type WorkspaceMetadata, type SubGraphRegistration, type AssertionCreatedMeta, type AssertionPromotedMeta, type AssertionUpdatedMeta, type AssertionDiscardedMeta } from './metadata.js'; +export { generateKCMetadata, generateTentativeMetadata, generateConfirmedFullMetadata, buildDeterministicTokenRows, compareRootIris, getTentativeStatusQuad, getConfirmedStatusQuad, generateOwnershipQuads, generateShareMetadata, generateWorkspaceMetadata, generateSubGraphRegistration, subGraphDeregistrationSparql, subGraphDiscoverySparql, subGraphWritersSparql, toHex, resolveUalByBatchId, updateMetaMerkleRoot, promoteUpdatedKaToPerCgId, restateKaPartition, restateLabelGraphForUpdate, readMaterializedVersion, shouldApplyMaterialization, writeMaterializedVersion, withMaterializationLock, compareMaterializedVersion, type MaterializedVersion, generateAssertionCreatedMetadata, generateAssertionPromotedMetadata, generateAssertionUpdatedMetadata, generateAssertionDiscardedMetadata, assertionStateQuad, assertionLayerQuad, deriveStatus, assertionLayerPointerQuad, stampLayerPointerSparql, type LifecycleMetadataOptions, WM_CURRENT_ASSERTION_PRED, SWM_CURRENT_ASSERTION_PRED, VM_CURRENT_ASSERTION_PRED, KA_ID_PRED, RESERVED_UAL_PRED, PROV_WAS_REVISION_OF, type KaStatus, type StatusPointers, type KCMetadata, type KAMetadata, type OnChainProvenance, type ShareMetadata, type WorkspaceMetadata, type SubGraphRegistration, type AssertionCreatedMeta, type AssertionPromotedMeta, type AssertionUpdatedMeta, type AssertionDiscardedMeta } from './metadata.js'; export { DKGPublisher, StaleWriteError, diff --git a/packages/publisher/src/metadata.ts b/packages/publisher/src/metadata.ts index 5e1d609de..fabca7406 100644 --- a/packages/publisher/src/metadata.ts +++ b/packages/publisher/src/metadata.ts @@ -311,6 +311,17 @@ export function generateConfirmedFullMetadata( * (lexicographically-sorted-by-rootEntity) tokenId order — both call sites sort * before minting. */ +/** + * GH #936 — the canonical root ordering used for deterministic compatibility + * tokenId assignment. The publisher (originator), the gossip path, and the + * chain-reconcile path MUST all assign `/` over roots sorted by + * THIS comparator, so every node derives the identical rootEntity→tokenId map. + * Shared here so the invariant is API-enforced rather than comment-enforced. + */ +export function compareRootIris(a: string, b: string): number { + return a < b ? -1 : a > b ? 1 : 0; +} + export function buildDeterministicTokenRows( ual: string, kaEntries: ReadonlyArray<{ tokenId: bigint; rootEntity: string }>, diff --git a/packages/storage/src/private-store.ts b/packages/storage/src/private-store.ts index 37495e3f3..ba38fa7c5 100644 --- a/packages/storage/src/private-store.ts +++ b/packages/storage/src/private-store.ts @@ -213,7 +213,7 @@ export class PrivateContentStore { // write) so the new commitment fully replaces it, then stamp the new // commitment marker. Repeated stores under the SAME commitment fall // through to the append+dedup path below (chunked/retry-safe). - await this.store.deleteBySubjectPrefix(graphUri, rootEntity); + await this.deleteRootPrivateSlice(graphUri, rootEntity); await this.store.deleteByPattern({ graph: graphUri, subject: markerSubject }); await this.store.insert([{ subject: markerSubject, @@ -366,12 +366,27 @@ export class PrivateContentStore { ): Promise { assertSafeIri(rootEntity); const graphUri = this.privateGraph(contextGraphId, subGraphName); - await this.store.deleteBySubjectPrefix(graphUri, rootEntity); + await this.deleteRootPrivateSlice(graphUri, rootEntity); const key = this.privateKey(contextGraphId, subGraphName); const entities = this.privateEntities.get(key); if (entities) entities.delete(rootEntity); } + /** + * Delete exactly ONE root's private slice from `graphUri`: the exact root + * subject plus its skolem children (`/.well-known/genid/…`), the SAME + * shape `getPrivateTriples` reads. We deliberately do NOT use a bare + * `deleteBySubjectPrefix(root)` — RDF root IRIs are not prefix-delimited, so a + * raw prefix would also delete sibling roots that share it (e.g. superseding + * `urn:device:1` would nuke `urn:device:10`'s private triples). The skolem + * prefix IS delimited (`…/.well-known/genid/`), so it is collision-safe. + */ + private async deleteRootPrivateSlice(graphUri: string, rootEntity: string): Promise { + assertSafeIri(rootEntity); + await this.store.deleteByPattern({ graph: graphUri, subject: rootEntity }); + await this.store.deleteBySubjectPrefix(graphUri, `${rootEntity}/.well-known/genid/`); + } + /** * GH #1078 — read the commitment id currently recorded for a root's private * slice (undefined when none has been recorded, i.e. legacy/append-only diff --git a/packages/storage/test/issue-1078-private-layer-scope.test.ts b/packages/storage/test/issue-1078-private-layer-scope.test.ts index ac5b6f37d..f0147428f 100644 --- a/packages/storage/test/issue-1078-private-layer-scope.test.ts +++ b/packages/storage/test/issue-1078-private-layer-scope.test.ts @@ -62,4 +62,31 @@ describe('GH #1078 — private payload storage must be scoped to the committing expect(serials).not.toContain('"OLD-0001"'); }, ); + + it( + 'superseding one root does NOT delete a sibling root whose IRI shares its prefix', + async () => { + const store = new OxigraphStore(); + const gm = new GraphManager(store); + const pcs = new PrivateContentStore(store, gm); + + const R1 = 'urn:gh1078:device:1'; + const R10 = 'urn:gh1078:device:10'; // shares the "urn:gh1078:device:1" prefix + + await pcs.storePrivateTriples(CG, R1, [{ subject: R1, predicate: 'https://schema.org/serialNumber', object: '"R1-OLD"', graph: '' }], undefined, 'c1'); + await pcs.storePrivateTriples(CG, R10, [{ subject: R10, predicate: 'https://schema.org/serialNumber', object: '"R10-KEEP"', graph: '' }], undefined, 'c10'); + + // Re-publish R1 under a NEW commitment → supersedes R1's slice. A bare + // prefix delete would also wipe R10 (STRSTARTS("urn:device:1")); the + // exact-root + skolem-prefix delete must leave R10 intact. + await pcs.storePrivateTriples(CG, R1, [{ subject: R1, predicate: 'https://schema.org/serialNumber', object: '"R1-NEW"', graph: '' }], undefined, 'c1-v2'); + + const r10 = (await pcs.getPrivateTriples(CG, R10)).map((q) => q.object); + expect(r10, 'sibling root R10 must survive R1 supersede').toContain('"R10-KEEP"'); + + const r1 = (await pcs.getPrivateTriples(CG, R1)).map((q) => q.object); + expect(r1).toContain('"R1-NEW"'); + expect(r1).not.toContain('"R1-OLD"'); + }, + ); }); From 15ce3547bf9b228cc09e2055b85556f8c7f2e0a1 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 19:20:18 +0200 Subject: [PATCH 12/14] =?UTF-8?q?test(#936):=20pin=20the=20exact=20canonic?= =?UTF-8?q?al=20root=E2=86=92tokenId=20map=20(guard=20vs=20oxigraph=20fals?= =?UTF-8?q?e-green)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit otReviewAgent: the two-replica equality assertion alone could false-green if oxigraph ever returned the unordered bindings pre-sorted. Additionally assert the exact lexicographic map (aaa→1, mmm→2, zzz→3), so a regression to positional assignment fails even under a stable store iteration order. Co-Authored-By: Claude Opus 4.8 --- .../agent/test/issue-936-tokenid-determinism.test.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/agent/test/issue-936-tokenid-determinism.test.ts b/packages/agent/test/issue-936-tokenid-determinism.test.ts index 2e45b88b5..db885b42d 100644 --- a/packages/agent/test/issue-936-tokenid-determinism.test.ts +++ b/packages/agent/test/issue-936-tokenid-determinism.test.ts @@ -120,5 +120,16 @@ describe('GH #936 — chain-driven reconcile must map each root to a determinist // identical on every replica. Today it is positional over a store-dependent // order, so the two replicas disagree on which root owns `/1`. expect(replicaA).toEqual(replicaB); + + // Pin the EXACT canonical (lexicographic-by-IRI) map, not just that the two + // replicas agree — otherwise, if oxigraph ever returned the unordered + // bindings already sorted, the old positional code would also produce + // identical maps and this test would be false-green. aaa < mmm < zzz. + const expectedCanonicalMap: Record = { + 'urn:gh936:aaa': '1', + 'urn:gh936:mmm': '2', + 'urn:gh936:zzz': '3', + }; + expect(replicaA).toEqual(expectedCanonicalMap); }); }); From 6cf929907d9bb3746c4d1c7bbeb115e622378d27 Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 19:30:23 +0200 Subject: [PATCH 13/14] test(#1098): cover the KACG nudge targeted self-prime (match binds, mismatch doesn't) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit otReviewAgent: the self-prime test only drove the periodic sweep. Add a focused case for the live onKARegisteredToContextGraph nudge's targeting via the shared selfPrimeSubscriptionOnChainId(targetOnChainId) helper — an unrelated subscribed CG (different on-chain id) is NOT bound for the event, the matching CG is. Co-Authored-By: Claude Opus 4.8 --- .../test/vm-reconcile-self-prime.test.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/agent/test/vm-reconcile-self-prime.test.ts b/packages/agent/test/vm-reconcile-self-prime.test.ts index b7c336ae2..ed3ee808e 100644 --- a/packages/agent/test/vm-reconcile-self-prime.test.ts +++ b/packages/agent/test/vm-reconcile-self-prime.test.ts @@ -22,6 +22,11 @@ import { DKGAgent } from '../src/index.js'; interface AgentInternals { runVmReconcileSweep(): Promise; + selfPrimeSubscriptionOnChainId( + localCgId: string, + sub: { subscribed: boolean; coreHosted?: boolean; onChainId?: string }, + targetOnChainId?: bigint, + ): Promise; subscribedContextGraphs: Map; reconcileCoalescer: { trigger: (cg: string) => void } | null; store: TripleStore; @@ -77,4 +82,37 @@ describe('GH #1098 — VM reconcile sweep self-primes onChainId for a pre-subscr expect(internals.subscribedContextGraphs.get(LOCAL)?.onChainId).toBe(ONCHAIN); expect(triggered).toContain(LOCAL); }); + + it('KACG nudge targeting: binds ONLY the unbound CG whose on-chain id matches the event, not an unrelated one', async () => { + // This exercises the SAME `selfPrimeSubscriptionOnChainId` helper the live + // onKARegisteredToContextGraph nudge delegates to, with a `targetOnChainId` + // (the event's CG id). The nudge loops subscribed-unbound CGs and binds the + // one whose resolved id matches the event — so an unrelated CG must NOT bind. + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'SelfPrimeTargeted', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + + const CG_MATCH = 'gh1098-match'; + const CG_OTHER = 'gh1098-other'; + const ON_MATCH = '500'; + const ON_OTHER = '600'; + const ontologyGraph = contextGraphDataGraphUri(SYSTEM_CONTEXT_GRAPHS.ONTOLOGY); + await internals.store.insert([ + { subject: `did:dkg:context-graph:${CG_MATCH}`, predicate: `${DKG_ONTOLOGY.DKG_CONTEXT_GRAPH}OnChainId`, object: `"${ON_MATCH}"`, graph: ontologyGraph }, + { subject: `did:dkg:context-graph:${CG_OTHER}`, predicate: `${DKG_ONTOLOGY.DKG_CONTEXT_GRAPH}OnChainId`, object: `"${ON_OTHER}"`, graph: ontologyGraph }, + ]); + internals.subscribedContextGraphs.set(CG_MATCH, { subscribed: true }); + internals.subscribedContextGraphs.set(CG_OTHER, { subscribed: true }); + + // Event for ON_MATCH: the other CG (resolves to ON_OTHER) must NOT bind. + const other = await internals.selfPrimeSubscriptionOnChainId(CG_OTHER, internals.subscribedContextGraphs.get(CG_OTHER)!, BigInt(ON_MATCH)); + expect(other).toBeNull(); + expect(internals.subscribedContextGraphs.get(CG_OTHER)?.onChainId).toBeUndefined(); + + // The matching CG binds. + const matched = await internals.selfPrimeSubscriptionOnChainId(CG_MATCH, internals.subscribedContextGraphs.get(CG_MATCH)!, BigInt(ON_MATCH)); + expect(matched).toBe(ON_MATCH); + expect(internals.subscribedContextGraphs.get(CG_MATCH)?.onChainId).toBe(ON_MATCH); + }); }); From 1cf195e331da3c04d471977b93d1115dbec9cf5a Mon Sep 17 00:00:00 2001 From: Bojan Date: Thu, 18 Jun 2026 19:55:21 +0200 Subject: [PATCH 14/14] fix: address bug-bot round-6 findings (KoOvA/KoOvD/KoOvH) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit KoOvA (🔴): mapLiftRequestToPublishOptions forced BOTH inline-encryption callbacks (encryptInlinePayload + encryptInlineChunked) to undefined for accessPolicy === 'public'. Previously the public branch forwarded resolved.encryptInlinePayload, so a resolver/default factory could push a public CG onto DKGPublisher's encrypted-inline path — silently encrypting public content at rest and breaking public ACK verification. KoOvD (🟡): added a CLI runtime-boundary regression (publisher-runner-private-encryption.test.ts) that drives a real ownerOnly async-lift publish through createPublisherRuntimeFromAgent and asserts publisher.publish receives the publishEncryptionFactory callback by reference — not the fail-closed mapper default. Also added prepareAsyncPublishPayload-seam precedence tests + exported isFailClosedInlineEncrypt / prepareAsyncPublishPayload from the package index. KoOvH (🟡): extracted the live onKARegisteredToContextGraph nudge body into DKGAgent.handleKARegisteredNudge so the event-handler branch is directly testable, and added tests exercising it: with multiple subscribed-but-unbound CGs and one matching ontology OnChainId only the match binds + reconciles; an already-bound CG reconciles directly. Verified: publisher 1185 passed, agent 1625 passed, CLI runner tests green. Co-Authored-By: Claude Opus 4.8 --- packages/agent/src/dkg-agent-lifecycle.ts | 34 +---- packages/agent/src/dkg-agent-swm-host.ts | 56 ++++++++ .../test/vm-reconcile-self-prime.test.ts | 65 +++++++++ ...ublisher-runner-private-encryption.test.ts | 124 ++++++++++++++++++ .../src/async-lift-publish-options.ts | 21 +-- packages/publisher/src/index.ts | 3 + ...ft-canonicalization-and-encryption.test.ts | 26 ++++ .../test/async-lift-publish-options.test.ts | 100 +++++++++++++- 8 files changed, 389 insertions(+), 40 deletions(-) create mode 100644 packages/cli/test/publisher-runner-private-encryption.test.ts diff --git a/packages/agent/src/dkg-agent-lifecycle.ts b/packages/agent/src/dkg-agent-lifecycle.ts index b8f8bda01..307dee87a 100644 --- a/packages/agent/src/dkg-agent-lifecycle.ts +++ b/packages/agent/src/dkg-agent-lifecycle.ts @@ -1510,37 +1510,9 @@ export class LifecycleSyncMethods extends DKGAgentBase { // is actually possible (chain + ordinal reads present). onKARegisteredToContextGraph: this.vmReconcileEnabled() ? async ({ contextGraphId: onChainId, kaId }) => { - const localCgId = this.resolveLocalCgIdByOnChainId(BigInt(onChainId)); - if (!localCgId) { - // GH #1098 — a pre-subscribed PUBLIC member peer may not have - // bound this CG's on-chain id yet (only curated CGs bind on the - // ContextGraphCreated event; ACK-signers bind via the storage-ACK - // hook). Find the subscribed-but-unbound CG whose locally-resolved - // on-chain id matches THIS event and bind + reconcile only it — - // targeted, not a global sweep, so an unrelated KA registration - // touches nothing. Uses the SAME self-prime helper as the - // periodic sweep (single bind/persist/cursor-reset path); the - // sweep remains the safety net for a CG whose quad hasn't arrived. - let targetOnChain: bigint | null = null; - try { targetOnChain = BigInt(onChainId); } catch { targetOnChain = null; } - if (targetOnChain !== null) { - for (const [lcg, sub] of this.subscribedContextGraphs) { - const bound = await this.selfPrimeSubscriptionOnChainId(lcg, sub, targetOnChain); - if (bound) { - this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> bound + reconcile pre-subscribed "${lcg}"`); - if (this.reconcileCoalescer) void this.reconcileCoalescer.trigger(lcg); - break; - } - } - } - return; // chain replay hasn't resolved the cleartext CG yet; periodic sweep is the safety net - } - const sub = this.subscribedContextGraphs.get(localCgId); - // Populate VM for CGs we member-subscribe to OR (Phase D) public - // CGs this Core hosts — a hosted Core fills its own gaps too. - if (!sub?.subscribed && !sub?.coreHosted) return; - this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> reconcile "${localCgId}"`); - if (this.reconcileCoalescer) void this.reconcileCoalescer.trigger(localCgId); + // GH #1098 — body extracted to `handleKARegisteredNudge` so the + // bind-only-the-matching-CG branch is directly testable. + await this.handleKARegisteredNudge(onChainId, kaId, ctx); } : undefined, }); diff --git a/packages/agent/src/dkg-agent-swm-host.ts b/packages/agent/src/dkg-agent-swm-host.ts index 5fcbdfb4a..be44828d3 100644 --- a/packages/agent/src/dkg-agent-swm-host.ts +++ b/packages/agent/src/dkg-agent-swm-host.ts @@ -2399,6 +2399,62 @@ export class SwmHostModeMethods extends DKGAgentBase { return resolved; } + /** + * GH #1098 (Phase B) — body of the live `onKARegisteredToContextGraph` nudge, + * extracted so the branch is directly testable. A + * `KnowledgeAssetRegisteredToContextGraph` event carries only `{ kaId, cgId }` + * (no ordinal), so this just triggers a coalesced reconcile for the matching + * local CG. Two cases: + * + * 1. The on-chain id is already bound to a local CG → trigger its reconcile + * (when subscribed or core-hosted). + * 2. The id is unbound but a pre-subscribed PUBLIC member CG resolves to it + * (subscribed BEFORE its first publish; only curated CGs bind on the + * ContextGraphCreated event and ACK-signers bind via the storage-ACK hook) + * → self-prime + bind ONLY the CG whose resolved id matches THIS event, + * then reconcile it. Unrelated subscribed-unbound CGs are left untouched. + * + * Best-effort and idempotent: a missed nudge heals on the periodic sweep. + * Returns the local CG id that was reconciled, or null if none matched. + */ + async handleKARegisteredNudge( + this: DKGAgent, + onChainId: string, + kaId: bigint, + ctx: OperationContext, + ): Promise { + let targetOnChain: bigint | null = null; + try { targetOnChain = BigInt(onChainId); } catch { targetOnChain = null; } + + const localCgId = targetOnChain === null ? null : this.resolveLocalCgIdByOnChainId(targetOnChain); + if (!localCgId) { + // Find the subscribed-but-unbound CG whose locally-resolved on-chain id + // matches THIS event and bind + reconcile only it — targeted, not a global + // sweep, so an unrelated KA registration touches nothing. Uses the SAME + // self-prime helper as the periodic sweep (single bind/persist/cursor-reset + // path); the sweep remains the safety net for a CG whose quad hasn't arrived. + if (targetOnChain !== null) { + for (const [lcg, sub] of this.subscribedContextGraphs) { + const bound = await this.selfPrimeSubscriptionOnChainId(lcg, sub, targetOnChain); + if (bound) { + this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> bound + reconcile pre-subscribed "${lcg}"`); + if (this.reconcileCoalescer) void this.reconcileCoalescer.trigger(lcg); + return lcg; + } + } + } + return null; // chain replay hasn't resolved the cleartext CG yet; periodic sweep is the safety net + } + + const sub = this.subscribedContextGraphs.get(localCgId); + // Populate VM for CGs we member-subscribe to OR (Phase D) public CGs this + // Core hosts — a hosted Core fills its own gaps too. + if (!sub?.subscribed && !sub?.coreHosted) return null; + this.log.info(ctx, `Phase B: KACG nudge cg=${onChainId} ka=${kaId} -> reconcile "${localCgId}"`); + if (this.reconcileCoalescer) void this.reconcileCoalescer.trigger(localCgId); + return localCgId; + } + /** * One reconcile pass for a single CG: build the injected deps and hand off to * the pure {@link reconcileContextGraph} orchestrator (which owns the cursor diff --git a/packages/agent/test/vm-reconcile-self-prime.test.ts b/packages/agent/test/vm-reconcile-self-prime.test.ts index ed3ee808e..ce6ffb7ec 100644 --- a/packages/agent/test/vm-reconcile-self-prime.test.ts +++ b/packages/agent/test/vm-reconcile-self-prime.test.ts @@ -16,6 +16,7 @@ import { DKG_ONTOLOGY, SYSTEM_CONTEXT_GRAPHS, contextGraphDataGraphUri, + createOperationContext, } from '@origintrail-official/dkg-core'; import type { TripleStore } from '@origintrail-official/dkg-storage'; import { DKGAgent } from '../src/index.js'; @@ -27,6 +28,7 @@ interface AgentInternals { sub: { subscribed: boolean; coreHosted?: boolean; onChainId?: string }, targetOnChainId?: bigint, ): Promise; + handleKARegisteredNudge(onChainId: string, kaId: bigint, ctx: unknown): Promise; subscribedContextGraphs: Map; reconcileCoalescer: { trigger: (cg: string) => void } | null; store: TripleStore; @@ -115,4 +117,67 @@ describe('GH #1098 — VM reconcile sweep self-primes onChainId for a pre-subscr expect(matched).toBe(ON_MATCH); expect(internals.subscribedContextGraphs.get(CG_MATCH)?.onChainId).toBe(ON_MATCH); }); + + it('live KACG nudge handler: with multiple subscribed-unbound CGs, binds + reconciles ONLY the one matching the event id', async () => { + // Exercises the EXACT branch the live `onKARegisteredToContextGraph` poller + // hook runs (extracted to `handleKARegisteredNudge`), not just the underlying + // self-prime helper — so the loop-and-target logic is covered end to end. + // Three pre-subscribed PUBLIC member CGs are unbound (the #1098 state); a KA + // registration arrives for ONE of their on-chain ids. Only that CG must bind + // and reconcile; the other two are left untouched. + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'KacgNudgeLive', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + + const CG_HIT = 'gh1098-nudge-hit'; + const CG_MISS_A = 'gh1098-nudge-miss-a'; + const CG_MISS_B = 'gh1098-nudge-miss-b'; + const ON_HIT = '7000'; + const ON_MISS_A = '7001'; + const ON_MISS_B = '7002'; + const ontologyGraph = contextGraphDataGraphUri(SYSTEM_CONTEXT_GRAPHS.ONTOLOGY); + await internals.store.insert([ + { subject: `did:dkg:context-graph:${CG_HIT}`, predicate: `${DKG_ONTOLOGY.DKG_CONTEXT_GRAPH}OnChainId`, object: `"${ON_HIT}"`, graph: ontologyGraph }, + { subject: `did:dkg:context-graph:${CG_MISS_A}`, predicate: `${DKG_ONTOLOGY.DKG_CONTEXT_GRAPH}OnChainId`, object: `"${ON_MISS_A}"`, graph: ontologyGraph }, + { subject: `did:dkg:context-graph:${CG_MISS_B}`, predicate: `${DKG_ONTOLOGY.DKG_CONTEXT_GRAPH}OnChainId`, object: `"${ON_MISS_B}"`, graph: ontologyGraph }, + ]); + internals.subscribedContextGraphs.set(CG_HIT, { subscribed: true }); + internals.subscribedContextGraphs.set(CG_MISS_A, { subscribed: true }); + internals.subscribedContextGraphs.set(CG_MISS_B, { subscribed: true }); + + const triggered: string[] = []; + internals.reconcileCoalescer = { trigger: (cg: string) => { triggered.push(cg); } }; + + // The event names ON_HIT's on-chain id. None is bound yet. + const reconciled = await internals.handleKARegisteredNudge(ON_HIT, 99n, createOperationContext('system')); + + expect(reconciled).toBe(CG_HIT); + expect(internals.subscribedContextGraphs.get(CG_HIT)?.onChainId).toBe(ON_HIT); + // The other two pre-subscribed CGs were NOT bound and NOT reconciled. + expect(internals.subscribedContextGraphs.get(CG_MISS_A)?.onChainId).toBeUndefined(); + expect(internals.subscribedContextGraphs.get(CG_MISS_B)?.onChainId).toBeUndefined(); + expect(triggered).toEqual([CG_HIT]); + }); + + it('live KACG nudge handler: an already-bound CG reconciles directly without a self-prime scan', async () => { + // The other live branch: the event id already resolves to a local CG. It must + // reconcile that CG straight away (subscribed or core-hosted), independent of + // the pre-subscribed self-prime loop. + const chain = new MockChainAdapter(); + agent = await DKGAgent.create({ name: 'KacgNudgeBound', chainAdapter: chain }); + stubNode(agent); + const internals = agent as unknown as AgentInternals; + + const CG_BOUND = 'gh1098-nudge-bound'; + const ON_BOUND = '8080'; + internals.subscribedContextGraphs.set(CG_BOUND, { subscribed: true, onChainId: ON_BOUND }); + + const triggered: string[] = []; + internals.reconcileCoalescer = { trigger: (cg: string) => { triggered.push(cg); } }; + + const reconciled = await internals.handleKARegisteredNudge(ON_BOUND, 1n, createOperationContext('system')); + expect(reconciled).toBe(CG_BOUND); + expect(triggered).toEqual([CG_BOUND]); + }); }); diff --git a/packages/cli/test/publisher-runner-private-encryption.test.ts b/packages/cli/test/publisher-runner-private-encryption.test.ts new file mode 100644 index 000000000..7b22644f0 --- /dev/null +++ b/packages/cli/test/publisher-runner-private-encryption.test.ts @@ -0,0 +1,124 @@ +/** + * GH #1121 / KoOvD — CLI runtime-boundary regression for the inline-encryption + * callback precedence. + * + * The original failure mode was at THIS boundary (`publisher-runner.ts`'s + * `publishExecutor`): the async-lift mapper pre-populates a fail-closed default + * `encryptInlinePayload` for non-public CGs (so plaintext can never silently + * ship), and the runner must let the REAL `publishEncryptionFactory` callback + * win over that default — otherwise every private async publish would invoke the + * throwing fail-closed default instead of encrypting. + * + * The publisher-level unit tests only assert the mapper's `resolved`-precedence; + * they do NOT cover the runner's `?? publishOptions.encryptInlinePayload` merge. + * This drives a real non-public (ownerOnly) async-lift publish end to end through + * `createPublisherRuntimeFromAgent` and asserts that the callback `publisher.publish` + * actually receives is the factory's closure — NOT the fail-closed mapper default. + * Hermetic — NoChainAdapter, in-memory store, no network. + */ +import { describe, expect, it, vi } from 'vitest'; +import { mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { ethers } from 'ethers'; +import { NoChainAdapter } from '@origintrail-official/dkg-chain'; +import { generateEd25519Keypair, TypedEventBus } from '@origintrail-official/dkg-core'; +import { + DKGPublisher, + isFailClosedInlineEncrypt, + type PublishOptions, +} from '@origintrail-official/dkg-publisher'; +import { createTripleStore } from '@origintrail-official/dkg-storage'; +import { addPublisherWallet } from '../src/publisher-wallets.js'; +import { createPublisherRuntimeFromAgent } from '../src/publisher-runner.js'; + +describe('publisher runner — private (non-public) inline-encryption callback handoff', () => { + it('passes the real publishEncryptionFactory callback to publisher.publish for an ownerOnly lift (not the fail-closed mapper default)', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-priv-enc-')); + const wallet = ethers.Wallet.createRandom(); + const store = await createTripleStore({ backend: 'oxigraph' }); + const keypair = await generateEd25519Keypair(); + await addPublisherWallet(dataDir, wallet.privateKey); + + const writer = new DKGPublisher({ + store, + chain: new NoChainAdapter(), + eventBus: new TypedEventBus(), + keypair, + publisherPrivateKey: wallet.privateKey, + }); + const write = await writer.writeToWorkspace('music-social', [ + { + subject: 'urn:local:/rihana', + predicate: 'http://schema.org/name', + object: '"Rihana"', + graph: '', + }, + ], { publisherPeerId: 'peer-1' }); + + // The agent-resolved, chainKey-bound AEAD closures. Stable references so the + // assertion can prove THESE reached publish (vs. the throwing fail-closed default). + const realInline = async (b: Uint8Array): Promise => new Uint8Array([...b, 0xee]); + const realChunked = async (): Promise<{ ciphertextChunksRoot: Uint8Array; ciphertextChunkCount: number; totalCiphertextBytes: number }> => ({ + ciphertextChunksRoot: ethers.getBytes(ethers.id('priv-enc-chunk-root')), + ciphertextChunkCount: 1, + totalCiphertextBytes: 1, + }); + + // Capture what the runner's publishExecutor hands to publisher.publish, then + // run the real publish (local finalization under NoChainAdapter). + let captured: PublishOptions | undefined; + const originalPublish = DKGPublisher.prototype.publish; + const publishSpy = vi + .spyOn(DKGPublisher.prototype, 'publish') + .mockImplementation(async function (this: DKGPublisher, opts: PublishOptions) { + captured ??= opts; + return originalPublish.call(this, opts); + }); + + try { + const runtime = await createPublisherRuntimeFromAgent({ + dataDir, + store, + keypair, + chainBase: undefined, + pollIntervalMs: 10, + errorBackoffMs: 10, + publishEncryptionFactory: () => ({ + encryptInlinePayload: realInline, + encryptInlineChunked: realChunked, + }), + }); + + const jobId = await runtime.publisher.lift({ + swmId: 'swm-main', + shareOperationId: write.shareOperationId, + roots: ['urn:local:/rihana'], + contextGraphId: 'music-social', + namespace: 'aloha', + scope: 'person-profile', + transitionType: 'CREATE', + authority: { type: 'owner', proofRef: 'proof:owner:1' }, + // The crux: a NON-public publish — the mapper pre-populates the fail-closed + // default, and the runner must override it with the real factory callback. + accessPolicy: 'ownerOnly', + }); + + const processed = await runtime.publisher.processNext(wallet.address); + expect(processed?.jobId).toBe(jobId); + + // The runner forwarded the lift's non-public access policy through. + expect(captured?.accessPolicy).toBe('ownerOnly'); + // publisher.publish received the REAL factory callbacks, by reference … + expect(captured?.encryptInlinePayload).toBe(realInline); + expect(captured?.encryptInlineChunked).toBe(realChunked); + // … and explicitly NOT the throwing fail-closed mapper default. + expect(isFailClosedInlineEncrypt(captured?.encryptInlinePayload)).toBe(false); + + await runtime.stop(); + } finally { + publishSpy.mockRestore(); + await store.close(); + } + }); +}); diff --git a/packages/publisher/src/async-lift-publish-options.ts b/packages/publisher/src/async-lift-publish-options.ts index 85d06dcab..4651c53a4 100644 --- a/packages/publisher/src/async-lift-publish-options.ts +++ b/packages/publisher/src/async-lift-publish-options.ts @@ -150,16 +150,21 @@ export function mapLiftRequestToPublishOptions(input: LiftPublishMappingInput): throw new Error('Lift publish mapping only allows allowedPeers when accessPolicy is allowList'); } - // GH #1121 — derive the inline-encryption path. Public CGs stay plaintext - // (undefined). Non-public CGs MUST carry an inline-encryption callback: use - // the agent-resolved one threaded through `resolved`, else the fail-closed - // default so a missing factory throws at publish time rather than leaking - // plaintext. The chunked callback is optional (the inline blob path is the - // floor) and only ever passes through the resolved value. + // GH #1121 — derive the inline-encryption path. Public CGs MUST stay + // plaintext: force BOTH inline callbacks to `undefined` so even a resolver + // that supplies a default encryption factory cannot push DKGPublisher onto + // the encrypted-inline path for a public CG (otherwise public content would + // be silently encrypted at rest and unreadable by readers). Non-public CGs + // MUST carry an inline-encryption callback: use the agent-resolved one + // threaded through `resolved`, else the fail-closed default so a missing + // factory throws at publish time rather than leaking plaintext. The chunked + // callback is optional (the inline blob path is the floor). const encryptInlinePayload = accessPolicy === 'public' - ? input.resolved.encryptInlinePayload + ? undefined : (input.resolved.encryptInlinePayload ?? failClosedInlineEncrypt); - const encryptInlineChunked = input.resolved.encryptInlineChunked; + const encryptInlineChunked = accessPolicy === 'public' + ? undefined + : input.resolved.encryptInlineChunked; // Request flags (enqueue-time caller intent) win over resolved hints (per-process defaults). const entityProofs = input.request.entityProofs ?? input.resolved.entityProofs; diff --git a/packages/publisher/src/index.ts b/packages/publisher/src/index.ts index 7b6cf3226..1a2f20178 100644 --- a/packages/publisher/src/index.ts +++ b/packages/publisher/src/index.ts @@ -191,6 +191,9 @@ export { } from './async-lift-runner.js'; export { mapLiftRequestToPublishOptions, + prepareAsyncPublishPayload, + isFailClosedInlineEncrypt, + type AsyncPreparedPublishPayload, type LiftResolvedPublishSlice, type LiftPublishMappingInput, } from './async-lift-publish-options.js'; diff --git a/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts b/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts index 745bcb117..63d9db74c 100644 --- a/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts +++ b/packages/publisher/test/async-lift-canonicalization-and-encryption.test.ts @@ -113,4 +113,30 @@ describe('GH #1121 — async lift carries an inline-encryption path for private // publisher onto the encrypted-inline path and break public ACK verification. expect(opts.encryptInlinePayload).toBeUndefined(); }); + + it('a PUBLIC async publish FORCES both inline callbacks to undefined even when the resolver supplies them', () => { + // Defence-in-depth for the public path: a resolver (or a per-process default + // encryption factory) might hand a non-public-looking callback through + // `resolved` even for a public CG. The mapper must NOT forward it — otherwise + // DKGPublisher's `useEncryptedInline` gate (`typeof inlineEncryptCb === 'function'`) + // would silently encrypt public content at rest, making it unreadable to + // public readers and breaking public ACK verification. Force BOTH to undefined. + const strayInline = async (b: Uint8Array): Promise => new Uint8Array([...b, 0xff]); + const strayChunked = async (): Promise => { /* would fan out members */ }; + const opts = mapLiftRequestToPublishOptions({ + request: { ...baseRequest, accessPolicy: 'public' }, + validation: { authorityProofRef: 'devnet-proof', transitionType: 'CREATE' }, + resolved: { + quads: [ + { subject: CALLER_ROOT, predicate: 'https://schema.org/name', object: '"Tenant A"', graph: 'g' }, + ], + accessPolicy: 'public', + encryptInlinePayload: strayInline, + encryptInlineChunked: strayChunked, + }, + }); + expect(opts.accessPolicy).toBe('public'); + expect(opts.encryptInlinePayload).toBeUndefined(); + expect(opts.encryptInlineChunked).toBeUndefined(); + }); }); diff --git a/packages/publisher/test/async-lift-publish-options.test.ts b/packages/publisher/test/async-lift-publish-options.test.ts index 347690a5c..a5297f1c3 100644 --- a/packages/publisher/test/async-lift-publish-options.test.ts +++ b/packages/publisher/test/async-lift-publish-options.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { mapLiftRequestToPublishOptions, prepareAsyncPublishPayload, type LiftPublishMappingInput } from '../src/async-lift-publish-options.js'; +import { mapLiftRequestToPublishOptions, prepareAsyncPublishPayload, isFailClosedInlineEncrypt, type LiftPublishMappingInput } from '../src/async-lift-publish-options.js'; describe('mapLiftRequestToPublishOptions', () => { function baseInput(): LiftPublishMappingInput { @@ -578,3 +578,101 @@ describe('mapLiftRequestToPublishOptions', () => { expect(options.precomputedAttestation).toBeUndefined(); }); }); + +// GH #1121 follow-up — the inline-encryption callback precedence/forcing that +// `async-lift-publisher-impl` depends on, asserted at the `prepareAsyncPublishPayload` +// seam the impl actually calls (not just the lower-level mapper). The impl threads +// `prepared.publishOptions` straight into `publishExecutor`/`publisher.publish`, so +// whatever lands here is exactly what the publisher's `useEncryptedInline` gate sees. +describe('prepareAsyncPublishPayload — inline-encryption callback handoff to publisher.publish', () => { + function baseInput(): LiftPublishMappingInput { + return { + request: { + swmId: 'swm-1', + shareOperationId: 'op-1', + roots: ['urn:local:/rihana'], + contextGraphId: 'music-social', + namespace: 'aloha', + scope: 'person-profile', + transitionType: 'CREATE', + authority: { type: 'owner', proofRef: 'proof:owner:1' }, + }, + validation: { + authorityProofRef: 'proof:owner:1', + priorVersion: undefined, + transitionType: 'CREATE', + }, + resolved: { + quads: [ + { + subject: 'did:dkg:music-social:rihana', + predicate: 'http://schema.org/name', + object: '"Rihana"', + graph: 'did:dkg:context-graph:music-social/_data', + }, + ], + }, + }; + } + + it('forwards the agent-resolved real encryption factory into publishOptions (NOT the fail-closed default) for a non-public publish', () => { + // KoOvD: the impl hands `prepared.publishOptions` to `publishExecutor`, which + // calls `publisher.publish`. If the mapper ever shadowed the real factory with + // the throwing fail-closed default, every private async publish would throw at + // encryption time. Pin that the REAL closure reaches the publish boundary. + const realInline = async (b: Uint8Array): Promise => new Uint8Array([...b, 0xff]); + const realChunked = async (): Promise => { /* member key fan-out */ }; + const prepared = prepareAsyncPublishPayload({ + ...baseInput(), + request: { ...baseInput().request, accessPolicy: 'ownerOnly' }, + resolved: { + ...baseInput().resolved, + accessPolicy: 'ownerOnly', + publisherPeerId: '12D3KooWPublisher', + encryptInlinePayload: realInline, + encryptInlineChunked: realChunked, + }, + }); + expect(prepared.publishOptions.accessPolicy).toBe('ownerOnly'); + expect(prepared.publishOptions.encryptInlinePayload).toBe(realInline); + expect(prepared.publishOptions.encryptInlineChunked).toBe(realChunked); + expect(isFailClosedInlineEncrypt(prepared.publishOptions.encryptInlinePayload)).toBe(false); + }); + + it('hands the fail-closed default to publisher.publish for a non-public publish with NO resolved factory (so it throws rather than leaking plaintext)', () => { + // When no factory is wired, the publish boundary must receive the throwing + // fail-closed marker — never `undefined`, which would silently ship plaintext. + const prepared = prepareAsyncPublishPayload({ + ...baseInput(), + request: { ...baseInput().request, accessPolicy: 'ownerOnly' }, + resolved: { + ...baseInput().resolved, + accessPolicy: 'ownerOnly', + publisherPeerId: '12D3KooWPublisher', + }, + }); + expect(prepared.publishOptions.encryptInlinePayload).toBeDefined(); + expect(isFailClosedInlineEncrypt(prepared.publishOptions.encryptInlinePayload)).toBe(true); + }); + + it('hands publisher.publish NO inline-encryption callbacks for a public publish, even when the resolver supplies them', () => { + // KoOvA at the handoff boundary: a stray resolver/default callback must not + // reach the publisher for a public CG, or its `useEncryptedInline` gate would + // encrypt public content at rest. Both inline callbacks must be undefined. + const strayInline = async (b: Uint8Array): Promise => new Uint8Array([...b, 0xff]); + const strayChunked = async (): Promise => { /* would fan out members */ }; + const prepared = prepareAsyncPublishPayload({ + ...baseInput(), + request: { ...baseInput().request, accessPolicy: 'public' }, + resolved: { + ...baseInput().resolved, + accessPolicy: 'public', + encryptInlinePayload: strayInline, + encryptInlineChunked: strayChunked, + }, + }); + expect(prepared.publishOptions.accessPolicy).toBe('public'); + expect(prepared.publishOptions.encryptInlinePayload).toBeUndefined(); + expect(prepared.publishOptions.encryptInlineChunked).toBeUndefined(); + }); +});