Skip to content

feat(chain): immediate RPC failover for reads and writes (multi-RPC) #1335

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Jurij89 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

feat(chain): immediate RPC failover for reads and writes (multi-RPC) #1335

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4ba46ee
feat(chain): immediate RPC failover for reads and writes (multi-RPC)
Jun 25, 2026
df7dc3f
fix(chain): address PR #1335 review — read-failover classifier, getRe…
Jun 26, 2026
8839a6b
fix(chain): PR #1335 review round 2 — deprecate getReadProvider, spli…
Jun 26, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 19 additions & 8 deletions packages/chain/src/evm-adapter-ack-sign.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,11 @@ export class AckSignMethods extends EVMChainAdapterBase {
'Verify cannot enforce ACK quorum without a real chain read — fix the adapter wiring or pass an explicit override.',
);
}
const value = Number(await this.contracts.parametersStorage.minimumRequiredSignatures());
const value = Number(await this.contractReadWithFailover(
'parametersStorage.minimumRequiredSignatures',
this.contracts.parametersStorage,
(c) => c.minimumRequiredSignatures(),
));
this.cachedMinRequiredSignatures = { value, cachedAt: now };
return value;
}
Expand All @@ -52,7 +56,9 @@ export class AckSignMethods extends EVMChainAdapterBase {
'Verify path cannot enforce sharding-table eligibility without it.',
);
}
return Boolean(await storage.nodeExists(identityId));
return Boolean(await this.contractReadWithFailover(
'shardingTableStorage.nodeExists', storage, (c) => c.nodeExists(identityId),
));
}

/**
Expand Down Expand Up @@ -80,16 +86,18 @@ export class AckSignMethods extends EVMChainAdapterBase {
if (!identityStorage) return { valid: false, reason: 'rpc-error' };

const keyHash = ethers.keccak256(ethers.solidityPacked(['address'], [recoveredAddress]));
const hasPurpose: boolean = await identityStorage.keyHasPurpose(
claimedIdentityId,
keyHash,
OPERATIONAL_KEY_PURPOSE,
const hasPurpose: boolean = await this.contractReadWithFailover(
'identityStorage.keyHasPurpose', identityStorage,
(c) => c.keyHasPurpose(claimedIdentityId, keyHash, OPERATIONAL_KEY_PURPOSE),
);
if (!hasPurpose) return { valid: false, reason: 'key-not-registered' };

const shardingTableStorage = await this.resolveContract('ShardingTableStorage');
if (!shardingTableStorage) return { valid: false, reason: 'rpc-error' };
const inST: boolean = Boolean(await shardingTableStorage.nodeExists(claimedIdentityId));
const inST: boolean = Boolean(await this.contractReadWithFailover(
'shardingTableStorage.nodeExists', shardingTableStorage,
(c) => c.nodeExists(claimedIdentityId),
));
if (!inST) return { valid: false, reason: 'not-in-sharding-table' };
return { valid: true };
} catch {
Expand Down Expand Up @@ -122,7 +130,10 @@ export class AckSignMethods extends EVMChainAdapterBase {
if (!identityStorage) return false;

const keyHash = ethers.keccak256(ethers.solidityPacked(['address'], [recoveredAddress]));
return identityStorage.keyHasPurpose(claimedIdentityId, keyHash, OPERATIONAL_KEY_PURPOSE);
return this.contractReadWithFailover(
'identityStorage.keyHasPurpose', identityStorage,
(c) => c.keyHasPurpose(claimedIdentityId, keyHash, OPERATIONAL_KEY_PURPOSE),
);
}

async signACKDigest(digest: Uint8Array): Promise<{ r: Uint8Array; vs: Uint8Array } | undefined> {
Expand Down
506 changes: 386 additions & 120 deletions packages/chain/src/evm-adapter-base.ts
View file
Open in desktop

Large diffs are not rendered by default.

32 changes: 32 additions & 0 deletions packages/chain/src/evm-adapter-constants.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,21 @@ export const MAX_PROBE_AGE_MS = 30_000;

export const RPC_READ_STALL_TIMEOUT_MS = 4_000;

/**
* Per-attempt deadline for WIDE `eth_getLogs` reads (the `evm-adapter-events.ts`
* `queryFilter` scans, which run over `[fromBlock ?? 0, toBlock]` — up to the
* event poller's 9,000-block window, and the full chain on a cold-start
* backfill). These legitimately take tens of seconds on a busy/slow chain, so
* the 4s `RPC_READ_STALL_TIMEOUT_MS` point-read cap would abort a healthy scan
* and fail it over across every endpoint → spurious `RPC_ENDPOINTS_EXHAUSTED`
* (which, in the poller, escapes before the cursor advances → a permanent
* stall). 30s still hard-bounds a genuinely hung backend on a multi-RPC node;
* it is passed as `multiAttemptTimeoutMs`, so single-RPC stays uncapped (#894).
* Larger than `KA_HIGH_WATER_PAGE_TIMEOUT_MS` (15s) because that bounds smaller
* 2,000-block pages, whereas this covers the wider 9,000-block poller window.
*/
export const RPC_LOG_SCAN_TIMEOUT_MS = 30_000;

export const RPC_TRANSACTION_POPULATION_ATTEMPT_TIMEOUT_MS = 10_000;

export const RPC_BROADCAST_ATTEMPT_TIMEOUT_MS = 10_000;
Expand All @@ -54,6 +69,23 @@ export const RPC_RECEIPT_POLL_INTERVAL_MS = 2_000;

export const RPC_RECEIPT_TIMEOUT_MS = 180_000;

/**
* Bounded "retry the whole endpoint set" for the BROADCAST phase (S2). After a
* full per-endpoint broadcast pass exhausts with a retryable error (e.g. a brief
* window where ALL endpoints 429), `sendSignedTransactionAndWait` re-broadcasts
* the SAME already-signed/already-WAL-checkpointed tx up to this many extra full
* passes, with `RPC_ENDPOINT_SET_RETRY_BACKOFF_MS` between passes, before
* surfacing `RPC_ENDPOINTS_EXHAUSTED`. Default `1` (one extra pass) keeps the
* "ride out a brief all-down blip" property without per-endpoint latency; `0`
* fails fast on the first full-pass exhaustion. Re-broadcasting the identical
* signed tx is idempotent (`isKnownTransactionError`), so this cannot double-
* submit or change the nonce — it is scoped to broadcast ONLY (receipt waiting
* owns its own deadline), so total lock-hold stays bounded.
*/
export const RPC_ENDPOINT_SET_RETRIES = 1;

export const RPC_ENDPOINT_SET_RETRY_BACKOFF_MS = 500;

export const ADMIN_KEY_PURPOSE = 1;

export const OPERATIONAL_KEY_PURPOSE = 2;
Expand Down
38 changes: 28 additions & 10 deletions packages/chain/src/evm-adapter-context-graph.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,9 @@ export class ContextGraphMethods extends EVMChainAdapterBase {
async isContextGraphActiveOnChain(contextGraphId: bigint): Promise<boolean> {
await this.init();
const cgs = this.requireContextGraphStorage();
return Boolean(await cgs.isContextGraphActive(contextGraphId));
return Boolean(await this.contractReadWithFailover(

@otReviewAgent otReviewAgent Jun 26, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Issue: This contractReadWithFailover(label, contract, c => c.method(...)) shape now recurs across the adapter modules, so every domain method has to carry transport labels, rebinding lambdas, and failover mechanics inline. That leaks the transport abstraction into business-facing code and makes future reads easy to implement inconsistently. Please hide this behind a narrower contract-view dispatcher or failover-bound contract facade, e.g. readContract(cgs, 'cgStorage.isContextGraphActive', 'isContextGraphActive', contextGraphId), so the call sites stay about the chain concept rather than the failover plumbing.

@Jurij89 Jurij89 Jun 26, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed — the inline contractReadWithFailover(label, contract, c => c.method(...)) shape leaking the transport abstraction into domain modules is worth hiding behind a narrower read facade (e.g. readContract(contract, label, method, ...args)). Tracked in #1336: the facade lands naturally with the failover-subsystem extraction, so deferring it to land together rather than half-introducing it here.

'cgStorage.isContextGraphActive', cgs, (c) => c.isContextGraphActive(contextGraphId),
));
}

async createOnChainContextGraph(params: CreateOnChainContextGraphParams): Promise<CreateOnChainContextGraphResult> {
Expand Down Expand Up @@ -470,7 +472,7 @@ export class ContextGraphMethods extends EVMChainAdapterBase {

// Unreachable below (kept for type-completeness until the mirror is removed);
// the unsupported-mirror guard above throws before any on-chain side effect.
const v10ChainId = (await this.provider.getNetwork()).chainId;
const v10ChainId = (await this.readWithFailover('getNetwork (chainId)', (p) => p.getNetwork())).chainId;
const v10KavAddress = await this.contracts.knowledgeAssetsLifecycle!.getAddress();
const authorTypedData = buildAuthorAttestationTypedData({
chainId: v10ChainId,
Expand Down Expand Up @@ -514,21 +516,27 @@ export class ContextGraphMethods extends EVMChainAdapterBase {
async getKAContextGraphId(kaId: bigint): Promise<bigint> {
await this.init();
const cgs = this.requireContextGraphStorage();
const cgId: bigint = await cgs.kaToContextGraph(kaId);
const cgId: bigint = await this.contractReadWithFailover(
'cgStorage.kaToContextGraph', cgs, (c) => c.kaToContextGraph(kaId),
);
return BigInt(cgId);
}

async getContextGraphKCCount(contextGraphId: bigint): Promise<bigint> {
await this.init();
const cgs = this.requireContextGraphStorage();
const count: bigint = await cgs.getContextGraphKaCount(contextGraphId);
const count: bigint = await this.contractReadWithFailover(
'cgStorage.getContextGraphKaCount', cgs, (c) => c.getContextGraphKaCount(contextGraphId),
);
return BigInt(count);
}

async getContextGraphKCAt(contextGraphId: bigint, index: bigint): Promise<bigint> {
await this.init();
const cgs = this.requireContextGraphStorage();
const kaId: bigint = await cgs.getContextGraphKaAt(contextGraphId, index);
const kaId: bigint = await this.contractReadWithFailover(
'cgStorage.getContextGraphKaAt', cgs, (c) => c.getContextGraphKaAt(contextGraphId, index),
);
return BigInt(kaId);
}

Expand All @@ -544,11 +552,15 @@ export class ContextGraphMethods extends EVMChainAdapterBase {
await this.init();
const cgs = this.requireContextGraphStorage();
try {
const raw: bigint = BigInt(await cgs.getAccessPolicy(contextGraphId));
const raw: bigint = BigInt(await this.contractReadWithFailover(
'cgStorage.getAccessPolicy', cgs, (c) => c.getAccessPolicy(contextGraphId),
));
return Number(raw);
} catch (primaryErr) {
try {
const cg = await cgs.getContextGraph(contextGraphId);
const cg = await this.contractReadWithFailover(
'cgStorage.getContextGraph', cgs, (c) => c.getContextGraph(contextGraphId),
);
const raw =
cg?.accessPolicy
?? (Array.isArray(cg) ? cg[5] : undefined);
Expand Down Expand Up @@ -581,7 +593,9 @@ export class ContextGraphMethods extends EVMChainAdapterBase {
}> {
await this.init();
const cgs = this.requireContextGraphStorage();
const result = await cgs.getPublishPolicy(contextGraphId);
const result = await this.contractReadWithFailover(
'cgStorage.getPublishPolicy', cgs, (c) => c.getPublishPolicy(contextGraphId),
);
// Ethers v6 returns named tuple as both array and object access;
// destructure positionally to stay robust against ABI naming
// changes.
Expand Down Expand Up @@ -609,7 +623,9 @@ export class ContextGraphMethods extends EVMChainAdapterBase {
async getContextGraphParticipantAgents(contextGraphId: bigint): Promise<string[]> {
await this.init();
const cgs = this.requireContextGraphStorage();
const raw: string[] = await cgs.getParticipantAgents(contextGraphId);
const raw: string[] = await this.contractReadWithFailover(
'cgStorage.getParticipantAgents', cgs, (c) => c.getParticipantAgents(contextGraphId),
);
return raw.map((addr: string) => ethers.getAddress(addr));
}

Expand All @@ -633,7 +649,9 @@ export class ContextGraphMethods extends EVMChainAdapterBase {
async getContextGraphNameHash(contextGraphId: bigint): Promise<string | null> {
await this.init();
const cgs = this.requireContextGraphStorage();
const raw: string = await cgs.getNameHash(contextGraphId);
const raw: string = await this.contractReadWithFailover(
'cgStorage.getNameHash', cgs, (c) => c.getNameHash(contextGraphId),
);
if (!raw || raw === ethers.ZeroHash) return null;
return raw.toLowerCase();
}
Expand Down
40 changes: 28 additions & 12 deletions packages/chain/src/evm-adapter-conviction.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@ export class ConvictionMethods extends EVMChainAdapterBase implements Conviction
if (!this.contracts.dkgPublishingConvictionNFT) return 0n;
if (!ethers.isAddress(agent)) return 0n;
try {
const id: bigint = await this.contracts.dkgPublishingConvictionNFT.agentToAccountId(agent);
const id: bigint = await this.contractReadWithFailover(
'pcaNFT.agentToAccountId', this.contracts.dkgPublishingConvictionNFT, (c) => c.agentToAccountId(agent),
);
return BigInt(id);
} catch (err: any) {
if (err?.code === 'CALL_EXCEPTION') return 0n;
Expand All @@ -56,7 +58,9 @@ export class ConvictionMethods extends EVMChainAdapterBase implements Conviction
// (committedTRAC, createdAtEpoch, expiresAtEpoch, createdAtTimestamp,
// expiresAtTimestamp, lockDurationEpochs, discountBps,
// lastSettledWindow, fullySwept). Pull index 5.
const tuple = await this.contracts.dkgPublishingConvictionNFT.accounts(accountId);
const tuple = await this.contractReadWithFailover(
'pcaNFT.accounts', this.contracts.dkgPublishingConvictionNFT, (c) => c.accounts(accountId),
);
const lock = tuple[5];
return Number(lock);
} catch (err: any) {
Expand Down Expand Up @@ -110,7 +114,7 @@ export class ConvictionMethods extends EVMChainAdapterBase implements Conviction
// wall clock first to mirror the contract exactly — otherwise the SDK
// would coerce, then fall through to full-price direct spend.
if (info.expiresAtTimestamp > 0) {
const latestBlock = await this.provider.getBlock('latest');
const latestBlock = await this.readWithFailover('conviction getBlock', (p) => p.getBlock('latest'));
const nowTs = latestBlock ? Number(latestBlock.timestamp) : Math.floor(Date.now() / 1000);
if (nowTs >= info.expiresAtTimestamp) return false;
}
Expand All @@ -124,10 +128,12 @@ export class ConvictionMethods extends EVMChainAdapterBase implements Conviction
if (!this.contracts.chronos) {
this.contracts.chronos = await this.resolveContract('Chronos');
}
const currentEpoch: bigint = BigInt(await this.contracts.chronos.getCurrentEpoch());
const remaining: bigint = await this.contracts.dkgPublishingConvictionNFT.getRemainingAllowance(
accountId,
currentEpoch,
const currentEpoch: bigint = BigInt(await this.contractReadWithFailover(
'chronos.getCurrentEpoch', this.contracts.chronos, (c) => c.getCurrentEpoch(),
));
const remaining: bigint = await this.contractReadWithFailover(
'pcaNFT.getRemainingAllowance', this.contracts.dkgPublishingConvictionNFT,
(c) => c.getRemainingAllowance(accountId, currentEpoch),
);
return BigInt(remaining) >= discountedCost;
} catch (err: any) {
Expand All @@ -139,7 +145,9 @@ export class ConvictionMethods extends EVMChainAdapterBase implements Conviction
async getPublishingConvictionAccountOwner(accountId: bigint): Promise<string> {
await this.init();
const nft = await this.resolveContract('DKGPublishingConvictionNFT');
const owner = await nft.ownerOf(accountId);
const owner = await this.contractReadWithFailover(
'pcaNFT.ownerOf', nft, (c) => c.ownerOf(accountId),
);
return ethers.getAddress(owner);
}

Expand Down Expand Up @@ -203,7 +211,9 @@ export class ConvictionMethods extends EVMChainAdapterBase implements Conviction
// createAccount() does transferFrom(msg.sender → stakingStorage,
// committedTRAC) — the signer must allow the NFT to pull the TRAC.
if (this.contracts.token) {
const allowance: bigint = await this.contracts.token.allowance(this.signer.address, nftAddress);
const allowance: bigint = await this.contractReadWithFailover(
'token.allowance', this.contracts.token, (c) => c.allowance(this.signer.address, nftAddress),
);
if (allowance < committedTRAC) {
await this.sendContractTransaction(
this.contracts.token,
Expand Down Expand Up @@ -259,7 +269,9 @@ export class ConvictionMethods extends EVMChainAdapterBase implements Conviction
// for a genuine account-missing revert so the route can disambiguate.
if (!this.contracts.dkgPublishingConvictionNFT) throw new PcaUnavailableError();
try {
const t = await this.contracts.dkgPublishingConvictionNFT.getAccountInfo(accountId);
const t = await this.contractReadWithFailover(
'pcaNFT.getAccountInfo', this.contracts.dkgPublishingConvictionNFT, (c) => c.getAccountInfo(accountId),
);
return {
owner: ethers.getAddress(t[0]),
committedTRAC: BigInt(t[1]),
Expand All @@ -286,7 +298,9 @@ export class ConvictionMethods extends EVMChainAdapterBase implements Conviction
const nft = this.requireConvictionNFT();
const nftAddress = await nft.getAddress();
if (this.contracts.token) {
const allowance: bigint = await this.contracts.token.allowance(this.signer.address, nftAddress);
const allowance: bigint = await this.contractReadWithFailover(
'token.allowance', this.contracts.token, (c) => c.allowance(this.signer.address, nftAddress),
);
if (allowance < amount) {
await this.sendContractTransaction(
this.contracts.token,
Expand Down Expand Up @@ -358,7 +372,9 @@ export class ConvictionMethods extends EVMChainAdapterBase implements Conviction
if (!this.contracts.dkgPublishingConvictionNFT) return false;
if (!ethers.isAddress(agent)) return false;
try {
return Boolean(await this.contracts.dkgPublishingConvictionNFT.isAgent(accountId, agent));
return Boolean(await this.contractReadWithFailover(
'pcaNFT.isAgent', this.contracts.dkgPublishingConvictionNFT, (c) => c.isAgent(accountId, agent),
));
} catch (err: any) {
if (err?.code === 'CALL_EXCEPTION') return false;
throw err;
Expand Down
Loading
Loading