fix: prevent 'nonce too low' on cannon build -- broadcast (strict-ordering RPCs)

.changeset/build-broadcast-nonce.md

@@ -0,0 +1,12 @@
+---
+"@usecannon/cli": patch
+"@usecannon/builder": patch
+---
+
+fix: prevent "nonce too low" during `cannon build -- broadcast` on strict-ordering RPCs
+
+Live private-key signers now use a viem `nonceManager`, so nonces are assigned from a
+serialized local counter that never regresses below an already-used nonce instead of a fresh
+`eth_getTransactionCount('pending')` read on every transaction. Every broadcast is also wrapped
+in a nonce-aware retry that re-reads the nonce and resubmits on a transient nonce error. Fixes
+repeated build aborts on eventually-consistent / load-balanced RPCs (e.g. MegaETH testnet).

packages/builder/src/create2.ts

@@ -1,6 +1,7 @@
 import Debug from 'debug';
 import * as viem from 'viem';
 import { CannonSigner, ChainBuilderRuntimeInfo } from './';
+import { sendTransactionWithNonceRetry } from './helpers';
 
 const debug = Debug('cannon:builder:create2');
 
@@ -40,11 +41,13 @@ export async function ensureArachnidCreate2Exists(
     }
 
     // now run the presigned deployment txn
-    const hash = await signer.wallet.sendTransaction({
-      account: signer.address,
-      chain: runtime.provider.chain,
-      data: '0x604580600e600039806000f350fe7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf3',
-    });
+    const hash = await sendTransactionWithNonceRetry(signer, runtime.chainId, async () =>
+      signer.wallet.sendTransaction({
+        account: signer.address,
+        chain: runtime.provider.chain,
+        data: '0x604580600e600039806000f350fe7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf3',
+      })
+    );
 
     await runtime.provider.waitForTransactionReceipt({ hash });
   }

packages/builder/src/helpers.test.ts

@@ -1,5 +1,11 @@
 import * as viem from 'viem';
-import { getDefaultStorage, getCannonContract, getBlockRetried } from './helpers';
+import {
+  getDefaultStorage,
+  getCannonContract,
+  getBlockRetried,
+  isNonceError,
+  sendTransactionWithNonceRetry,
+} from './helpers';
 import { IPFSLoader, InMemoryLoader } from './loader';
 import { InMemoryRegistry, FallbackRegistry } from './registry';
 import { CannonStorage } from './runtime';
@@ -114,3 +120,66 @@ describe('helpers.test.ts', () => {
     });
   });
 });
+
+describe('isNonceError', () => {
+  it('matches viem nonce error messages, including nested causes', () => {
+    expect(isNonceError(new Error('nonce too low: next nonce 5, tx nonce 4'))).toBe(true);
+    expect(isNonceError(new Error('Nonce provided for the transaction is higher'))).toBe(true);
+    expect(isNonceError({ cause: { details: 'nonce too low' } } as any)).toBe(true);
+    expect(isNonceError(new Error('execution reverted'))).toBe(false);
+    expect(isNonceError(undefined)).toBe(false);
+  });
+
+  it('matches "already known" only in a transaction context', () => {
+    expect(isNonceError(new Error('transaction already known'))).toBe(true);
+    expect(isNonceError(new Error('already known transaction (hash 0x123)'))).toBe(true);
+    // must not retry unrelated "already known" failures
+    expect(isNonceError(new Error('contract already known in registry'))).toBe(false);
+  });
+});
+
+describe('sendTransactionWithNonceRetry', () => {
+  function fakeSigner() {
+    const reset = jest.fn();
+    const signer = {
+      address: '0x000000000000000000000000000000000000abcd',
+      wallet: { account: { address: '0x000000000000000000000000000000000000abcd', nonceManager: { reset } } },
+    } as any;
+    return { signer, reset };
+  }
+
+  it('retries a nonce-too-low failure then resolves, resetting the nonce manager', async () => {
+    const { signer, reset } = fakeSigner();
+    const send = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('nonce too low: next nonce 5, tx nonce 4'))
+      .mockResolvedValueOnce('0xhash');
+
+    const hash = await sendTransactionWithNonceRetry(signer, 6343, send);
+
+    expect(hash).toBe('0xhash');
+    expect(send).toHaveBeenCalledTimes(2);
+    expect(reset).toHaveBeenCalledWith({ address: signer.address, chainId: 6343 });
+  });
+
+  it('rethrows a non-nonce error without retrying', async () => {
+    const { signer } = fakeSigner();
+    const send = jest.fn().mockRejectedValue(new Error('execution reverted: boom'));
+
+    await expect(sendTransactionWithNonceRetry(signer, 6343, send)).rejects.toThrow('execution reverted');
+    expect(send).toHaveBeenCalledTimes(1);
+  });
+
+  it('rejects with the last error after all retries are exhausted', async () => {
+    const { signer } = fakeSigner();
+    const send = jest
+      .fn()
+      .mockRejectedValueOnce(new Error('nonce too low: next nonce 5, tx nonce 4'))
+      .mockRejectedValueOnce(new Error('nonce too low: next nonce 6, tx nonce 5'))
+      .mockRejectedValueOnce(new Error('nonce too low: next nonce 7, tx nonce 6'))
+      .mockRejectedValueOnce(new Error('nonce too low: next nonce 8, tx nonce 7'));
+
+    await expect(sendTransactionWithNonceRetry(signer, 6343, send)).rejects.toThrow('nonce too low');
+    expect(send).toHaveBeenCalledTimes(4); // 1 initial + 3 retries
+  });
+});

packages/builder/src/helpers.ts

@@ -8,6 +8,7 @@ import { PackageReference } from './package-reference';
 import { OnChainRegistry, FallbackRegistry, InMemoryRegistry } from './registry';
 import { CannonStorage } from './runtime';
 import { getContractFromPath } from './util';
+import type { CannonSigner } from './types';
 
 export function getDefaultStorage() {
   const registryChainIds = DEFAULT_REGISTRY_CONFIG.map((registry) => registry.chainId);
@@ -77,3 +78,42 @@ export async function getBlockRetried(provider: viem.PublicClient, blockHash: vi
     return provider.getBlock({ blockHash }).catch(retry);
   });
 }
+
+const NONCE_ERROR_RE =
+  /nonce too low|nonce too high|nonce provided.*is (?:too )?(?:low|high|higher|lower)|invalid (?:transaction )?nonce|replacement transaction underpriced|(?:transaction|tx).*already known|already known.*(?:transaction|tx)/i;
+
+// Detects whether an error (or any error in its `cause` chain) is a nonce-related RPC error.
+// viem wraps RPC errors, so the underlying message can live a few levels down in `.cause`.
+export function isNonceError(err: unknown): boolean {
+  let e: any = err;
+  for (let i = 0; e && i < 5; i++) {
+    const msg = e.details ?? e.shortMessage ?? e.message ?? '';
+    if (NONCE_ERROR_RE.test(String(msg))) return true;
+    e = e.cause;
+  }
+  return false;
+}
+
+// Defense-in-depth around a broadcast: runs the prepare+send closure and, on a nonce-class
+// error, resets the signer's viem nonce manager (so the next prepare re-reads the chain) and
+// retries with exponential backoff. The closure is responsible for (re-)preparing the
+// transaction each attempt so a fresh nonce is assigned. Non-nonce errors propagate immediately,
+// so the happy path is unchanged.
+export async function sendTransactionWithNonceRetry(
+  signer: CannonSigner,
+  chainId: number,
+  prepareAndSend: () => Promise<viem.Hash>
+): Promise<viem.Hash> {
+  return (await promiseRetry({ retries: 3, minTimeout: 250, factor: 2 }, (retry) =>
+    prepareAndSend().catch((err: unknown) => {
+      if (!isNonceError(err)) throw err;
+      const account: any = (signer.wallet as any).account;
+      try {
+        account?.nonceManager?.reset?.({ address: account.address, chainId });
+      } catch {
+        // best-effort: the nonce manager re-reads the chain on the next prepare regardless
+      }
+      return retry(err);
+    })
+  )) as viem.Hash;
+}

packages/builder/src/steps/deploy.ts

@@ -10,7 +10,7 @@ import { ChainArtifacts, ChainBuilderContext, ContractArtifact } from '../types'
 import { encodeDeployData, getContractDefinitionFromPath, getMergedAbiFromContractPaths } from '../util';
 import { template } from '../utils/template';
 import { CannonAction } from '../actions';
-import { getBlockRetried } from '../helpers';
+import { getBlockRetried, sendTransactionWithNonceRetry } from '../helpers';
 
 const debug = Debug('cannon:builder:deploy');
 
@@ -315,8 +315,10 @@ const deploySpec = {
           const fullCreate2Txn = _.assign(create2Txn, overrides, { account: signer.wallet.account || signer.address });
           debug('final create2 txn', fullCreate2Txn);
 
-          const preparedTxn = await runtime.provider.prepareTransactionRequest(fullCreate2Txn);
-          const hash = await signer.wallet.sendTransaction(preparedTxn as any);
+          const hash = await sendTransactionWithNonceRetry(signer, runtime.chainId, async () => {
+            const preparedTxn = await runtime.provider.prepareTransactionRequest(fullCreate2Txn);
+            return signer.wallet.sendTransaction(preparedTxn as any);
+          });
           receipt = await runtime.provider.waitForTransactionReceipt({ hash });
           debug('arachnid create2 complete', receipt);
         }
@@ -365,11 +367,12 @@ const deploySpec = {
               'The CREATE2 contract seems to be failing in the constructor. However, we were not able to get a stack trace.'
             );
           } else {
-            const preparedTxn = await runtime.provider.prepareTransactionRequest(
-              _.assign(txn, overrides, { account: signer.wallet.account || signer.address })
-            );
-
-            const hash = await signer.wallet.sendTransaction(preparedTxn as any);
+            const hash = await sendTransactionWithNonceRetry(signer, runtime.chainId, async () => {
+              const preparedTxn = await runtime.provider.prepareTransactionRequest(
+                _.assign(txn, overrides, { account: signer.wallet.account || signer.address })
+              );
+              return signer.wallet.sendTransaction(preparedTxn as any);
+            });
             receipt = await runtime.provider.waitForTransactionReceipt({ hash });
             deployAddress = receipt.contractAddress!;
           }

packages/builder/src/steps/diamond.ts

@@ -4,6 +4,7 @@ import * as viem from 'viem';
 import { z } from 'zod';
 import { ARACHNID_DEFAULT_DEPLOY_ADDR, ensureArachnidCreate2Exists, makeArachnidCreate2Txn } from '../create2';
 import { mergeTemplateAccesses } from '../access-recorder';
+import { sendTransactionWithNonceRetry } from '../helpers';
 import { ChainBuilderRuntime } from '../runtime';
 import { diamondSchema } from '../schemas';
 import { ContractArtifact, ContractMap, PackageState } from '../types';
@@ -170,7 +171,7 @@ const diamondStep = {
     ]);
 
     // finally, do the cut operation
-    const updateFacets = [];
+    const updateFacets: any[] = [];
 
     for (const facetName of deployedContracts) {
       updateFacets.push({
@@ -194,18 +195,19 @@ const diamondStep = {
 
     // todo: what to do about the owner of the proxy changing unexpectedly?
     try {
-      const preparedTxn = await runtime.provider.prepareTransactionRequest({
-        account: ownerSigner.wallet.account || ownerSigner.address,
-        to: proxyAddress,
-        data: viem.encodeFunctionData({
-          abi: (await import('../abis/diamond/DiamondWipeAndPaveFacet.json')).abi,
-          functionName: 'diamondWipeAndPave',
-          args: [updateFacets, config.diamondArgs.init, config.diamondArgs.initCalldata],
-        }),
-        ...config.overrides,
-      } as any);
-
-      const txn = await ownerSigner.wallet.sendTransaction(preparedTxn as any);
+      const txn = await sendTransactionWithNonceRetry(ownerSigner, runtime.chainId, async () => {
+        const preparedTxn = await runtime.provider.prepareTransactionRequest({
+          account: ownerSigner.wallet.account || ownerSigner.address,
+          to: proxyAddress,
+          data: viem.encodeFunctionData({
+            abi: (await import('../abis/diamond/DiamondWipeAndPaveFacet.json')).abi,
+            functionName: 'diamondWipeAndPave',
+            args: [updateFacets, config.diamondArgs.init, config.diamondArgs.initCalldata],
+          }),
+          ...config.overrides,
+        } as any);
+        return ownerSigner.wallet.sendTransaction(preparedTxn as any);
+      });
 
       const receipt = await runtime.provider.waitForTransactionReceipt({ hash: txn });
       debug('got receipt', receipt);
@@ -298,8 +300,8 @@ async function firstTimeDeploy(
     const bytecode = await runtime.provider.getCode({ address: addr });
 
     if (!bytecode) {
-      const hash = await signer.wallet.sendTransaction(
-        _.assign({ account: signer.wallet.account || signer.address }, create2Txn as any)
+      const hash = await sendTransactionWithNonceRetry(signer, runtime.chainId, async () =>
+        signer.wallet.sendTransaction(_.assign({ account: signer.wallet.account || signer.address }, create2Txn as any))
       );
       const receipt = await runtime.provider.waitForTransactionReceipt({ hash });
       const block = await runtime.provider.getBlock({ blockHash: receipt.blockHash });

packages/builder/src/steps/invoke.ts

@@ -21,7 +21,7 @@ import {
   getMergedAbiFromContractPaths,
 } from '../util';
 import { template, getTemplateMatches, isTemplateString } from '../utils/template';
-import { getBlockRetried } from '../helpers';
+import { getBlockRetried, sendTransactionWithNonceRetry } from '../helpers';
 import { isStepPath, isStepName } from '../utils/matchers';
 import { CannonAction } from '../actions';
 
@@ -142,23 +142,27 @@ async function runTxn(
 
     const callSigner = await runtime.getSigner(address);
 
-    const preparedTxn = await runtime.provider.prepareTransactionRequest({
-      account: callSigner.wallet.account || callSigner.address,
-      to: contract.address,
-      data: encodeFunctionData({ abi: [neededFuncAbi], functionName: neededFuncAbi.name, args: config.args }),
-      value: config.value,
-      ...overrides,
+    txn = await sendTransactionWithNonceRetry(callSigner, runtime.chainId, async () => {
+      const preparedTxn = await runtime.provider.prepareTransactionRequest({
+        account: callSigner.wallet.account || callSigner.address,
+        to: contract.address,
+        data: encodeFunctionData({ abi: [neededFuncAbi], functionName: neededFuncAbi.name, args: config.args }),
+        value: config.value,
+        ...overrides,
+      });
+      return callSigner.wallet.sendTransaction(preparedTxn as any);
     });
-    txn = await callSigner.wallet.sendTransaction(preparedTxn as any);
   } else {
-    const preparedTxn = await runtime.provider.prepareTransactionRequest({
-      account: signer.wallet.account || signer.address,
-      to: contract.address,
-      data: encodeFunctionData({ abi: [neededFuncAbi], functionName: neededFuncAbi.name, args: config.args }),
-      value: config.value,
-      ...overrides,
+    txn = await sendTransactionWithNonceRetry(signer, runtime.chainId, async () => {
+      const preparedTxn = await runtime.provider.prepareTransactionRequest({
+        account: signer.wallet.account || signer.address,
+        to: contract.address,
+        data: encodeFunctionData({ abi: [neededFuncAbi], functionName: neededFuncAbi.name, args: config.args }),
+        value: config.value,
+        ...overrides,
+      });
+      return signer.wallet.sendTransaction(preparedTxn as any);
     });
-    txn = await signer.wallet.sendTransaction(preparedTxn as any);
   }
 
   const receipt = await runtime.provider.waitForTransactionReceipt({ hash: txn });

packages/builder/src/steps/router.ts

@@ -8,7 +8,7 @@ import { CannonError } from '../error';
 import { mergeTemplateAccesses } from '../access-recorder';
 import { routerSchema } from '../schemas';
 import { ContractMap } from '../types';
-import { getBlockRetried } from '../helpers';
+import { getBlockRetried, sendTransactionWithNonceRetry } from '../helpers';
 import {
   encodeDeployData,
   getContractDefinitionFromPath,
@@ -279,9 +279,10 @@ const routerStep = {
         const fullCreate2Txn = _.assign(create2Txn, overrides, { account: signer.wallet.account || signer.address });
         debug('final create2 txn', fullCreate2Txn);
 
-        const preparedTxn = await runtime.provider.prepareTransactionRequest(fullCreate2Txn);
-
-        const hash = await signer.wallet.sendTransaction(preparedTxn as any);
+        const hash = await sendTransactionWithNonceRetry(signer, runtime.chainId, async () => {
+          const preparedTxn = await runtime.provider.prepareTransactionRequest(fullCreate2Txn);
+          return signer.wallet.sendTransaction(preparedTxn as any);
+        });
         receipt = await runtime.provider.waitForTransactionReceipt({ hash });
         debug('arachnid create2 complete', receipt);
       }
@@ -291,11 +292,12 @@ const routerStep = {
         : await runtime.getDefaultSigner!(txn, config.salt);
       debug('using deploy signer with address', signer.address);
 
-      const preparedTxn = await signer.wallet.prepareTransactionRequest(
-        _.assign(txn, overrides, { account: signer.wallet.account || signer.address })
-      );
-
-      const hash = await signer.wallet.sendTransaction(preparedTxn as any);
+      const hash = await sendTransactionWithNonceRetry(signer, runtime.chainId, async () => {
+        const preparedTxn = await signer.wallet.prepareTransactionRequest(
+          _.assign(txn, overrides, { account: signer.wallet.account || signer.address })
+        );
+        return signer.wallet.sendTransaction(preparedTxn as any);
+      });
       receipt = await runtime.provider.waitForTransactionReceipt({ hash });
       deployAddress = receipt.contractAddress!;
     }