Skip to content

fix: optimize Block::from_filecoin_tipset #7062

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hanabi1224 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: optimize Block::from_filecoin_tipset #7062

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/beacon/drand.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,7 @@ impl DrandBeacon {
fil_round_time: interval,
fil_gen_time: genesis_ts,
verified_beacons: SizeTrackingLruCache::new_with_metrics(
"verified_beacons".into(),
"verified_beacons",
CACHE_SIZE,
),
}
Expand Down
2 changes: 1 addition & 1 deletion src/chain/store/chain_store.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -596,7 +596,7 @@ pub struct MessagesInTipsetCache {
impl MessagesInTipsetCache {
pub fn new(capacity: NonZeroUsize) -> Self {
Self {
cache: SizeTrackingLruCache::new_with_metrics("msg_in_tipset".into(), capacity),
cache: SizeTrackingLruCache::new_with_metrics("msg_in_tipset", capacity),
}
}

Expand Down
3 changes: 1 addition & 2 deletions src/chain/store/index.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,7 @@ pub enum ResolveNullTipset {
impl ChainIndex {
pub fn new(db: impl Into<DbImpl>) -> Self {
let db = db.into();
let ts_cache =
SizeTrackingLruCache::new_with_metrics("tipset".into(), DEFAULT_TIPSET_CACHE_SIZE);
let ts_cache = SizeTrackingLruCache::new_with_metrics("tipset", DEFAULT_TIPSET_CACHE_SIZE);
Self {
ts_cache,
db,
Expand Down
4 changes: 2 additions & 2 deletions src/chain_sync/bad_block_cache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ impl ShallowClone for BadBlockCache {
impl BadBlockCache {
pub fn new(cap: NonZeroUsize) -> Self {
Self {
cache: SizeTrackingLruCache::new_with_metrics("bad_block".into(), cap),
cache: SizeTrackingLruCache::new_with_metrics("bad_block", cap),
}
}

Expand Down Expand Up @@ -81,7 +81,7 @@ impl Default for SeenBlockCache {
impl SeenBlockCache {
pub fn new(cap: NonZeroUsize) -> Self {
Self {
cache: SizeTrackingLruCache::new_with_metrics("seen_gossip_block".into(), cap),
cache: SizeTrackingLruCache::new_with_metrics("seen_gossip_block", cap),
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/db/blockstore_with_read_cache.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ mod tests {
records.push((key, record));
}
let cache = Arc::new(LruBlockstoreReadCache::new_without_metrics_registry(
"test_blockstore_read_cache".into(),
"test_blockstore_read_cache",
CACHE_SIZE.try_into().unwrap(),
));
let db = BlockstoreWithReadCache::new(
Expand Down
2 changes: 1 addition & 1 deletion src/db/car/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ impl ZstdFrameCache {
ZstdFrameCache {
max_size,
current_size: Arc::new(AtomicUsize::new(0)),
lru: SizeTrackingLruCache::unbounded_with_metrics("zstd_frame".into()),
lru: SizeTrackingLruCache::unbounded_with_metrics("zstd_frame"),
}
}

Expand Down
8 changes: 4 additions & 4 deletions src/message_pool/msgpool/msg_pool.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,11 @@ pub(in crate::message_pool) struct Caches {
impl Caches {
pub(in crate::message_pool) fn new() -> Self {
Self {
bls_sig: SizeTrackingLruCache::new_with_metrics("bls_sig".into(), BLS_SIG_CACHE_SIZE),
sig_val: SizeTrackingLruCache::new_with_metrics("sig_val".into(), SIG_VAL_CACHE_SIZE),
key: SizeTrackingLruCache::new_with_metrics("mpool_key".into(), KEY_CACHE_SIZE),
bls_sig: SizeTrackingLruCache::new_with_metrics("bls_sig", BLS_SIG_CACHE_SIZE),
sig_val: SizeTrackingLruCache::new_with_metrics("sig_val", SIG_VAL_CACHE_SIZE),
key: SizeTrackingLruCache::new_with_metrics("mpool_key", KEY_CACHE_SIZE),
state_nonce: SizeTrackingLruCache::new_with_metrics(
"state_nonce".into(),
"state_nonce",
STATE_NONCE_CACHE_SIZE,
),
}
Expand Down
4 changes: 2 additions & 2 deletions src/rpc/methods/chain.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ use crate::lotus_json::{HasLotusJson, LotusJson, lotus_json_with_self};
use crate::lotus_json::{assert_all_snapshots, assert_unchanged_via_json};
use crate::message::{ChainMessage, SignedMessage};
use crate::prelude::*;
use crate::rpc::eth::Block as EthBlock;
use crate::rpc::eth::{
EthLog, TxInfo, eth_logs_with_filter, types::ApiHeaders, types::EthFilterSpec,
Block as EthBlock, EthLog, TxInfo, eth_logs_with_filter, types::ApiHeaders,
types::EthFilterSpec,
};
use crate::rpc::f3::F3ExportLatestSnapshot;
use crate::rpc::types::*;
Expand Down
192 changes: 111 additions & 81 deletions src/rpc/methods/eth.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ use crate::shim::gas::GasOutputs;
use crate::shim::message::Message;
use crate::shim::trace::{CallReturn, ExecutionEvent};
use crate::shim::{clock::ChainEpoch, state_tree::StateTree};
use crate::state_manager::cache::ForestLruCache;
use crate::state_manager::{ExecutedMessage, ExecutedTipset, TipsetState, VMFlush};
use crate::utils::cache::SizeTrackingLruCache;
use crate::utils::db::BlockstoreExt as _;
use crate::utils::encoding::from_slice_with_fallback;
use crate::utils::get_size::{CidWrapper, big_int_heap_size_helper};
Expand Down Expand Up @@ -477,95 +477,125 @@ impl Block {
ctx: Ctx,
tipset: crate::blocks::Tipset,
tx_info: TxInfo,
) -> Result<Self> {
static ETH_BLOCK_CACHE: LazyLock<SizeTrackingLruCache<CidWrapper, Block>> =
) -> Result<Arc<Self>> {
static ETH_BLOCK_HASH_TX_CACHE: LazyLock<ForestLruCache<CidWrapper, Arc<Block>>> =
LazyLock::new(|| {
const DEFAULT_CACHE_SIZE: NonZeroUsize = nonzero!(500usize);
let cache_size = std::env::var("FOREST_ETH_BLOCK_CACHE_SIZE")
Copy link
Copy Markdown
Member

@LesnyRumcajs LesnyRumcajs May 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this must be removed from the docs, no?

Copy link
Copy Markdown
Member

@LesnyRumcajs LesnyRumcajs May 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah no, I see it's now used for full and hash.

.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(DEFAULT_CACHE_SIZE);
SizeTrackingLruCache::new_with_metrics("eth_block".into(), cache_size)
ForestLruCache::with_size("eth_block_hash_tx", Block::block_cache_size())
});
Comment on lines +481 to 484
Copy link
Copy Markdown
Contributor

@coderabbitai coderabbitai Bot May 15, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major | ⚡ Quick win

FOREST_ETH_BLOCK_CACHE_SIZE now budgets two independent caches.

Both eth_block_hash_tx and eth_block_full_tx are initialized with Block::block_cache_size(), so a setting of N now allows up to 2N cached blocks. On this path that can materially increase resident memory compared to the previous single-cache budget.

Please either split the existing budget across the two caches or introduce separate knobs with explicit names.

Also applies to: 508-510, 582-590

🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@src/rpc/methods/eth.rs` around lines 481 - 484, The two caches
ETH_BLOCK_HASH_TX_CACHE and ETH_BLOCK_FULL_TX_CACHE are both using
Block::block_cache_size(), doubling the intended budget; update initialization
to either split the existing FOREST_ETH_BLOCK_CACHE_SIZE between them (e.g., use
Block::block_cache_size() / 2 for each) or introduce separate configuration
knobs (e.g., FOREST_ETH_BLOCK_HASH_CACHE_SIZE and
FOREST_ETH_BLOCK_FULL_CACHE_SIZE) and use those values when constructing
ForestLruCache in ETH_BLOCK_HASH_TX_CACHE and ETH_BLOCK_FULL_TX_CACHE (and the
other similar cache inits around the same file) so the total memory matches the
intended budget and each cache uses the correct size parameter.


let block_cid = tipset.key().cid()?;
let mut block = if let Some(b) = ETH_BLOCK_CACHE.get_cloned(&block_cid.into()) {
b
} else {
let parent_cid = tipset.parents().cid()?;
let block_number = EthInt64(tipset.epoch());
let block_hash: EthHash = block_cid.into();
match tx_info {
TxInfo::Full => Self::from_filecoin_tipset_with_full_tx(ctx, tipset).await,
TxInfo::Hash => {
let block_cid = tipset.key().cid()?;
ETH_BLOCK_HASH_TX_CACHE
.get_or_else(&block_cid.into(), async move || {
let block_with_full_tx =
Self::from_filecoin_tipset_with_full_tx(ctx, tipset).await?;
Ok(Arc::new(
Arc::unwrap_or_clone(block_with_full_tx)
.downcast_full_transaction_to_hash(),
))
})
.await
}
}
}

let ExecutedTipset {
state_root,
executed_messages,
..
} = ctx.state_manager.load_executed_tipset(&tipset).await?;
let has_transactions = !executed_messages.is_empty();
let state_tree = ctx.state_manager.get_state_tree(&state_root)?;

let mut full_transactions = vec![];
let mut gas_used = 0;
for (
i,
ExecutedMessage {
message, receipt, ..
},
) in executed_messages.iter().enumerate()
{
let ti = EthUint64(i as u64);
gas_used += receipt.gas_used();
let mut tx = match message {
ChainMessage::Signed(smsg) => new_eth_tx_from_signed_message(
smsg,
&state_tree,
ctx.chain_config().eth_chain_id,
)?,
ChainMessage::Unsigned(msg) => {
let tx = eth_tx_from_native_message(
msg,
async fn from_filecoin_tipset_with_full_tx(
ctx: Ctx,
tipset: crate::blocks::Tipset,
) -> Result<Arc<Self>> {
static ETH_BLOCK_FULL_TX_CACHE: LazyLock<ForestLruCache<CidWrapper, Arc<Block>>> =
LazyLock::new(|| {
ForestLruCache::with_size("eth_block_full_tx", Block::block_cache_size())
});

let block_cid = tipset.key().cid()?;
ETH_BLOCK_FULL_TX_CACHE
.get_or_else(&block_cid.into(), async move || {
let parent_cid = tipset.parents().cid()?;
let block_number = EthInt64(tipset.epoch());
let block_hash: EthHash = block_cid.into();

let ExecutedTipset {
state_root,
executed_messages,
..
} = ctx.state_manager.load_executed_tipset(&tipset).await?;
let has_transactions = !executed_messages.is_empty();
let state_tree = ctx.state_manager.get_state_tree(&state_root)?;

let mut full_transactions = vec![];
let mut gas_used = 0;
for (
i,
ExecutedMessage {
message, receipt, ..
},
) in executed_messages.iter().enumerate()
{
let ti = EthUint64(i as u64);
gas_used += receipt.gas_used();
let mut tx = match message {
ChainMessage::Signed(smsg) => new_eth_tx_from_signed_message(
smsg,
&state_tree,
ctx.chain_config().eth_chain_id,
)?;
ApiEthTx {
hash: msg.cid().into(),
..tx
)?,
ChainMessage::Unsigned(msg) => {
let tx = eth_tx_from_native_message(
msg,
&state_tree,
ctx.chain_config().eth_chain_id,
)?;
ApiEthTx {
hash: msg.cid().into(),
..tx
}
}
}
};
tx.block_hash = block_hash;
tx.block_number = block_number;
tx.transaction_index = ti;
full_transactions.push(tx);
}
};
tx.block_hash = block_hash;
tx.block_number = block_number;
tx.transaction_index = ti;
full_transactions.push(tx);
}

let b = Block {
hash: block_hash,
number: block_number,
parent_hash: parent_cid.into(),
timestamp: EthUint64(tipset.block_headers().first().timestamp),
base_fee_per_gas: tipset
.block_headers()
.first()
.parent_base_fee
.clone()
.into(),
gas_used: EthUint64(gas_used),
transactions: Transactions::Full(full_transactions),
..Block::new(has_transactions, tipset.len())
};
ETH_BLOCK_CACHE.push(block_cid.into(), b.clone());
b
};
Ok(Arc::new(Block {
hash: block_hash,
number: block_number,
parent_hash: parent_cid.into(),
timestamp: EthUint64(tipset.block_headers().first().timestamp),
base_fee_per_gas: tipset
.block_headers()
.first()
.parent_base_fee
.clone()
.into(),
gas_used: EthUint64(gas_used),
transactions: Transactions::Full(full_transactions),
..Block::new(has_transactions, tipset.len())
}))
})
.await
}

if tx_info == TxInfo::Hash
&& let Transactions::Full(transactions) = &block.transactions
{
block.transactions =
fn block_cache_size() -> NonZeroUsize {
const DEFAULT_CACHE_SIZE: NonZeroUsize = nonzero!(500usize);
static CACHE_SIZE: std::sync::LazyLock<NonZeroUsize> = std::sync::LazyLock::new(|| {
std::env::var("FOREST_ETH_BLOCK_CACHE_SIZE")
.ok()
.and_then(|s| s.parse().ok())
.unwrap_or(DEFAULT_CACHE_SIZE)
});
*CACHE_SIZE
}

fn downcast_full_transaction_to_hash(mut self) -> Self {
if let Transactions::Full(transactions) = &self.transactions {
self.transactions =
Transactions::Hash(transactions.iter().map(|tx| tx.hash.to_string()).collect())
}

Ok(block)
self
}
}

Expand Down Expand Up @@ -1396,7 +1426,7 @@ impl RpcMethod<2> for EthGetBlockByHash {
const PERMISSION: Permission = Permission::Read;

type Params = (EthHash, bool);
type Ok = Block;
type Ok = Arc<Block>;

async fn handle(
ctx: Ctx,
Expand Down Expand Up @@ -1424,7 +1454,7 @@ impl RpcMethod<2> for EthGetBlockByNumber {
Some("Retrieves a block by its number or a special tag.");

type Params = (BlockNumberOrPredefined, bool);
type Ok = Block;
type Ok = Arc<Block>;

async fn handle(
ctx: Ctx,
Expand Down
2 changes: 1 addition & 1 deletion src/rpc/methods/eth/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -453,7 +453,7 @@ pub struct FilterID(EthHash);
lotus_json_with_self!(FilterID);

#[derive(Clone, Serialize, Deserialize, PartialEq, Debug)]
pub struct ApiHeaders(#[serde(with = "crate::lotus_json")] pub Block);
pub struct ApiHeaders(#[serde(with = "crate::lotus_json")] pub Arc<Block>);

lotus_json_with_self!(ApiHeaders);

Expand Down
2 changes: 1 addition & 1 deletion src/rpc/methods/f3.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ impl GetPowerTable {
// The RAM overhead on mainnet is ~14MiB
const BLOCKSTORE_CACHE_CAP: NonZeroUsize = nonzero!(65536_usize);
static BLOCKSTORE_CACHE: LazyLock<LruBlockstoreReadCache> = LazyLock::new(|| {
LruBlockstoreReadCache::new_with_metrics("get_powertable".into(), BLOCKSTORE_CACHE_CAP)
LruBlockstoreReadCache::new_with_metrics("get_powertable", BLOCKSTORE_CACHE_CAP)
});
let db = BlockstoreWithReadCache::new(
ctx.db_owned(),
Expand Down
Loading
Loading