diff --git a/crates/cold/src/traits.rs b/crates/cold/src/traits.rs index c0d8994..8da36b7 100644 --- a/crates/cold/src/traits.rs +++ b/crates/cold/src/traits.rs @@ -71,13 +71,20 @@ impl BlockData { impl From for BlockData { fn from(block: ExecutedBlock) -> Self { - Self::new( - block.header, - block.transactions, - block.receipts, - block.signet_events, - block.zenith_header, - ) + // Destructure so adding a new `ExecutedBlock` field is a compile + // error here until the author decides whether it belongs in cold. + // `bundle` and `journal_hash` are hot-only: they live in + // `PlainAccountState`/`PlainStorageState` and `JournalHashes`. + let ExecutedBlock { + header, + transactions, + receipts, + signet_events, + zenith_header, + bundle: _, + journal_hash: _, + } = block; + Self::new(header, transactions, receipts, signet_events, zenith_header) } } diff --git a/crates/hot-mdbx/src/db_info.rs b/crates/hot-mdbx/src/db_info.rs index 1092b7f..49e9fff 100644 --- a/crates/hot-mdbx/src/db_info.rs +++ b/crates/hot-mdbx/src/db_info.rs @@ -108,6 +108,26 @@ impl FixedSizeInfo { _ => None, } } + + /// Canonical mapping from the `(dual_key, fixed_val)` size hints accepted + /// by [`queue_raw_create`] to the [`FixedSizeInfo`] implied by them. + /// Used both when persisting FSI for a newly-created table and when + /// pre-populating the open-time cache with a compile-time fallback for + /// tables that pre-date their on-disk metadata entry. + /// + /// [`queue_raw_create`]: signet_hot::model::HotKvWrite::queue_raw_create + pub(crate) const fn from_create_args( + dual_key: Option, + fixed_val: Option, + ) -> Self { + match (dual_key, fixed_val) { + (Some(key2_size), Some(value_size)) => { + Self::DupFixed { key2_size, total_size: key2_size + value_size } + } + (Some(key2_size), None) => Self::DupSort { key2_size }, + (None, _) => Self::None, + } + } } impl ValSer for FixedSizeInfo { @@ -208,6 +228,7 @@ mod tests { ("TableG", FixedSizeInfo::None), ("TableH", FixedSizeInfo::None), ("TableI", FixedSizeInfo::None), + ("TableJ", FixedSizeInfo::None), ]; let cache = FsiCache::new(known); @@ -233,6 +254,7 @@ mod tests { ("T7", FixedSizeInfo::None), ("T8", FixedSizeInfo::None), ("T9", FixedSizeInfo::None), + ("T10", FixedSizeInfo::None), ]; let cache = FsiCache::new(known); diff --git a/crates/hot-mdbx/src/lib.rs b/crates/hot-mdbx/src/lib.rs index 9349604..5ced2dc 100644 --- a/crates/hot-mdbx/src/lib.rs +++ b/crates/hot-mdbx/src/lib.rs @@ -80,25 +80,9 @@ mod utils; use signet_hot::{ model::{HotKv, HotKvError, HotKvWrite}, - tables::{ - AccountChangeSets, AccountsHistory, Bytecodes, HeaderNumbers, Headers, NUM_TABLES, - PlainAccountState, PlainStorageState, StorageChangeSets, StorageHistory, Table, - }, + tables::{NUM_TABLES, STANDARD_TABLES}, }; -/// The known table names, used to pre-populate the FSI cache at open time. -const KNOWN_TABLE_NAMES: [&str; NUM_TABLES] = [ - Headers::NAME, - HeaderNumbers::NAME, - Bytecodes::NAME, - PlainAccountState::NAME, - PlainStorageState::NAME, - AccountsHistory::NAME, - AccountChangeSets::NAME, - StorageHistory::NAME, - StorageChangeSets::NAME, -]; - /// 1 KB in bytes pub const KILOBYTE: usize = 1024; /// 1 MB in bytes @@ -458,13 +442,41 @@ fn create_tables_and_populate_cache(env: &Environment) -> Result( tx: &Tx, ) -> Result<[(&'static str, FixedSizeInfo); NUM_TABLES], MdbxError> { let mut known = [("", FixedSizeInfo::None); NUM_TABLES]; - for (i, &name) in KNOWN_TABLE_NAMES.iter().enumerate() { - known[i] = (name, tx.read_fsi_from_table(name)?); + for (index, table) in STANDARD_TABLES.iter().enumerate() { + let fsi = match tx.read_fsi_from_table(table.name) { + Ok(fsi) => fsi, + Err(MdbxError::UnknownTable(_)) => { + let expected = + FixedSizeInfo::from_create_args(table.dual_key_size, table.fixed_val_size); + tracing::warn!( + target: "storage::db::mdbx", + table = table.name, + ?expected, + "FSI metadata entry missing for known table; falling back to compile-time \ + expected value. Fires once per open per missing table until a RW open \ + creates the table and persists its FSI; RO-only consumers of a \ + pre-upgrade database will see this on every open." + ); + expected + } + Err(error) => return Err(error), + }; + known[index] = (table.name, fsi); } Ok(known) } diff --git a/crates/hot-mdbx/src/test_utils.rs b/crates/hot-mdbx/src/test_utils.rs index 91d93c5..10d22b2 100644 --- a/crates/hot-mdbx/src/test_utils.rs +++ b/crates/hot-mdbx/src/test_utils.rs @@ -54,6 +54,7 @@ mod tests { use signet_hot::{ KeySer, MAX_KEY_SIZE, ValSer, conformance::{conformance, test_unwind_conformance}, + db::{HotDbRead, UnsafeDbWrite}, model::{ DualKeyTraverse, DualTableTraverse, HotKv, HotKvRead, HotKvWrite, TableTraverse, TableTraverseMut, @@ -1968,6 +1969,52 @@ mod tests { } } + /// Regression test: opening a database written by an older binary that + /// pre-dates a known table must succeed in both RO and RW modes, and a + /// subsequent RW open must re-create the missing table and persist its + /// FSI normally. + #[test] + #[serial] + fn open_tolerates_pre_upgrade_db() { + let dir = tempdir().unwrap(); + + // Phase 1: open RW (auto-creates all tables and FSIs), then forget + // JournalHashes entirely to mimic a pre-upgrade DB. + { + let args = DatabaseArguments::new(); + let db = DatabaseEnv::open(dir.path(), DatabaseEnvKind::RW, args).unwrap(); + let writer: Tx = db.tx_rw().unwrap(); + // SAFETY: no Cursor or Database references to JournalHashes + // exist in this scope. + unsafe { + writer.forget_table(tables::JournalHashes::NAME).unwrap(); + } + writer.raw_commit().unwrap(); + } + + // Phase 2: reopening RO must succeed despite the missing table. + { + let args = DatabaseArguments::new(); + DatabaseEnv::open(dir.path(), DatabaseEnvKind::RO, args) + .expect("RO open must tolerate a pre-upgrade DB"); + } + + // Phase 3: the next RW open must re-create the table and FSI, and + // subsequent reads and writes against it must work normally. + { + let args = DatabaseArguments::new(); + let db = DatabaseEnv::open(dir.path(), DatabaseEnvKind::RW, args) + .expect("RW open must tolerate a pre-upgrade DB"); + let hash = B256::repeat_byte(0x42); + let writer: Tx = db.tx_rw().unwrap(); + writer.put_journal_hash(7, &hash).unwrap(); + writer.raw_commit().unwrap(); + + let reader: Tx = db.reader().unwrap(); + assert_eq!(reader.get_journal_hash(7).unwrap(), Some(hash)); + } + } + /// Regression test: FSI must survive a database close/reopen cycle. /// /// This exercises the `store_fsi` / `read_fsi_from_table` path with an empty cache. diff --git a/crates/hot-mdbx/src/tx.rs b/crates/hot-mdbx/src/tx.rs index e62415f..c8c0d44 100644 --- a/crates/hot-mdbx/src/tx.rs +++ b/crates/hot-mdbx/src/tx.rs @@ -142,6 +142,28 @@ impl Tx { Ok(()) } + + /// Removes a table from the environment as if it had never been created: + /// drops the named sub-database and erases its FSI metadata entry. Used + /// by tests to simulate a database written by an older binary that + /// pre-dates the table. Does not invalidate the in-memory `FsiCache`; + /// callers must reopen the parent `DatabaseEnv` after commit. + /// + /// # Safety + /// + /// Caller must ensure no [`Cursor`] or other references to `table` + /// exist for the lifetime of this transaction. See + /// [`signet_libmdbx::tx::Tx::drop_db`]. + #[cfg(test)] + pub(crate) unsafe fn forget_table(&self, table: &'static str) -> Result<(), MdbxError> { + let metadata = self.inner.open_db(None)?; + self.inner + .del(metadata, fsi_name_to_key(table).as_slice(), None) + .map_err(MdbxError::Mdbx)?; + let db = self.inner.open_db(Some(table))?; + // SAFETY: forwarded from this function's safety contract. + unsafe { self.inner.drop_db(db) }.map_err(MdbxError::Mdbx) + } } fn fsi_name_to_key(name: &'static str) -> B256 { @@ -288,22 +310,14 @@ macro_rules! impl_hot_kv_write { dual_key: Option, fixed_val: Option, ) -> Result<(), Self::Error> { - let mut flags = signet_libmdbx::DatabaseFlags::default(); + let fsi = FixedSizeInfo::from_create_args(dual_key, fixed_val); - let mut fsi = FixedSizeInfo::None; - - if let Some(key2_size) = dual_key { + let mut flags = signet_libmdbx::DatabaseFlags::default(); + if fsi.is_dupsort() { flags.set(signet_libmdbx::DatabaseFlags::DUP_SORT, true); - if let Some(value_size) = fixed_val { - flags.set(signet_libmdbx::DatabaseFlags::DUP_FIXED, true); - fsi = FixedSizeInfo::DupFixed { - key2_size, - total_size: key2_size + value_size, - }; - } else { - // DUPSORT without DUP_FIXED - variable value size - fsi = FixedSizeInfo::DupSort { key2_size }; - } + } + if fsi.is_dup_fixed() { + flags.set(signet_libmdbx::DatabaseFlags::DUP_FIXED, true); } self.inner.create_db(Some(table), flags)?; diff --git a/crates/hot/src/conformance/mod.rs b/crates/hot/src/conformance/mod.rs index 2639865..4fb40ba 100644 --- a/crates/hot/src/conformance/mod.rs +++ b/crates/hot/src/conformance/mod.rs @@ -29,6 +29,7 @@ pub fn conformance(hot_kv: &T) { test_storage_history(hot_kv); test_account_changes(hot_kv); test_storage_changes(hot_kv); + test_journal_hash_roundtrip(hot_kv); test_missing_reads(hot_kv); test_cursor_iter_k2(hot_kv); test_cursor_iter_k2_single(hot_kv); diff --git a/crates/hot/src/conformance/roundtrip.rs b/crates/hot/src/conformance/roundtrip.rs index cba2c76..4795f46 100644 --- a/crates/hot/src/conformance/roundtrip.rs +++ b/crates/hot/src/conformance/roundtrip.rs @@ -251,6 +251,42 @@ pub fn test_storage_changes(hot_kv: &T) { } } +/// Test writing, reading, and overwriting journal hashes. +pub fn test_journal_hash_roundtrip(hot_kv: &T) { + let hash_a = b256!("0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + let hash_b = b256!("0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + let hash_c = b256!("0xcccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"); + + // Write hashes at two block numbers. + { + let writer = hot_kv.writer().unwrap(); + writer.put_journal_hash(7, &hash_a).unwrap(); + writer.put_journal_hash(8, &hash_b).unwrap(); + writer.commit().unwrap(); + } + + // Read back. + { + let reader = hot_kv.reader().unwrap(); + assert_eq!(reader.get_journal_hash(7).unwrap(), Some(hash_a)); + assert_eq!(reader.get_journal_hash(8).unwrap(), Some(hash_b)); + assert_eq!(reader.get_journal_hash(9).unwrap(), None); + } + + // Overwrite block 7 - producers retrying a block must be able to replace + // the previous entry. + { + let writer = hot_kv.writer().unwrap(); + writer.put_journal_hash(7, &hash_c).unwrap(); + writer.commit().unwrap(); + } + { + let reader = hot_kv.reader().unwrap(); + assert_eq!(reader.get_journal_hash(7).unwrap(), Some(hash_c)); + assert_eq!(reader.get_journal_hash(8).unwrap(), Some(hash_b)); + } +} + /// Test that missing reads return None pub fn test_missing_reads(hot_kv: &T) { let missing_addr = address!("0x9999999999999999999999999999999999999999"); @@ -288,4 +324,7 @@ pub fn test_missing_reads(hot_kv: &T) { // Missing storage change assert!(reader.get_storage_change(999999, &missing_addr, &missing_slot).unwrap().is_none()); + + // Missing journal hash + assert!(reader.get_journal_hash(999999).unwrap().is_none()); } diff --git a/crates/hot/src/conformance/unwind.rs b/crates/hot/src/conformance/unwind.rs index 3471c6d..a0ad872 100644 --- a/crates/hot/src/conformance/unwind.rs +++ b/crates/hot/src/conformance/unwind.rs @@ -228,6 +228,7 @@ pub fn make_account_info(nonce: u64, balance: U256, code_hash: Option) -> /// - Headers and header number mappings /// - Account and storage change sets /// - Account and storage history indices +/// - Journal hashes pub fn test_unwind_conformance(store_a: &Kv, store_b: &Kv) { // Test addresses let addr1 = address!("0x1111111111111111111111111111111111111111"); @@ -412,10 +413,21 @@ pub fn test_unwind_conformance(store_a: &Kv, store_b: &Kv) { blocks.push((sealed, bundle)); } - // Store A: Append all 5 blocks, then unwind to block 1 + // Journal hashes, one per block; verifies unwind_above also drops these. + let journal_hashes: Vec = (0..blocks.len()) + .map(|index| { + let byte = u8::try_from(index + 1).expect("test block count fits in u8"); + B256::from([byte; 32]) + }) + .collect(); + + // Store A: Append all 5 blocks plus journal hashes, then unwind to block 1 { let writer = store_a.writer().unwrap(); writer.append_blocks(blocks.iter().map(|(h, b)| (h, b))).unwrap(); + for ((header, _), hash) in blocks.iter().zip(&journal_hashes) { + writer.put_journal_hash(header.number, hash).unwrap(); + } writer.commit().unwrap(); } { @@ -424,10 +436,13 @@ pub fn test_unwind_conformance(store_a: &Kv, store_b: &Kv) { writer.commit().unwrap(); } - // Store B: Append only blocks 0, 1 + // Store B: Append only blocks 0, 1 plus their journal hashes { let writer = store_b.writer().unwrap(); writer.append_blocks(blocks[0..2].iter().map(|(h, b)| (h, b))).unwrap(); + for ((header, _), hash) in blocks[0..2].iter().zip(&journal_hashes[0..2]) { + writer.put_journal_hash(header.number, hash).unwrap(); + } writer.commit().unwrap(); } @@ -454,6 +469,12 @@ pub fn test_unwind_conformance(store_a: &Kv, store_b: &Kv) { collect_single_table::(&reader_b), ); + assert_single_tables_equal::( + "JournalHashes", + collect_single_table::(&reader_a), + collect_single_table::(&reader_b), + ); + // Note: Bytecodes are not removed on unwind (they're content-addressed), // so store_a may have more bytecodes than store_b. We skip this comparison. // assert_single_tables_equal::(...) diff --git a/crates/hot/src/db/consistent.rs b/crates/hot/src/db/consistent.rs index c180d65..17664e2 100644 --- a/crates/hot/src/db/consistent.rs +++ b/crates/hot/src/db/consistent.rs @@ -148,8 +148,22 @@ pub trait HistoryWrite: UnsafeDbWrite + UnsafeHistoryWrite { /// - Headers and header number mappings /// - Account and storage change sets /// - Account and storage history indices + /// - Journal hashes fn unwind_above(&self, block: BlockNumber) -> Result<(), HistoryError> { - let first_block = block + 1; + // Nothing can sit above `u64::MAX`; bail out before any of the + // `block + 1` arithmetic below has a chance to overflow. + let Some(first_block) = block.checked_add(1) else { + return Ok(()); + }; + + // Clean journal hashes independently of the Headers table. Direct + // callers of `put_journal_hash` are not forced to pair writes with a + // header, so the upper bound here is `u64::MAX` rather than + // `last_block_number()`. The range delete is a no-op when the table + // has no entries above `block`. + self.traverse_mut::()? + .delete_range_inclusive(first_block..=u64::MAX)?; + let Some(last_block) = self.last_block_number()? else { return Ok(()); }; @@ -345,3 +359,78 @@ pub trait HistoryWrite: UnsafeDbWrite + UnsafeHistoryWrite { } impl HistoryWrite for T where T: UnsafeDbWrite + UnsafeHistoryWrite {} + +#[cfg(test)] +mod tests { + use super::HistoryWrite; + use crate::{ + db::{HistoryRead, HotDbRead, UnsafeDbWrite, UnsafeHistoryWrite}, + mem::MemKv, + model::HotKv, + }; + use alloy::{ + consensus::{Header, Sealable}, + primitives::{Address, B256, U256}, + }; + use signet_storage_types::Account; + + /// Regression: `unwind_above(u64::MAX)` must be a no-op rather than + /// overflowing `block + 1` and wiping data in `JournalHashes`, + /// `Headers`/`HeaderNumbers`, and the change-set tables (which the + /// wrapped `first_block = 0` would have caused to be range-deleted). + #[test] + fn unwind_above_u64_max_is_noop() { + let store = MemKv::new(); + let hash = B256::with_last_byte(0x42); + let header = Header { number: 7, ..Default::default() }.seal_slow(); + let header_hash = header.hash(); + let address = Address::with_last_byte(0xab); + let account = Account { nonce: 1, balance: U256::from(99), bytecode_hash: None }; + + let writer = store.writer().unwrap(); + writer.put_header(&header).unwrap(); + writer.write_account_prestate(7, address, &account).unwrap(); + writer.put_journal_hash(7, &hash).unwrap(); + writer.commit().unwrap(); + + let writer = store.writer().unwrap(); + writer.unwind_above(u64::MAX).unwrap(); + writer.commit().unwrap(); + + let reader = store.reader().unwrap(); + assert_eq!(reader.get_journal_hash(7).unwrap(), Some(hash)); + assert_eq!(reader.get_header(7).unwrap().expect("header survives").number, 7); + assert_eq!(reader.get_header_number(&header_hash).unwrap(), Some(7)); + assert_eq!(reader.get_account_change(7, &address).unwrap(), Some(account)); + } + + /// Boundary: `unwind_above(u64::MAX - 1)` must delete every table's + /// entry at `u64::MAX`. Exercises the inclusive upper bound of the range + /// delete on `JournalHashes`, `Headers`/`HeaderNumbers`, and the change + /// sets at the extreme. + #[test] + fn unwind_above_below_u64_max_deletes_max_entry() { + let store = MemKv::new(); + let hash = B256::with_last_byte(0xab); + let header = Header { number: u64::MAX, ..Default::default() }.seal_slow(); + let header_hash = header.hash(); + let address = Address::with_last_byte(0xcd); + let account = Account { nonce: 3, balance: U256::from(7), bytecode_hash: None }; + + let writer = store.writer().unwrap(); + writer.put_header(&header).unwrap(); + writer.write_account_prestate(u64::MAX, address, &account).unwrap(); + writer.put_journal_hash(u64::MAX, &hash).unwrap(); + writer.commit().unwrap(); + + let writer = store.writer().unwrap(); + writer.unwind_above(u64::MAX - 1).unwrap(); + writer.commit().unwrap(); + + let reader = store.reader().unwrap(); + assert!(reader.get_journal_hash(u64::MAX).unwrap().is_none()); + assert!(reader.get_header(u64::MAX).unwrap().is_none()); + assert!(reader.get_header_number(&header_hash).unwrap().is_none()); + assert!(reader.get_account_change(u64::MAX, &address).unwrap().is_none()); + } +} diff --git a/crates/hot/src/db/inconsistent.rs b/crates/hot/src/db/inconsistent.rs index 9a47a2b..2121386 100644 --- a/crates/hot/src/db/inconsistent.rs +++ b/crates/hot/src/db/inconsistent.rs @@ -102,6 +102,11 @@ pub trait UnsafeDbWrite: HotKvWrite + super::sealed::Sealed { self.queue_delete::(hash) } + /// Write the keccak256 of the wire-encoded `Journal::V1` bytes for a block. + fn put_journal_hash(&self, number: u64, hash: &B256) -> Result<(), Self::Error> { + self.queue_put::(&number, hash) + } + /// Commit the write transaction. fn commit(self) -> Result<(), Self::Error> where diff --git a/crates/hot/src/db/read.rs b/crates/hot/src/db/read.rs index b6adf47..80e65b4 100644 --- a/crates/hot/src/db/read.rs +++ b/crates/hot/src/db/read.rs @@ -44,6 +44,12 @@ pub trait HotDbRead: HotKvRead + super::sealed::Sealed { }; self.get_header(number) } + + /// Read the keccak256 of the wire-encoded `Journal::V1` bytes for a + /// block, if one was recorded. + fn get_journal_hash(&self, number: u64) -> Result, Self::Error> { + self.get::(&number) + } } impl HotDbRead for T where T: HotKvRead {} diff --git a/crates/hot/src/model/traits.rs b/crates/hot/src/model/traits.rs index 7d4ee1c..03c446a 100644 --- a/crates/hot/src/model/traits.rs +++ b/crates/hot/src/model/traits.rs @@ -6,7 +6,7 @@ use crate::{ revm::{RevmRead, RevmWrite}, }, ser::{KeySer, MAX_KEY_SIZE, ValSer}, - tables::{DualKey, SingleKey, Table}, + tables::{DualKey, STANDARD_TABLES, SingleKey, Table}, }; use std::borrow::Cow; @@ -394,11 +394,8 @@ pub trait HotKvWrite: HotKvRead { /// Queue creation of all standard hot storage tables. /// - /// This creates the 9 predefined tables used by the history and state - /// subsystems: [`Headers`], [`HeaderNumbers`], [`Bytecodes`], - /// [`PlainAccountState`], [`PlainStorageState`], [`AccountsHistory`], - /// [`AccountChangeSets`], [`StorageHistory`], and - /// [`StorageChangeSets`]. + /// Iterates over [`STANDARD_TABLES`] and calls [`queue_raw_create`] + /// for each entry. /// /// This is expected to be a no-op if the tables already exist, as /// [`queue_raw_create`] is required to be idempotent. @@ -407,28 +404,11 @@ pub trait HotKvWrite: HotKvRead { /// [`raw_commit`] after this method to persist the tables. /// /// [`queue_raw_create`]: Self::queue_raw_create - /// /// [`raw_commit`]: Self::raw_commit - /// [`Headers`]: crate::tables::Headers - /// [`HeaderNumbers`]: crate::tables::HeaderNumbers - /// [`Bytecodes`]: crate::tables::Bytecodes - /// [`PlainAccountState`]: crate::tables::PlainAccountState - /// [`PlainStorageState`]: crate::tables::PlainStorageState - /// [`AccountsHistory`]: crate::tables::AccountsHistory - /// [`AccountChangeSets`]: crate::tables::AccountChangeSets - /// [`StorageHistory`]: crate::tables::StorageHistory - /// [`StorageChangeSets`]: crate::tables::StorageChangeSets fn queue_db_init(&self) -> Result<(), Self::Error> { - use crate::tables; - self.queue_create::()?; - self.queue_create::()?; - self.queue_create::()?; - self.queue_create::()?; - self.queue_create::()?; - self.queue_create::()?; - self.queue_create::()?; - self.queue_create::()?; - self.queue_create::()?; + for table in &STANDARD_TABLES { + self.queue_raw_create(table.name, table.dual_key_size, table.fixed_val_size)?; + } Ok(()) } diff --git a/crates/hot/src/tables/definitions.rs b/crates/hot/src/tables/definitions.rs index 6d6af5e..a9dd7d5 100644 --- a/crates/hot/src/tables/definitions.rs +++ b/crates/hot/src/tables/definitions.rs @@ -46,3 +46,8 @@ table! { /// Records storage states before transactions, keyed by (address, block number). This table is used to rollback storage states. As such, appends and unwinds are always full replacements, never merges. StorageChangeSets<(u64, Address) => U256 => U256> is 32, FullReplacements } + +table! { + /// Records the keccak256 of the wire-encoded `Journal::V1` bytes emitted for each rollup block, keyed by block number. Opt-in per block: callers that do not produce a journal simply do not write an entry. + JournalHashes B256> +} diff --git a/crates/hot/src/tables/mod.rs b/crates/hot/src/tables/mod.rs index 24edd22..4c24b3c 100644 --- a/crates/hot/src/tables/mod.rs +++ b/crates/hot/src/tables/mod.rs @@ -5,16 +5,61 @@ mod macros; mod definitions; pub use definitions::*; -/// The number of standard hot storage tables created by -/// [`queue_db_init`](crate::model::HotKvWrite::queue_db_init). Update this -/// constant whenever a table is added to or removed from `queue_db_init`. -pub const NUM_TABLES: usize = 9; +/// The number of standard hot storage tables. Tied to the length of +/// [`STANDARD_TABLES`]. +pub const NUM_TABLES: usize = STANDARD_TABLES.len(); use crate::{ DeserError, KeySer, MAX_FIXED_VAL_SIZE, MAX_KEY_SIZE, ValSer, model::{DualKeyValue, KeyValue}, }; +/// Compile-time metadata for a standard hot storage table. The two `Option` +/// fields mirror the arguments accepted by +/// [`queue_raw_create`](crate::model::HotKvWrite::queue_raw_create) so the +/// same record drives both table creation and backend-side FSI inference. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct StandardTable { + /// Table name. + pub name: &'static str, + /// Size of the second key for DUPSORT tables, `None` for single-keyed + /// tables. Mirrors [`Table::DUAL_KEY_SIZE`]. + pub dual_key_size: Option, + /// Mirrors [`Table::FIXED_VAL_SIZE`]: `Some(size)` when the table's value + /// type is a fixed-size primitive of `size <= MAX_FIXED_VAL_SIZE` bytes, + /// `None` otherwise. Backends combine this with [`Self::dual_key_size`]: + /// `(Some, Some)` implies DUP_FIXED, `(Some, None)` implies plain DUPSORT, + /// and `(None, _)` implies neither (today the macro forces this arm to + /// `(None, None)` for single-keyed tables). + pub fixed_val_size: Option, +} + +impl StandardTable { + /// Derive a [`StandardTable`] from a [`Table`] type's associated + /// constants. + pub const fn from_table() -> Self { + Self { name: T::NAME, dual_key_size: T::DUAL_KEY_SIZE, fixed_val_size: T::FIXED_VAL_SIZE } + } +} + +/// The canonical list of standard hot storage tables created by +/// [`queue_db_init`](crate::model::HotKvWrite::queue_db_init). Backends may +/// also iterate over this list to pre-populate per-table caches at open +/// time. Update this constant whenever a standard table is added or +/// removed. +pub const STANDARD_TABLES: [StandardTable; 10] = [ + StandardTable::from_table::(), + StandardTable::from_table::(), + StandardTable::from_table::(), + StandardTable::from_table::(), + StandardTable::from_table::(), + StandardTable::from_table::(), + StandardTable::from_table::(), + StandardTable::from_table::(), + StandardTable::from_table::(), + StandardTable::from_table::(), +]; + /// Trait for table definitions. /// /// Tables are compile-time definitions of key-value pairs stored in hot diff --git a/crates/storage/src/lib.rs b/crates/storage/src/lib.rs index c389e22..a0f4fb4 100644 --- a/crates/storage/src/lib.rs +++ b/crates/storage/src/lib.rs @@ -88,6 +88,7 @@ pub use signet_cold::{ pub use signet_cold_mdbx::MdbxColdBackend; pub use signet_hot::{ HistoryError, HistoryRead, HistoryWrite, HotKv, + db::HotDbRead, model::{HotKvRead, RevmRead, RevmWrite}, }; pub use signet_hot_mdbx::{DatabaseArguments, DatabaseEnv}; diff --git a/crates/storage/src/unified.rs b/crates/storage/src/unified.rs index f525d12..6ae20c0 100644 --- a/crates/storage/src/unified.rs +++ b/crates/storage/src/unified.rs @@ -9,6 +9,7 @@ use alloy::primitives::BlockNumber; use signet_cold::{BlockData, ColdReceipt, ColdStorage, ColdStorageBackend, ColdStorageError}; use signet_hot::{ HistoryRead, HistoryWrite, HotKv, + db::UnsafeDbWrite, model::{HotKvReadError, HotKvWrite, RevmRead}, }; use signet_storage_types::{ExecutedBlock, SealedHeader}; @@ -195,6 +196,14 @@ impl UnifiedStorage { writer .append_blocks(blocks.iter().map(|b| (&b.header, &b.bundle))) .map_err(|e| e.map_db(|e| e.into_hot_kv_error()))?; + // Persist journal hashes alongside headers in the same transaction. + // Blocks without a journal (e.g. block-only nodes) skip the write. + blocks + .iter() + .filter_map(|block| block.journal_hash.as_ref().map(|hash| (block.header.number, hash))) + .try_for_each(|(height, hash)| { + writer.put_journal_hash(height, hash).map_err(|e| e.into_hot_kv_error()) + })?; writer.raw_commit().map_err(|e| e.into_hot_kv_error())?; Ok(()) diff --git a/crates/storage/tests/unified.rs b/crates/storage/tests/unified.rs index ed238be..668adba 100644 --- a/crates/storage/tests/unified.rs +++ b/crates/storage/tests/unified.rs @@ -5,7 +5,7 @@ use alloy::{ primitives::{Address, B256, Signature, TxKind, U256}, }; use signet_cold::{ColdStorage, HeaderSpecifier, mem::MemColdBackend}; -use signet_hot::{HistoryRead, HistoryWrite, HotKv, mem::MemKv, model::HotKvWrite}; +use signet_hot::{HistoryRead, HistoryWrite, HotKv, db::HotDbRead, mem::MemKv, model::HotKvWrite}; use signet_storage::UnifiedStorage; use signet_storage_types::{ ExecutedBlock, ExecutedBlockBuilder, Receipt, RecoveredTx, SealedHeader, TransactionSigned, @@ -170,6 +170,78 @@ async fn drain_above_empty_when_at_tip() { cancel.cancel(); } +#[tokio::test] +async fn append_blocks_persists_journal_hashes() { + let hot = MemKv::new(); + let cancel = CancellationToken::new(); + let cold_handle = ColdStorage::new(MemColdBackend::new(), cancel.clone()); + let storage = UnifiedStorage::new(hot.clone(), cold_handle); + + // Three blocks: 0 with a hash, 1 without, 2 with a hash. + let hash_0 = B256::with_last_byte(0xa0); + let hash_2 = B256::with_last_byte(0xc2); + let journal_hashes = [Some(hash_0), None, Some(hash_2)]; + + let mut parent_hash = B256::ZERO; + let mut blocks = Vec::with_capacity(3); + for (number, journal_hash) in journal_hashes.into_iter().enumerate() { + let header = Header { number: number as u64, parent_hash, ..Default::default() }; + let sealed: SealedHeader = header.seal_slow(); + parent_hash = sealed.hash(); + let mut builder = ExecutedBlockBuilder::new().header(sealed).bundle(BundleState::default()); + if let Some(hash) = journal_hash { + builder = builder.journal_hash(hash); + } + blocks.push(builder.build().unwrap()); + } + + storage.append_blocks(blocks).await.unwrap(); + + let reader = hot.reader().unwrap(); + assert_eq!(reader.get_journal_hash(0).unwrap(), Some(hash_0)); + assert_eq!(reader.get_journal_hash(1).unwrap(), None); + assert_eq!(reader.get_journal_hash(2).unwrap(), Some(hash_2)); + + cancel.cancel(); +} + +#[tokio::test] +async fn unwind_above_drops_journal_hashes() { + let hot = MemKv::new(); + let cancel = CancellationToken::new(); + let cold_handle = ColdStorage::new(MemColdBackend::new(), cancel.clone()); + let storage = UnifiedStorage::new(hot.clone(), cold_handle); + + // Three blocks, each carrying a journal hash. + let hashes = + [B256::with_last_byte(0xa0), B256::with_last_byte(0xa1), B256::with_last_byte(0xa2)]; + + let mut parent_hash = B256::ZERO; + let mut blocks = Vec::with_capacity(3); + for (number, hash) in hashes.iter().enumerate() { + let header = Header { number: number as u64, parent_hash, ..Default::default() }; + let sealed: SealedHeader = header.seal_slow(); + parent_hash = sealed.hash(); + let block = ExecutedBlockBuilder::new() + .header(sealed) + .bundle(BundleState::default()) + .journal_hash(*hash) + .build() + .unwrap(); + blocks.push(block); + } + + storage.append_blocks(blocks).await.unwrap(); + storage.unwind_above(0).await.unwrap(); + + let reader = hot.reader().unwrap(); + assert_eq!(reader.get_journal_hash(0).unwrap(), Some(hashes[0])); + assert_eq!(reader.get_journal_hash(1).unwrap(), None); + assert_eq!(reader.get_journal_hash(2).unwrap(), None); + + cancel.cancel(); +} + #[tokio::test] async fn drain_above_cold_lag() { let hot = MemKv::new(); diff --git a/crates/types/src/execution.rs b/crates/types/src/execution.rs index c546554..c1a747e 100644 --- a/crates/types/src/execution.rs +++ b/crates/types/src/execution.rs @@ -4,7 +4,7 @@ //! needed by both hot and cold storage systems for a single executed block. use crate::{DbSignetEvent, DbZenithHeader, Receipt, RecoveredTx, SealedHeader}; -use alloy::primitives::BlockNumber; +use alloy::primitives::{B256, BlockNumber}; use core::fmt; use trevm::revm::database::BundleState; @@ -42,21 +42,16 @@ pub struct ExecutedBlock { pub signet_events: Vec, /// The zenith header, if present. pub zenith_header: Option, + /// keccak256 of the wire-encoded `Journal::V1` bytes emitted for this + /// block, when produced. Persisted into the `JournalHashes` hot table by + /// `append_blocks` so producing and syncing nodes can re-seed the rolling + /// previous-journal hash across restarts and reverts. `None` for callers + /// that do not produce a journal (e.g. block-only nodes); no entry is + /// written in that case. + pub journal_hash: Option, } impl ExecutedBlock { - /// Create a new executed block. - pub const fn new( - header: SealedHeader, - bundle: BundleState, - transactions: Vec, - receipts: Vec, - signet_events: Vec, - zenith_header: Option, - ) -> Self { - Self { header, bundle, transactions, receipts, signet_events, zenith_header } - } - /// Get the block number. pub fn block_number(&self) -> BlockNumber { self.header.number @@ -85,6 +80,7 @@ pub struct ExecutedBlockBuilder { receipts: Vec, signet_events: Vec, zenith_header: Option, + journal_hash: Option, } impl ExecutedBlockBuilder { @@ -129,6 +125,14 @@ impl ExecutedBlockBuilder { self } + /// Set the journal hash (keccak256 of the wire-encoded `Journal::V1`). + /// Leave the method uncalled for block-only nodes that do not produce + /// a journal - the field defaults to `None`. + pub const fn journal_hash(mut self, hash: B256) -> Self { + self.journal_hash = Some(hash); + self + } + /// Build the [`ExecutedBlock`]. /// /// # Errors @@ -142,6 +146,7 @@ impl ExecutedBlockBuilder { receipts: self.receipts, signet_events: self.signet_events, zenith_header: self.zenith_header, + journal_hash: self.journal_hash, }) } }