diff --git a/firewood/src/db.rs b/firewood/src/db.rs index 757fcfbcd..45b754c3d 100644 --- a/firewood/src/db.rs +++ b/firewood/src/db.rs @@ -21,10 +21,11 @@ use parking_lot::{Mutex, RwLock}; #[cfg(feature = "eth")] use primitive_types::U256; use shale::compact::CompactSpace; -use shale::ShaleStore; use shale::{ compact::CompactSpaceHeader, CachedStore, ObjPtr, ShaleError, SpaceId, Storable, StoredView, }; +use shale::{Obj, ShaleStore}; +use std::mem::size_of; use std::{ collections::VecDeque, error::Error, @@ -466,6 +467,8 @@ pub struct Db { // #[metered(registry = DbMetrics, visibility = pub)] impl Db { + const PARAM_SIZE: u64 = size_of::() as u64; + /// Open a database. pub fn new>(db_path: P, cfg: &DbConfig) -> Result { // TODO: make sure all fds are released at the end @@ -516,7 +519,7 @@ impl Db { let header_bytes = unsafe { std::slice::from_raw_parts( &header as *const DbParams as *const u8, - std::mem::size_of::(), + size_of::(), ) .to_vec() }; @@ -525,7 +528,7 @@ impl Db { } // read DbParams - let mut header_bytes = [0; std::mem::size_of::()]; + let mut header_bytes = [0; size_of::()]; nix::sys::uio::pread(fd0, &mut header_bytes, 0).map_err(DbError::System)?; drop(file0); let params: DbParams = cast_slice(&header_bytes)[0]; @@ -641,13 +644,24 @@ impl Db { merkle: get_sub_universe_from_empty_delta(&data_cache.merkle), blob: get_sub_universe_from_empty_delta(&data_cache.blob), }; + + let db_header_ref = Db::get_db_header_ref(&base.merkle.meta)?; + + let merkle_payload_header_ref = + Db::get_payload_header_ref(&base.merkle.meta, Db::PARAM_SIZE + DbHeader::MSIZE)?; + + let blob_payload_header_ref = Db::get_payload_header_ref(&base.blob.meta, 0)?; + + let header_refs = ( + db_header_ref, + merkle_payload_header_ref, + blob_payload_header_ref, + ); + let base_revision = Db::new_revision( - &base.merkle.meta, - &base.blob.meta, - base.merkle.meta.clone(), - base.merkle.payload.clone(), - base.blob.meta.clone(), - base.blob.payload.clone(), + header_refs, + (base.merkle.meta.clone(), base.merkle.payload.clone()), + (base.blob.meta.clone(), base.blob.payload.clone()), params.payload_regn_nbit, cfg.payload_max_walk, &cfg.rev, @@ -683,7 +697,7 @@ impl Db { payload_regn_nbit: u64, cfg: &DbConfig, ) -> Result<(Universe>, DbRev), DbError> { - let mut offset = std::mem::size_of::() as u64; + let mut offset = Db::PARAM_SIZE; let db_header: ObjPtr = ObjPtr::new_from_addr(offset); offset += DbHeader::MSIZE; let merkle_payload_header: ObjPtr = ObjPtr::new_from_addr(offset); @@ -727,13 +741,25 @@ impl Db { ), }; - let mut rev: DbRev> = Db::new_revision( + let db_header_ref = Db::get_db_header_ref(store.merkle.meta.as_ref())?; + + let merkle_payload_header_ref = Db::get_payload_header_ref( store.merkle.meta.as_ref(), - store.blob.meta.as_ref(), - store.merkle.meta.clone(), - store.merkle.payload.clone(), - store.blob.meta.clone(), - store.blob.payload.clone(), + Db::PARAM_SIZE + DbHeader::MSIZE, + )?; + + let blob_payload_header_ref = Db::get_payload_header_ref(store.blob.meta.as_ref(), 0)?; + + let header_refs = ( + db_header_ref, + merkle_payload_header_ref, + blob_payload_header_ref, + ); + + let mut rev: DbRev> = Db::new_revision( + header_refs, + (store.merkle.meta.clone(), store.merkle.payload.clone()), + (store.blob.meta.clone(), store.blob.payload.clone()), payload_regn_nbit, cfg.payload_max_walk, &cfg.rev, @@ -743,46 +769,50 @@ impl Db { Ok((store, rev)) } + fn get_payload_header_ref( + meta_ref: &K, + header_offset: u64, + ) -> Result, DbError> { + let payload_header = ObjPtr::::new_from_addr(header_offset); + StoredView::ptr_to_obj( + meta_ref, + payload_header, + shale::compact::CompactHeader::MSIZE, + ) + .map_err(Into::into) + } + + fn get_db_header_ref(meta_ref: &K) -> Result, DbError> { + let db_header = ObjPtr::::new_from_addr(Db::PARAM_SIZE); + StoredView::ptr_to_obj(meta_ref, db_header, DbHeader::MSIZE).map_err(Into::into) + } + fn new_revision( - merkle_meta_ref: &K, - blob_meta_ref: &K, - merkle_meta: impl Into>, - merkle_payload: impl Into>, - blob_meta: impl Into>, - blob_payload: impl Into>, + header_refs: ( + Obj, + Obj, + Obj, + ), + merkle: (impl Into>, impl Into>), + _blob: (impl Into>, impl Into>), payload_regn_nbit: u64, payload_max_walk: u64, cfg: &DbRevConfig, ) -> Result>, DbError> { - // Set up the storage layout - let mut offset = std::mem::size_of::() as u64; - // DbHeader starts after DbParams in merkle meta space - let db_header: ObjPtr = ObjPtr::new_from_addr(offset); - offset += DbHeader::MSIZE; - // Merkle CompactHeader starts after DbHeader in merkle meta space - let merkle_payload_header: ObjPtr = ObjPtr::new_from_addr(offset); - offset += CompactSpaceHeader::MSIZE; - assert!(offset <= SPACE_RESERVED); + // TODO: This should be a compile time check + const DB_OFFSET: u64 = Db::PARAM_SIZE; + let merkle_offset = DB_OFFSET + DbHeader::MSIZE; + assert!(merkle_offset + CompactSpaceHeader::MSIZE <= SPACE_RESERVED); - let mut db_header_ref = - StoredView::ptr_to_obj(merkle_meta_ref, db_header, DbHeader::MSIZE).unwrap(); + let mut db_header_ref = header_refs.0; + let merkle_payload_header_ref = header_refs.1; - let merkle_payload_header_ref = StoredView::ptr_to_obj( - merkle_meta_ref, - merkle_payload_header, - shale::compact::CompactHeader::MSIZE, - )?; - - #[cfg(feature = "eth")] - let blob_payload_header_ref = StoredView::ptr_to_obj( - blob_meta_ref, - blob_payload_header, - shale::compact::CompactHeader::MSIZE, - )?; + let merkle_meta = merkle.0.into(); + let merkle_payload = merkle.1.into(); let merkle_space = shale::compact::CompactSpace::new( - merkle_meta.into(), - merkle_payload.into(), + merkle_meta, + merkle_payload, merkle_payload_header_ref, shale::ObjCache::new(cfg.merkle_ncached_objs), payload_max_walk, @@ -791,15 +821,22 @@ impl Db { .unwrap(); #[cfg(feature = "eth")] - let blob_space = shale::compact::CompactSpace::new( - blob_meta.into(), - blob_payload.into(), - blob_payload_header_ref, - shale::ObjCache::new(cfg.blob_ncached_objs), - payload_max_walk, - payload_regn_nbit, - ) - .unwrap(); + let blob_space = { + let blob = _blob; + let blob_payload_header_ref = header_refs.2; + let blob_meta = blob.0.into(); + let blob_payload = blob.1.into(); + + shale::compact::CompactSpace::new( + blob_meta, + blob_payload, + header_refs.2, + shale::ObjCache::new(cfg.blob_ncached_objs), + payload_max_walk, + payload_regn_nbit, + ) + .unwrap() + }; if db_header_ref.acc_root.is_null() { let mut err = Ok(()); @@ -958,20 +995,33 @@ impl Db { drop(inner_lock); let cfg = cfg.as_ref().unwrap_or(&self.rev_cfg); - Some(Revision { + + let db_header_ref = Db::get_db_header_ref(&space.merkle.meta).unwrap(); + + let merkle_payload_header_ref = + Db::get_payload_header_ref(&space.merkle.meta, Db::PARAM_SIZE + DbHeader::MSIZE) + .unwrap(); + + let blob_payload_header_ref = Db::get_payload_header_ref(&space.blob.meta, 0).unwrap(); + + let header_refs = ( + db_header_ref, + merkle_payload_header_ref, + blob_payload_header_ref, + ); + + Revision { rev: Db::new_revision( - &space.merkle.meta, - &space.blob.meta, - space.merkle.meta.clone(), - space.merkle.payload.clone(), - space.blob.meta.clone(), - space.blob.payload.clone(), + header_refs, + (space.merkle.meta.clone(), space.merkle.payload.clone()), + (space.blob.meta.clone(), space.blob.payload.clone()), self.payload_regn_nbit, 0, cfg, ) .unwrap(), - }) + } + .into() } } @@ -1112,13 +1162,26 @@ impl Proposal { let m = Arc::clone(&self.m); let r = Arc::clone(&self.r); let cfg = self.cfg.clone(); - let mut rev = Db::new_revision( + + let db_header_ref = Db::get_db_header_ref(store.merkle.meta.as_ref())?; + + let merkle_payload_header_ref = Db::get_payload_header_ref( store.merkle.meta.as_ref(), - store.blob.meta.as_ref(), - store.merkle.meta.clone(), - store.merkle.payload.clone(), - store.blob.meta.clone(), - store.blob.payload.clone(), + Db::PARAM_SIZE + DbHeader::MSIZE, + )?; + + let blob_payload_header_ref = Db::get_payload_header_ref(store.blob.meta.as_ref(), 0)?; + + let header_refs = ( + db_header_ref, + merkle_payload_header_ref, + blob_payload_header_ref, + ); + + let mut rev = Db::new_revision( + header_refs, + (store.merkle.meta.clone(), store.merkle.payload.clone()), + (store.blob.meta.clone(), store.blob.payload.clone()), cfg.payload_regn_nbit, cfg.payload_max_walk, &cfg.rev, @@ -1274,13 +1337,24 @@ impl Proposal { merkle: get_sub_universe_from_empty_delta(&rev_inner.data_cache.merkle), blob: get_sub_universe_from_empty_delta(&rev_inner.data_cache.blob), }; + + let db_header_ref = Db::get_db_header_ref(&base.merkle.meta)?; + + let merkle_payload_header_ref = + Db::get_payload_header_ref(&base.merkle.meta, Db::PARAM_SIZE + DbHeader::MSIZE)?; + + let blob_payload_header_ref = Db::get_payload_header_ref(&base.blob.meta, 0)?; + + let header_refs = ( + db_header_ref, + merkle_payload_header_ref, + blob_payload_header_ref, + ); + let base_revision = Db::new_revision( - &base.merkle.meta, - &base.blob.meta, - base.merkle.meta.clone(), - base.merkle.payload.clone(), - base.blob.meta.clone(), - base.blob.payload.clone(), + header_refs, + (base.merkle.meta.clone(), base.merkle.payload.clone()), + (base.blob.meta.clone(), base.blob.payload.clone()), 0, self.cfg.payload_max_walk, &self.cfg.rev,