storage: refactor and rocksdb implementation

Cargo.toml

@@ -21,13 +21,17 @@ members = [
     "crates/groth16",
     "crates/errors",
     "crates/sp1",
+    "crates/storage",
+    "crates/config",
 ]
 default-members = [
     "crates/prism",
     "crates/common",
     "crates/nova",
     "crates/groth16",
     "crates/errors",
+    "crates/storage",
+    "crates/config",
 ]
 resolver = "2"
 
@@ -78,10 +82,14 @@ bincode = "1.3.3"
 sp1-zkvm = { version = "1.2.0" }
 sp1-sdk = { version = "1.2.0" }
 prism-common = { path = "crates/common" }
+prism-config = { path = "crates/config" }
+prism-storage = { path = "crates/storage" }
 prism-nova = { path = "crates/nova" }
 prism-errors = { path = "crates/errors" }
 prism-main = { path = "crates/prism" }
 prism-groth16 = { path = "crates/groth16" }
+rocksdb = {version = "0.21.0", features = ["multi-threaded-cf"]}
+
 
 [patch.crates-io]
 sha2-v0-10-8 = { git = "https://github.com/sp1-patches/RustCrypto-hashes", package = "sha2", branch = "patch-sha2-v0.10.8" }

crates/config/Cargo.toml

@@ -0,0 +1,55 @@
+[package]
+name = "prism-config"
+name.workspace = true
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+readme.workspace = true
+
+[dependencies]
+axum = { workspace = true }
+borsh = { workspace = true }
+tower-http = { workspace = true }
+utoipa = { workspace = true }
+utoipa-swagger-ui = { workspace = true }
+async-trait = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+redis = { workspace = true }
+ed25519-dalek = { workspace = true }
+ed25519 = { workspace = true }
+base64 = { workspace = true }
+tokio = { workspace = true }
+bellman = { workspace = true }
+bincode = { workspace = true }
+bls12_381 = { workspace = true }
+rand = { workspace = true }
+hex = { workspace = true }
+ff = { workspace = true }
+log = { workspace = true }
+pretty_env_logger = { workspace = true }
+clap = { workspace = true }
+config = { workspace = true }
+thiserror = { workspace = true }
+indexed-merkle-tree = { workspace = true }
+dotenvy = { workspace = true }
+celestia-rpc = { workspace = true }
+celestia-types = { workspace = true }
+mockall = { workspace = true }
+keystore-rs = { workspace = true }
+toml = { workspace = true }
+dirs = { workspace = true }
+anyhow = { workspace = true }
+jmt = { workspace = true }
+sha2 = { workspace = true }
+auto_impl = { workspace = true }
+prism-common = { workspace = true }
+prism-storage = { workspace = true }
+prism-errors = { workspace = true }
+prism-groth16 = { workspace = true }
+sp1-sdk = { workspace = true }

crates/prism/src/cfg.rs → crates/config/src/lib.rs

@@ -12,6 +12,7 @@ use serde::{Deserialize, Serialize};
 use std::{fs, path::Path, sync::Arc};
 
 use crate::da::{celestia::CelestiaConnection, DataAvailabilityLayer};
+use prism_storage::RedisConfig;
 
 #[derive(Clone, Debug, Subcommand, Deserialize)]
 pub enum Commands {
@@ -97,6 +98,7 @@ impl Default for WebServerConfig {
     }
 }
 
+
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct RedisConfig {
     pub connection_string: String,
@@ -300,7 +302,7 @@ pub async fn initialize_da_layer(
             unreachable!() // This line should never be reached due to the return in the last iteration
         }
         DALayerOption::InMemory => {
-            let (da_layer, _height_rx, _block_rx) = InMemoryDataAvailabilityLayer::new(1);
+            let (da_layer, _height_rx, _block_rx) = InMemoryDataAvailabilityLayer::new(30);
             Ok(Arc::new(da_layer) as Arc<dyn DataAvailabilityLayer + 'static>)
         }
         DALayerOption::None => Err(anyhow!(PrismError::ConfigError(

crates/prism/Cargo.toml

@@ -49,6 +49,7 @@ jmt = { workspace = true }
 sha2 = { workspace = true }
 auto_impl = { workspace = true }
 prism-common = { workspace = true }
+prism-storage = { workspace = true }
 prism-errors = { workspace = true }
 prism-groth16 = { workspace = true }
 sp1-sdk = { workspace = true }

crates/prism/src/lib.rs

@@ -2,7 +2,6 @@ pub mod cfg;
 pub mod consts;
 pub mod da;
 pub mod node_types;
-pub mod storage;
 pub mod utils;
 pub mod webserver;
 #[macro_use]

crates/prism/src/main.rs

@@ -2,7 +2,6 @@ mod cfg;
 pub mod consts;
 pub mod da;
 mod node_types;
-pub mod storage;
 mod utils;
 mod webserver;
 
@@ -13,13 +12,15 @@ use keystore_rs::{KeyChain, KeyStore, KeyStoreType};
 use crate::cfg::{CommandLineArgs, Commands};
 use node_types::{lightclient::LightClient, sequencer::Sequencer, NodeType};
 use std::sync::Arc;
-use storage::RedisConnection;
+use prism_storage::RedisConnection;
 
 #[macro_use]
 extern crate log;
 
 pub const PRISM_ELF: &[u8] = include_bytes!("../../../elf/riscv32im-succinct-zkvm-elf");
 
+
+
 /// The main function that initializes and runs a prism client.
 #[tokio::main()]
 async fn main() -> std::io::Result<()> {

crates/prism/src/node_types/sequencer.rs

@@ -24,9 +24,10 @@ use crate::{
     consts::{CHANNEL_BUFFER_SIZE, DA_RETRY_COUNT, DA_RETRY_INTERVAL},
     da::{DataAvailabilityLayer, FinalizedEpoch},
     node_types::NodeType,
-    storage::Database,
     webserver::{OperationInput, WebServer},
 };
+
+use prism_storage::Database;
 use prism_common::{
     hashchain::{Hashchain, HashchainEntry},
     operation::{AccountSource, Operation},
@@ -470,8 +471,8 @@ mod tests {
     use crate::{
         cfg::{Config, RedisConfig},
         da::memory::InMemoryDataAvailabilityLayer,
-        storage::RedisConnection,
     };
+    use prism_storage::RedisConnection;
     use base64::{engine::general_purpose::STANDARD as engine, Engine as _};
     use keystore_rs::create_signing_key;
     use serial_test::serial;

crates/storage/Cargo.toml

@@ -0,0 +1,30 @@
+[package]
+name = "prism-storage"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+description.workspace = true
+homepage.workspace = true
+repository.workspace = true
+license.workspace = true
+keywords.workspace = true
+readme.workspace = true
+
+[dependencies]
+serde = { workspace = true }
+serde_json = { workspace = true }
+redis = { workspace = true }
+mockall = { workspace = true }
+tokio = { workspace = true }
+log = { workspace = true }
+anyhow = { workspace = true }
+bincode = { workspace = true }
+hex = { workspace = true }
+jmt = { workspace = true }
+pretty_env_logger = { workspace = true }
+config = { workspace = true }
+thiserror = { workspace = true }
+prism-errors = { workspace = true }
+prism-common = { workspace = true }
+auto_impl = {workspace = true}
+rocksdb = {workspace = true}