Skip to content

Commit

Permalink
feat: refactor transport opts structure
Browse files Browse the repository at this point in the history 
Signed-off-by: Diwakar Sharma <[email protected]>
  • Loading branch information
@dsharma-dc
dsharma-dc committed Aug 13, 2024
1 parent 7e27a13 commit 1b7f911
Show file tree
Hide file tree
Showing 2 changed files with 113 additions and 147 deletions.
256 changes: 111 additions & 145 deletions io-engine/src/subsys/config/opts.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,109 @@ impl Display for NvmfTgtTransport {
}
}

/// Nvmf settings for the transports
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct NvmfTransportOpts {
/// max queue depth
max_queue_depth: u16,
/// max qpairs per controller
max_qpairs_per_ctrl: u16,
/// encapsulated data size
in_capsule_data_size: u32,
/// max IO size
max_io_size: u32,
/// IO unit size
io_unit_size: u32,
/// max admin queue depth per admin queue
max_aq_depth: u32,
/// num of shared buffers
num_shared_buf: u32,
/// cache size
buf_cache_size: u32,
/// dif
dif_insert_or_strip: bool,
/// abort execution timeout
abort_timeout_sec: u32,
/// acceptor poll rate, microseconds
acceptor_poll_rate: u32,
/// Use zero-copy operations if the underlying bdev supports them
zcopy: bool,
/// ACK timeout in milliseconds
ack_timeout: u32,
/// Size of RDMA data WR pool
data_wr_pool_size: u32,
}

impl NvmfTransportOpts {
pub fn new(xprt: NvmfTgtTransport) -> Self {
xprt.xprt_nvmf_opts()
}
}

impl Default for NvmfTransportOpts {
fn default() -> Self {
NvmfTransportOpts::new(NvmfTgtTransport::Tcp)
}
}

/// This can be extended to configure transport specific options
/// like tcp_transport_opts and rdma_transport_opts if required later.
pub trait TransportOpts {
fn xprt_nvmf_opts(&self) -> NvmfTransportOpts;
}

impl TransportOpts for NvmfTgtTransport {
fn xprt_nvmf_opts(&self) -> NvmfTransportOpts {
match self {
NvmfTgtTransport::Rdma => NvmfTransportOpts {
max_queue_depth: try_from_env("NVMF_RDMA_MAX_QUEUE_DEPTH", 128),
in_capsule_data_size: 8192,
max_io_size: 131_072,
io_unit_size: 8192,
max_qpairs_per_ctrl: try_from_env(
"NVMF_RDMA_MAX_QPAIRS_PER_CTRL",
32,
),
num_shared_buf: try_from_env("NVMF_RDMA_NUM_SHARED_BUF", 2047),
buf_cache_size: try_from_env("NVMF_RDMA_BUF_CACHE_SIZE", 64),
dif_insert_or_strip: false,
max_aq_depth: 32,
abort_timeout_sec: 1,
acceptor_poll_rate: try_from_env(
"NVMF_ACCEPTOR_POLL_RATE",
10_000,
),
zcopy: try_from_env("NVMF_ZCOPY", 1) == 1,
ack_timeout: try_from_env("NVMF_ACK_TIMEOUT", 0),
data_wr_pool_size: try_from_env("NVMF_DATA_WR_POOL_SIZE", 4095),
},
NvmfTgtTransport::Tcp => NvmfTransportOpts {
max_queue_depth: try_from_env("NVMF_TCP_MAX_QUEUE_DEPTH", 32),
in_capsule_data_size: 4096,
max_io_size: 131_072,
io_unit_size: 131_072,
max_qpairs_per_ctrl: try_from_env(
"NVMF_TCP_MAX_QPAIRS_PER_CTRL",
32,
),
num_shared_buf: try_from_env("NVMF_TCP_NUM_SHARED_BUF", 2047),
buf_cache_size: try_from_env("NVMF_TCP_BUF_CACHE_SIZE", 64),
dif_insert_or_strip: false,
max_aq_depth: 32,
abort_timeout_sec: 1,
acceptor_poll_rate: try_from_env(
"NVMF_ACCEPTOR_POLL_RATE",
10_000,
),
zcopy: try_from_env("NVMF_ZCOPY", 1) == 1,
ack_timeout: try_from_env("NVMF_ACK_TIMEOUT", 10_000),
data_wr_pool_size: try_from_env("NVMF_DATA_WR_POOL_SIZE", 0),
},
}
}
}

#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct NvmfTgtConfig {
Expand All @@ -114,13 +217,13 @@ pub struct NvmfTgtConfig {
/// NVMF target Command Retry Delay in x100 ms.
pub crdt: [u16; TARGET_CRDT_LEN],
/// TCP transport options
pub opts_tcp: NvmfTcpTransportOpts,
pub opts_tcp: NvmfTransportOpts,
/// NVMF target interface (ip, mac, name or subnet).
pub interface: Option<String>,
/// Enable RDMA for NVMF target or not
pub rdma: Option<bool>,
/// RDMA transport options
pub opts_rdma: NvmfRdmaTransportOpts,
pub opts_rdma: NvmfTransportOpts,
}

impl From<NvmfTgtConfig> for Box<spdk_nvmf_target_opts> {
Expand Down Expand Up @@ -148,10 +251,10 @@ impl Default for NvmfTgtConfig {
name: "mayastor_target".to_string(),
max_namespaces: 2048,
crdt: args.nvmf_tgt_crdt,
opts_tcp: NvmfTcpTransportOpts::default(),
opts_tcp: NvmfTransportOpts::new(NvmfTgtTransport::Tcp),
interface: None,
rdma: None,
opts_rdma: NvmfRdmaTransportOpts::default(),
opts_rdma: NvmfTransportOpts::new(NvmfTgtTransport::Rdma),
}
}
}
Expand All @@ -162,70 +265,6 @@ impl GetOpts for NvmfTgtConfig {
}
}

/// Settings for the TCP transport
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct NvmfTcpTransportOpts {
/// max queue depth
max_queue_depth: u16,
/// max qpairs per controller
max_qpairs_per_ctrl: u16,
/// encapsulated data size
in_capsule_data_size: u32,
/// max IO size
max_io_size: u32,
/// IO unit size
io_unit_size: u32,
/// max admin queue depth per admin queue
max_aq_depth: u32,
/// num of shared buffers
num_shared_buf: u32,
/// cache size
buf_cache_size: u32,
/// dif
dif_insert_or_strip: bool,
/// abort execution timeout
abort_timeout_sec: u32,
/// acceptor poll rate, microseconds
acceptor_poll_rate: u32,
/// Use zero-copy operations if the underlying bdev supports them
zcopy: bool,
/// ACK timeout in milliseconds
ack_timeout: u32,
/// Size of RDMA data WR pool
data_wr_pool_size: u32,
}

/// Settings for the RDMA transport
#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct NvmfRdmaTransportOpts {
/// max queue depth
max_queue_depth: u16,
/// max qpairs per controller
max_qpairs_per_ctrl: u16,
/// encapsulated data size
in_capsule_data_size: u32,
/// max IO size
max_io_size: u32,
/// IO unit size
io_unit_size: u32,
/// max admin queue depth per admin queue
max_aq_depth: u32,
/// num of shared buffers
num_shared_buf: u32,
/// cache size
buf_cache_size: u32,
/// dif
dif_insert_or_strip: bool,
/// abort execution timeout
abort_timeout_sec: u32,
/// acceptor poll rate, microseconds
acceptor_poll_rate: u32,
/// Use zero-copy operations if the underlying bdev supports them
zcopy: bool,
}

/// try to read an env variable or returns the default when not found
pub(crate) fn try_from_env<T>(name: &str, default: T) -> T
where
Expand Down Expand Up @@ -317,86 +356,11 @@ where
}
}

impl Default for NvmfTcpTransportOpts {
fn default() -> Self {
Self {
max_queue_depth: try_from_env("NVMF_TCP_MAX_QUEUE_DEPTH", 32),
in_capsule_data_size: 4096,
max_io_size: 131_072,
io_unit_size: 131_072,
max_qpairs_per_ctrl: try_from_env(
"NVMF_TCP_MAX_QPAIRS_PER_CTRL",
32,
),
num_shared_buf: try_from_env("NVMF_TCP_NUM_SHARED_BUF", 2047),
buf_cache_size: try_from_env("NVMF_TCP_BUF_CACHE_SIZE", 64),
dif_insert_or_strip: false,
max_aq_depth: 32,
abort_timeout_sec: 1,
acceptor_poll_rate: try_from_env("NVMF_ACCEPTOR_POLL_RATE", 10_000),
zcopy: try_from_env("NVMF_ZCOPY", 1) == 1,
ack_timeout: try_from_env("NVMF_ACK_TIMEOUT", 0),
data_wr_pool_size: try_from_env("NVMF_DATA_WR_POOL_SIZE", 0),
}
}
}

// todo: Tune the defaults by experiments or recommendations, if required.
impl Default for NvmfRdmaTransportOpts {
fn default() -> Self {
Self {
max_queue_depth: try_from_env("NVMF_RDMA_MAX_QUEUE_DEPTH", 128),
in_capsule_data_size: 8192,
max_io_size: 131_072,
io_unit_size: 8192,
max_qpairs_per_ctrl: try_from_env(
"NVMF_RDMA_MAX_QPAIRS_PER_CTRL",
32,
),
num_shared_buf: try_from_env("NVMF_RDMA_NUM_SHARED_BUF", 2047),
buf_cache_size: try_from_env("NVMF_RDMA_BUF_CACHE_SIZE", 64),
dif_insert_or_strip: false,
max_aq_depth: 32,
abort_timeout_sec: 1,
acceptor_poll_rate: try_from_env("NVMF_ACCEPTOR_POLL_RATE", 10_000),
zcopy: try_from_env("NVMF_ZCOPY", 1) == 1,
}
}
}

/// we cannot add derives for YAML to these structs directly, so we need to
/// copy them. The upside though, is that if the FFI structures change, we will
/// know about it during compile time.
impl From<NvmfTcpTransportOpts> for spdk_nvmf_transport_opts {
fn from(o: NvmfTcpTransportOpts) -> Self {
struct_size_init!(
Self {
max_queue_depth: o.max_queue_depth,
max_qpairs_per_ctrlr: o.max_qpairs_per_ctrl,
in_capsule_data_size: o.in_capsule_data_size,
max_io_size: o.max_io_size,
io_unit_size: o.io_unit_size,
max_aq_depth: o.max_aq_depth,
num_shared_buffers: o.num_shared_buf,
buf_cache_size: o.buf_cache_size,
dif_insert_or_strip: o.dif_insert_or_strip,
reserved29: Default::default(),
abort_timeout_sec: o.abort_timeout_sec,
association_timeout: 120000,
transport_specific: std::ptr::null(),
acceptor_poll_rate: o.acceptor_poll_rate,
zcopy: o.zcopy,
reserved61: Default::default(),
ack_timeout: o.ack_timeout,
data_wr_pool_size: o.data_wr_pool_size,
},
opts_size
)
}
}

impl From<NvmfRdmaTransportOpts> for spdk_nvmf_transport_opts {
fn from(o: NvmfRdmaTransportOpts) -> Self {
impl From<NvmfTransportOpts> for spdk_nvmf_transport_opts {
fn from(o: NvmfTransportOpts) -> Self {
Self {
max_queue_depth: o.max_queue_depth,
max_qpairs_per_ctrlr: o.max_qpairs_per_ctrl,
Expand All @@ -414,6 +378,8 @@ impl From<NvmfRdmaTransportOpts> for spdk_nvmf_transport_opts {
opts_size: std::mem::size_of::<spdk_nvmf_transport_opts>() as u64,
acceptor_poll_rate: o.acceptor_poll_rate,
zcopy: o.zcopy,
ack_timeout: o.ack_timeout,
data_wr_pool_size: o.data_wr_pool_size,
reserved61: Default::default(),
}
}
Expand Down
4 changes: 2 additions & 2 deletions io-engine/src/subsys/nvmf/target.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ use spdk_rs::libspdk::{

use crate::{
constants::NVME_CONTROLLER_MODEL_ID,
core::{Cores, MayastorEnvironment, Mthread, Reactor, Reactors},
core::{Cores, MayastorEnvironment, Mthread, Reactors},
ffihelper::{AsStr, FfiResult},
subsys::{
nvmf::{
Expand Down Expand Up @@ -444,7 +444,7 @@ impl Target {

Reactors::master().send_future(async move {
let nqn = discovery.get_nqn();
if let Err(error) = discovery.start().await {
if let Err(error) = discovery.start(false).await {
error!("Error starting subsystem '{nqn}': {error}");
}

Expand Down

0 comments on commit 1b7f911

Please sign in to comment.