Merge #1691

1691: feat: add RDMA option to cli args and configs r=dsharma-dc a=dsharma-dc



Co-authored-by: Diwakar Sharma <[email protected]>

io-engine/src/bin/io-engine.rs

@@ -106,6 +106,11 @@ fn start_tokio_runtime(args: &MayastorCliArgs) {
         warn!("Nexus channel debug is enabled");
     }
 
+    if args.rdma {
+        env::set_var("ENABLE_RDMA", "true");
+        warn!("RDMA is enabled for Mayastor NVMEoF target");
+    }
+
     print_feature!("Async QPair connection", "spdk-async-qpair-connect");
     print_feature!("Fault injection", "fault-injection");
 

io-engine/src/core/env.rs

@@ -269,6 +269,9 @@ pub struct MayastorCliArgs {
     /// # Warning: Don't use this in production.
     #[clap(long, env = "MAYASTOR_DELAY", hide = true, value_parser = delay_compat)]
     pub developer_delay: bool,
+    /// Enables RDMA between initiator and Mayastor Nvmf target.
+    #[clap(long = "enable-rdma", env = "ENABLE_RDMA", value_parser = delay_compat)]
+    pub rdma: bool,
 }
 
 fn delay_compat(s: &str) -> Result<bool, String> {
@@ -336,6 +339,7 @@ impl Default for MayastorCliArgs {
             lvm: false,
             snap_rebuild: false,
             developer_delay: false,
+            rdma: false,
         }
     }
 }
@@ -416,6 +420,7 @@ pub struct MayastorEnvironment {
     skip_sig_handler: bool,
     enable_io_all_thrd_nexus_channels: bool,
     developer_delay: bool,
+    rdma: bool,
 }
 
 impl Default for MayastorEnvironment {
@@ -464,6 +469,7 @@ impl Default for MayastorEnvironment {
             skip_sig_handler: false,
             enable_io_all_thrd_nexus_channels: false,
             developer_delay: false,
+            rdma: false,
         }
     }
 }
@@ -605,6 +611,7 @@ impl MayastorEnvironment {
             api_versions: args.api_versions,
             skip_sig_handler: args.skip_sig_handler,
             developer_delay: args.developer_delay,
+            rdma: args.rdma,
             enable_io_all_thrd_nexus_channels: args
                 .enable_io_all_thrd_nexus_channels,
             ..Default::default()
@@ -958,7 +965,7 @@ impl MayastorEnvironment {
         if let Some(delay) = cfg.eal_opts.developer_delay {
             self.developer_delay = delay;
         }
-        if let Some(interface) = &cfg.nvmf_tcp_tgt_conf.interface {
+        if let Some(interface) = &cfg.nvmf_tgt_conf.interface {
             self.nvmf_tgt_interface = Some(interface.clone());
         }
         self.clone().setup_static();

io-engine/src/subsys/config/mod.rs

@@ -138,7 +138,7 @@ pub struct Config {
     pub source: Option<String>,
     /// these options are not set/copied but are applied
     /// on target creation.
-    pub nvmf_tcp_tgt_conf: NvmfTgtConfig,
+    pub nvmf_tgt_conf: NvmfTgtConfig,
     /// options specific to NVMe bdev types
     pub nvme_bdev_opts: NvmeBdevOpts,
     /// generic bdev options
@@ -208,7 +208,7 @@ impl Config {
         // are immutable, we can copy them with any locks held
         Config {
             source: self.source.clone(),
-            nvmf_tcp_tgt_conf: self.nvmf_tcp_tgt_conf.get(),
+            nvmf_tgt_conf: self.nvmf_tgt_conf.get(),
             nvme_bdev_opts: self.nvme_bdev_opts.get(),
             bdev_opts: self.bdev_opts.get(),
             nexus_opts: self.nexus_opts.get(),

io-engine/src/subsys/config/opts.rs

@@ -97,6 +97,8 @@ pub struct NvmfTgtConfig {
     pub opts: NvmfTcpTransportOpts,
     /// NVMF target interface (ip, mac, name or subnet).
     pub interface: Option<String>,
+    /// Enable RDMA for NVMF target or not
+    pub rdma: Option<bool>,
 }
 
 impl From<NvmfTgtConfig> for Box<spdk_nvmf_target_opts> {
@@ -121,6 +123,7 @@ impl Default for NvmfTgtConfig {
             crdt: args.nvmf_tgt_crdt,
             opts: NvmfTcpTransportOpts::default(),
             interface: None,
+            rdma: None,
         }
     }
 }

io-engine/src/subsys/nvmf/target.rs

@@ -109,7 +109,7 @@ impl Target {
     fn init(&mut self) -> Result<()> {
         let cfg = Config::get();
         let tgt_ptr: Box<spdk_nvmf_target_opts> =
-            cfg.nvmf_tcp_tgt_conf.clone().into();
+            cfg.nvmf_tgt_conf.clone().into();
 
         let tgt =
             unsafe { spdk_nvmf_tgt_create(&*tgt_ptr as *const _ as *mut _) };

io-engine/src/subsys/nvmf/transport.rs

@@ -34,7 +34,7 @@ static TCP_TRANSPORT: Lazy<CString> =
 
 pub async fn add_tcp_transport() -> Result<(), Error> {
     let cfg = Config::get();
-    let mut opts = cfg.nvmf_tcp_tgt_conf.opts.into();
+    let mut opts = cfg.nvmf_tgt_conf.opts.into();
     let transport = unsafe {
         spdk_nvmf_transport_create(TCP_TRANSPORT.as_ptr(), &mut opts)
     };

test/grpc/test_nexus.js

@@ -40,7 +40,7 @@ const configNvmfTarget = `
 nexus_opts:
   nvmf_nexus_port: 4422
   nvmf_replica_port: 8420
-nvmf_tcp_tgt_conf:
+nvmf_tgt_conf:
   max_namespaces: 2
 `;