diff --git a/Cargo.lock b/Cargo.lock index ce860f7a1..990216c53 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -380,6 +380,29 @@ dependencies = [ "which", ] +[[package]] +name = "bindgen" +version = "0.69.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" +dependencies = [ + "bitflags 2.4.0", + "cexpr", + "clang-sys", + "itertools 0.11.0", + "lazy_static", + "lazycell", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn 2.0.38", + "which", +] + [[package]] name = "bit-vec" version = "0.6.3" @@ -924,7 +947,7 @@ dependencies = [ name = "devinfo" version = "1.0.0" dependencies = [ - "bindgen", + "bindgen 0.68.1", "nix", "semver", "snafu", @@ -1891,7 +1914,7 @@ dependencies = [ name = "libnvme-rs" version = "0.1.0" dependencies = [ - "bindgen", + "bindgen 0.68.1", "cc", "glob", "libc", @@ -2990,10 +3013,10 @@ dependencies = [ [[package]] name = "spdk-rs" -version = "0.1.0" +version = "0.2.0" dependencies = [ "async-trait", - "bindgen", + "bindgen 0.69.4", "cc", "futures", "nix", diff --git a/ci.nix b/ci.nix index b0b4bbbb6..3fa791f12 100644 --- a/ci.nix +++ b/ci.nix @@ -1,129 +1,72 @@ -{ nospdk ? false, norust ? false, spdk_rel ? false, asan ? false }: +{ rust ? "stable" +, spdk ? "develop" +, spdk-path ? null +} @ args: let sources = import ./nix/sources.nix; + pkgs = import sources.nixpkgs { - overlays = - [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { }) ]; + overlays = [ + (_: _: { inherit sources; }) + (import ./nix/overlay.nix { }) + ]; }; -in -with pkgs; -let - nospdk_moth = - "You have requested environment without SPDK, you should provide it!"; - norust_moth = - "You have requested environment without RUST, you should provide it!"; - norustc_msg = "no rustc, use rustup tool to install it"; - channel = import ./nix/lib/rust.nix { inherit sources; }; + # python environment for test/python - pytest_inputs = python3.withPackages + pytest_inputs = with pkgs; python3.withPackages (ps: with ps; [ virtualenv grpcio grpcio-tools asyncssh black ]); - spdk = if (!spdk_rel) then libspdk-dev else libspdk; -in -mkShell { - name = "io-engine-dev-shell"; - # fortify does not work with -O0 which is used by spdk when --enable-debug - hardeningDisable = [ "fortify" ]; - buildInputs = [ - autoconf - automake - clang - cowsay - docker - docker-compose - e2fsprogs - etcd - fio - gdb - git - gnuplot - kubernetes-helm - libaio - libbsd - libnvme - libpcap - libunwind - liburing - llvmPackages.bintools - llvmPackages.libclang - meson - ninja - nodejs-16_x - numactl - nvme-cli - openssl - pkg-config - pre-commit - procps - pytest_inputs - python3 - udev - utillinux - xfsprogs - yasm - ] ++ (if (nospdk) then [ spdk.buildInputs ] else [ spdk ]) - ++ pkgs.lib.optional (!norust && asan) channel.asan - ++ pkgs.lib.optional (!norust && !asan) channel.stable - ++ pkgs.lib.optional (!norust) channel.nightly; - RUST_NIGHTLY_PATH = channel.nightly; - LIBCLANG_PATH = io-engine.LIBCLANG_PATH; - PROTOC = io-engine.PROTOC; - PROTOC_INCLUDE = io-engine.PROTOC_INCLUDE; - SPDK_PATH = if nospdk then null else "${spdk}"; - FIO_SPDK = if nospdk then null else "${spdk}/fio/spdk_nvme"; - ETCD_BIN = "${etcd}/bin/etcd"; - LVM_BINS = "${lvm2.bin}/bin"; + shellAttrs = import ./spdk-rs/nix/shell { + inherit rust; + inherit spdk; + inherit spdk-path; + inherit sources; + inherit pkgs; - IO_ENGINE_DIR = if asan then "target/x86_64-unknown-linux-gnu/debug" else "target/debug"; + cfg = { + buildInputs = with pkgs; [ + docker + docker-compose + e2fsprogs + etcd + gdb + git + gnuplot + kubernetes-helm + nodejs-16_x + numactl + pytest_inputs + udev + xfsprogs + ]; - # ASAN-related Cargo settings. - ASAN_ENABLE = if asan then "1" else null; - ASAN_OPTIONS = if asan then "detect_leaks=0" else null; - ASAN_BUILD_ENV = if asan then "shell" else null; - RUSTFLAGS = if asan then "-Zsanitizer=address" else null; - CARGO_BUILD_RUSTFLAGS = if asan then "-Zbuild-std" else null; - CARGO_BUILD_TARGET = if asan then "x86_64-unknown-linux-gnu" else null; - CARGO_PROFILE_DEV_PANIC = if asan then "unwind" else null; - RUST_BACKTRACE = if asan then "full" else null; + shellEnv = with pkgs; { + PROTOC = io-engine.PROTOC; + PROTOC_INCLUDE = io-engine.PROTOC_INCLUDE; + ETCD_BIN = "${etcd}/bin/etcd"; + LVM_BINS = "${lvm2.bin}/bin"; + }; - shellHook = '' - export FIO="$(which fio 2> /dev/null)" - ${pkgs.lib.optionalString (asan) "export LLVM_SYMBOLIZER_DIR=$(dirname $(realpath $(which llvm-symbolizer)))"} + shellHook = '' + # SRCDIR is needed by docker-compose files as it requires absolute paths + export SRCDIR=`pwd` - ${pkgs.lib.optionalString (asan) "echo 'AddressSanitizer is enabled, forcing nightly rustc.'"} - ${pkgs.lib.optionalString (asan) "echo ' ASAN_ENABLE :' $\{ASAN_ENABLE\}"} - ${pkgs.lib.optionalString (asan) "echo ' ASAN_OPTIONS :' $\{ASAN_OPTIONS\}"} - ${pkgs.lib.optionalString (asan) "echo ' RUSTFLAGS :' $\{RUSTFLAGS\}"} - ${pkgs.lib.optionalString (asan) "echo ' CARGO_BUILD_RUSTFLAGS :' $\{CARGO_BUILD_RUSTFLAGS\}"} - ${pkgs.lib.optionalString (asan) "echo ' CARGO_BUILD_TARGET :' $\{CARGO_BUILD_TARGET\}"} - ${pkgs.lib.optionalString (asan) "echo ' CARGO_PROFILE_DEV_PANIC :' $\{CARGO_PROFILE_DEV_PANIC\}"} - ${pkgs.lib.optionalString (asan) "echo ' RUST_BACKTRACE :' $\{RUST_BACKTRACE\}"} - ${pkgs.lib.optionalString (asan) "echo ' LLVM_SYMBOLIZER_DIR :' $\{LLVM_SYMBOLIZER_DIR\}"} - ${pkgs.lib.optionalString (asan) "echo"} + export PATH="$PATH:$(pwd)/scripts/nix-sudo" - echo 'FIO version :' $(fio --version 2> /dev/null) - echo 'FIO path :' $FIO - ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK version :' $(echo $SPDK_PATH | sed 's/.*libspdk-//g')"} - ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK path :' $SPDK_PATH"} - ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK FIO plugin :' $FIO_SPDK"} - echo 'LVM path :' $LVM_BINS - ${pkgs.lib.optionalString (!norust) "echo 'Rust version :' $(rustc --version 2> /dev/null || echo '${norustc_msg}')"} - ${pkgs.lib.optionalString (!norust) "echo 'Rust path :' $(which rustc 2> /dev/null || echo '${norustc_msg}')"} - echo 'I/O engine dir :' $IO_ENGINE_DIR - ${pkgs.lib.optionalString (nospdk) "cowsay ${nospdk_moth}"} - ${pkgs.lib.optionalString (nospdk) "export CFLAGS=-msse4"} - ${pkgs.lib.optionalString (nospdk) "echo"} - ${pkgs.lib.optionalString (norust) "cowsay ${norust_moth}"} - ${pkgs.lib.optionalString (norust) "echo 'Hint: use rustup tool.'"} - ${pkgs.lib.optionalString (norust) "echo"} + export IO_ENGINE_DIR="$RUST_TARGET_DEBUG" + ''; - # SRCDIR is needed by docker-compose files as it requires absolute paths - export SRCDIR=`pwd` - if [ -z "$CI" ]; then - echo - pre-commit install - pre-commit install --hook commit-msg - fi - export PATH=$PATH:$(pwd)/scripts/nix-sudo - ''; -} + shellInfoHook = '' + echo + echo "PROTOC : $PROTOC" + echo "PROTOC_INCLUDE : $PROTOC_INCLUDE" + echo "ETCD_BIN : $ETCD_BIN" + echo "LVM path : $LVM_BINS" + echo "I/O engine dir : $IO_ENGINE_DIR" + ''; + }; + }; +in + pkgs.mkShell shellAttrs // { + name = "io-engine-dev-shell"; + } diff --git a/ide.nix b/ide.nix new file mode 100644 index 000000000..18b89c00c --- /dev/null +++ b/ide.nix @@ -0,0 +1,9 @@ +{ rust ? "none" +, spdk ? "develop" +, spdk-path ? null +} @ args: + import ./ci.nix { + inherit rust; + inherit spdk; + inherit spdk-path; + } diff --git a/io-engine-tests/src/error_bdev.rs b/io-engine-tests/src/error_bdev.rs index c2d24ee4e..89a0d51c7 100644 --- a/io-engine-tests/src/error_bdev.rs +++ b/io-engine-tests/src/error_bdev.rs @@ -44,6 +44,7 @@ pub fn inject_error(error_device: &str, op: u32, mode: u32, count: u32) { error_num: count, corrupt_offset: 0, corrupt_value: 0, + error_qd: 0, }; unsafe { diff --git a/io-engine/src/bdev/null_bdev.rs b/io-engine/src/bdev/null_bdev.rs index 4e34c3e44..b1ace5b36 100644 --- a/io-engine/src/bdev/null_bdev.rs +++ b/io-engine/src/bdev/null_bdev.rs @@ -134,10 +134,12 @@ impl CreateDestroy for Null { }); } + let uuid = spdk_rs::Uuid::generate().into_raw(); + let cname = self.name.clone().into_cstring(); let opts = spdk_rs::libspdk::spdk_null_bdev_opts { name: cname.as_ptr(), - uuid: std::ptr::null(), + uuid: &uuid, num_blocks: self.num_blocks, block_size: self.blk_size, physical_block_size: 0, diff --git a/io-engine/src/bdev/nvme.rs b/io-engine/src/bdev/nvme.rs index 891e824fc..b6ec14b3e 100644 --- a/io-engine/src/bdev/nvme.rs +++ b/io-engine/src/bdev/nvme.rs @@ -10,13 +10,9 @@ use snafu::ResultExt; use url::Url; use spdk_rs::{ + bdevs::bdev_nvme_delete_async, ffihelper::copy_str_with_null, - libspdk::{ - bdev_nvme_create, - bdev_nvme_delete, - nvme_path_id, - spdk_nvme_transport_id, - }, + libspdk::{bdev_nvme_create, spdk_nvme_transport_id}, }; use crate::{ @@ -136,25 +132,20 @@ impl CreateDestroy for NVMe { if let Some(mut bdev) = UntypedBdev::lookup_by_name(&self.get_name()) { bdev.remove_alias(self.url.as_ref()); - let mut path_id = nvme_path_id::default(); - copy_str_with_null(&self.name, &mut path_id.trid.traddr); - path_id.trid.trtype = spdk_rs::libspdk::SPDK_NVME_TRANSPORT_PCIE; + let res = bdev_nvme_delete_async(&self.name, None).await; - let errno = unsafe { - bdev_nvme_delete( - self.name.clone().into_cstring().as_ptr(), - &path_id, - ) - }; - if errno != 0 { + // Restore the alias in the case the deletion failed. + if !res.is_ok_and(|e| e.is_ok()) { UntypedBdev::lookup_by_name(&self.get_name()) .map(|mut b| b.add_alias(self.url.as_ref())); } - errno_result_from_i32((), errno).context( - bdev_api::DestroyBdevFailed { - name: self.name.clone(), - }, - ) + + res.context(bdev_api::BdevCommandCanceled { + name: self.name.clone(), + })? + .context(bdev_api::DestroyBdevFailed { + name: self.name.clone(), + }) } else { Err(BdevError::BdevNotFound { name: self.get_name(), diff --git a/io-engine/src/bdev/nvmf.rs b/io-engine/src/bdev/nvmf.rs index 4aacc02c4..7da651c61 100644 --- a/io-engine/src/bdev/nvmf.rs +++ b/io-engine/src/bdev/nvmf.rs @@ -11,10 +11,10 @@ use snafu::ResultExt; use url::Url; use spdk_rs::{ + bdevs::bdev_nvme_delete_async, ffihelper::copy_str_with_null, libspdk::{ bdev_nvme_create, - bdev_nvme_delete, spdk_nvme_transport_id, SPDK_NVME_IO_FLAGS_PRCHK_GUARD, SPDK_NVME_IO_FLAGS_PRCHK_REFTAG, @@ -195,31 +195,46 @@ impl CreateDestroy for Nvmf { name: self.name.clone(), })?; + // Remove partially created nvme bdev which doesn't show up in + // the list of bdevs if bdev_count == 0 { - error!("No nvme bdev created, no namespaces?"); - // Remove partially created nvme bdev which doesn't show up in - // the list of bdevs - let errno = - unsafe { bdev_nvme_delete(cname.as_ptr(), std::ptr::null()) }; - info!( - "removed partially created bdev {}, returned {}", - self.name, errno + error!( + "No nvme bdev created after creating {n}, no namespaces?", + n = self.name ); + + match bdev_nvme_delete_async(&self.name, None).await { + Ok(_) => { + info!("Removed partially created bdev {n}", n = self.name) + } + Err(e) => { + error!( + "Failed to remove partially created bdev {n}: {e:?}", + n = self.name + ) + } + } + return Err(BdevError::BdevNotFound { name: self.name.clone(), }); } + if let Some(mut bdev) = UntypedBdev::lookup_by_name(&self.get_name()) { if let Some(u) = self.uuid { if bdev.uuid_as_string() != u.hyphenated().to_string() { - error!("Connected to device {} but expect to connect to {} instead", bdev.uuid_as_string(), u.hyphenated().to_string()); + error!( + "Connected to device {dev} but expect to connect to {s} instead", + dev = bdev.uuid_as_string(), + s = u.hyphenated().to_string(), + ); } }; if !bdev.add_alias(&self.alias) { error!( - "Failed to add alias {} to device {}", - self.alias, - self.get_name() + "Failed to add alias {alias} to device {name}", + alias = self.alias, + name = self.get_name() ); } }; @@ -235,20 +250,15 @@ impl CreateDestroy for Nvmf { match UntypedBdev::lookup_by_name(&self.get_name()) { Some(mut bdev) => { bdev.remove_alias(&self.alias); - let cname = CString::new(self.name.clone()).unwrap(); - - let errno = unsafe { - bdev_nvme_delete(cname.as_ptr(), std::ptr::null()) - }; - async { - errno_result_from_i32((), errno).context( - bdev_api::DestroyBdevFailed { - name: self.name.clone(), - }, - ) - } - .await + bdev_nvme_delete_async(&self.name, None) + .await + .context(bdev_api::BdevCommandCanceled { + name: self.name.clone(), + })? + .context(bdev_api::DestroyBdevFailed { + name: self.name.clone(), + }) } None => Err(BdevError::BdevNotFound { name: self.get_name(), diff --git a/io-engine/src/bdev/nvmx/controller.rs b/io-engine/src/bdev/nvmx/controller.rs index 1751a91f6..4878f3b17 100644 --- a/io-engine/src/bdev/nvmx/controller.rs +++ b/io-engine/src/bdev/nvmx/controller.rs @@ -4,6 +4,7 @@ use std::{ convert::From, fmt, + mem::zeroed, os::raw::c_void, ptr::NonNull, sync::{Arc, Mutex}, @@ -777,7 +778,7 @@ impl<'a> Drop for NvmeController<'a> { } extern "C" fn aer_cb(ctx: *mut c_void, cpl: *const spdk_nvme_cpl) { - let mut event = spdk_nvme_async_event_completion::default(); + let mut event: spdk_nvme_async_event_completion = unsafe { zeroed() }; if !nvme_cpl_succeeded(cpl) { warn!("AER request execute failed"); @@ -1017,7 +1018,7 @@ pub(crate) fn connected_attached_cb( pub(crate) mod options { use spdk_rs::ffihelper::copy_str_with_null; - use std::mem::size_of; + use std::mem::{size_of, zeroed}; use spdk_rs::libspdk::{ spdk_nvme_ctrlr_get_default_ctrlr_opts, @@ -1037,7 +1038,7 @@ pub(crate) mod options { impl Default for NvmeControllerOpts { fn default() -> Self { - let mut default = spdk_nvme_ctrlr_opts::default(); + let mut default: spdk_nvme_ctrlr_opts = unsafe { zeroed() }; unsafe { spdk_nvme_ctrlr_get_default_ctrlr_opts( diff --git a/io-engine/src/bdev/nvmx/handle.rs b/io-engine/src/bdev/nvmx/handle.rs index c138bf505..73a618dde 100644 --- a/io-engine/src/bdev/nvmx/handle.rs +++ b/io-engine/src/bdev/nvmx/handle.rs @@ -1,4 +1,9 @@ -use std::{alloc::Layout, mem::ManuallyDrop, os::raw::c_void, sync::Arc}; +use std::{ + alloc::Layout, + mem::{zeroed, ManuallyDrop}, + os::raw::c_void, + sync::Arc, +}; use async_trait::async_trait; use futures::channel::oneshot; @@ -1118,12 +1123,12 @@ impl BlockDeviceHandle for NvmeDeviceHandle { // Fill max-size ranges until the remaining blocks fit into one range. while remaining > SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS { unsafe { - let mut range = spdk_nvme_dsm_range::default(); - - range.attributes.raw = 0; - range.length = - SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS as u32; - range.starting_lba = offset; + let range = spdk_nvme_dsm_range { + attributes: zeroed(), + length: SPDK_NVME_DATASET_MANAGEMENT_RANGE_MAX_BLOCKS + as u32, + starting_lba: offset, + }; *dsm_ranges.add(range_id) = range; } @@ -1135,11 +1140,11 @@ impl BlockDeviceHandle for NvmeDeviceHandle { // Setup range that describes the remaining blocks and schedule unmap. let rc = unsafe { - let mut range = spdk_nvme_dsm_range::default(); - - range.attributes.raw = 0; - range.length = remaining as u32; - range.starting_lba = offset; + let range = spdk_nvme_dsm_range { + attributes: zeroed(), + length: remaining as u32, + starting_lba: offset, + }; *dsm_ranges.add(range_id) = range; diff --git a/io-engine/src/bdev/nvmx/qpair.rs b/io-engine/src/bdev/nvmx/qpair.rs index 141364ce9..9684c2ce7 100644 --- a/io-engine/src/bdev/nvmx/qpair.rs +++ b/io-engine/src/bdev/nvmx/qpair.rs @@ -24,6 +24,7 @@ use spdk_rs::libspdk::{ #[cfg(feature = "spdk-async-qpair-connect")] use std::{os::raw::c_void, time::Duration}; +use std::mem::zeroed; #[cfg(feature = "spdk-async-qpair-connect")] use spdk_rs::{ @@ -110,7 +111,7 @@ impl Drop for QPair { fn get_default_options( ctrlr_handle: SpdkNvmeController, ) -> spdk_nvme_io_qpair_opts { - let mut opts = spdk_nvme_io_qpair_opts::default(); + let mut opts: spdk_nvme_io_qpair_opts = unsafe { zeroed() }; let default_opts = nvme_bdev_running_config(); unsafe { diff --git a/io-engine/src/bin/spdk.rs b/io-engine/src/bin/spdk.rs index 13776741e..8862ecf78 100644 --- a/io-engine/src/bin/spdk.rs +++ b/io-engine/src/bin/spdk.rs @@ -10,6 +10,7 @@ use std::{ ffi::{c_void, CString}, io::{Error, ErrorKind}, iter::Iterator, + mem::zeroed, os::raw::{c_char, c_int}, ptr::null_mut, vec::Vec, @@ -35,7 +36,7 @@ fn main() -> Result<(), std::io::Error> { .collect::>(); c_args.push(std::ptr::null()); - let mut opts: spdk_app_opts = Default::default(); + let mut opts: spdk_app_opts = unsafe { zeroed() }; unsafe { spdk_app_opts_init( diff --git a/io-engine/src/core/env.rs b/io-engine/src/core/env.rs index e214b2238..154d8f0bf 100644 --- a/io-engine/src/core/env.rs +++ b/io-engine/src/core/env.rs @@ -20,25 +20,36 @@ use futures::{channel::oneshot, future}; use http::Uri; use once_cell::sync::{Lazy, OnceCell}; use snafu::Snafu; +use tokio::runtime::Builder; +use version_info::{package_description, version_info_str}; + use spdk_rs::{ libspdk::{ spdk_app_shutdown_cb, + spdk_env_dpdk_post_init, + spdk_env_fini, + spdk_log_close, spdk_log_level, spdk_log_open, + spdk_log_set_flag, spdk_log_set_level, spdk_log_set_print_level, spdk_pci_addr, + spdk_rpc_finish, + spdk_rpc_initialize, spdk_rpc_set_state, + spdk_subsystem_fini, + spdk_subsystem_init, spdk_thread_lib_fini, spdk_thread_send_critical_msg, + spdk_trace_cleanup, + spdk_env_dpdk_rte_eal_init, SPDK_LOG_DEBUG, SPDK_LOG_INFO, SPDK_RPC_RUNTIME, }, spdk_rs_log, }; -use tokio::runtime::Builder; -use version_info::{package_description, version_info_str}; use crate::{ bdev::{bdev_io_ctx_pool_init, nexus, nvme_io_ctx_pool_init}, @@ -345,26 +356,6 @@ pub static GLOBAL_RC: Lazy>> = pub static SIG_RECEIVED: Lazy = Lazy::new(|| AtomicBool::new(false)); -// FFI functions that are needed to initialize the environment -extern "C" { - pub fn rte_eal_init(argc: i32, argv: *mut *mut libc::c_char) -> i32; - pub fn spdk_trace_cleanup(); - pub fn spdk_env_dpdk_post_init(legacy_mem: bool) -> i32; - pub fn spdk_env_fini(); - pub fn spdk_log_close(); - pub fn spdk_log_set_flag(name: *const c_char, enable: bool) -> i32; - pub fn spdk_rpc_finish(); - pub fn spdk_rpc_initialize(listen: *mut libc::c_char); - pub fn spdk_subsystem_fini( - f: Option, - ctx: *mut c_void, - ); - pub fn spdk_subsystem_init( - f: Option, - ctx: *mut c_void, - ); -} - #[derive(Debug, Snafu)] pub enum EnvError { #[snafu(display("Failed to install signal handler"))] @@ -777,7 +768,7 @@ impl MayastorEnvironment { debug!("EAL arguments {:?}", args); if unsafe { - rte_eal_init( + spdk_env_dpdk_rte_eal_init( (cargs.len() as libc::c_int) - 1, cargs.as_ptr() as *mut *mut c_char, ) @@ -802,7 +793,7 @@ impl MayastorEnvironment { unsafe { for flag in &self.log_component { let cflag = CString::new(flag.as_str()).unwrap(); - if spdk_log_set_flag(cflag.as_ptr(), true) != 0 { + if spdk_log_set_flag(cflag.as_ptr()) != 0 { error!("Failed to set SPDK log flag: {:?}", cflag); } } @@ -928,7 +919,10 @@ impl MayastorEnvironment { } else { info!("RPC server listening at: {}", ctx.rpc.to_str().unwrap()); unsafe { - spdk_rpc_initialize(ctx.rpc.as_ptr() as *mut c_char); + spdk_rpc_initialize( + ctx.rpc.as_ptr() as *mut c_char, + std::ptr::null_mut(), + ); spdk_rpc_set_state(SPDK_RPC_RUNTIME); }; diff --git a/io-engine/src/lvs/lvol_snapshot.rs b/io-engine/src/lvs/lvol_snapshot.rs index 100bc4c6c..733ddf61f 100644 --- a/io-engine/src/lvs/lvol_snapshot.rs +++ b/io-engine/src/lvs/lvol_snapshot.rs @@ -1,6 +1,7 @@ use std::{ convert::TryFrom, ffi::{c_ushort, c_void, CString}, + mem::zeroed, os::raw::c_char, }; @@ -221,7 +222,7 @@ impl SnapshotOps for Lvol { cb_arg: *mut c_void, ) -> Result<(), LvsError> { let mut attr_descrs: [spdk_xattr_descriptor; SnapshotXattrs::COUNT] = - [spdk_xattr_descriptor::default(); SnapshotXattrs::COUNT]; + [unsafe { zeroed() }; SnapshotXattrs::COUNT]; // Vector to keep allocated CStrings before snapshot creation // is complete to guarantee validity of attribute buffers @@ -385,7 +386,7 @@ impl SnapshotOps for Lvol { cb_arg: *mut c_void, ) -> Result<(), LvsError> { let mut attr_descrs: [spdk_xattr_descriptor; CloneXattrs::COUNT] = - [spdk_xattr_descriptor::default(); CloneXattrs::COUNT]; + [unsafe { zeroed() }; CloneXattrs::COUNT]; // Vector to keep allocated CStrings before snapshot creation // is complete to guarantee validity of attribute buffers diff --git a/io-engine/src/subsys/config/mod.rs b/io-engine/src/subsys/config/mod.rs index 84c5b8c3a..38ddd2391 100644 --- a/io-engine/src/subsys/config/mod.rs +++ b/io-engine/src/subsys/config/mod.rs @@ -6,7 +6,7 @@ //! spell out the YAML spec for a given sub component. Serde will fill //! in the default when missing, which are defined within the individual //! options. -use std::{fmt::Display, fs, io::Write, path::Path}; +use std::{fmt::Display, fs, io::Write, mem::zeroed, path::Path}; use futures::FutureExt; use once_cell::sync::OnceCell; @@ -104,13 +104,16 @@ impl ConfigSubsystem { pub fn new() -> Self { static MAYASTOR_SUBSYS: &str = "MayastorConfig"; debug!("creating Mayastor subsystem..."); - let mut ss = Box::::default(); + let mut ss = spdk_subsystem { + name: unsafe { zeroed() }, + init: Some(Self::init), + fini: Some(Self::fini), + write_config_json: Some(Self::config), + tailq: unsafe { zeroed() }, + }; ss.name = std::ffi::CString::new(MAYASTOR_SUBSYS).unwrap().into_raw(); - ss.init = Some(Self::init); - ss.fini = Some(Self::fini); - ss.write_config_json = Some(Self::config); - Self(Box::into_raw(ss)) + Self(Box::into_raw(Box::new(ss))) } } diff --git a/io-engine/src/subsys/config/opts.rs b/io-engine/src/subsys/config/opts.rs index aece4fcda..7133f3c60 100644 --- a/io-engine/src/subsys/config/opts.rs +++ b/io-engine/src/subsys/config/opts.rs @@ -23,11 +23,14 @@ use spdk_rs::{ spdk_sock_impl_opts, spdk_sock_impl_set_opts, }, + struct_size_init, }; use std::{ convert::TryFrom, fmt::{Debug, Display}, + mem::zeroed, + ptr::null_mut, str::FromStr, }; @@ -98,11 +101,14 @@ pub struct NvmfTgtConfig { impl From for Box { fn from(o: NvmfTgtConfig) -> Self { - let mut out = Self::default(); + let mut out = spdk_nvmf_target_opts { + name: unsafe { zeroed() }, + max_subsystems: o.max_namespaces, + crdt: o.crdt, + discovery_filter: 0, + }; copy_str_with_null(&o.name, &mut out.name); - out.max_subsystems = o.max_namespaces; - out.crdt = o.crdt; - out + Box::new(out) } } @@ -346,7 +352,7 @@ pub struct NvmeBdevOpts { impl GetOpts for NvmeBdevOpts { fn get(&self) -> Self { - let opts = spdk_bdev_nvme_opts::default(); + let opts: spdk_bdev_nvme_opts = unsafe { zeroed() }; unsafe { bdev_nvme_get_opts(&opts as *const _ as *mut spdk_bdev_nvme_opts) }; @@ -466,6 +472,8 @@ impl From<&NvmeBdevOpts> for spdk_bdev_nvme_opts { nvme_error_stat: false, rdma_srq_size: 0, io_path_stat: false, + allow_accel_sequence: false, + rdma_max_cq_size: 0, } } } @@ -477,17 +485,26 @@ pub struct BdevOpts { bdev_io_pool_size: u32, /// number of bdev IO structures cached per thread bdev_io_cache_size: u32, + /// Size of the per-thread iobuf small cache. + iobuf_small_cache_size: u32, + /// Size of the per-thread iobuf large cache. + iobuf_large_cache_size: u32, } impl GetOpts for BdevOpts { fn get(&self) -> Self { - let mut opts = spdk_bdev_opts::default(); - unsafe { - spdk_bdev_get_opts( - &mut opts, - std::mem::size_of::() as u64, - ) - }; + let mut opts = struct_size_init!( + spdk_bdev_opts { + bdev_io_pool_size: 0, + bdev_io_cache_size: 0, + bdev_auto_examine: false, + reserved9: unsafe { zeroed() }, + iobuf_small_cache_size: 0, + iobuf_large_cache_size: 0, + }, + opts_size + ); + unsafe { spdk_bdev_get_opts(&mut opts, opts.opts_size) }; opts.into() } @@ -507,6 +524,8 @@ impl Default for BdevOpts { Self { bdev_io_pool_size: try_from_env("BDEV_IO_POOL_SIZE", 65535), bdev_io_cache_size: try_from_env("BDEV_IO_CACHE_SIZE", 512), + iobuf_small_cache_size: try_from_env("BUF_SMALL_CACHE_SIZE", 128), + iobuf_large_cache_size: try_from_env("BUF_LARGE_CACHE_SIZE", 16), } } } @@ -516,20 +535,25 @@ impl From for BdevOpts { Self { bdev_io_pool_size: o.bdev_io_pool_size, bdev_io_cache_size: o.bdev_io_cache_size, + iobuf_small_cache_size: o.iobuf_small_cache_size, + iobuf_large_cache_size: o.iobuf_large_cache_size, } } } impl From<&BdevOpts> for spdk_bdev_opts { fn from(o: &BdevOpts) -> Self { - Self { - bdev_io_pool_size: o.bdev_io_pool_size, - bdev_io_cache_size: o.bdev_io_cache_size, - bdev_auto_examine: false, - reserved9: Default::default(), - opts_size: std::mem::size_of::() as u64, - reserved: Default::default(), - } + struct_size_init!( + Self { + bdev_io_pool_size: o.bdev_io_pool_size, + bdev_io_cache_size: o.bdev_io_cache_size, + bdev_auto_examine: false, + reserved9: Default::default(), + iobuf_small_cache_size: o.iobuf_small_cache_size, + iobuf_large_cache_size: o.iobuf_large_cache_size, + }, + opts_size + ) } } @@ -573,7 +597,25 @@ impl Default for PosixSocketOpts { impl GetOpts for PosixSocketOpts { fn get(&self) -> Self { - let opts = spdk_sock_impl_opts::default(); + let opts = spdk_sock_impl_opts { + recv_buf_size: 0, + send_buf_size: 0, + enable_recv_pipe: false, + enable_zerocopy_send: false, + enable_quickack: false, + enable_placement_id: 0, + enable_zerocopy_send_server: false, + enable_zerocopy_send_client: false, + zerocopy_threshold: 0, + tls_version: 0, + enable_ktls: false, + psk_key: null_mut(), + psk_key_size: 0, + psk_identity: null_mut(), + get_key: None, + get_key_ctx: null_mut(), + tls_cipher_suites: null_mut(), + }; unsafe { let name = std::ffi::CString::new("posix").unwrap(); @@ -655,7 +697,12 @@ pub struct IoBufOpts { impl GetOpts for IoBufOpts { fn get(&self) -> Self { - let mut opts = spdk_iobuf_opts::default(); + let mut opts = spdk_iobuf_opts { + small_pool_count: 0, + large_pool_count: 0, + small_bufsize: 0, + large_bufsize: 0, + }; unsafe { spdk_iobuf_get_opts(&mut opts) }; opts.into() } diff --git a/io-engine/src/subsys/mod.rs b/io-engine/src/subsys/mod.rs index 0518e9c0d..ac6b09060 100644 --- a/io-engine/src/subsys/mod.rs +++ b/io-engine/src/subsys/mod.rs @@ -22,6 +22,7 @@ use spdk_rs::libspdk::{ spdk_add_subsystem_depend, spdk_subsystem_depend, }; +use std::mem::zeroed; pub use registration::{ registration_grpc::Registration, @@ -39,11 +40,13 @@ pub mod registration; pub(crate) fn register_subsystem() { unsafe { spdk_add_subsystem(ConfigSubsystem::new().0) } unsafe { - let mut depend = Box::::default(); - depend.name = b"mayastor_nvmf_tgt\0" as *const u8 as *mut _; - depend.depends_on = b"bdev\0" as *const u8 as *mut _; + let depend = spdk_subsystem_depend { + name: b"mayastor_nvmf_tgt\0" as *const u8 as *mut _, + depends_on: b"bdev\0" as *const u8 as *mut _, + tailq: zeroed(), + }; spdk_add_subsystem(Nvmf::new().0); - spdk_add_subsystem_depend(Box::into_raw(depend)); + spdk_add_subsystem_depend(Box::into_raw(Box::new(depend))); } RegistrationSubsystem::register(); } diff --git a/io-engine/src/subsys/nvmf/admin_cmd.rs b/io-engine/src/subsys/nvmf/admin_cmd.rs index 551393acc..c7dbe264b 100644 --- a/io-engine/src/subsys/nvmf/admin_cmd.rs +++ b/io-engine/src/subsys/nvmf/admin_cmd.rs @@ -37,9 +37,9 @@ use spdk_rs::{ spdk_nvmf_bdev_ctrlr_nvme_passthru_admin, spdk_nvmf_request, spdk_nvmf_request_complete, + spdk_nvmf_request_copy_to_buf, spdk_nvmf_request_get_bdev, spdk_nvmf_request_get_cmd, - spdk_nvmf_request_get_data, spdk_nvmf_request_get_response, spdk_nvmf_request_get_subsystem, spdk_nvmf_set_custom_admin_cmd_hdlr, @@ -131,38 +131,37 @@ pub fn set_snapshot_time(cmd: &mut spdk_nvme_cmd) -> u64 { fn decode_snapshot_params( req: *mut spdk_nvmf_request, ) -> Option { - let encoded_msg = unsafe { - let mut val = std::ptr::null_mut(); - let mut size: u32 = 0; + const ITEM_SZ: usize = std::mem::size_of::(); + + let mut val: Vec = Vec::with_capacity(ITEM_SZ * 2); - spdk_nvmf_request_get_data( + let encoded_msg = unsafe { + let bytes_copied = spdk_nvmf_request_copy_to_buf( req, - &mut val as *mut *mut c_void, - &mut size as *mut u32, - ); + val.as_mut_ptr() as _, + val.capacity() as u64, + ) as usize; info!( "## length = {}, iov_cnt = {}, size = {}", (*req).length, (*req).iovcnt, - size, + bytes_copied, ); - std::slice::from_raw_parts(val as *const u8, size as usize) + std::slice::from_raw_parts(val.as_ptr(), bytes_copied) }; + let decoded_msg = bincode::deserialize::(encoded_msg); + // Decode versioned snapshot creation request. - let decoded_msg = - match bincode::deserialize::(encoded_msg) { - Err(e) => { - error!( - "Failed to deserialize snapshot creation message: {}", - e - ); - return None; - } - Ok(msg) => msg, - }; + let decoded_msg = match decoded_msg { + Err(e) => { + error!("Failed to deserialize snapshot creation message: {:?}", e); + return None; + } + Ok(msg) => msg, + }; let snapshot_params = match decoded_msg { NvmeSnapshotMessage::V1(v1) => v1.params().clone(), diff --git a/io-engine/src/subsys/nvmf/mod.rs b/io-engine/src/subsys/nvmf/mod.rs index 2117a506d..3bdb0884d 100644 --- a/io-engine/src/subsys/nvmf/mod.rs +++ b/io-engine/src/subsys/nvmf/mod.rs @@ -8,7 +8,7 @@ //! //! As connections come on, we randomly schedule them across cores by putting //! the qpair in a poll group that is allocated during reactor start. -use std::cell::RefCell; +use std::{cell::RefCell, mem::zeroed}; use nix::errno::Errno; use snafu::Snafu; @@ -124,11 +124,13 @@ impl Nvmf { pub fn new() -> Self { debug!("Creating NVMF subsystem..."); - let mut ss = Box::::default(); - ss.name = b"mayastor_nvmf_tgt\x00" as *const u8 as *const libc::c_char; - ss.init = Some(Self::init); - ss.fini = Some(Self::fini); - ss.write_config_json = None; - Self(Box::into_raw(ss)) + let ss = spdk_subsystem { + name: b"mayastor_nvmf_tgt\x00" as *const u8 as *const libc::c_char, + init: Some(Self::init), + fini: Some(Self::fini), + write_config_json: None, + tailq: unsafe { zeroed() }, + }; + Self(Box::into_raw(Box::new(ss))) } } diff --git a/io-engine/src/subsys/nvmf/subsystem.rs b/io-engine/src/subsys/nvmf/subsystem.rs index e913580b2..c9e26acf8 100644 --- a/io-engine/src/subsys/nvmf/subsystem.rs +++ b/io-engine/src/subsys/nvmf/subsystem.rs @@ -2,7 +2,7 @@ use std::{ convert::TryFrom, ffi::{c_void, CString}, fmt::{self, Debug, Display, Formatter}, - mem::size_of, + mem::zeroed, ptr::{self, NonNull}, }; @@ -12,9 +12,7 @@ use nix::errno::Errno; use spdk_rs::{ libspdk::{ nvmf_subsystem_find_listener, - nvmf_subsystem_set_ana_state, nvmf_subsystem_set_cntlid_range, - spdk_bdev_nvme_opts, spdk_nvmf_ctrlr_set_cpl_error_cb, spdk_nvmf_ns_get_bdev, spdk_nvmf_ns_opts, @@ -42,6 +40,7 @@ use spdk_rs::{ spdk_nvmf_subsystem_resume, spdk_nvmf_subsystem_set_allow_any_host, spdk_nvmf_subsystem_set_ana_reporting, + spdk_nvmf_subsystem_set_ana_state, spdk_nvmf_subsystem_set_event_cb, spdk_nvmf_subsystem_set_mn, spdk_nvmf_subsystem_set_sn, @@ -55,6 +54,7 @@ use spdk_rs::{ SPDK_NVMF_SUBTYPE_DISCOVERY, SPDK_NVMF_SUBTYPE_NVME, }, + struct_size_init, NvmeStatus, NvmfController, NvmfSubsystemEvent, @@ -519,10 +519,19 @@ impl NvmfSubsystem { where T: spdk_rs::BdevOps, { - let opts = spdk_nvmf_ns_opts { - nguid: *bdev.uuid().as_bytes(), - ..Default::default() - }; + let opts = struct_size_init!( + spdk_nvmf_ns_opts { + nsid: 0, + nguid: *bdev.uuid().as_bytes(), + eui64: unsafe { zeroed() }, + uuid: Default::default(), + reserved44: unsafe { zeroed() }, + anagrpid: 0, + reserved60: unsafe { zeroed() }, + }, + opts_size + ); + let bdev_cname = CString::new(bdev.name()).unwrap(); let ptpl = ptpl.map(|ptpl| { CString::new(ptpl.to_string_lossy().to_string()).unwrap() @@ -536,7 +545,7 @@ impl NvmfSubsystem { self.0.as_ptr(), bdev_cname.as_ptr(), &opts as *const _, - size_of::() as u64, + opts.opts_size, ptpl_ptr, ) }; @@ -979,7 +988,7 @@ impl NvmfSubsystem { let (s, r) = oneshot::channel::(); unsafe { - nvmf_subsystem_set_ana_state( + spdk_nvmf_subsystem_set_ana_state( self.0.as_ptr(), trid_replica.as_ptr(), ana_state, diff --git a/io-engine/src/subsys/nvmf/target.rs b/io-engine/src/subsys/nvmf/target.rs index 56c23f281..7607242f0 100644 --- a/io-engine/src/subsys/nvmf/target.rs +++ b/io-engine/src/subsys/nvmf/target.rs @@ -1,7 +1,8 @@ use std::{ cell::RefCell, ffi::{c_void, CString}, - ptr::NonNull, + mem::zeroed, + ptr::{null, NonNull}, }; use nix::errno::Errno; @@ -225,7 +226,13 @@ impl Target { fn listen(&mut self) -> Result<()> { let cfg = Config::get(); let trid_nexus = TransportId::new(cfg.nexus_opts.nvmf_nexus_port); - let mut opts = spdk_nvmf_listen_opts::default(); + let mut opts = spdk_nvmf_listen_opts { + opts_size: 0, + transport_specific: null(), + secure_channel: false, + reserved1: unsafe { zeroed() }, + ana_state: 0, + }; unsafe { spdk_nvmf_listen_opts_init( &mut opts, diff --git a/io-engine/src/subsys/registration/mod.rs b/io-engine/src/subsys/registration/mod.rs index 16b77e545..4b82917f0 100644 --- a/io-engine/src/subsys/registration/mod.rs +++ b/io-engine/src/subsys/registration/mod.rs @@ -13,7 +13,7 @@ use spdk_rs::libspdk::{ spdk_subsystem_fini_next, spdk_subsystem_init_next, }; -use std::convert::TryFrom; +use std::{convert::TryFrom, mem::zeroed}; macro_rules! default_addr { () => { @@ -70,13 +70,15 @@ impl RegistrationSubsystem { fn new() -> Self { info!("creating Mayastor registration subsystem..."); - let mut ss = Box::::default(); - ss.name = b"mayastor_grpc_registration\x00" as *const u8 - as *const libc::c_char; - ss.init = Some(Self::init); - ss.fini = Some(Self::fini); - ss.write_config_json = None; - Self(Box::into_raw(ss)) + let ss = spdk_subsystem { + name: b"mayastor_grpc_registration\x00" as *const u8 + as *const libc::c_char, + init: Some(Self::init), + fini: Some(Self::fini), + write_config_json: None, + tailq: unsafe { zeroed() }, + }; + Self(Box::into_raw(Box::new(ss))) } /// register the subsystem with spdk diff --git a/io-engine/tests/block_device_nvmf.rs b/io-engine/tests/block_device_nvmf.rs index 919f0223f..4408566f2 100755 --- a/io-engine/tests/block_device_nvmf.rs +++ b/io-engine/tests/block_device_nvmf.rs @@ -38,6 +38,7 @@ use std::{ use spdk_rs::{AsIoVecs, DmaBuf}; pub mod common; + use io_engine::{ constants::NVME_CONTROLLER_MODEL_ID, core::{DeviceEventListener, DeviceEventSink, ReadOptions}, @@ -266,7 +267,7 @@ fn flag_callback_invocation() { false, true, Ordering::Acquire, - Ordering::Relaxed + Ordering::Relaxed, ), Ok(false), "Callback is called more than once" @@ -279,7 +280,7 @@ fn check_callback_invocation() { true, false, Ordering::Acquire, - Ordering::Relaxed + Ordering::Relaxed, ), Ok(true), "Callback has not been called" @@ -1899,5 +1900,5 @@ async fn nvmf_device_hot_remove() { .await .expect_err("Device has been successfully created for controller without namespaces"); }) - .await; + .await; } diff --git a/nix/lib/rust.nix b/nix/lib/rust.nix deleted file mode 100644 index 4ad077933..000000000 --- a/nix/lib/rust.nix +++ /dev/null @@ -1,12 +0,0 @@ -{ sources ? import ../sources.nix }: -let - pkgs = - import sources.nixpkgs { overlays = [ (import sources.rust-overlay) ]; }; - nightly_version = "2023-08-25"; - stable_version = "1.72.0"; -in -with pkgs; rec { - nightly = rust-bin.nightly.${nightly_version}.default; - stable = rust-bin.stable.${stable_version}.default; - asan = rust-bin.nightly.${nightly_version}.default; -} diff --git a/nix/overlay.nix b/nix/overlay.nix index a23923cdc..2b08b995d 100644 --- a/nix/overlay.nix +++ b/nix/overlay.nix @@ -4,19 +4,14 @@ let img_prefix = if product_prefix == "" then config.product_prefix else product_prefix; in self: super: rec { - fio = super.callPackage ./pkgs/fio { }; sourcer = super.callPackage ./lib/sourcer.nix { }; images = super.callPackage ./pkgs/images { inherit img_tag img_org img_prefix; }; - libnvme = super.callPackage ./pkgs/libnvme { }; - libspdk = (super.callPackage ./pkgs/libspdk { with-fio = false; }).release; - libspdk-fio = (super.callPackage ./pkgs/libspdk { with-fio = true; multi-outputs = true; }).release; - libspdk-dev = (super.callPackage ./pkgs/libspdk { with-fio = true; }).debug; io-engine = (super.callPackage ./pkgs/io-engine { inherit tag sourcer; }).release; io-engine-adhoc = (super.callPackage ./pkgs/io-engine { inherit tag; }).adhoc; io-engine-dev = (super.callPackage ./pkgs/io-engine { inherit tag; }).debug; mkContainerEnv = super.callPackage ./lib/mkContainerEnv.nix { }; ms-buildenv = super.callPackage ./pkgs/ms-buildenv { }; - nvme-cli = super.callPackage ./pkgs/nvme-cli { }; nvmet-cli = super.callPackage ./pkgs/nvmet-cli { }; units = (super.callPackage ./pkgs/io-engine/units.nix { inherit tag sourcer; }); } +// (import ../spdk-rs/nix/overlay.nix { } self super) diff --git a/nix/pkgs/fio/default.nix b/nix/pkgs/fio/default.nix deleted file mode 100644 index bea485c9e..000000000 --- a/nix/pkgs/fio/default.nix +++ /dev/null @@ -1,58 +0,0 @@ -{ lib -, stdenv -, fetchFromGitHub -, makeWrapper -, libaio -, python3 -, zlib -, withGnuplot ? false -, gnuplot ? null -}: - -stdenv.mkDerivation rec { - pname = "fio"; - version = "3.37"; - - src = fetchFromGitHub { - owner = "axboe"; - repo = "fio"; - rev = "fio-${version}"; - sha256 = "sha256-dKHTxVglH10aV44RuSeIFATn83DVdmCYtuaiS3b0+zo="; - }; - - buildInputs = [ python3 zlib ] - ++ lib.optional (!stdenv.isDarwin) libaio; - - nativeBuildInputs = [ makeWrapper ]; - - strictDeps = true; - - enableParallelBuilding = true; - - postPatch = '' - substituteInPlace Makefile \ - --replace "mandir = /usr/share/man" "mandir = \$(prefix)/man" \ - --replace "sharedir = /usr/share/fio" "sharedir = \$(prefix)/share/fio" - substituteInPlace tools/plot/fio2gnuplot --replace /usr/share/fio $out/share/fio - ''; - - preInstall = '' - mkdir -p $dev/include - cp -p --parents $(find . -name "*.h") $dev/include - ''; - - postInstall = lib.optionalString withGnuplot '' - wrapProgram $out/bin/fio2gnuplot \ - --prefix PATH : ${lib.makeBinPath [ gnuplot ]} - ''; - - outputs = [ "out" "dev" ]; - setOutputFlags = false; - - meta = with lib; { - description = "Flexible IO Tester - an IO benchmark tool"; - homepage = "https://git.kernel.dk/cgit/fio/"; - license = licenses.gpl2; - platforms = platforms.unix; - }; -} diff --git a/nix/pkgs/io-engine/cargo-package.nix b/nix/pkgs/io-engine/cargo-package.nix index bf03e2518..47282fa32 100644 --- a/nix/pkgs/io-engine/cargo-package.nix +++ b/nix/pkgs/io-engine/cargo-package.nix @@ -5,6 +5,7 @@ , lib , libaio , libbsd +, libexecinfo , libnvme , libspdk , libspdk-dev @@ -30,7 +31,7 @@ }: let version = versions.version; - channel = import ../../lib/rust.nix { inherit sources; }; + channel = import ../../../spdk-rs/nix/lib/rust.nix { inherit sources; }; rustPlatform = makeRustPlatform { rustc = channel.stable; cargo = channel.stable; @@ -75,6 +76,7 @@ let buildInputs = [ libaio libbsd + libexecinfo libnvme libpcap libunwind @@ -102,12 +104,12 @@ in cargoBuildFlags = "--bin io-engine --bin io-engine-client --bin casperf"; buildType = "release"; buildInputs = buildProps.buildInputs ++ [ libspdk ]; - SPDK_PATH = "${libspdk}"; + SPDK_ROOT_DIR = "${libspdk}"; }); debug = rustPlatform.buildRustPackage (buildProps // { cargoBuildFlags = "--workspace --bins --exclude io-engine-bench"; buildType = "debug"; buildInputs = buildProps.buildInputs ++ [ libspdk-dev ]; - SPDK_PATH = "${libspdk-dev}"; + SPDK_ROOT_DIR = "${libspdk-dev}"; }); } diff --git a/nix/pkgs/libnvme/default.nix b/nix/pkgs/libnvme/default.nix deleted file mode 100644 index c434f8b2b..000000000 --- a/nix/pkgs/libnvme/default.nix +++ /dev/null @@ -1,19 +0,0 @@ -{ stdenv, lib, fetchgit, json_c, libuuid, meson, ninja, openssl, pkg-config, python3, sources }: - -stdenv.mkDerivation rec { - version = sources.libnvme.rev; - name = "libnvme-${version}"; - - src = sources.libnvme; - - nativeBuildInputs = [ json_c libuuid meson ninja openssl pkg-config python3 ]; - meta = { - description = "Userspace NVMe library"; - longDescription = '' - This is the libnvme development C library. libnvme provides type defintions for NVMe specification structures, enumerations, and bit fields, helper functions to construct, dispatch, and decode commands and payloads, and utilities to connect, scan, and manage nvme devices on a Linux system. - ''; - homepage = "https://github.com/linux-nvme/libnvme"; - licenses = lib.licenses.lgpl21Plus; - maintainers = [ "jonathan.teh@mayadata.io" ]; - }; -} diff --git a/nix/pkgs/libspdk/default.nix b/nix/pkgs/libspdk/default.nix deleted file mode 100644 index 1f6797466..000000000 --- a/nix/pkgs/libspdk/default.nix +++ /dev/null @@ -1,218 +0,0 @@ -{ binutils -, cunit -, fetchFromGitHub -, pkg-config -, lcov -, lib -, libaio -, libbpf -, libbsd -, libelf -, libexecinfo -, libpcap -, liburing -, libuuid -, nasm -, cmake -, fio -, ninja -, jansson -, meson -, ncurses -, numactl -, openssl -, python3 -, stdenv -, libtool -, yasm -, targetPlatform -, buildPlatform -, buildPackages -, llvmPackages -, pkgs -, gcc -, zlib -, autoconf -, automake -, with-fio -, multi-outputs ? false -}: -let - fio-include = "${fio.dev}/include"; - fio-output = if multi-outputs then "fio" else "out"; - - # Derivation attributes for production version of libspdk - # - # How to update to new libspdk commit: - # 1. Push your SPDK commit to openebs repo. - # 2. Copy git commit hash to 'rev' field, and copy first 9 digits of it - # to 'version' field. - # 3. Leave old SHA256 intact. - # 4. Login to your machine and run 'sudo nix store gc' to get rid of stale - # copies of SPDK package. - # 5. Enter your mayastor 'nix-shell'. - # 6. Wait until SPDK pkg build fails with - # "hash mismatch in fixed-output derivation" error. - # 7. Copy SHA256 from 'got' of the error message to 'sha256' field. - # 8. 'nix-shell' build must now succeed. - drvAttrs = rec { - version = "23.05-a56cdf9"; - - src = fetchFromGitHub { - owner = "openebs"; - repo = "spdk"; - rev = "a56cdf9f8e3c0fe7db82dcadfdb6806743167635"; - sha256 = "sha256-Dgnn09dUaAUS6d+Og/DLcIhe5xeHsDBDydHuNyE3pIo="; - fetchSubmodules = true; - }; - - nativeBuildInputs = [ - meson - ninja - pkg-config - python3 - llvmPackages.clang - gcc - cmake - ]; - - buildInputs = [ - autoconf - automake - binutils - jansson - libaio - libbpf - libbsd - libelf - libexecinfo - libpcap - libtool - liburing - libuuid - nasm - ncurses - numactl - openssl - (python3.withPackages (ps: with ps; [ pyelftools ])) - yasm - zlib - ]; - - configureFlags = (if (targetPlatform.config == "x86_64-unknown-linux-gnu") then - [ - "--target-arch=nehalem" - "--without-shared" - "--without-crypto" - ] - else if (targetPlatform.config == "aarch64-unknown-linux-gnu") then - [ - "--target-arch=armv8-a+crypto" - ] - else - [ ] - ) ++ - (if (targetPlatform.config != buildPlatform.config) then [ "--cross-prefix=${targetPlatform.config}" ] else [ ]) ++ - (if with-fio then [ "--with-fio=${fio-include}" ] else [ ]) ++ - [ - "--with-uring" - "--without-uring-zns" - "--disable-unit-tests" - "--disable-tests" - ]; - - configurePhase = '' - patchShebangs ./. > /dev/null - export AS=yasm - ./configure ${builtins.concatStringsSep " " configureFlags} - ''; - enableParallelBuilding = true; - - hardeningDisable = [ "all" ]; - - buildPhase = (if (targetPlatform.config == "aarch64-unknown-linux-gnu") then - [ - "DPDKBUILD_FLAGS=-Dplatform=generic" - ] - else - [ ] - ) ++ - [ - "make -j`nproc`" - ]; - - installPhase = '' - echo "installing SPDK to $out" - mkdir -p $out/lib/pkgconfig - mkdir $out/bin - - pushd include - find . -type f -name "*.h" -exec install -vD "{}" $out/include/{} \; - popd - - pushd lib - find . -type f -name "*.h" -exec install -vD "{}" $out/include/spdk/lib/{} \; - popd - - # copy private headers from bdev modules needed for creating of bdevs - pushd module - find . -type f -name "*.h" -exec install -vD "{}" $out/include/spdk/module/{} \; - popd - - find . -executable -type f -name 'bdevperf' -exec install -vD "{}" $out/bin \; - - # copy libraries - install -v build/lib/*.a $out/lib/ - install -v build/lib/pkgconfig/*.pc $out/lib/pkgconfig/ - install -v dpdk/build/lib/*.a $out/lib/ - install -v dpdk/build/lib/pkgconfig/*.pc $out/lib/pkgconfig/ - '' + lib.optionalString targetPlatform.isx86_64 '' - install -v isa-l/.libs/*.a $out/lib/ - install -v isa-l/*.pc $out/lib/pkgconfig/ - install -v isa-l-crypto/.libs/*.a $out/lib/ - install -v isa-l-crypto/*.pc $out/lib/pkgconfig/ - - # fix paths in pkg config files - build_dir=`pwd` - for i in `ls $out/lib/pkgconfig/*.pc`; - do - echo "fixing pkg config paths in '$i' ..." - sed -i "s,$build_dir/build/lib,$out/lib,g" $i - sed -i "s,$build_dir/dpdk/build,$out,g" $i - sed -i "s,$build_dir/intel-ipsec-mb/lib,$out/lib,g" $i - done - '' + lib.optionalString (with-fio && !multi-outputs) '' - mkdir $out/fio - cp build/fio/spdk_* $out/fio - '' + lib.optionalString (with-fio && multi-outputs) '' - mkdir $fio - cp build/fio/spdk_* $fio - ''; - - outputs = [ "out" ] ++ lib.optional (fio-output != "out") fio-output; - }; -in -{ - release = llvmPackages.stdenv.mkDerivation (drvAttrs // { - pname = "libspdk"; - dontStrip = false; - }); - debug = llvmPackages.stdenv.mkDerivation (drvAttrs // { - pname = "libspdk-dev"; - dontStrip = true; - nativeBuildInputs = drvAttrs.nativeBuildInputs ++ [ cunit lcov ]; - buildInputs = drvAttrs.buildInputs ++ [ cunit lcov ]; - configurePhase = '' - patchShebangs ./. > /dev/null - export AS=yasm - ./configure ${builtins.concatStringsSep " " (drvAttrs.configureFlags ++ - [ - "--enable-debug" - ])} - ''; - installPhase = drvAttrs.installPhase + '' - echo "Copying test files" - cp -ar test $out/test - ''; - }); -} diff --git a/nix/pkgs/nvme-cli/default.nix b/nix/pkgs/nvme-cli/default.nix deleted file mode 100644 index 2c8bbf038..000000000 --- a/nix/pkgs/nvme-cli/default.nix +++ /dev/null @@ -1,23 +0,0 @@ -{ stdenv, lib, libuuid, pkg-config, sources }: - -stdenv.mkDerivation rec { - version = sources.nvme-cli.rev; - name = "nvme-cli-${version}"; - src = sources.nvme-cli; - - nativeBuildInputs = [ pkg-config ]; - buildInputs = [ libuuid ]; - - makeFlags = [ "DESTDIR=$(out)" "PREFIX=" ]; - - # To omit the hostnqn and hostid files that are impure and should be unique - # for each target host: - installTargets = [ "install-spec" ]; - - meta = with lib; { - description = sources.nvme-cli.description; - homepage = sources.nvme-cli.homepage; - license = licenses.gpl2Plus; - platforms = platforms.linux; - }; -} diff --git a/nix/sources.json b/nix/sources.json deleted file mode 100644 index 00b706d74..000000000 --- a/nix/sources.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "libnvme": { - "branch": "master", - "description": "nvme client library", - "homepage": "https://github.com/linux-nvme/libnvme", - "owner": "linux-nvme", - "repo": "libnvme", - "rev": "v1.0-rc3", - "sha256": "0wkr0axmm4wqjncb6x7fjyg0wxjdi9f2qd3df6ph671qh5ralrjy", - "type": "tarball", - "url": "https://github.com/linux-nvme/libnvme/archive/v1.0-rc3.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "nvme-cli": { - "branch": "master", - "description": "NVM-Express user space tooling for Linux.", - "homepage": "https://github.com/linux-nvme/nvme-cli", - "owner": "linux-nvme", - "repo": "nvme-cli", - "rev": "v1.16", - "sha256": "130x5cf6kkcnyg5qd35igii249ysfjnbxp1pxfwkickmqg3d007z", - "type": "tarball", - "url": "https://github.com/linux-nvme/nvme-cli/archive/refs/tags/v1.16.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "niv": { - "branch": "master", - "description": "Easy dependency management for Nix projects", - "homepage": "https://github.com/nmattia/niv", - "owner": "nmattia", - "repo": "niv", - "rev": "1819632b5823e0527da28ad82fecd6be5136c1e9", - "sha256": "08jz17756qchq0zrqmapcm33nr4ms9f630mycc06i6zkfwl5yh5i", - "type": "tarball", - "url": "https://github.com/nmattia/niv/archive/1819632b5823e0527da28ad82fecd6be5136c1e9.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "nixpkgs": { - "branch": "master", - "description": "A read-only mirror of NixOS/nixpkgs tracking the released channels. Send issues and PRs to", - "homepage": "https://github.com/NixOS/nixpkgs", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "22.11", - "sha256": "11w3wn2yjhaa5pv20gbfbirvjq6i3m7pqrq2msf0g7cv44vijwgw", - "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/22.11.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - }, - "rust-overlay": { - "branch": "master", - "description": "Pure and reproducible nix overlay for binary distributed rust toolchains", - "homepage": "", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "673e2d3d2a3951adc6f5e3351c9fce6ad130baed", - "sha256": "sha256:1vgjkaikv8lwm3kmb4jbc96kxhdy38nmf1v4s7nmx6j4bd8pmlyd", - "type": "tarball", - "url": "https://github.com/oxalica/rust-overlay/archive/673e2d3d2a3951adc6f5e3351c9fce6ad130baed.tar.gz", - "url_template": "https://github.com///archive/.tar.gz" - } -} diff --git a/nix/sources.nix b/nix/sources.nix index ddf8be247..28a48153a 100644 --- a/nix/sources.nix +++ b/nix/sources.nix @@ -1,136 +1,3 @@ -# This file has been generated by Niv. -let - # - # The fetchers. fetch_ fetches specs of type . - # - fetch_file = pkgs: spec: - if spec.builtin or true then - builtins_fetchurl { inherit (spec) url sha256; } - else - pkgs.fetchurl { inherit (spec) url sha256; }; - - fetch_tarball = pkgs: spec: - if spec.builtin or true then - builtins_fetchTarball { inherit (spec) url sha256; } - else - pkgs.fetchzip { inherit (spec) url sha256; }; - - fetch_git = spec: - builtins.fetchGit { url = spec.repo; inherit (spec) rev ref; }; - - fetch_builtin-tarball = spec: - builtins.trace - '' - WARNING: - The niv type "builtin-tarball" will soon be deprecated. You should - instead use `builtin = true`. - - $ niv modify -a type=tarball -a builtin=true - '' - builtins_fetchTarball - { inherit (spec) url sha256; }; - - fetch_builtin-url = spec: - builtins.trace - '' - WARNING: - The niv type "builtin-url" will soon be deprecated. You should - instead use `builtin = true`. - - $ niv modify -a type=file -a builtin=true - '' - (builtins_fetchurl { inherit (spec) url sha256; }); - - # - # Various helpers - # - - # The set of packages used when specs are fetched using non-builtins. - mkPkgs = sources: - let - sourcesNixpkgs = - import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { }; - hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; - hasThisAsNixpkgsPath = == ./.; - in - if builtins.hasAttr "nixpkgs" sources - then sourcesNixpkgs - else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then - import { } - else - abort - '' - Please specify either (through -I or NIX_PATH=nixpkgs=...) or - add a package called "nixpkgs" to your sources.json. - ''; - - # The actual fetching function. - fetch = pkgs: name: spec: - - if ! builtins.hasAttr "type" spec then - abort "ERROR: niv spec ${name} does not have a 'type' attribute" - else if spec.type == "file" then fetch_file pkgs spec - else if spec.type == "tarball" then fetch_tarball pkgs spec - else if spec.type == "git" then fetch_git spec - else if spec.type == "builtin-tarball" then fetch_builtin-tarball spec - else if spec.type == "builtin-url" then fetch_builtin-url spec - else - abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; - - # Ports of functions for older nix versions - - # a Nix version of mapAttrs if the built-in doesn't exist - mapAttrs = builtins.mapAttrs or ( - f: set: with builtins; - listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) - ); - - # fetchTarball version that is compatible between all the versions of Nix - builtins_fetchTarball = { url, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchTarball; - in - if lessThan nixVersion "1.12" then - fetchTarball { inherit url; } - else - fetchTarball attrs; - - # fetchurl version that is compatible between all the versions of Nix - builtins_fetchurl = { url, sha256 }@attrs: - let - inherit (builtins) lessThan nixVersion fetchurl; - in - if lessThan nixVersion "1.12" then - fetchurl { inherit url; } - else - fetchurl attrs; - - # Create the final "sources" from the config - mkSources = config: - mapAttrs - ( - name: spec: - if builtins.hasAttr "outPath" spec - then - abort - "The values in sources.json should not have an 'outPath' attribute" - else - spec // { outPath = fetch config.pkgs name spec; } - ) - config.sources; - - # The "config" used by the fetchers - mkConfig = - { sourcesFile ? ./sources.json - , sources ? builtins.fromJSON (builtins.readFile sourcesFile) - , pkgs ? mkPkgs sources - }: rec { - # The sources, i.e. the attribute set of spec name to spec - inherit sources; - - # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers - inherit pkgs; - }; -in -mkSources (mkConfig { }) // { __functor = _: settings: mkSources (mkConfig settings); } +# Redirects to `spdk-rs`. +import ../spdk-rs/nix/sources.nix diff --git a/shell.nix b/shell.nix index f2f95bc53..7088c8aa2 100644 --- a/shell.nix +++ b/shell.nix @@ -1,92 +1,9 @@ -{ nospdk ? false, spdk_rel ? false }: -let - sources = import ./nix/sources.nix; - pkgs = import sources.nixpkgs { - overlays = - [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { }) ]; - }; -in -with pkgs; -let - nospdk_moth = - "You have requested environment without SPDK, you should provide it!"; - norustc_msg = "no rustc, use rustup tool to install it"; - channel = import ./nix/lib/rust.nix { inherit sources; }; - # python environment for test/python - pytest_inputs = python3.withPackages - (ps: with ps; [ virtualenv grpcio grpcio-tools asyncssh black ]); - spdk = if (!spdk_rel) then libspdk-dev else libspdk; -in -mkShell { - name = "io-engine-dev-shell"; - # fortify does not work with -O0 which is used by spdk when --enable-debug - hardeningDisable = [ "fortify" ]; - buildInputs = [ - autoconf - automake - clang - cowsay - etcd - fio - gnuplot - libaio - libbsd - libnvme - libpcap - libunwind - liburing - llvmPackages.bintools - llvmPackages.libclang - meson - ninja - nodejs-16_x - numactl - nvme-cli - openssl - pkg-config - pre-commit - procps - pytest_inputs - python3 - udev - utillinux - xfsprogs - yasm - ] ++ (if (nospdk) then [ spdk.buildInputs ] else [ spdk ]); - - LIBCLANG_PATH = io-engine.LIBCLANG_PATH; - PROTOC = io-engine.PROTOC; - PROTOC_INCLUDE = io-engine.PROTOC_INCLUDE; - SPDK_PATH = if nospdk then null else "${spdk}"; - FIO_SPDK = if nospdk then null else "${spdk}/fio/spdk_nvme"; - ETCD_BIN = "${etcd}/bin/etcd"; - ETCDCTL_API = "3"; - LVM_BINS = "${lvm2.bin}/bin"; - - IO_ENGINE_DIR = "target/debug"; - - shellHook = '' - export FIO="$(which fio 2> /dev/null)" - echo 'FIO version :' $(fio --version 2> /dev/null) - echo 'FIO path :' $FIO - ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK version :' $(echo $SPDK_PATH | sed 's/.*libspdk-//g')"} - ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK path :' $SPDK_PATH"} - ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK FIO plugin :' $FIO_SPDK"} - echo 'LVM path :' $LVM_BINS - echo 'Rust version :' $(rustc --version 2> /dev/null || echo '${norustc_msg}') - echo 'Rust path :' $(which rustc 2> /dev/null || echo '${norustc_msg}') - echo 'I/O engine dir :' $IO_ENGINE_DIR - ${pkgs.lib.optionalString (nospdk) "cowsay ${nospdk_moth}"} - ${pkgs.lib.optionalString (nospdk) "export CFLAGS=-msse4"} - ${pkgs.lib.optionalString (nospdk) "echo"} - - # SRCDIR is needed by docker-compose files as it requires absolute paths - export SRCDIR=`pwd` - if [ -z "$CI" ]; then - echo - pre-commit install - pre-commit install --hook commit-msg - fi - export PATH=$PATH:$(pwd)/scripts/nix-sudo - ''; -} +{ rust ? "stable" +, spdk ? "develop" +, spdk-path ? null +} @ args: + import ./ci.nix { + inherit rust; + inherit spdk; + inherit spdk-path; + } diff --git a/spdk-rs b/spdk-rs index a1efae6c1..ea6ef4ef3 160000 --- a/spdk-rs +++ b/spdk-rs @@ -1 +1 @@ -Subproject commit a1efae6c1d8c6eaf4f6ce54b5c919f664fd466f3 +Subproject commit ea6ef4ef351429fd76cbaec20d0735dcfa2588e3 diff --git a/test/python/common/fio_spdk.py b/test/python/common/fio_spdk.py index ce7550403..51be9d893 100644 --- a/test/python/common/fio_spdk.py +++ b/test/python/common/fio_spdk.py @@ -25,7 +25,7 @@ def __init__(self, name, rw, uris, runtime=15): def build(self) -> str: spdk_fio_path = os.environ.get("FIO_SPDK") if spdk_fio_path is None: - spdk_path = os.environ.get("SPDK_PATH") + spdk_path = os.environ.get("SPDK_ROOT_DIR") if spdk_path is None: spdk_path = os.getcwd() + "/../../spdk-rs/spdk/build" spdk_fio_path = "{}/fio/spdk_nvme".format(spdk_path)