Merge #1663

1663: Upgrading to SPDK 24.01 and refactoring Nix scripts r=dsavitskiy a=dsavitskiy



Co-authored-by: Dmitry Savitskiy <[email protected]>

ci.nix

@@ -1,129 +1,72 @@
-{ nospdk ? false, norust ? false, spdk_rel ? false, asan ? false }:
+{ rust ? "stable"
+, spdk ? "develop"
+, spdk-path ? null
+} @ args:
 let
   sources = import ./nix/sources.nix;
+
   pkgs = import sources.nixpkgs {
-    overlays =
-      [ (_: _: { inherit sources; }) (import ./nix/overlay.nix { }) ];
+    overlays = [
+      (_: _: { inherit sources; })
+      (import ./nix/overlay.nix { })
+    ];
   };
-in
-with pkgs;
-let
-  nospdk_moth =
-    "You have requested environment without SPDK, you should provide it!";
-  norust_moth =
-    "You have requested environment without RUST, you should provide it!";
-  norustc_msg = "no rustc, use rustup tool to install it";
-  channel = import ./nix/lib/rust.nix { inherit sources; };
+
   # python environment for test/python
-  pytest_inputs = python3.withPackages
+  pytest_inputs = with pkgs; python3.withPackages
     (ps: with ps; [ virtualenv grpcio grpcio-tools asyncssh black ]);
-  spdk = if (!spdk_rel) then libspdk-dev else libspdk;
-in
-mkShell {
-  name = "io-engine-dev-shell";
-  # fortify does not work with -O0 which is used by spdk when --enable-debug
-  hardeningDisable = [ "fortify" ];
-  buildInputs = [
-    autoconf
-    automake
-    clang
-    cowsay
-    docker
-    docker-compose
-    e2fsprogs
-    etcd
-    fio
-    gdb
-    git
-    gnuplot
-    kubernetes-helm
-    libaio
-    libbsd
-    libnvme
-    libpcap
-    libunwind
-    liburing
-    llvmPackages.bintools
-    llvmPackages.libclang
-    meson
-    ninja
-    nodejs-16_x
-    numactl
-    nvme-cli
-    openssl
-    pkg-config
-    pre-commit
-    procps
-    pytest_inputs
-    python3
-    udev
-    utillinux
-    xfsprogs
-    yasm
-  ] ++ (if (nospdk) then [ spdk.buildInputs ] else [ spdk ])
-  ++ pkgs.lib.optional (!norust && asan) channel.asan
-  ++ pkgs.lib.optional (!norust && !asan) channel.stable
-  ++ pkgs.lib.optional (!norust) channel.nightly;
 
-  RUST_NIGHTLY_PATH = channel.nightly;
-  LIBCLANG_PATH = io-engine.LIBCLANG_PATH;
-  PROTOC = io-engine.PROTOC;
-  PROTOC_INCLUDE = io-engine.PROTOC_INCLUDE;
-  SPDK_PATH = if nospdk then null else "${spdk}";
-  FIO_SPDK = if nospdk then null else "${spdk}/fio/spdk_nvme";
-  ETCD_BIN = "${etcd}/bin/etcd";
-  LVM_BINS = "${lvm2.bin}/bin";
+  shellAttrs = import ./spdk-rs/nix/shell {
+    inherit rust;
+    inherit spdk;
+    inherit spdk-path;
+    inherit sources;
+    inherit pkgs;
 
-  IO_ENGINE_DIR = if asan then "target/x86_64-unknown-linux-gnu/debug" else "target/debug";
+    cfg = {
+      buildInputs = with pkgs; [
+        docker
+        docker-compose
+        e2fsprogs
+        etcd
+        gdb
+        git
+        gnuplot
+        kubernetes-helm
+        nodejs-16_x
+        numactl
+        pytest_inputs
+        udev
+        xfsprogs
+      ];
 
-  # ASAN-related Cargo settings.
-  ASAN_ENABLE = if asan then "1" else null;
-  ASAN_OPTIONS = if asan then "detect_leaks=0" else null;
-  ASAN_BUILD_ENV = if asan then "shell" else null;
-  RUSTFLAGS = if asan then "-Zsanitizer=address" else null;
-  CARGO_BUILD_RUSTFLAGS = if asan then "-Zbuild-std" else null;
-  CARGO_BUILD_TARGET = if asan then "x86_64-unknown-linux-gnu" else null;
-  CARGO_PROFILE_DEV_PANIC = if asan then "unwind" else null;
-  RUST_BACKTRACE = if asan then "full" else null;
+      shellEnv = with pkgs; {
+        PROTOC = io-engine.PROTOC;
+        PROTOC_INCLUDE = io-engine.PROTOC_INCLUDE;
+        ETCD_BIN = "${etcd}/bin/etcd";
+        LVM_BINS = "${lvm2.bin}/bin";
+      };
 
-  shellHook = ''
-    export FIO="$(which fio 2> /dev/null)"
-    ${pkgs.lib.optionalString (asan) "export LLVM_SYMBOLIZER_DIR=$(dirname $(realpath $(which llvm-symbolizer)))"}
+      shellHook = ''
+        # SRCDIR is needed by docker-compose files as it requires absolute paths
+        export SRCDIR=`pwd`
 
-    ${pkgs.lib.optionalString (asan) "echo 'AddressSanitizer is enabled, forcing nightly rustc.'"}
-    ${pkgs.lib.optionalString (asan) "echo '  ASAN_ENABLE             :' $\{ASAN_ENABLE\}"}
-    ${pkgs.lib.optionalString (asan) "echo '  ASAN_OPTIONS            :' $\{ASAN_OPTIONS\}"}
-    ${pkgs.lib.optionalString (asan) "echo '  RUSTFLAGS               :' $\{RUSTFLAGS\}"}
-    ${pkgs.lib.optionalString (asan) "echo '  CARGO_BUILD_RUSTFLAGS   :' $\{CARGO_BUILD_RUSTFLAGS\}"}
-    ${pkgs.lib.optionalString (asan) "echo '  CARGO_BUILD_TARGET      :' $\{CARGO_BUILD_TARGET\}"}
-    ${pkgs.lib.optionalString (asan) "echo '  CARGO_PROFILE_DEV_PANIC :' $\{CARGO_PROFILE_DEV_PANIC\}"}
-    ${pkgs.lib.optionalString (asan) "echo '  RUST_BACKTRACE          :' $\{RUST_BACKTRACE\}"}
-    ${pkgs.lib.optionalString (asan) "echo '  LLVM_SYMBOLIZER_DIR     :' $\{LLVM_SYMBOLIZER_DIR\}"}
-    ${pkgs.lib.optionalString (asan) "echo"}
+        export PATH="$PATH:$(pwd)/scripts/nix-sudo"
 
-                                         echo 'FIO version     :' $(fio --version 2> /dev/null)
-                                         echo 'FIO path        :' $FIO
-    ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK version    :' $(echo $SPDK_PATH | sed 's/.*libspdk-//g')"}
-    ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK path       :' $SPDK_PATH"}
-    ${pkgs.lib.optionalString (!nospdk) "echo 'SPDK FIO plugin :' $FIO_SPDK"}
-    echo 'LVM path        :' $LVM_BINS
-    ${pkgs.lib.optionalString (!norust) "echo 'Rust version    :' $(rustc --version 2> /dev/null || echo '${norustc_msg}')"}
-    ${pkgs.lib.optionalString (!norust) "echo 'Rust path       :' $(which rustc 2> /dev/null || echo '${norustc_msg}')"}
-    echo 'I/O engine dir  :' $IO_ENGINE_DIR
-    ${pkgs.lib.optionalString (nospdk) "cowsay ${nospdk_moth}"}
-    ${pkgs.lib.optionalString (nospdk) "export CFLAGS=-msse4"}
-    ${pkgs.lib.optionalString (nospdk) "echo"}
-    ${pkgs.lib.optionalString (norust) "cowsay ${norust_moth}"}
-    ${pkgs.lib.optionalString (norust) "echo 'Hint: use rustup tool.'"}
-    ${pkgs.lib.optionalString (norust) "echo"}
+        export IO_ENGINE_DIR="$RUST_TARGET_DEBUG"
+      '';
 
-    # SRCDIR is needed by docker-compose files as it requires absolute paths
-    export SRCDIR=`pwd`
-    if [ -z "$CI" ]; then
-      echo
-      pre-commit install
-      pre-commit install --hook commit-msg
-    fi
-    export PATH=$PATH:$(pwd)/scripts/nix-sudo
-  '';
-}
+      shellInfoHook = ''
+        echo
+        echo "PROTOC          : $PROTOC"
+        echo "PROTOC_INCLUDE  : $PROTOC_INCLUDE"
+        echo "ETCD_BIN        : $ETCD_BIN"
+        echo "LVM path        : $LVM_BINS"
+        echo "I/O engine dir  : $IO_ENGINE_DIR"
+      '';
+    };
+  };
+in
+  pkgs.mkShell shellAttrs // {
+    name = "io-engine-dev-shell";
+  }

ide.nix

@@ -0,0 +1,9 @@
+{ rust ? "none"
+, spdk ? "develop"
+, spdk-path ? null
+} @ args:
+  import ./ci.nix {
+    inherit rust;
+    inherit spdk;
+    inherit spdk-path;
+  }

io-engine-tests/src/error_bdev.rs

@@ -44,6 +44,7 @@ pub fn inject_error(error_device: &str, op: u32, mode: u32, count: u32) {
         error_num: count,
         corrupt_offset: 0,
         corrupt_value: 0,
+        error_qd: 0,
     };
 
     unsafe {

io-engine/src/bdev/null_bdev.rs

@@ -134,10 +134,12 @@ impl CreateDestroy for Null {
             });
         }
 
+        let uuid = spdk_rs::Uuid::generate().into_raw();
+
         let cname = self.name.clone().into_cstring();
         let opts = spdk_rs::libspdk::spdk_null_bdev_opts {
             name: cname.as_ptr(),
-            uuid: std::ptr::null(),
+            uuid: &uuid,
             num_blocks: self.num_blocks,
             block_size: self.blk_size,
             physical_block_size: 0,

io-engine/src/bdev/nvme.rs

@@ -10,13 +10,9 @@ use snafu::ResultExt;
 use url::Url;
 
 use spdk_rs::{
+    bdevs::bdev_nvme_delete_async,
     ffihelper::copy_str_with_null,
-    libspdk::{
-        bdev_nvme_create,
-        bdev_nvme_delete,
-        nvme_path_id,
-        spdk_nvme_transport_id,
-    },
+    libspdk::{bdev_nvme_create, spdk_nvme_transport_id},
 };
 
 use crate::{
@@ -136,25 +132,20 @@ impl CreateDestroy for NVMe {
         if let Some(mut bdev) = UntypedBdev::lookup_by_name(&self.get_name()) {
             bdev.remove_alias(self.url.as_ref());
 
-            let mut path_id = nvme_path_id::default();
-            copy_str_with_null(&self.name, &mut path_id.trid.traddr);
-            path_id.trid.trtype = spdk_rs::libspdk::SPDK_NVME_TRANSPORT_PCIE;
+            let res = bdev_nvme_delete_async(&self.name, None).await;
 
-            let errno = unsafe {
-                bdev_nvme_delete(
-                    self.name.clone().into_cstring().as_ptr(),
-                    &path_id,
-                )
-            };
-            if errno != 0 {
+            // Restore the alias in the case the deletion failed.
+            if !res.is_ok_and(|e| e.is_ok()) {
                 UntypedBdev::lookup_by_name(&self.get_name())
                     .map(|mut b| b.add_alias(self.url.as_ref()));
             }
-            errno_result_from_i32((), errno).context(
-                bdev_api::DestroyBdevFailed {
-                    name: self.name.clone(),
-                },
-            )
+
+            res.context(bdev_api::BdevCommandCanceled {
+                name: self.name.clone(),
+            })?
+            .context(bdev_api::DestroyBdevFailed {
+                name: self.name.clone(),
+            })
         } else {
             Err(BdevError::BdevNotFound {
                 name: self.get_name(),