Try #1459:

Jenkinsfile

@@ -140,6 +140,7 @@ pipeline {
           anyOf {
             branch 'master'
             branch 'release/*'
+            branch 'trying'
             expression { run_linter == false }
           }
         }

io-engine-tests/src/compose/rpc/v1.rs

@@ -51,6 +51,7 @@ pub struct RpcHandle {
     pub host: host::HostRpcClient<Channel>,
     pub nexus: nexus::NexusRpcClient<Channel>,
     pub snapshot: snapshot::SnapshotRpcClient<Channel>,
+    pub testing: testing::TestingRpcClient<Channel>,
 }
 
 impl RpcHandle {
@@ -99,11 +100,17 @@ impl RpcHandle {
             nexus::NexusRpcClient::connect(format!("http://{endpoint}"))
                 .await
                 .unwrap();
+
         let snapshot =
             snapshot::SnapshotRpcClient::connect(format!("http://{endpoint}"))
                 .await
                 .unwrap();
 
+        let testing =
+            testing::TestingRpcClient::connect(format!("http://{endpoint}"))
+                .await
+                .unwrap();
+
         Ok(Self {
             name,
             endpoint,
@@ -114,6 +121,7 @@ impl RpcHandle {
             host,
             nexus,
             snapshot,
+            testing,
         })
     }
 }

io-engine-tests/src/lib.rs

@@ -32,6 +32,7 @@ pub mod nvme;
 pub mod nvmf;
 pub mod pool;
 pub mod replica;
+pub mod testing;
 
 pub use compose::MayastorTest;
 

io-engine-tests/src/nexus.rs

@@ -9,16 +9,12 @@ use super::{
             ChildStateReason,
             CreateNexusRequest,
             DestroyNexusRequest,
-            InjectNexusFaultRequest,
-            InjectedFault,
-            ListInjectedNexusFaultsRequest,
             ListNexusOptions,
             Nexus,
             PublishNexusRequest,
             RebuildHistoryRecord,
             RebuildHistoryRequest,
             RemoveChildNexusRequest,
-            RemoveInjectedNexusFaultRequest,
         },
         SharedRpcHandle,
         Status,
@@ -277,6 +273,16 @@ impl NexusBuilder {
         self.online_child_bdev(&self.replica_uri(r)).await
     }
 
+    pub async fn online_child_replica_wait(
+        &self,
+        r: &ReplicaBuilder,
+        d: Duration,
+    ) -> Result<(), Status> {
+        self.online_child_replica(r).await?;
+        self.wait_replica_state(r, ChildState::Online, None, d)
+            .await
+    }
+
     pub async fn offline_child_bdev(
         &self,
         bdev: &str,
@@ -301,53 +307,22 @@ impl NexusBuilder {
         self.offline_child_bdev(&self.replica_uri(r)).await
     }
 
-    pub async fn inject_nexus_fault(
-        &self,
-        inj_uri: &str,
-    ) -> Result<(), Status> {
-        self.rpc()
-            .lock()
-            .await
-            .nexus
-            .inject_nexus_fault(InjectNexusFaultRequest {
-                uuid: self.uuid(),
-                uri: inj_uri.to_owned(),
-            })
-            .await
-            .map(|r| r.into_inner())
-    }
-
-    pub async fn remove_injected_nexus_fault(
+    pub async fn offline_child_replica_wait(
         &self,
-        inj_uri: &str,
+        r: &ReplicaBuilder,
+        d: Duration,
     ) -> Result<(), Status> {
-        self.rpc()
-            .lock()
-            .await
-            .nexus
-            .remove_injected_nexus_fault(RemoveInjectedNexusFaultRequest {
-                uuid: self.uuid(),
-                uri: inj_uri.to_owned(),
-            })
-            .await
-            .map(|r| r.into_inner())
-    }
-
-    pub async fn list_injected_faults(
-        &self,
-    ) -> Result<Vec<InjectedFault>, Status> {
-        self.rpc()
-            .lock()
-            .await
-            .nexus
-            .list_injected_nexus_faults(ListInjectedNexusFaultsRequest {
-                uuid: self.uuid(),
-            })
-            .await
-            .map(|r| r.into_inner().injections)
+        self.offline_child_replica(r).await?;
+        self.wait_replica_state(
+            r,
+            ChildState::Degraded,
+            Some(ChildStateReason::ByClient),
+            d,
+        )
+        .await
     }
 
-    pub async fn inject_fault_at_replica(
+    pub async fn add_injection_at_replica(
         &self,
         r: &ReplicaBuilder,
         inj_params: &str,
@@ -362,7 +337,7 @@ impl NexusBuilder {
         })?;
 
         let inj_uri = format!("inject://{dev}?{inj_params}",);
-        self.inject_nexus_fault(&inj_uri).await?;
+        super::testing::add_injection(self.rpc(), &inj_uri).await?;
 
         Ok(inj_uri)
     }

io-engine-tests/src/testing.rs

@@ -0,0 +1,49 @@
+use super::compose::rpc::v1::{
+    testing::{
+        AddInjectionRequest,
+        Injection,
+        ListInjectionsRequest,
+        RemoveInjectionRequest,
+    },
+    SharedRpcHandle,
+    Status,
+};
+
+pub async fn add_injection(
+    rpc: SharedRpcHandle,
+    inj_uri: &str,
+) -> Result<(), Status> {
+    rpc.lock()
+        .await
+        .testing
+        .add_injection(AddInjectionRequest {
+            uri: inj_uri.to_owned(),
+        })
+        .await
+        .map(|r| r.into_inner())
+}
+
+pub async fn remove_injection(
+    rpc: SharedRpcHandle,
+    inj_uri: &str,
+) -> Result<(), Status> {
+    rpc.lock()
+        .await
+        .testing
+        .remove_injection(RemoveInjectionRequest {
+            uri: inj_uri.to_owned(),
+        })
+        .await
+        .map(|r| r.into_inner())
+}
+
+pub async fn list_injections(
+    rpc: SharedRpcHandle,
+) -> Result<Vec<Injection>, Status> {
+    rpc.lock()
+        .await
+        .testing
+        .list_injections(ListInjectionsRequest {})
+        .await
+        .map(|r| r.into_inner().injections)
+}

io-engine/Cargo.toml

@@ -9,8 +9,8 @@ build = "build.rs"
 
 [features]
 default = ["spdk-async-qpair-connect", "spdk-subsystem-events"]
-io-engine-testing = ["nexus-fault-injection"]
-nexus-fault-injection = [] # Enables nexus-level fault injection code.
+io-engine-testing = ["fault-injection"]
+fault-injection = [] # Enables fault injection code.
 nexus-io-tracing = [] # Enables nexus I/O tracing code.
 spdk-async-qpair-connect = []
 spdk-subsystem-events = []