Merge #1457

1457: test: fix python tests r=tiagolobocastro a=tiagolobocastro

Now that the nexus stays shutdown on all children failure, dd might not complete with error. To address this, add smaller delay and ctrl loss timeout. Also add missing sudo for xfs_info.

Co-authored-by: Tiago Castro <[email protected]>

Jenkinsfile

@@ -155,8 +155,7 @@ pipeline {
           }
           post {
             always {
-              // in case of abnormal termination of any nvmf test
-              sh 'sudo nvme disconnect-all'
+              sh 'nix-shell --run "./scripts/clean-cargo-tests.sh" ci.nix'
               sh 'sudo ./scripts/check-coredumps.sh --since "${START_DATE}"'
             }
           }
@@ -190,6 +189,7 @@ pipeline {
           }
           post {
             always {
+              sh 'nix-shell --run "./scripts/clean-cargo-tests.sh" ci.nix'
               junit '*-xunit-report.xml'
               sh 'sudo ./scripts/check-coredumps.sh --since "${START_DATE}"'
             }

scripts/cargo-test.sh

@@ -1,24 +1,20 @@
 #!/usr/bin/env bash
 
-SCRIPTDIR=$(dirname "$0")
+SCRIPTDIR="$(realpath "$(dirname "$0")")"
 
 cleanup_handler() {
-  for c in $(docker ps -a --filter "label=io.mayastor.test.name" --format '{{.ID}}') ; do
-    docker kill "$c" || true
-    docker rm "$c" || true
-  done
-
-  for n in $(docker network ls --filter "label=io.mayastor.test.name" --format '{{.ID}}') ; do
-    docker network rm "$n" || true
-  done
+  ERROR=$?
+  "$SCRIPTDIR"/clean-cargo-tests.sh || true
+  if [ $ERROR != 0 ]; then exit $ERROR; fi
 }
 
-trap cleanup_handler ERR INT QUIT TERM HUP
-
 echo "running cargo-test..."
 echo "rustc version:"
 rustc --version
 
+leanup_handler
+trap cleanup_handler INT QUIT TERM HUP EXIT
+
 set -euxo pipefail
 export PATH=$PATH:${HOME}/.cargo/bin
 ( cd jsonrpc && cargo test )

scripts/clean-cargo-tests.sh

@@ -0,0 +1,19 @@
+SCRIPT_DIR="$(dirname "$0")"
+ROOT_DIR=$(realpath "$SCRIPT_DIR/..")
+
+sudo nvme disconnect-all
+
+for c in $(docker ps -a --filter "label=io.mayastor.test.name" --format '{{.ID}}') ; do
+  docker kill "$c"
+  docker rm "$c"
+done
+
+for n in $(docker network ls --filter "label=io.mayastor.test.name" --format '{{.ID}}') ; do
+  docker network rm "$n" || ( sudo systemctl restart docker && docker network rm "$n" )
+done
+
+# Kill's processes running off the workspace cargo binary location
+ps aux | grep "$ROOT_DIR/target" | grep -v -e sudo -e grep | awk '{ print $2 }' | xargs -I% sudo kill -9 %
+
+exit 0
+

scripts/grpc-test.sh

@@ -1,5 +1,16 @@
 #!/usr/bin/env bash
 
+SCRIPTDIR="$(realpath "$(dirname "$0")")"
+
+cleanup_handler() {
+  ERROR=$?
+  "$SCRIPTDIR"/clean-cargo-tests.sh || true
+  if [ $ERROR != 0 ]; then exit $ERROR; fi
+}
+
+cleanup_handler
+trap cleanup_handler INT QUIT TERM HUP EXIT
+
 set -euxo pipefail
 
 export PATH="$PATH:${HOME}/.cargo/bin"

test/python/common/nvme.py

@@ -56,13 +56,15 @@ async def nvme_remote_discover(remote, uri):
         raise ValueError("uri {} is not discovered".format(u.path[1:]))
 
 
-def nvme_connect(uri):
+def nvme_connect(uri, delay=10, tmo=600):
     u = urlparse(uri)
     port = u.port
     host = u.hostname
     nqn = u.path[1:]
 
-    command = "sudo nvme connect -t tcp -s {0} -a {1} -n {2}".format(port, host, nqn)
+    command = (
+        f"sudo nvme connect -t tcp -s {port} -a {host} -n {nqn} -c {delay} -l {tmo}"
+    )
     subprocess.run(command, check=True, shell=True, capture_output=False)
     time.sleep(1)
     command = "sudo nvme list -v -o json"

test/python/tests/nexus/test_multi_nexus.py

@@ -7,6 +7,7 @@
 import pytest
 import asyncio
 import uuid as guid
+import pytest_asyncio
 
 NEXUS_COUNT = 15
 DESTROY_COUNT = 7
@@ -114,7 +115,7 @@ def connect_devices(create_nexuses):
         nvme_disconnect(nexus)
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
 async def mount_devices(connect_devices):
     "Create and mount a filesystem on each nvmf connected device."
     for dev in connect_devices:
@@ -126,6 +127,7 @@ async def mount_devices(connect_devices):
 
     for dev in connect_devices:
         await run_cmd_async(f"sudo umount /mnt{dev}")
+    await run_cmd_async(f"sudo rm -rf /mnt/dev")
 
 
 @pytest.mark.asyncio

test/python/tests/nexus/test_nexus.py

@@ -447,7 +447,7 @@ def test_nexus_preempt_key(
 
     # verify write error with nexus on ms3
     uri = create_nexus_v2
-    dev = nvme_connect(uri)
+    dev = nvme_connect(uri, 1, 1)
     job = "sudo dd if=/dev/urandom of={0} bs=512 count=1".format(dev)
 
     try:

test/python/tests/nexus_fault/test_nexus_fault.py

@@ -122,9 +122,9 @@ def _(mounted_nexus):
     """the initiator filesystem should not be shutdown."""
     try:
         # xfs_info should still be working as the fs is not shutdown
-        run_cmd(f"xfs_info {mounted_nexus}")
-    except:
-        pytest.fail(f"Filesystem on {mounted_nexus} should not be shutdown")
+        run_cmd(f"sudo xfs_info {mounted_nexus}")
+    except Exception as e:
+        pytest.fail(f"Filesystem on {mounted_nexus} should not be shutdown: {e}")
 
 
 @pytest.fixture(scope="module")

test/python/v1/nexus/test_multi_nexus.py

@@ -7,6 +7,7 @@
 import pytest
 import asyncio
 import uuid as guid
+import pytest_asyncio
 
 NEXUS_COUNT = 15
 DESTROY_COUNT = 7
@@ -139,7 +140,7 @@ def connect_devices(create_nexuses):
         nvme_disconnect(nexus)
 
 
-@pytest.fixture
+@pytest_asyncio.fixture
 async def mount_devices(connect_devices):
     "Create and mount a filesystem on each nvmf connected device."
     for dev in connect_devices:
@@ -151,6 +152,7 @@ async def mount_devices(connect_devices):
 
     for dev in connect_devices:
         await run_cmd_async(f"sudo umount /mnt{dev}")
+    await run_cmd_async(f"sudo rm -rf /mnt/dev")
 
 
 @pytest.mark.asyncio

test/python/v1/nexus/test_nexus.py

@@ -435,7 +435,7 @@ def test_nexus_preempt_key(
 
     # verify write error with nexus on ms3
     uri = create_nexus
-    dev = nvme_connect(uri)
+    dev = nvme_connect(uri, 1, 1)
     job = "sudo dd if=/dev/urandom of={0} bs=512 count=1".format(dev)
 
     try: