updated e2e_test.sh to run test packages concurrently.

tools/cd_scripts/e2e_test.sh

@@ -44,6 +44,7 @@ set -x
 cd ~/
 cp /details.txt .
 touch logs.txt
+touch logs-hns.txt
 
 echo User: $USER &>> ~/logs.txt
 echo Current Working Directory: $(pwd)  &>> ~/logs.txt
@@ -139,28 +140,216 @@ git checkout $(sed -n 2p ~/details.txt) |& tee -a ~/logs.txt
 
 #run tests with testbucket flag
 set +e
-# TODO: Running both tests in parallel leads to more test failures. Since Louhi runs on a smaller machine, this does not significantly reduce execution time.
-# We can include this task as part of the parallel e2e test project in the Louhi pipeline.
+# Test directory arrays
+TEST_DIR_PARALLEL=(
+  "local_file"
+  "log_rotation"
+  "mounting"
+  "read_cache"
+  "gzip"
+  "write_large_files"
+  "rename_dir_limit"
+  "read_large_files"
+  "explicit_dir"
+  "implicit_dir"
+  "interrupt"
+  "operations"
+  "log_content"
+  "kernel_list_cache"
+  "concurrent_operations"
+)
 
-# Run tests on HNS bucket
-GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 -short --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt)-hns --timeout=60m &>> ~/logs-hns.txt
-if [ $? -ne 0 ];
-then
-    echo "Test failures detected" &>> ~/logs-hns.txt
-else
-    touch success-hns.txt
-    gsutil cp success-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
-fi
-gsutil cp ~/logs-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
+TEST_DIR_HNS_PARALLEL_GROUP=(
+  "implicit_dir"
+  "rename_dir_limit"
+  "operations"
+  "local_file"
+  "gzip"
+  "interrupt"
+  "log_content"
+  "read_large_files"
+  "write_large_files"
+  "log_rotation"
+  "read_cache"
+  "mounting"
+  "kernel_list_cache"
+  "concurrent_operations"
+)
+
+# These tests never become parallel as they are changing bucket permissions.
+TEST_DIR_NON_PARALLEL=(
+  "readonly"
+  "managed_folders"
+  "readonly_creds"
+  "list_large_dir"
+)
+
+TEST_DIR_HNS_NON_PARALLEL=(
+  "readonly"
+  "readonly_creds"
+  "managed_folders"
+  "list_large_dir"
+)
+
+# Create a temporary file to store the log file name.
+TEST_LOGS_FILE=$(mktemp)
+
+GO_TEST_SHORT_FLAG="-short"
+INTEGRATION_TEST_TIMEOUT=300m
+
+function run_non_parallel_tests() {
+  local exit_code=0
+  local -n test_array=$1
+  local BUCKET_NAME=$2
+  for test_dir_np in "${test_array[@]}"
+  do
+    test_path_non_parallel="./tools/integration_tests/$test_dir_np"
+    # To make it clear whether tests are running on a flat or HNS bucket, We kept the log file naming
+    # convention to include the bucket name as a suffix (e.g., package_name_bucket_name).
+    local log_file="/tmp/${test_dir_np}_${BUCKET_NAME}.log"
+    echo $log_file >> $TEST_LOGS_FILE
+    # Executing integration tests
+    GODEBUG=asyncpreemptoff=1 go test $test_path_non_parallel -p 1 $GO_TEST_SHORT_FLAG --integrationTest -v --testbucket=$BUCKET_NAME --testInstalledPackage=true -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1
+    exit_code_non_parallel=$?
+    if [ $exit_code_non_parallel != 0 ]; then
+      exit_code=$exit_code_non_parallel
+    fi
+  done
+  return $exit_code
+}
+
+function run_parallel_tests() {
+  local exit_code=0
+  local -n test_array=$1
+  local BUCKET_NAME=$2
+  local pids=()
+  for test_dir_p in "${test_array[@]}"
+  do
+    test_path_parallel="./tools/integration_tests/$test_dir_p"
+    # To make it clear whether tests are running on a flat or HNS bucket, We kept the log file naming
+    # convention to include the bucket name as a suffix (e.g., package_name_bucket_name).
+    local log_file="/tmp/${test_dir_p}_${BUCKET_NAME}.log"
+    echo $log_file >> $TEST_LOGS_FILE
+    # Executing integration tests
+    GODEBUG=asyncpreemptoff=1 go test $test_path_parallel $GO_TEST_SHORT_FLAG -p 1 --integrationTest -v --testbucket=$BUCKET_NAME --testInstalledPackage=true -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 &
+    pid=$!  # Store the PID of the background process
+    pids+=("$pid")  # Optionally add the PID to an array for later
+  done
+  # Wait for processes and collect exit codes
+  for pid in "${pids[@]}"; do
+    wait $pid
+    exit_code_parallel=$?
+    if [ $exit_code_parallel != 0 ]; then
+      exit_code=$exit_code_parallel
+    fi
+  done
+  return $exit_code
+}
+
+function run_e2e_tests_for_flat_bucket() {
+  flat_bucket_name_non_parallel=$(sed -n 3p ~/details.txt)
+  echo "Flat Bucket name to run tests sequentially: "$flat_bucket_name_non_parallel
+
+  flat_bucket_name_parallel=$(sed -n 3p ~/details.txt)-parallel
+  echo "Flat Bucket name to run tests parallelly: "$flat_bucket_name_parallel
+
+  echo "Running parallel tests..."
+  run_parallel_tests TEST_DIR_PARALLEL "$flat_bucket_name_parallel" &
+  parallel_tests_pid=$!
+
+ echo "Running non parallel tests ..."
+ run_non_parallel_tests TEST_DIR_NON_PARALLEL "$flat_bucket_name_non_parallel" &
+ non_parallel_tests_pid=$!
+
+ # Wait for all tests to complete.
+ wait $parallel_tests_pid
+ parallel_tests_exit_code=$?
+ wait $non_parallel_tests_pid
+ non_parallel_tests_exit_code=$?
+
+ if [ $non_parallel_tests_exit_code != 0 ] || [ $parallel_tests_exit_code != 0 ];
+ then
+   return 1
+ fi
+ return 0
+}
 
-# Run tests on FLAT bucket
-GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 -short --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt) --timeout=60m &>> ~/logs.txt
-if [ $? -ne 0 ];
+function run_e2e_tests_for_hns_bucket(){
+  hns_bucket_name_non_parallel=$(sed -n 3p ~/details.txt)-hns
+  echo "HNS Bucket name to run tests sequentially: "$hns_bucket_name_non_parallel
+
+  hns_bucket_name_parallel=$(sed -n 3p ~/details.txt)-hns-parallel
+  echo "HNS Bucket name to run tests parallelly: "$hns_bucket_name_parallel
+
+   echo "Running tests for HNS bucket"
+   run_parallel_tests TEST_DIR_HNS_PARALLEL_GROUP "$hns_bucket_name_parallel" &
+   parallel_tests_hns_group_pid=$!
+   run_non_parallel_tests TEST_DIR_HNS_NON_PARALLEL "$hns_bucket_name_non_parallel" &
+   non_parallel_tests_hns_group_pid=$!
+
+   # Wait for all tests to complete.
+   wait $parallel_tests_hns_group_pid
+   parallel_tests_hns_group_exit_code=$?
+   wait $non_parallel_tests_hns_group_pid
+   non_parallel_tests_hns_group_exit_code=$?
+
+   if [ $parallel_tests_hns_group_exit_code != 0 ] || [ $non_parallel_tests_hns_group_exit_code != 0 ];
+   then
+    return 1
+   fi
+   return 0
+}
+
+function gather_test_logs() {
+  readarray -t test_logs_array < "$TEST_LOGS_FILE"
+  rm "$TEST_LOGS_FILE"
+  for test_log_file in "${test_logs_array[@]}"
+  do
+    log_file=${test_log_file}
+    if [ -f "$log_file" ]; then
+      if [[ "$test_log_file" == *"hns"* ]]; then
+        output_file="$HOME/logs-hns.txt"
+      else
+        output_file="$HOME/logs.txt"
+      fi
+
+      echo "=== Log for ${test_log_file} ===" >> "$output_file"
+      cat "$log_file" >> "$output_file"
+      echo "=========================================" >> "$output_file"
+    fi
+  done
+}
+
+#run integration tests
+run_e2e_tests_for_hns_bucket &
+e2e_tests_hns_bucket_pid=$!
+
+run_e2e_tests_for_flat_bucket &
+e2e_tests_flat_bucket_pid=$!
+
+wait $e2e_tests_flat_bucket_pid
+e2e_tests_flat_bucket_status=$?
+
+wait $e2e_tests_hns_bucket_pid
+e2e_tests_hns_bucket_status=$?
+
+gather_test_logs
+
+if [ $e2e_tests_flat_bucket_status != 0 ]
 then
     echo "Test failures detected" &>> ~/logs.txt
 else
     touch success.txt
     gsutil cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
 fi
 gsutil cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
+
+if [ $e2e_tests_hns_bucket_status != 0 ];
+then
+    echo "Test failures detected" &>> ~/logs-hns.txt
+else
+    touch success-hns.txt
+    gsutil cp success-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
+fi
+gsutil cp ~/logs-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/
 '