diff --git a/tools/cd_scripts/e2e_test.sh b/tools/cd_scripts/e2e_test.sh index f230622c83..38194ac267 100755 --- a/tools/cd_scripts/e2e_test.sh +++ b/tools/cd_scripts/e2e_test.sh @@ -44,6 +44,7 @@ set -x cd ~/ cp /details.txt . touch logs.txt +touch logs-hns.txt echo User: $USER &>> ~/logs.txt echo Current Working Directory: $(pwd) &>> ~/logs.txt @@ -139,23 +140,202 @@ git checkout $(sed -n 2p ~/details.txt) |& tee -a ~/logs.txt #run tests with testbucket flag set +e -# TODO: Running both tests in parallel leads to more test failures. Since Louhi runs on a smaller machine, this does not significantly reduce execution time. -# We can include this task as part of the parallel e2e test project in the Louhi pipeline. +# Test directory arrays +TEST_DIR_PARALLEL=( + "local_file" + "log_rotation" + "mounting" + "read_cache" + "gzip" + "write_large_files" + "rename_dir_limit" + "read_large_files" + "explicit_dir" + "implicit_dir" + "interrupt" + "operations" + "log_content" + "kernel_list_cache" + "concurrent_operations" +) -# Run tests on HNS bucket -GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 -short --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt)-hns --timeout=60m &>> ~/logs-hns.txt -if [ $? -ne 0 ]; -then - echo "Test failures detected" &>> ~/logs-hns.txt -else - touch success-hns.txt - gsutil cp success-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ -fi -gsutil cp ~/logs-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ +TEST_DIR_HNS_PARALLEL_GROUP=( + "implicit_dir" + "rename_dir_limit" + "operations" + "local_file" + "gzip" + "interrupt" + "log_content" + "read_large_files" + "write_large_files" + "log_rotation" + "read_cache" + "mounting" + "kernel_list_cache" + "concurrent_operations" +) + +# These tests never become parallel as they are changing bucket permissions. +TEST_DIR_NON_PARALLEL=( + "readonly" + "managed_folders" + "readonly_creds" + "list_large_dir" +) + +TEST_DIR_HNS_NON_PARALLEL=( + "readonly" + "readonly_creds" + "managed_folders" + "list_large_dir" +) + +# Create a temporary file to store the log file name. +TEST_LOGS_FILE=$(mktemp) + +GO_TEST_SHORT_FLAG="-short" +INTEGRATION_TEST_TIMEOUT=300m + +function run_non_parallel_tests() { + local exit_code=0 + local -n test_array=$1 + local BUCKET_NAME=$2 + for test_dir_np in "${test_array[@]}" + do + test_path_non_parallel="./tools/integration_tests/$test_dir_np" + # To make it clear whether tests are running on a flat or HNS bucket, We kept the log file naming + # convention to include the bucket name as a suffix (e.g., package_name_bucket_name). + local log_file="/tmp/${test_dir_np}_${BUCKET_NAME}.log" + echo $log_file >> $TEST_LOGS_FILE + # Executing integration tests + GODEBUG=asyncpreemptoff=1 go test $test_path_non_parallel -p 1 $GO_TEST_SHORT_FLAG --integrationTest -v --testbucket=$BUCKET_NAME --testInstalledPackage=true -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 + exit_code_non_parallel=$? + if [ $exit_code_non_parallel != 0 ]; then + exit_code=$exit_code_non_parallel + fi + done + return $exit_code +} + +function run_parallel_tests() { + local exit_code=0 + local -n test_array=$1 + local BUCKET_NAME=$2 + local pids=() + for test_dir_p in "${test_array[@]}" + do + test_path_parallel="./tools/integration_tests/$test_dir_p" + # To make it clear whether tests are running on a flat or HNS bucket, We kept the log file naming + # convention to include the bucket name as a suffix (e.g., package_name_bucket_name). + local log_file="/tmp/${test_dir_p}_${BUCKET_NAME}.log" + echo $log_file >> $TEST_LOGS_FILE + # Executing integration tests + GODEBUG=asyncpreemptoff=1 go test $test_path_parallel $GO_TEST_SHORT_FLAG -p 1 --integrationTest -v --testbucket=$BUCKET_NAME --testInstalledPackage=true -timeout $INTEGRATION_TEST_TIMEOUT > "$log_file" 2>&1 & + pid=$! # Store the PID of the background process + pids+=("$pid") # Optionally add the PID to an array for later + done + # Wait for processes and collect exit codes + for pid in "${pids[@]}"; do + wait $pid + exit_code_parallel=$? + if [ $exit_code_parallel != 0 ]; then + exit_code=$exit_code_parallel + fi + done + return $exit_code +} + +function run_e2e_tests_for_flat_bucket() { + flat_bucket_name_non_parallel=$(sed -n 3p ~/details.txt) + echo "Flat Bucket name to run tests sequentially: "$flat_bucket_name_non_parallel + + flat_bucket_name_parallel=$(sed -n 3p ~/details.txt)-parallel + echo "Flat Bucket name to run tests parallelly: "$flat_bucket_name_parallel + + echo "Running parallel tests..." + run_parallel_tests TEST_DIR_PARALLEL "$flat_bucket_name_parallel" & + parallel_tests_pid=$! + + echo "Running non parallel tests ..." + run_non_parallel_tests TEST_DIR_NON_PARALLEL "$flat_bucket_name_non_parallel" & + non_parallel_tests_pid=$! + + # Wait for all tests to complete. + wait $parallel_tests_pid + parallel_tests_exit_code=$? + wait $non_parallel_tests_pid + non_parallel_tests_exit_code=$? + + if [ $non_parallel_tests_exit_code != 0 ] || [ $parallel_tests_exit_code != 0 ]; + then + return 1 + fi + return 0 +} -# Run tests on FLAT bucket -GODEBUG=asyncpreemptoff=1 CGO_ENABLED=0 go test ./tools/integration_tests/... -p 1 -short --integrationTest -v --testbucket=$(sed -n 3p ~/details.txt) --timeout=60m &>> ~/logs.txt -if [ $? -ne 0 ]; +function run_e2e_tests_for_hns_bucket(){ + hns_bucket_name_non_parallel=$(sed -n 3p ~/details.txt)-hns + echo "HNS Bucket name to run tests sequentially: "$hns_bucket_name_non_parallel + + hns_bucket_name_parallel=$(sed -n 3p ~/details.txt)-hns-parallel + echo "HNS Bucket name to run tests parallelly: "$hns_bucket_name_parallel + + echo "Running tests for HNS bucket" + run_parallel_tests TEST_DIR_HNS_PARALLEL_GROUP "$hns_bucket_name_parallel" & + parallel_tests_hns_group_pid=$! + run_non_parallel_tests TEST_DIR_HNS_NON_PARALLEL "$hns_bucket_name_non_parallel" & + non_parallel_tests_hns_group_pid=$! + + # Wait for all tests to complete. + wait $parallel_tests_hns_group_pid + parallel_tests_hns_group_exit_code=$? + wait $non_parallel_tests_hns_group_pid + non_parallel_tests_hns_group_exit_code=$? + + if [ $parallel_tests_hns_group_exit_code != 0 ] || [ $non_parallel_tests_hns_group_exit_code != 0 ]; + then + return 1 + fi + return 0 +} + +function gather_test_logs() { + readarray -t test_logs_array < "$TEST_LOGS_FILE" + rm "$TEST_LOGS_FILE" + for test_log_file in "${test_logs_array[@]}" + do + log_file=${test_log_file} + if [ -f "$log_file" ]; then + if [[ "$test_log_file" == *"hns"* ]]; then + output_file="$HOME/logs-hns.txt" + else + output_file="$HOME/logs.txt" + fi + + echo "=== Log for ${test_log_file} ===" >> "$output_file" + cat "$log_file" >> "$output_file" + echo "=========================================" >> "$output_file" + fi + done +} + +#run integration tests +run_e2e_tests_for_hns_bucket & +e2e_tests_hns_bucket_pid=$! + +run_e2e_tests_for_flat_bucket & +e2e_tests_flat_bucket_pid=$! + +wait $e2e_tests_flat_bucket_pid +e2e_tests_flat_bucket_status=$? + +wait $e2e_tests_hns_bucket_pid +e2e_tests_hns_bucket_status=$? + +gather_test_logs + +if [ $e2e_tests_flat_bucket_status != 0 ] then echo "Test failures detected" &>> ~/logs.txt else @@ -163,4 +343,13 @@ else gsutil cp success.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ fi gsutil cp ~/logs.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ + +if [ $e2e_tests_hns_bucket_status != 0 ]; +then + echo "Test failures detected" &>> ~/logs-hns.txt +else + touch success-hns.txt + gsutil cp success-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ +fi +gsutil cp ~/logs-hns.txt gs://gcsfuse-release-packages/v$(sed -n 1p ~/details.txt)/$(sed -n 3p ~/details.txt)/ '