From 4e89ab6832e9edae1a6e1f0c102b6dea6d8bcc07 Mon Sep 17 00:00:00 2001 From: Nitin Garg <113666283+gargnitingoogle@users.noreply.github.com> Date: Fri, 10 Nov 2023 16:47:46 +0530 Subject: [PATCH] Remove chmod calls for running scripts (#1489) * Remove chmod calls for running scripts As in the previous commit, we now have exec-mode set on all script files, and thus chmod for running them is not needed anymore. --- perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh | 3 --- .../scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh | 2 -- .../continuous_test/gcp_ubuntu/periodic_experiments/build.sh | 3 --- perfmetrics/scripts/continuous_test/ml_tests/build.sh | 1 - .../scripts/continuous_test/ml_tests/pytorch/dino/build.sh | 1 - .../scripts/continuous_test/ml_tests/tf/resnet/build.sh | 1 - perfmetrics/scripts/ml_tests/pytorch/dino/Dockerfile | 1 - .../scripts/ml_tests/pytorch/dino/setup_host_and_run_model.sh | 2 -- perfmetrics/scripts/ml_tests/run_image_recognition_models.py | 1 - perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile | 1 - .../scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh | 2 -- perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh | 2 -- 12 files changed, 20 deletions(-) diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh index 2f7cdf0bc4..4ce7c47205 100755 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh @@ -24,7 +24,6 @@ cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse" echo "Building and installing gcsfuse" # Get the latest commitId of yesterday in the log file. Build gcsfuse and run commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H) -chmod +x perfmetrics/scripts/build_and_install_gcsfuse.sh ./perfmetrics/scripts/build_and_install_gcsfuse.sh $commitId # Mounting gcs bucket @@ -45,12 +44,10 @@ LOG_FILE_FIO_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs.txt GCSFUSE_FIO_FLAGS="$GCSFUSE_FLAGS --log-file $LOG_FILE_FIO_TESTS --stackdriver-export-interval=30s" # Executing perf tests -chmod +x run_load_test_and_fetch_metrics.sh ./run_load_test_and_fetch_metrics.sh "$GCSFUSE_FIO_FLAGS" "$UPLOAD_FLAGS" # ls_metrics test. This test does gcsfuse mount with the passed flags first and then does the testing. LOG_FILE_LIST_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-list-logs.txt GCSFUSE_LIST_FLAGS="$GCSFUSE_FLAGS --log-file $LOG_FILE_LIST_TESTS" cd "./ls_metrics" -chmod +x run_ls_benchmark.sh ./run_ls_benchmark.sh "$GCSFUSE_LIST_FLAGS" "$UPLOAD_FLAGS" diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh index a7af5f6152..b0a868e479 100755 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/e2e_tests/build.sh @@ -23,10 +23,8 @@ cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse" echo "Building and installing gcsfuse..." # Get the latest commitId of yesterday in the log file. Build gcsfuse and run commitId=$(git log --before='yesterday 23:59:59' --max-count=1 --pretty=%H) -chmod +x perfmetrics/scripts/build_and_install_gcsfuse.sh ./perfmetrics/scripts/build_and_install_gcsfuse.sh $commitId echo "Running e2e tests on installed package...." -chmod +x perfmetrics/scripts/run_e2e_tests.sh # $1 argument is refering to value of testInstalledPackage ./perfmetrics/scripts/run_e2e_tests.sh $RUN_E2E_TESTS_ON_INSTALLED_PACKAGE diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/periodic_experiments/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/periodic_experiments/build.sh index ba6f44b2f6..353dab0da9 100755 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/periodic_experiments/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/periodic_experiments/build.sh @@ -62,7 +62,6 @@ CONFIG_FILE_FLAGS_COMPRESSED_JSON=$(echo "$CONFIG_FILE_FLAGS_JSON" | jq -c .) echo "Building and installing gcsfuse" # Get the latest commitId of yesterday in the log file. Build gcsfuse and run -chmod +x perfmetrics/scripts/build_and_install_gcsfuse.sh ./perfmetrics/scripts/build_and_install_gcsfuse.sh $BRANCH cd "./perfmetrics/scripts/" @@ -84,13 +83,11 @@ fi # Executing perf tests LOG_FILE_FIO_TESTS="${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs${EXPERIMENT_NUMBER}.txt" GCSFUSE_FIO_FLAGS="$GCSFUSE_FLAGS --log-file $LOG_FILE_FIO_TESTS --log-format \"text\" --stackdriver-export-interval=30s" -chmod +x run_load_test_and_fetch_metrics.sh ./run_load_test_and_fetch_metrics.sh "$GCSFUSE_FIO_FLAGS" "$UPLOAD_FLAGS" # ls_metrics test. This test does gcsfuse mount with the passed flags first and then does the testing. LOG_FILE_LIST_TESTS="${KOKORO_ARTIFACTS_DIR}/gcsfuse-list-logs${EXPERIMENT_NUMBER}.txt" GCSFUSE_LIST_FLAGS="$GCSFUSE_FLAGS --log-file $LOG_FILE_LIST_TESTS --log-format \"text\" --stackdriver-export-interval=30s" cd "./ls_metrics" -chmod +x run_ls_benchmark.sh ./run_ls_benchmark.sh "$GCSFUSE_LIST_FLAGS" "$UPLOAD_FLAGS" rm -f $CONFIG_FILE_YML diff --git a/perfmetrics/scripts/continuous_test/ml_tests/build.sh b/perfmetrics/scripts/continuous_test/ml_tests/build.sh index f0a8cc2e5d..902d2049d9 100755 --- a/perfmetrics/scripts/continuous_test/ml_tests/build.sh +++ b/perfmetrics/scripts/continuous_test/ml_tests/build.sh @@ -14,7 +14,6 @@ source ~/.bashrc cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts" echo "Setting up a machine" -chmod +x ml_tests/setup.sh source ml_tests/setup.sh echo "Running ML model automation script" diff --git a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/dino/build.sh b/perfmetrics/scripts/continuous_test/ml_tests/pytorch/dino/build.sh index 3d421c8fbe..93c106c6e6 100755 --- a/perfmetrics/scripts/continuous_test/ml_tests/pytorch/dino/build.sh +++ b/perfmetrics/scripts/continuous_test/ml_tests/pytorch/dino/build.sh @@ -10,6 +10,5 @@ TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/ml_tests/pytorch/dino/setup cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/" -chmod +x run_and_manage_test.sh source run_and_manage_test.sh $VM_NAME $ZONE_NAME $ARTIFACTS_BUCKET_PATH $TEST_SCRIPT_PATH diff --git a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/build.sh b/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/build.sh index 6b61202d8f..4a361607ef 100755 --- a/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/build.sh +++ b/perfmetrics/scripts/continuous_test/ml_tests/tf/resnet/build.sh @@ -10,6 +10,5 @@ TEST_SCRIPT_PATH="github/gcsfuse/perfmetrics/scripts/ml_tests/tf/resnet/setup_ho cd "${KOKORO_ARTIFACTS_DIR}/github/gcsfuse/perfmetrics/scripts/continuous_test/ml_tests/" -chmod +x run_and_manage_test.sh source run_and_manage_test.sh $VM_NAME $ZONE_NAME $ARTIFACTS_BUCKET_PATH $TEST_SCRIPT_PATH diff --git a/perfmetrics/scripts/ml_tests/pytorch/dino/Dockerfile b/perfmetrics/scripts/ml_tests/pytorch/dino/Dockerfile index a044153515..8f0a34ed85 100644 --- a/perfmetrics/scripts/ml_tests/pytorch/dino/Dockerfile +++ b/perfmetrics/scripts/ml_tests/pytorch/dino/Dockerfile @@ -25,7 +25,6 @@ WORKDIR "/pytorch_dino/" RUN git clone "https://github.com/facebookresearch/dino" COPY perfmetrics/scripts/ml_tests/pytorch/dino/setup_container.sh ./ -RUN chmod +x setup_container.sh RUN mkdir -p "run_artifacts" RUN mkdir -p "gcsfuse_data" diff --git a/perfmetrics/scripts/ml_tests/pytorch/dino/setup_host_and_run_model.sh b/perfmetrics/scripts/ml_tests/pytorch/dino/setup_host_and_run_model.sh index 3f0616f690..3c710f2e14 100755 --- a/perfmetrics/scripts/ml_tests/pytorch/dino/setup_host_and_run_model.sh +++ b/perfmetrics/scripts/ml_tests/pytorch/dino/setup_host_and_run_model.sh @@ -6,7 +6,6 @@ set -e cd "$HOME/github/gcsfuse/perfmetrics/scripts" echo "Setting up the machine with Docker and Nvidia Driver" -chmod +x ml_tests/setup_host.sh source ml_tests/setup_host.sh cd "$HOME/github/gcsfuse" @@ -20,7 +19,6 @@ sudo docker run --gpus all --name=pytorch_automation_container --privileged -d - --shm-size=128g pytorch-gcsfuse:latest # Setup the log_rotation. -chmod +x perfmetrics/scripts/ml_tests/setup_log_rotation.sh source perfmetrics/scripts/ml_tests/setup_log_rotation.sh $HOME/github/gcsfuse/container_artifacts/gcsfuse.log # Wait for the script completion as well as logs output. diff --git a/perfmetrics/scripts/ml_tests/run_image_recognition_models.py b/perfmetrics/scripts/ml_tests/run_image_recognition_models.py index 34675cccc3..95abd2401d 100644 --- a/perfmetrics/scripts/ml_tests/run_image_recognition_models.py +++ b/perfmetrics/scripts/ml_tests/run_image_recognition_models.py @@ -136,7 +136,6 @@ def _run_model(directory_name, data_path, data_read_method, ml_model_path, req_f end_time = int(time.time()) os.system(f'''cd .. - chmod +x populate_metrics.sh ./populate_metrics.sh {start_time} {end_time} ''') diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile b/perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile index 73126499ad..053ea4e473 100644 --- a/perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile +++ b/perfmetrics/scripts/ml_tests/tf/resnet/Dockerfile @@ -30,7 +30,6 @@ WORKDIR "/tf_test/" COPY ./perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/setup_container.sh . COPY ./perfmetrics/scripts/ml_tests/tf/resnet/setup_scripts/resnet_runner.py . -RUN chmod +x setup_container.sh RUN mkdir -p "myBucket" ENTRYPOINT ["/bin/bash", "-c", "./setup_container.sh"] diff --git a/perfmetrics/scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh b/perfmetrics/scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh index a25852381c..ca8c7f9ae1 100755 --- a/perfmetrics/scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh +++ b/perfmetrics/scripts/ml_tests/tf/resnet/setup_host_and_run_model.sh @@ -6,7 +6,6 @@ set -e cd "$HOME/github/gcsfuse/perfmetrics/scripts" echo "Setting up the machine with Docker and Nvidia Driver..." -chmod +x ml_tests/setup_host.sh source ml_tests/setup_host.sh cd "$HOME/github/gcsfuse/" @@ -21,7 +20,6 @@ sudo docker run --gpus all --name tf_model_container --privileged -d \ -v $HOME/github/gcsfuse/container_artifacts/output:/home/output:rw,rshared --shm-size=24g tf-dlc-gcsfuse:latest # Setup the log_rotation. -chmod +x perfmetrics/scripts/ml_tests/setup_log_rotation.sh source perfmetrics/scripts/ml_tests/setup_log_rotation.sh $HOME/github/gcsfuse/container_artifacts/logs/gcsfuse.log # Wait for the script completion as well as logs output. diff --git a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh index 00048c1a96..3926c671dc 100755 --- a/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh +++ b/perfmetrics/scripts/presubmit_test/pr_perf_test/build.sh @@ -53,7 +53,6 @@ function execute_perf_test() { # The VM will itself exit if the gcsfuse mount fails. go run . $GCSFUSE_FLAGS $BUCKET_NAME $MOUNT_POINT # Running FIO test - chmod +x perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh ./perfmetrics/scripts/presubmit/run_load_test_on_presubmit.sh sudo umount gcs } @@ -115,7 +114,6 @@ then git checkout pr/$KOKORO_GITHUB_PULL_REQUEST_NUMBER echo "Running e2e tests...." - chmod +x perfmetrics/scripts/run_e2e_tests.sh # $1 argument is refering to value of testInstalledPackage. ./perfmetrics/scripts/run_e2e_tests.sh $RUN_E2E_TESTS_ON_INSTALLED_PACKAGE fi