diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh index 1fd1abfd8d..07cdf53137 100755 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh @@ -92,7 +92,7 @@ LOG_FILE_RENAME_TESTS_FLAT=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-rename-flat.txt GCSFUSE_HNS_FLAGS="$COMMON_MOUNT_FLAGS --log-file $LOG_FILE_RENAME_TESTS_HNS --stackdriver-export-interval=30s" GCSFUSE_FLAT_FLAGS="$COMMON_MOUNT_FLAGS --log-file $LOG_FILE_RENAME_TESTS_FLAT --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s" ./run_rename_benchmark.sh $GCSFUSE_HNS_FLAGS $GCSFUSE_FLAT_FLAGS $UPLOAD_FLAGS - +cd ".." # TODO: Testing for hns bucket with client protocol set to grpc. To be done when # includeFolderAsPrefixes is supported in grpc. diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg index af8a3adfcf..4a0d05fa24 100644 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg @@ -14,12 +14,12 @@ action { define_artifacts { - regex: "gcsfuse-logs-fio-hns.txt" - regex: "gcsfuse-logs-ls-hns.txt" - regex: "gcsfuse-logs-fio-flat.txt" - regex: "gcsfuse-logs-ls-flat.txt" - regex: "gcsfuse-logs-rename-hns.txt" - regex: "gcsfuse-logs-rename-flat.txt" + regex: "gcsfuse-logs-fio-hns\.txt" + regex: "gcsfuse-logs-ls-hns\.txt" + regex: "gcsfuse-logs-fio-flat\.txt" + regex: "gcsfuse-logs-ls-flat\.txt" + regex: "gcsfuse-logs-rename-hns\.txt" + regex: "gcsfuse-logs-rename-flat\.txt" regex: "github/gcsfuse/perfmetrics/scripts/fio-output.json" strip_prefix: "github/gcsfuse/perfmetrics/scripts" } diff --git a/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py b/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py index 16228ef53e..c8934cd70e 100644 --- a/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py +++ b/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py @@ -13,8 +13,8 @@ # limitations under the License. # To run the script,run in terminal: -# python3 renaming_benchmark.py config.json bucket_type [--upload_gs] \ -# [--num_samples NUM_SAMPLES] +# python3 renaming_benchmark.py dir-config.json bucket_type gcsfuse_mount_flags \ +# [--upload_gs] [--num_samples NUM_SAMPLES] # where dir-config.json file contains the directory structure details for the test. import os @@ -295,7 +295,7 @@ def _record_time_of_operation(mount_point, dir, num_samples): time_interval_for_vm_metrics={} # Collecting metrics for non-nested folders. for folder in dir["folders"]["folder_structure"]: - log.info("Testing started for {}".folder["name"]) + log.info("Testing started for {}".format(folder["name"])) results[folder["name"]],time_interval = _record_time_for_folder_rename(mount_point,folder,num_samples) time_interval_for_vm_metrics[folder["name"]]=[time_interval[0][0],time_interval[-1][-1]] @@ -371,8 +371,6 @@ def _parse_arguments(argv): 'gcsfuse_flags', help='Gcsfuse flags for mounting the tests bucket.', action='store', - nargs=1, - required=True, ) parser.add_argument( '--upload_gs', diff --git a/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh b/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh index cb76a4796f..8ca46d4829 100755 --- a/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh +++ b/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh @@ -29,6 +29,7 @@ echo "Installing Ops Agent on Vm" curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh sudo bash add-google-cloud-ops-agent-repo.sh --also-install +echo "Upgrading gcloud version" ../upgrade_gcloud.sh GCSFUSE_HNS_FLAGS=$1 @@ -36,9 +37,6 @@ GCSFUSE_FLAT_FLAGS=$2 UPLOAD_FLAGS=$3 gsutil cp gs://periodic-perf-tests/creds.json ../gsheet/ -echo "Upgrading gcloud version" -../upgrade_gcloud.sh - # Uncomment the following 2 lines to run the benchmark on flat bucket. #echo "Running renaming benchmark on flat bucket" #python3 renaming_benchmark.py config-flat.json flat "$GCSFUSE_FLAT_FLAGS" "$UPLOAD_FLAGS" diff --git a/perfmetrics/scripts/upgrade_gcloud.sh b/perfmetrics/scripts/upgrade_gcloud.sh index bee0176b7e..36e12d445a 100755 --- a/perfmetrics/scripts/upgrade_gcloud.sh +++ b/perfmetrics/scripts/upgrade_gcloud.sh @@ -13,15 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +set -e sudo apt-get update # Upgrade gcloud version. # Kokoro machine's outdated gcloud version prevents the use of the "gcloud storage" feature. gcloud version wget -O gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz -q -sudo tar xzf gcloud.tar.gz && sudo cp -r google-cloud-sdk /usr/local && sudo rm -r google-cloud-sdk +sudo tar xzf gcloud.tar.gz && sudo mv google-cloud-sdk /usr/local sudo /usr/local/google-cloud-sdk/install.sh export PATH=/usr/local/google-cloud-sdk/bin:$PATH echo 'export PATH=/usr/local/google-cloud-sdk/bin:$PATH' >> ~/.bashrc gcloud version && rm gcloud.tar.gz -sudo /usr/local/google-cloud-sdk/bin/gcloud components update -sudo /usr/local/google-cloud-sdk/bin/gcloud components install alpha +sudo gcloud components update +sudo gcloud components install alpha