upgrading gcloud version and adding debug logs to rename perf tests

perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh

@@ -84,9 +84,11 @@ LIST_CONFIG_FILE="config-hns.json"
 run_load_test_and_fetch_metrics "$GCSFUSE_FIO_FLAGS" "$BUCKET_NAME" "$SPREADSHEET_ID"
 run_ls_benchmark "$GCSFUSE_LS_FLAGS" "$SPREADSHEET_ID" "$LIST_CONFIG_FILE"
 
-#Running the rename benchmark script.
+# Running the rename benchmark script.
 cd "./hns_rename_folders_metrics"
-./run_rename_benchmark.sh $UPLOAD_FLAGS
+# Rename perf tests are only performed daily on HNS bucket by default.Thus , creating log file with hns suffix only.
+LOG_FILE_RENAME_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-rename-hns.txt
+./run_rename_benchmark.sh $UPLOAD_FLAGS $LOG_FILE_RENAME_TESTS
 
 
 # TODO: Testing for hns bucket with client protocol set to grpc. To be done when

perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py

@@ -40,6 +40,7 @@
 WORKSHEET_VM_METRICS_FLAT = 'vm_metrics_flat'
 WORKSHEET_VM_METRICS_HNS = 'vm_metrics_hns'
 SPREADSHEET_ID = '1UVEvsf49eaDJdTGLQU1rlNTIAxg8PZoNQCy_GX6Nw-A'
+LOG_FILE='/tmp/gcsfuse-logs-rename.txt'
 INSTANCE=socket.gethostname()
 PERIOD_SEC=120
 
@@ -344,12 +345,12 @@ def _perform_testing(dir, test_type, num_samples):
     # Creating config file for mounting with hns enabled.
     with open("/tmp/config.yml",'w') as mount_config:
       mount_config.write("enable-hns: true")
-    mount_flags="--config-file=/tmp/config.yml --stackdriver-export-interval=30s"
+    mount_flags="--config-file=/tmp/config.yml --debug_gcs --debug_fuse --log-format \"text\" --log-file {} --stackdriver-export-interval=30s".format(LOG_FILE)
   else :
     # Creating config file for mounting with hns disabled.
     with open("/tmp/config.yml",'w') as mount_config:
       mount_config.write("enable-hns: false")
-    mount_flags = "--config-file=/tmp/config.yml  --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s"
+    mount_flags = "--config-file=/tmp/config.yml --debug_gcs --debug_fuse --log-format \"text\" --log-file {}  --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s".format(LOG_FILE)
 
   # Mounting the gcs bucket.
   bucket_name = mount_gcs_bucket(dir["name"], mount_flags, log)
@@ -391,6 +392,13 @@ def _parse_arguments(argv):
       required=False,
       type=int,
   )
+  parser.add_argument(
+      '--log_file',
+      help='Provide the log file path',
+      action= 'store',
+      required=False,
+      type=str,
+  )
 
   return parser.parse_args(argv[1:])
 
@@ -510,6 +518,8 @@ def _run_rename_benchmark(test_type,dir_config,num_samples,upload_gs):
                     'python3 renaming_benchmark.py  [--upload_gs] [--num_samples NUM_SAMPLES] config_file bucket_type')
 
   args = _parse_arguments(argv)
+  if args.log_file :
+    LOG_FILE = args.log_file
   check_dependencies(['gcloud', 'gcsfuse'], log)
   _run_rename_benchmark(args.bucket_type, args.config_file, args.num_samples,
                           args.upload_gs)

perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark_test.py

@@ -133,7 +133,7 @@ def test_perform_testing_flat(self, mock_log, mock_record_time_of_operation,
     test_type = "flat"
     num_samples = 4
     results = {}
-    mount_flags = "--config-file=/tmp/config.yml  --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s"
+    mount_flags = "--config-file=/tmp/config.yml --debug_gcs --debug_fuse --log-format \"text\" --log-file /tmp/gcsfuse-logs-rename.txt  --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s"
     mock_mount_gcs_bucket.return_value="flat_bucket"
     mock_record_time_of_operation.return_value = [{"test_folder": [0.1, 0.2, 0.3, 0.4]},[[0.1,0.4]]]
     expected_results = {"test_folder": [0.1, 0.2, 0.3, 0.4]}
@@ -170,7 +170,7 @@ def test_perform_testing_hns(self, mock_log, mock_record_time_of_operation,
     test_type = "hns"
     num_samples = 4
     results = {}
-    mount_flags = "--config-file=/tmp/config.yml --stackdriver-export-interval=30s"
+    mount_flags = "--config-file=/tmp/config.yml --debug_gcs --debug_fuse --log-format \"text\" --log-file /tmp/gcsfuse-logs-rename.txt --stackdriver-export-interval=30s"
     mock_mount_gcs_bucket.return_value="hns_bucket"
     mock_record_time_of_operation.return_value = [{"test_folder": [0.1, 0.2, 0.3, 0.4]},[[0.1,0.4]]]
     expected_results = {"test_folder": [0.1, 0.2, 0.3, 0.4]}

perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh

@@ -29,6 +29,10 @@ echo "Installing Ops Agent on Vm"
 curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh
 sudo bash add-google-cloud-ops-agent-repo.sh --also-install
 
+cd "../"
+./upgrade_gcloud.sh
+cd "hns_rename_folders_metrics"
+
 UPLOAD_FLAGS=$1
 gsutil cp gs://periodic-perf-tests/creds.json ../gsheet/
 
@@ -37,4 +41,5 @@ gsutil cp gs://periodic-perf-tests/creds.json ../gsheet/
 #python3 renaming_benchmark.py config-flat.json flat "$UPLOAD_FLAGS"
 
 echo "Running renaming benchmark on HNS bucket"
-python3 renaming_benchmark.py config-hns.json hns $UPLOAD_FLAGS
+LOG_FILE_HNS=$2
+python3 renaming_benchmark.py config-hns.json hns $UPLOAD_FLAGS --log_file $LOG_FILE_HNS

perfmetrics/scripts/upgrade_gcloud.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+sudo apt-get update
+# Upgrade gcloud version.
+# Kokoro machine's outdated gcloud version prevents the use of the "gcloud storage" feature.
+gcloud version
+wget -O gcloud.tar.gz https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz -q
+sudo tar xzf gcloud.tar.gz && sudo cp -r google-cloud-sdk /usr/local && sudo rm -r google-cloud-sdk
+sudo /usr/local/google-cloud-sdk/install.sh
+export PATH=/usr/local/google-cloud-sdk/bin:$PATH
+echo 'export PATH=/usr/local/google-cloud-sdk/bin:$PATH' >> ~/.bashrc
+gcloud version && rm gcloud.tar.gz
+sudo /usr/local/google-cloud-sdk/bin/gcloud components update
+sudo /usr/local/google-cloud-sdk/bin/gcloud components install alpha