diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh index f68377b275..0b42df52ea 100755 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh @@ -84,9 +84,11 @@ LIST_CONFIG_FILE="config-hns.json" run_load_test_and_fetch_metrics "$GCSFUSE_FIO_FLAGS" "$BUCKET_NAME" "$SPREADSHEET_ID" run_ls_benchmark "$GCSFUSE_LS_FLAGS" "$SPREADSHEET_ID" "$LIST_CONFIG_FILE" -#Running the rename benchmark script. +# Running the rename benchmark script. cd "./hns_rename_folders_metrics" -./run_rename_benchmark.sh $UPLOAD_FLAGS +# Rename perf tests are only performed daily on HNS bucket by default.Thus , creating log file with hns suffix only. +LOG_FILE_RENAME_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-rename-hns.txt +./run_rename_benchmark.sh $UPLOAD_FLAGS $LOG_FILE_RENAME_TESTS # TODO: Testing for hns bucket with client protocol set to grpc. To be done when diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg index 8c4967c5f7..20d8f478c2 100644 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg @@ -18,6 +18,7 @@ action { regex: "gcsfuse-logs-ls-hns.txt" regex: "gcsfuse-logs-fio-flat.txt" regex: "gcsfuse-logs-ls-flat.txt" + regex: "gcsfuse-logs-rename-hns.txt" regex: "github/gcsfuse/perfmetrics/scripts/fio-output.json" strip_prefix: "github/gcsfuse/perfmetrics/scripts" } diff --git a/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py b/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py index 4aeb76c3ea..85ca4bd0f9 100644 --- a/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py +++ b/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py @@ -40,6 +40,7 @@ WORKSHEET_VM_METRICS_FLAT = 'vm_metrics_flat' WORKSHEET_VM_METRICS_HNS = 'vm_metrics_hns' SPREADSHEET_ID = '1UVEvsf49eaDJdTGLQU1rlNTIAxg8PZoNQCy_GX6Nw-A' +LOG_FILE='/tmp/gcsfuse-logs-rename.txt' INSTANCE=socket.gethostname() PERIOD_SEC=120 @@ -344,12 +345,12 @@ def _perform_testing(dir, test_type, num_samples): # Creating config file for mounting with hns enabled. with open("/tmp/config.yml",'w') as mount_config: mount_config.write("enable-hns: true") - mount_flags="--config-file=/tmp/config.yml --stackdriver-export-interval=30s" + mount_flags="--config-file=/tmp/config.yml --log-severity=trace --log-format \"text\" --log-file {} --stackdriver-export-interval=30s".format(LOG_FILE) else : # Creating config file for mounting with hns disabled. with open("/tmp/config.yml",'w') as mount_config: mount_config.write("enable-hns: false") - mount_flags = "--config-file=/tmp/config.yml --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s" + mount_flags = "--config-file=/tmp/config.yml --log-severity=trace --log-format \"text\" --log-file {} --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s".format(LOG_FILE) # Mounting the gcs bucket. bucket_name = mount_gcs_bucket(dir["name"], mount_flags, log) @@ -391,6 +392,13 @@ def _parse_arguments(argv): required=False, type=int, ) + parser.add_argument( + '--log_file', + help='Provide the log file path', + action= 'store', + required=False, + type=str, + ) return parser.parse_args(argv[1:]) @@ -510,6 +518,8 @@ def _run_rename_benchmark(test_type,dir_config,num_samples,upload_gs): 'python3 renaming_benchmark.py [--upload_gs] [--num_samples NUM_SAMPLES] config_file bucket_type') args = _parse_arguments(argv) + if args.log_file : + LOG_FILE = args.log_file check_dependencies(['gcloud', 'gcsfuse'], log) _run_rename_benchmark(args.bucket_type, args.config_file, args.num_samples, args.upload_gs) diff --git a/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark_test.py b/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark_test.py index 86e06d7e22..41cbc2fc88 100644 --- a/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark_test.py +++ b/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark_test.py @@ -133,7 +133,7 @@ def test_perform_testing_flat(self, mock_log, mock_record_time_of_operation, test_type = "flat" num_samples = 4 results = {} - mount_flags = "--config-file=/tmp/config.yml --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s" + mount_flags = "--config-file=/tmp/config.yml --log-severity=trace --log-format \"text\" --log-file /tmp/gcsfuse-logs-rename.txt --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s" mock_mount_gcs_bucket.return_value="flat_bucket" mock_record_time_of_operation.return_value = [{"test_folder": [0.1, 0.2, 0.3, 0.4]},[[0.1,0.4]]] expected_results = {"test_folder": [0.1, 0.2, 0.3, 0.4]} @@ -170,7 +170,7 @@ def test_perform_testing_hns(self, mock_log, mock_record_time_of_operation, test_type = "hns" num_samples = 4 results = {} - mount_flags = "--config-file=/tmp/config.yml --stackdriver-export-interval=30s" + mount_flags = "--config-file=/tmp/config.yml --log-severity=trace --log-format \"text\" --log-file /tmp/gcsfuse-logs-rename.txt --stackdriver-export-interval=30s" mock_mount_gcs_bucket.return_value="hns_bucket" mock_record_time_of_operation.return_value = [{"test_folder": [0.1, 0.2, 0.3, 0.4]},[[0.1,0.4]]] expected_results = {"test_folder": [0.1, 0.2, 0.3, 0.4]} diff --git a/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh b/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh index fd1f102ed2..e28c7fc8e3 100755 --- a/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh +++ b/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh @@ -29,6 +29,10 @@ echo "Installing Ops Agent on Vm" curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh sudo bash add-google-cloud-ops-agent-repo.sh --also-install +cd "../" +./upgrade_gcloud.sh +cd "hns_rename_folders_metrics" + UPLOAD_FLAGS=$1 gsutil cp gs://periodic-perf-tests/creds.json ../gsheet/ @@ -39,4 +43,5 @@ echo "Upgrading gcloud version" #python3 renaming_benchmark.py config-flat.json flat "$UPLOAD_FLAGS" echo "Running renaming benchmark on HNS bucket" -python3 renaming_benchmark.py config-hns.json hns $UPLOAD_FLAGS +LOG_FILE_HNS=$2 +python3 renaming_benchmark.py config-hns.json hns $UPLOAD_FLAGS --log_file $LOG_FILE_HNS