From d4c59a1684df057c1ff14dd6f8a3f1c5554e84e1 Mon Sep 17 00:00:00 2001 From: anushkadhn Date: Thu, 19 Sep 2024 05:43:09 +0000 Subject: [PATCH] adding debug logs to rename perf tests --- .../continuous_test/gcp_ubuntu/build.sh | 9 +++-- .../continuous_test/gcp_ubuntu/continuous.cfg | 1 + .../renaming_benchmark.py | 36 ++++++------------- .../run_rename_benchmark.sh | 14 ++++---- 4 files changed, 25 insertions(+), 35 deletions(-) diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh index 0b42df52ea..4a424c081a 100755 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/build.sh @@ -86,9 +86,12 @@ run_ls_benchmark "$GCSFUSE_LS_FLAGS" "$SPREADSHEET_ID" "$LIST_CONFIG_FILE" # Running the rename benchmark script. cd "./hns_rename_folders_metrics" -# Rename perf tests are only performed daily on HNS bucket by default.Thus , creating log file with hns suffix only. -LOG_FILE_RENAME_TESTS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-rename-hns.txt -./run_rename_benchmark.sh $UPLOAD_FLAGS $LOG_FILE_RENAME_TESTS +# Rename perf tests are only performed daily on HNS bucket by default. +LOG_FILE_RENAME_TESTS_HNS=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-rename-hns.txt +LOG_FILE_RENAME_TESTS_FLAT=${KOKORO_ARTIFACTS_DIR}/gcsfuse-logs-rename-flat.txt +GCSFUSE_HNS_FLAGS="--log-severity=trace --log-format \"text\" --log-file $LOG_FILE_RENAME_TESTS_HNS --stackdriver-export-interval=30s" +GCSFUSE_FLAT_FLAGS="--log-severity=trace --log-format \"text\" --log-file $LOG_FILE_RENAME_TESTS_FLAT --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s" +./run_rename_benchmark.sh $GCSFUSE_HNS_FLAGS $GCSFUSE_FLAT_FLAGS $UPLOAD_FLAGS # TODO: Testing for hns bucket with client protocol set to grpc. To be done when diff --git a/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg b/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg index 20d8f478c2..af8a3adfcf 100644 --- a/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg +++ b/perfmetrics/scripts/continuous_test/gcp_ubuntu/continuous.cfg @@ -19,6 +19,7 @@ action { regex: "gcsfuse-logs-fio-flat.txt" regex: "gcsfuse-logs-ls-flat.txt" regex: "gcsfuse-logs-rename-hns.txt" + regex: "gcsfuse-logs-rename-flat.txt" regex: "github/gcsfuse/perfmetrics/scripts/fio-output.json" strip_prefix: "github/gcsfuse/perfmetrics/scripts" } diff --git a/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py b/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py index 85ca4bd0f9..13d84fb356 100644 --- a/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py +++ b/perfmetrics/scripts/hns_rename_folders_metrics/renaming_benchmark.py @@ -40,7 +40,6 @@ WORKSHEET_VM_METRICS_FLAT = 'vm_metrics_flat' WORKSHEET_VM_METRICS_HNS = 'vm_metrics_hns' SPREADSHEET_ID = '1UVEvsf49eaDJdTGLQU1rlNTIAxg8PZoNQCy_GX6Nw-A' -LOG_FILE='/tmp/gcsfuse-logs-rename.txt' INSTANCE=socket.gethostname() PERIOD_SEC=120 @@ -307,7 +306,7 @@ def _record_time_of_operation(mount_point, dir, num_samples): return results,time_interval_for_vm_metrics -def _perform_testing(dir, test_type, num_samples): +def _perform_testing(dir, mount_flags, num_samples): """ This function performs rename operations and records time of operation . Args: @@ -341,17 +340,6 @@ def _perform_testing(dir, test_type, num_samples): test_type : flat or hns. num_samples: Number of samples to collect for each test. """ - if test_type == "hns": - # Creating config file for mounting with hns enabled. - with open("/tmp/config.yml",'w') as mount_config: - mount_config.write("enable-hns: true") - mount_flags="--config-file=/tmp/config.yml --log-severity=trace --log-format \"text\" --log-file {} --stackdriver-export-interval=30s".format(LOG_FILE) - else : - # Creating config file for mounting with hns disabled. - with open("/tmp/config.yml",'w') as mount_config: - mount_config.write("enable-hns: false") - mount_flags = "--config-file=/tmp/config.yml --log-severity=trace --log-format \"text\" --log-file {} --implicit-dirs --rename-dir-limit=1000000 --stackdriver-export-interval=30s".format(LOG_FILE) - # Mounting the gcs bucket. bucket_name = mount_gcs_bucket(dir["name"], mount_flags, log) # Record time of operation and populate the results dict. @@ -377,6 +365,13 @@ def _parse_arguments(argv): action='store', choices=['hns','flat'] ) + parser.add_argument( + 'gcsfuse_flags', + help='Gcsfuse flags for mounting the tests bucket.', + action='store', + nargs=1, + required=True, + ) parser.add_argument( '--upload_gs', help='Upload the results to the Google Sheet.', @@ -392,13 +387,6 @@ def _parse_arguments(argv): required=False, type=int, ) - parser.add_argument( - '--log_file', - help='Provide the log file path', - action= 'store', - required=False, - type=str, - ) return parser.parse_args(argv[1:]) @@ -451,7 +439,7 @@ def _get_upload_value_for_vm_metrics(vm_metrics): return upload_values -def _run_rename_benchmark(test_type,dir_config,num_samples,upload_gs): +def _run_rename_benchmark(test_type,dir_config,mount_flags,num_samples,upload_gs): with open(os.path.abspath(dir_config)) as file: dir_str = json.load(file) @@ -468,7 +456,7 @@ def _run_rename_benchmark(test_type,dir_config,num_samples,upload_gs): sys.exit(1) # Getting latency related metrics - results,time_intervals=_perform_testing(dir_str, test_type, num_samples) + results,time_intervals=_perform_testing(dir_str, mount_flags, num_samples) parsed_metrics = _parse_results(dir_str, results, num_samples) upload_values = _get_values_to_export(dir_str, parsed_metrics, test_type) @@ -518,8 +506,6 @@ def _run_rename_benchmark(test_type,dir_config,num_samples,upload_gs): 'python3 renaming_benchmark.py [--upload_gs] [--num_samples NUM_SAMPLES] config_file bucket_type') args = _parse_arguments(argv) - if args.log_file : - LOG_FILE = args.log_file check_dependencies(['gcloud', 'gcsfuse'], log) - _run_rename_benchmark(args.bucket_type, args.config_file, args.num_samples, + _run_rename_benchmark(args.bucket_type, args.config_file,args.gcsfuse_flags, args.num_samples, args.upload_gs) diff --git a/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh b/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh index e28c7fc8e3..8e57457e25 100755 --- a/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh +++ b/perfmetrics/scripts/hns_rename_folders_metrics/run_rename_benchmark.sh @@ -29,19 +29,19 @@ echo "Installing Ops Agent on Vm" curl -sSO https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh sudo bash add-google-cloud-ops-agent-repo.sh --also-install -cd "../" -./upgrade_gcloud.sh -cd "hns_rename_folders_metrics" +../upgrade_gcloud.sh -UPLOAD_FLAGS=$1 +GCSFUSE_HNS_FLAGS=$1 +GCSFUSE_FLAT_FLAGS=$2 +UPLOAD_FLAGS=$3 gsutil cp gs://periodic-perf-tests/creds.json ../gsheet/ echo "Upgrading gcloud version" ../upgrade_gcloud.sh +# Uncomment the following 2 lines to run the benchmark on flat bucket #echo "Running renaming benchmark on flat bucket" -#python3 renaming_benchmark.py config-flat.json flat "$UPLOAD_FLAGS" +#python3 renaming_benchmark.py config-flat.json flat "$GCSFUSE_FLAT_FLAGS" "$UPLOAD_FLAGS" echo "Running renaming benchmark on HNS bucket" -LOG_FILE_HNS=$2 -python3 renaming_benchmark.py config-hns.json hns $UPLOAD_FLAGS --log_file $LOG_FILE_HNS +python3 renaming_benchmark.py config-hns.json hns "$GCSFUSE_HNS_FLAGS" $UPLOAD_FLAGS