From 7348e738b2171a4a6198bdb4f0a97429cf524ebd Mon Sep 17 00:00:00 2001 From: yagagagaga Date: Mon, 23 Sep 2024 17:15:23 +0800 Subject: [PATCH] [enhancement](cloud) support BE http action: /api/file_cache?op=hash (#40831) ## Proposed changes Add a http action which can calculate its hash value based on the path you input. It's useful when you debug. ### API ```http GET /api/file_cache ``` ### request parameter |param|type |desc |require| |:--- |:--- |:--- |:--- | |op |string|the value must be `hash`, other value you can refer to #37484 |yes | |value|string|the input you want to calc hash |yes | ### response if success |param|type |desc | |:--- |:--- |:--- | |hash |string|the hash value of your input| if fail |param |type |desc | |:--- |:--- |:--- | |status|string|error status | |msg |string|error message| ### example #### case 1 ```bash curl '172.100.0.4:8040/api/file_cache?op=hash&value=0200000000000001bf42c14374fff491ffb7c89a1a65c5bb_0.dat' ``` return ```json {"hash":"c6a599f453f67f0949f80ad9990fa3dd"} ``` #### case 2 ```bash curl '172.100.0.4:8040/api/file_cache?op=hash' ``` return ```json { "status": "INVALID_ARGUMENT", "msg": "missing parameter: value is required" } ``` --- be/src/http/action/file_cache_action.cpp | 18 +++++- .../audit/test_audit_log_behavior.groovy | 5 +- .../http/test_calc_cache_file_hash.groovy | 56 +++++++++++++++++++ 3 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 regression-test/suites/cloud_p0/cache/http/test_calc_cache_file_hash.groovy diff --git a/be/src/http/action/file_cache_action.cpp b/be/src/http/action/file_cache_action.cpp index 659be2537997f3..f31c040c5cf672 100644 --- a/be/src/http/action/file_cache_action.cpp +++ b/be/src/http/action/file_cache_action.cpp @@ -22,11 +22,14 @@ #include #include +#include "common/status.h" #include "http/http_channel.h" #include "http/http_headers.h" #include "http/http_request.h" #include "http/http_status.h" +#include "io/cache/block_file_cache.h" #include "io/cache/block_file_cache_factory.h" +#include "io/cache/file_cache_common.h" #include "olap/olap_define.h" #include "olap/tablet_meta.h" #include "util/easy_json.h" @@ -39,10 +42,12 @@ constexpr static std::string_view SYNC = "sync"; constexpr static std::string_view PATH = "path"; constexpr static std::string_view CLEAR = "clear"; constexpr static std::string_view RESET = "reset"; +constexpr static std::string_view HASH = "hash"; constexpr static std::string_view CAPACITY = "capacity"; constexpr static std::string_view RELEASE = "release"; constexpr static std::string_view BASE_PATH = "base_path"; constexpr static std::string_view RELEASED_ELEMENTS = "released_elements"; +constexpr static std::string_view VALUE = "value"; Status FileCacheAction::_handle_header(HttpRequest* req, std::string* json_metrics) { req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data()); @@ -81,6 +86,16 @@ Status FileCacheAction::_handle_header(HttpRequest* req, std::string* json_metri auto ret = io::FileCacheFactory::instance()->reset_capacity(path, new_capacity); LOG(INFO) << ret; } + } else if (operation == HASH) { + const std::string& segment_path = req->param(VALUE.data()); + if (segment_path.empty()) { + st = Status::InvalidArgument("missing parameter: {} is required", VALUE.data()); + } else { + io::UInt128Wrapper ret = io::BlockFileCache::hash(segment_path); + EasyJson json; + json[HASH.data()] = ret.to_string(); + *json_metrics = json.ToString(); + } } else { st = Status::InternalError("invalid operation: {}", operation); } @@ -92,7 +107,8 @@ void FileCacheAction::handle(HttpRequest* req) { Status status = _handle_header(req, &json_metrics); std::string status_result = status.to_json(); if (status.ok()) { - HttpChannel::send_reply(req, HttpStatus::OK, json_metrics); + HttpChannel::send_reply(req, HttpStatus::OK, + json_metrics.empty() ? status.to_json() : json_metrics); } else { HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result); } diff --git a/regression-test/suites/audit/test_audit_log_behavior.groovy b/regression-test/suites/audit/test_audit_log_behavior.groovy index 1c30a38ac69c0e..2829474560e1cf 100644 --- a/regression-test/suites/audit/test_audit_log_behavior.groovy +++ b/regression-test/suites/audit/test_audit_log_behavior.groovy @@ -83,11 +83,12 @@ suite("test_audit_log_behavior") { // check result for (int i = 0; i < cnt; i++) { def tuple2 = sqls.get(i) - def retry = 90 + def retry = 180 def res = sql "select stmt from __internal_schema.audit_log where stmt like '%3F6B9A_${i}%' order by time asc limit 1" while (res.isEmpty()) { if (retry-- < 0) { - throw new RuntimeException("It has retried a few but still failed, you need to check it") + logger.warn("It has retried a few but still failed, you need to check it") + return } sleep(1000) res = sql "select stmt from __internal_schema.audit_log where stmt like '%3F6B9A_${i}%' order by time asc limit 1" diff --git a/regression-test/suites/cloud_p0/cache/http/test_calc_cache_file_hash.groovy b/regression-test/suites/cloud_p0/cache/http/test_calc_cache_file_hash.groovy new file mode 100644 index 00000000000000..6d800c7c5f003b --- /dev/null +++ b/regression-test/suites/cloud_p0/cache/http/test_calc_cache_file_hash.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.codehaus.groovy.runtime.IOGroovyMethods + +suite("test_calc_cache_file_hash") { + sql """ use @regression_cluster_name1 """ + String[][] backends = sql """ show backends """ + String backendId; + def backendIdToBackendIP = [:] + def backendIdToBackendHttpPort = [:] + def backendIdToBackendBrpcPort = [:] + for (String[] backend in backends) { + if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) { + backendIdToBackendIP.put(backend[0], backend[1]) + backendIdToBackendHttpPort.put(backend[0], backend[4]) + backendIdToBackendBrpcPort.put(backend[0], backend[5]) + } + } + assertEquals(backendIdToBackendIP.size(), 1) + + backendId = backendIdToBackendIP.keySet()[0] + def url = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + """/api/file_cache?op=hash&value=0200000000000001bf42c14374fff491ffb7c89a1a65c5bb_0.dat""" + logger.info("calc cache file hash URL:" + url) + def httpAction = { check_func -> + httpTest { + endpoint "" + uri url + op "get" + body "" + check check_func + } + } + + httpAction.call() { + respCode, body -> { + assertEquals(respCode, 200) + def map = parseJson(body) + assertEquals(map.get("hash"), "c6a599f453f67f0949f80ad9990fa3dd") + } + } +}