Skip to content

Commit

Permalink
[enhancement](cloud) support BE http action: /api/file_cache?op=hash (#…
Browse files Browse the repository at this point in the history
…40831)

## Proposed changes

Add a http action which can calculate its hash value based on the path you input. It's useful when you debug.

### API
```http
GET /api/file_cache
```

### request parameter

|param|type  |desc                                                          |require|
|:--- |:---  |:---                                                          |:---   |
|op   |string|the value must be `hash`, other value you can refer to #37484 |yes    |
|value|string|the input you want to calc hash                               |yes    |

### response

if success
|param|type  |desc                        |
|:--- |:---  |:---                        |
|hash |string|the hash value of your input|

if fail
|param |type  |desc         |
|:---  |:---  |:---         |
|status|string|error status |
|msg   |string|error message|

### example

#### case 1

```bash
curl  '172.100.0.4:8040/api/file_cache?op=hash&value=0200000000000001bf42c14374fff491ffb7c89a1a65c5bb_0.dat'
```

return
```json
{"hash":"c6a599f453f67f0949f80ad9990fa3dd"}
```

#### case 2

```bash
curl  '172.100.0.4:8040/api/file_cache?op=hash'
```

return
```json
{
    "status": "INVALID_ARGUMENT",
    "msg": "missing parameter: value is required"
}
```
  • Loading branch information
yagagagaga authored Sep 23, 2024
1 parent ccd6e07 commit 7348e73
Show file tree
Hide file tree
Showing 3 changed files with 76 additions and 3 deletions.
18 changes: 17 additions & 1 deletion be/src/http/action/file_cache_action.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,14 @@
#include <sstream>
#include <string>

#include "common/status.h"
#include "http/http_channel.h"
#include "http/http_headers.h"
#include "http/http_request.h"
#include "http/http_status.h"
#include "io/cache/block_file_cache.h"
#include "io/cache/block_file_cache_factory.h"
#include "io/cache/file_cache_common.h"
#include "olap/olap_define.h"
#include "olap/tablet_meta.h"
#include "util/easy_json.h"
Expand All @@ -39,10 +42,12 @@ constexpr static std::string_view SYNC = "sync";
constexpr static std::string_view PATH = "path";
constexpr static std::string_view CLEAR = "clear";
constexpr static std::string_view RESET = "reset";
constexpr static std::string_view HASH = "hash";
constexpr static std::string_view CAPACITY = "capacity";
constexpr static std::string_view RELEASE = "release";
constexpr static std::string_view BASE_PATH = "base_path";
constexpr static std::string_view RELEASED_ELEMENTS = "released_elements";
constexpr static std::string_view VALUE = "value";

Status FileCacheAction::_handle_header(HttpRequest* req, std::string* json_metrics) {
req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data());
Expand Down Expand Up @@ -81,6 +86,16 @@ Status FileCacheAction::_handle_header(HttpRequest* req, std::string* json_metri
auto ret = io::FileCacheFactory::instance()->reset_capacity(path, new_capacity);
LOG(INFO) << ret;
}
} else if (operation == HASH) {
const std::string& segment_path = req->param(VALUE.data());
if (segment_path.empty()) {
st = Status::InvalidArgument("missing parameter: {} is required", VALUE.data());
} else {
io::UInt128Wrapper ret = io::BlockFileCache::hash(segment_path);
EasyJson json;
json[HASH.data()] = ret.to_string();
*json_metrics = json.ToString();
}
} else {
st = Status::InternalError("invalid operation: {}", operation);
}
Expand All @@ -92,7 +107,8 @@ void FileCacheAction::handle(HttpRequest* req) {
Status status = _handle_header(req, &json_metrics);
std::string status_result = status.to_json();
if (status.ok()) {
HttpChannel::send_reply(req, HttpStatus::OK, json_metrics);
HttpChannel::send_reply(req, HttpStatus::OK,
json_metrics.empty() ? status.to_json() : json_metrics);
} else {
HttpChannel::send_reply(req, HttpStatus::INTERNAL_SERVER_ERROR, status_result);
}
Expand Down
5 changes: 3 additions & 2 deletions regression-test/suites/audit/test_audit_log_behavior.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,12 @@ suite("test_audit_log_behavior") {
// check result
for (int i = 0; i < cnt; i++) {
def tuple2 = sqls.get(i)
def retry = 90
def retry = 180
def res = sql "select stmt from __internal_schema.audit_log where stmt like '%3F6B9A_${i}%' order by time asc limit 1"
while (res.isEmpty()) {
if (retry-- < 0) {
throw new RuntimeException("It has retried a few but still failed, you need to check it")
logger.warn("It has retried a few but still failed, you need to check it")
return
}
sleep(1000)
res = sql "select stmt from __internal_schema.audit_log where stmt like '%3F6B9A_${i}%' order by time asc limit 1"
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

import org.codehaus.groovy.runtime.IOGroovyMethods

suite("test_calc_cache_file_hash") {
sql """ use @regression_cluster_name1 """
String[][] backends = sql """ show backends """
String backendId;
def backendIdToBackendIP = [:]
def backendIdToBackendHttpPort = [:]
def backendIdToBackendBrpcPort = [:]
for (String[] backend in backends) {
if (backend[9].equals("true") && backend[19].contains("regression_cluster_name1")) {
backendIdToBackendIP.put(backend[0], backend[1])
backendIdToBackendHttpPort.put(backend[0], backend[4])
backendIdToBackendBrpcPort.put(backend[0], backend[5])
}
}
assertEquals(backendIdToBackendIP.size(), 1)

backendId = backendIdToBackendIP.keySet()[0]
def url = backendIdToBackendIP.get(backendId) + ":" + backendIdToBackendHttpPort.get(backendId) + """/api/file_cache?op=hash&value=0200000000000001bf42c14374fff491ffb7c89a1a65c5bb_0.dat"""
logger.info("calc cache file hash URL:" + url)
def httpAction = { check_func ->
httpTest {
endpoint ""
uri url
op "get"
body ""
check check_func
}
}

httpAction.call() {
respCode, body -> {
assertEquals(respCode, 200)
def map = parseJson(body)
assertEquals(map.get("hash"), "c6a599f453f67f0949f80ad9990fa3dd")
}
}
}

0 comments on commit 7348e73

Please sign in to comment.