From 52a5ef0e6af3d5271edc9f58ffcfae5c770cdca1 Mon Sep 17 00:00:00 2001 From: xueweizhang Date: Tue, 19 Dec 2023 11:44:33 +0800 Subject: [PATCH] [improvement](http) add show_table_data http api (#28380) In some cases, users need to get the data size of single replica of a table, and evaluate certain actions based on this, such as estimating the precise backup size. Signed-off-by: nextdreamblue --- .../http-actions/fe/show-table-data-action.md | 163 ++++++++++++++++++ docs/sidebars.json | 1 + .../http-actions/fe/show-table-data-action.md | 163 ++++++++++++++++++ .../doris/httpv2/rest/RestBaseController.java | 1 + .../apache/doris/httpv2/rest/ShowAction.java | 74 ++++++++ 5 files changed, 402 insertions(+) create mode 100644 docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md create mode 100644 docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md diff --git a/docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md b/docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md new file mode 100644 index 00000000000000..9cdbe2c49ad729 --- /dev/null +++ b/docs/en/docs/admin-manual/http-actions/fe/show-table-data-action.md @@ -0,0 +1,163 @@ +--- +{ + "title": "Show Table Data Action", + "language": "en" +} +--- + + + +# Show Table Data Action + +## Request + +`GET /api/show_table_data` + +## Description + +Used to get the data size of all tables in all databases under all internal catalog, or the data size of the specified database or table. Unit byte. + +## Path parameters + +NULL + +## Query parameters + +* `db` + + Optional. If specified, get the data size of the tables under the specified database. + +* `table` + + Optional. If specified, get the data size of the specified table. + +* `single_replica` + + Optional. If specified, get the data size of the single replica of the table. + +## Request body + +NULL + +## Response + +1. The data size of all tables in the specified database. + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244, + "revenue0":0, + "customer":1906421482 + } + }, + "count":0 + } + ``` + +2. The data size of the specified table of the specified db. + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244 + } + }, + "count":0 + } + ``` + +3. The data size of the single replica of the specified table of the specified db. + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":3008182748 + } + }, + "count":0 + } + ``` + +## Examples + +1. The data size of all tables in the specified database. + + ``` + GET /api/show_table_data?db=tpch + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244, + "revenue0":0, + "customer":1906421482 + } + }, + "count":0 + } + ``` + +2. The data size of the specified table of the specified db. + + ``` + GET /api/show_table_data?db=tpch&table=partsupp + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244 + } + }, + "count":0 + } + ``` +3. The data size of the single replica of the specified table of the specified db. + + ``` + GET /api/show_table_data?db=tpch&table=partsupp&single_replica=true + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":3008182748 + } + }, + "count":0 + } + ``` diff --git a/docs/sidebars.json b/docs/sidebars.json index 35dc65bf34ac54..99a1145693770e 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -1265,6 +1265,7 @@ "admin-manual/http-actions/fe/show-meta-info-action", "admin-manual/http-actions/fe/show-proc-action", "admin-manual/http-actions/fe/show-runtime-info-action", + "admin-manual/http-actions/fe/show-table-data-action", "admin-manual/http-actions/fe/statement-execution-action", "admin-manual/http-actions/fe/table-query-plan-action", "admin-manual/http-actions/fe/table-row-count-action", diff --git a/docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md b/docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md new file mode 100644 index 00000000000000..97bb3dcf3b8650 --- /dev/null +++ b/docs/zh-CN/docs/admin-manual/http-actions/fe/show-table-data-action.md @@ -0,0 +1,163 @@ +--- +{ + "title": "Show Table Data Action", + "language": "zh-CN" +} +--- + + + +# Show Table Data Action + +## Request + +`GET /api/show_table_data` + +## Description + +用于获取所有internal源下所有数据库所有表的数据量，或者指定数据库或指定表的数据量。单位字节。 + +## Path parameters + +无 + +## Query parameters + +* `db` + + 可选。如果指定，则获取指定数据库下表的数据量。 + +* `table` + + 可选。如果指定，则获取指定表的数据量。 + +* `single_replica` + + 可选。如果指定，则获取表单副本所占用的数据量。 + +## Request body + +无 + +## Response + +1. 指定数据库所有表的数据量。 + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244, + "revenue0":0, + "customer":1906421482 + } + }, + "count":0 + } + ``` + +2. 指定数据库指定表的数据量。 + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244 + } + }, + "count":0 + } + ``` + +3. 指定数据库指定表单副本的数据量。 + + ``` + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":3008182748 + } + }, + "count":0 + } + ``` + +## Examples + +1. 获取指定数据库的数据量 + + ``` + GET /api/show_table_data?db=tpch + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244, + "revenue0":0, + "customer":1906421482 + } + }, + "count":0 + } + ``` + +2. 指定数据库指定表的数据量。 + + ``` + GET /api/show_table_data?db=tpch&table=partsupp + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":9024548244 + } + }, + "count":0 + } + ``` +3. 指定数据库指定表单副本的数据量。 + + ``` + GET /api/show_table_data?db=tpch&table=partsupp&single_replica=true + + Response: + { + "msg":"success", + "code":0, + "data":{ + "tpch":{ + "partsupp":3008182748 + } + }, + "count":0 + } + ``` diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java index ac388349889ee8..cd1ce13451d0ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/RestBaseController.java @@ -55,6 +55,7 @@ public class RestBaseController extends BaseController { protected static final String LABEL_KEY = "label"; protected static final String TXN_ID_KEY = "txn_id"; protected static final String TXN_OPERATION_KEY = "txn_operation"; + protected static final String SINGLE_REPLICA_KEY = "single_replica"; private static final Logger LOG = LogManager.getLogger(RestBaseController.class); public ActionAuthorizationInfo executeCheckPassword(HttpServletRequest request, diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java index c1dc8f43802d7f..6db2a7e7b51d3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java @@ -23,6 +23,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf.TableType; +import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.proc.ProcNodeInterface; @@ -208,6 +209,39 @@ public Object show_data(HttpServletRequest request, HttpServletResponse response return ResponseEntityBuilder.ok(oneEntry); } + @RequestMapping(path = "/api/show_table_data", method = RequestMethod.GET) + public Object show_table_data(HttpServletRequest request, HttpServletResponse response) { + if (Config.enable_all_http_auth) { + executeCheckPassword(request, response); + checkGlobalAuth(ConnectContext.get().getCurrentUserIdentity(), PrivPredicate.ADMIN); + } + + String dbName = request.getParameter(DB_KEY); + String tableName = request.getParameter(TABLE_KEY); + String singleReplica = request.getParameter(SINGLE_REPLICA_KEY); + boolean singleReplicaBool = Boolean.parseBoolean(singleReplica); + Map> oneEntry = Maps.newHashMap(); + if (dbName != null) { + String fullDbName = getFullDbName(dbName); + DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(fullDbName); + if (db == null) { + return ResponseEntityBuilder.okWithCommonError("database " + fullDbName + " not found."); + } + Map tablesEntry = getDataSizeOfTables(db, tableName, singleReplicaBool); + oneEntry.put(ClusterNamespace.getNameFromFullName(fullDbName), tablesEntry); + } else { + for (long dbId : Env.getCurrentInternalCatalog().getDbIds()) { + DatabaseIf db = Env.getCurrentInternalCatalog().getDbNullable(dbId); + if (db == null || !(db instanceof Database) || ((Database) db).isMysqlCompatibleDatabase()) { + continue; + } + Map tablesEntry = getDataSizeOfTables(db, tableName, singleReplicaBool); + oneEntry.put(ClusterNamespace.getNameFromFullName(db.getFullName()), tablesEntry); + } + } + return ResponseEntityBuilder.ok(oneEntry); + } + private Map getHaInfo() throws IOException { HashMap feInfo = new HashMap(); feInfo.put("role", Env.getCurrentEnv().getFeType().toString()); @@ -289,6 +323,46 @@ public long getDataSizeOfDatabase(DatabaseIf db) { return totalSize; } + public Map getDataSizeOfTables(DatabaseIf db, String tableName, boolean singleReplica) { + Map oneEntry = Maps.newHashMap(); + db.readLock(); + try { + if (Strings.isNullOrEmpty(tableName)) { + List tables = db.getTables(); + for (Table table : tables) { + Map tableEntry = getDataSizeOfTable(table, singleReplica); + oneEntry.putAll(tableEntry); + } + } else { + Table table = ((Database) db).getTableNullable(tableName); + if (table == null) { + return oneEntry; + } + Map tableEntry = getDataSizeOfTable(table, singleReplica); + oneEntry.putAll(tableEntry); + } + } finally { + db.readUnlock(); + } + return oneEntry; + } + + public Map getDataSizeOfTable(Table table, boolean singleReplica) { + Map oneEntry = Maps.newHashMap(); + if (table.getType() == TableType.VIEW || table.getType() == TableType.ODBC) { + oneEntry.put(table.getName(), 0L); + } else if (table.getType() == TableType.OLAP) { + table.readLock(); + try { + long tableSize = ((OlapTable) table).getDataSize(singleReplica); + oneEntry.put(table.getName(), tableSize); + } finally { + table.readUnlock(); + } + } + return oneEntry; + } + private Map getDataSize() { Map result = new HashMap(); List dbNames = Env.getCurrentInternalCatalog().getDbNames();