From 76b0393cc9f9add9fff6b178ac2b7da6baf53f09 Mon Sep 17 00:00:00 2001 From: daidai <2017501503@qq.com> Date: Wed, 9 Oct 2024 19:06:35 +0800 Subject: [PATCH] [Enhancement](tvf)catalog tvf implements user permission checks and hides sensitive information (#41497) (#41599) bp #41497 --- be/src/vec/exec/scan/vmeta_scanner.cpp | 1 + .../apache/doris/datasource/CatalogMgr.java | 85 +++++++++++-------- .../tablefunction/MetadataGenerator.java | 9 +- .../tvf/test_catalogs_tvf.out | 40 +++++++++ .../tvf/test_catalogs_tvf.groovy | 68 ++++++++++++++- 5 files changed, 162 insertions(+), 41 deletions(-) diff --git a/be/src/vec/exec/scan/vmeta_scanner.cpp b/be/src/vec/exec/scan/vmeta_scanner.cpp index 74fed8c80c73cd..180d18b0c2c746 100644 --- a/be/src/vec/exec/scan/vmeta_scanner.cpp +++ b/be/src/vec/exec/scan/vmeta_scanner.cpp @@ -400,6 +400,7 @@ Status VMetaScanner::_build_catalogs_metadata_request(const TMetaScanRange& meta // create TMetadataTableRequestParams TMetadataTableRequestParams metadata_table_params; metadata_table_params.__set_metadata_type(TMetadataType::CATALOGS); + metadata_table_params.__set_current_user_ident(_user_identity); request->__set_metada_table_params(metadata_table_params); return Status::OK(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index d9b118b03c5d59..7f63ea92fd3cdc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -24,6 +24,7 @@ import org.apache.doris.analysis.DropCatalogStmt; import org.apache.doris.analysis.ShowCatalogStmt; import org.apache.doris.analysis.ShowCreateCatalogStmt; +import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.EnvFactory; @@ -371,30 +372,27 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) matcher = PatternMatcherWrapper.createMysqlPattern(showStmt.getPattern(), CaseSensibility.CATALOG.getCaseSensibility()); } - for (CatalogIf catalog : nameToCatalog.values()) { - if (Env.getCurrentEnv().getAccessManager() - .checkCtlPriv(ConnectContext.get(), catalog.getName(), PrivPredicate.SHOW)) { - String name = catalog.getName(); - // Filter catalog name - if (matcher != null && !matcher.match(name)) { - continue; - } - List row = Lists.newArrayList(); - row.add(String.valueOf(catalog.getId())); - row.add(name); - row.add(catalog.getType()); - if (name.equals(currentCtlg)) { - row.add("Yes"); - } else { - row.add("No"); - } - Map props = catalog.getProperties(); - String createTime = props.getOrDefault(ExternalCatalog.CREATE_TIME, FeConstants.null_string); - row.add(createTime); - row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime())); - row.add(catalog.getComment()); - rows.add(row); + for (CatalogIf catalog : listCatalogsWithCheckPriv(ConnectContext.get().getCurrentUserIdentity())) { + String name = catalog.getName(); + // Filter catalog name + if (matcher != null && !matcher.match(name)) { + continue; + } + List row = Lists.newArrayList(); + row.add(String.valueOf(catalog.getId())); + row.add(name); + row.add(catalog.getType()); + if (name.equals(currentCtlg)) { + row.add("Yes"); + } else { + row.add("No"); } + Map props = catalog.getProperties(); + String createTime = props.getOrDefault(ExternalCatalog.CREATE_TIME, FeConstants.null_string); + row.add(createTime); + row.add(TimeUtils.longToTimeString(catalog.getLastUpdateTime())); + row.add(catalog.getComment()); + rows.add(row); // sort by catalog name rows.sort((x, y) -> { @@ -414,18 +412,8 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) if (!Strings.isNullOrEmpty(catalog.getResource())) { rows.add(Arrays.asList("resource", catalog.getResource())); } - // use tree map to maintain display order, making it easier to view properties - Map sortedMap = new TreeMap<>(catalog.getProperties()).descendingMap(); - for (Map.Entry elem : sortedMap.entrySet()) { - if (PrintableMap.HIDDEN_KEY.contains(elem.getKey())) { - continue; - } - if (PrintableMap.SENSITIVE_KEY.contains(elem.getKey())) { - rows.add(Arrays.asList(elem.getKey(), PrintableMap.PASSWORD_MASK)); - } else { - rows.add(Arrays.asList(elem.getKey(), elem.getValue())); - } - } + Map sortedMap = getCatalogPropertiesWithPrintable(catalog); + sortedMap.forEach((k, v) -> rows.add(Arrays.asList(k, v))); } } finally { readUnlock(); @@ -434,6 +422,25 @@ public ShowResultSet showCatalogs(ShowCatalogStmt showStmt, String currentCtlg) return new ShowResultSet(showStmt.getMetaData(), rows); } + public static Map getCatalogPropertiesWithPrintable(CatalogIf catalog) { + // use tree map to maintain display order, making it easier to view properties + Map sortedMap = new TreeMap<>(); + catalog.getProperties().forEach( + (key, value) -> { + if (PrintableMap.HIDDEN_KEY.contains(key)) { + return; + } + if (PrintableMap.SENSITIVE_KEY.contains(key)) { + sortedMap.put(key, PrintableMap.PASSWORD_MASK); + } else { + sortedMap.put(key, value); + } + } + ); + return sortedMap; + } + + public ShowResultSet showCreateCatalog(ShowCreateCatalogStmt showStmt) throws AnalysisException { List> rows = Lists.newArrayList(); readLock(); @@ -538,6 +545,14 @@ public List listCatalogs() { return nameToCatalog.values().stream().collect(Collectors.toList()); } + public List listCatalogsWithCheckPriv(UserIdentity userIdentity) { + return nameToCatalog.values().stream().filter( + catalog -> Env.getCurrentEnv().getAccessManager() + .checkCtlPriv(userIdentity, catalog.getName(), PrivPredicate.SHOW) + ).collect(Collectors.toList()); + } + + /** * Reply for alter catalog props event. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 466d324c36951d..a1cac53599556d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -47,6 +47,7 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.InternalCatalog; @@ -513,17 +514,17 @@ private static TFetchSchemaTableDataResult frontendsDisksMetadataResult(TMetadat private static TFetchSchemaTableDataResult catalogsMetadataResult(TMetadataTableRequestParams params) { TFetchSchemaTableDataResult result = new TFetchSchemaTableDataResult(); - List info = Env.getCurrentEnv().getCatalogMgr().listCatalogs(); - List dataBatch = Lists.newArrayList(); + UserIdentity currentUserIdentity = UserIdentity.fromThrift(params.getCurrentUserIdent()); + List info = Env.getCurrentEnv().getCatalogMgr().listCatalogsWithCheckPriv(currentUserIdentity); + List dataBatch = Lists.newArrayList(); for (CatalogIf catalog : info) { TRow trow = new TRow(); trow.addToColumnValue(new TCell().setLongVal(catalog.getId())); trow.addToColumnValue(new TCell().setStringVal(catalog.getName())); trow.addToColumnValue(new TCell().setStringVal(catalog.getType())); - Map properties = catalog.getProperties(); - + Map properties = CatalogMgr.getCatalogPropertiesWithPrintable(catalog); for (Map.Entry entry : properties.entrySet()) { TRow subTrow = new TRow(trow); subTrow.addToColumnValue(new TCell().setStringVal(entry.getKey())); diff --git a/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out b/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out index 310da69d76670f..7f97c337999b02 100644 --- a/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out +++ b/regression-test/data/external_table_p0/tvf/test_catalogs_tvf.out @@ -8,3 +8,43 @@ catalog_test_hive00 hms type hms -- !create -- catalog_test_es00 es type es +-- !test_10 -- +catalog_tvf_test_dlf hms dlf.catalog.id 987654321 + +-- !test_11 -- +catalog_tvf_test_dlf hms dlf.secret_key *XXX + +-- !test_12 -- +catalog_tvf_test_dlf hms dlf.access_key AAAAAAAAAAAAAAAAAAAAAA + +-- !test_13 -- +catalog_tvf_test_dlf hms dlf.uid 123456789 + +-- !test_14 -- +catalog_tvf_test_dlf hms type hms + +-- !test_15 -- + +-- !test_16 -- +internal internal NULL NULL + +-- !test_17 -- +catalog_tvf_test_dlf hms dlf.secret_key *XXX + +-- !test_18 -- +catalog_tvf_test_dlf hms dlf.access_key AAAAAAAAAAAAAAAAAAAAAA + +-- !test_19 -- +catalog_tvf_test_dlf hms dlf.uid 123456789 + +-- !test_20 -- +catalog_tvf_test_dlf hms type hms + +-- !test_21 -- + +-- !test_22 -- + +-- !test_23 -- + +-- !test_24 -- + diff --git a/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy b/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy index a59953cf567300..748a7e49d14c77 100644 --- a/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_catalogs_tvf.groovy @@ -18,12 +18,12 @@ suite("test_catalogs_tvf","p0,external,tvf,external_docker") { List> table = sql """ select * from catalogs(); """ assertTrue(table.size() > 0) - assertEquals(5, table[0].size) + assertEquals(5, table[0].size()) table = sql """ select CatalogId,CatalogName from catalogs();""" assertTrue(table.size() > 0) - assertTrue(table[0].size == 2) + assertTrue(table[0].size() == 2) table = sql """ select * from catalogs() where CatalogId=0;""" @@ -76,4 +76,68 @@ suite("test_catalogs_tvf","p0,external,tvf,external_docker") { // check exception exception "catalogs table-valued-function does not support any params" } + + sql """ drop catalog if exists catalog_tvf_test_dlf """ + + sql """ + CREATE CATALOG catalog_tvf_test_dlf PROPERTIES ( + "type"="hms", + "hive.metastore.type" = "dlf", + "dlf.proxy.mode" = "DLF_ONLY", + "dlf.endpoint" = "dlf-vpc.cn-beijing.aliyuncs.com", + "dlf.region" = "cn-beijing", + "dlf.uid" = "123456789", + "dlf.catalog.id" = "987654321", + "dlf.access_key" = "AAAAAAAAAAAAAAAAAAAAAA", + "dlf.secret_key" = "BBBBBBBBBBBBBBBBBBBBBB" + );""" + + order_qt_test_10 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.catalog.id" """ + order_qt_test_11 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.secret_key" """ + order_qt_test_12 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.access_key" """ + order_qt_test_13 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.uid" """ + order_qt_test_14 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "type" """ + + + def user = 'catalog_user_test' + def pwd = 'C123_567p' + try_sql("DROP USER ${user}") + + sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'""" + sql """GRANT SELECT_PRIV on `internal`.``.`` to '${user}'""" + + + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql """ switch internal """ + order_qt_test_15 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "type" """ + order_qt_test_16 """ select CatalogName,CatalogType,Property,Value from catalogs() """ + } + + sql """GRANT SELECT_PRIV on `catalog_tvf_test_dlf`.``.`` to '${user}'""" + + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql """ switch internal """ + + order_qt_test_17 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.secret_key" """ + order_qt_test_18 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.access_key" """ + order_qt_test_19 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.uid" """ + order_qt_test_20 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "type" """ + } + + sql """REVOKE SELECT_PRIV on `catalog_tvf_test_dlf`.``.`` FROM '${user}'""" + + + connect(user=user, password="${pwd}", url=context.config.jdbcUrl) { + sql """ switch internal """ + + order_qt_test_21 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.secret_key" """ + order_qt_test_22 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.access_key" """ + order_qt_test_23 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "dlf.uid" """ + order_qt_test_24 """ select CatalogName,CatalogType,Property,Value from catalogs() where CatalogName = "catalog_tvf_test_dlf" and Property= "type" """ + } + + + }