From a833914bfcf0237349b8e6ede11a8fb2c3a7589c Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Mon, 7 Oct 2024 23:12:01 +0800 Subject: [PATCH] Handling Comments --- .../datasource/jdbc/JdbcExternalCatalog.java | 26 +++++++++-- .../jdbc/client/JdbcGbaseClient.java | 4 +- .../mapping/DefaultIdentifierMapping.java | 43 ++++++++--------- .../datasource/mapping/IdentifierMapping.java | 46 ++++++++++++++++++- 4 files changed, 85 insertions(+), 34 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 9d837fcac6328c..55eb0fa7698399 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -58,6 +58,7 @@ import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; +import java.util.stream.Collectors; @Getter public class JdbcExternalCatalog extends ExternalCatalog { @@ -245,12 +246,16 @@ protected void initLocalObjectsImpl() { @Override protected List listDatabaseNames() { - return identifierMapping.fromRemoteDatabaseName(jdbcClient.getDatabaseNameList()); + return getMappedDatabaseNames(); } @Override protected void buildDatabaseMapping() { - identifierMapping.fromRemoteDatabaseName(jdbcClient.getDatabaseNameList()); + getMappedDatabaseNames(); + } + + private List getMappedDatabaseNames() { + return identifierMapping.fromRemoteDatabaseName(jdbcClient.getDatabaseNameList()); } protected String getRemoteDatabaseName(String dbName) { @@ -287,9 +292,20 @@ public List listColumns(String dbName, String tblName) { makeSureInitialized(); String remoteDbName = getRemoteDatabaseName(dbName); String remoteTblName = getRemoteTableName(dbName, tblName); - return identifierMapping.fromRemoteColumnName(remoteDbName, remoteTblName, - jdbcClient.getColumnsFromJdbc(remoteDbName, - remoteTblName)); + + List remoteColumns = jdbcClient.getColumnsFromJdbc(remoteDbName, remoteTblName); + + List remoteColumnNames = remoteColumns.stream() + .map(Column::getName) + .collect(Collectors.toList()); + List localColumnNames = identifierMapping.fromRemoteColumnName(remoteDbName, remoteTblName, + remoteColumnNames); + + for (int i = 0; i < remoteColumns.size(); i++) { + remoteColumns.get(i).setName(localColumnNames.get(i)); + } + + return remoteColumns; } protected Map getRemoteColumnNames(String dbName, String tblName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java index da2e9a9a0bb938..6f82150466ad42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcGbaseClient.java @@ -86,12 +86,10 @@ protected ResultSet getRemoteColumns(DatabaseMetaData databaseMetaData, String c } @Override - public List getJdbcColumnsInfo(String localDbName, String localTableName) { + public List getJdbcColumnsInfo(String remoteDbName, String remoteTableName) { Connection conn = getConnection(); ResultSet rs = null; List tableSchema = Lists.newArrayList(); - String remoteDbName = getRemoteDatabaseName(localDbName); - String remoteTableName = getRemoteTableName(localDbName, localTableName); try { DatabaseMetaData databaseMetaData = conn.getMetaData(); String catalogName = getCatalogName(conn); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/DefaultIdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/DefaultIdentifierMapping.java index 4847cd86e6d79c..2ce8604084899f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/DefaultIdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/DefaultIdentifierMapping.java @@ -33,6 +33,7 @@ import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; public class DefaultIdentifierMapping implements IdentifierMapping { private static final Logger LOG = LogManager.getLogger(DefaultIdentifierMapping.class); @@ -125,12 +126,12 @@ public List fromRemoteTableName(String remoteDbName, List remote } @Override - public List fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, - List remoteColumns) { - // If mapping is not required, return the original input + public List fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, + List remoteColumnNames) { if (!isLowerCaseMetaNames && isMappingInvalid()) { - return remoteColumns; + return remoteColumnNames; } + JsonNode tablesNode = readAndParseJson(metaNamesMapping, "columns"); Map columnNameMapping = Maps.newTreeMap(); @@ -145,19 +146,17 @@ public List fromRemoteColumnName(String remoteDatabaseName, String remot } } } + localColumnToRemoteColumn.putIfAbsent(remoteDatabaseName, new ConcurrentHashMap<>()); localColumnToRemoteColumn.get(remoteDatabaseName).putIfAbsent(remoteTableName, new ConcurrentHashMap<>()); - List remoteColumnNames = Lists.newArrayList(); - for (Column remoteColumn : remoteColumns) { - remoteColumnNames.add(remoteColumn.getName()); - } - Map> result = nameListToMapping(remoteColumnNames, localColumnToRemoteColumn.get(remoteDatabaseName).get(remoteTableName), columnNameMapping, isLowerCaseMetaNames); + List localColumnNames = result.get("localNames"); List conflictNames = result.get("conflictNames"); + if (!conflictNames.isEmpty()) { throw new RuntimeException( "Conflict column names found in remote database/schema: " + remoteDatabaseName @@ -166,10 +165,7 @@ public List fromRemoteColumnName(String remoteDatabaseName, String remot + ". Please set lower_case_meta_names to false or" + " use meta_name_mapping to specify the column names."); } - for (int i = 0; i < remoteColumns.size(); i++) { - remoteColumns.get(i).setName(localColumnNames.get(i)); - } - return remoteColumns; + return localColumnNames; } @Override @@ -178,7 +174,7 @@ public String toRemoteDatabaseName(String localDatabaseName) { if (!isLowerCaseMetaNames && isMappingInvalid()) { return localDatabaseName; } - return getRequiredMapping(localDBToRemoteDB, localDatabaseName, "database", localDatabaseName); + return getRequiredMapping(localDBToRemoteDB, localDatabaseName, "database"); } @Override @@ -189,7 +185,7 @@ public String toRemoteTableName(String remoteDatabaseName, String localTableName } Map tableMap = localTableToRemoteTable.computeIfAbsent(remoteDatabaseName, k -> new ConcurrentHashMap<>()); - return getRequiredMapping(tableMap, localTableName, "table", localTableName); + return getRequiredMapping(tableMap, localTableName, "table"); } @Override @@ -209,12 +205,11 @@ public Map toRemoteColumnNames(String remoteDatabaseName, String return columnMap; } - private V getRequiredMapping(Map map, K key, String typeName, String entityName) { + private V getRequiredMapping(Map map, K key, String typeName) { V value = map.get(key); if (value == null) { - LOG.warn("No remote {} found for {}: {}. Please refresh this catalog.", typeName, typeName, entityName); - throw new RuntimeException("No remote " + typeName + " found for " + typeName + ": " + entityName - + ". Please refresh this catalog."); + LOG.warn("No remote {} found for: {}. Please refresh this catalog.", typeName, key); + throw new RuntimeException("No remote" + typeName + "found for: " + key + ". Please refresh this catalog."); } return value; } @@ -232,7 +227,7 @@ private JsonNode readAndParseJson(String jsonPath, String nodeName) { private Map> nameListToMapping(List remoteNames, ConcurrentHashMap localNameToRemoteName, Map nameMapping, boolean isLowerCaseMetaNames) { - List filteredDatabaseNames = Lists.newArrayList(); + List filteredNames = Lists.newArrayList(); Set lowerCaseNames = Sets.newHashSet(); Map> nameMap = Maps.newHashMap(); List conflictNames = Lists.newArrayList(); @@ -244,14 +239,12 @@ private Map> nameListToMapping(List remoteNames, localNameToRemoteName.computeIfAbsent(localName, k -> name); if (isLowerCaseMetaNames && !lowerCaseNames.add(localName)) { - if (nameMap.containsKey(localName)) { - nameMap.get(localName).add(mappedName); - } + nameMap.get(localName).add(mappedName); } else { nameMap.putIfAbsent(localName, Lists.newArrayList(Collections.singletonList(mappedName))); } - filteredDatabaseNames.add(localName); + filteredNames.add(localName); } for (List conflictNameList : nameMap.values()) { @@ -261,7 +254,7 @@ private Map> nameListToMapping(List remoteNames, } Map> result = Maps.newConcurrentMap(); - result.put("localNames", filteredDatabaseNames); + result.put("localNames", filteredNames); result.put("conflictNames", conflictNames); return result; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java index 7745a25d27da47..6090e5d9ec814e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/mapping/IdentifierMapping.java @@ -23,15 +23,59 @@ import java.util.Map; public interface IdentifierMapping { + + /** + * Maps a list of remote database names to their corresponding local database names. + * + * @param remoteDatabaseNames the list of remote database names to be mapped + * @return a list of corresponding local database names + */ List fromRemoteDatabaseName(List remoteDatabaseNames); + /** + * Maps a list of remote table names in a specified remote database to their corresponding local table names. + * + * @param remoteDatabaseName the name of the remote database where the tables reside + * @param remoteTableNames the list of remote table names to be mapped + * @return a list of corresponding local table names + */ List fromRemoteTableName(String remoteDatabaseName, List remoteTableNames); - List fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, List remoteColumns); + /** + * Maps a list of remote columns in a specified remote database and table to their corresponding local columns. + * + * @param remoteDatabaseName the name of the remote database + * @param remoteTableName the name of the remote table where the columns reside + * @param remoteColumnNames the list of remote column names to be mapped + * @return a list of corresponding local columns + */ + List fromRemoteColumnName(String remoteDatabaseName, String remoteTableName, + List remoteColumnNames); + /** + * Maps a local database name to its corresponding remote database name. + * + * @param localDatabaseName the name of the local database to be mapped + * @return the corresponding remote database name + */ String toRemoteDatabaseName(String localDatabaseName); + /** + * Maps a local table name in a specified remote database to its corresponding remote table name. + * + * @param remoteDatabaseName the name of the remote database where the table resides + * @param localTableName the name of the local table to be mapped + * @return the corresponding remote table name + */ String toRemoteTableName(String remoteDatabaseName, String localTableName); + /** + * Maps local column names in a specified remote database and table to their corresponding remote column names. + * + * @param remoteDatabaseName the name of the remote database + * @param remoteTableName the name of the remote table + * @return a map of local column names to corresponding remote column names + */ Map toRemoteColumnNames(String remoteDatabaseName, String remoteTableName); } +