Skip to content

Commit

Permalink
Revert "[branch-2.1][improvement](jdbc catalog) Optimize JdbcCatalog …
Browse files Browse the repository at this point in the history 
…case mapping stability" (#41588)

Reverts #41330
  • Loading branch information
@zy-kkk
zy-kkk authored Oct 9, 2024
1 parent e218fd2 commit ade86c0
Show file tree
Hide file tree
Showing 10 changed files with 409 additions and 371 deletions.
18 changes: 0 additions & 18 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,6 @@
import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase;
import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase;
import org.apache.doris.datasource.jdbc.JdbcExternalDatabase;
import org.apache.doris.datasource.mapping.IdentifierMapping;
import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase;
import org.apache.doris.datasource.metacache.MetaCache;
import org.apache.doris.datasource.operations.ExternalMetadataOps;
Expand Down Expand Up @@ -143,9 +142,6 @@ public abstract class ExternalCatalog
protected Optional<Boolean> useMetaCache = Optional.empty();
protected MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache;

protected IdentifierMapping identifierMapping;
private boolean mappingsInitialized = false;

public ExternalCatalog() {
}

Expand Down Expand Up @@ -178,10 +174,6 @@ protected List<String> listDatabaseNames() {
}
}

// only for forward to master
protected void buildDatabaseMapping() {
}

// Will be called when creating catalog(so when as replaying)
// to add some default properties if missing.
public void setDefaultPropsIfMissing(boolean isReplay) {
Expand Down Expand Up @@ -210,10 +202,6 @@ public void checkWhenCreating() throws DdlException {
*/
public abstract List<String> listTableNames(SessionContext ctx, String dbName);

// only for forward to master
protected void buildTableMapping(SessionContext ctx, String dbName) {
}

/**
* check if the specified table exist.
*
Expand Down Expand Up @@ -278,10 +266,6 @@ public final synchronized void makeSureInitialized() {
}
initialized = true;
}
if (!mappingsInitialized) {
buildDatabaseMapping();
mappingsInitialized = true;
}
}

protected final void initLocalObjects() {
Expand Down Expand Up @@ -407,7 +391,6 @@ private List<String> getFilteredDatabaseNames() {
public void onRefresh(boolean invalidCache) {
this.objectCreated = false;
this.initialized = false;
this.mappingsInitialized = false;
synchronized (this.propLock) {
this.convertedProperties = null;
}
Expand Down Expand Up @@ -733,7 +716,6 @@ public void gsonPostProcess() throws IOException {
}
this.propLock = new byte[0];
this.initialized = false;
this.mappingsInitialized = false;
setDefaultPropsIfMissing(true);
}

Expand Down
7 changes: 0 additions & 7 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,8 +91,6 @@ public abstract class ExternalDatabase<T extends ExternalTable>

private MetaCache<T> metaCache;

private boolean mappingsInitialized = false;

/**
* Create external database.
*
Expand All @@ -119,7 +117,6 @@ public void setTableExtCatalog(ExternalCatalog extCatalog) {

public void setUnInitialized(boolean invalidCache) {
this.initialized = false;
this.mappingsInitialized = false;
this.invalidCacheInInit = invalidCache;
if (extCatalog.getUseMetaCache().isPresent()) {
if (extCatalog.getUseMetaCache().get() && metaCache != null) {
Expand Down Expand Up @@ -173,10 +170,6 @@ public final synchronized void makeSureInitialized() {
}
initialized = true;
}
if (!mappingsInitialized) {
extCatalog.buildTableMapping(null, name);
mappingsInitialized = true;
}
}

public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {
Expand Down
56 changes: 10 additions & 46 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.apache.doris.datasource.jdbc.client.JdbcClient;
import org.apache.doris.datasource.jdbc.client.JdbcClientConfig;
import org.apache.doris.datasource.jdbc.client.JdbcClientException;
import org.apache.doris.datasource.mapping.DefaultIdentifierMapping;
import org.apache.doris.proto.InternalService;
import org.apache.doris.proto.InternalService.PJdbcTestConnectionRequest;
import org.apache.doris.proto.InternalService.PJdbcTestConnectionResult;
Expand Down Expand Up @@ -119,16 +118,19 @@ public void onRefresh(boolean invalidCache) {
super.onRefresh(invalidCache);
if (jdbcClient != null) {
jdbcClient.closeClient();
jdbcClient = null;
}
}

@Override
public void onRefreshCache(boolean invalidCache) {
onRefresh(invalidCache);
}

@Override
public void onClose() {
super.onClose();
if (jdbcClient != null) {
jdbcClient.closeClient();
jdbcClient = null;
}
}

Expand Down Expand Up @@ -229,6 +231,8 @@ protected void initLocalObjectsImpl() {
.setDriverUrl(getDriverUrl())
.setDriverClass(getDriverClass())
.setOnlySpecifiedDatabase(getOnlySpecifiedDatabase())
.setIsLowerCaseMetaNames(getLowerCaseMetaNames())
.setMetaNamesMapping(getMetaNamesMapping())
.setIncludeDatabaseMap(getIncludeDatabaseMap())
.setExcludeDatabaseMap(getExcludeDatabaseMap())
.setConnectionPoolMinSize(getConnectionPoolMinSize())
Expand All @@ -238,62 +242,22 @@ protected void initLocalObjectsImpl() {
.setConnectionPoolKeepAlive(isConnectionPoolKeepAlive());

jdbcClient = JdbcClient.createJdbcClient(jdbcClientConfig);
identifierMapping = new DefaultIdentifierMapping(Boolean.parseBoolean(getLowerCaseMetaNames()),
getMetaNamesMapping());
}

@Override
protected List<String> listDatabaseNames() {
return identifierMapping.fromRemoteDatabaseName(jdbcClient.getDatabaseNameList());
}

@Override
protected void buildDatabaseMapping() {
identifierMapping.fromRemoteDatabaseName(jdbcClient.getDatabaseNameList());
}

protected String getRemoteDatabaseName(String dbName) {
return identifierMapping.toRemoteDatabaseName(dbName);
return jdbcClient.getDatabaseNameList();
}

@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
String remoteDbName = getRemoteDatabaseName(dbName);
return identifierMapping.fromRemoteTableName(remoteDbName, jdbcClient.getTablesNameList(remoteDbName));
}

@Override
protected void buildTableMapping(SessionContext ctx, String dbName) {
String remoteDbName = getRemoteDatabaseName(dbName);
identifierMapping.fromRemoteTableName(getRemoteDatabaseName(dbName),
jdbcClient.getTablesNameList(remoteDbName));
}

protected String getRemoteTableName(String dbName, String tblName) {
return identifierMapping.toRemoteTableName(getRemoteDatabaseName(dbName), tblName);
return jdbcClient.getTablesNameList(dbName);
}

@Override
public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
makeSureInitialized();
String remoteDbName = getRemoteDatabaseName(dbName);
String remoteTblName = getRemoteTableName(dbName, tblName);
return jdbcClient.isTableExist(remoteDbName, remoteTblName);
}

public List<Column> listColumns(String dbName, String tblName) {
makeSureInitialized();
String remoteDbName = getRemoteDatabaseName(dbName);
String remoteTblName = getRemoteTableName(dbName, tblName);
return identifierMapping.fromRemoteColumnName(remoteDbName, remoteTblName,
jdbcClient.getColumnsFromJdbc(remoteDbName,
remoteTblName));
}

protected Map<String, String> getRemoteColumnNames(String dbName, String tblName) {
return identifierMapping.toRemoteColumnNames(getRemoteDatabaseName(dbName),
getRemoteTableName(dbName, tblName));
return jdbcClient.isTableExist(dbName, tblName);
}

@Override
Expand Down
25 changes: 8 additions & 17 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.thrift.TTableDescriptor;

import com.google.common.collect.Maps;
import org.apache.commons.text.StringSubstitutor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
Expand Down Expand Up @@ -87,29 +86,21 @@ public TTableDescriptor toThrift() {

@Override
public Optional<SchemaCacheValue> initSchema() {
return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).listColumns(dbName, name)));
return Optional.of(new SchemaCacheValue(((JdbcExternalCatalog) catalog).getJdbcClient()
.getColumnsFromJdbc(dbName, name)));
}

private JdbcTable toJdbcTable() {
List<Column> schema = getFullSchema();
JdbcExternalCatalog jdbcCatalog = (JdbcExternalCatalog) catalog;
String fullTableName = this.dbName + "." + this.name;
JdbcTable jdbcTable = new JdbcTable(this.id, fullTableName, schema, TableType.JDBC_EXTERNAL_TABLE);
jdbcCatalog.configureJdbcTable(jdbcTable, fullTableName);
String fullDbName = this.dbName + "." + this.name;
JdbcTable jdbcTable = new JdbcTable(this.id, fullDbName, schema, TableType.JDBC_EXTERNAL_TABLE);
jdbcCatalog.configureJdbcTable(jdbcTable, fullDbName);

// Set remote properties
jdbcTable.setRemoteDatabaseName(jdbcCatalog.getRemoteDatabaseName(this.dbName));
jdbcTable.setRemoteTableName(jdbcCatalog.getRemoteTableName(this.dbName, this.name));
Map<String, String> remoteColumnNames = jdbcCatalog.getRemoteColumnNames(this.dbName, this.name);
if (!remoteColumnNames.isEmpty()) {
jdbcTable.setRemoteColumnNames(remoteColumnNames);
} else {
remoteColumnNames = Maps.newHashMap();
for (Column column : schema) {
remoteColumnNames.put(column.getName(), column.getName());
}
jdbcTable.setRemoteColumnNames(remoteColumnNames);
}
jdbcTable.setRemoteDatabaseName(jdbcCatalog.getJdbcClient().getRemoteDatabaseName(this.dbName));
jdbcTable.setRemoteTableName(jdbcCatalog.getJdbcClient().getRemoteTableName(this.dbName, this.name));
jdbcTable.setRemoteColumnNames(jdbcCatalog.getJdbcClient().getRemoteColumnNames(this.dbName, this.name));

return jdbcTable;
}
Expand Down
45 changes: 45 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcIdentifierMapping.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.datasource.jdbc;

import org.apache.doris.datasource.jdbc.client.JdbcClient;
import org.apache.doris.datasource.mapping.IdentifierMapping;

public class JdbcIdentifierMapping extends IdentifierMapping {
private final JdbcClient jdbcClient;

public JdbcIdentifierMapping(boolean isLowerCaseMetaNames, String metaNamesMapping, JdbcClient jdbcClient) {
super(isLowerCaseMetaNames, metaNamesMapping);
this.jdbcClient = jdbcClient;
}

@Override
protected void loadDatabaseNames() {
jdbcClient.getDatabaseNameList();
}

@Override
protected void loadTableNames(String localDbName) {
jdbcClient.getTablesNameList(localDbName);
}

@Override
protected void loadColumnNames(String localDbName, String localTableName) {
jdbcClient.getColumnsFromJdbc(localDbName, localTableName);
}
}
Loading

0 comments on commit ade86c0

Please sign in to comment.