diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index de3c8c2c662135..3588a3a0bffddb 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1979,16 +1979,20 @@ public class Config extends ConfigBase { * Max cache num of hive partition. * Decrease this value if FE's memory is small */ - @ConfField(mutable = false, masterOnly = false) - public static long max_hive_partition_cache_num = 100000; + @ConfField(description = {"Hive Metastore 表级别分区缓存的最大数量。", + "Max cache number of partition at table level in Hive Metastore."}) + public static long max_hive_partition_cache_num = 10000; - @ConfField(mutable = false, masterOnly = false, description = {"Hive表名缓存的最大数量。", - "Max cache number of hive table name list."}) - public static long max_hive_table_cache_num = 1000; + @ConfField(description = {"Hudi/Iceberg 表级别缓存的最大数量。", + "Max cache number of hudi/iceberg table."}) + public static long max_external_table_cache_num = 1000; - @ConfField(mutable = false, masterOnly = false, description = { - "Hive分区表缓存的最大数量", "Max cache number of hive partition table" - }) + @ConfField(description = {"External Catalog 中,Database 和 Table 的实例缓存的最大数量。", + "Max cache number of database and table instance in external catalog."}) + public static long max_meta_object_cache_num = 1000; + + @ConfField(description = {"Hive分区表缓存的最大数量", + "Max cache number of hive partition table"}) public static long max_hive_partition_table_cache_num = 1000; @ConfField(mutable = false, masterOnly = false, description = {"获取Hive分区值时候的最大返回数量,-1代表没有限制。", diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 4f587ab7465958..2dfcec1d8aef4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -238,7 +238,7 @@ public final synchronized void makeSureInitialized() { name, OptionalLong.of(86400L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), - Config.max_hive_table_cache_num, + Config.max_meta_object_cache_num, ignored -> getFilteredDatabaseNames(), dbName -> Optional.ofNullable( buildDbForInit(dbName, Util.genIdByName(name, dbName), logType)), @@ -660,8 +660,6 @@ protected ExternalDatabase buildDbForInit(String dbName return new IcebergExternalDatabase(this, dbId, dbName); case MAX_COMPUTE: return new MaxComputeExternalDatabase(this, dbId, dbName); - //case HUDI: - //return new HudiExternalDatabase(this, dbId, dbName); case LAKESOUL: return new LakeSoulExternalDatabase(this, dbId, dbName); case TEST: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index b564a17ce8687b..dc6f9aaea73d8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -145,7 +145,7 @@ public final synchronized void makeSureInitialized() { name, OptionalLong.of(86400L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), - Config.max_hive_table_cache_num, + Config.max_meta_object_cache_num, ignored -> listTableNames(), tableName -> Optional.ofNullable( buildTableForInit(tableName, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index da88a03f2ebabe..631362a5b41701 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -177,7 +177,7 @@ public void setNewFileCache() { CacheFactory fileCacheFactory = new CacheFactory( OptionalLong.of(fileMetaCacheTtlSecond >= HMSExternalCatalog.FILE_META_CACHE_TTL_DISABLE_CACHE - ? fileMetaCacheTtlSecond : 86400L), + ? fileMetaCacheTtlSecond : 28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L), Config.max_external_file_cache_num, false, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java index d9c1c208271767..2372b88e0db32e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiCachedPartitionProcessor.java @@ -52,9 +52,9 @@ public HudiCachedPartitionProcessor(long catalogId, ExecutorService executor) { this.catalogId = catalogId; this.executor = executor; CacheFactory partitionCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), - Config.max_hive_table_cache_num, + Config.max_external_table_cache_num, false, null); this.partitionCache = partitionCacheFactory.buildCache(key -> new TablePartitionValues(), null, executor); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java index dc11a6cacc24cf..13bd9650978f81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataCache.java @@ -55,17 +55,17 @@ public class IcebergMetadataCache { public IcebergMetadataCache(ExecutorService executor) { CacheFactory snapshotListCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), - Config.max_hive_table_cache_num, + Config.max_external_table_cache_num, false, null); this.snapshotListCache = snapshotListCacheFactory.buildCache(key -> loadSnapshots(key), null, executor); CacheFactory tableCacheFactory = new CacheFactory( - OptionalLong.of(86400L), + OptionalLong.of(28800L), OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60), - Config.max_hive_table_cache_num, + Config.max_external_table_cache_num, false, null); this.tableCache = tableCacheFactory.buildCache(key -> loadTable(key), null, executor); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index c251db3a5c19f7..e3ad8668fb5525 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -57,7 +57,7 @@ public MetaCache(String name, CacheFactory namesCacheFactory = new CacheFactory( expireAfterWriteSec, refreshAfterWriteSec, - maxSize, + 1, // names cache has one and only one entry true, null); CacheFactory objCacheFactory = new CacheFactory(