diff --git a/docs/en/docs/faq/install-faq.md b/docs/en/docs/faq/install-faq.md index af78496c782479..81a816c1354d3c 100644 --- a/docs/en/docs/faq/install-faq.md +++ b/docs/en/docs/faq/install-faq.md @@ -163,7 +163,7 @@ In many cases, we need to troubleshoot problems through logs. The format and vie Doris supports one BE node to configure multiple storage paths. Usually, one storage path can be configured for each disk. At the same time, Doris supports storage media properties that specify paths, such as SSD or HDD. SSD stands for high-speed storage device and HDD stands for low-speed storage device. -If doris cluster has only one storage medium type, the practice is not specify storage medium in be.conf configuration file. ```Failed to find enough host with storage medium and tag```, generally we got this error for only config SSD medium in be.conf, but default parameter ```default_storage_medium``` in fe is HDD, so there is no HDD storage medium in cluster. There are several ways to fix this, one is modify the parameter in fe.conf and restart fe; the other way is take the SSD config in be.conf away,and the third way is add properties when create table ```{"storage_medium" = "ssd"}``` +If the cluster only has one type of medium, such as all HDD or all SSD, the best practice is not to explicitly specify the medium property in be.conf. If encountering the error ```Failed to find enough host with storage medium and tag``` mentioned above, it is generally because be.conf only configures the SSD medium, while the table creation stage explicitly specifies ```properties {"storage_medium" = "hdd"}```; similarly, if be.conf only configures the HDD medium, and the table creation stage explicitly specifies ```properties {"storage_medium" = "ssd"}```, the same error will occur. The solution is to modify the properties parameter in the table creation to match the configuration; or remove the explicit configuration of SSD/HDD in be.conf. By specifying the storage medium properties of the path, we can take advantage of Doris's hot and cold data partition storage function to store hot data in SSD at the partition level, while cold data is automatically transferred to HDD. diff --git a/docs/zh-CN/docs/faq/install-faq.md b/docs/zh-CN/docs/faq/install-faq.md index 4dca37ae39eef1..d9d2c78e4242b0 100644 --- a/docs/zh-CN/docs/faq/install-faq.md +++ b/docs/zh-CN/docs/faq/install-faq.md @@ -163,7 +163,7 @@ Observer 角色和这个单词的含义一样，仅仅作为观察者来同步 Doris支持一个BE节点配置多个存储路径。通常情况下，每块盘配置一个存储路径即可。同时，Doris支持指定路径的存储介质属性，如SSD或HDD。SSD代表高速存储设备，HDD代表低速存储设备。 -如果集群只有一种介质比如都是HDD或者都是SSD，最佳实践是不用在be.conf中显式指定介质属性。如果遇到上述报错```Failed to find enough host with storage medium and tag```，一般是因为be.conf中只配置了SSD的介质，而fe中参数default_storage_medium默认为HDD，因此建表时会发现没有HDD介质的存储而报错。解决方案可以修改此FE配置并重启FE生效；或者将be.conf中SSD的显式配置去掉；或者建表时增加properties参数 ```properties {"storage_medium" = "ssd"}```均可 +如果集群只有一种介质比如都是HDD或者都是SSD，最佳实践是不用在be.conf中显式指定介质属性。如果遇到上述报错```Failed to find enough host with storage medium and tag```，一般是因为be.conf中只配置了SSD的介质，而建表阶段中显式指定了```properties {"storage_medium" = "hdd"}```；同理如果be.conf只配置了HDD的介质，而而建表阶段中显式指定了```properties {"storage_medium" = "ssd"}```也会出现上述错误。解决方案可以修改建表的properties参数与配置匹配；或者将be.conf中SSD/HDD的显式配置去掉即可。 通过指定路径的存储介质属性，我们可以利用Doris的冷热数据分区存储功能，在分区级别将热数据存储在SSD中，而冷数据会自动转移到HDD中。 diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index a0237f03d827de..6bd606a378304f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -1091,7 +1091,7 @@ private Partition resetPartitionForRestore(OlapTable localTbl, OlapTable remoteT // replicas try { Map> beIds = Env.getCurrentSystemInfo() - .selectBackendIdsForReplicaCreation(replicaAlloc, null); + .selectBackendIdsForReplicaCreation(replicaAlloc, null, false, false); for (Map.Entry> entry : beIds.entrySet()) { for (Long beId : entry.getValue()) { long newReplicaId = env.getNextId(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java index 6137af9424d181..7028e5e449ad9c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java @@ -49,6 +49,7 @@ public class DataProperty implements Writable, GsonPostProcessable { private String storagePolicy; @SerializedName(value = "isMutable") private boolean isMutable = true; + private boolean storageMediumSpecified; private DataProperty() { // for persist @@ -97,6 +98,10 @@ public String getStoragePolicy() { return storagePolicy; } + public boolean isStorageMediumSpecified() { + return storageMediumSpecified; + } + public boolean isMutable() { return isMutable; } @@ -105,6 +110,10 @@ public void setMutable(boolean mutable) { isMutable = mutable; } + public void setStorageMediumSpecified(boolean isSpecified) { + storageMediumSpecified = isSpecified; + } + public static DataProperty read(DataInput in) throws IOException { if (Env.getCurrentEnvJournalVersion() >= FeMetaVersion.VERSION_108) { String json = Text.readString(in); @@ -164,4 +173,5 @@ public void gsonPostProcess() throws IOException { // storagePolicy is a newly added field, it may be null when replaying from old version. this.storagePolicy = Strings.nullToEmpty(this.storagePolicy); } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 086c4bb7ec0630..27c1a99c6861ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -578,7 +578,7 @@ public Status resetIdsForRestore(Env env, Database db, ReplicaAllocation restore try { Map> tag2beIds = Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation( - replicaAlloc, null); + replicaAlloc, null, false, false); for (Map.Entry> entry3 : tag2beIds.entrySet()) { for (Long beId : entry3.getValue()) { long newReplicaId = env.getNextId(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java index 7d0e5833fb1e89..54c059448228c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/DynamicPartitionUtil.java @@ -228,7 +228,7 @@ private static void checkReplicationNum(String val, Database db) throws DdlExcep ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_FORMAT, val); } ReplicaAllocation replicaAlloc = new ReplicaAllocation(Short.valueOf(val)); - Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null); + Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null, false, true); } private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, int hotPartitionNum, @@ -237,13 +237,14 @@ private static void checkReplicaAllocation(ReplicaAllocation replicaAlloc, int h ErrorReport.reportDdlException(ErrorCode.ERROR_DYNAMIC_PARTITION_REPLICATION_NUM_ZERO); } - Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null); + Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, null, false, true); if (hotPartitionNum <= 0) { return; } try { - Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, TStorageMedium.SSD); + Env.getCurrentSystemInfo().selectBackendIdsForReplicaCreation(replicaAlloc, TStorageMedium.SSD, false, + true); } catch (DdlException e) { throw new DdlException("Failed to find enough backend for ssd storage medium. When setting " + DynamicPartitionProperty.HOT_PARTITION_NUM + " > 0, the hot partitions will store " diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index beb6d3b893a0b3..dd137077dbbf23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -140,12 +140,6 @@ public class PropertyAnalyzer { public static final String PROPERTIES_ENABLE_DUPLICATE_WITHOUT_KEYS_BY_DEFAULT = "enable_duplicate_without_keys_by_default"; - - private static final Logger LOG = LogManager.getLogger(PropertyAnalyzer.class); - private static final String COMMA_SEPARATOR = ","; - private static final double MAX_FPP = 0.05; - private static final double MIN_FPP = 0.0001; - // For unique key data model, the feature Merge-on-Write will leverage a primary // key index and a delete-bitmap to mark duplicate keys as deleted in load stage, // which can avoid the merging cost in read stage, and accelerate the aggregation @@ -153,6 +147,10 @@ public class PropertyAnalyzer { // For the detail design, see the [DISP-018](https://cwiki.apache.org/confluence/ // display/DORIS/DSIP-018%3A+Support+Merge-On-Write+implementation+for+UNIQUE+KEY+data+model) public static final String ENABLE_UNIQUE_KEY_MERGE_ON_WRITE = "enable_unique_key_merge_on_write"; + private static final Logger LOG = LogManager.getLogger(PropertyAnalyzer.class); + private static final String COMMA_SEPARATOR = ","; + private static final double MAX_FPP = 0.05; + private static final double MIN_FPP = 0.0001; /** * check and replace members of DataProperty by properties. @@ -172,6 +170,7 @@ public static DataProperty analyzeDataProperty(Map properties, f long cooldownTimestamp = oldDataProperty.getCooldownTimeMs(); String newStoragePolicy = oldDataProperty.getStoragePolicy(); boolean hasStoragePolicy = false; + boolean storageMediumSpecified = false; for (Map.Entry entry : properties.entrySet()) { String key = entry.getKey(); @@ -179,8 +178,10 @@ public static DataProperty analyzeDataProperty(Map properties, f if (key.equalsIgnoreCase(PROPERTIES_STORAGE_MEDIUM)) { if (value.equalsIgnoreCase(TStorageMedium.SSD.name())) { storageMedium = TStorageMedium.SSD; + storageMediumSpecified = true; } else if (value.equalsIgnoreCase(TStorageMedium.HDD.name())) { storageMedium = TStorageMedium.HDD; + storageMediumSpecified = true; } else { throw new AnalysisException("Invalid storage medium: " + value); } @@ -247,7 +248,12 @@ public static DataProperty analyzeDataProperty(Map properties, f boolean mutable = PropertyAnalyzer.analyzeBooleanProp(properties, PROPERTIES_MUTABLE, true); properties.remove(PROPERTIES_MUTABLE); - return new DataProperty(storageMedium, cooldownTimestamp, newStoragePolicy, mutable); + DataProperty dataProperty = new DataProperty(storageMedium, cooldownTimestamp, newStoragePolicy, mutable); + // check the state of data property + if (storageMediumSpecified) { + dataProperty.setStorageMediumSpecified(true); + } + return dataProperty; } public static short analyzeShortKeyColumnCount(Map properties) throws AnalysisException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 8e996a110b0c23..9e1e3a8396f406 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -222,19 +222,14 @@ public InternalCatalog() { fullNameToDb.put(db.getFullName(), db); } - @Override - public long getId() { - return INTERNAL_CATALOG_ID; - } - @Override public String getType() { return "internal"; } @Override - public String getComment() { - return "Doris internal catalog"; + public long getId() { + return INTERNAL_CATALOG_ID; } @Override @@ -247,6 +242,10 @@ public List getDbNames() { return Lists.newArrayList(fullNameToDb.keySet()); } + public List getDbIds() { + return Lists.newArrayList(idToDb.keySet()); + } + @Nullable @Override public Database getDbNullable(String dbName) { @@ -278,16 +277,6 @@ public Database getDbNullable(long dbId) { return idToDb.get(dbId); } - public TableName getTableNameByTableId(Long tableId) { - for (Database db : fullNameToDb.values()) { - Table table = db.getTableNullable(tableId); - if (table != null) { - return new TableName("", db.getFullName(), table.getName()); - } - } - return null; - } - @Override public Map getProperties() { return Maps.newHashMap(); @@ -303,6 +292,21 @@ public void modifyCatalogProps(Map props) { LOG.warn("Ignore the modify catalog props in build-in catalog."); } + @Override + public String getComment() { + return "Doris internal catalog"; + } + + public TableName getTableNameByTableId(Long tableId) { + for (Database db : fullNameToDb.values()) { + Table table = db.getTableNullable(tableId); + if (table != null) { + return new TableName("", db.getFullName(), table.getName()); + } + } + return null; + } + // Use tryLock to avoid potential dead lock private boolean tryLock(boolean mustLock) { while (true) { @@ -336,10 +340,6 @@ private boolean tryLock(boolean mustLock) { } } - public List getDbIds() { - return Lists.newArrayList(idToDb.keySet()); - } - public List getDbs() { return Lists.newArrayList(idToDb.values()); } @@ -1225,7 +1225,7 @@ public void createTableAsSelect(CreateTableAsSelectStmt stmt) throws DdlExceptio ColumnDef columnDef; if (resultExpr.getSrcSlotRef() == null) { columnDef = new ColumnDef(name, typeDef, false, null, - true, false, new DefaultValue(false, null), ""); + true, false, new DefaultValue(false, null), ""); } else { Column column = resultExpr.getSrcSlotRef().getDesc().getColumn(); boolean setDefault = StringUtils.isNotBlank(column.getDefaultValue()); @@ -1517,7 +1517,7 @@ public void addPartition(Database db, String tableName, AddPartitionClause addPa olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy, idGeneratorBuffer, olapTable.disableAutoCompaction(), olapTable.enableSingleReplicaCompaction(), olapTable.skipWriteIndexOnLoad(), olapTable.storeRowColumn(), olapTable.isDynamicSchema(), - binlogConfig); + binlogConfig, dataProperty.isStorageMediumSpecified()); // check again olapTable = db.getOlapTableOrDdlException(tableName); @@ -1744,7 +1744,8 @@ private Partition createPartitionWithIndices(String clusterName, long dbId, long DataSortInfo dataSortInfo, boolean enableUniqueKeyMergeOnWrite, String storagePolicy, IdGeneratorBuffer idGeneratorBuffer, boolean disableAutoCompaction, boolean enableSingleReplicaCompaction, boolean skipWriteIndexOnLoad, - boolean storeRowColumn, boolean isDynamicSchema, BinlogConfig binlogConfig) throws DdlException { + boolean storeRowColumn, boolean isDynamicSchema, BinlogConfig binlogConfig, + boolean isStorageMediumSpecified) throws DdlException { // create base index first. Preconditions.checkArgument(baseIndexId != -1); MaterializedIndex baseIndex = new MaterializedIndex(baseIndexId, IndexState.NORMAL); @@ -1783,7 +1784,7 @@ private Partition createPartitionWithIndices(String clusterName, long dbId, long int schemaHash = indexMeta.getSchemaHash(); TabletMeta tabletMeta = new TabletMeta(dbId, tableId, partitionId, indexId, schemaHash, storageMedium); createTablets(clusterName, index, ReplicaState.NORMAL, distributionInfo, version, replicaAlloc, tabletMeta, - tabletIdSet, idGeneratorBuffer); + tabletIdSet, idGeneratorBuffer, isStorageMediumSpecified); boolean ok = false; String errMsg = null; @@ -2190,7 +2191,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep List rollupColumns = Env.getCurrentEnv().getMaterializedViewHandler() .checkAndPrepareMaterializedView(addRollupClause, olapTable, baseRollupIndex, false); short rollupShortKeyColumnCount = Env.calcShortKeyColumnCount(rollupColumns, alterClause.getProperties(), - true/*isKeysRequired*/); + true/*isKeysRequired*/); int rollupSchemaHash = Util.generateSchemaHash(); long rollupIndexId = idGeneratorBuffer.getNextId(); olapTable.setIndexMeta(rollupIndexId, addRollupClause.getRollupName(), rollupColumns, schemaVersion, @@ -2269,7 +2270,6 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep "Database " + db.getFullName() + " create unpartitioned table " + tableName + " increasing " + totalReplicaNum + " of replica exceeds quota[" + db.getReplicaQuota() + "]"); } - // create partition Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(), olapTable.getId(), olapTable.getBaseIndexId(), partitionId, partitionName, olapTable.getIndexIdToMeta(), partitionDistributionInfo, partitionInfo.getDataProperty(partitionId).getStorageMedium(), @@ -2278,7 +2278,8 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy, idGeneratorBuffer, olapTable.disableAutoCompaction(), olapTable.enableSingleReplicaCompaction(), skipWriteIndexOnLoad, - storeRowColumn, isDynamicSchema, binlogConfigForTask); + storeRowColumn, isDynamicSchema, binlogConfigForTask, + partitionInfo.getDataProperty(partitionId).isStorageMediumSpecified()); olapTable.addPartition(partition); } else if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { @@ -2329,22 +2330,23 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep && !Strings.isNullOrEmpty(partionStoragePolicy)) { throw new AnalysisException( "Can not create UNIQUE KEY table that enables Merge-On-write" - + " with storage policy(" + partionStoragePolicy + ")"); + + " with storage policy(" + partionStoragePolicy + ")"); } if (!partionStoragePolicy.equals("")) { storagePolicy = partionStoragePolicy; } Env.getCurrentEnv().getPolicyMgr().checkStoragePolicyExist(storagePolicy); - Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(), olapTable.getId(), - olapTable.getBaseIndexId(), entry.getValue(), entry.getKey(), olapTable.getIndexIdToMeta(), - partitionDistributionInfo, dataProperty.getStorageMedium(), - partitionInfo.getReplicaAllocation(entry.getValue()), versionInfo, bfColumns, bfFpp, - tabletIdSet, olapTable.getCopiedIndexes(), isInMemory, storageFormat, - partitionInfo.getTabletType(entry.getValue()), compressionType, + + Partition partition = createPartitionWithIndices(db.getClusterName(), db.getId(), + olapTable.getId(), olapTable.getBaseIndexId(), entry.getValue(), entry.getKey(), + olapTable.getIndexIdToMeta(), partitionDistributionInfo, + dataProperty.getStorageMedium(), partitionInfo.getReplicaAllocation(entry.getValue()), + versionInfo, bfColumns, bfFpp, tabletIdSet, olapTable.getCopiedIndexes(), isInMemory, + storageFormat, partitionInfo.getTabletType(entry.getValue()), compressionType, olapTable.getDataSortInfo(), olapTable.getEnableUniqueKeyMergeOnWrite(), storagePolicy, idGeneratorBuffer, olapTable.disableAutoCompaction(), - olapTable.enableSingleReplicaCompaction(), skipWriteIndexOnLoad, - storeRowColumn, isDynamicSchema, binlogConfigForTask); + olapTable.enableSingleReplicaCompaction(), skipWriteIndexOnLoad, storeRowColumn, + isDynamicSchema, binlogConfigForTask, dataProperty.isStorageMediumSpecified()); olapTable.addPartition(partition); } } else { @@ -2380,7 +2382,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep // register or remove table from DynamicPartition after table created DynamicPartitionUtil.registerOrRemoveDynamicPartitionTable(db.getId(), olapTable, false); Env.getCurrentEnv().getDynamicPartitionScheduler() - .executeDynamicPartitionFirstTime(db.getId(), olapTable.getId()); + .executeDynamicPartitionFirstTime(db.getId(), olapTable.getId()); Env.getCurrentEnv().getDynamicPartitionScheduler() .createOrUpdateRuntimeInfo(tableId, DynamicPartitionScheduler.LAST_UPDATE_TIME, TimeUtils.getCurrentFormatTime()); @@ -2528,7 +2530,8 @@ private void createJdbcTable(Database db, CreateTableStmt stmt) throws DdlExcept @VisibleForTesting public void createTablets(String clusterName, MaterializedIndex index, ReplicaState replicaState, DistributionInfo distributionInfo, long version, ReplicaAllocation replicaAlloc, TabletMeta tabletMeta, - Set tabletIdSet, IdGeneratorBuffer idGeneratorBuffer) throws DdlException { + Set tabletIdSet, IdGeneratorBuffer idGeneratorBuffer, boolean isStorageMediumSpecified) + throws DdlException { ColocateTableIndex colocateIndex = Env.getCurrentColocateIndex(); Map>> backendsPerBucketSeq = null; GroupId groupId = null; @@ -2588,10 +2591,12 @@ public void createTablets(String clusterName, MaterializedIndex index, ReplicaSt } else { if (!Config.disable_storage_medium_check) { chosenBackendIds = Env.getCurrentSystemInfo() - .selectBackendIdsForReplicaCreation(replicaAlloc, tabletMeta.getStorageMedium()); + .selectBackendIdsForReplicaCreation(replicaAlloc, tabletMeta.getStorageMedium(), + isStorageMediumSpecified, false); } else { chosenBackendIds = Env.getCurrentSystemInfo() - .selectBackendIdsForReplicaCreation(replicaAlloc, null); + .selectBackendIdsForReplicaCreation(replicaAlloc, null, + isStorageMediumSpecified, false); } } @@ -2764,7 +2769,8 @@ public void truncateTable(TruncateTableStmt truncateTableStmt) throws DdlExcepti olapTable.getPartitionInfo().getDataProperty(oldPartitionId).getStoragePolicy(), idGeneratorBuffer, olapTable.disableAutoCompaction(), olapTable.enableSingleReplicaCompaction(), olapTable.skipWriteIndexOnLoad(), - olapTable.storeRowColumn(), olapTable.isDynamicSchema(), binlogConfig); + olapTable.storeRowColumn(), olapTable.isDynamicSchema(), binlogConfig, + copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).isStorageMediumSpecified()); newPartitions.add(newPartition); } } catch (DdlException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java index 12483ff2f745a6..ace77c1ca85aa0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java +++ b/fe/fe-core/src/main/java/org/apache/doris/system/SystemInfoService.java @@ -528,11 +528,14 @@ public Map> getBeIdRoundRobinForReplicaCreation( * * @param replicaAlloc * @param storageMedium + * @param isStorageMediumSpecified + * @param isOnlyForCheck set true if only used for check available backend * @return return the selected backend ids group by tag. * @throws DdlException */ public Map> selectBackendIdsForReplicaCreation( - ReplicaAllocation replicaAlloc, TStorageMedium storageMedium) + ReplicaAllocation replicaAlloc, TStorageMedium storageMedium, boolean isStorageMediumSpecified, + boolean isOnlyForCheck) throws DdlException { Map copiedBackends = Maps.newHashMap(idToBackendRef); Map> chosenBackendIds = Maps.newHashMap(); @@ -557,6 +560,14 @@ public Map> selectBackendIdsForReplicaCreation( BeSelectionPolicy policy = builder.build(); List beIds = selectBackendIdsByPolicy(policy, entry.getValue()); + // first time empty, retry with different storage medium + // if only for check, no need to retry different storage medium to get backend + if (beIds.isEmpty() && storageMedium != null && !isStorageMediumSpecified && !isOnlyForCheck) { + storageMedium = (storageMedium == TStorageMedium.HDD) ? TStorageMedium.SSD : TStorageMedium.HDD; + policy = builder.setStorageMedium(storageMedium).build(); + beIds = selectBackendIdsByPolicy(policy, entry.getValue()); + } + // after retry different storage medium, it's still empty if (beIds.isEmpty()) { LOG.error("failed backend(s) for policy:" + policy); String errorReplication = "replication tag: " + entry.getKey() diff --git a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java index 5317c122e7e6f7..dcc76fdfcddd72 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java @@ -153,7 +153,8 @@ public void setUp() throws Exception { new Expectations() { { - systemInfoService.selectBackendIdsForReplicaCreation((ReplicaAllocation) any, (TStorageMedium) any); + systemInfoService.selectBackendIdsForReplicaCreation((ReplicaAllocation) any, (TStorageMedium) any, + false, true); minTimes = 0; result = new Delegate() { public synchronized List selectBackendIdsForReplicaCreation( diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableElasticOnStorageMediumTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableElasticOnStorageMediumTest.java new file mode 100644 index 00000000000000..8b09b4b65b5fd3 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableElasticOnStorageMediumTest.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.common.DdlException; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TStorageMedium; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; + +import java.util.List; + +public class CreateTableElasticOnStorageMediumTest extends TestWithFeService { + + @Override + protected void runAfterAll() throws Exception { + Env.getCurrentEnv().clear(); + } + + public void setStorageMediumToSSDTest() throws Exception { + SystemInfoService clusterInfo = Env.getCurrentEnv().getClusterInfo(); + List allBackends = clusterInfo.getAllBackends(); + // set all backends' storage medium to SSD + for (Backend backend : allBackends) { + if (backend.hasPathHash()) { + backend.getDisks().values().stream() + .peek(diskInfo -> diskInfo.setStorageMedium(TStorageMedium.SSD)); + } + } + createDatabase("db1"); + + String sql1 = "CREATE TABLE IF NOT EXISTS db1.t1 (pk INT, v1 INT sum) AGGREGATE KEY (pk) " + + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1');"; + Assertions.assertDoesNotThrow(() -> createTables(sql1)); + String sql2 = "CREATE TABLE IF NOT EXISTS db1.t2 (pk INT, v1 INT sum) AGGREGATE KEY (pk) " + + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1', 'storage_medium' = 'ssd');"; + Assertions.assertDoesNotThrow(() -> createTables(sql2)); + String sql3 = "CREATE TABLE IF NOT EXISTS db1.t3 (pk INT, v1 INT sum) AGGREGATE KEY (pk) " + + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1', 'storage_medium' = 'hdd');"; + Assertions.assertThrows(DdlException.class, () -> createTables(sql3)); + } + + public void setStorageMediumToHDDTest() throws Exception { + SystemInfoService clusterInfo = Env.getCurrentEnv().getClusterInfo(); + List allBackends = clusterInfo.getAllBackends(); + // set all backends' storage medium to SSD + for (Backend backend : allBackends) { + if (backend.hasPathHash()) { + backend.getDisks().values().stream() + .peek(diskInfo -> diskInfo.setStorageMedium(TStorageMedium.HDD)); + } + } + createDatabase("db1"); + + String sql1 = "CREATE TABLE IF NOT EXISTS db1.t4 (pk INT, v1 INT sum) AGGREGATE KEY (pk) " + + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1');"; + Assertions.assertDoesNotThrow(() -> createTables(sql1)); + String sql2 = "CREATE TABLE IF NOT EXISTS db1.t5 (pk INT, v1 INT sum) AGGREGATE KEY (pk) " + + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1', 'storage_medium' = 'hdd');"; + Assertions.assertDoesNotThrow(() -> createTables(sql2)); + String sql3 = "CREATE TABLE IF NOT EXISTS db1.t6 (pk INT, v1 INT sum) AGGREGATE KEY (pk) " + + "DISTRIBUTED BY HASH(pk) BUCKETS 1 PROPERTIES ('replication_num' = '1', 'storage_medium' = 'ssd');"; + Assertions.assertThrows(DdlException.class, () -> createTables(sql3)); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java index 891c48b75fbc51..6020369d50b9b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/ModifyBackendTest.java @@ -82,7 +82,7 @@ public void testModifyBackendTag() throws Exception { CreateTableStmt createStmt = (CreateTableStmt) UtFrameUtils.parseAndAnalyzeStmt(createStr, connectContext); ExceptionChecker.expectThrowsWithMsg(DdlException.class, "Failed to find enough backend, please check the replication num,replication tag and storage medium.\n" + "Create failed replications:\n" - + "replication tag: {\"location\" : \"default\"}, replication num: 1, storage medium: HDD", + + "replication tag: {\"location\" : \"default\"}, replication num: 1, storage medium: SSD", () -> DdlExecutor.execute(Env.getCurrentEnv(), createStmt)); createStr = "create table test.tbl1(\n" + "k1 int\n" + ") distributed by hash(k1)\n" + "buckets 3 properties(\n" diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/RoundRobinCreateTabletTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/RoundRobinCreateTabletTest.java index 029ce462dd9f49..fb65a13e743961 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/RoundRobinCreateTabletTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/RoundRobinCreateTabletTest.java @@ -113,7 +113,7 @@ public void testCreateTablets() { try { Env.getCurrentEnv().getInternalCatalog().createTablets(clusterName, index, ReplicaState.NORMAL, distributionInfo, 0, replicaAlloc, tabletMeta, - tabletIdSet, idGeneratorBuffer); + tabletIdSet, idGeneratorBuffer, false); } catch (Exception e) { System.out.println("failed to create tablets " + e.getMessage()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java index f043e9000671df..bf57f21f02fe93 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/load/sync/canal/CanalSyncDataTest.java @@ -150,7 +150,7 @@ public void setUp() throws Exception { result = execPlanFragmentParams; systemInfoService.selectBackendIdsForReplicaCreation((ReplicaAllocation) any, - (TStorageMedium) any); + (TStorageMedium) any, false, true); minTimes = 0; result = backendIds; diff --git a/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java b/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java index 17b9b6591fcd85..172fbd559456c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/system/SystemInfoServiceTest.java @@ -373,7 +373,7 @@ public void testSelectBackendIdsForReplicaCreation() throws Exception { Map beCounterMap = Maps.newHashMap(); for (int i = 0; i < 10000; ++i) { Map> res = infoService.selectBackendIdsForReplicaCreation(replicaAlloc, - TStorageMedium.HDD); + TStorageMedium.HDD, false, false); Assert.assertEquals(3, res.get(Tag.DEFAULT_BACKEND_TAG).size()); for (Long beId : res.get(Tag.DEFAULT_BACKEND_TAG)) { beCounterMap.put(beId, beCounterMap.getOrDefault(beId, 0) + 1);