From 385739564d0c2d057632bb6de21d90f4b9942829 Mon Sep 17 00:00:00 2001 From: wangbo Date: Fri, 17 May 2024 17:34:08 +0800 Subject: [PATCH 001/111] [test](executor) Add workload group upgrade test #35007 --- .../workload_manager_p0/test_check_wg.out | 6 ++++ .../suites/workload_manager_p0/load.groovy | 33 +++++++++++++++++++ .../workload_manager_p0/test_check_wg.groovy | 19 +++++++++++ .../workload_manager_p0/test_curd_wlg.groovy | 7 ---- 4 files changed, 58 insertions(+), 7 deletions(-) create mode 100644 regression-test/data/workload_manager_p0/test_check_wg.out create mode 100644 regression-test/suites/workload_manager_p0/load.groovy create mode 100644 regression-test/suites/workload_manager_p0/test_check_wg.groovy diff --git a/regression-test/data/workload_manager_p0/test_check_wg.out b/regression-test/data/workload_manager_p0/test_check_wg.out new file mode 100644 index 000000000000000..1f0a440ba0ba983 --- /dev/null +++ b/regression-test/data/workload_manager_p0/test_check_wg.out @@ -0,0 +1,6 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select_wg -- +normal +upgrade_g1 +upgrade_g2 + diff --git a/regression-test/suites/workload_manager_p0/load.groovy b/regression-test/suites/workload_manager_p0/load.groovy new file mode 100644 index 000000000000000..bdbce4093adab98 --- /dev/null +++ b/regression-test/suites/workload_manager_p0/load.groovy @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_create_upgrade_wg") { + sql "ADMIN SET FRONTEND CONFIG ('enable_workload_group' = 'true');" + + sql "create workload group if not exists upgrade_g1 " + + "properties ( " + + " 'cpu_share'='11', " + + " 'memory_limit'='0.2%', " + + " 'enable_memory_overcommit'='true' " + + ");" + + sql "create workload group if not exists upgrade_g2 " + + "properties ( " + + " 'cpu_share'='12', " + + " 'memory_limit'='0.3%', " + + " 'enable_memory_overcommit'='true' " + + ");" +} \ No newline at end of file diff --git a/regression-test/suites/workload_manager_p0/test_check_wg.groovy b/regression-test/suites/workload_manager_p0/test_check_wg.groovy new file mode 100644 index 000000000000000..6c81338e703b892 --- /dev/null +++ b/regression-test/suites/workload_manager_p0/test_check_wg.groovy @@ -0,0 +1,19 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +suite("test_check_wg") { + qt_select_wg "select name from information_schema.workload_groups where name in ('upgrade_g1','normal','upgrade_g2');" +} \ No newline at end of file diff --git a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy index 62eb762ff9b8b60..4b54b9224c378d4 100644 --- a/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy +++ b/regression-test/suites/workload_manager_p0/test_curd_wlg.groovy @@ -76,13 +76,6 @@ suite("test_crud_wlg") { sql "ADMIN SET FRONTEND CONFIG ('enable_alter_queue_prop_sync' = 'true');" sql "ADMIN SET FRONTEND CONFIG ('query_queue_update_interval_ms' = '100');" - sql "create workload group if not exists normal " + - "properties ( " + - " 'cpu_share'='1024', " + - " 'memory_limit'='50%', " + - " 'enable_memory_overcommit'='true' " + - ");" - // reset normal group property sql "alter workload group normal properties ( 'cpu_share'='1024' );" sql "alter workload group normal properties ( 'memory_limit'='50%' );" From b76cfcd007b1548bb8303acb50d4a36a16ad9c6e Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Fri, 17 May 2024 22:54:21 +0800 Subject: [PATCH 002/111] [refactor](mtmv) Materialization context and mtmv decoupling (#34093) (#34916) Decoupling the MTMV from the materialization context. Change MaterializationContext to abstract which is the materialization desc. It now has AsyncMaterializationContext sub class, can also has other type of MaterializationContext such as SyncMaterializationContext and so on. --- .../java/org/apache/doris/mtmv/MTMVCache.java | 5 +- .../apache/doris/nereids/NereidsPlanner.java | 16 +- .../mv/AbstractMaterializedViewRule.java | 82 +++--- .../mv/AsyncMaterializationContext.java | 140 ++++++++++ .../mv/InitMaterializationContextHook.java | 19 +- .../mv/MaterializationContext.java | 250 ++++++++++-------- .../exploration/mv/MaterializedViewUtils.java | 3 + .../rules/exploration/mv/StructInfo.java | 7 +- 8 files changed, 351 insertions(+), 171 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java index 154bd4ec7f17c19..8bd87e2e149a5b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java @@ -42,9 +42,10 @@ */ public class MTMVCache { - // the materialized view plan which should be optimized by the same rules to query + // The materialized view plan which should be optimized by the same rules to query + // and will remove top sink and unused sort private final Plan logicalPlan; - // for stable output order, we should use original plan + // The original plan of mv def sql private final Plan originalPlan; public MTMVCache(Plan logicalPlan, Plan originalPlan) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java index 6f65ee9538cc960..2a354eb9c4b1501 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java @@ -23,7 +23,6 @@ import org.apache.doris.analysis.StatementBase; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; -import org.apache.doris.catalog.MTMV; import org.apache.doris.common.NereidsException; import org.apache.doris.common.Pair; import org.apache.doris.common.profile.SummaryProfile; @@ -53,7 +52,6 @@ import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalSqlCache; -import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalEmptyRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; @@ -467,11 +465,16 @@ public String getExplainString(ExplainOptions explainOptions) { plan = optimizedPlan.shape(""); break; case MEMO_PLAN: + StringBuilder materializationStringBuilder = new StringBuilder(); + materializationStringBuilder.append("materializationContexts:").append("\n"); + for (MaterializationContext ctx : cascadesContext.getMaterializationContexts()) { + materializationStringBuilder.append("\n").append(ctx).append("\n"); + } plan = cascadesContext.getMemo().toString() + "\n\n========== OPTIMIZED PLAN ==========\n" + optimizedPlan.treeString() + "\n\n========== MATERIALIZATIONS ==========\n" - + MaterializationContext.toDetailString(cascadesContext.getMaterializationContexts()); + + materializationStringBuilder; break; case ALL_PLAN: plan = "========== PARSED PLAN " @@ -488,14 +491,9 @@ public String getExplainString(ExplainOptions explainOptions) { + optimizedPlan.treeString(); break; default: - List materializationListChosenByCbo = this.getPhysicalPlan() - .collectToList(node -> node instanceof PhysicalCatalogRelation - && ((PhysicalCatalogRelation) node).getTable() instanceof MTMV).stream() - .map(node -> (MTMV) ((PhysicalCatalogRelation) node).getTable()) - .collect(Collectors.toList()); plan = super.getExplainString(explainOptions) + MaterializationContext.toSummaryString(cascadesContext.getMaterializationContexts(), - materializationListChosenByCbo); + this.getPhysicalPlan()); } if (statementContext != null && !statementContext.getHints().isEmpty()) { String hint = getHintExplainString(statementContext.getHints()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 7a08797ca550ebe..dcff9db69db86f4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -237,14 +237,14 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca // Rewrite query by view rewrittenPlan = rewriteQueryByView(matchMode, queryStructInfo, viewStructInfo, viewToQuerySlotMapping, rewrittenPlan, materializationContext); - if (rewrittenPlan == null) { - continue; - } rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext, childContext -> { Rewriter.getWholeTreeRewriter(childContext).execute(); return childContext.getRewritePlan(); }, rewrittenPlan, queryPlan); + if (rewrittenPlan == null) { + continue; + } // check the partitions used by rewritten plan is valid or not Multimap, Partition> invalidPartitionsQueryUsed = calcUsedInvalidMvPartitions(rewrittenPlan, materializationContext, cascadesContext); @@ -287,9 +287,10 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } // For rewrittenPlan which contains materialized view should remove invalid partition ids List children = Lists.newArrayList( - rewrittenPlan.accept(new InvalidPartitionRemover(), Pair.of(materializationContext.getMTMV(), - invalidPartitionsQueryUsed.values().stream() - .map(Partition::getId).collect(Collectors.toSet()))), + rewrittenPlan.accept(new InvalidPartitionRemover(), Pair.of( + materializationContext.getMaterializationQualifier(), + invalidPartitionsQueryUsed.values().stream().map(Partition::getId) + .collect(Collectors.toSet()))), StructInfo.addFilterOnTableScan(queryPlan, filterOnOriginPlan, cascadesContext)); // Union query materialized view and source table rewrittenPlan = new LogicalUnion(Qualifier.ALL, @@ -387,40 +388,43 @@ protected Multimap, Partition> calcUsedIn Plan rewrittenPlan, MaterializationContext materializationContext, CascadesContext cascadesContext) { - // check partition is valid or not - MTMV mtmv = materializationContext.getMTMV(); - PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo(); - if (PartitionType.UNPARTITIONED.equals(mvPartitionInfo.getType())) { - // if not partition, if rewrite success, it means mv is available - return ImmutableMultimap.of(); - } - // check mv related table partition is valid or not - MTMVPartitionInfo mvCustomPartitionInfo = mtmv.getMvPartitionInfo(); - BaseTableInfo relatedPartitionTable = mvCustomPartitionInfo.getRelatedTableInfo(); - if (relatedPartitionTable == null) { - return ImmutableMultimap.of(); + if (materializationContext instanceof AsyncMaterializationContext) { + // check partition is valid or not + MTMV mtmv = ((AsyncMaterializationContext) materializationContext).getMtmv(); + PartitionInfo mvPartitionInfo = mtmv.getPartitionInfo(); + if (PartitionType.UNPARTITIONED.equals(mvPartitionInfo.getType())) { + // if not partition, if rewrite success, it means mv is available + return ImmutableMultimap.of(); + } + // check mv related table partition is valid or not + MTMVPartitionInfo mvCustomPartitionInfo = mtmv.getMvPartitionInfo(); + BaseTableInfo relatedPartitionTable = mvCustomPartitionInfo.getRelatedTableInfo(); + if (relatedPartitionTable == null) { + return ImmutableMultimap.of(); + } + // get mv valid partitions + Set mvDataValidPartitionIdSet = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, + cascadesContext.getConnectContext(), System.currentTimeMillis()).stream() + .map(Partition::getId) + .collect(Collectors.toSet()); + // get partitions query used + Set mvPartitionSetQueryUsed = rewrittenPlan.collectToList(node -> node instanceof LogicalOlapScan + && Objects.equals(((CatalogRelation) node).getTable().getName(), mtmv.getName())) + .stream() + .map(node -> ((LogicalOlapScan) node).getSelectedPartitionIds()) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + // get invalid partition ids + Set invalidMvPartitionIdSet = new HashSet<>(mvPartitionSetQueryUsed); + invalidMvPartitionIdSet.removeAll(mvDataValidPartitionIdSet); + ImmutableMultimap.Builder, Partition> invalidPartitionMapBuilder = + ImmutableMultimap.builder(); + Pair partitionInfo = Pair.of(mvCustomPartitionInfo, mvPartitionInfo); + invalidMvPartitionIdSet.forEach(invalidPartitionId -> + invalidPartitionMapBuilder.put(partitionInfo, mtmv.getPartition(invalidPartitionId))); + return invalidPartitionMapBuilder.build(); } - // get mv valid partitions - Set mvDataValidPartitionIdSet = MTMVRewriteUtil.getMTMVCanRewritePartitions(mtmv, - cascadesContext.getConnectContext(), System.currentTimeMillis()).stream() - .map(Partition::getId) - .collect(Collectors.toSet()); - // get partitions query used - Set mvPartitionSetQueryUsed = rewrittenPlan.collectToList(node -> node instanceof LogicalOlapScan - && Objects.equals(((CatalogRelation) node).getTable().getName(), mtmv.getName())) - .stream() - .map(node -> ((LogicalOlapScan) node).getSelectedPartitionIds()) - .flatMap(Collection::stream) - .collect(Collectors.toSet()); - // get invalid partition ids - Set invalidMvPartitionIdSet = new HashSet<>(mvPartitionSetQueryUsed); - invalidMvPartitionIdSet.removeAll(mvDataValidPartitionIdSet); - ImmutableMultimap.Builder, Partition> invalidPartitionMapBuilder = - ImmutableMultimap.builder(); - Pair partitionInfo = Pair.of(mvCustomPartitionInfo, mvPartitionInfo); - invalidMvPartitionIdSet.forEach(invalidPartitionId -> - invalidPartitionMapBuilder.put(partitionInfo, mtmv.getPartition(invalidPartitionId))); - return invalidPartitionMapBuilder.build(); + return ImmutableMultimap.of(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java new file mode 100644 index 000000000000000..1c0d854dd90096b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AsyncMaterializationContext.java @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.exploration.mv; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.Table; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.trees.plans.ObjectId; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Relation; +import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.collect.Multimap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * Async context for query rewrite by materialized view + */ +public class AsyncMaterializationContext extends MaterializationContext { + + private static final Logger LOG = LogManager.getLogger(AsyncMaterializationContext.class); + private final MTMV mtmv; + + /** + * MaterializationContext, this contains necessary info for query rewriting by mv + */ + public AsyncMaterializationContext(MTMV mtmv, Plan mvPlan, Plan mvOriginalPlan, List baseTables, + List
baseViews, CascadesContext cascadesContext) { + super(mvPlan, mvOriginalPlan, MaterializedViewUtils.generateMvScanPlan(mtmv, cascadesContext), cascadesContext); + this.mtmv = mtmv; + } + + public MTMV getMtmv() { + return mtmv; + } + + @Override + Plan doGenerateMvPlan(CascadesContext cascadesContext) { + return MaterializedViewUtils.generateMvScanPlan(this.mtmv, cascadesContext); + } + + @Override + List getMaterializationQualifier() { + return this.mtmv.getFullQualifiers(); + } + + @Override + String getStringInfo() { + StringBuilder failReasonBuilder = new StringBuilder("[").append("\n"); + for (Map.Entry>> reasonEntry : this.failReason.asMap().entrySet()) { + failReasonBuilder + .append("\n") + .append("ObjectId : ").append(reasonEntry.getKey()).append(".\n"); + for (Pair reason : reasonEntry.getValue()) { + failReasonBuilder.append("Summary : ").append(reason.key()).append(".\n") + .append("Reason : ").append(reason.value()).append(".\n"); + } + } + failReasonBuilder.append("\n").append("]"); + return Utils.toSqlString("MaterializationContext[" + getMaterializationQualifier() + "]", + "rewriteSuccess", this.success, + "failReason", failReasonBuilder.toString()); + } + + @Override + boolean isFinalChosen(Relation relation) { + if (!(relation instanceof PhysicalCatalogRelation)) { + return false; + } + return ((PhysicalCatalogRelation) relation).getTable() instanceof MTMV; + } + + public Plan getMvScanPlan() { + return mvScanPlan; + } + + public List
getBaseTables() { + return baseTables; + } + + public List
getBaseViews() { + return baseViews; + } + + public ExpressionMapping getMvExprToMvScanExprMapping() { + return mvExprToMvScanExprMapping; + } + + public boolean isAvailable() { + return available; + } + + public Plan getMvPlan() { + return mvPlan; + } + + public Multimap> getFailReason() { + return failReason; + } + + public boolean isEnableRecordFailureDetail() { + return enableRecordFailureDetail; + } + + public void setSuccess(boolean success) { + this.success = success; + this.failReason.clear(); + } + + public StructInfo getStructInfo() { + return structInfo; + } + + public boolean isSuccess() { + return success; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 08c312b4737c8c7..311932fa1b7cf7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -20,7 +20,9 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.AnalysisException; import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.PlannerHook; @@ -29,6 +31,7 @@ import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -75,10 +78,18 @@ public void initMaterializationContext(CascadesContext cascadesContext) { return; } for (MTMV materializedView : availableMTMVs) { - cascadesContext.addMaterializationContext( - MaterializationContext.fromMaterializedView(materializedView, - MaterializedViewUtils.generateMvScanPlan(materializedView, cascadesContext), - cascadesContext)); + MTMVCache mtmvCache = null; + try { + mtmvCache = materializedView.getOrGenerateCache(); + } catch (AnalysisException e) { + LOG.warn("MaterializationContext init mv cache generate fail", e); + } + if (mtmvCache == null) { + continue; + } + cascadesContext.addMaterializationContext(new AsyncMaterializationContext(materializedView, + mtmvCache.getLogicalPlan(), mtmvCache.getOriginalPlan(), ImmutableList.of(), ImmutableList.of(), + cascadesContext)); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 1af63dab21fc3c8..261e3bb85f9f1d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -18,102 +18,97 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.analysis.StatementBase; -import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Table; -import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; -import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.algebra.Relation; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; import org.apache.doris.nereids.util.ExpressionUtils; -import org.apache.doris.nereids.util.Utils; -import com.google.common.collect.ImmutableList; +import com.google.common.collect.HashMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Multimap; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.BitSet; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; /** - * Maintain the context for query rewrite by materialized view + * Abstract context for query rewrite by materialized view */ -public class MaterializationContext { - +public abstract class MaterializationContext { private static final Logger LOG = LogManager.getLogger(MaterializationContext.class); - - private final MTMV mtmv; - private final List
baseTables; - private final List
baseViews; - // Group ids that are rewritten by this mv to reduce rewrite times - private final Set matchedFailGroups = new HashSet<>(); - private final Set matchedSuccessGroups = new HashSet<>(); - // if rewrite by mv fail, record the reason, if success the failReason should be empty. - // The key is the query belonged group expression objectId, the value is the fail reason - private final Map> failReason = new LinkedHashMap<>(); + protected List
baseTables; + protected List
baseViews; + // The plan of mv def sql + protected Plan mvPlan; + // The original plan of mv def sql + protected Plan originalMvPlan; // Should regenerate when materialization is already rewritten successfully because one query may hit repeatedly // make sure output is different in multi using - private Plan mvScanPlan; - // generated expressions form mv scan plan - private ExpressionMapping mvExprToMvScanExprMapping; - private List mvPlanOutputShuttledExpressions; - private boolean available = true; - // the mv plan from cache at present, record it to make sure query rewrite by mv is right when cache change. - private Plan mvPlan; - // mark rewrite success or not - private boolean success = false; - private boolean enableRecordFailureDetail = false; - private StructInfo structInfo; + protected Plan mvScanPlan; + // The mvPlan output shuttled expression, this is used by generate field mvExprToMvScanExprMapping + protected List mvPlanOutputShuttledExpressions; + // Generated mapping from mv plan out shuttled expr to mv scan plan out slot mapping, this is used for later used + protected ExpressionMapping mvExprToMvScanExprMapping; + // This mark the materialization context is available or not, + // will not be used in query transparent rewritten if false + protected boolean available = true; + // Mark the mv plan in the context is already rewritten successfully or not + protected boolean success = false; + // Mark enable record failure detail info or not, because record failure detail info is performance-depleting + protected final boolean enableRecordFailureDetail; + // The mv plan struct info + protected final StructInfo structInfo; + // Group id set that are rewritten unsuccessfully by this mv for reducing rewrite times + protected final Set matchedFailGroups = new HashSet<>(); + // Group id set that are rewritten successfully by this mv for reducing rewrite times + protected final Set matchedSuccessGroups = new HashSet<>(); + // Record the reason, if rewrite by mv fail. The failReason should be empty if success. + // The key is the query belonged group expression objectId, the value is the fail reasons because + // for one materialization query may be multi when nested materialized view. + protected final Multimap> failReason = HashMultimap.create(); /** * MaterializationContext, this contains necessary info for query rewriting by mv */ - public MaterializationContext(MTMV mtmv, Plan mvScanPlan, List
baseTables, List
baseViews, - CascadesContext cascadesContext) { - this.mtmv = mtmv; + public MaterializationContext(Plan mvPlan, Plan originalMvPlan, Plan mvScanPlan, CascadesContext cascadesContext) { + this.mvPlan = mvPlan; + this.originalMvPlan = originalMvPlan; this.mvScanPlan = mvScanPlan; - this.baseTables = baseTables; - this.baseViews = baseViews; + StatementBase parsedStatement = cascadesContext.getStatementContext().getParsedStatement(); this.enableRecordFailureDetail = parsedStatement != null && parsedStatement.isExplain() && ExplainLevel.MEMO_PLAN == parsedStatement.getExplainOptions().getExplainLevel(); - MTMVCache mtmvCache = null; - try { - mtmvCache = mtmv.getOrGenerateCache(); - } catch (AnalysisException e) { - LOG.warn("MaterializationContext init mv cache generate fail", e); - } - if (mtmvCache == null) { - this.available = false; - return; - } + this.mvPlanOutputShuttledExpressions = ExpressionUtils.shuttleExpressionWithLineage( - mtmvCache.getOriginalPlan().getOutput(), - mtmvCache.getOriginalPlan(), + originalMvPlan.getOutput(), + originalMvPlan, new BitSet()); // mv output expression shuttle, this will be used to expression rewrite this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, - this.mvScanPlan.getExpressions()); + this.mvScanPlan.getOutput()); // copy the plan from cache, which the plan in cache may change - this.mvPlan = mtmvCache.getLogicalPlan(); List viewStructInfos = MaterializedViewUtils.extractStructInfo( - mtmvCache.getLogicalPlan(), cascadesContext, new BitSet()); + mvPlan, cascadesContext, new BitSet()); if (viewStructInfos.size() > 1) { // view struct info should only have one, log error and use the first struct info - LOG.warn(String.format("view strut info is more than one, mv name is %s, mv plan is %s", - mtmv.getName(), mvPlan.treeString())); + LOG.warn(String.format("view strut info is more than one, materialization name is %s, mv plan is %s", + getMaterializationQualifier(), getMvPlan().treeString())); } this.structInfo = viewStructInfos.get(0); } @@ -131,22 +126,49 @@ public void addMatchedGroup(GroupId groupId, boolean rewriteSuccess) { } /** - * Try to generate scan plan for materialized view - * if MaterializationContext is already rewritten by materialized view, then should generate in real time - * when query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output + * Try to generate scan plan for materialization + * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later + * query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output * should be different */ public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) { if (!this.matchedSuccessGroups.isEmpty()) { - this.mvScanPlan = MaterializedViewUtils.generateMvScanPlan(this.mtmv, cascadesContext); + this.mvScanPlan = doGenerateMvPlan(cascadesContext); // mv output expression shuttle, this will be used to expression rewrite this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, this.mvScanPlan.getExpressions()); } } - public MTMV getMTMV() { - return mtmv; + /** + * Try to generate scan plan for materialization + * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later + * query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output + * should be different + */ + abstract Plan doGenerateMvPlan(CascadesContext cascadesContext); + + /** + * Get materialization unique qualifier which identify it + */ + abstract List getMaterializationQualifier(); + + /** + * Get String info which is used for to string + */ + abstract String getStringInfo(); + + /** + * Calc the relation is chosen finally or not + */ + abstract boolean isFinalChosen(Relation relation); + + public Plan getMvPlan() { + return mvPlan; + } + + public Plan getOriginalMvPlan() { + return originalMvPlan; } public Plan getMvScanPlan() { @@ -169,11 +191,7 @@ public boolean isAvailable() { return available; } - public Plan getMvPlan() { - return mvPlan; - } - - public Map> getFailReason() { + public Multimap> getFailReason() { return failReason; } @@ -183,6 +201,7 @@ public boolean isEnableRecordFailureDetail() { public void setSuccess(boolean success) { this.success = success; + // TODO clear the fail message by according planId ? this.failReason.clear(); } @@ -190,8 +209,12 @@ public StructInfo getStructInfo() { return structInfo; } + public boolean isSuccess() { + return success; + } + /** - * recordFailReason + * Record fail reason when in rewriting */ public void recordFailReason(StructInfo structInfo, String summary, Supplier failureReasonSupplier) { // record it's rewritten @@ -207,65 +230,54 @@ public void recordFailReason(StructInfo structInfo, String summary, Supplier> reason : this.failReason.entrySet()) { - failReasonBuilder - .append("\n") - .append("ObjectId : ").append(reason.getKey()).append(".\n") - .append("Summary : ").append(reason.getValue().key()).append(".\n") - .append("Reason : ").append(reason.getValue().value()).append(".\n"); - } - failReasonBuilder.append("\n").append("]"); - return Utils.toSqlString("MaterializationContext[" + mtmv.getName() + "]", - "rewriteSuccess", this.success, - "failReason", failReasonBuilder.toString()); - } - - /** - * toString, this contains summary and detail info. - */ - public static String toDetailString(List materializationContexts) { - StringBuilder builder = new StringBuilder(); - builder.append("materializationContexts:").append("\n"); - for (MaterializationContext ctx : materializationContexts) { - builder.append("\n").append(ctx).append("\n"); - } - return builder.toString(); + return getStringInfo(); } /** - * toSummaryString, this contains only summary info. + * ToSummaryString, this contains only summary info. */ public static String toSummaryString(List materializationContexts, - List chosenMaterializationNames) { + PhysicalPlan physicalPlan) { if (materializationContexts.isEmpty()) { return ""; } - Set materializationChosenNameSet = chosenMaterializationNames.stream() - .map(MTMV::getName) + Set rewrittenSuccessMaterializationSet = materializationContexts.stream() + .filter(MaterializationContext::isSuccess) .collect(Collectors.toSet()); + Set> chosenMaterializationQualifiers = new HashSet<>(); + physicalPlan.accept(new DefaultPlanVisitor() { + @Override + public Void visitPhysicalRelation(PhysicalRelation physicalRelation, Void context) { + for (MaterializationContext rewrittenContext : rewrittenSuccessMaterializationSet) { + if (rewrittenContext.isFinalChosen(physicalRelation)) { + chosenMaterializationQualifiers.add(rewrittenContext.getMaterializationQualifier()); + } + } + return null; + } + }, null); + StringBuilder builder = new StringBuilder(); builder.append("\nMaterializedView"); // rewrite success and chosen builder.append("\nMaterializedViewRewriteSuccessAndChose:\n"); - if (!materializationChosenNameSet.isEmpty()) { - builder.append(" Names: ").append(String.join(", ", materializationChosenNameSet)); + if (!chosenMaterializationQualifiers.isEmpty()) { + builder.append(" Names: "); + chosenMaterializationQualifiers.forEach(materializationQualifier -> + builder.append(generateQualifierName(materializationQualifier)).append(", ")); } // rewrite success but not chosen builder.append("\nMaterializedViewRewriteSuccessButNotChose:\n"); - Set rewriteSuccessButNotChoseNameSet = materializationContexts.stream() - .filter(materializationContext -> materializationContext.isSuccess() - && !materializationChosenNameSet.contains(materializationContext.getMTMV().getName())) - .map(materializationContext -> materializationContext.getMTMV().getName()) + Set> rewriteSuccessButNotChoseQualifiers = rewrittenSuccessMaterializationSet.stream() + .map(MaterializationContext::getMaterializationQualifier) + .filter(materializationQualifier -> !chosenMaterializationQualifiers.contains(materializationQualifier)) .collect(Collectors.toSet()); - if (!rewriteSuccessButNotChoseNameSet.isEmpty()) { - builder.append(" Names: ").append(String.join(", ", rewriteSuccessButNotChoseNameSet)); + if (!rewriteSuccessButNotChoseQualifiers.isEmpty()) { + builder.append(" Names: "); + rewriteSuccessButNotChoseQualifiers.forEach(materializationQualifier -> + builder.append(generateQualifierName(materializationQualifier)).append(", ")); } // rewrite fail builder.append("\nMaterializedViewRewriteFail:"); @@ -274,7 +286,7 @@ public static String toSummaryString(List materializatio Set failReasonSet = ctx.getFailReason().values().stream().map(Pair::key).collect(ImmutableSet.toImmutableSet()); builder.append("\n") - .append(" Name: ").append(ctx.getMTMV().getName()) + .append(" Name: ").append(generateQualifierName(ctx.getMaterializationQualifier())) .append("\n") .append(" FailSummary: ").append(String.join(", ", failReasonSet)); } @@ -282,12 +294,24 @@ public static String toSummaryString(List materializatio return builder.toString(); } - /** - * MaterializationContext fromMaterializedView - */ - public static MaterializationContext fromMaterializedView(MTMV materializedView, Plan mvScanPlan, - CascadesContext cascadesContext) { - return new MaterializationContext(materializedView, mvScanPlan, ImmutableList.of(), ImmutableList.of(), - cascadesContext); + private static String generateQualifierName(List qualifiers) { + return String.join("#", qualifiers); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + MaterializationContext context = (MaterializationContext) o; + return getMaterializationQualifier().equals(context.getMaterializationQualifier()); + } + + @Override + public int hashCode() { + return Objects.hash(getMaterializationQualifier()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 6863a7e01b14f1c..608f3b5f5f051ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -203,6 +203,9 @@ public static Plan rewriteByRules( CascadesContext cascadesContext, Function planRewriter, Plan rewrittenPlan, Plan originPlan) { + if (originPlan == null || rewrittenPlan == null) { + return null; + } if (originPlan.getOutputSet().size() != rewrittenPlan.getOutputSet().size()) { return rewrittenPlan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java index 7f6b5f0d291d963..1ba0c9964eb0abd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.rules.exploration.mv; -import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.TableIf; @@ -630,11 +629,11 @@ private Boolean doVisit(Plan plan, PlanCheckContext checkContext) { /** * Add predicates on base table when materialized view scan contains invalid partitions */ - public static class InvalidPartitionRemover extends DefaultPlanRewriter>> { + public static class InvalidPartitionRemover extends DefaultPlanRewriter, Set>> { // materialized view scan is always LogicalOlapScan, so just handle LogicalOlapScan @Override - public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Pair> context) { - if (olapScan.getTable().getName().equals(context.key().getName())) { + public Plan visitLogicalOlapScan(LogicalOlapScan olapScan, Pair, Set> context) { + if (olapScan.getTable().getFullQualifiers().equals(context.key())) { List selectedPartitionIds = olapScan.getSelectedPartitionIds(); return olapScan.withSelectedPartitionIds(selectedPartitionIds.stream() .filter(partitionId -> !context.value().contains(partitionId)) From 691f3c5ee7a36a4ae374dbef731f25ce5c2bf20d Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Sat, 11 May 2024 11:31:24 +0800 Subject: [PATCH 003/111] [Performance](Variant) Improve load performance for variant type (#33890) 1. remove phmap for padding rows 2. add SimpleFieldVisitorToScarlarType for short circuit type deducing 3. correct type coercion for conflict types bettween integers 4. improve nullable column performance 5. remove shared_ptr dependancy for DataType use TypeIndex instead 6. Optimization by caching the order of fields (which is almost always the same) and a quick check to match the next expected field, instead of searching the hash table. benchmark: In clickbench data, load performance: 12m36.799s ->7m10.934s about 43% latency reduce In variant_p2/performance.groovy: 3min44s20 -> 1min15s80 about 66% latency reducy --- be/src/vec/columns/column_object.cpp | 189 +++++++++++++----- be/src/vec/columns/column_object.h | 28 ++- be/src/vec/common/schema_util.cpp | 57 +++--- be/src/vec/common/schema_util.h | 4 +- be/src/vec/core/field.h | 5 + be/src/vec/json/parse2column.cpp | 22 +- .../suites/variant_p2/performance.groovy | 36 ++++ 7 files changed, 248 insertions(+), 93 deletions(-) create mode 100644 regression-test/suites/variant_p2/performance.groovy diff --git a/be/src/vec/columns/column_object.cpp b/be/src/vec/columns/column_object.cpp index ddb5bee6e019595..3bae978f4d3f6d4 100644 --- a/be/src/vec/columns/column_object.cpp +++ b/be/src/vec/columns/column_object.cpp @@ -45,6 +45,7 @@ #include "util/defer_op.h" #include "util/simd/bits.h" #include "vec/aggregate_functions/aggregate_function.h" +#include "vec/aggregate_functions/helpers.h" #include "vec/columns/column.h" #include "vec/columns/column_array.h" #include "vec/columns/column_nullable.h" @@ -56,6 +57,7 @@ #include "vec/common/field_visitors.h" #include "vec/common/schema_util.h" #include "vec/common/string_buffer.hpp" +#include "vec/common/string_ref.h" #include "vec/core/column_with_type_and_name.h" #include "vec/core/field.h" #include "vec/core/types.h" @@ -68,6 +70,7 @@ #include "vec/data_types/data_type_nothing.h" #include "vec/data_types/data_type_nullable.h" #include "vec/data_types/get_least_supertype.h" +#include "vec/json/path_in_data.h" #ifdef __AVX2__ #include "util/jsonb_parser_simd.h" @@ -78,23 +81,22 @@ namespace doris::vectorized { namespace { -DataTypePtr create_array_of_type(DataTypePtr type, size_t num_dimensions, bool is_nullable) { - const DataTypeNullable* nullable = typeid_cast(type.get()); - if ((nullable && - typeid_cast(nullable->get_nested_type().get())) || - typeid_cast(type.get())) { +DataTypePtr create_array_of_type(TypeIndex type, size_t num_dimensions, bool is_nullable) { + if (type == ColumnObject::MOST_COMMON_TYPE_ID) { // JSONB type MUST NOT wrapped in ARRAY column, it should be top level. // So we ignored num_dimensions. - return type; + return is_nullable ? make_nullable(std::make_shared()) + : std::make_shared(); } + DataTypePtr result = DataTypeFactory::instance().create_data_type(type, is_nullable); for (size_t i = 0; i < num_dimensions; ++i) { - type = std::make_shared(std::move(type)); + result = std::make_shared(result); if (is_nullable) { // wrap array with nullable - type = make_nullable(type); + result = make_nullable(result); } } - return type; + return result; } DataTypePtr get_base_type_of_array(const DataTypePtr& type) { @@ -149,6 +151,63 @@ class FieldVisitorToNumberOfDimensions : public StaticVisitor { } }; +// Visitor that allows to get type of scalar field +// but exclude fields contain complex field.This is a faster version +// for FieldVisitorToScalarType which does not support complex field. +class SimpleFieldVisitorToScalarType : public StaticVisitor { +public: + size_t operator()(const Array& x) { + throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "Array type is not supported"); + } + size_t operator()(const UInt64& x) { + if (x <= std::numeric_limits::max()) { + type = TypeIndex::Int8; + } else if (x <= std::numeric_limits::max()) { + type = TypeIndex::Int16; + } else if (x <= std::numeric_limits::max()) { + type = TypeIndex::Int32; + } else { + type = TypeIndex::Int64; + } + return 1; + } + size_t operator()(const Int64& x) { + if (x <= std::numeric_limits::max() && x >= std::numeric_limits::min()) { + type = TypeIndex::Int8; + } else if (x <= std::numeric_limits::max() && + x >= std::numeric_limits::min()) { + type = TypeIndex::Int16; + } else if (x <= std::numeric_limits::max() && + x >= std::numeric_limits::min()) { + type = TypeIndex::Int32; + } else { + type = TypeIndex::Int64; + } + return 1; + } + size_t operator()(const JsonbField& x) { + type = TypeIndex::JSONB; + return 1; + } + size_t operator()(const Null&) { + have_nulls = true; + return 1; + } + template + size_t operator()(const T&) { + type = TypeId>::value; + return 1; + } + void get_scalar_type(TypeIndex* data_type) const { *data_type = type; } + bool contain_nulls() const { return have_nulls; } + + bool need_convert_field() const { return false; } + +private: + TypeIndex type = TypeIndex::Nothing; + bool have_nulls; +}; + /// Visitor that allows to get type of scalar field /// or least common type of scalars in array. /// More optimized version of FieldToDataType. @@ -208,8 +267,10 @@ class FieldVisitorToScalarType : public StaticVisitor { type_indexes.insert(TypeId>::value); return 0; } - void get_scalar_type(DataTypePtr* type) const { - get_least_supertype(type_indexes, type); + void get_scalar_type(TypeIndex* type) const { + DataTypePtr data_type; + get_least_supertype(type_indexes, &data_type); + *type = data_type->get_type_id(); } bool contain_nulls() const { return have_nulls; } bool need_convert_field() const { return field_types.size() > 1; } @@ -221,20 +282,30 @@ class FieldVisitorToScalarType : public StaticVisitor { }; } // namespace -void get_field_info(const Field& field, FieldInfo* info) { - FieldVisitorToScalarType to_scalar_type_visitor; + +template +void get_field_info_impl(const Field& field, FieldInfo* info) { + Visitor to_scalar_type_visitor; apply_visitor(to_scalar_type_visitor, field); - DataTypePtr type = nullptr; - to_scalar_type_visitor.get_scalar_type(&type); + TypeIndex type_id; + to_scalar_type_visitor.get_scalar_type(&type_id); // array item's dimension may missmatch, eg. [1, 2, [1, 2, 3]] *info = { - type, + type_id, to_scalar_type_visitor.contain_nulls(), to_scalar_type_visitor.need_convert_field(), apply_visitor(FieldVisitorToNumberOfDimensions(), field), }; } +void get_field_info(const Field& field, FieldInfo* info) { + if (field.is_complex_field()) { + get_field_info_impl(field, info); + } else { + get_field_info_impl(field, info); + } +} + ColumnObject::Subcolumn::Subcolumn(MutableColumnPtr&& data_, DataTypePtr type, bool is_nullable_, bool is_root_) : least_common_type(type), is_nullable(is_nullable_), is_root(is_root_) { @@ -285,8 +356,8 @@ void ColumnObject::Subcolumn::add_new_column_part(DataTypePtr type) { } void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) { - auto base_type = std::move(info.scalar_type); - if (is_nothing(base_type)) { + auto base_type = WhichDataType(info.scalar_type_id); + if (base_type.is_nothing()) { insertDefault(); return; } @@ -295,7 +366,7 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) { if (is_nothing(least_common_type.get_base())) { column_dim = value_dim; } - if (is_nothing(base_type)) { + if (base_type.is_nothing()) { value_dim = column_dim; } bool type_changed = false; @@ -305,29 +376,30 @@ void ColumnObject::Subcolumn::insert(Field field, FieldInfo info) { "Dimension of types mismatched between inserted value and column, " "expected:{}, but meet:{} for type:{}", column_dim, value_dim, least_common_type.get()->get_name()); - base_type = std::make_shared(); + base_type = MOST_COMMON_TYPE_ID; value_dim = 0; type_changed = true; } - if (is_nullable && !is_nothing(base_type)) { - base_type = make_nullable(base_type); - } - - const auto& least_common_base_type = least_common_type.get_base(); if (data.empty()) { - add_new_column_part(create_array_of_type(std::move(base_type), value_dim, is_nullable)); - } else if (!least_common_base_type->equals(*base_type) && !is_nothing(base_type)) { - if (!schema_util::is_conversion_required_between_integers(*base_type, - *least_common_base_type)) { + add_new_column_part(create_array_of_type(base_type.idx, value_dim, is_nullable)); + } else if (least_common_type.get_type_id() != base_type.idx && !base_type.is_nothing()) { + if (schema_util::is_conversion_required_between_integers(base_type.idx, + least_common_type.get_type_id())) { + LOG_EVERY_N(INFO, 100) << "Conversion between " << getTypeName(base_type.idx) << " and " + << getTypeName(least_common_type.get_type_id()); + DataTypePtr base_data_type; + TypeIndex base_data_type_id; get_least_supertype( - DataTypes {std::move(base_type), least_common_base_type}, &base_type); + TypeIndexSet {base_type.idx, least_common_type.get_base_type_id()}, + &base_data_type); type_changed = true; + base_data_type_id = base_data_type->get_type_id(); if (is_nullable) { - base_type = make_nullable(base_type); + base_data_type = make_nullable(base_data_type); } - if (!least_common_base_type->equals(*base_type)) { + if (!least_common_type.get_base()->equals(*base_data_type)) { add_new_column_part( - create_array_of_type(std::move(base_type), value_dim, is_nullable)); + create_array_of_type(base_data_type_id, value_dim, is_nullable)); } } } @@ -578,6 +650,14 @@ ColumnObject::Subcolumn::LeastCommonType::LeastCommonType(DataTypePtr type_) if (!WhichDataType(type).is_nothing()) { least_common_type_serder = type->get_serde(); } + type_id = type->is_nullable() ? assert_cast(type.get()) + ->get_nested_type() + ->get_type_id() + : type->get_type_id(); + base_type_id = base_type->is_nullable() ? assert_cast(base_type.get()) + ->get_nested_type() + ->get_type_id() + : base_type->get_type_id(); } ColumnObject::ColumnObject(bool is_nullable_, bool create_root_) @@ -677,14 +757,12 @@ void ColumnObject::try_insert(const Field& field) { return; } const auto& object = field.get(); - phmap::flat_hash_set inserted; size_t old_size = size(); for (const auto& [key_str, value] : object) { PathInData key; if (!key_str.empty()) { key = PathInData(key_str); } - inserted.insert(key_str); if (!has_subcolumn(key)) { bool succ = add_sub_column(key, old_size); if (!succ) { @@ -700,7 +778,7 @@ void ColumnObject::try_insert(const Field& field) { subcolumn->insert(value); } for (auto& entry : subcolumns) { - if (!inserted.contains(entry->path.get_path())) { + if (old_size == entry->data.size()) { entry->data.insertDefault(); } } @@ -749,16 +827,6 @@ Status ColumnObject::try_insert_indices_from(const IColumn& src, const int* indi return Status::OK(); } -FieldInfo ColumnObject::Subcolumn::get_subcolumn_field_info() const { - const auto& base_type = least_common_type.get_base(); - return FieldInfo { - .scalar_type = base_type, - .have_nulls = base_type->is_nullable(), - .need_convert = false, - .num_dimensions = least_common_type.get_dimensions(), - }; -} - void ColumnObject::insert_range_from(const IColumn& src, size_t start, size_t length) { #ifndef NDEBUG check_consistency(); @@ -809,6 +877,33 @@ const ColumnObject::Subcolumn* ColumnObject::get_subcolumn(const PathInData& key return &node->data; } +const ColumnObject::Subcolumn* ColumnObject::get_subcolumn_with_cache(const PathInData& key, + size_t key_index) const { + // Optimization by caching the order of fields (which is almost always the same) + // and a quick check to match the next expected field, instead of searching the hash table. + if (_prev_positions.size() > key_index && _prev_positions[key_index].second != nullptr && + key == _prev_positions[key_index].first) { + return _prev_positions[key_index].second; + } + const auto* subcolumn = get_subcolumn(key); + if (key_index >= _prev_positions.size()) { + _prev_positions.resize(key_index + 1); + } + if (subcolumn != nullptr) { + _prev_positions[key_index] = std::make_pair(key, subcolumn); + } + return subcolumn; +} + +ColumnObject::Subcolumn* ColumnObject::get_subcolumn(const PathInData& key, size_t key_index) { + return const_cast(get_subcolumn_with_cache(key, key_index)); +} + +const ColumnObject::Subcolumn* ColumnObject::get_subcolumn(const PathInData& key, + size_t key_index) const { + return get_subcolumn_with_cache(key, key_index); +} + ColumnObject::Subcolumn* ColumnObject::get_subcolumn(const PathInData& key) { const auto* node = subcolumns.find_leaf(key); if (node == nullptr) { @@ -1238,6 +1333,7 @@ void ColumnObject::finalize(bool ignore_sparse) { } std::swap(subcolumns, new_subcolumns); doc_structure = nullptr; + _prev_positions.clear(); } void ColumnObject::finalize() { @@ -1356,6 +1452,7 @@ void ColumnObject::clear() { Subcolumns empty; std::swap(empty, subcolumns); num_rows = 0; + _prev_positions.clear(); } void ColumnObject::revise_to(int target_num_rows) { diff --git a/be/src/vec/columns/column_object.h b/be/src/vec/columns/column_object.h index 8573428ff2bba29..55abd534dd145bd 100644 --- a/be/src/vec/columns/column_object.h +++ b/be/src/vec/columns/column_object.h @@ -35,6 +35,7 @@ #include "common/status.h" #include "olap/tablet_schema.h" +#include "util/jsonb_document.h" #include "vec/columns/column.h" #include "vec/columns/subcolumn_tree.h" #include "vec/common/cow.h" @@ -62,8 +63,8 @@ namespace doris::vectorized { /// It allows to recreate field with different number /// of dimensions or nullability. struct FieldInfo { - /// The common type of of all scalars in field. - DataTypePtr scalar_type; + /// The common type id of of all scalars in field. + TypeIndex scalar_type_id; /// Do we have NULL scalar in field. bool have_nulls; /// If true then we have scalars with different types in array and @@ -72,6 +73,7 @@ struct FieldInfo { /// Number of dimension in array. 0 if field is scalar. size_t num_dimensions; }; + void get_field_info(const Field& field, FieldInfo* info); /** A column that represents object with dynamic set of subcolumns. * Subcolumns are identified by paths in document and are stored in @@ -91,6 +93,7 @@ class ColumnObject final : public COWHelper { // Using jsonb type as most common type, since it's adopted all types of json using MostCommonType = DataTypeJsonb; + constexpr static TypeIndex MOST_COMMON_TYPE_ID = TypeIndex::JSONB; class Subcolumn { public: Subcolumn() = default; @@ -147,8 +150,6 @@ class ColumnObject final : public COWHelper { /// Returns last inserted field. Field get_last_field() const; - FieldInfo get_subcolumn_field_info() const; - /// Returns single column if subcolumn in finalizes. /// Otherwise -- undefined behaviour. IColumn& get_finalized_column(); @@ -176,6 +177,10 @@ class ColumnObject final : public COWHelper { const DataTypePtr& get_base() const { return base_type; } + const TypeIndex& get_type_id() const { return type_id; } + + const TypeIndex& get_base_type_id() const { return base_type_id; } + size_t get_dimensions() const { return num_dimensions; } void remove_nullable() { type = doris::vectorized::remove_nullable(type); } @@ -185,6 +190,8 @@ class ColumnObject final : public COWHelper { private: DataTypePtr type; DataTypePtr base_type; + TypeIndex type_id; + TypeIndex base_type_id; size_t num_dimensions = 0; DataTypeSerDeSPtr least_common_type_serder; }; @@ -227,6 +234,10 @@ class ColumnObject final : public COWHelper { // used for quickly row store encoding ColumnPtr rowstore_column; + using SubColumnWithName = std::pair; + // Cached search results for previous row (keyed as index in JSON object) - used as a hint. + mutable std::vector _prev_positions; + public: static constexpr auto COLUMN_NAME_DUMMY = "_dummy"; @@ -289,6 +300,9 @@ class ColumnObject final : public COWHelper { // return null if not found const Subcolumn* get_subcolumn(const PathInData& key) const; + // return null if not found + const Subcolumn* get_subcolumn(const PathInData& key, size_t index_hint) const; + /** More efficient methods of manipulation */ [[noreturn]] IColumn& get_data() { LOG(FATAL) << "Not implemented method get_data()"; @@ -302,6 +316,12 @@ class ColumnObject final : public COWHelper { // return null if not found Subcolumn* get_subcolumn(const PathInData& key); + // return null if not found + Subcolumn* get_subcolumn(const PathInData& key, size_t index_hint); + + // return null if not found + const Subcolumn* get_subcolumn_with_cache(const PathInData& key, size_t index_hint) const; + void incr_num_rows() { ++num_rows; } void incr_num_rows(size_t n) { num_rows += n; } diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index 5c7a2f8482a7388..2f9e5ded2126987 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -109,36 +109,41 @@ Array create_empty_array_field(size_t num_dimensions) { return array; } -bool is_conversion_required_between_integers(const IDataType& lhs, const IDataType& rhs) { +size_t get_size_of_interger(TypeIndex type) { + switch (type) { + case TypeIndex::Int8: + return sizeof(int8_t); + case TypeIndex::Int16: + return sizeof(int16_t); + case TypeIndex::Int32: + return sizeof(int32_t); + case TypeIndex::Int64: + return sizeof(int64_t); + case TypeIndex::Int128: + return sizeof(int128_t); + case TypeIndex::UInt8: + return sizeof(uint8_t); + case TypeIndex::UInt16: + return sizeof(uint16_t); + case TypeIndex::UInt32: + return sizeof(uint32_t); + case TypeIndex::UInt64: + return sizeof(uint64_t); + case TypeIndex::UInt128: + return sizeof(uint128_t); + default: + LOG(FATAL) << "Unknown integer type: " << getTypeName(type); + return 0; + } +} + +bool is_conversion_required_between_integers(const TypeIndex& lhs, const TypeIndex& rhs) { WhichDataType which_lhs(lhs); WhichDataType which_rhs(rhs); bool is_native_int = which_lhs.is_native_int() && which_rhs.is_native_int(); bool is_native_uint = which_lhs.is_native_uint() && which_rhs.is_native_uint(); - return (is_native_int || is_native_uint) && - lhs.get_size_of_value_in_memory() <= rhs.get_size_of_value_in_memory(); -} - -bool is_conversion_required_between_integers(FieldType lhs, FieldType rhs) { - // We only support signed integers for semi-structure data at present - // TODO add unsigned integers - if (lhs == FieldType::OLAP_FIELD_TYPE_BIGINT) { - return !(rhs == FieldType::OLAP_FIELD_TYPE_TINYINT || - rhs == FieldType::OLAP_FIELD_TYPE_SMALLINT || - rhs == FieldType::OLAP_FIELD_TYPE_INT || rhs == FieldType::OLAP_FIELD_TYPE_BIGINT); - } - if (lhs == FieldType::OLAP_FIELD_TYPE_INT) { - return !(rhs == FieldType::OLAP_FIELD_TYPE_TINYINT || - rhs == FieldType::OLAP_FIELD_TYPE_SMALLINT || - rhs == FieldType::OLAP_FIELD_TYPE_INT); - } - if (lhs == FieldType::OLAP_FIELD_TYPE_SMALLINT) { - return !(rhs == FieldType::OLAP_FIELD_TYPE_TINYINT || - rhs == FieldType::OLAP_FIELD_TYPE_SMALLINT); - } - if (lhs == FieldType::OLAP_FIELD_TYPE_TINYINT) { - return !(rhs == FieldType::OLAP_FIELD_TYPE_TINYINT); - } - return true; + return (!is_native_int && !is_native_uint) || + get_size_of_interger(lhs) > get_size_of_interger(rhs); } Status cast_column(const ColumnWithTypeAndName& arg, const DataTypePtr& type, ColumnPtr* result) { diff --git a/be/src/vec/common/schema_util.h b/be/src/vec/common/schema_util.h index e6ed60480f5ac1d..078081593c549bc 100644 --- a/be/src/vec/common/schema_util.h +++ b/be/src/vec/common/schema_util.h @@ -34,6 +34,7 @@ #include "vec/columns/column_object.h" #include "vec/core/columns_with_type_and_name.h" #include "vec/core/field.h" +#include "vec/core/types.h" #include "vec/data_types/data_type.h" #include "vec/json/path_in_data.h" @@ -66,8 +67,7 @@ Status cast_column(const ColumnWithTypeAndName& arg, const DataTypePtr& type, Co /// If both of types are signed/unsigned integers and size of left field type /// is less than right type, we don't need to convert field, /// because all integer fields are stored in Int64/UInt64. -bool is_conversion_required_between_integers(const IDataType& lhs, const IDataType& rhs); -bool is_conversion_required_between_integers(FieldType lhs, FieldType rhs); +bool is_conversion_required_between_integers(const TypeIndex& lhs, const TypeIndex& rhs); struct ExtraInfo { // -1 indicates it's not a Frontend generated column diff --git a/be/src/vec/core/field.h b/be/src/vec/core/field.h index 356216e70741d3d..de2d544e7e0a8f0 100644 --- a/be/src/vec/core/field.h +++ b/be/src/vec/core/field.h @@ -493,6 +493,11 @@ class Field { return *this; } + bool is_complex_field() const { + return which == Types::Array || which == Types::Map || which == Types::Tuple || + which == Types::VariantMap; + } + Field& operator=(Field&& rhs) { if (this != &rhs) { if (which != rhs.which) { diff --git a/be/src/vec/json/parse2column.cpp b/be/src/vec/json/parse2column.cpp index cc3c649bb70faa2..0f61e24dad7c4b1 100644 --- a/be/src/vec/json/parse2column.cpp +++ b/be/src/vec/json/parse2column.cpp @@ -148,36 +148,28 @@ void parse_json_to_variant(IColumn& column, const char* src, size_t length, } auto& [paths, values] = *result; assert(paths.size() == values.size()); - phmap::flat_hash_set paths_set; - size_t num_rows = column_object.size(); + size_t old_num_rows = column_object.size(); for (size_t i = 0; i < paths.size(); ++i) { FieldInfo field_info; get_field_info(values[i], &field_info); - if (is_nothing(field_info.scalar_type)) { + if (WhichDataType(field_info.scalar_type_id).is_nothing()) { continue; } - if (!paths_set.insert(paths[i].get_path()).second) { - // return Status::DataQualityError( - // fmt::format("Object has ambiguous path {}, {}", paths[i].get_path())); - throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "Object has ambiguous path {}", - paths[i].get_path()); - } - - if (!column_object.has_subcolumn(paths[i])) { - column_object.add_sub_column(paths[i], num_rows); + if (column_object.get_subcolumn(paths[i], i) == nullptr) { + column_object.add_sub_column(paths[i], old_num_rows); } - auto* subcolumn = column_object.get_subcolumn(paths[i]); + auto* subcolumn = column_object.get_subcolumn(paths[i], i); if (!subcolumn) { throw doris::Exception(ErrorCode::INVALID_ARGUMENT, "Failed to find sub column {}", paths[i].get_path()); } - assert(subcolumn->size() == num_rows); + DCHECK_EQ(subcolumn->size(), old_num_rows); subcolumn->insert(std::move(values[i]), std::move(field_info)); } // /// Insert default values to missed subcolumns. const auto& subcolumns = column_object.get_subcolumns(); for (const auto& entry : subcolumns) { - if (!paths_set.contains(entry->path.get_path())) { + if (entry->data.size() == old_num_rows) { entry->data.insertDefault(); } } diff --git a/regression-test/suites/variant_p2/performance.groovy b/regression-test/suites/variant_p2/performance.groovy new file mode 100644 index 000000000000000..1f10dd90c042ff8 --- /dev/null +++ b/regression-test/suites/variant_p2/performance.groovy @@ -0,0 +1,36 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("regression_test_variant_performance", "p2"){ + sql """CREATE TABLE IF NOT EXISTS var_perf ( + k bigint, + v variant + + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY RANDOM BUCKETS 4 + properties("replication_num" = "1", "disable_auto_compaction" = "false"); + """ + sql """ + insert into var_perf + SELECT *, '{"field1":348,"field2":596,"field3":781,"field4":41,"field5":922,"field6":84,"field7":222,"field8":312,"field9":490,"field10":715,"field11":837,"field12":753,"field13":171,"field14":727,"field15":739,"field16":545,"field17":964,"field18":540,"field19":685,"field20":828,"field21":157,"field22":404,"field23":287,"field24":481,"field25":476,"field26":559,"field27":144,"field28":545,"field29":70,"field30":668,"field31":820,"field32":193,"field33":465,"field34":347,"field35":898,"field36":705,"field37":754,"field38":866,"field39":752,"field40":303,"field41":214,"field42":41,"field43":609,"field44":487,"field45":832,"field46":832,"field47":134,"field48":964,"field49":919,"field50":670,"field51":767,"field52":334,"field53":506,"field54":838,"field55":510,"field56":770,"field57":168,"field58":701,"field59":961,"field60":927,"field61":375,"field62":939,"field63":464,"field64":420,"field65":212,"field66":882,"field67":344,"field68":724,"field69":997,"field70":198,"field71":739,"field72":628,"field73":563,"field74":979,"field75":563,"field76":891,"field77":496,"field78":442,"field79":847,"field80":771,"field81":229,"field82":1023,"field83":184,"field84":563,"field85":980,"field86":191,"field87":426,"field88":527,"field89":945,"field90":552,"field91":454,"field92":728,"field93":631,"field94":191,"field95":148,"field96":679,"field97":955,"field98":934,"field99":258,"field100":442}' + from numbers("number" = "10000000") + union all + SELECT *, '{"field1":201,"field2":465,"field3":977,"field4":101112,"field5":131415,"field6":216,"field7":192021,"field8":822324,"field9":525627,"field10":928930,"field11":413233,"field12":243536,"field13":373839,"field14":404142,"field15":434445,"field16":1464748,"field17":495051,"field18":525354,"field19":565657,"field20":1585960,"field21":616263,"field22":646566,"field23":676869,"field24":707172,"field25":737475,"field26":767778,"field27":798081,"field28":828384,"field29":858687,"field30":888990,"field31":919293,"field32":949596,"field33":979899,"field34":100101,"field35":103104,"field36":106107,"field37":109110,"field38":112113,"field39":115116,"field40":118119,"field41":121122,"field42":124125,"field43":127128,"field44":130131,"field45":133134,"field46":136137,"field47":139140,"field48":142143,"field49":145146,"field50":148149,"field51":151152,"field52":154155,"field53":157158,"field54":160161,"field55":163164,"field56":166167,"field57":169170,"field58":172173,"field59":175176,"field60":178179,"field61":181182,"field62":184185,"field63":187188,"field64":190191,"field65":193194,"field66":196197,"field67":199200,"field68":202203,"field69":205206,"field70":208209,"field71":211212,"field72":214215,"field73":217218,"field74":220221,"field75":223224,"field76":226227,"field77":229230,"field78":232233,"field79":235236,"field80":238239,"field81":241242,"field82":244245,"field83":247248,"field84":250251,"field85":253254,"field86":256257,"field87":259260,"field88":262263,"field89":265266,"field90":268269,"field91":271272,"field92":274275,"field93":277278,"field94":280281,"field95":283284,"field96":286287,"field97":289290,"field98":292293,"field99":295296,"field100":298299}' + from numbers("number" = "10000000") + """ +} \ No newline at end of file From 7e967e53b83d451dff4dd14da4162a53aba398da Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Sat, 11 May 2024 14:05:22 +0800 Subject: [PATCH 004/111] Fix failed p2 hive statistics case. (#34663) --- .../test_hive_partition_column_analyze.groovy | 390 +++++++++--------- .../hive/test_hive_partition_statistic.groovy | 3 + 2 files changed, 200 insertions(+), 193 deletions(-) diff --git a/regression-test/suites/external_table_p2/hive/test_hive_partition_column_analyze.groovy b/regression-test/suites/external_table_p2/hive/test_hive_partition_column_analyze.groovy index 336e2e8b7844cce..d4b1fa3eca44be7 100644 --- a/regression-test/suites/external_table_p2/hive/test_hive_partition_column_analyze.groovy +++ b/regression-test/suites/external_table_p2/hive/test_hive_partition_column_analyze.groovy @@ -31,199 +31,203 @@ suite("test_hive_partition_column_analyze", "p2,external,hive,external_remote,ex """ logger.info("catalog " + catalog_name + " created") - // Test analyze table without init. - sql """analyze table ${catalog_name}.multi_partition.multi_partition_parquet (event_day) with sync""" - sql """analyze table ${catalog_name}.multi_partition.multi_partition_orc (event_day) with sync""" - - sql """switch ${catalog_name};""" - logger.info("switched to catalog " + catalog_name) - sql """use multi_partition;""" - def result = sql """show column stats multi_partition_parquet (event_day)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "event_day") - assertEquals(result[0][2], "3.83714205E8") - assertEquals(result[0][3], "99949.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "3.83714205E9") - assertEquals(result[0][6], "10.0") - assertEquals(result[0][7], "\'1749-09-24\'") - assertEquals(result[0][8], "\'2023-05-26\'") - - result = sql """show column stats multi_partition_orc (event_day)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "event_day") - assertEquals(result[0][2], "1.9007155E8") - assertEquals(result[0][3], "99949.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "1.9007155E9") - assertEquals(result[0][6], "10.0") - assertEquals(result[0][7], "\'1749-09-24\'") - assertEquals(result[0][8], "\'2023-05-26\'") - - sql """analyze table ${catalog_name}.partition_type.tinyint_partition (tinyint_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.smallint_partition (smallint_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.int_partition (int_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.bigint_partition (bigint_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.char_partition (char_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.varchar_partition (varchar_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.string_partition (string_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.date_partition (date_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.float_partition (float_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.double_partition (double_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.decimal_partition (decimal_part) with sync""" - sql """analyze table ${catalog_name}.partition_type.two_partition (part1, part2) with sync""" - - sql """use partition_type;""" - - result = sql """show column stats tinyint_partition (tinyint_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "tinyint_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "141474.0") - assertEquals(result[0][6], "1.0") - assertEquals(result[0][7], "1") - assertEquals(result[0][8], "100") - - result = sql """show column stats smallint_partition (smallint_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "smallint_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "282948.0") - assertEquals(result[0][6], "2.0") - assertEquals(result[0][7], "1") - assertEquals(result[0][8], "100") - - result = sql """show column stats int_partition (int_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "int_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "565896.0") - assertEquals(result[0][6], "4.0") - assertEquals(result[0][7], "1") - assertEquals(result[0][8], "100") - - result = sql """show column stats bigint_partition (bigint_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "bigint_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "1131792.0") - assertEquals(result[0][6], "8.0") - assertEquals(result[0][7], "1") - assertEquals(result[0][8], "100") - - result = sql """show column stats char_partition (char_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "char_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "101.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "2829480.0") - assertEquals(result[0][6], "20.0") - assertEquals(result[0][7], "\'1 \'") - assertEquals(result[0][8], "\'a \'") - - result = sql """show column stats varchar_partition (varchar_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "varchar_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "271630.0") - assertEquals(result[0][6], "1.9199994345250717") - assertEquals(result[0][7], "\'1\'") - assertEquals(result[0][8], "\'99\'") - - result = sql """show column stats string_partition (string_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "string_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "271630.0") - assertEquals(result[0][6], "1.9199994345250717") - assertEquals(result[0][7], "\'1\'") - assertEquals(result[0][8], "\'99\'") - - result = sql """show column stats date_partition (date_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "date_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "565896.0") - assertEquals(result[0][6], "4.0") - assertEquals(result[0][7], "\'2001-10-12\'") - assertEquals(result[0][8], "\'2100-10-12\'") - - result = sql """show column stats float_partition (float_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "float_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "565896.0") - assertEquals(result[0][6], "4.0") - assertEquals(result[0][7], "296.3103") - assertEquals(result[0][8], "32585.627") - - result = sql """show column stats double_partition (double_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "double_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "1131792.0") - assertEquals(result[0][6], "8.0") - assertEquals(result[0][7], "115.14474") - assertEquals(result[0][8], "32761.14458") - - result = sql """show column stats decimal_partition (decimal_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "decimal_part") - assertEquals(result[0][2], "141474.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "1131792.0") - assertEquals(result[0][6], "8.0") - assertEquals(result[0][7], "243.2868") - assertEquals(result[0][8], "32527.1543") - - sql """analyze table ${catalog_name}.partition_type.decimal_partition (decimal_part) with sync with sql""" - result = sql """show column stats decimal_partition (decimal_part)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "decimal_part") - assertEquals(result[0][2], "100000.0") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][5], "800000.0") - assertEquals(result[0][6], "8.0") - assertEquals(result[0][7], "243.2868") - assertEquals(result[0][8], "32527.1543") - - result = sql """show column stats two_partition (part1)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "part1") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][7], "1") - assertEquals(result[0][8], "100") - - result = sql """show column stats two_partition (part2)""" - assertEquals(result.size(), 1) - assertEquals(result[0][0], "part2") - assertEquals(result[0][3], "100.0") - assertEquals(result[0][4], "0.0") - assertEquals(result[0][7], "\'1\'") - assertEquals(result[0][8], "\'99\'") - + try { + sql """set global enable_get_row_count_from_file_list=true""" + // Test analyze table without init. + sql """analyze table ${catalog_name}.multi_partition.multi_partition_parquet (event_day) with sync""" + sql """analyze table ${catalog_name}.multi_partition.multi_partition_orc (event_day) with sync""" + + sql """switch ${catalog_name};""" + logger.info("switched to catalog " + catalog_name) + sql """use multi_partition;""" + def result = sql """show column stats multi_partition_parquet (event_day)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "event_day") + assertEquals(result[0][2], "3.83714205E8") + assertEquals(result[0][3], "99949.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "3.83714205E9") + assertEquals(result[0][6], "10.0") + assertEquals(result[0][7], "\'1749-09-24\'") + assertEquals(result[0][8], "\'2023-05-26\'") + + result = sql """show column stats multi_partition_orc (event_day)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "event_day") + assertEquals(result[0][2], "1.9007155E8") + assertEquals(result[0][3], "99949.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "1.9007155E9") + assertEquals(result[0][6], "10.0") + assertEquals(result[0][7], "\'1749-09-24\'") + assertEquals(result[0][8], "\'2023-05-26\'") + + sql """analyze table ${catalog_name}.partition_type.tinyint_partition (tinyint_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.smallint_partition (smallint_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.int_partition (int_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.bigint_partition (bigint_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.char_partition (char_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.varchar_partition (varchar_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.string_partition (string_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.date_partition (date_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.float_partition (float_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.double_partition (double_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.decimal_partition (decimal_part) with sync""" + sql """analyze table ${catalog_name}.partition_type.two_partition (part1, part2) with sync""" + + sql """use partition_type;""" + + result = sql """show column stats tinyint_partition (tinyint_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "tinyint_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "141474.0") + assertEquals(result[0][6], "1.0") + assertEquals(result[0][7], "1") + assertEquals(result[0][8], "100") + + result = sql """show column stats smallint_partition (smallint_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "smallint_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "282948.0") + assertEquals(result[0][6], "2.0") + assertEquals(result[0][7], "1") + assertEquals(result[0][8], "100") + + result = sql """show column stats int_partition (int_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "int_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "565896.0") + assertEquals(result[0][6], "4.0") + assertEquals(result[0][7], "1") + assertEquals(result[0][8], "100") + + result = sql """show column stats bigint_partition (bigint_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "bigint_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "1131792.0") + assertEquals(result[0][6], "8.0") + assertEquals(result[0][7], "1") + assertEquals(result[0][8], "100") + + result = sql """show column stats char_partition (char_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "char_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "101.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "2829480.0") + assertEquals(result[0][6], "20.0") + assertEquals(result[0][7], "\'1 \'") + assertEquals(result[0][8], "\'a \'") + + result = sql """show column stats varchar_partition (varchar_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "varchar_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "271630.0") + assertEquals(result[0][6], "1.9199994345250717") + assertEquals(result[0][7], "\'1\'") + assertEquals(result[0][8], "\'99\'") + + result = sql """show column stats string_partition (string_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "string_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "271630.0") + assertEquals(result[0][6], "1.9199994345250717") + assertEquals(result[0][7], "\'1\'") + assertEquals(result[0][8], "\'99\'") + + result = sql """show column stats date_partition (date_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "date_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "565896.0") + assertEquals(result[0][6], "4.0") + assertEquals(result[0][7], "\'2001-10-12\'") + assertEquals(result[0][8], "\'2100-10-12\'") + + result = sql """show column stats float_partition (float_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "float_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "565896.0") + assertEquals(result[0][6], "4.0") + assertEquals(result[0][7], "296.3103") + assertEquals(result[0][8], "32585.627") + + result = sql """show column stats double_partition (double_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "double_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "1131792.0") + assertEquals(result[0][6], "8.0") + assertEquals(result[0][7], "115.14474") + assertEquals(result[0][8], "32761.14458") + + result = sql """show column stats decimal_partition (decimal_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "decimal_part") + assertEquals(result[0][2], "141474.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "1131792.0") + assertEquals(result[0][6], "8.0") + assertEquals(result[0][7], "243.2868") + assertEquals(result[0][8], "32527.1543") + + sql """analyze table ${catalog_name}.partition_type.decimal_partition (decimal_part) with sync with sql""" + result = sql """show column stats decimal_partition (decimal_part)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "decimal_part") + assertEquals(result[0][2], "100000.0") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][5], "800000.0") + assertEquals(result[0][6], "8.0") + assertEquals(result[0][7], "243.2868") + assertEquals(result[0][8], "32527.1543") + + result = sql """show column stats two_partition (part1)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "part1") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][7], "1") + assertEquals(result[0][8], "100") + + result = sql """show column stats two_partition (part2)""" + assertEquals(result.size(), 1) + assertEquals(result[0][0], "part2") + assertEquals(result[0][3], "100.0") + assertEquals(result[0][4], "0.0") + assertEquals(result[0][7], "\'1\'") + assertEquals(result[0][8], "\'99\'") + } finally { + sql """set global enable_get_row_count_from_file_list=false""" + } sql """drop catalog ${catalog_name}""" } } diff --git a/regression-test/suites/external_table_p2/hive/test_hive_partition_statistic.groovy b/regression-test/suites/external_table_p2/hive/test_hive_partition_statistic.groovy index 73b6db1c6461567..6f398a62e3cd831 100644 --- a/regression-test/suites/external_table_p2/hive/test_hive_partition_statistic.groovy +++ b/regression-test/suites/external_table_p2/hive/test_hive_partition_statistic.groovy @@ -18,6 +18,8 @@ suite("test_hive_partition_statistic", "p2,external,hive,external_remote,external_remote_hive") { String enabled = context.config.otherConfigs.get("enableExternalHiveTest") if (enabled != null && enabled.equalsIgnoreCase("true")) { + logger.info("This feature has not been supported yet, skip it.") + /** String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") String catalog_name = "test_hive_partition_statistic" @@ -48,6 +50,7 @@ suite("test_hive_partition_statistic", "p2,external,hive,external_remote,externa order_qt_2 """select part_id, count, ndv, null_count, min, max from internal.__internal_schema.column_statistics where catalog_id='$ctlId' and part_id='event_day=1956-09-07'""" sql """drop catalog ${catalog_name}"""; + **/ } } From 876248aa4e382d6268b44811909e6fecc8002ed0 Mon Sep 17 00:00:00 2001 From: Mryange <59914473+Mryange@users.noreply.github.com> Date: Sat, 11 May 2024 14:17:43 +0800 Subject: [PATCH 005/111] [fix](function) json_object can not input null value (#34591) --- be/src/vec/functions/function_json.cpp | 24 ++++++++++++++++++- .../test_query_json_object.groovy | 4 ++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/be/src/vec/functions/function_json.cpp b/be/src/vec/functions/function_json.cpp index 4418561d8814201..e7c2fc1781dfc5f 100644 --- a/be/src/vec/functions/function_json.cpp +++ b/be/src/vec/functions/function_json.cpp @@ -619,6 +619,7 @@ struct ExecuteReducer { struct FunctionJsonArrayImpl { static constexpr auto name = "json_array"; + static constexpr auto must_not_null = false; template using Reducer = ExecuteReducer; @@ -654,7 +655,7 @@ struct FunctionJsonArrayImpl { struct FunctionJsonObjectImpl { static constexpr auto name = "json_object"; - + static constexpr auto must_not_null = true; template using Reducer = ExecuteReducer; @@ -743,6 +744,9 @@ class FunctionJsonAlwaysNotNullable : public IFunction { data_columns.push_back(assert_cast(column_ptrs.back().get())); } } + if (SpecificImpl::must_not_null) { + RETURN_IF_ERROR(check_keys_all_not_null(nullmaps, input_rows_count, arguments.size())); + } execute(data_columns, *assert_cast(result_column.get()), input_rows_count, nullmaps); block.get_by_position(result).column = std::move(result_column); @@ -774,6 +778,24 @@ class FunctionJsonAlwaysNotNullable : public IFunction { result_column.insert_data(buf.GetString(), buf.GetSize()); } } + + static Status check_keys_all_not_null(const std::vector& nullmaps, int size, + size_t args) { + for (int i = 0; i < args; i += 2) { + const auto* null_map = nullmaps[i]; + if (null_map) { + const bool not_null_num = + simd::count_zero_num((int8_t*)null_map->get_data().data(), size); + if (not_null_num < size) { + return Status::InternalError( + "function {} can not input null value , JSON documents may not contain " + "NULL member names.", + name); + } + } + } + return Status::OK(); + } }; struct FunctionJsonQuoteImpl { diff --git a/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_object.groovy b/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_object.groovy index 2a12d69ffe0bc2d..e4ebe8392760dbe 100644 --- a/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_object.groovy +++ b/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_object.groovy @@ -41,5 +41,9 @@ suite("test_query_json_object", "query") { sql "insert into ${tableName} values(4,null,null,'test','2022-01-01 11:11:11');" sql "insert into ${tableName} values(5,1,true,'test','2022-01-01 11:11:11');" qt_sql1 "select json_object('k0',k0,'k1',k1,'k2',k2,'k3',k3,'k4',k4,'k5', null,'k6','k6') from ${tableName} order by k0;" + test { + sql """select k0,json_object(k3,123) from ${tableName} order by k0;""" + exception "[CANCELLED][INTERNAL_ERROR] function json_object can not input null value , JSON documents may not contain NULL member names." + } sql "DROP TABLE ${tableName};" } From 6f91e9cc4dcb670f34fa381ac91a7c3dcf424c6c Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Sat, 11 May 2024 17:23:19 +0800 Subject: [PATCH 006/111] [fix](test) fix s3 load test failed (#34671) --- .../broker_load/test_s3_load_with_load_parallelism.groovy | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/regression-test/suites/load_p0/broker_load/test_s3_load_with_load_parallelism.groovy b/regression-test/suites/load_p0/broker_load/test_s3_load_with_load_parallelism.groovy index be0d7f9c34822e5..b1d1782b5012188 100644 --- a/regression-test/suites/load_p0/broker_load/test_s3_load_with_load_parallelism.groovy +++ b/regression-test/suites/load_p0/broker_load/test_s3_load_with_load_parallelism.groovy @@ -45,8 +45,10 @@ suite("test_s3_load_with_load_parallelism", "load_p0") { // attributesList.add(new LoadAttributes("s3://doris-build-1308700295/regression/load/data/enclose_not_trim_quotes.csv", // "${tableName}", "", "COLUMNS TERMINATED BY \",\"", "FORMAT AS \"CSV\"", "(k1,k2,v1,v2,v3,v4)", // "PROPERTIES (\"enclose\" = \"\\\"\", \"escape\" = \"\\\\\")").addProperties("trim_double_quotes", "false")) + + def bucket = getS3BucketName() - attributesList.add(new LoadAttributes("s3://test-for-student-1308700295/regression/segcompaction/segcompaction.orc", + attributesList.add(new LoadAttributes("s3://${bucket}/regression/segcompaction/segcompaction.orc", "${tableName}", "", "", "FORMAT AS \"ORC\"", "(col_0, col_1, col_2, col_3, col_4, col_5, col_6, col_7, col_8, col_9, col_10, col_11, col_12, col_13, col_14, col_15, col_16, col_17, col_18, col_19, col_20, col_21, col_22, col_23, col_24, col_25, col_26, col_27, col_28, col_29, col_30, col_31, col_32, col_33, col_34, col_35, col_36, col_37, col_38, col_39, col_40, col_41, col_42, col_43, col_44, col_45, col_46, col_47, col_48, col_49)", "").addProperties("load_parallelism", "3")) def ak = getS3AK() From e2614d453a802eafa953748468dcb9ca291e8a96 Mon Sep 17 00:00:00 2001 From: walter Date: Sat, 11 May 2024 19:54:59 +0800 Subject: [PATCH 007/111] [case](regression) Add hdfs backup restore case (#34716) --- .../test_backup_restore_hdfs.groovy | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 regression-test/suites/backup_restore/test_backup_restore_hdfs.groovy diff --git a/regression-test/suites/backup_restore/test_backup_restore_hdfs.groovy b/regression-test/suites/backup_restore/test_backup_restore_hdfs.groovy new file mode 100644 index 000000000000000..239fbde6b53de1f --- /dev/null +++ b/regression-test/suites/backup_restore/test_backup_restore_hdfs.groovy @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_backup_restore_hdfs", "backup_restore") { + String suiteName = "test_backup_restore_hdfs" + String repoName = "${suiteName}_repo" + String dbName = "${suiteName}_db" + String tableName = "${suiteName}_table" + String snapshotName = "${suiteName}_snapshot_" + UUID.randomUUID().toString().replace('-', '') + + if (!enableHdfs()) { + logger.info("hdfs is not enabled, skip this case") + return + } + + def syncer = getSyncer() + syncer.createHdfsRepository(repoName) + + // create repository for fs broker: + // + // String hdfsFs = getHdfsFs() + // String hdfsUser = getHdfsUser() + // String dataDir = getHdfsDataDir() + // String prefix = "test" + + // try_sql "DROP REPOSITORY `${repoName}`" + // sql """ + // CREATE REPOSITORY `${repoName}` + // WITH broker `broker_0` + // ON LOCATION "${dataDir}/${prefix}/${repoName}" + // PROPERTIES + // ( + // "fs.defaultFS" = "${hdfsFs}", + // "hadoop.username" = "${hdfsUser}" + // ) + // """ + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName}" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0") + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + + List values = [] + for (int i = 1; i <= 10; ++i) { + values.add("(${i}, ${i})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + syncer.waitSnapshotFinish(dbName) + + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "TRUNCATE TABLE ${dbName}.${tableName}" + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON ( `${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + sql "DROP TABLE ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" + sql "DROP REPOSITORY `${repoName}`" +} + From b3b848f862d857d5fa439ff808c59aabd56e9cf7 Mon Sep 17 00:00:00 2001 From: jakevin Date: Mon, 13 May 2024 10:17:29 +0800 Subject: [PATCH 008/111] [feature](Nereids): eliminate useless project (#34611) --- .../processor/post/ColumnPruningPostProcessor.java | 11 +++++++++-- .../doris/nereids/trees/plans/algebra/Project.java | 10 ++++++++++ .../nereids/trees/plans/logical/LogicalProject.java | 10 ---------- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ColumnPruningPostProcessor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ColumnPruningPostProcessor.java index ff362b7bc9e0105..e8c3cf4a6ac8670 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ColumnPruningPostProcessor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/ColumnPruningPostProcessor.java @@ -40,9 +40,16 @@ }) public class ColumnPruningPostProcessor extends PlanPostProcessor { @Override - public PhysicalProject visitPhysicalProject(PhysicalProject project, CascadesContext ctx) { + public Plan visitPhysicalProject(PhysicalProject project, CascadesContext ctx) { Plan child = project.child(); Plan newChild = child.accept(this, ctx); + if (project.isAllSlots()) { + Set projects = project.getProjects().stream().map(Slot.class::cast).collect(Collectors.toSet()); + Set outputSet = newChild.getOutputSet(); + if (outputSet.equals(projects)) { + return ((AbstractPhysicalPlan) newChild).copyStatsAndGroupIdFrom(project); + } + } if (newChild instanceof AbstractPhysicalJoin) { AbstractPhysicalJoin join = (AbstractPhysicalJoin) newChild; Plan left = join.left(); @@ -91,7 +98,7 @@ public PhysicalProject visitPhysicalProject(PhysicalProject proj } if (newLeft != left || newRight != right) { - return (PhysicalProject) project.withChildren(join.withChildren(newLeft, newRight)); + return project.withChildren(join.withChildren(newLeft, newRight)); } else { return project; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java index 7fa62f7628fc2dc..15b095bee3bd288 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/algebra/Project.java @@ -117,4 +117,14 @@ static List findProject( return expr; }); } + + /** isAllSlots */ + default boolean isAllSlots() { + for (NamedExpression project : getProjects()) { + if (!project.isSlot()) { + return false; + } + } + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java index de23bc6f5b07c8d..24d938636aa0cfe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalProject.java @@ -114,16 +114,6 @@ public List getExcepts() { return excepts; } - /** isAllSlots */ - public boolean isAllSlots() { - for (NamedExpression project : projects) { - if (!project.isSlot()) { - return false; - } - } - return true; - } - @Override public List computeOutput() { Builder slots = ImmutableList.builderWithExpectedSize(projects.size()); From 5012ddd87a2a671bc2aed465732adc2def929993 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 13 May 2024 12:05:22 +0800 Subject: [PATCH 009/111] [fix](Nereids) fix sql cache return old value when truncate partition (#34698) 1. fix sql cache return old value when truncate partition 2. use expire_sql_cache_in_fe_second to control the expire time of the sql cache which in the NereidsSqlCacheManager --- .../java/org/apache/doris/common/Config.java | 8 +++- .../java/org/apache/doris/catalog/Env.java | 4 +- .../doris/common/NereidsSqlCacheManager.java | 38 +++++++++---------- .../apache/doris/qe/cache/CacheAnalyzer.java | 2 +- .../cache/parse_sql_from_sql_cache.groovy | 38 ++++++++++++++++++- 5 files changed, 65 insertions(+), 25 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 6fc20578ec00040..d44ff4a0dd4db0b 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1306,12 +1306,18 @@ public class Config extends ConfigBase { * Minimum interval between last version when caching results, * This parameter distinguishes between offline and real-time updates */ + @ConfField(mutable = true, masterOnly = false) + public static int cache_last_version_interval_second = 30; + + /** + * Expire sql sql in frontend time + */ @ConfField( mutable = true, masterOnly = false, callbackClassString = "org.apache.doris.common.NereidsSqlCacheManager$UpdateConfig" ) - public static int cache_last_version_interval_second = 30; + public static int expire_sql_cache_in_fe_second = 300; /** * Set the maximum number of rows that can be cached diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 4839769e0f889d6..d27ae1473235686 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -771,9 +771,7 @@ public Env(boolean isCheckpointCatalog) { this.mtmvService = new MTMVService(); this.insertOverwriteManager = new InsertOverwriteManager(); this.dnsCache = new DNSCache(); - this.sqlCacheManager = new NereidsSqlCacheManager( - Config.sql_cache_manage_num, Config.cache_last_version_interval_second - ); + this.sqlCacheManager = new NereidsSqlCacheManager(); } public static void destroyCheckpoint() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java index 8989375c07f7d21..cf6280650f06f64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/NereidsSqlCacheManager.java @@ -74,8 +74,11 @@ public class NereidsSqlCacheManager { // value: SqlCacheContext private volatile Cache sqlCaches; - public NereidsSqlCacheManager(int sqlCacheNum, long cacheIntervalSeconds) { - sqlCaches = buildSqlCaches(sqlCacheNum, cacheIntervalSeconds); + public NereidsSqlCacheManager() { + sqlCaches = buildSqlCaches( + Config.sql_cache_manage_num, + Config.expire_sql_cache_in_fe_second + ); } public static synchronized void updateConfig() { @@ -90,22 +93,24 @@ public static synchronized void updateConfig() { Cache sqlCaches = buildSqlCaches( Config.sql_cache_manage_num, - Config.cache_last_version_interval_second + Config.expire_sql_cache_in_fe_second ); sqlCaches.putAll(sqlCacheManager.sqlCaches.asMap()); sqlCacheManager.sqlCaches = sqlCaches; } - private static Cache buildSqlCaches(int sqlCacheNum, long cacheIntervalSeconds) { - sqlCacheNum = sqlCacheNum < 0 ? 100 : sqlCacheNum; - cacheIntervalSeconds = cacheIntervalSeconds < 0 ? 30 : cacheIntervalSeconds; - - return Caffeine.newBuilder() - .maximumSize(sqlCacheNum) - .expireAfterAccess(Duration.ofSeconds(cacheIntervalSeconds)) + private static Cache buildSqlCaches(int sqlCacheNum, long expireAfterAccessSeconds) { + Caffeine cacheBuilder = Caffeine.newBuilder() // auto evict cache when jvm memory too low - .softValues() - .build(); + .softValues(); + if (sqlCacheNum > 0) { + cacheBuilder = cacheBuilder.maximumSize(sqlCacheNum); + } + if (expireAfterAccessSeconds > 0) { + cacheBuilder = cacheBuilder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSeconds)); + } + + return cacheBuilder.build(); } /** tryAddFeCache */ @@ -237,9 +242,6 @@ public Optional tryParseSql(ConnectContext connectContext, Stri } private boolean tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext) { - long latestPartitionTime = sqlCacheContext.getLatestPartitionTime(); - long latestPartitionVersion = sqlCacheContext.getLatestPartitionVersion(); - if (sqlCacheContext.hasUnsupportedTables()) { return true; } @@ -255,7 +257,7 @@ private boolean tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext) { long cacheTableTime = scanTable.latestTimestamp; long currentTableVersion = olapTable.getVisibleVersion(); long cacheTableVersion = scanTable.latestVersion; - // some partitions have been dropped, or delete or update or insert rows into new partition? + // some partitions have been dropped, or delete or updated or replaced, or insert rows into new partition? if (currentTableTime > cacheTableTime || (currentTableTime == cacheTableTime && currentTableVersion > cacheTableVersion)) { return true; @@ -264,9 +266,7 @@ private boolean tablesOrDataChanged(Env env, SqlCacheContext sqlCacheContext) { for (Long scanPartitionId : scanTable.getScanPartitions()) { Partition partition = olapTable.getPartition(scanPartitionId); // partition == null: is this partition truncated? - if (partition == null || partition.getVisibleVersionTime() > latestPartitionTime - || (partition.getVisibleVersionTime() == latestPartitionTime - && partition.getVisibleVersion() > latestPartitionVersion)) { + if (partition == null) { return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java index 85f37094b027255..47fccfcd37a9c9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/cache/CacheAnalyzer.java @@ -701,11 +701,11 @@ private CacheTable buildCacheTableForOlapScanNode(OlapScanNode node) { scanTables.add(scanTable); for (Long partitionId : node.getSelectedPartitionIds()) { Partition partition = olapTable.getPartition(partitionId); + scanTable.addScanPartition(partitionId); if (partition.getVisibleVersionTime() >= cacheTable.latestPartitionTime) { cacheTable.latestPartitionId = partition.getId(); cacheTable.latestPartitionTime = partition.getVisibleVersionTime(); cacheTable.latestPartitionVersion = partition.getVisibleVersion(); - scanTable.addScanPartition(partitionId); } } return cacheTable; diff --git a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy index b672e30cb62ae77..b75d35f8c51abac 100644 --- a/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy +++ b/regression-test/suites/nereids_p0/cache/parse_sql_from_sql_cache.groovy @@ -32,7 +32,6 @@ suite("parse_sql_from_sql_cache") { } } - sql "ADMIN SET FRONTEND CONFIG ('cache_last_version_interval_second' = '10')" combineFutures( @@ -656,6 +655,43 @@ suite("parse_sql_from_sql_cache") { assertHasCache "select * from test_use_plan_cache20 where id=999" def result6 = sql "select * from test_use_plan_cache20 where id=999" assertTrue(result6.isEmpty()) + }), + extraThread("test_truncate_partition", { + sql "drop table if exists test_use_plan_cache21" + sql """create table test_use_plan_cache21 ( + id int, + dt int + ) + partition by range(dt) + ( + partition dt1 values [('1'), ('2')), + partition dt2 values [('2'), ('3')) + ) + distributed by hash(id) + properties('replication_num'='1')""" + + + + sql "insert into test_use_plan_cache21 values('2', '2')" + sleep(100) + sql "insert into test_use_plan_cache21 values('1', '1')" + + // after partition changed 10s, the sql cache can be used + sleep(10000) + + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + sql "set enable_sql_cache=true" + + assertNoCache "select * from test_use_plan_cache21" + def result1 = sql "select * from test_use_plan_cache21" + assertTrue(result1.size() == 2) + assertHasCache "select * from test_use_plan_cache21" + + sql "truncate table test_use_plan_cache21 partition dt2" + assertNoCache "select * from test_use_plan_cache21" + def result2 = sql "select * from test_use_plan_cache21" + assertTrue(result2.size() == 1) }) ).get() } From c71d0b6b223d092d98bc42b3b81f3d5636e9e63e Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Mon, 13 May 2024 14:42:04 +0800 Subject: [PATCH 010/111] [fix](Nereids) cast from json should always nullable (#34707) --- .../doris/nereids/trees/expressions/Cast.java | 2 ++ .../suites/query_p0/cast/test_cast.groovy | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java index 76cb1826e551135..62bd3639b5a244b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Cast.java @@ -79,6 +79,8 @@ public boolean nullable() { return true; } else if (!childDataType.isTimeLikeType() && targetType.isTimeLikeType()) { return true; + } else if (childDataType.isJsonType()) { + return true; } else { return child().nullable(); } diff --git a/regression-test/suites/query_p0/cast/test_cast.groovy b/regression-test/suites/query_p0/cast/test_cast.groovy index c0df52b39e7ddf8..2fe4d53eb80a099 100644 --- a/regression-test/suites/query_p0/cast/test_cast.groovy +++ b/regression-test/suites/query_p0/cast/test_cast.groovy @@ -145,4 +145,22 @@ suite('test_cast', "arrow_flight_sql") { sql "select * from ${tbl} where case when k0 = 101 then 'true' else 1 end" result([[101]]) } + + sql "DROP TABLE IF EXISTS test_json" + sql """ + CREATE TABLE IF NOT EXISTS test_json ( + id INT not null, + j JSON not null + ) + DUPLICATE KEY(id) + DISTRIBUTED BY HASH(id) BUCKETS 10 + PROPERTIES("replication_num" = "1"); + """ + + sql """ + INSERT INTO test_json VALUES(26, '{"k1":"v1", "k2": 200}'); + """ + sql "sync" + sql "Select cast(j as int) from test_json" + sql "DROP TABLE IF EXISTS test_json" } From 46bf43130f78de6cbdce2bbd597bc40822e94b8e Mon Sep 17 00:00:00 2001 From: Mryange <59914473+Mryange@users.noreply.github.com> Date: Mon, 13 May 2024 15:05:53 +0800 Subject: [PATCH 011/111] [test](case) error format case in test_query_json_object (#34722) error format case in test_query_json_object --- .../sql_functions/json_function/test_query_json_object.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_object.groovy b/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_object.groovy index e4ebe8392760dbe..c3b1e35b5e663a8 100644 --- a/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_object.groovy +++ b/regression-test/suites/query_p0/sql_functions/json_function/test_query_json_object.groovy @@ -43,7 +43,7 @@ suite("test_query_json_object", "query") { qt_sql1 "select json_object('k0',k0,'k1',k1,'k2',k2,'k3',k3,'k4',k4,'k5', null,'k6','k6') from ${tableName} order by k0;" test { sql """select k0,json_object(k3,123) from ${tableName} order by k0;""" - exception "[CANCELLED][INTERNAL_ERROR] function json_object can not input null value , JSON documents may not contain NULL member names." + exception "function json_object can not input null value , JSON documents may not contain NULL member names." } sql "DROP TABLE ${tableName};" } From 1545d96617c6bf5cac7605cbc9884f777e5ab650 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Mon, 13 May 2024 15:12:12 +0800 Subject: [PATCH 012/111] [WIP](test) remove enable_nereids_planner in regression cases (part 4) (#34642) before PR are #34417 #34490 #34558 --- .../test_group_having_alias.out | 2 - ...t_array_functions_array_intersect_sort.out | 110 ++---------------- .../javaudf_p0/test_javaudf_with_decimal.out | 8 +- .../correctness_p0/test_cast_decimal.groovy | 14 --- .../test_group_having_alias.groovy | 20 ---- ...rray_functions_array_intersect_sort.groovy | 97 +++------------ ..._with_complex_type_insertinto_doris.groovy | 1 - .../tvf/test_tvf_view_count_p2.groovy | 16 +-- .../test_javaudf_with_decimal.groovy | 37 +----- ...ray_with_single_row_max_than_uint16.groovy | 1 - 10 files changed, 32 insertions(+), 274 deletions(-) diff --git a/regression-test/data/correctness_p0/test_group_having_alias.out b/regression-test/data/correctness_p0/test_group_having_alias.out index 727040875a76443..0b711d92ad6d6fb 100644 --- a/regression-test/data/correctness_p0/test_group_having_alias.out +++ b/regression-test/data/correctness_p0/test_group_having_alias.out @@ -10,8 +10,6 @@ 202245 202245 --- !sql -- - -- !sql -- 202245 3 diff --git a/regression-test/data/datatype_p0/nested_types/query/array_functions/test_array_functions_array_intersect_sort.out b/regression-test/data/datatype_p0/nested_types/query/array_functions/test_array_functions_array_intersect_sort.out index 0127b7b25db4eec..0ede6c3b7a226e5 100644 --- a/regression-test/data/datatype_p0/nested_types/query/array_functions/test_array_functions_array_intersect_sort.out +++ b/regression-test/data/datatype_p0/nested_types/query/array_functions/test_array_functions_array_intersect_sort.out @@ -1,109 +1,43 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !oLd_sql -- +-- !nereid_sql -- array_intersect-array-sort --- !old_sql -- -[1, 2] -[1, 2] -[1] -[] - --- !old_sql -- -[] -[] -[] -[] - --- !old_sql -- -[] -[] -[] -[] - --- !old_sql -- -[1, 2] -[1, 2] -[1] -[] - --- !old_sql -- -[1, 2] -[1, 2] -[1, 2] -[1, 2] - --- !old_sql -- -[] -[] -[] -[] - --- !old_sql -- +-- !nereid_sql -- [1, 2] [1, 2] [1] [] --- !old_sql -- +-- !nereid_sql -- [] [] [] [] --- !old_sql -- +-- !nereid_sql -- [] [] [] [] --- !old_sql -- +-- !nereid_sql -- [1, 2] [1, 2] [1] [] --- !old_sql -- +-- !nereid_sql -- [1, 2] [1, 2] [1, 2] [1, 2] --- !old_sql -- -[] -[] -[] -[] - --- !old_sql -- -[] - --- !old_sql -- -[] - --- !old_sql -- -["a"] - --- !old_sql -- +-- !nereid_sql -- [] - --- !old_sql -- -[1] - --- !old_sql -- [] - --- !old_sql -- -[2] - --- !old_sql -- [] - --- !old_sql -- [] --- !nereid_sql -- -array_intersect-array-sort - -- !nereid_sql -- [1, 2] [1, 2] @@ -141,53 +75,29 @@ array_intersect-array-sort [] -- !nereid_sql -- -[1, 2] -[1, 2] -[1] [] -- !nereid_sql -- [] -[] -[] -[] -- !nereid_sql -- -[] -[] -[] -[] +["a"] -- !nereid_sql -- -[1, 2] -[1, 2] -[1] [] -- !nereid_sql -- -[1, 2] -[1, 2] -[1, 2] -[1, 2] +[1] -- !nereid_sql -- [] -[] -[] -[] -- !nereid_sql -- -[] +[2] -- !nereid_sql -- [] --- !nereid_sql -- -["a"] - -- !nereid_sql -- [] --- !nereid_sql -- -[1] - diff --git a/regression-test/data/javaudf_p0/test_javaudf_with_decimal.out b/regression-test/data/javaudf_p0/test_javaudf_with_decimal.out index 4a05a63689f3aac..abcbe67e12c9e67 100644 --- a/regression-test/data/javaudf_p0/test_javaudf_with_decimal.out +++ b/regression-test/data/javaudf_p0/test_javaudf_with_decimal.out @@ -1,13 +1,9 @@ -- This file is automatically generated. You should know what you did if you want to edit this --- !select_1 -- +-- !getarrscale -- [1.123, 1.123] 3 [2.123, 2.123] 3 --- !select_2 -- -{1.123:1.123457} 9 -{2.123:2.123457} 9 - --- !select_3 -- +-- !retscale -- 1 {1.0000000000:1.1110000000} 2 {2.0000000000:2.2220000000} diff --git a/regression-test/suites/correctness_p0/test_cast_decimal.groovy b/regression-test/suites/correctness_p0/test_cast_decimal.groovy index 21a1ab6d0c3ee4d..ec9a8434c468f07 100644 --- a/regression-test/suites/correctness_p0/test_cast_decimal.groovy +++ b/regression-test/suites/correctness_p0/test_cast_decimal.groovy @@ -16,20 +16,6 @@ // under the License. suite("test_cast_decimal") { - sql """ - set enable_nereids_planner=true; - """ - - explain { - sql """select cast(32123.34212456734 as decimal(3,2));""" - contains "cast(32123.34212456734 as DECIMALV3(3, 2))" - } - - - sql """ - set enable_nereids_planner=false; - """ - explain { sql """select cast(32123.34212456734 as decimal(3,2));""" contains "CAST(32123.34212456734 AS DECIMALV3(3, 2))" diff --git a/regression-test/suites/correctness_p0/test_group_having_alias.groovy b/regression-test/suites/correctness_p0/test_group_having_alias.groovy index 43f5abf4fcb1cd7..7712e9ff20377ee 100644 --- a/regression-test/suites/correctness_p0/test_group_having_alias.groovy +++ b/regression-test/suites/correctness_p0/test_group_having_alias.groovy @@ -71,22 +71,6 @@ ORDER BY date; """ - sql """set group_by_and_having_use_alias_first=true""" - sql "set enable_nereids_planner=false" // nereids not support it - - qt_sql """ - SELECT - date_format(date, '%x%v') AS `date`, - count(date) AS `diff_days` - FROM `tb_holiday` - WHERE `date` between 20221111 AND 20221116 - GROUP BY date - HAVING date = 20221111 - ORDER BY date; - """ - - sql "set enable_nereids_planner=default" // nereids not support bellow sql - qt_sql """ SELECT date_format(date, '%x%v') AS `date2`, @@ -107,8 +91,6 @@ ORDER BY date; """ - sql """set group_by_and_having_use_alias_first=false""" - sql """ CREATE TABLE `test_having_alias_tb` ( `id` int(11) NULL, @@ -133,8 +115,6 @@ qt_case5 """ SELECT id, max(v1) v1 FROM test_having_alias_tb GROUP BY 1 having count(distinct v1)>1 ORDER BY id; """ qt_case_cir2273 """ select sum( id * 0.5 ) val from test_having_alias_tb having val > 0; """ - sql """set enable_nereids_planner=true""" - sql """set enable_fallback_to_original_planner=false""" qt_case6 """ SELECT date_format(date, '%x%v') AS `date` FROM `tb_holiday` WHERE `date` between 20221111 AND 20221116 HAVING date = 202245 ORDER BY date; """ diff --git a/regression-test/suites/datatype_p0/nested_types/query/array_functions/test_array_functions_array_intersect_sort.groovy b/regression-test/suites/datatype_p0/nested_types/query/array_functions/test_array_functions_array_intersect_sort.groovy index 46c43dacce3cc8f..804834a2a11d85e 100644 --- a/regression-test/suites/datatype_p0/nested_types/query/array_functions/test_array_functions_array_intersect_sort.groovy +++ b/regression-test/suites/datatype_p0/nested_types/query/array_functions/test_array_functions_array_intersect_sort.groovy @@ -16,68 +16,6 @@ // under the License. suite("test_array_functions_array_intersect_sort", "p0") { - - sql "set enable_nereids_planner=false;" - // ========= array_intersect =========== - // with sort - qt_oLd_sql "SELECT 'array_intersect-array-sort';" - sql "drop table if exists tbl_array_intersect;" - sql "create table tbl_array_intersect (date Date, arr Array) ENGINE=OLAP DISTRIBUTED BY HASH(date) BUCKETS 1 PROPERTIES('replication_num' = '1');" - - sql "insert into tbl_array_intersect values ('2019-01-01', [1,2,3]);" - sql "insert into tbl_array_intersect values ('2019-01-02', [1,2]);" - sql "insert into tbl_array_intersect values ('2019-01-03', [1]);" - sql "insert into tbl_array_intersect values ('2019-01-04', []);" - - order_qt_old_sql "SELECT array_sort(array_intersect(arr, [1,2])) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect(arr, [])) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect([], arr)) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect([1,2], arr)) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect([1,2], [1,2,3,4])) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect([], [])) from tbl_array_intersect order by date;" - - - order_qt_old_sql "SELECT array_sort(array_intersect(arr, [1,2])) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect(arr, [])) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect([], arr)) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect([1,2], arr)) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect([1,2], [1,2,3,4])) from tbl_array_intersect order by date;" - order_qt_old_sql "SELECT array_sort(array_intersect([], [])) from tbl_array_intersect order by date;" - - - order_qt_old_sql "SELECT array_sort(array_intersect([-100], [156]));" - order_qt_old_sql "SELECT array_sort(array_intersect([1], [257]));" - - order_qt_old_sql "SELECT array_sort(array_intersect(['a', 'b', 'c'], ['a', 'a']));" - order_qt_old_sql "SELECT array_sort(array_intersect([1, 1], [2, 2]));" - order_qt_old_sql "SELECT array_sort(array_intersect([1, 1], [1, 2]));" - // nereids not support array_intersect with three argument - order_qt_old_sql "SELECT array_sort(array_intersect([1, 1, 1], [3], [2, 2, 2]));" - order_qt_old_sql "SELECT array_sort(array_intersect([1, 2], [1, 2], [2]));" - order_qt_old_sql "SELECT array_sort(array_intersect([1, 1], [2, 1], [2, 2], [1]));" - order_qt_old_sql "SELECT array_sort(array_intersect([1, 1], [2, 1], [2, 2], [2, 2, 2]));" - - // nereids not support array_intersect with one argument - try { - sql """ - SELECT array_sort(array_intersect([])); - """ - } catch (Exception ex) { - assertTrue(ex.getMessage().contains("No matching function with signature: array_intersect(ARRAY)")) - } - try { - sql """ - SELECT array_sort(array_intersect([1, 2, 3])); - """ - } catch (Exception ex) { - assertTrue(ex.getMessage().contains("No matching function with signature: array_intersect(ARRAY)")) - } - - // nereids - sql "set enable_nereids_planner=true;" - sql "set enable_fallback_to_original_planner=false;" - - // ========= array_intersect =========== // with sort qt_nereid_sql "SELECT 'array_intersect-array-sort';" @@ -111,26 +49,17 @@ suite("test_array_functions_array_intersect_sort", "p0") { order_qt_nereid_sql "SELECT array_sort(array_intersect(['a', 'b', 'c'], ['a', 'a']));" order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 1], [2, 2]));" order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 1], [1, 2]));" - // nereids not support array_intersect with three argument -// order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 1, 1], [3], [2, 2, 2]));" -// order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 2], [1, 2], [2]));" -// order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 1], [2, 1], [2, 2], [1]));" -// order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 1], [2, 1], [2, 2], [2, 2, 2]));" -// -// // nereids not support array_intersect with one argument -// try { -// sql """ -// SELECT array_sort(array_intersect([])); -// """ -// } catch (Exception ex) { -// assertTrue(ex.getMessage().contains("No matching function with signature: array_intersect(ARRAY)")) -// } -// try { -// sql """ -// SELECT array_sort(array_intersect([1, 2, 3])); -// """ -// } catch (Exception ex) { -// assertTrue(ex.getMessage().contains("No matching function with signature: array_intersect(ARRAY)")) -// } - + order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 1, 1], [3], [2, 2, 2]));" + order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 2], [1, 2], [2]));" + order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 1], [2, 1], [2, 2], [1]));" + order_qt_nereid_sql "SELECT array_sort(array_intersect([1, 1], [2, 1], [2, 2], [2, 2, 2]));" + + test { + sql "SELECT array_sort(array_intersect([]))" + exception "Can not found function 'array_intersect' which has 1 arity" + } + test { + sql "SELECT array_sort(array_intersect([1, 2, 3]))" + exception "Can not found function 'array_intersect' which has 1 arity" + } } diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy index c8bd8a8c2d6b4d5..596ed79eec9dc2f 100644 --- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy @@ -19,7 +19,6 @@ import org.junit.Assert // This suit test the `backends` tvf suite("test_local_tvf_with_complex_type_insertinto_doris", "p0") { - sql """set enable_nereids_planner=false""" List> backends = sql """ select * from backends(); """ assertTrue(backends.size() > 0) def be_id = backends[0][0] diff --git a/regression-test/suites/external_table_p2/tvf/test_tvf_view_count_p2.groovy b/regression-test/suites/external_table_p2/tvf/test_tvf_view_count_p2.groovy index 32b9bac9c70404c..1e2fbb295604b37 100644 --- a/regression-test/suites/external_table_p2/tvf/test_tvf_view_count_p2.groovy +++ b/regression-test/suites/external_table_p2/tvf/test_tvf_view_count_p2.groovy @@ -30,19 +30,11 @@ suite("test_tvf_view_count_p2", "p2,external,tvf,external_remote,external_remote "hadoop.username" = "hadoop", "format"="parquet");""" - def result = sql """explain verbose select count(1) from tvf_view_count;""" - def contain0 = false; - def contain1 = false; - for (String value : result) { - if (value.contains("SlotDescriptor{id=0,")) { - contain0 = true; - } - if (value.contains("SlotDescriptor{id=1,")) { - contain1 = true; - } + explain { + "select count(1) from tvf_view_count" + contains "SlotDescriptor{id=0," + notContains "SlotDescriptor{id=1," } - assertTrue(contain0) - assertFalse(contain1) sql """drop database if exists test_tvf_view_count_p2""" } diff --git a/regression-test/suites/javaudf_p0/test_javaudf_with_decimal.groovy b/regression-test/suites/javaudf_p0/test_javaudf_with_decimal.groovy index bf2887283190b14..d8aedea8f1d3a5f 100644 --- a/regression-test/suites/javaudf_p0/test_javaudf_with_decimal.groovy +++ b/regression-test/suites/javaudf_p0/test_javaudf_with_decimal.groovy @@ -15,18 +15,11 @@ // specific language governing permissions and limitations // under the License. -import org.codehaus.groovy.runtime.IOGroovyMethods - -import java.nio.charset.StandardCharsets -import java.nio.file.Files -import java.nio.file.Paths - suite("test_javaudf_with_decimal") { def jarPath = """${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar""" - log.info("Jar path: ${jarPath}".toString()) + log.info("Jar path: ${jarPath}") try { try_sql("drop function IF EXISTS getarrscale(Array);") - try_sql("drop function IF EXISTS getmapscale(Map);") try_sql("drop function IF EXISTS retscale(int);") try_sql("drop table IF EXISTS dbwithDecimal;") sql """ @@ -45,32 +38,16 @@ suite("test_javaudf_with_decimal") { sql """ INSERT INTO dbwithDecimal VALUES(1,[1.123,1.123456],{1.123:1.123456789}); """ sql """ INSERT INTO dbwithDecimal VALUES(2,[2.123,2.123456],{2.123:2.123456789}); """ - sql """ - CREATE FUNCTION getarrscale(Array) RETURNS int PROPERTIES ( "file"="file://${jarPath}", "symbol"="org.apache.doris.udf.MyArrayDecimal", "always_nullable"="true", "type"="JAVA_UDF" ); - """ - - sql """ - - CREATE FUNCTION getmapscale(Map) RETURNS int PROPERTIES ( - "file"="file://${jarPath}", - "symbol"="org.apache.doris.udf.MyMapDecimal", - "always_nullable"="true", - "type"="JAVA_UDF" - ); - - """ - sql """ - CREATE FUNCTION retscale(int) RETURNS Map PROPERTIES ( "file"="file://${jarPath}", "symbol"="org.apache.doris.udf.MyMapRetDecimal", @@ -79,18 +56,10 @@ suite("test_javaudf_with_decimal") { ); """ - sql """ - set enable_nereids_planner=false; - """ - - qt_select_1 """ select arr,getarrscale(arr) from dbwithDecimal order by id; """ - - qt_select_2 """ select mp,getmapscale(mp) from dbwithDecimal order by id ; """ - - qt_select_3 """ select id,retscale(id) from dbwithDecimal order by id; """ + qt_getarrscale """ select arr,getarrscale(arr) from dbwithDecimal order by id; """ + qt_retscale """ select id,retscale(id) from dbwithDecimal order by id; """ } finally { try_sql("drop function IF EXISTS getarrscale(Array);") - try_sql("drop function IF EXISTS getmapscale(Map);") try_sql("drop function IF EXISTS retscale(int);") try_sql("drop table IF EXISTS dbwithDecimal;") } diff --git a/regression-test/suites/query_p0/test_array_with_single_row_max_than_uint16.groovy b/regression-test/suites/query_p0/test_array_with_single_row_max_than_uint16.groovy index 6875c836d37ac23..0294b215bf12055 100644 --- a/regression-test/suites/query_p0/test_array_with_single_row_max_than_uint16.groovy +++ b/regression-test/suites/query_p0/test_array_with_single_row_max_than_uint16.groovy @@ -17,7 +17,6 @@ suite("test_array_with_single_row_max_than_uint16", "query") { // define a sql table - sql """set enable_nereids_planner=false""" List> backends = sql """ select * from backends(); """ assertTrue(backends.size() > 0) def be_id = backends[0][0] From b6409f5584107db86ad9c3b09dde556663c6532a Mon Sep 17 00:00:00 2001 From: qiye Date: Mon, 13 May 2024 15:27:51 +0800 Subject: [PATCH 013/111] [improvement](inverted index) Disable the use of skipping write index on load (#34719) When `skip_write_index_on_load` is turned on, users will get an error when querying for the latest data(not compacted), giving them a bad experience. And we can use `inverted_index_ram_dir_enable = true` and `inverted_index_storage_format=V2` to reduce IO and CPU consumption. So we disable it now. 1. Disable setting `skip_write_index_on_load` to `true` in create table stmt. 2. Disable setting `skip_write_index_on_load` to `true` in alter table properties stmt. You can still alter `skip_write_index_on_load` to `false`. Co-authored-by: Luennng --- .../apache/doris/analysis/ModifyTablePropertiesClause.java | 5 +++++ .../java/org/apache/doris/common/util/PropertyAnalyzer.java | 3 ++- .../java/org/apache/doris/datasource/InternalCatalog.java | 3 +-- regression-test/suites/datatype_p0/scalar_types/load.groovy | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java index c2bc7bc7d0dea0f..e0aa994c6d3a6c0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ModifyTablePropertiesClause.java @@ -240,6 +240,11 @@ public void analyze(Analyzer analyzer) throws AnalysisException { this.needTableStable = false; this.opType = AlterOpType.MODIFY_TABLE_PROPERTY_SYNC; } else if (properties.containsKey(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD)) { + if (properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true")) { + throw new AnalysisException( + "Property " + + PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD + " is forbidden now"); + } if (!properties.get(PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("true") && !properties.get(PropertyAnalyzer .PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD).equalsIgnoreCase("false")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index ba4434349cbf96a..07236da2ee30d9d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -705,7 +705,8 @@ public static Boolean analyzeSkipWriteIndexOnLoad(Map properties } properties.remove(PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD); if (value.equalsIgnoreCase("true")) { - return true; + throw new AnalysisException("Property " + PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD + + " is forbidden now."); } else if (value.equalsIgnoreCase("false")) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index ee3b6ee965104a0..41ef985f17f8e65 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2405,8 +2405,7 @@ private void createOlapTable(Database db, CreateTableStmt stmt) throws UserExcep olapTable.setStoreRowColumn(storeRowColumn); // set skip inverted index on load - boolean skipWriteIndexOnLoad = PropertyAnalyzer.analyzeBooleanProp(properties, - PropertyAnalyzer.PROPERTIES_SKIP_WRITE_INDEX_ON_LOAD, false); + boolean skipWriteIndexOnLoad = PropertyAnalyzer.analyzeSkipWriteIndexOnLoad(properties); olapTable.setSkipWriteIndexOnLoad(skipWriteIndexOnLoad); boolean isMutable = PropertyAnalyzer.analyzeBooleanProp(properties, PropertyAnalyzer.PROPERTIES_MUTABLE, true); diff --git a/regression-test/suites/datatype_p0/scalar_types/load.groovy b/regression-test/suites/datatype_p0/scalar_types/load.groovy index 91cd4eb1f1f1d64..4c7c7ee93666de0 100644 --- a/regression-test/suites/datatype_p0/scalar_types/load.groovy +++ b/regression-test/suites/datatype_p0/scalar_types/load.groovy @@ -448,7 +448,7 @@ suite("test_scalar_types_load", "p0") { DUPLICATE KEY(`k1`) COMMENT 'OLAP' DISTRIBUTED BY HASH(`k1`) BUCKETS 10 - PROPERTIES("replication_num" = "1", "skip_write_index_on_load" = "true"); + PROPERTIES("replication_num" = "1", "skip_write_index_on_load" = "false"); """ // insert data into dup table with index From 80dd027ce2e28a6fb5c6c1b9eaaf7c7ff5f2a068 Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Mon, 13 May 2024 23:46:27 +0800 Subject: [PATCH 014/111] [opt](join) For left semi/anti join without mark join conjunct and without other conjucnts, stop probing after matching one row (#34703) --- .../vec/common/hash_table/join_hash_table.h | 31 +++++++++++++------ .../exec/join/process_hash_table_probe_impl.h | 3 +- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/be/src/vec/common/hash_table/join_hash_table.h b/be/src/vec/common/hash_table/join_hash_table.h index 10ca6c9b2ddaa5d..a869ad419ad530f 100644 --- a/be/src/vec/common/hash_table/join_hash_table.h +++ b/be/src/vec/common/hash_table/join_hash_table.h @@ -89,7 +89,7 @@ class JoinHashTable { auto find_batch(const Key* __restrict keys, const uint32_t* __restrict build_idx_map, int probe_idx, uint32_t build_idx, int probe_rows, uint32_t* __restrict probe_idxs, bool& probe_visited, - uint32_t* __restrict build_idxs) { + uint32_t* __restrict build_idxs, bool has_mark_join_conjunct = false) { if constexpr (JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN || JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) { if (_empty_build_side) { @@ -100,12 +100,25 @@ class JoinHashTable { if constexpr (with_other_conjuncts || (is_mark_join && JoinOpType != TJoinOp::RIGHT_SEMI_JOIN)) { - constexpr bool null_aware_without_other_conjuncts = - (JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN || - JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN) && - !with_other_conjuncts; - return _find_batch_conjunct( + if constexpr (!with_other_conjuncts) { + constexpr bool is_null_aware_join = + JoinOpType == TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN || + JoinOpType == TJoinOp::NULL_AWARE_LEFT_SEMI_JOIN; + constexpr bool is_left_half_join = JoinOpType == TJoinOp::LEFT_SEMI_JOIN || + JoinOpType == TJoinOp::LEFT_ANTI_JOIN; + + /// For null aware join or left half(semi/anti) join without other conjuncts and without + /// mark join conjunct. + /// If one row on probe side has one match in build side, we should stop searching the + /// hash table for this row. + if (is_null_aware_join || (is_left_half_join && !has_mark_join_conjunct)) { + return _find_batch_conjunct( + keys, build_idx_map, probe_idx, build_idx, probe_rows, probe_idxs, + build_idxs); + } + } + + return _find_batch_conjunct( keys, build_idx_map, probe_idx, build_idx, probe_rows, probe_idxs, build_idxs); } @@ -314,7 +327,7 @@ class JoinHashTable { return std::tuple {probe_idx, 0U, matched_cnt}; } - template + template auto _find_batch_conjunct(const Key* __restrict keys, const uint32_t* __restrict build_idx_map, int probe_idx, uint32_t build_idx, int probe_rows, uint32_t* __restrict probe_idxs, uint32_t* __restrict build_idxs) { @@ -345,7 +358,7 @@ class JoinHashTable { probe_idxs[matched_cnt] = probe_idx; matched_cnt++; - if constexpr (null_aware_without_other_conjuncts) { + if constexpr (only_need_to_match_one) { build_idx = 0; break; } diff --git a/be/src/vec/exec/join/process_hash_table_probe_impl.h b/be/src/vec/exec/join/process_hash_table_probe_impl.h index b4212405aeda170..9a15df637aaf60a 100644 --- a/be/src/vec/exec/join/process_hash_table_probe_impl.h +++ b/be/src/vec/exec/join/process_hash_table_probe_impl.h @@ -213,6 +213,7 @@ Status ProcessHashTableProbe::do_process(HashTableType& hash } auto& mcol = mutable_block.mutable_columns(); + const bool has_mark_join_conjunct = !_parent->_mark_join_conjuncts.empty(); int current_offset = 0; if constexpr ((JoinOpType == doris::TJoinOp::NULL_AWARE_LEFT_ANTI_JOIN || @@ -258,7 +259,7 @@ Status ProcessHashTableProbe::do_process(HashTableType& hash need_null_map_for_probe && ignore_null > (hash_table_ctx.keys, hash_table_ctx.bucket_nums.data(), probe_index, build_index, probe_rows, _probe_indexs.data(), - _probe_visited, _build_indexs.data()); + _probe_visited, _build_indexs.data(), has_mark_join_conjunct); probe_index = new_probe_idx; build_index = new_build_idx; current_offset = new_current_offset; From 6c515e0c76a38116332e998ec1650228941d6d3b Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Tue, 14 May 2024 09:50:29 +0800 Subject: [PATCH 015/111] [fix](group commit) Make compatibility issues on serializing and deserializing wal file more clear (#34793) --- be/src/agent/be_exec_version_manager.h | 1 + be/src/vec/exec/format/wal/wal_reader.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/be/src/agent/be_exec_version_manager.h b/be/src/agent/be_exec_version_manager.h index 32a520cc4bd223a..248bcfc7e1ab830 100644 --- a/be/src/agent/be_exec_version_manager.h +++ b/be/src/agent/be_exec_version_manager.h @@ -77,5 +77,6 @@ constexpr inline int BeExecVersionManager::min_be_exec_version = 0; /// functional constexpr inline int BITMAP_SERDE = 3; constexpr inline int USE_NEW_SERDE = 4; // release on DORIS version 2.1 +constexpr inline int OLD_WAL_SERDE = 3; // use to solve compatibility issues, see pr #32299 } // namespace doris diff --git a/be/src/vec/exec/format/wal/wal_reader.cpp b/be/src/vec/exec/format/wal/wal_reader.cpp index 1fbd395ad269d5f..0d15c198b0e3a1e 100644 --- a/be/src/vec/exec/format/wal/wal_reader.cpp +++ b/be/src/vec/exec/format/wal/wal_reader.cpp @@ -17,6 +17,7 @@ #include "wal_reader.h" +#include "agent/be_exec_version_manager.h" #include "common/logging.h" #include "common/sync_point.h" #include "gutil/strings/split.h" @@ -52,9 +53,10 @@ Status WalReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { // in the latest version. So if a wal is created by 2.1.0 (wal version=0 && be_exec_version=3), // it should upgrade the be_exec_version to 4 to use the new way to deserialize pblock to solve // compatibility issues.see https://github.com/apache/doris/pull/32299 - if (_version == 0 && pblock.has_be_exec_version() && pblock.be_exec_version() == 3) { + if (_version == 0 && pblock.has_be_exec_version() && + pblock.be_exec_version() == OLD_WAL_SERDE) { VLOG_DEBUG << "need to set be_exec_version to 4 to solve compatibility issues"; - pblock.set_be_exec_version(4); + pblock.set_be_exec_version(USE_NEW_SERDE); } if (st.is()) { LOG(INFO) << "read eof on wal:" << _wal_path; From f7801948ad81d968f77d7be9835d64d8fedc5cca Mon Sep 17 00:00:00 2001 From: camby Date: Tue, 14 May 2024 10:13:28 +0800 Subject: [PATCH 016/111] fix backup and restore failed between force_replication_allocation setted and not setted clusters (#34608) --- .../src/main/java/org/apache/doris/analysis/RestoreStmt.java | 5 +++++ .../src/main/java/org/apache/doris/backup/BackupJobInfo.java | 4 ++++ .../src/main/java/org/apache/doris/backup/RestoreJob.java | 4 ++++ 3 files changed, 13 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java index 9f5f6ee72537bd0..21e417631dda18a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RestoreStmt.java @@ -20,6 +20,7 @@ import org.apache.doris.backup.Repository; import org.apache.doris.catalog.ReplicaAllocation; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.UserException; @@ -168,6 +169,10 @@ public void analyzeProperties() throws AnalysisException { ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, "Invalid reserve_replica value: " + copiedProperties.get(PROP_RESERVE_REPLICA)); } + // force set reserveReplica to false, do not keep the origin allocation + if (reserveReplica && !Config.force_olap_table_replication_allocation.isEmpty()) { + reserveReplica = false; + } copiedProperties.remove(PROP_RESERVE_REPLICA); } // reserve dynamic partition enable diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java index 7108315cee7dc99..aa127961e3d217e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java @@ -32,6 +32,7 @@ import org.apache.doris.catalog.TableIf.TableType; import org.apache.doris.catalog.Tablet; import org.apache.doris.catalog.View; +import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.Version; import org.apache.doris.common.io.Text; @@ -99,6 +100,8 @@ public class BackupJobInfo implements Writable { public int minorVersion; @SerializedName("patch_version") public int patchVersion; + @SerializedName("is_force_replication_allocation") + public boolean isForceReplicationAllocation; @SerializedName("tablet_be_map") public Map tabletBeMap = Maps.newHashMap(); @@ -607,6 +610,7 @@ public static BackupJobInfo fromCatalog(long backupTime, String label, String db jobInfo.majorVersion = Version.DORIS_BUILD_VERSION_MAJOR; jobInfo.minorVersion = Version.DORIS_BUILD_VERSION_MINOR; jobInfo.patchVersion = Version.DORIS_BUILD_VERSION_PATCH; + jobInfo.isForceReplicationAllocation = !Config.force_olap_table_replication_allocation.isEmpty(); Collection
tbls = backupMeta.getTables().values(); // tbls diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java index 5eb4950466f1b42..2af0242db333520 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java @@ -189,6 +189,10 @@ public RestoreJob(String label, String backupTs, long dbId, String dbName, Backu this.state = RestoreJobState.PENDING; this.metaVersion = metaVersion; this.reserveReplica = reserveReplica; + // if backup snapshot is come from a cluster with force replication allocation, ignore the origin allocation + if (jobInfo.isForceReplicationAllocation) { + this.reserveReplica = false; + } this.reserveDynamicPartitionEnable = reserveDynamicPartitionEnable; this.isBeingSynced = isBeingSynced; properties.put(PROP_RESERVE_REPLICA, String.valueOf(reserveReplica)); From 30a036e7a4862782076db186934d64f5e4aef3a9 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Tue, 14 May 2024 10:46:29 +0800 Subject: [PATCH 017/111] [feature](mtmv) create mtmv support partitions rollup (#31812) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit if create MTMV `date_trunc(`xxx`,'month')` when related table is `range` partition,and have 3 partitions: ``` 20200101-20200102 20200102-20200103 20200201-20200202 ``` then MTMV will have 2 partitions: ``` 20200101-20200201 20200201-20200301 ``` when related table is `list` partition,and have 3 partitions: ``` (20200101,20200102) (20200103) (20200201) ``` then MTMV will have 2 partitions: ``` (20200101,20200102，20200103) (20200201) ``` --- .../org/apache/doris/nereids/DorisParser.g4 | 7 +- .../java/org/apache/doris/catalog/MTMV.java | 40 +-- .../doris/job/extensions/mtmv/MTMVTask.java | 6 +- .../mtmv/MTMVPartitionExprDateTrunc.java | 228 ++++++++++++++ .../doris/mtmv/MTMVPartitionExprFactory.java | 39 +++ .../doris/mtmv/MTMVPartitionExprService.java | 60 ++++ .../apache/doris/mtmv/MTMVPartitionInfo.java | 39 ++- .../apache/doris/mtmv/MTMVPartitionUtil.java | 79 +++-- ...VRelatedPartitionDescGeneratorService.java | 38 +++ ...MTMVRelatedPartitionDescInitGenerator.java | 34 ++ ...PartitionDescOnePartitionColGenerator.java | 67 ++++ ...MVRelatedPartitionDescRollUpGenerator.java | 142 +++++++++ ...elatedPartitionDescSyncLimitGenerator.java | 142 +++++++++ .../apache/doris/mtmv/MTMVRelatedTableIf.java | 29 -- .../java/org/apache/doris/mtmv/MTMVUtil.java | 100 +----- .../mtmv/RelatedPartitionDescResult.java | 53 ++++ .../nereids/parser/LogicalPlanBuilder.java | 24 +- .../plans/commands/info/CreateMTMVInfo.java | 86 +----- .../info/MTMVPartitionDefinition.java | 215 +++++++++++++ ...latedPartitionDescRollUpGeneratorTest.java | 152 +++++++++ ...edPartitionDescSyncLimitGeneratorTest.java | 94 ++++++ .../org/apache/doris/mtmv/MTMVUtilTest.java | 65 ---- .../test_hive_limit_partition_mtmv.out | 14 + .../mtmv_p0/test_rollup_partition_mtmv.out | 16 + .../test_hive_limit_partition_mtmv.groovy | 129 +++++--- .../mtmv_p0/test_rollup_partition_mtmv.groovy | 292 ++++++++++++++++++ 26 files changed, 1808 insertions(+), 382 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprFactory.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprService.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorService.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mtmv/RelatedPartitionDescResult.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGeneratorTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java create mode 100644 regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out create mode 100644 regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 9562bed5f2177fc..c78f11b0007f3d0 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -101,7 +101,7 @@ statementBase (REFRESH refreshMethod? refreshTrigger?)? ((DUPLICATE)? KEY keys=identifierList)? (COMMENT STRING_LITERAL)? - (PARTITION BY LEFT_PAREN partitionKey = identifier RIGHT_PAREN)? + (PARTITION BY LEFT_PAREN mvPartition RIGHT_PAREN)? (DISTRIBUTED BY (HASH hashKeys=identifierList | RANDOM) (BUCKETS (INTEGER_VALUE | AUTO))?)? propertyClause? AS query #createMTMV @@ -225,6 +225,11 @@ refreshMethod : COMPLETE | AUTO ; +mvPartition + : partitionKey = identifier + | partitionExpr = functionCallExpression + ; + identifierOrStringLiteral : identifier | STRING_LITERAL diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index c76f1a253f2fe22..a3050bc3ac6b70c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -30,6 +30,7 @@ import org.apache.doris.mtmv.MTMVJobManager; import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVPartitionUtil; import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVRefreshState; import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; @@ -38,7 +39,6 @@ import org.apache.doris.mtmv.MTMVRefreshSnapshot; import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVStatus; -import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Maps; @@ -51,7 +51,6 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; -import java.util.HashMap; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; @@ -313,35 +312,6 @@ public Map generateMvPartitionDescs() { return result; } - /** - * generateRelatedPartitionDescs - *

- * Different partitions may generate the same PartitionKeyDesc through logical calculations - * (such as selecting only one column, or rolling up partitions), so it is a one to many relationship - * - * @return related PartitionKeyDesc ==> relatedPartitionIds - * @throws AnalysisException - */ - public Map> generateRelatedPartitionDescs() throws AnalysisException { - if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { - return Maps.newHashMap(); - } - Map> res = new HashMap<>(); - int relatedColPos = mvPartitionInfo.getRelatedColPos(); - Map relatedPartitionItems = mvPartitionInfo.getRelatedTable() - .getPartitionItemsByTimeFilter(relatedColPos, - MTMVUtil.generateMTMVPartitionSyncConfigByProperties(mvProperties)); - for (Entry entry : relatedPartitionItems.entrySet()) { - PartitionKeyDesc partitionKeyDesc = entry.getValue().toPartitionKeyDesc(relatedColPos); - if (res.containsKey(partitionKeyDesc)) { - res.get(partitionKeyDesc).add(entry.getKey()); - } else { - res.put(partitionKeyDesc, Sets.newHashSet(entry.getKey())); - } - } - return res; - } - /** * Calculate the partition and associated partition mapping relationship of the MTMV * It is the result of real-time comparison calculation, so there may be some costs, @@ -354,13 +324,19 @@ public Map> calculatePartitionMappings() throws AnalysisExceptio if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { return Maps.newHashMap(); } + long start = System.currentTimeMillis(); Map> res = Maps.newHashMap(); - Map> relatedPartitionDescs = generateRelatedPartitionDescs(); + Map> relatedPartitionDescs = MTMVPartitionUtil + .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties); Map mvPartitionItems = getAndCopyPartitionItems(); for (Entry entry : mvPartitionItems.entrySet()) { res.put(entry.getKey(), relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Sets.newHashSet())); } + if (LOG.isDebugEnabled()) { + LOG.debug("calculatePartitionMappings use [{}] mills, mvName is [{}]", + System.currentTimeMillis() - start, name); + } return res; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index efcdc4f564e9a83..7a4c5277591525c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -170,7 +170,7 @@ public void run() throws JobException { // Now, the MTMV first ensures consistency with the data in the cache. // To be completely consistent with hive, you need to manually refresh the cache // refreshHmsTable(); - if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE) { + if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { MTMVPartitionUtil.alignMvPartition(mtmv); } Map> partitionMappings = mtmv.calculatePartitionMappings(); @@ -225,7 +225,7 @@ private void exec(ConnectContext ctx, Set refreshPartitionIds, lastQueryId = DebugUtil.printId(queryId); // if SELF_MANAGE mv, only have default partition, will not have partitionItem, so we give empty set UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand - .from(mtmv, mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE + .from(mtmv, mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE ? refreshPartitionIds : Sets.newHashSet(), tableWithPartKey); executor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, ctx.getStatementContext())); ctx.setExecutor(executor); @@ -400,7 +400,7 @@ protected void closeOrReleaseResources() { private Map getIncrementalTableMap() throws AnalysisException { Map tableWithPartKey = Maps.newHashMap(); - if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE) { + if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { tableWithPartKey .put(mtmv.getMvPartitionInfo().getRelatedTable(), mtmv.getMvPartitionInfo().getRelatedCol()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java new file mode 100644 index 000000000000000..e1991ab2921815c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java @@ -0,0 +1,228 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.PartitionExprUtil; +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.StringUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +public class MTMVPartitionExprDateTrunc implements MTMVPartitionExprService { + private static Set timeUnits = ImmutableSet.of("year", "month", "day"); + private String timeUnit; + + public MTMVPartitionExprDateTrunc(FunctionCallExpr functionCallExpr) throws AnalysisException { + List paramsExprs = functionCallExpr.getParams().exprs(); + if (paramsExprs.size() != 2) { + throw new AnalysisException("date_trunc params exprs size should be 2."); + } + Expr param = paramsExprs.get(1); + if (!(param instanceof StringLiteral)) { + throw new AnalysisException("date_trunc param of time unit is not string literal."); + } + this.timeUnit = param.getStringValue().toLowerCase(); + } + + @Override + public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + if (!timeUnits.contains(this.timeUnit)) { + throw new AnalysisException( + String.format("timeUnit not support: %s, only support: %s", this.timeUnit, timeUnits)); + } + MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable(); + PartitionType partitionType = relatedTable.getPartitionType(); + if (partitionType == PartitionType.RANGE) { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + if (!partitionColumnType.isDateType()) { + throw new AnalysisException( + "partitionColumnType should be date/datetime " + + "when PartitionType is range and expr is date_trunc"); + } + } + } + + @Override + public String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map mvProperties) + throws AnalysisException { + String res = null; + Optional dateFormat = getDateFormat(mvProperties); + List> inValues = partitionKeyDesc.getInValues(); + for (int i = 0; i < inValues.size(); i++) { + // mtmv only support one partition column + PartitionValue partitionValue = inValues.get(i).get(0); + if (partitionValue.isNullPartition()) { + throw new AnalysisException("date trunc not support null partition value"); + } + String identity = dateTrunc(partitionValue.getStringValue(), dateFormat, false).toString(); + if (i == 0) { + res = identity; + } else { + if (!Objects.equals(res, identity)) { + throw new AnalysisException( + String.format("partition values not equal, res: %s, identity: %s", res, + identity)); + } + } + } + return res; + } + + private Optional getDateFormat(Map mvProperties) { + Optional dateFormat = + StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)) + ? Optional.empty() + : Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)); + return dateFormat; + } + + @Override + public PartitionKeyDesc generateRollUpPartitionKeyDesc(PartitionKeyDesc partitionKeyDesc, + MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + // mtmv only support one partition column + Preconditions.checkState(partitionKeyDesc.getLowerValues().size() == 1, + "only support one partition column"); + DateTimeV2Literal beginTime = dateTrunc( + partitionKeyDesc.getLowerValues().get(0).getStringValue(), + Optional.empty(), false); + + PartitionValue lowerValue = new PartitionValue(dateTimeToStr(beginTime, partitionColumnType)); + PartitionValue upperValue = getUpperValue(partitionKeyDesc.getUpperValues().get(0), beginTime, + partitionColumnType); + return PartitionKeyDesc.createFixed( + Collections.singletonList(lowerValue), + Collections.singletonList(upperValue)); + } + + private PartitionValue getUpperValue(PartitionValue upperValue, DateTimeV2Literal beginTruncTime, + Type partitionColumnType) throws AnalysisException { + if (upperValue.isMax()) { + throw new AnalysisException("date trunc not support MAXVALUE partition"); + } + // begin time and end time dateTrunc should has same result + DateTimeV2Literal endTruncTime = dateTrunc(upperValue.getStringValue(), Optional.empty(), true); + if (!Objects.equals(beginTruncTime, endTruncTime)) { + throw new AnalysisException( + String.format("partition values not equal, beginTruncTime: %s, endTruncTime: %s", beginTruncTime, + endTruncTime)); + } + DateTimeV2Literal endTime = dateIncrement(beginTruncTime); + return new PartitionValue(dateTimeToStr(endTime, partitionColumnType)); + } + + private DateTimeV2Literal dateTrunc(String value, + Optional dateFormat, boolean isUpper) throws AnalysisException { + DateTimeV2Literal dateTimeLiteral = strToDate(value, dateFormat); + // for (2020-01-31,2020-02-01),if not -1, lower value and upper value will not same after rollup + if (isUpper) { + dateTimeLiteral = (DateTimeV2Literal) DateTimeArithmetic.secondsSub(dateTimeLiteral, new IntegerLiteral(1)); + } + Expression expression = DateTimeExtractAndTransform.dateTrunc(dateTimeLiteral, new VarcharLiteral(timeUnit)); + if (!(expression instanceof DateTimeV2Literal)) { + throw new AnalysisException("dateTrunc() should return DateLiteral, expression: " + expression); + } + return (DateTimeV2Literal) expression; + } + + private DateTimeV2Literal strToDate(String value, + Optional dateFormat) throws AnalysisException { + try { + return new DateTimeV2Literal(value); + } catch (Exception e) { + if (!dateFormat.isPresent()) { + throw e; + } + Expression strToDate = DateTimeExtractAndTransform + .strToDate(new VarcharLiteral(value), + new VarcharLiteral(dateFormat.get())); + if (strToDate instanceof DateV2Literal) { + DateV2Literal dateV2Literal = (DateV2Literal) strToDate; + return new DateTimeV2Literal(dateV2Literal.getYear(), dateV2Literal.getMonth(), dateV2Literal.getDay(), + 0, 0, 0); + } else if (strToDate instanceof DateTimeV2Literal) { + return (DateTimeV2Literal) strToDate; + } else { + throw new AnalysisException( + String.format("strToDate failed, stringValue: %s, dateFormat: %s", value, + dateFormat)); + } + } + } + + private DateTimeV2Literal dateIncrement(DateTimeV2Literal value) throws AnalysisException { + Expression result; + switch (timeUnit) { + case "year": + result = value.plusYears(1L); + break; + case "month": + result = value.plusMonths(1L); + break; + case "day": + result = value.plusDays(1L); + break; + default: + throw new AnalysisException("MTMV partition roll up not support timeUnit: " + timeUnit); + } + if (!(result instanceof DateTimeV2Literal)) { + throw new AnalysisException("sub() should return DateTimeLiteral, result: " + result); + } + return (DateTimeV2Literal) result; + } + + private String dateTimeToStr(DateTimeV2Literal literal, + Type partitionColumnType) throws AnalysisException { + if (partitionColumnType.isDate() || partitionColumnType.isDateV2()) { + return String.format(PartitionExprUtil.DATE_FORMATTER, literal.getYear(), literal.getMonth(), + literal.getDay()); + } else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2()) { + return String.format(PartitionExprUtil.DATETIME_FORMATTER, + literal.getYear(), literal.getMonth(), literal.getDay(), + literal.getHour(), literal.getMinute(), literal.getSecond()); + } else { + throw new AnalysisException( + "MTMV not support partition with column type : " + partitionColumnType); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprFactory.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprFactory.java new file mode 100644 index 000000000000000..0fc9a067fad88ea --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprFactory.java @@ -0,0 +1,39 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.common.AnalysisException; + +/** + * MTMV Partition Expr Factory + */ +public class MTMVPartitionExprFactory { + public static MTMVPartitionExprService getExprService(Expr expr) throws AnalysisException { + if (!(expr instanceof FunctionCallExpr)) { + throw new AnalysisException("now mtmv partition only support FunctionCallExpr"); + } + FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; + String fnName = functionCallExpr.getFnName().getFunction().toLowerCase(); + if ("date_trunc".equals(fnName)) { + return new MTMVPartitionExprDateTrunc(functionCallExpr); + } + throw new AnalysisException("MTMV partition not support function name: " + fnName); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprService.java new file mode 100644 index 000000000000000..e6974343ef2444e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprService.java @@ -0,0 +1,60 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.common.AnalysisException; + +import java.util.Map; + +/** + * Interface for materialized view partitioning function + */ +public interface MTMVPartitionExprService { + + /** + * for list partition, get identity by expr + * + * @param partitionKeyDesc + * @param mvProperties + * @return + * @throws AnalysisException + */ + String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map mvProperties) + throws AnalysisException; + + /** + * for range partition, get roll up PartitionKeyDesc by expr + * + * @param partitionKeyDesc + * @param mvPartitionInfo + * @return + * @throws AnalysisException + */ + PartitionKeyDesc generateRollUpPartitionKeyDesc( + PartitionKeyDesc partitionKeyDesc, MTMVPartitionInfo mvPartitionInfo) + throws AnalysisException; + + /** + * Check if user input is legal + * + * @param mtmvPartitionInfo + * @throws AnalysisException + */ + void analyze(MTMVPartitionInfo mtmvPartitionInfo) throws AnalysisException; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java index 348f6e4de996716..7ca1b7e3e63748b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java @@ -17,10 +17,16 @@ package org.apache.doris.mtmv; +import org.apache.doris.analysis.Expr; +import org.apache.doris.catalog.Column; import org.apache.doris.common.AnalysisException; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; import com.google.gson.annotations.SerializedName; +import java.util.List; + /** * MTMVPartitionInfo */ @@ -28,17 +34,24 @@ public class MTMVPartitionInfo { public enum MTMVPartitionType { FOLLOW_BASE_TABLE, + EXPR, SELF_MANAGE } + public static final ImmutableSet MTMV_PARTITION_FUNCTIONS = new ImmutableSortedSet.Builder( + String.CASE_INSENSITIVE_ORDER).add("date_trunc") + .build(); + @SerializedName("pt") - MTMVPartitionType partitionType; + private MTMVPartitionType partitionType; @SerializedName("rt") - BaseTableInfo relatedTable; + private BaseTableInfo relatedTable; @SerializedName("rc") - String relatedCol; + private String relatedCol; @SerializedName("pc") - String partitionCol; + private String partitionCol; + @SerializedName("expr") + private Expr expr; public MTMVPartitionInfo() { } @@ -89,6 +102,14 @@ public void setPartitionCol(String partitionCol) { this.partitionCol = partitionCol; } + public Expr getExpr() { + return expr; + } + + public void setExpr(Expr expr) { + this.expr = expr; + } + /** * Get the position of relatedCol in the relatedTable partition column * @@ -99,7 +120,15 @@ public int getRelatedColPos() throws AnalysisException { if (partitionType == MTMVPartitionType.SELF_MANAGE) { throw new AnalysisException("partitionType is: " + partitionType); } - return MTMVPartitionUtil.getPos(getRelatedTable(), relatedCol); + List partitionColumns = getRelatedTable().getPartitionColumns(); + for (int i = 0; i < partitionColumns.size(); i++) { + if (partitionColumns.get(i).getName().equalsIgnoreCase(relatedCol)) { + return i; + } + } + throw new AnalysisException( + String.format("getRelatedColPos error, relatedCol: %s, partitionColumns: %s", relatedCol, + partitionColumns)); } // toString() is not easy to find where to call the method diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index b80c5fc283f4485..cd0312c419e814c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -27,12 +27,13 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.Partition; -import org.apache.doris.catalog.PartitionItem; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -56,6 +57,18 @@ public class MTMVPartitionUtil { private static final Pattern PARTITION_NAME_PATTERN = Pattern.compile("[^a-zA-Z0-9,]"); private static final String PARTITION_NAME_PREFIX = "p_"; + private static final List partitionDescGenerators = ImmutableList + .of( + // It is necessary to maintain this order, + // because some impl deal `PartitionItem`, and some impl deal `PartitionDesc` + // for example: if `MTMVRelatedPartitionDescOnePartitionColGenerator` not generate `PartitionDesc`, + // `MTMVRelatedPartitionDescRollUpGenerator` will not have parameter + new MTMVRelatedPartitionDescInitGenerator(), + new MTMVRelatedPartitionDescSyncLimitGenerator(), + new MTMVRelatedPartitionDescOnePartitionColGenerator(), + new MTMVRelatedPartitionDescRollUpGenerator() + ); + /** * Determine whether the partition is sync with retated partition and other baseTables * @@ -71,7 +84,7 @@ public static boolean isMTMVPartitionSync(MTMV mtmv, Long partitionId, Set Set tables, Set excludedTriggerTables) throws AnalysisException { boolean isSyncWithPartition = true; - if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE) { + if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable(); // if follow base table, not need compare with related table, only should compare with related partition excludedTriggerTables.add(relatedTable.getName()); @@ -96,7 +109,8 @@ public static boolean isMTMVPartitionSync(MTMV mtmv, Long partitionId, Set public static void alignMvPartition(MTMV mtmv) throws DdlException, AnalysisException { Map mtmvPartitionDescs = mtmv.generateMvPartitionDescs(); - Set relatedPartitionDescs = mtmv.generateRelatedPartitionDescs().keySet(); + Set relatedPartitionDescs = generateRelatedPartitionDescs(mtmv.getMvPartitionInfo(), + mtmv.getMvProperties()).keySet(); // drop partition of mtmv for (Entry entry : mtmvPartitionDescs.entrySet()) { if (!relatedPartitionDescs.contains(entry.getValue())) { @@ -115,19 +129,18 @@ public static void alignMvPartition(MTMV mtmv) /** * getPartitionDescsByRelatedTable when create MTMV * - * @param relatedTable * @param tableProperties - * @param relatedCol + * @param mvPartitionInfo * @return * @throws AnalysisException */ - public static List getPartitionDescsByRelatedTable(MTMVRelatedTableIf relatedTable, - Map tableProperties, String relatedCol, Map mvProperties) + public static List getPartitionDescsByRelatedTable( + Map tableProperties, MTMVPartitionInfo mvPartitionInfo, Map mvProperties) throws AnalysisException { - HashMap partitionProperties = Maps.newHashMap(); List res = Lists.newArrayList(); - Set relatedPartitionDescs = getRelatedPartitionDescs(relatedTable, relatedCol, - MTMVUtil.generateMTMVPartitionSyncConfigByProperties(mvProperties)); + HashMap partitionProperties = Maps.newHashMap(); + Set relatedPartitionDescs = generateRelatedPartitionDescs(mvPartitionInfo, mvProperties) + .keySet(); for (PartitionKeyDesc partitionKeyDesc : relatedPartitionDescs) { SinglePartitionDesc singlePartitionDesc = new SinglePartitionDesc(true, generatePartitionName(partitionKeyDesc), @@ -139,28 +152,18 @@ public static List getPartitionDescsByRelatedTable(MTMVRelated return res; } - private static Set getRelatedPartitionDescs(MTMVRelatedTableIf relatedTable, String relatedCol, - MTMVPartitionSyncConfig config) - throws AnalysisException { - int pos = getPos(relatedTable, relatedCol); - Set res = Sets.newHashSet(); - for (Entry entry : relatedTable.getPartitionItemsByTimeFilter(pos, config).entrySet()) { - PartitionKeyDesc partitionKeyDesc = entry.getValue().toPartitionKeyDesc(pos); - res.add(partitionKeyDesc); + public static Map> generateRelatedPartitionDescs(MTMVPartitionInfo mvPartitionInfo, + Map mvProperties) throws AnalysisException { + long start = System.currentTimeMillis(); + RelatedPartitionDescResult result = new RelatedPartitionDescResult(); + for (MTMVRelatedPartitionDescGeneratorService service : partitionDescGenerators) { + service.apply(mvPartitionInfo, mvProperties, result); } - return res; - } - - public static int getPos(MTMVRelatedTableIf relatedTable, String relatedCol) throws AnalysisException { - List partitionColumns = relatedTable.getPartitionColumns(); - for (int i = 0; i < partitionColumns.size(); i++) { - if (partitionColumns.get(i).getName().equalsIgnoreCase(relatedCol)) { - return i; - } + if (LOG.isDebugEnabled()) { + LOG.debug("generateRelatedPartitionDescs use [{}] mills, mvPartitionInfo is [{}]", + System.currentTimeMillis() - start, mvPartitionInfo); } - throw new AnalysisException( - String.format("getRelatedColPos error, relatedCol: %s, partitionColumns: %s", relatedCol, - partitionColumns)); + return result.getDescs(); } public static List getPartitionNamesByIds(MTMV mtmv, Collection ids) throws AnalysisException { @@ -258,7 +261,7 @@ private static List getPartitionUnSyncTables(MTMV mtmv, Long partitionId if (!mtmvRelatedTableIf.needAutoRefresh()) { continue; } - if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE && mtmv + if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE && mtmv .getMvPartitionInfo().getRelatedTableInfo().equals(baseTableInfo)) { if (CollectionUtils.isEmpty(relatedPartitionIds)) { throw new AnalysisException("can not found related partition"); @@ -469,7 +472,7 @@ private static MTMVRefreshPartitionSnapshot generatePartitionSnapshot(MTMV mtmv, Set baseTables, Set relatedPartitionIds) throws AnalysisException { MTMVRefreshPartitionSnapshot refreshPartitionSnapshot = new MTMVRefreshPartitionSnapshot(); - if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE) { + if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable(); for (Long relatedPartitionId : relatedPartitionIds) { MTMVSnapshotIf partitionSnapshot = relatedTable @@ -479,7 +482,7 @@ private static MTMVRefreshPartitionSnapshot generatePartitionSnapshot(MTMV mtmv, } } for (BaseTableInfo baseTableInfo : baseTables) { - if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE && mtmv + if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE && mtmv .getMvPartitionInfo().getRelatedTableInfo().equals(baseTableInfo)) { continue; } @@ -491,4 +494,14 @@ private static MTMVRefreshPartitionSnapshot generatePartitionSnapshot(MTMV mtmv, } return refreshPartitionSnapshot; } + + public static Type getPartitionColumnType(MTMVRelatedTableIf relatedTable, String col) throws AnalysisException { + List partitionColumns = relatedTable.getPartitionColumns(); + for (Column column : partitionColumns) { + if (column.getName().equals(col)) { + return column.getType(); + } + } + throw new AnalysisException("can not getPartitionColumnType by:" + col); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorService.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorService.java new file mode 100644 index 000000000000000..09d85576b5cba45 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorService.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.common.AnalysisException; + +import java.util.Map; + +/** + * Interface for a series of processes to generate PartitionDesc + */ +public interface MTMVRelatedPartitionDescGeneratorService { + /** + * generate related table PartitionDesc + * + * @param mvPartitionInfo PartitionInfo of MTMV + * @param mvProperties properties of MTMV + * @param lastResult the processing result of the previous process + * @throws AnalysisException + */ + void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, + RelatedPartitionDescResult lastResult) throws AnalysisException; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java new file mode 100644 index 000000000000000..13b58239376116e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescInitGenerator.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.common.AnalysisException; + +import java.util.Map; + +/** + * get all related partition descs + */ +public class MTMVRelatedPartitionDescInitGenerator implements MTMVRelatedPartitionDescGeneratorService { + + @Override + public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, + RelatedPartitionDescResult lastResult) throws AnalysisException { + lastResult.setItems(mvPartitionInfo.getRelatedTable().getAndCopyPartitionItems()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java new file mode 100644 index 000000000000000..ab14f302e756b82 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; + +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +/** + * For example, the related table is partitioned by `date` and `region`, with the following 6 partitions + *

+ * 20200101 beijing + * 20200101 shanghai + * 20200102 beijing + * 20200102 shanghai + * 20200103 beijing + * 20200103 shanghai + *

+ * If the MTMV is partitioned by `date`, then the MTMV will have three partitions: 20200101, 202000102, 20200103 + *

+ * If the MTMV is partitioned by `region`, then the MTMV will have two partitions: beijing, shanghai + */ +public class MTMVRelatedPartitionDescOnePartitionColGenerator implements MTMVRelatedPartitionDescGeneratorService { + + @Override + public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, + RelatedPartitionDescResult lastResult) throws AnalysisException { + if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { + return; + } + Map> res = Maps.newHashMap(); + Map relatedPartitionItems = lastResult.getItems(); + int relatedColPos = mvPartitionInfo.getRelatedColPos(); + for (Entry entry : relatedPartitionItems.entrySet()) { + PartitionKeyDesc partitionKeyDesc = entry.getValue().toPartitionKeyDesc(relatedColPos); + if (res.containsKey(partitionKeyDesc)) { + res.get(partitionKeyDesc).add(entry.getKey()); + } else { + res.put(partitionKeyDesc, Sets.newHashSet(entry.getKey())); + } + } + lastResult.setDescs(res); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java new file mode 100644 index 000000000000000..e9b4b1fe6a5262d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; + +import com.google.common.base.Preconditions; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +/** + * Roll up some partitions into one partition + */ +public class MTMVRelatedPartitionDescRollUpGenerator implements MTMVRelatedPartitionDescGeneratorService { + + @Override + public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, + RelatedPartitionDescResult lastResult) throws AnalysisException { + if (mvPartitionInfo.getPartitionType() != MTMVPartitionType.EXPR) { + return; + } + MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable(); + PartitionType partitionType = relatedTable.getPartitionType(); + if (partitionType == PartitionType.RANGE) { + lastResult.setDescs(rollUpRange(lastResult.getDescs(), mvPartitionInfo)); + } else if (partitionType == PartitionType.LIST) { + lastResult.setDescs(rollUpList(lastResult.getDescs(), mvPartitionInfo, mvProperties)); + } else { + throw new AnalysisException("only RANGE/LIST partition support roll up"); + } + } + + /** + * when related table has 3 partitions:(20200101),(20200102),(20200201) + *

+ * if expr is `date_trunc(month)` + * then,MTMV will have 2 partitions (20200101,20200102),(20200201) + *

+ * if expr is `date_trunc(year)` + * then,MTMV will have 1 partitions (20200101,20200102,20200201) + * + * @param relatedPartitionDescs + * @param mvPartitionInfo + * @return + * @throws AnalysisException + */ + public Map> rollUpList(Map> relatedPartitionDescs, + MTMVPartitionInfo mvPartitionInfo, Map mvProperties) throws AnalysisException { + Map> identityToValues = Maps.newHashMap(); + Map> identityToPartitionIds = Maps.newHashMap(); + MTMVPartitionExprService exprSerice = MTMVPartitionExprFactory.getExprService(mvPartitionInfo.getExpr()); + + for (Entry> entry : relatedPartitionDescs.entrySet()) { + String rollUpIdentity = exprSerice.getRollUpIdentity(entry.getKey(), mvProperties); + Preconditions.checkNotNull(rollUpIdentity); + if (identityToValues.containsKey(rollUpIdentity)) { + identityToValues.get(rollUpIdentity).addAll(getStringValues(entry.getKey())); + identityToPartitionIds.get(rollUpIdentity).addAll(entry.getValue()); + } else { + identityToValues.put(rollUpIdentity, getStringValues(entry.getKey())); + identityToPartitionIds.put(rollUpIdentity, entry.getValue()); + } + } + Map> result = Maps.newHashMap(); + for (Entry> entry : identityToValues.entrySet()) { + result.put(PartitionKeyDesc.createIn(getPartitionValues(entry.getValue())), + identityToPartitionIds.get(entry.getKey())); + } + return result; + } + + private List> getPartitionValues(Set strings) { + List> inValues = Lists.newArrayList(); + for (String value : strings) { + inValues.add(Lists.newArrayList(new PartitionValue(value))); + } + return inValues; + } + + private Set getStringValues(PartitionKeyDesc partitionKeyDesc) { + List> inValues = partitionKeyDesc.getInValues(); + Set res = Sets.newHashSet(); + for (List list : inValues) { + res.add(list.get(0).getStringValue()); + } + return res; + } + + /** + * when related table has 3 partitions:(20200101-20200102),(20200102-20200103),(20200201-20200202) + *

+ * if expr is `date_trunc(month)` + * then,MTMV will have 2 partitions (20200101-20200201),(20200101-20200301) + *

+ * if expr is `date_trunc(year)` + * then,MTMV will have 1 partitions (20200101-20210101) + * + * @param relatedPartitionDescs + * @param mvPartitionInfo + * @return + * @throws AnalysisException + */ + public Map> rollUpRange(Map> relatedPartitionDescs, + MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + Map> result = Maps.newHashMap(); + MTMVPartitionExprService exprSerice = MTMVPartitionExprFactory.getExprService(mvPartitionInfo.getExpr()); + for (Entry> entry : relatedPartitionDescs.entrySet()) { + PartitionKeyDesc rollUpDesc = exprSerice.generateRollUpPartitionKeyDesc(entry.getKey(), mvPartitionInfo); + if (result.containsKey(rollUpDesc)) { + result.get(rollUpDesc).addAll(entry.getValue()); + } else { + result.put(rollUpDesc, entry.getValue()); + } + } + return result; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java new file mode 100644 index 000000000000000..e031071192e3087 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java @@ -0,0 +1,142 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeAcquire; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import com.google.common.collect.Maps; +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; + +/** + * Only focus on partial partitions of related tables + */ +public class MTMVRelatedPartitionDescSyncLimitGenerator implements MTMVRelatedPartitionDescGeneratorService { + + @Override + public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, + RelatedPartitionDescResult lastResult) throws AnalysisException { + Map partitionItems = lastResult.getItems(); + MTMVPartitionSyncConfig config = generateMTMVPartitionSyncConfigByProperties(mvProperties); + if (config.getSyncLimit() <= 0) { + return; + } + long nowTruncSubSec = getNowTruncSubSec(config.getTimeUnit(), config.getSyncLimit()); + Optional dateFormat = config.getDateFormat(); + Map res = Maps.newHashMap(); + int relatedColPos = mvPartitionInfo.getRelatedColPos(); + for (Entry entry : partitionItems.entrySet()) { + if (entry.getValue().isGreaterThanSpecifiedTime(relatedColPos, dateFormat, nowTruncSubSec)) { + res.put(entry.getKey(), entry.getValue()); + } + } + lastResult.setItems(res); + } + + /** + * Generate MTMVPartitionSyncConfig based on mvProperties + * + * @param mvProperties + * @return + */ + public MTMVPartitionSyncConfig generateMTMVPartitionSyncConfigByProperties( + Map mvProperties) { + int syncLimit = StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT)) ? -1 + : Integer.parseInt(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT)); + MTMVPartitionSyncTimeUnit timeUnit = + StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT)) + ? MTMVPartitionSyncTimeUnit.DAY : MTMVPartitionSyncTimeUnit + .valueOf(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT).toUpperCase()); + Optional dateFormat = + StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)) + ? Optional.empty() + : Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)); + return new MTMVPartitionSyncConfig(syncLimit, timeUnit, dateFormat); + } + + /** + * Obtain the minimum second from `syncLimit` `timeUnit` ago + * + * @param timeUnit + * @param syncLimit + * @return + * @throws AnalysisException + */ + public long getNowTruncSubSec(MTMVPartitionSyncTimeUnit timeUnit, int syncLimit) + throws AnalysisException { + if (syncLimit < 1) { + throw new AnalysisException("Unexpected syncLimit, syncLimit: " + syncLimit); + } + // get current time + Expression now = DateTimeAcquire.now(); + if (!(now instanceof DateTimeLiteral)) { + throw new AnalysisException("now() should return DateTimeLiteral, now: " + now); + } + DateTimeLiteral nowLiteral = (DateTimeLiteral) now; + // date trunc + now = DateTimeExtractAndTransform + .dateTrunc(nowLiteral, new VarcharLiteral(timeUnit.name())); + if (!(now instanceof DateTimeLiteral)) { + throw new AnalysisException("dateTrunc() should return DateTimeLiteral, now: " + now); + } + nowLiteral = (DateTimeLiteral) now; + // date sub + if (syncLimit > 1) { + nowLiteral = dateSub(nowLiteral, timeUnit, syncLimit - 1); + } + return ((IntegerLiteral) DateTimeExtractAndTransform.unixTimestamp(nowLiteral)).getValue(); + } + + + private DateTimeLiteral dateSub( + org.apache.doris.nereids.trees.expressions.literal.DateLiteral date, MTMVPartitionSyncTimeUnit timeUnit, + int num) + throws AnalysisException { + IntegerLiteral integerLiteral = new IntegerLiteral(num); + Expression result; + switch (timeUnit) { + case DAY: + result = DateTimeArithmetic.dateSub(date, integerLiteral); + break; + case YEAR: + result = DateTimeArithmetic.yearsSub(date, integerLiteral); + break; + case MONTH: + result = DateTimeArithmetic.monthsSub(date, integerLiteral); + break; + default: + throw new AnalysisException("MTMV partition limit not support timeUnit: " + timeUnit.name()); + } + if (!(result instanceof DateTimeLiteral)) { + throw new AnalysisException("sub() should return DateTimeLiteral, result: " + result); + } + return (DateTimeLiteral) result; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java index 1adfe315a8baa02..ec99a04d73f9142 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java @@ -24,12 +24,8 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import com.google.common.collect.Maps; - import java.util.List; import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; import java.util.Set; /** @@ -44,31 +40,6 @@ public interface MTMVRelatedTableIf extends TableIf { */ Map getAndCopyPartitionItems(); - /** - * Obtain a list of partitions filtered by time - * - * @param pos The position of the partition column to be checked in all partition columns - * @param config - * @return - * @throws AnalysisException - */ - default Map getPartitionItemsByTimeFilter(int pos, MTMVPartitionSyncConfig config) - throws AnalysisException { - Map partitionItems = getAndCopyPartitionItems(); - if (config.getSyncLimit() <= 0) { - return partitionItems; - } - long nowTruncSubSec = MTMVUtil.getNowTruncSubSec(config.getTimeUnit(), config.getSyncLimit()); - Optional dateFormat = config.getDateFormat(); - Map res = Maps.newHashMap(); - for (Entry entry : partitionItems.entrySet()) { - if (entry.getValue().isGreaterThanSpecifiedTime(pos, dateFormat, nowTruncSubSec)) { - res.put(entry.getKey(), entry.getValue()); - } - } - return res; - } - /** * getPartitionType LIST/RANGE/UNPARTITIONED * diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java index 01033a9615a45d6..ddbe763fdfb845a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java @@ -25,22 +25,15 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; -import org.apache.doris.common.util.PropertyAnalyzer; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeAcquire; -import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; -import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; -import org.apache.commons.lang3.StringUtils; - -import java.util.Map; import java.util.Optional; import java.util.Set; @@ -61,6 +54,20 @@ public static TableIf getTable(BaseTableInfo baseTableInfo) throws AnalysisExcep return table; } + public static MTMVRelatedTableIf getRelatedTable(BaseTableInfo baseTableInfo) { + TableIf relatedTable = null; + try { + relatedTable = MTMVUtil.getTable(baseTableInfo); + } catch (org.apache.doris.common.AnalysisException e) { + throw new org.apache.doris.nereids.exceptions.AnalysisException(e.getMessage(), e); + } + if (!(relatedTable instanceof MTMVRelatedTableIf)) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "base table for partitioning only can be OlapTable or HMSTable"); + } + return (MTMVRelatedTableIf) relatedTable; + } + public static MTMV getMTMV(long dbId, long mtmvId) throws DdlException, MetaNotFoundException { Database db = Env.getCurrentInternalCatalog().getDbOrDdlException(dbId); return (MTMV) db.getTableOrMetaException(mtmvId, TableType.MATERIALIZED_VIEW); @@ -82,64 +89,6 @@ public static boolean mtmvContainsExternalTable(MTMV mtmv) { return false; } - /** - * Obtain the minimum second from `syncLimit` `timeUnit` ago - * - * @param timeUnit - * @param syncLimit - * @return - * @throws AnalysisException - */ - public static long getNowTruncSubSec(MTMVPartitionSyncTimeUnit timeUnit, int syncLimit) - throws AnalysisException { - if (syncLimit < 1) { - throw new AnalysisException("Unexpected syncLimit, syncLimit: " + syncLimit); - } - // get current time - Expression now = DateTimeAcquire.now(); - if (!(now instanceof DateTimeLiteral)) { - throw new AnalysisException("now() should return DateTimeLiteral, now: " + now); - } - DateTimeLiteral nowLiteral = (DateTimeLiteral) now; - // date trunc - now = DateTimeExtractAndTransform - .dateTrunc(nowLiteral, new VarcharLiteral(timeUnit.name())); - if (!(now instanceof DateTimeLiteral)) { - throw new AnalysisException("dateTrunc() should return DateTimeLiteral, now: " + now); - } - nowLiteral = (DateTimeLiteral) now; - // date sub - if (syncLimit > 1) { - nowLiteral = dateSub(nowLiteral, timeUnit, syncLimit - 1); - } - return ((IntegerLiteral) DateTimeExtractAndTransform.unixTimestamp(nowLiteral)).getValue(); - } - - private static DateTimeLiteral dateSub( - org.apache.doris.nereids.trees.expressions.literal.DateLiteral date, MTMVPartitionSyncTimeUnit timeUnit, - int num) - throws AnalysisException { - IntegerLiteral integerLiteral = new IntegerLiteral(num); - Expression result; - switch (timeUnit) { - case DAY: - result = DateTimeArithmetic.dateSub(date, integerLiteral); - break; - case YEAR: - result = DateTimeArithmetic.yearsSub(date, integerLiteral); - break; - case MONTH: - result = DateTimeArithmetic.monthsSub(date, integerLiteral); - break; - default: - throw new AnalysisException("MTMV partition limit not support timeUnit: " + timeUnit.name()); - } - if (!(result instanceof DateTimeLiteral)) { - throw new AnalysisException("sub() should return DateTimeLiteral, result: " + result); - } - return (DateTimeLiteral) result; - } - /** * Convert LiteralExpr to second * @@ -177,25 +126,4 @@ public static long getExprTimeSec(org.apache.doris.analysis.LiteralExpr expr, Op expr.getStringValue(), dateFormat)); } } - - /** - * Generate MTMVPartitionSyncConfig based on mvProperties - * - * @param mvProperties - * @return - */ - public static MTMVPartitionSyncConfig generateMTMVPartitionSyncConfigByProperties( - Map mvProperties) { - int syncLimit = StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT)) ? -1 - : Integer.parseInt(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT)); - MTMVPartitionSyncTimeUnit timeUnit = - StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT)) - ? MTMVPartitionSyncTimeUnit.DAY : MTMVPartitionSyncTimeUnit - .valueOf(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT).toUpperCase()); - Optional dateFormat = - StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)) - ? Optional.empty() - : Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)); - return new MTMVPartitionSyncConfig(syncLimit, timeUnit, dateFormat); - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/RelatedPartitionDescResult.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/RelatedPartitionDescResult.java new file mode 100644 index 000000000000000..068cf1522a71d19 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/RelatedPartitionDescResult.java @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.catalog.PartitionItem; + +import com.google.common.collect.Maps; + +import java.util.Map; +import java.util.Set; + +public class RelatedPartitionDescResult { + // PartitionKeyDesc to relatedTable partition ids(Different partitions may have the same PartitionKeyDesc) + private Map> descs; + private Map items; + + public RelatedPartitionDescResult() { + this.descs = Maps.newHashMap(); + this.items = Maps.newHashMap(); + } + + public Map> getDescs() { + return descs; + } + + public void setDescs(Map> descs) { + this.descs = descs; + } + + public Map getItems() { + return items; + } + + public void setItems(Map items) { + this.items = items; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 76bd59970856d4d..2263951e097784f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -33,7 +33,6 @@ import org.apache.doris.common.Pair; import org.apache.doris.job.common.IntervalUnit; import org.apache.doris.load.loadv2.LoadTask; -import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; import org.apache.doris.mtmv.MTMVRefreshEnum.BuildMode; import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; @@ -127,6 +126,7 @@ import org.apache.doris.nereids.DorisParser.MapLiteralContext; import org.apache.doris.nereids.DorisParser.MultiStatementsContext; import org.apache.doris.nereids.DorisParser.MultipartIdentifierContext; +import org.apache.doris.nereids.DorisParser.MvPartitionContext; import org.apache.doris.nereids.DorisParser.NamedExpressionContext; import org.apache.doris.nereids.DorisParser.NamedExpressionSeqContext; import org.apache.doris.nereids.DorisParser.NullLiteralContext; @@ -406,6 +406,7 @@ import org.apache.doris.nereids.trees.plans.commands.info.InPartition; import org.apache.doris.nereids.trees.plans.commands.info.IndexDefinition; import org.apache.doris.nereids.trees.plans.commands.info.LessThanPartition; +import org.apache.doris.nereids.trees.plans.commands.info.MTMVPartitionDefinition; import org.apache.doris.nereids.trees.plans.commands.info.PartitionDefinition; import org.apache.doris.nereids.trees.plans.commands.info.PartitionDefinition.MaxValue; import org.apache.doris.nereids.trees.plans.commands.info.PauseMTMVInfo; @@ -635,22 +636,29 @@ public CreateMTMVCommand visitCreateMTMV(CreateMTMVContext ctx) { desc, properties, logicalPlan, querySql, new MTMVRefreshInfo(buildMode, refreshMethod, refreshTriggerInfo), ctx.cols == null ? Lists.newArrayList() : visitSimpleColumnDefs(ctx.cols), - visitMTMVPartitionInfo(ctx.partitionKey) + visitMTMVPartitionInfo(ctx.mvPartition()) )); } /** - * get MTMVPartitionInfo + * get MTMVPartitionDefinition * - * @param ctx IdentifierContext - * @return MTMVPartitionInfo + * @param ctx MvPartitionContext + * @return MTMVPartitionDefinition */ - public MTMVPartitionInfo visitMTMVPartitionInfo(IdentifierContext ctx) { + public MTMVPartitionDefinition visitMTMVPartitionInfo(MvPartitionContext ctx) { + MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition(); if (ctx == null) { - return new MTMVPartitionInfo(MTMVPartitionType.SELF_MANAGE); + mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.SELF_MANAGE); + } else if (ctx.partitionKey != null) { + mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.FOLLOW_BASE_TABLE); + mtmvPartitionDefinition.setPartitionCol(ctx.partitionKey.getText()); } else { - return new MTMVPartitionInfo(MTMVPartitionType.FOLLOW_BASE_TABLE, ctx.getText()); + mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.EXPR); + Expression functionCallExpression = visitFunctionCallExpression(ctx.partitionExpr); + mtmvPartitionDefinition.setFunctionCallExpression(functionCallExpression); } + return mtmvPartitionDefinition; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index b56265b999a1982..7d86ff9a8b1f9b4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -30,10 +30,8 @@ import org.apache.doris.catalog.PartitionType; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.View; -import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.FeNameFormat; -import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.mtmv.EnvInfo; import org.apache.doris.mtmv.MTMVPartitionInfo; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; @@ -45,14 +43,12 @@ import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.mysql.privilege.PrivPredicate; -import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; import org.apache.doris.nereids.trees.TreeNode; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; @@ -78,7 +74,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -103,9 +98,10 @@ public class CreateMTMVInfo { private final List columns = Lists.newArrayList(); private final List simpleColumnDefinitions; private final EnvInfo envInfo; - private final MTMVPartitionInfo mvPartitionInfo; + private final MTMVPartitionDefinition mvPartitionDefinition; private PartitionDesc partitionDesc; private MTMVRelation relation; + private MTMVPartitionInfo mvPartitionInfo; /** * constructor for create MTMV @@ -116,7 +112,7 @@ public CreateMTMVInfo(boolean ifNotExists, TableNameInfo mvName, LogicalPlan logicalQuery, String querySql, MTMVRefreshInfo refreshInfo, List simpleColumnDefinitions, - MTMVPartitionInfo mvPartitionInfo) { + MTMVPartitionDefinition mvPartitionDefinition) { this.ifNotExists = Objects.requireNonNull(ifNotExists, "require ifNotExists object"); this.mvName = Objects.requireNonNull(mvName, "require mvName object"); this.keys = Utils.copyRequiredList(keys); @@ -130,8 +126,8 @@ public CreateMTMVInfo(boolean ifNotExists, TableNameInfo mvName, .requireNonNull(simpleColumnDefinitions, "require simpleColumnDefinitions object"); this.envInfo = new EnvInfo(ConnectContext.get().getCurrentCatalog().getId(), ConnectContext.get().getCurrentDbId()); - this.mvPartitionInfo = Objects - .requireNonNull(mvPartitionInfo, "require mtmvPartitionInfo object"); + this.mvPartitionDefinition = Objects + .requireNonNull(mvPartitionDefinition, "require mtmvPartitionInfo object"); } /** @@ -212,7 +208,9 @@ public void analyzeQuery(ConnectContext ctx) { } getRelation(planner); getColumns(plan); - analyzePartition(planner, ctx); + this.mvPartitionInfo = mvPartitionDefinition + .analyzeAndTransferToMTMVPartitionInfo(planner, ctx, logicalQuery); + this.partitionDesc = generatePartitionDesc(ctx); } private void getRelation(NereidsPlanner planner) { @@ -235,67 +233,17 @@ private void getRelation(NereidsPlanner planner) { this.relation = MTMVPlanUtil.generateMTMVRelation(plan); } - private void analyzePartition(NereidsPlanner planner, ConnectContext ctx) { - if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.FOLLOW_BASE_TABLE) { - - CascadesContext cascadesContext = planner.getCascadesContext(); - SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); - Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); - // Should not make table without data to empty relation when analyze the related table, - // so add disable rules - sessionVariable.setDisableNereidsRules(MTMV_PLANER_DISABLE_RULES); - cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - try { - Plan mvRewrittenPlan = - planner.plan(logicalQuery, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); - Optional relatedTableInfo = MaterializedViewUtils - .getRelatedTableInfo(mvPartitionInfo.getPartitionCol(), mvRewrittenPlan); - if (!relatedTableInfo.isPresent() || !relatedTableInfo.get().isPctPossible()) { - throw new AnalysisException("Unable to find a suitable base table for partitioning"); - } - TableIf relatedTable = null; - try { - relatedTable = MTMVUtil.getTable(relatedTableInfo.get().getTableInfo()); - } catch (org.apache.doris.common.AnalysisException e) { - throw new AnalysisException(e.getMessage(), e); - } - if (!(relatedTable instanceof MTMVRelatedTableIf)) { - throw new AnalysisException("base table for partitioning only can be OlapTable or HMSTable"); - } - MTMVRelatedTableIf mtmvBaseRealtedTable = (MTMVRelatedTableIf) relatedTable; - Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); - try { - partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames()); - } catch (DdlException e) { - throw new AnalysisException(e.getMessage(), e); - } - - if (!partitionColumnNames.contains(relatedTableInfo.get().getColumn())) { - throw new AnalysisException("error related column: " + relatedTableInfo.get().getColumn()); - } - if (!(mtmvBaseRealtedTable instanceof HMSExternalTable) - && partitionColumnNames.size() != 1) { - throw new AnalysisException("only hms table support multi column partition."); - } - mvPartitionInfo.setRelatedTable(relatedTableInfo.get().getTableInfo()); - mvPartitionInfo.setRelatedCol(relatedTableInfo.get().getColumn()); - partitionDesc = generatePartitionDesc(mtmvBaseRealtedTable, ctx); - } finally { - // after operate, roll back the disable rules - sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); - cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); - } + private PartitionDesc generatePartitionDesc(ConnectContext ctx) { + if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { + return null; } - } - - private PartitionDesc generatePartitionDesc(MTMVRelatedTableIf relatedTable, ConnectContext ctx) { + MTMVRelatedTableIf relatedTable = MTMVUtil.getRelatedTable(mvPartitionInfo.getRelatedTableInfo()); List allPartitionDescs = null; try { allPartitionDescs = MTMVPartitionUtil - .getPartitionDescsByRelatedTable(relatedTable, properties, mvPartitionInfo.getRelatedCol(), - mvProperties); + .getPartitionDescsByRelatedTable(properties, mvPartitionInfo, mvProperties); } catch (org.apache.doris.common.AnalysisException e) { - throw new AnalysisException("getPartitionDescsByRelatedTable failed", e); + throw new AnalysisException(e.getMessage(), e); } if (allPartitionDescs.size() > ctx.getSessionVariable().getCreateTablePartitionMaxNum()) { throw new AnalysisException(String.format( @@ -316,7 +264,7 @@ private PartitionDesc generatePartitionDesc(MTMVRelatedTableIf relatedTable, Con return null; } } catch (org.apache.doris.common.AnalysisException e) { - throw new AnalysisException("can not generate partitionDesc", e); + throw new AnalysisException(e.getMessage(), e); } } @@ -333,7 +281,7 @@ private void analyzeBaseTables(Plan plan) { throw new AnalysisException("can not contain VIEW"); } } catch (org.apache.doris.common.AnalysisException e) { - LOG.warn("can not get table, ", e); + LOG.warn(e.getMessage(), e); } } } @@ -362,7 +310,7 @@ private void getColumns(Plan plan) { try { FeNameFormat.checkColumnName(colName); } catch (org.apache.doris.common.AnalysisException e) { - throw new AnalysisException(e.getMessage()); + throw new AnalysisException(e.getMessage(), e); } if (colNames.contains(colName)) { throw new AnalysisException("repeat cols:" + colName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java new file mode 100644 index 000000000000000..09b30063b9de06b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/Expr.java +// and modified by Doris + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.FunctionParams; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.mtmv.MTMVPartitionExprFactory; +import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVRelatedTableIf; +import org.apache.doris.mtmv.MTMVUtil; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.Sets; + +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * MTMVPartitionDefinition + */ +public class MTMVPartitionDefinition { + private MTMVPartitionType partitionType; + private String partitionCol; + private Expression functionCallExpression; + + /** + * analyzeAndTransferToMTMVPartitionInfo + * + * @param planner planner + * @param ctx ctx + * @param logicalQuery logicalQuery + * @return MTMVPartitionInfo + */ + public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx, + LogicalPlan logicalQuery) { + MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(partitionType); + if (this.partitionType == MTMVPartitionType.SELF_MANAGE) { + return mtmvPartitionInfo; + } + String partitionColName; + if (this.partitionType == MTMVPartitionType.EXPR) { + Expr expr; + if (functionCallExpression instanceof UnboundFunction) { + UnboundFunction function = (UnboundFunction) functionCallExpression; + expr = new FunctionCallExpr(function.getName(), + new FunctionParams(convertToLegacyArguments(function.children()))); + } else { + throw new AnalysisException( + "unsupported auto partition expr " + functionCallExpression.toString()); + } + partitionColName = getColNameFromExpr(expr); + mtmvPartitionInfo.setExpr(expr); + } else { + partitionColName = this.partitionCol; + } + mtmvPartitionInfo.setPartitionCol(partitionColName); + RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, logicalQuery, partitionColName); + mtmvPartitionInfo.setRelatedCol(relatedTableInfo.getColumn()); + mtmvPartitionInfo.setRelatedTable(relatedTableInfo.getTableInfo()); + if (this.partitionType == MTMVPartitionType.EXPR) { + try { + MTMVPartitionExprFactory.getExprService(mtmvPartitionInfo.getExpr()).analyze(mtmvPartitionInfo); + } catch (org.apache.doris.common.AnalysisException e) { + throw new AnalysisException(e.getMessage(), e); + } + } + return mtmvPartitionInfo; + } + + /** + * getColNameFromExpr + * + * @param expr expr + * @return String + */ + public static String getColNameFromExpr(Expr expr) { + if (!(expr instanceof FunctionCallExpr)) { + throw new AnalysisException( + "auto create partition only support function call expr is: " + + MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS); + } + FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; + List paramsExpr = functionCallExpr.getParams().exprs(); + String name = functionCallExpr.getFnName().getFunction(); + if (MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS.contains(name)) { + for (Expr param : paramsExpr) { + if (param instanceof SlotRef) { + return ((SlotRef) param).getColumnName(); + } + } + throw new AnalysisException("can not find colName"); + } else { + throw new AnalysisException( + "auto create partition only support function call expr is: " + + MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS); + } + } + + private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, LogicalPlan + logicalQuery, + String partitionColName) { + CascadesContext cascadesContext = planner.getCascadesContext(); + SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); + Set tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); + // Should not make table without data to empty relation when analyze the related table, + // so add disable rules + sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); + cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + try { + Plan mvRewrittenPlan = + planner.plan(logicalQuery, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN); + Optional relatedTableInfo = MaterializedViewUtils + .getRelatedTableInfo(partitionColName, mvRewrittenPlan); + if (!relatedTableInfo.isPresent() || !relatedTableInfo.get().isPctPossible()) { + throw new AnalysisException("Unable to find a suitable base table for partitioning"); + } + MTMVRelatedTableIf mtmvBaseRealtedTable = MTMVUtil.getRelatedTable(relatedTableInfo.get().getTableInfo()); + Set partitionColumnNames = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + try { + partitionColumnNames.addAll(mtmvBaseRealtedTable.getPartitionColumnNames()); + } catch (DdlException e) { + throw new AnalysisException(e.getMessage(), e); + } + + if (!partitionColumnNames.contains(relatedTableInfo.get().getColumn())) { + throw new AnalysisException("error related column: " + relatedTableInfo.get().getColumn()); + } + if (!(mtmvBaseRealtedTable instanceof HMSExternalTable) + && partitionColumnNames.size() != 1) { + throw new AnalysisException("only hms table support multi column partition."); + } + return relatedTableInfo.get(); + } finally { + // after operate, roll back the disable rules + sessionVariable.setDisableNereidsRules(String.join(",", tempDisableRules)); + cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); + } + } + + private static List convertToLegacyArguments(List children) { + return children.stream().map(child -> { + if (child instanceof UnboundSlot) { + return new SlotRef(null, ((UnboundSlot) child).getName()); + } else if (child instanceof Literal) { + return new StringLiteral(((Literal) child).getStringValue()); + } else { + throw new AnalysisException("unsupported argument " + child.toString()); + } + }).collect(Collectors.toList()); + } + + public MTMVPartitionType getPartitionType() { + return partitionType; + } + + public void setPartitionType(MTMVPartitionType partitionType) { + this.partitionType = partitionType; + } + + public String getPartitionCol() { + return partitionCol; + } + + public void setPartitionCol(String partitionCol) { + this.partitionCol = partitionCol; + } + + public Expression getFunctionCallExpression() { + return functionCallExpression; + } + + public void setFunctionCallExpression(Expression functionCallExpression) { + this.functionCallExpression = functionCallExpression; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGeneratorTest.java new file mode 100644 index 000000000000000..b866100b63d4e72 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGeneratorTest.java @@ -0,0 +1,152 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class MTMVRelatedPartitionDescRollUpGeneratorTest { + @Mocked + private MTMVPartitionUtil mtmvPartitionUtil; + @Mocked + private MTMVPartitionInfo mtmvPartitionInfo; + + @Test + public void testRollUpRange() throws AnalysisException { + FunctionCallExpr expr = new FunctionCallExpr("date_trunc", + Lists.newArrayList(new SlotRef(null, null), new StringLiteral("month"))); + new Expectations() { + { + mtmvPartitionUtil.getPartitionColumnType((MTMVRelatedTableIf) any, (String) any); + minTimes = 0; + result = Type.DATE; + + mtmvPartitionInfo.getRelatedTable(); + minTimes = 0; + result = null; + + mtmvPartitionInfo.getExpr(); + minTimes = 0; + result = null; + + mtmvPartitionInfo.getPartitionType(); + minTimes = 0; + result = MTMVPartitionType.EXPR; + + mtmvPartitionInfo.getExpr(); + minTimes = 0; + result = expr; + } + }; + MTMVRelatedPartitionDescRollUpGenerator generator = new MTMVRelatedPartitionDescRollUpGenerator(); + Map> relatedPartitionDescs = Maps.newHashMap(); + PartitionKeyDesc desc20200101 = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2020-01-01")), + Lists.newArrayList(new PartitionValue("2020-01-02"))); + PartitionKeyDesc desc20200102 = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2020-01-02")), + Lists.newArrayList(new PartitionValue("2020-01-03"))); + PartitionKeyDesc desc20200201 = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2020-02-01")), + Lists.newArrayList(new PartitionValue("2020-02-02"))); + relatedPartitionDescs.put(desc20200101, Sets.newHashSet(1L)); + relatedPartitionDescs.put(desc20200102, Sets.newHashSet(2L)); + relatedPartitionDescs.put(desc20200201, Sets.newHashSet(3L)); + Map> res = generator.rollUpRange(relatedPartitionDescs, + mtmvPartitionInfo); + + PartitionKeyDesc expectDesc202001 = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2020-01-01")), + Lists.newArrayList(new PartitionValue("2020-02-01"))); + PartitionKeyDesc expectDesc202002 = PartitionKeyDesc.createFixed( + Lists.newArrayList(new PartitionValue("2020-02-01")), + Lists.newArrayList(new PartitionValue("2020-03-01"))); + Assert.assertEquals(2, res.size()); + Assert.assertEquals(Sets.newHashSet(1L, 2L), res.get(expectDesc202001)); + Assert.assertEquals(Sets.newHashSet(3L), res.get(expectDesc202002)); + } + + @Test + public void testRollUpList() throws AnalysisException { + FunctionCallExpr expr = new FunctionCallExpr("date_trunc", + Lists.newArrayList(new SlotRef(null, null), new StringLiteral("month"))); + new Expectations() { + { + mtmvPartitionUtil.getPartitionColumnType((MTMVRelatedTableIf) any, (String) any); + minTimes = 0; + result = Type.DATE; + + mtmvPartitionInfo.getRelatedTable(); + minTimes = 0; + result = null; + + mtmvPartitionInfo.getExpr(); + minTimes = 0; + result = null; + + mtmvPartitionInfo.getPartitionType(); + minTimes = 0; + result = MTMVPartitionType.EXPR; + + mtmvPartitionInfo.getExpr(); + minTimes = 0; + result = expr; + } + }; + MTMVRelatedPartitionDescRollUpGenerator generator = new MTMVRelatedPartitionDescRollUpGenerator(); + Map> relatedPartitionDescs = Maps.newHashMap(); + relatedPartitionDescs.put(generateInDesc("2020-01-01"), Sets.newHashSet(1L)); + relatedPartitionDescs.put(generateInDesc("2020-01-02"), Sets.newHashSet(2L)); + relatedPartitionDescs.put(generateInDesc("2020-02-01"), Sets.newHashSet(3L)); + Map> res = generator.rollUpList(relatedPartitionDescs, + mtmvPartitionInfo, Maps.newHashMap()); + + PartitionKeyDesc expectDesc202001 = generateInDesc("2020-01-01", "2020-01-02"); + PartitionKeyDesc expectDesc202002 = generateInDesc("2020-02-01"); + Assert.assertEquals(2, res.size()); + Assert.assertEquals(Sets.newHashSet(1L, 2L), res.get(expectDesc202001)); + Assert.assertEquals(Sets.newHashSet(3L), res.get(expectDesc202002)); + } + + private PartitionKeyDesc generateInDesc(String... values) { + List> partitionValues = Lists.newArrayList(); + for (String value : values) { + List partitionValue = Lists.newArrayList(new PartitionValue(value)); + partitionValues.add(partitionValue); + } + return PartitionKeyDesc.createIn(partitionValues); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java new file mode 100644 index 000000000000000..666a7947bcb691e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGeneratorTest.java @@ -0,0 +1,94 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeAcquire; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; + +import com.google.common.collect.Maps; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.Assert; +import org.junit.Test; + +import java.util.Map; + + +public class MTMVRelatedPartitionDescSyncLimitGeneratorTest { + @Mocked + private DateTimeAcquire dateTimeAcquire; + + @Test + public void testGenerateMTMVPartitionSyncConfigByProperties() throws AnalysisException { + MTMVRelatedPartitionDescSyncLimitGenerator generator = new MTMVRelatedPartitionDescSyncLimitGenerator(); + Map mvProperties = Maps.newHashMap(); + MTMVPartitionSyncConfig config = generator + .generateMTMVPartitionSyncConfigByProperties(mvProperties); + Assert.assertEquals(-1, config.getSyncLimit()); + Assert.assertFalse(config.getDateFormat().isPresent()); + Assert.assertEquals(MTMVPartitionSyncTimeUnit.DAY, config.getTimeUnit()); + + mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT, "1"); + config = generator.generateMTMVPartitionSyncConfigByProperties(mvProperties); + Assert.assertEquals(1, config.getSyncLimit()); + Assert.assertFalse(config.getDateFormat().isPresent()); + Assert.assertEquals(MTMVPartitionSyncTimeUnit.DAY, config.getTimeUnit()); + + mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT, "month"); + config = generator.generateMTMVPartitionSyncConfigByProperties(mvProperties); + Assert.assertEquals(1, config.getSyncLimit()); + Assert.assertFalse(config.getDateFormat().isPresent()); + Assert.assertEquals(MTMVPartitionSyncTimeUnit.MONTH, config.getTimeUnit()); + + mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT, "%Y%m%d"); + config = generator.generateMTMVPartitionSyncConfigByProperties(mvProperties); + Assert.assertEquals(1, config.getSyncLimit()); + Assert.assertEquals("%Y%m%d", config.getDateFormat().get()); + Assert.assertEquals(MTMVPartitionSyncTimeUnit.MONTH, config.getTimeUnit()); + } + + @Test + public void testGetNowTruncSubSec() throws AnalysisException { + MTMVRelatedPartitionDescSyncLimitGenerator generator = new MTMVRelatedPartitionDescSyncLimitGenerator(); + DateTimeLiteral dateTimeLiteral = new DateTimeLiteral("2020-02-03 20:10:10"); + new Expectations() { + { + dateTimeAcquire.now(); + minTimes = 0; + result = dateTimeLiteral; + } + }; + long nowTruncSubSec = generator.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.DAY, 1); + // 2020-02-03 + Assert.assertEquals(1580659200L, nowTruncSubSec); + nowTruncSubSec = generator.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.MONTH, 1); + // 2020-02-01 + Assert.assertEquals(1580486400L, nowTruncSubSec); + nowTruncSubSec = generator.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.YEAR, 1); + // 2020-01-01 + Assert.assertEquals(1577808000L, nowTruncSubSec); + nowTruncSubSec = generator.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.MONTH, 3); + // 2019-12-01 + Assert.assertEquals(1575129600L, nowTruncSubSec); + nowTruncSubSec = generator.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.DAY, 4); + // 2020-01-31 + Assert.assertEquals(1580400000L, nowTruncSubSec); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilTest.java index 2e6df56cd1a4b53..2a1b2d0c1401e73 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVUtilTest.java @@ -23,51 +23,13 @@ import org.apache.doris.analysis.StringLiteral; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.util.PropertyAnalyzer; -import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeAcquire; -import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; -import com.google.common.collect.Maps; -import mockit.Expectations; -import mockit.Mocked; import org.junit.Assert; import org.junit.Test; -import java.util.Map; import java.util.Optional; public class MTMVUtilTest { - @Mocked - private DateTimeAcquire dateTimeAcquire; - - @Test - public void testGenerateMTMVPartitionSyncConfigByProperties() throws AnalysisException { - Map mvProperties = Maps.newHashMap(); - MTMVPartitionSyncConfig config = MTMVUtil - .generateMTMVPartitionSyncConfigByProperties(mvProperties); - Assert.assertEquals(-1, config.getSyncLimit()); - Assert.assertFalse(config.getDateFormat().isPresent()); - Assert.assertEquals(MTMVPartitionSyncTimeUnit.DAY, config.getTimeUnit()); - - mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_SYNC_LIMIT, "1"); - config = MTMVUtil.generateMTMVPartitionSyncConfigByProperties(mvProperties); - Assert.assertEquals(1, config.getSyncLimit()); - Assert.assertFalse(config.getDateFormat().isPresent()); - Assert.assertEquals(MTMVPartitionSyncTimeUnit.DAY, config.getTimeUnit()); - - mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_TIME_UNIT, "month"); - config = MTMVUtil.generateMTMVPartitionSyncConfigByProperties(mvProperties); - Assert.assertEquals(1, config.getSyncLimit()); - Assert.assertFalse(config.getDateFormat().isPresent()); - Assert.assertEquals(MTMVPartitionSyncTimeUnit.MONTH, config.getTimeUnit()); - - mvProperties.put(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT, "%Y%m%d"); - config = MTMVUtil.generateMTMVPartitionSyncConfigByProperties(mvProperties); - Assert.assertEquals(1, config.getSyncLimit()); - Assert.assertEquals("%Y%m%d", config.getDateFormat().get()); - Assert.assertEquals(MTMVPartitionSyncTimeUnit.MONTH, config.getTimeUnit()); - } - @Test public void testGetExprTimeSec() throws AnalysisException { LiteralExpr expr = new DateLiteral("2020-01-01"); @@ -83,31 +45,4 @@ public void testGetExprTimeSec() throws AnalysisException { exprTimeSec = MTMVUtil.getExprTimeSec(expr, Optional.empty()); Assert.assertEquals(253402185600L, exprTimeSec); } - - @Test - public void testGetNowTruncSubSec() throws AnalysisException { - DateTimeLiteral dateTimeLiteral = new DateTimeLiteral("2020-02-03 20:10:10"); - new Expectations() { - { - dateTimeAcquire.now(); - minTimes = 0; - result = dateTimeLiteral; - } - }; - long nowTruncSubSec = MTMVUtil.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.DAY, 1); - // 2020-02-03 - Assert.assertEquals(1580659200L, nowTruncSubSec); - nowTruncSubSec = MTMVUtil.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.MONTH, 1); - // 2020-02-01 - Assert.assertEquals(1580486400L, nowTruncSubSec); - nowTruncSubSec = MTMVUtil.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.YEAR, 1); - // 2020-01-01 - Assert.assertEquals(1577808000L, nowTruncSubSec); - nowTruncSubSec = MTMVUtil.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.MONTH, 3); - // 2019-12-01 - Assert.assertEquals(1575129600L, nowTruncSubSec); - nowTruncSubSec = MTMVUtil.getNowTruncSubSec(MTMVPartitionSyncTimeUnit.DAY, 4); - // 2020-01-31 - Assert.assertEquals(1580400000L, nowTruncSubSec); - } } diff --git a/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out b/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out index 5b63ad2f7b68802..1a8731d6a1c4461 100644 --- a/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out +++ b/regression-test/data/mtmv_p0/test_hive_limit_partition_mtmv.out @@ -4,17 +4,31 @@ 2 sh 20380101 3 bj 20200101 4 sh 20200101 +5 bj 20380102 -- !mtmv_complete -- 1 20380101 bj 2 20380101 sh +5 20380102 bj + +-- !mtmv_datetrunc -- +1 20380101 bj +2 20380101 sh +5 20380102 bj -- !select_base_table -- 1 bj 20380101 2 sh 20380101 3 bj 20200101 4 sh 20200101 +5 bj 20380102 -- !mtmv_complete -- 1 20380101 bj 2 20380101 sh +5 20380102 bj + +-- !mtmv_datetrunc -- +1 20380101 bj +2 20380101 sh +5 20380102 bj diff --git a/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out new file mode 100644 index 000000000000000..5552dac72c30050 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_rollup_partition_mtmv.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !date_list_month -- +1 2020-01-01 +2 2020-01-02 +3 2020-02-01 + +-- !string_list_month -- +1 2020==01==01 +2 2020==01==02 +3 2020==02==01 + +-- !date_range_month -- +1 2020-01-01 +2 2020-01-02 +3 2020-02-01 + diff --git a/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy index 06390437a79e160..1e6c49bb50af0e9 100644 --- a/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_limit_partition_mtmv.groovy @@ -45,11 +45,13 @@ suite("test_hive_limit_partition_mtmv", "p0,external,hive,external_docker,extern partition(region="sh",day="20380101") partition(region="bj",day="20200101") partition(region="sh",day="20200101") + partition(region="bj",day="20380102") """ def insert_str1 = """insert into ${hive_database}.${hive_table} PARTITION(region="bj",day="20380101") values(1)""" def insert_str2 = """insert into ${hive_database}.${hive_table} PARTITION(region="sh",day="20380101") values(2)""" def insert_str3 = """insert into ${hive_database}.${hive_table} PARTITION(region="bj",day="20200101") values(3)""" def insert_str4 = """insert into ${hive_database}.${hive_table} PARTITION(region="sh",day="20200101") values(4)""" + def insert_str5 = """insert into ${hive_database}.${hive_table} PARTITION(region="bj",day="20380102") values(5)""" logger.info("hive sql: " + drop_table_str) hive_docker """ ${drop_table_str} """ @@ -69,12 +71,15 @@ suite("test_hive_limit_partition_mtmv", "p0,external,hive,external_docker,extern hive_docker """ ${insert_str3} """ logger.info("hive sql: " + insert_str4) hive_docker """ ${insert_str4} """ + logger.info("hive sql: " + insert_str5) + hive_docker """ ${insert_str5} """ // prepare catalog String hms_port = context.config.otherConfigs.get(hivePrefix + "HmsPort") String catalog_name = "test_${hivePrefix}_limit_partition_mtmv_catalog" String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + sql """drop catalog if exists ${catalog_name}""" sql """create catalog if not exists ${catalog_name} properties ( "type"="hms", @@ -83,7 +88,6 @@ suite("test_hive_limit_partition_mtmv", "p0,external,hive,external_docker,extern order_qt_select_base_table "SELECT * FROM ${catalog_name}.${hive_database}.${hive_table}" - // string type def mvName = "test_hive_limit_partition_mtmv" def dbName = "regression_test_mtmv_p0" @@ -105,62 +109,85 @@ suite("test_hive_limit_partition_mtmv", "p0,external,hive,external_docker,extern """ def showPartitionsResult = sql """show partitions from ${mvName}""" logger.info("showPartitionsResult: " + showPartitionsResult.toString()) - assertEquals(1, showPartitionsResult.size()) + assertEquals(2, showPartitionsResult.size()) assertTrue(showPartitionsResult.toString().contains("p_20380101")) + assertTrue(showPartitionsResult.toString().contains("p_20380102")) - // refresh complete - sql """ - REFRESH MATERIALIZED VIEW ${mvName} complete + // date trunc + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`day`,'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'partition_sync_limit'='2', + 'partition_sync_time_unit'='MONTH', + 'partition_date_format'='%Y%m%d' + ) + AS + SELECT k1,day,region FROM ${catalog_name}.${hive_database}.${hive_table}; """ - def jobName = getJobName(dbName, mvName); - waitingMTMVTaskFinished(jobName) - order_qt_mtmv_complete "SELECT * FROM ${mvName} order by k1,day,region" + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(1, showPartitionsResult.size()) + assertTrue(showPartitionsResult.toString().contains("_20380101")) + assertTrue(showPartitionsResult.toString().contains("_20380102")) + // refresh complete + sql """ + REFRESH MATERIALIZED VIEW ${mvName} complete + """ + jobName = getJobName(dbName, mvName); + waitingMTMVTaskFinished(jobName) + order_qt_mtmv_datetrunc "SELECT * FROM ${mvName} order by k1,day,region" - // date type - sql """drop materialized view if exists ${mvName};""" - create_table_str = """ CREATE TABLE ${hive_database}.${hive_table} ( - `k1` int) - PARTITIONED BY ( - `region` string, - `day` date - ) - STORED AS ORC; - """ - add_partition_str = """ - alter table ${hive_database}.${hive_table} add if not exists - partition(region="bj",day="2038-01-01") - partition(region="sh",day="2038-01-01") - partition(region="bj",day="2020-01-01") - partition(region="sh",day="2020-01-01") - """ - logger.info("hive sql: " + drop_table_str) - hive_docker """ ${drop_table_str} """ - logger.info("hive sql: " + create_table_str) - hive_docker """ ${create_table_str} """ - logger.info("hive sql: " + add_partition_str) - hive_docker """ ${add_partition_str} """ - sql """REFRESH catalog ${catalog_name}""" - sql """ - CREATE MATERIALIZED VIEW ${mvName} - BUILD DEFERRED REFRESH AUTO ON MANUAL - partition by(`day`) - DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ( - 'replication_num' = '1', - 'partition_sync_limit'='2', - 'partition_sync_time_unit'='YEAR' - ) - AS - SELECT k1,day,region FROM ${catalog_name}.${hive_database}.${hive_table}; - """ - showPartitionsResult = sql """show partitions from ${mvName}""" - logger.info("showPartitionsResult: " + showPartitionsResult.toString()) - assertEquals(1, showPartitionsResult.size()) - assertTrue(showPartitionsResult.toString().contains("p_20380101")) - sql """drop materialized view if exists ${mvName};""" - sql """drop catalog if exists ${catalog_name}""" + // date type + sql """drop materialized view if exists ${mvName};""" + create_table_str = """ CREATE TABLE ${hive_database}.${hive_table} ( + `k1` int) + PARTITIONED BY ( + `region` string, + `day` date + ) + STORED AS ORC; + """ + add_partition_str = """ + alter table ${hive_database}.${hive_table} add if not exists + partition(region="bj",day="2038-01-01") + partition(region="sh",day="2038-01-01") + partition(region="bj",day="2020-01-01") + partition(region="sh",day="2020-01-01") + """ + logger.info("hive sql: " + drop_table_str) + hive_docker """ ${drop_table_str} """ + logger.info("hive sql: " + create_table_str) + hive_docker """ ${create_table_str} """ + logger.info("hive sql: " + add_partition_str) + hive_docker """ ${add_partition_str} """ + + sql """REFRESH catalog ${catalog_name}""" + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(`day`) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'partition_sync_limit'='2', + 'partition_sync_time_unit'='YEAR' + ) + AS + SELECT k1,day,region FROM ${catalog_name}.${hive_database}.${hive_table}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(1, showPartitionsResult.size()) + assertTrue(showPartitionsResult.toString().contains("p_20380101")) + sql """drop materialized view if exists ${mvName};""" + sql """drop catalog if exists ${catalog_name}""" } } diff --git a/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy new file mode 100644 index 000000000000000..3a1bfe9f4fdd5a0 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_rollup_partition_mtmv.groovy @@ -0,0 +1,292 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.junit.Assert; + +suite("test_rollup_partition_mtmv") { + def tableName = "t_test_rollup_partition_mtmv_user" + def mvName = "multi_mv_test_rollup_partition_mtmv" + def dbName = "regression_test_mtmv_p0" + + // list partition date type + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` DATE NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY list(`k2`) + ( + PARTITION p_20200101 VALUES IN ("2020-01-01"), + PARTITION p_20200102 VALUES IN ("2020-01-02"), + PARTITION p_20200201 VALUES IN ("2020-02-01") + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableName} values(1,"2020-01-01"),(2,"2020-01-02"),(3,"2020-02-01"); + """ + + // list date month + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`,'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * FROM ${tableName}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(2, showPartitionsResult.size()) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + def jobName = getJobName(dbName, mvName); + log.info(jobName) + waitingMTMVTaskFinished(jobName) + order_qt_date_list_month "SELECT * FROM ${mvName} order by k1,k2" + + sql """drop materialized view if exists ${mvName};""" + // list date year + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`,'year')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * FROM ${tableName}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(1, showPartitionsResult.size()) + + // list string month + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` varchar(200) NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY list(`k2`) + ( + PARTITION p_20200101 VALUES IN ("2020==01==01"), + PARTITION p_20200102 VALUES IN ("2020==01==02"), + PARTITION p_20200201 VALUES IN ("2020==02==01") + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableName} values(1,"2020==01==01"),(2,"2020==01==02"),(3,"2020==02==01"); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`,'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1', + 'partition_date_format'='%Y==%m==%d' + ) + AS + SELECT * FROM ${tableName}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(2, showPartitionsResult.size()) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + jobName = getJobName(dbName, mvName); + log.info(jobName) + waitingMTMVTaskFinished(jobName) + order_qt_string_list_month "SELECT * FROM ${mvName} order by k1,k2" + + + // range date month + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` DATE NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY range(`k2`) + ( + PARTITION p_20200101 VALUES [("2020-01-01"),("2020-01-02")), + PARTITION p_20200102 VALUES [("2020-01-02"),("2020-01-03")), + PARTITION p_20200201 VALUES [("2020-02-01"),("2020-02-02")) + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + sql """ + insert into ${tableName} values(1,"2020-01-01"),(2,"2020-01-02"),(3,"2020-02-01"); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`,'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * FROM ${tableName}; + """ + showPartitionsResult = sql """show partitions from ${mvName}""" + logger.info("showPartitionsResult: " + showPartitionsResult.toString()) + assertEquals(2, showPartitionsResult.size()) + + sql """ + REFRESH MATERIALIZED VIEW ${mvName} AUTO + """ + jobName = getJobName(dbName, mvName); + log.info(jobName) + waitingMTMVTaskFinished(jobName) + order_qt_date_range_month "SELECT * FROM ${mvName} order by k1,k2" + + // not support MAXVALUE + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` DATE NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY range(`k2`) + ( + PARTITION p_20200101 VALUES [("2020-01-01"),("2020-01-02")), + PARTITION p_20200102 VALUES [("2020-01-02"),("2020-01-03")), + PARTITION p_20200201 VALUES [("2020-02-01"),(MAXVALUE)) + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`,'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + + // range not support other data type + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` int NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY range(`k2`) + ( + PARTITION p_1 VALUES [(1),(2)) + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`,'month')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } + + // not support trunc hour + sql """drop table if exists `${tableName}`""" + sql """drop materialized view if exists ${mvName};""" + sql """ + CREATE TABLE `${tableName}` ( + `k1` LARGEINT NOT NULL COMMENT '\"用户id\"', + `k2` DATE NOT NULL COMMENT '\"数据灌入日期时间\"' + ) ENGINE=OLAP + DUPLICATE KEY(`k1`) + COMMENT 'OLAP' + PARTITION BY range(`k2`) + ( + PARTITION p_20200101 VALUES [("2020-01-01"),("2020-01-02")), + PARTITION p_20200102 VALUES [("2020-01-02"),("2020-01-03")), + PARTITION p_20200201 VALUES [("2020-02-01"),("2020-02-02")) + ) + DISTRIBUTED BY HASH(`k1`) BUCKETS 2 + PROPERTIES ('replication_num' = '1') ; + """ + + try { + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by (date_trunc(`k2`,'hour')) + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS + SELECT * FROM ${tableName}; + """ + Assert.fail(); + } catch (Exception e) { + log.info(e.getMessage()) + } +} From 38bac76b376c317a536a5d4b604594107dbea6f5 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Tue, 14 May 2024 10:51:15 +0800 Subject: [PATCH 018/111] [opt](mtmv) Cache materialization check result for performance (#34301) Need check materialization sql pattern in different abstract rule when rewrite by materialized view. Such as the subClass of AbstractMaterializedViewJoinRule, MaterializedViewScanRule, AbstractMaterializedViewAggregateRule. This check result can be cached when has checked, this can avoid unnecessary repeat check --- .../org/apache/doris/nereids/memo/Memo.java | 46 +++++++++++++++++++ .../mv/AbstractMaterializedViewRule.java | 40 ++++++++++++++-- 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java index d7d46ecc15e4395..09a0ca69840dfed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Memo.java @@ -30,6 +30,7 @@ import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.properties.RequestPropertyDeriver; import org.apache.doris.nereids.properties.RequirePropertiesSupplier; +import org.apache.doris.nereids.rules.exploration.mv.AbstractMaterializedViewRule; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.GroupPlan; import org.apache.doris.nereids.trees.plans.LeafPlan; @@ -53,6 +54,7 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -74,6 +76,10 @@ public class Memo { private static long stateId = 0; private final ConnectContext connectContext; private final AtomicLong refreshVersion = new AtomicLong(1); + private final Map, Set> materializationCheckSuccessMap = + new LinkedHashMap<>(); + private final Map, Set> materializationCheckFailMap = + new LinkedHashMap<>(); private final IdGenerator groupIdGenerator = GroupId.createGenerator(); private final Map groups = Maps.newLinkedHashMap(); // we could not use Set, because Set does not have get method. @@ -127,6 +133,46 @@ public long getRefreshVersion() { return refreshVersion.get(); } + /** + * Record materialization check result for performance + */ + public void recordMaterializationCheckResult(Class target, + Long checkedMaterializationId, boolean isSuccess) { + if (isSuccess) { + Set checkedSet = materializationCheckSuccessMap.get(target); + if (checkedSet == null) { + checkedSet = new HashSet<>(); + materializationCheckSuccessMap.put(target, checkedSet); + } + checkedSet.add(checkedMaterializationId); + } else { + Set checkResultSet = materializationCheckFailMap.get(target); + if (checkResultSet == null) { + checkResultSet = new HashSet<>(); + materializationCheckFailMap.put(target, checkResultSet); + } + checkResultSet.add(checkedMaterializationId); + } + } + + /** + * Get the info for materialization context is checked + * + * @return if true, check successfully, if false check fail, if null not checked + */ + public Boolean materializationHasChecked(Class target, + long materializationId) { + Set checkSuccessSet = materializationCheckSuccessMap.get(target); + if (checkSuccessSet != null && checkSuccessSet.contains(materializationId)) { + return true; + } + Set checkFailSet = materializationCheckFailMap.get(target); + if (checkFailSet != null && checkFailSet.contains(materializationId)) { + return false; + } + return null; + } + private Plan skipProject(Plan plan, Group targetGroup) { // Some top project can't be eliminated if (plan instanceof LogicalProject && ((LogicalProject) plan).canEliminate()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index dcff9db69db86f4..41157a6c86346f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -108,11 +108,7 @@ public List rewrite(Plan queryPlan, CascadesContext cascadesContext) { continue; } // check mv plan is valid or not - boolean valid = checkPattern(context.getStructInfo()) && context.getStructInfo().isValid(); - if (!valid) { - context.recordFailReason(context.getStructInfo(), - "View struct info is invalid", () -> String.format(", view plan is %s", - context.getStructInfo().getOriginalPlan().treeString())); + if (!isMaterializationValid(cascadesContext, context)) { continue; } // get query struct infos according to the view strut info, if valid query struct infos is empty, bail out @@ -668,6 +664,40 @@ protected boolean checkIfRewritten(Plan plan, MaterializationContext context) { && context.alreadyRewrite(plan.getGroupExpression().get().getOwnerGroup().getGroupId()); } + // check mv plan is valid or not, this can use cache for performance + private boolean isMaterializationValid(CascadesContext cascadesContext, MaterializationContext context) { + long materializationId = context.getMaterializationQualifier().hashCode(); + Boolean cachedCheckResult = cascadesContext.getMemo().materializationHasChecked(this.getClass(), + materializationId); + if (cachedCheckResult == null) { + // need check in real time + boolean checkResult = checkPattern(context.getStructInfo()); + if (!checkResult) { + context.recordFailReason(context.getStructInfo(), + "View struct info is invalid", () -> String.format("view plan is %s", + context.getStructInfo().getOriginalPlan().treeString())); + cascadesContext.getMemo().recordMaterializationCheckResult(this.getClass(), materializationId, + false); + return false; + } else { + cascadesContext.getMemo().recordMaterializationCheckResult(this.getClass(), + materializationId, true); + } + } else if (!cachedCheckResult) { + context.recordFailReason(context.getStructInfo(), + "View struct info is invalid", () -> String.format("view plan is %s", + context.getStructInfo().getOriginalPlan().treeString())); + return false; + } + if (!context.getStructInfo().isValid()) { + context.recordFailReason(context.getStructInfo(), + "View struct info is invalid", () -> String.format("view plan is %s", + context.getStructInfo().getOriginalPlan().treeString())); + return false; + } + return true; + } + /** * Query and mv match node */ From 6b1c441258a237185769da69a7d0c32e0a2b3c8d Mon Sep 17 00:00:00 2001 From: huanghaibin <284824253@qq.com> Date: Tue, 14 May 2024 16:48:27 +0800 Subject: [PATCH 019/111] [fix](group_commit) Wal reader should check block length to avoid reading empty block (#34792) --- be/src/olap/wal/wal_reader.cpp | 3 +++ be/src/vec/exec/format/wal/wal_reader.cpp | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/be/src/olap/wal/wal_reader.cpp b/be/src/olap/wal/wal_reader.cpp index 9e4618b2bc1000b..fa96f0c5a0bad92 100644 --- a/be/src/olap/wal/wal_reader.cpp +++ b/be/src/olap/wal/wal_reader.cpp @@ -69,6 +69,9 @@ Status WalReader::read_block(PBlock& block) { file_reader->read_at(_offset, {row_len_buf, WalWriter::LENGTH_SIZE}, &bytes_read)); _offset += WalWriter::LENGTH_SIZE; size_t block_len = decode_fixed64_le(row_len_buf); + if (block_len == 0) { + return Status::DataQualityError("fail to read wal {} ,block is empty", _file_name); + } // read block std::string block_buf; block_buf.resize(block_len); diff --git a/be/src/vec/exec/format/wal/wal_reader.cpp b/be/src/vec/exec/format/wal/wal_reader.cpp index 0d15c198b0e3a1e..5010f1912abf121 100644 --- a/be/src/vec/exec/format/wal/wal_reader.cpp +++ b/be/src/vec/exec/format/wal/wal_reader.cpp @@ -68,6 +68,11 @@ Status WalReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { LOG(WARNING) << "Failed to read wal on path = " << _wal_path; return st; } + int be_exec_version = pblock.has_be_exec_version() ? pblock.be_exec_version() : 0; + if (!BeExecVersionManager::check_be_exec_version(be_exec_version)) { + return Status::DataQualityError("check be exec version fail when reading wal file {}", + _wal_path); + } vectorized::Block src_block; RETURN_IF_ERROR(src_block.deserialize(pblock)); //convert to dst block From 1e53a2a81d19e6a1c2b67b33c6b300e34c789ba8 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Tue, 14 May 2024 17:04:24 +0800 Subject: [PATCH 020/111] [Improve](inverted index) improve query performance by not using output index result column (#34281) --- be/src/olap/rowset/segment_v2/segment_iterator.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index 3dbb8b694ae706e..a74fb305795c310 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -2265,7 +2265,9 @@ Status SegmentIterator::_next_batch_internal(vectorized::Block* block) { RETURN_IF_ERROR(_convert_to_expected_type(_first_read_column_ids)); RETURN_IF_ERROR(_convert_to_expected_type(_non_predicate_columns)); _output_non_pred_columns(block); - _output_index_result_column(nullptr, 0, block); + if (!_enable_common_expr_pushdown || !_remaining_conjunct_roots.empty()) { + _output_index_result_column(nullptr, 0, block); + } } else { uint16_t selected_size = _current_batch_rows_read; uint16_t sel_rowid_idx[selected_size]; From 4a8df535537e8eab8fa2ad54934a185e17d4e660 Mon Sep 17 00:00:00 2001 From: Pxl Date: Wed, 15 May 2024 10:25:22 +0800 Subject: [PATCH 021/111] [Chore](rollup) check duplicate column name when create table with rollup (#34827) check duplicate column name when create table with rollup --- .../plans/commands/info/CreateTableInfo.java | 4 ++++ .../plans/commands/info/RollupDefinition.java | 14 ++++++++++- .../test_dup_mv_useless.groovy | 23 +++++++++++++++++++ .../nereids_p0/create_table/ddl/table.sql | 2 +- 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index a4cf08efe744afa..585d1da0b101d58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -459,6 +459,10 @@ public void validate(ConnectContext ctx) { } } } + + for (RollupDefinition rollup : rollups) { + rollup.validate(); + } } else { // mysql, broker and hive do not need key desc if (keysType != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RollupDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RollupDefinition.java index 6c3857279f5e341..b01e380e9048279 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RollupDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RollupDefinition.java @@ -18,12 +18,15 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.analysis.AddRollupClause; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.Maps; +import com.google.common.collect.Sets; import java.util.List; import java.util.Map; +import java.util.Set; /** * rollup definition @@ -41,7 +44,16 @@ public RollupDefinition(String name, List cols, List dupKeys, Ma this.properties = Maps.newHashMap(properties); } - public void validate() { + /** + * check rollup validity + */ + public void validate() throws AnalysisException { + Set colSet = Sets.newHashSet(); + for (String col : cols) { + if (!colSet.add(col)) { + throw new AnalysisException("rollup has duplicate column name " + col); + } + } } public AddRollupClause translateToCatalogStyle() { diff --git a/regression-test/suites/mv_p0/test_mv_useless/test_dup_mv_useless.groovy b/regression-test/suites/mv_p0/test_mv_useless/test_dup_mv_useless.groovy index 7640b6e18064f6d..d91cafbe93d83a7 100644 --- a/regression-test/suites/mv_p0/test_mv_useless/test_dup_mv_useless.groovy +++ b/regression-test/suites/mv_p0/test_mv_useless/test_dup_mv_useless.groovy @@ -48,4 +48,27 @@ suite ("test_dup_mv_useless") { createMV("create materialized view k1_k2_u21 as select k2,k1 from ${testTable} group by k2,k1 order by k2,k1;") createMV("create materialized view k1_k2_sumk3 as select k1,k2,sum(k3) from ${testTable} group by k1,k2;") sql "insert into ${testTable} select 4,4,4;" + + test { + sql """ + create table test_rollup ( + `id` int not null, + `kbool` boolean not null, + `ktint` tinyint(4) not null, + `ksint` smallint(6) not null, + `kint` int(11) not null, + `kbint` bigint(20) not null, + `klint` largeint(40) not null + ) engine=OLAP + duplicate key(id, kbool, ktint) + distributed by random buckets auto + rollup ( + r1 (id, ktint, kbool, ktint, kbint) duplicate key(id) + ) + properties ( + "replication_num"="1" + ); + """ + exception "duplicate column name" + } } diff --git a/regression-test/suites/nereids_p0/create_table/ddl/table.sql b/regression-test/suites/nereids_p0/create_table/ddl/table.sql index bcac1168b731a68..c1f244071780791 100644 --- a/regression-test/suites/nereids_p0/create_table/ddl/table.sql +++ b/regression-test/suites/nereids_p0/create_table/ddl/table.sql @@ -238,7 +238,7 @@ create table test_rollup ( duplicate key(id, kbool, ktint) distributed by random buckets auto rollup ( - r1 (id, ktint, kbool, ktint, kbint) duplicate key(id) + r1 (id, ktint, kbool, kbint) duplicate key(id) ) properties ( "replication_num"="1" From 71caf88ec1563e01329fa0202f2a7cf854d94f6d Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Wed, 15 May 2024 13:57:28 +0800 Subject: [PATCH 022/111] [opt](mtmv) Optimize the logic of slot mapping generate for performance (#34597) Slot mapping is used for materialized view rewritting given the relation mapping, the slot mapping is the same Optimize the slot mapping genarate logic Cache the slot mapping in materialization context by realation mapping key --- .../mv/AbstractMaterializedViewRule.java | 9 +++- .../mv/LogicalCompatibilityContext.java | 7 +-- .../mv/MaterializationContext.java | 13 +++++ .../mv/mapping/ExpressionMapping.java | 23 --------- .../rules/exploration/mv/mapping/Mapping.java | 24 ++++++--- .../mv/mapping/RelationMapping.java | 18 +++++++ .../exploration/mv/mapping/SlotMapping.java | 50 +++++++++++-------- 7 files changed, 86 insertions(+), 58 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 41157a6c86346f0..a88362e66976ec8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -179,7 +179,12 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca return rewriteResults; } for (RelationMapping queryToViewTableMapping : queryToViewTableMappings) { - SlotMapping queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + SlotMapping queryToViewSlotMapping = + materializationContext.getSlotMappingFromCache(queryToViewTableMapping); + if (queryToViewSlotMapping == null) { + queryToViewSlotMapping = SlotMapping.generate(queryToViewTableMapping); + materializationContext.addSlotMappingToCache(queryToViewTableMapping, queryToViewSlotMapping); + } if (queryToViewSlotMapping == null) { materializationContext.recordFailReason(queryStructInfo, "Query to view slot mapping is null", () -> ""); @@ -187,7 +192,7 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca } SlotMapping viewToQuerySlotMapping = queryToViewSlotMapping.inverse(); LogicalCompatibilityContext compatibilityContext = LogicalCompatibilityContext.from( - queryToViewTableMapping, queryToViewSlotMapping, queryStructInfo, viewStructInfo); + queryToViewTableMapping, viewToQuerySlotMapping, queryStructInfo, viewStructInfo); ComparisonResult comparisonResult = StructInfo.isGraphLogicalEquals(queryStructInfo, viewStructInfo, compatibilityContext); if (comparisonResult.isInvalid()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index b4ed509f300bf70..25bafeb64c16f48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -128,7 +128,7 @@ public ObjectId getPlanNodeId() { * this make expression mapping between query and view by relation and the slot in relation mapping */ public static LogicalCompatibilityContext from(RelationMapping relationMapping, - SlotMapping queryToViewSlotMapping, + SlotMapping viewToQuerySlotMapping, StructInfo queryStructInfo, StructInfo viewStructInfo) { // init node mapping @@ -147,11 +147,8 @@ public static LogicalCompatibilityContext from(RelationMapping relationMapping, queryToViewNodeMapping.put(queryStructInfoNode, viewStructInfoNode); } } - // init expression mapping - Map viewToQuerySlotMapping = queryToViewSlotMapping.inverse() - .toSlotReferenceMap(); return new LogicalCompatibilityContext(queryToViewNodeMapping, - viewToQuerySlotMapping, + viewToQuerySlotMapping.toSlotReferenceMap(), queryStructInfo, viewStructInfo); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 261e3bb85f9f1d4..2f0d04e1141f195 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -23,6 +23,8 @@ import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.memo.GroupId; import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.ObjectId; import org.apache.doris.nereids.trees.plans.Plan; @@ -40,8 +42,10 @@ import org.apache.logging.log4j.Logger; import java.util.BitSet; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.function.Supplier; @@ -52,6 +56,7 @@ */ public abstract class MaterializationContext { private static final Logger LOG = LogManager.getLogger(MaterializationContext.class); + public final Map queryToMvSlotMappingCache = new HashMap<>(); protected List

baseTables; protected List
baseViews; // The plan of mv def sql @@ -140,6 +145,14 @@ public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) { } } + public void addSlotMappingToCache(RelationMapping relationMapping, SlotMapping slotMapping) { + queryToMvSlotMappingCache.put(relationMapping, slotMapping); + } + + public SlotMapping getSlotMappingFromCache(RelationMapping relationMapping) { + return queryToMvSlotMappingCache.get(relationMapping); + } + /** * Try to generate scan plan for materialization * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java index 5a5bfedfe170ce1..8c77eacfaf0945b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/ExpressionMapping.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ArrayListMultimap; -import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.Lists; import com.google.common.collect.Multimap; @@ -32,7 +31,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Map.Entry; /** * Expression mapping, maybe one expression map to multi expression @@ -104,27 +102,6 @@ public static ExpressionMapping generate( return new ExpressionMapping(expressionMultiMap); } - @Override - public Mapping chainedFold(Mapping target) { - - ImmutableMultimap.Builder foldedMappingBuilder = - ImmutableMultimap.builder(); - - Multimap targetMapping - = ((ExpressionMapping) target).getExpressionMapping(); - for (Entry> exprMapping : - this.getExpressionMapping().asMap().entrySet()) { - Collection valueExpressions = exprMapping.getValue(); - valueExpressions.forEach(valueExpr -> { - if (targetMapping.containsKey(valueExpr)) { - targetMapping.get(valueExpr).forEach( - targetValue -> foldedMappingBuilder.put(exprMapping.getKey(), targetValue)); - } - }); - } - return new ExpressionMapping(foldedMappingBuilder.build()); - } - @Override public String toString() { return Utils.toSqlString("ExpressionMapping", "expressionMapping", expressionMapping); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java index 18fa282267c2cc8..e14b79a52eaade3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/Mapping.java @@ -22,6 +22,8 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation; +import java.util.HashMap; +import java.util.Map; import java.util.Objects; import javax.annotation.Nullable; @@ -38,10 +40,15 @@ public static final class MappedRelation { public final RelationId relationId; public final CatalogRelation belongedRelation; + // Generate eagerly, will be used to generate slot mapping + private final Map slotNameToSlotMap = new HashMap<>(); public MappedRelation(RelationId relationId, CatalogRelation belongedRelation) { this.relationId = relationId; this.belongedRelation = belongedRelation; + for (Slot slot : belongedRelation.getOutput()) { + slotNameToSlotMap.put(slot.getName(), slot); + } } public static MappedRelation of(RelationId relationId, CatalogRelation belongedRelation) { @@ -56,6 +63,10 @@ public CatalogRelation getBelongedRelation() { return belongedRelation; } + public Map getSlotNameToSlotMap() { + return slotNameToSlotMap; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -72,6 +83,11 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(relationId); } + + @Override + public String toString() { + return "MappedRelation{" + "relationId=" + relationId + ", slotNameToSlotMap=" + slotNameToSlotMap + '}'; + } } /** @@ -141,12 +157,4 @@ public String toString() { return "MappedSlot{" + "slot=" + slot + '}'; } } - - /** Chain fold tow mapping, such as this mapping is {[a -> b]}, the target mapping is - * {[b -> c]} after chain fold, this result will be {[a -> c]}, if the value side in this mapping - * can get the key in the target mapping, will lose the mapping - */ - protected Mapping chainedFold(Mapping target) { - return null; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java index b5494c01b1b9243..eb53923da53cf8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/RelationMapping.java @@ -32,6 +32,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.Set; /** @@ -143,4 +144,21 @@ public static RelationMapping merge(List> private static Long getTableQualifier(TableIf tableIf) { return tableIf.getId(); } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + RelationMapping that = (RelationMapping) o; + return Objects.equals(mappedRelationMap, that.mappedRelationMap); + } + + @Override + public int hashCode() { + return Objects.hash(mappedRelationMap); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java index f95bcedd2fab5e2..8384b3e094f61ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/SlotMapping.java @@ -22,10 +22,11 @@ import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.HashMap; import java.util.Map; -import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -33,11 +34,15 @@ */ public class SlotMapping extends Mapping { + public static final Logger LOG = LogManager.getLogger(SlotMapping.class); + private final BiMap relationSlotMap; private Map slotReferenceMap; - public SlotMapping(BiMap relationSlotMap) { + public SlotMapping(BiMap relationSlotMap, + Map slotReferenceMap) { this.relationSlotMap = relationSlotMap; + this.slotReferenceMap = slotReferenceMap; } public BiMap getRelationSlotMap() { @@ -45,11 +50,12 @@ public BiMap getRelationSlotMap() { } public SlotMapping inverse() { - return SlotMapping.of(relationSlotMap.inverse()); + return SlotMapping.of(relationSlotMap.inverse(), null); } - public static SlotMapping of(BiMap relationSlotMap) { - return new SlotMapping(relationSlotMap); + public static SlotMapping of(BiMap relationSlotMap, + Map slotReferenceMap) { + return new SlotMapping(relationSlotMap, slotReferenceMap); } /** @@ -58,26 +64,31 @@ public static SlotMapping of(BiMap relationSlotMap) { @Nullable public static SlotMapping generate(RelationMapping relationMapping) { BiMap relationSlotMap = HashBiMap.create(); + Map slotReferenceMap = new HashMap<>(); BiMap mappedRelationMap = relationMapping.getMappedRelationMap(); for (Map.Entry mappedRelationEntry : mappedRelationMap.entrySet()) { - Map targetNameSlotMap = - mappedRelationEntry.getValue().getBelongedRelation().getOutput().stream() - .collect(Collectors.toMap(Slot::getName, slot -> slot)); - for (Slot sourceSlot : mappedRelationEntry.getKey().getBelongedRelation().getOutput()) { - Slot targetSlot = targetNameSlotMap.get(sourceSlot.getName()); + MappedRelation sourceRelation = mappedRelationEntry.getKey(); + Map sourceSlotNameToSlotMap = sourceRelation.getSlotNameToSlotMap(); + + MappedRelation targetRelation = mappedRelationEntry.getValue(); + Map targetSlotNameSlotMap = targetRelation.getSlotNameToSlotMap(); + + for (String sourceSlotName : sourceSlotNameToSlotMap.keySet()) { + Slot targetSlot = targetSlotNameSlotMap.get(sourceSlotName); // source slot can not map from target, bail out if (targetSlot == null) { + LOG.warn(String.format("SlotMapping generate is null, source relation is %s, " + + "target relation is %s", sourceRelation, targetRelation)); return null; } - relationSlotMap.put(MappedSlot.of(sourceSlot, mappedRelationEntry.getKey().getBelongedRelation()), - MappedSlot.of(targetSlot, mappedRelationEntry.getValue().getBelongedRelation())); + Slot sourceSlot = sourceSlotNameToSlotMap.get(sourceSlotName); + relationSlotMap.put(MappedSlot.of(sourceSlot, + sourceRelation.getBelongedRelation()), + MappedSlot.of(targetSlot, targetRelation.getBelongedRelation())); + slotReferenceMap.put((SlotReference) sourceSlot, (SlotReference) targetSlot); } } - return SlotMapping.of(relationSlotMap); - } - - public Map toMappedSlotMap() { - return (Map) this.getRelationSlotMap(); + return SlotMapping.of(relationSlotMap, slotReferenceMap); } /** @@ -87,12 +98,11 @@ public Map toSlotReferenceMap() { if (this.slotReferenceMap != null) { return this.slotReferenceMap; } - Map slotReferenceSlotReferenceMap = new HashMap<>(); + this.slotReferenceMap = new HashMap<>(); for (Map.Entry entry : this.getRelationSlotMap().entrySet()) { - slotReferenceSlotReferenceMap.put((SlotReference) entry.getKey().getSlot(), + this.slotReferenceMap.put((SlotReference) entry.getKey().getSlot(), (SlotReference) entry.getValue().getSlot()); } - this.slotReferenceMap = slotReferenceSlotReferenceMap; return this.slotReferenceMap; } From 5719f6ff0c6119b752fef130d023d0309571022f Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Wed, 15 May 2024 16:39:42 +0800 Subject: [PATCH 023/111] [fix](planner) fix date_xxx functions without complete function signature (#34761) Problem: When using current_date as input of functions like date_sub, fold constant would failed cause of missing of function signature in Planner Solved: Add complete function signature of functions like date_sub --- .../org/apache/doris/rewrite/FEFunctions.java | 68 +++++++++++++++++++ .../test_date_function.groovy | 45 ++++++++++++ 2 files changed, 113 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java index f5d03d68b263cfc..520cdeefff0aad6 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java +++ b/fe/fe-core/src/main/java/org/apache/doris/rewrite/FEFunctions.java @@ -104,6 +104,11 @@ public static DateLiteral dateAdd(LiteralExpr date, LiteralExpr day) throws Anal return daysAdd(date, day); } + @FEFunction(name = "date_add", argTypes = { "DATE", "INT" }, returnType = "DATE") + public static DateLiteral dateAddDate(LiteralExpr date, LiteralExpr day) throws AnalysisException { + return daysAdd(date, day); + } + @FEFunction(name = "date_add", argTypes = { "DATEV2", "INT" }, returnType = "DATEV2") public static DateLiteral dateAddDateV2(LiteralExpr date, LiteralExpr day) throws AnalysisException { return daysAdd(date, day); @@ -135,6 +140,12 @@ public static DateLiteral yearsAdd(LiteralExpr date, LiteralExpr year) throws An return dateLiteral.plusYears(year.getLongValue()); } + @FEFunction(name = "years_add", argTypes = { "DATE", "INT" }, returnType = "DATE") + public static DateLiteral yearsAddDate(LiteralExpr date, LiteralExpr year) throws AnalysisException { + DateLiteral dateLiteral = (DateLiteral) date; + return dateLiteral.plusYears((int) year.getLongValue()); + } + @FEFunction(name = "years_add", argTypes = { "DATEV2", "INT" }, returnType = "DATEV2") public static DateLiteral yearsAddDateV2(LiteralExpr date, LiteralExpr year) throws AnalysisException { DateLiteral dateLiteral = (DateLiteral) date; @@ -153,6 +164,12 @@ public static DateLiteral monthsAdd(LiteralExpr date, LiteralExpr month) throws return dateLiteral.plusMonths(month.getLongValue()); } + @FEFunction(name = "months_add", argTypes = { "DATE", "INT" }, returnType = "DATE") + public static DateLiteral monthsAddDate(LiteralExpr date, LiteralExpr month) throws AnalysisException { + DateLiteral dateLiteral = (DateLiteral) date; + return dateLiteral.plusMonths((int) month.getLongValue()); + } + @FEFunction(name = "months_add", argTypes = { "DATEV2", "INT" }, returnType = "DATEV2") public static DateLiteral monthsAddDateV2(LiteralExpr date, LiteralExpr month) throws AnalysisException { DateLiteral dateLiteral = (DateLiteral) date; @@ -171,6 +188,12 @@ public static DateLiteral daysAdd(LiteralExpr date, LiteralExpr day) throws Anal return dateLiteral.plusDays(day.getLongValue()); } + @FEFunction(name = "days_add", argTypes = { "DATE", "INT" }, returnType = "DATE") + public static DateLiteral daysAddDate(LiteralExpr date, LiteralExpr day) throws AnalysisException { + DateLiteral dateLiteral = (DateLiteral) date; + return dateLiteral.plusDays((int) day.getLongValue()); + } + @FEFunction(name = "days_add", argTypes = { "DATEV2", "INT" }, returnType = "DATEV2") public static DateLiteral daysAddDateV2(LiteralExpr date, LiteralExpr day) throws AnalysisException { DateLiteral dateLiteral = (DateLiteral) date; @@ -224,26 +247,71 @@ public static DateLiteral dateSub(LiteralExpr date, LiteralExpr day) throws Anal return dateAdd(date, new IntLiteral(-(int) day.getLongValue())); } + @FEFunction(name = "date_sub", argTypes = { "DATE", "INT" }, returnType = "DATE") + public static DateLiteral dateSubDate(LiteralExpr date, LiteralExpr day) throws AnalysisException { + return daysAdd(date, new IntLiteral(-(int) day.getLongValue())); + } + @FEFunction(name = "date_sub", argTypes = { "DATEV2", "INT" }, returnType = "DATEV2") public static DateLiteral dateSubDateV2(LiteralExpr date, LiteralExpr day) throws AnalysisException { return dateAdd(date, new IntLiteral(-(int) day.getLongValue())); } + @FEFunction(name = "date_sub", argTypes = { "DATETIMEV2", "INT" }, returnType = "DATETIMEV2") + public static DateLiteral dateSubDateTimeV2(LiteralExpr date, LiteralExpr day) throws AnalysisException { + return daysAdd(date, new IntLiteral(-(int) day.getLongValue())); + } + @FEFunction(name = "years_sub", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") public static DateLiteral yearsSub(LiteralExpr date, LiteralExpr year) throws AnalysisException { return yearsAdd(date, new IntLiteral(-(int) year.getLongValue())); } + @FEFunction(name = "years_sub", argTypes = { "DATE", "INT" }, returnType = "DATE") + public static DateLiteral yearsSubDate(LiteralExpr date, LiteralExpr year) throws AnalysisException { + return yearsAdd(date, new IntLiteral(-(int) year.getLongValue())); + } + + @FEFunction(name = "years_sub", argTypes = { "DATEV2", "INT" }, returnType = "DATEV2") + public static DateLiteral yearsSubDateV2(LiteralExpr date, LiteralExpr year) throws AnalysisException { + return yearsAdd(date, new IntLiteral(-(int) year.getLongValue())); + } + + @FEFunction(name = "years_sub", argTypes = { "DATETIMEV2", "INT" }, returnType = "DATETIMEV2") + public static DateLiteral yearsSubDateTimeV2(LiteralExpr date, LiteralExpr year) throws AnalysisException { + return yearsAdd(date, new IntLiteral(-(int) year.getLongValue())); + } + @FEFunction(name = "months_sub", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") public static DateLiteral monthsSub(LiteralExpr date, LiteralExpr month) throws AnalysisException { return monthsAdd(date, new IntLiteral(-(int) month.getLongValue())); } + @FEFunction(name = "months_sub", argTypes = { "DATE", "INT" }, returnType = "DATE") + public static DateLiteral monthsSubDate(LiteralExpr date, LiteralExpr month) throws AnalysisException { + return monthsAdd(date, new IntLiteral(-(int) month.getLongValue())); + } + + @FEFunction(name = "months_sub", argTypes = { "DATEV2", "INT" }, returnType = "DATEV2") + public static DateLiteral monthsSubDateV2(LiteralExpr date, LiteralExpr month) throws AnalysisException { + return monthsAdd(date, new IntLiteral(-(int) month.getLongValue())); + } + + @FEFunction(name = "months_sub", argTypes = { "DATETIMEV2", "INT" }, returnType = "DATETIMEV2") + public static DateLiteral monthsSubDateTimeV2(LiteralExpr date, LiteralExpr month) throws AnalysisException { + return monthsAdd(date, new IntLiteral(-(int) month.getLongValue())); + } + @FEFunction(name = "days_sub", argTypes = { "DATETIME", "INT" }, returnType = "DATETIME") public static DateLiteral daysSub(LiteralExpr date, LiteralExpr day) throws AnalysisException { return daysAdd(date, new IntLiteral(-(int) day.getLongValue())); } + @FEFunction(name = "days_sub", argTypes = { "DATE", "INT" }, returnType = "DATE") + public static DateLiteral daysSubDate(LiteralExpr date, LiteralExpr day) throws AnalysisException { + return daysAdd(date, new IntLiteral(-(int) day.getLongValue())); + } + @FEFunction(name = "days_sub", argTypes = { "DATETIMEV2", "INT" }, returnType = "DATETIMEV2") public static DateLiteral daysSubDateTimeV2(LiteralExpr date, LiteralExpr day) throws AnalysisException { return daysAdd(date, new IntLiteral(-(int) day.getLongValue())); diff --git a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy index 04de3abef0646ce..49db2bb4c809a44 100644 --- a/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy +++ b/regression-test/suites/query_p0/sql_functions/datetime_functions/test_date_function.groovy @@ -239,6 +239,51 @@ suite("test_date_function") { contains "2025-01-16" } + explain { + sql """select * from ${tableName} where test_datetime >= date_sub('2024-01-16',INTERVAL 1 day);""" + contains "2024-01-15" + } + + explain { + sql """select * from ${tableName} where test_datetime >= months_sub('2024-02-16',1);""" + contains "2024-01-16" + } + explain { + sql """select * from ${tableName} where test_datetime >= years_sub('2024-01-16',1);""" + contains "2023-01-16" + } + + explain { + sql """select * from ${tableName} where test_datetime >= date_add(cast('2024-01-16' as DATE),INTERVAL 1 day);""" + contains "2024-01-17" + } + explain { + sql """select * from ${tableName} where test_datetime >= adddate(cast('2024-01-16' as DATE),INTERVAL 1 day);""" + contains "2024-01-17" + } + explain { + sql """select * from ${tableName} where test_datetime >= months_add(cast('2024-01-16' as DATE),1);""" + contains "2024-02-16" + } + explain { + sql """select * from ${tableName} where test_datetime >= years_add(cast('2024-01-16' as DATE),1);""" + contains "2025-01-16" + } + + explain { + sql """select * from ${tableName} where test_datetime >= date_sub(cast('2024-01-16' as DATE),INTERVAL 1 day);""" + contains "2024-01-15" + } + + explain { + sql """select * from ${tableName} where test_datetime >= months_sub(cast('2024-02-16' as DATE),1);""" + contains "2024-01-16" + } + explain { + sql """select * from ${tableName} where test_datetime >= years_sub(cast('2024-01-16' as DATE),1);""" + contains "2023-01-16" + } + // DATE_FORMAT sql """ truncate table ${tableName} """ sql """ insert into ${tableName} values ("2009-10-04 22:23:00") """ From 8264078a9a8e38f83f89b14ef2e10e6298c48804 Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Wed, 15 May 2024 17:54:30 +0800 Subject: [PATCH 024/111] [fix](nereids)4 phase agg may lost parameter in some case (#34816) --- .../doris/nereids/rules/implementation/AggregateStrategies.java | 2 +- regression-test/suites/nereids_syntax_p0/agg_4_phase.groovy | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java index edbd28677b4a00c..e1095df7bab0b97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/AggregateStrategies.java @@ -1766,7 +1766,7 @@ private List> fourPhaseAggregateWithDistin AggregateFunction nonDistinct = aggregateFunction .withDistinctAndChildren(false, ImmutableList.copyOf(aggChild)); AggregateExpression nonDistinctAggExpr = new AggregateExpression(nonDistinct, - distinctLocalParam, aggregateFunction.child(0)); + distinctLocalParam, aggregateFunction); return nonDistinctAggExpr; } else { needUpdateSlot.add(aggregateFunction); diff --git a/regression-test/suites/nereids_syntax_p0/agg_4_phase.groovy b/regression-test/suites/nereids_syntax_p0/agg_4_phase.groovy index 4a3e60afb73e850..19cac99c153fa7b 100644 --- a/regression-test/suites/nereids_syntax_p0/agg_4_phase.groovy +++ b/regression-test/suites/nereids_syntax_p0/agg_4_phase.groovy @@ -56,4 +56,6 @@ suite("agg_4_phase") { contains ":VAGGREGATE (update serialize)" } qt_4phase (test_sql) + + sql """select GROUP_CONCAT(distinct name, " ") from agg_4_phase_tbl;""" } \ No newline at end of file From a07876e807d10161537a84c699a09586c3ac9c30 Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Wed, 15 May 2024 18:02:18 +0800 Subject: [PATCH 025/111] [fix](planner)correlated predicate should include isnull predicate (#34833) --- .../apache/doris/analysis/StmtRewriter.java | 3 +- .../test_subquery_with_agg.groovy | 41 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java index 62737131b62d720..93823cf398c2dcc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java @@ -957,7 +957,8 @@ private static boolean containsCorrelatedPredicate(Expr root, List tupl * query block (i.e. is not bound by the given 'tupleIds'). */ private static boolean isCorrelatedPredicate(Expr expr, List tupleIds) { - return (expr instanceof BinaryPredicate || expr instanceof SlotRef) && !expr.isBoundByTupleIds(tupleIds); + return (expr instanceof BinaryPredicate || expr instanceof SlotRef + || expr instanceof IsNullPredicate) && !expr.isBoundByTupleIds(tupleIds); } /** diff --git a/regression-test/suites/correctness_p0/test_subquery_with_agg.groovy b/regression-test/suites/correctness_p0/test_subquery_with_agg.groovy index e0592830ffe9210..a962d64dcbc637b 100644 --- a/regression-test/suites/correctness_p0/test_subquery_with_agg.groovy +++ b/regression-test/suites/correctness_p0/test_subquery_with_agg.groovy @@ -82,4 +82,45 @@ suite("test_subquery_with_agg") { drop table if exists agg_subquery_table; """ + sql """drop table if exists subquery_table_xyz;""" + sql """CREATE TABLE `subquery_table_xyz` ( + `phone`bigint(20) NULL + ) ENGINE=OLAP + DUPLICATE KEY(`phone`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`phone`) BUCKETS 3 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + sql """WITH tmp1 AS + (SELECT DISTINCT phone + FROM subquery_table_xyz oua + WHERE (NOT EXISTS + (SELECT 1 + FROM subquery_table_xyz o1 + WHERE oua.phone = o1.phone + AND phone IS NOT NULL))), + tmp2 AS + (SELECT DISTINCT phone + FROM subquery_table_xyz oua + WHERE (NOT EXISTS + (SELECT 1 + FROM subquery_table_xyz o1 + WHERE oua.phone = o1.phone + and phone IS NOT NULL))), + tmp3 AS + (SELECT DISTINCT phone + FROM subquery_table_xyz oua + WHERE (NOT EXISTS + (SELECT 1 + FROM subquery_table_xyz o1 + WHERE oua.phone = o1.phone and + phone IS NOT NULL))) + SELECT COUNT(DISTINCT tmp1.phone) + FROM tmp1 + JOIN tmp2 + ON tmp1.phone = tmp2.phone + JOIN tmp3 + ON tmp2.phone = tmp3.phone;""" + } From 849eeb39e9c25645498966c681c7d2c115db8262 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 15 May 2024 18:26:11 +0800 Subject: [PATCH 026/111] [fix](load) skip sending cancel rpc if VNodeChannel is not inited (#34897) --- be/src/vec/sink/writer/vtablet_writer.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp index c2a029e7870686a..3fb17850eff6843 100644 --- a/be/src/vec/sink/writer/vtablet_writer.cpp +++ b/be/src/vec/sink/writer/vtablet_writer.cpp @@ -841,6 +841,10 @@ void VNodeChannel::cancel(const std::string& cancel_msg) { // we don't need to wait last rpc finished, cause closure's release/reset will join. // But do we need brpc::StartCancel(call_id)? _cancel_with_msg(cancel_msg); + // if not inited, _stub will be nullptr, skip sending cancel rpc + if (!_inited) { + return; + } auto request = std::make_shared(); request->set_allocated_id(&_parent->_load_id); From 4b96f9834fd213e7baedec78fea9757eedc386d3 Mon Sep 17 00:00:00 2001 From: Kaijie Chen Date: Wed, 15 May 2024 18:26:41 +0800 Subject: [PATCH 027/111] [fix](move-memtable) change brpc connection type to single (#34883) --- be/src/util/brpc_client_cache.h | 6 +++++- be/src/vec/sink/load_stream_stub.cpp | 6 +++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/be/src/util/brpc_client_cache.h b/be/src/util/brpc_client_cache.h index 2e1d0508ba3a642..290f2cc3e047476 100644 --- a/be/src/util/brpc_client_cache.h +++ b/be/src/util/brpc_client_cache.h @@ -111,7 +111,8 @@ class BrpcClientCache { std::shared_ptr get_new_client_no_cache(const std::string& host_port, const std::string& protocol = "baidu_std", - const std::string& connect_type = "") { + const std::string& connect_type = "", + const std::string& connection_group = "") { brpc::ChannelOptions options; if constexpr (std::is_same_v) { options.protocol = config::function_service_protocol; @@ -121,6 +122,9 @@ class BrpcClientCache { if (connect_type != "") { options.connection_type = connect_type; } + if (connection_group != "") { + options.connection_group = connection_group; + } options.connect_timeout_ms = 2000; options.max_retry = 10; diff --git a/be/src/vec/sink/load_stream_stub.cpp b/be/src/vec/sink/load_stream_stub.cpp index 155ce2de349ab6c..92670c1c9300909 100644 --- a/be/src/vec/sink/load_stream_stub.cpp +++ b/be/src/vec/sink/load_stream_stub.cpp @@ -175,9 +175,9 @@ Status LoadStreamStub::open(BrpcClientCache* client_cache, *request.add_tablets() = tablet; } POpenLoadStreamResponse response; - // use "pooled" connection to avoid conflicts between streaming rpc and regular rpc, - // see: https://github.com/apache/brpc/issues/392 - const auto& stub = client_cache->get_new_client_no_cache(host_port, "baidu_std", "pooled"); + // set connection_group "streaming" to distinguish with non-streaming connections + const auto& stub = + client_cache->get_new_client_no_cache(host_port, "baidu_std", "single", "streaming"); stub->open_load_stream(&cntl, &request, &response, nullptr); for (const auto& resp : response.tablet_schemas()) { auto tablet_schema = std::make_unique(); From dff61715461bf8e6e3736cc4ed14e601c93a3feb Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Wed, 15 May 2024 23:00:34 +0800 Subject: [PATCH 028/111] [fix](auto inc) db_id and table_id should be int64_t instead of int32_t (#34912) --- be/src/olap/iterators.h | 2 +- be/src/olap/rowset/segment_v2/column_reader.h | 2 +- be/src/olap/rowset/segment_v2/segment_iterator.h | 4 ++-- be/src/olap/schema_cache.cpp | 4 ++-- be/src/olap/schema_cache.h | 4 ++-- be/src/olap/tablet_schema.h | 12 ++++++------ be/src/olap/utils.h | 4 ++-- be/src/runtime/descriptors.h | 4 ++-- be/src/vec/sink/autoinc_buffer.cpp | 7 +++++++ 9 files changed, 25 insertions(+), 18 deletions(-) diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h index 95e905185ace2c8..deb14ff554f6589 100644 --- a/be/src/olap/iterators.h +++ b/be/src/olap/iterators.h @@ -115,7 +115,7 @@ class StorageReadOptions { RuntimeState* runtime_state = nullptr; RowsetId rowset_id; Version version; - int32_t tablet_id = 0; + int64_t tablet_id = 0; // slots that cast may be eliminated in storage layer std::map target_cast_type_for_variants; RowRanges row_ranges; diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 9ef9d53d6624667..5d25758e17f25ae 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -599,7 +599,7 @@ class RowIdColumnIterator : public ColumnIterator { private: rowid_t _current_rowid = 0; - int32_t _tablet_id = 0; + int64_t _tablet_id = 0; RowsetId _rowset_id; int32_t _segment_id = 0; }; diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.h b/be/src/olap/rowset/segment_v2/segment_iterator.h index 50c195fb720a166..23d3d76dfc6ede3 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.h +++ b/be/src/olap/rowset/segment_v2/segment_iterator.h @@ -124,7 +124,7 @@ class SegmentIterator : public RowwiseIterator { bool is_lazy_materialization_read() const override { return _lazy_materialization_read; } uint64_t data_id() const override { return _segment->id(); } RowsetId rowset_id() const { return _segment->rowset_id(); } - int32_t tablet_id() const { return _tablet_id; } + int64_t tablet_id() const { return _tablet_id; } bool update_profile(RuntimeProfile* profile) override { bool updated = false; @@ -486,7 +486,7 @@ class SegmentIterator : public RowwiseIterator { // used to collect filter information. std::vector _filter_info_id; bool _record_rowids = false; - int32_t _tablet_id = 0; + int64_t _tablet_id = 0; std::set _output_columns; std::unique_ptr _path_reader; diff --git a/be/src/olap/schema_cache.cpp b/be/src/olap/schema_cache.cpp index 7bf6b592c6e570e..765e95cdc31d925 100644 --- a/be/src/olap/schema_cache.cpp +++ b/be/src/olap/schema_cache.cpp @@ -40,7 +40,7 @@ SchemaCache* SchemaCache::instance() { } // format: tabletId-unique_id1-uniqueid2...-version-type -std::string SchemaCache::get_schema_key(int32_t tablet_id, const TabletSchemaSPtr& schema, +std::string SchemaCache::get_schema_key(int64_t tablet_id, const TabletSchemaSPtr& schema, const std::vector& column_ids, int32_t version, Type type) { if (column_ids.empty() || schema->column(column_ids[0]).unique_id() < 0) { @@ -57,7 +57,7 @@ std::string SchemaCache::get_schema_key(int32_t tablet_id, const TabletSchemaSPt } // format: tabletId-unique_id1-uniqueid2...-version-type -std::string SchemaCache::get_schema_key(int32_t tablet_id, const std::vector& columns, +std::string SchemaCache::get_schema_key(int64_t tablet_id, const std::vector& columns, int32_t version, Type type) { if (columns.empty() || columns[0].col_unique_id < 0) { return ""; diff --git a/be/src/olap/schema_cache.h b/be/src/olap/schema_cache.h index b94fcf8ccdbb2f6..047132e6568038d 100644 --- a/be/src/olap/schema_cache.h +++ b/be/src/olap/schema_cache.h @@ -53,10 +53,10 @@ class SchemaCache : public LRUCachePolicy { static void create_global_instance(size_t capacity); // get cache schema key, delimiter with SCHEMA_DELIMITER - static std::string get_schema_key(int32_t tablet_id, const TabletSchemaSPtr& schema, + static std::string get_schema_key(int64_t tablet_id, const TabletSchemaSPtr& schema, const std::vector& column_ids, int32_t version, Type type); - static std::string get_schema_key(int32_t tablet_id, const std::vector& columns, + static std::string get_schema_key(int64_t tablet_id, const std::vector& columns, int32_t version, Type type); // Get a shared cached schema from cache, schema_key is a subset of column unique ids diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index 33942035d29fa42..ef2a6808d455376 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -368,10 +368,10 @@ class TabletSchema { } std::string auto_increment_column() const { return _auto_increment_column; } - void set_table_id(int32_t table_id) { _table_id = table_id; } - int32_t table_id() const { return _table_id; } - void set_db_id(int32_t db_id) { _db_id = db_id; } - int32_t db_id() const { return _db_id; } + void set_table_id(int64_t table_id) { _table_id = table_id; } + int64_t table_id() const { return _table_id; } + void set_db_id(int64_t db_id) { _db_id = db_id; } + int64_t db_id() const { return _db_id; } void build_current_tablet_schema(int64_t index_id, int32_t version, const OlapTableIndexSchema* index, const TabletSchema& out_tablet_schema); @@ -485,8 +485,8 @@ class TabletSchema { int32_t _sequence_col_idx = -1; int32_t _version_col_idx = -1; int32_t _schema_version = -1; - int32_t _table_id = -1; - int32_t _db_id = -1; + int64_t _table_id = -1; + int64_t _db_id = -1; bool _disable_auto_compaction = false; bool _enable_single_replica_compaction = false; int64_t _mem_size = 0; diff --git a/be/src/olap/utils.h b/be/src/olap/utils.h index 4539aef288f005d..27b7b77eea14a90 100644 --- a/be/src/olap/utils.h +++ b/be/src/olap/utils.h @@ -296,9 +296,9 @@ struct RowLocation { }; struct GlobalRowLoacation { - GlobalRowLoacation(uint32_t tid, RowsetId rsid, uint32_t sid, uint32_t rid) + GlobalRowLoacation(int64_t tid, RowsetId rsid, uint32_t sid, uint32_t rid) : tablet_id(tid), row_location(rsid, sid, rid) {} - uint32_t tablet_id; + int64_t tablet_id; RowLocation row_location; bool operator==(const GlobalRowLoacation& rhs) const { diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h index d407a0b779f39c3..c6c01c88cf58520 100644 --- a/be/src/runtime/descriptors.h +++ b/be/src/runtime/descriptors.h @@ -146,13 +146,13 @@ class TableDescriptor { ::doris::TTableType::type table_type() const { return _table_type; } const std::string& name() const { return _name; } const std::string& database() const { return _database; } - int32_t table_id() const { return _table_id; } + int64_t table_id() const { return _table_id; } private: ::doris::TTableType::type _table_type; std::string _name; std::string _database; - int32_t _table_id; + int64_t _table_id; int _num_cols; int _num_clustering_cols; }; diff --git a/be/src/vec/sink/autoinc_buffer.cpp b/be/src/vec/sink/autoinc_buffer.cpp index 39613159aca6c88..c7c096ec6e8c1e8 100644 --- a/be/src/vec/sink/autoinc_buffer.cpp +++ b/be/src/vec/sink/autoinc_buffer.cpp @@ -19,6 +19,8 @@ #include +#include + #include "common/status.h" #include "runtime/client_cache.h" #include "runtime/exec_env.h" @@ -69,6 +71,11 @@ Status AutoIncIDBuffer::sync_request_ids(size_t length, } DCHECK_LE(length, _front_buffer.second); + if (length > _front_buffer.second) { + return Status::RpcError("auto inc sync result length > front buffer. " + + std::to_string(length) + " vs " + + std::to_string(_front_buffer.second)); + } result->emplace_back(_front_buffer.first, length); _front_buffer.first += length; _front_buffer.second -= length; From 0febfc10e4548ea704d3e3f54029996525d8f107 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Wed, 15 May 2024 23:19:26 +0800 Subject: [PATCH 029/111] [Fix](inverted index) fix wrong fs in inverted_index_file_writer (#34903) --- be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp index 9714480318e4a00..5727a452bf1d905 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_file_writer.cpp @@ -52,7 +52,7 @@ Result InvertedIndexFileWriter::open(const TabletIndex* index (_index_file_dir / _segment_file_name).native(), index_id, index_suffix); bool exists = false; - auto st = _fs->exists(lfs_index_path.c_str(), &exists); + auto st = _lfs->exists(lfs_index_path.c_str(), &exists); if (!st.ok()) { LOG(ERROR) << "index_path:" << lfs_index_path << " exists error:" << st; return ResultError(st); From 5d1f5968ebe460eb93aee2c2fca1ae02371fecce Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Thu, 16 May 2024 16:42:54 +0800 Subject: [PATCH 030/111] [fix](case)fix PolicyTest testMergeFilterNereidsPlanner failed (#34637) The results of explain may have some order that is not fixed --- .../src/test/java/org/apache/doris/policy/PolicyTest.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java index 3169a57159f3079..ddf0d68fd2b171b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/policy/PolicyTest.java @@ -283,7 +283,10 @@ public void testMergeFilterNereidsPlanner() throws Exception { createPolicy("CREATE ROW POLICY test_row_policy4 ON test.table1 AS PERMISSIVE TO test_policy USING (k2 = 1)"); String queryStr = "EXPLAIN select * from test.table1"; String explainString = getSQLPlanOrErrorMsg(queryStr); - Assertions.assertTrue(explainString.contains("k2[#1] IN (1, 2) AND (k1[#0] = 1)")); + System.out.println(explainString); + Assertions.assertTrue(explainString.contains("IN (1, 2)") || explainString.contains("IN (2, 1)")); + Assertions.assertTrue(explainString.contains("AND")); + Assertions.assertTrue(explainString.contains("= 1)")); dropPolicy("DROP ROW POLICY test_row_policy1 ON test.table1"); dropPolicy("DROP ROW POLICY test_row_policy3 ON test.table1"); dropPolicy("DROP ROW POLICY test_row_policy4 ON test.table1"); From b51a4212d682ffbfb1f789290d777edce8707037 Mon Sep 17 00:00:00 2001 From: meiyi Date: Thu, 16 May 2024 17:54:39 +0800 Subject: [PATCH 031/111] [fix](txn insert) Fix txn insert values error when connect to follower fe (#34950) --- .../insert/BatchInsertIntoTableCommand.java | 4 +-- .../plans/commands/insert/InsertUtils.java | 29 ++++++++++++++----- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java index 6d4431eec271b40..4399cd57db48ed4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BatchInsertIntoTableCommand.java @@ -33,7 +33,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.commands.Command; -import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync; +import org.apache.doris.nereids.trees.plans.commands.NoForward; import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; @@ -59,7 +59,7 @@ /** * insert into values with in txn model. */ -public class BatchInsertIntoTableCommand extends Command implements ForwardWithSync, Explainable { +public class BatchInsertIntoTableCommand extends Command implements NoForward, Explainable { public static final Logger LOG = LogManager.getLogger(BatchInsertIntoTableCommand.class); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index ad974e9e7bc005a..8293183eeb200e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -24,7 +24,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.UserException; +import org.apache.doris.common.Config; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; @@ -54,11 +54,14 @@ import org.apache.doris.proto.InternalService; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.InsertStreamTxnExecutor; +import org.apache.doris.qe.MasterTxnExecutor; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.service.FrontendOptions; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileType; +import org.apache.doris.thrift.TLoadTxnBeginRequest; +import org.apache.doris.thrift.TLoadTxnBeginResult; import org.apache.doris.thrift.TMergeType; import org.apache.doris.thrift.TStreamLoadPutRequest; import org.apache.doris.thrift.TTxnParams; @@ -182,15 +185,25 @@ private static void beginBatchInsertTransaction(ConnectContext ctx, txnEntry.setDb(dbObj); String label = txnEntry.getLabel(); try { - long txnId = Env.getCurrentGlobalTransactionMgr().beginTransaction( - txnConf.getDbId(), Lists.newArrayList(tblObj.getId()), - label, new TransactionState.TxnCoordinator( - TransactionState.TxnSourceType.FE, FrontendOptions.getLocalHostAddress()), - sourceType, timeoutSecond); - txnConf.setTxnId(txnId); + long txnId; String token = Env.getCurrentEnv().getLoadManager().getTokenManager().acquireToken(); + if (Config.isCloudMode() || Env.getCurrentEnv().isMaster()) { + txnId = Env.getCurrentGlobalTransactionMgr().beginTransaction( + txnConf.getDbId(), Lists.newArrayList(tblObj.getId()), + label, new TransactionState.TxnCoordinator( + TransactionState.TxnSourceType.FE, FrontendOptions.getLocalHostAddress()), + sourceType, timeoutSecond); + } else { + MasterTxnExecutor masterTxnExecutor = new MasterTxnExecutor(ctx); + TLoadTxnBeginRequest request = new TLoadTxnBeginRequest(); + request.setDb(txnConf.getDb()).setTbl(txnConf.getTbl()).setToken(token) + .setLabel(label).setUser("").setUserIp("").setPasswd(""); + TLoadTxnBeginResult result = masterTxnExecutor.beginTxn(request); + txnId = result.getTxnId(); + } + txnConf.setTxnId(txnId); txnConf.setToken(token); - } catch (UserException e) { + } catch (Exception e) { throw new AnalysisException(e.getMessage(), e); } From db273d578f0756fa318f06cc77bdf9f5eed21745 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Thu, 16 May 2024 19:56:11 +0800 Subject: [PATCH 032/111] [Fix](tablet id) use int64_t instead of int32_t or uint32_t for tablet_id (#34962) --- be/src/olap/rowset/segment_v2/column_reader.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 5d25758e17f25ae..9c889b848fda293 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -557,7 +557,7 @@ class ArrayFileColumnIterator final : public ColumnIterator { class RowIdColumnIterator : public ColumnIterator { public: RowIdColumnIterator() = delete; - RowIdColumnIterator(int32_t tid, RowsetId rid, int32_t segid) + RowIdColumnIterator(int64_t tid, RowsetId rid, int32_t segid) : _tablet_id(tid), _rowset_id(rid), _segment_id(segid) {} Status seek_to_first() override { From 6f5abfd23fcdf959168c7bed76fe7514f3e363f3 Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Fri, 17 May 2024 09:33:47 +0800 Subject: [PATCH 033/111] [regression-test](fix) fix case bug, using test_insert_dft_tbl in multiple test cases #34983 --- regression-test/suites/load_p0/insert/test_insert.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/load_p0/insert/test_insert.groovy b/regression-test/suites/load_p0/insert/test_insert.groovy index 25b010f242d0998..1ef37f843f78878 100644 --- a/regression-test/suites/load_p0/insert/test_insert.groovy +++ b/regression-test/suites/load_p0/insert/test_insert.groovy @@ -49,7 +49,7 @@ suite("test_insert") { qt_sql1 "select * from ${insert_tbl} order by 1, 2, 3, 4" - def insert_tbl_dft = "test_insert_dft_tbl" + def insert_tbl_dft = "test_insert_dft2_tbl" sql """ DROP TABLE IF EXISTS ${insert_tbl_dft}""" // `k7` should be float type, and bug exists now, https://github.com/apache/doris/pull/20867 From eb7eaee386280e9854863eb03258126d70d99f48 Mon Sep 17 00:00:00 2001 From: zhiqiang Date: Fri, 17 May 2024 10:44:31 +0800 Subject: [PATCH 034/111] [fix](function) money format (#34680) --- be/src/vec/functions/function_string.h | 180 ++++++++++++------ .../function/function_money_format_test.cpp | 92 +++++++++ .../string_functions/test_string_function.out | 4 +- .../string_functions/test_money_format.out | 154 +++++++++++++++ .../string_functions/test_money_format.groovy | 77 ++++++++ 5 files changed, 452 insertions(+), 55 deletions(-) create mode 100644 be/test/vec/function/function_money_format_test.cpp create mode 100644 regression-test/data/query_p0/sql_functions/string_functions/test_money_format.out create mode 100644 regression-test/suites/query_p0/sql_functions/string_functions/test_money_format.groovy diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h index 61339bbd9cf089f..6dbd27041876198 100644 --- a/be/src/vec/functions/function_string.h +++ b/be/src/vec/functions/function_string.h @@ -25,15 +25,18 @@ #include #include #include +#include #include #include #include +#include #include #include #include #include #include #include +#include #include #include @@ -65,6 +68,7 @@ #include "vec/core/field.h" #include "vec/core/types.h" #include "vec/data_types/data_type.h" +#include "vec/functions/round.h" #include "vec/io/io_helper.h" #ifndef USE_LIBCPP @@ -2959,18 +2963,97 @@ class FunctionMoneyFormat : public IFunction { namespace MoneyFormat { +constexpr size_t MAX_FORMAT_LEN_DEC32() { + // Decimal(9, 0) + // Double the size to avoid some unexpected bug. + return 2 * (1 + 9 + (9 / 3) + 3); +} + +constexpr size_t MAX_FORMAT_LEN_DEC64() { + // Decimal(18, 0) + // Double the size to avoid some unexpected bug. + return 2 * (1 + 18 + (18 / 3) + 3); +} + +constexpr size_t MAX_FORMAT_LEN_DEC128V2() { + // DecimalV2 has at most 27 digits + // Double the size to avoid some unexpected bug. + return 2 * (1 + 27 + (27 / 3) + 3); +} + +constexpr size_t MAX_FORMAT_LEN_DEC128V3() { + // Decimal(38, 0) + // Double the size to avoid some unexpected bug. + return 2 * (1 + 39 + (39 / 3) + 3); +} + +constexpr size_t MAX_FORMAT_LEN_INT64() { + // INT_MIN = -9223372036854775807 + // Double the size to avoid some unexpected bug. + return 2 * (1 + 20 + (20 / 3) + 3); +} + +constexpr size_t MAX_FORMAT_LEN_INT128() { + // INT128_MIN = -170141183460469231731687303715884105728 + return 2 * (1 + 39 + (39 / 3) + 3); +} + template -StringRef do_money_format(FunctionContext* context, const T int_value, - const int32_t frac_value = 0) { +StringRef do_money_format(FunctionContext* context, UInt32 scale, T int_value, T frac_value) { + static_assert(std::is_integral::value); + const bool is_negative = int_value < 0 || frac_value < 0; + + // do round to frac_part + // magic number 2: since we need to round frac_part to 2 digits + if (scale > 2) { + DCHECK(scale <= 38); + // do rounding, so we need to reserve 3 digits. + auto multiplier = common::exp10_i128(std::abs(static_cast(scale - 3))); + // do devide first to avoid overflow + // after round frac_value will be positive by design. + frac_value = std::abs(frac_value / multiplier) + 5; + frac_value /= 10; + } else if (scale < 2) { + DCHECK(frac_value < 100); + // since scale <= 2, overflow is impossiable + frac_value = frac_value * common::exp10_i32(2 - scale); + } + + if (frac_value == 100) { + if (is_negative) { + int_value -= 1; + } else { + int_value += 1; + } + frac_value = 0; + } + + bool append_sign_manually = false; + if (is_negative && int_value == 0) { + // when int_value is 0, result of SimpleItoaWithCommas will contains just zero + // for Decimal like -0.1234, this will leads to problem, because negative sign is discarded. + // this is why we introduce argument append_sing_manually. + append_sign_manually = true; + } + char local[N]; char* p = SimpleItoaWithCommas(int_value, local, sizeof(local)); - int32_t string_val_len = local + sizeof(local) - p + 3; - StringRef result = context->create_temp_string_val(string_val_len); + const Int32 integer_str_len = N - (p - local); + const Int32 frac_str_len = 2; + const Int32 whole_decimal_str_len = + (append_sign_manually ? 1 : 0) + integer_str_len + 1 + frac_str_len; + + StringRef result = context->create_temp_string_val(whole_decimal_str_len); char* result_data = const_cast(result.data); - memcpy(result_data, p, string_val_len - 3); - *(result_data + string_val_len - 3) = '.'; - *(result_data + string_val_len - 2) = '0' + (frac_value / 10); - *(result_data + string_val_len - 1) = '0' + (frac_value % 10); + + if (append_sign_manually) { + memset(result_data, '-', 1); + } + + memcpy(result_data + (append_sign_manually ? 1 : 0), p, integer_str_len); + *(result_data + whole_decimal_str_len - 3) = '.'; + *(result_data + whole_decimal_str_len - 2) = '0' + std::abs(frac_value / 10); + *(result_data + whole_decimal_str_len - 1) = '0' + std::abs(frac_value % 10); return result; }; @@ -3010,9 +3093,9 @@ struct MoneyFormatDoubleImpl { const auto* data_column = assert_cast*>(col_ptr.get()); // when scale is above 38, we will go here for (size_t i = 0; i < input_rows_count; i++) { - // truncate to 2 decimal places, keep same with mysql + // round to 2 decimal places double value = - MathFunctions::my_double_round(data_column->get_element(i), 2, false, true); + MathFunctions::my_double_round(data_column->get_element(i), 2, false, false); StringRef str = MoneyFormat::do_money_format(context, fmt::format("{:.2f}", value)); result_column->insert_data(str.data, str.size); } @@ -3027,7 +3110,9 @@ struct MoneyFormatInt64Impl { const auto* data_column = assert_cast*>(col_ptr.get()); for (size_t i = 0; i < input_rows_count; i++) { Int64 value = data_column->get_element(i); - StringRef str = MoneyFormat::do_money_format(context, value); + StringRef str = + MoneyFormat::do_money_format( + context, 0, value, 0); result_column->insert_data(str.data, str.size); } } @@ -3039,9 +3124,14 @@ struct MoneyFormatInt128Impl { static void execute(FunctionContext* context, ColumnString* result_column, const ColumnPtr col_ptr, size_t input_rows_count) { const auto* data_column = assert_cast*>(col_ptr.get()); + // SELECT money_format(170141183460469231731687303715884105728/*INT128_MAX + 1*/) will + // get "170,141,183,460,469,231,731,687,303,715,884,105,727.00" in doris, + // see https://github.com/apache/doris/blob/788abf2d7c3c7c2d57487a9608e889e7662d5fb2/be/src/vec/data_types/data_type_number_base.cpp#L124 for (size_t i = 0; i < input_rows_count; i++) { Int128 value = data_column->get_element(i); - StringRef str = MoneyFormat::do_money_format(context, value); + StringRef str = + MoneyFormat::do_money_format( + context, 0, value, 0); result_column->insert_data(str.data, str.size); } } @@ -3056,70 +3146,54 @@ struct MoneyFormatDecimalImpl { size_t input_rows_count) { if (auto* decimalv2_column = check_and_get_column>(*col_ptr)) { for (size_t i = 0; i < input_rows_count; i++) { - DecimalV2Value value = DecimalV2Value(decimalv2_column->get_element(i)); - - DecimalV2Value rounded(0); - value.round(&rounded, 2, HALF_UP); - - StringRef str = MoneyFormat::do_money_format( - context, rounded.int_value(), abs(rounded.frac_value() / 10000000)); + const Decimal128V2& dec128 = decimalv2_column->get_element(i); + DecimalV2Value value = DecimalV2Value(dec128.value); + // unified_frac_value has 3 digits + auto unified_frac_value = value.frac_value() / 1000000; + StringRef str = + MoneyFormat::do_money_format( + context, 3, value.int_value(), unified_frac_value); result_column->insert_data(str.data, str.size); } } else if (auto* decimal32_column = check_and_get_column>(*col_ptr)) { const UInt32 scale = decimal32_column->get_scale(); - // scale is up to 9, so exp10_i32 is enough - const auto multiplier = common::exp10_i32(std::abs(static_cast(scale - 2))); for (size_t i = 0; i < input_rows_count; i++) { - Decimal32 frac_part = decimal32_column->get_fractional_part(i); - if (scale > 2) { - int delta = ((frac_part % multiplier) << 1) > multiplier; - frac_part = frac_part / multiplier + delta; - } else if (scale < 2) { - frac_part = frac_part * multiplier; - } - - StringRef str = MoneyFormat::do_money_format( - context, decimal32_column->get_whole_part(i), frac_part); + const Decimal32& frac_part = decimal32_column->get_fractional_part(i); + const Decimal32& whole_part = decimal32_column->get_whole_part(i); + StringRef str = + MoneyFormat::do_money_format( + context, scale, static_cast(whole_part.value), + static_cast(frac_part.value)); result_column->insert_data(str.data, str.size); } } else if (auto* decimal64_column = check_and_get_column>(*col_ptr)) { const UInt32 scale = decimal64_column->get_scale(); - // 9 < scale <= 18 - const auto multiplier = common::exp10_i64(std::abs(static_cast(scale - 2))); for (size_t i = 0; i < input_rows_count; i++) { - Decimal64 frac_part = decimal64_column->get_fractional_part(i); - if (scale > 2) { - int delta = ((frac_part % multiplier) << 1) > multiplier; - frac_part = frac_part / multiplier + delta; - } else if (scale < 2) { - frac_part = frac_part * multiplier; - } + const Decimal64& frac_part = decimal64_column->get_fractional_part(i); + const Decimal64& whole_part = decimal64_column->get_whole_part(i); - StringRef str = MoneyFormat::do_money_format( - context, decimal64_column->get_whole_part(i), frac_part); + StringRef str = + MoneyFormat::do_money_format( + context, scale, whole_part.value, frac_part.value); result_column->insert_data(str.data, str.size); } } else if (auto* decimal128_column = check_and_get_column>(*col_ptr)) { const UInt32 scale = decimal128_column->get_scale(); - // 18 < scale <= 38 - const auto multiplier = common::exp10_i128(std::abs(static_cast(scale - 2))); for (size_t i = 0; i < input_rows_count; i++) { - Decimal128V3 frac_part = decimal128_column->get_fractional_part(i); - if (scale > 2) { - int delta = ((frac_part % multiplier) << 1) > multiplier; - frac_part = frac_part / multiplier + delta; - } else if (scale < 2) { - frac_part = frac_part * multiplier; - } + const Decimal128V3& frac_part = decimal128_column->get_fractional_part(i); + const Decimal128V3& whole_part = decimal128_column->get_whole_part(i); - StringRef str = MoneyFormat::do_money_format<__int128, 53>( - context, decimal128_column->get_whole_part(i), frac_part); + StringRef str = + MoneyFormat::do_money_format( + context, scale, whole_part.value, frac_part.value); result_column->insert_data(str.data, str.size); } diff --git a/be/test/vec/function/function_money_format_test.cpp b/be/test/vec/function/function_money_format_test.cpp new file mode 100644 index 000000000000000..6a17caab3b3ab3f --- /dev/null +++ b/be/test/vec/function/function_money_format_test.cpp @@ -0,0 +1,92 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include + +#include +#include + +#include "function_test_util.h" +#include "runtime/decimalv2_value.h" +#include "runtime/define_primitive_type.h" +#include "runtime/runtime_state.h" +#include "runtime/types.h" +#include "vec/columns/column_decimal.h" +#include "vec/columns/column_string.h" +#include "vec/core/block.h" +#include "vec/core/types.h" +#include "vec/data_types/data_type_decimal.h" +#include "vec/functions/function_string.h" + +namespace doris::vectorized { +TEST(function_money_format_test, money_format_with_decimalV2) { + // why not using + std::multimap input_dec_str_and_expected_str = { + {std::string("123.12"), std::string("123.12")}, + {std::string("-123.12"), std::string("-123.12")}, + {std::string("-0.12434"), std::string("-0.12")}, + {std::string("-0.12534"), std::string("-0.13")}, + {std::string("-123456789.12434"), std::string("-123,456,789.12")}, + {std::string("-123456789.12534"), std::string("-123,456,789.13")}, + {std::string("0.999999999"), std::string("1.00")}, + {std::string("-0.999999999"), std::string("-1.00")}, + {std::string("999999999999999999.994999999"), + std::string("999,999,999,999,999,999.99")}, + {std::string("-999999999999999999.994999999"), + std::string("-999,999,999,999,999,999.99")}, + {std::string("-999999999999999999.995999999"), + std::string("-1,000,000,000,000,000,000.00")}}; + + auto money_format = FunctionMoneyFormat::create(); + std::unique_ptr runtime_state = std::make_unique(); + TypeDescriptor return_type = {PrimitiveType::TYPE_VARCHAR}; + TypeDescriptor arg_type = {PrimitiveType::TYPE_DECIMALV2}; + std::vector arg_types = {arg_type}; + + auto context = FunctionContext::create_context(runtime_state.get(), return_type, arg_types); + + Block block; + ColumnNumbers arguments = {0}; + size_t result_idx = 1; + auto col_dec_v2 = ColumnDecimal::create(0, 9); + auto col_res_expected = ColumnString::create(); + for (const auto& input_and_expected : input_dec_str_and_expected_str) { + DecimalV2Value dec_v2_value(input_and_expected.first); + col_dec_v2->insert_value(Decimal128V2(dec_v2_value.value())); + col_res_expected->insert_data(input_and_expected.second.c_str(), + input_and_expected.second.size()); + } + + block.insert({std::move(col_dec_v2), std::make_shared>(10, 1), + "col_dec_v2"}); + block.insert({nullptr, std::make_shared(), "col_res"}); + + Status exec_status = money_format->execute_impl(context.get(), block, arguments, result_idx, + block.get_by_position(0).column->size()); + + // Check result + auto col_res = block.get_by_position(result_idx).column; + for (size_t i = 0; i < col_res->size(); ++i) { + auto res = col_res->get_data_at(i); + auto res_expected = col_res_expected->get_data_at(i); + EXPECT_EQ(res.debug_string(), res_expected.debug_string()) + << "res " << res.debug_string() << ' ' << "res_expected " + << res_expected.debug_string(); + } +} + +}; // namespace doris::vectorized \ No newline at end of file diff --git a/regression-test/data/nereids_p0/sql_functions/string_functions/test_string_function.out b/regression-test/data/nereids_p0/sql_functions/string_functions/test_string_function.out index 090ec198a7b6b41..e8305c284ff520f 100644 --- a/regression-test/data/nereids_p0/sql_functions/string_functions/test_string_function.out +++ b/regression-test/data/nereids_p0/sql_functions/string_functions/test_string_function.out @@ -189,10 +189,10 @@ ab d 1.12 -- !sql_float64 -- -1.12 +1.13 -- !sql_float64 -- -1.12 +1.13 -- !sql -- true diff --git a/regression-test/data/query_p0/sql_functions/string_functions/test_money_format.out b/regression-test/data/query_p0/sql_functions/string_functions/test_money_format.out new file mode 100644 index 000000000000000..55a3609aa918c35 --- /dev/null +++ b/regression-test/data/query_p0/sql_functions/string_functions/test_money_format.out @@ -0,0 +1,154 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !money_format -- +\N + +-- !money_format -- +0.00 + +-- !money_format -- +0.00 + +-- !money_format -- +-123.13 + +-- !money_format -- +-17,014,116.00 + +-- !money_format -- +1,123.46 + +-- !money_format -- +1,123.40 + +-- !money_format -- +1.12 + +-- !money_format_dec32_2_1 -- +-0.10 + +-- !money_format_dec32_3_2 -- +-0.11 + +-- !money_format_dec32_4_3 -- +-0.11 + +-- !money_format_dec32_4_3 -- +-0.12 + +-- !money_format_dec32_9_0 -- +999,999,999.00 + +-- !money_format_dec32_9_0_negative -- +-999,999,999.00 + +-- !money_format_dec32_9_1 -- +99,999,999.90 + +-- !money_format_dec32_9_1_negative -- +-9,999,999.90 + +-- !money_format_dec32_9_2 -- +9,999,999.99 + +-- !money_format_dec32_9_2_negative -- +-9,999,999.99 + +-- !money_format_dec32_9_9 -- +1.00 + +-- !money_format_dec32_9_9_negative -- +-1.00 + +-- !money_format_dec64_18_0 -- +999,999,999,999,999,999.00 + +-- !money_format_dec64_18_0_negative -- +-999,999,999,999,999,999.00 + +-- !money_format_dec64_18_1 -- +99,999,999,999,999,999.90 + +-- !money_format_dec64_18_1_negative -- +-99,999,999,999,999,999.90 + +-- !money_format_dec64_18_2 -- +999,999,999,999,999.99 + +-- !money_format_dec64_18_2_negative -- +-999,999,999,999,999.99 + +-- !money_format_dec64_18_17 -- +10.00 + +-- !money_format_dec64_18_17_negative -- +-10.00 + +-- !money_format_dec64_18_18 -- +1.00 + +-- !money_format_dec64_18_18_negative -- +-1.00 + +-- !money_format_dec128_38_0 -- +99,999,999,999,999,999,999,999,999,999,999,999,999.00 + +-- !money_format_dec128_38_0_negative -- +-99,999,999,999,999,999,999,999,999,999,999,999,999.00 + +-- !money_format_dec128_38_1 -- +9,999,999,999,999,999,999,999,999,999,999,999,999.90 + +-- !money_format_dec128_38_1_negative -- +-9,999,999,999,999,999,999,999,999,999,999,999,999.90 + +-- !money_format_dec128_38_2 -- +999,999,999,999,999,999,999,999,999,999,999,999.99 + +-- !money_format_dec128_38_2_negative -- +-999,999,999,999,999,999,999,999,999,999,999,999.99 + +-- !money_format_dec128_38_38 -- +1.00 + +-- !money_format_dec128_38_38_negative -- +-1.00 + +-- !money_format_interger -- +1.00 + +-- !money_format_interger -- +-1.00 + +-- !money_format_interger -- +1,233,456,789.00 + +-- !money_format_interger -- +-1,233,456,789.00 + +-- !money_format_interger -- +9,223,372,036,854,775,807.00 + +-- !money_format_interger -- +-9,223,372,036,854,775,808.00 + +-- !money_format_interger_int128_min -- +-170,141,183,460,469,231,731,687,303,715,884,105,728.00 + +-- !money_format_interger_int128_max -- +170,141,183,460,469,231,731,687,303,715,884,105,727.00 + +-- !money_format_double -- +1.2323 1.23 + +-- !money_format_double -- +1.2353 1.24 + +-- !money_format_double -- +-1.2353 -1.24 + +-- !money_format_double -- +-1.234567892353E8 -123,456,789.24 + +-- !money_format_double -- +-0.2353 -0.24 + diff --git a/regression-test/suites/query_p0/sql_functions/string_functions/test_money_format.groovy b/regression-test/suites/query_p0/sql_functions/string_functions/test_money_format.groovy new file mode 100644 index 000000000000000..b30062d3052a6f0 --- /dev/null +++ b/regression-test/suites/query_p0/sql_functions/string_functions/test_money_format.groovy @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_money_format") { + qt_money_format "SELECT money_format(NULL);" + qt_money_format "SELECT money_format(0);" + qt_money_format "SELECT money_format(0.000);" + qt_money_format "select money_format(-123.125);" + qt_money_format "select money_format(-17014116);" + qt_money_format "select money_format(1123.456);" + qt_money_format "select money_format(1123.4);" + qt_money_format "select money_format(1.1249);" + + qt_money_format_dec32_2_1 "select money_format(-0.1);" + qt_money_format_dec32_3_2 "select money_format(-0.11);" + qt_money_format_dec32_4_3 "select money_format(-0.114);" + qt_money_format_dec32_4_3 "select money_format(-0.115);" + + qt_money_format_dec32_9_0 """select money_format(cast(concat(repeat('9', 9)) as DECIMALV3(9, 0)));""" + qt_money_format_dec32_9_0_negative """select money_format(cast(concat('-', repeat('9', 9)) as DECIMALV3(9, 0)));""" + qt_money_format_dec32_9_1 """select money_format(cast(concat(repeat('9', 8), '.', repeat('9', 1)) as DECIMALV3(9, 1)));""" + qt_money_format_dec32_9_1_negative """select money_format(cast(concat('-',repeat('9', 7), '.', repeat('9', 1)) as DECIMALV3(9, 1)));""" + qt_money_format_dec32_9_2 """select money_format(cast(concat(repeat('9', 7), '.', repeat('9', 2)) as DECIMALV3(9, 2)));""" + qt_money_format_dec32_9_2_negative """select money_format(cast(concat('-', repeat('9', 7), '.', repeat('9', 2)) as DECIMALV3(9, 2)));""" + qt_money_format_dec32_9_9 """select money_format(cast(concat('0.', repeat('9', 9)) as DECIMALV3(9, 9)));""" + qt_money_format_dec32_9_9_negative """select money_format(cast(concat('-', '0.', repeat('9', 9)) as DECIMALV3(9, 9)));""" + + qt_money_format_dec64_18_0 """select money_format(cast(concat(repeat('9', 18)) as DECIMALV3(18, 0)));""" + qt_money_format_dec64_18_0_negative """select money_format(cast(concat('-', repeat('9', 18)) as DECIMALV3(18, 0)));""" + qt_money_format_dec64_18_1 """select money_format(cast(concat(repeat('9', 17), '.', repeat('9', 1)) as DECIMALV3(18, 1)));""" + qt_money_format_dec64_18_1_negative """select money_format(cast(concat('-',repeat('9', 17), '.', repeat('9', 1)) as DECIMALV3(18, 1)));""" + qt_money_format_dec64_18_2 """select money_format(cast(concat(repeat('9', 15), '.', repeat('9', 2)) as DECIMALV3(18, 2)));""" + qt_money_format_dec64_18_2_negative """select money_format(cast(concat('-', repeat('9', 15), '.', repeat('9', 2)) as DECIMALV3(18, 2)));""" + qt_money_format_dec64_18_17 """select money_format(cast(concat('9.', repeat('9', 17)) as DECIMALV3(18, 17)))""" + qt_money_format_dec64_18_17_negative """select money_format(cast(concat('-', '9.', repeat('9', 17)) as DECIMALV3(18, 17)))""" + qt_money_format_dec64_18_18 """select money_format(cast(concat('0.', repeat('9', 18)) as DECIMALV3(18, 18)));""" + qt_money_format_dec64_18_18_negative """select money_format(cast(concat('-', '0.', repeat('9', 18)) as DECIMALV3(18, 18)));""" + + qt_money_format_dec128_38_0 """select money_format(cast(concat(repeat('9', 38)) as DECIMALV3(38, 0)));""" + qt_money_format_dec128_38_0_negative """select money_format(cast(concat('-', repeat('9', 38)) as DECIMALV3(38, 0)));""" + qt_money_format_dec128_38_1 """select money_format(cast(concat(repeat('9', 37), '.', repeat('9', 1)) as DECIMALV3(38, 1)));""" + qt_money_format_dec128_38_1_negative """select money_format(cast(concat('-',repeat('9', 37), '.', repeat('9', 1)) as DECIMALV3(38, 1)));""" + qt_money_format_dec128_38_2 """select money_format(cast(concat(repeat('9', 36), '.', repeat('9', 2)) as DECIMALV3(38, 2)));""" + qt_money_format_dec128_38_2_negative """select money_format(cast(concat('-', repeat('9', 36), '.', repeat('9', 2)) as DECIMALV3(38, 2)));""" + qt_money_format_dec128_38_38 """select money_format(cast(concat('0.', repeat('9', 38)) as DECIMALV3(38, 38)));""" + qt_money_format_dec128_38_38_negative """select money_format(cast(concat('-', '0.', repeat('9', 38)) as DECIMALV3(38, 38)));""" + + qt_money_format_interger "select money_format(1);" + qt_money_format_interger "select money_format(-1);" + qt_money_format_interger "select money_format(1233456789);" + qt_money_format_interger "select money_format(-1233456789);" + qt_money_format_interger """select money_format(cast("9223372036854775807" as BigInt))""" + qt_money_format_interger """select money_format(cast("-9223372036854775808" as BigInt))""" + qt_money_format_interger_int128_min """select money_format(-170141183460469231731687303715884105728);""" + qt_money_format_interger_int128_max """select money_format(170141183460469231731687303715884105727);""" + + qt_money_format_double """select cast("1.2323" as Double), money_format(cast("1.2323" as Double));""" + qt_money_format_double """select cast("1.2353" as Double), money_format(cast("1.2353" as Double));""" + qt_money_format_double """select cast("-1.2353" as Double), money_format(cast("-1.2353" as Double));""" + qt_money_format_double """select cast("-123456789.2353" as Double), money_format(cast("-123456789.2353" as Double));""" + qt_money_format_double """select cast("-0.2353" as Double), money_format(cast("-0.2353" as Double));""" +} + From 73419c243116300511f76d1964f7373d540c0673 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 17 May 2024 10:55:10 +0800 Subject: [PATCH 035/111] [enhance](mtmv)MTMV support hive table determine whether the data is sync (#34845) Previously supported, this PR only turns on the switch --- .../apache/doris/datasource/hive/HMSExternalTable.java | 2 +- regression-test/suites/mtmv_p0/test_hive_mtmv.groovy | 8 -------- .../suites/mtmv_p0/test_hive_refresh_mtmv.groovy | 8 ++++---- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index c2099a1acc865f3..57dee5debb0b26d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -958,7 +958,7 @@ private List> transferPartitionItemToPartitionValues(PartitionItem @Override public boolean needAutoRefresh() { - return false; + return true; } @Override diff --git a/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy index 8a91a45c5f8b4a9..872d7bf8ec03698 100644 --- a/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_mtmv.groovy @@ -61,14 +61,6 @@ suite("test_hive_mtmv", "p0,external,hive,external_docker,external_docker_hive") waitingMTMVTaskFinished(jobName) order_qt_refresh_one_partition "SELECT * FROM ${mvName} order by id" - //refresh other partitions - // current, for hive, auto refresh will not change data - sql """ - REFRESH MATERIALIZED VIEW ${mvName} AUTO - """ - waitingMTMVTaskFinished(jobName) - order_qt_refresh_other_partition "SELECT * FROM ${mvName} order by id" - //refresh complete sql """ REFRESH MATERIALIZED VIEW ${mvName} complete diff --git a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy index 755c6c15b0d2d4d..ac19ecc06056649 100644 --- a/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_hive_refresh_mtmv.groovy @@ -95,7 +95,7 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke assertTrue(showPartitionsResult.toString().contains("p_2020")) sql """ - REFRESH MATERIALIZED VIEW ${mvName} complete + REFRESH MATERIALIZED VIEW ${mvName} auto """ def jobName = getJobName(dbName, mvName); waitingMTMVTaskFinished(jobName) @@ -108,7 +108,7 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke REFRESH catalog ${catalog_name} """ sql """ - REFRESH MATERIALIZED VIEW ${mvName} complete + REFRESH MATERIALIZED VIEW ${mvName} auto """ waitingMTMVTaskFinished(jobName) order_qt_mtmv_2 "SELECT * FROM ${mvName} order by user_id" @@ -124,7 +124,7 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke REFRESH catalog ${catalog_name} """ sql """ - REFRESH MATERIALIZED VIEW ${mvName} complete + REFRESH MATERIALIZED VIEW ${mvName} auto """ waitingMTMVTaskFinished(jobName) showPartitionsResult = sql """show partitions from ${mvName}""" @@ -143,7 +143,7 @@ suite("test_hive_refresh_mtmv", "p0,external,hive,external_docker,external_docke REFRESH catalog ${catalog_name} """ sql """ - REFRESH MATERIALIZED VIEW ${mvName} complete + REFRESH MATERIALIZED VIEW ${mvName} auto """ waitingMTMVTaskFinished(jobName) showPartitionsResult = sql """show partitions from ${mvName}""" From 5b72dd1217ee54acd692fea2489caa0a2091757c Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Fri, 17 May 2024 11:24:57 +0800 Subject: [PATCH 036/111] [chore](test) remove useless drop table in test_list_partition_datatype (#34930) --- .../list_partition/test_list_partition_datatype.groovy | 1 - 1 file changed, 1 deletion(-) diff --git a/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy b/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy index 1de3f6588b6016b..d06a0195adbf33a 100644 --- a/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy +++ b/regression-test/suites/partition_p0/list_partition/test_list_partition_datatype.groovy @@ -499,7 +499,6 @@ suite("test_list_partition_datatype", "p0") { assertTrue(ret.size() == 1) try_sql "DROP TABLE IF EXISTS test_list_partition_ddl_tbl_1" try_sql "DROP TABLE IF EXISTS test_list_partition_empty_tb" - try_sql "DROP TABLE IF EXISTS test_list_partition_select_tb" try_sql "DROP TABLE IF EXISTS test_list_partition_tb2_char" // try_sql "DROP TABLE IF EXISTS test_list_partition_tb3_char" } From cc11e50200f092006c287684384c2e85f7d94995 Mon Sep 17 00:00:00 2001 From: seawinde <149132972+seawinde@users.noreply.github.com> Date: Fri, 17 May 2024 11:39:39 +0800 Subject: [PATCH 037/111] [fix](mtmv)Fix slot desc wrong in query rewrite by materialized view when query is complex (#34904) --- .../java/org/apache/doris/catalog/MTMV.java | 8 +- .../mv/AbstractMaterializedViewRule.java | 2 +- .../mv/InitMaterializationContextHook.java | 5 +- .../mv/MaterializationContext.java | 13 +- .../exploration/mv/MaterializedViewUtils.java | 7 +- .../trees/plans/visitor/TableCollector.java | 14 +- .../mv/nested/ddl/customer_create.sql | 15 + .../mv/nested/ddl/customer_delete.sql | 1 + .../mv/nested/ddl/date_create.sql | 24 + .../mv/nested/ddl/date_delete.sql | 1 + .../mv/nested/ddl/lineorder_create.sql | 24 + .../mv/nested/ddl/lineorder_delete.sql | 1 + .../mv/nested/ddl/lineorder_flat_create.sql | 45 ++ .../mv/nested/ddl/lineorder_flat_delete.sql | 1 + .../mv/nested/ddl/part_create.sql | 16 + .../mv/nested/ddl/part_delete.sql | 1 + .../mv/nested/ddl/supplier_create.sql | 14 + .../mv/nested/ddl/supplier_delete.sql | 1 + .../mv/nested/nested_materialized_view.groovy | 750 +++++++++++++++++- 19 files changed, 920 insertions(+), 23 deletions(-) create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_delete.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_create.sql create mode 100644 regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_delete.sql diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index a3050bc3ac6b70c..fb32741afa384a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -40,6 +40,7 @@ import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVStatus; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.Maps; import com.google.common.collect.Sets; @@ -267,12 +268,15 @@ public Set getExcludedTriggerTables() { } } - public MTMVCache getOrGenerateCache() throws AnalysisException { + /** + * Called when in query, Should use one connection context in query + */ + public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws AnalysisException { if (cache == null) { writeMvLock(); try { if (cache == null) { - this.cache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this)); + this.cache = MTMVCache.from(this, connectionContext); } } finally { writeMvUnlock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index a88362e66976ec8..2274ad441f85ac1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -323,9 +323,9 @@ protected List doRewrite(StructInfo queryStructInfo, CascadesContext casca continue; } recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext); + rewriteResults.add(rewrittenPlan); // if rewrite successfully, try to regenerate mv scan because it maybe used again materializationContext.tryReGenerateMvScanPlan(cascadesContext); - rewriteResults.add(rewrittenPlan); } return rewriteResults; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index 311932fa1b7cf7a..d91fa5cb1a972aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -63,6 +63,9 @@ public void initMaterializationContext(CascadesContext cascadesContext) { } Plan rewritePlan = cascadesContext.getRewritePlan(); TableCollectorContext collectorContext = new TableCollectorContext(Sets.newHashSet(), true); + // Keep use one connection context when in query, if new connect context, + // the ConnectionContext.get() will change + collectorContext.setConnectContext(cascadesContext.getConnectContext()); rewritePlan.accept(TableCollector.INSTANCE, collectorContext); Set collectedTables = collectorContext.getCollectedTables(); if (collectedTables.isEmpty()) { @@ -80,7 +83,7 @@ public void initMaterializationContext(CascadesContext cascadesContext) { for (MTMV materializedView : availableMTMVs) { MTMVCache mtmvCache = null; try { - mtmvCache = materializedView.getOrGenerateCache(); + mtmvCache = materializedView.getOrGenerateCache(cascadesContext.getConnectContext()); } catch (AnalysisException e) { LOG.warn("MaterializationContext init mv cache generate fail", e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java index 2f0d04e1141f195..c2c2448b24c371c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java @@ -134,15 +134,14 @@ public void addMatchedGroup(GroupId groupId, boolean rewriteSuccess) { * Try to generate scan plan for materialization * if MaterializationContext is already rewritten successfully, then should generate new scan plan in later * query rewrite, because one plan may hit the materialized view repeatedly and the mv scan output - * should be different + * should be different. + * This method should be called when query rewrite successfully */ public void tryReGenerateMvScanPlan(CascadesContext cascadesContext) { - if (!this.matchedSuccessGroups.isEmpty()) { - this.mvScanPlan = doGenerateMvPlan(cascadesContext); - // mv output expression shuttle, this will be used to expression rewrite - this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, - this.mvScanPlan.getExpressions()); - } + this.mvScanPlan = doGenerateMvPlan(cascadesContext); + // mv output expression shuttle, this will be used to expression rewrite + this.mvExprToMvScanExprMapping = ExpressionMapping.generate(this.mvPlanOutputShuttledExpressions, + this.mvScanPlan.getOutput()); } public void addSlotMappingToCache(RelationMapping relationMapping, SlotMapping slotMapping) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 608f3b5f5f051ec..f1a90fb940c4531 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -184,12 +184,13 @@ public static Plan generateMvScanPlan(MTMV materializedView, CascadesContext cas LogicalOlapScan mvScan = new LogicalOlapScan( cascadesContext.getStatementContext().getNextRelationId(), materializedView, - ImmutableList.of(materializedView.getQualifiedDbName()), - // this must be empty, or it will be used to sample + materializedView.getFullQualifiers(), ImmutableList.of(), + materializedView.getBaseIndexId(), + PreAggStatus.on(), + // this must be empty, or it will be used to sample ImmutableList.of(), Optional.empty()); - mvScan = mvScan.withMaterializedIndexSelected(PreAggStatus.on(), materializedView.getBaseIndexId()); List mvProjects = mvScan.getOutput().stream().map(NamedExpression.class::cast) .collect(Collectors.toList()); return new LogicalProject(mvProjects, mvScan); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java index 94b1eb89c2f1082..5ab6b7ef015a17b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation; import org.apache.doris.nereids.trees.plans.physical.PhysicalCatalogRelation; import org.apache.doris.nereids.trees.plans.visitor.TableCollector.TableCollectorContext; +import org.apache.doris.qe.ConnectContext; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -72,7 +73,9 @@ private void expandMvAndCollect(MTMV mtmv, TableCollectorContext context) { if (!context.isExpand()) { return; } - MTMVCache expandedMv = MTMVCache.from(mtmv, MTMVPlanUtil.createMTMVContext(mtmv)); + // Make sure use only one connection context when in query to avoid ConnectionContext.get() wrong + MTMVCache expandedMv = MTMVCache.from(mtmv, context.getConnectContext() == null + ? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext()); expandedMv.getLogicalPlan().accept(this, context); } @@ -85,6 +88,7 @@ public static final class TableCollectorContext { private final Set targetTableTypes; // if expand the mv or not private final boolean expand; + private ConnectContext connectContext; public TableCollectorContext(Set targetTableTypes, boolean expand) { this.targetTableTypes = targetTableTypes; @@ -102,5 +106,13 @@ public Set getTargetTableTypes() { public boolean isExpand() { return expand; } + + public ConnectContext getConnectContext() { + return connectContext; + } + + public void setConnectContext(ConnectContext connectContext) { + this.connectContext = connectContext; + } } } diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_create.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_create.sql new file mode 100644 index 000000000000000..d13101b70e01bf9 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_create.sql @@ -0,0 +1,15 @@ +CREATE TABLE IF NOT EXISTS `customer` ( + `c_custkey` int(11) NOT NULL COMMENT "", + `c_name` varchar(26) NOT NULL COMMENT "", + `c_address` varchar(41) NOT NULL COMMENT "", + `c_city` varchar(11) NOT NULL COMMENT "", + `c_nation` varchar(16) NOT NULL COMMENT "", + `c_region` varchar(13) NOT NULL COMMENT "", + `c_phone` varchar(16) NOT NULL COMMENT "", + `c_mktsegment` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`c_custkey`) +DISTRIBUTED BY HASH(`c_custkey`) BUCKETS 1 +PROPERTIES ( +"replication_num" = "1" +); \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_delete.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_delete.sql new file mode 100644 index 000000000000000..fe22a226fedf85e --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/customer_delete.sql @@ -0,0 +1 @@ +truncate table customer; \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_create.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_create.sql new file mode 100644 index 000000000000000..cfab87b1d8bc085 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_create.sql @@ -0,0 +1,24 @@ +CREATE TABLE IF NOT EXISTS `date` ( + `d_datekey` int(11) NOT NULL COMMENT "", + `d_date` varchar(20) NOT NULL COMMENT "", + `d_dayofweek` varchar(10) NOT NULL COMMENT "", + `d_month` varchar(11) NOT NULL COMMENT "", + `d_year` int(11) NOT NULL COMMENT "", + `d_yearmonthnum` int(11) NOT NULL COMMENT "", + `d_yearmonth` varchar(9) NOT NULL COMMENT "", + `d_daynuminweek` int(11) NOT NULL COMMENT "", + `d_daynuminmonth` int(11) NOT NULL COMMENT "", + `d_daynuminyear` int(11) NOT NULL COMMENT "", + `d_monthnuminyear` int(11) NOT NULL COMMENT "", + `d_weeknuminyear` int(11) NOT NULL COMMENT "", + `d_sellingseason` varchar(14) NOT NULL COMMENT "", + `d_lastdayinweekfl` int(11) NOT NULL COMMENT "", + `d_lastdayinmonthfl` int(11) NOT NULL COMMENT "", + `d_holidayfl` int(11) NOT NULL COMMENT "", + `d_weekdayfl` int(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`d_datekey`) +DISTRIBUTED BY HASH(`d_datekey`) BUCKETS 1 +PROPERTIES ( +"replication_num" = "1" +); \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_delete.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_delete.sql new file mode 100644 index 000000000000000..12933cbbad92daa --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/date_delete.sql @@ -0,0 +1 @@ +truncate table `date`; \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_create.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_create.sql new file mode 100644 index 000000000000000..c066ea8a0d70c81 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_create.sql @@ -0,0 +1,24 @@ +CREATE TABLE IF NOT EXISTS `lineorder` ( + `lo_orderkey` bigint(20) NOT NULL COMMENT "", + `lo_linenumber` bigint(20) NOT NULL COMMENT "", + `lo_custkey` int(11) NOT NULL COMMENT "", + `lo_partkey` int(11) NOT NULL COMMENT "", + `lo_suppkey` int(11) NOT NULL COMMENT "", + `lo_orderdate` int(11) NOT NULL COMMENT "", + `lo_orderpriority` varchar(16) NOT NULL COMMENT "", + `lo_shippriority` int(11) NOT NULL COMMENT "", + `lo_quantity` bigint(20) NOT NULL COMMENT "", + `lo_extendedprice` bigint(20) NOT NULL COMMENT "", + `lo_ordtotalprice` bigint(20) NOT NULL COMMENT "", + `lo_discount` bigint(20) NOT NULL COMMENT "", + `lo_revenue` bigint(20) NOT NULL COMMENT "", + `lo_supplycost` bigint(20) NOT NULL COMMENT "", + `lo_tax` bigint(20) NOT NULL COMMENT "", + `lo_commitdate` bigint(20) NOT NULL COMMENT "", + `lo_shipmode` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`lo_orderkey`, `lo_linenumber`) +DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 1 +PROPERTIES ( +"replication_num" = "1" +); \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_delete.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_delete.sql new file mode 100644 index 000000000000000..329e040060edc63 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_delete.sql @@ -0,0 +1 @@ +truncate table lineorder; \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_create.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_create.sql new file mode 100644 index 000000000000000..35faee0786cb89a --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_create.sql @@ -0,0 +1,45 @@ +CREATE TABLE IF NOT EXISTS `lineorder_flat` ( + `LO_ORDERDATE` date NOT NULL COMMENT "", + `LO_ORDERKEY` int(11) NOT NULL COMMENT "", + `LO_LINENUMBER` tinyint(4) NOT NULL COMMENT "", + `LO_CUSTKEY` int(11) NOT NULL COMMENT "", + `LO_PARTKEY` int(11) NOT NULL COMMENT "", + `LO_SUPPKEY` int(11) NOT NULL COMMENT "", + `LO_ORDERPRIORITY` varchar(100) NOT NULL COMMENT "", + `LO_SHIPPRIORITY` tinyint(4) NOT NULL COMMENT "", + `LO_QUANTITY` tinyint(4) NOT NULL COMMENT "", + `LO_EXTENDEDPRICE` int(11) NOT NULL COMMENT "", + `LO_ORDTOTALPRICE` int(11) NOT NULL COMMENT "", + `LO_DISCOUNT` tinyint(4) NOT NULL COMMENT "", + `LO_REVENUE` int(11) NOT NULL COMMENT "", + `LO_SUPPLYCOST` int(11) NOT NULL COMMENT "", + `LO_TAX` tinyint(4) NOT NULL COMMENT "", + `LO_COMMITDATE` date NOT NULL COMMENT "", + `LO_SHIPMODE` varchar(100) NOT NULL COMMENT "", + `C_NAME` varchar(100) NOT NULL COMMENT "", + `C_ADDRESS` varchar(100) NOT NULL COMMENT "", + `C_CITY` varchar(100) NOT NULL COMMENT "", + `C_NATION` varchar(100) NOT NULL COMMENT "", + `C_REGION` varchar(100) NOT NULL COMMENT "", + `C_PHONE` varchar(100) NOT NULL COMMENT "", + `C_MKTSEGMENT` varchar(100) NOT NULL COMMENT "", + `S_NAME` varchar(100) NOT NULL COMMENT "", + `S_ADDRESS` varchar(100) NOT NULL COMMENT "", + `S_CITY` varchar(100) NOT NULL COMMENT "", + `S_NATION` varchar(100) NOT NULL COMMENT "", + `S_REGION` varchar(100) NOT NULL COMMENT "", + `S_PHONE` varchar(100) NOT NULL COMMENT "", + `P_NAME` varchar(100) NOT NULL COMMENT "", + `P_MFGR` varchar(100) NOT NULL COMMENT "", + `P_CATEGORY` varchar(100) NOT NULL COMMENT "", + `P_BRAND` varchar(100) NOT NULL COMMENT "", + `P_COLOR` varchar(100) NOT NULL COMMENT "", + `P_TYPE` varchar(100) NOT NULL COMMENT "", + `P_SIZE` tinyint(4) NOT NULL COMMENT "", + `P_CONTAINER` varchar(100) NOT NULL COMMENT "" +) ENGINE=OLAP +UNIQUE KEY(`LO_ORDERDATE`, `LO_ORDERKEY`, `LO_LINENUMBER`) +DISTRIBUTED BY HASH(`LO_ORDERKEY`) BUCKETS 1 +PROPERTIES ( +"replication_num" = "1" +); \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_delete.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_delete.sql new file mode 100644 index 000000000000000..13f6c32bb65a60d --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/lineorder_flat_delete.sql @@ -0,0 +1 @@ +truncate table lineorder_flat; \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_create.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_create.sql new file mode 100644 index 000000000000000..2405026a3cd1528 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_create.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS `part` ( + `p_partkey` int(11) NOT NULL COMMENT "", + `p_name` varchar(23) NOT NULL COMMENT "", + `p_mfgr` varchar(7) NOT NULL COMMENT "", + `p_category` varchar(8) NOT NULL COMMENT "", + `p_brand` varchar(10) NOT NULL COMMENT "", + `p_color` varchar(12) NOT NULL COMMENT "", + `p_type` varchar(26) NOT NULL COMMENT "", + `p_size` int(11) NOT NULL COMMENT "", + `p_container` varchar(11) NOT NULL COMMENT "" +) +UNIQUE KEY (`p_partkey`) +DISTRIBUTED BY HASH(`p_partkey`) BUCKETS 1 +PROPERTIES ( +"replication_num" = "1" +); \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_delete.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_delete.sql new file mode 100644 index 000000000000000..02c6abd2539add4 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/part_delete.sql @@ -0,0 +1 @@ +truncate table `part`; \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_create.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_create.sql new file mode 100644 index 000000000000000..1580a8fcca4084f --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_create.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS `supplier` ( + `s_suppkey` int(11) NOT NULL COMMENT "", + `s_name` varchar(26) NOT NULL COMMENT "", + `s_address` varchar(26) NOT NULL COMMENT "", + `s_city` varchar(11) NOT NULL COMMENT "", + `s_nation` varchar(16) NOT NULL COMMENT "", + `s_region` varchar(13) NOT NULL COMMENT "", + `s_phone` varchar(16) NOT NULL COMMENT "" +) +UNIQUE KEY (`s_suppkey`) +DISTRIBUTED BY HASH(`s_suppkey`) BUCKETS 1 +PROPERTIES ( +"replication_num" = "1" +); \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_delete.sql b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_delete.sql new file mode 100644 index 000000000000000..39e663134cabd00 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested/ddl/supplier_delete.sql @@ -0,0 +1 @@ +truncate table `supplier`; \ No newline at end of file diff --git a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy index 5a5ddc5b3c717ad..f7b47ef39596634 100644 --- a/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy @@ -16,22 +16,102 @@ // under the License. suite("nested_materialized_view") { + + + def compare_res = { def stmt -> + sql "SET enable_materialized_view_rewrite=false" + def origin_res = sql stmt + logger.info("origin_res: " + origin_res) + sql "SET enable_materialized_view_rewrite=true" + def mv_origin_res = sql stmt + logger.info("mv_origin_res: " + mv_origin_res) + assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size())) + for (int row = 0; row < mv_origin_res.size(); row++) { + assertTrue(mv_origin_res[row].size() == origin_res[row].size()) + for (int col = 0; col < mv_origin_res[row].size(); col++) { + assertTrue(mv_origin_res[row][col] == origin_res[row][col]) + } + } + } + + // ssb_sf1_p1 is writted to test unique key table merge correctly. + // It creates unique key table and sets bucket num to 1 in order to make sure that + // many rowsets will be created during loading and then the merge process will be triggered. + + def tables = ["customer", "lineorder", "part", "date", "supplier"] + def columns = ["""c_custkey,c_name,c_address,c_city,c_nation,c_region,c_phone,c_mktsegment,no_use""", + """lo_orderkey,lo_linenumber,lo_custkey,lo_partkey,lo_suppkey,lo_orderdate,lo_orderpriority, + lo_shippriority,lo_quantity,lo_extendedprice,lo_ordtotalprice,lo_discount, + lo_revenue,lo_supplycost,lo_tax,lo_commitdate,lo_shipmode,lo_dummy""", + """p_partkey,p_name,p_mfgr,p_category,p_brand,p_color,p_type,p_size,p_container,p_dummy""", + """d_datekey,d_date,d_dayofweek,d_month,d_year,d_yearmonthnum,d_yearmonth, + d_daynuminweek,d_daynuminmonth,d_daynuminyear,d_monthnuminyear,d_weeknuminyear, + d_sellingseason,d_lastdayinweekfl,d_lastdayinmonthfl,d_holidayfl,d_weekdayfl,d_dummy""", + """s_suppkey,s_name,s_address,s_city,s_nation,s_region,s_phone,s_dummy"""] + + for (String table in tables) { + sql new File("""${context.file.parent}/ddl/${table}_create.sql""").text + sql new File("""${context.file.parent}/ddl/${table}_delete.sql""").text + } + def i = 0 + for (String tableName in tables) { + streamLoad { + // a default db 'regression_test' is specified in + // ${DORIS_HOME}/conf/regression-conf.groovy + table tableName + + // default label is UUID: + // set 'label' UUID.randomUUID().toString() + + // default column_separator is specify in doris fe config, usually is '\t'. + // this line change to ',' + set 'column_separator', '|' + set 'compress_type', 'GZ' + set 'columns', columns[i] + + + // relate to ${DORIS_HOME}/regression-test/data/demo/streamload_input.csv. + // also, you can stream load a http stream, e.g. http://xxx/some.csv + file """${getS3Url()}/regression/ssb/sf1/${tableName}.tbl.gz""" + + time 10000 // limit inflight 10s + + // stream load action will check result, include Success status, and NumberTotalRows == NumberLoadedRows + + // if declared a check callback, the default check condition will ignore. + // So you must check all condition + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(json.NumberTotalRows, json.NumberLoadedRows) + assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0) + } + } + i++ + } + sql """ sync """ + + String db = context.config.getDbNameByFile(context.file) sql "use ${db}" sql "SET enable_nereids_planner=true" sql "set runtime_filter_mode=OFF" sql "SET enable_fallback_to_original_planner=false" - sql "SET enable_materialized_view_rewrite=true" sql "SET enable_nereids_timeout = false" + sql "SET enable_materialized_view_rewrite=true" sql "SET enable_materialized_view_nest_rewrite = true" def create_mtmv = { db_name, mv_name, mv_sql -> sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" sql""" - CREATE MATERIALIZED VIEW ${mv_name} + CREATE MATERIALIZED VIEW ${mv_name} BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 - PROPERTIES ('replication_num' = '1') + PROPERTIES ('replication_num' = '1') AS ${mv_sql} """ @@ -50,8 +130,8 @@ suite("nested_materialized_view") { o_orderstatus CHAR(1) NOT NULL, o_totalprice DECIMALV3(15,2) NOT NULL, o_orderdate DATE NOT NULL, - o_orderpriority CHAR(15) NOT NULL, - o_clerk CHAR(15) NOT NULL, + o_orderpriority CHAR(15) NOT NULL, + o_clerk CHAR(15) NOT NULL, o_shippriority INTEGER NOT NULL, o_comment VARCHAR(79) NOT NULL ) @@ -104,7 +184,7 @@ suite("nested_materialized_view") { ps_suppkey INTEGER NOT NULL, ps_availqty INTEGER NOT NULL, ps_supplycost DECIMALV3(15,2) NOT NULL, - ps_comment VARCHAR(199) NOT NULL + ps_comment VARCHAR(199) NOT NULL ) DUPLICATE KEY(ps_partkey, ps_suppkey) DISTRIBUTED BY HASH(ps_partkey) BUCKETS 3 @@ -130,7 +210,7 @@ suite("nested_materialized_view") { (3, 1, 'o', 33.5, '2023-12-10', 'a', 'b', 1, 'yy'), (4, 2, 'o', 43.2, '2023-12-11', 'c','d',2, 'mm'), (5, 2, 'o', 56.2, '2023-12-12', 'c','d',2, 'mi'), - (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); + (5, 2, 'o', 1.2, '2023-12-12', 'c','d',2, 'mi'); """ sql """ @@ -192,7 +272,661 @@ suite("nested_materialized_view") { sql("${query1_0}") contains("mv1_0_inner_mv(mv1_0_inner_mv)") } - order_qt_query1_1_after "${query1_0}" + + // complex nest mv rewrite + create_mtmv(db, "mv1_a", """ + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19930423 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate;""") + + create_mtmv(db, "mv2_a", """ + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19930423 + group by + lo_custkey, + lo_orderdate;""") + + create_mtmv(db, "mv4_a", """ + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19930423 + group by + lo_partkey, + lo_orderdate;""") + + create_mtmv(db, "mv_all_6_a", """ + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value4, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum +from + mv1_a t1 + left join mv2_a t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join mv4_a t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join mv4_a t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring'; + """) + + create_mtmv(db, "mv1_b", """ +select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19980421 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate;""") + + create_mtmv(db, "mv2_b", """ + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19980421 + group by + lo_custkey, + lo_orderdate;""") + + create_mtmv(db, "mv4_b", """ + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19980421 + group by + lo_partkey, + lo_orderdate;""") + + create_mtmv(db, "mv_all_6_b", """ + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value4, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum +from + mv1_b t1 + left join mv2_b t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join mv4_b t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join mv4_b t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring'; + """) + + + create_mtmv(db, "mv1_c", """ + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940413 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate;""") + + create_mtmv(db, "mv2_c", """ + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940413 + group by + lo_custkey, + lo_orderdate;""") + + create_mtmv(db, "mv4_c", """ + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940413 + group by + lo_partkey, + lo_orderdate;""") + + create_mtmv(db, "mv_all_6_c", """ + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value4, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum +from + mv1_c t1 + left join mv2_c t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join mv4_c t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join mv4_c t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring'; + """) + + + create_mtmv(db, "mv1_d", """ + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940218 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate;""") + + create_mtmv(db, "mv2_d", """ + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940218 + group by + lo_custkey, + lo_orderdate;""") + + create_mtmv(db, "mv4_d", """ + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 and lo_orderdate = 19940218 + group by + lo_partkey, + lo_orderdate;""") + + create_mtmv(db, "mv_all_6_d", """ + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value4, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum +from + mv1_d t1 + left join mv2_d t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join mv4_d t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join mv4_d t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring'; + """) + + def query2_0 = """ +select * from ( + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value3, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum + from + ( + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19930423 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate + ) t1 + left join ( + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19930423 + group by + lo_custkey, + lo_orderdate + ) t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value3 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19930423 + group by + lo_partkey, + lo_orderdate + ) t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19930423 + group by + lo_partkey, + lo_orderdate + ) t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring' + union all + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value3, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum + from + ( + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19980421 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate + ) t1 + left join ( + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19980421 + group by + lo_custkey, + lo_orderdate + ) t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value3 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19980421 + group by + lo_partkey, + lo_orderdate + ) t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19980421 + group by + lo_partkey, + lo_orderdate + ) t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring' + union ALL + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value3, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum + from + ( + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940413 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate + ) t1 + left join ( + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940413 + group by + lo_custkey, + lo_orderdate + ) t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value3 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940413 + group by + lo_partkey, + lo_orderdate + ) t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940413 + group by + lo_partkey, + lo_orderdate + ) t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring' + UNION ALL + select + '测试1' as nm, + '测试2' as t_nm, + t1.sum_value1, + t1.sum_value1 - t2.sum_value2, + t1.sum_value1 - t3.sum_value3, + t1.sum_value1 - t4.sum_value4, + t1.lo_custkey, + t5.p_name, + t6.d_yearmonthnum + from + ( + select + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value1 + FROM + lineorder + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940218 + group by + lo_custkey, + lo_partkey, + lo_orderkey, + lo_orderdate + ) t1 + left join ( + select + lo_custkey, + lo_orderdate, + sum(lo_revenue * lo_discount) as sum_value2 + FROM + lineorder + inner join customer on lo_custkey = c_custkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940218 + group by + lo_custkey, + lo_orderdate + ) t2 on t1.lo_custkey = t2.lo_custkey + and t1.lo_orderdate = t2.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value3 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940218 + group by + lo_partkey, + lo_orderdate + ) t3 on t1.lo_partkey = t3.lo_partkey + and t1.lo_orderdate = t3.lo_orderdate + left join ( + select + lo_partkey, + lo_orderdate, + sum(lo_extendedprice * lo_discount) as sum_value4 + FROM + lineorder + inner join supplier on lo_suppkey = s_suppkey + inner join date on lo_orderdate = d_datekey + where + d_daynuminweek > 0 + and lo_orderdate = 19940218 + group by + lo_partkey, + lo_orderdate + ) t4 on t1.lo_partkey = t4.lo_partkey + and t1.lo_orderdate = t4.lo_orderdate + left join part t5 on t1.lo_partkey = t5.p_partkey + and t5.p_name = 'forest chiffon' + left join date t6 on t1.lo_orderdate = t6.d_datekey + and t6.d_sellingseason = 'Spring' +) t order by 1,2,3,4,5,6,7,8,9; + """ + + sql "SET enable_materialized_view_rewrite= true" + sql "SET enable_materialized_view_nest_rewrite = true" + explain { + sql("${query2_0}") + check {result -> + result.contains("mv_all_6_a(mv_all_6_a)") && result.contains("mv_all_6_b(mv_all_6_b)") + && result.contains("mv_all_6_c(mv_all_6_c)") && result.contains("mv_all_6_d(mv_all_6_d)") + } + } + // Compare result when before and after mv rewrite + compare_res(query2_0) } From 05605d99a982d70aaf8c07bea673e86089677ef0 Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Fri, 17 May 2024 11:49:22 +0800 Subject: [PATCH 038/111] [opt](routine-load) optimize routine load task allocation algorithm (#34778) --- .../load/routineload/RoutineLoadManager.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java index 7eb7806f5158a0e..356262f8c2af98f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadManager.java @@ -488,6 +488,7 @@ public long getAvailableBeForTask(long jobId, long previousBeId) throws LoadExce readLock(); try { Map beIdToConcurrentTasks = getBeCurrentTasksNumMap(); + int previousBeIdleTaskNum = 0; // 1. Find if the given BE id has more than half of available slots if (previousBeId != -1L && availableBeIds.contains(previousBeId)) { @@ -495,22 +496,22 @@ public long getAvailableBeForTask(long jobId, long previousBeId) throws LoadExce Backend previousBackend = Env.getCurrentSystemInfo().getBackend(previousBeId); // check previousBackend is not null && load available if (previousBackend != null && previousBackend.isLoadAvailable()) { - int idleTaskNum = 0; if (!beIdToMaxConcurrentTasks.containsKey(previousBeId)) { - idleTaskNum = 0; + previousBeIdleTaskNum = 0; } else if (beIdToConcurrentTasks.containsKey(previousBeId)) { - idleTaskNum = beIdToMaxConcurrentTasks.get(previousBeId) + previousBeIdleTaskNum = beIdToMaxConcurrentTasks.get(previousBeId) - beIdToConcurrentTasks.get(previousBeId); } else { - idleTaskNum = beIdToMaxConcurrentTasks.get(previousBeId); + previousBeIdleTaskNum = beIdToMaxConcurrentTasks.get(previousBeId); } - if (idleTaskNum > (Config.max_routine_load_task_num_per_be >> 1)) { + if (previousBeIdleTaskNum == Config.max_routine_load_task_num_per_be) { return previousBeId; } } } - // 2. The given BE id does not have available slots, find a BE with min tasks + // 2. we believe that the benefits of load balance outweigh the benefits of object pool cache, + // so we try to find the one with the most idle slots as much as possible // 3. The previous BE is not in cluster && is not load available, find a new BE with min tasks int idleTaskNum = 0; long resultBeId = -1L; @@ -530,6 +531,11 @@ public long getAvailableBeForTask(long jobId, long previousBeId) throws LoadExce maxIdleSlotNum = Math.max(maxIdleSlotNum, idleTaskNum); } } + // 4. on the basis of selecting the maximum idle slot be, + // try to reuse the object cache as much as possible + if (previousBeIdleTaskNum == maxIdleSlotNum) { + return previousBeId; + } return resultBeId; } finally { readUnlock(); From 274c96b12dbacac234c1f2b86492f8383b5c1929 Mon Sep 17 00:00:00 2001 From: kkop Date: Fri, 17 May 2024 13:44:47 +0800 Subject: [PATCH 039/111] [enhancement](regression-test) modify a key type tests (#34600) Co-authored-by: cjj2010 <2449402815@qq.com> --- ...est_unique_schema_key_change_modify.groovy | 583 +++++++++++++++++- 1 file changed, 580 insertions(+), 3 deletions(-) diff --git a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify.groovy b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify.groovy index ace0626b61e3379..c13d1468ada0dcc 100644 --- a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify.groovy +++ b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify.groovy @@ -24,7 +24,7 @@ suite("test_unique_schema_key_change_modify","p0") { sql """ DROP TABLE IF EXISTS ${tbName} """ def getTableStatusSql = " SHOW ALTER TABLE COLUMN WHERE IndexName='${tbName}' ORDER BY createtime DESC LIMIT 1 " - initTable = " CREATE TABLE IF NOT EXISTS ${tbName}\n" + + def initTable = " CREATE TABLE IF NOT EXISTS ${tbName}\n" + " (\n" + " `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" + " `username` VARCHAR(50) NOT NULL COMMENT \"用户昵称\",\n" + @@ -43,7 +43,7 @@ suite("test_unique_schema_key_change_modify","p0") { " \"enable_unique_key_merge_on_write\" = \"true\"\n" + " );" - initTableData = "insert into ${tbName} values(123456789, 'Alice', 0, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," + + def initTableData = "insert into ${tbName} values(123456789, 'Alice', 0, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," + " (234567890, 'Bob', 0, 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-02-02 12:00:00')," + " (345678901, 'Carol', 1, 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou', '2022-03-03 14:00:00')," + " (456789012, 'Dave', 0, 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-04-04 16:00:00')," + @@ -52,7 +52,7 @@ suite("test_unique_schema_key_change_modify","p0") { " (789012345, 'Grace', 0, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-07-07 22:00:00');" //TODO Test the unique model by modify a key type from BOOLEAN to TINYINT - errorMessage="errCode = 2, detailMessage = Can not change BOOLEAN to TINYINT" + def errorMessage="errCode = 2, detailMessage = Can not change BOOLEAN to TINYINT" expectException({ sql initTable sql initTableData @@ -953,5 +953,582 @@ suite("test_unique_schema_key_change_modify","p0") { }, insertSql, true, "${tbName}") }, errorMessage) + /** + * Test the unique model by modify a key type from FLOAT to other type + */ + sql """ DROP TABLE IF EXISTS ${tbName} """ + initTable = " CREATE TABLE IF NOT EXISTS ${tbName}\n" + + " (\n" + + " `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" + + " `username` VARCHAR(50) NOT NULL COMMENT \"用户昵称\",\n" + + " `score` FLOAT COMMENT \"分数\",\n" + + " `city` VARCHAR(20) COMMENT \"用户所在城市\",\n" + + " `age` SMALLINT COMMENT \"用户年龄\",\n" + + " `sex` TINYINT COMMENT \"用户性别\",\n" + + " `phone` LARGEINT COMMENT \"用户电话\",\n" + + " `address` VARCHAR(500) COMMENT \"用户地址\",\n" + + " `register_time` DATETIME COMMENT \"用户注册时间\"\n" + + " )\n" + + " UNIQUE KEY(`user_id`, `username`,`score`)\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 1\n" + + " PROPERTIES (\n" + + " \"replication_allocation\" = \"tag.location.default: 1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + " );" + + initTableData = "insert into ${tbName} values(123456789, 'Alice', 1.8, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," + + " (234567890, 'Bob', 1.89, 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-02-02 12:00:00')," + + " (345678901, 'Carol', 2.6, 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou', '2022-03-03 14:00:00')," + + " (456789012, 'Dave', 3.9, 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-04-04 16:00:00')," + + " (567890123, 'Eve', 4.2, 'Chengdu', 27, 0, 13572468091, 'No. 654 Street, Chengdu', '2022-05-05 18:00:00')," + + " (678901234, 'Frank', 2.5, 'Hangzhou', 32, 1, 13467985213, 'No. 321 Street, Hangzhou', '2022-06-06 20:00:00')," + + " (789012345, 'Grace', 2.1, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-07-07 22:00:00');" + + //TODO Test the unique model by modify a key type from FLOAT to BOOLEAN + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score BOOLEAN key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', false, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + // TODO Test the unique model by modify a key type from FLOAT to TINYINT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score TINYINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 2, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //Test the unique model by modify a key type from FLOAT to SMALLINT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score SMALLINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 3, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //Test the unique model by modify a key type from FLOAT to INT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + + sql """ alter table ${tbName} MODIFY column score INT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 4, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //Test the unique model by modify a key type from FLOAT to BIGINT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score BIGINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 545645, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //Test the unique model by modify a key type from FLOAT to LARGEINT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score FLOAT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 156546, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //Test the unique model by modify a key type from FLOAT to DOUBLE + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score DOUBLE key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.23, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //TODO Test the unique model by modify a key type from FLOAT to DECIMAL + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score DECIMAL(38,0) key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.23, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + + }, errorMessage) + + //TODO Test the unique model by modify a key type from FLOAT to CHAR + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score CHAR(15) key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //TODO Test the unique model by modify a key type from FLOAT to VARCHAR + //Test the unique model by modify a key type from FLOAT to VARCHAR + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score VARCHAR(100) key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //TODO Test the unique model by modify a key type from FLOAT to STRING + //Test the unique model by modify a key type from FLOAT to STRING + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score STRING key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + /** + * Test the unique model by modify a key type from DOUBLE to other type + */ + sql """ DROP TABLE IF EXISTS ${tbName} """ + initTable = " CREATE TABLE IF NOT EXISTS ${tbName}\n" + + " (\n" + + " `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" + + " `username` VARCHAR(50) NOT NULL COMMENT \"用户昵称\",\n" + + " `score` DOUBLE COMMENT \"分数\",\n" + + " `city` VARCHAR(20) COMMENT \"用户所在城市\",\n" + + " `age` SMALLINT COMMENT \"用户年龄\",\n" + + " `sex` TINYINT COMMENT \"用户性别\",\n" + + " `phone` LARGEINT COMMENT \"用户电话\",\n" + + " `address` VARCHAR(500) COMMENT \"用户地址\",\n" + + " `register_time` DATETIME COMMENT \"用户注册时间\"\n" + + " )\n" + + " UNIQUE KEY(`user_id`, `username`,`score`)\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 1\n" + + " PROPERTIES (\n" + + " \"replication_allocation\" = \"tag.location.default: 1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + " );" + + initTableData = "insert into ${tbName} values(123456789, 'Alice', 1.8, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," + + " (234567890, 'Bob', 1.89, 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-02-02 12:00:00')," + + " (345678901, 'Carol', 2.6, 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou', '2022-03-03 14:00:00')," + + " (456789012, 'Dave', 3.9, 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-04-04 16:00:00')," + + " (567890123, 'Eve', 4.2, 'Chengdu', 27, 0, 13572468091, 'No. 654 Street, Chengdu', '2022-05-05 18:00:00')," + + " (678901234, 'Frank', 2.5, 'Hangzhou', 32, 1, 13467985213, 'No. 321 Street, Hangzhou', '2022-06-06 20:00:00')," + + " (789012345, 'Grace', 2.1, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-07-07 22:00:00');" + + //Test the unique model by modify a key type from DOUBLE to BOOLEAN + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score BOOLEAN key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', false, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + // Test the unique model by modify a key type from DOUBLE to TINYINT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score TINYINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 2, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //Test the unique model by modify a key type from DOUBLE to SMALLINT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score SMALLINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 3, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //Test the unique model by modify a key type from DOUBLE to INT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + + sql """ alter table ${tbName} MODIFY column score INT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 4, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //Test the unique model by modify a key type from DOUBLE to BIGINT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score BIGINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 545645, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //Test the unique model by modify a key type from DOUBLE to LARGEINT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score FLOAT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 156546, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //Test the unique model by modify a key type from DOUBLE to FLOAT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score FLOAT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.23, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + // Test the unique model by modify a key type from DOUBLE to DECIMAL + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score DECIMAL(38,0) key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.23, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + + }, errorMessage) + + //TODO Test the unique model by modify a key type from DOUBLE to CHAR + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score CHAR(15) key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //TODO Test the unique model by modify a key type from DOUBLE to VARCHAR + //Test the unique model by modify a key type from FLOAT to VARCHAR + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score VARCHAR(100) key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //Test the unique model by modify a key type from DOUBLE to STRING + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column score STRING key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + /** + * Test the unique model by modify a key type from DECIMAL to other type + */ + sql """ DROP TABLE IF EXISTS ${tbName} """ + initTable = " CREATE TABLE IF NOT EXISTS ${tbName}\n" + + " (\n" + + " `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" + + " `username` VARCHAR(50) NOT NULL COMMENT \"用户昵称\",\n" + + " `rice` DECIMAL(38,10) COMMENT \"米粒\",\n" + + " `city` VARCHAR(20) COMMENT \"用户所在城市\",\n" + + " `age` SMALLINT COMMENT \"用户年龄\",\n" + + " `sex` TINYINT COMMENT \"用户性别\",\n" + + " `phone` LARGEINT COMMENT \"用户电话\",\n" + + " `address` VARCHAR(500) COMMENT \"用户地址\",\n" + + " `register_time` DATETIME COMMENT \"用户注册时间\"\n" + + " )\n" + + " UNIQUE KEY(`user_id`, `username`,`rice`)\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 1\n" + + " PROPERTIES (\n" + + " \"replication_allocation\" = \"tag.location.default: 1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + " );" + + initTableData = "insert into ${tbName} values(123456789, 'Alice', 1.8, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," + + " (234567890, 'Bob', 1.89, 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-02-02 12:00:00')," + + " (345678901, 'Carol', 2.6, 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou', '2022-03-03 14:00:00')," + + " (456789012, 'Dave', 3.9, 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-04-04 16:00:00')," + + " (567890123, 'Eve', 4.2, 'Chengdu', 27, 0, 13572468091, 'No. 654 Street, Chengdu', '2022-05-05 18:00:00')," + + " (678901234, 'Frank', 2.5, 'Hangzhou', 32, 1, 13467985213, 'No. 321 Street, Hangzhou', '2022-06-06 20:00:00')," + + " (789012345, 'Grace', 2.1, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-07-07 22:00:00');" + + //Test the unique model by modify a key type from DECIMAL to BOOLEAN + errorMessage = "errCode = 2, detailMessage = Can not change DECIMAL128 to BOOLEAN" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice BOOLEAN key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', false, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + // Test the unique model by modify a key type from DECIMAL to TINYINT + errorMessage = "errCode = 2, detailMessage = Can not change DECIMAL128 to TINYINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice TINYINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 2, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //Test the unique model by modify a key type from DECIMAL to SMALLINT + errorMessage = "errCode = 2, detailMessage = Can not change DECIMAL128 to SMALLINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice SMALLINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 3, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //Test the unique model by modify a key type from DECIMAL to INT + errorMessage = "errCode = 2, detailMessage = Can not change DECIMAL128 to INT" + expectException({ + sql initTable + sql initTableData + + sql """ alter table ${tbName} MODIFY column rice INT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 4, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //Test the unique model by modify a key type from DECIMAL to BIGINT + errorMessage = "errCode = 2, detailMessage = Can not change DECIMAL128 to BIGINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice BIGINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 545645, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //Test the unique model by modify a key type from DECIMAL to LARGEINT + errorMessage = "errCode = 2, detailMessage = Can not change DECIMAL128 to LARGEINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice LARGEINT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 156546, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + //Test the unique model by modify a key type from DECIMAL to FLOAT + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice FLOAT key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.23, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + + // Test the unique model by modify a key type from DECIMAL to DOUBLE + errorMessage = "errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice DOUBLE key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.23, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + + }, errorMessage) + + //Test the unique model by modify a key type from DECIMAL to CHAR + errorMessage = "errCode = 2, detailMessage = Can not change DECIMAL128 to CHAR" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice CHAR(15) key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) + + //Test the unique model by modify a key type from DECIMAL to VARCHAR + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice VARCHAR(100) key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, false, "${tbName}") + + sql """ DROP TABLE IF EXISTS ${tbName2} """ + initTable2 = " CREATE TABLE IF NOT EXISTS ${tbName2}\n" + + " (\n" + + " `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" + + " `username` VARCHAR(50) NOT NULL COMMENT \"用户昵称\",\n" + + " `rice` VARCHAR(100) COMMENT \"米粒\",\n" + + " `city` VARCHAR(20) COMMENT \"用户所在城市\",\n" + + " `age` SMALLINT COMMENT \"用户年龄\",\n" + + " `sex` TINYINT COMMENT \"用户性别\",\n" + + " `phone` LARGEINT COMMENT \"用户电话\",\n" + + " `address` VARCHAR(500) COMMENT \"用户地址\",\n" + + " `register_time` DATETIME COMMENT \"用户注册时间\"\n" + + " )\n" + + " UNIQUE KEY(`user_id`, `username`,`rice`)\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 1\n" + + " PROPERTIES (\n" + + " \"replication_allocation\" = \"tag.location.default: 1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + " );" + + initTableData2 = "insert into ${tbName2} values(123456789, 'Alice', '1.8000000000', 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," + + " (234567890, 'Bob', '1.8900000000', 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-02-02 12:00:00')," + + " (345678901, 'Carol', '2.6000000000', 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou', '2022-03-03 14:00:00')," + + " (456789012, 'Dave', '3.9000000000', 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-04-04 16:00:00')," + + " (567890123, 'Eve', '4.2000000000', 'Chengdu', 27, 0, 13572468091, 'No. 654 Street, Chengdu', '2022-05-05 18:00:00')," + + " (678901234, 'Frank', '2.5000000000', 'Hangzhou', 32, 1, 13467985213, 'No. 321 Street, Hangzhou', '2022-06-06 20:00:00')," + + " (123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," + + " (789012345, 'Grace', '2.1000000000', 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-07-07 22:00:00');" + + sql initTable2 + sql initTableData2 + checkTableData("${tbName}","${tbName2}","rice") + sql """ DROP TABLE IF EXISTS ${tbName} """ + + //Test the unique model by modify a key type from DECIMAL to STRING + errorMessage = "errCode = 2, detailMessage = String Type should not be used in key column[rice]." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column rice STRING key """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 'asd', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true, "${tbName}") + }, errorMessage) } From 437c1a1ba40fe6a1e545fbd64b3501eff220e30a Mon Sep 17 00:00:00 2001 From: kkop Date: Fri, 17 May 2024 13:48:18 +0800 Subject: [PATCH 040/111] [enhancement](regression-test) modify a key type tests (#34717) Co-authored-by: cjj2010 <2449402815@qq.com> --- ...est_unique_schema_key_change_modify.groovy | 4 +- ...st_unique_schema_key_change_modify1.groovy | 547 ++++++++++++++++++ 2 files changed, 549 insertions(+), 2 deletions(-) create mode 100644 regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify1.groovy diff --git a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify.groovy b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify.groovy index c13d1468ada0dcc..f33e1f24adeefe5 100644 --- a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify.groovy +++ b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify.groovy @@ -16,8 +16,8 @@ // under the License. suite("test_unique_schema_key_change_modify","p0") { - def tbName = "test_unique_schema_key_change_modify" - def tbName2 = "test_unique_schema_key_change_modify_1" + def tbName = "test_unique_schema_key_change_modify_1" + def tbName2 = "test_unique_schema_key_change_modify_2" /** * Test the unique model by modify a value type */ diff --git a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify1.groovy b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify1.groovy new file mode 100644 index 000000000000000..5576362a311a2a5 --- /dev/null +++ b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_modify1.groovy @@ -0,0 +1,547 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_unique_schema_key_change_modify1","p0") { + def tbName = "test_unique_schema_key_change_modify_3" + def tbName2 = "test_unique_schema_key_change_modify_4" + /** + * Test the unique model by modify a value type + */ + + /** + * Test the unique model by modify a key type from DATE to other type + */ + + sql """ DROP TABLE IF EXISTS ${tbName} """ + def getTableStatusSql = " SHOW ALTER TABLE COLUMN WHERE IndexName='${tbName}' ORDER BY createtime DESC LIMIT 1 " + def initTable = " CREATE TABLE IF NOT EXISTS ${tbName}\n" + + " (\n" + + " `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" + + " `username` VARCHAR(50) NOT NULL COMMENT \"用户昵称\",\n" + + " `login_time` DATE COMMENT \"用户登陆时间\",\n" + + " `is_teacher` BOOLEAN COMMENT \"是否是老师\",\n" + + " `city` VARCHAR(20) COMMENT \"用户所在城市\",\n" + + " `age` SMALLINT COMMENT \"用户年龄\",\n" + + " `sex` TINYINT COMMENT \"用户性别\",\n" + + " `phone` LARGEINT COMMENT \"用户电话\",\n" + + " `address` VARCHAR(500) COMMENT \"用户地址\",\n" + + " `register_time` DATETIME COMMENT \"用户注册时间\"\n" + + " )\n" + + " UNIQUE KEY(`user_id`, `username`, `login_time`)\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 1\n" + + " PROPERTIES (\n" + + " \"replication_allocation\" = \"tag.location.default: 1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + " );" + + def initTableData = "insert into ${tbName} values(123456789, 'Alice', '2022-01-01', 0, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00')," + + " (234567890, 'Bob', '2022-01-01 12:00:00', 0, 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-02-02 12:00:00')," + + " (345678901, 'Carol', '2022-01-01 12:00:00', 1, 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou', '2022-03-03 14:00:00')," + + " (456789012, 'Dave', '2022-01-01 12:00:00', 0, 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-04-04 16:00:00')," + + " (567890123, 'Eve', '2022-01-01 12:00:00', 0, 'Chengdu', 27, 0, 13572468091, 'No. 654 Street, Chengdu', '2022-05-05 18:00:00')," + + " (678901234, 'Frank', '2022-01-01 12:00:00', 1, 'Hangzhou', 32, 1, 13467985213, 'No. 321 Street, Hangzhou', '2022-06-06 20:00:00')," + + " (789012345, 'Grace', '2022-01-01 12:00:00', 0, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-07-07 22:00:00');" + + //TODO Test the unique model by modify a key type from DATE to BOOLEAN + def errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to BOOLEAN" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time BOOLEAN KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 0, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + //TODO Test the unique model by modify a key type from DATE to TINYINT + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to TINYINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time TINYINT KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + //TODO Test the unique model by modify a key type from DATE to SMALLINT + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to SMALLINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time SMALLINT KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + //TODO Test the unique model by modify a key type from DATE to INT + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to INT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time INT KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + + //TODO Test the unique model by modify a key type from DATE to BIGINT + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to BIGINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time BIGINT KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + + },errorMessage) + + + //TODO Test the unique model by modify a key type from DATE to FLOAT + errorMessage="errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time FLOAT KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.0, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + + //TODO Test the unique model by modify a key type from DATE to DECIMAL + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to DECIMAL32" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time DECIMAL KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.0, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + //TODO Test the unique model by modify a key type from DATE to CHAR + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to CHAR" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time CHAR KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', '1', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + + },errorMessage) + + //TODO Test the unique model by modify a key type from DATE to STRING + errorMessage="errCode = 2, detailMessage = String Type should not be used in key column[login_time]." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time STRING KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', '1', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + //TODO Test the unique model by modify a key type from DATE to VARCHAR + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to VARCHAR" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time VARCHAR(32) KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', '1', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + + /** + * Test the unique model by modify a key type from DATETIME to other type + */ + + + initTable = " CREATE TABLE IF NOT EXISTS ${tbName}\n" + + " (\n" + + " `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" + + " `username` VARCHAR(50) NOT NULL COMMENT \"用户昵称\",\n" + + " `login_time` DATETIME COMMENT \"用户登陆时间\",\n" + + " `is_teacher` BOOLEAN COMMENT \"是否是老师\",\n" + + " `city` VARCHAR(20) COMMENT \"用户所在城市\",\n" + + " `age` SMALLINT COMMENT \"用户年龄\",\n" + + " `sex` TINYINT COMMENT \"用户性别\",\n" + + " `phone` LARGEINT COMMENT \"用户电话\",\n" + + " `address` VARCHAR(500) COMMENT \"用户地址\",\n" + + " `register_time` DATETIME COMMENT \"用户注册时间\"\n" + + " )\n" + + " UNIQUE KEY(`user_id`, `username`, `login_time`)\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 1\n" + + " PROPERTIES (\n" + + " \"replication_allocation\" = \"tag.location.default: 1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + " );" + + initTableData = "insert into ${tbName} values(123456789, 'Alice', 0, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01', '2022-01-01 10:00:00')," + + " (234567890, 'Bob', 0, 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-01-01', '2022-02-02 12:00:00')," + + " (345678901, 'Carol', 1, 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou','2022-01-01', '2022-03-03 14:00:00')," + + " (456789012, 'Dave', 0, 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-01-01', '2022-04-04 16:00:00')," + + " (567890123, 'Eve', 0, 'Chengdu', 27, 0, 13572468091, 'No. 654 Street, Chengdu', '2022-01-01', '2022-05-05 18:00:00')," + + " (678901234, 'Frank', 1, 'Hangzhou', 32, 1, 13467985213, 'No. 321 Street, Hangzhou', '2022-01-01', '2022-06-06 20:00:00')," + + " (789012345, 'Grace', 0, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-01-01', '2022-07-07 22:00:00');" + + //TODO Test the unique model by modify a key type from DATETIME to BOOLEAN + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to BOOLEAN" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time BOOLEAN KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 0, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + //TODO Test the unique model by modify a key type from DATETIME to TINYINT + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to TINYINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time TINYINT KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + //TODO Test the unique model by modify a key type from DATETIME to SMALLINT + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to SMALLINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time SMALLINT KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + //TODO Test the unique model by modify a key type from DATETIME to INT + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to INT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time INT KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + + //TODO Test the unique model by modify a key type from DATETIME to BIGINT + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to BIGINT" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time BIGINT KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + + },errorMessage) + + + //TODO Test the unique model by modify a key type from DATETIME to FLOAT + errorMessage="errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time FLOAT KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.0, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + + //TODO Test the unique model by modify a key type from DATETIME to DECIMAL + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to DECIMAL32" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time DECIMAL KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.0, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + //TODO Test the unique model by modify a key type from DATETIME to CHAR + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to CHAR" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time CHAR KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', '1', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + + },errorMessage) + + //TODO Test the unique model by modify a key type from DATETIME to STRING + errorMessage="errCode = 2, detailMessage = String Type should not be used in key column[login_time]." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time STRING KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', '1', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + //TODO Test the unique model by modify a key type from DATETIME to VARCHAR + errorMessage="errCode = 2, detailMessage = Can not change DATEV2 to VARCHAR" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column login_time VARCHAR(32) KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', '1', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + /** + * Test the unique model by modify a key type from CHAR to other type + */ + + + initTable = " CREATE TABLE IF NOT EXISTS ${tbName}\n" + + " (\n" + + " `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" + + " `username` CHAR(255) NOT NULL COMMENT \"用户昵称\",\n" + + " `login_time` DATETIME COMMENT \"用户登陆时间\",\n" + + " `is_teacher` BOOLEAN COMMENT \"是否是老师\",\n" + + " `city` VARCHAR(20) COMMENT \"用户所在城市\",\n" + + " `age` SMALLINT COMMENT \"用户年龄\",\n" + + " `sex` TINYINT COMMENT \"用户性别\",\n" + + " `phone` LARGEINT COMMENT \"用户电话\",\n" + + " `address` VARCHAR(500) COMMENT \"用户地址\",\n" + + " `register_time` DATETIME COMMENT \"用户注册时间\"\n" + + " )\n" + + " UNIQUE KEY(`user_id`, `username`, `login_time`)\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 1\n" + + " PROPERTIES (\n" + + " \"replication_allocation\" = \"tag.location.default: 1\",\n" + + " \"enable_unique_key_merge_on_write\" = \"true\"\n" + + " );" + + initTableData = "insert into ${tbName} values(123456789, 'Alice', 0, 'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01', '2022-01-01 10:00:00')," + + " (234567890, 'Bob', 0, 'Shanghai', 30, 1, 13998765432, 'No. 456 Street, Shanghai', '2022-01-01', '2022-02-02 12:00:00')," + + " (345678901, 'Carol', 1, 'Guangzhou', 28, 0, 13724681357, 'No. 789 Street, Guangzhou','2022-01-01', '2022-03-03 14:00:00')," + + " (456789012, 'Dave', 0, 'Shenzhen', 35, 1, 13680864279, 'No. 987 Street, Shenzhen', '2022-01-01', '2022-04-04 16:00:00')," + + " (567890123, 'Eve', 0, 'Chengdu', 27, 0, 13572468091, 'No. 654 Street, Chengdu', '2022-01-01', '2022-05-05 18:00:00')," + + " (678901234, 'Frank', 1, 'Hangzhou', 32, 1, 13467985213, 'No. 321 Street, Hangzhou', '2022-01-01', '2022-06-06 20:00:00')," + + " (789012345, 'Grace', 0, 'Xian', 29, 0, 13333333333, 'No. 222 Street, Xian', '2022-01-01', '2022-07-07 22:00:00');" + + //TODO Test the unique model by modify a key type from CHAR to BOOLEAN + errorMessage="errCode = 2, detailMessage = Can not change VARCHAR to BOOLEAN" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username BOOLEAN KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 0, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + //TODO Test the unique model by modify a key type from CHAR to TINYINT + errorMessage="errCode = 2, detailMessage = Can not change default value" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username TINYINT KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + //TODO Test the unique model by modify a key type from CHAR to SMALLINT + errorMessage="errCode = 2, detailMessage = Can not change default value" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username SMALLINT KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + //TODO Test the unique model by modify a key type from CHAR to INT + errorMessage="errCode = 2, detailMessage = Can not change default value" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username INT KEY DEFAULT "1" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + + //TODO Test the unique model by modify a key type from CHAR to BIGINT + errorMessage="errCode = 2, detailMessage = Can not change default value" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username BIGINT KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + + },errorMessage) + + + //TODO Test the unique model by modify a key type from CHAR to FLOAT + errorMessage="errCode = 2, detailMessage = Float or double can not used as a key, use decimal instead." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username FLOAT KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.0, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + + //TODO Test the unique model by modify a key type from CHAR to DECIMAL + errorMessage="errCode = 2, detailMessage = Can not change VARCHAR to DECIMAL32" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username DECIMAL KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', 1.0, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + //TODO Test the unique model by modify a key type from CHAR to DATETIME + errorMessage="errCode = 2, detailMessage = date literal [0] is invalid: null" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username DATETIME KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', '1', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + + },errorMessage) + + //TODO Test the unique model by modify a key type from CHAR to STRING + errorMessage="errCode = 2, detailMessage = String Type should not be used in key column[username]." + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username STRING KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', '1', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + + + //TODO Test the unique model by modify a key type from CHAR to VARCHAR + errorMessage="errCode = 2, detailMessage = Can not change default value" + expectException({ + sql initTable + sql initTableData + sql """ alter table ${tbName} MODIFY column username VARCHAR(32) KEY DEFAULT "0" """ + insertSql = "insert into ${tbName} values(123456689, 'Alice', '1', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); " + waitForSchemaChangeDone({ + sql getTableStatusSql + time 60 + }, insertSql, true,"${tbName}") + },errorMessage) + +} From 9b5028785d13a588ab4307e626a5ef26b60ad145 Mon Sep 17 00:00:00 2001 From: xueweizhang Date: Fri, 17 May 2024 14:19:04 +0800 Subject: [PATCH 041/111] [fix](prepare) fix datetimev2 return err when binary_row_format (#34662) fix datetimev2 return err when binary_row_format. before pr, Backend return datetimev2 alwary by to_string. fix datatimev2 return metadata loss scale. --- .../schema_scanner/schema_columns_scanner.cpp | 27 +++++++++-- be/src/util/mysql_row_buffer.cpp | 46 +++++++++++-------- be/src/util/mysql_row_buffer.h | 4 +- .../serde/data_type_datetimev2_serde.cpp | 4 +- .../apache/doris/mysql/MysqlSerializer.java | 2 + .../test_compaction_uniq_keys_row_store.out | 32 ++++++------- .../scalar_types/sql/infomation_schema.out | 2 +- 7 files changed, 71 insertions(+), 46 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_columns_scanner.cpp b/be/src/exec/schema_scanner/schema_columns_scanner.cpp index 56a6c5f256e0f47..763f24b9e531ce3 100644 --- a/be/src/exec/schema_scanner/schema_columns_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_columns_scanner.cpp @@ -512,7 +512,9 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { { int64_t srcs[columns_num]; for (int i = 0; i < columns_num; ++i) { - if (_desc_result.columns[i].columnDesc.__isset.columnPrecision) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (_desc_result.columns[i].columnDesc.__isset.columnPrecision && + data_type != TPrimitiveType::DATETIMEV2) { srcs[i] = _desc_result.columns[i].columnDesc.columnPrecision; datas[i] = srcs + i; } else { @@ -525,7 +527,9 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { { int64_t srcs[columns_num]; for (int i = 0; i < columns_num; ++i) { - if (_desc_result.columns[i].columnDesc.__isset.columnScale) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (_desc_result.columns[i].columnDesc.__isset.columnScale && + data_type != TPrimitiveType::DATETIMEV2) { srcs[i] = _desc_result.columns[i].columnDesc.columnScale; datas[i] = srcs + i; } else { @@ -535,7 +539,20 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { RETURN_IF_ERROR(fill_dest_column_for_range(block, 11, datas)); } // DATETIME_PRECISION - { RETURN_IF_ERROR(fill_dest_column_for_range(block, 12, null_datas)); } + { + std::vector srcs(columns_num); + for (int i = 0; i < columns_num; ++i) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (_desc_result.columns[i].columnDesc.__isset.columnScale && + data_type == TPrimitiveType::DATETIMEV2) { + srcs[i] = _desc_result.columns[i].columnDesc.columnScale; + datas[i] = srcs.data() + i; + } else { + datas[i] = nullptr; + } + } + RETURN_IF_ERROR(fill_dest_column_for_range(block, 12, datas)); + } // CHARACTER_SET_NAME { RETURN_IF_ERROR(fill_dest_column_for_range(block, 13, null_datas)); } // COLLATION_NAME @@ -605,7 +622,9 @@ Status SchemaColumnsScanner::_fill_block_impl(vectorized::Block* block) { { int64_t srcs[columns_num]; for (int i = 0; i < columns_num; ++i) { - if (_desc_result.columns[i].columnDesc.__isset.columnScale) { + int data_type = _desc_result.columns[i].columnDesc.columnType; + if (_desc_result.columns[i].columnDesc.__isset.columnScale && + data_type != TPrimitiveType::DATETIMEV2) { srcs[i] = _desc_result.columns[i].columnDesc.columnScale; datas[i] = srcs + i; } else { diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp index a15fa37a0f31a32..3e4aa332cea44b7 100644 --- a/be/src/util/mysql_row_buffer.cpp +++ b/be/src/util/mysql_row_buffer.cpp @@ -385,19 +385,25 @@ int MysqlRowBuffer::push_timev2(double data, int scale) { template template -int MysqlRowBuffer::push_vec_datetime(DateType& data) { +int MysqlRowBuffer::push_vec_datetime(DateType& data, int scale) { if (is_binary_format && !_dynamic_mode) { - return push_datetime(data); + return push_datetime(data, scale); } char buf[64]; - char* pos = data.to_string(buf); + char* pos = nullptr; + if constexpr (std::is_same_v> || + std::is_same_v>) { + pos = data.to_string(buf, scale); + } else { + pos = data.to_string(buf); + } return push_string(buf, pos - buf - 1); } template template -int MysqlRowBuffer::push_datetime(const DateType& data) { +int MysqlRowBuffer::push_datetime(const DateType& data, int scale) { if (is_binary_format && !_dynamic_mode) { char buff[12], *pos; size_t length; @@ -410,16 +416,6 @@ int MysqlRowBuffer::push_datetime(const DateType& data) { pos[4] = (uchar)data.hour(); pos[5] = (uchar)data.minute(); pos[6] = (uchar)data.second(); - if constexpr (std::is_same_v> || - std::is_same_v>) { - int4store(pos + 7, data.microsecond()); - if (data.microsecond()) { - length = 11; - } - } else { - int4store(pos + 7, 0); - } - if (data.hour() || data.minute() || data.second()) { length = 7; } else if (data.year() || data.month() || data.day()) { @@ -427,6 +423,14 @@ int MysqlRowBuffer::push_datetime(const DateType& data) { } else { length = 0; } + if constexpr (std::is_same_v> || + std::is_same_v>) { + if (scale > 0 || data.microsecond()) { + int4store(pos + 7, data.microsecond()); + length = 11; + } + } + buff[0] = (char)length; // Length is stored first return append(buff, length + 1); } @@ -511,14 +515,16 @@ template class MysqlRowBuffer; template class MysqlRowBuffer; template int MysqlRowBuffer::push_vec_datetime>( - DateV2Value& value); + DateV2Value& value, int scale); template int MysqlRowBuffer::push_vec_datetime>( - DateV2Value& value); -template int MysqlRowBuffer::push_vec_datetime(VecDateTimeValue& value); + DateV2Value& value, int scale); +template int MysqlRowBuffer::push_vec_datetime(VecDateTimeValue& value, + int scale); template int MysqlRowBuffer::push_vec_datetime>( - DateV2Value& value); + DateV2Value& value, int scale); template int MysqlRowBuffer::push_vec_datetime>( - DateV2Value& value); -template int MysqlRowBuffer::push_vec_datetime(VecDateTimeValue& value); + DateV2Value& value, int scale); +template int MysqlRowBuffer::push_vec_datetime(VecDateTimeValue& value, + int scale); } // namespace doris diff --git a/be/src/util/mysql_row_buffer.h b/be/src/util/mysql_row_buffer.h index 6f12fda1ecb399d..b740efa7764ed1f 100644 --- a/be/src/util/mysql_row_buffer.h +++ b/be/src/util/mysql_row_buffer.h @@ -76,7 +76,7 @@ class MysqlRowBuffer { int push_time(double data); int push_timev2(double data, int scale); template - int push_datetime(const DateType& data); + int push_datetime(const DateType& data, int scale); int push_decimal(const DecimalV2Value& data, int round_scale); int push_ipv4(const IPv4Value& ipv4_val); int push_ipv6(const IPv6Value& ipv6_val); @@ -84,7 +84,7 @@ class MysqlRowBuffer { int push_null(); template - int push_vec_datetime(DateType& data); + int push_vec_datetime(DateType& data, int scale = -1); // this function reserved size, change the pos step size, return old pos // Becareful when use the returned pointer. diff --git a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp index 56aeb3c29cabedb..73e859f985a5cf4 100644 --- a/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp +++ b/be/src/vec/data_types/serde/data_type_datetimev2_serde.cpp @@ -178,10 +178,8 @@ Status DataTypeDateTimeV2SerDe::_write_column_to_mysql(const IColumn& column, int row_idx, bool col_const) const { auto& data = assert_cast&>(column).get_data(); const auto col_index = index_check_const(row_idx, col_const); - char buf[64]; DateV2Value date_val = binary_cast>(data[col_index]); - char* pos = date_val.to_string(buf, scale); // _nesting_level >= 2 means this datetimev2 is in complex type // and we should add double quotes if (_nesting_level >= 2) { @@ -189,7 +187,7 @@ Status DataTypeDateTimeV2SerDe::_write_column_to_mysql(const IColumn& column, return Status::InternalError("pack mysql buffer failed."); } } - if (UNLIKELY(0 != result.push_string(buf, pos - buf - 1))) { + if (UNLIKELY(0 != result.push_vec_datetime(date_val, scale))) { return Status::InternalError("pack mysql buffer failed."); } if (_nesting_level >= 2) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java index 228f3891ce90e58..1454dca3c3db9a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/MysqlSerializer.java @@ -298,6 +298,8 @@ public int getMysqlDecimals(Type type) { case DECIMAL64: case DECIMAL128: case DECIMAL256: + case TIMEV2: + case DATETIMEV2: return ((ScalarType) type).decimalScale(); case FLOAT: case DOUBLE: diff --git a/regression-test/data/compaction/test_compaction_uniq_keys_row_store.out b/regression-test/data/compaction/test_compaction_uniq_keys_row_store.out index e49aea97d906df6..cedf0dbe9bd696a 100644 --- a/regression-test/data/compaction/test_compaction_uniq_keys_row_store.out +++ b/regression-test/data/compaction/test_compaction_uniq_keys_row_store.out @@ -1,49 +1,49 @@ -- This file is automatically generated. You should know what you did if you want to edit this -- !point_select -- -1 2017-10-01 2017-10-01 2017-10-01 11:11:11.021 2017-10-01 11:11:11.011000 Beijing 10 1 2020-01-01 00:00:00 2020-01-01 00:00:00 2017-10-01 11:11:11.170 2017-10-01 11:11:11.110111 2020-01-01 00:00:00 1 30 20 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.021 2017-10-01T11:11:11.011 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2017-10-01T11:11:11.170 2017-10-01T11:11:11.110111 2020-01-01T00:00 1 30 20 -- !point_select -- -1 2017-10-01 2017-10-01 2017-10-01 11:11:11.022 2017-10-01 11:11:11.012000 Beijing 10 1 2020-01-02 00:00:00 2020-01-02 00:00:00 2017-10-01 11:11:11.160 2017-10-01 11:11:11.100111 2020-01-02 00:00:00 1 31 19 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.022 2017-10-01T11:11:11.012 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 -- !point_select -- -2 2017-10-01 2017-10-01 2017-10-01 11:11:11.023 2017-10-01 11:11:11.013000 Beijing 10 1 2020-01-02 00:00:00 2020-01-02 00:00:00 2017-10-01 11:11:11.150 2017-10-01 11:11:11.130111 2020-01-02 00:00:00 1 31 21 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.023 2017-10-01T11:11:11.013 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.150 2017-10-01T11:11:11.130111 2020-01-02T00:00 1 31 21 -- !point_select -- -2 2017-10-01 2017-10-01 2017-10-01 11:11:11.024 2017-10-01 11:11:11.014000 Beijing 10 1 2020-01-03 00:00:00 2020-01-03 00:00:00 2017-10-01 11:11:11.140 2017-10-01 11:11:11.120111 2020-01-03 00:00:00 1 32 20 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.024 2017-10-01T11:11:11.014 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.140 2017-10-01T11:11:11.120111 2020-01-03T00:00 1 32 20 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.025 2017-10-01 11:11:11.015000 Beijing 10 1 2020-01-03 00:00:00 2020-01-03 00:00:00 2017-10-01 11:11:11.100 2017-10-01 11:11:11.140111 2020-01-03 00:00:00 1 32 22 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.025 2017-10-01T11:11:11.015 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.100 2017-10-01T11:11:11.140111 2020-01-03T00:00 1 32 22 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.026 2017-10-01 11:11:11.016000 Beijing 10 1 2020-01-04 00:00:00 2020-01-04 00:00:00 2017-10-01 11:11:11.110 2017-10-01 11:11:11.150111 2020-01-04 00:00:00 1 33 21 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.026 2017-10-01T11:11:11.016 Beijing 10 1 2020-01-04T00:00 2020-01-04T00:00 2017-10-01T11:11:11.110 2017-10-01T11:11:11.150111 2020-01-04T00:00 1 33 21 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.027 2017-10-01 11:11:11.017000 Beijing 10 1 1970-01-01 00:00:00 1970-01-01 00:00:00 1970-01-01 00:00:00.111 1970-01-01 00:00:00.000000 2020-01-05 00:00:00 1 34 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.027 2017-10-01T11:11:11.017 Beijing 10 1 1970-01-01T00:00 1970-01-01T00:00 1970-01-01T00:00:00.111 1970-01-01T00:00 2020-01-05T00:00 1 34 20 -- !point_select -- -4 2017-10-01 2017-10-01 2017-10-01 11:11:11.028 2017-10-01 11:11:11.018000 Beijing 10 1 1970-01-01 00:00:00 1970-01-01 00:00:00 1970-01-01 00:00:00.111 1970-01-01 00:00:00.000000 2020-01-05 00:00:00 1 34 20 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.028 2017-10-01T11:11:11.018 Beijing 10 1 1970-01-01T00:00 1970-01-01T00:00 1970-01-01T00:00:00.111 1970-01-01T00:00 2020-01-05T00:00 1 34 20 -- !point_select -- -1 2017-10-01 2017-10-01 2017-10-01 11:11:11.021 2017-10-01 11:11:11.011000 Beijing 10 1 2020-01-01 00:00:00 2020-01-01 00:00:00 2017-10-01 11:11:11.170 2017-10-01 11:11:11.110111 2020-01-01 00:00:00 1 30 20 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.021 2017-10-01T11:11:11.011 Beijing 10 1 2020-01-01T00:00 2020-01-01T00:00 2017-10-01T11:11:11.170 2017-10-01T11:11:11.110111 2020-01-01T00:00 1 30 20 -- !point_select -- -1 2017-10-01 2017-10-01 2017-10-01 11:11:11.022 2017-10-01 11:11:11.012000 Beijing 10 1 2020-01-02 00:00:00 2020-01-02 00:00:00 2017-10-01 11:11:11.160 2017-10-01 11:11:11.100111 2020-01-02 00:00:00 1 31 19 +1 2017-10-01 2017-10-01 2017-10-01T11:11:11.022 2017-10-01T11:11:11.012 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.160 2017-10-01T11:11:11.100111 2020-01-02T00:00 1 31 19 -- !point_select -- -2 2017-10-01 2017-10-01 2017-10-01 11:11:11.023 2017-10-01 11:11:11.013000 Beijing 10 1 2020-01-02 00:00:00 2020-01-02 00:00:00 2017-10-01 11:11:11.150 2017-10-01 11:11:11.130111 2020-01-02 00:00:00 1 31 21 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.023 2017-10-01T11:11:11.013 Beijing 10 1 2020-01-02T00:00 2020-01-02T00:00 2017-10-01T11:11:11.150 2017-10-01T11:11:11.130111 2020-01-02T00:00 1 31 21 -- !point_select -- -2 2017-10-01 2017-10-01 2017-10-01 11:11:11.024 2017-10-01 11:11:11.014000 Beijing 10 1 2020-01-03 00:00:00 2020-01-03 00:00:00 2017-10-01 11:11:11.140 2017-10-01 11:11:11.120111 2020-01-03 00:00:00 1 32 20 +2 2017-10-01 2017-10-01 2017-10-01T11:11:11.024 2017-10-01T11:11:11.014 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.140 2017-10-01T11:11:11.120111 2020-01-03T00:00 1 32 20 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.025 2017-10-01 11:11:11.015000 Beijing 10 1 2020-01-03 00:00:00 2020-01-03 00:00:00 2017-10-01 11:11:11.100 2017-10-01 11:11:11.140111 2020-01-03 00:00:00 1 32 22 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.025 2017-10-01T11:11:11.015 Beijing 10 1 2020-01-03T00:00 2020-01-03T00:00 2017-10-01T11:11:11.100 2017-10-01T11:11:11.140111 2020-01-03T00:00 1 32 22 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.026 2017-10-01 11:11:11.016000 Beijing 10 1 2020-01-04 00:00:00 2020-01-04 00:00:00 2017-10-01 11:11:11.110 2017-10-01 11:11:11.150111 2020-01-04 00:00:00 1 33 21 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.026 2017-10-01T11:11:11.016 Beijing 10 1 2020-01-04T00:00 2020-01-04T00:00 2017-10-01T11:11:11.110 2017-10-01T11:11:11.150111 2020-01-04T00:00 1 33 21 -- !point_select -- -3 2017-10-01 2017-10-01 2017-10-01 11:11:11.027 2017-10-01 11:11:11.017000 Beijing 10 1 1970-01-01 00:00:00 1970-01-01 00:00:00 1970-01-01 00:00:00.111 1970-01-01 00:00:00.000000 2020-01-05 00:00:00 1 34 20 +3 2017-10-01 2017-10-01 2017-10-01T11:11:11.027 2017-10-01T11:11:11.017 Beijing 10 1 1970-01-01T00:00 1970-01-01T00:00 1970-01-01T00:00:00.111 1970-01-01T00:00 2020-01-05T00:00 1 34 20 -- !point_select -- -4 2017-10-01 2017-10-01 2017-10-01 11:11:11.028 2017-10-01 11:11:11.018000 Beijing 10 1 1970-01-01 00:00:00 1970-01-01 00:00:00 1970-01-01 00:00:00.111 1970-01-01 00:00:00.000000 2020-01-05 00:00:00 1 34 20 +4 2017-10-01 2017-10-01 2017-10-01T11:11:11.028 2017-10-01T11:11:11.018 Beijing 10 1 1970-01-01T00:00 1970-01-01T00:00 1970-01-01T00:00:00.111 1970-01-01T00:00 2020-01-05T00:00 1 34 20 diff --git a/regression-test/data/datatype_p0/scalar_types/sql/infomation_schema.out b/regression-test/data/datatype_p0/scalar_types/sql/infomation_schema.out index a5c5eaad199399f..97e08bc652c685d 100644 --- a/regression-test/data/datatype_p0/scalar_types/sql/infomation_schema.out +++ b/regression-test/data/datatype_p0/scalar_types/sql/infomation_schema.out @@ -14,7 +14,7 @@ internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_decimal internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_date 12 \N YES date \N \N \N \N \N \N \N date \N \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_datetime 13 \N YES datetime \N \N \N \N \N \N \N datetime \N \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_datev2 14 \N YES date \N \N \N \N \N \N \N date \N \N \N \N -internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_datetimev2 15 \N YES datetime \N \N 18 0 \N \N \N datetime \N 0 \N \N +internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_datetimev2 15 \N YES datetime \N \N \N \N 0 \N \N datetime \N \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_char 16 \N YES char 15 60 \N \N \N \N \N char(15) 15 \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_varchar 17 \N YES varchar 100 400 \N \N \N \N \N varchar(100) 100 \N \N \N internal regression_test_datatype_p0_scalar_types tbl_scalar_types_dup c_string 18 \N YES varchar 2147483643 8589934572 \N \N \N \N \N string 2147483643 \N \N \N From e66dd5886015e29ebb65b06a092558defe836bd5 Mon Sep 17 00:00:00 2001 From: airborne12 Date: Fri, 17 May 2024 14:40:08 +0800 Subject: [PATCH 042/111] [Improve](inverted index) improve performance by introducing bulk contains for bitmap in _build_index_result_column (#34831) --- be/src/olap/rowset/segment_v2/segment_iterator.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp b/be/src/olap/rowset/segment_v2/segment_iterator.cpp index a74fb305795c310..bb91a14bd48799e 100644 --- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp +++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp @@ -2510,12 +2510,13 @@ void SegmentIterator::_build_index_result_column(const uint16_t* sel_rowid_idx, vectorized::ColumnUInt8::Container& vec_match_pred = index_result_column->get_data(); vec_match_pred.resize(block->rows()); size_t idx_in_selected = 0; + roaring::BulkContext bulk_context; for (uint32_t i = 0; i < _current_batch_rows_read; i++) { auto rowid = _block_rowids[i]; if (sel_rowid_idx == nullptr || (idx_in_selected < select_size && i == sel_rowid_idx[idx_in_selected])) { - if (index_result.contains(rowid)) { + if (index_result.containsBulk(bulk_context, rowid)) { vec_match_pred[idx_in_selected] = true; } else { vec_match_pred[idx_in_selected] = false; From bfd875eae396c10d7c615e9f0071602a189ee809 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B0=A2=E5=81=A5?= Date: Fri, 17 May 2024 15:06:49 +0800 Subject: [PATCH 043/111] [opt](nereids) lazy get expression map when comparing hypergraph (#34753) --- .../exploration/mv/HyperGraphComparator.java | 114 ++++++++++++------ .../mv/LogicalCompatibilityContext.java | 31 +---- 2 files changed, 80 insertions(+), 65 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java index 0c9dc9c33deece3..d180254c7fe1a6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/HyperGraphComparator.java @@ -46,6 +46,7 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -120,30 +121,37 @@ private ComparisonResult isLogicCompatible() { } // 3 try to construct a map which can be mapped from edge to edge - Map queryToView = constructQueryToViewMapWithExpr(); - if (!makeViewJoinCompatible(queryToView)) { + Map queryToViewJoinEdge = constructQueryToViewJoinMapWithExpr(); + if (!makeViewJoinCompatible(queryToViewJoinEdge)) { return ComparisonResult.newInvalidResWithErrorMessage("Join types are not compatible\n"); } refreshViewEdges(); // 4 compare them by expression and nodes. Note compare edges after inferring for nodes - boolean matchNodes = queryToView.entrySet().stream() + boolean matchNodes = queryToViewJoinEdge.entrySet().stream() + .allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue())); + if (!matchNodes) { + return ComparisonResult.newInvalidResWithErrorMessage("Join nodes are not compatible\n"); + } + Map queryToViewFilterEdge = constructQueryToViewFilterMapWithExpr(); + matchNodes = queryToViewFilterEdge.entrySet().stream() .allMatch(e -> compareEdgeWithNode(e.getKey(), e.getValue())); if (!matchNodes) { return ComparisonResult.newInvalidResWithErrorMessage("Join nodes are not compatible\n"); } - queryToView.forEach(this::compareEdgeWithExpr); + queryToViewJoinEdge.forEach(this::compareJoinEdgeWithExpr); + queryToViewFilterEdge.forEach(this::compareFilterEdgeWithExpr); // 5 process residual edges - Sets.difference(getQueryJoinEdgeSet(), queryToView.keySet()) + Sets.difference(getQueryJoinEdgeSet(), queryToViewJoinEdge.keySet()) .forEach(e -> pullUpQueryExprWithEdge.put(e, e.getExpressions())); - Sets.difference(getQueryFilterEdgeSet(), queryToView.keySet()) + Sets.difference(getQueryFilterEdgeSet(), queryToViewFilterEdge.keySet()) .forEach(e -> pullUpQueryExprWithEdge.put(e, e.getExpressions())); - Sets.difference(getViewJoinEdgeSet(), Sets.newHashSet(queryToView.values())) + Sets.difference(getViewJoinEdgeSet(), Sets.newHashSet(queryToViewJoinEdge.values())) .stream() .filter(e -> !LongBitmap.isOverlap(e.getReferenceNodes(), eliminateViewNodesMap)) .forEach(e -> pullUpViewExprWithEdge.put(e, e.getExpressions())); - Sets.difference(getViewFilterEdgeSet(), Sets.newHashSet(queryToView.values())) + Sets.difference(getViewFilterEdgeSet(), Sets.newHashSet(queryToViewFilterEdge.values())) .stream() .filter(e -> !LongBitmap.isOverlap(e.getReferenceNodes(), eliminateViewNodesMap)) .forEach(e -> pullUpViewExprWithEdge.put(e, e.getExpressions())); @@ -238,7 +246,7 @@ private boolean compareNodeWithExpr(StructInfoNode query, StructInfoNode view) { int size = queryExprSetList.size(); for (int i = 0; i < size; i++) { Set mappingQueryExprSet = queryExprSetList.get(i).stream() - .map(e -> logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(e)) + .map(logicalCompatibilityContext::getViewNodeExprFromQuery) .collect(Collectors.toSet()); if (!mappingQueryExprSet.equals(viewExprSetList.get(i))) { return false; @@ -350,12 +358,6 @@ private Set getQueryFilterEdgeSet() { return ImmutableSet.copyOf(queryHyperGraph.getFilterEdges()); } - private List getQueryEdges() { - return ImmutableList.builder() - .addAll(getQueryJoinEdges()) - .addAll(getQueryFilterEdges()).build(); - } - private boolean makeViewJoinCompatible(Map queryToView) { for (Entry entry : queryToView.entrySet()) { if (entry.getKey() instanceof JoinEdge && entry.getValue() instanceof JoinEdge) { @@ -384,37 +386,50 @@ private List getViewFilterEdges() { return viewHyperGraph.getFilterEdges(); } - private List getViewEdges() { - return ImmutableList.builder() - .addAll(getViewJoinEdges()) - .addAll(getViewFilterEdges()).build(); - } - - private Map getQueryToViewExprMap() { - return logicalCompatibilityContext.getQueryToViewAllExpressionMapping(); - } - private Map getQueryToViewNodeIdMap() { return logicalCompatibilityContext.getQueryToViewNodeIDMapping(); } - private Map constructQueryToViewMapWithExpr() { - Map viewExprToEdge = getViewEdges().stream() + private Map constructQueryToViewJoinMapWithExpr() { + Map viewExprToEdge = getViewJoinEdges().stream() .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); - Map queryExprToEdge = getQueryEdges().stream() + Map queryExprToEdge = getQueryJoinEdges().stream() .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); - return queryExprToEdge.entrySet().stream() - .filter(entry -> viewExprToEdge.containsKey(getViewExprFromQueryExpr(entry.getKey()))) - .map(entry -> Pair.of(entry.getValue(), - viewExprToEdge.get(getViewExprFromQueryExpr(entry.getKey())))) - .distinct() - .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + + HashMap edgeMap = new HashMap<>(); + for (Entry entry : queryExprToEdge.entrySet()) { + if (edgeMap.containsKey(entry.getValue())) { + continue; + } + Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(entry.getKey()); + if (viewExprToEdge.containsKey(viewExpr)) { + edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr))); + } + } + return edgeMap; } - private Expression getViewExprFromQueryExpr(Expression query) { - return logicalCompatibilityContext.getQueryToViewAllExpressionMapping().get(query); + private Map constructQueryToViewFilterMapWithExpr() { + Map viewExprToEdge = getViewFilterEdges().stream() + .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) + .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + Map queryExprToEdge = getQueryFilterEdges().stream() + .flatMap(e -> e.getExpressions().stream().map(expr -> Pair.of(expr, e))) + .collect(ImmutableMap.toImmutableMap(p -> p.first, p -> p.second)); + + HashMap edgeMap = new HashMap<>(); + for (Entry entry : queryExprToEdge.entrySet()) { + if (edgeMap.containsKey(entry.getValue())) { + continue; + } + Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(entry.getKey()); + if (viewExprToEdge.containsKey(viewExpr)) { + edgeMap.put(entry.getValue(), Objects.requireNonNull(viewExprToEdge.get(viewExpr))); + } + } + return edgeMap; } private void refreshViewEdges() { @@ -510,16 +525,35 @@ private long rewriteQueryNodeMap(long bitmap) { return newBitmap; } - private void compareEdgeWithExpr(Edge query, Edge view) { + private void compareJoinEdgeWithExpr(Edge query, Edge view) { + Set queryExprSet = query.getExpressionSet(); + Set viewExprSet = view.getExpressionSet(); + + Set exprMappedOfView = new HashSet<>(); + List residualQueryExpr = new ArrayList<>(); + for (Expression queryExpr : queryExprSet) { + Expression viewExpr = logicalCompatibilityContext.getViewJoinExprFromQuery(queryExpr); + if (viewExprSet.contains(viewExpr)) { + exprMappedOfView.add(viewExpr); + } else { + residualQueryExpr.add(queryExpr); + } + } + List residualViewExpr = ImmutableList.copyOf(Sets.difference(viewExprSet, exprMappedOfView)); + pullUpQueryExprWithEdge.put(query, residualQueryExpr); + pullUpViewExprWithEdge.put(query, residualViewExpr); + } + + private void compareFilterEdgeWithExpr(Edge query, Edge view) { Set queryExprSet = query.getExpressionSet(); Set viewExprSet = view.getExpressionSet(); Set exprMappedOfView = new HashSet<>(); List residualQueryExpr = new ArrayList<>(); for (Expression queryExpr : queryExprSet) { - if (getQueryToViewExprMap().containsKey(queryExpr) && viewExprSet.contains( - getQueryToViewExprMap().get(queryExpr))) { - exprMappedOfView.add(getQueryToViewExprMap().get(queryExpr)); + Expression viewExpr = logicalCompatibilityContext.getViewFilterExprFromQuery(queryExpr); + if (viewExprSet.contains(viewExpr)) { + exprMappedOfView.add(viewExpr); } else { residualQueryExpr.add(queryExpr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java index 25bafeb64c16f48..ca13c9701dabc2c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/LogicalCompatibilityContext.java @@ -92,35 +92,16 @@ public BiMap getQueryToViewNodeIDMapping() { return queryToViewNodeIDMapping; } - /** - * Get all expression mapping in query to view - */ - @Deprecated - public BiMap getQueryToViewAllExpressionMapping() { - if (queryToViewAllExpressionMapping != null) { - return queryToViewAllExpressionMapping; - } - queryToViewAllExpressionMapping = HashBiMap.create(); - queryToViewAllExpressionMapping.putAll(getQueryToViewJoinEdgeExpressionMapping()); - queryToViewAllExpressionMapping.putAll(getQueryToViewNodeExpressionMapping()); - queryToViewAllExpressionMapping.putAll(getQueryToViewFilterEdgeExpressionMapping()); - return queryToViewAllExpressionMapping; - } - - public BiMap getQueryToViewJoinEdgeExpressionMapping() { - return queryToViewJoinEdgeExpressionMappingSupplier.get(); - } - - public BiMap getQueryToViewNodeExpressionMapping() { - return queryToViewNodeExpressionMappingSupplier.get(); + public Expression getViewJoinExprFromQuery(Expression queryJoinExpr) { + return queryToViewJoinEdgeExpressionMappingSupplier.get().get(queryJoinExpr); } - public BiMap getQueryToViewFilterEdgeExpressionMapping() { - return queryToViewFilterEdgeExpressionMappingSupplier.get(); + public Expression getViewFilterExprFromQuery(Expression queryJoinExpr) { + return queryToViewFilterEdgeExpressionMappingSupplier.get().get(queryJoinExpr); } - public ObjectId getPlanNodeId() { - return planNodeId; + public Expression getViewNodeExprFromQuery(Expression queryJoinExpr) { + return queryToViewNodeExpressionMappingSupplier.get().get(queryJoinExpr); } /** From 81bcb9d4909916dd4bdaee45228838e9e405c69d Mon Sep 17 00:00:00 2001 From: Xujian Duan <50550370+DarvenDuan@users.noreply.github.com> Date: Fri, 17 May 2024 16:53:37 +0800 Subject: [PATCH 044/111] [opt](planner)(Nereids) support auto aggregation for random distributed table (#33630) support auto aggregation for querying detail data of random distributed table: the same key column will return only one row. --- .../apache/doris/analysis/StmtRewriter.java | 236 ++++++++++++++++ .../doris/nereids/jobs/executor/Analyzer.java | 3 + .../apache/doris/nereids/rules/RuleType.java | 5 +- .../BuildAggForRandomDistributedTable.java | 257 ++++++++++++++++++ .../org/apache/doris/qe/StmtExecutor.java | 9 +- .../select_random_distributed_tbl.out | 217 +++++++++++++++ .../select_random_distributed_tbl.groovy | 134 +++++++++ 7 files changed, 857 insertions(+), 4 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BuildAggForRandomDistributedTable.java create mode 100644 regression-test/data/query_p0/aggregate/select_random_distributed_tbl.out create mode 100644 regression-test/suites/query_p0/aggregate/select_random_distributed_tbl.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java index 93823cf398c2dcc..8fcd54b4a1d8285 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/StmtRewriter.java @@ -20,9 +20,16 @@ package org.apache.doris.analysis; +import org.apache.doris.catalog.AggStateType; +import org.apache.doris.catalog.AggregateType; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DistributionInfo; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.FunctionSet; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.TableAliasGenerator; @@ -1366,4 +1373,233 @@ public static boolean rewriteByPolicy(StatementBase statementBase, Analyzer anal } return reAnalyze; } + + /** + * + * @param column the column of SlotRef + * @param selectList new selectList for selectStmt + * @param groupByExprs group by Exprs for selectStmt + * @return true if ref can be rewritten + */ + private static boolean rewriteSelectList(Column column, SelectList selectList, ArrayList groupByExprs) { + SlotRef slot = new SlotRef(null, column.getName()); + if (column.isKey()) { + selectList.addItem(new SelectListItem(slot, column.getName())); + groupByExprs.add(slot); + return true; + } else if (column.isAggregated()) { + FunctionCallExpr func = generateAggFunction(slot, column); + if (func != null) { + selectList.addItem(new SelectListItem(func, column.getName())); + return true; + } + } + return false; + } + + /** + * rewrite stmt for querying random distributed table, construct an aggregation node for pre-agg + * * CREATE TABLE `tbl` ( + * `k1` BIGINT NULL DEFAULT "10", + * `k3` SMALLINT NULL, + * `a` BIGINT SUM NULL DEFAULT "0" + * ) ENGINE=OLAP + * AGGREGATE KEY(`k1`, `k2`) + * DISTRIBUTED BY RANDOM BUCKETS 1 + * PROPERTIES ( + * "replication_allocation" = "tag.location.default: 1" + * ) + * e.g., + * original: select * from tbl + * rewrite: select * from (select k1, k2, sum(pv) from tbl group by k1, k2) t + * do not rewrite if no need two phase agg: + * e.g., + * 1. select max(k1) from tbl + * 2. select sum(a) from tbl + * + * @param statementBase stmt to rewrite + * @param analyzer the analyzer + * @return true if rewritten + * @throws UserException + */ + public static boolean rewriteForRandomDistribution(StatementBase statementBase, Analyzer analyzer) + throws UserException { + boolean reAnalyze = false; + if (!(statementBase instanceof SelectStmt)) { + return false; + } + SelectStmt selectStmt = (SelectStmt) statementBase; + for (int i = 0; i < selectStmt.fromClause.size(); i++) { + TableRef tableRef = selectStmt.fromClause.get(i); + // Recursively rewrite subquery + if (tableRef instanceof InlineViewRef) { + InlineViewRef viewRef = (InlineViewRef) tableRef; + if (rewriteForRandomDistribution(viewRef.getQueryStmt(), viewRef.getAnalyzer())) { + reAnalyze = true; + } + continue; + } + TableIf table = tableRef.getTable(); + if (!(table instanceof OlapTable)) { + continue; + } + // only rewrite random distributed AGG_KEY table + OlapTable olapTable = (OlapTable) table; + if (olapTable.getKeysType() != KeysType.AGG_KEYS) { + continue; + } + DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); + if (distributionInfo.getType() != DistributionInfo.DistributionInfoType.RANDOM) { + continue; + } + + // check agg function and column agg type + AggregateInfo aggInfo = selectStmt.getAggInfo(); + GroupByClause groupByClause = selectStmt.getGroupByClause(); + boolean aggTypeMatch = true; + if (aggInfo != null || groupByClause != null) { + if (aggInfo != null) { + ArrayList aggExprs = aggInfo.getAggregateExprs(); + if (aggExprs.stream().anyMatch(expr -> !aggTypeMatch(expr.getFnName().getFunction(), expr))) { + aggTypeMatch = false; + } + List groupExprs = aggInfo.getGroupingExprs(); + if (groupExprs.stream().anyMatch(expr -> !isKeyOrConstantExpr(expr))) { + aggTypeMatch = false; + } + } + if (groupByClause != null) { + List groupByExprs = groupByClause.getGroupingExprs(); + if (groupByExprs.stream().anyMatch(expr -> !isKeyOrConstantExpr(expr))) { + aggTypeMatch = false; + } + } + if (aggTypeMatch) { + continue; + } + } + // construct a new InlineViewRef for pre-agg + boolean canRewrite = true; + SelectList selectList = new SelectList(); + ArrayList groupingExprs = new ArrayList<>(); + List columns = olapTable.getBaseSchema(); + for (Column col : columns) { + if (!rewriteSelectList(col, selectList, groupingExprs)) { + canRewrite = false; + break; + } + } + if (!canRewrite) { + continue; + } + Expr whereClause = selectStmt.getWhereClause() == null ? null : selectStmt.getWhereClause().clone(); + SelectStmt newSelectSmt = new SelectStmt(selectList, + new FromClause(Lists.newArrayList(tableRef)), + whereClause, + new GroupByClause(groupingExprs, GroupByClause.GroupingType.GROUP_BY), + null, + null, + LimitElement.NO_LIMIT); + InlineViewRef inlineViewRef = new InlineViewRef(tableRef.getAliasAsName().getTbl(), newSelectSmt); + inlineViewRef.setJoinOp(tableRef.getJoinOp()); + inlineViewRef.setLeftTblRef(tableRef.getLeftTblRef()); + inlineViewRef.setOnClause(tableRef.getOnClause()); + tableRef.setOnClause(null); + tableRef.setLeftTblRef(null); + tableRef.setOnClause(null); + if (selectStmt.fromClause.size() > i + 1) { + selectStmt.fromClause.get(i + 1).setLeftTblRef(inlineViewRef); + } + selectStmt.fromClause.set(i, inlineViewRef); + selectStmt.analyze(analyzer); + reAnalyze = true; + } + return reAnalyze; + } + + /** + * check if the agg type of functionCall match the agg type of column + * @param functionName the functionName of functionCall + * @param expr FunctionCallExpr + * @return true if agg type match + */ + private static boolean aggTypeMatch(String functionName, Expr expr) { + if (expr.getChildren().isEmpty()) { + if (expr instanceof SlotRef) { + Column col = ((SlotRef) expr).getDesc().getColumn(); + if (col.isKey()) { + return functionName.equalsIgnoreCase("MAX") + || functionName.equalsIgnoreCase("MIN"); + } + if (col.isAggregated()) { + AggregateType aggType = col.getAggregationType(); + // agg type not mach + if (aggType == AggregateType.GENERIC) { + return col.getType().isAggStateType(); + } + if (aggType == AggregateType.HLL_UNION) { + return functionName.equalsIgnoreCase(FunctionSet.HLL_UNION) + || functionName.equalsIgnoreCase(FunctionSet.HLL_UNION_AGG); + } + if (aggType == AggregateType.BITMAP_UNION) { + return functionName.equalsIgnoreCase(FunctionSet.BITMAP_UNION) + || functionName.equalsIgnoreCase(FunctionSet.BITMAP_UNION_COUNT) + || functionName.equalsIgnoreCase(FunctionSet.BITMAP_INTERSECT); + } + return functionName.equalsIgnoreCase(aggType.name()); + } + } + return false; + } + List children = expr.getChildren(); + return children.stream().allMatch(child -> aggTypeMatch(functionName, child)); + } + + /** + * check if the columns in expr is key column or constant, if group by clause contains value column, need rewrite + * + * @param expr expr to check + * @return true if all columns is key column or constant + */ + private static boolean isKeyOrConstantExpr(Expr expr) { + if (expr instanceof SlotRef) { + Column col = ((SlotRef) expr).getDesc().getColumn(); + return col.isKey(); + } else if (expr.isConstant()) { + return true; + } + List children = expr.getChildren(); + return children.stream().allMatch(StmtRewriter::isKeyOrConstantExpr); + } + + /** + * generate aggregation function according to the aggType of column + * + * @param slot slot of column + * @return aggFunction generated + */ + private static FunctionCallExpr generateAggFunction(SlotRef slot, Column column) { + AggregateType aggregateType = column.getAggregationType(); + switch (aggregateType) { + case SUM: + case MAX: + case MIN: + case HLL_UNION: + case BITMAP_UNION: + case QUANTILE_UNION: + FunctionName funcName = new FunctionName(aggregateType.toString().toLowerCase()); + return new FunctionCallExpr(funcName, new FunctionParams(false, Lists.newArrayList(slot))); + case GENERIC: + Type type = column.getType(); + if (!type.isAggStateType()) { + return null; + } + AggStateType aggState = (AggStateType) type; + // use AGGREGATE_FUNCTION_UNION to aggregate multiple agg_state into one + FunctionName functionName = new FunctionName(aggState.getFunctionName() + "_union"); + return new FunctionCallExpr(functionName, new FunctionParams(false, Lists.newArrayList(slot))); + default: + return null; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index ac0a44210717fb4..77d23464e65965e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -26,6 +26,7 @@ import org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver; import org.apache.doris.nereids.rules.analysis.BindSink; import org.apache.doris.nereids.rules.analysis.BindSlotWithPaths; +import org.apache.doris.nereids.rules.analysis.BuildAggForRandomDistributedTable; import org.apache.doris.nereids.rules.analysis.CheckAfterBind; import org.apache.doris.nereids.rules.analysis.CheckAnalysis; import org.apache.doris.nereids.rules.analysis.CheckPolicy; @@ -176,6 +177,8 @@ private static List buildAnalyzeJobs(Optional c topDown(new EliminateGroupByConstant()), topDown(new SimplifyAggGroupBy()), + // run BuildAggForRandomDistributedTable before NormalizeAggregate in order to optimize the agg plan + topDown(new BuildAggForRandomDistributedTable()), topDown(new NormalizeAggregate()), topDown(new HavingToFilter()), bottomUp(new SemiJoinCommute()), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index c688d7d5b3fdee5..c4eb7fe9b063480 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -313,7 +313,10 @@ public enum RuleType { // topn opts DEFER_MATERIALIZE_TOP_N_RESULT(RuleTypeClass.REWRITE), - + // pre agg for random distributed table + BUILD_AGG_FOR_RANDOM_DISTRIBUTED_TABLE_PROJECT_SCAN(RuleTypeClass.REWRITE), + BUILD_AGG_FOR_RANDOM_DISTRIBUTED_TABLE_FILTER_SCAN(RuleTypeClass.REWRITE), + BUILD_AGG_FOR_RANDOM_DISTRIBUTED_TABLE_AGG_SCAN(RuleTypeClass.REWRITE), // exploration rules TEST_EXPLORATION(RuleTypeClass.EXPLORATION), OR_EXPANSION(RuleTypeClass.EXPLORATION), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BuildAggForRandomDistributedTable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BuildAggForRandomDistributedTable.java new file mode 100644 index 000000000000000..86c89e49d3d6a51 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BuildAggForRandomDistributedTable.java @@ -0,0 +1,257 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.analysis; + +import org.apache.doris.catalog.AggStateType; +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.FunctionRegistry; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Type; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.AggCombinerFunctionBuilder; +import org.apache.doris.nereids.trees.expressions.functions.FunctionBuilder; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.HllFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.HllUnion; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.QuantileUnion; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + +/** + * build agg plan for querying random distributed table + */ +public class BuildAggForRandomDistributedTable implements AnalysisRuleFactory { + + @Override + public List buildRules() { + return ImmutableList.of( + // Project(Scan) -> project(agg(scan)) + logicalProject(logicalOlapScan()).when(project -> isRandomDistributedTbl(project.child())) + .then(project -> preAggForRandomDistribution(project, project.child())) + .toRule(RuleType.BUILD_AGG_FOR_RANDOM_DISTRIBUTED_TABLE_PROJECT_SCAN), + // agg(scan) -> agg(agg(scan)), agg(agg) may optimized by MergeAggregate + logicalAggregate(logicalOlapScan()).when(agg -> isRandomDistributedTbl(agg.child())).whenNot(agg -> { + Set functions = agg.getAggregateFunctions(); + List groupByExprs = agg.getGroupByExpressions(); + // check if need generate an inner agg plan or not + // should not rewrite twice if we had rewritten olapScan to aggregate(olapScan) + return functions.stream().allMatch(this::aggTypeMatch) && groupByExprs.stream() + .allMatch(this::isKeyOrConstantExpr); + }) + .then(agg -> preAggForRandomDistribution(agg, agg.child())) + .toRule(RuleType.BUILD_AGG_FOR_RANDOM_DISTRIBUTED_TABLE_AGG_SCAN), + // filter(scan) -> filter(agg(scan)) + logicalFilter(logicalOlapScan()).when(filter -> isRandomDistributedTbl(filter.child())) + .then(filter -> preAggForRandomDistribution(filter, filter.child())) + .toRule(RuleType.BUILD_AGG_FOR_RANDOM_DISTRIBUTED_TABLE_FILTER_SCAN)); + + } + + /** + * check the olapTable of olapScan is randomDistributed table + * + * @param olapScan olap scan plan + * @return true if olapTable is randomDistributed table + */ + private boolean isRandomDistributedTbl(LogicalOlapScan olapScan) { + OlapTable olapTable = olapScan.getTable(); + KeysType keysType = olapTable.getKeysType(); + DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); + return keysType == KeysType.AGG_KEYS && distributionInfo.getType() == DistributionInfoType.RANDOM; + } + + /** + * add LogicalAggregate above olapScan for preAgg + * + * @param logicalPlan parent plan of olapScan + * @param olapScan olap scan plan, it may be LogicalProject, LogicalFilter, LogicalAggregate + * @return rewritten plan + */ + private Plan preAggForRandomDistribution(LogicalPlan logicalPlan, LogicalOlapScan olapScan) { + OlapTable olapTable = olapScan.getTable(); + List childOutputSlots = olapScan.computeOutput(); + List groupByExpressions = new ArrayList<>(); + List outputExpressions = new ArrayList<>(); + List columns = olapTable.getBaseSchema(); + + for (Column col : columns) { + // use exist slot in the plan + SlotReference slot = SlotReference.fromColumn(olapTable, col, col.getName(), olapScan.getQualifier()); + ExprId exprId = slot.getExprId(); + for (Slot childSlot : childOutputSlots) { + if (childSlot instanceof SlotReference && ((SlotReference) childSlot).getName() == col.getName()) { + exprId = childSlot.getExprId(); + slot = slot.withExprId(exprId); + break; + } + } + if (col.isKey()) { + groupByExpressions.add(slot); + outputExpressions.add(slot); + } else { + Expression function = generateAggFunction(slot, col); + // DO NOT rewrite + if (function == null) { + return logicalPlan; + } + Alias alias = new Alias(exprId, function, col.getName()); + outputExpressions.add(alias); + } + } + LogicalAggregate aggregate = new LogicalAggregate<>(groupByExpressions, outputExpressions, + olapScan); + return logicalPlan.withChildren(aggregate); + } + + /** + * generate aggregation function according to the aggType of column + * + * @param slot slot of column + * @return aggFunction generated + */ + private Expression generateAggFunction(SlotReference slot, Column column) { + AggregateType aggregateType = column.getAggregationType(); + switch (aggregateType) { + case SUM: + return new Sum(slot); + case MAX: + return new Max(slot); + case MIN: + return new Min(slot); + case HLL_UNION: + return new HllUnion(slot); + case BITMAP_UNION: + return new BitmapUnion(slot); + case QUANTILE_UNION: + return new QuantileUnion(slot); + case GENERIC: + Type type = column.getType(); + if (!type.isAggStateType()) { + return null; + } + AggStateType aggState = (AggStateType) type; + // use AGGREGATE_FUNCTION_UNION to aggregate multiple agg_state into one + String funcName = aggState.getFunctionName() + AggCombinerFunctionBuilder.UNION_SUFFIX; + FunctionRegistry functionRegistry = Env.getCurrentEnv().getFunctionRegistry(); + FunctionBuilder builder = functionRegistry.findFunctionBuilder(funcName, slot); + return builder.build(funcName, ImmutableList.of(slot)).first; + default: + return null; + } + } + + /** + * if the agg type of AggregateFunction is as same as the agg type of column, DO NOT need to rewrite + * + * @param function agg function to check + * @return true if agg type match + */ + private boolean aggTypeMatch(AggregateFunction function) { + List children = function.children(); + if (function.getName().equalsIgnoreCase("count")) { + Count count = (Count) function; + // do not rewrite for count distinct for key column + if (count.isDistinct()) { + return children.stream().allMatch(this::isKeyOrConstantExpr); + } + if (count.isStar()) { + return false; + } + } + return children.stream().allMatch(child -> aggTypeMatch(function, child)); + } + + /** + * check if the agg type of functionCall match the agg type of column + * + * @param function the functionCall + * @param expression expr to check + * @return true if agg type match + */ + private boolean aggTypeMatch(AggregateFunction function, Expression expression) { + if (expression.children().isEmpty()) { + if (expression instanceof SlotReference && ((SlotReference) expression).getColumn().isPresent()) { + Column col = ((SlotReference) expression).getColumn().get(); + String functionName = function.getName(); + if (col.isKey()) { + return functionName.equalsIgnoreCase("max") || functionName.equalsIgnoreCase("min"); + } + if (col.isAggregated()) { + AggregateType aggType = col.getAggregationType(); + // agg type not mach + if (aggType == AggregateType.GENERIC) { + return col.getType().isAggStateType(); + } + if (aggType == AggregateType.HLL_UNION) { + return function instanceof HllFunction; + } + if (aggType == AggregateType.BITMAP_UNION) { + return function instanceof BitmapFunction; + } + return functionName.equalsIgnoreCase(aggType.name()); + } + } + return false; + } + List children = expression.children(); + return children.stream().allMatch(child -> aggTypeMatch(function, child)); + } + + /** + * check if the columns in expr is key column or constant, if group by clause contains value column, need rewrite + * + * @param expr expr to check + * @return true if all columns is key column or constant + */ + private boolean isKeyOrConstantExpr(Expression expr) { + if (expr instanceof SlotReference && ((SlotReference) expr).getColumn().isPresent()) { + Column col = ((SlotReference) expr).getColumn().get(); + return col.isKey(); + } else if (expr.isConstant()) { + return true; + } + List children = expr.children(); + return children.stream().allMatch(this::isKeyOrConstantExpr); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java index 1cf499f0c6c1800..c2b0e9dd44440c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java @@ -1305,21 +1305,24 @@ private void analyzeAndGenerateQueryPlan(TQueryOptions tQueryOptions) throws Use reAnalyze = true; } if (parsedStmt instanceof SelectStmt) { - if (StmtRewriter.rewriteByPolicy(parsedStmt, analyzer)) { + if (StmtRewriter.rewriteByPolicy(parsedStmt, analyzer) + || StmtRewriter.rewriteForRandomDistribution(parsedStmt, analyzer)) { reAnalyze = true; } } if (parsedStmt instanceof SetOperationStmt) { List operands = ((SetOperationStmt) parsedStmt).getOperands(); for (SetOperationStmt.SetOperand operand : operands) { - if (StmtRewriter.rewriteByPolicy(operand.getQueryStmt(), analyzer)) { + if (StmtRewriter.rewriteByPolicy(operand.getQueryStmt(), analyzer) + || StmtRewriter.rewriteForRandomDistribution(operand.getQueryStmt(), analyzer)) { reAnalyze = true; } } } if (parsedStmt instanceof InsertStmt) { QueryStmt queryStmt = ((InsertStmt) parsedStmt).getQueryStmt(); - if (queryStmt != null && StmtRewriter.rewriteByPolicy(queryStmt, analyzer)) { + if (queryStmt != null && (StmtRewriter.rewriteByPolicy(queryStmt, analyzer) + || StmtRewriter.rewriteForRandomDistribution(queryStmt, analyzer))) { reAnalyze = true; } } diff --git a/regression-test/data/query_p0/aggregate/select_random_distributed_tbl.out b/regression-test/data/query_p0/aggregate/select_random_distributed_tbl.out new file mode 100644 index 000000000000000..1afb2a067626326 --- /dev/null +++ b/regression-test/data/query_p0/aggregate/select_random_distributed_tbl.out @@ -0,0 +1,217 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_1 -- +1 a 6 3 1 \N \N \N +2 b 15 6 4 \N \N \N + +-- !sql_2 -- +1 a 6 3 1 \N \N \N +2 b 15 6 4 \N \N \N + +-- !sql_3 -- +2 a 6 +3 b 15 + +-- !sql_4 -- +1 a 7 +2 b 16 + +-- !sql_5 -- +1 6 3 1 2.0 3 3 2.0 +2 15 6 4 5.0 3 3 5.0 + +-- !sql_6 -- +2 + +-- !sql_7 -- +2 + +-- !sql_8 -- +2 + +-- !sql_9 -- +15 + +-- !sql_10 -- +9 + +-- !sql_11 -- +5 + +-- !sql_12 -- +1 6 +4 15 + +-- !sql_13 -- +2 + +-- !sql_14 -- +2 + +-- !sql_15 -- +2 + +-- !sql_16 -- +2 + +-- !sql_1 -- +1 a 6 3 1 \N \N \N +2 b 15 6 4 \N \N \N + +-- !sql_2 -- +1 a 6 3 1 \N \N \N +2 b 15 6 4 \N \N \N + +-- !sql_3 -- +2 a 6 +3 b 15 + +-- !sql_4 -- +1 a 7 +2 b 16 + +-- !sql_5 -- +1 6 3 1 2.0 3 3 2.0 +2 15 6 4 5.0 3 3 5.0 + +-- !sql_6 -- +2 + +-- !sql_7 -- +2 + +-- !sql_8 -- +2 + +-- !sql_9 -- +15 + +-- !sql_10 -- +9 + +-- !sql_11 -- +5 + +-- !sql_12 -- +1 6 +4 15 + +-- !sql_13 -- +2 + +-- !sql_14 -- +2 + +-- !sql_15 -- +2 + +-- !sql_16 -- +2 + +-- !sql_1 -- +1 a 6 3 1 \N \N \N +2 b 15 6 4 \N \N \N + +-- !sql_2 -- +1 a 6 3 1 \N \N \N +2 b 15 6 4 \N \N \N + +-- !sql_3 -- +2 a 6 +3 b 15 + +-- !sql_4 -- +1 a 7 +2 b 16 + +-- !sql_5 -- +1 6 3 1 2.0 3 3 2.0 +2 15 6 4 5.0 3 3 5.0 + +-- !sql_6 -- +2 + +-- !sql_7 -- +2 + +-- !sql_8 -- +2 + +-- !sql_9 -- +15 + +-- !sql_10 -- +9 + +-- !sql_11 -- +5 + +-- !sql_12 -- +1 6 +4 15 + +-- !sql_13 -- +2 + +-- !sql_14 -- +2 + +-- !sql_15 -- +2 + +-- !sql_16 -- +2 + +-- !sql_1 -- +1 a 6 3 1 \N \N \N +2 b 15 6 4 \N \N \N + +-- !sql_2 -- +1 a 6 3 1 \N \N \N +2 b 15 6 4 \N \N \N + +-- !sql_3 -- +2 a 6 +3 b 15 + +-- !sql_4 -- +1 a 7 +2 b 16 + +-- !sql_5 -- +1 6 3 1 2.0 3 3 2.0 +2 15 6 4 5.0 3 3 5.0 + +-- !sql_6 -- +2 + +-- !sql_7 -- +2 + +-- !sql_8 -- +2 + +-- !sql_9 -- +15 + +-- !sql_10 -- +9 + +-- !sql_11 -- +5 + +-- !sql_12 -- +1 6 +4 15 + +-- !sql_13 -- +2 + +-- !sql_14 -- +2 + +-- !sql_15 -- +2 + +-- !sql_16 -- +2 + diff --git a/regression-test/suites/query_p0/aggregate/select_random_distributed_tbl.groovy b/regression-test/suites/query_p0/aggregate/select_random_distributed_tbl.groovy new file mode 100644 index 000000000000000..ff0df74589a9b19 --- /dev/null +++ b/regression-test/suites/query_p0/aggregate/select_random_distributed_tbl.groovy @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("select_random_distributed_tbl") { + def tableName = "random_distributed_tbl_test" + + sql "drop table if exists ${tableName};" + sql "set enable_agg_state=true;" + sql """ admin set frontend config("enable_quantile_state_type"="true"); """ + sql """ + CREATE TABLE ${tableName} + ( + `k1` LARGEINT NOT NULL, + `k2` VARCHAR(20) NULL, + `v_sum` BIGINT SUM NULL DEFAULT "0", + `v_max` INT MAX NULL DEFAULT "0", + `v_min` INT MIN NULL DEFAULT "99999", + `v_generic` AGG_STATE GENERIC, + `v_hll` HLL HLL_UNION NOT NULL, + `v_bitmap` BITMAP BITMAP_UNION NOT NULL, + `v_quantile_union` QUANTILE_STATE QUANTILE_UNION NOT NULL + ) ENGINE=OLAP + AGGREGATE KEY(`k1`, `k2`) + COMMENT 'OLAP' + DISTRIBUTED BY RANDOM BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql """ insert into ${tableName} values(1,"a",1,1,1,avg_state(1),hll_hash(1),bitmap_hash(1),to_quantile_state(1, 2048)) """ + sql """ insert into ${tableName} values(1,"a",2,2,2,avg_state(2),hll_hash(2),bitmap_hash(2),to_quantile_state(2, 2048)) """ + sql """ insert into ${tableName} values(1,"a",3,3,3,avg_state(3),hll_hash(3),bitmap_hash(3),to_quantile_state(3, 2048)) """ + sql """ insert into ${tableName} values(2,"b",4,4,4,avg_state(4),hll_hash(4),bitmap_hash(4),to_quantile_state(4, 2048)) """ + sql """ insert into ${tableName} values(2,"b",5,5,5,avg_state(5),hll_hash(5),bitmap_hash(5),to_quantile_state(5, 2048)) """ + sql """ insert into ${tableName} values(2,"b",6,6,6,avg_state(6),hll_hash(6),bitmap_hash(6),to_quantile_state(6, 2048)) """ + + for (int i = 0; i < 2; ++i) { + if (i == 0) { + // test legacy planner + sql "set enable_nereids_planner = false;" + } else if (i == 1) { + // test nereids planner + sql "set enable_nereids_planner = true;" + } + + def whereStr = "" + for (int j = 0; j < 2; ++j) { + if (j == 1) { + // test with filter + whereStr = "where k1 > 0" + } + def sql1 = "select * except (v_generic) from ${tableName} ${whereStr} order by k1, k2" + qt_sql_1 "${sql1}" + def res1 = sql """ explain ${sql1} """ + assertTrue(res1.toString().contains("VAGGREGATE")) + + def sql2 = "select k1 ,k2 ,v_sum ,v_max ,v_min ,v_hll ,v_bitmap ,v_quantile_union from ${tableName} ${whereStr} order by k1, k2" + qt_sql_2 "${sql2}" + def res2 = sql """ explain ${sql2} """ + assertTrue(res2.toString().contains("VAGGREGATE")) + + def sql3 = "select k1+1, k2, v_sum from ${tableName} ${whereStr} order by k1, k2" + qt_sql_3 "${sql3}" + def res3 = sql """ explain ${sql3} """ + assertTrue(res3.toString().contains("VAGGREGATE")) + + def sql4 = "select k1, k2, v_sum+1 from ${tableName} ${whereStr} order by k1, k2" + qt_sql_4 "${sql4}" + def res4 = sql """ explain ${sql4} """ + assertTrue(res4.toString().contains("VAGGREGATE")) + + def sql5 = """ select k1, sum(v_sum), max(v_max), min(v_min), avg_merge(v_generic), + hll_union_agg(v_hll), bitmap_union_count(v_bitmap), quantile_percent(quantile_union(v_quantile_union),0.5) + from ${tableName} ${whereStr} group by k1 order by k1 """ + qt_sql_5 "${sql5}" + + def sql6 = "select count(1) from ${tableName} ${whereStr}" + qt_sql_6 "${sql6}" + + def sql7 = "select count(*) from ${tableName} ${whereStr}" + qt_sql_7 "${sql7}" + + def sql8 = "select max(k1) from ${tableName} ${whereStr}" + qt_sql_8 "${sql8}" + def res8 = sql """ explain ${sql8} """ + // no pre agg + assertFalse(res8.toString().contains("sum")) + + def sql9 = "select max(v_sum) from ${tableName} ${whereStr}" + qt_sql_9 "${sql9}" + def res9 = sql """ explain ${sql9} """ + assertTrue(res9.toString().contains("sum")) + + def sql10 = "select sum(v_max) from ${tableName} ${whereStr}" + qt_sql_10 "${sql10}" + + def sql11 = "select sum(v_min) from ${tableName} ${whereStr}" + qt_sql_11 "${sql11}" + + // test group by value + def sql12 = "select v_min, sum(v_sum) from ${tableName} ${whereStr} group by v_min order by v_min" + qt_sql_12 "${sql12}" + + def sql13 = "select count(k1) from ${tableName} ${whereStr}" + qt_sql_13 "${sql13}" + + def sql14 = "select count(distinct k1) from ${tableName} ${whereStr}" + qt_sql_14 "${sql14}" + + def sql15 = "select count(v_sum) from ${tableName} ${whereStr}" + qt_sql_15 "${sql15}" + + def sql16 = "select count(distinct v_sum) from ${tableName} ${whereStr}" + qt_sql_16 "${sql16}" + } + } + + sql "drop table ${tableName};" +} \ No newline at end of file From e3e5f18f26af29edd0ab7363789d33b60fbc9adf Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Fri, 17 May 2024 17:16:14 +0800 Subject: [PATCH 045/111] [Fix](Json type) correct cast result for json type (#34764) --- be/src/util/jsonb_document.h | 2 +- be/src/vec/functions/function_cast.h | 79 ++++++++----------- be/test/vec/function/function_jsonb_test.cpp | 15 ++-- .../data/datatype_p0/json/json_cast.out | 43 ++++++++++ .../json_p0/test_json_load_and_function.out | 52 ++++++------ ...test_json_load_unique_key_and_function.out | 12 +-- .../jsonb_p0/test_jsonb_load_and_function.out | 52 ++++++------ ...est_jsonb_load_unique_key_and_function.out | 12 +-- .../nereids_function_p0/scalar_function/J.out | 52 ++++++------ .../json_p0/test_json_load_and_function.out | 52 ++++++------ ...test_json_load_unique_key_and_function.out | 12 +-- .../jsonb_p0/test_jsonb_load_and_function.out | 52 ++++++------ ...est_jsonb_load_unique_key_and_function.out | 12 +-- regression-test/data/variant_p0/load.out | 17 +++- .../suites/datatype_p0/json/json_cast.groovy | 34 ++++++++ regression-test/suites/variant_p0/load.groovy | 25 ++++-- 16 files changed, 307 insertions(+), 216 deletions(-) create mode 100644 regression-test/data/datatype_p0/json/json_cast.out create mode 100644 regression-test/suites/datatype_p0/json/json_cast.groovy diff --git a/be/src/util/jsonb_document.h b/be/src/util/jsonb_document.h index 73b6a44b38af06a..4d71cc9f6b31f74 100644 --- a/be/src/util/jsonb_document.h +++ b/be/src/util/jsonb_document.h @@ -498,7 +498,7 @@ class JsonbValue { bool isNull() const { return (type_ == JsonbType::T_Null); } bool isTrue() const { return (type_ == JsonbType::T_True); } bool isFalse() const { return (type_ == JsonbType::T_False); } - bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64(); } + bool isInt() const { return isInt8() || isInt16() || isInt32() || isInt64() || isInt128(); } bool isInt8() const { return (type_ == JsonbType::T_Int8); } bool isInt16() const { return (type_ == JsonbType::T_Int16); } bool isInt32() const { return (type_ == JsonbType::T_Int32); } diff --git a/be/src/vec/functions/function_cast.h b/be/src/vec/functions/function_cast.h index d19cb4d857496f7..7f3f74217b4dd93 100644 --- a/be/src/vec/functions/function_cast.h +++ b/be/src/vec/functions/function_cast.h @@ -89,6 +89,7 @@ #include "vec/data_types/data_type_struct.h" #include "vec/data_types/data_type_time.h" #include "vec/data_types/data_type_time_v2.h" +#include "vec/data_types/serde/data_type_serde.h" #include "vec/functions/function.h" #include "vec/functions/function_convert_tz.h" #include "vec/functions/function_helpers.h" @@ -762,29 +763,43 @@ struct ConvertImplGenericToJsonb { auto column_string = ColumnString::create(); JsonbWriter writer; + ColumnUInt8::MutablePtr col_null_map_to = ColumnUInt8::create(col_from.size()); + ColumnUInt8::Container* vec_null_map_to = &col_null_map_to->get_data(); + DataTypeSerDe::FormatOptions format_options; + format_options.converted_from_string = true; + DataTypeSerDeSPtr from_serde = type.get_serde(); + DataTypeSerDeSPtr to_serde = data_type_to->get_serde(); + auto col_to = data_type_to->create_column(); + auto tmp_col = ColumnString::create(); + vectorized::DataTypeSerDe::FormatOptions options; for (size_t i = 0; i < input_rows_count; i++) { // convert to string tmp_col->clear(); VectorBufferWriter write_buffer(*tmp_col.get()); - type.to_string(col_from, i, write_buffer); + Status st = + from_serde->serialize_column_to_json(col_from, i, i + 1, write_buffer, options); + // if serialized failed, will return null + (*vec_null_map_to)[i] = !st.ok(); + if (!st.ok()) { + col_to->insert_default(); + continue; + } write_buffer.commit(); writer.reset(); auto str_ref = tmp_col->get_data_at(0); - ReadBuffer read_buffer((char*)(str_ref.data), str_ref.size); + Slice data((char*)(str_ref.data), str_ref.size); // first try to parse string - Status st = data_type_to->from_string(read_buffer, column_string.get()); + st = to_serde->deserialize_one_cell_from_json(*col_to, data, format_options); + // if parsing failed, will return null + (*vec_null_map_to)[i] = !st.ok(); if (!st.ok()) { - // write raw string to jsonb - writer.writeStartString(); - writer.writeString(str_ref.data, str_ref.size); - writer.writeEndString(); - column_string->insert_data(writer.getOutput()->getBuffer(), - writer.getOutput()->getSize()); + col_to->insert_default(); } } - block.replace_by_position(result, std::move(column_string)); + block.replace_by_position( + result, ColumnNullable::create(std::move(col_to), std::move(col_null_map_to))); return Status::OK(); } }; @@ -845,39 +860,13 @@ struct ConvertImplFromJsonb { null_map[i] = 1; res[i] = 0; } - } else if constexpr (type_index == TypeIndex::Int8) { - if (value->isInt8()) { - res[i] = (int8_t)((const JsonbIntVal*)value)->val(); - } else { - null_map[i] = 1; - res[i] = 0; - } - } else if constexpr (type_index == TypeIndex::Int16) { - if (value->isInt8() || value->isInt16()) { - res[i] = (int16_t)((const JsonbIntVal*)value)->val(); - } else { - null_map[i] = 1; - res[i] = 0; - } - } else if constexpr (type_index == TypeIndex::Int32) { - if (value->isInt8() || value->isInt16() || value->isInt32()) { - res[i] = (int32_t)((const JsonbIntVal*)value)->val(); - } else { - null_map[i] = 1; - res[i] = 0; - } - } else if constexpr (type_index == TypeIndex::Int64) { - if (value->isInt8() || value->isInt16() || value->isInt32() || - value->isInt64()) { - res[i] = (int64_t)((const JsonbIntVal*)value)->val(); - } else { - null_map[i] = 1; - res[i] = 0; - } - } else if constexpr (type_index == TypeIndex::Int128) { - if (value->isInt8() || value->isInt16() || value->isInt32() || - value->isInt64() || value->isInt128()) { - res[i] = (int128_t)((const JsonbIntVal*)value)->val(); + } else if constexpr (type_index == TypeIndex::Int8 || + type_index == TypeIndex::Int16 || + type_index == TypeIndex::Int32 || + type_index == TypeIndex::Int64 || + type_index == TypeIndex::Int128) { + if (value->isInt()) { + res[i] = ((const JsonbIntVal*)value)->val(); } else { null_map[i] = 1; res[i] = 0; @@ -885,8 +874,7 @@ struct ConvertImplFromJsonb { } else if constexpr (type_index == TypeIndex::Float64) { if (value->isDouble()) { res[i] = ((const JsonbDoubleVal*)value)->val(); - } else if (value->isInt8() || value->isInt16() || value->isInt32() || - value->isInt64()) { + } else if (value->isInt()) { res[i] = ((const JsonbIntVal*)value)->val(); } else { null_map[i] = 1; @@ -2082,7 +2070,6 @@ class FunctionCast final : public IFunctionBase { const auto& col_with_type_and_name = block.get_by_position(arguments[0]); auto& from_type = col_with_type_and_name.type; auto& col_from = col_with_type_and_name.column; - // set variant root column/type to from column/type auto variant = ColumnObject::create(true /*always nullable*/); variant->create_root(from_type, col_from->assume_mutable()); diff --git a/be/test/vec/function/function_jsonb_test.cpp b/be/test/vec/function/function_jsonb_test.cpp index d1c7ef121a47729..91206028ef16006 100644 --- a/be/test/vec/function/function_jsonb_test.cpp +++ b/be/test/vec/function/function_jsonb_test.cpp @@ -1271,11 +1271,11 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) { {{STRING("null"), static_cast(TypeIndex::Int8)}, Null()}, {{STRING("true"), static_cast(TypeIndex::Int8)}, Null()}, {{STRING("false"), static_cast(TypeIndex::Int8)}, Null()}, - {{STRING("100"), static_cast(TypeIndex::Int8)}, TINYINT(100)}, //int8 - {{STRING("10000"), static_cast(TypeIndex::Int8)}, Null()}, // int16 - {{STRING("1000000000"), static_cast(TypeIndex::Int8)}, Null()}, // int32 + {{STRING("100"), static_cast(TypeIndex::Int8)}, TINYINT(100)}, //int8 + {{STRING("10000"), static_cast(TypeIndex::Int8)}, TINYINT(16)}, // int16 + {{STRING("1000000000"), static_cast(TypeIndex::Int8)}, TINYINT(0)}, // int32 {{STRING("1152921504606846976"), static_cast(TypeIndex::Int8)}, - Null()}, // int64 + TINYINT(0)}, // int64 {{STRING("6.18"), static_cast(TypeIndex::Int8)}, Null()}, // double {{STRING(R"("abcd")"), static_cast(TypeIndex::Int8)}, Null()}, // string {{STRING("{}"), static_cast(TypeIndex::Int8)}, Null()}, // empty object @@ -1306,9 +1306,10 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) { {{STRING("false"), static_cast(TypeIndex::Int16)}, Null()}, {{STRING("100"), static_cast(TypeIndex::Int16)}, SMALLINT(100)}, //int8 {{STRING("10000"), static_cast(TypeIndex::Int16)}, SMALLINT(10000)}, // int16 - {{STRING("1000000000"), static_cast(TypeIndex::Int16)}, Null()}, // int32 + {{STRING("1000000000"), static_cast(TypeIndex::Int16)}, + SMALLINT(-13824)}, // int32 {{STRING("1152921504606846976"), static_cast(TypeIndex::Int16)}, - Null()}, // int64 + SMALLINT(0)}, // int64 {{STRING("6.18"), static_cast(TypeIndex::Int16)}, Null()}, // double {{STRING(R"("abcd")"), static_cast(TypeIndex::Int16)}, Null()}, // string {{STRING("{}"), static_cast(TypeIndex::Int16)}, Null()}, // empty object @@ -1342,7 +1343,7 @@ TEST(FunctionJsonbTEST, JsonbCastToOtherTest) { {{STRING("1000000000"), static_cast(TypeIndex::Int32)}, INT(1000000000)}, // int32 {{STRING("1152921504606846976"), static_cast(TypeIndex::Int32)}, - Null()}, // int64 + INT(0)}, // int64 {{STRING("6.18"), static_cast(TypeIndex::Int32)}, Null()}, // double {{STRING(R"("abcd")"), static_cast(TypeIndex::Int32)}, Null()}, // string {{STRING("{}"), static_cast(TypeIndex::Int32)}, Null()}, // empty object diff --git a/regression-test/data/datatype_p0/json/json_cast.out b/regression-test/data/datatype_p0/json/json_cast.out new file mode 100644 index 000000000000000..73809a890243a86 --- /dev/null +++ b/regression-test/data/datatype_p0/json/json_cast.out @@ -0,0 +1,43 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql1 -- +10 + +-- !sql2 -- +23 + +-- !sql3 -- +-28649 + +-- !sql4 -- +-679213870 + +-- !sql5 -- +1234 + +-- !sql6 -- +-46 + +-- !sql7 -- +true + +-- !sql8 -- +\N + +-- !sql9 -- +1000.1111 + +-- !sql10 -- +\N + +-- !sql11 -- +["CXO0N: 1045901740","HMkTa: 1348450505","44 HHD: 915015173","j9WoJ: -1517316688"] + +-- !sql12 -- +111111 + +-- !sql13 -- +111111 + +-- !sql14 -- +1.1111 + diff --git a/regression-test/data/json_p0/test_json_load_and_function.out b/regression-test/data/json_p0/test_json_load_and_function.out index 40e195544ea1712..d8c03d5c3efbefd 100644 --- a/regression-test/data/json_p0/test_json_load_and_function.out +++ b/regression-test/data/json_p0/test_json_load_and_function.out @@ -5151,8 +5151,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5166,9 +5166,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5178,7 +5178,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5192,9 +5192,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5218,9 +5218,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5244,9 +5244,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -5307,8 +5307,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5322,9 +5322,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5334,7 +5334,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5348,9 +5348,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5374,9 +5374,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5400,9 +5400,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/json_p0/test_json_load_unique_key_and_function.out b/regression-test/data/json_p0/test_json_load_unique_key_and_function.out index f63959c2e02aa38..3cb34304af49f3b 100644 --- a/regression-test/data/json_p0/test_json_load_unique_key_and_function.out +++ b/regression-test/data/json_p0/test_json_load_unique_key_and_function.out @@ -3959,8 +3959,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -3983,7 +3983,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -4097,8 +4097,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -4121,7 +4121,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out index 8a3b573cebdba2b..9e7eddb936eac31 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_and_function.out @@ -6942,8 +6942,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6957,9 +6957,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -6969,7 +6969,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6983,9 +6983,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7009,9 +7009,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7035,9 +7035,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -7098,8 +7098,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7113,9 +7113,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7125,7 +7125,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7139,9 +7139,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7165,9 +7165,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7191,9 +7191,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out b/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out index eca0a75867aba9d..a9bec25de0904f8 100644 --- a/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out +++ b/regression-test/data/jsonb_p0/test_jsonb_load_unique_key_and_function.out @@ -5546,8 +5546,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5570,7 +5570,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5684,8 +5684,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5708,7 +5708,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N diff --git a/regression-test/data/nereids_function_p0/scalar_function/J.out b/regression-test/data/nereids_function_p0/scalar_function/J.out index 5785ef0c4837fc7..25f57c51c168a38 100644 --- a/regression-test/data/nereids_function_p0/scalar_function/J.out +++ b/regression-test/data/nereids_function_p0/scalar_function/J.out @@ -6942,8 +6942,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6957,9 +6957,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -6969,7 +6969,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6983,9 +6983,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7009,9 +7009,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7035,9 +7035,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -7098,8 +7098,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7113,9 +7113,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7125,7 +7125,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7139,9 +7139,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7165,9 +7165,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7191,9 +7191,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out b/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out index e12fb79e3808988..b474baa8102dd7d 100644 --- a/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out +++ b/regression-test/data/nereids_p0/json_p0/test_json_load_and_function.out @@ -5148,8 +5148,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5163,9 +5163,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5175,7 +5175,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5189,9 +5189,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5215,9 +5215,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5241,9 +5241,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -5304,8 +5304,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5319,9 +5319,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5331,7 +5331,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5345,9 +5345,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5371,9 +5371,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -5397,9 +5397,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/nereids_p0/json_p0/test_json_load_unique_key_and_function.out b/regression-test/data/nereids_p0/json_p0/test_json_load_unique_key_and_function.out index f63959c2e02aa38..3cb34304af49f3b 100644 --- a/regression-test/data/nereids_p0/json_p0/test_json_load_unique_key_and_function.out +++ b/regression-test/data/nereids_p0/json_p0/test_json_load_unique_key_and_function.out @@ -3959,8 +3959,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -3983,7 +3983,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -4097,8 +4097,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -4121,7 +4121,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N diff --git a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out index 8e77ddb11c18e49..442e68cadb2a088 100644 --- a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out +++ b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_and_function.out @@ -6942,8 +6942,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6957,9 +6957,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -6969,7 +6969,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -6983,9 +6983,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7009,9 +7009,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7035,9 +7035,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.2524337771678448E19 30 -9223372036854775808 -9.223372036854776E18 -31 18446744073709551615 \N +31 18446744073709551615 1.8446744073709552E19 -- !select -- 1 \N \N @@ -7098,8 +7098,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7113,9 +7113,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -17778 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7125,7 +7125,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -7139,9 +7139,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N -30 -9223372036854775808 \N -31 18446744073709551615 \N +29 12524337771678448270 -1209615730 +30 -9223372036854775808 0 +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7165,9 +7165,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 -5922406302031103346 30 -9223372036854775808 -9223372036854775808 -31 18446744073709551615 \N +31 18446744073709551615 -1 -- !select -- 1 \N \N @@ -7191,9 +7191,9 @@ 26 \N \N 27 {"k1":"v1","k2":200} \N 28 {"a.b.c":{"k1.a1":"v31","k2":300},"a":"niu"} \N -29 12524337771678448270 \N +29 12524337771678448270 1.25243377716784e+19 30 -9223372036854775808 -9.22337203685478e+18 -31 18446744073709551615 \N +31 18446744073709551615 1.84467440737096e+19 -- !select -- 1 \N \N diff --git a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.out b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.out index 632badf32e1de87..f69d695d455bb2a 100644 --- a/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.out +++ b/regression-test/data/nereids_p0/jsonb_p0/test_jsonb_load_unique_key_and_function.out @@ -5546,8 +5546,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5570,7 +5570,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5684,8 +5684,8 @@ 4 false \N 5 100 100 6 10000 10000 -7 1000000000 \N -8 1152921504606846976 \N +7 1000000000 -13824 +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N @@ -5708,7 +5708,7 @@ 5 100 100 6 10000 10000 7 1000000000 1000000000 -8 1152921504606846976 \N +8 1152921504606846976 0 9 6.18 \N 10 "abcd" \N 11 {} \N diff --git a/regression-test/data/variant_p0/load.out b/regression-test/data/variant_p0/load.out index cc1a74c7041acbe..d8f145d5fc4a2ee 100644 --- a/regression-test/data/variant_p0/load.out +++ b/regression-test/data/variant_p0/load.out @@ -212,11 +212,14 @@ [123] -- !sql_25 -- -50000 55000.00000000545 6150000 +50000 54999.9999999998 6150000 -- !sql_26 -- 5000 +-- !sql_27 -- +16 + -- !sql_29_1 -- 1 {"kxxxx":123} {"xxxxyyyy":123} 1 {"kyyyy":"123"} {"kxkxkxkx":[123]} @@ -322,3 +325,15 @@ -- !sql_31 -- kaana +-- !sql_39 -- +1 array +2 string +3 bigint +4 double + +-- !sql_39 -- +["CXO0N: 1045901740", "HMkTa: 1348450505", "44 HHD: 915015173", "j9WoJ: -1517316688"] + +-- !sql_39 -- +[1] + diff --git a/regression-test/suites/datatype_p0/json/json_cast.groovy b/regression-test/suites/datatype_p0/json/json_cast.groovy new file mode 100644 index 000000000000000..7646b4f7ccbf273 --- /dev/null +++ b/regression-test/suites/datatype_p0/json/json_cast.groovy @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_json_type_cast", "p0") { + qt_sql1 "SELECT CAST(CAST(10 AS JSON) as INT)" + qt_sql2 "SELECT CAST(CAST(102423 AS JSON) as TINYINT)" + qt_sql3 "SELECT CAST(CAST(102423 AS JSON) as SMALLINT)" + qt_sql4 "SELECT CAST(CAST(102400001234 AS JSON) as INT)" + qt_sql5 "SELECT CAST(CAST(102400001234 AS JSON) as SMALLINT)" + qt_sql6 "SELECT CAST(CAST(102400001234 AS JSON) as TINYINT)" + qt_sql7 "SELECT CAST(CAST(102400001234 AS JSON) as BOOLEAN)" + qt_sql8 "SELECT CAST(CAST(1000.1111 AS JSON) as INT)" + qt_sql9 "SELECT CAST(CAST(1000.1111 AS JSON) as DOUBLE)" + qt_sql10 "SELECT CAST(CAST(1000.1111 AS JSON) as BOOLEAN)" + + qt_sql11 """select cast('["CXO0N: 1045901740", "HMkTa: 1348450505", "44 HHD: 915015173", "j9WoJ: -1517316688"]' as json);""" + qt_sql12 """select cast("111111" as json)""" + qt_sql13 """select cast(111111 as json)""" + qt_sql14 """select cast(1.1111 as json)""" +} \ No newline at end of file diff --git a/regression-test/suites/variant_p0/load.groovy b/regression-test/suites/variant_p0/load.groovy index b85dd686f842463..77a975687ee351a 100644 --- a/regression-test/suites/variant_p0/load.groovy +++ b/regression-test/suites/variant_p0/load.groovy @@ -206,13 +206,14 @@ suite("regression_test_variant", "nonConcurrent"){ load_json_data.call(table_name, """${getS3Url() + '/load/ghdata_sample.json'}""") qt_sql_26 "select count() from ${table_name}" - // FIXME: this case it not passed - // // 8. json empty string - // // table_name = "empty_string" - // // create_table table_name - // // sql """INSERT INTO empty_string VALUES (1, ''), (2, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');""" - // // sql """INSERT INTO empty_string VALUES (3, null), (4, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');""" - // // qt_sql_27 "SELECT * FROM ${table_name} ORDER BY k;" + // 8. json empty string + table_name = "empty_string" + create_table table_name + sql """INSERT INTO empty_string VALUES (1, ''), (2, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');""" + sql """INSERT INTO empty_string VALUES (3, null), (4, '{"k1": 1, "k2": "v1"}'), (3, '{}'), (4, '{"k1": 2}');""" + sql """INSERT INTO empty_string VALUES (3, null), (4, null), (3, '{}'), (4, '{"k1": 2}');""" + sql """INSERT INTO empty_string VALUES (3, ''), (4, null), (3, '{}'), (4, null);""" + qt_sql_27 "SELECT count() FROM ${table_name};" // // // 9. btc data // // table_name = "btcdata" @@ -393,6 +394,16 @@ suite("regression_test_variant", "nonConcurrent"){ qt_sql_31 """select cast(v['xxxx'] as string) from sparse_columns where cast(v['xxxx'] as string) != 'null' order by k limit 1;""" sql "truncate table sparse_columns" set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") + + // test cast + table_name = "variant_cast" + create_table.call(table_name, "DUPLICATE", "1") + sql """ + insert into variant_cast values(1,'["CXO0N: 1045901740", "HMkTa: 1348450505", "44 HHD: 915015173", "j9WoJ: -1517316688"]'),(2,'"[1]"'),(3,'123456'),(4,'1.11111') + """ + qt_sql_39 "select k, json_type(cast(v as json), '\$') from variant_cast order by k" + qt_sql_39 "select cast(v as array) from variant_cast where k = 1 order by k" + qt_sql_39 "select cast(v as string) from variant_cast where k = 2 order by k" } finally { // reset flags set_be_config.call("variant_ratio_of_defaults_as_sparse_column", "0.95") From 435147d449c5b1bbab59c882f0a8ae68adc3c845 Mon Sep 17 00:00:00 2001 From: zhangdong <493738387@qq.com> Date: Fri, 17 May 2024 17:31:22 +0800 Subject: [PATCH 046/111] [enhance](mtmv) MTMV deal partition use name instead of id (#34910) partition id will change when insert overwrite When the materialized view runs a task, if the base table is in insert overwrite, the materialized view task may report an error: partition not found by partitionId Upgrade compatibility: Hive currently does not support automatic refresh, so it has no impact --- .../java/org/apache/doris/catalog/MTMV.java | 22 ++-- .../org/apache/doris/catalog/OlapTable.java | 27 ++++- .../datasource/hive/HMSExternalTable.java | 36 ++++-- .../doris/job/extensions/mtmv/MTMVTask.java | 41 ++++--- .../doris/mtmv/MTMVMaxTimestampSnapshot.java | 19 ++- .../apache/doris/mtmv/MTMVPartitionUtil.java | 112 +++++++++--------- ...PartitionDescOnePartitionColGenerator.java | 6 +- ...MVRelatedPartitionDescRollUpGenerator.java | 20 ++-- ...elatedPartitionDescSyncLimitGenerator.java | 6 +- .../apache/doris/mtmv/MTMVRelatedTableIf.java | 6 +- .../apache/doris/mtmv/MTMVRewriteUtil.java | 6 +- .../mtmv/RelatedPartitionDescResult.java | 12 +- .../commands/UpdateMvByPartitionCommand.java | 26 ++-- .../doris/mtmv/MTMVPartitionUtilTest.java | 18 +-- ...latedPartitionDescRollUpGeneratorTest.java | 28 ++--- .../doris/mtmv/MTMVRewriteUtilTest.java | 12 +- .../org/apache/doris/mtmv/MTMVTaskTest.java | 46 ++++--- 17 files changed, 236 insertions(+), 207 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index fb32741afa384a0..d95c2388a288d4e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -305,12 +305,12 @@ public MTMVRefreshSnapshot getRefreshSnapshot() { /** * generateMvPartitionDescs * - * @return mvPartitionId ==> mvPartitionKeyDesc + * @return mvPartitionName ==> mvPartitionKeyDesc */ - public Map generateMvPartitionDescs() { - Map mtmvItems = getAndCopyPartitionItems(); - Map result = Maps.newHashMap(); - for (Entry entry : mtmvItems.entrySet()) { + public Map generateMvPartitionDescs() { + Map mtmvItems = getAndCopyPartitionItems(); + Map result = Maps.newHashMap(); + for (Entry entry : mtmvItems.entrySet()) { result.put(entry.getKey(), entry.getValue().toPartitionKeyDesc()); } return result; @@ -321,19 +321,19 @@ public Map generateMvPartitionDescs() { * It is the result of real-time comparison calculation, so there may be some costs, * so it should be called with caution * - * @return mvPartitionId ==> relationPartitionIds + * @return mvPartitionName ==> relationPartitionNames * @throws AnalysisException */ - public Map> calculatePartitionMappings() throws AnalysisException { + public Map> calculatePartitionMappings() throws AnalysisException { if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { return Maps.newHashMap(); } long start = System.currentTimeMillis(); - Map> res = Maps.newHashMap(); - Map> relatedPartitionDescs = MTMVPartitionUtil + Map> res = Maps.newHashMap(); + Map> relatedPartitionDescs = MTMVPartitionUtil .generateRelatedPartitionDescs(mvPartitionInfo, mvProperties); - Map mvPartitionItems = getAndCopyPartitionItems(); - for (Entry entry : mvPartitionItems.entrySet()) { + Map mvPartitionItems = getAndCopyPartitionItems(); + for (Entry entry : mvPartitionItems.entrySet()) { res.put(entry.getKey(), relatedPartitionDescs.getOrDefault(entry.getValue().toPartitionKeyDesc(), Sets.newHashSet())); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 9cf9be00915bb68..0d3477b3a260cd4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -95,6 +95,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Objects; import java.util.Optional; import java.util.Set; @@ -1037,6 +1038,17 @@ public Partition getPartition(long partitionId) { return partition; } + public PartitionItem getPartitionItemOrAnalysisException(String partitionName) throws AnalysisException { + Partition partition = nameToPartition.get(partitionName); + if (partition == null) { + partition = tempPartitions.getPartition(partitionName); + } + if (partition == null) { + throw new AnalysisException("partition not found: " + partitionName); + } + return partitionInfo.getItem(partition.getId()); + } + public Partition getPartitionOrAnalysisException(long partitionId) throws AnalysisException { Partition partition = idToPartition.get(partitionId); if (partition == null) { @@ -2657,10 +2669,17 @@ public PartitionType getPartitionType() { } @Override - public Map getAndCopyPartitionItems() { + public Map getAndCopyPartitionItems() { readLock(); try { - return Maps.newHashMap(getPartitionInfo().getIdToItem(false)); + Map res = Maps.newHashMap(); + for (Entry entry : getPartitionInfo().getIdToItem(false).entrySet()) { + Partition partition = idToPartition.get(entry.getKey()); + if (partition != null) { + res.put(partition.getName(), entry.getValue()); + } + } + return res; } finally { readUnlock(); } @@ -2672,8 +2691,8 @@ public List getPartitionColumns() { } @Override - public MTMVSnapshotIf getPartitionSnapshot(long partitionId) throws AnalysisException { - long visibleVersion = getPartitionOrAnalysisException(partitionId).getVisibleVersion(); + public MTMVSnapshotIf getPartitionSnapshot(String partitionName) throws AnalysisException { + long visibleVersion = getPartitionOrAnalysisException(partitionName).getVisibleVersion(); return new MTMVVersionSnapshot(visibleVersion); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index 57dee5debb0b26d..e5624fb58b5c326 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -878,12 +878,22 @@ public Set getPartitionColumnNames() { } @Override - public Map getAndCopyPartitionItems() { + public Map getAndCopyPartitionItems() { HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .getMetaStoreCache((HMSExternalCatalog) getCatalog()); HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( getDbName(), getName(), getPartitionColumnTypes()); - return hivePartitionValues.getIdToPartitionItem(); + Map res = Maps.newHashMap(); + Map idToPartitionItem = hivePartitionValues.getIdToPartitionItem(); + for (Entry entry : idToPartitionItem.entrySet()) { + try { + res.put(getPartitionName(entry.getKey()), entry.getValue()); + } catch (AnalysisException e) { + LOG.info("can not get partitionName by: " + entry.getKey()); + } + + } + return res; } @Override @@ -905,35 +915,35 @@ private HiveMetaStoreCache.HivePartitionValues getHivePartitionValues() { } @Override - public MTMVSnapshotIf getPartitionSnapshot(long partitionId) throws AnalysisException { - long partitionLastModifyTime = getPartitionLastModifyTime(partitionId); + public MTMVSnapshotIf getPartitionSnapshot(String partitionName) throws AnalysisException { + long partitionLastModifyTime = getPartitionLastModifyTime(partitionName); return new MTMVTimestampSnapshot(partitionLastModifyTime); } @Override public MTMVSnapshotIf getTableSnapshot() throws AnalysisException { if (getPartitionType() == PartitionType.UNPARTITIONED) { - return new MTMVMaxTimestampSnapshot(-1L, getLastDdlTime()); + return new MTMVMaxTimestampSnapshot(getName(), getLastDdlTime()); } - long partitionId = 0L; + String partitionName = ""; long maxVersionTime = 0L; long visibleVersionTime; - for (Entry entry : getAndCopyPartitionItems().entrySet()) { + for (Entry entry : getAndCopyPartitionItems().entrySet()) { visibleVersionTime = getPartitionLastModifyTime(entry.getKey()); if (visibleVersionTime > maxVersionTime) { maxVersionTime = visibleVersionTime; - partitionId = entry.getKey(); + partitionName = entry.getKey(); } } - return new MTMVMaxTimestampSnapshot(partitionId, maxVersionTime); + return new MTMVMaxTimestampSnapshot(partitionName, maxVersionTime); } - private long getPartitionLastModifyTime(long partitionId) throws AnalysisException { - return getPartitionById(partitionId).getLastModifiedTime(); + private long getPartitionLastModifyTime(String partitionName) throws AnalysisException { + return getPartitionByName(partitionName).getLastModifiedTime(); } - private HivePartition getPartitionById(long partitionId) throws AnalysisException { - PartitionItem item = getAndCopyPartitionItems().get(partitionId); + private HivePartition getPartitionByName(String partitionName) throws AnalysisException { + PartitionItem item = getAndCopyPartitionItems().get(partitionName); List> partitionValuesList = transferPartitionItemToPartitionValues(item); List partitions = getPartitionsByPartitionValues(partitionValuesList); if (partitions.size() != 1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index 7a4c5277591525c..71051be3958ae53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -173,30 +173,28 @@ public void run() throws JobException { if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { MTMVPartitionUtil.alignMvPartition(mtmv); } - Map> partitionMappings = mtmv.calculatePartitionMappings(); - List needRefreshPartitionIds = calculateNeedRefreshPartitions(partitionMappings); - this.needRefreshPartitions = MTMVPartitionUtil.getPartitionNamesByIds(mtmv, needRefreshPartitionIds); - this.refreshMode = generateRefreshMode(needRefreshPartitionIds); + Map> partitionMappings = mtmv.calculatePartitionMappings(); + this.needRefreshPartitions = calculateNeedRefreshPartitions(partitionMappings); + this.refreshMode = generateRefreshMode(needRefreshPartitions); if (refreshMode == MTMVTaskRefreshMode.NOT_REFRESH) { return; } Map tableWithPartKey = getIncrementalTableMap(); this.completedPartitions = Lists.newCopyOnWriteArrayList(); int refreshPartitionNum = mtmv.getRefreshPartitionNum(); - long execNum = (needRefreshPartitionIds.size() / refreshPartitionNum) + ((needRefreshPartitionIds.size() + long execNum = (needRefreshPartitions.size() / refreshPartitionNum) + ((needRefreshPartitions.size() % refreshPartitionNum) > 0 ? 1 : 0); this.partitionSnapshots = Maps.newConcurrentMap(); for (int i = 0; i < execNum; i++) { int start = i * refreshPartitionNum; int end = start + refreshPartitionNum; - Set execPartitionIds = Sets.newHashSet(needRefreshPartitionIds - .subList(start, end > needRefreshPartitionIds.size() ? needRefreshPartitionIds.size() : end)); + Set execPartitionNames = Sets.newHashSet(needRefreshPartitions + .subList(start, end > needRefreshPartitions.size() ? needRefreshPartitions.size() : end)); // need get names before exec - List execPartitionNames = MTMVPartitionUtil.getPartitionNamesByIds(mtmv, execPartitionIds); Map execPartitionSnapshots = MTMVPartitionUtil - .generatePartitionSnapshots(mtmv, relation.getBaseTables(), execPartitionIds, + .generatePartitionSnapshots(mtmv, relation.getBaseTables(), execPartitionNames, partitionMappings); - exec(ctx, execPartitionIds, tableWithPartKey); + exec(ctx, execPartitionNames, tableWithPartKey); completedPartitions.addAll(execPartitionNames); partitionSnapshots.putAll(execPartitionSnapshots); } @@ -218,15 +216,15 @@ public void run() throws JobException { } } - private void exec(ConnectContext ctx, Set refreshPartitionIds, - Map tableWithPartKey) + private void exec(ConnectContext ctx, Set refreshPartitionNames, + Map tableWithPartKey) throws Exception { TUniqueId queryId = generateQueryId(); lastQueryId = DebugUtil.printId(queryId); // if SELF_MANAGE mv, only have default partition, will not have partitionItem, so we give empty set UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand .from(mtmv, mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE - ? refreshPartitionIds : Sets.newHashSet(), tableWithPartKey); + ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey); executor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, ctx.getStatementContext())); ctx.setExecutor(executor); ctx.setQueryId(queryId); @@ -252,7 +250,7 @@ public synchronized void onSuccess() throws JobException { } @Override - protected synchronized void executeCancelLogic() { + protected synchronized void executeCancelLogic() { LOG.info("mtmv task cancel, taskId: {}", super.getTaskId()); if (executor != null) { executor.cancel(); @@ -407,29 +405,30 @@ private Map getIncrementalTableMap() throws AnalysisException { return tableWithPartKey; } - private MTMVTaskRefreshMode generateRefreshMode(List needRefreshPartitionIds) { + private MTMVTaskRefreshMode generateRefreshMode(List needRefreshPartitionIds) { if (CollectionUtils.isEmpty(needRefreshPartitionIds)) { return MTMVTaskRefreshMode.NOT_REFRESH; - } else if (needRefreshPartitionIds.size() == mtmv.getPartitionIds().size()) { + } else if (needRefreshPartitionIds.size() == mtmv.getPartitionNames().size()) { return MTMVTaskRefreshMode.COMPLETE; } else { return MTMVTaskRefreshMode.PARTIAL; } } - public List calculateNeedRefreshPartitions(Map> partitionMappings) throws AnalysisException { + public List calculateNeedRefreshPartitions(Map> partitionMappings) + throws AnalysisException { // check whether the user manually triggers it if (taskContext.getTriggerMode() == MTMVTaskTriggerMode.MANUAL) { if (taskContext.isComplete()) { - return mtmv.getPartitionIds(); + return Lists.newArrayList(mtmv.getPartitionNames()); } else if (!CollectionUtils .isEmpty(taskContext.getPartitions())) { - return MTMVPartitionUtil.getPartitionsIdsByNames(mtmv, taskContext.getPartitions()); + return taskContext.getPartitions(); } } // if refreshMethod is COMPLETE, we must FULL refresh, avoid external table MTMV always not refresh if (mtmv.getRefreshInfo().getRefreshMethod() == RefreshMethod.COMPLETE) { - return mtmv.getPartitionIds(); + return Lists.newArrayList(mtmv.getPartitionNames()); } // check if data is fresh // We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation() @@ -441,7 +440,7 @@ public List calculateNeedRefreshPartitions(Map> partitionM } // current, if partitionType is SELF_MANAGE, we can only FULL refresh if (mtmv.getMvPartitionInfo().getPartitionType() == MTMVPartitionType.SELF_MANAGE) { - return mtmv.getPartitionIds(); + return Lists.newArrayList(mtmv.getPartitionNames()); } // We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation() // to avoid rebuilding the baseTable and causing a change in the tableId diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVMaxTimestampSnapshot.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVMaxTimestampSnapshot.java index 53f9df542cfbd87..0a127857b485923 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVMaxTimestampSnapshot.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVMaxTimestampSnapshot.java @@ -25,17 +25,17 @@ * so the update time is used instead of the version */ public class MTMVMaxTimestampSnapshot implements MTMVSnapshotIf { - // partitionId corresponding to timestamp - // The reason why both timestamp and partitionId are stored is to avoid + // partitionName corresponding to timestamp + // The reason why both timestamp and partitionName are stored is to avoid // deleting the partition corresponding to timestamp - @SerializedName("p") - private long partitionId; + @SerializedName("pn") + private String partitionName; // The maximum modify time in all partitions @SerializedName("t") private long timestamp; - public MTMVMaxTimestampSnapshot(long partitionId, long timestamp) { - this.partitionId = partitionId; + public MTMVMaxTimestampSnapshot(String partitionName, long timestamp) { + this.partitionName = partitionName; this.timestamp = timestamp; } @@ -48,19 +48,18 @@ public boolean equals(Object o) { return false; } MTMVMaxTimestampSnapshot that = (MTMVMaxTimestampSnapshot) o; - return partitionId == that.partitionId - && timestamp == that.timestamp; + return timestamp == that.timestamp && Objects.equal(partitionName, that.partitionName); } @Override public int hashCode() { - return Objects.hashCode(partitionId, timestamp); + return Objects.hashCode(partitionName, timestamp); } @Override public String toString() { return "MTMVMaxTimestampSnapshot{" - + "partitionId=" + partitionId + + "partitionName='" + partitionName + '\'' + ", timestamp=" + timestamp + '}'; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index cd0312c419e814c..f2b7b146ee5eb21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -73,14 +73,14 @@ public class MTMVPartitionUtil { * Determine whether the partition is sync with retated partition and other baseTables * * @param mtmv - * @param partitionId - * @param relatedPartitionIds + * @param partitionName + * @param relatedPartitionNames * @param tables * @param excludedTriggerTables * @return * @throws AnalysisException */ - public static boolean isMTMVPartitionSync(MTMV mtmv, Long partitionId, Set relatedPartitionIds, + public static boolean isMTMVPartitionSync(MTMV mtmv, String partitionName, Set relatedPartitionNames, Set tables, Set excludedTriggerTables) throws AnalysisException { boolean isSyncWithPartition = true; @@ -88,14 +88,14 @@ public static boolean isMTMVPartitionSync(MTMV mtmv, Long partitionId, Set MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable(); // if follow base table, not need compare with related table, only should compare with related partition excludedTriggerTables.add(relatedTable.getName()); - if (CollectionUtils.isEmpty(relatedPartitionIds)) { + if (CollectionUtils.isEmpty(relatedPartitionNames)) { LOG.warn("can not found related partition, partitionId: {}, mtmvName: {}, relatedTableName: {}", - partitionId, mtmv.getName(), relatedTable.getName()); + partitionName, mtmv.getName(), relatedTable.getName()); return false; } - isSyncWithPartition = isSyncWithPartitions(mtmv, partitionId, relatedTable, relatedPartitionIds); + isSyncWithPartition = isSyncWithPartitions(mtmv, partitionName, relatedTable, relatedPartitionNames); } - return isSyncWithPartition && isSyncWithAllBaseTables(mtmv, partitionId, tables, excludedTriggerTables); + return isSyncWithPartition && isSyncWithAllBaseTables(mtmv, partitionName, tables, excludedTriggerTables); } @@ -108,11 +108,11 @@ public static boolean isMTMVPartitionSync(MTMV mtmv, Long partitionId, Set */ public static void alignMvPartition(MTMV mtmv) throws DdlException, AnalysisException { - Map mtmvPartitionDescs = mtmv.generateMvPartitionDescs(); + Map mtmvPartitionDescs = mtmv.generateMvPartitionDescs(); Set relatedPartitionDescs = generateRelatedPartitionDescs(mtmv.getMvPartitionInfo(), mtmv.getMvProperties()).keySet(); // drop partition of mtmv - for (Entry entry : mtmvPartitionDescs.entrySet()) { + for (Entry entry : mtmvPartitionDescs.entrySet()) { if (!relatedPartitionDescs.contains(entry.getValue())) { dropPartition(mtmv, entry.getKey()); } @@ -152,7 +152,7 @@ public static List getPartitionDescsByRelatedTable( return res; } - public static Map> generateRelatedPartitionDescs(MTMVPartitionInfo mvPartitionInfo, + public static Map> generateRelatedPartitionDescs(MTMVPartitionInfo mvPartitionInfo, Map mvProperties) throws AnalysisException { long start = System.currentTimeMillis(); RelatedPartitionDescResult result = new RelatedPartitionDescResult(); @@ -219,11 +219,11 @@ public static boolean isMTMVSync(MTMV mtmv) { * @throws AnalysisException */ public static boolean isMTMVSync(MTMV mtmv, Set tables, Set excludeTables, - Map> partitionMappings) + Map> partitionMappings) throws AnalysisException { - List partitionIds = mtmv.getPartitionIds(); - for (Long partitionId : partitionIds) { - if (!isMTMVPartitionSync(mtmv, partitionId, partitionMappings.get(partitionId), tables, + Set partitionNames = mtmv.getPartitionNames(); + for (String partitionName : partitionNames) { + if (!isMTMVPartitionSync(mtmv, partitionName, partitionMappings.get(partitionName), tables, excludeTables)) { return false; } @@ -236,20 +236,22 @@ public static boolean isMTMVSync(MTMV mtmv, Set tables, Set UnSyncTableNames + * @return partitionName ==> UnSyncTableNames * @throws AnalysisException */ public static Map> getPartitionsUnSyncTables(MTMV mtmv, List partitionIds) throws AnalysisException { Map> res = Maps.newHashMap(); - Map> partitionMappings = mtmv.calculatePartitionMappings(); + Map> partitionMappings = mtmv.calculatePartitionMappings(); for (Long partitionId : partitionIds) { - res.put(partitionId, getPartitionUnSyncTables(mtmv, partitionId, partitionMappings.get(partitionId))); + String partitionName = mtmv.getPartitionOrAnalysisException(partitionId).getName(); + res.put(partitionId, getPartitionUnSyncTables(mtmv, partitionName, partitionMappings.get(partitionName))); } return res; } - private static List getPartitionUnSyncTables(MTMV mtmv, Long partitionId, Set relatedPartitionIds) + private static List getPartitionUnSyncTables(MTMV mtmv, String partitionName, + Set relatedPartitionNames) throws AnalysisException { List res = Lists.newArrayList(); for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTables()) { @@ -263,16 +265,16 @@ private static List getPartitionUnSyncTables(MTMV mtmv, Long partitionId } if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE && mtmv .getMvPartitionInfo().getRelatedTableInfo().equals(baseTableInfo)) { - if (CollectionUtils.isEmpty(relatedPartitionIds)) { + if (CollectionUtils.isEmpty(relatedPartitionNames)) { throw new AnalysisException("can not found related partition"); } - boolean isSyncWithPartition = isSyncWithPartitions(mtmv, partitionId, mtmvRelatedTableIf, - relatedPartitionIds); + boolean isSyncWithPartition = isSyncWithPartitions(mtmv, partitionName, mtmvRelatedTableIf, + relatedPartitionNames); if (!isSyncWithPartition) { res.add(mtmvRelatedTableIf.getName()); } } else { - if (!isSyncWithBaseTable(mtmv, partitionId, baseTableInfo)) { + if (!isSyncWithBaseTable(mtmv, partitionName, baseTableInfo)) { res.add(table.getName()); } } @@ -287,18 +289,18 @@ private static List getPartitionUnSyncTables(MTMV mtmv, Long partitionId * @param baseTables * @return */ - public static List getMTMVNeedRefreshPartitions(MTMV mtmv, Set baseTables, - Map> partitionMappings) { - List partitionIds = mtmv.getPartitionIds(); - List res = Lists.newArrayList(); - for (Long partitionId : partitionIds) { + public static List getMTMVNeedRefreshPartitions(MTMV mtmv, Set baseTables, + Map> partitionMappings) { + Set partitionNames = mtmv.getPartitionNames(); + List res = Lists.newArrayList(); + for (String partitionName : partitionNames) { try { - if (!isMTMVPartitionSync(mtmv, partitionId, partitionMappings.get(partitionId), baseTables, + if (!isMTMVPartitionSync(mtmv, partitionName, partitionMappings.get(partitionName), baseTables, mtmv.getExcludedTriggerTables())) { - res.add(partitionId); + res.add(partitionName); } } catch (AnalysisException e) { - res.add(partitionId); + res.add(partitionName); LOG.warn("check isMTMVPartitionSync failed", e); } } @@ -309,23 +311,21 @@ public static List getMTMVNeedRefreshPartitions(MTMV mtmv, Set relatedPartitionIds) throws AnalysisException { + Set relatedPartitionNames) throws AnalysisException { if (!relatedTable.needAutoRefresh()) { return true; } - String mtmvPartitionName = mtmv.getPartitionName(mtmvPartitionId); - for (Long relatedPartitionId : relatedPartitionIds) { + for (String relatedPartitionName : relatedPartitionNames) { MTMVSnapshotIf relatedPartitionCurrentSnapshot = relatedTable - .getPartitionSnapshot(relatedPartitionId); - String relatedPartitionName = relatedTable.getPartitionName(relatedPartitionId); + .getPartitionSnapshot(relatedPartitionName); if (!mtmv.getRefreshSnapshot() .equalsWithRelatedPartition(mtmvPartitionName, relatedPartitionName, relatedPartitionCurrentSnapshot)) { @@ -355,15 +355,14 @@ public static String generatePartitionName(PartitionKeyDesc desc) { * drop partition of mtmv * * @param mtmv - * @param partitionId + * @param partitionName */ - private static void dropPartition(MTMV mtmv, Long partitionId) throws AnalysisException, DdlException { + private static void dropPartition(MTMV mtmv, String partitionName) throws DdlException { if (!mtmv.writeLockIfExist()) { return; } try { - Partition partition = mtmv.getPartitionOrAnalysisException(partitionId); - DropPartitionClause dropPartitionClause = new DropPartitionClause(false, partition.getName(), false, false); + DropPartitionClause dropPartitionClause = new DropPartitionClause(false, partitionName, false, false); Env.getCurrentEnv().dropPartition((Database) mtmv.getDatabase(), mtmv, dropPartitionClause); } finally { mtmv.writeUnlock(); @@ -394,12 +393,12 @@ private static void addPartition(MTMV mtmv, PartitionKeyDesc oldPartitionKeyDesc /** * Determine is sync, ignoring excludedTriggerTables and non OlapTanle * - * @param mtmvPartitionId + * @param mtmvPartitionName * @param tables * @param excludedTriggerTables * @return */ - private static boolean isSyncWithAllBaseTables(MTMV mtmv, long mtmvPartitionId, Set tables, + private static boolean isSyncWithAllBaseTables(MTMV mtmv, String mtmvPartitionName, Set tables, Set excludedTriggerTables) throws AnalysisException { for (BaseTableInfo baseTableInfo : tables) { TableIf table = null; @@ -412,7 +411,7 @@ private static boolean isSyncWithAllBaseTables(MTMV mtmv, long mtmvPartitionId, if (excludedTriggerTables.contains(table.getName())) { continue; } - boolean syncWithBaseTable = isSyncWithBaseTable(mtmv, mtmvPartitionId, baseTableInfo); + boolean syncWithBaseTable = isSyncWithBaseTable(mtmv, mtmvPartitionName, baseTableInfo); if (!syncWithBaseTable) { return false; } @@ -420,7 +419,7 @@ private static boolean isSyncWithAllBaseTables(MTMV mtmv, long mtmvPartitionId, return true; } - private static boolean isSyncWithBaseTable(MTMV mtmv, long mtmvPartitionId, BaseTableInfo baseTableInfo) + private static boolean isSyncWithBaseTable(MTMV mtmv, String mtmvPartitionName, BaseTableInfo baseTableInfo) throws AnalysisException { TableIf table = null; try { @@ -440,7 +439,6 @@ private static boolean isSyncWithBaseTable(MTMV mtmv, long mtmvPartitionId, Base return true; } MTMVSnapshotIf baseTableCurrentSnapshot = baseTable.getTableSnapshot(); - String mtmvPartitionName = mtmv.getPartitionName(mtmvPartitionId); return mtmv.getRefreshSnapshot() .equalsWithBaseTable(mtmvPartitionName, baseTable.getId(), baseTableCurrentSnapshot); } @@ -450,35 +448,35 @@ private static boolean isSyncWithBaseTable(MTMV mtmv, long mtmvPartitionId, Base * * @param mtmv * @param baseTables - * @param partitionIds + * @param partitionNames * @param partitionMappings * @return * @throws AnalysisException */ public static Map generatePartitionSnapshots(MTMV mtmv, - Set baseTables, Set partitionIds, - Map> partitionMappings) + Set baseTables, Set partitionNames, + Map> partitionMappings) throws AnalysisException { Map res = Maps.newHashMap(); - for (Long partitionId : partitionIds) { - res.put(mtmv.getPartitionName(partitionId), - generatePartitionSnapshot(mtmv, baseTables, partitionMappings.get(partitionId))); + for (String partitionName : partitionNames) { + res.put(partitionName, + generatePartitionSnapshot(mtmv, baseTables, partitionMappings.get(partitionName))); } return res; } private static MTMVRefreshPartitionSnapshot generatePartitionSnapshot(MTMV mtmv, - Set baseTables, Set relatedPartitionIds) + Set baseTables, Set relatedPartitionNames) throws AnalysisException { MTMVRefreshPartitionSnapshot refreshPartitionSnapshot = new MTMVRefreshPartitionSnapshot(); if (mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE) { MTMVRelatedTableIf relatedTable = mtmv.getMvPartitionInfo().getRelatedTable(); - for (Long relatedPartitionId : relatedPartitionIds) { + for (String relatedPartitionName : relatedPartitionNames) { MTMVSnapshotIf partitionSnapshot = relatedTable - .getPartitionSnapshot(relatedPartitionId); + .getPartitionSnapshot(relatedPartitionName); refreshPartitionSnapshot.getPartitions() - .put(relatedTable.getPartitionName(relatedPartitionId), partitionSnapshot); + .put(relatedPartitionName, partitionSnapshot); } } for (BaseTableInfo baseTableInfo : baseTables) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java index ab14f302e756b82..c5dad9bdb418910 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescOnePartitionColGenerator.java @@ -51,10 +51,10 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope if (mvPartitionInfo.getPartitionType() == MTMVPartitionType.SELF_MANAGE) { return; } - Map> res = Maps.newHashMap(); - Map relatedPartitionItems = lastResult.getItems(); + Map> res = Maps.newHashMap(); + Map relatedPartitionItems = lastResult.getItems(); int relatedColPos = mvPartitionInfo.getRelatedColPos(); - for (Entry entry : relatedPartitionItems.entrySet()) { + for (Entry entry : relatedPartitionItems.entrySet()) { PartitionKeyDesc partitionKeyDesc = entry.getValue().toPartitionKeyDesc(relatedColPos); if (res.containsKey(partitionKeyDesc)) { res.get(partitionKeyDesc).add(entry.getKey()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java index e9b4b1fe6a5262d..76e20ef70f5d924 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java @@ -69,27 +69,27 @@ public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvPrope * @return * @throws AnalysisException */ - public Map> rollUpList(Map> relatedPartitionDescs, + public Map> rollUpList(Map> relatedPartitionDescs, MTMVPartitionInfo mvPartitionInfo, Map mvProperties) throws AnalysisException { Map> identityToValues = Maps.newHashMap(); - Map> identityToPartitionIds = Maps.newHashMap(); + Map> identityToPartitionNames = Maps.newHashMap(); MTMVPartitionExprService exprSerice = MTMVPartitionExprFactory.getExprService(mvPartitionInfo.getExpr()); - for (Entry> entry : relatedPartitionDescs.entrySet()) { + for (Entry> entry : relatedPartitionDescs.entrySet()) { String rollUpIdentity = exprSerice.getRollUpIdentity(entry.getKey(), mvProperties); Preconditions.checkNotNull(rollUpIdentity); if (identityToValues.containsKey(rollUpIdentity)) { identityToValues.get(rollUpIdentity).addAll(getStringValues(entry.getKey())); - identityToPartitionIds.get(rollUpIdentity).addAll(entry.getValue()); + identityToPartitionNames.get(rollUpIdentity).addAll(entry.getValue()); } else { identityToValues.put(rollUpIdentity, getStringValues(entry.getKey())); - identityToPartitionIds.put(rollUpIdentity, entry.getValue()); + identityToPartitionNames.put(rollUpIdentity, entry.getValue()); } } - Map> result = Maps.newHashMap(); + Map> result = Maps.newHashMap(); for (Entry> entry : identityToValues.entrySet()) { result.put(PartitionKeyDesc.createIn(getPartitionValues(entry.getValue())), - identityToPartitionIds.get(entry.getKey())); + identityToPartitionNames.get(entry.getKey())); } return result; } @@ -125,11 +125,11 @@ private Set getStringValues(PartitionKeyDesc partitionKeyDesc) { * @return * @throws AnalysisException */ - public Map> rollUpRange(Map> relatedPartitionDescs, + public Map> rollUpRange(Map> relatedPartitionDescs, MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { - Map> result = Maps.newHashMap(); + Map> result = Maps.newHashMap(); MTMVPartitionExprService exprSerice = MTMVPartitionExprFactory.getExprService(mvPartitionInfo.getExpr()); - for (Entry> entry : relatedPartitionDescs.entrySet()) { + for (Entry> entry : relatedPartitionDescs.entrySet()) { PartitionKeyDesc rollUpDesc = exprSerice.generateRollUpPartitionKeyDesc(entry.getKey(), mvPartitionInfo); if (result.containsKey(rollUpDesc)) { result.get(rollUpDesc).addAll(entry.getValue()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java index e031071192e3087..c6fb331631f5235 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescSyncLimitGenerator.java @@ -43,16 +43,16 @@ public class MTMVRelatedPartitionDescSyncLimitGenerator implements MTMVRelatedPa @Override public void apply(MTMVPartitionInfo mvPartitionInfo, Map mvProperties, RelatedPartitionDescResult lastResult) throws AnalysisException { - Map partitionItems = lastResult.getItems(); + Map partitionItems = lastResult.getItems(); MTMVPartitionSyncConfig config = generateMTMVPartitionSyncConfigByProperties(mvProperties); if (config.getSyncLimit() <= 0) { return; } long nowTruncSubSec = getNowTruncSubSec(config.getTimeUnit(), config.getSyncLimit()); Optional dateFormat = config.getDateFormat(); - Map res = Maps.newHashMap(); + Map res = Maps.newHashMap(); int relatedColPos = mvPartitionInfo.getRelatedColPos(); - for (Entry entry : partitionItems.entrySet()) { + for (Entry entry : partitionItems.entrySet()) { if (entry.getValue().isGreaterThanSpecifiedTime(relatedColPos, dateFormat, nowTruncSubSec)) { res.put(entry.getKey(), entry.getValue()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java index ec99a04d73f9142..8aee7741cee0a08 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedTableIf.java @@ -38,7 +38,7 @@ public interface MTMVRelatedTableIf extends TableIf { * * @return partitionId->PartitionItem */ - Map getAndCopyPartitionItems(); + Map getAndCopyPartitionItems(); /** * getPartitionType LIST/RANGE/UNPARTITIONED @@ -65,11 +65,11 @@ public interface MTMVRelatedTableIf extends TableIf { /** * getPartitionSnapshot * - * @param partitionId + * @param partitionName * @return partition snapshot at current time * @throws AnalysisException */ - MTMVSnapshotIf getPartitionSnapshot(long partitionId) throws AnalysisException; + MTMVSnapshotIf getPartitionSnapshot(String partitionName) throws AnalysisException; /** * getTableSnapshot diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index f0199169859211a..03a1aefeba18708 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -66,7 +66,7 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne && mtmv.getStatus().getRefreshState() == MTMVRefreshState.SUCCESS)) { return res; } - Map> partitionMappings = null; + Map> partitionMappings = null; // check gracePeriod long gracePeriodMills = mtmv.getGracePeriod(); for (Partition partition : allPartitions) { @@ -79,8 +79,8 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne if (partitionMappings == null) { partitionMappings = mtmv.calculatePartitionMappings(); } - if (MTMVPartitionUtil.isMTMVPartitionSync(mtmv, partition.getId(), - partitionMappings.get(partition.getId()), mtmvRelation.getBaseTables(), + if (MTMVPartitionUtil.isMTMVPartitionSync(mtmv, partition.getName(), + partitionMappings.get(partition.getName()), mtmvRelation.getBaseTables(), Sets.newHashSet())) { res.add(partition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/RelatedPartitionDescResult.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/RelatedPartitionDescResult.java index 068cf1522a71d19..b349722a76d6b1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/RelatedPartitionDescResult.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/RelatedPartitionDescResult.java @@ -27,27 +27,27 @@ public class RelatedPartitionDescResult { // PartitionKeyDesc to relatedTable partition ids(Different partitions may have the same PartitionKeyDesc) - private Map> descs; - private Map items; + private Map> descs; + private Map items; public RelatedPartitionDescResult() { this.descs = Maps.newHashMap(); this.items = Maps.newHashMap(); } - public Map> getDescs() { + public Map> getDescs() { return descs; } - public void setDescs(Map> descs) { + public void setDescs(Map> descs) { this.descs = descs; } - public Map getItems() { + public Map getItems() { return items; } - public void setItems(Map items) { + public void setItems(Map items) { this.items = items; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index ac8db7f97621a47..7c97b0f881a73b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; @@ -56,6 +57,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Lists; import com.google.common.collect.Range; import com.google.common.collect.Sets; @@ -79,16 +81,16 @@ private UpdateMvByPartitionCommand(LogicalPlan logicalQuery) { * Construct command * * @param mv materialize view - * @param partitionIds update partitions in mv and tables + * @param partitionNames update partitions in mv and tables * @param tableWithPartKey the partitions key for different table * @return command */ - public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionIds, + public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionNames, Map tableWithPartKey) throws UserException { NereidsParser parser = new NereidsParser(); Map> predicates = - constructTableWithPredicates(mv, partitionIds, tableWithPartKey); - List parts = constructPartsForMv(mv, partitionIds); + constructTableWithPredicates(mv, partitionNames, tableWithPartKey); + List parts = constructPartsForMv(partitionNames); Plan plan = parser.parseSingle(mv.getQuerySql()); plan = plan.accept(new PredicateAdder(), predicates); if (plan instanceof Sink) { @@ -99,17 +101,17 @@ public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionIds, return new UpdateMvByPartitionCommand(sink); } - private static List constructPartsForMv(MTMV mv, Set partitionIds) { - return partitionIds.stream() - .map(id -> mv.getPartition(id).getName()) - .collect(ImmutableList.toImmutableList()); + private static List constructPartsForMv(Set partitionNames) { + return Lists.newArrayList(partitionNames); } private static Map> constructTableWithPredicates(MTMV mv, - Set partitionIds, Map tableWithPartKey) { - Set items = partitionIds.stream() - .map(id -> mv.getPartitionInfo().getItem(id)) - .collect(ImmutableSet.toImmutableSet()); + Set partitionNames, Map tableWithPartKey) throws AnalysisException { + Set items = Sets.newHashSet(); + for (String partitionName : partitionNames) { + PartitionItem partitionItem = mv.getPartitionItemOrAnalysisException(partitionName); + items.add(partitionItem); + } ImmutableMap.Builder> builder = new ImmutableMap.Builder<>(); tableWithPartKey.forEach((table, colName) -> builder.put(table, constructPredicates(items, colName)) diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java index bf819553e861c83..261b750c79698d4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java @@ -71,13 +71,13 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc minTimes = 0; result = Lists.newArrayList(p1); - mtmv.getPartitionIds(); + mtmv.getPartitionNames(); minTimes = 0; - result = Lists.newArrayList(1L); + result = Sets.newHashSet("name1"); - p1.getId(); + p1.getName(); minTimes = 0; - result = 1L; + result = "name1"; mtmv.getMvPartitionInfo(); minTimes = 0; @@ -101,7 +101,7 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc mtmv.getPartitionName(anyLong); minTimes = 0; - result = "p1"; + result = "name1"; mtmv.getRefreshSnapshot(); minTimes = 0; @@ -119,13 +119,13 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc minTimes = 0; result = true; - baseOlapTable.getPartitionSnapshot(anyLong); + baseOlapTable.getPartitionSnapshot(anyString); minTimes = 0; result = baseSnapshotIf; baseOlapTable.getPartitionName(anyLong); minTimes = 0; - result = "p1"; + result = "name1"; refreshSnapshot.equalsWithRelatedPartition(anyString, anyString, (MTMVSnapshotIf) any); minTimes = 0; @@ -156,7 +156,7 @@ public void testIsMTMVSyncNotSync() { @Test public void testIsSyncWithPartition() throws AnalysisException { boolean isSyncWithPartition = MTMVPartitionUtil - .isSyncWithPartitions(mtmv, 1L, baseOlapTable, Sets.newHashSet(2L)); + .isSyncWithPartitions(mtmv, "name1", baseOlapTable, Sets.newHashSet("name2")); Assert.assertTrue(isSyncWithPartition); } @@ -170,7 +170,7 @@ public void testIsSyncWithPartitionNotSync() throws AnalysisException { } }; boolean isSyncWithPartition = MTMVPartitionUtil - .isSyncWithPartitions(mtmv, 1L, baseOlapTable, Sets.newHashSet(2L)); + .isSyncWithPartitions(mtmv, "name1", baseOlapTable, Sets.newHashSet("name2")); Assert.assertFalse(isSyncWithPartition); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGeneratorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGeneratorTest.java index b866100b63d4e72..15036c5e497516f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGeneratorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGeneratorTest.java @@ -72,7 +72,7 @@ public void testRollUpRange() throws AnalysisException { } }; MTMVRelatedPartitionDescRollUpGenerator generator = new MTMVRelatedPartitionDescRollUpGenerator(); - Map> relatedPartitionDescs = Maps.newHashMap(); + Map> relatedPartitionDescs = Maps.newHashMap(); PartitionKeyDesc desc20200101 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("2020-01-01")), Lists.newArrayList(new PartitionValue("2020-01-02"))); @@ -82,10 +82,10 @@ public void testRollUpRange() throws AnalysisException { PartitionKeyDesc desc20200201 = PartitionKeyDesc.createFixed( Lists.newArrayList(new PartitionValue("2020-02-01")), Lists.newArrayList(new PartitionValue("2020-02-02"))); - relatedPartitionDescs.put(desc20200101, Sets.newHashSet(1L)); - relatedPartitionDescs.put(desc20200102, Sets.newHashSet(2L)); - relatedPartitionDescs.put(desc20200201, Sets.newHashSet(3L)); - Map> res = generator.rollUpRange(relatedPartitionDescs, + relatedPartitionDescs.put(desc20200101, Sets.newHashSet("name1")); + relatedPartitionDescs.put(desc20200102, Sets.newHashSet("name2")); + relatedPartitionDescs.put(desc20200201, Sets.newHashSet("name3")); + Map> res = generator.rollUpRange(relatedPartitionDescs, mtmvPartitionInfo); PartitionKeyDesc expectDesc202001 = PartitionKeyDesc.createFixed( @@ -95,8 +95,8 @@ public void testRollUpRange() throws AnalysisException { Lists.newArrayList(new PartitionValue("2020-02-01")), Lists.newArrayList(new PartitionValue("2020-03-01"))); Assert.assertEquals(2, res.size()); - Assert.assertEquals(Sets.newHashSet(1L, 2L), res.get(expectDesc202001)); - Assert.assertEquals(Sets.newHashSet(3L), res.get(expectDesc202002)); + Assert.assertEquals(Sets.newHashSet("name1", "name2"), res.get(expectDesc202001)); + Assert.assertEquals(Sets.newHashSet("name3"), res.get(expectDesc202002)); } @Test @@ -127,18 +127,18 @@ public void testRollUpList() throws AnalysisException { } }; MTMVRelatedPartitionDescRollUpGenerator generator = new MTMVRelatedPartitionDescRollUpGenerator(); - Map> relatedPartitionDescs = Maps.newHashMap(); - relatedPartitionDescs.put(generateInDesc("2020-01-01"), Sets.newHashSet(1L)); - relatedPartitionDescs.put(generateInDesc("2020-01-02"), Sets.newHashSet(2L)); - relatedPartitionDescs.put(generateInDesc("2020-02-01"), Sets.newHashSet(3L)); - Map> res = generator.rollUpList(relatedPartitionDescs, + Map> relatedPartitionDescs = Maps.newHashMap(); + relatedPartitionDescs.put(generateInDesc("2020-01-01"), Sets.newHashSet("name1")); + relatedPartitionDescs.put(generateInDesc("2020-01-02"), Sets.newHashSet("name2")); + relatedPartitionDescs.put(generateInDesc("2020-02-01"), Sets.newHashSet("name3")); + Map> res = generator.rollUpList(relatedPartitionDescs, mtmvPartitionInfo, Maps.newHashMap()); PartitionKeyDesc expectDesc202001 = generateInDesc("2020-01-01", "2020-01-02"); PartitionKeyDesc expectDesc202002 = generateInDesc("2020-02-01"); Assert.assertEquals(2, res.size()); - Assert.assertEquals(Sets.newHashSet(1L, 2L), res.get(expectDesc202001)); - Assert.assertEquals(Sets.newHashSet(3L), res.get(expectDesc202002)); + Assert.assertEquals(Sets.newHashSet("name1", "name2"), res.get(expectDesc202001)); + Assert.assertEquals(Sets.newHashSet("name3"), res.get(expectDesc202002)); } private PartitionKeyDesc generateInDesc(String... values) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java index 8de7ed75ccd5144..864478933c0f14e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRewriteUtilTest.java @@ -103,7 +103,8 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc minTimes = 0; result = true; - MTMVPartitionUtil.isMTMVPartitionSync((MTMV) any, anyLong, (Set) any, (Set) any, + MTMVPartitionUtil.isMTMVPartitionSync((MTMV) any, anyString, (Set) any, + (Set) any, (Set) any); minTimes = 0; result = true; @@ -130,7 +131,8 @@ public void testGetMTMVCanRewritePartitionsInGracePeriod() throws AnalysisExcept minTimes = 0; result = 2L; - MTMVPartitionUtil.isMTMVPartitionSync((MTMV) any, anyLong, (Set) any, (Set) any, + MTMVPartitionUtil.isMTMVPartitionSync((MTMV) any, anyString, (Set) any, + (Set) any, (Set) any); minTimes = 0; result = false; @@ -150,7 +152,8 @@ public void testGetMTMVCanRewritePartitionsNotInGracePeriod() throws AnalysisExc minTimes = 0; result = 1L; - MTMVPartitionUtil.isMTMVPartitionSync((MTMV) any, anyLong, (Set) any, (Set) any, + MTMVPartitionUtil.isMTMVPartitionSync((MTMV) any, anyString, (Set) any, + (Set) any, (Set) any); minTimes = 0; result = false; @@ -180,7 +183,8 @@ public void testGetMTMVCanRewritePartitionsDisableMaterializedViewRewrite() { public void testGetMTMVCanRewritePartitionsNotSync() throws AnalysisException { new Expectations() { { - MTMVPartitionUtil.isMTMVPartitionSync((MTMV) any, anyLong, (Set) any, (Set) any, + MTMVPartitionUtil.isMTMVPartitionSync((MTMV) any, anyString, (Set) any, + (Set) any, (Set) any); minTimes = 0; result = false; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java index 3bc603395005730..512bd6099f025b0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTaskTest.java @@ -42,11 +42,9 @@ import java.util.Set; public class MTMVTaskTest { - private long poneId = 1L; private String poneName = "p1"; - private long ptwoId = 2L; private String ptwoName = "p2"; - private List allPartitionIds = Lists.newArrayList(poneId, ptwoId); + private List allPartitionNames = Lists.newArrayList(poneName, ptwoName); private MTMVRelation relation = new MTMVRelation(Sets.newHashSet(), Sets.newHashSet(), Sets.newHashSet()); @Mocked @@ -70,9 +68,9 @@ public void setUp() minTimes = 0; result = mtmv; - mtmv.getPartitionIds(); + mtmv.getPartitionNames(); minTimes = 0; - result = allPartitionIds; + result = Sets.newHashSet(poneName, ptwoName); mtmv.getMvPartitionInfo(); minTimes = 0; @@ -82,12 +80,12 @@ public void setUp() minTimes = 0; result = MTMVPartitionType.FOLLOW_BASE_TABLE; - mtmvPartitionUtil.getPartitionsIdsByNames(mtmv, Lists.newArrayList(poneName)); - minTimes = 0; - result = poneId; + // mtmvPartitionUtil.getPartitionsIdsByNames(mtmv, Lists.newArrayList(poneName)); + // minTimes = 0; + // result = poneId; mtmvPartitionUtil.isMTMVSync(mtmv, (Set) any, (Set) any, - (Map>) any); + (Map>) any); minTimes = 0; result = true; @@ -106,16 +104,16 @@ public void setUp() public void testCalculateNeedRefreshPartitionsManualComplete() throws AnalysisException { MTMVTaskContext context = new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL, null, true); MTMVTask task = new MTMVTask(mtmv, relation, context); - List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); - Assert.assertEquals(allPartitionIds, result); + List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); + Assert.assertEquals(allPartitionNames, result); } @Test public void testCalculateNeedRefreshPartitionsManualPartitions() throws AnalysisException { MTMVTaskContext context = new MTMVTaskContext(MTMVTaskTriggerMode.MANUAL, Lists.newArrayList(poneName), false); MTMVTask task = new MTMVTask(mtmv, relation, context); - List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); - Assert.assertEquals(Lists.newArrayList(poneId), result); + List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); + Assert.assertEquals(Lists.newArrayList(poneName), result); } @Test @@ -129,7 +127,7 @@ public void testCalculateNeedRefreshPartitionsSystem() throws AnalysisException }; MTMVTaskContext context = new MTMVTaskContext(MTMVTaskTriggerMode.SYSTEM); MTMVTask task = new MTMVTask(mtmv, relation, context); - List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); + List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); Assert.assertTrue(CollectionUtils.isEmpty(result)); } @@ -137,8 +135,8 @@ public void testCalculateNeedRefreshPartitionsSystem() throws AnalysisException public void testCalculateNeedRefreshPartitionsSystemComplete() throws AnalysisException { MTMVTaskContext context = new MTMVTaskContext(MTMVTaskTriggerMode.SYSTEM); MTMVTask task = new MTMVTask(mtmv, relation, context); - List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); - Assert.assertEquals(allPartitionIds, result); + List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); + Assert.assertEquals(allPartitionNames, result); } @Test @@ -146,15 +144,15 @@ public void testCalculateNeedRefreshPartitionsSystemNotSyncComplete() throws Ana new Expectations() { { mtmvPartitionUtil.isMTMVSync(mtmv, (Set) any, (Set) any, - (Map>) any); + (Map>) any); minTimes = 0; result = false; } }; MTMVTaskContext context = new MTMVTaskContext(MTMVTaskTriggerMode.SYSTEM); MTMVTask task = new MTMVTask(mtmv, relation, context); - List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); - Assert.assertEquals(allPartitionIds, result); + List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); + Assert.assertEquals(allPartitionNames, result); } @Test @@ -162,7 +160,7 @@ public void testCalculateNeedRefreshPartitionsSystemNotSyncAuto() throws Analysi new Expectations() { { mtmvPartitionUtil - .isMTMVSync(mtmv, (Set) any, (Set) any, (Map>) any); + .isMTMVSync(mtmv, (Set) any, (Set) any, (Map>) any); minTimes = 0; result = false; @@ -171,14 +169,14 @@ public void testCalculateNeedRefreshPartitionsSystemNotSyncAuto() throws Analysi result = RefreshMethod.AUTO; mtmvPartitionUtil - .getMTMVNeedRefreshPartitions(mtmv, (Set) any, (Map>) any); + .getMTMVNeedRefreshPartitions(mtmv, (Set) any, (Map>) any); minTimes = 0; - result = Lists.newArrayList(ptwoId); + result = Lists.newArrayList(ptwoName); } }; MTMVTaskContext context = new MTMVTaskContext(MTMVTaskTriggerMode.SYSTEM); MTMVTask task = new MTMVTask(mtmv, relation, context); - List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); - Assert.assertEquals(Lists.newArrayList(ptwoId), result); + List result = task.calculateNeedRefreshPartitions(Maps.newHashMap()); + Assert.assertEquals(Lists.newArrayList(ptwoName), result); } } From a59f9c3fa158ea24da288efba4022261c4826f9e Mon Sep 17 00:00:00 2001 From: xueweizhang Date: Fri, 17 May 2024 18:07:31 +0800 Subject: [PATCH 047/111] [fix](planner) fix unrequired slot bug when join node introduced by #25204 (#34923) before fix, join node will retain some slots, which are not materialized and unrequired. join node need remove these slots and not make them be output slots. Signed-off-by: nextdreamblue --- .../apache/doris/planner/JoinNodeBase.java | 6 ++ .../test_inlineview_with_project.out | 9 +- .../test_inlineview_with_project.groovy | 84 +++++++++++++++++++ 3 files changed, 96 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java index fbe12b3d6a04053..91a3c26e7708fa3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/JoinNodeBase.java @@ -171,6 +171,9 @@ protected void computeOutputTuple(Analyzer analyzer) throws UserException { boolean needSetToNullable = getChild(0) instanceof JoinNodeBase && analyzer.isOuterJoined(leftTupleDesc.getId()); for (SlotDescriptor leftSlotDesc : leftTupleDesc.getSlots()) { + if (!isMaterializedByChild(leftSlotDesc, getChild(0).getOutputSmap())) { + continue; + } SlotDescriptor outputSlotDesc = analyzer.getDescTbl().copySlotDescriptor(outputTupleDesc, leftSlotDesc); if (leftNullable) { @@ -191,6 +194,9 @@ protected void computeOutputTuple(Analyzer analyzer) throws UserException { boolean needSetToNullable = getChild(1) instanceof JoinNodeBase && analyzer.isOuterJoined(rightTupleDesc.getId()); for (SlotDescriptor rightSlotDesc : rightTupleDesc.getSlots()) { + if (!isMaterializedByChild(rightSlotDesc, getChild(1).getOutputSmap())) { + continue; + } SlotDescriptor outputSlotDesc = analyzer.getDescTbl().copySlotDescriptor(outputTupleDesc, rightSlotDesc); if (rightNullable) { diff --git a/regression-test/data/correctness_p0/test_inlineview_with_project.out b/regression-test/data/correctness_p0/test_inlineview_with_project.out index e6ed3ee5706de91..1f85a6dff31f53c 100644 --- a/regression-test/data/correctness_p0/test_inlineview_with_project.out +++ b/regression-test/data/correctness_p0/test_inlineview_with_project.out @@ -14,9 +14,9 @@ 3 -- !select5 -- -1 -2 -3 +1.0 +2.0 +3.0 -- !select5 -- 1 @@ -29,3 +29,6 @@ -- !select5 -- 3 +-- !select6 -- +2020-01-01 + diff --git a/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy b/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy index f13c45a53287592..e769aa8c8ea9c5c 100644 --- a/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy +++ b/regression-test/suites/correctness_p0/test_inlineview_with_project.groovy @@ -665,4 +665,88 @@ suite("test_inlineview_with_project") { sql """DROP TABLE IF EXISTS `dr_user_test_t1`;""" sql """DROP TABLE IF EXISTS `dr_user_test_t2`;""" + + sql """ + drop table if exists dws_mf_wms_join_t1; + """ + + sql """ + drop table if exists dws_mf_wms_join_t2; + """ + + + sql """CREATE TABLE `dws_mf_wms_join_t1` ( + `ddate` DATE NULL COMMENT '日期字段', + `game_id` VARCHAR(65533) NULL, + `main_currency_stock` BIGINT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`ddate`) + DISTRIBUTED BY HASH(`ddate`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + + sql """CREATE TABLE `dws_mf_wms_join_t2` ( + `game_id` VARCHAR(65533) NULL, + ) ENGINE=OLAP + DUPLICATE KEY(`game_id`) + DISTRIBUTED BY HASH(`game_id`) BUCKETS 10 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + );""" + + sql """insert into dws_mf_wms_join_t1 values('2020-01-01','12345',100);""" + sql """insert into dws_mf_wms_join_t2 values('12345');""" + + qt_select6 """SELECT + a1.ddate + FROM + ( + SELECT + aaa.ddate + FROM + ( + SELECT + aa.ddate, + CONCAT('main', aa.main_currency_stock) AS arr, + ROW_NUMBER() OVER ( + PARTITION BY aa.ddate + ) AS rn + FROM + ( + SELECT + ddate, + main_currency_stock, + game_id + FROM + dws_mf_wms_join_t1 a + ) aa + LEFT JOIN ( + SELECT + game_id + FROM + dws_mf_wms_join_t2 + ) b ON aa.game_id = b.game_id + ) aaa + CROSS JOIN ( + select + 1 as newarr + ) b + WHERE + rn = 1 + ) a1 + GROUP BY + GROUPING SETS ( + ( + a1.ddate + ) + );""" + + sql """ + drop table if exists dws_mf_wms_join_join_t1; + """ + + sql """ + drop table if exists dws_mf_wms_join_join_t2; + """ } From 89d5f2e816a75c8f36bf8653e067927eb37d08d8 Mon Sep 17 00:00:00 2001 From: slothever <18522955+wsjz@users.noreply.github.com> Date: Fri, 17 May 2024 22:43:59 +0800 Subject: [PATCH 048/111] [fix](multi-catalog)remove http scheme in oss endpoint (#34907) remove http scheme in oss endpoint, scheme maybe appear in url (http://bucket.http//.region.aliyuncs.com) if use http client --- .../apache/doris/datasource/property/PropertyConverter.java | 2 +- .../doris/datasource/property/PropertyConverterTest.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index bccd3147a9b5eaa..e3ba2bdce27cea0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -493,7 +493,7 @@ private static void putNewPropertiesForCompatibility(Map props, } private static String getOssEndpoint(String region, boolean publicAccess) { - String prefix = "http://oss-"; + String prefix = "oss-"; String suffix = ".aliyuncs.com"; if (!publicAccess) { suffix = "-internal" + suffix; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java index 90a15db4aaa99ca..4e4b0408641c15e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java @@ -336,7 +336,7 @@ public void testDlfPropertiesConverter() throws Exception { Map hdProps = catalog.getCatalogProperty().getHadoopProperties(); Assertions.assertEquals("akk", hdProps.get(OssProperties.ACCESS_KEY)); Assertions.assertEquals("skk", hdProps.get(OssProperties.SECRET_KEY)); - Assertions.assertEquals("http://oss-cn-beijing-internal.aliyuncs.com", + Assertions.assertEquals("oss-cn-beijing-internal.aliyuncs.com", hdProps.get(OssProperties.ENDPOINT)); String queryDlf2 = "create catalog hms_dlf2 properties (\n" @@ -392,7 +392,7 @@ public void testDlfPropertiesConverter() throws Exception { Map hdProps3 = catalog3.getCatalogProperty().getHadoopProperties(); Assertions.assertEquals("akk", hdProps3.get(OssProperties.ACCESS_KEY)); Assertions.assertEquals("skk", hdProps3.get(OssProperties.SECRET_KEY)); - Assertions.assertEquals("http://oss-cn-beijing-internal.aliyuncs.com", hdProps3.get(OssProperties.ENDPOINT)); + Assertions.assertEquals("oss-cn-beijing-internal.aliyuncs.com", hdProps3.get(OssProperties.ENDPOINT)); } @Test From 22f85be71266679a6ca4de7657af99cbe7f2ef4e Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Fri, 17 May 2024 22:52:43 +0800 Subject: [PATCH 049/111] [fix](hive-ctas) support create hive table with full quolified name (#34984) Before, when executing `create table hive.db.table as select` to create table in hive catalog, if current catalog is not hive catalog, the default engine name will be filled with `olap`, which is wrong. This PR will fill the default engine name base on specified catalog. --- .../plans/commands/info/CreateTableInfo.java | 17 +++++++--- .../hive/ddl/test_hive_ctas.out | 33 +++++++++++++++++++ .../hive/ddl/test_hive_ctas.groovy | 14 ++++++++ 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 585d1da0b101d58..3d4607517d95508 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -40,6 +40,7 @@ import org.apache.doris.common.util.InternalDatabaseUtil; import org.apache.doris.common.util.ParseUtil; import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.es.EsUtil; import org.apache.doris.datasource.hive.HMSExternalCatalog; @@ -548,16 +549,21 @@ public void validate(ConnectContext ctx) { } private void paddingEngineName(String ctlName, ConnectContext ctx) { + Preconditions.checkArgument(!Strings.isNullOrEmpty(ctlName)); if (Strings.isNullOrEmpty(engineName)) { - if (InternalCatalog.INTERNAL_CATALOG_NAME.equals(ctlName)) { + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(ctlName); + if (catalog == null) { + throw new AnalysisException("Unknown catalog: " + ctlName); + } + + if (catalog instanceof InternalCatalog) { engineName = "olap"; - } else if (ctx.getCurrentCatalog() instanceof HMSExternalCatalog) { + } else if (catalog instanceof HMSExternalCatalog) { engineName = "hive"; - } else if (ctx.getCurrentCatalog() instanceof IcebergExternalCatalog) { + } else if (catalog instanceof IcebergExternalCatalog) { engineName = "iceberg"; } else { - // set to olap by default - engineName = "olap"; + throw new AnalysisException("Current catalog does not support create table: " + ctlName); } } } @@ -782,3 +788,4 @@ public void setIsExternal(boolean isExternal) { this.isExternal = isExternal; } } + diff --git a/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out b/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out index 39d711b2df0e6c1..d9fa227a59c0d23 100644 --- a/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out +++ b/regression-test/data/external_table_p0/hive/ddl/test_hive_ctas.out @@ -39,6 +39,14 @@ \N 11 value_for_pt1 \N 22 value_for_pt11 +-- !qualified_table1 -- +11 value_for_pt1 +22 value_for_pt11 + +-- !qualified_table2 -- +11 value_for_pt1 +22 value_for_pt11 + -- !ctas_types_01 -- true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 @@ -85,6 +93,14 @@ true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default \N 11 value_for_pt1 \N 22 value_for_pt11 +-- !qualified_table1 -- +11 value_for_pt1 +22 value_for_pt11 + +-- !qualified_table2 -- +11 value_for_pt1 +22 value_for_pt11 + -- !ctas_types_01 -- true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 @@ -131,6 +147,14 @@ true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default \N 11 value_for_pt1 \N 22 value_for_pt11 +-- !qualified_table1 -- +11 value_for_pt1 +22 value_for_pt11 + +-- !qualified_table2 -- +11 value_for_pt1 +22 value_for_pt11 + -- !ctas_types_01 -- true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 @@ -177,8 +201,17 @@ true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default \N 11 value_for_pt1 \N 22 value_for_pt11 +-- !qualified_table1 -- +11 value_for_pt1 +22 value_for_pt11 + +-- !qualified_table2 -- +11 value_for_pt1 +22 value_for_pt11 + -- !ctas_types_01 -- true 127 32767 2147483647 9223372036854775807 default 22.12345 3.141592653 99999.9999 default default 2023-05-29 2023-05-29T23:19:34 -- !ctas_types_02 -- true 127 32767 2147483647 default 22.12345 3.141592653 99999.9999 default + diff --git a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy index 3bead6d0a3e9537..d7d9b33c4742b6d 100644 --- a/regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy +++ b/regression-test/suites/external_table_p0/hive/ddl/test_hive_ctas.groovy @@ -392,6 +392,20 @@ suite("test_hive_ctas", "p0,external,hive,external_docker,external_docker_hive") exception "errCode = 2, detailMessage = insert into cols should be corresponding to the query output" } sql """ DROP TABLE IF EXISTS ${catalog_name}.test_no_err.ctas_o2 """ + + // test ctas with qualified table name + sql """drop table if exists ${catalog_name}.test_no_err.qualified_table1""" + sql """use internal.test_ctas_olap""" + sql """create table ${catalog_name}.test_no_err.qualified_table1 as SELECT col1,pt1 as col2 FROM ${catalog_name}.test_ctas.part_ctas_src WHERE col1>0;""" + order_qt_qualified_table1 """select * from ${catalog_name}.test_no_err.qualified_table1""" + + sql """drop table if exists ${catalog_name}.test_no_err.qualified_table2""" + sql """switch ${catalog_name}""" + sql """create table test_no_err.qualified_table2 as SELECT col1,pt1 as col2 FROM ${catalog_name}.test_ctas.part_ctas_src WHERE col1>0;""" + order_qt_qualified_table2 """select * from ${catalog_name}.test_no_err.qualified_table2""" + + sql """drop table if exists ${catalog_name}.test_no_err.qualified_table1""" + sql """drop table if exists ${catalog_name}.test_no_err.qualified_table2""" sql """ DROP DATABASE IF EXISTS test_no_err """ } finally { From a6a398d7a4fdf35f223d6796aef37abb7a332201 Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Sun, 19 May 2024 14:20:57 +0800 Subject: [PATCH 050/111] [Fix](function) remove datev2 signature of microsecond #35017 --- .../apache/doris/analysis/FunctionCallExpr.java | 16 ++++++++-------- .../functions/scalar/Microsecond.java | 4 +--- .../datetime_functions/test_date_function.out | 3 +++ .../datetime_functions/test_date_function.groovy | 1 + 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java index 5253ff7fde15e79..4aaaf6508d8f521 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java @@ -1815,7 +1815,7 @@ && collectChildReturnTypes()[0].isDecimalV3()) { .toSql()); } } - if (fn.getFunctionName().getFunction().equals("timediff")) { + if (fn.getFunctionName().getFunction().equalsIgnoreCase("timediff")) { fn.getReturnType().getPrimitiveType().setTimeType(); ScalarType left = (ScalarType) argTypes[0]; ScalarType right = (ScalarType) argTypes[1]; @@ -1826,17 +1826,17 @@ && collectChildReturnTypes()[0].isDecimalV3()) { } } - if (fn.getFunctionName().getFunction().equals("from_microsecond")) { + if (fn.getFunctionName().getFunction().equalsIgnoreCase("from_microsecond")) { Type ret = ScalarType.createDatetimeV2Type(6); fn.setReturnType(ret); } - if (fn.getFunctionName().getFunction().equals("from_millisecond")) { + if (fn.getFunctionName().getFunction().equalsIgnoreCase("from_millisecond")) { Type ret = ScalarType.createDatetimeV2Type(3); fn.setReturnType(ret); } - if (fn.getFunctionName().getFunction().equals("from_second")) { + if (fn.getFunctionName().getFunction().equalsIgnoreCase("from_second")) { Type ret = ScalarType.createDatetimeV2Type(0); fn.setReturnType(ret); } @@ -1863,7 +1863,7 @@ && collectChildReturnTypes()[0].isDecimalV3()) { } } - if (fn.getFunctionName().getFunction().equals("struct_element")) { + if (fn.getFunctionName().getFunction().equalsIgnoreCase("struct_element")) { if (children.size() < 2) { throw new AnalysisException(fnName.getFunction() + " needs two parameters: " + this.toSql()); } @@ -1888,7 +1888,7 @@ && collectChildReturnTypes()[0].isDecimalV3()) { } } - if (fn.getFunctionName().getFunction().equals("sha2")) { + if (fn.getFunctionName().getFunction().equalsIgnoreCase("sha2")) { if ((children.size() != 2) || (getChild(1).isConstant() == false) || !(getChild(1) instanceof IntLiteral)) { throw new AnalysisException( @@ -2046,7 +2046,7 @@ && collectChildReturnTypes()[0].isDecimalV3()) { * SELECT str_to_date("2020-09-01", "%Y-%m-%d %H:%i:%s"); * Return type is DATETIME */ - if (fn.getFunctionName().getFunction().equals("str_to_date")) { + if (fn.getFunctionName().getFunction().equalsIgnoreCase("str_to_date")) { Expr child1Result = getChild(1).getResultValue(false); if (child1Result instanceof StringLiteral) { if (DateLiteral.hasTimePart(child1Result.getStringValue())) { @@ -2085,7 +2085,7 @@ && collectChildReturnTypes()[0].isDecimalV3()) { // cast(xx as char(N)/varchar(N)) will be handled as substr(cast(xx as char, varchar), 1, N), // but type is varchar(*), we change it to varchar(N); - if (fn.getFunctionName().getFunction().equals("substr") + if (fn.getFunctionName().getFunction().equalsIgnoreCase("substr") && children.size() == 3 && children.get(1) instanceof IntLiteral && children.get(2) instanceof IntLiteral) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Microsecond.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Microsecond.java index 6d3bf51b55d53d7..4e610e21bedb8da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Microsecond.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/Microsecond.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.DateTimeV2Type; -import org.apache.doris.nereids.types.DateV2Type; import org.apache.doris.nereids.types.IntegerType; import com.google.common.base.Preconditions; @@ -39,8 +38,7 @@ public class Microsecond extends ScalarFunction implements UnaryExpression, ExplicitlyCastableSignature, PropagateNullableOnDateLikeV2Args { public static final List SIGNATURES = ImmutableList.of( - FunctionSignature.ret(IntegerType.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT), - FunctionSignature.ret(IntegerType.INSTANCE).args(DateV2Type.INSTANCE) + FunctionSignature.ret(IntegerType.INSTANCE).args(DateTimeV2Type.SYSTEM_DEFAULT) ); /** diff --git a/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out b/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out index 46a6256615cfbd9..d71b9968051c365 100644 --- a/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out +++ b/regression-test/data/nereids_p0/sql_functions/datetime_functions/test_date_function.out @@ -422,6 +422,9 @@ February -- !sql -- 767891 +-- !sql_ms_use_date -- +0 + -- !sql -- 1 2022-08-01 2 2022-08-01 diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy index 58413e29cee4b5d..d1e8fcab8052ec3 100644 --- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy @@ -428,6 +428,7 @@ suite("test_date_function") { sql """ insert into ${tableName} values('1999-01-02 10:11:12.767891') """ qt_sql """ select microsecond(k1) from ${tableName}; """ + qt_sql_ms_use_date "select microsecond(cast(k1 as date)) from ${tableName};" // qt_sql """ select count(*) from (select * from numbers("number" = "200")) tmp1 WHERE 0 <= UNIX_TIMESTAMP(); """ From 7aff10b93bad45b7a8e191be130a25c1d96f0b6e Mon Sep 17 00:00:00 2001 From: airborne12 Date: Mon, 20 May 2024 09:52:22 +0800 Subject: [PATCH 051/111] [Fix](inverted index) fix race condition for column reader load inverted index reader (#34922) (#35040) --- be/src/olap/rowset/segment_v2/column_reader.cpp | 11 +++++++---- be/src/olap/rowset/segment_v2/column_reader.h | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp b/be/src/olap/rowset/segment_v2/column_reader.cpp index 9d19d1a64b4e30f..6d5c28b8d6dc84c 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.cpp +++ b/be/src/olap/rowset/segment_v2/column_reader.cpp @@ -262,9 +262,12 @@ Status ColumnReader::new_inverted_index_iterator( std::shared_ptr index_file_reader, const TabletIndex* index_meta, const StorageReadOptions& read_options, std::unique_ptr* iterator) { RETURN_IF_ERROR(_ensure_inverted_index_loaded(index_file_reader, index_meta)); - if (_inverted_index) { - RETURN_IF_ERROR(_inverted_index->new_iterator(read_options.stats, - read_options.runtime_state, iterator)); + { + std::shared_lock rlock(_load_index_lock); + if (_inverted_index) { + RETURN_IF_ERROR(_inverted_index->new_iterator(read_options.stats, + read_options.runtime_state, iterator)); + } } return Status::OK(); } @@ -536,7 +539,7 @@ Status ColumnReader::_load_bitmap_index(bool use_page_cache, bool kept_in_memory Status ColumnReader::_load_inverted_index_index( std::shared_ptr index_file_reader, const TabletIndex* index_meta) { - std::lock_guard wlock(_load_index_lock); + std::unique_lock wlock(_load_index_lock); if (_inverted_index && index_meta && _inverted_index->get_index_id() == index_meta->index_id()) { diff --git a/be/src/olap/rowset/segment_v2/column_reader.h b/be/src/olap/rowset/segment_v2/column_reader.h index 9c889b848fda293..30f916d00cdaa97 100644 --- a/be/src/olap/rowset/segment_v2/column_reader.h +++ b/be/src/olap/rowset/segment_v2/column_reader.h @@ -249,7 +249,7 @@ class ColumnReader { // meta for various column indexes (null if the index is absent) std::unique_ptr _segment_zone_map; - mutable std::mutex _load_index_lock; + mutable std::shared_mutex _load_index_lock; std::unique_ptr _zone_map_index; std::unique_ptr _ordinal_index; std::unique_ptr _bitmap_index; From 665650857961b335376d97e8611069da31e00050 Mon Sep 17 00:00:00 2001 From: TengJianPing <18241664+jacktengg@users.noreply.github.com> Date: Mon, 20 May 2024 10:10:11 +0800 Subject: [PATCH 052/111] [fix](agg) fix DCHECK failure of agg when failed to alloc memory (#35011) * [fix](agg) fix DCHECK failure of agg when failed to alloc memory * add comment --- be/src/vec/common/string_buffer.hpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/be/src/vec/common/string_buffer.hpp b/be/src/vec/common/string_buffer.hpp index 84f0dffbff71fd1..3be04ad15e64a48 100644 --- a/be/src/vec/common/string_buffer.hpp +++ b/be/src/vec/common/string_buffer.hpp @@ -40,10 +40,12 @@ class BufferWritable final { _now_offset += 1; } - inline void commit() { - ColumnString::check_chars_length(_offsets.back() + _now_offset, 0); - _offsets.push_back(_offsets.back() + _now_offset); + void commit() { + auto now_offset = _now_offset; _now_offset = 0; + // the following code may throw exception, and DCHECK in destructor will fail if _now_offset is not reset to 0 + ColumnString::check_chars_length(_offsets.back() + now_offset, 0); + _offsets.push_back(_offsets.back() + now_offset); } ~BufferWritable() { DCHECK(_now_offset == 0); } From 7c29a964e510d2c7ccaa31a0348ec6fff2310f55 Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Mon, 20 May 2024 10:28:22 +0800 Subject: [PATCH 053/111] [Fix](Nereids) fix leading with multi level of brace pairs (#34169) (#35043) fix leading with multi level of brace pairs example: leading(t1 {{t2 t3} {t4 t5}} t6) can be reduced to leading(t1 {t2 t3 {t4 t5}} t6) also update cases which remove project node from explain shape plan --- .../doris/nereids/hint/LeadingHint.java | 44 +- .../nereids_hint_tpcds_p0/shape/query64.out | 18 +- .../data/nereids_p0/hint/fix_leading.out | 59 +- .../data/nereids_p0/hint/multi_leading.out | 575 ++++++---------- .../data/nereids_p0/hint/test_leading.out | 613 +++++++----------- .../suites/nereids_p0/hint/fix_leading.groovy | 8 + .../nereids_p0/hint/multi_leading.groovy | 1 + .../nereids_p0/hint/test_leading.groovy | 1 + 8 files changed, 496 insertions(+), 823 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java index 3ef5217566a44d1..4d58990fbc09a19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java @@ -55,7 +55,7 @@ public class LeadingHint extends Hint { private List parameters; private final List tablelist = new ArrayList<>(); - private final List levellist = new ArrayList<>(); + private final List levelList = new ArrayList<>(); private final Map distributeHints = new HashMap<>(); @@ -120,18 +120,46 @@ public LeadingHint(String hintName, List parameters, String originalStri } } else { tablelist.add(parameter); - levellist.add(level); + levelList.add(level); } lastParameter = parameter; } + normalizeLevelList(); + } + + private void removeGap(int left, int right, int gap) { + for (int i = left; i <= right; i++) { + levelList.set(i, levelList.get(i) - (gap - 1)); + } + } + + // when we write leading like: leading(t1 {{t2 t3} {t4 t5}} t6) + // levelList would like 0 2 2 3 3 0, it could be reduced to 0 1 1 2 2 0 like leading(t1 {t2 t3 {t4 t5}} t6) + // gap is like 0 to 2 or 3 to 0 in upper example, and this function is to remove gap when we use a lot of braces + private void normalizeLevelList() { + int leftIndex = 0; + // at lease two tables were needed + for (int i = 1; i < levelList.size(); i++) { + if ((levelList.get(i) - levelList.get(leftIndex)) > 1) { + int rightIndex = i; + for (int j = i; j < levelList.size(); j++) { + if ((levelList.get(rightIndex) - levelList.get(j)) > 1) { + removeGap(i, rightIndex, Math.min(levelList.get(i) - levelList.get(leftIndex), + levelList.get(rightIndex) - levelList.get(j))); + } + rightIndex = j; + } + } + leftIndex = i; + } } public List getTablelist() { return tablelist; } - public List getLevellist() { - return levellist; + public List getLevelList() { + return levelList; } public Map getRelationIdToScanMap() { @@ -485,10 +513,10 @@ public Plan generateLeadingJoinPlan() { } logicalPlan = makeFilterPlanIfExist(getFilters(), logicalPlan); assert (logicalPlan != null); - stack.push(Pair.of(getLevellist().get(index), Pair.of(logicalPlan, index))); - int stackTopLevel = getLevellist().get(index++); + stack.push(Pair.of(getLevelList().get(index), Pair.of(logicalPlan, index))); + int stackTopLevel = getLevelList().get(index++); while (index < getTablelist().size()) { - int currentLevel = getLevellist().get(index); + int currentLevel = getLevelList().get(index); if (currentLevel == stackTopLevel) { // should return error if can not found table logicalPlan = getLogicalPlanByName(getTablelist().get(index++)); @@ -531,7 +559,7 @@ public Plan generateLeadingJoinPlan() { logicalJoin.setBitmap(LongBitmap.or(getBitmap(newStackTop.second.first), getBitmap(logicalPlan))); if (stackTopLevel > 0) { if (index < getTablelist().size()) { - if (stackTopLevel > getLevellist().get(index)) { + if (stackTopLevel > getLevelList().get(index)) { stackTopLevel--; } } else { diff --git a/regression-test/data/nereids_hint_tpcds_p0/shape/query64.out b/regression-test/data/nereids_hint_tpcds_p0/shape/query64.out index 35bfa59388ea4dd..ceca7b3b1caee17 100644 --- a/regression-test/data/nereids_hint_tpcds_p0/shape/query64.out +++ b/regression-test/data/nereids_hint_tpcds_p0/shape/query64.out @@ -43,25 +43,25 @@ PhysicalCteAnchor ( cteId=CTEId#1 ) --------------------------------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_cdemo_sk = cd1.cd_demo_sk)) otherCondition=(( not (cd_marital_status = cd_marital_status))) build RFs:RF6 cd_demo_sk->[ss_cdemo_sk] ----------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF5 c_customer_sk->[ss_customer_sk] +--------------------------------------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=() build RFs:RF5 cd_demo_sk->[c_current_cdemo_sk] ----------------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF5 RF6 RF8 RF9 RF10 RF11 RF12 RF16 RF17 RF18 RF19 -----------------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((customer.c_current_cdemo_sk = cd2.cd_demo_sk)) otherCondition=() build RFs:RF4 cd_demo_sk->[c_current_cdemo_sk] +--------------------------------------------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((store_sales.ss_customer_sk = customer.c_customer_sk)) otherCondition=() build RFs:RF4 c_customer_sk->[ss_customer_sk] +----------------------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------------------------------PhysicalOlapScan[store_sales] apply RFs: RF4 RF6 RF8 RF9 RF10 RF11 RF12 RF16 RF17 RF18 RF19 ----------------------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------------------------------------------------PhysicalProject --------------------------------------------------------------------------------------------------hashJoin[INNER_JOIN] hashCondition=((customer.c_first_sales_date_sk = d2.d_date_sk)) otherCondition=() build RFs:RF3 d_date_sk->[c_first_sales_date_sk] ----------------------------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF3 RF4 RF7 RF13 RF14 +--------------------------------------------------------------------------------------------------------PhysicalOlapScan[customer] apply RFs: RF3 RF5 RF7 RF13 RF14 ----------------------------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------------------------------------------------------PhysicalProject --------------------------------------------------------------------------------------------------------PhysicalOlapScan[date_dim] -----------------------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------------------------------------------------------------------------PhysicalProject ---------------------------------------------------------------------------------------------------PhysicalOlapScan[customer_demographics] +----------------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] +------------------------------------------------------------------------------------------PhysicalProject +--------------------------------------------------------------------------------------------PhysicalOlapScan[customer_demographics] ----------------------------------------------------------------------------------PhysicalDistribute[DistributionSpecHash] ------------------------------------------------------------------------------------PhysicalProject --------------------------------------------------------------------------------------PhysicalOlapScan[customer_demographics] diff --git a/regression-test/data/nereids_p0/hint/fix_leading.out b/regression-test/data/nereids_p0/hint/fix_leading.out index 3acc6a7d7909bdc..987db0245fcb8f8 100644 --- a/regression-test/data/nereids_p0/hint/fix_leading.out +++ b/regression-test/data/nereids_p0/hint/fix_leading.out @@ -236,48 +236,45 @@ SyntaxError: PhysicalResultSink --hashAgg[GLOBAL] ----hashAgg[LOCAL] -------PhysicalProject ---------NestedLoopJoin[RIGHT_OUTER_JOIN](c3 > 500) -----------PhysicalDistribute[DistributionSpecGather] -------------PhysicalProject ---------------NestedLoopJoin[LEFT_OUTER_JOIN](c1 < 200)(c1 > 500) -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------filter((t2.c2 > 500)) -----------------------PhysicalOlapScan[t2] -----------PhysicalDistribute[DistributionSpecGather] -------------PhysicalProject ---------------PhysicalOlapScan[t3] +------NestedLoopJoin[RIGHT_OUTER_JOIN](c3 > 500) +--------PhysicalDistribute[DistributionSpecGather] +----------NestedLoopJoin[LEFT_OUTER_JOIN](c1 < 200)(c1 > 500) +------------PhysicalOlapScan[t1] +------------PhysicalDistribute[DistributionSpecReplicated] +--------------filter((t2.c2 > 500)) +----------------PhysicalOlapScan[t2] +--------PhysicalDistribute[DistributionSpecGather] +----------PhysicalOlapScan[t3] Hint log: Used: leading(t1 t2 t3 ) UnUsed: SyntaxError: --- !select5_1 -- +-- !select6_1 -- PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------NestedLoopJoin[LEFT_OUTER_JOIN](t3.c3 > 500) -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------NestedLoopJoin[LEFT_OUTER_JOIN](t1.c1 > 500) -------------------PhysicalProject ---------------------filter((t1.c1 < 200)) -----------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------filter((t2.c2 > 500)) -------------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t6.c6)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t1.c1 = t3.c3) and (t1.c1 = t4.c4) and (t1.c1 = t5.c5)) otherCondition=() +------------PhysicalOlapScan[t1] +------------PhysicalDistribute[DistributionSpecHash] +--------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4) and (t2.c2 = t5.c5) and (t3.c3 = t4.c4) and (t3.c3 = t5.c5)) otherCondition=() +----------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +------------------PhysicalOlapScan[t2] +------------------PhysicalDistribute[DistributionSpecHash] +--------------------PhysicalOlapScan[t3] +----------------PhysicalDistribute[DistributionSpecHash] +------------------hashJoin[INNER_JOIN] hashCondition=((t4.c4 = t5.c5)) otherCondition=() +--------------------PhysicalOlapScan[t4] +--------------------PhysicalDistribute[DistributionSpecHash] +----------------------PhysicalOlapScan[t5] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t6] Hint log: -Used: +Used: leading(t1 { { t2 t3 } { t4 t5 } } t6 ) UnUsed: -SyntaxError: leading(t1 t2) Msg:leading should have all tables in query block, missing tables: t3 +SyntaxError: diff --git a/regression-test/data/nereids_p0/hint/multi_leading.out b/regression-test/data/nereids_p0/hint/multi_leading.out index 71db5aec524b10c..4f7056de310d1b4 100644 --- a/regression-test/data/nereids_p0/hint/multi_leading.out +++ b/regression-test/data/nereids_p0/hint/multi_leading.out @@ -4,42 +4,32 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t1.c1)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------filter((t2.c2 > 300)) ---------------------PhysicalOlapScan[t2] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------filter((cte.c1 > 300)) -----------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t1.c1)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() +------------filter((t2.c2 > 300)) +--------------PhysicalOlapScan[t2] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------filter((t1.c1 > 300)) -------------------PhysicalOlapScan[t1] +--------------filter((cte.c1 > 300)) +----------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------filter((t1.c1 > 300)) +--------------PhysicalOlapScan[t1] -- !sql1_2 -- PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t1.c1)) otherCondition=() -------------PhysicalProject ---------------filter((t1.c1 > 300)) -----------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------filter((t2.c2 > 300)) -----------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------filter((cte.c1 > 300)) -------------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t1.c1)) otherCondition=() +----------filter((t1.c1 > 300)) +------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() +--------------filter((t2.c2 > 300)) +----------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecHash] +----------------filter((cte.c1 > 300)) +------------------PhysicalOlapScan[t1] Hint log: Used: leading(t2 t1 ) leading(t1 cte ) @@ -51,20 +41,15 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t1.c1)) otherCondition=() -------------PhysicalProject ---------------filter((t1.c1 > 300)) -----------------PhysicalOlapScan[t1] -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------filter((cte.c1 > 300)) ---------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------filter((t2.c2 > 300)) -----------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t1.c1)) otherCondition=() +----------filter((t1.c1 > 300)) +------------PhysicalOlapScan[t1] +----------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() +------------filter((cte.c1 > 300)) +--------------PhysicalOlapScan[t1] +------------PhysicalDistribute[DistributionSpecHash] +--------------filter((t2.c2 > 300)) +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t1 t2 ) leading(t1 cte ) @@ -76,20 +61,15 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t1.c1)) otherCondition=() -------------PhysicalProject ---------------filter((t1.c1 > 300)) -----------------PhysicalOlapScan[t1] -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------filter((cte.c1 > 300)) ---------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------filter((t2.c2 > 300)) -----------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t1.c1)) otherCondition=() +----------filter((t1.c1 > 300)) +------------PhysicalOlapScan[t1] +----------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() +------------filter((cte.c1 > 300)) +--------------PhysicalOlapScan[t1] +------------PhysicalDistribute[DistributionSpecHash] +--------------filter((t2.c2 > 300)) +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t1 t2 ) leading(t1 cte ) @@ -113,36 +93,26 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] -- !sql2_2 -- PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t3 alias1 ) @@ -154,18 +124,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t2] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t2] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 t1 ) @@ -177,18 +142,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(t2 t1 ) leading(t3 alias1 ) @@ -212,56 +172,38 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalOlapScan[t1] ---------------------PhysicalDistribute[DistributionSpecHash] -----------------------PhysicalProject -------------------------PhysicalOlapScan[t2] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t2] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] +--------------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t2] -- !sql3_2 -- PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t3] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------------PhysicalProject -------------------------PhysicalOlapScan[t1] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +------------PhysicalOlapScan[t3] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t1] +--------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +----------------PhysicalOlapScan[t1] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(t2 t1 ) leading(t3 alias1 cte ) @@ -273,28 +215,19 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] ---------------------PhysicalDistribute[DistributionSpecHash] -----------------------PhysicalProject -------------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] +--------------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t2 t1 ) @@ -306,28 +239,19 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t3] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() -----------------------PhysicalProject -------------------------PhysicalOlapScan[t2] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = cte.c11)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +------------PhysicalOlapScan[t3] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t1] +--------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t2.c2)) otherCondition=() +----------------PhysicalOlapScan[t2] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((cte.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(t2 t1 ) leading(t2 t1 ) leading(t2 t1 ) leading(t3 alias1 cte ) @@ -351,46 +275,32 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = alias2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() -----------------------PhysicalProject -------------------------PhysicalOlapScan[t2] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[t4] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = alias2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() +----------------PhysicalOlapScan[t2] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalOlapScan[t4] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] -- !sql4_1 -- PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = alias2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = alias2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecHash] +----------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() +------------------PhysicalOlapScan[t2] ------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() -------------------------PhysicalProject ---------------------------PhysicalOlapScan[t2] -------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[t4] +--------------------PhysicalOlapScan[t4] Hint log: Used: leading(t3 alias1 ) @@ -402,23 +312,16 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2)) otherCondition=() -----------------PhysicalProject -------------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] ---------------------PhysicalDistribute[DistributionSpecHash] -----------------------PhysicalProject -------------------------PhysicalOlapScan[t4] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2)) otherCondition=() +------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t4] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: leading(alias2 t1 ) @@ -430,23 +333,16 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = alias2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() -----------------------PhysicalProject -------------------------PhysicalOlapScan[t4] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = alias2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() +----------------PhysicalOlapScan[t4] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t4 t2 ) @@ -458,113 +354,22 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2)) otherCondition=() -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() -----------------------PhysicalProject -------------------------PhysicalOlapScan[t2] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[t4] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2)) otherCondition=() +--------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() +----------------PhysicalOlapScan[t2] +----------------PhysicalDistribute[DistributionSpecHash] +------------------PhysicalOlapScan[t4] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(alias2 t1 ) leading(t3 alias1 ) UnUsed: SyntaxError: --- !sql4_5 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = alias2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() -----------------------PhysicalProject -------------------------PhysicalOlapScan[t2] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[t4] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: leading(t3 alias1) -SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause - --- !sql4_6 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() -----------------------PhysicalProject -------------------------PhysicalOlapScan[t2] -----------------------PhysicalDistribute[DistributionSpecHash] -------------------------PhysicalProject ---------------------------PhysicalOlapScan[t4] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] - -Hint log: -Used: -UnUsed: leading(alias2 t1) -SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause - --- !sql4_7 -- -PhysicalResultSink ---hashAgg[GLOBAL] -----PhysicalDistribute[DistributionSpecGather] -------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((alias1.c1 = alias2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t4.c4)) otherCondition=() -------------------------PhysicalProject ---------------------------PhysicalOlapScan[t2] -------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------PhysicalProject -----------------------------PhysicalOlapScan[t4] - -Hint log: -Used: leading(t3 alias1 ) -UnUsed: leading(alias2 t1) -SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause - -- !sql4_res_0 -- 6224 @@ -592,55 +397,45 @@ SyntaxError: leading(t4 t2) Msg:one query block can only have one leading clause -- !sql5_1 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------PhysicalOlapScan[t1] +----PhysicalOlapScan[t1] --PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] -------PhysicalProject ---------NestedLoopJoin[INNER_JOIN](cast(sum(c11) as DOUBLE) > (cast(0.05 as DOUBLE) * avg(c11))) -----------PhysicalProject -------------hashAgg[GLOBAL] ---------------PhysicalDistribute[DistributionSpecHash] -----------------hashAgg[LOCAL] -------------------PhysicalDistribute[DistributionSpecExecutionAny] +------NestedLoopJoin[INNER_JOIN](cast(sum(c11) as DOUBLE) > (cast(0.05 as DOUBLE) * avg(c11))) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------PhysicalDistribute[DistributionSpecReplicated] +----------hashAgg[GLOBAL] +------------PhysicalDistribute[DistributionSpecGather] +--------------hashAgg[LOCAL] +----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = cte.c11)) otherCondition=() +------------------PhysicalOlapScan[t1] +------------------PhysicalDistribute[DistributionSpecHash] --------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------PhysicalDistribute[DistributionSpecReplicated] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecGather] -------------------hashAgg[LOCAL] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = cte.c11)) otherCondition=() -------------------------PhysicalOlapScan[t1] -------------------------PhysicalDistribute[DistributionSpecHash] ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -- !sql5_2 -- PhysicalCteAnchor ( cteId=CTEId#0 ) --PhysicalCteProducer ( cteId=CTEId#0 ) -----PhysicalProject -------PhysicalOlapScan[t1] +----PhysicalOlapScan[t1] --PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] -------PhysicalProject ---------NestedLoopJoin[INNER_JOIN](cast(sum(c11) as DOUBLE) > (cast(0.05 as DOUBLE) * avg(c11))) -----------PhysicalProject -------------hashAgg[GLOBAL] ---------------PhysicalDistribute[DistributionSpecHash] -----------------hashAgg[LOCAL] +------NestedLoopJoin[INNER_JOIN](cast(sum(c11) as DOUBLE) > (cast(0.05 as DOUBLE) * avg(c11))) +--------hashAgg[GLOBAL] +----------PhysicalDistribute[DistributionSpecHash] +------------hashAgg[LOCAL] +--------------PhysicalDistribute[DistributionSpecExecutionAny] +----------------PhysicalCteConsumer ( cteId=CTEId#0 ) +--------PhysicalDistribute[DistributionSpecReplicated] +----------hashAgg[GLOBAL] +------------PhysicalDistribute[DistributionSpecGather] +--------------hashAgg[LOCAL] +----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = cte.c11)) otherCondition=() ------------------PhysicalDistribute[DistributionSpecExecutionAny] --------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -----------PhysicalDistribute[DistributionSpecReplicated] -------------PhysicalProject ---------------hashAgg[GLOBAL] -----------------PhysicalDistribute[DistributionSpecGather] -------------------hashAgg[LOCAL] ---------------------PhysicalProject -----------------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = cte.c11)) otherCondition=() -------------------------PhysicalDistribute[DistributionSpecExecutionAny] ---------------------------PhysicalCteConsumer ( cteId=CTEId#0 ) -------------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------------PhysicalOlapScan[t1] +------------------PhysicalDistribute[DistributionSpecReplicated] +--------------------PhysicalOlapScan[t1] Hint log: Used: leading(cte t1 ) diff --git a/regression-test/data/nereids_p0/hint/test_leading.out b/regression-test/data/nereids_p0/hint/test_leading.out index 31df4aafe732ba0..058fdf8b52fb0ae 100644 --- a/regression-test/data/nereids_p0/hint/test_leading.out +++ b/regression-test/data/nereids_p0/hint/test_leading.out @@ -2152,13 +2152,10 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +----------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecReplicated] +------------PhysicalOlapScan[t2] Hint log: Used: [broadcast]_2 @@ -2170,19 +2167,14 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: [broadcast]_2 [shuffle]_3 @@ -2194,19 +2186,14 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[RIGHT_OUTER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: [broadcast]_2 [shuffle]_3 @@ -2218,18 +2205,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecReplicated] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t1 shuffle t2 broadcast t3 ) @@ -2241,18 +2223,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +----------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t3] Hint log: Used: leading(t1 shuffle { t2 broadcast t3 } ) @@ -2264,18 +2241,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t3] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +----------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t3] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t1 shuffle { t3 broadcast t2 } ) @@ -2287,18 +2259,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t2] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t2] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecReplicated] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 shuffle t1 broadcast t3 ) @@ -2310,17 +2277,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 shuffle { t1 broadcast t3 } ) @@ -2332,17 +2295,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t3] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t3] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(t2 shuffle { t3 broadcast t1 } ) @@ -2354,18 +2313,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecReplicated] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t1 t2 broadcast t3 ) @@ -2377,18 +2331,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +----------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t3] Hint log: Used: leading(t1 { t2 broadcast t3 } ) @@ -2400,18 +2349,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t3] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +----------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t3] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t1 { t3 broadcast t2 } ) @@ -2423,18 +2367,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t2] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t2] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecReplicated] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 t1 broadcast t3 ) @@ -2446,17 +2385,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 { t1 broadcast t3 } ) @@ -2468,17 +2403,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t3] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t3] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(t2 { t3 broadcast t1 } ) @@ -2490,18 +2421,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t1 shuffle t2 t3 ) @@ -2513,18 +2439,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +----------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t3] Hint log: Used: leading(t1 shuffle { t2 t3 } ) @@ -2536,18 +2457,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t3] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +----------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t3] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t1 shuffle { t3 t2 } ) @@ -2559,18 +2475,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t2] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t2] ------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 shuffle t1 t3 ) @@ -2582,17 +2493,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 shuffle { t1 t3 } ) @@ -2604,17 +2511,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t3] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2) and (t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t3.c3)) otherCondition=() +--------------PhysicalOlapScan[t3] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(t2 shuffle { t3 t1 } ) @@ -2626,18 +2529,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] ------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecReplicated] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t1 broadcast t2 t3 ) @@ -2649,18 +2547,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t2] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t2] +------------PhysicalDistribute[DistributionSpecReplicated] +--------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 broadcast t1 t3 ) @@ -2672,18 +2565,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecReplicated] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t3 broadcast { t1 t2 } ) @@ -2695,18 +2583,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecReplicated] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(t3 broadcast { t2 t1 } ) @@ -2718,18 +2601,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t1] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t1] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecReplicated] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t1 shuffle t2 broadcast t3 ) @@ -2741,18 +2619,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t2] -----------------PhysicalDistribute[DistributionSpecHash] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t2] +------------PhysicalDistribute[DistributionSpecHash] +--------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecReplicated] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 shuffle t1 broadcast t3 ) @@ -2764,18 +2637,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t3 shuffle { t1 broadcast t2 } ) @@ -2787,18 +2655,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(t3 shuffle { t2 broadcast t1 } ) @@ -2810,19 +2673,14 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecReplicated] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------PhysicalDistribute[DistributionSpecHash] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecReplicated] +----------------PhysicalOlapScan[t2] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t1 broadcast t2 shuffle t3 ) @@ -2834,18 +2692,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -----------------PhysicalProject -------------------PhysicalOlapScan[t2] -----------------PhysicalDistribute[DistributionSpecReplicated] -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------PhysicalDistribute[DistributionSpecHash] ---------------PhysicalProject -----------------PhysicalOlapScan[t3] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +------------PhysicalOlapScan[t2] +------------PhysicalDistribute[DistributionSpecReplicated] +--------------PhysicalOlapScan[t1] +----------PhysicalDistribute[DistributionSpecHash] +------------PhysicalOlapScan[t3] Hint log: Used: leading(t2 broadcast t1 shuffle t3 ) @@ -2857,18 +2710,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t1] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t2] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecReplicated] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t1] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t2] Hint log: Used: leading(t3 broadcast { t1 shuffle t2 } ) @@ -2880,18 +2728,13 @@ PhysicalResultSink --hashAgg[GLOBAL] ----PhysicalDistribute[DistributionSpecGather] ------hashAgg[LOCAL] ---------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[t3] -------------PhysicalDistribute[DistributionSpecReplicated] ---------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() -------------------PhysicalProject ---------------------PhysicalOlapScan[t2] -------------------PhysicalDistribute[DistributionSpecHash] ---------------------PhysicalProject -----------------------PhysicalOlapScan[t1] +--------hashJoin[INNER_JOIN] hashCondition=((t2.c2 = t3.c3)) otherCondition=() +----------PhysicalOlapScan[t3] +----------PhysicalDistribute[DistributionSpecReplicated] +------------hashJoin[INNER_JOIN] hashCondition=((t1.c1 = t2.c2)) otherCondition=() +--------------PhysicalOlapScan[t2] +--------------PhysicalDistribute[DistributionSpecHash] +----------------PhysicalOlapScan[t1] Hint log: Used: leading(t3 broadcast { t2 shuffle t1 } ) diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index 4c88fb2ac30f62a..172b02f9e5a7143 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -27,6 +27,8 @@ suite("fix_leading") { sql 'set enable_nereids_planner=true' sql 'set enable_fallback_to_original_planner=false' sql 'set runtime_filter_mode=OFF' + sql "set ignore_shape_nodes='PhysicalProject'" + // create tables sql """drop table if exists t1;""" @@ -38,6 +40,8 @@ suite("fix_leading") { sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');""" sql """create table t3 (c3 int, c33 int) distributed by hash(c3) buckets 3 properties('replication_num' = '1');""" sql """create table t4 (c4 int, c44 int) distributed by hash(c4) buckets 3 properties('replication_num' = '1');""" + sql """create table t5 (c5 int, c55 int) distributed by hash(c5) buckets 3 properties('replication_num' = '1');""" + sql """create table t6 (c6 int, c66 int) distributed by hash(c6) buckets 3 properties('replication_num' = '1');""" streamLoad { table "t1" @@ -176,4 +180,8 @@ suite("fix_leading") { sql """shape plan select /*+ leading(t1 t2)*/ count(*) from t1 left join t2 on c1 > 500 and c2 >500 right join t3 on c3 > 500 and c1 < 200;""" contains("SyntaxError: leading(t1 t2) Msg:leading should have all tables in query block, missing tables: t3") } + + // check brace problem + qt_select6_1 """explain shape plan select /*+ leading(t1 {{t2 t3}{t4 t5}} t6) */ count(*) from t1 join t2 on c1 = c2 join t3 on c1 = c3 join t4 on c1 = c4 join t5 on c1 = c5 join t6 on c1 = c6;""" + } diff --git a/regression-test/suites/nereids_p0/hint/multi_leading.groovy b/regression-test/suites/nereids_p0/hint/multi_leading.groovy index f716b5c98d05e0c..b1d6b010de999d2 100644 --- a/regression-test/suites/nereids_p0/hint/multi_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/multi_leading.groovy @@ -24,6 +24,7 @@ suite("multi_leading") { // setting planner to nereids sql 'set enable_nereids_planner=true' + sql "set ignore_shape_nodes='PhysicalProject'" sql 'set enable_fallback_to_original_planner=false' sql 'set runtime_filter_mode=OFF' diff --git a/regression-test/suites/nereids_p0/hint/test_leading.groovy b/regression-test/suites/nereids_p0/hint/test_leading.groovy index d1e11144b527c79..3908f55504b2542 100644 --- a/regression-test/suites/nereids_p0/hint/test_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/test_leading.groovy @@ -25,6 +25,7 @@ suite("test_leading") { // setting planner to nereids sql 'set enable_nereids_planner=true' + sql "set ignore_shape_nodes='PhysicalProject'" sql 'set enable_fallback_to_original_planner=false' sql 'set runtime_filter_mode=OFF' From 5ac4ea2cd958e848fe4f2a9a5fd26e2070463c21 Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Mon, 20 May 2024 10:40:10 +0800 Subject: [PATCH 054/111] [Fix](Nereids) fix leading hint with update of alias name (#34434) (#35046) Problem: when using leading like leading(tbl1 tbl2) in "select * from (select tbl1.c1 from t1 as tbl1 join t2 as tbl2) join t3 as tbl2 on tbl2.c3 != 101;", in which tbl2.c3 means t3.c3 but not t2.c3 Causes and solved: when finding columns in condition, leading hint would find tbl2.c3's RelationId, and when we collect RelationId and aliasName we should update it if aliasName is repeat --- .../doris/nereids/hint/LeadingHint.java | 4 ++ .../data/nereids_hint_tpch_p0/shape/q15.out | 33 ++++++++-------- .../nereids_hint_tpch_p0/shape/q15.groovy | 2 +- .../suites/nereids_p0/hint/fix_leading.groovy | 38 +++++++++++++++++++ 4 files changed, 60 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java index 4d58990fbc09a19..d21ee19df6d6b49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/LeadingHint.java @@ -216,6 +216,10 @@ public void putRelationIdAndTableName(Pair relationIdTableNa pair.second = relationIdTableNamePair.second; isUpdate = true; } + if (pair.second.equals(relationIdTableNamePair.second)) { + pair.first = relationIdTableNamePair.first; + isUpdate = true; + } } if (!isUpdate) { relationIdAndTableName.add(relationIdTableNamePair); diff --git a/regression-test/data/nereids_hint_tpch_p0/shape/q15.out b/regression-test/data/nereids_hint_tpch_p0/shape/q15.out index c059df5018f3420..bb2463ee3f36840 100644 --- a/regression-test/data/nereids_hint_tpch_p0/shape/q15.out +++ b/regression-test/data/nereids_hint_tpch_p0/shape/q15.out @@ -5,23 +5,24 @@ PhysicalResultSink ----PhysicalDistribute[DistributionSpecGather] ------PhysicalQuickSort[LOCAL_SORT] --------PhysicalProject -----------hashJoin[INNER_JOIN] hashCondition=((supplier.s_suppkey = revenue0.supplier_no)) otherCondition=() -------------PhysicalProject ---------------PhysicalOlapScan[supplier] +----------hashJoin[INNER_JOIN] hashCondition=((revenue0.total_revenue = max(total_revenue))) otherCondition=() ------------PhysicalDistribute[DistributionSpecHash] --------------PhysicalProject -----------------hashJoin[INNER_JOIN] hashCondition=((revenue0.total_revenue = max(total_revenue))) otherCondition=() -------------------hashAgg[GLOBAL] ---------------------PhysicalDistribute[DistributionSpecGather] -----------------------hashAgg[LOCAL] -------------------------PhysicalProject ---------------------------hashAgg[GLOBAL] -----------------------------PhysicalDistribute[DistributionSpecHash] -------------------------------hashAgg[LOCAL] ---------------------------------PhysicalProject -----------------------------------filter((lineitem.l_shipdate < '1996-04-01') and (lineitem.l_shipdate >= '1996-01-01')) -------------------------------------PhysicalOlapScan[lineitem] -------------------PhysicalDistribute[DistributionSpecReplicated] +----------------hashJoin[INNER_JOIN] hashCondition=((supplier.s_suppkey = revenue0.supplier_no)) otherCondition=() +------------------PhysicalProject +--------------------PhysicalOlapScan[supplier] +------------------PhysicalDistribute[DistributionSpecHash] +--------------------PhysicalProject +----------------------hashAgg[GLOBAL] +------------------------PhysicalDistribute[DistributionSpecHash] +--------------------------hashAgg[LOCAL] +----------------------------PhysicalProject +------------------------------filter((lineitem.l_shipdate < '1996-04-01') and (lineitem.l_shipdate >= '1996-01-01')) +--------------------------------PhysicalOlapScan[lineitem] +------------PhysicalDistribute[DistributionSpecHash] +--------------hashAgg[GLOBAL] +----------------PhysicalDistribute[DistributionSpecGather] +------------------hashAgg[LOCAL] --------------------PhysicalProject ----------------------hashAgg[GLOBAL] ------------------------PhysicalDistribute[DistributionSpecHash] @@ -31,7 +32,7 @@ PhysicalResultSink --------------------------------PhysicalOlapScan[lineitem] Hint log: -Used: leading(revenue0 supplier ) +Used: leading(supplier revenue0 ) UnUsed: SyntaxError: diff --git a/regression-test/suites/nereids_hint_tpch_p0/shape/q15.groovy b/regression-test/suites/nereids_hint_tpch_p0/shape/q15.groovy index 23cf07c71e162ab..7fed5551352ebbf 100644 --- a/regression-test/suites/nereids_hint_tpch_p0/shape/q15.groovy +++ b/regression-test/suites/nereids_hint_tpch_p0/shape/q15.groovy @@ -37,7 +37,7 @@ suite("q15") { qt_select """ explain shape plan select - /*+ leading(revenue0 supplier) */ + /*+ leading(supplier revenue0) */ s_suppkey, s_name, s_address, diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index 172b02f9e5a7143..4681fa16e062509 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -184,4 +184,42 @@ suite("fix_leading") { // check brace problem qt_select6_1 """explain shape plan select /*+ leading(t1 {{t2 t3}{t4 t5}} t6) */ count(*) from t1 join t2 on c1 = c2 join t3 on c1 = c3 join t4 on c1 = c4 join t5 on c1 = c5 join t6 on c1 = c6;""" + // check filter in duplicated aliasName + explain { + sql """shape plan SELECT + t1.c2 AS c4 + FROM + ( + SELECT + /*+ leading( { tbl2 tbl3 } tbl1 ) */ + tbl3.c3 AS c4, + 4 AS c2 + FROM + t1 AS tbl1 + INNER JOIN t2 AS tbl2 ON tbl2.c2 >= tbl1.c1 + OR tbl2.c2 < (5 * 1) + INNER JOIN t3 AS tbl3 ON tbl2.c2 >= tbl2.c2 + WHERE + ( + tbl1.c1 <> tbl3.c3 + ) + ORDER BY + 2, + 4, + 1, + 3 ASC + LIMIT + 5 OFFSET 10 + ) AS t1 + INNER JOIN t4 AS tbl2 ON tbl2.c4 != (7 * 1) + WHERE + NOT ( + t1.c2 > tbl2.c4 + ) + ORDER BY + 1 DESC + LIMIT + 5;""" + contains("Used: leading({ tbl2 tbl3 } tbl1 )") + } } From be50139eb14fcea2a87184c7ffecd7b9b3ea50e6 Mon Sep 17 00:00:00 2001 From: LiBinfeng <46676950+LiBinfeng-01@users.noreply.github.com> Date: Mon, 20 May 2024 10:44:22 +0800 Subject: [PATCH 055/111] [Fix](Nereids) fix leading with cte and same subqueryalias name (#34838) (#35047) fix leading with cte and same subqueryalias name Example: with tbl1 as select t1.c1 from t1 select tbl2.c2 from (select / * + leading(t2 tbl1) * / tbl1.c1, t2.c2 from tbl1 join t2) as tbl2 join t3; Reason: in this case, before getting analyzed preprocess would change subquery tbl2 to cte plan, and this cte plan should be in upper level cte plan, but not in logical result sink plan --- .../pre/PullUpSubqueryAliasToCTE.java | 11 +- .../data/nereids_p0/hint/multi_leading.out | 2 +- regression-test/data/nereids_p0/hint/t5.csv | 1000 +++++++++++++++++ regression-test/data/nereids_p0/hint/t6.csv | 1000 +++++++++++++++++ .../suites/nereids_p0/hint/fix_leading.groovy | 45 + 5 files changed, 2056 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/nereids_p0/hint/t5.csv create mode 100644 regression-test/data/nereids_p0/hint/t6.csv diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java index 0101a346478bf89..8e8889f5e62df25 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/PullUpSubqueryAliasToCTE.java @@ -79,6 +79,15 @@ public Plan visitLogicalCTE(LogicalCTE logicalCTE, StatementCont subQueryAlias = new LogicalSubQueryAlias<>(subQueryAlias.getAlias(), newSubQueryAlias); } } - return visitChildren(this, logicalCTE, context); + Plan cte = visitChildren(this, logicalCTE, context); + if (!aliasQueries.isEmpty()) { + LogicalCTE newLogicalCTE = (LogicalCTE) cte; + List> subQueryAliasesOfCte = new ArrayList<>(); + subQueryAliasesOfCte.addAll(logicalCTE.getAliasQueries()); + subQueryAliasesOfCte.addAll(aliasQueries); + aliasQueries = new ArrayList<>(); + return new LogicalCTE<>(subQueryAliasesOfCte, (LogicalPlan) newLogicalCTE.child()); + } + return cte; } } diff --git a/regression-test/data/nereids_p0/hint/multi_leading.out b/regression-test/data/nereids_p0/hint/multi_leading.out index 4f7056de310d1b4..241522a192bd545 100644 --- a/regression-test/data/nereids_p0/hint/multi_leading.out +++ b/regression-test/data/nereids_p0/hint/multi_leading.out @@ -254,7 +254,7 @@ PhysicalResultSink ----------------PhysicalOlapScan[t1] Hint log: -Used: leading(t2 t1 ) leading(t2 t1 ) leading(t2 t1 ) leading(t3 alias1 cte ) +Used: leading(t2 t1 ) leading(t2 t1 ) leading(t3 alias1 cte ) UnUsed: SyntaxError: diff --git a/regression-test/data/nereids_p0/hint/t5.csv b/regression-test/data/nereids_p0/hint/t5.csv new file mode 100644 index 000000000000000..cfae19cec058c91 --- /dev/null +++ b/regression-test/data/nereids_p0/hint/t5.csv @@ -0,0 +1,1000 @@ +379|311 +347|\N +319|233 +286|355 +30|154 +38|418 +270|294 +\N|162 +241|248 +299|102 +87|154 +482|337 +385|165 +291|308 +49|153 +394|392 +250|249 +487|186 +35|\N +309|167 +427|157 +178|428 +436|112 +412|56 +144|218 +33|488 +48|142 +455|403 +211|232 +355|185 +167|294 +238|116 +191|86 +160|399 +288|321 +494|442 +7|220 +291|486 +496|321 +292|351 +231|214 +392|266 +299|324 +212|176 +17|34 +242|341 +452|27 +\N|155 +363|258 +363|268 +221|\N +361|358 +21|251 +209|232 +439|209 +25|423 +283|271 +364|104 +105|19 +212|115 +224|250 +342|376 +261|27 +19|88 +213|372 +268|426 +102|466 +126|97 +305|312 +476|355 +409|318 +215|375 +234|318 +353|451 +313|11 +296|315 +393|244 +285|46 +289|209 +420|264 +346|482 +317|258 +437|72 +48|468 +88|22 +498|102 +496|85 +456|\N +389|419 +22|144 +473|313 +110|399 +87|240 +235|178 +265|182 +\N|169 +69|69 +222|334 +495|211 +16|95 +98|453 +191|18 +206|248 +433|202 +445|138 +498|391 +65|494 +305|278 +436|219 +\N|217 +150|206 +467|439 +56|381 +174|231 +64|57 +33|228 +241|290 +215|120 +230|228 +306|323 +108|287 +310|70 +372|171 +400|167 +74|32 +468|\N +258|15 +167|164 +237|256 +273|28 +167|333 +289|50 +73|461 +16|56 +332|181 +133|353 +302|265 +154|35 +500|432 +223|456 +327|434 +306|387 +473|144 +93|300 +106|121 +52|152 +108|456 +277|55 +445|451 +244|253 +123|257 +425|473 +131|341 +370|161 +330|78 +321|306 +127|66 +429|130 +\N|34 +4|260 +106|166 +298|407 +366|55 +451|\N +495|384 +178|59 +52|220 +182|165 +336|469 +56|147 +36|151 +17|475 +451|469 +189|404 +124|487 +3|165 +305|392 +43|375 +12|321 +25|\N +371|227 +283|457 +413|125 +\N|251 +101|358 +\N|340 +198|463 +78|407 +168|378 +361|357 +135|37 +38|333 +256|122 +352|37 +21|261 +270|149 +6|466 +488|\N +249|388 +60|297 +\N|399 +466|303 +198|344 +415|314 +500|243 +385|439 +159|286 +130|244 +457|370 +157|206 +249|442 +302|250 +76|475 +112|11 +\N|33 +467|327 +285|229 +365|486 +171|481 +285|82 +225|329 +11|380 +211|328 +387|109 +473|221 +435|297 +352|454 +340|292 +73|346 +183|125 +397|458 +454|369 +227|326 +68|1 +336|63 +\N|40 +94|478 +118|237 +109|161 +370|395 +33|253 +238|285 +406|40 +207|247 +4|453 +346|1 +177|286 +355|48 +348|462 +308|334 +46|6 +220|237 +67|293 +187|117 +178|53 +459|101 +10|31 +444|55 +408|272 +340|34 +14|461 +121|36 +255|494 +497|345 +142|227 +90|423 +101|2 +451|281 +335|230 +166|256 +\N|\N +53|460 +65|429 +417|212 +330|472 +170|215 +216|206 +135|455 +401|48 +99|294 +\N|396 +283|370 +27|184 +472|255 +145|471 +183|335 +\N|140 +284|131 +\N|313 +125|326 +117|31 +347|191 +329|81 +25|431 +494|142 +37|\N +\N|\N +485|226 +264|217 +439|208 +457|243 +9|105 +202|408 +102|349 +446|286 +32|\N +416|13 +437|138 +181|189 +24|166 +448|240 +120|89 +\N|262 +\N|123 +305|120 +\N|88 +476|247 +360|5 +476|162 +305|401 +135|264 +435|376 +64|402 +187|198 +377|240 +\N|146 +430|87 +157|50 +282|397 +\N|57 +140|333 +312|196 +113|355 +10|385 +57|73 +175|\N +421|89 +186|149 +384|184 +363|406 +189|348 +436|\N +374|275 +123|276 +85|465 +\N|89 +416|328 +30|252 +254|\N +243|359 +197|382 +283|273 +30|198 +276|433 +300|132 +446|368 +205|493 +472|229 +\N|443 +207|475 +136|154 +138|374 +71|27 +65|261 +337|116 +290|499 +326|321 +323|226 +286|201 +65|\N +478|56 +\N|403 +313|36 +324|411 +123|131 +148|429 +400|373 +371|125 +476|452 +19|491 +105|7 +457|93 +113|473 +243|489 +130|23 +43|418 +466|422 +485|270 +88|419 +\N|366 +454|205 +384|473 +26|104 +280|45 +59|144 +225|154 +\N|420 +362|4 +268|315 +\N|306 +119|41 +88|392 +437|73 +208|38 +\N|396 +137|121 +248|485 +203|76 +121|72 +365|306 +406|62 +14|134 +418|152 +233|224 +286|342 +408|405 +387|306 +313|468 +459|6 +89|120 +452|375 +4|117 +398|150 +91|80 +401|360 +396|148 +39|222 +137|485 +77|361 +480|\N +121|423 +90|147 +3|299 +462|81 +262|457 +417|218 +\N|493 +64|372 +91|332 +69|\N +\N|70 +136|422 +51|465 +341|342 +485|104 +419|123 +377|326 +\N|196 +468|232 +298|210 +319|412 +258|112 +312|481 +9|332 +222|475 +263|18 +191|178 +336|246 +323|158 +133|165 +158|40 +257|352 +7|47 +245|234 +100|405 +19|263 +189|85 +309|244 +113|404 +92|208 +171|102 +425|305 +\N|48 +455|398 +449|96 +\N|260 +376|121 +85|439 +84|255 +141|491 +3|483 +284|168 +289|81 +45|106 +61|123 +304|210 +304|139 +393|106 +266|410 +364|23 +\N|274 +\N|88 +208|43 +239|277 +455|\N +494|102 +207|61 +137|286 +88|451 +486|317 +259|432 +387|165 +126|263 +272|395 +25|82 +425|0 +140|256 +108|101 +115|379 +259|92 +243|\N +115|460 +97|197 +\N|203 +199|1 +275|275 +35|432 +220|94 +99|96 +120|85 +44|156 +373|95 +351|425 +475|410 +219|\N +114|150 +51|\N +234|376 +266|293 +280|132 +132|122 +450|475 +232|493 +278|110 +301|273 +147|136 +279|50 +311|179 +430|393 +14|82 +205|133 +198|343 +139|145 +470|142 +5|191 +429|351 +54|\N +485|24 +203|\N +359|146 +35|288 +126|234 +47|392 +170|487 +361|125 +322|335 +54|345 +453|9 +268|64 +117|402 +467|241 +\N|203 +431|490 +332|258 +322|\N +123|76 +313|200 +354|272 +\N|231 +102|444 +266|\N +226|463 +66|377 +194|\N +310|362 +209|314 +239|346 +317|130 +112|444 +12|254 +350|172 +369|402 +482|398 +427|171 +65|409 +304|135 +254|416 +79|359 +204|286 +\N|87 +175|159 +272|306 +265|441 +54|40 +336|311 +173|97 +84|220 +184|383 +196|192 +27|98 +162|4 +104|181 +153|320 +153|\N +106|329 +34|70 +228|227 +284|144 +453|284 +287|199 +82|\N +53|104 +436|340 +344|256 +410|215 +145|373 +95|427 +71|241 +101|436 +69|203 +389|231 +290|134 +279|103 +7|256 +405|132 +498|371 +\N|3 +438|101 +15|17 +338|297 +299|140 +62|\N +188|\N +138|55 +445|135 +425|26 +\N|398 +341|331 +160|313 +\N|76 +355|228 +163|5 +111|217 +62|151 +460|495 +411|437 +200|421 +163|\N +440|426 +151|281 +493|352 +352|103 +445|465 +176|380 +\N|198 +195|219 +468|\N +380|439 +61|307 +403|388 +\N|359 +69|410 +354|456 +491|80 +310|374 +162|426 +277|81 +176|463 +226|469 +446|79 +477|26 +86|231 +379|456 +414|251 +373|114 +499|416 +490|104 +65|237 +\N|493 +450|189 +1|432 +33|178 +160|158 +480|362 +135|420 +73|286 +149|402 +244|472 +305|181 +227|98 +412|209 +59|255 +49|198 +105|483 +445|\N +59|24 +237|178 +453|37 +54|111 +465|456 +94|395 +406|420 +413|38 +85|436 +61|57 +15|437 +95|\N +460|155 +471|230 +482|499 +302|389 +\N|456 +168|157 +284|386 +385|211 +461|490 +65|229 +335|128 +415|420 +374|12 +315|90 +103|226 +20|277 +101|233 +237|180 +101|399 +268|242 +164|249 +373|4 +81|141 +325|221 +\N|\N +438|302 +206|339 +30|90 +\N|\N +30|149 +49|183 +13|83 +83|27 +411|\N +91|495 +260|239 +435|329 +330|457 +318|284 +298|327 +230|50 +68|139 +55|120 +40|117 +185|\N +351|483 +242|199 +45|243 +341|491 +38|473 +289|474 +301|486 +59|272 +477|157 +108|379 +\N|232 +223|9 +388|187 +\N|211 +436|47 +500|410 +361|383 +228|455 +26|13 +\N|459 +396|153 +348|387 +420|39 +34|402 +183|249 +323|80 +36|147 +114|246 +457|383 +262|425 +253|460 +284|287 +48|486 +452|478 +327|362 +253|475 +190|217 +124|11 +67|485 +451|27 +478|256 +331|371 +88|451 +247|177 +479|221 +405|169 +85|315 +292|232 +100|44 +131|302 +\N|21 +204|284 +104|411 +482|444 +248|233 +425|159 +297|414 +18|385 +112|454 +22|164 +210|314 +267|116 +263|446 +171|294 +186|423 +181|339 +303|277 +51|485 +290|67 +\N|127 +63|143 +387|\N +\N|211 +286|69 +14|499 +100|430 +179|441 +473|22 +74|271 +106|294 +339|172 +58|423 +193|219 +267|59 +126|178 +356|185 +121|130 +413|103 +370|496 +364|196 +178|5 +124|437 +446|127 +151|313 +451|468 +419|106 +99|291 +\N|439 +327|493 +386|146 +\N|163 +\N|94 +327|278 +66|386 +\N|492 +461|146 +61|164 +273|146 +398|479 +499|65 +68|20 +163|65 +\N|58 +180|454 +401|5 +109|312 +268|475 +297|362 +137|178 +59|297 +74|\N +250|38 +137|244 +330|355 +456|62 +\N|292 +419|432 +166|214 +176|72 +167|459 +469|308 +425|299 +134|371 +98|336 +228|387 +409|119 +205|313 +308|317 +443|125 +259|432 +\N|95 +143|395 +224|416 +217|108 +29|386 +224|30 +378|43 +83|343 +43|202 +159|376 +495|425 +395|205 +459|161 +402|500 +304|84 +362|392 +176|187 +443|459 +226|185 +21|445 +500|\N +299|74 +0|208 +18|72 +483|320 +436|181 +338|47 +25|301 +461|54 +420|325 +219|148 +271|320 +378|45 +131|\N +97|142 +300|147 +165|228 +173|322 +337|247 +11|214 +325|199 +205|195 +\N|412 +211|36 +91|85 +281|97 +212|246 +258|354 +185|208 +11|310 +493|2 +238|327 +\N|480 +281|193 +442|373 +342|50 +357|171 +151|255 +477|273 +\N|251 +492|424 +185|325 +450|401 +241|435 +260|280 +500|122 +413|51 +76|360 +163|69 +186|200 +232|15 +259|179 +199|59 +261|82 +210|257 +453|275 +116|48 +464|382 +\N|485 +11|110 +302|324 +14|54 +85|1 +163|83 +\N|238 +356|244 +465|55 +329|455 +162|471 +316|459 +223|363 +258|321 +495|472 +373|56 +38|471 +36|430 +161|88 +459|258 +418|195 +105|319 +91|407 +365|55 +86|433 +240|467 +363|163 +343|68 +233|297 +484|282 diff --git a/regression-test/data/nereids_p0/hint/t6.csv b/regression-test/data/nereids_p0/hint/t6.csv new file mode 100644 index 000000000000000..a473205388be190 --- /dev/null +++ b/regression-test/data/nereids_p0/hint/t6.csv @@ -0,0 +1,1000 @@ +141|422 +310|497 +10|317 +427|312 +286|263 +180|19 +265|224 +436|342 +455|379 +285|357 +490|79 +222|239 +124|22 +270|\N +190|266 +207|78 +380|63 +\N|9 +191|150 +6|385 +236|\N +8|182 +454|285 +141|160 +267|341 +215|322 +337|423 +488|105 +171|137 +120|314 +342|33 +295|351 +173|212 +80|2 +336|248 +383|423 +\N|122 +484|444 +\N|217 +134|10 +61|225 +362|115 +\N|\N +96|129 +404|194 +446|310 +127|299 +307|178 +477|428 +285|109 +434|292 +366|71 +384|469 +455|82 +220|423 +442|402 +355|474 +468|427 +435|90 +91|277 +61|463 +318|417 +\N|144 +328|150 +448|253 +69|258 +0|494 +93|195 +4|71 +16|374 +417|348 +463|\N +487|228 +479|399 +333|183 +109|17 +\N|343 +403|407 +141|136 +476|253 +396|287 +141|83 +363|181 +13|361 +\N|124 +235|49 +479|63 +316|428 +306|376 +293|364 +230|326 +237|361 +462|236 +139|245 +401|135 +\N|99 +13|194 +122|297 +81|332 +465|\N +51|367 +102|\N +221|\N +110|374 +97|198 +39|224 +330|365 +421|335 +72|164 +366|375 +360|375 +19|429 +282|179 +235|84 +465|198 +144|448 +335|108 +412|216 +267|328 +390|197 +388|266 +305|30 +427|131 +104|\N +284|349 +48|268 +404|81 +483|304 +371|62 +\N|345 +435|321 +137|26 +252|173 +389|394 +346|193 +37|140 +97|67 +175|343 +304|398 +219|471 +231|128 +\N|387 +167|411 +181|223 +166|240 +480|286 +293|365 +247|319 +267|363 +372|65 +239|32 +197|95 +24|\N +410|131 +180|196 +385|474 +175|272 +35|322 +170|46 +410|399 +168|456 +9|384 +262|493 +468|194 +141|227 +420|384 +95|183 +201|18 +320|456 +212|424 +69|280 +365|121 +25|\N +162|36 +479|223 +0|102 +101|277 +68|254 +343|150 +241|245 +367|360 +143|329 +241|157 +297|69 +369|281 +293|374 +450|170 +236|117 +226|327 +2|390 +101|278 +138|74 +91|311 +286|86 +59|218 +308|269 +203|370 +\N|446 +219|139 +212|175 +254|243 +135|104 +384|413 +231|234 +49|458 +300|183 +4|491 +177|105 +71|362 +154|188 +10|321 +266|\N +5|270 +160|202 +249|216 +472|21 +457|362 +481|291 +192|122 +203|353 +176|492 +\N|24 +419|114 +21|376 +300|185 +154|500 +12|64 +197|251 +75|367 +116|335 +47|278 +160|183 +52|403 +138|300 +343|301 +71|226 +32|485 +134|\N +449|236 +94|193 +240|414 +99|88 +138|372 +70|219 +6|348 +486|466 +99|27 +274|\N +394|500 +319|139 +401|242 +67|160 +4|93 +376|200 +471|447 +480|274 +\N|163 +401|469 +115|156 +26|286 +20|370 +167|410 +189|\N +69|411 +67|339 +193|66 +381|296 +159|341 +123|153 +\N|373 +123|190 +195|212 +20|354 +498|61 +164|154 +413|449 +34|246 +329|236 +\N|399 +287|299 +324|208 +138|32 +213|299 +256|264 +339|6 +95|\N +434|\N +335|331 +316|85 +355|436 +450|123 +345|343 +220|343 +165|160 +426|\N +155|385 +353|286 +398|294 +8|103 +326|384 +494|398 +247|120 +\N|435 +328|20 +436|156 +111|363 +91|456 +111|217 +24|470 +62|240 +147|\N +190|307 +384|364 +496|105 +286|225 +96|352 +452|258 +155|258 +288|86 +419|185 +366|45 +351|388 +68|431 +337|421 +381|408 +370|274 +279|98 +303|323 +77|90 +248|\N +234|20 +472|494 +186|257 +34|184 +40|321 +53|160 +368|123 +\N|164 +224|346 +80|474 +121|47 +483|68 +48|77 +42|334 +2|192 +368|231 +288|314 +290|168 +453|382 +239|324 +29|157 +57|21 +39|37 +201|208 +478|444 +270|59 +201|195 +435|425 +498|85 +169|224 +80|370 +251|224 +58|\N +463|89 +368|433 +60|128 +452|407 +19|239 +373|211 +85|468 +339|444 +141|133 +458|172 +218|486 +374|246 +14|246 +108|378 +181|39 +124|450 +272|209 +265|370 +324|61 +172|\N +\N|285 +338|118 +77|315 +101|281 +345|476 +95|172 +32|150 +91|475 +447|327 +447|29 +19|54 +143|\N +362|142 +181|479 +219|11 +451|127 +244|275 +493|155 +490|435 +254|30 +480|159 +45|97 +299|494 +90|141 +201|340 +338|71 +166|325 +429|499 +160|424 +497|412 +267|362 +139|69 +165|67 +393|332 +101|311 +415|196 +156|376 +118|259 +140|307 +316|361 +37|196 +402|285 +28|372 +139|321 +206|159 +308|161 +453|95 +189|166 +132|82 +367|413 +266|219 +359|177 +\N|248 +82|428 +230|97 +246|418 +138|\N +38|447 +91|292 +2|33 +296|228 +181|189 +\N|28 +138|283 +35|39 +84|113 +416|248 +183|\N +444|371 +290|278 +465|173 +31|268 +383|456 +474|21 +410|423 +192|468 +306|202 +62|263 +282|283 +472|43 +23|431 +154|339 +201|156 +309|163 +180|311 +1|276 +478|382 +217|441 +\N|409 +13|267 +471|464 +282|390 +264|441 +28|269 +270|174 +217|46 +232|23 +272|164 +21|299 +460|69 +177|387 +406|269 +32|157 +175|238 +303|21 +408|122 +357|441 +329|162 +356|205 +84|108 +1|375 +131|301 +354|433 +94|283 +427|141 +389|370 +463|145 +454|446 +372|\N +278|284 +316|63 +160|261 +399|275 +187|241 +480|266 +57|181 +324|266 +147|36 +228|306 +52|188 +120|52 +468|310 +68|458 +394|134 +\N|325 +144|217 +41|215 +73|226 +399|374 +278|77 +288|337 +451|64 +381|495 +265|348 +385|360 +111|479 +336|89 +179|472 +163|211 +106|465 +495|10 +294|455 +215|428 +209|131 +161|297 +437|292 +244|119 +11|442 +242|498 +151|0 +491|112 +302|262 +17|442 +337|499 +353|69 +339|52 +165|290 +15|494 +447|69 +343|141 +250|45 +309|164 +148|377 +226|469 +138|426 +66|260 +115|458 +272|17 +61|233 +62|500 +361|252 +181|143 +283|387 +333|460 +190|480 +103|89 +87|30 +80|104 +320|456 +259|\N +40|458 +403|327 +461|216 +168|394 +159|358 +401|138 +133|101 +308|353 +94|120 +30|25 +408|54 +120|58 +469|202 +276|217 +187|319 +11|22 +199|481 +\N|83 +118|187 +178|18 +259|230 +86|374 +83|253 +151|293 +427|63 +100|416 +428|63 +13|491 +197|53 +250|104 +324|328 +94|31 +132|221 +3|235 +463|76 +88|199 +113|104 +377|213 +\N|175 +2|438 +213|434 +44|358 +77|480 +423|128 +108|108 +303|160 +25|321 +321|466 +300|170 +475|359 +40|330 +65|10 +239|235 +108|389 +73|149 +379|123 +25|11 +56|132 +279|45 +\N|218 +422|498 +348|285 +320|329 +350|433 +479|362 +39|7 +\N|481 +342|122 +477|166 +163|444 +408|362 +214|128 +225|161 +428|\N +123|234 +174|457 +\N|250 +171|266 +499|294 +101|\N +248|88 +\N|302 +196|313 +398|204 +146|32 +460|\N +8|97 +255|213 +199|379 +250|36 +495|460 +88|273 +329|75 +179|198 +147|386 +216|155 +\N|200 +259|88 +155|3 +103|394 +46|\N +331|85 +155|144 +54|461 +93|216 +\N|413 +\N|335 +452|355 +449|398 +206|67 +225|411 +370|70 +91|107 +421|367 +306|36 +327|161 +375|20 +405|197 +482|308 +243|209 +36|290 +349|35 +309|425 +47|202 +399|249 +\N|5 +138|164 +456|464 +267|192 +398|73 +\N|418 +\N|474 +162|363 +300|38 +\N|37 +230|20 +469|187 +140|319 +\N|198 +230|456 +446|464 +\N|394 +426|399 +192|296 +386|345 +158|65 +436|263 +256|492 +224|410 +239|43 +187|57 +457|448 +13|288 +49|366 +234|435 +455|20 +306|34 +60|7 +210|485 +\N|399 +101|356 +96|410 +\N|455 +249|396 +448|186 +130|310 +168|489 +222|480 +104|146 +349|431 +379|336 +233|409 +53|415 +181|236 +120|239 +437|460 +117|352 +399|316 +224|159 +408|310 +463|140 +251|427 +239|213 +255|\N +380|\N +325|268 +\N|411 +204|\N +230|373 +64|484 +226|371 +404|464 +53|81 +188|479 +216|254 +17|172 +329|233 +75|377 +114|233 +124|\N +411|244 +219|173 +254|168 +96|\N +325|84 +321|109 +74|14 +311|168 +81|221 +489|466 +194|406 +297|234 +291|498 +442|408 +94|238 +183|193 +189|3 +352|434 +93|240 +475|\N +88|15 +407|\N +318|199 +216|9 +48|462 +450|167 +159|31 +100|313 +246|100 +\N|312 +290|457 +419|465 +478|348 +128|205 +130|238 +123|114 +351|391 +342|195 +67|148 +224|180 +106|432 +\N|29 +279|342 +93|94 +302|347 +67|213 +207|326 +197|90 +120|257 +163|\N +208|489 +386|287 +362|142 +394|455 +383|361 +50|54 +87|452 +72|340 +60|458 +219|353 +280|331 +222|405 +16|283 +\N|496 +256|475 +170|397 +188|386 +164|225 +178|238 +440|141 +424|184 +173|393 +339|205 +498|\N +229|281 +302|417 +42|223 +72|183 +155|256 +297|390 +241|107 +195|208 +128|411 +400|396 +319|69 +180|479 +429|99 +269|96 +460|228 +261|342 +454|247 +365|\N +496|437 +362|323 +485|433 +469|139 +443|320 +300|204 +219|447 +322|359 +183|320 +251|113 +31|388 +364|45 +171|68 +436|175 +72|385 +147|258 +6|389 +316|87 +39|381 +143|155 +387|17 +342|229 +48|134 +164|35 +460|442 +493|267 +395|400 +100|244 +292|5 +400|296 +144|326 +493|111 +391|500 +44|50 +230|361 +168|56 +210|\N +156|354 +457|470 +137|405 +238|119 +71|198 +54|29 +498|348 +29|201 +104|477 +378|121 +382|370 +498|160 +256|393 +165|22 +66|20 +57|385 +314|90 +150|63 +16|195 +445|220 +340|213 +145|29 +404|88 +94|235 +253|11 +199|464 +\N|318 +167|75 +127|49 +186|136 +274|198 +494|465 +96|347 +191|\N +431|421 +\N|113 +260|\N +116|259 +220|216 +113|455 +453|146 +2|444 +302|34 +\N|53 +302|37 +318|434 +73|485 +444|60 +287|301 +117|92 +167|323 +476|393 +21|229 +307|140 +130|440 +88|220 +361|47 +297|147 +307|290 +67|251 +8|311 +248|137 +79|463 +188|231 +445|62 +430|223 +\N|78 +485|365 +9|149 +304|302 +266|166 +\N|235 +346|8 +309|202 +475|420 +493|237 +211|\N +479|250 +425|\N +95|57 +3|242 +420|277 +\N|28 +132|194 +447|411 +480|210 +1|239 +378|101 +338|92 +491|250 +408|91 +31|153 +422|308 +283|364 +489|243 +167|493 +37|81 +491|285 diff --git a/regression-test/suites/nereids_p0/hint/fix_leading.groovy b/regression-test/suites/nereids_p0/hint/fix_leading.groovy index 4681fa16e062509..ae17b17c53a2379 100644 --- a/regression-test/suites/nereids_p0/hint/fix_leading.groovy +++ b/regression-test/suites/nereids_p0/hint/fix_leading.groovy @@ -79,6 +79,24 @@ suite("fix_leading") { time 10000 } + streamLoad { + table "t5" + db "fix_leading" + set 'column_separator', '|' + set 'format', 'csv' + file 't5.csv' + time 10000 + } + + streamLoad { + table "t6" + db "fix_leading" + set 'column_separator', '|' + set 'format', 'csv' + file 't6.csv' + time 10000 + } + // bug fix 1: {t1 t2}{t3 t4} miss levels qt_select1 """explain shape plan select /*+ leading({t1 t2}{t3 t4}) */ * from t1 join t2 on c2 = c2 join t3 on c1 = c3 join t4 on c1 = c4;""" @@ -222,4 +240,31 @@ suite("fix_leading") { 5;""" contains("Used: leading({ tbl2 tbl3 } tbl1 )") } + + // check cte as input in alias leading query + explain { + sql """shape plan WITH tbl1 AS ( + SELECT + tbl1.c1 AS c111, + tbl2.c2 as c222 + FROM + t1 AS tbl1 + RIGHT JOIN t2 AS tbl2 ON tbl1.c1 = tbl2.c2 + ) + SELECT + tbl3.c3, + tbl2.c2 + FROM + ( + SELECT + /*+ leading( tbl2 tbl1 ) */ + tbl1.c111 AS c1, + tbl2.c2 AS c2 + FROM + t2 AS tbl2 + JOIN tbl1 ON tbl2.c2 = tbl1.c111 + ) AS tbl2 + RIGHT JOIN t3 AS tbl3 ON tbl2.c2 = tbl3.c3;""" + contains("Used: leading(tbl2 tbl1 )") + } } From a43c6eca2279f8d3f65a5bcbdb01ea7a86ba7775 Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Mon, 20 May 2024 13:28:26 +0800 Subject: [PATCH 056/111] [chore](femetaversion) add a check in fe code to avoid fe meta version changed during pick PR (#35039) * [chore](femetaversion) add a check in fe code to avoid fe meta version changed during pick PR * f * f --------- Co-authored-by: yiguolei --- .../src/main/java/org/apache/doris/DorisFE.java | 11 +++++++++++ gensrc/script/gen_build_version.sh | 3 +++ 2 files changed, 14 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java index dfadde6626c4086..bc6a95e6db3800d 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java +++ b/fe/fe-core/src/main/java/org/apache/doris/DorisFE.java @@ -21,6 +21,7 @@ import org.apache.doris.common.CommandLineOptions; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.LdapConfig; import org.apache.doris.common.Log4jConfig; import org.apache.doris.common.ThreadPoolManager; @@ -77,6 +78,16 @@ public class DorisFE { private static FileLock processFileLock; public static void main(String[] args) { + // Every doris version should have a final meta version, it should not change + // between small releases. Add a check here to avoid mistake. + if (Version.DORIS_FE_META_VERSION > 0 + && FeMetaVersion.VERSION_CURRENT != Version.DORIS_FE_META_VERSION) { + System.err.println("This release's fe meta version should be " + + Version.DORIS_FE_META_VERSION + + " but it is " + FeMetaVersion.VERSION_CURRENT + + ". It should not change, or FE could not rollback in this version"); + return; + } StartupOptions options = new StartupOptions(); options.enableHttpServer = true; options.enableQeService = true; diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index 3112824867abc7e..5efb18166da1f12 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -35,6 +35,8 @@ build_version_rc_version="rc09" build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}-${build_version_rc_version}" +# This version is used to check FeMetaVersion is not changed during release +build_fe_meta_version=0 unset LANG unset LC_CTYPE @@ -132,6 +134,7 @@ public class Version { public static final String DORIS_BUILD_TIME = "${build_time}"; public static final String DORIS_BUILD_INFO = "${build_info}"; public static final String DORIS_JAVA_COMPILE_VERSION = "${java_version_str}"; + public static final int DORIS_FE_META_VERSION = ${build_fe_meta_version}; public static void main(String[] args) { System.out.println("doris_build_version_prefix: " + DORIS_BUILD_VERSION_PREFIX); From 5fa5ea87835d8325ab63efcf2cb1ace7835da34f Mon Sep 17 00:00:00 2001 From: yiguolei Date: Mon, 20 May 2024 13:30:15 +0800 Subject: [PATCH 057/111] add check fe meta version to 129 for branch2.1 --- gensrc/script/gen_build_version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gensrc/script/gen_build_version.sh b/gensrc/script/gen_build_version.sh index 5efb18166da1f12..6a0c0ba6bcd7cb1 100755 --- a/gensrc/script/gen_build_version.sh +++ b/gensrc/script/gen_build_version.sh @@ -36,7 +36,7 @@ build_version_rc_version="rc09" build_version="${build_version_prefix}-${build_version_major}.${build_version_minor}.${build_version_patch}-${build_version_rc_version}" # This version is used to check FeMetaVersion is not changed during release -build_fe_meta_version=0 +build_fe_meta_version=129 unset LANG unset LC_CTYPE From 42425808a10c508a85925a178525672c9c8f8558 Mon Sep 17 00:00:00 2001 From: abmdocrt Date: Mon, 20 May 2024 15:43:46 +0800 Subject: [PATCH 058/111] [Cherry-Pick](branch-2.1) Pick "Fix multiple replica partial update auto inc data inconsistency problem #34788" (#35056) * [Fix](auto inc) Fix multiple replica partial update auto inc data inconsistency problem (#34788) * **Problem:** For tables with auto-increment columns, updating partial columns can cause data inconsistency among replicas. **Cause:** Previously, the implementation for updating partial columns in tables with auto-increment columns was done independently on each BE (Backend), leading to potential inconsistencies in the auto-increment column values generated by each BE. **Solution:** Before distributing blocks, determine if the update involves partial columns of a table with an auto-increment column. If so, add the auto-increment column to the last column of the block. After distributing to each BE, each BE will check if the data key for the partial column update exists. If it exists, the previous auto-increment column value is used; if not, the auto-increment column value from the last column of the block is used. This ensures that the auto-increment column values are consistent across different BEs. * 2 * [Fix](regression-test) Fix auto inc partial update unstable regression test (#34940) --- be/src/exec/tablet_info.cpp | 3 + be/src/exec/tablet_info.h | 2 + be/src/olap/delta_writer_v2.cpp | 3 +- be/src/olap/memtable.cpp | 13 +- be/src/olap/memtable.h | 2 + be/src/olap/partial_update_info.h | 10 +- .../olap/rowset/segment_v2/segment_writer.cpp | 30 +-- .../olap/rowset/segment_v2/segment_writer.h | 7 +- .../segment_v2/vertical_segment_writer.cpp | 27 +- .../segment_v2/vertical_segment_writer.h | 6 +- be/src/olap/rowset_builder.cpp | 3 +- be/src/runtime/runtime_state.h | 7 + be/src/vec/sink/vtablet_block_convertor.cpp | 49 +++- be/src/vec/sink/vtablet_block_convertor.h | 7 +- be/src/vec/sink/writer/vtablet_writer.cpp | 6 +- be/src/vec/sink/writer/vtablet_writer_v2.cpp | 6 +- .../apache/doris/planner/OlapTableSink.java | 1 + gensrc/proto/descriptors.proto | 1 + gensrc/thrift/Descriptors.thrift | 1 + ..._inc_partial_update_consistency_insert.out | 97 +++++++ ...partial_update_consistency_stream_load.out | 95 +++++++ ...c_partial_update_consistency_insert.groovy | 221 +++++++++++++++ ...tial_update_consistency_stream_load.groovy | 255 ++++++++++++++++++ 23 files changed, 788 insertions(+), 64 deletions(-) create mode 100644 regression-test/data/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_insert.out create mode 100644 regression-test/data/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_stream_load.out create mode 100644 regression-test/suites/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_insert.groovy create mode 100644 regression-test/suites/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_stream_load.groovy diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp index ff8c272fb222467..62ff0b2fccee817 100644 --- a/be/src/exec/tablet_info.cpp +++ b/be/src/exec/tablet_info.cpp @@ -121,6 +121,7 @@ Status OlapTableSchemaParam::init(const POlapTableSchemaParam& pschema) { _is_strict_mode = pschema.is_strict_mode(); if (_is_partial_update) { _auto_increment_column = pschema.auto_increment_column(); + _auto_increment_column_unique_id = pschema.auto_increment_column_unique_id(); } _timestamp_ms = pschema.timestamp_ms(); _timezone = pschema.timezone(); @@ -186,6 +187,7 @@ Status OlapTableSchemaParam::init(const TOlapTableSchemaParam& tschema) { } if (_is_partial_update) { _auto_increment_column = tschema.auto_increment_column; + _auto_increment_column_unique_id = tschema.auto_increment_column_unique_id; } for (const auto& tcolumn : tschema.partial_update_input_columns) { @@ -258,6 +260,7 @@ void OlapTableSchemaParam::to_protobuf(POlapTableSchemaParam* pschema) const { pschema->set_partial_update(_is_partial_update); pschema->set_is_strict_mode(_is_strict_mode); pschema->set_auto_increment_column(_auto_increment_column); + pschema->set_auto_increment_column_unique_id(_auto_increment_column_unique_id); pschema->set_timestamp_ms(_timestamp_ms); pschema->set_timezone(_timezone); for (auto col : _partial_update_input_columns) { diff --git a/be/src/exec/tablet_info.h b/be/src/exec/tablet_info.h index 20f4fa51fc64b1d..fcba8fd82623bb9 100644 --- a/be/src/exec/tablet_info.h +++ b/be/src/exec/tablet_info.h @@ -93,6 +93,7 @@ class OlapTableSchemaParam { return _partial_update_input_columns; } std::string auto_increment_coulumn() const { return _auto_increment_column; } + int32_t auto_increment_column_unique_id() const { return _auto_increment_column_unique_id; } void set_timestamp_ms(int64_t timestamp_ms) { _timestamp_ms = timestamp_ms; } int64_t timestamp_ms() const { return _timestamp_ms; } void set_timezone(std::string timezone) { _timezone = timezone; } @@ -113,6 +114,7 @@ class OlapTableSchemaParam { std::set _partial_update_input_columns; bool _is_strict_mode = false; std::string _auto_increment_column; + int32_t _auto_increment_column_unique_id; int64_t _timestamp_ms = 0; std::string _timezone; }; diff --git a/be/src/olap/delta_writer_v2.cpp b/be/src/olap/delta_writer_v2.cpp index 51cef7e9f585593..5cfc260d1b584b4 100644 --- a/be/src/olap/delta_writer_v2.cpp +++ b/be/src/olap/delta_writer_v2.cpp @@ -238,7 +238,8 @@ void DeltaWriterV2::_build_current_tablet_schema(int64_t index_id, _partial_update_info->init(*_tablet_schema, table_schema_param->is_partial_update(), table_schema_param->partial_update_input_columns(), table_schema_param->is_strict_mode(), - table_schema_param->timestamp_ms(), table_schema_param->timezone()); + table_schema_param->timestamp_ms(), table_schema_param->timezone(), + table_schema_param->auto_increment_coulumn()); } } // namespace doris diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 123eb7d82646125..2676bf7a32e93fd 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -68,16 +68,22 @@ MemTable::MemTable(int64_t tablet_id, const TabletSchema* tablet_schema, _query_thread_context.init(); _arena = std::make_unique(); _vec_row_comparator = std::make_shared(_tablet_schema); - // TODO: Support ZOrderComparator in the future - _init_columns_offset_by_slot_descs(slot_descs, tuple_desc); _num_columns = _tablet_schema->num_columns(); if (partial_update_info != nullptr) { _is_partial_update = partial_update_info->is_partial_update; if (_is_partial_update) { _num_columns = partial_update_info->partial_update_input_columns.size(); + if (partial_update_info->is_schema_contains_auto_inc_column && + !partial_update_info->is_input_columns_contains_auto_inc_column) { + _is_partial_update_and_auto_inc = true; + _num_columns += 1; + } } } + // TODO: Support ZOrderComparator in the future + _init_columns_offset_by_slot_descs(slot_descs, tuple_desc); } + void MemTable::_init_columns_offset_by_slot_descs(const std::vector* slot_descs, const TupleDescriptor* tuple_desc) { for (auto slot_desc : *slot_descs) { @@ -89,6 +95,9 @@ void MemTable::_init_columns_offset_by_slot_descs(const std::vector& partial_update_cols, bool is_strict_mode, - int64_t timestamp_ms, const std::string& timezone) { + int64_t timestamp_ms, const std::string& timezone, + const std::string& auto_increment_column) { is_partial_update = partial_update; partial_update_input_columns = partial_update_cols; this->timestamp_ms = timestamp_ms; @@ -42,8 +43,13 @@ struct PartialUpdateInfo { } else { update_cids.emplace_back(i); } + if (auto_increment_column == tablet_column.name()) { + is_schema_contains_auto_inc_column = true; + } } this->is_strict_mode = is_strict_mode; + is_input_columns_contains_auto_inc_column = + is_partial_update && partial_update_input_columns.contains(auto_increment_column); } bool is_partial_update {false}; @@ -56,5 +62,7 @@ struct PartialUpdateInfo { bool is_strict_mode {false}; int64_t timestamp_ms {0}; std::string timezone; + bool is_input_columns_contains_auto_inc_column = false; + bool is_schema_contains_auto_inc_column = false; }; } // namespace doris diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index b25435684789957..83e93631ab142d4 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -112,13 +112,6 @@ SegmentWriter::SegmentWriter(io::FileWriter* file_writer, uint32_t segment_id, const auto& column = _tablet_schema->column(_tablet_schema->sequence_col_idx()); _seq_coder = get_key_coder(column.type()); } - if (!_tablet_schema->auto_increment_column().empty()) { - _auto_inc_id_buffer = - vectorized::GlobalAutoIncBuffers::GetInstance()->get_auto_inc_buffer( - _tablet_schema->db_id(), _tablet_schema->table_id(), - _tablet_schema->column(_tablet_schema->auto_increment_column()) - .unique_id()); - } // encode the rowid into the primary key index if (!_tablet_schema->cluster_key_idxes().empty()) { const auto* type_info = get_scalar_type_info(); @@ -559,7 +552,7 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* // read and fill block auto mutable_full_columns = full_block.mutate_columns(); RETURN_IF_ERROR(fill_missing_columns(mutable_full_columns, use_default_or_null_flag, - has_default_or_nullable, segment_start_pos)); + has_default_or_nullable, segment_start_pos, block)); full_block.set_columns(std::move(mutable_full_columns)); // row column should be filled here if (_tablet_schema->store_row_column()) { @@ -618,7 +611,8 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* Status SegmentWriter::fill_missing_columns(vectorized::MutableColumns& mutable_full_columns, const std::vector& use_default_or_null_flag, bool has_default_or_nullable, - const size_t& segment_start_pos) { + const size_t& segment_start_pos, + const vectorized::Block* block) { if constexpr (!std::is_same_v) { // TODO(plat1ko): cloud mode return Status::NotSupported("fill_missing_columns"); @@ -712,18 +706,6 @@ Status SegmentWriter::fill_missing_columns(vectorized::MutableColumns& mutable_f } } - // deal with partial update auto increment column when there no key in old block. - if (!_tablet_schema->auto_increment_column().empty()) { - if (_auto_inc_id_allocator.total_count < use_default_or_null_flag.size()) { - std::vector> res; - RETURN_IF_ERROR( - _auto_inc_id_buffer->sync_request_ids(use_default_or_null_flag.size(), &res)); - for (auto [start, length] : res) { - _auto_inc_id_allocator.insert_ids(start, length); - } - } - } - // fill all missing value from mutable_old_columns, need to consider default value and null value for (auto idx = 0; idx < use_default_or_null_flag.size(); idx++) { // `use_default_or_null_flag[idx] == true` doesn't mean that we should read values from the old row @@ -751,7 +733,11 @@ Status SegmentWriter::fill_missing_columns(vectorized::MutableColumns& mutable_f FieldType::OLAP_FIELD_TYPE_BIGINT); auto auto_inc_column = assert_cast( mutable_full_columns[cids_missing[i]].get()); - auto_inc_column->insert(_auto_inc_id_allocator.next_id()); + auto_inc_column->insert( + (assert_cast( + block->get_by_name("__PARTIAL_UPDATE_AUTO_INC_COLUMN__") + .column.get())) + ->get_element(idx)); } else { // If the control flow reaches this branch, the column neither has default value // nor is nullable. It means that the row's delete sign is marked, and the value diff --git a/be/src/olap/rowset/segment_v2/segment_writer.h b/be/src/olap/rowset/segment_v2/segment_writer.h index 1adb94aad21950d..2f26d6158ee70bf 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.h +++ b/be/src/olap/rowset/segment_v2/segment_writer.h @@ -40,7 +40,6 @@ #include "olap/tablet_schema.h" #include "util/faststring.h" #include "util/slice.h" -#include "vec/sink/autoinc_buffer.h" namespace doris { namespace vectorized { @@ -130,7 +129,8 @@ class SegmentWriter { void set_mow_context(std::shared_ptr mow_context); Status fill_missing_columns(vectorized::MutableColumns& mutable_full_columns, const std::vector& use_default_or_null_flag, - bool has_default_or_nullable, const size_t& segment_start_pos); + bool has_default_or_nullable, const size_t& segment_start_pos, + const vectorized::Block* block); private: DISALLOW_COPY_AND_ASSIGN(SegmentWriter); @@ -226,9 +226,6 @@ class SegmentWriter { // group every rowset-segment row id to speed up reader PartialUpdateReadPlan _rssid_to_rid; std::map _rsid_to_rowset; - - std::shared_ptr _auto_inc_id_buffer = nullptr; - vectorized::AutoIncIDAllocator _auto_inc_id_allocator; }; } // namespace segment_v2 diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp index 5cadc9aac6e8f97..5d2ddedb204ed28 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp @@ -103,11 +103,6 @@ VerticalSegmentWriter::VerticalSegmentWriter(io::FileWriter* file_writer, uint32 const auto& column = _tablet_schema->column(_tablet_schema->sequence_col_idx()); _seq_coder = get_key_coder(column.type()); } - if (!_tablet_schema->auto_increment_column().empty()) { - _auto_inc_id_buffer = vectorized::GlobalAutoIncBuffers::GetInstance()->get_auto_inc_buffer( - _tablet_schema->db_id(), _tablet_schema->table_id(), - _tablet_schema->column(_tablet_schema->auto_increment_column()).unique_id()); - } if (_tablet_schema->has_inverted_index()) { _inverted_index_file_writer = std::make_unique( _file_writer->fs(), _file_writer->path().parent_path(), @@ -493,7 +488,7 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da // read and fill block auto mutable_full_columns = full_block.mutate_columns(); RETURN_IF_ERROR(_fill_missing_columns(mutable_full_columns, use_default_or_null_flag, - has_default_or_nullable, segment_start_pos)); + has_default_or_nullable, segment_start_pos, data.block)); // row column should be filled here if (_tablet_schema->store_row_column()) { // convert block to row store format @@ -552,7 +547,7 @@ Status VerticalSegmentWriter::_append_block_with_partial_content(RowsInBlock& da Status VerticalSegmentWriter::_fill_missing_columns( vectorized::MutableColumns& mutable_full_columns, const std::vector& use_default_or_null_flag, bool has_default_or_nullable, - const size_t& segment_start_pos) { + const size_t& segment_start_pos, const vectorized::Block* block) { if constexpr (!std::is_same_v) { // TODO(plat1ko): CloudStorageEngine return Status::NotSupported("fill_missing_columns"); @@ -645,18 +640,6 @@ Status VerticalSegmentWriter::_fill_missing_columns( } } - // deal with partial update auto increment column when there no key in old block. - if (!_tablet_schema->auto_increment_column().empty()) { - if (_auto_inc_id_allocator.total_count < use_default_or_null_flag.size()) { - std::vector> res; - RETURN_IF_ERROR( - _auto_inc_id_buffer->sync_request_ids(use_default_or_null_flag.size(), &res)); - for (auto [start, length] : res) { - _auto_inc_id_allocator.insert_ids(start, length); - } - } - } - // fill all missing value from mutable_old_columns, need to consider default value and null value for (auto idx = 0; idx < use_default_or_null_flag.size(); idx++) { // `use_default_or_null_flag[idx] == true` doesn't mean that we should read values from the old row @@ -684,7 +667,11 @@ Status VerticalSegmentWriter::_fill_missing_columns( FieldType::OLAP_FIELD_TYPE_BIGINT); auto auto_inc_column = assert_cast( mutable_full_columns[missing_cids[i]].get()); - auto_inc_column->insert(_auto_inc_id_allocator.next_id()); + auto_inc_column->insert( + (assert_cast( + block->get_by_name("__PARTIAL_UPDATE_AUTO_INC_COLUMN__") + .column.get())) + ->get_element(idx)); } else { // If the control flow reaches this branch, the column neither has default value // nor is nullable. It means that the row's delete sign is marked, and the value diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h index 7bc6bf7c4fa8783..02e7170ff511ac4 100644 --- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.h +++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.h @@ -38,7 +38,6 @@ #include "olap/tablet_schema.h" #include "util/faststring.h" #include "util/slice.h" -#include "vec/sink/autoinc_buffer.h" namespace doris { namespace vectorized { @@ -145,7 +144,8 @@ class VerticalSegmentWriter { Status _append_block_with_partial_content(RowsInBlock& data); Status _fill_missing_columns(vectorized::MutableColumns& mutable_full_columns, const std::vector& use_default_or_null_flag, - bool has_default_or_nullable, const size_t& segment_start_pos); + bool has_default_or_nullable, const size_t& segment_start_pos, + const vectorized::Block* block); private: uint32_t _segment_id; @@ -193,8 +193,6 @@ class VerticalSegmentWriter { std::map _rsid_to_rowset; std::vector _batched_blocks; - std::shared_ptr _auto_inc_id_buffer = nullptr; - vectorized::AutoIncIDAllocator _auto_inc_id_allocator; }; } // namespace segment_v2 diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index 4355bb6f964f524..2153a9ad1a8e87a 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -377,7 +377,8 @@ void BaseRowsetBuilder::_build_current_tablet_schema(int64_t index_id, _partial_update_info->init(*_tablet_schema, table_schema_param->is_partial_update(), table_schema_param->partial_update_input_columns(), table_schema_param->is_strict_mode(), - table_schema_param->timestamp_ms(), table_schema_param->timezone()); + table_schema_param->timestamp_ms(), table_schema_param->timezone(), + table_schema_param->auto_increment_coulumn()); } } // namespace doris diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h index abef8615b1e47dc..9f483fdc26eb5a3 100644 --- a/be/src/runtime/runtime_state.h +++ b/be/src/runtime/runtime_state.h @@ -42,6 +42,7 @@ #include "runtime/task_execution_context.h" #include "util/debug_util.h" #include "util/runtime_profile.h" +#include "vec/columns/columns_number.h" namespace doris { class IRuntimeFilter; @@ -628,6 +629,10 @@ class RuntimeState { int task_num() const { return _task_num; } + vectorized::ColumnInt64* partial_update_auto_inc_column() { + return _partial_update_auto_inc_column; + }; + private: Status create_error_log_file(); @@ -755,6 +760,8 @@ class RuntimeState { // prohibit copies RuntimeState(const RuntimeState&); + + vectorized::ColumnInt64* _partial_update_auto_inc_column; }; #define RETURN_IF_CANCELLED(state) \ diff --git a/be/src/vec/sink/vtablet_block_convertor.cpp b/be/src/vec/sink/vtablet_block_convertor.cpp index 678c899d980bb5c..d93a654728d4467 100644 --- a/be/src/vec/sink/vtablet_block_convertor.cpp +++ b/be/src/vec/sink/vtablet_block_convertor.cpp @@ -19,6 +19,7 @@ #include #include +#include #include #include @@ -46,6 +47,7 @@ #include "vec/common/assert_cast.h" #include "vec/core/block.h" #include "vec/core/types.h" +#include "vec/data_types/data_type.h" #include "vec/data_types/data_type_decimal.h" #include "vec/data_types/data_type_nullable.h" #include "vec/exprs/vexpr.h" @@ -66,8 +68,21 @@ Status OlapTableBlockConvertor::validate_and_convert_block( output_vexpr_ctxs, *input_block, block.get())); } - // fill the valus for auto-increment columns - if (_auto_inc_col_idx.has_value()) { + if (_is_partial_update_and_auto_inc) { + // If this load is partial update and this table has a auto inc column, + // e.g. table schema: k1, v1, v2(auto inc) + // 1. insert columns include auto inc column + // e.g. insert into table (k1, v2) value(a, 1); + // we do nothing. + // 2. insert columns do not include auto inc column + // e.g. insert into table (k1, v1) value(a, a); + // we need to fill auto_inc_cols by creating a new column. + if (!_auto_inc_col_idx.has_value()) { + RETURN_IF_ERROR(_partial_update_fill_auto_inc_cols(block.get(), rows)); + } + } else if (_auto_inc_col_idx.has_value()) { + // fill the valus for auto-increment columns + DCHECK_EQ(_is_partial_update_and_auto_inc, false); RETURN_IF_ERROR(_fill_auto_inc_cols(block.get(), rows)); } @@ -91,8 +106,16 @@ Status OlapTableBlockConvertor::validate_and_convert_block( return Status::OK(); } -void OlapTableBlockConvertor::init_autoinc_info(int64_t db_id, int64_t table_id, int batch_size) { +void OlapTableBlockConvertor::init_autoinc_info(int64_t db_id, int64_t table_id, int batch_size, + bool is_partial_update_and_auto_inc, + int32_t auto_increment_column_unique_id) { _batch_size = batch_size; + if (is_partial_update_and_auto_inc) { + _is_partial_update_and_auto_inc = is_partial_update_and_auto_inc; + _auto_inc_id_buffer = GlobalAutoIncBuffers::GetInstance()->get_auto_inc_buffer( + db_id, table_id, auto_increment_column_unique_id); + return; + } for (size_t idx = 0; idx < _output_tuple_desc->slots().size(); idx++) { if (_output_tuple_desc->slots()[idx]->is_auto_increment()) { _auto_inc_col_idx = idx; @@ -522,4 +545,24 @@ Status OlapTableBlockConvertor::_fill_auto_inc_cols(vectorized::Block* block, si return Status::OK(); } +Status OlapTableBlockConvertor::_partial_update_fill_auto_inc_cols(vectorized::Block* block, + size_t rows) { + auto dst_column = vectorized::ColumnInt64::create(); + vectorized::ColumnInt64::Container& dst_values = dst_column->get_data(); + size_t null_value_count = rows; + std::vector> res; + RETURN_IF_ERROR(_auto_inc_id_buffer->sync_request_ids(null_value_count, &res)); + for (auto [start, length] : res) { + _auto_inc_id_allocator.insert_ids(start, length); + } + + for (size_t i = 0; i < rows; i++) { + dst_values.emplace_back(_auto_inc_id_allocator.next_id()); + } + block->insert(vectorized::ColumnWithTypeAndName(std::move(dst_column), + std::make_shared>(), + "__PARTIAL_UPDATE_AUTO_INC_COLUMN__")); + return Status::OK(); +} + } // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/sink/vtablet_block_convertor.h b/be/src/vec/sink/vtablet_block_convertor.h index 4eaaef3869cccbe..0db340ce6c27d43 100644 --- a/be/src/vec/sink/vtablet_block_convertor.h +++ b/be/src/vec/sink/vtablet_block_convertor.h @@ -53,7 +53,9 @@ class OlapTableBlockConvertor { int64_t num_filtered_rows() const { return _num_filtered_rows; } - void init_autoinc_info(int64_t db_id, int64_t table_id, int batch_size); + void init_autoinc_info(int64_t db_id, int64_t table_id, int batch_size, + bool is_partial_update_and_auto_inc = false, + int32_t auto_increment_column_unique_id = -1); AutoIncIDAllocator& auto_inc_id_allocator() { return _auto_inc_id_allocator; } @@ -82,6 +84,8 @@ class OlapTableBlockConvertor { Status _fill_auto_inc_cols(vectorized::Block* block, size_t rows); + Status _partial_update_fill_auto_inc_cols(vectorized::Block* block, size_t rows); + TupleDescriptor* _output_tuple_desc = nullptr; std::map, DecimalV2Value> _max_decimalv2_val; @@ -105,6 +109,7 @@ class OlapTableBlockConvertor { std::optional _auto_inc_col_idx; std::shared_ptr _auto_inc_id_buffer = nullptr; AutoIncIDAllocator _auto_inc_id_allocator; + bool _is_partial_update_and_auto_inc = false; }; } // namespace doris::vectorized \ No newline at end of file diff --git a/be/src/vec/sink/writer/vtablet_writer.cpp b/be/src/vec/sink/writer/vtablet_writer.cpp index 3fb17850eff6843..0a7238e9f5d915c 100644 --- a/be/src/vec/sink/writer/vtablet_writer.cpp +++ b/be/src/vec/sink/writer/vtablet_writer.cpp @@ -1178,8 +1178,10 @@ Status VTabletWriter::_init(RuntimeState* state, RuntimeProfile* profile) { } _block_convertor = std::make_unique(_output_tuple_desc); - _block_convertor->init_autoinc_info(_schema->db_id(), _schema->table_id(), - _state->batch_size()); + _block_convertor->init_autoinc_info( + _schema->db_id(), _schema->table_id(), _state->batch_size(), + _schema->is_partial_update() && !_schema->auto_increment_coulumn().empty(), + _schema->auto_increment_column_unique_id()); _output_row_desc = _pool->add(new RowDescriptor(_output_tuple_desc, false)); // add all counter diff --git a/be/src/vec/sink/writer/vtablet_writer_v2.cpp b/be/src/vec/sink/writer/vtablet_writer_v2.cpp index b883d8e87c9e7c3..c1b43722c33b99d 100644 --- a/be/src/vec/sink/writer/vtablet_writer_v2.cpp +++ b/be/src/vec/sink/writer/vtablet_writer_v2.cpp @@ -213,8 +213,10 @@ Status VTabletWriterV2::_init(RuntimeState* state, RuntimeProfile* profile) { } _block_convertor = std::make_unique(_output_tuple_desc); - _block_convertor->init_autoinc_info(_schema->db_id(), _schema->table_id(), - _state->batch_size()); + _block_convertor->init_autoinc_info( + _schema->db_id(), _schema->table_id(), _state->batch_size(), + _schema->is_partial_update() && !_schema->auto_increment_coulumn().empty(), + _schema->auto_increment_column_unique_id()); _output_row_desc = _pool->add(new RowDescriptor(_output_tuple_desc, false)); // add all counter diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java index d180c7233985cc0..ada7c6b770bab05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapTableSink.java @@ -312,6 +312,7 @@ public TOlapTableSchemaParam createSchema(long dbId, OlapTable table, Analyzer a for (Column col : table.getFullSchema()) { if (col.isAutoInc()) { schemaParam.setAutoIncrementColumn(col.getName()); + schemaParam.setAutoIncrementColumnUniqueId(col.getUniqueId()); } } } diff --git a/gensrc/proto/descriptors.proto b/gensrc/proto/descriptors.proto index 9d6945becc05831..13c069f414fc2b9 100644 --- a/gensrc/proto/descriptors.proto +++ b/gensrc/proto/descriptors.proto @@ -72,5 +72,6 @@ message POlapTableSchemaParam { optional string auto_increment_column = 10; optional int64 timestamp_ms = 11 [default = 0]; optional string timezone = 12; + optional int32 auto_increment_column_unique_id = 13; }; diff --git a/gensrc/thrift/Descriptors.thrift b/gensrc/thrift/Descriptors.thrift index d82f74d771c0169..ef7a8451684982e 100644 --- a/gensrc/thrift/Descriptors.thrift +++ b/gensrc/thrift/Descriptors.thrift @@ -245,6 +245,7 @@ struct TOlapTableSchemaParam { 9: optional list partial_update_input_columns 10: optional bool is_strict_mode = false 11: optional string auto_increment_column + 12: optional i32 auto_increment_column_unique_id = -1 } struct TTabletLocation { diff --git a/regression-test/data/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_insert.out b/regression-test/data/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_insert.out new file mode 100644 index 000000000000000..79a082991187dab --- /dev/null +++ b/regression-test/data/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_insert.out @@ -0,0 +1,97 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1_1 -- +Alice 200 +Beata 700 +Bob 100 +Carter 500 +Doris 800 +Nereids 900 +Smith 600 +Test 400 +Tom 300 + +-- !select1_2 -- + +-- !select1_3 -- +Alice 200 +Beata 723 +Bob 123 +Carter 523 +Doris 800 +Nereids 923 +Smith 600 +Test 400 +Tom 323 + +-- !select1_4 -- + +-- !select2_1 -- +Alice 200 +Beata 700 +Bob 100 +Carter 500 +Doris 800 +Nereids 900 +Smith 600 +Test 400 +Tom 300 + +-- !select2_2 -- + +-- !select2_3 -- +Alice 200 +Beata 700 +Bob 100 +Carter 500 +Doris 800 +Nereids 900 +Smith 600 +Test 400 +Tom 300 + +-- !select2_4 -- + +-- !select3_1 -- +Alice 200 +Beata 700 +Bob 100 +Carter 500 +Doris 800 +Nereids 900 +Smith 600 +Test 400 +Tom 300 + +-- !select3_2 -- + +-- !select3_3 -- +Alice 200 +Beata 9996 +Bob 9990 +Carter 9994 +Doris 800 +Nereids 9998 +Smith 600 +Test 400 +Tom 9992 + +-- !select3_4 -- + +-- !select3_5 -- +Alice 200 +BBBBeata 9996 +BBBBob 9990 +Beata 9996 +Bob 9990 +CCCCarter 9994 +Carter 9994 +Doris 800 +NNNNereids 9998 +Nereids 9998 +Smith 600 +TTTTom 9992 +Test 400 +Tom 9992 + +-- !select3_6 -- + diff --git a/regression-test/data/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_stream_load.out b/regression-test/data/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_stream_load.out new file mode 100644 index 000000000000000..a31e438f292301e --- /dev/null +++ b/regression-test/data/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_stream_load.out @@ -0,0 +1,95 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !select1_1 -- +1 Bob 100 +2 Alice 200 +3 Tom 300 +4 Test 400 +5 Carter 500 +6 Smith 600 +7 Beata 700 +8 Doris 800 +9 Nereids 900 + +-- !select1_1 -- +Alice 200 +Beata 723 +Bob 123 +Carter 523 +Doris 800 +Nereids 923 +Smith 600 +Test 400 +Tom 323 + +-- !select1_2 -- + +-- !select2_1 -- +Alice 200 +Beata 700 +Bob 100 +Carter 500 +Doris 800 +Nereids 900 +Smith 600 +Test 400 +Tom 300 + +-- !select2_2 -- + +-- !select2_3 -- +Alice 200 +Beata 700 +Bob 100 +Carter 500 +Doris 800 +Nereids 900 +Smith 600 +Test 400 +Tom 300 + +-- !select2_4 -- + +-- !select3_1 -- +Alice 200 +Beata 700 +Bob 100 +Carter 500 +Doris 800 +Nereids 900 +Smith 600 +Test 400 +Tom 300 + +-- !select3_2 -- + +-- !select3_3 -- +Alice 200 +Beata 9996 +Bob 9990 +Carter 9994 +Doris 800 +Nereids 9998 +Smith 600 +Test 400 +Tom 9992 + +-- !select3_4 -- + +-- !select3_5 -- +Alice 200 +BBeata 9996 +BBob 9990 +Beata 9996 +Bob 9990 +CCarter 9994 +Carter 9994 +Doris 800 +NNereids 9998 +Nereids 9998 +Smith 600 +TTom 9992 +Test 400 +Tom 9992 + +-- !select3_6 -- + diff --git a/regression-test/suites/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_insert.groovy b/regression-test/suites/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_insert.groovy new file mode 100644 index 000000000000000..4e2c8a5cbd86688 --- /dev/null +++ b/regression-test/suites/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_insert.groovy @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_unique_table_auto_inc_partial_update_correct_insert") { + + def backends = sql_return_maparray('show backends') + def replicaNum = 0 + def targetBackend = null + for (def be : backends) { + def alive = be.Alive.toBoolean() + def decommissioned = be.SystemDecommissioned.toBoolean() + if (alive && !decommissioned) { + replicaNum++ + targetBackend = be + } + } + assertTrue(replicaNum > 0) + + def check_data_correct = { def tableName -> + def old_result = sql "select id from ${tableName} order by id;" + logger.info("first result: " + old_result) + for (int i = 1; i<30; ++i){ + def new_result = sql "select id from ${tableName} order by id;" + logger.info("new result: " + new_result) + for (int j = 0; j 1;" + check_data_correct(table1) + + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + + // 1, 123 + // 3, 323 + // 5, 523 + // 7, 723 + // 9, 923 + sql "insert into ${table1} (id, value) values (1,123),(3,323),(5,523),(7,723),(9,923)" + qt_select1_3 "select name, value from ${table1} order by name, value;" + qt_select1_4 "select id, count(*) from ${table1} group by id having count(*) > 1;" + check_data_correct(table1) + sql "drop table if exists ${table1};" + + // test for partial update, auto inc col is value, update auto inc col + def table2 = "unique_auto_inc_col_value_partial_update_insert" + sql "drop table if exists ${table2}" + sql """ + CREATE TABLE IF NOT EXISTS `${table2}` ( + `name` varchar(65533) NOT NULL COMMENT "用户姓名", + `value` int(11) NOT NULL COMMENT "用户得分", + `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT "用户 ID" + ) ENGINE=OLAP + UNIQUE KEY(`name`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`name`) BUCKETS 3 + PROPERTIES ( + "replication_num" = "${replicaNum}", + "in_memory" = "false", + "storage_format" = "V2", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + + + // Bob, 100 + // Alice, 200 + // Tom, 300 + // Test, 400 + // Carter, 500 + // Smith, 600 + // Beata, 700 + // Doris, 800 + // Nereids, 900 + sql "insert into ${table2} (name, value) values ('Bob',100)" + sql "insert into ${table2} (name, value) values ('Alice',200)" + sql "insert into ${table2} (name, value) values ('Tom',300)" + sql "insert into ${table2} (name, value) values ('Test',400)" + sql "insert into ${table2} (name, value) values ('Carter',500)" + sql "insert into ${table2} (name, value) values ('Smith',600)" + sql "insert into ${table2} (name, value) values ('Beata',700)" + sql "insert into ${table2} (name, value) values ('Doris',800)" + sql "insert into ${table2} (name, value) values ('Nereids',900)" + qt_select2_1 "select name, value from ${table2} order by name, value;" + qt_select2_2 "select id, count(*) from ${table2} group by id having count(*) > 1;" + check_data_correct(table2) + + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + // Bob, 9990 + // Tom, 9992 + // Carter, 9994 + // Beata, 9996 + // Nereids, 9998 + sql "insert into ${table2} (name, id) values ('Bob',9990),('Tom',9992),('Carter',9994),('Beata',9996),('Nereids',9998)" + qt_select2_3 "select name, value from ${table2} order by name, value;" + qt_select2_4 "select id, count(*) from ${table2} group by id having count(*) > 1;" + check_data_correct(table2) + sql "drop table if exists ${table2};" + + // test for partial update, auto inc col is value, update other col + def table3 = "unique_auto_inc_col_value_partial_update_insert" + sql "drop table if exists ${table3}" + sql """ + CREATE TABLE IF NOT EXISTS `${table3}` ( + `name` varchar(65533) NOT NULL COMMENT "用户姓名", + `value` int(11) NOT NULL COMMENT "用户得分", + `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT "用户 ID" + ) ENGINE=OLAP + UNIQUE KEY(`name`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`name`) BUCKETS 3 + PROPERTIES ( + "replication_num" = "${replicaNum}", + "in_memory" = "false", + "storage_format" = "V2", + "enable_unique_key_merge_on_write" = "true" + ) + """ + sql "set enable_unique_key_partial_update=false;" + sql "set enable_insert_strict=true;" + + // Bob, 100 + // Alice, 200 + // Tom, 300 + // Test, 400 + // Carter, 500 + // Smith, 600 + // Beata, 700 + // Doris, 800 + // Nereids, 900 + sql "insert into ${table2} (name, value) values ('Bob',100),('Alice',200),('Tom',300),('Test',400),('Carter',500),('Smith',600),('Beata',700),('Doris',800),('Nereids',900)" + qt_select3_1 "select name, value from ${table3} order by name, value;" + qt_select3_2 "select id, count(*) from ${table3} group by id having count(*) > 1;" + check_data_correct(table3) + + sql "set enable_unique_key_partial_update=true;" + sql "set enable_insert_strict=false;" + // Bob, 9990 + // Tom, 9992 + // Carter, 9994 + // Beata, 9996 + // Nereids, 9998 + sql "insert into ${table2} (name, value) values ('Bob',9990)" + sql "insert into ${table2} (name, value) values ('Tom',9992)" + sql "insert into ${table2} (name, value) values ('Carter',9994)" + sql "insert into ${table2} (name, value) values ('Beata',9996)" + sql "insert into ${table2} (name, value) values ('Nereids',9998)" + qt_select3_3 "select name, value from ${table3} order by name, value;" + qt_select3_4 "select id, count(*) from ${table3} group by id having count(*) > 1;" + check_data_correct(table3) + + sql "insert into ${table2} (name, value) values ('BBBBob',9990)" + sql "insert into ${table2} (name, value) values ('TTTTom',9992)" + sql "insert into ${table2} (name, value) values ('CCCCarter',9994)" + sql "insert into ${table2} (name, value) values ('BBBBeata',9996)" + sql "insert into ${table2} (name, value) values ('NNNNereids',9998)" + qt_select3_5 "select name, value from ${table3} order by name, value;" + qt_select3_6 "select id, count(*) from ${table3} group by id having count(*) > 1;" + check_data_correct(table3) + sql "drop table if exists ${table3};" +} + diff --git a/regression-test/suites/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_stream_load.groovy b/regression-test/suites/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_stream_load.groovy new file mode 100644 index 000000000000000..474794deb9bb9f2 --- /dev/null +++ b/regression-test/suites/data_model_p0/unique/test_unique_table_auto_inc_partial_update_consistency_stream_load.groovy @@ -0,0 +1,255 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_unique_table_auto_inc_partial_update_correct_stream_load") { + + def backends = sql_return_maparray('show backends') + def replicaNum = 0 + def targetBackend = null + for (def be : backends) { + def alive = be.Alive.toBoolean() + def decommissioned = be.SystemDecommissioned.toBoolean() + if (alive && !decommissioned) { + replicaNum++ + targetBackend = be + } + } + assertTrue(replicaNum > 0) + + def check_data_correct = { def tableName -> + def old_result = sql "select id from ${tableName} order by id;" + logger.info("first result: " + old_result) + for (int i = 1; i<30; ++i){ + def new_result = sql "select id from ${tableName} order by id;" + logger.info("new result: " + new_result) + for (int j = 0; j 1;" + check_data_correct(table1) + sql "drop table if exists ${table1};" + + // test for partial update, auto inc col is value, update auto inc col + def table2 = "unique_auto_inc_col_value_partial_update_stream_load" + sql "drop table if exists ${table2}" + sql """ + CREATE TABLE IF NOT EXISTS `${table2}` ( + `name` varchar(65533) NOT NULL COMMENT "用户姓名", + `value` int(11) NOT NULL COMMENT "用户得分", + `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT "用户 ID" + ) ENGINE=OLAP + UNIQUE KEY(`name`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`name`) BUCKETS 3 + PROPERTIES ( + "replication_num" = "${replicaNum}", + "in_memory" = "false", + "storage_format" = "V2", + "enable_unique_key_merge_on_write" = "true" + ) + """ + + // Bob, 100 + // Alice, 200 + // Tom, 300 + // Test, 400 + // Carter, 500 + // Smith, 600 + // Beata, 700 + // Doris, 800 + // Nereids, 900 + streamLoad { + table "${table2}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'name, value' + + file 'auto_inc_basic.csv' + time 10000 // limit inflight 10s + } + sql "sync" + qt_select2_1 "select name, value from ${table2} order by name, value;" + qt_select2_2 "select id, count(*) from ${table2} group by id having count(*) > 1;" + check_data_correct(table2) + + // Bob, 9990 + // Tom, 9992 + // Carter, 9994 + // Beata, 9996 + // Nereids, 9998 + streamLoad { + table "${table2}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'name, id' + set 'partial_columns', 'true' + + file 'auto_inc_partial_update2.csv' + time 10000 + } + sql "sync" + qt_select2_3 "select name, value from ${table2} order by name, value;" + qt_select2_4 "select id, count(*) from ${table2} group by id having count(*) > 1;" + check_data_correct(table2) + sql "drop table if exists ${table2};" + + // test for partial update, auto inc col is value, update other col + def table3 = "unique_auto_inc_col_value_partial_update_stream_load" + sql "drop table if exists ${table3}" + sql """ + CREATE TABLE IF NOT EXISTS `${table3}` ( + `name` varchar(65533) NOT NULL COMMENT "用户姓名", + `value` int(11) NOT NULL COMMENT "用户得分", + `id` BIGINT NOT NULL AUTO_INCREMENT COMMENT "用户 ID" + ) ENGINE=OLAP + UNIQUE KEY(`name`) + COMMENT "OLAP" + DISTRIBUTED BY HASH(`name`) BUCKETS 3 + PROPERTIES ( + "replication_num" = "${replicaNum}", + "in_memory" = "false", + "storage_format" = "V2", + "enable_unique_key_merge_on_write" = "true" + ) + """ + streamLoad { + table "${table3}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'name, value' + + file 'auto_inc_basic.csv' + time 10000 // limit inflight 10s + } + sql "sync" + qt_select3_1 "select name, value from ${table3} order by name, value;" + qt_select3_2 "select id, count(*) from ${table3} group by id having count(*) > 1;" + check_data_correct(table3) + + // Bob, 9990 + // Tom, 9992 + // Carter, 9994 + // Beata, 9996 + // Nereids, 9998 + streamLoad { + table "${table3}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'name, value' + set 'partial_columns', 'true' + + file 'auto_inc_partial_update2.csv' + time 10000 + } + sql "sync" + qt_select3_3 "select name, value from ${table3} order by name, value;" + qt_select3_4 "select id, count(*) from ${table3} group by id having count(*) > 1;" + check_data_correct(table3) + // BBob, 9990 + // TTom, 9992 + // CCarter, 9994 + // BBeata, 9996 + // NNereids, 9998 + streamLoad { + table "${table3}" + + set 'column_separator', ',' + set 'format', 'csv' + set 'columns', 'name, value' + set 'partial_columns', 'true' + + file 'auto_inc_partial_update3.csv' + time 10000 + } + sql "sync" + qt_select3_5 "select name, value from ${table3} order by name, value;" + qt_select3_6 "select id, count(*) from ${table3} group by id having count(*) > 1;" + check_data_correct(table3) + sql "drop table if exists ${table3};" +} + From cb5e5c5aa7b40272fd455dfc378c249fcd3a6b2d Mon Sep 17 00:00:00 2001 From: zy-kkk Date: Mon, 20 May 2024 16:47:23 +0800 Subject: [PATCH 059/111] [branch-2.1](test) fix dual test #35067 --- regression-test/suites/query_p0/dual/dual.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/query_p0/dual/dual.groovy b/regression-test/suites/query_p0/dual/dual.groovy index eb001305b47e61f..42e4ce7ba8c1564 100644 --- a/regression-test/suites/query_p0/dual/dual.groovy +++ b/regression-test/suites/query_p0/dual/dual.groovy @@ -70,7 +70,7 @@ suite('dual') { // Test error handling when table does not exist test { sql "select 1 from `dual`" - exception "Table [dual] does not exist in database [regression_test_query_p0_dual]" + exception "Unknown table 'dual'" } // Disable and enable Nereids planner to check behavior differences @@ -90,4 +90,4 @@ suite('dual') { sql "select 1, a from dual" exception "Unknown column 'a' in 'table list'" } -} \ No newline at end of file +} From dbaa27ebbe30aa6e0f17b609c2b63db0818c4a7c Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Mon, 20 May 2024 16:50:50 +0800 Subject: [PATCH 060/111] [fix](agg) memory leak issue in agg operator (#35037) (#35055) --- be/src/pipeline/pipeline_x/dependency.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/be/src/pipeline/pipeline_x/dependency.cpp b/be/src/pipeline/pipeline_x/dependency.cpp index ec890dac7f1a807..ba47f9355988db2 100644 --- a/be/src/pipeline/pipeline_x/dependency.cpp +++ b/be/src/pipeline/pipeline_x/dependency.cpp @@ -193,6 +193,12 @@ Status AggSharedState::reset_hash_table() { } }); + if (hash_table.has_null_key_data()) { + auto st = _destroy_agg_status( + hash_table.template get_null_key_data()); + RETURN_IF_ERROR(st); + } + aggregate_data_container.reset(new vectorized::AggregateDataContainer( sizeof(typename HashTableType::key_type), ((total_size_of_aggregate_states + align_aggregate_states - 1) / From aba00d7146bd8a235b8a3b514ca598181a59b6a4 Mon Sep 17 00:00:00 2001 From: wangbo Date: Mon, 20 May 2024 20:36:29 +0800 Subject: [PATCH 061/111] [Fix](executor)Fix workload reg test #35082 --- regression-test/data/workload_manager_p0/test_check_wg.out | 3 +++ .../suites/workload_manager_p0/test_check_wg.groovy | 3 ++- .../workload_manager_p0/test_workload_sched_policy.groovy | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/regression-test/data/workload_manager_p0/test_check_wg.out b/regression-test/data/workload_manager_p0/test_check_wg.out index 1f0a440ba0ba983..dde3373fe6b99c5 100644 --- a/regression-test/data/workload_manager_p0/test_check_wg.out +++ b/regression-test/data/workload_manager_p0/test_check_wg.out @@ -1,4 +1,7 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !select_count -- +3 + -- !select_wg -- normal upgrade_g1 diff --git a/regression-test/suites/workload_manager_p0/test_check_wg.groovy b/regression-test/suites/workload_manager_p0/test_check_wg.groovy index 6c81338e703b892..5001bce10799595 100644 --- a/regression-test/suites/workload_manager_p0/test_check_wg.groovy +++ b/regression-test/suites/workload_manager_p0/test_check_wg.groovy @@ -15,5 +15,6 @@ // specific language governing permissions and limitations // under the License. suite("test_check_wg") { - qt_select_wg "select name from information_schema.workload_groups where name in ('upgrade_g1','normal','upgrade_g2');" + qt_select_count "select count(1) from information_schema.workload_groups where name in ('upgrade_g1','normal','upgrade_g2')" + qt_select_wg "select name from information_schema.workload_groups where name in ('upgrade_g1','normal','upgrade_g2') order by name;" } \ No newline at end of file diff --git a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy index c51a10f89b5f650..6dc1a28d463a487 100644 --- a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy +++ b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy @@ -121,6 +121,8 @@ suite("test_workload_sched_policy") { sql """grant ADMIN_PRIV on *.*.* to test_workload_sched_user""" // 1 create test_set_var_policy + sql """drop workload policy if exists test_set_var_policy;""" + sql """drop workload policy if exists test_set_var_policy2;""" sql "create workload policy test_set_var_policy conditions(username='test_workload_sched_user')" + "actions(set_session_variable 'parallel_pipeline_task_num=33');" def result1 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) { From 6fe533eedebd35b2783e4a3eb59431d66b528680 Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Mon, 20 May 2024 21:11:22 +0800 Subject: [PATCH 062/111] [branch-2.1](routine-load) fix routine load case fail #35084 --- .../routine_load/test_routine_load.groovy | 90 +++++++++---------- 1 file changed, 45 insertions(+), 45 deletions(-) diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy index 87eae26a48aa447..03a3d40b786c6d0 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy @@ -279,9 +279,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_send_batch_parallelism "select * from ${tableName1} order by k00,k01" + qt_sql_send_batch_parallelism "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_send_batch_parallelism "select * from ${tableName1} order by k00" + qt_sql_send_batch_parallelism "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -366,9 +366,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_desired_concurrent_number "select * from ${tableName1} order by k00,k01" + qt_sql_desired_concurrent_number "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_desired_concurrent_number "select * from ${tableName1} order by k00" + qt_sql_desired_concurrent_number "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -612,9 +612,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_num_as_string "select * from ${tableName1} order by k00,k01" + qt_sql_num_as_string "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_num_as_string "select * from ${tableName1} order by k00" + qt_sql_num_as_string "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -693,9 +693,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_exec_mem_limit "select * from ${tableName1} order by k00,k01" + qt_sql_exec_mem_limit "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_exec_mem_limit "select * from ${tableName1} order by k00" + qt_sql_exec_mem_limit "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -815,9 +815,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_timezone_shanghai "select * from ${tableName1} order by k00,k01" + qt_sql_timezone_shanghai "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_timezone_shanghai "select * from ${tableName1} order by k00" + qt_sql_timezone_shanghai "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -893,9 +893,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_strict_mode "select * from ${tableName1} order by k00,k01" + qt_sql_strict_mode "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_strict_mode "select * from ${tableName1} order by k00" + qt_sql_strict_mode "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -974,9 +974,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_max_error_number "select * from ${tableName1} order by k00,k01" + qt_sql_max_error_number "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_max_error_number "select * from ${tableName1} order by k00" + qt_sql_max_error_number "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -1056,9 +1056,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_max_filter_ratio "select * from ${tableName1} order by k00,k01" + qt_sql_max_filter_ratio "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_max_filter_ratio "select * from ${tableName1} order by k00" + qt_sql_max_filter_ratio "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -1115,9 +1115,9 @@ suite("test_routine_load","p0") { def tableName1 = "routine_load_" + tableName if (i <= 3) { - qt_sql_load_to_single_tablet "select * from ${tableName1} order by k00,k01" + qt_sql_load_to_single_tablet "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_load_to_single_tablet "select * from ${tableName1} order by k00" + qt_sql_load_to_single_tablet "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -1173,9 +1173,9 @@ suite("test_routine_load","p0") { sleep(10000) def tableName1 = "routine_load_" + tableName if (i <= 3) { - qt_sql_column_separator "select * from ${tableName1} order by k00,k01" + qt_sql_column_separator "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_column_separator "select * from ${tableName1} order by k00" + qt_sql_column_separator "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -1252,9 +1252,9 @@ suite("test_routine_load","p0") { count++ } if (i <= 3) { - qt_sql_json "select * from ${tableName1} order by k00,k01" + qt_sql_json "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_json "select * from ${tableName1} order by k00" + qt_sql_json "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -1373,9 +1373,9 @@ suite("test_routine_load","p0") { count++ } if (i <= 3) { - qt_sql_json_jsonpath "select * from ${tableName1} order by k00,k01" + qt_sql_json_jsonpath "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_json_jsonpath "select * from ${tableName1} order by k00" + qt_sql_json_jsonpath "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -1470,9 +1470,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_disable_simdjson_reader "select * from ${tableName1} order by k00,k01" + qt_disable_simdjson_reader "select * from ${tableName1} order by k00,k01,k02" } else { - qt_disable_simdjson_reader "select * from ${tableName1} order by k00" + qt_disable_simdjson_reader "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -1553,9 +1553,9 @@ suite("test_routine_load","p0") { // count++ // } // if (i <= 3) { - // qt_sql_json_strip_outer_array "select * from ${tableName1} order by k00,k01" + // qt_sql_json_strip_outer_array "select * from ${tableName1} order by k00,k01,k02" // } else { - // qt_sql_json_strip_outer_array "select * from ${tableName1} order by k00" + // qt_sql_json_strip_outer_array "select * from ${tableName1} order by k00,k01,k02" // } // sql "stop routine load for ${jobs[i]}" @@ -1631,9 +1631,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_multi_table_one_data "select * from ${tableName1} order by k00,k01" + qt_sql_multi_table_one_data "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_multi_table_one_data "select * from ${tableName1} order by k00" + qt_sql_multi_table_one_data "select * from ${tableName1} order by k00,k01,k02" } i++ @@ -1710,9 +1710,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_multi_table "select * from ${tableName1} order by k00,k01" + qt_sql_multi_table "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_multi_table "select * from ${tableName1} order by k00" + qt_sql_multi_table "select * from ${tableName1} order by k00,k01,k02" } i++ @@ -1792,9 +1792,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_show_command "select * from ${tableName1} order by k00,k01" + qt_show_command "select * from ${tableName1} order by k00,k01,k02" } else { - qt_show_command "select * from ${tableName1} order by k00" + qt_show_command "select * from ${tableName1} order by k00,k01,k02" } def res = sql "SHOW ROUTINE LOAD TASK WHERE JobName = \"${jobs[i]}\"" @@ -1879,9 +1879,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_pause_and_resume_command "select * from ${tableName1} order by k00,k01" + qt_pause_and_resume_command "select * from ${tableName1} order by k00,k01,k02" } else { - qt_pause_and_resume_command "select * from ${tableName1} order by k00" + qt_pause_and_resume_command "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -1995,9 +1995,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_update_command "select * from ${tableName1} order by k00,k01" + qt_update_command "select * from ${tableName1} order by k00,k01,k02" } else { - qt_update_command "select * from ${tableName1} order by k00" + qt_update_command "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -2077,9 +2077,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_preceding_filter "select * from ${tableName1} order by k00,k01" + qt_sql_preceding_filter "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_preceding_filter "select * from ${tableName1} order by k00" + qt_sql_preceding_filter "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -2158,9 +2158,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_where "select * from ${tableName1} order by k00,k01" + qt_sql_where "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_where "select * from ${tableName1} order by k00" + qt_sql_where "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -2248,9 +2248,9 @@ suite("test_routine_load","p0") { } if (i <= 3) { - qt_sql_delete "select * from ${tableName1} order by k00,k01" + qt_sql_delete "select * from ${tableName1} order by k00,k01,k02" } else { - qt_sql_delete "select * from ${tableName1} order by k00" + qt_sql_delete "select * from ${tableName1} order by k00,k01,k02" } sql "stop routine load for ${jobs[i]}" @@ -2320,7 +2320,7 @@ suite("test_routine_load","p0") { sleep(5000) count++ } - qt_sql_squence "select * from routine_load_uniq_tbl_basic_sequence order by k00,k01" + qt_sql_squence "select * from routine_load_uniq_tbl_basic_sequence order by k00,k01,k02" sql "stop routine load for ${job}" } finally { sql new File("""${context.file.parent}/ddl/uniq_tbl_basic_drop_sequence.sql""").text From 45c145fdf7b6c350a7f34b80942bd91e99b3a690 Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Mon, 20 May 2024 21:49:53 +0800 Subject: [PATCH 063/111] [fix](Nereids) LogicalPlanDeepCopier copy scan conjuncts in wrong way (#35077) pick from master #35076 intro by PR #34933 This PR attempts to address the issue of losing conjuncts when performing a deep copy of the outer structure. However, the timing of copying the conjuncts is incorrect, resulting in the inability to map slots within the conjuncts to the output of the outer structure. --- .../trees/copier/LogicalPlanDeepCopier.java | 73 +++---------------- .../trees/plans/logical/LogicalEsScan.java | 18 +---- .../logical/LogicalExternalRelation.java | 68 +++++++++++++++++ .../trees/plans/logical/LogicalFileScan.java | 16 +--- .../trees/plans/logical/LogicalJdbcScan.java | 19 +---- .../trees/plans/logical/LogicalOdbcScan.java | 19 +---- .../trees/plans/visitor/RelationVisitor.java | 13 +++- 7 files changed, 102 insertions(+), 124 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java index 9f87d6a60a69bee..277f5ae345a6f84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -39,17 +39,14 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; -import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; -import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; import org.apache.doris.nereids.trees.plans.logical.LogicalIntersect; -import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; import org.apache.doris.nereids.trees.plans.logical.LogicalLimit; -import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPartitionTopN; @@ -187,69 +184,23 @@ public Plan visitLogicalDeferMaterializeOlapScan(LogicalDeferMaterializeOlapScan .collect(ImmutableSet.toImmutableSet()); SlotReference newRowId = (SlotReference) ExpressionDeepCopier.INSTANCE .deepCopy(deferMaterializeOlapScan.getColumnIdSlot(), context); - LogicalDeferMaterializeOlapScan newMaterializeOlapScan = - new LogicalDeferMaterializeOlapScan(newScan, newSlotIds, newRowId); - return newMaterializeOlapScan; + return new LogicalDeferMaterializeOlapScan(newScan, newSlotIds, newRowId); } @Override - public Plan visitLogicalFileScan(LogicalFileScan fileScan, DeepCopierContext context) { - if (context.getRelationReplaceMap().containsKey(fileScan.getRelationId())) { - return context.getRelationReplaceMap().get(fileScan.getRelationId()); - } - Set conjuncts = fileScan.getConjuncts().stream() - .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) - .collect(ImmutableSet.toImmutableSet()); - LogicalFileScan newFileScan = fileScan.withConjuncts(conjuncts) - .withRelationId(StatementScopeIdGenerator.newRelationId()); - updateReplaceMapWithOutput(fileScan, newFileScan, context.exprIdReplaceMap); - context.putRelation(fileScan.getRelationId(), newFileScan); - return newFileScan; - } - - @Override - public Plan visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, DeepCopierContext context) { - if (context.getRelationReplaceMap().containsKey(jdbcScan.getRelationId())) { - return context.getRelationReplaceMap().get(jdbcScan.getRelationId()); - } - Set conjuncts = jdbcScan.getConjuncts().stream() - .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) - .collect(ImmutableSet.toImmutableSet()); - LogicalJdbcScan newJdbcScan = jdbcScan.withConjuncts(conjuncts) - .withRelationId(StatementScopeIdGenerator.newRelationId()); - updateReplaceMapWithOutput(jdbcScan, newJdbcScan, context.exprIdReplaceMap); - context.putRelation(jdbcScan.getRelationId(), newJdbcScan); - return newJdbcScan; - } - - @Override - public Plan visitLogicalOdbcScan(LogicalOdbcScan odbcScan, DeepCopierContext context) { - if (context.getRelationReplaceMap().containsKey(odbcScan.getRelationId())) { - return context.getRelationReplaceMap().get(odbcScan.getRelationId()); - } - Set conjuncts = odbcScan.getConjuncts().stream() - .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) - .collect(ImmutableSet.toImmutableSet()); - LogicalOdbcScan newOdbcScan = odbcScan.withConjuncts(conjuncts) - .withRelationId(StatementScopeIdGenerator.newRelationId()); - updateReplaceMapWithOutput(odbcScan, newOdbcScan, context.exprIdReplaceMap); - context.putRelation(odbcScan.getRelationId(), newOdbcScan); - return newOdbcScan; - } - - @Override - public Plan visitLogicalEsScan(LogicalEsScan esScan, DeepCopierContext context) { - if (context.getRelationReplaceMap().containsKey(esScan.getRelationId())) { - return context.getRelationReplaceMap().get(esScan.getRelationId()); + public Plan visitLogicalExternalRelation(LogicalExternalRelation relation, + DeepCopierContext context) { + if (context.getRelationReplaceMap().containsKey(relation.getRelationId())) { + return context.getRelationReplaceMap().get(relation.getRelationId()); } - Set conjuncts = esScan.getConjuncts().stream() + LogicalExternalRelation newRelation = relation.withRelationId(StatementScopeIdGenerator.newRelationId()); + updateReplaceMapWithOutput(relation, newRelation, context.exprIdReplaceMap); + Set conjuncts = relation.getConjuncts().stream() .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) .collect(ImmutableSet.toImmutableSet()); - LogicalEsScan newEsScan = esScan.withConjuncts(conjuncts) - .withRelationId(StatementScopeIdGenerator.newRelationId()); - updateReplaceMapWithOutput(esScan, newEsScan, context.exprIdReplaceMap); - context.putRelation(esScan.getRelationId(), newEsScan); - return newEsScan; + newRelation = newRelation.withConjuncts(conjuncts); + context.putRelation(relation.getRelationId(), newRelation); + return newRelation; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java index 66882354e204153..ea278aa203c438f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java @@ -31,16 +31,13 @@ import com.google.common.collect.ImmutableSet; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; /** * Logical scan for external es catalog. */ -public class LogicalEsScan extends LogicalCatalogRelation { - - private final Set conjuncts; +public class LogicalEsScan extends LogicalExternalRelation { /** * Constructor for LogicalEsScan. @@ -48,8 +45,7 @@ public class LogicalEsScan extends LogicalCatalogRelation { public LogicalEsScan(RelationId id, ExternalTable table, List qualifier, Optional groupExpression, Optional logicalProperties, Set conjuncts) { - super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties); - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); + super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); } public LogicalEsScan(RelationId id, ExternalTable table, List qualifier) { @@ -83,6 +79,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr conjuncts); } + @Override public LogicalEsScan withConjuncts(Set conjuncts) { return new LogicalEsScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), conjuncts); @@ -98,13 +95,4 @@ public LogicalEsScan withRelationId(RelationId relationId) { public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalEsScan(this, context); } - - @Override - public boolean equals(Object o) { - return super.equals(o) && Objects.equals(conjuncts, ((LogicalEsScan) o).conjuncts); - } - - public Set getConjuncts() { - return this.conjuncts; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java new file mode 100644 index 000000000000000..bc6313f52910447 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; + +import com.google.common.collect.ImmutableSet; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * abstract class catalog relation for logical relation + */ +public abstract class LogicalExternalRelation extends LogicalCatalogRelation { + + // TODO remove this conjuncts when old planner is removed + protected final Set conjuncts; + + public LogicalExternalRelation(RelationId relationId, PlanType type, TableIf table, List qualifier, + Set conjuncts, + Optional groupExpression, Optional logicalProperties) { + super(relationId, type, table, qualifier, groupExpression, logicalProperties); + this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); + } + + public abstract LogicalExternalRelation withConjuncts(Set conjuncts); + + @Override + public abstract LogicalExternalRelation withRelationId(RelationId relationId); + + public Set getConjuncts() { + return conjuncts; + } + + @Override + public boolean equals(Object o) { + return super.equals(o) && Objects.equals(conjuncts, ((LogicalExternalRelation) o).conjuncts); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalExternalRelation(this, context); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index 0b61dd24dac3cff..f7bdae5c2f3bf30 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -42,10 +42,8 @@ /** * Logical file scan for external catalog. */ -public class LogicalFileScan extends LogicalCatalogRelation { +public class LogicalFileScan extends LogicalExternalRelation { - // TODO remove this conjuncts when old planner is removed - private final Set conjuncts; private final SelectedPartitions selectedPartitions; private final Optional tableSample; @@ -55,9 +53,7 @@ public class LogicalFileScan extends LogicalCatalogRelation { public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, Optional groupExpression, Optional logicalProperties, Set conjuncts, SelectedPartitions selectedPartitions, Optional tableSample) { - super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, - groupExpression, logicalProperties); - this.conjuncts = conjuncts; + super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; } @@ -68,10 +64,6 @@ public LogicalFileScan(RelationId id, ExternalTable table, List qualifie Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample); } - public Set getConjuncts() { - return conjuncts; - } - public SelectedPartitions getSelectedPartitions() { return selectedPartitions; } @@ -108,6 +100,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample); } + @Override public LogicalFileScan withConjuncts(Set conjuncts) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample); @@ -131,8 +124,7 @@ public R accept(PlanVisitor visitor, C context) { @Override public boolean equals(Object o) { - return super.equals(o) && Objects.equals(conjuncts, ((LogicalFileScan) o).conjuncts) - && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions); + return super.equals(o) && Objects.equals(selectedPartitions, ((LogicalFileScan) o).selectedPartitions); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java index 1f295f37da7c8d1..cde2b6be242a085 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java @@ -33,16 +33,13 @@ import com.google.common.collect.ImmutableSet; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; /** * Logical scan for external jdbc catalog and jdbc table. */ -public class LogicalJdbcScan extends LogicalCatalogRelation { - - private final Set conjuncts; +public class LogicalJdbcScan extends LogicalExternalRelation { /** * Constructor for LogicalJdbcScan. @@ -51,9 +48,7 @@ public LogicalJdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, Optional logicalProperties, Set conjuncts) { - super(id, PlanType.LOGICAL_JDBC_SCAN, table, qualifier, - groupExpression, logicalProperties); - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); + super(id, PlanType.LOGICAL_JDBC_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); } public LogicalJdbcScan(RelationId id, TableIf table, List qualifier) { @@ -81,6 +76,7 @@ public LogicalJdbcScan withGroupExpression(Optional groupExpres Optional.of(getLogicalProperties()), conjuncts); } + @Override public LogicalJdbcScan withConjuncts(Set conjuncts) { return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), conjuncts); @@ -101,13 +97,4 @@ public LogicalJdbcScan withRelationId(RelationId relationId) { public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalJdbcScan(this, context); } - - @Override - public boolean equals(Object o) { - return super.equals(o) && Objects.equals(conjuncts, ((LogicalJdbcScan) o).conjuncts); - } - - public Set getConjuncts() { - return this.conjuncts; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java index 1e57ad80c45601a..414cb335af15b63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java @@ -32,24 +32,19 @@ import com.google.common.collect.ImmutableSet; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; /** * Logical scan for external odbc table. */ -public class LogicalOdbcScan extends LogicalCatalogRelation { - - private final Set conjuncts; +public class LogicalOdbcScan extends LogicalExternalRelation { public LogicalOdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, Optional logicalProperties, Set conjuncts) { - super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier, - groupExpression, logicalProperties); - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); + super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); } public LogicalOdbcScan(RelationId id, TableIf table, List qualifier) { @@ -77,6 +72,7 @@ public LogicalOdbcScan withGroupExpression(Optional groupExpres Optional.of(getLogicalProperties()), conjuncts); } + @Override public LogicalOdbcScan withConjuncts(Set conjuncts) { return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), conjuncts); @@ -97,13 +93,4 @@ public LogicalOdbcScan withRelationId(RelationId relationId) { public R accept(PlanVisitor visitor, C context) { return visitor.visitLogicalOdbcScan(this, context); } - - @Override - public boolean equals(Object o) { - return super.equals(o) && Objects.equals(conjuncts, ((LogicalOdbcScan) o).conjuncts); - } - - public Set getConjuncts() { - return this.conjuncts; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java index 95f9c6c96d6f730..046964c351d60ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; @@ -92,20 +93,24 @@ default R visitLogicalEmptyRelation(LogicalEmptyRelation emptyRelation, C contex return visitLogicalRelation(emptyRelation, context); } + default R visitLogicalExternalRelation(LogicalExternalRelation relation, C context) { + return visitLogicalCatalogRelation(relation, context); + } + default R visitLogicalEsScan(LogicalEsScan esScan, C context) { - return visitLogicalCatalogRelation(esScan, context); + return visitLogicalExternalRelation(esScan, context); } default R visitLogicalFileScan(LogicalFileScan fileScan, C context) { - return visitLogicalCatalogRelation(fileScan, context); + return visitLogicalExternalRelation(fileScan, context); } default R visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, C context) { - return visitLogicalCatalogRelation(jdbcScan, context); + return visitLogicalExternalRelation(jdbcScan, context); } default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) { - return visitLogicalCatalogRelation(odbcScan, context); + return visitLogicalExternalRelation(odbcScan, context); } default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) { From 8ca399ab920e5719baacf9740f1cd2fad8d5dd6c Mon Sep 17 00:00:00 2001 From: HappenLee Date: Tue, 21 May 2024 12:50:05 +0800 Subject: [PATCH 064/111] [exec](pipeline) runtime filter wait time (#35108) --- be/src/exprs/runtime_filter.h | 2 ++ be/src/pipeline/pipeline_x/dependency.cpp | 21 +++++++++++++++++ be/src/pipeline/pipeline_x/dependency.h | 17 +++++++++----- be/src/vec/exec/runtime_filter_consumer.cpp | 25 ++++++++++++++++++--- 4 files changed, 57 insertions(+), 8 deletions(-) diff --git a/be/src/exprs/runtime_filter.h b/be/src/exprs/runtime_filter.h index 781f7ac34fffce3..4733d39e29893f3 100644 --- a/be/src/exprs/runtime_filter.h +++ b/be/src/exprs/runtime_filter.h @@ -244,6 +244,8 @@ class IRuntimeFilter { bool has_remote_target() const { return _has_remote_target; } + bool has_local_target() const { return _has_local_target; } + bool is_ready() const { return (!_enable_pipeline_exec && _rf_state == RuntimeFilterState::READY) || (_enable_pipeline_exec && diff --git a/be/src/pipeline/pipeline_x/dependency.cpp b/be/src/pipeline/pipeline_x/dependency.cpp index ba47f9355988db2..093e26ff8540e9e 100644 --- a/be/src/pipeline/pipeline_x/dependency.cpp +++ b/be/src/pipeline/pipeline_x/dependency.cpp @@ -125,6 +125,27 @@ Dependency* RuntimeFilterDependency::is_blocked_by(PipelineXTask* task) { return ready ? nullptr : this; } +// should check rf timeout in two case: +// 1. the rf is ready just remove the wait queue +// 2. if the rf have local dependency, the rf should start wait when all local dependency is ready +bool RuntimeFilterTimer::should_be_check_timeout() { + if (!_parent->ready() && !_local_runtime_filter_dependencies.empty()) { + bool all_ready = true; + for (auto& dep : _local_runtime_filter_dependencies) { + if (!dep->ready()) { + all_ready = false; + break; + } + } + if (all_ready) { + _local_runtime_filter_dependencies.clear(); + _registration_time = MonotonicMillis(); + } + return all_ready; + } + return true; +} + void RuntimeFilterTimer::call_timeout() { _parent->set_ready(); } diff --git a/be/src/pipeline/pipeline_x/dependency.h b/be/src/pipeline/pipeline_x/dependency.h index 693bde10f36ce82..525a6dea562963d 100644 --- a/be/src/pipeline/pipeline_x/dependency.h +++ b/be/src/pipeline/pipeline_x/dependency.h @@ -112,6 +112,7 @@ class Dependency : public std::enable_shared_from_this { BasicSharedState* shared_state() { return _shared_state; } void set_shared_state(BasicSharedState* shared_state) { _shared_state = shared_state; } virtual std::string debug_string(int indentation_level = 0); + bool ready() const { return _ready; } // Start the watcher. We use it to count how long this dependency block the current pipeline task. void start_watcher() { _watcher.start(); } @@ -256,11 +257,19 @@ class RuntimeFilterTimer { int64_t registration_time() const { return _registration_time; } int32_t wait_time_ms() const { return _wait_time_ms; } + void set_local_runtime_filter_dependencies( + const std::vector>& deps) { + _local_runtime_filter_dependencies = deps; + } + + bool should_be_check_timeout(); + private: friend struct RuntimeFilterTimerQueue; std::shared_ptr _parent = nullptr; + std::vector> _local_runtime_filter_dependencies; std::mutex _lock; - const int64_t _registration_time; + int64_t _registration_time; const int32_t _wait_time_ms; }; @@ -283,11 +292,9 @@ struct RuntimeFilterTimerQueue { ~RuntimeFilterTimerQueue() = default; RuntimeFilterTimerQueue() { _thread = std::thread(&RuntimeFilterTimerQueue::start, this); } - void push_filter_timer(std::shared_ptr filter) { push(filter); } - - void push(std::shared_ptr filter) { + void push_filter_timer(std::vector>&& filter) { std::unique_lock lc(_que_lock); - _que.push_back(filter); + _que.insert(_que.end(), filter.begin(), filter.end()); cv.notify_all(); } diff --git a/be/src/vec/exec/runtime_filter_consumer.cpp b/be/src/vec/exec/runtime_filter_consumer.cpp index 2913fad3d8d8d1c..66fd0297c980b6d 100644 --- a/be/src/vec/exec/runtime_filter_consumer.cpp +++ b/be/src/vec/exec/runtime_filter_consumer.cpp @@ -81,17 +81,36 @@ void RuntimeFilterConsumer::init_runtime_filter_dependency( runtime_filter_dependencies, const int id, const int node_id, const std::string& name) { runtime_filter_dependencies.resize(_runtime_filter_descs.size()); + std::vector> runtime_filter_timers( + _runtime_filter_descs.size()); + std::vector> + local_runtime_filter_dependencies; + for (size_t i = 0; i < _runtime_filter_descs.size(); ++i) { IRuntimeFilter* runtime_filter = _runtime_filter_ctxs[i].runtime_filter; runtime_filter_dependencies[i] = std::make_shared( id, node_id, name, _state->get_query_ctx(), runtime_filter); _runtime_filter_ctxs[i].runtime_filter_dependency = runtime_filter_dependencies[i].get(); - auto filter_timer = std::make_shared( + runtime_filter_timers[i] = std::make_shared( runtime_filter->registration_time(), runtime_filter->wait_time_ms(), runtime_filter_dependencies[i]); - runtime_filter->set_filter_timer(filter_timer); - ExecEnv::GetInstance()->runtime_filter_timer_queue()->push_filter_timer(filter_timer); + runtime_filter->set_filter_timer(runtime_filter_timers[i]); + if (runtime_filter->has_local_target()) { + local_runtime_filter_dependencies.emplace_back(runtime_filter_dependencies[i]); + } + } + + // The gloabl runtime filter timer need set local runtime filter dependencies. + // start to wait before the local runtime filter ready + for (size_t i = 0; i < _runtime_filter_descs.size(); ++i) { + IRuntimeFilter* runtime_filter = _runtime_filter_ctxs[i].runtime_filter; + if (!runtime_filter->has_local_target()) { + runtime_filter_timers[i]->set_local_runtime_filter_dependencies( + local_runtime_filter_dependencies); + } } + ExecEnv::GetInstance()->runtime_filter_timer_queue()->push_filter_timer( + std::move(runtime_filter_timers)); } Status RuntimeFilterConsumer::_acquire_runtime_filter(bool pipeline_x) { From 26d5c503394d7a5d38178f41cecbe51af19556d4 Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Sun, 19 May 2024 20:47:10 +0800 Subject: [PATCH 065/111] [Bug](Variant) fix incorrect use of column index in TabletSchema (#35019) --- be/src/vec/common/schema_util.cpp | 4 +- .../test_alter_add_drop_column.groovy | 53 +++++++++++++++++++ 2 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 regression-test/suites/variant_p0/schema_change/test_alter_add_drop_column.groovy diff --git a/be/src/vec/common/schema_util.cpp b/be/src/vec/common/schema_util.cpp index 2f9e5ded2126987..49e901e08463c41 100644 --- a/be/src/vec/common/schema_util.cpp +++ b/be/src/vec/common/schema_util.cpp @@ -395,9 +395,9 @@ void inherit_root_attributes(TabletSchemaSPtr& schema) { col.type() != FieldType::OLAP_FIELD_TYPE_DOUBLE && col.type() != FieldType::OLAP_FIELD_TYPE_FLOAT) { // above types are not supported in bf - col.set_is_bf_column(schema->column(col.parent_unique_id()).is_bf_column()); + col.set_is_bf_column(schema->column_by_uid(col.parent_unique_id()).is_bf_column()); } - col.set_aggregation_method(schema->column(col.parent_unique_id()).aggregation()); + col.set_aggregation_method(schema->column_by_uid(col.parent_unique_id()).aggregation()); auto it = variants_index_meta.find(col.parent_unique_id()); // variant has no index meta, ignore if (it == variants_index_meta.end()) { diff --git a/regression-test/suites/variant_p0/schema_change/test_alter_add_drop_column.groovy b/regression-test/suites/variant_p0/schema_change/test_alter_add_drop_column.groovy new file mode 100644 index 000000000000000..6a38ec2804e557d --- /dev/null +++ b/regression-test/suites/variant_p0/schema_change/test_alter_add_drop_column.groovy @@ -0,0 +1,53 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("regression_test_variant_add_drop_column", "variant_type"){ + def table_name = "variant_add_drop_column" + sql "DROP TABLE IF EXISTS ${table_name}" + sql """ + CREATE TABLE IF NOT EXISTS ${table_name} ( + k bigint, + v variant + ) + DUPLICATE KEY(`k`) + DISTRIBUTED BY HASH(k) BUCKETS 1 + properties("replication_num" = "1", "bloom_filter_columns" = "v"); + """ + sql """insert into variant_add_drop_column values (1, '{"a" : 12345,"b" : 2}')""" + + sql "alter table variant_add_drop_column add column v2 variant default null" + sql "alter table variant_add_drop_column add column t1 datetime default null" + sql "alter table variant_add_drop_column add column t2 datetime default null" + sql """insert into variant_add_drop_column values (1, '{"a" : 12345234567,"b" : 2}', '{"xxx" : 1}', "2021-01-01 01:01:01", "2021-01-01 01:01:01")""" + sql "alter table variant_add_drop_column add column i1 int default null" + sql """insert into variant_add_drop_column values (1, '{"a" : 12345,"b" : 2}', '{"xxx" : 1}', "2021-01-01 01:01:01", "2021-01-01 01:01:01", 12345)""" + sql "alter table variant_add_drop_column drop column t1" + sql """insert into variant_add_drop_column values (1, '{"a" : 12345,"b" : 2}', '{"xxx" : 1}', "2021-01-01 01:01:01", 12345)""" + sql "alter table variant_add_drop_column drop column t2" + sql """insert into variant_add_drop_column values (1, '{"a" : 12345,"b" : 2}', '{"xxx" : 1}', 12345)""" + sql "alter table variant_add_drop_column drop column i1" + sql """insert into variant_add_drop_column values (1, '{"a" : 12345,"b" : 2}', '{"xxx" : 1}')""" + sql "alter table variant_add_drop_column drop column v" + sql """insert into variant_add_drop_column values (1, '{"a" : 12345,"b" : 2}')""" + sql "alter table variant_add_drop_column add column v variant default null" + sql """insert into variant_add_drop_column values (1, '{"a" : 12345,"b" : 2}', '{"a" : 12345,"b" : 2}')""" + sql "alter table variant_add_drop_column add column v3 variant default null" + sql """insert into variant_add_drop_column values (1, '{"a" : 12345,"b" : 2}', '{"a" : 12345,"b" : 2}', '{"a" : 12345,"b" : 2}')""" + sql "alter table variant_add_drop_column drop column v" + sql "alter table variant_add_drop_column drop column v2" + sql """insert into variant_add_drop_column values (1, '{"a" : 12345,"b" : 2}')""" +} \ No newline at end of file From 518b143caa4c1a94cc89156558886400b8c00a27 Mon Sep 17 00:00:00 2001 From: minghong Date: Mon, 20 May 2024 10:06:51 +0800 Subject: [PATCH 066/111] [feat](Nereids)choose agg mv in cbo #35020 --- .../org/apache/doris/nereids/cost/CostModelV1.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java index d469dd3b40b179e..87e0f8ab9fabad3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.cost; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.PlanContext; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.DistributionSpecGather; @@ -97,8 +99,17 @@ public Cost visit(Plan plan, PlanContext context) { @Override public Cost visitPhysicalOlapScan(PhysicalOlapScan physicalOlapScan, PlanContext context) { + OlapTable table = physicalOlapScan.getTable(); Statistics statistics = context.getStatisticsWithCheck(); - return CostV1.ofCpu(context.getSessionVariable(), statistics.getRowCount()); + double rows = statistics.getRowCount(); + double aggMvBonus = 0.0; + if (table.getBaseIndexId() != physicalOlapScan.getSelectedIndexId()) { + if (table.getIndexMetaByIndexId(physicalOlapScan.getSelectedIndexId()) + .getKeysType().equals(KeysType.AGG_KEYS)) { + aggMvBonus = rows > 1.0 ? 1.0 : rows * 0.5; + } + } + return CostV1.ofCpu(context.getSessionVariable(), rows - aggMvBonus); } @Override From 5872173901392e89cad84c461504269cf54b2db8 Mon Sep 17 00:00:00 2001 From: zhangstar333 <87313068+zhangstar333@users.noreply.github.com> Date: Mon, 20 May 2024 11:38:02 +0800 Subject: [PATCH 067/111] [improve](function) add limit check for lpad/rpad function input big value of length (#34810) --- be/src/vec/functions/function_string.h | 8 ++++++++ regression-test/data/query_p1/test_big_pad.out | 4 ++++ regression-test/suites/query_p1/test_big_pad.groovy | 7 +++++++ 3 files changed, 19 insertions(+) create mode 100644 regression-test/data/query_p1/test_big_pad.out diff --git a/be/src/vec/functions/function_string.h b/be/src/vec/functions/function_string.h index 6dbd27041876198..fbaed751c7d0b18 100644 --- a/be/src/vec/functions/function_string.h +++ b/be/src/vec/functions/function_string.h @@ -1714,6 +1714,14 @@ class FunctionStringPad : public IFunction { res_chars, res_offsets); continue; } + if (col_len_data[i] > context->state()->repeat_max_num()) { + return Status::InvalidArgument( + " {} function the length argument is {} exceeded maximum default " + "value: {}." + "if you really need this length, you could change the session variable " + "set repeat_max_num = xxx.", + get_name(), col_len_data[i], context->state()->repeat_max_num()); + } // make compatible with mysql. return empty string if pad is empty if (pad_char_size == 0) { diff --git a/regression-test/data/query_p1/test_big_pad.out b/regression-test/data/query_p1/test_big_pad.out new file mode 100644 index 000000000000000..5c622cdf66ce25e --- /dev/null +++ b/regression-test/data/query_p1/test_big_pad.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_rpad -- +15000 + diff --git a/regression-test/suites/query_p1/test_big_pad.groovy b/regression-test/suites/query_p1/test_big_pad.groovy index 635881dc80d113b..b96a380ac8bb5a7 100644 --- a/regression-test/suites/query_p1/test_big_pad.groovy +++ b/regression-test/suites/query_p1/test_big_pad.groovy @@ -30,11 +30,18 @@ suite("test_big_pad") { distributed BY hash(k1) buckets 3 properties("replication_num" = "1"); """ + test { + sql "select rpad('a',15000,'asd');" + exception "rpad function the length argument is 15000 exceeded maximum default value" + } + sql """ set repeat_max_num = 2000000001 """ // default value is 10000 + qt_sql_rpad"select length(rpad('a',15000,'asd'));" sql "insert into d_table values(1,2000000000,1,'a'),(1,2000000000,1,'a'),(1,2000000000,1,'a');" test { sql "select rpad('a',k2,'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') from d_table;" exception "string column length is too large" } + } From f3762322c803b32ae1349a5c9326190f12295377 Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Mon, 20 May 2024 14:08:13 +0800 Subject: [PATCH 068/111] [opt](nereids)new way to set pre-agg status (#34738) --- .../doris/nereids/jobs/executor/Rewriter.java | 4 + .../apache/doris/nereids/rules/RuleType.java | 15 + .../nereids/rules/analysis/BindRelation.java | 2 +- .../rules/rewrite/AdjustPreAggStatus.java | 748 ++++++++++++++++++ .../AbstractSelectMaterializedIndexRule.java | 12 +- .../SelectMaterializedIndexWithAggregate.java | 18 +- ...lectMaterializedIndexWithoutAggregate.java | 20 +- .../nereids/trees/plans/PreAggStatus.java | 15 +- .../trees/plans/logical/LogicalOlapScan.java | 14 +- .../rewrite/mv/SelectRollupIndexTest.java | 30 +- .../nereids/trees/plans/PlanToStringTest.java | 2 +- 11 files changed, 823 insertions(+), 57 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustPreAggStatus.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index b18f8a67a3a8ab9..bb75713e249e2b9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -35,6 +35,7 @@ import org.apache.doris.nereids.rules.rewrite.AddDefaultLimit; import org.apache.doris.nereids.rules.rewrite.AdjustConjunctsReturnType; import org.apache.doris.nereids.rules.rewrite.AdjustNullable; +import org.apache.doris.nereids.rules.rewrite.AdjustPreAggStatus; import org.apache.doris.nereids.rules.rewrite.AggScalarSubQueryToWindowFunction; import org.apache.doris.nereids.rules.rewrite.BuildAggForUnion; import org.apache.doris.nereids.rules.rewrite.CTEInline; @@ -390,6 +391,9 @@ public class Rewriter extends AbstractBatchJobExecutor { bottomUp(RuleSet.PUSH_DOWN_FILTERS), custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT, EliminateUnnecessaryProject::new) ), + topic("adjust preagg status", + topDown(new AdjustPreAggStatus()) + ), topic("topn optimize", topDown(new DeferMaterializeTopNResult()) ), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index c4eb7fe9b063480..14a959b43550e53 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -241,6 +241,21 @@ public enum RuleType { MATERIALIZED_INDEX_PROJECT_SCAN(RuleTypeClass.REWRITE), MATERIALIZED_INDEX_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE), MATERIALIZED_INDEX_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_FILTER_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_PROJECT_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_REPEAT_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_REPEAT_FILTER_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_REPEAT_PROJECT_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_REPEAT_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_AGG_REPEAT_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_FILTER_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_PROJECT_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE), + PREAGG_STATUS_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE), REDUCE_AGGREGATE_CHILD_OUTPUT_ROWS(RuleTypeClass.REWRITE), OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 84a3021ecd1740b..c52f80523a90db9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -206,7 +206,7 @@ private LogicalPlan makeOlapScan(TableIf table, UnboundRelation unboundRelation, } PreAggStatus preAggStatus = olapTable.getIndexMetaByIndexId(indexId).getKeysType().equals(KeysType.DUP_KEYS) - ? PreAggStatus.on() + ? PreAggStatus.unset() : PreAggStatus.off("For direct index scan."); scan = new LogicalOlapScan(unboundRelation.getRelationId(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustPreAggStatus.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustPreAggStatus.java new file mode 100644 index 000000000000000..a0c0b56dd71c994 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustPreAggStatus.java @@ -0,0 +1,748 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.AggregateType; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MaterializedIndexMeta; +import org.apache.doris.common.Pair; +import org.apache.doris.nereids.annotation.Developing; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.CaseWhen; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.VirtualSlotReference; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnion; +import org.apache.doris.nereids.trees.expressions.functions.agg.BitmapUnionCount; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.HllUnion; +import org.apache.doris.nereids.trees.expressions.functions.agg.HllUnionAgg; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PreAggStatus; +import org.apache.doris.nereids.trees.plans.algebra.Project; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * AdjustPreAggStatus + */ +@Developing +public class AdjustPreAggStatus implements RewriteRuleFactory { + /////////////////////////////////////////////////////////////////////////// + // All the patterns + /////////////////////////////////////////////////////////////////////////// + @Override + public List buildRules() { + return ImmutableList.of( + // Aggregate(Scan) + logicalAggregate(logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet)) + .thenApplyNoThrow(ctx -> { + LogicalAggregate agg = ctx.root; + LogicalOlapScan scan = agg.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, Optional.empty()); + List groupByExpressions = agg.getGroupByExpressions(); + Set predicates = ImmutableSet.of(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(scan.withPreAggStatus(preAggStatus)); + }).toRule(RuleType.PREAGG_STATUS_AGG_SCAN), + + // Aggregate(Filter(Scan)) + logicalAggregate( + logicalFilter(logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet))) + .thenApplyNoThrow(ctx -> { + LogicalAggregate> agg = ctx.root; + LogicalFilter filter = agg.child(); + LogicalOlapScan scan = filter.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, Optional.empty()); + List groupByExpressions = + agg.getGroupByExpressions(); + Set predicates = filter.getConjuncts(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(filter + .withChildren(scan.withPreAggStatus(preAggStatus))); + }).toRule(RuleType.PREAGG_STATUS_AGG_FILTER_SCAN), + + // Aggregate(Project(Scan)) + logicalAggregate(logicalProject( + logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet))) + .thenApplyNoThrow(ctx -> { + LogicalAggregate> agg = + ctx.root; + LogicalProject project = agg.child(); + LogicalOlapScan scan = project.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, + Optional.of(project)); + List groupByExpressions = + ExpressionUtils.replace(agg.getGroupByExpressions(), + project.getAliasToProducer()); + Set predicates = ImmutableSet.of(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(project + .withChildren(scan.withPreAggStatus(preAggStatus))); + }).toRule(RuleType.PREAGG_STATUS_AGG_PROJECT_SCAN), + + // Aggregate(Project(Filter(Scan))) + logicalAggregate(logicalProject(logicalFilter( + logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet)))) + .thenApplyNoThrow(ctx -> { + LogicalAggregate>> agg = ctx.root; + LogicalProject> project = agg.child(); + LogicalFilter filter = project.child(); + LogicalOlapScan scan = filter.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, Optional.of(project)); + List groupByExpressions = + ExpressionUtils.replace(agg.getGroupByExpressions(), + project.getAliasToProducer()); + Set predicates = filter.getConjuncts(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(project.withChildren(filter + .withChildren(scan.withPreAggStatus(preAggStatus)))); + }).toRule(RuleType.PREAGG_STATUS_AGG_PROJECT_FILTER_SCAN), + + // Aggregate(Filter(Project(Scan))) + logicalAggregate(logicalFilter(logicalProject( + logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet)))) + .thenApplyNoThrow(ctx -> { + LogicalAggregate>> agg = ctx.root; + LogicalFilter> filter = + agg.child(); + LogicalProject project = filter.child(); + LogicalOlapScan scan = project.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, Optional.of(project)); + List groupByExpressions = + ExpressionUtils.replace(agg.getGroupByExpressions(), + project.getAliasToProducer()); + Set predicates = ExpressionUtils.replace( + filter.getConjuncts(), project.getAliasToProducer()); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(filter.withChildren(project + .withChildren(scan.withPreAggStatus(preAggStatus)))); + }).toRule(RuleType.PREAGG_STATUS_AGG_FILTER_PROJECT_SCAN), + + // Aggregate(Repeat(Scan)) + logicalAggregate( + logicalRepeat(logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet))) + .thenApplyNoThrow(ctx -> { + LogicalAggregate> agg = ctx.root; + LogicalRepeat repeat = agg.child(); + LogicalOlapScan scan = repeat.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, Optional.empty()); + List groupByExpressions = nonVirtualGroupByExprs(agg); + Set predicates = ImmutableSet.of(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(repeat + .withChildren(scan.withPreAggStatus(preAggStatus))); + }).toRule(RuleType.PREAGG_STATUS_AGG_REPEAT_SCAN), + + // Aggregate(Repeat(Filter(Scan))) + logicalAggregate(logicalRepeat(logicalFilter( + logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet)))) + .thenApplyNoThrow(ctx -> { + LogicalAggregate>> agg = ctx.root; + LogicalRepeat> repeat = agg.child(); + LogicalFilter filter = repeat.child(); + LogicalOlapScan scan = filter.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, Optional.empty()); + List groupByExpressions = + nonVirtualGroupByExprs(agg); + Set predicates = filter.getConjuncts(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(repeat.withChildren(filter + .withChildren(scan.withPreAggStatus(preAggStatus)))); + }).toRule(RuleType.PREAGG_STATUS_AGG_REPEAT_FILTER_SCAN), + + // Aggregate(Repeat(Project(Scan))) + logicalAggregate(logicalRepeat(logicalProject( + logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet)))) + .thenApplyNoThrow(ctx -> { + LogicalAggregate>> agg = ctx.root; + LogicalRepeat> repeat = agg.child(); + LogicalProject project = repeat.child(); + LogicalOlapScan scan = project.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, Optional.empty()); + List groupByExpressions = + ExpressionUtils.replace(nonVirtualGroupByExprs(agg), + project.getAliasToProducer()); + Set predicates = ImmutableSet.of(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(repeat.withChildren(project + .withChildren(scan.withPreAggStatus(preAggStatus)))); + }).toRule(RuleType.PREAGG_STATUS_AGG_REPEAT_PROJECT_SCAN), + + // Aggregate(Repeat(Project(Filter(Scan)))) + logicalAggregate(logicalRepeat(logicalProject(logicalFilter( + logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet))))) + .thenApplyNoThrow(ctx -> { + LogicalAggregate>>> agg + = ctx.root; + LogicalRepeat>> repeat = agg.child(); + LogicalProject> project = repeat.child(); + LogicalFilter filter = project.child(); + LogicalOlapScan scan = filter.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, Optional.empty()); + List groupByExpressions = + ExpressionUtils.replace(nonVirtualGroupByExprs(agg), + project.getAliasToProducer()); + Set predicates = filter.getConjuncts(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(repeat + .withChildren(project.withChildren(filter.withChildren( + scan.withPreAggStatus(preAggStatus))))); + }).toRule(RuleType.PREAGG_STATUS_AGG_REPEAT_PROJECT_FILTER_SCAN), + + // Aggregate(Repeat(Filter(Project(Scan)))) + logicalAggregate(logicalRepeat(logicalFilter(logicalProject( + logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet))))) + .thenApplyNoThrow(ctx -> { + LogicalAggregate>>> agg + = ctx.root; + LogicalRepeat>> repeat = agg.child(); + LogicalFilter> filter = repeat.child(); + LogicalProject project = filter.child(); + LogicalOlapScan scan = project.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = + extractAggFunctionAndReplaceSlot(agg, Optional.of(project)); + List groupByExpressions = + ExpressionUtils.replace(nonVirtualGroupByExprs(agg), + project.getAliasToProducer()); + Set predicates = ExpressionUtils.replace( + filter.getConjuncts(), project.getAliasToProducer()); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return agg.withChildren(repeat + .withChildren(filter.withChildren(project.withChildren( + scan.withPreAggStatus(preAggStatus))))); + }).toRule(RuleType.PREAGG_STATUS_AGG_REPEAT_FILTER_PROJECT_SCAN), + + // Filter(Project(Scan)) + logicalFilter(logicalProject( + logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet))) + .thenApplyNoThrow(ctx -> { + LogicalFilter> filter = ctx.root; + LogicalProject project = filter.child(); + LogicalOlapScan scan = project.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = ImmutableList.of(); + List groupByExpressions = ImmutableList.of(); + Set predicates = ExpressionUtils.replace( + filter.getConjuncts(), project.getAliasToProducer()); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return filter.withChildren(project + .withChildren(scan.withPreAggStatus(preAggStatus))); + }).toRule(RuleType.PREAGG_STATUS_FILTER_PROJECT_SCAN), + + // Filter(Scan) + logicalFilter(logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet)) + .thenApplyNoThrow(ctx -> { + LogicalFilter filter = ctx.root; + LogicalOlapScan scan = filter.child(); + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = ImmutableList.of(); + List groupByExpressions = ImmutableList.of(); + Set predicates = filter.getConjuncts(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return filter.withChildren(scan.withPreAggStatus(preAggStatus)); + }).toRule(RuleType.PREAGG_STATUS_FILTER_SCAN), + + // only scan. + logicalOlapScan().when(LogicalOlapScan::isPreAggStatusUnSet) + .thenApplyNoThrow(ctx -> { + LogicalOlapScan scan = ctx.root; + PreAggStatus preAggStatus = checkKeysType(scan); + if (preAggStatus == PreAggStatus.unset()) { + List aggregateFunctions = ImmutableList.of(); + List groupByExpressions = ImmutableList.of(); + Set predicates = ImmutableSet.of(); + preAggStatus = checkPreAggStatus(scan, predicates, + aggregateFunctions, groupByExpressions); + } + return scan.withPreAggStatus(preAggStatus); + }).toRule(RuleType.PREAGG_STATUS_SCAN)); + } + + /////////////////////////////////////////////////////////////////////////// + // Set pre-aggregation status. + /////////////////////////////////////////////////////////////////////////// + + /** + * Do aggregate function extraction and replace aggregate function's input slots by underlying project. + *

+ * 1. extract aggregate functions in aggregate plan. + *

+ * 2. replace aggregate function's input slot by underlying project expression if project is present. + *

+ * For example: + * 

+     * input arguments:
+     * agg: Aggregate(sum(v) as sum_value)
+     * underlying project: Project(a + b as v)
+     *
+     * output:
+     * sum(a + b)
+     *
+ */ + private List extractAggFunctionAndReplaceSlot(LogicalAggregate agg, + Optional> project) { + Optional> slotToProducerOpt = + project.map(Project::getAliasToProducer); + return agg.getOutputExpressions().stream() + // extract aggregate functions. + .flatMap(e -> e.>collect(AggregateFunction.class::isInstance) + .stream()) + // replace aggregate function's input slot by its producing expression. + .map(expr -> slotToProducerOpt + .map(slotToExpressions -> (AggregateFunction) ExpressionUtils.replace(expr, + slotToExpressions)) + .orElse(expr)) + .collect(Collectors.toList()); + } + + private PreAggStatus checkKeysType(LogicalOlapScan olapScan) { + long selectIndexId = olapScan.getSelectedIndexId(); + MaterializedIndexMeta meta = olapScan.getTable().getIndexMetaByIndexId(selectIndexId); + if (meta.getKeysType() == KeysType.DUP_KEYS || (meta.getKeysType() == KeysType.UNIQUE_KEYS + && olapScan.getTable().getEnableUniqueKeyMergeOnWrite())) { + return PreAggStatus.on(); + } else { + return PreAggStatus.unset(); + } + } + + private PreAggStatus checkPreAggStatus(LogicalOlapScan olapScan, Set predicates, + List aggregateFuncs, List groupingExprs) { + Set outputSlots = olapScan.getOutputSet(); + Pair, Set> splittedSlots = splitSlots(outputSlots); + Set keySlots = splittedSlots.first; + Set valueSlots = splittedSlots.second; + Preconditions.checkState(outputSlots.size() == keySlots.size() + valueSlots.size(), + "output slots contains no key or value slots"); + + Set groupingExprsInputSlots = ExpressionUtils.getInputSlotSet(groupingExprs); + if (groupingExprsInputSlots.retainAll(keySlots)) { + return PreAggStatus + .off(String.format("Grouping expression %s contains non-key column %s", + groupingExprs, groupingExprsInputSlots)); + } + + Set predicateInputSlots = ExpressionUtils.getInputSlotSet(predicates); + if (predicateInputSlots.retainAll(keySlots)) { + return PreAggStatus.off(String.format("Predicate %s contains non-key column %s", + predicates, predicateInputSlots)); + } + + return checkAggregateFunctions(aggregateFuncs, groupingExprsInputSlots); + } + + private Pair, Set> splitSlots(Set slots) { + Set keySlots = Sets.newHashSetWithExpectedSize(slots.size()); + Set valueSlots = Sets.newHashSetWithExpectedSize(slots.size()); + for (Slot slot : slots) { + if (slot instanceof SlotReference && ((SlotReference) slot).getColumn().isPresent()) { + if (((SlotReference) slot).getColumn().get().isKey()) { + keySlots.add((SlotReference) slot); + } else { + valueSlots.add((SlotReference) slot); + } + } + } + return Pair.of(keySlots, valueSlots); + } + + private static Expression removeCast(Expression expression) { + while (expression instanceof Cast) { + expression = ((Cast) expression).child(); + } + return expression; + } + + private PreAggStatus checkAggWithKeyAndValueSlots(AggregateFunction aggFunc, + Set keySlots, Set valueSlots) { + Expression child = aggFunc.child(0); + List conditionExps = new ArrayList<>(); + List returnExps = new ArrayList<>(); + + // ignore cast + while (child instanceof Cast) { + if (!((Cast) child).getDataType().isNumericType()) { + return PreAggStatus.off(String.format("%s is not numeric CAST.", child.toSql())); + } + child = child.child(0); + } + // step 1: extract all condition exprs and return exprs + if (child instanceof If) { + conditionExps.add(child.child(0)); + returnExps.add(removeCast(child.child(1))); + returnExps.add(removeCast(child.child(2))); + } else if (child instanceof CaseWhen) { + CaseWhen caseWhen = (CaseWhen) child; + // WHEN THEN + for (WhenClause whenClause : caseWhen.getWhenClauses()) { + conditionExps.add(whenClause.getOperand()); + returnExps.add(removeCast(whenClause.getResult())); + } + // ELSE + returnExps.add(removeCast(caseWhen.getDefaultValue().orElse(new NullLiteral()))); + } else { + // currently, only IF and CASE WHEN are supported + returnExps.add(removeCast(child)); + } + + // step 2: check condition expressions + Set inputSlots = ExpressionUtils.getInputSlotSet(conditionExps); + inputSlots.retainAll(valueSlots); + if (!inputSlots.isEmpty()) { + return PreAggStatus + .off(String.format("some columns in condition %s is not key.", conditionExps)); + } + + return KeyAndValueSlotsAggChecker.INSTANCE.check(aggFunc, returnExps); + } + + private PreAggStatus checkAggregateFunctions(List aggregateFuncs, + Set groupingExprsInputSlots) { + PreAggStatus preAggStatus = aggregateFuncs.isEmpty() && groupingExprsInputSlots.isEmpty() + ? PreAggStatus.off("No aggregate on scan.") + : PreAggStatus.on(); + for (AggregateFunction aggFunc : aggregateFuncs) { + if (aggFunc.children().size() == 1 && aggFunc.child(0) instanceof Slot) { + Slot aggSlot = (Slot) aggFunc.child(0); + if (aggSlot instanceof SlotReference + && ((SlotReference) aggSlot).getColumn().isPresent()) { + if (((SlotReference) aggSlot).getColumn().get().isKey()) { + preAggStatus = OneKeySlotAggChecker.INSTANCE.check(aggFunc); + } else { + preAggStatus = OneValueSlotAggChecker.INSTANCE.check(aggFunc, + ((SlotReference) aggSlot).getColumn().get().getAggregationType()); + } + } else { + preAggStatus = PreAggStatus.off( + String.format("aggregate function %s use unknown slot %s from scan", + aggFunc, aggSlot)); + } + } else { + Set aggSlots = aggFunc.getInputSlots(); + Pair, Set> splitSlots = splitSlots(aggSlots); + preAggStatus = + checkAggWithKeyAndValueSlots(aggFunc, splitSlots.first, splitSlots.second); + } + if (preAggStatus.isOff()) { + return preAggStatus; + } + } + return preAggStatus; + } + + private List nonVirtualGroupByExprs(LogicalAggregate agg) { + return agg.getGroupByExpressions().stream() + .filter(expr -> !(expr instanceof VirtualSlotReference)) + .collect(ImmutableList.toImmutableList()); + } + + private static class OneValueSlotAggChecker + extends ExpressionVisitor { + public static final OneValueSlotAggChecker INSTANCE = new OneValueSlotAggChecker(); + + public PreAggStatus check(AggregateFunction aggFun, AggregateType aggregateType) { + return aggFun.accept(INSTANCE, aggregateType); + } + + @Override + public PreAggStatus visit(Expression expr, AggregateType aggregateType) { + return PreAggStatus.off(String.format("%s is not aggregate function.", expr.toSql())); + } + + @Override + public PreAggStatus visitAggregateFunction(AggregateFunction aggregateFunction, + AggregateType aggregateType) { + return PreAggStatus + .off(String.format("%s is not supported.", aggregateFunction.toSql())); + } + + @Override + public PreAggStatus visitMax(Max max, AggregateType aggregateType) { + if (aggregateType == AggregateType.MAX && !max.isDistinct()) { + return PreAggStatus.on(); + } else { + return PreAggStatus + .off(String.format("%s is not match agg mode %s or has distinct param", + max.toSql(), aggregateType)); + } + } + + @Override + public PreAggStatus visitMin(Min min, AggregateType aggregateType) { + if (aggregateType == AggregateType.MIN && !min.isDistinct()) { + return PreAggStatus.on(); + } else { + return PreAggStatus + .off(String.format("%s is not match agg mode %s or has distinct param", + min.toSql(), aggregateType)); + } + } + + @Override + public PreAggStatus visitSum(Sum sum, AggregateType aggregateType) { + if (aggregateType == AggregateType.SUM && !sum.isDistinct()) { + return PreAggStatus.on(); + } else { + return PreAggStatus + .off(String.format("%s is not match agg mode %s or has distinct param", + sum.toSql(), aggregateType)); + } + } + + @Override + public PreAggStatus visitBitmapUnionCount(BitmapUnionCount bitmapUnionCount, + AggregateType aggregateType) { + if (aggregateType == AggregateType.BITMAP_UNION) { + return PreAggStatus.on(); + } else { + return PreAggStatus.off("invalid bitmap_union_count: " + bitmapUnionCount.toSql()); + } + } + + @Override + public PreAggStatus visitBitmapUnion(BitmapUnion bitmapUnion, AggregateType aggregateType) { + if (aggregateType == AggregateType.BITMAP_UNION) { + return PreAggStatus.on(); + } else { + return PreAggStatus.off("invalid bitmapUnion: " + bitmapUnion.toSql()); + } + } + + @Override + public PreAggStatus visitHllUnionAgg(HllUnionAgg hllUnionAgg, AggregateType aggregateType) { + if (aggregateType == AggregateType.HLL_UNION) { + return PreAggStatus.on(); + } else { + return PreAggStatus.off("invalid hllUnionAgg: " + hllUnionAgg.toSql()); + } + } + + @Override + public PreAggStatus visitHllUnion(HllUnion hllUnion, AggregateType aggregateType) { + if (aggregateType == AggregateType.HLL_UNION) { + return PreAggStatus.on(); + } else { + return PreAggStatus.off("invalid hllUnion: " + hllUnion.toSql()); + } + } + } + + private static class OneKeySlotAggChecker extends ExpressionVisitor { + public static final OneKeySlotAggChecker INSTANCE = new OneKeySlotAggChecker(); + + public PreAggStatus check(AggregateFunction aggFun) { + return aggFun.accept(INSTANCE, null); + } + + @Override + public PreAggStatus visit(Expression expr, Void context) { + return PreAggStatus.off(String.format("%s is not aggregate function.", expr.toSql())); + } + + @Override + public PreAggStatus visitAggregateFunction(AggregateFunction aggregateFunction, + Void context) { + return PreAggStatus.off(String.format("Aggregate function %s contains key column %s", + aggregateFunction.toSql(), aggregateFunction.child(0).toSql())); + } + + @Override + public PreAggStatus visitMax(Max max, Void context) { + return PreAggStatus.on(); + } + + @Override + public PreAggStatus visitMin(Min min, Void context) { + return PreAggStatus.on(); + } + + @Override + public PreAggStatus visitCount(Count count, Void context) { + if (count.isDistinct()) { + return PreAggStatus.on(); + } else { + return PreAggStatus.off(String.format("%s is not distinct.", count.toSql())); + } + } + } + + private static class KeyAndValueSlotsAggChecker + extends ExpressionVisitor> { + public static final KeyAndValueSlotsAggChecker INSTANCE = new KeyAndValueSlotsAggChecker(); + + public PreAggStatus check(AggregateFunction aggFun, List returnValues) { + return aggFun.accept(INSTANCE, returnValues); + } + + @Override + public PreAggStatus visit(Expression expr, List returnValues) { + return PreAggStatus.off(String.format("%s is not aggregate function.", expr.toSql())); + } + + @Override + public PreAggStatus visitAggregateFunction(AggregateFunction aggregateFunction, + List returnValues) { + return PreAggStatus + .off(String.format("%s is not supported.", aggregateFunction.toSql())); + } + + @Override + public PreAggStatus visitSum(Sum sum, List returnValues) { + for (Expression value : returnValues) { + if (!(isAggTypeMatched(value, AggregateType.SUM) || value.isZeroLiteral() + || value.isNullLiteral())) { + return PreAggStatus.off(String.format("%s is not supported.", sum.toSql())); + } + } + return PreAggStatus.on(); + } + + @Override + public PreAggStatus visitMax(Max max, List returnValues) { + for (Expression value : returnValues) { + if (!(isAggTypeMatched(value, AggregateType.MAX) || isKeySlot(value) + || value.isNullLiteral())) { + return PreAggStatus.off(String.format("%s is not supported.", max.toSql())); + } + } + return PreAggStatus.on(); + } + + @Override + public PreAggStatus visitMin(Min min, List returnValues) { + for (Expression value : returnValues) { + if (!(isAggTypeMatched(value, AggregateType.MIN) || isKeySlot(value) + || value.isNullLiteral())) { + return PreAggStatus.off(String.format("%s is not supported.", min.toSql())); + } + } + return PreAggStatus.on(); + } + + @Override + public PreAggStatus visitCount(Count count, List returnValues) { + if (count.isDistinct()) { + for (Expression value : returnValues) { + if (!(isKeySlot(value) || value.isZeroLiteral() || value.isNullLiteral())) { + return PreAggStatus + .off(String.format("%s is not supported.", count.toSql())); + } + } + return PreAggStatus.on(); + } else { + return PreAggStatus.off(String.format("%s is not supported.", count.toSql())); + } + } + + private boolean isKeySlot(Expression expression) { + return expression instanceof SlotReference + && ((SlotReference) expression).getColumn().isPresent() + && ((SlotReference) expression).getColumn().get().isKey(); + } + + private boolean isAggTypeMatched(Expression expression, AggregateType aggregateType) { + return expression instanceof SlotReference + && ((SlotReference) expression).getColumn().isPresent() + && ((SlotReference) expression).getColumn().get() + .getAggregationType() == aggregateType; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java index 03bef1a6b47f81e..f1b2d1233cff2a5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/AbstractSelectMaterializedIndexRule.java @@ -88,17 +88,7 @@ protected boolean shouldSelectIndexWithAgg(LogicalOlapScan scan) { case AGG_KEYS: case UNIQUE_KEYS: case DUP_KEYS: - // SelectMaterializedIndexWithAggregate(R1) run before SelectMaterializedIndexWithoutAggregate(R2) - // if R1 selects baseIndex and preAggStatus is off - // we should give a chance to R2 to check if some prefix-index can be selected - // so if R1 selects baseIndex and preAggStatus is off, we keep scan's index unselected in order to - // let R2 to get a chance to do its work - // at last, after R1, the scan may be the 4 status - // 1. preAggStatus is ON and baseIndex is selected, it means select baseIndex is correct. - // 2. preAggStatus is ON and some other Index is selected, this is correct, too. - // 3. preAggStatus is OFF, no index is selected, it means R2 could get a chance to run - // so we check the preAggStatus and if some index is selected to make sure R1 can be run only once - return scan.getPreAggStatus().isOn() && !scan.isIndexSelected(); + return !scan.isIndexSelected(); default: return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java index f28b3952fa9309a..cb03a0c5840116d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java @@ -211,7 +211,7 @@ public List buildRules() { result.exprRewriteMap.projectExprMap); LogicalProject newProject = new LogicalProject<>( generateNewOutputsWithMvOutputs(mvPlan, newProjectList), - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); + scan.withMaterializedIndexSelected(result.indexId)); return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext), new ReplaceExpressions(slotContext) .replace( @@ -259,9 +259,6 @@ public List buildRules() { filter.getExpressions(), project.getExpressions() )) ); - if (mvPlanWithoutAgg.getSelectedIndexId() == result.indexId) { - mvPlanWithoutAgg = mvPlanWithoutAgg.withPreAggStatus(result.preAggStatus); - } SlotContext slotContextWithoutAgg = generateBaseScanExprToMvExpr(mvPlanWithoutAgg); return agg.withChildren(new LogicalProject( @@ -535,7 +532,7 @@ public List buildRules() { result.exprRewriteMap.projectExprMap); LogicalProject newProject = new LogicalProject<>( generateNewOutputsWithMvOutputs(mvPlan, newProjectList), - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId)); + scan.withMaterializedIndexSelected(result.indexId)); return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext), new ReplaceExpressions(slotContext).replace(new LogicalAggregate<>( @@ -552,16 +549,7 @@ public List buildRules() { } private static LogicalOlapScan createLogicalOlapScan(LogicalOlapScan scan, SelectResult result) { - LogicalOlapScan mvPlan; - if (result.preAggStatus.isOff()) { - // we only set preAggStatus and make index unselected to let SelectMaterializedIndexWithoutAggregate - // have a chance to run and select proper index - mvPlan = scan.withPreAggStatus(result.preAggStatus); - } else { - mvPlan = - scan.withMaterializedIndexSelected(result.preAggStatus, result.indexId); - } - return mvPlan; + return scan.withMaterializedIndexSelected(result.indexId); } /////////////////////////////////////////////////////////////////////////// diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java index e05a1eda3e63fb7..f99eff25fcc3aaa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithoutAggregate.java @@ -27,7 +27,6 @@ import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; @@ -185,7 +184,7 @@ public static LogicalOlapScan select( break; case DUP_KEYS: if (table.getIndexIdToMeta().size() == 1) { - return scan.withMaterializedIndexSelected(PreAggStatus.on(), baseIndexId); + return scan.withMaterializedIndexSelected(baseIndexId); } break; default: @@ -210,19 +209,10 @@ public static LogicalOlapScan select( // this is fail-safe for select mv // select baseIndex if bestIndex's slots' data types are different from baseIndex bestIndex = isSameDataType(scan, bestIndex, requiredSlots.get()) ? bestIndex : baseIndexId; - return scan.withMaterializedIndexSelected(PreAggStatus.on(), bestIndex); + return scan.withMaterializedIndexSelected(bestIndex); } else { - final PreAggStatus preAggStatus; - if (preAggEnabledByHint(scan)) { - // PreAggStatus could be enabled by pre-aggregation hint for agg-keys and unique-keys. - preAggStatus = PreAggStatus.on(); - } else { - // if PreAggStatus is OFF, we use the message from SelectMaterializedIndexWithAggregate - preAggStatus = scan.getPreAggStatus().isOff() ? scan.getPreAggStatus() - : PreAggStatus.off("No aggregate on scan."); - } if (table.getIndexIdToMeta().size() == 1) { - return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId); + return scan.withMaterializedIndexSelected(baseIndexId); } int baseIndexKeySize = table.getKeyColumnsByIndexId(table.getBaseIndexId()).size(); // No aggregate on scan. @@ -235,13 +225,13 @@ public static LogicalOlapScan select( if (candidates.size() == 1) { // `candidates` only have base index. - return scan.withMaterializedIndexSelected(preAggStatus, baseIndexId); + return scan.withMaterializedIndexSelected(baseIndexId); } else { long bestIndex = selectBestIndex(candidates, scan, predicatesSupplier.get()); // this is fail-safe for select mv // select baseIndex if bestIndex's slots' data types are different from baseIndex bestIndex = isSameDataType(scan, bestIndex, requiredSlots.get()) ? bestIndex : baseIndexId; - return scan.withMaterializedIndexSelected(preAggStatus, bestIndex); + return scan.withMaterializedIndexSelected(bestIndex); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PreAggStatus.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PreAggStatus.java index 7affac49b2bc097..8ba99c2c07f0eb1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PreAggStatus.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PreAggStatus.java @@ -26,10 +26,11 @@ public class PreAggStatus { private enum Status { - ON, OFF + ON, OFF, UNSET } private static final PreAggStatus PRE_AGG_ON = new PreAggStatus(Status.ON, ""); + private static final PreAggStatus PRE_AGG_UNSET = new PreAggStatus(Status.UNSET, ""); private final Status status; private final String offReason; @@ -46,6 +47,10 @@ public boolean isOff() { return status == Status.OFF; } + public boolean isUnset() { + return status == Status.UNSET; + } + public String getOffReason() { return offReason; } @@ -58,6 +63,10 @@ public PreAggStatus offOrElse(Supplier supplier) { } } + public static PreAggStatus unset() { + return PRE_AGG_UNSET; + } + public static PreAggStatus on() { return PRE_AGG_ON; } @@ -70,8 +79,10 @@ public static PreAggStatus off(String reason) { public String toString() { if (status == Status.ON) { return "ON"; - } else { + } else if (status == Status.OFF) { return "OFF, " + offReason; + } else { + return "UNSET"; } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index d0d91f1cf8dafb6..714f540524f1a84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -126,7 +126,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, ImmutableList.of(), - -1, false, PreAggStatus.on(), ImmutableList.of(), ImmutableList.of(), + -1, false, PreAggStatus.unset(), ImmutableList.of(), ImmutableList.of(), Maps.newHashMap(), Optional.empty(), false, false); } @@ -134,7 +134,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, L List hints, Optional tableSample) { this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, tabletIds, - -1, false, PreAggStatus.on(), ImmutableList.of(), hints, Maps.newHashMap(), + -1, false, PreAggStatus.unset(), ImmutableList.of(), hints, Maps.newHashMap(), tableSample, false, false); } @@ -143,7 +143,7 @@ public LogicalOlapScan(RelationId id, OlapTable table, List qualifier, L this(id, table, qualifier, Optional.empty(), Optional.empty(), // must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1' specifiedPartitions, false, tabletIds, - -1, false, PreAggStatus.on(), specifiedPartitions, hints, Maps.newHashMap(), + -1, false, PreAggStatus.unset(), specifiedPartitions, hints, Maps.newHashMap(), tableSample, false, false); } @@ -275,11 +275,11 @@ public LogicalOlapScan withSelectedPartitionIds(List selectedPartitionIds) hints, cacheSlotWithSlotName, tableSample, directMvScan, projectPulledUp); } - public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, long indexId) { + public LogicalOlapScan withMaterializedIndexSelected(long indexId) { return new LogicalOlapScan(relationId, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, - indexId, true, preAgg, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, + indexId, true, PreAggStatus.unset(), manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, projectPulledUp); } @@ -432,6 +432,10 @@ public boolean isDirectMvScan() { return directMvScan; } + public boolean isPreAggStatusUnSet() { + return preAggStatus.isUnset(); + } + private List createSlotsVectorized(List columns) { List qualified = qualified(); Object[] slots = new Object[columns.size()]; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectRollupIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectRollupIndexTest.java index 0686edba64e01e6..45552bfc2fae6eb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectRollupIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/mv/SelectRollupIndexTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.nereids.rules.analysis.LogicalSubQueryAliasToLogicalProject; +import org.apache.doris.nereids.rules.rewrite.AdjustPreAggStatus; import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject; import org.apache.doris.nereids.trees.plans.PreAggStatus; @@ -110,6 +111,7 @@ public void testMatchingBase() { PlanChecker.from(connectContext) .analyze(" select k1, sum(v1) from t group by k1") .applyTopDown(new SelectMaterializedIndexWithAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { Assertions.assertTrue(scan.getPreAggStatus().isOn()); Assertions.assertEquals("t", scan.getSelectedMaterializedIndexName().get()); @@ -122,6 +124,7 @@ void testAggFilterScan() { PlanChecker.from(connectContext) .analyze("select k2, sum(v1) from t where k3=0 group by k2") .applyTopDown(new SelectMaterializedIndexWithAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { Assertions.assertTrue(scan.getPreAggStatus().isOn()); Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName().get()); @@ -139,8 +142,7 @@ void testTranslate() { public void testTranslateWhenPreAggIsOff() { singleTableTest("select k2, min(v1) from t group by k2", scan -> { Assertions.assertFalse(scan.isPreAggregation()); - Assertions.assertEquals("Aggregate operator don't match, " - + "aggregate function: min(v1), column aggregate type: SUM", + Assertions.assertEquals("min(v1) is not match agg mode SUM or has distinct param", scan.getReasonOfPreAggregation()); }); } @@ -150,6 +152,7 @@ public void testWithEqualFilter() { PlanChecker.from(connectContext) .analyze("select k2, sum(v1) from t where k3=0 group by k2") .applyTopDown(new SelectMaterializedIndexWithAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { Assertions.assertTrue(scan.getPreAggStatus().isOn()); Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName().get()); @@ -162,6 +165,7 @@ public void testWithNonEqualFilter() { PlanChecker.from(connectContext) .analyze("select k2, sum(v1) from t where k3>0 group by k2") .applyTopDown(new SelectMaterializedIndexWithAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { Assertions.assertTrue(scan.getPreAggStatus().isOn()); Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName().get()); @@ -174,6 +178,7 @@ public void testWithFilter() { PlanChecker.from(connectContext) .analyze("select k2, sum(v1) from t where k2>3 group by k2") .applyTopDown(new SelectMaterializedIndexWithAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { Assertions.assertTrue(scan.getPreAggStatus().isOn()); Assertions.assertEquals("r1", scan.getSelectedMaterializedIndexName().get()); @@ -193,6 +198,7 @@ public void testWithFilterAndProject() { .applyBottomUp(new MergeProjects()) .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { Assertions.assertTrue(scan.getPreAggStatus().isOn()); Assertions.assertEquals("r2", scan.getSelectedMaterializedIndexName().get()); @@ -210,6 +216,7 @@ public void testNoAggregate() { .analyze("select k1, v1 from t") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOff()); @@ -224,11 +231,11 @@ public void testAggregateTypeNotMatch() { .analyze("select k1, min(v1) from t group by k1") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOff()); - Assertions.assertEquals("Aggregate operator don't match, " - + "aggregate function: min(v1), column aggregate type: SUM", preAgg.getOffReason()); + Assertions.assertEquals("min(v1) is not match agg mode SUM or has distinct param", preAgg.getOffReason()); return true; })); } @@ -239,10 +246,11 @@ public void testInvalidSlotInAggFunction() { .analyze("select k1, sum(v1 + 1) from t group by k1") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOff()); - Assertions.assertEquals("do not support compound expression [(v1 + 1)] in SUM.", + Assertions.assertEquals("sum((v1 + 1)) is not supported.", preAgg.getOffReason()); return true; })); @@ -254,10 +262,11 @@ public void testKeyColumnInAggFunction() { .analyze("select k1, sum(k2) from t group by k1") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOff()); - Assertions.assertEquals("Aggregate function sum(k2) contains key column k2.", + Assertions.assertEquals("Aggregate function sum(k2) contains key column k2", preAgg.getOffReason()); return true; })); @@ -269,6 +278,7 @@ public void testMaxCanUseKeyColumn() { .analyze("select k2, max(k3) from t group by k2") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOn()); @@ -283,6 +293,7 @@ public void testMinCanUseKeyColumn() { .analyze("select k2, min(k3) from t group by k2") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOn()); @@ -297,6 +308,7 @@ public void testMinMaxCanUseKeyColumnWithBaseTable() { .analyze("select k1, min(k2), max(k2) from t group by k1") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOn()); @@ -311,6 +323,8 @@ public void testFilterAggWithBaseTable() { .analyze("select k1 from t where k1 = 0 group by k1") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new MergeProjects()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOn()); @@ -325,6 +339,7 @@ public void testDuplicatePreAggOn() { .analyze("select k1, sum(k1) from duplicate_tbl group by k1") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOn()); @@ -338,6 +353,7 @@ public void testDuplicatePreAggOnEvenWithoutAggregate() { .analyze("select k1, v1 from duplicate_tbl") .applyTopDown(new SelectMaterializedIndexWithAggregate()) .applyTopDown(new SelectMaterializedIndexWithoutAggregate()) + .applyTopDown(new AdjustPreAggStatus()) .matches(logicalOlapScan().when(scan -> { PreAggStatus preAgg = scan.getPreAggStatus(); Assertions.assertTrue(preAgg.isOn()); @@ -402,7 +418,7 @@ public void testCountDistinctKeyColumn() { public void testCountDistinctValueColumn() { singleTableTest("select k1, count(distinct v1) from t group by k1", scan -> { Assertions.assertFalse(scan.isPreAggregation()); - Assertions.assertEquals("Count distinct is only valid for key columns, but meet count(DISTINCT v1).", + Assertions.assertEquals("count(DISTINCT v1) is not supported.", scan.getReasonOfPreAggregation()); Assertions.assertEquals("t", scan.getSelectedIndexName()); }); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java index 44cb6c296af30de..0a6eb7e0c592eff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanToStringTest.java @@ -84,7 +84,7 @@ public void testLogicalOlapScan() { Assertions.assertTrue( plan.toString().matches("LogicalOlapScan \\( qualified=db\\.table, " + "indexName=, " - + "selectedIndexId=-1, preAgg=ON \\)")); + + "selectedIndexId=-1, preAgg=UNSET \\)")); } @Test From b4a798240a84dfb9b7b5953f45de14c4c68e2aaf Mon Sep 17 00:00:00 2001 From: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com> Date: Mon, 20 May 2024 20:05:23 +0800 Subject: [PATCH 069/111] [fix](inverted_index) donot use int32_t for index id to avoid overflow (#35062) --- be/src/olap/rowset/beta_rowset.cpp | 2 +- be/src/olap/rowset/beta_rowset.h | 2 +- be/src/olap/rowset/rowset.h | 2 +- be/src/olap/tablet.cpp | 2 +- be/src/olap/tablet.h | 2 +- be/src/olap/tablet_schema.cpp | 4 ++-- be/src/olap/tablet_schema.h | 4 ++-- be/src/olap/task/index_builder.cpp | 2 +- be/src/olap/task/index_builder.h | 2 +- be/test/testutil/mock_rowset.h | 2 +- 10 files changed, 12 insertions(+), 12 deletions(-) diff --git a/be/src/olap/rowset/beta_rowset.cpp b/be/src/olap/rowset/beta_rowset.cpp index fc5275eca5317c7..decb172956c1cd7 100644 --- a/be/src/olap/rowset/beta_rowset.cpp +++ b/be/src/olap/rowset/beta_rowset.cpp @@ -262,7 +262,7 @@ void BetaRowset::do_close() { Status BetaRowset::link_files_to(const std::string& dir, RowsetId new_rowset_id, size_t new_rowset_start_seg_id, - std::set* without_index_uids) { + std::set* without_index_uids) { DCHECK(is_local()); auto fs = _rowset_meta->fs(); if (!fs) { diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h index 404a45a1be1cde1..abf03da7f4588c7 100644 --- a/be/src/olap/rowset/beta_rowset.h +++ b/be/src/olap/rowset/beta_rowset.h @@ -71,7 +71,7 @@ class BetaRowset final : public Rowset { Status link_files_to(const std::string& dir, RowsetId new_rowset_id, size_t new_rowset_start_seg_id = 0, - std::set* without_index_uids = nullptr) override; + std::set* without_index_uids = nullptr) override; Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) override; diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h index e7675c1916a235c..dffa4a7a3ee3e2e 100644 --- a/be/src/olap/rowset/rowset.h +++ b/be/src/olap/rowset/rowset.h @@ -205,7 +205,7 @@ class Rowset : public std::enable_shared_from_this { // hard link all files in this rowset to `dir` to form a new rowset with id `new_rowset_id`. virtual Status link_files_to(const std::string& dir, RowsetId new_rowset_id, size_t new_rowset_start_seg_id = 0, - std::set* without_index_uids = nullptr) = 0; + std::set* without_index_uids = nullptr) = 0; // copy all files to `dir` virtual Status copy_files_to(const std::string& dir, const RowsetId& new_rowset_id) = 0; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index 61a1961edf9db53..f0867b4a19de08b 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -1373,7 +1373,7 @@ std::vector Tablet::pick_first_consecutive_empty_rowsets(int li } std::vector Tablet::pick_candidate_rowsets_to_build_inverted_index( - const std::set& alter_index_uids, bool is_drop_op) { + const std::set& alter_index_uids, bool is_drop_op) { std::vector candidate_rowsets; { std::shared_lock rlock(_meta_lock); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 7a959f7272fe0c3..6eb63648b9a2c8f 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -290,7 +290,7 @@ class Tablet final : public BaseTablet { std::vector pick_candidate_rowsets_to_base_compaction(); std::vector pick_candidate_rowsets_to_full_compaction(); std::vector pick_candidate_rowsets_to_build_inverted_index( - const std::set& alter_index_uids, bool is_drop_op); + const std::set& alter_index_uids, bool is_drop_op); // used for single compaction to get the local versions // Single compaction does not require remote rowsets and cannot violate the cooldown semantics diff --git a/be/src/olap/tablet_schema.cpp b/be/src/olap/tablet_schema.cpp index 0e9376e09cfff88..d423ec4d7092ac7 100644 --- a/be/src/olap/tablet_schema.cpp +++ b/be/src/olap/tablet_schema.cpp @@ -1273,7 +1273,7 @@ bool TabletSchema::has_inverted_index(const TabletColumn& col) const { return false; } -bool TabletSchema::has_inverted_index_with_index_id(int32_t index_id, +bool TabletSchema::has_inverted_index_with_index_id(int64_t index_id, const std::string& suffix_name) const { for (size_t i = 0; i < _indexes.size(); i++) { if (_indexes[i].index_type() == IndexType::INVERTED && @@ -1285,7 +1285,7 @@ bool TabletSchema::has_inverted_index_with_index_id(int32_t index_id, } const TabletIndex* TabletSchema::get_inverted_index_with_index_id( - int32_t index_id, const std::string& suffix_name) const { + int64_t index_id, const std::string& suffix_name) const { for (size_t i = 0; i < _indexes.size(); i++) { if (_indexes[i].index_type() == IndexType::INVERTED && _indexes[i].get_index_suffix() == suffix_name && _indexes[i].index_id() == index_id) { diff --git a/be/src/olap/tablet_schema.h b/be/src/olap/tablet_schema.h index ef2a6808d455376..384772375d171f0 100644 --- a/be/src/olap/tablet_schema.h +++ b/be/src/olap/tablet_schema.h @@ -346,8 +346,8 @@ class TabletSchema { } std::vector get_indexes_for_column(const TabletColumn& col) const; bool has_inverted_index(const TabletColumn& col) const; - bool has_inverted_index_with_index_id(int32_t index_id, const std::string& suffix_path) const; - const TabletIndex* get_inverted_index_with_index_id(int32_t index_id, + bool has_inverted_index_with_index_id(int64_t index_id, const std::string& suffix_path) const; + const TabletIndex* get_inverted_index_with_index_id(int64_t index_id, const std::string& suffix_name) const; const TabletIndex* get_inverted_index(const TabletColumn& col) const; const TabletIndex* get_inverted_index(int32_t col_unique_id, diff --git a/be/src/olap/task/index_builder.cpp b/be/src/olap/task/index_builder.cpp index 09f745833d1c2f6..a62763b8af8da99 100644 --- a/be/src/olap/task/index_builder.cpp +++ b/be/src/olap/task/index_builder.cpp @@ -59,7 +59,7 @@ Status IndexBuilder::update_inverted_index_info() { LOG(INFO) << "begin to update_inverted_index_info, tablet=" << _tablet->tablet_id() << ", is_drop_op=" << _is_drop_op; // index ids that will not be linked - std::set without_index_uids; + std::set without_index_uids; _output_rowsets.reserve(_input_rowsets.size()); _pending_rs_guards.reserve(_input_rowsets.size()); for (auto&& input_rowset : _input_rowsets) { diff --git a/be/src/olap/task/index_builder.h b/be/src/olap/task/index_builder.h index 1ed16886315478e..75f3b5d2ff0ccbf 100644 --- a/be/src/olap/task/index_builder.h +++ b/be/src/olap/task/index_builder.h @@ -69,7 +69,7 @@ class IndexBuilder { std::vector _columns; std::vector _alter_inverted_indexes; bool _is_drop_op; - std::set _alter_index_ids; + std::set _alter_index_ids; std::vector _input_rowsets; std::vector _output_rowsets; std::vector _pending_rs_guards; diff --git a/be/test/testutil/mock_rowset.h b/be/test/testutil/mock_rowset.h index 50065ebe6b43d44..7761b2c6b03a2f5 100644 --- a/be/test/testutil/mock_rowset.h +++ b/be/test/testutil/mock_rowset.h @@ -30,7 +30,7 @@ class MockRowset : public Rowset { Status remove() override { return Status::NotSupported("MockRowset not support this method."); } Status link_files_to(const std::string& dir, RowsetId new_rowset_id, size_t start_seg_id, - std::set* without_index_uids) override { + std::set* without_index_uids) override { return Status::NotSupported("MockRowset not support this method."); } From 944d9bd4bdcc0cacd96b45d8008d90bb943b2c9a Mon Sep 17 00:00:00 2001 From: HappenLee Date: Mon, 20 May 2024 20:56:20 +0800 Subject: [PATCH 070/111] [exec](performance) opt the topn nullable column order performance in Heap Sort (#35042) --- be/src/vec/columns/column_nullable.cpp | 47 ++++++++++++++++++++++++++ be/src/vec/columns/column_nullable.h | 4 +++ 2 files changed, 51 insertions(+) diff --git a/be/src/vec/columns/column_nullable.cpp b/be/src/vec/columns/column_nullable.cpp index e80e9d9b636d52e..fea85917d723558 100644 --- a/be/src/vec/columns/column_nullable.cpp +++ b/be/src/vec/columns/column_nullable.cpp @@ -422,6 +422,53 @@ int ColumnNullable::compare_at(size_t n, size_t m, const IColumn& rhs_, return get_nested_column().compare_at(n, m, nullable_rhs.get_nested_column(), null_direction_hint); } +void ColumnNullable::compare_internal(size_t rhs_row_id, const IColumn& rhs, int nan_direction_hint, + int direction, std::vector& cmp_res, + uint8* __restrict filter) const { + const auto& rhs_null_column = assert_cast(rhs); + const bool right_is_null = rhs.is_null_at(rhs_row_id); + const bool left_contains_null = has_null(); + if (!left_contains_null && !right_is_null) { + get_nested_column().compare_internal(rhs_row_id, rhs_null_column.get_nested_column(), + nan_direction_hint, direction, cmp_res, filter); + } else { + auto sz = this->size(); + DCHECK(cmp_res.size() == sz); + + size_t begin = simd::find_zero(cmp_res, 0); + while (begin < sz) { + size_t end = simd::find_one(cmp_res, begin + 1); + if (right_is_null) { + for (size_t row_id = begin; row_id < end; row_id++) { + if (!is_null_at(row_id)) { + if ((-nan_direction_hint * direction) < 0) { + filter[row_id] = 1; + cmp_res[row_id] = 1; + } else if ((-nan_direction_hint * direction) > 0) { + cmp_res[row_id] = 1; + } + } + } + } else { + for (size_t row_id = begin; row_id < end; row_id++) { + if (is_null_at(row_id)) { + if (nan_direction_hint * direction < 0) { + filter[row_id] = 1; + cmp_res[row_id] = 1; + } else if (nan_direction_hint * direction > 0) { + cmp_res[row_id] = 1; + } + } + } + } + begin = simd::find_zero(cmp_res, end + 1); + } + if (!right_is_null) { + get_nested_column().compare_internal(rhs_row_id, rhs_null_column.get_nested_column(), + nan_direction_hint, direction, cmp_res, filter); + } + } +} void ColumnNullable::get_permutation(bool reverse, size_t limit, int null_direction_hint, Permutation& res) const { diff --git a/be/src/vec/columns/column_nullable.h b/be/src/vec/columns/column_nullable.h index 8842cc67ef251c9..01e2394f71e2702 100644 --- a/be/src/vec/columns/column_nullable.h +++ b/be/src/vec/columns/column_nullable.h @@ -212,6 +212,10 @@ class ColumnNullable final : public COWHelper { ColumnPtr permute(const Permutation& perm, size_t limit) const override; // ColumnPtr index(const IColumn & indexes, size_t limit) const override; int compare_at(size_t n, size_t m, const IColumn& rhs_, int null_direction_hint) const override; + + void compare_internal(size_t rhs_row_id, const IColumn& rhs, int nan_direction_hint, + int direction, std::vector& cmp_res, + uint8* __restrict filter) const override; void get_permutation(bool reverse, size_t limit, int null_direction_hint, Permutation& res) const override; void reserve(size_t n) override; From c0fd98abe50c75d500524feaac9ed74a52835c6e Mon Sep 17 00:00:00 2001 From: Tiewei Fang <43782773+BePPPower@users.noreply.github.com> Date: Tue, 21 May 2024 09:57:15 +0800 Subject: [PATCH 071/111] [Fix](tvf) Fix that tvf reading empty files in compressed formats. (#34926) 1. Fix the issue with tvf reading empty compressed files. 2. move two test cases (`test_local_tvf_compression` and `test_s3_tvf_compression`) from p2 to p0 --- be/src/exec/decompressor.cpp | 27 +++++++++--- .../new_plain_text_line_reader.cpp | 3 +- .../ExternalFileTableValuedFunction.java | 40 +++++++++++++++++- .../tvf/compress/test_empty_snappy.snappy | Bin 0 -> 4 bytes .../tvf/compress/test_tvf.csv.bz2 | Bin .../tvf/compress/test_tvf.csv.deflate | Bin .../tvf/compress/test_tvf.csv.gz | Bin .../tvf/compress/test_tvf.csv.lz4 | Bin .../tvf/compress/test_tvf.csv.snappy | Bin .../tvf/test_local_tvf_compression.out | 2 + .../tvf/test_s3_tvf_compression.out | 0 .../tvf/test_local_tvf_compression.groovy | 14 +++++- .../tvf/test_s3_tvf_compression.groovy | 2 +- 13 files changed, 78 insertions(+), 10 deletions(-) create mode 100644 regression-test/data/external_table_p0/tvf/compress/test_empty_snappy.snappy rename regression-test/data/{external_table_p2 => external_table_p0}/tvf/compress/test_tvf.csv.bz2 (100%) rename regression-test/data/{external_table_p2 => external_table_p0}/tvf/compress/test_tvf.csv.deflate (100%) rename regression-test/data/{external_table_p2 => external_table_p0}/tvf/compress/test_tvf.csv.gz (100%) rename regression-test/data/{external_table_p2 => external_table_p0}/tvf/compress/test_tvf.csv.lz4 (100%) rename regression-test/data/{external_table_p2 => external_table_p0}/tvf/compress/test_tvf.csv.snappy (100%) rename regression-test/data/{external_table_p2 => external_table_p0}/tvf/test_local_tvf_compression.out (99%) rename regression-test/data/{external_table_p2 => external_table_p0}/tvf/test_s3_tvf_compression.out (100%) rename regression-test/suites/{external_table_p2 => external_table_p0}/tvf/test_local_tvf_compression.groovy (93%) rename regression-test/suites/{external_table_p2 => external_table_p0}/tvf/test_s3_tvf_compression.groovy (98%) diff --git a/be/src/exec/decompressor.cpp b/be/src/exec/decompressor.cpp index c9f16f10e7aa158..d8d02c9cf9e32cb 100644 --- a/be/src/exec/decompressor.cpp +++ b/be/src/exec/decompressor.cpp @@ -557,6 +557,10 @@ Status SnappyBlockDecompressor::decompress(uint8_t* input, size_t input_len, // See: // https://github.com/apache/hadoop/blob/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/main/native/src/codec/SnappyCodec.cc while (remaining_input_size > 0) { + if (remaining_input_size < 4) { + *more_input_bytes = 4 - remaining_input_size; + break; + } // Read uncompressed size uint32_t uncompressed_block_len = Decompressor::_read_int32(src); int64_t remaining_output_len = output_max_len - uncompressed_total_len; @@ -566,12 +570,24 @@ Status SnappyBlockDecompressor::decompress(uint8_t* input, size_t input_len, break; } + if (uncompressed_block_len == 0) { + remaining_input_size -= sizeof(uint32_t); + break; + } + + if (remaining_input_size <= 2 * sizeof(uint32_t)) { + // The remaining input size should be larger then + // +1 means we need at least 1 bytes of compressed data. + *more_input_bytes = 2 * sizeof(uint32_t) + 1 - remaining_input_size; + break; + } + // Read compressed size - size_t tmp_src_size = remaining_input_size - sizeof(uint32_t); + size_t tmp_remaining_size = remaining_input_size - 2 * sizeof(uint32_t); size_t compressed_len = _read_int32(src + sizeof(uint32_t)); - if (compressed_len == 0 || compressed_len > tmp_src_size) { + if (compressed_len > tmp_remaining_size) { // Need more input data - *more_input_bytes = compressed_len - tmp_src_size; + *more_input_bytes = compressed_len - tmp_remaining_size; break; } @@ -590,8 +606,9 @@ Status SnappyBlockDecompressor::decompress(uint8_t* input, size_t input_len, // Decompress if (!snappy::RawUncompress(reinterpret_cast(src), compressed_len, reinterpret_cast(output))) { - return Status::InternalError("snappy block decompress failed. uncompressed_len: {}", - uncompressed_block_len); + return Status::InternalError( + "snappy block decompress failed. uncompressed_len: {}, compressed_len: {}", + uncompressed_block_len, compressed_len); } output += uncompressed_block_len; diff --git a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp index fefd5ecae6784a0..8dce6e589afde23 100644 --- a/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp +++ b/be/src/vec/exec/format/file_reader/new_plain_text_line_reader.cpp @@ -444,7 +444,8 @@ Status NewPlainTextLineReader::read_line(const uint8_t** ptr, size_t* size, bool std::stringstream ss; ss << "decompress made no progress." << " input_read_bytes: " << input_read_bytes - << " decompressed_len: " << decompressed_len; + << " decompressed_len: " << decompressed_len + << " input len: " << (_input_buf_limit - _input_buf_pos); LOG(WARNING) << ss.str(); return Status::InternalError(ss.str()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 260f7b2df44c209..0a295e1b55538d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -482,7 +482,7 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti // get first file, used to parse table schema TBrokerFileStatus firstFile = null; for (TBrokerFileStatus fileStatus : fileStatuses) { - if (fileStatus.isIsDir() || fileStatus.size == 0) { + if (isFileContentEmpty(fileStatus)) { continue; } firstFile = fileStatus; @@ -514,5 +514,43 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti return InternalService.PFetchTableSchemaRequest.newBuilder() .setFileScanRange(ByteString.copyFrom(new TSerializer().serialize(fileScanRange))).build(); } + + private boolean isFileContentEmpty(TBrokerFileStatus fileStatus) { + if (fileStatus.isIsDir() || fileStatus.size == 0) { + return true; + } + if (Util.isCsvFormat(fileFormatType) || fileFormatType == TFileFormatType.FORMAT_JSON) { + int magicNumberBytes = 0; + switch (compressionType) { + case GZ: + magicNumberBytes = 20; + break; + case LZO: + case LZOP: + magicNumberBytes = 42; + break; + case DEFLATE: + magicNumberBytes = 8; + break; + case SNAPPYBLOCK: + case LZ4BLOCK: + case LZ4FRAME: + magicNumberBytes = 4; + break; + case BZ2: + magicNumberBytes = 14; + break; + case UNKNOWN: + case PLAIN: + default: + break; + } + // fileStatus.size may be -1 in http_stream + if (fileStatus.size >= 0 && fileStatus.size <= magicNumberBytes) { + return true; + } + } + return false; + } } diff --git a/regression-test/data/external_table_p0/tvf/compress/test_empty_snappy.snappy b/regression-test/data/external_table_p0/tvf/compress/test_empty_snappy.snappy new file mode 100644 index 0000000000000000000000000000000000000000..593f4708db84ac8fd0f5cc47c634f38c013fe9e4 GIT binary patch literal 4 LcmZQzU|;|M00aO5 literal 0 HcmV?d00001 diff --git a/regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.bz2 b/regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.bz2 similarity index 100% rename from regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.bz2 rename to regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.bz2 diff --git a/regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.deflate b/regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.deflate similarity index 100% rename from regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.deflate rename to regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.deflate diff --git a/regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.gz b/regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.gz similarity index 100% rename from regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.gz rename to regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.gz diff --git a/regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.lz4 b/regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.lz4 similarity index 100% rename from regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.lz4 rename to regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.lz4 diff --git a/regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.snappy b/regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.snappy similarity index 100% rename from regression-test/data/external_table_p2/tvf/compress/test_tvf.csv.snappy rename to regression-test/data/external_table_p0/tvf/compress/test_tvf.csv.snappy diff --git a/regression-test/data/external_table_p2/tvf/test_local_tvf_compression.out b/regression-test/data/external_table_p0/tvf/test_local_tvf_compression.out similarity index 99% rename from regression-test/data/external_table_p2/tvf/test_local_tvf_compression.out rename to regression-test/data/external_table_p0/tvf/test_local_tvf_compression.out index 19699b0dc5d29d8..8120427ea6cec54 100644 --- a/regression-test/data/external_table_p2/tvf/test_local_tvf_compression.out +++ b/regression-test/data/external_table_p0/tvf/test_local_tvf_compression.out @@ -148,3 +148,5 @@ -- !snappy_2 -- +-- !snappy_empty -- + diff --git a/regression-test/data/external_table_p2/tvf/test_s3_tvf_compression.out b/regression-test/data/external_table_p0/tvf/test_s3_tvf_compression.out similarity index 100% rename from regression-test/data/external_table_p2/tvf/test_s3_tvf_compression.out rename to regression-test/data/external_table_p0/tvf/test_s3_tvf_compression.out diff --git a/regression-test/suites/external_table_p2/tvf/test_local_tvf_compression.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy similarity index 93% rename from regression-test/suites/external_table_p2/tvf/test_local_tvf_compression.groovy rename to regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy index 343afab56f9a511..420887fb41d6b66 100644 --- a/regression-test/suites/external_table_p2/tvf/test_local_tvf_compression.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_compression.groovy @@ -17,11 +17,11 @@ import org.junit.Assert // specific language governing permissions and limitations // under the License. -suite("test_local_tvf_compression", "p2,external,tvf,external_remote,external_remote_tvf") { +suite("test_local_tvf_compression", "p0,tvf") { List> backends = sql """ show backends """ assertTrue(backends.size() > 0) def be_id = backends[0][0] - def dataFilePath = context.config.dataPath + "/external_table_p2/tvf/compress" + def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/compress" def outFilePath="/compress" @@ -136,6 +136,16 @@ suite("test_local_tvf_compression", "p2,external,tvf,external_remote,external_re "compress_type" ="${compress_type}block") where c2="abcd" order by c3 limit 22 ; """ + // test empty snapppy file + qt_snappy_empty """ + select * from local( + "file_path" = "${outFilePath}/test_empty_snappy.${compress_type}", + "backend_id" = "${be_id}", + "format" = "csv", + "column_separator" = ",", + "compress_type" ="${compress_type}block"); + """ + // test error case test { sql """ diff --git a/regression-test/suites/external_table_p2/tvf/test_s3_tvf_compression.groovy b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_compression.groovy similarity index 98% rename from regression-test/suites/external_table_p2/tvf/test_s3_tvf_compression.groovy rename to regression-test/suites/external_table_p0/tvf/test_s3_tvf_compression.groovy index 279fcb5e8a5a36f..5eb63c94ddc8ded 100644 --- a/regression-test/suites/external_table_p2/tvf/test_s3_tvf_compression.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_s3_tvf_compression.groovy @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite("test_s3_tvf_compression", "p2,external,tvf,external_remote,external_remote_tvf") { +suite("test_s3_tvf_compression", "p0") { String ak = getS3AK() String sk = getS3SK() From 74d66e96502d825df50b2b45ad4c9de4f9032920 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 21 May 2024 10:57:58 +0800 Subject: [PATCH 072/111] [Fix](parquet-reader) Fix Timestamp Int96 min-max statistics is incorrect when was written by some old parquet writers by disable it. (#35041) Parquet INT96 timestamp values were compared incorrectly for the purposes of producing statistics by older parquet writers, so PARQUET-1065 deprecated them. The result is that any writer that produced stats was producing unusable incorrect values, except the special case where min == max and an incorrect ordering would not be material to the result. PARQUET-1026 made binary stats available and valid in that special case. --- be/src/vec/exec/format/parquet/parquet_pred_cmp.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/be/src/vec/exec/format/parquet/parquet_pred_cmp.h b/be/src/vec/exec/format/parquet/parquet_pred_cmp.h index 1f80aa7ecd58d7a..916f3f64ee690e8 100644 --- a/be/src/vec/exec/format/parquet/parquet_pred_cmp.h +++ b/be/src/vec/exec/format/parquet/parquet_pred_cmp.h @@ -257,6 +257,17 @@ class ParquetPredicate { ParquetInt96 datetime96_max = *reinterpret_cast(encoded_max.data()); int64_t micros_max = datetime96_max.to_timestamp_micros(); + + // From Trino: Parquet INT96 timestamp values were compared incorrectly + // for the purposes of producing statistics by older parquet writers, + // so PARQUET-1065 deprecated them. The result is that any writer that produced stats + // was producing unusable incorrect values, except the special case where min == max + // and an incorrect ordering would not be material to the result. + // PARQUET-1026 made binary stats available and valid in that special case. + if (micros_min != micros_max) { + return false; + } + if constexpr (std::is_same_v || std::is_same_v>) { min_value.from_unixtime(micros_min / 1000000, ctz); From fb28d0b18598360a6def809ae7df08a91edf5a10 Mon Sep 17 00:00:00 2001 From: shee <13843187+qzsee@users.noreply.github.com> Date: Tue, 21 May 2024 10:59:07 +0800 Subject: [PATCH 073/111] [BUG] fix scan range boundary handling is incorrect (#34832) fix scan range boundary handling is incorrect Co-authored-by: shizhiqiang03 --- be/src/exec/olap_common.h | 5 ++- .../query_p0/scan_range/test_scan_range.out | 10 +++++ .../scan_range/test_scan_range.groovy | 45 +++++++++++++++++++ 3 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/query_p0/scan_range/test_scan_range.out create mode 100644 regression-test/suites/query_p0/scan_range/test_scan_range.groovy diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h index 5d6377b9c7c9336..46983c5390ebcc9 100644 --- a/be/src/exec/olap_common.h +++ b/be/src/exec/olap_common.h @@ -702,14 +702,15 @@ bool ColumnValueRange::convert_to_close_range( bool is_empty = false; if (!is_begin_include()) { - if (_low_value == TYPE_MIN) { + if (_low_value == TYPE_MAX) { is_empty = true; } else { ++_low_value; } } + if (!is_end_include()) { - if (_high_value == TYPE_MAX) { + if (_high_value == TYPE_MIN) { is_empty = true; } else { --_high_value; diff --git a/regression-test/data/query_p0/scan_range/test_scan_range.out b/regression-test/data/query_p0/scan_range/test_scan_range.out new file mode 100644 index 000000000000000..e4df16ef06ca8e3 --- /dev/null +++ b/regression-test/data/query_p0/scan_range/test_scan_range.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql_1 -- +1 + +-- !sql_2 -- +1 + +-- !sql_3 -- + +-- !sql_4 -- diff --git a/regression-test/suites/query_p0/scan_range/test_scan_range.groovy b/regression-test/suites/query_p0/scan_range/test_scan_range.groovy new file mode 100644 index 000000000000000..e011a5095a59ec1 --- /dev/null +++ b/regression-test/suites/query_p0/scan_range/test_scan_range.groovy @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_scan_range", "query,p0") { + + sql "use test_query_db" + + def tableName = "test_scan_range_tbl" + + sql """ + CREATE TABLE `${tableName}` ( + `k1` INT NULL, + `k2` INT NULL + ) ENGINE=OLAP + DUPLICATE KEY(`k1`, `k2`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" + ); + """ + + sql "insert into ${tableName} values(1,1)" + + qt_sql_1 "select k1 from ${tableName} where k1 > -2147483648" + + qt_sql_2 "select k1 from ${tableName} where k1 < 2147483647" + + qt_sql_3 "select k1 from ${tableName} where k1 < -2147483648" + + qt_sql_4 "select k1 from ${tableName} where k1 > 2147483647" +} From 44bb2bb639ef65e9206434c515db61f1bfccb727 Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Tue, 21 May 2024 11:56:09 +0800 Subject: [PATCH 074/111] [opt](routine-load) do not schedule invalid task (#34918) --- .../doris/load/routineload/RoutineLoadTaskScheduler.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java index 6c0baf5bdffe767..d4d5d5512d31e38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/routineload/RoutineLoadTaskScheduler.java @@ -123,6 +123,9 @@ private void scheduleOneTask(RoutineLoadTaskInfo routineLoadTaskInfo) throws Exc } try { + if (routineLoadManager.getJob(routineLoadTaskInfo.getJobId()).isFinal()) { + return; + } // check if topic has more data to consume if (!routineLoadTaskInfo.hasMoreDataToConsume()) { needScheduleTasksQueue.addLast(routineLoadTaskInfo); From 5019aa03e938e163a5dbdb849e01a619de681c7b Mon Sep 17 00:00:00 2001 From: Xin Liao Date: Tue, 21 May 2024 15:30:49 +0800 Subject: [PATCH 075/111] [enhancement](be-meta) disable sync rocksdb by default for better performance (#32714) (#35122) --- be/src/common/config.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index ebb30e9d8e1db5d..3a9e1b89f55bec1 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -601,7 +601,7 @@ DEFINE_mInt32(result_buffer_cancelled_interval_time, "300"); DEFINE_mInt32(priority_queue_remaining_tasks_increased_frequency, "512"); // sync tablet_meta when modifying meta -DEFINE_mBool(sync_tablet_meta, "true"); +DEFINE_mBool(sync_tablet_meta, "false"); // default thrift rpc timeout ms DEFINE_mInt32(thrift_rpc_timeout_ms, "60000"); From 706c9c473b0c66334d3cc18c2c259b4875702995 Mon Sep 17 00:00:00 2001 From: yujun Date: Tue, 21 May 2024 15:49:16 +0800 Subject: [PATCH 076/111] [fix](autobucket) calc bucket num exclude today's partition #34304 #35129 --- .../clone/DynamicPartitionScheduler.java | 30 ++++++++----------- .../doris/common/util/AutoBucketUtils.java | 1 + 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java index 96c8b2b13b13de8..3fa73930906403b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java @@ -185,38 +185,28 @@ private static long getNextPartitionSize(ArrayList historyPartitionsSize) } } - private static int getBucketsNum(DynamicPartitionProperty property, OlapTable table) { + private static int getBucketsNum(DynamicPartitionProperty property, OlapTable table, String nowPartitionName) { if (!table.isAutoBucket()) { return property.getBuckets(); } // auto bucket // get all history partitions - List partitions = Lists.newArrayList(); + ArrayList partitionSizeArray = Lists.newArrayList(); RangePartitionInfo info = (RangePartitionInfo) (table.getPartitionInfo()); List> idToItems = new ArrayList<>(info.getIdToItem(false).entrySet()); idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem) o.getValue()).getItems().upperEndpoint())); for (Map.Entry idToItem : idToItems) { Partition partition = table.getPartition(idToItem.getKey()); - if (partition != null) { - partitions.add(partition); - } - } - - // no exist history partition - if (partitions.size() == 0) { - return property.getBuckets(); - } - - ArrayList partitionSizeArray = Lists.newArrayList(); - for (Partition partition : partitions) { - if (partition.getVisibleVersion() >= 2) { + // exclude current partition because its data isn't enough one week/day/hour. + if (partition != null && !partition.getName().equals(nowPartitionName) + && partition.getVisibleVersion() >= 2) { partitionSizeArray.add(partition.getAllDataSize(true)); } } // no exist history partition data - if (partitionSizeArray.size() == 0) { + if (partitionSizeArray.isEmpty()) { return property.getBuckets(); } @@ -249,6 +239,12 @@ private ArrayList getAddPartitionClause(Database db, OlapTab int hotPartitionNum = dynamicPartitionProperty.getHotPartitionNum(); String storagePolicyName = dynamicPartitionProperty.getStoragePolicy(); + String nowPartitionPrevBorder = DynamicPartitionUtil.getPartitionRangeString( + dynamicPartitionProperty, now, 0, partitionFormat); + String nowPartitionName = dynamicPartitionProperty.getPrefix() + + DynamicPartitionUtil.getFormattedPartitionName(dynamicPartitionProperty.getTimeZone(), + nowPartitionPrevBorder, dynamicPartitionProperty.getTimeUnit()); + for (; idx <= dynamicPartitionProperty.getEnd(); idx++) { String prevBorder = DynamicPartitionUtil.getPartitionRangeString( dynamicPartitionProperty, now, idx, partitionFormat); @@ -322,7 +318,7 @@ private ArrayList getAddPartitionClause(Database db, OlapTab DistributionDesc distributionDesc = null; DistributionInfo distributionInfo = olapTable.getDefaultDistributionInfo(); - int bucketsNum = getBucketsNum(dynamicPartitionProperty, olapTable); + int bucketsNum = getBucketsNum(dynamicPartitionProperty, olapTable, nowPartitionName); if (distributionInfo.getType() == DistributionInfo.DistributionInfoType.HASH) { HashDistributionInfo hashDistributionInfo = (HashDistributionInfo) distributionInfo; List distColumnNames = new ArrayList<>(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java index 55a07b89eb8f335..9185d82dde72ae1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/AutoBucketUtils.java @@ -93,6 +93,7 @@ public static int getBucketsNum(long partitionSize) { if (bucketsNum < bucketsNumByPartitionSize && bucketsNum < beNum) { bucketsNum = beNum; } + bucketsNum = Math.min(bucketsNum, Config.autobucket_max_buckets); logger.debug("AutoBucketsUtil: final bucketsNum {}", bucketsNum); return bucketsNum; } From 428a6fd6ab06d3b6337251b76917de4b543e3f52 Mon Sep 17 00:00:00 2001 From: yujun Date: Tue, 21 May 2024 15:49:31 +0800 Subject: [PATCH 077/111] fix test_decommission_with_replica_num_fail (#35123) --- .../test_decommission_with_replica_num_fail.groovy | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy b/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy index 02bd0788bcee7d7..0fb3ebbbbbadb57 100644 --- a/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy +++ b/regression-test/suites/alter_p0/test_decommission_with_replica_num_fail.groovy @@ -15,8 +15,7 @@ // specific language governing permissions and limitations // under the License. -suite('test_decommission_with_replica_num_fail') { - +suite('test_decommission_with_replica_num_fail', 'nonConcurrent') { def tbl = 'test_decommission_with_replica_num_fail' def backends = sql_return_maparray('show backends') def replicaNum = 0 @@ -44,6 +43,11 @@ suite('test_decommission_with_replica_num_fail') { "replication_num" = "${replicaNum}" ); """ + + // fix set force_olap_table_replication_num + sql "ALTER TABLE ${tbl} SET ('default.replication_num' = '${replicaNum}')" + sql "ALTER TABLE ${tbl} MODIFY PARTITION (*) SET ('replication_num' = '${replicaNum}')" + try { test { sql "ALTER SYSTEM DECOMMISSION BACKEND '${targetBackend.Host}:${targetBackend.HeartbeatPort}'" @@ -51,6 +55,10 @@ suite('test_decommission_with_replica_num_fail') { } } finally { sql "CANCEL DECOMMISSION BACKEND '${targetBackend.Host}:${targetBackend.HeartbeatPort}'" + backends = sql_return_maparray('show backends') + for (def be : backends) { + logger.info("backend=${be}") + } } sql "DROP TABLE IF EXISTS ${tbl} FORCE" } From b0ecf76131c8d66e18b8e38fb07b67f72cb4bb84 Mon Sep 17 00:00:00 2001 From: lide Date: Tue, 21 May 2024 16:34:48 +0800 Subject: [PATCH 078/111] [fix][build](audit-loader) Fix a build error for AuditLoaderPlugin. (#35119) --- .../java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java index ed4b7efc6513afe..4d53622b814bdbb 100755 --- a/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java +++ b/fe_plugins/auditloader/src/main/java/org/apache/doris/plugin/audit/AuditLoaderPlugin.java @@ -19,7 +19,7 @@ import org.apache.doris.common.Config; import org.apache.doris.catalog.Env; -import org.apache.doris.plugin.AuditEvent; +import org.apache.doris.plugin.audit.AuditEvent; import org.apache.doris.plugin.AuditPlugin; import org.apache.doris.plugin.Plugin; import org.apache.doris.plugin.PluginContext; From 367603a6c9e1073fe2507164d46a3e16cd7fb86c Mon Sep 17 00:00:00 2001 From: lide Date: Tue, 21 May 2024 16:35:42 +0800 Subject: [PATCH 079/111] [security] fix fastjson security issues. (#35120) cherry pick from #34627 Co-authored-by: derenli --- samples/doris-demo/flink-demo-v1.1/pom.xml | 2 +- samples/doris-demo/flink-demo/pom.xml | 2 +- samples/doris-demo/spark-demo/pom.xml | 2 +- samples/doris-demo/spring-jdbc-demo/pom.xml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/samples/doris-demo/flink-demo-v1.1/pom.xml b/samples/doris-demo/flink-demo-v1.1/pom.xml index 87054383ec11b40..d1f3380347336c2 100644 --- a/samples/doris-demo/flink-demo-v1.1/pom.xml +++ b/samples/doris-demo/flink-demo-v1.1/pom.xml @@ -30,7 +30,7 @@ under the License. 2.12 1.8 1.15.3 - 1.2.62 + 1.2.83 2.8.3 compile 1.7.30 diff --git a/samples/doris-demo/flink-demo/pom.xml b/samples/doris-demo/flink-demo/pom.xml index 586a86c68df8c5c..a934bc02faff318 100644 --- a/samples/doris-demo/flink-demo/pom.xml +++ b/samples/doris-demo/flink-demo/pom.xml @@ -31,7 +31,7 @@ under the License. 2.12 1.8 1.12.2 - 1.2.62 + 1.2.83 2.8.3 compile diff --git a/samples/doris-demo/spark-demo/pom.xml b/samples/doris-demo/spark-demo/pom.xml index f10211d327bf455..e1752b4302ca809 100644 --- a/samples/doris-demo/spark-demo/pom.xml +++ b/samples/doris-demo/spark-demo/pom.xml @@ -73,7 +73,7 @@ under the License. com.alibaba fastjson - 1.2.78 + 1.2.83 com.fasterxml.jackson.core diff --git a/samples/doris-demo/spring-jdbc-demo/pom.xml b/samples/doris-demo/spring-jdbc-demo/pom.xml index 2c221f8eb37dd99..34b6049ee757c72 100644 --- a/samples/doris-demo/spring-jdbc-demo/pom.xml +++ b/samples/doris-demo/spring-jdbc-demo/pom.xml @@ -38,7 +38,7 @@ under the License. UTF-8 1.8 1.3.2 - 1.2.70 + 1.2.83 1.1.14 2.5 From 0599cb2efd1a6dcc1c609db35fa0a63c99a73545 Mon Sep 17 00:00:00 2001 From: GoGoWen <82132356+GoGoWen@users.noreply.github.com> Date: Tue, 21 May 2024 16:48:08 +0800 Subject: [PATCH 080/111] fix replica's remote data size set to data size (#35098) fix replica's remote data size set to data size --- .../src/main/java/org/apache/doris/clone/TabletSchedCtx.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java index 706e346c0463ce4..8401ec17bbd0435 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java +++ b/fe/fe-core/src/main/java/org/apache/doris/clone/TabletSchedCtx.java @@ -1178,7 +1178,7 @@ public void finishCloneTask(CloneTask cloneTask, TFinishTaskRequest request) } replica.updateVersionInfo(reportedTablet.getVersion(), reportedTablet.getDataSize(), - reportedTablet.getDataSize(), reportedTablet.getRowCount()); + reportedTablet.getRemoteDataSize(), reportedTablet.getRowCount()); if (replica.getLastFailedVersion() > partition.getCommittedVersion() && reportedTablet.getVersion() >= partition.getCommittedVersion() //&& !(reportedTablet.isSetVersionMiss() && reportedTablet.isVersionMiss() From 037de3deddfe1d9f724d165ba63cd2000a3cfcb3 Mon Sep 17 00:00:00 2001 From: wangbo Date: Tue, 21 May 2024 19:04:04 +0800 Subject: [PATCH 081/111] [Fix](executor)Fix workload policy test #35146 --- .../test_workload_sched_policy.groovy | 33 +++++++++++-------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy index 6dc1a28d463a487..d3f9b35426ad49a 100644 --- a/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy +++ b/regression-test/suites/workload_manager_p0/test_workload_sched_policy.groovy @@ -115,7 +115,6 @@ suite("test_workload_sched_policy") { qt_select_policy_tvf_after_drop "select name,condition,action,priority,enabled,version from information_schema.workload_policy where name in('be_policy','fe_policy','set_action_policy','test_cancel_policy') order by name;" // test workload policy - sql "ADMIN SET FRONTEND CONFIG ('workload_sched_policy_interval_ms' = '500');" sql """drop user if exists test_workload_sched_user""" sql """create user test_workload_sched_user identified by '12345'""" sql """grant ADMIN_PRIV on *.*.* to test_workload_sched_user""" @@ -123,36 +122,42 @@ suite("test_workload_sched_policy") { // 1 create test_set_var_policy sql """drop workload policy if exists test_set_var_policy;""" sql """drop workload policy if exists test_set_var_policy2;""" + sql "drop workload group if exists test_set_session_wg;" + sql "drop workload group if exists test_set_session_wg2;" + sql "create workload group test_set_session_wg properties('cpu_share'='1024');" + sql "create workload group test_set_session_wg2 properties('cpu_share'='1024');" + sql "create workload policy test_set_var_policy conditions(username='test_workload_sched_user')" + - "actions(set_session_variable 'parallel_pipeline_task_num=33');" + "actions(set_session_variable 'workload_group=test_set_session_wg');" + def result1 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) { logger.info("begin sleep 15s to wait") Thread.sleep(15000) - sql "show variables like '%parallel_pipeline_task_num%';" + sql "show variables like 'workload_group';" } - assertEquals("parallel_pipeline_task_num", result1[0][0]) - assertEquals("33", result1[0][1]) + assertEquals("workload_group", result1[0][0]) + assertEquals("test_set_session_wg", result1[0][1]) // 2 create test_set_var_policy2 with higher priority sql "create workload policy test_set_var_policy2 conditions(username='test_workload_sched_user') " + - "actions(set_session_variable 'parallel_pipeline_task_num=22') properties('priority'='10');" + "actions(set_session_variable 'workload_group=test_set_session_wg2') properties('priority'='10');" def result2 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) { Thread.sleep(3000) - sql "show variables like '%parallel_pipeline_task_num%';" + sql "show variables like 'workload_group';" } - assertEquals("parallel_pipeline_task_num", result2[0][0]) - assertEquals("22", result2[0][1]) + assertEquals("workload_group", result2[0][0]) + assertEquals("test_set_session_wg2", result2[0][1]) // 3 disable test_set_var_policy2 sql "alter workload policy test_set_var_policy2 properties('enabled'='false');" def result3 = connect(user = 'test_workload_sched_user', password = '12345', url = context.config.jdbcUrl) { Thread.sleep(3000) - sql "show variables like '%parallel_pipeline_task_num%';" + sql "show variables like 'workload_group';" } - assertEquals("parallel_pipeline_task_num", result3[0][0]) - assertEquals("33", result3[0][1]) - - sql "ADMIN SET FRONTEND CONFIG ('workload_sched_policy_interval_ms' = '10000');" + assertEquals("workload_group", result3[0][0]) + assertEquals("test_set_session_wg", result3[0][1]) + sql "drop workload group if exists test_set_session_wg;" + sql "drop workload group if exists test_set_session_wg2;" sql "drop workload policy if exists test_set_var_policy;" sql "drop workload policy if exists test_set_var_policy2;" From 98f8eb5c43754c633610487bdfc389e78ad495c5 Mon Sep 17 00:00:00 2001 From: Ashin Gau Date: Tue, 21 May 2024 22:27:07 +0800 Subject: [PATCH 082/111] [opt](split) get file splits in batch mode (#34032) (#35107) bp #34032 --- be/src/common/config.cpp | 1 + be/src/common/config.h | 2 + be/src/pipeline/exec/file_scan_operator.cpp | 57 ++---- be/src/pipeline/exec/file_scan_operator.h | 5 +- be/src/vec/exec/scan/new_file_scan_node.cpp | 60 ++---- be/src/vec/exec/scan/new_file_scan_node.h | 4 +- .../vec/exec/scan/split_source_connector.cpp | 86 ++++++++ be/src/vec/exec/scan/split_source_connector.h | 115 +++++++++++ be/src/vec/exec/scan/vfile_scanner.cpp | 67 ++++--- be/src/vec/exec/scan/vfile_scanner.h | 15 +- be/test/vec/exec/vwal_scanner_test.cpp | 29 ++- .../java/org/apache/doris/catalog/Env.java | 10 +- .../doris/datasource/FileQueryScanNode.java | 185 +++++++++++------- .../apache/doris/datasource/FileScanNode.java | 19 +- .../doris/datasource/SplitAssignment.java | 85 ++++++++ .../doris/datasource/SplitGenerator.java | 68 +++++++ .../apache/doris/datasource/SplitSource.java | 95 +++++++++ .../doris/datasource/SplitSourceManager.java | 78 ++++++++ .../doris/datasource/SplitToScanRange.java | 34 ++++ .../datasource/hive/source/HiveScanNode.java | 74 ++++++- .../datasource/hudi/source/HudiScanNode.java | 120 +++++++++--- .../maxcompute/source/MaxComputeScanNode.java | 2 +- .../org/apache/doris/planner/ScanNode.java | 13 +- .../java/org/apache/doris/qe/Coordinator.java | 11 +- .../org/apache/doris/qe/SessionVariable.java | 17 ++ .../doris/service/FrontendServiceImpl.java | 21 ++ gensrc/thrift/FrontendService.thrift | 10 + gensrc/thrift/PlanNodes.thrift | 7 +- 28 files changed, 1046 insertions(+), 244 deletions(-) create mode 100644 be/src/vec/exec/scan/split_source_connector.cpp create mode 100644 be/src/vec/exec/scan/split_source_connector.h create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSource.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/SplitToScanRange.java diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 3a9e1b89f55bec1..11600c4b5e9b438 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -237,6 +237,7 @@ DEFINE_Validator(doris_scanner_thread_pool_thread_num, [](const int config) -> b } return true; }); +DEFINE_Int32(remote_split_source_batch_size, "1024"); DEFINE_Int32(doris_max_remote_scanner_thread_pool_thread_num, "-1"); // number of olap scanner thread pool queue size DEFINE_Int32(doris_scanner_thread_pool_queue_size, "102400"); diff --git a/be/src/common/config.h b/be/src/common/config.h index 6d22ec483cbe4c7..15450a4b7164904 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -279,6 +279,8 @@ DECLARE_mInt64(doris_blocking_priority_queue_wait_timeout_ms); // number of scanner thread pool size for olap table // and the min thread num of remote scanner thread pool DECLARE_mInt32(doris_scanner_thread_pool_thread_num); +// number of batch size to fetch the remote split source +DECLARE_mInt32(remote_split_source_batch_size); // max number of remote scanner thread pool size // if equal to -1, value is std::max(512, CpuInfo::num_cores() * 10) DECLARE_Int32(doris_max_remote_scanner_thread_pool_thread_num); diff --git a/be/src/pipeline/exec/file_scan_operator.cpp b/be/src/pipeline/exec/file_scan_operator.cpp index f81781481df3f07..59dbbe8d1a52a3e 100644 --- a/be/src/pipeline/exec/file_scan_operator.cpp +++ b/be/src/pipeline/exec/file_scan_operator.cpp @@ -31,7 +31,7 @@ namespace doris::pipeline { Status FileScanLocalState::_init_scanners(std::list* scanners) { - if (_scan_ranges.empty()) { + if (_split_source->num_scan_ranges() == 0) { _eos = true; _scan_dependency->set_ready(); return Status::OK(); @@ -40,13 +40,12 @@ Status FileScanLocalState::_init_scanners(std::list* s auto& p = _parent->cast(); size_t shard_num = std::min( config::doris_scanner_thread_pool_thread_num / state()->query_parallel_instance_num(), - _scan_ranges.size()); + _max_scanners); shard_num = std::max(shard_num, (size_t)1); _kv_cache.reset(new vectorized::ShardedKVCache(shard_num)); - for (auto& scan_range : _scan_ranges) { + for (int i = 0; i < _max_scanners; ++i) { std::unique_ptr scanner = vectorized::VFileScanner::create_unique( - state(), this, p._limit_per_scanner, - scan_range.scan_range.ext_scan_range.file_scan_range, _scanner_profile.get(), + state(), this, p._limit_per_scanner, _split_source, _scanner_profile.get(), _kv_cache.get()); RETURN_IF_ERROR( scanner->prepare(_conjuncts, &_colname_to_value_range, &_colname_to_slot_id)); @@ -62,47 +61,25 @@ std::string FileScanLocalState::name_suffix() const { void FileScanLocalState::set_scan_ranges(RuntimeState* state, const std::vector& scan_ranges) { - int max_scanners = + _max_scanners = config::doris_scanner_thread_pool_thread_num / state->query_parallel_instance_num(); - max_scanners = std::max(std::max(max_scanners, state->parallel_scan_max_scanners_count()), 1); + _max_scanners = std::max(std::max(_max_scanners, state->parallel_scan_max_scanners_count()), 1); // For select * from table limit 10; should just use one thread. if (should_run_serial()) { - max_scanners = 1; + _max_scanners = 1; } - if (scan_ranges.size() <= max_scanners) { - _scan_ranges = scan_ranges; - } else { - // There is no need for the number of scanners to exceed the number of threads in thread pool. - // scan_ranges is sorted by path(as well as partition path) in FE, so merge scan ranges in order. - // In the insert statement, reading data in partition order can reduce the memory usage of BE - // and prevent the generation of smaller tables. - _scan_ranges.resize(max_scanners); - int num_ranges = scan_ranges.size() / max_scanners; - int num_add_one = scan_ranges.size() - num_ranges * max_scanners; - int scan_index = 0; - int range_index = 0; - for (int i = 0; i < num_add_one; ++i) { - _scan_ranges[scan_index] = scan_ranges[range_index++]; - auto& ranges = - _scan_ranges[scan_index++].scan_range.ext_scan_range.file_scan_range.ranges; - for (int j = 0; j < num_ranges; j++) { - auto& merged_ranges = - scan_ranges[range_index++].scan_range.ext_scan_range.file_scan_range.ranges; - ranges.insert(ranges.end(), merged_ranges.begin(), merged_ranges.end()); - } + if (scan_ranges.size() == 1) { + auto scan_range = scan_ranges[0].scan_range.ext_scan_range.file_scan_range; + if (scan_range.__isset.split_source) { + auto split_source = scan_range.split_source; + _split_source = std::make_shared( + state, split_source.split_source_id, split_source.num_splits); } - for (int i = num_add_one; i < max_scanners; ++i) { - _scan_ranges[scan_index] = scan_ranges[range_index++]; - auto& ranges = - _scan_ranges[scan_index++].scan_range.ext_scan_range.file_scan_range.ranges; - for (int j = 0; j < num_ranges - 1; j++) { - auto& merged_ranges = - scan_ranges[range_index++].scan_range.ext_scan_range.file_scan_range.ranges; - ranges.insert(ranges.end(), merged_ranges.begin(), merged_ranges.end()); - } - } - LOG(INFO) << "Merge " << scan_ranges.size() << " scan ranges to " << _scan_ranges.size(); } + if (_split_source == nullptr) { + _split_source = std::make_shared(scan_ranges); + } + _max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges()); if (scan_ranges.size() > 0 && scan_ranges[0].scan_range.ext_scan_range.file_scan_range.__isset.params) { // for compatibility. diff --git a/be/src/pipeline/exec/file_scan_operator.h b/be/src/pipeline/exec/file_scan_operator.h index 4d0c38b2850b4aa..776396a3abb9825 100644 --- a/be/src/pipeline/exec/file_scan_operator.h +++ b/be/src/pipeline/exec/file_scan_operator.h @@ -27,7 +27,7 @@ #include "pipeline/exec/scan_operator.h" #include "pipeline/pipeline_x/operator.h" #include "vec/exec/format/format_common.h" -#include "vec/exec/scan/vscan_node.h" +#include "vec/exec/scan/split_source_connector.h" namespace doris { class ExecNode; @@ -57,7 +57,8 @@ class FileScanLocalState final : public ScanLocalState { std::string name_suffix() const override; private: - std::vector _scan_ranges; + std::shared_ptr _split_source = nullptr; + int _max_scanners; // A in memory cache to save some common components // of the this scan node. eg: // 1. iceberg delete file diff --git a/be/src/vec/exec/scan/new_file_scan_node.cpp b/be/src/vec/exec/scan/new_file_scan_node.cpp index eed7cfaaec6f1eb..d0b0f3cce9b2549 100644 --- a/be/src/vec/exec/scan/new_file_scan_node.cpp +++ b/be/src/vec/exec/scan/new_file_scan_node.cpp @@ -26,6 +26,7 @@ #include "common/config.h" #include "common/object_pool.h" +#include "runtime/client_cache.h" #include "vec/exec/scan/vfile_scanner.h" #include "vec/exec/scan/vscanner.h" @@ -36,6 +37,8 @@ class RuntimeState; namespace doris::vectorized { +using apache::thrift::transport::TTransportException; + NewFileScanNode::NewFileScanNode(ObjectPool* pool, const TPlanNode& tnode, const DescriptorTbl& descs) : VScanNode(pool, tnode, descs) { @@ -60,47 +63,25 @@ Status NewFileScanNode::prepare(RuntimeState* state) { void NewFileScanNode::set_scan_ranges(RuntimeState* state, const std::vector& scan_ranges) { - int max_scanners = + _max_scanners = config::doris_scanner_thread_pool_thread_num / state->query_parallel_instance_num(); - max_scanners = std::max(std::max(max_scanners, state->parallel_scan_max_scanners_count()), 1); + _max_scanners = std::max(std::max(_max_scanners, state->parallel_scan_max_scanners_count()), 1); // For select * from table limit 10; should just use one thread. if (should_run_serial()) { - max_scanners = 1; + _max_scanners = 1; } - if (scan_ranges.size() <= max_scanners) { - _scan_ranges = scan_ranges; - } else { - // There is no need for the number of scanners to exceed the number of threads in thread pool. - // scan_ranges is sorted by path(as well as partition path) in FE, so merge scan ranges in order. - // In the insert statement, reading data in partition order can reduce the memory usage of BE - // and prevent the generation of smaller tables. - _scan_ranges.resize(max_scanners); - int num_ranges = scan_ranges.size() / max_scanners; - int num_add_one = scan_ranges.size() - num_ranges * max_scanners; - int scan_index = 0; - int range_index = 0; - for (int i = 0; i < num_add_one; ++i) { - _scan_ranges[scan_index] = scan_ranges[range_index++]; - auto& ranges = - _scan_ranges[scan_index++].scan_range.ext_scan_range.file_scan_range.ranges; - for (int j = 0; j < num_ranges; j++) { - auto& merged_ranges = - scan_ranges[range_index++].scan_range.ext_scan_range.file_scan_range.ranges; - ranges.insert(ranges.end(), merged_ranges.begin(), merged_ranges.end()); - } - } - for (int i = num_add_one; i < max_scanners; ++i) { - _scan_ranges[scan_index] = scan_ranges[range_index++]; - auto& ranges = - _scan_ranges[scan_index++].scan_range.ext_scan_range.file_scan_range.ranges; - for (int j = 0; j < num_ranges - 1; j++) { - auto& merged_ranges = - scan_ranges[range_index++].scan_range.ext_scan_range.file_scan_range.ranges; - ranges.insert(ranges.end(), merged_ranges.begin(), merged_ranges.end()); - } + if (scan_ranges.size() == 1) { + auto scan_range = scan_ranges[0].scan_range.ext_scan_range.file_scan_range; + if (scan_range.__isset.split_source) { + auto split_source = scan_range.split_source; + _split_source = std::make_shared( + state, split_source.split_source_id, split_source.num_splits); } - LOG(INFO) << "Merge " << scan_ranges.size() << " scan ranges to " << _scan_ranges.size(); } + if (_split_source == nullptr) { + _split_source = std::make_shared(scan_ranges); + } + _max_scanners = std::min(_max_scanners, _split_source->num_scan_ranges()); if (scan_ranges.size() > 0 && scan_ranges[0].scan_range.ext_scan_range.file_scan_range.__isset.params) { // for compatibility. @@ -125,20 +106,19 @@ Status NewFileScanNode::_process_conjuncts() { } Status NewFileScanNode::_init_scanners(std::list* scanners) { - if (_scan_ranges.empty()) { + if (_split_source->num_scan_ranges() == 0) { _eos = true; return Status::OK(); } size_t shard_num = std::min( config::doris_scanner_thread_pool_thread_num / _state->query_parallel_instance_num(), - _scan_ranges.size()); + _max_scanners); shard_num = std::max(shard_num, (size_t)1); _kv_cache.reset(new ShardedKVCache(shard_num)); - for (auto& scan_range : _scan_ranges) { + for (int i = 0; i < _max_scanners; ++i) { std::unique_ptr scanner = - VFileScanner::create_unique(_state, this, _limit_per_scanner, - scan_range.scan_range.ext_scan_range.file_scan_range, + VFileScanner::create_unique(_state, this, _limit_per_scanner, _split_source, runtime_profile(), _kv_cache.get()); RETURN_IF_ERROR( scanner->prepare(_conjuncts, &_colname_to_value_range, &_colname_to_slot_id)); diff --git a/be/src/vec/exec/scan/new_file_scan_node.h b/be/src/vec/exec/scan/new_file_scan_node.h index e6d9fc1e621d983..24af199c96030c2 100644 --- a/be/src/vec/exec/scan/new_file_scan_node.h +++ b/be/src/vec/exec/scan/new_file_scan_node.h @@ -25,6 +25,7 @@ #include "common/status.h" #include "vec/exec/format/format_common.h" +#include "vec/exec/scan/split_source_connector.h" #include "vec/exec/scan/vscan_node.h" namespace doris { @@ -58,7 +59,8 @@ class NewFileScanNode : public VScanNode { Status _init_scanners(std::list* scanners) override; private: - std::vector _scan_ranges; + std::shared_ptr _split_source = nullptr; + int _max_scanners; // A in memory cache to save some common components // of the this scan node. eg: // 1. iceberg delete file diff --git a/be/src/vec/exec/scan/split_source_connector.cpp b/be/src/vec/exec/scan/split_source_connector.cpp new file mode 100644 index 000000000000000..2d35d3796bcc7a2 --- /dev/null +++ b/be/src/vec/exec/scan/split_source_connector.cpp @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "vec/exec/scan/split_source_connector.h" + +#include "runtime/exec_env.h" +#include "runtime/query_context.h" + +namespace doris::vectorized { + +using apache::thrift::transport::TTransportException; + +Status LocalSplitSourceConnector::get_next(bool* has_next, TFileRangeDesc* range) { + std::lock_guard l(_range_lock); + *has_next = false; + if (_scan_index < _scan_ranges.size()) { + auto& ranges = _scan_ranges[_scan_index].scan_range.ext_scan_range.file_scan_range.ranges; + if (_range_index < ranges.size()) { + *has_next = true; + *range = ranges[_range_index++]; + if (_range_index == ranges.size()) { + _scan_index++; + _range_index = 0; + } + } + } + return Status::OK(); +} + +Status RemoteSplitSourceConnector::get_next(bool* has_next, TFileRangeDesc* range) { + std::lock_guard l(_range_lock); + *has_next = false; + if (_scan_index == _scan_ranges.size() && !_last_batch) { + Status coord_status; + FrontendServiceConnection coord(_state->exec_env()->frontend_client_cache(), + _state->get_query_ctx()->coord_addr, &coord_status); + RETURN_IF_ERROR(coord_status); + TFetchSplitBatchRequest request; + request.__set_split_source_id(_split_source_id); + request.__set_max_num_splits(config::remote_split_source_batch_size); + TFetchSplitBatchResult result; + try { + coord->fetchSplitBatch(result, request); + } catch (std::exception& e1) { + LOG(WARNING) << "Failed to get batch of split source: {}, try to reopen" << e1.what(); + RETURN_IF_ERROR(coord.reopen()); + try { + coord->fetchSplitBatch(result, request); + } catch (std::exception& e2) { + return Status::IOError("Failed to get batch of split source: {}", e2.what()); + } + } + _last_batch = result.splits.empty(); + _scan_ranges = result.splits; + _scan_index = 0; + _range_index = 0; + } + if (_scan_index < _scan_ranges.size()) { + auto& ranges = _scan_ranges[_scan_index].scan_range.ext_scan_range.file_scan_range.ranges; + if (_range_index < ranges.size()) { + *has_next = true; + *range = ranges[_range_index++]; + if (_range_index == ranges.size()) { + _scan_index++; + _range_index = 0; + } + } + } + return Status::OK(); +} + +} // namespace doris::vectorized diff --git a/be/src/vec/exec/scan/split_source_connector.h b/be/src/vec/exec/scan/split_source_connector.h new file mode 100644 index 000000000000000..cf358846b30f34d --- /dev/null +++ b/be/src/vec/exec/scan/split_source_connector.h @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "common/config.h" +#include "runtime/client_cache.h" +#include "runtime/runtime_state.h" + +namespace doris::vectorized { + +/* + * Multiple scanners within a scan node share a split source. + * Each scanner call `get_next` to get the next scan range. A fast scanner will immediately obtain + * the next scan range, so there is no situation of data skewing. + */ +class SplitSourceConnector { +public: + SplitSourceConnector() = default; + virtual ~SplitSourceConnector() = default; + + /** + * Get the next scan range. has_next should be to true to fetch the next scan range. + * @param has_next whether exists the next scan range + * @param range the obtained next scan range + */ + virtual Status get_next(bool* has_next, TFileRangeDesc* range) = 0; + + virtual int num_scan_ranges() = 0; + + virtual TFileScanRangeParams* get_params() = 0; +}; + +/** + * The file splits are already assigned in `TFileScanRange.ranges`. Scan node has need to + * fetch the scan ranges from frontend. + * + * In cases where the number of files is small, the splits are directly transmitted to backend. + */ +class LocalSplitSourceConnector : public SplitSourceConnector { +private: + std::mutex _range_lock; + std::vector _scan_ranges; + int _scan_index = 0; + int _range_index = 0; + +public: + LocalSplitSourceConnector(const std::vector& scan_ranges) + : _scan_ranges(scan_ranges) {} + + Status get_next(bool* has_next, TFileRangeDesc* range) override; + + int num_scan_ranges() override { return _scan_ranges.size(); } + + TFileScanRangeParams* get_params() override { + if (_scan_ranges.size() > 0 && + _scan_ranges[0].scan_range.ext_scan_range.file_scan_range.__isset.params) { + // for compatibility. + return &_scan_ranges[0].scan_range.ext_scan_range.file_scan_range.params; + } + LOG(FATAL) << "Unreachable, params is got by file_scan_range_params_map"; + } +}; + +/** + * The file splits are lazily generated in frontend, and saved as a split source in frontend. + * The scan node needs to fetch file splits from the frontend service. Each split source is identified by + * a unique ID, and the ID is stored in `TFileScanRange.split_source.split_source_id` + * + * In the case of a large number of files, backend can scan data while obtaining splits information. + */ +class RemoteSplitSourceConnector : public SplitSourceConnector { +private: + std::mutex _range_lock; + RuntimeState* _state; + int64 _split_source_id; + int _num_splits; + + std::vector _scan_ranges; + bool _last_batch = false; + int _scan_index = 0; + int _range_index = 0; + +public: + RemoteSplitSourceConnector(RuntimeState* state, int64 split_source_id, int num_splits) + : _state(state), _split_source_id(split_source_id), _num_splits(num_splits) {} + + Status get_next(bool* has_next, TFileRangeDesc* range) override; + + /* + * Remote split source is fetched in batch mode, and the splits are generated while scanning, + * so the number of scan ranges may not be accurate. + */ + int num_scan_ranges() override { return _num_splits; } + + TFileScanRangeParams* get_params() override { + LOG(FATAL) << "Unreachable, params is got by file_scan_range_params_map"; + } +}; + +} // namespace doris::vectorized diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index 03a79c0a18b359a..398621b08ea5409 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -87,25 +87,23 @@ namespace doris::vectorized { using namespace ErrorCode; VFileScanner::VFileScanner(RuntimeState* state, NewFileScanNode* parent, int64_t limit, - const TFileScanRange& scan_range, RuntimeProfile* profile, - ShardedKVCache* kv_cache) + std::shared_ptr split_source, + RuntimeProfile* profile, ShardedKVCache* kv_cache) : VScanner(state, static_cast(parent), limit, profile), - _ranges(scan_range.ranges), - _next_range(0), + _split_source(split_source), _cur_reader(nullptr), _cur_reader_eof(false), _kv_cache(kv_cache), _strict_mode(false) { - if (scan_range.params.__isset.strict_mode) { - _strict_mode = scan_range.params.strict_mode; - } - if (state->get_query_ctx() != nullptr && state->get_query_ctx()->file_scan_range_params_map.count(parent->id()) > 0) { _params = &(state->get_query_ctx()->file_scan_range_params_map[parent->id()]); } else { - CHECK(scan_range.__isset.params); - _params = &(scan_range.params); + // old fe thrift protocol + _params = _split_source->get_params(); + } + if (_params->__isset.strict_mode) { + _strict_mode = _params->strict_mode; } // For load scanner, there are input and output tuple. @@ -116,25 +114,24 @@ VFileScanner::VFileScanner(RuntimeState* state, NewFileScanNode* parent, int64_t } VFileScanner::VFileScanner(RuntimeState* state, pipeline::FileScanLocalState* local_state, - int64_t limit, const TFileScanRange& scan_range, RuntimeProfile* profile, - ShardedKVCache* kv_cache) + int64_t limit, + std::shared_ptr split_source, + RuntimeProfile* profile, ShardedKVCache* kv_cache) : VScanner(state, local_state, limit, profile), - _ranges(scan_range.ranges), - _next_range(0), + _split_source(split_source), _cur_reader(nullptr), _cur_reader_eof(false), _kv_cache(kv_cache), _strict_mode(false) { - if (scan_range.params.__isset.strict_mode) { - _strict_mode = scan_range.params.strict_mode; - } - if (state->get_query_ctx() != nullptr && state->get_query_ctx()->file_scan_range_params_map.count(local_state->parent_id()) > 0) { _params = &(state->get_query_ctx()->file_scan_range_params_map[local_state->parent_id()]); } else { - CHECK(scan_range.__isset.params); - _params = &(scan_range.params); + // old fe thrift protocol + _params = _split_source->get_params(); + } + if (_params->__isset.strict_mode) { + _strict_mode = _params->strict_mode; } // For load scanner, there are input and output tuple. @@ -284,7 +281,13 @@ void VFileScanner::_get_slot_ids(VExpr* expr, std::vector* slot_ids) { Status VFileScanner::open(RuntimeState* state) { RETURN_IF_CANCELLED(state); RETURN_IF_ERROR(VScanner::open(state)); - RETURN_IF_ERROR(_init_expr_ctxes()); + RETURN_IF_ERROR(_split_source->get_next(&_first_scan_range, &_current_range)); + if (_first_scan_range) { + RETURN_IF_ERROR(_init_expr_ctxes()); + } else { + // there's no scan range in split source. stop scanner directly. + _scanner_eof = true; + } return Status::OK(); } @@ -735,19 +738,21 @@ Status VFileScanner::_get_next_reader() { if (_cur_reader) { _cur_reader->collect_profile_before_close(); RETURN_IF_ERROR(_cur_reader->close()); + _state->update_num_finished_scan_range(1); } _cur_reader.reset(nullptr); _src_block_init = false; - if (_next_range >= _ranges.size() || _should_stop) { + bool has_next = _first_scan_range; + if (!_first_scan_range) { + RETURN_IF_ERROR(_split_source->get_next(&has_next, &_current_range)); + } + _first_scan_range = false; + if (!has_next || _should_stop) { _scanner_eof = true; - _state->update_num_finished_scan_range(1); return Status::OK(); } - if (_next_range != 0) { - _state->update_num_finished_scan_range(1); - } - const TFileRangeDesc& range = _ranges[_next_range++]; + const TFileRangeDesc& range = _current_range; _current_range_path = range.path; // create reader for specific format @@ -995,7 +1000,7 @@ Status VFileScanner::_generate_fill_columns() { _partition_col_descs.clear(); _missing_col_descs.clear(); - const TFileRangeDesc& range = _ranges.at(_next_range - 1); + const TFileRangeDesc& range = _current_range; if (range.__isset.columns_from_path && !_partition_slot_descs.empty()) { for (const auto& slot_desc : _partition_slot_descs) { if (slot_desc) { @@ -1038,8 +1043,6 @@ Status VFileScanner::_generate_fill_columns() { } Status VFileScanner::_init_expr_ctxes() { - DCHECK(!_ranges.empty()); - std::map full_src_index_map; std::map full_src_slot_map; std::map partition_name_to_key_index_map; @@ -1055,8 +1058,8 @@ Status VFileScanner::_init_expr_ctxes() { // All ranges in _ranges vector should have identical columns_from_path_keys // because they are all file splits for the same external table. // So here use the first element of _ranges to fill the partition_name_to_key_index_map - if (_ranges[0].__isset.columns_from_path_keys) { - std::vector key_map = _ranges[0].columns_from_path_keys; + if (_current_range.__isset.columns_from_path_keys) { + std::vector key_map = _current_range.columns_from_path_keys; if (!key_map.empty()) { for (size_t i = 0; i < key_map.size(); i++) { partition_name_to_key_index_map.emplace(key_map[i], i); diff --git a/be/src/vec/exec/scan/vfile_scanner.h b/be/src/vec/exec/scan/vfile_scanner.h index bdaefc595e3a2fb..43c1a8b13da51a5 100644 --- a/be/src/vec/exec/scan/vfile_scanner.h +++ b/be/src/vec/exec/scan/vfile_scanner.h @@ -64,12 +64,12 @@ class VFileScanner : public VScanner { static constexpr const char* NAME = "VFileScanner"; VFileScanner(RuntimeState* state, NewFileScanNode* parent, int64_t limit, - const TFileScanRange& scan_range, RuntimeProfile* profile, - ShardedKVCache* kv_cache); + std::shared_ptr split_source, + RuntimeProfile* profile, ShardedKVCache* kv_cache); VFileScanner(RuntimeState* state, pipeline::FileScanLocalState* parent, int64_t limit, - const TFileScanRange& scan_range, RuntimeProfile* profile, - ShardedKVCache* kv_cache); + std::shared_ptr split_source, + RuntimeProfile* profile, ShardedKVCache* kv_cache); Status open(RuntimeState* state) override; @@ -99,8 +99,9 @@ class VFileScanner : public VScanner { protected: const TFileScanRangeParams* _params = nullptr; - const std::vector& _ranges; - int _next_range; + std::shared_ptr _split_source; + bool _first_scan_range = false; + TFileRangeDesc _current_range; std::unique_ptr _cur_reader; bool _cur_reader_eof; @@ -243,7 +244,7 @@ class VFileScanner : public VScanner { // Otherwise, the cache miss rate will be high bool _shoudl_enable_file_meta_cache() { return config::max_external_file_meta_cache_num > 0 && - _ranges.size() < config::max_external_file_meta_cache_num / 3; + _split_source->num_scan_ranges() < config::max_external_file_meta_cache_num / 3; } }; } // namespace doris::vectorized diff --git a/be/test/vec/exec/vwal_scanner_test.cpp b/be/test/vec/exec/vwal_scanner_test.cpp index a362504a4bfb372..d9d753d4d2d3d68 100644 --- a/be/test/vec/exec/vwal_scanner_test.cpp +++ b/be/test/vec/exec/vwal_scanner_test.cpp @@ -35,6 +35,32 @@ namespace doris { namespace vectorized { + +class TestSplitSourceConnector : public SplitSourceConnector { +private: + std::mutex _range_lock; + TFileScanRange _scan_range; + int _range_index = 0; + +public: + TestSplitSourceConnector(const TFileScanRange& scan_range) : _scan_range(scan_range) {} + + Status get_next(bool* has_next, TFileRangeDesc* range) override { + std::lock_guard l(_range_lock); + if (_range_index < _scan_range.ranges.size()) { + *has_next = true; + *range = _scan_range.ranges[_range_index++]; + } else { + *has_next = false; + } + return Status::OK(); + } + + int num_scan_ranges() override { return _scan_range.ranges.size(); } + + TFileScanRangeParams* get_params() override { return &_scan_range.params; } +}; + class VWalScannerTest : public testing::Test { public: VWalScannerTest() : _runtime_state(TQueryGlobals()) { @@ -266,7 +292,8 @@ void VWalScannerTest::init() { } void VWalScannerTest::generate_scanner(std::shared_ptr& scanner) { - scanner = std::make_shared(&_runtime_state, _scan_node.get(), -1, _scan_range, + auto split_source = std::make_shared(_scan_range); + scanner = std::make_shared(&_runtime_state, _scan_node.get(), -1, split_source, _profile, _kv_cache.get()); scanner->_is_load = false; vectorized::VExprContextSPtrs _conjuncts; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index d27ae1473235686..d0223e626bc98b8 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -132,6 +132,7 @@ import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalMetaIdMgr; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.SplitSourceManager; import org.apache.doris.datasource.es.EsExternalCatalog; import org.apache.doris.datasource.es.EsRepository; import org.apache.doris.datasource.hive.HiveTransactionMgr; @@ -535,6 +536,8 @@ public class Env { private final NereidsSqlCacheManager sqlCacheManager; + private final SplitSourceManager splitSourceManager; + public List getFrontendInfos() { List res = new ArrayList<>(); @@ -772,6 +775,7 @@ public Env(boolean isCheckpointCatalog) { this.insertOverwriteManager = new InsertOverwriteManager(); this.dnsCache = new DNSCache(); this.sqlCacheManager = new NereidsSqlCacheManager(); + this.splitSourceManager = new SplitSourceManager(); } public static void destroyCheckpoint() { @@ -1712,7 +1716,7 @@ private void startNonMasterDaemonThreads() { workloadGroupMgr.start(); workloadSchedPolicyMgr.start(); workloadRuntimeStatusMgr.start(); - + splitSourceManager.start(); } private void transferToNonMaster(FrontendNodeType newType) { @@ -6058,6 +6062,10 @@ public NereidsSqlCacheManager getSqlCacheManager() { return sqlCacheManager; } + public SplitSourceManager getSplitSourceManager() { + return splitSourceManager; + } + public void alterMTMVRefreshInfo(AlterMTMVRefreshInfo info) { AlterMTMV alter = new AlterMTMV(info.getMvName(), info.getRefreshInfo(), MTMVAlterOpType.ALTER_REFRESH_INFO); this.alter.processAlterMTMV(alter, false); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 7afb04831ce0294..ff25464b4c9522f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -60,12 +60,12 @@ import org.apache.doris.thrift.TScanRange; import org.apache.doris.thrift.TScanRangeLocation; import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.doris.thrift.TSplitSource; import org.apache.doris.thrift.TTableFormatFileDesc; import org.apache.doris.thrift.TTextSerdeType; import org.apache.doris.thrift.TTransactionalHiveDeleteDeltaDesc; import org.apache.doris.thrift.TTransactionalHiveDesc; -import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -124,6 +124,7 @@ public void init(Analyzer analyzer) throws UserException { */ @Override public void init() throws UserException { + super.init(); if (ConnectContext.get().getExecutor() != null) { ConnectContext.get().getExecutor().getSummaryProfile().setInitScanNodeStartTime(); } @@ -268,14 +269,6 @@ public void createScanRangeLocations() throws UserException { if (ConnectContext.get().getExecutor() != null) { ConnectContext.get().getExecutor().getSummaryProfile().setGetSplitsStartTime(); } - List inputSplits = getSplits(); - if (ConnectContext.get().getExecutor() != null) { - ConnectContext.get().getExecutor().getSummaryProfile().setGetSplitsFinishTime(); - } - this.inputSplitsNum = inputSplits.size(); - if (inputSplits.isEmpty() && !(getLocationType() == TFileType.FILE_STREAM)) { - return; - } TFileFormatType fileFormatType = getFileFormatType(); if (fileFormatType == TFileFormatType.FORMAT_ORC) { genSlotToSchemaIdMapForOrc(); @@ -317,72 +310,66 @@ public void createScanRangeLocations() throws UserException { } List pathPartitionKeys = getPathPartitionKeys(); - - Multimap assignment = backendPolicy.computeScanRangeAssignment(inputSplits); - for (Backend backend : assignment.keySet()) { - Collection splits = assignment.get(backend); - for (Split split : splits) { - FileSplit fileSplit = (FileSplit) split; - TFileType locationType; - if (fileSplit instanceof IcebergSplit - && ((IcebergSplit) fileSplit).getConfig().containsKey(HMSExternalCatalog.BIND_BROKER_NAME)) { - locationType = TFileType.FILE_BROKER; - } else { - locationType = getLocationType(fileSplit.getPath().toString()); - } - + if (isBatchMode()) { + // File splits are generated lazily, and fetched by backends while scanning. + // Only provide the unique ID of split source to backend. + SplitAssignment splitAssignment = new SplitAssignment(backendPolicy, this); + splitAssignment.init(); + if (ConnectContext.get().getExecutor() != null) { + ConnectContext.get().getExecutor().getSummaryProfile().setGetSplitsFinishTime(); + } + if (splitAssignment.getCurrentAssignment().isEmpty() && !(getLocationType() == TFileType.FILE_STREAM)) { + return; + } + inputSplitsNum = splitAssignment.numApproximateSplits(); + + TFileType locationType; + FileSplit fileSplit = (FileSplit) splitAssignment.getCurrentAssignment().values().iterator().next(); + if (fileSplit instanceof IcebergSplit + && ((IcebergSplit) fileSplit).getConfig().containsKey(HMSExternalCatalog.BIND_BROKER_NAME)) { + locationType = TFileType.FILE_BROKER; + } else { + locationType = getLocationType(fileSplit.getPath().toString()); + } + totalFileSize = fileSplit.getLength() * inputSplitsNum; + // Not accurate, only used to estimate concurrency. + int numSplitsPerBE = splitAssignment.numApproximateSplits() / backendPolicy.numBackends(); + for (Backend backend : backendPolicy.getBackends()) { + SplitSource splitSource = new SplitSource( + this::splitToScanRange, backend, locationProperties, splitAssignment, pathPartitionKeys); + splitSources.add(splitSource); + Env.getCurrentEnv().getSplitSourceManager().registerSplitSource(splitSource); TScanRangeLocations curLocations = newLocations(); - // If fileSplit has partition values, use the values collected from hive partitions. - // Otherwise, use the values in file path. - boolean isACID = false; - if (fileSplit instanceof HiveSplit) { - HiveSplit hiveSplit = (HiveSplit) fileSplit; - isACID = hiveSplit.isACID(); - } - List partitionValuesFromPath = fileSplit.getPartitionValues() == null - ? BrokerUtil.parseColumnsFromPath(fileSplit.getPath().toString(), pathPartitionKeys, - false, isACID) : fileSplit.getPartitionValues(); - - TFileRangeDesc rangeDesc = createFileRangeDesc(fileSplit, partitionValuesFromPath, pathPartitionKeys, - locationType); - TFileCompressType fileCompressType = getFileCompressType(fileSplit); - rangeDesc.setCompressType(fileCompressType); - if (isACID) { - HiveSplit hiveSplit = (HiveSplit) fileSplit; - hiveSplit.setTableFormatType(TableFormatType.TRANSACTIONAL_HIVE); - TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); - tableFormatFileDesc.setTableFormatType(hiveSplit.getTableFormatType().value()); - AcidInfo acidInfo = (AcidInfo) hiveSplit.getInfo(); - TTransactionalHiveDesc transactionalHiveDesc = new TTransactionalHiveDesc(); - transactionalHiveDesc.setPartition(acidInfo.getPartitionLocation()); - List deleteDeltaDescs = new ArrayList<>(); - for (DeleteDeltaInfo deleteDeltaInfo : acidInfo.getDeleteDeltas()) { - TTransactionalHiveDeleteDeltaDesc deleteDeltaDesc = new TTransactionalHiveDeleteDeltaDesc(); - deleteDeltaDesc.setDirectoryLocation(deleteDeltaInfo.getDirectoryLocation()); - deleteDeltaDesc.setFileNames(deleteDeltaInfo.getFileNames()); - deleteDeltaDescs.add(deleteDeltaDesc); - } - transactionalHiveDesc.setDeleteDeltas(deleteDeltaDescs); - tableFormatFileDesc.setTransactionalHiveParams(transactionalHiveDesc); - rangeDesc.setTableFormatParams(tableFormatFileDesc); - } - - setScanParams(rangeDesc, fileSplit); - - curLocations.getScanRange().getExtScanRange().getFileScanRange().addToRanges(rangeDesc); + TSplitSource tSource = new TSplitSource(); + tSource.setSplitSourceId(splitSource.getUniqueId()); + tSource.setNumSplits(numSplitsPerBE); + curLocations.getScanRange().getExtScanRange().getFileScanRange().setSplitSource(tSource); TScanRangeLocation location = new TScanRangeLocation(); - setLocationPropertiesIfNecessary(backend, locationType, locationProperties); location.setBackendId(backend.getId()); location.setServer(new TNetworkAddress(backend.getHost(), backend.getBePort())); curLocations.addToLocations(location); - if (LOG.isDebugEnabled()) { - LOG.debug("assign to backend {} with table split: {} ({}, {}), location: {}", - curLocations.getLocations().get(0).getBackendId(), fileSplit.getPath(), - fileSplit.getStart(), fileSplit.getLength(), - Joiner.on("|").join(fileSplit.getHosts())); - } + // So there's only one scan range for each backend. + // Each backend only starts up one ScanNode instance. + // However, even one ScanNode instance can provide maximum scanning concurrency. scanRangeLocations.add(curLocations); - this.totalFileSize += fileSplit.getLength(); + setLocationPropertiesIfNecessary(backend, locationType, locationProperties); + } + } else { + List inputSplits = getSplits(); + if (ConnectContext.get().getExecutor() != null) { + ConnectContext.get().getExecutor().getSummaryProfile().setGetSplitsFinishTime(); + } + inputSplitsNum = inputSplits.size(); + if (inputSplits.isEmpty() && !(getLocationType() == TFileType.FILE_STREAM)) { + return; + } + Multimap assignment = backendPolicy.computeScanRangeAssignment(inputSplits); + for (Backend backend : assignment.keySet()) { + Collection splits = assignment.get(backend); + for (Split split : splits) { + scanRangeLocations.add(splitToScanRange(backend, locationProperties, split, pathPartitionKeys)); + totalFileSize += split.getLength(); + } } } @@ -395,6 +382,66 @@ public void createScanRangeLocations() throws UserException { } } + private TScanRangeLocations splitToScanRange( + Backend backend, + Map locationProperties, + Split split, + List pathPartitionKeys) throws UserException { + FileSplit fileSplit = (FileSplit) split; + TFileType locationType; + if (fileSplit instanceof IcebergSplit + && ((IcebergSplit) fileSplit).getConfig().containsKey(HMSExternalCatalog.BIND_BROKER_NAME)) { + locationType = TFileType.FILE_BROKER; + } else { + locationType = getLocationType(fileSplit.getPath().toString()); + } + + TScanRangeLocations curLocations = newLocations(); + // If fileSplit has partition values, use the values collected from hive partitions. + // Otherwise, use the values in file path. + boolean isACID = false; + if (fileSplit instanceof HiveSplit) { + HiveSplit hiveSplit = (HiveSplit) fileSplit; + isACID = hiveSplit.isACID(); + } + List partitionValuesFromPath = fileSplit.getPartitionValues() == null + ? BrokerUtil.parseColumnsFromPath(fileSplit.getPath().toString(), pathPartitionKeys, + false, isACID) : fileSplit.getPartitionValues(); + + TFileRangeDesc rangeDesc = createFileRangeDesc(fileSplit, partitionValuesFromPath, pathPartitionKeys, + locationType); + TFileCompressType fileCompressType = getFileCompressType(fileSplit); + rangeDesc.setCompressType(fileCompressType); + if (isACID) { + HiveSplit hiveSplit = (HiveSplit) fileSplit; + hiveSplit.setTableFormatType(TableFormatType.TRANSACTIONAL_HIVE); + TTableFormatFileDesc tableFormatFileDesc = new TTableFormatFileDesc(); + tableFormatFileDesc.setTableFormatType(hiveSplit.getTableFormatType().value()); + AcidInfo acidInfo = (AcidInfo) hiveSplit.getInfo(); + TTransactionalHiveDesc transactionalHiveDesc = new TTransactionalHiveDesc(); + transactionalHiveDesc.setPartition(acidInfo.getPartitionLocation()); + List deleteDeltaDescs = new ArrayList<>(); + for (DeleteDeltaInfo deleteDeltaInfo : acidInfo.getDeleteDeltas()) { + TTransactionalHiveDeleteDeltaDesc deleteDeltaDesc = new TTransactionalHiveDeleteDeltaDesc(); + deleteDeltaDesc.setDirectoryLocation(deleteDeltaInfo.getDirectoryLocation()); + deleteDeltaDesc.setFileNames(deleteDeltaInfo.getFileNames()); + deleteDeltaDescs.add(deleteDeltaDesc); + } + transactionalHiveDesc.setDeleteDeltas(deleteDeltaDescs); + tableFormatFileDesc.setTransactionalHiveParams(transactionalHiveDesc); + rangeDesc.setTableFormatParams(tableFormatFileDesc); + } + + setScanParams(rangeDesc, fileSplit); + curLocations.getScanRange().getExtScanRange().getFileScanRange().addToRanges(rangeDesc); + TScanRangeLocation location = new TScanRangeLocation(); + setLocationPropertiesIfNecessary(backend, locationType, locationProperties); + location.setBackendId(backend.getId()); + location.setServer(new TNetworkAddress(backend.getHost(), backend.getBePort())); + curLocations.addToLocations(location); + return curLocations; + } + private void setLocationPropertiesIfNecessary(Backend selectedBackend, TFileType locationType, Map locationProperties) throws UserException { if (locationType == TFileType.FILE_HDFS || locationType == TFileType.FILE_BROKER) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java index 71008e21088a95b..227f636e67b9cd7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java @@ -69,6 +69,7 @@ public abstract class FileScanNode extends ExternalScanNode { protected long totalFileSize = 0; protected long totalPartitionNum = 0; protected long readPartitionNum = 0; + protected long fileSplitSize; public FileScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType, boolean needCheckColumnPriv) { @@ -76,6 +77,11 @@ public FileScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, St this.needCheckColumnPriv = needCheckColumnPriv; } + @Override + public void init() throws UserException { + this.fileSplitSize = ConnectContext.get().getSessionVariable().getFileSplitSize(); + } + @Override protected void toThrift(TPlanNode planNode) { planNode.setPushDownAggTypeOpt(pushDownAggNoGroupingOp); @@ -233,18 +239,17 @@ protected List splitFile(Path path, long blockSize, BlockLocation[] block result.add(splitCreator.create(path, 0, length, length, modificationTime, hosts, partitionValues)); return result; } - long splitSize = ConnectContext.get().getSessionVariable().getFileSplitSize(); - if (splitSize <= 0) { - splitSize = blockSize; + if (fileSplitSize <= 0) { + fileSplitSize = blockSize; } // Min split size is DEFAULT_SPLIT_SIZE(128MB). - splitSize = Math.max(splitSize, DEFAULT_SPLIT_SIZE); + fileSplitSize = Math.max(fileSplitSize, DEFAULT_SPLIT_SIZE); long bytesRemaining; - for (bytesRemaining = length; (double) bytesRemaining / (double) splitSize > 1.1D; - bytesRemaining -= splitSize) { + for (bytesRemaining = length; (double) bytesRemaining / (double) fileSplitSize > 1.1D; + bytesRemaining -= fileSplitSize) { int location = getBlockIndex(blockLocations, length - bytesRemaining); String[] hosts = location == -1 ? null : blockLocations[location].getHosts(); - result.add(splitCreator.create(path, length - bytesRemaining, splitSize, + result.add(splitCreator.create(path, length - bytesRemaining, fileSplitSize, length, modificationTime, hosts, partitionValues)); } if (bytesRemaining != 0L) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java new file mode 100644 index 000000000000000..f41eaba7dd88200 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitAssignment.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.UserException; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.spi.Split; +import org.apache.doris.system.Backend; + +import com.google.common.collect.ArrayListMultimap; +import com.google.common.collect.Multimap; + +import java.util.Collection; + +/** + * When file splits are supplied in batch mode, splits are generated lazily and assigned in each call of `getNextBatch`. + * `SplitGenerator` provides the file splits, and `FederationBackendPolicy` assigns these splits to backends. + */ +public class SplitAssignment { + // magic number to estimate how many splits are allocated to BE in each batch + private static final int NUM_SPLITS_PER_BE = 1024; + // magic number to estimate how many splits are generated of each partition in each batch. + private static final int NUM_SPLITS_PER_PARTITION = 10; + + private final FederationBackendPolicy backendPolicy; + private final SplitGenerator splitGenerator; + // Store the current assignment of file splits + private final Multimap assignment; + private final int maxBatchSize; + + public SplitAssignment(FederationBackendPolicy backendPolicy, SplitGenerator splitGenerator) { + this.backendPolicy = backendPolicy; + this.splitGenerator = splitGenerator; + this.assignment = ArrayListMultimap.create(); + int numPartitions = ConnectContext.get().getSessionVariable().getNumPartitionsInBatchMode(); + maxBatchSize = Math.min(NUM_SPLITS_PER_PARTITION * numPartitions, + NUM_SPLITS_PER_BE * backendPolicy.numBackends()); + } + + public void init() throws UserException { + if (assignment.isEmpty() && splitGenerator.hasNext()) { + assignment.putAll(backendPolicy.computeScanRangeAssignment(splitGenerator.getNextBatch(maxBatchSize))); + } + } + + public Multimap getCurrentAssignment() { + return assignment; + } + + public int numApproximateSplits() { + return splitGenerator.numApproximateSplits(); + } + + public synchronized Collection getNextBatch(Backend backend) throws UserException { + // Each call should consume all splits + Collection splits = assignment.removeAll(backend); + while (splits.isEmpty()) { + // Get the next batch of splits, and assign to backends + // If there is data skewing, it maybe causes splits to accumulate on some BE + if (!splitGenerator.hasNext()) { + return splits; + } + // todo: In each batch, it's to find the optimal assignment for partial splits, + // how to solve the global data skew? + assignment.putAll(backendPolicy.computeScanRangeAssignment(splitGenerator.getNextBatch(maxBatchSize))); + splits = assignment.removeAll(backend); + } + return splits; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java new file mode 100644 index 000000000000000..b819c7f9ef20f54 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.NotImplementedException; +import org.apache.doris.common.UserException; +import org.apache.doris.spi.Split; + +import java.util.List; + +/** + * The Producer(e.g. ScanNode) that provides the file splits in lazy and batch mode. + * The consumer should call `getNextBatch` to fetch the next batch of splits. + */ +public interface SplitGenerator { + /** + * Get the next batch of splits. If the producer(e.g. ScanNode) doesn't support batch mode, + * should throw user exceptions. + */ + default List getNextBatch(int maxBatchSize) throws UserException { + throw new NotImplementedException("Should implement getNextBatch if in batch mode."); + } + + /** + * Get all file splits if the producer doesn't support batch mode. + */ + default List getSplits() throws UserException { + // todo: remove this interface if batch mode is stable + throw new NotImplementedException("Scan node sub class need to implement getSplits interface."); + } + + /** + * `getNextBatch` should return empty list even if `hasNext` returns false. + */ + default boolean hasNext() { + return false; + } + + /** + * Whether the producer(e.g. ScanNode) support batch mode. + */ + default boolean isBatchMode() { + return false; + } + + /** + * Because file splits are generated lazily, the exact number of splits may not be known, + * provide an estimated value to show in describe statement. + */ + default int numApproximateSplits() { + return -1; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSource.java new file mode 100644 index 000000000000000..74e6aa88ba32324 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSource.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.UserException; +import org.apache.doris.spi.Split; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TScanRangeLocations; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; + +/** + * If there are many files, splitting these files into scan ranges will consume a lot of time. + * Even the simplest queries(e.g. select * from large_table limit 1) can get stuck or crash due to the split process. + * Furthermore, during the splitting process, the backend did not do anything. + * It is completely possible to split files whiling scanning data on the ready splits at once. + * `SplitSource` introduce a lazy and batch mode to provide the file splits. Each `SplitSource` has a unique ID, + * which is used by backends to call `FrontendServiceImpl#fetchSplitBatch` to fetch splits batch by batch. + * `SplitSource`s are managed by `SplitSourceManager`, which stores `SplitSource` as a weak reference, and clean + * the split source when its related scan node is GC. + */ +public class SplitSource { + private static final AtomicLong UNIQUE_ID_GENERATOR = new AtomicLong(0); + + private final long uniqueId; + private final SplitToScanRange splitToScanRange; + private final Backend backend; + private final Map locationProperties; + private final List pathPartitionKeys; + private final SplitAssignment splitAssignment; + private Iterator splitIterator = null; + private boolean isLastBatch = false; + + public SplitSource( + SplitToScanRange splitToScanRange, + Backend backend, + Map locationProperties, + SplitAssignment splitAssignment, + List pathPartitionKeys) { + this.uniqueId = UNIQUE_ID_GENERATOR.getAndIncrement(); + this.splitToScanRange = splitToScanRange; + this.backend = backend; + this.locationProperties = locationProperties; + this.pathPartitionKeys = pathPartitionKeys; + this.splitAssignment = splitAssignment; + } + + public long getUniqueId() { + return uniqueId; + } + + /** + * Get the next batch of file splits. If there's no more split, return empty list. + */ + public synchronized List getNextBatch(int maxBatchSize) throws UserException { + if (isLastBatch) { + return Collections.emptyList(); + } + List scanRanges = new ArrayList<>(maxBatchSize); + for (int i = 0; i < maxBatchSize; i++) { + if (splitIterator == null || !splitIterator.hasNext()) { + Collection splits = splitAssignment.getNextBatch(backend); + if (splits.isEmpty()) { + isLastBatch = true; + return scanRanges; + } + splitIterator = splits.iterator(); + } + scanRanges.add(splitToScanRange.getScanRange( + backend, locationProperties, splitIterator.next(), pathPartitionKeys)); + } + return scanRanges; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java new file mode 100644 index 000000000000000..83a7436df9ab824 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitSourceManager.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.util.MasterDaemon; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.lang.ref.ReferenceQueue; +import java.lang.ref.WeakReference; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * `SplitSource` is obtained by RPC call of `FrontendServiceImpl#fetchSplitBatch`. + * Each `SplitSource` is reference by its unique ID. `SplitSourceManager` provides the register, get, and remove + * function to manage the split sources. In order to clean the split source when the query finished, + * `SplitSource` is stored as a weak reference, and use `ReferenceQueue` to remove split source when GC. + */ +public class SplitSourceManager extends MasterDaemon { + private static final Logger LOG = LogManager.getLogger(SplitSourceManager.class); + + public static class SplitSourceReference extends WeakReference { + private final long uniqueId; + + public SplitSourceReference(SplitSource splitSource, ReferenceQueue queue) { + super(splitSource, queue); + uniqueId = splitSource.getUniqueId(); + } + + public long getUniqueId() { + return uniqueId; + } + } + + private final ReferenceQueue splitsRefQueue = new ReferenceQueue<>(); + private final Map> splits = new ConcurrentHashMap<>(); + + public void registerSplitSource(SplitSource splitSource) { + splits.put(splitSource.getUniqueId(), new SplitSourceReference(splitSource, splitsRefQueue)); + } + + public void removeSplitSource(long uniqueId) { + splits.remove(uniqueId); + } + + public SplitSource getSplitSource(long uniqueId) { + return splits.get(uniqueId).get(); + } + + @Override + protected void runAfterCatalogReady() { + while (true) { + try { + SplitSourceReference reference = (SplitSourceReference) splitsRefQueue.remove(); + removeSplitSource(reference.getUniqueId()); + } catch (Exception e) { + LOG.warn("Failed to clean split source", e); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitToScanRange.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitToScanRange.java new file mode 100644 index 000000000000000..0e890252857583c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/SplitToScanRange.java @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.common.UserException; +import org.apache.doris.spi.Split; +import org.apache.doris.system.Backend; +import org.apache.doris.thrift.TScanRangeLocations; + +import java.util.List; +import java.util.Map; + +public interface SplitToScanRange { + TScanRangeLocations getScanRange( + Backend backend, + Map locationProperties, + Split split, + List pathPartitionKeys) throws UserException; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index 0dbaa8b462b8ff6..12024c2561642df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -63,8 +63,10 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -95,6 +97,11 @@ public class HiveScanNode extends FileQueryScanNode { @Setter private SelectedPartitions selectedPartitions = null; + private boolean partitionInit = false; + private List prunedPartitions; + private Iterator prunedPartitionsIter; + private int numSplitsPerPartition = NUM_SPLITS_PER_PARTITION; + /** * * External file scan node for Query Hive table * needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column priv @@ -195,14 +202,21 @@ protected List getPartitions() throws AnalysisException { } @Override - protected List getSplits() throws UserException { + public List getSplits() throws UserException { long start = System.currentTimeMillis(); try { + if (!partitionInit) { + prunedPartitions = getPartitions(); + partitionInit = true; + } HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog()); String bindBrokerName = hmsTable.getCatalog().bindBrokerName(); List allFiles = Lists.newArrayList(); - getFileSplitByPartitions(cache, getPartitions(), allFiles, bindBrokerName); + getFileSplitByPartitions(cache, prunedPartitions, allFiles, bindBrokerName); + if (ConnectContext.get().getExecutor() != null) { + ConnectContext.get().getExecutor().getSummaryProfile().setGetPartitionFilesFinishTime(); + } if (LOG.isDebugEnabled()) { LOG.debug("get #{} files for table: {}.{}, cost: {} ms", allFiles.size(), hmsTable.getDbName(), hmsTable.getName(), @@ -217,6 +231,59 @@ protected List getSplits() throws UserException { } } + @Override + public List getNextBatch(int maxBatchSize) throws UserException { + try { + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog()); + String bindBrokerName = hmsTable.getCatalog().bindBrokerName(); + List allFiles = Lists.newArrayList(); + int numPartitions = 0; + while (allFiles.size() < maxBatchSize && prunedPartitionsIter.hasNext()) { + List partitions = new ArrayList<>(NUM_PARTITIONS_PER_LOOP); + for (int i = 0; i < NUM_PARTITIONS_PER_LOOP && prunedPartitionsIter.hasNext(); ++i) { + partitions.add(prunedPartitionsIter.next()); + numPartitions++; + } + getFileSplitByPartitions(cache, partitions, allFiles, bindBrokerName); + } + if (allFiles.size() / numPartitions > numSplitsPerPartition) { + numSplitsPerPartition = allFiles.size() / numPartitions; + } + return allFiles; + } catch (Throwable t) { + LOG.warn("get file split failed for table: {}", hmsTable.getName(), t); + throw new UserException( + "get file split failed for table: " + hmsTable.getName() + ", err: " + Util.getRootCauseMessage(t), + t); + } + } + + @Override + public boolean hasNext() { + return prunedPartitionsIter.hasNext(); + } + + @Override + public boolean isBatchMode() { + if (!partitionInit) { + try { + prunedPartitions = getPartitions(); + } catch (Exception e) { + return false; + } + prunedPartitionsIter = prunedPartitions.iterator(); + partitionInit = true; + } + int numPartitions = ConnectContext.get().getSessionVariable().getNumPartitionsInBatchMode(); + return numPartitions >= 0 && prunedPartitions.size() >= numPartitions; + } + + @Override + public int numApproximateSplits() { + return numSplitsPerPartition * prunedPartitions.size(); + } + private void getFileSplitByPartitions(HiveMetaStoreCache cache, List partitions, List allFiles, String bindBrokerName) throws IOException { List fileCaches; @@ -225,9 +292,6 @@ private void getFileSplitByPartitions(HiveMetaStoreCache cache, List hiveFileStatuses = selectFiles(fileCaches); splitAllFiles(allFiles, hiveFileStatuses); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 94748e7e4273fac..8dd853a48f373b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -35,6 +35,7 @@ import org.apache.doris.datasource.hudi.HudiUtils; import org.apache.doris.planner.ListPartitionPrunerV2; import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.thrift.TExplainLevel; @@ -68,6 +69,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Map; @@ -97,6 +99,14 @@ public class HudiScanNode extends HiveScanNode { private boolean incrementalRead = false; private IncrementalRelation incrementalRelation; + private boolean partitionInit = false; + private HoodieTimeline timeline; + private Option snapshotTimestamp; + private String queryInstant; + private List prunedPartitions; + private Iterator prunedPartitionsIter; + private int numSplitsPerPartition = NUM_SPLITS_PER_PARTITION; + /** * External file scan node for Query Hudi table * needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column priv @@ -193,6 +203,22 @@ protected void doInitialize() throws UserException { } else { incrementalRelation = null; } + + timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); + if (desc.getRef().getTableSnapshot() != null) { + queryInstant = desc.getRef().getTableSnapshot().getTime(); + snapshotTimestamp = Option.of(queryInstant); + } else { + Option snapshotInstant = timeline.lastInstant(); + if (!snapshotInstant.isPresent()) { + prunedPartitions = Collections.emptyList(); + prunedPartitionsIter = prunedPartitions.iterator(); + partitionInit = true; + return; + } + queryInstant = snapshotInstant.get().getTimestamp(); + snapshotTimestamp = Option.empty(); + } } @Override @@ -300,32 +326,8 @@ private List getIncrementalSplits() { incrementalRelation.getEndTs())).collect(Collectors.toList()); } - @Override - public List getSplits() throws UserException { - if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { - return getIncrementalSplits(); - } - - HoodieTimeline timeline = hudiClient.getCommitsAndCompactionTimeline().filterCompletedInstants(); - String queryInstant; - Option snapshotTimestamp; - if (desc.getRef().getTableSnapshot() != null) { - queryInstant = desc.getRef().getTableSnapshot().getTime(); - snapshotTimestamp = Option.of(queryInstant); - } else { - Option snapshotInstant = timeline.lastInstant(); - if (!snapshotInstant.isPresent()) { - return Collections.emptyList(); - } - queryInstant = snapshotInstant.get().getTimestamp(); - snapshotTimestamp = Option.empty(); - } - // Non partition table will get one dummy partition - List partitions = HiveMetaStoreClientHelper.ugiDoAs( - HiveMetaStoreClientHelper.getConfiguration(hmsTable), - () -> getPrunedPartitions(hudiClient, snapshotTimestamp)); + private void getPartitionSplits(List partitions, List splits) { Executor executor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); - List splits = Collections.synchronizedList(new ArrayList<>()); CountDownLatch countDownLatch = new CountDownLatch(partitions.size()); partitions.forEach(partition -> executor.execute(() -> { String globPath; @@ -370,9 +372,69 @@ public List getSplits() throws UserException { } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } + } + + @Override + public List getSplits() throws UserException { + if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { + return getIncrementalSplits(); + } + if (!partitionInit) { + prunedPartitions = HiveMetaStoreClientHelper.ugiDoAs( + HiveMetaStoreClientHelper.getConfiguration(hmsTable), + () -> getPrunedPartitions(hudiClient, snapshotTimestamp)); + partitionInit = true; + } + List splits = Collections.synchronizedList(new ArrayList<>()); + getPartitionSplits(prunedPartitions, splits); + return splits; + } + + @Override + public List getNextBatch(int maxBatchSize) throws UserException { + List splits = Collections.synchronizedList(new ArrayList<>()); + int numPartitions = 0; + while (splits.size() < maxBatchSize && prunedPartitionsIter.hasNext()) { + List partitions = new ArrayList<>(NUM_PARTITIONS_PER_LOOP); + for (int i = 0; i < NUM_PARTITIONS_PER_LOOP && prunedPartitionsIter.hasNext(); ++i) { + partitions.add(prunedPartitionsIter.next()); + numPartitions++; + } + getPartitionSplits(partitions, splits); + } + if (splits.size() / numPartitions > numSplitsPerPartition) { + numSplitsPerPartition = splits.size() / numPartitions; + } return splits; } + @Override + public boolean hasNext() { + return prunedPartitionsIter.hasNext(); + } + + @Override + public boolean isBatchMode() { + if (incrementalRead && !incrementalRelation.fallbackFullTableScan()) { + return false; + } + if (!partitionInit) { + // Non partition table will get one dummy partition + prunedPartitions = HiveMetaStoreClientHelper.ugiDoAs( + HiveMetaStoreClientHelper.getConfiguration(hmsTable), + () -> getPrunedPartitions(hudiClient, snapshotTimestamp)); + prunedPartitionsIter = prunedPartitions.iterator(); + partitionInit = true; + } + int numPartitions = ConnectContext.get().getSessionVariable().getNumPartitionsInBatchMode(); + return numPartitions >= 0 && prunedPartitions.size() >= numPartitions; + } + + @Override + public int numApproximateSplits() { + return numSplitsPerPartition * prunedPartitions.size(); + } + private HudiSplit generateHudiSplit(FileSlice fileSlice, List partitionValues, String queryInstant) { Optional baseFile = fileSlice.getBaseFile().toJavaOptional(); String filePath = baseFile.map(BaseFile::getPath).orElse(""); @@ -404,7 +466,11 @@ private HudiSplit generateHudiSplit(FileSlice fileSlice, List partitionV @Override public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { - return super.getNodeExplainString(prefix, detailLevel) - + String.format("%shudiNativeReadSplits=%d/%d\n", prefix, noLogsSplitNum.get(), inputSplitsNum); + if (isBatchMode()) { + return super.getNodeExplainString(prefix, detailLevel); + } else { + return super.getNodeExplainString(prefix, detailLevel) + + String.format("%shudiNativeReadSplits=%d/%d\n", prefix, noLogsSplitNum.get(), inputSplitsNum); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java index 279031d7f9b773a..c17a1d3e3f0d489 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java @@ -116,7 +116,7 @@ protected Map getLocationProperties() throws UserException { } @Override - protected List getSplits() throws UserException { + public List getSplits() throws UserException { List result = new ArrayList<>(); com.aliyun.odps.Table odpsTable = table.getOdpsTable(); if (desc.getSlots().isEmpty() || odpsTable.getFileNum() <= 0) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index ea20916e482ec3d..86a19cf6709e168 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -39,13 +39,13 @@ import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.common.AnalysisException; -import org.apache.doris.common.NotImplementedException; import org.apache.doris.common.UserException; import org.apache.doris.datasource.FederationBackendPolicy; import org.apache.doris.datasource.FileScanNode; +import org.apache.doris.datasource.SplitGenerator; +import org.apache.doris.datasource.SplitSource; import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.spi.Split; import org.apache.doris.statistics.StatisticalType; import org.apache.doris.statistics.query.StatsDelta; import org.apache.doris.system.Backend; @@ -75,8 +75,10 @@ /** * Representation of the common elements of all scan nodes. */ -public abstract class ScanNode extends PlanNode { +public abstract class ScanNode extends PlanNode implements SplitGenerator { private static final Logger LOG = LogManager.getLogger(ScanNode.class); + protected static final int NUM_SPLITS_PER_PARTITION = 10; + protected static final int NUM_PARTITIONS_PER_LOOP = 100; protected final TupleDescriptor desc; // for distribution prunner protected Map columnFilters = Maps.newHashMap(); @@ -85,6 +87,7 @@ public abstract class ScanNode extends PlanNode { protected String sortColumn = null; protected Analyzer analyzer; protected List scanRangeLocations = Lists.newArrayList(); + protected List splitSources = Lists.newArrayList(); protected PartitionInfo partitionsInfo = null; // create a mapping between output slot's id and project expr @@ -123,10 +126,6 @@ public void setSortColumn(String column) { sortColumn = column; } - protected List getSplits() throws UserException { - throw new NotImplementedException("Scan node sub class need to implement getSplits interface."); - } - /** * cast expr to SlotDescriptor type */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 540c3598d0c2a29..de4a9a80e4de6fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -1472,7 +1472,8 @@ public void cancel(Types.PPlanFragmentCancelReason cancelReason) { // Print an error stack here to know why send cancel again. LOG.warn("Query {} already in abnormal status {}, but received cancel again," + "so that send cancel to BE again", - DebugUtil.printId(queryId), queryStatus.toString(), new Exception()); + DebugUtil.printId(queryId), queryStatus.toString(), + new Exception("cancel failed")); } else { queryStatus.updateStatus(TStatusCode.CANCELLED, "cancelled"); } @@ -2462,12 +2463,16 @@ private void updateScanRangeNumByScanRange(TScanRangeParams param) { if (externalScanRange != null) { TFileScanRange fileScanRange = externalScanRange.getFileScanRange(); if (fileScanRange != null) { - scanRangeNum += fileScanRange.getRanges().size(); + if (fileScanRange.isSetRanges()) { + scanRangeNum += fileScanRange.getRanges().size(); + } else if (fileScanRange.isSetSplitSource()) { + scanRangeNum += fileScanRange.getSplitSource().getNumSplits(); + } } } TPaloScanRange paloScanRange = scanRange.getPaloScanRange(); if (paloScanRange != null) { - scanRangeNum = scanRangeNum + 1; + scanRangeNum += 1; } // TODO: more ranges? } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index cfa399324918c3a..2af7725e029e997 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -413,6 +413,8 @@ public class SessionVariable implements Serializable, Writable { // Split size for ExternalFileScanNode. Default value 0 means use the block size of HDFS/S3. public static final String FILE_SPLIT_SIZE = "file_split_size"; + public static final String NUM_PARTITIONS_IN_BATCH_MODE = "num_partitions_in_batch_mode"; + /** * use insert stmt as the unified backend for all loads */ @@ -1429,6 +1431,13 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { @VariableMgr.VarAttr(name = FILE_SPLIT_SIZE, needForward = true) public long fileSplitSize = 0; + @VariableMgr.VarAttr( + name = NUM_PARTITIONS_IN_BATCH_MODE, + description = {"如果分区数量超过阈值，BE将通过batch方式获取scan ranges", + "If the number of partitions exceeds the threshold, scan ranges will be got through batch mode."}, + needForward = true) + public int numPartitionsInBatchMode = 1024; + @VariableMgr.VarAttr( name = ENABLE_PARQUET_LAZY_MAT, description = {"控制 parquet reader 是否启用延迟物化技术。默认为 true。", @@ -2633,6 +2642,14 @@ public void setFileSplitSize(long fileSplitSize) { this.fileSplitSize = fileSplitSize; } + public int getNumPartitionsInBatchMode() { + return numPartitionsInBatchMode; + } + + public void setNumSplitsInBatchMode(int numPartitionsInBatchMode) { + this.numPartitionsInBatchMode = numPartitionsInBatchMode; + } + public boolean isEnableParquetLazyMat() { return enableParquetLazyMat; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java index 6b262baa2e1923c..b0abcc671417d36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java @@ -68,6 +68,7 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.SplitSource; import org.apache.doris.insertoverwrite.InsertOverwriteManager; import org.apache.doris.insertoverwrite.InsertOverwriteUtil; import org.apache.doris.load.routineload.ErrorReason; @@ -140,6 +141,8 @@ import org.apache.doris.thrift.TFetchResourceResult; import org.apache.doris.thrift.TFetchSchemaTableDataRequest; import org.apache.doris.thrift.TFetchSchemaTableDataResult; +import org.apache.doris.thrift.TFetchSplitBatchRequest; +import org.apache.doris.thrift.TFetchSplitBatchResult; import org.apache.doris.thrift.TFinishTaskRequest; import org.apache.doris.thrift.TFrontendPingFrontendRequest; import org.apache.doris.thrift.TFrontendPingFrontendResult; @@ -209,6 +212,7 @@ import org.apache.doris.thrift.TRestoreSnapshotResult; import org.apache.doris.thrift.TRollbackTxnRequest; import org.apache.doris.thrift.TRollbackTxnResult; +import org.apache.doris.thrift.TScanRangeLocations; import org.apache.doris.thrift.TSchemaTableName; import org.apache.doris.thrift.TShowProcessListRequest; import org.apache.doris.thrift.TShowProcessListResult; @@ -956,6 +960,23 @@ public TReportExecStatusResult reportExecStatus(TReportExecStatusParams params) return QeProcessorImpl.INSTANCE.reportExecStatus(params, getClientAddr()); } + @Override + public TFetchSplitBatchResult fetchSplitBatch(TFetchSplitBatchRequest request) throws TException { + TFetchSplitBatchResult result = new TFetchSplitBatchResult(); + SplitSource splitSource = + Env.getCurrentEnv().getSplitSourceManager().getSplitSource(request.getSplitSourceId()); + if (splitSource == null) { + throw new TException("Split source " + request.getSplitSourceId() + " is released"); + } + try { + List locations = splitSource.getNextBatch(request.getMaxNumSplits()); + result.setSplits(locations); + return result; + } catch (Exception e) { + throw new TException("Failed to get split source " + request.getSplitSourceId(), e); + } + } + @Override public TMasterResult finishTask(TFinishTaskRequest request) throws TException { return masterImpl.finishTask(request); diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index d2996f2a5dcc074..69045525a5ac97d 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -1420,6 +1420,15 @@ struct TShowUserResult { 1: optional list> userinfo_list } +struct TFetchSplitBatchRequest { + 1: optional i64 split_source_id + 2: optional i32 max_num_splits +} + +struct TFetchSplitBatchResult { + 1: optional list splits +} + service FrontendService { TGetDbsResult getDbNames(1: TGetDbsParams params) TGetTablesResult getTableNames(1: TGetTablesParams params) @@ -1509,4 +1518,5 @@ service FrontendService { TShowProcessListResult showProcessList(1: TShowProcessListRequest request) TShowUserResult showUser(1: TShowUserRequest request) + TFetchSplitBatchResult fetchSplitBatch(1: TFetchSplitBatchRequest request) } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 6494c4cf1d31378..8f9b9e0da54704a 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -432,6 +432,11 @@ struct TFileRangeDesc { 12: optional string fs_name } +struct TSplitSource { + 1: optional i64 split_source_id + 2: optional i32 num_splits +} + // TFileScanRange represents a set of descriptions of a file and the rules for reading and converting it. // TFileScanRangeParams: describe how to read and convert file // list: file location and range @@ -442,12 +447,12 @@ struct TFileScanRange { // file_scan_params in TExecPlanFragmentParams will always be set in query request, // and TFileScanRangeParams here is used for some other request such as fetch table schema for tvf. 2: optional TFileScanRangeParams params + 3: optional TSplitSource split_source } // Scan range for external datasource, such as file on hdfs, es datanode, etc. struct TExternalScanRange { 1: optional TFileScanRange file_scan_range - // TODO: add more scan range type? } enum TDataGenFunctionName { From 903ff3202103093eadcf0f0672843eb6719fcb94 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Tue, 21 May 2024 22:31:47 +0800 Subject: [PATCH 083/111] [opt](fe) exit FE when transfer to (non)master failed (#34809) (#35158) bp #34809 --- be/src/vec/exec/scan/vfile_scanner.cpp | 3 +- .../java/org/apache/doris/catalog/Env.java | 275 ++++++++++-------- .../org/apache/doris/service/FeServer.java | 2 +- 3 files changed, 148 insertions(+), 132 deletions(-) diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp b/be/src/vec/exec/scan/vfile_scanner.cpp index 398621b08ea5409..fca2c301c647110 100644 --- a/be/src/vec/exec/scan/vfile_scanner.cpp +++ b/be/src/vec/exec/scan/vfile_scanner.cpp @@ -958,8 +958,7 @@ Status VFileScanner::_get_next_reader() { COUNTER_UPDATE(_empty_file_counter, 1); continue; } else if (!init_status.ok()) { - return Status::InternalError("failed to init reader for file {}, err: {}", range.path, - init_status.to_string()); + return Status::InternalError("failed to init reader, err: {}", init_status.to_string()); } _name_to_col_type.clear(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index d0223e626bc98b8..5dffb93121c1041 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -1444,137 +1444,147 @@ private void getHelperNodeFromDeployManager() throws Exception { @SuppressWarnings({"checkstyle:WhitespaceAfter", "checkstyle:LineLength"}) private void transferToMaster() { - // stop replayer - if (replayer != null) { - replayer.exit(); - try { - replayer.join(); - } catch (InterruptedException e) { - LOG.warn("got exception when stopping the replayer thread", e); + try { + // stop replayer + if (replayer != null) { + replayer.exit(); + try { + replayer.join(); + } catch (InterruptedException e) { + LOG.warn("got exception when stopping the replayer thread", e); + } + replayer = null; } - replayer = null; - } - // set this after replay thread stopped. to avoid replay thread modify them. - isReady.set(false); - canRead.set(false); + // set this after replay thread stopped. to avoid replay thread modify them. + isReady.set(false); + canRead.set(false); - toMasterProgress = "open editlog"; - editLog.open(); + toMasterProgress = "open editlog"; + editLog.open(); - if (Config.edit_log_type.equalsIgnoreCase("bdb")) { - if (!haProtocol.fencing()) { - LOG.error("fencing failed. will exit."); - System.exit(-1); + if (Config.edit_log_type.equalsIgnoreCase("bdb")) { + if (!haProtocol.fencing()) { + LOG.error("fencing failed. will exit."); + System.exit(-1); + } } - } - toMasterProgress = "replay journal"; - long replayStartTime = System.currentTimeMillis(); - // replay journals. -1 means replay all the journals larger than current journal id. - replayJournal(-1); - long replayEndTime = System.currentTimeMillis(); - LOG.info("finish replay in " + (replayEndTime - replayStartTime) + " msec"); + toMasterProgress = "replay journal"; + long replayStartTime = System.currentTimeMillis(); + // replay journals. -1 means replay all the journals larger than current journal id. + replayJournal(-1); + long replayEndTime = System.currentTimeMillis(); + LOG.info("finish replay in " + (replayEndTime - replayStartTime) + " msec"); - checkCurrentNodeExist(); + checkCurrentNodeExist(); - checkBeExecVersion(); + checkBeExecVersion(); - toMasterProgress = "roll editlog"; - editLog.rollEditLog(); + toMasterProgress = "roll editlog"; + editLog.rollEditLog(); - // Log meta_version - long journalVersion = MetaContext.get().getMetaVersion(); - if (journalVersion < FeConstants.meta_version) { - toMasterProgress = "log meta version"; - editLog.logMetaVersion(FeConstants.meta_version); - MetaContext.get().setMetaVersion(FeConstants.meta_version); - } + // Log meta_version + long journalVersion = MetaContext.get().getMetaVersion(); + if (journalVersion < FeConstants.meta_version) { + toMasterProgress = "log meta version"; + editLog.logMetaVersion(FeConstants.meta_version); + MetaContext.get().setMetaVersion(FeConstants.meta_version); + } - // Log the first frontend - if (isFirstTimeStartUp) { - // if isFirstTimeStartUp is true, frontends must contains this Node. - Frontend self = frontends.get(nodeName); - Preconditions.checkNotNull(self); - // OP_ADD_FIRST_FRONTEND is emitted, so it can write to BDBJE even if canWrite is false - editLog.logAddFirstFrontend(self); + // Log the first frontend + if (isFirstTimeStartUp) { + // if isFirstTimeStartUp is true, frontends must contains this Node. + Frontend self = frontends.get(nodeName); + Preconditions.checkNotNull(self); + // OP_ADD_FIRST_FRONTEND is emitted, so it can write to BDBJE even if canWrite is false + editLog.logAddFirstFrontend(self); - initLowerCaseTableNames(); - // Set initial root password if master FE first time launch. - auth.setInitialRootPassword(Config.initial_root_password); - } else { - if (journalVersion <= FeMetaVersion.VERSION_114) { - // if journal version is less than 114, which means it is upgraded from version before 2.0. - // When upgrading from 1.2 to 2.0, we need to make sure that the parallelism of query remain unchanged - // when switch to pipeline engine, otherwise it may impact the load of entire cluster - // because the default parallelism of pipeline engine is higher than previous version. - // so set parallel_pipeline_task_num to parallel_fragment_exec_instance_num - int newVal = VariableMgr.newSessionVariable().parallelExecInstanceNum; - VariableMgr.refreshDefaultSessionVariables("1.x to 2.x", SessionVariable.PARALLEL_PIPELINE_TASK_NUM, - String.valueOf(newVal)); - - // similar reason as above, need to upgrade broadcast scale factor during 1.2 to 2.x - // if the default value has been upgraded - double newBcFactorVal = VariableMgr.newSessionVariable().getBroadcastRightTableScaleFactor(); - VariableMgr.refreshDefaultSessionVariables("1.x to 2.x", - SessionVariable.BROADCAST_RIGHT_TABLE_SCALE_FACTOR, - String.valueOf(newBcFactorVal)); - - // similar reason as above, need to upgrade enable_nereids_planner to true - VariableMgr.refreshDefaultSessionVariables("1.x to 2.x", SessionVariable.ENABLE_NEREIDS_PLANNER, - "true"); - } - if (journalVersion <= FeMetaVersion.VERSION_123) { - VariableMgr.refreshDefaultSessionVariables("2.0 to 2.1", SessionVariable.ENABLE_NEREIDS_DML, "true"); - VariableMgr.refreshDefaultSessionVariables("2.0 to 2.1", - SessionVariable.FRAGMENT_TRANSMISSION_COMPRESSION_CODEC, "none"); - if (VariableMgr.newSessionVariable().nereidsTimeoutSecond == 5) { + initLowerCaseTableNames(); + // Set initial root password if master FE first time launch. + auth.setInitialRootPassword(Config.initial_root_password); + } else { + if (journalVersion <= FeMetaVersion.VERSION_114) { + // if journal version is less than 114, which means it is upgraded from version before 2.0. + // When upgrading from 1.2 to 2.0, + // we need to make sure that the parallelism of query remain unchanged + // when switch to pipeline engine, otherwise it may impact the load of entire cluster + // because the default parallelism of pipeline engine is higher than previous version. + // so set parallel_pipeline_task_num to parallel_fragment_exec_instance_num + int newVal = VariableMgr.newSessionVariable().parallelExecInstanceNum; + VariableMgr.refreshDefaultSessionVariables("1.x to 2.x", + SessionVariable.PARALLEL_PIPELINE_TASK_NUM, + String.valueOf(newVal)); + + // similar reason as above, need to upgrade broadcast scale factor during 1.2 to 2.x + // if the default value has been upgraded + double newBcFactorVal = VariableMgr.newSessionVariable().getBroadcastRightTableScaleFactor(); + VariableMgr.refreshDefaultSessionVariables("1.x to 2.x", + SessionVariable.BROADCAST_RIGHT_TABLE_SCALE_FACTOR, + String.valueOf(newBcFactorVal)); + + // similar reason as above, need to upgrade enable_nereids_planner to true + VariableMgr.refreshDefaultSessionVariables("1.x to 2.x", SessionVariable.ENABLE_NEREIDS_PLANNER, + "true"); + } + if (journalVersion <= FeMetaVersion.VERSION_123) { + VariableMgr.refreshDefaultSessionVariables("2.0 to 2.1", SessionVariable.ENABLE_NEREIDS_DML, + "true"); VariableMgr.refreshDefaultSessionVariables("2.0 to 2.1", - SessionVariable.NEREIDS_TIMEOUT_SECOND, "30"); + SessionVariable.FRAGMENT_TRANSMISSION_COMPRESSION_CODEC, "none"); + if (VariableMgr.newSessionVariable().nereidsTimeoutSecond == 5) { + VariableMgr.refreshDefaultSessionVariables("2.0 to 2.1", + SessionVariable.NEREIDS_TIMEOUT_SECOND, "30"); + } } } - } - getPolicyMgr().createDefaultStoragePolicy(); + getPolicyMgr().createDefaultStoragePolicy(); - // MUST set master ip before starting checkpoint thread. - // because checkpoint thread need this info to select non-master FE to push image + // MUST set master ip before starting checkpoint thread. + // because checkpoint thread need this info to select non-master FE to push image - toMasterProgress = "log master info"; - this.masterInfo = new MasterInfo(Env.getCurrentEnv().getSelfNode().getHost(), - Config.http_port, - Config.rpc_port); - editLog.logMasterInfo(masterInfo); - LOG.info("logMasterInfo:{}", masterInfo); + toMasterProgress = "log master info"; + this.masterInfo = new MasterInfo(Env.getCurrentEnv().getSelfNode().getHost(), + Config.http_port, + Config.rpc_port); + editLog.logMasterInfo(masterInfo); + LOG.info("logMasterInfo:{}", masterInfo); - // for master, the 'isReady' is set behind. - // but we are sure that all metadata is replayed if we get here. - // so no need to check 'isReady' flag in this method - postProcessAfterMetadataReplayed(false); + // for master, the 'isReady' is set behind. + // but we are sure that all metadata is replayed if we get here. + // so no need to check 'isReady' flag in this method + postProcessAfterMetadataReplayed(false); - insertOverwriteManager.allTaskFail(); + insertOverwriteManager.allTaskFail(); - toMasterProgress = "start daemon threads"; + toMasterProgress = "start daemon threads"; - // start all daemon threads that only running on MASTER FE - startMasterOnlyDaemonThreads(); - // start other daemon threads that should running on all FE - startNonMasterDaemonThreads(); + // start all daemon threads that only running on MASTER FE + startMasterOnlyDaemonThreads(); + // start other daemon threads that should running on all FE + startNonMasterDaemonThreads(); - MetricRepo.init(); + MetricRepo.init(); - toMasterProgress = "finished"; - canRead.set(true); - isReady.set(true); - checkLowerCaseTableNames(); + toMasterProgress = "finished"; + canRead.set(true); + isReady.set(true); + checkLowerCaseTableNames(); - String msg = "master finished to replay journal, can write now."; - Util.stdoutWithTime(msg); - LOG.info(msg); - // for master, there are some new thread pools need to register metric - ThreadPoolManager.registerAllThreadPoolMetric(); - if (analysisManager != null) { - analysisManager.getStatisticsCache().preHeat(); + String msg = "master finished to replay journal, can write now."; + Util.stdoutWithTime(msg); + LOG.info(msg); + // for master, there are some new thread pools need to register metric + ThreadPoolManager.registerAllThreadPoolMetric(); + if (analysisManager != null) { + analysisManager.getStatisticsCache().preHeat(); + } + } catch (Throwable e) { + // When failed to transfer to master, we need to exit the process. + // Otherwise, the process will be in an unknown state. + LOG.error("failed to transfer to master. progress: {}", toMasterProgress, e); + System.exit(-1); } } @@ -1722,36 +1732,43 @@ private void startNonMasterDaemonThreads() { private void transferToNonMaster(FrontendNodeType newType) { isReady.set(false); - if (feType == FrontendNodeType.OBSERVER || feType == FrontendNodeType.FOLLOWER) { - Preconditions.checkState(newType == FrontendNodeType.UNKNOWN); - LOG.warn("{} to UNKNOWN, still offer read service", feType.name()); - // not set canRead here, leave canRead as what is was. - // if meta out of date, canRead will be set to false in replayer thread. - metaReplayState.setTransferToUnknown(); - return; - } + try { + if (feType == FrontendNodeType.OBSERVER || feType == FrontendNodeType.FOLLOWER) { + Preconditions.checkState(newType == FrontendNodeType.UNKNOWN); + LOG.warn("{} to UNKNOWN, still offer read service", feType.name()); + // not set canRead here, leave canRead as what is was. + // if meta out of date, canRead will be set to false in replayer thread. + metaReplayState.setTransferToUnknown(); + return; + } - // transfer from INIT/UNKNOWN to OBSERVER/FOLLOWER + // transfer from INIT/UNKNOWN to OBSERVER/FOLLOWER - if (replayer == null) { - createReplayer(); - replayer.start(); - } + if (replayer == null) { + createReplayer(); + replayer.start(); + } - // 'isReady' will be set to true in 'setCanRead()' method - if (!postProcessAfterMetadataReplayed(true)) { - // the state has changed, exit early. - return; - } + // 'isReady' will be set to true in 'setCanRead()' method + if (!postProcessAfterMetadataReplayed(true)) { + // the state has changed, exit early. + return; + } - checkLowerCaseTableNames(); + checkLowerCaseTableNames(); - startNonMasterDaemonThreads(); + startNonMasterDaemonThreads(); - MetricRepo.init(); + MetricRepo.init(); - if (analysisManager != null) { - analysisManager.getStatisticsCache().preHeat(); + if (analysisManager != null) { + analysisManager.getStatisticsCache().preHeat(); + } + } catch (Throwable e) { + // When failed to transfer to non-master, we need to exit the process. + // Otherwise, the process will be in an unknown state. + LOG.error("failed to transfer to non-master.", e); + System.exit(-1); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FeServer.java b/fe/fe-core/src/main/java/org/apache/doris/service/FeServer.java index 6c61c9cdd488ab8..542c993a4383b99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/service/FeServer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/service/FeServer.java @@ -44,7 +44,7 @@ public FeServer(int port) { public void start() throws IOException { FrontendServiceImpl service = new FrontendServiceImpl(ExecuteEnv.getInstance()); - Logger feServiceLogger = LogManager.getLogger(FrontendServiceImpl.class); + Logger feServiceLogger = LogManager.getLogger(FeServer.class); FrontendService.Iface instance = (FrontendService.Iface) Proxy.newProxyInstance( FrontendServiceImpl.class.getClassLoader(), FrontendServiceImpl.class.getInterfaces(), From 009ab77c25eafbd833f76236e606e235b7300c6d Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Tue, 21 May 2024 22:46:37 +0800 Subject: [PATCH 084/111] [feature](iceberg)Support write to iceberg for 2.1 (#35103) #34257 #33629 bp: #34257 #33629 --- .../iceberg/IcebergExternalCatalog.java | 5 +- .../iceberg/IcebergExternalTable.java | 16 + .../iceberg/IcebergTransaction.java | 186 ++++++++++ .../datasource/iceberg/IcebergUtils.java | 57 +++ .../iceberg/source/IcebergApiSource.java | 11 +- .../iceberg/source/IcebergHMSSource.java | 12 +- .../iceberg/source/IcebergSource.java | 3 - .../UnboundBaseExternalTableSink.java | 117 +++++++ .../analyzer/UnboundHiveTableSink.java | 91 +---- .../analyzer/UnboundIcebergTableSink.java | 84 +++++ .../analyzer/UnboundTableSinkCreator.java | 9 + .../translator/PhysicalPlanTranslator.java | 25 +- .../TurnOffPageCacheForInsertIntoSelect.java | 8 + .../properties/RequestPropertyDeriver.java | 12 + .../apache/doris/nereids/rules/RuleSet.java | 2 + .../apache/doris/nereids/rules/RuleType.java | 2 + .../nereids/rules/analysis/BindSink.java | 68 +++- ...rgTableSinkToPhysicalIcebergTableSink.java | 48 +++ .../doris/nereids/trees/plans/PlanType.java | 2 + ...BaseExternalTableInsertCommandContext.java | 33 ++ .../BaseExternalTableInsertExecutor.java | 161 +++++++++ .../insert/HiveInsertCommandContext.java | 11 +- .../commands/insert/HiveInsertExecutor.java | 110 +----- .../insert/IcebergInsertExecutor.java | 70 ++++ .../insert/InsertIntoTableCommand.java | 8 +- .../plans/commands/insert/InsertUtils.java | 6 +- .../logical/LogicalIcebergTableSink.java | 150 ++++++++ .../PhysicalBaseExternalTableSink.java | 79 +++++ .../plans/physical/PhysicalHiveTableSink.java | 50 +-- .../physical/PhysicalIcebergTableSink.java | 133 +++++++ .../trees/plans/visitor/SinkVisitor.java | 15 + .../planner/BaseExternalTableDataSink.java | 97 ++++++ .../org/apache/doris/planner/DataSink.java | 3 + .../apache/doris/planner/HiveTableSink.java | 72 +--- .../doris/planner/IcebergTableSink.java | 146 ++++++++ .../java/org/apache/doris/qe/Coordinator.java | 17 + .../IcebergTransactionManager.java | 69 ++++ .../TransactionManagerFactory.java | 5 + .../hive/HiveDDLAndDMLPlanTest.java | 27 +- .../doris/datasource/hive/HmsCommitTest.java | 1 - .../iceberg/IcebergTransactionTest.java | 328 ++++++++++++++++++ gensrc/thrift/DataSinks.thrift | 39 +++ gensrc/thrift/FrontendService.thrift | 6 +- 43 files changed, 2075 insertions(+), 319 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundBaseExternalTableSink.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSink.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergTableSinkToPhysicalIcebergTableSink.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertCommandContext.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergTableSink.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBaseExternalTableSink.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 49ca6721f498beb..6f79afd5de5d7fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -22,6 +22,7 @@ import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.PropertyConverter; +import org.apache.doris.transaction.TransactionManagerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.s3a.Constants; @@ -51,7 +52,9 @@ public IcebergExternalCatalog(long catalogId, String name, String comment) { @Override protected void initLocalObjectsImpl() { initCatalog(); - metadataOps = ExternalMetadataOperations.newIcebergMetadataOps(this, catalog); + IcebergMetadataOps ops = ExternalMetadataOperations.newIcebergMetadataOps(this, catalog); + transactionManager = TransactionManagerFactory.createIcebergTransactionManager(ops); + metadataOps = ops; } public Catalog getCatalog() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index de9c3814fd68b90..06864bfe6a6e9a9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -28,9 +28,14 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import org.apache.iceberg.PartitionField; +import org.apache.iceberg.Table; + import java.util.HashMap; import java.util.List; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; public class IcebergExternalTable extends ExternalTable { @@ -83,4 +88,15 @@ public long fetchRowCount() { makeSureInitialized(); return IcebergUtils.getIcebergRowCount(getCatalog(), getDbName(), getName()); } + + public Table getIcebergTable() { + return IcebergUtils.getIcebergTable(getCatalog(), getDbName(), getName()); + } + + @Override + public Set getPartitionNames() { + getIcebergTable(); + return IcebergUtils.getIcebergTable(getCatalog(), getDbName(), getName()) + .spec().fields().stream().map(PartitionField::name).collect(Collectors.toSet()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java new file mode 100644 index 000000000000000..2e1ad8ab1e1a2e5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -0,0 +1,186 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/trinodb/trino/blob/438/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java +// and modified by Doris + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.common.UserException; +import org.apache.doris.thrift.TFileContent; +import org.apache.doris.thrift.TIcebergCommitData; +import org.apache.doris.transaction.Transaction; + +import com.google.common.base.VerifyException; +import com.google.common.collect.Lists; +import org.apache.iceberg.AppendFiles; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.FileContent; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +public class IcebergTransaction implements Transaction { + + private static final Logger LOG = LogManager.getLogger(IcebergTransaction.class); + private final IcebergMetadataOps ops; + private org.apache.iceberg.Transaction transaction; + private final List commitDataList = Lists.newArrayList(); + + public IcebergTransaction(IcebergMetadataOps ops) { + this.ops = ops; + } + + public void updateIcebergCommitData(List commitDataList) { + synchronized (this) { + this.commitDataList.addAll(commitDataList); + } + } + + public void beginInsert(String dbName, String tbName) { + Table icebergTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbName)); + transaction = icebergTable.newTransaction(); + } + + public void finishInsert() { + Table icebergTable = transaction.table(); + AppendFiles appendFiles = transaction.newAppend(); + + for (CommitTaskData task : convertToCommitTaskData()) { + DataFiles.Builder builder = DataFiles.builder(icebergTable.spec()) + .withPath(task.getPath()) + .withFileSizeInBytes(task.getFileSizeInBytes()) + .withFormat(IcebergUtils.getFileFormat(icebergTable)) + .withMetrics(task.getMetrics()); + + if (icebergTable.spec().isPartitioned()) { + List partitionValues = task.getPartitionValues() + .orElseThrow(() -> new VerifyException("No partition data for partitioned table")); + builder.withPartitionValues(partitionValues); + } + appendFiles.appendFile(builder.build()); + } + + // in appendFiles.commit, it will generate metadata(manifest and snapshot) + // after appendFiles.commit, in current transaction, you can already see the new snapshot + appendFiles.commit(); + } + + public List convertToCommitTaskData() { + List commitTaskData = new ArrayList<>(); + for (TIcebergCommitData data : this.commitDataList) { + commitTaskData.add(new CommitTaskData( + data.getFilePath(), + data.getFileSize(), + new Metrics( + data.getRowCount(), + Collections.EMPTY_MAP, + Collections.EMPTY_MAP, + Collections.EMPTY_MAP, + Collections.EMPTY_MAP + ), + data.isSetPartitionValues() ? Optional.of(data.getPartitionValues()) : Optional.empty(), + convertToFileContent(data.getFileContent()), + data.isSetReferencedDataFiles() ? Optional.of(data.getReferencedDataFiles()) : Optional.empty() + )); + } + return commitTaskData; + } + + private FileContent convertToFileContent(TFileContent content) { + if (content.equals(TFileContent.DATA)) { + return FileContent.DATA; + } else if (content.equals(TFileContent.POSITION_DELETES)) { + return FileContent.POSITION_DELETES; + } else { + return FileContent.EQUALITY_DELETES; + } + } + + @Override + public void commit() throws UserException { + // Externally readable + // Manipulate the relevant data so that others can also see the latest table, such as: + // 1. hadoop: it will change the version number information in 'version-hint.text' + // 2. hive: it will change the table properties, the most important thing is to revise 'metadata_location' + // 3. and so on ... + transaction.commitTransaction(); + } + + @Override + public void rollback() { + + } + + public long getUpdateCnt() { + return commitDataList.stream().mapToLong(TIcebergCommitData::getRowCount).sum(); + } + + public static class CommitTaskData { + private final String path; + private final long fileSizeInBytes; + private final Metrics metrics; + private final Optional> partitionValues; + private final FileContent content; + private final Optional> referencedDataFiles; + + public CommitTaskData(String path, + long fileSizeInBytes, + Metrics metrics, + Optional> partitionValues, + FileContent content, + Optional> referencedDataFiles) { + this.path = path; + this.fileSizeInBytes = fileSizeInBytes; + this.metrics = metrics; + this.partitionValues = partitionValues; + this.content = content; + this.referencedDataFiles = referencedDataFiles; + } + + public String getPath() { + return path; + } + + public long getFileSizeInBytes() { + return fileSizeInBytes; + } + + public Metrics getMetrics() { + return metrics; + } + + public Optional> getPartitionValues() { + return partitionValues; + } + + public FileContent getContent() { + return content; + } + + public Optional> getReferencedDataFiles() { + return referencedDataFiles; + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 05519f845957066..9c57fc8e940335d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -46,6 +46,7 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; +import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.thrift.TExprOpcode; import com.google.common.collect.Lists; @@ -53,6 +54,7 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.expressions.And; import org.apache.iceberg.expressions.Expression; import org.apache.iceberg.expressions.Expressions; @@ -61,6 +63,7 @@ import org.apache.iceberg.expressions.Unbound; import org.apache.iceberg.types.Type.TypeID; import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.LocationUtil; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -90,6 +93,13 @@ public Integer initialValue() { public static final String TOTAL_POSITION_DELETES = "total-position-deletes"; public static final String TOTAL_EQUALITY_DELETES = "total-equality-deletes"; + // nickname in flink and spark + public static final String WRITE_FORMAT = "write-format"; + public static final String COMPRESSION_CODEC = "compression-codec"; + + // nickname in spark + public static final String SPARK_SQL_COMPRESSION_CODEC = "spark.sql.iceberg.compression-codec"; + public static Expression convertToIcebergExpr(Expr expr, Schema schema) { if (expr == null) { return null; @@ -573,4 +583,51 @@ public static long getIcebergRowCount(ExternalCatalog catalog, String dbName, St } return -1; } + + public static String getFileFormat(Table table) { + Map properties = table.properties(); + if (properties.containsKey(WRITE_FORMAT)) { + return properties.get(WRITE_FORMAT); + } + if (properties.containsKey(TableProperties.DEFAULT_FILE_FORMAT)) { + return properties.get(TableProperties.DEFAULT_FILE_FORMAT); + } + return TableProperties.DEFAULT_FILE_FORMAT_DEFAULT; + } + + public static String getFileCompress(Table table) { + Map properties = table.properties(); + if (properties.containsKey(COMPRESSION_CODEC)) { + return properties.get(COMPRESSION_CODEC); + } else if (properties.containsKey(SPARK_SQL_COMPRESSION_CODEC)) { + return properties.get(SPARK_SQL_COMPRESSION_CODEC); + } + String fileFormat = getFileFormat(table); + if (fileFormat.equalsIgnoreCase("parquet")) { + return properties.getOrDefault( + TableProperties.PARQUET_COMPRESSION, TableProperties.PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0); + } else if (fileFormat.equalsIgnoreCase("orc")) { + return properties.getOrDefault( + TableProperties.ORC_COMPRESSION, TableProperties.ORC_COMPRESSION_DEFAULT); + } + throw new NotSupportedException("Unsupported file format: " + fileFormat); + } + + public static String dataLocation(Table table) { + Map properties = table.properties(); + if (properties.containsKey(TableProperties.WRITE_LOCATION_PROVIDER_IMPL)) { + throw new NotSupportedException( + "Table " + table.name() + " specifies " + properties.get(TableProperties.WRITE_LOCATION_PROVIDER_IMPL) + + " as a location provider. " + + "Writing to Iceberg tables with custom location provider is not supported."); + } + String dataLocation = properties.get(TableProperties.WRITE_DATA_LOCATION); + if (dataLocation == null) { + dataLocation = properties.get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION); + if (dataLocation == null) { + dataLocation = String.format("%s/data", LocationUtil.stripTrailingSlash(table.location())); + } + } + return dataLocation; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java index 42e0d709e05757e..e590e918344d7d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java @@ -24,11 +24,11 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.planner.ColumnRange; import org.apache.doris.thrift.TFileAttributes; import org.apache.iceberg.Table; -import org.apache.iceberg.TableProperties; import java.util.Map; @@ -61,14 +61,7 @@ public TupleDescriptor getDesc() { @Override public String getFileFormat() { - Map properties = originTable.properties(); - if (properties.containsKey(TableProperties.DEFAULT_FILE_FORMAT)) { - return properties.get(TableProperties.DEFAULT_FILE_FORMAT); - } - if (properties.containsKey(FLINK_WRITE_FORMAT)) { - return properties.get(FLINK_WRITE_FORMAT); - } - return TableProperties.DEFAULT_FILE_FORMAT_DEFAULT; + return IcebergUtils.getFileFormat(originTable); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java index 632120e5c452ec3..06b785a15f890ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java @@ -26,12 +26,11 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.source.HiveScanNode; +import org.apache.doris.datasource.iceberg.IcebergUtils; import org.apache.doris.planner.ColumnRange; import org.apache.doris.thrift.TFileAttributes; import org.apache.doris.thrift.TFileTextScanRangeParams; -import org.apache.iceberg.TableProperties; - import java.util.Map; public class IcebergHMSSource implements IcebergSource { @@ -59,14 +58,7 @@ public TupleDescriptor getDesc() { @Override public String getFileFormat() throws DdlException, MetaNotFoundException { - Map properties = hmsTable.getRemoteTable().getParameters(); - if (properties.containsKey(TableProperties.DEFAULT_FILE_FORMAT)) { - return properties.get(TableProperties.DEFAULT_FILE_FORMAT); - } - if (properties.containsKey(FLINK_WRITE_FORMAT)) { - return properties.get(FLINK_WRITE_FORMAT); - } - return TableProperties.DEFAULT_FILE_FORMAT_DEFAULT; + return IcebergUtils.getFileFormat(icebergTable); } public org.apache.iceberg.Table getIcebergTable() throws MetaNotFoundException { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSource.java index 270a4d4df18f612..b4b1bf2a805d121 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSource.java @@ -27,9 +27,6 @@ public interface IcebergSource { - // compatible with flink, which is "write.format.default" in spark - String FLINK_WRITE_FORMAT = "write-format"; - TupleDescriptor getDesc(); org.apache.iceberg.Table getIcebergTable() throws MetaNotFoundException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundBaseExternalTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundBaseExternalTableSink.java new file mode 100644 index 000000000000000..cfdefc59872d4e7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundBaseExternalTableSink.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.exceptions.UnboundException; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.UnboundLogicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; +import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; +import org.apache.doris.nereids.util.Utils; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Represent an external table sink plan node that has not been bound. + */ +public abstract class UnboundBaseExternalTableSink extends UnboundLogicalSink + implements Unbound, Sink, BlockFuncDepsPropagation { + List hints; + List partitions; + + /** + * constructor + */ + public UnboundBaseExternalTableSink(List nameParts, + PlanType type, + List outputExprs, + Optional groupExpression, + Optional logicalProperties, + List colNames, + DMLCommandType dmlCommandType, + CHILD_TYPE child, + List hints, + List partitions) { + super(nameParts, type, outputExprs, groupExpression, + logicalProperties, colNames, dmlCommandType, child); + this.hints = Utils.copyRequiredList(hints); + this.partitions = Utils.copyRequiredList(partitions); + } + + public List getColNames() { + return colNames; + } + + public List getPartitions() { + return partitions; + } + + public List getHints() { + return hints; + } + + @Override + public UnboundBaseExternalTableSink withOutputExprs(List outputExprs) { + throw new UnboundException("could not call withOutputExprs on " + this.getClass().getSimpleName()); + } + + @Override + public List getExpressions() { + throw new UnsupportedOperationException(this.getClass().getSimpleName() + " don't support getExpression()"); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + UnboundBaseExternalTableSink that = (UnboundBaseExternalTableSink) o; + return Objects.equals(nameParts, that.nameParts) + && Objects.equals(colNames, that.colNames) + && Objects.equals(hints, that.hints) + && Objects.equals(partitions, that.partitions); + } + + @Override + public int hashCode() { + return Objects.hash(nameParts, colNames, hints, partitions); + } + + @Override + public LogicalProperties computeLogicalProperties() { + return UnboundLogicalProperties.INSTANCE; + } + + @Override + public List computeOutput() { + throw new UnboundException("output"); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundHiveTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundHiveTableSink.java index 0b56c2b681d0873..4ffbc0230a005e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundHiveTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundHiveTableSink.java @@ -17,36 +17,23 @@ package org.apache.doris.nereids.analyzer; -import org.apache.doris.nereids.exceptions.UnboundException; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.properties.UnboundLogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.NamedExpression; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.algebra.Sink; import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; -import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import java.util.List; -import java.util.Objects; import java.util.Optional; /** - * Represent an hive table sink plan node that has not been bound. + * Represent a hive table sink plan node that has not been bound. */ -public class UnboundHiveTableSink extends UnboundLogicalSink - implements Unbound, Sink, BlockFuncDepsPropagation { - private final List hints; - private final List partitions; +public class UnboundHiveTableSink extends UnboundBaseExternalTableSink { public UnboundHiveTableSink(List nameParts, List colNames, List hints, List partitions, CHILD_TYPE child) { @@ -57,28 +44,21 @@ public UnboundHiveTableSink(List nameParts, List colNames, List< /** * constructor */ - public UnboundHiveTableSink(List nameParts, List colNames, List hints, + public UnboundHiveTableSink(List nameParts, + List colNames, + List hints, List partitions, DMLCommandType dmlCommandType, Optional groupExpression, Optional logicalProperties, CHILD_TYPE child) { super(nameParts, PlanType.LOGICAL_UNBOUND_HIVE_TABLE_SINK, ImmutableList.of(), groupExpression, - logicalProperties, colNames, dmlCommandType, child); - this.hints = Utils.copyRequiredList(hints); - this.partitions = Utils.copyRequiredList(partitions); - } - - public List getColNames() { - return colNames; - } - - public List getPartitions() { - return partitions; + logicalProperties, colNames, dmlCommandType, child, hints, partitions); } - public List getHints() { - return hints; + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitUnboundHiveTableSink(this, context); } @Override @@ -86,64 +66,19 @@ public Plan withChildren(List children) { Preconditions.checkArgument(children.size() == 1, "UnboundHiveTableSink only accepts one child"); return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions, - dmlCommandType, groupExpression, Optional.empty(), children.get(0)); - } - - @Override - public UnboundHiveTableSink withOutputExprs(List outputExprs) { - throw new UnboundException("could not call withOutputExprs on UnboundHiveTableSink"); - } - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitUnboundHiveTableSink(this, context); - } - - @Override - public List getExpressions() { - throw new UnsupportedOperationException(this.getClass().getSimpleName() + " don't support getExpression()"); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - UnboundHiveTableSink that = (UnboundHiveTableSink) o; - return Objects.equals(nameParts, that.nameParts) - && Objects.equals(colNames, that.colNames) - && Objects.equals(hints, that.hints) - && Objects.equals(partitions, that.partitions); - } - - @Override - public int hashCode() { - return Objects.hash(nameParts, colNames, hints, partitions); + dmlCommandType, groupExpression, Optional.empty(), children.get(0)); } @Override public Plan withGroupExpression(Optional groupExpression) { return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions, - dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child()); + dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, - Optional logicalProperties, List children) { + Optional logicalProperties, List children) { return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions, - dmlCommandType, groupExpression, logicalProperties, children.get(0)); - } - - @Override - public LogicalProperties computeLogicalProperties() { - return UnboundLogicalProperties.INSTANCE; - } - - @Override - public List computeOutput() { - throw new UnboundException("output"); + dmlCommandType, groupExpression, logicalProperties, children.get(0)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSink.java new file mode 100644 index 000000000000000..a540e3a9067d53f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundIcebergTableSink.java @@ -0,0 +1,84 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.analyzer; + +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Optional; + +/** + * Represent an iceberg table sink plan node that has not been bound. + */ +public class UnboundIcebergTableSink extends UnboundBaseExternalTableSink { + + public UnboundIcebergTableSink(List nameParts, List colNames, List hints, + List partitions, CHILD_TYPE child) { + this(nameParts, colNames, hints, partitions, DMLCommandType.NONE, + Optional.empty(), Optional.empty(), child); + } + + /** + * constructor + */ + public UnboundIcebergTableSink(List nameParts, + List colNames, + List hints, + List partitions, + DMLCommandType dmlCommandType, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { + super(nameParts, PlanType.LOGICAL_UNBOUND_HIVE_TABLE_SINK, ImmutableList.of(), groupExpression, + logicalProperties, colNames, dmlCommandType, child, hints, partitions); + } + + @Override + public Plan withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, + "UnboundHiveTableSink only accepts one child"); + return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, Optional.empty(), children.get(0)); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitUnboundIcebergTableSink(this, context); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child()); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, groupExpression, logicalProperties, children.get(0)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java index 974bec90a2cdcca..fd70401f25d3c68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundTableSinkCreator.java @@ -22,6 +22,7 @@ import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.trees.plans.Plan; @@ -53,6 +54,8 @@ public static LogicalSink createUnboundTableSink(List na return new UnboundTableSink<>(nameParts, colNames, hints, partitions, query); } else if (curCatalog instanceof HMSExternalCatalog) { return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions, query); + } else if (curCatalog instanceof IcebergExternalCatalog) { + return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, query); } throw new UserException("Load data to " + curCatalog.getClass().getSimpleName() + " is not supported."); } @@ -72,6 +75,9 @@ public static LogicalSink createUnboundTableSink(List na } else if (curCatalog instanceof HMSExternalCatalog) { return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions, dmlCommandType, Optional.empty(), Optional.empty(), plan); + } else if (curCatalog instanceof IcebergExternalCatalog) { + return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, Optional.empty(), Optional.empty(), plan); } throw new RuntimeException("Load data to " + curCatalog.getClass().getSimpleName() + " is not supported."); } @@ -101,6 +107,9 @@ public static LogicalSink createUnboundTableSinkMaybeOverwrite(L } else if (curCatalog instanceof HMSExternalCatalog && !isAutoDetectPartition) { return new UnboundHiveTableSink<>(nameParts, colNames, hints, partitions, dmlCommandType, Optional.empty(), Optional.empty(), plan); + } else if (curCatalog instanceof IcebergExternalCatalog && !isAutoDetectPartition) { + return new UnboundIcebergTableSink<>(nameParts, colNames, hints, partitions, + dmlCommandType, Optional.empty(), Optional.empty(), plan); } throw new AnalysisException( "Auto overwrite data to " + curCatalog.getClass().getSimpleName() + " is not supported." diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 6748e8df249349b..d3bf9e4d737b65a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -121,6 +121,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalHashAggregate; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalHiveTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalIntersect; import org.apache.doris.nereids.trees.plans.physical.PhysicalJdbcScan; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; @@ -166,6 +167,7 @@ import org.apache.doris.planner.HashJoinNode; import org.apache.doris.planner.HashJoinNode.DistributionMode; import org.apache.doris.planner.HiveTableSink; +import org.apache.doris.planner.IcebergTableSink; import org.apache.doris.planner.IntersectNode; import org.apache.doris.planner.JoinNodeBase; import org.apache.doris.planner.MultiCastDataSink; @@ -455,7 +457,28 @@ public PlanFragment visitPhysicalHiveTableSink(PhysicalHiveTableSink icebergTableSink, + PlanTranslatorContext context) { + PlanFragment rootFragment = icebergTableSink.child().accept(this, context); + rootFragment.setOutputPartition(DataPartition.UNPARTITIONED); + + TupleDescriptor hiveTuple = context.generateTupleDesc(); + List targetTableColumns = icebergTableSink.getTargetTable().getFullSchema(); + for (Column column : targetTableColumns) { + SlotDescriptor slotDesc = context.addSlotDesc(hiveTuple); + slotDesc.setIsMaterialized(true); + slotDesc.setType(column.getType()); + slotDesc.setColumn(column); + slotDesc.setIsNullable(column.isAllowNull()); + slotDesc.setAutoInc(column.isAutoInc()); + } + IcebergTableSink sink = new IcebergTableSink((IcebergExternalTable) icebergTableSink.getTargetTable()); rootFragment.setSink(sink); return rootFragment; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPageCacheForInsertIntoSelect.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPageCacheForInsertIntoSelect.java index 67f0c1c3ba902f4..ab817c2f1d7c564 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPageCacheForInsertIntoSelect.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/pre/TurnOffPageCacheForInsertIntoSelect.java @@ -25,6 +25,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalHiveTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; import org.apache.doris.qe.SessionVariable; import org.apache.doris.qe.VariableMgr; @@ -59,6 +60,13 @@ public Plan visitLogicalHiveTableSink(LogicalHiveTableSink table return tableSink; } + @Override + public Plan visitLogicalIcebergTableSink( + LogicalIcebergTableSink tableSink, StatementContext context) { + turnOffPageCache(context); + return tableSink; + } + private void turnOffPageCache(StatementContext context) { SessionVariable sessionVariable = context.getConnectContext().getSessionVariable(); // set temporary session value, and then revert value in the 'finally block' of StmtExecutor#execute diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java index 16086f7e2950033..750707c52c48140 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/RequestPropertyDeriver.java @@ -39,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalHashJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalHiveTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalLimit; import org.apache.doris.nereids.trees.plans.physical.PhysicalNestedLoopJoin; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; @@ -140,6 +141,17 @@ public Void visitPhysicalHiveTableSink(PhysicalHiveTableSink hiv return null; } + @Override + public Void visitPhysicalIcebergTableSink( + PhysicalIcebergTableSink icebergTableSink, PlanContext context) { + if (connectContext != null && !connectContext.getSessionVariable().enableStrictConsistencyDml) { + addRequestPropertyToChildren(PhysicalProperties.ANY); + } else { + addRequestPropertyToChildren(icebergTableSink.getRequirePhysicalProperties()); + } + return null; + } + @Override public Void visitPhysicalResultSink(PhysicalResultSink physicalResultSink, PlanContext context) { addRequestPropertyToChildren(PhysicalProperties.GATHER); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index d317b1e87385213..1525172d5afcf72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -66,6 +66,7 @@ import org.apache.doris.nereids.rules.implementation.LogicalFilterToPhysicalFilter; import org.apache.doris.nereids.rules.implementation.LogicalGenerateToPhysicalGenerate; import org.apache.doris.nereids.rules.implementation.LogicalHiveTableSinkToPhysicalHiveTableSink; +import org.apache.doris.nereids.rules.implementation.LogicalIcebergTableSinkToPhysicalIcebergTableSink; import org.apache.doris.nereids.rules.implementation.LogicalIntersectToPhysicalIntersect; import org.apache.doris.nereids.rules.implementation.LogicalJdbcScanToPhysicalJdbcScan; import org.apache.doris.nereids.rules.implementation.LogicalJoinToHashJoin; @@ -191,6 +192,7 @@ public class RuleSet { .add(new LogicalGenerateToPhysicalGenerate()) .add(new LogicalOlapTableSinkToPhysicalOlapTableSink()) .add(new LogicalHiveTableSinkToPhysicalHiveTableSink()) + .add(new LogicalIcebergTableSinkToPhysicalIcebergTableSink()) .add(new LogicalFileSinkToPhysicalFileSink()) .add(new LogicalResultSinkToPhysicalResultSink()) .add(new LogicalDeferMaterializeResultSinkToPhysicalDeferMaterializeResultSink()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 14a959b43550e53..d3dac32355ccae8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -31,6 +31,7 @@ public enum RuleType { // **** make sure BINDING_UNBOUND_LOGICAL_PLAN is the lowest priority in the rewrite rules. **** BINDING_RESULT_SINK(RuleTypeClass.REWRITE), BINDING_INSERT_HIVE_TABLE(RuleTypeClass.REWRITE), + BINDING_INSERT_ICEBERG_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_TARGET_TABLE(RuleTypeClass.REWRITE), BINDING_INSERT_FILE(RuleTypeClass.REWRITE), BINDING_ONE_ROW_RELATION_SLOT(RuleTypeClass.REWRITE), @@ -413,6 +414,7 @@ public enum RuleType { LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_OLAP_TABLE_SINK_TO_PHYSICAL_OLAP_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_HIVE_TABLE_SINK_TO_PHYSICAL_HIVE_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), + LOGICAL_ICEBERG_TABLE_SINK_TO_PHYSICAL_ICEBERG_TABLE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_RESULT_SINK_TO_PHYSICAL_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_DEFER_MATERIALIZE_RESULT_SINK_TO_PHYSICAL_DEFER_MATERIALIZE_RESULT_SINK_RULE(RuleTypeClass.IMPLEMENTATION), LOGICAL_FILE_SINK_TO_PHYSICAL_FILE_SINK_RULE(RuleTypeClass.IMPLEMENTATION), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 7a56d2523e0c333..63f6db8fad93e92 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -29,8 +29,11 @@ import org.apache.doris.common.Pair; import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; +import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -54,6 +57,7 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; import org.apache.doris.nereids.trees.plans.logical.LogicalHiveTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; @@ -102,7 +106,9 @@ public List buildRules() { }) ), // TODO: bind hive taget table - RuleType.BINDING_INSERT_HIVE_TABLE.build(unboundHiveTableSink().thenApply(this::bindHiveTableSink)) + RuleType.BINDING_INSERT_HIVE_TABLE.build(unboundHiveTableSink().thenApply(this::bindHiveTableSink)), + RuleType.BINDING_INSERT_ICEBERG_TABLE.build( + unboundIcebergTableSink().thenApply(this::bindIcebergTableSink)) ); } @@ -393,12 +399,53 @@ private Plan bindHiveTableSink(MatchingContext> ctx) Column column = table.getColumn(cn); if (column == null) { throw new AnalysisException(String.format("column %s is not found in table %s", - cn, table.getName())); + cn, table.getName())); } return column; }).collect(ImmutableList.toImmutableList()); } LogicalHiveTableSink boundSink = new LogicalHiveTableSink<>( + database, + table, + bindColumns, + child.getOutput().stream() + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()), + sink.getDMLCommandType(), + Optional.empty(), + Optional.empty(), + child); + // we need to insert all the columns of the target table + if (boundSink.getCols().size() != child.getOutput().size()) { + throw new AnalysisException("insert into cols should be corresponding to the query output"); + } + Map columnToOutput = getColumnToOutput(ctx, table, false, + boundSink, child); + LogicalProject fullOutputProject = getOutputProjectByCoercion(table.getFullSchema(), child, columnToOutput); + return boundSink.withChildAndUpdateOutput(fullOutputProject); + } + + private Plan bindIcebergTableSink(MatchingContext> ctx) { + UnboundIcebergTableSink sink = ctx.root; + Pair pair = bind(ctx.cascadesContext, sink); + IcebergExternalDatabase database = pair.first; + IcebergExternalTable table = pair.second; + LogicalPlan child = ((LogicalPlan) sink.child()); + + List bindColumns; + if (sink.getColNames().isEmpty()) { + bindColumns = table.getBaseSchema(true).stream().collect(ImmutableList.toImmutableList()); + } else { + bindColumns = sink.getColNames().stream().map(cn -> { + Column column = table.getColumn(cn); + if (column == null) { + throw new AnalysisException(String.format("column %s is not found in table %s", + cn, table.getName())); + } + return column; + }).collect(ImmutableList.toImmutableList()); + } + LogicalIcebergTableSink boundSink = new LogicalIcebergTableSink<>( database, table, bindColumns, @@ -442,11 +489,26 @@ private Pair bind(CascadesContext cascade Pair, TableIf> pair = RelationUtil.getDbAndTable(tableQualifier, cascadesContext.getConnectContext().getEnv()); if (pair.second instanceof HMSExternalTable) { - return Pair.of(((HMSExternalDatabase) pair.first), (HMSExternalTable) pair.second); + HMSExternalTable table = (HMSExternalTable) pair.second; + if (table.getDlaType() == HMSExternalTable.DLAType.HIVE) { + return Pair.of(((HMSExternalDatabase) pair.first), table); + } } throw new AnalysisException("the target table of insert into is not a Hive table"); } + private Pair bind(CascadesContext cascadesContext, + UnboundIcebergTableSink sink) { + List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), + sink.getNameParts()); + Pair, TableIf> pair = RelationUtil.getDbAndTable(tableQualifier, + cascadesContext.getConnectContext().getEnv()); + if (pair.second instanceof IcebergExternalTable) { + return Pair.of(((IcebergExternalDatabase) pair.first), (IcebergExternalTable) pair.second); + } + throw new AnalysisException("the target table of insert into is not an iceberg table"); + } + private List bindPartitionIds(OlapTable table, List partitions, boolean temp) { return partitions.isEmpty() ? ImmutableList.of() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergTableSinkToPhysicalIcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergTableSinkToPhysicalIcebergTableSink.java new file mode 100644 index 000000000000000..c520ef83f2730cc --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalIcebergTableSinkToPhysicalIcebergTableSink.java @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.implementation; + +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink; + +import java.util.Optional; + +/** + * Implementation rule that convert logical IcebergTableSink to physical IcebergTableSink. + */ +public class LogicalIcebergTableSinkToPhysicalIcebergTableSink extends OneImplementationRuleFactory { + @Override + public Rule build() { + return logicalIcebergTableSink().thenApply(ctx -> { + LogicalIcebergTableSink sink = ctx.root; + return new PhysicalIcebergTableSink<>( + sink.getDatabase(), + sink.getTargetTable(), + sink.getCols(), + sink.getOutputExprs(), + Optional.empty(), + sink.getLogicalProperties(), + null, + null, + sink.child()); + }).toRule(RuleType.LOGICAL_ICEBERG_TABLE_SINK_TO_PHYSICAL_ICEBERG_TABLE_SINK_RULE); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java index 4aeecdfd507799a..71a7c24c9b64c12 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java @@ -48,6 +48,7 @@ public enum PlanType { LOGICAL_FILE_SINK, LOGICAL_OLAP_TABLE_SINK, LOGICAL_HIVE_TABLE_SINK, + LOGICAL_ICEBERG_TABLE_SINK, LOGICAL_RESULT_SINK, LOGICAL_UNBOUND_OLAP_TABLE_SINK, LOGICAL_UNBOUND_HIVE_TABLE_SINK, @@ -100,6 +101,7 @@ public enum PlanType { PHYSICAL_FILE_SINK, PHYSICAL_OLAP_TABLE_SINK, PHYSICAL_HIVE_TABLE_SINK, + PHYSICAL_ICEBERG_TABLE_SINK, PHYSICAL_RESULT_SINK, // physical others diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertCommandContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertCommandContext.java new file mode 100644 index 000000000000000..eb2b45e767c3462 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertCommandContext.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +/** + * For Base External Table + */ +public class BaseExternalTableInsertCommandContext extends InsertCommandContext { + protected boolean overwrite = false; + + public boolean isOverwrite() { + return overwrite; + } + + public void setOverwrite(boolean overwrite) { + this.overwrite = overwrite; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java new file mode 100644 index 000000000000000..5a660826601492a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java @@ -0,0 +1,161 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.UserException; +import org.apache.doris.common.profile.SummaryProfile; +import org.apache.doris.common.util.DebugUtil; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; +import org.apache.doris.planner.BaseExternalTableDataSink; +import org.apache.doris.planner.DataSink; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryState; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.transaction.TransactionManager; +import org.apache.doris.transaction.TransactionStatus; +import org.apache.doris.transaction.TransactionType; + +import com.google.common.base.Strings; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Optional; + +/** + * Insert executor for base external table + */ +public abstract class BaseExternalTableInsertExecutor extends AbstractInsertExecutor { + private static final Logger LOG = LogManager.getLogger(BaseExternalTableInsertExecutor.class); + private static final long INVALID_TXN_ID = -1L; + protected long txnId = INVALID_TXN_ID; + protected TransactionStatus txnStatus = TransactionStatus.ABORTED; + protected final TransactionManager transactionManager; + protected final String catalogName; + protected Optional summaryProfile = Optional.empty(); + + /** + * constructor + */ + public BaseExternalTableInsertExecutor(ConnectContext ctx, ExternalTable table, + String labelName, NereidsPlanner planner, + Optional insertCtx) { + super(ctx, table, labelName, planner, insertCtx); + catalogName = table.getCatalog().getName(); + transactionManager = table.getCatalog().getTransactionManager(); + + if (ConnectContext.get().getExecutor() != null) { + summaryProfile = Optional.of(ConnectContext.get().getExecutor().getSummaryProfile()); + } + } + + public long getTxnId() { + return txnId; + } + + /** + * collect commit infos from BEs + */ + protected abstract void setCollectCommitInfoFunc(); + + /** + * At this time, FE has successfully collected all commit information from BEs. + * Before commit this txn, commit information need to be analyzed and processed. + */ + protected abstract void doBeforeCommit() throws UserException; + + /** + * The type of the current transaction + */ + protected abstract TransactionType transactionType(); + + @Override + public void beginTransaction() { + txnId = transactionManager.begin(); + setCollectCommitInfoFunc(); + } + + @Override + protected void onComplete() throws UserException { + if (ctx.getState().getStateType() == QueryState.MysqlStateType.ERR) { + LOG.warn("errors when abort txn. {}", ctx.getQueryIdentifier()); + } else { + doBeforeCommit(); + summaryProfile.ifPresent(profile -> profile.setTransactionBeginTime(transactionType())); + transactionManager.commit(txnId); + summaryProfile.ifPresent(SummaryProfile::setTransactionEndTime); + txnStatus = TransactionStatus.COMMITTED; + Env.getCurrentEnv().getRefreshManager().refreshTable( + catalogName, + table.getDatabase().getFullName(), + table.getName(), + true); + } + } + + @Override + protected void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink physicalSink) { + try { + ((BaseExternalTableDataSink) sink).bindDataSink(insertCtx); + } catch (Exception e) { + throw new AnalysisException(e.getMessage(), e); + } + } + + @Override + protected void onFail(Throwable t) { + errMsg = t.getMessage() == null ? "unknown reason" : t.getMessage(); + String queryId = DebugUtil.printId(ctx.queryId()); + // if any throwable being thrown during insert operation, first we should abort this txn + LOG.warn("insert [{}] with query id {} failed", labelName, queryId, t); + StringBuilder sb = new StringBuilder(t.getMessage()); + if (txnId != INVALID_TXN_ID) { + LOG.warn("insert [{}] with query id {} abort txn {} failed", labelName, queryId, txnId); + if (!Strings.isNullOrEmpty(coordinator.getTrackingUrl())) { + sb.append(". url: ").append(coordinator.getTrackingUrl()); + } + } + ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, t.getMessage()); + transactionManager.rollback(txnId); + } + + @Override + protected void afterExec(StmtExecutor executor) { + StringBuilder sb = new StringBuilder(); + sb.append("{"); + sb.append("'status':'") + .append(ctx.isTxnModel() ? TransactionStatus.PREPARE.name() : txnStatus.name()); + sb.append("', 'txnId':'").append(txnId).append("'"); + if (!Strings.isNullOrEmpty(errMsg)) { + sb.append(", 'err':'").append(errMsg).append("'"); + } + sb.append("}"); + ctx.getState().setOk(loadedRows, 0, sb.toString()); + // set insert result in connection context, + // so that user can use `show insert result` to get info of the last insert operation. + ctx.setOrUpdateInsertResult(txnId, labelName, database.getFullName(), table.getName(), + txnStatus, loadedRows, 0); + // update it, so that user can get loaded rows in fe.audit.log + ctx.updateReturnRows((int) loadedRows); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertCommandContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertCommandContext.java index 49d5a12c2bb7a0d..1e68a5cd2207333 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertCommandContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertCommandContext.java @@ -20,19 +20,10 @@ /** * For Hive Table */ -public class HiveInsertCommandContext extends InsertCommandContext { - private boolean overwrite = false; +public class HiveInsertCommandContext extends BaseExternalTableInsertCommandContext { private String writePath; private String queryId; - public boolean isOverwrite() { - return overwrite; - } - - public void setOverwrite(boolean overwrite) { - this.overwrite = overwrite; - } - public String getWritePath() { return writePath; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java index 9421af795033208..dea731f9af5baf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java @@ -17,47 +17,26 @@ package org.apache.doris.nereids.trees.plans.commands.insert; -import org.apache.doris.catalog.Env; -import org.apache.doris.common.ErrorCode; import org.apache.doris.common.UserException; -import org.apache.doris.common.profile.SummaryProfile; import org.apache.doris.common.util.DebugUtil; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSTransaction; import org.apache.doris.nereids.NereidsPlanner; -import org.apache.doris.nereids.exceptions.AnalysisException; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.physical.PhysicalHiveTableSink; -import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; -import org.apache.doris.planner.DataSink; -import org.apache.doris.planner.HiveTableSink; -import org.apache.doris.planner.PlanFragment; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.QueryState; -import org.apache.doris.qe.StmtExecutor; import org.apache.doris.thrift.TUniqueId; -import org.apache.doris.transaction.TransactionManager; -import org.apache.doris.transaction.TransactionStatus; import org.apache.doris.transaction.TransactionType; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.util.Optional; /** - * Insert executor for olap table + * Insert executor for hive table */ -public class HiveInsertExecutor extends AbstractInsertExecutor { +public class HiveInsertExecutor extends BaseExternalTableInsertExecutor { private static final Logger LOG = LogManager.getLogger(HiveInsertExecutor.class); - private static final long INVALID_TXN_ID = -1L; - private long txnId = INVALID_TXN_ID; - private TransactionStatus txnStatus = TransactionStatus.ABORTED; - private final TransactionManager transactionManager; - private final String catalogName; - private Optional summaryProfile = Optional.empty(); /** * constructor @@ -66,36 +45,14 @@ public HiveInsertExecutor(ConnectContext ctx, HMSExternalTable table, String labelName, NereidsPlanner planner, Optional insertCtx) { super(ctx, table, labelName, planner, insertCtx); - catalogName = table.getCatalog().getName(); - transactionManager = table.getCatalog().getTransactionManager(); - - if (ConnectContext.get().getExecutor() != null) { - summaryProfile = Optional.of(ConnectContext.get().getExecutor().getSummaryProfile()); - } - } - - public long getTxnId() { - return txnId; } @Override - public void beginTransaction() { - txnId = transactionManager.begin(); + public void setCollectCommitInfoFunc() { HMSTransaction transaction = (HMSTransaction) transactionManager.getTransaction(txnId); coordinator.setHivePartitionUpdateFunc(transaction::updateHivePartitionUpdates); } - @Override - protected void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink physicalSink) { - HiveTableSink hiveTableSink = (HiveTableSink) sink; - PhysicalHiveTableSink physicalHiveSink = (PhysicalHiveTableSink) physicalSink; - try { - hiveTableSink.bindDataSink(physicalHiveSink.getCols(), insertCtx); - } catch (Exception e) { - throw new AnalysisException(e.getMessage(), e); - } - } - @Override protected void beforeExec() { // check params @@ -109,61 +66,16 @@ protected void beforeExec() { } @Override - protected void onComplete() throws UserException { - if (ctx.getState().getStateType() == QueryState.MysqlStateType.ERR) { - LOG.warn("errors when abort txn. {}", ctx.getQueryIdentifier()); - } else { - HMSTransaction transaction = (HMSTransaction) transactionManager.getTransaction(txnId); - loadedRows = transaction.getUpdateCnt(); - String dbName = ((HMSExternalTable) table).getDbName(); - String tbName = table.getName(); - transaction.finishInsertTable(dbName, tbName); - summaryProfile.ifPresent(profile -> profile.setTransactionBeginTime(TransactionType.HMS)); - transactionManager.commit(txnId); - summaryProfile.ifPresent(SummaryProfile::setTransactionEndTime); - txnStatus = TransactionStatus.COMMITTED; - Env.getCurrentEnv().getRefreshManager().refreshTable( - catalogName, - dbName, - tbName, - true); - } - } - - @Override - protected void onFail(Throwable t) { - errMsg = t.getMessage() == null ? "unknown reason" : t.getMessage(); - String queryId = DebugUtil.printId(ctx.queryId()); - // if any throwable being thrown during insert operation, first we should abort this txn - LOG.warn("insert [{}] with query id {} failed", labelName, queryId, t); - StringBuilder sb = new StringBuilder(t.getMessage()); - if (txnId != INVALID_TXN_ID) { - LOG.warn("insert [{}] with query id {} abort txn {} failed", labelName, queryId, txnId); - if (!Strings.isNullOrEmpty(coordinator.getTrackingUrl())) { - sb.append(". url: ").append(coordinator.getTrackingUrl()); - } - } - ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, t.getMessage()); - transactionManager.rollback(txnId); + protected void doBeforeCommit() throws UserException { + HMSTransaction transaction = (HMSTransaction) transactionManager.getTransaction(txnId); + loadedRows = transaction.getUpdateCnt(); + String dbName = ((HMSExternalTable) table).getDbName(); + String tbName = table.getName(); + transaction.finishInsertTable(dbName, tbName); } @Override - protected void afterExec(StmtExecutor executor) { - StringBuilder sb = new StringBuilder(); - sb.append("{"); - sb.append("'status':'") - .append(ctx.isTxnModel() ? TransactionStatus.PREPARE.name() : txnStatus.name()); - sb.append("', 'txnId':'").append(txnId).append("'"); - if (!Strings.isNullOrEmpty(errMsg)) { - sb.append(", 'err':'").append(errMsg).append("'"); - } - sb.append("}"); - ctx.getState().setOk(loadedRows, 0, sb.toString()); - // set insert result in connection context, - // so that user can use `show insert result` to get info of the last insert operation. - ctx.setOrUpdateInsertResult(txnId, labelName, database.getFullName(), table.getName(), - txnStatus, loadedRows, 0); - // update it, so that user can get loaded rows in fe.audit.log - ctx.updateReturnRows((int) loadedRows); + protected TransactionType transactionType() { + return TransactionType.HMS; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java new file mode 100644 index 000000000000000..cbb291fb3b718b0 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/IcebergInsertExecutor.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergTransaction; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.transaction.TransactionType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Optional; + +/** + * Insert executor for iceberg table + */ +public class IcebergInsertExecutor extends BaseExternalTableInsertExecutor { + private static final Logger LOG = LogManager.getLogger(IcebergInsertExecutor.class); + + /** + * constructor + */ + public IcebergInsertExecutor(ConnectContext ctx, IcebergExternalTable table, + String labelName, NereidsPlanner planner, + Optional insertCtx) { + super(ctx, table, labelName, planner, insertCtx); + } + + @Override + public void setCollectCommitInfoFunc() { + IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); + coordinator.setIcebergCommitDataFunc(transaction::updateIcebergCommitData); + } + + @Override + protected void doBeforeCommit() throws UserException { + IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); + loadedRows = transaction.getUpdateCnt(); + transaction.finishInsert(); + } + + @Override + protected TransactionType transactionType() { + return TransactionType.ICEBERG; + } + + @Override + protected void beforeExec() { + IcebergTransaction transaction = (IcebergTransaction) transactionManager.getTransaction(txnId); + transaction.beginInsert(((IcebergExternalTable) table).getDbName(), table.getName()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java index 7c9ea67860c991c..003c2e591165cc8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java @@ -24,6 +24,7 @@ import org.apache.doris.common.ErrorReport; import org.apache.doris.common.util.ProfileManager.ProfileType; import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.load.loadv2.LoadStatistic; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.NereidsPlanner; @@ -37,6 +38,7 @@ import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalHiveTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; @@ -177,9 +179,13 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor insertExecutor = new HiveInsertExecutor(ctx, hiveExternalTable, label, planner, Optional.of(insertCtx.orElse((new HiveInsertCommandContext())))); // set hive query options + } else if (physicalSink instanceof PhysicalIcebergTableSink) { + IcebergExternalTable icebergExternalTable = (IcebergExternalTable) targetTableIf; + insertExecutor = new IcebergInsertExecutor(ctx, icebergExternalTable, label, planner, + Optional.of(insertCtx.orElse((new BaseExternalTableInsertCommandContext())))); } else { // TODO: support other table types - throw new AnalysisException("insert into command only support olap table"); + throw new AnalysisException("insert into command only support [olap, hive, iceberg] table"); } insertExecutor.beginTransaction(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 8293183eeb200e8..cabdfc203e5d5e8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -28,6 +28,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; +import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -369,8 +370,11 @@ public static TableIf getTargetTable(Plan plan, ConnectContext ctx) { unboundTableSink = (UnboundTableSink) plan; } else if (plan instanceof UnboundHiveTableSink) { unboundTableSink = (UnboundHiveTableSink) plan; + } else if (plan instanceof UnboundIcebergTableSink) { + unboundTableSink = (UnboundIcebergTableSink) plan; } else { - throw new AnalysisException("the root of plan should be UnboundTableSink or UnboundHiveTableSink" + throw new AnalysisException("the root of plan should be" + + " [UnboundTableSink, UnboundHiveTableSink, UnboundIcebergTableSink]," + " but it is " + plan.getType()); } List tableQualifier = RelationUtil.getQualifierName(ctx, unboundTableSink.getNameParts()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergTableSink.java new file mode 100644 index 000000000000000..d121645b231c034 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalIcebergTableSink.java @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.logical; + +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.PropagateFuncDeps; +import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.nereids.util.Utils; + +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * logical hive table sink for insert command + */ +public class LogicalIcebergTableSink extends LogicalTableSink + implements Sink, PropagateFuncDeps { + // bound data sink + private final IcebergExternalDatabase database; + private final IcebergExternalTable targetTable; + private final DMLCommandType dmlCommandType; + + /** + * constructor + */ + public LogicalIcebergTableSink(IcebergExternalDatabase database, + IcebergExternalTable targetTable, + List cols, + List outputExprs, + DMLCommandType dmlCommandType, + Optional groupExpression, + Optional logicalProperties, + CHILD_TYPE child) { + super(PlanType.LOGICAL_ICEBERG_TABLE_SINK, outputExprs, groupExpression, logicalProperties, cols, child); + this.database = Objects.requireNonNull(database, "database != null in LogicalHiveTableSink"); + this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in LogicalHiveTableSink"); + this.dmlCommandType = dmlCommandType; + } + + public Plan withChildAndUpdateOutput(Plan child) { + List output = child.getOutput().stream() + .map(NamedExpression.class::cast) + .collect(ImmutableList.toImmutableList()); + return new LogicalIcebergTableSink<>(database, targetTable, cols, output, + dmlCommandType, Optional.empty(), Optional.empty(), child); + } + + @Override + public Plan withChildren(List children) { + Preconditions.checkArgument(children.size() == 1, "LogicalHiveTableSink only accepts one child"); + return new LogicalIcebergTableSink<>(database, targetTable, cols, outputExprs, + dmlCommandType, Optional.empty(), Optional.empty(), children.get(0)); + } + + public LogicalIcebergTableSink withOutputExprs(List outputExprs) { + return new LogicalIcebergTableSink<>(database, targetTable, cols, outputExprs, + dmlCommandType, Optional.empty(), Optional.empty(), child()); + } + + public IcebergExternalDatabase getDatabase() { + return database; + } + + public IcebergExternalTable getTargetTable() { + return targetTable; + } + + public DMLCommandType getDmlCommandType() { + return dmlCommandType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + LogicalIcebergTableSink that = (LogicalIcebergTableSink) o; + return dmlCommandType == that.dmlCommandType + && Objects.equals(database, that.database) + && Objects.equals(targetTable, that.targetTable) && Objects.equals(cols, that.cols); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), database, targetTable, cols, dmlCommandType); + } + + @Override + public String toString() { + return Utils.toSqlString("LogicalHiveTableSink[" + id.asInt() + "]", + "outputExprs", outputExprs, + "database", database.getFullName(), + "targetTable", targetTable.getName(), + "cols", cols, + "dmlCommandType", dmlCommandType + ); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitLogicalIcebergTableSink(this, context); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new LogicalIcebergTableSink<>(database, targetTable, cols, outputExprs, + dmlCommandType, groupExpression, Optional.of(getLogicalProperties()), child()); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return new LogicalIcebergTableSink<>(database, targetTable, cols, outputExprs, + dmlCommandType, groupExpression, logicalProperties, children.get(0)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBaseExternalTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBaseExternalTableSink.java new file mode 100644 index 000000000000000..82483c63a40412a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalBaseExternalTableSink.java @@ -0,0 +1,79 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.algebra.Sink; +import org.apache.doris.nereids.util.Utils; +import org.apache.doris.statistics.Statistics; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** abstract physical external table sink */ +public abstract class PhysicalBaseExternalTableSink extends PhysicalTableSink + implements Sink { + + protected final ExternalDatabase database; + protected final ExternalTable targetTable; + protected final List cols; + + /** + * constructor + */ + public PhysicalBaseExternalTableSink(PlanType type, + ExternalDatabase database, + ExternalTable targetTable, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, + Statistics statistics, + CHILD_TYPE child) { + super(type, outputExprs, groupExpression, + logicalProperties, physicalProperties, statistics, child); + this.database = Objects.requireNonNull( + database, "database != null in " + this.getClass().getSimpleName()); + this.targetTable = Objects.requireNonNull( + targetTable, "targetTable != null in " + this.getClass().getSimpleName()); + this.cols = Utils.copyRequiredList(cols); + } + + public ExternalTable getTargetTable() { + return targetTable; + } + + @Override + public List getExpressions() { + return ImmutableList.of(); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHiveTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHiveTableSink.java index 58141e61bf88850..4a7febf3d1d9af9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHiveTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHiveTableSink.java @@ -25,30 +25,20 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.algebra.Sink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; -import com.google.common.collect.ImmutableList; - import java.util.ArrayList; import java.util.List; -import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; -/** abstract physical hive sink */ -public class PhysicalHiveTableSink extends PhysicalTableSink implements Sink { - - private final HMSExternalDatabase database; - private final HMSExternalTable targetTable; - private final List cols; +/** physical hive sink */ +public class PhysicalHiveTableSink extends PhysicalBaseExternalTableSink { /** * constructor @@ -76,28 +66,14 @@ public PhysicalHiveTableSink(HMSExternalDatabase database, PhysicalProperties physicalProperties, Statistics statistics, CHILD_TYPE child) { - super(PlanType.PHYSICAL_HIVE_TABLE_SINK, outputExprs, groupExpression, + super(PlanType.PHYSICAL_HIVE_TABLE_SINK, database, targetTable, cols, outputExprs, groupExpression, logicalProperties, physicalProperties, statistics, child); - this.database = Objects.requireNonNull(database, "database != null in PhysicalHiveTableSink"); - this.targetTable = Objects.requireNonNull(targetTable, "targetTable != null in PhysicalHiveTableSink"); - this.cols = Utils.copyRequiredList(cols); - } - - public HMSExternalDatabase getDatabase() { - return database; - } - - public HMSExternalTable getTargetTable() { - return targetTable; - } - - public List getCols() { - return cols; } @Override public Plan withChildren(List children) { - return new PhysicalHiveTableSink<>(database, targetTable, cols, outputExprs, groupExpression, + return new PhysicalHiveTableSink<>( + (HMSExternalDatabase) database, (HMSExternalTable) targetTable, cols, outputExprs, groupExpression, getLogicalProperties(), physicalProperties, statistics, children.get(0)); } @@ -106,27 +82,25 @@ public R accept(PlanVisitor visitor, C context) { return visitor.visitPhysicalHiveTableSink(this, context); } - @Override - public List getExpressions() { - return ImmutableList.of(); - } - @Override public Plan withGroupExpression(Optional groupExpression) { - return new PhysicalHiveTableSink<>(database, targetTable, cols, outputExprs, + return new PhysicalHiveTableSink<>( + (HMSExternalDatabase) database, (HMSExternalTable) targetTable, cols, outputExprs, groupExpression, getLogicalProperties(), child()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalHiveTableSink<>(database, targetTable, cols, outputExprs, + return new PhysicalHiveTableSink<>( + (HMSExternalDatabase) database, (HMSExternalTable) targetTable, cols, outputExprs, groupExpression, logicalProperties.get(), children.get(0)); } @Override public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { - return new PhysicalHiveTableSink<>(database, targetTable, cols, outputExprs, + return new PhysicalHiveTableSink<>( + (HMSExternalDatabase) database, (HMSExternalTable) targetTable, cols, outputExprs, groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); } @@ -135,7 +109,7 @@ public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalPr */ @Override public PhysicalProperties getRequirePhysicalProperties() { - Set hivePartitionKeys = targetTable.getPartitionColumnNames(); + Set hivePartitionKeys = ((HMSExternalTable) targetTable).getPartitionColumnNames(); if (!hivePartitionKeys.isEmpty()) { List columnIdx = new ArrayList<>(); List fullSchema = targetTable.getFullSchema(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java new file mode 100644 index 000000000000000..2dd467c5f61948d --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalIcebergTableSink.java @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.physical; + +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.iceberg.IcebergExternalDatabase; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.nereids.memo.GroupExpression; +import org.apache.doris.nereids.properties.DistributionSpecTableSinkHashPartitioned; +import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.PlanType; +import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; +import org.apache.doris.statistics.Statistics; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** physical iceberg sink */ +public class PhysicalIcebergTableSink extends PhysicalBaseExternalTableSink { + + /** + * constructor + */ + public PhysicalIcebergTableSink(IcebergExternalDatabase database, + IcebergExternalTable targetTable, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + CHILD_TYPE child) { + this(database, targetTable, cols, outputExprs, groupExpression, logicalProperties, + PhysicalProperties.GATHER, null, child); + } + + /** + * constructor + */ + public PhysicalIcebergTableSink(IcebergExternalDatabase database, + IcebergExternalTable targetTable, + List cols, + List outputExprs, + Optional groupExpression, + LogicalProperties logicalProperties, + PhysicalProperties physicalProperties, + Statistics statistics, + CHILD_TYPE child) { + super(PlanType.PHYSICAL_ICEBERG_TABLE_SINK, database, targetTable, cols, outputExprs, groupExpression, + logicalProperties, physicalProperties, statistics, child); + } + + @Override + public Plan withChildren(List children) { + return new PhysicalIcebergTableSink<>( + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, + cols, outputExprs, groupExpression, + getLogicalProperties(), physicalProperties, statistics, children.get(0)); + } + + @Override + public R accept(PlanVisitor visitor, C context) { + return visitor.visitPhysicalIcebergTableSink(this, context); + } + + @Override + public Plan withGroupExpression(Optional groupExpression) { + return new PhysicalIcebergTableSink<>( + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, + groupExpression, getLogicalProperties(), child()); + } + + @Override + public Plan withGroupExprLogicalPropChildren(Optional groupExpression, + Optional logicalProperties, List children) { + return new PhysicalIcebergTableSink<>( + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, + groupExpression, logicalProperties.get(), children.get(0)); + } + + @Override + public PhysicalPlan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { + return new PhysicalIcebergTableSink<>( + (IcebergExternalDatabase) database, (IcebergExternalTable) targetTable, cols, outputExprs, + groupExpression, getLogicalProperties(), physicalProperties, statistics, child()); + } + + /** + * get output physical properties + */ + @Override + public PhysicalProperties getRequirePhysicalProperties() { + Set partitionNames = targetTable.getPartitionNames(); + if (!partitionNames.isEmpty()) { + List columnIdx = new ArrayList<>(); + List fullSchema = targetTable.getFullSchema(); + for (int i = 0; i < fullSchema.size(); i++) { + Column column = fullSchema.get(i); + if (partitionNames.contains(column.getName())) { + columnIdx.add(i); + } + } + // mapping partition id + List exprIds = columnIdx.stream() + .map(idx -> child().getOutput().get(idx).getExprId()) + .collect(Collectors.toList()); + DistributionSpecTableSinkHashPartitioned shuffleInfo = new DistributionSpecTableSinkHashPartitioned(); + shuffleInfo.setOutputColExprIds(exprIds); + return new PhysicalProperties(shuffleInfo); + } + return PhysicalProperties.SINK_RANDOM_PARTITIONED; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java index 976592b461e7b6d..e0b8a1dddc1706d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java @@ -18,12 +18,14 @@ package org.apache.doris.nereids.trees.plans.visitor; import org.apache.doris.nereids.analyzer.UnboundHiveTableSink; +import org.apache.doris.nereids.analyzer.UnboundIcebergTableSink; import org.apache.doris.nereids.analyzer.UnboundResultSink; import org.apache.doris.nereids.analyzer.UnboundTableSink; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalFileSink; import org.apache.doris.nereids.trees.plans.logical.LogicalHiveTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; import org.apache.doris.nereids.trees.plans.logical.LogicalSink; @@ -31,6 +33,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeResultSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalFileSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalHiveTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalIcebergTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink; import org.apache.doris.nereids.trees.plans.physical.PhysicalSink; @@ -61,6 +64,10 @@ default R visitUnboundHiveTableSink(UnboundHiveTableSink unbound return visitLogicalSink(unboundTableSink, context); } + default R visitUnboundIcebergTableSink(UnboundIcebergTableSink unboundTableSink, C context) { + return visitLogicalSink(unboundTableSink, context); + } + default R visitUnboundResultSink(UnboundResultSink unboundResultSink, C context) { return visitLogicalSink(unboundResultSink, context); } @@ -85,6 +92,10 @@ default R visitLogicalHiveTableSink(LogicalHiveTableSink hiveTab return visitLogicalTableSink(hiveTableSink, context); } + default R visitLogicalIcebergTableSink(LogicalIcebergTableSink icebergTableSink, C context) { + return visitLogicalTableSink(icebergTableSink, context); + } + default R visitLogicalResultSink(LogicalResultSink logicalResultSink, C context) { return visitLogicalSink(logicalResultSink, context); } @@ -114,6 +125,10 @@ default R visitPhysicalHiveTableSink(PhysicalHiveTableSink hiveT return visitPhysicalTableSink(hiveTableSink, context); } + default R visitPhysicalIcebergTableSink(PhysicalIcebergTableSink icebergTableSink, C context) { + return visitPhysicalTableSink(icebergTableSink, context); + } + default R visitPhysicalResultSink(PhysicalResultSink physicalResultSink, C context) { return visitPhysicalSink(physicalResultSink, context); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java new file mode 100644 index 000000000000000..92350e95013c524 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/BaseExternalTableDataSink.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/DataSink.java +// and modified by Doris + +package org.apache.doris.planner; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext; +import org.apache.doris.thrift.TDataSink; +import org.apache.doris.thrift.TFileCompressType; +import org.apache.doris.thrift.TFileFormatType; + +import java.util.Optional; +import java.util.Set; + +public abstract class BaseExternalTableDataSink extends DataSink { + + protected TDataSink tDataSink; + + @Override + protected TDataSink toThrift() { + return tDataSink; + } + + @Override + public PlanNodeId getExchNodeId() { + return null; + } + + @Override + public DataPartition getOutputPartition() { + return DataPartition.RANDOM; + } + + /** + * File format types supported by the current table + */ + protected abstract Set supportedFileFormatTypes(); + + protected TFileFormatType getTFileFormatType(String format) throws AnalysisException { + TFileFormatType fileFormatType = TFileFormatType.FORMAT_UNKNOWN; + String lowerCase = format.toLowerCase(); + if (lowerCase.contains("orc")) { + fileFormatType = TFileFormatType.FORMAT_ORC; + } else if (lowerCase.contains("parquet")) { + fileFormatType = TFileFormatType.FORMAT_PARQUET; + } else if (lowerCase.contains("text")) { + fileFormatType = TFileFormatType.FORMAT_CSV_PLAIN; + } + if (!supportedFileFormatTypes().contains(fileFormatType)) { + throw new AnalysisException("Unsupported input format type: " + format); + } + return fileFormatType; + } + + protected TFileCompressType getTFileCompressType(String compressType) { + if ("snappy".equalsIgnoreCase(compressType)) { + return TFileCompressType.SNAPPYBLOCK; + } else if ("lz4".equalsIgnoreCase(compressType)) { + return TFileCompressType.LZ4BLOCK; + } else if ("lzo".equalsIgnoreCase(compressType)) { + return TFileCompressType.LZO; + } else if ("zlib".equalsIgnoreCase(compressType)) { + return TFileCompressType.ZLIB; + } else if ("zstd".equalsIgnoreCase(compressType)) { + return TFileCompressType.ZSTD; + } else if ("uncompressed".equalsIgnoreCase(compressType)) { + return TFileCompressType.PLAIN; + } else { + // try to use plain type to decompress parquet or orc file + return TFileCompressType.PLAIN; + } + } + + /** + * check sink params and generate thrift data sink to BE + * @param insertCtx insert info context + * @throws AnalysisException if source file format cannot be read + */ + public abstract void bindDataSink(Optional insertCtx) throws AnalysisException; +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java index c769bbea782d0a3..8d6daa2f8b72b49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataSink.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.odbc.sink.OdbcTableSink; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TExplainLevel; @@ -70,6 +71,8 @@ public static DataSink createDataSink(TableIf table) throws AnalysisException { return new OdbcTableSink((OdbcTable) table); } else if (table instanceof HMSExternalTable) { return new HiveTableSink((HMSExternalTable) table); + } else if (table instanceof IcebergExternalTable) { + return new IcebergTableSink((IcebergExternalTable) table); } else { throw new AnalysisException("Unknown table type " + table.getType()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java index 6374dfddc6636a7..efa9bd9b8f83461 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java @@ -31,7 +31,6 @@ import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TDataSinkType; import org.apache.doris.thrift.TExplainLevel; -import org.apache.doris.thrift.TFileCompressType; import org.apache.doris.thrift.TFileFormatType; import org.apache.doris.thrift.TFileType; import org.apache.doris.thrift.THiveBucket; @@ -50,16 +49,24 @@ import java.util.Set; import java.util.stream.Collectors; -public class HiveTableSink extends DataSink { +public class HiveTableSink extends BaseExternalTableDataSink { - private HMSExternalTable targetTable; - protected TDataSink tDataSink; + private final HMSExternalTable targetTable; + private static final HashSet supportedTypes = new HashSet() {{ + add(TFileFormatType.FORMAT_ORC); + add(TFileFormatType.FORMAT_PARQUET); + }}; public HiveTableSink(HMSExternalTable targetTable) { super(); this.targetTable = targetTable; } + @Override + protected Set supportedFileFormatTypes() { + return supportedTypes; + } + @Override public String getExplainString(String prefix, TExplainLevel explainLevel) { StringBuilder strBuilder = new StringBuilder(); @@ -72,26 +79,7 @@ public String getExplainString(String prefix, TExplainLevel explainLevel) { } @Override - protected TDataSink toThrift() { - return tDataSink; - } - - @Override - public PlanNodeId getExchNodeId() { - return null; - } - - @Override - public DataPartition getOutputPartition() { - return DataPartition.RANDOM; - } - - /** - * check sink params and generate thrift data sink to BE - * @param insertCtx insert info context - * @throws AnalysisException if source file format cannot be read - */ - public void bindDataSink(List insertCols, Optional insertCtx) + public void bindDataSink(Optional insertCtx) throws AnalysisException { THiveTableSink tSink = new THiveTableSink(); tSink.setDbName(targetTable.getDbName()); @@ -124,7 +112,7 @@ public void bindDataSink(List insertCols, Optional bucketInfo.setBucketCount(sd.getNumBuckets()); tSink.setBucketInfo(bucketInfo); - TFileFormatType formatType = getFileFormatType(sd); + TFileFormatType formatType = getTFileFormatType(sd.getInputFormat()); tSink.setFileFormat(formatType); setCompressType(tSink, formatType); @@ -180,23 +168,7 @@ private void setCompressType(THiveTableSink tSink, TFileFormatType formatType) { compressType = "uncompressed"; break; } - - if ("snappy".equalsIgnoreCase(compressType)) { - tSink.setCompressionType(TFileCompressType.SNAPPYBLOCK); - } else if ("lz4".equalsIgnoreCase(compressType)) { - tSink.setCompressionType(TFileCompressType.LZ4BLOCK); - } else if ("lzo".equalsIgnoreCase(compressType)) { - tSink.setCompressionType(TFileCompressType.LZO); - } else if ("zlib".equalsIgnoreCase(compressType)) { - tSink.setCompressionType(TFileCompressType.ZLIB); - } else if ("zstd".equalsIgnoreCase(compressType)) { - tSink.setCompressionType(TFileCompressType.ZSTD); - } else if ("uncompressed".equalsIgnoreCase(compressType)) { - tSink.setCompressionType(TFileCompressType.PLAIN); - } else { - // try to use plain type to decompress parquet or orc file - tSink.setCompressionType(TFileCompressType.PLAIN); - } + tSink.setCompressionType(getTFileCompressType(compressType)); } private void setPartitionValues(THiveTableSink tSink) throws AnalysisException { @@ -207,7 +179,7 @@ private void setPartitionValues(THiveTableSink tSink) throws AnalysisException { for (org.apache.hadoop.hive.metastore.api.Partition partition : hivePartitions) { THivePartition hivePartition = new THivePartition(); StorageDescriptor sd = partition.getSd(); - hivePartition.setFileFormat(getFileFormatType(sd)); + hivePartition.setFileFormat(getTFileFormatType(sd.getInputFormat())); hivePartition.setValues(partition.getValues()); THiveLocationParams locationParams = new THiveLocationParams(); @@ -222,20 +194,6 @@ private void setPartitionValues(THiveTableSink tSink) throws AnalysisException { tSink.setPartitions(partitions); } - private TFileFormatType getFileFormatType(StorageDescriptor sd) throws AnalysisException { - TFileFormatType fileFormatType; - if (sd.getInputFormat().toLowerCase().contains("orc")) { - fileFormatType = TFileFormatType.FORMAT_ORC; - } else if (sd.getInputFormat().toLowerCase().contains("parquet")) { - fileFormatType = TFileFormatType.FORMAT_PARQUET; - } else if (sd.getInputFormat().toLowerCase().contains("text")) { - fileFormatType = TFileFormatType.FORMAT_CSV_PLAIN; - } else { - throw new AnalysisException("Unsupported input format type: " + sd.getInputFormat()); - } - return fileFormatType; - } - protected TDataSinkType getDataSinkType() { return TDataSinkType.HIVE_TABLE_SINK; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java new file mode 100644 index 000000000000000..659be7cb1fed983 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/IcebergTableSink.java @@ -0,0 +1,146 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.planner; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.LocationPath; +import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.nereids.trees.plans.commands.insert.BaseExternalTableInsertCommandContext; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertCommandContext; +import org.apache.doris.thrift.TDataSink; +import org.apache.doris.thrift.TDataSinkType; +import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TFileFormatType; +import org.apache.doris.thrift.TIcebergTableSink; +import org.apache.doris.thrift.TSortField; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; +import org.apache.iceberg.NullOrder; +import org.apache.iceberg.PartitionSpecParser; +import org.apache.iceberg.SchemaParser; +import org.apache.iceberg.SortDirection; +import org.apache.iceberg.SortField; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.types.Types; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +public class IcebergTableSink extends BaseExternalTableDataSink { + + private final IcebergExternalTable targetTable; + private static final HashSet supportedTypes = new HashSet() {{ + add(TFileFormatType.FORMAT_ORC); + add(TFileFormatType.FORMAT_PARQUET); + }}; + + public IcebergTableSink(IcebergExternalTable targetTable) { + super(); + this.targetTable = targetTable; + } + + @Override + protected Set supportedFileFormatTypes() { + return supportedTypes; + } + + @Override + public String getExplainString(String prefix, TExplainLevel explainLevel) { + StringBuilder strBuilder = new StringBuilder(); + strBuilder.append(prefix).append("ICEBERG TABLE SINK\n"); + if (explainLevel == TExplainLevel.BRIEF) { + return strBuilder.toString(); + } + // TODO: explain partitions + return strBuilder.toString(); + } + + @Override + public void bindDataSink(Optional insertCtx) + throws AnalysisException { + + TIcebergTableSink tSink = new TIcebergTableSink(); + + Table icebergTable = targetTable.getIcebergTable(); + + tSink.setDbName(targetTable.getDbName()); + tSink.setTbName(targetTable.getName()); + + // schema + tSink.setSchemaJson(SchemaParser.toJson(icebergTable.schema())); + + // partition spec + if (icebergTable.spec().isPartitioned()) { + tSink.setPartitionSpecsJson(Maps.transformValues(icebergTable.specs(), PartitionSpecParser::toJson)); + tSink.setPartitionSpecId(icebergTable.spec().specId()); + } + + // sort order + if (icebergTable.sortOrder().isSorted()) { + SortOrder sortOrder = icebergTable.sortOrder(); + Set baseColumnFieldIds = icebergTable.schema().columns().stream() + .map(Types.NestedField::fieldId) + .collect(ImmutableSet.toImmutableSet()); + ImmutableList.Builder sortFields = ImmutableList.builder(); + for (SortField sortField : sortOrder.fields()) { + if (!sortField.transform().isIdentity()) { + continue; + } + if (!baseColumnFieldIds.contains(sortField.sourceId())) { + continue; + } + TSortField tSortField = new TSortField(); + tSortField.setSourceColumnId(sortField.sourceId()); + tSortField.setAscending(sortField.direction().equals(SortDirection.ASC)); + tSortField.setNullFirst(sortField.nullOrder().equals(NullOrder.NULLS_FIRST)); + sortFields.add(tSortField); + } + tSink.setSortFields(sortFields.build()); + } + + // file info + tSink.setFileFormat(getTFileFormatType(IcebergUtils.getFileFormat(icebergTable))); + tSink.setCompressionType(getTFileCompressType(IcebergUtils.getFileCompress(icebergTable))); + + // hadoop config + HashMap props = new HashMap<>(icebergTable.properties()); + Map catalogProps = targetTable.getCatalog().getProperties(); + props.putAll(catalogProps); + tSink.setHadoopConfig(props); + + // location + LocationPath locationPath = new LocationPath(IcebergUtils.dataLocation(icebergTable), catalogProps); + tSink.setOutputPath(locationPath.toStorageLocation().toString()); + tSink.setOriginalOutputPath(locationPath.toString()); + tSink.setFileType(locationPath.getTFileTypeForBE()); + + if (insertCtx.isPresent()) { + BaseExternalTableInsertCommandContext context = (BaseExternalTableInsertCommandContext) insertCtx.get(); + tSink.setOverwrite(context.isOverwrite()); + } + tDataSink = new TDataSink(TDataSinkType.ICEBERG_TABLE_SINK); + tDataSink.setIcebergTableSink(tSink); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index de4a9a80e4de6fa..1c272b1dbb1926b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -93,6 +93,7 @@ import org.apache.doris.thrift.TFileScanRange; import org.apache.doris.thrift.TFileScanRangeParams; import org.apache.doris.thrift.THivePartitionUpdate; +import org.apache.doris.thrift.TIcebergCommitData; import org.apache.doris.thrift.TNetworkAddress; import org.apache.doris.thrift.TPaloScanRange; import org.apache.doris.thrift.TPipelineFragmentParams; @@ -246,6 +247,9 @@ public class Coordinator implements CoordInterface { // Collect all hivePartitionUpdates obtained from be Consumer> hivePartitionUpdateFunc; + // Collect all icebergCommitData obtained from be + Consumer> icebergCommitDataFunc; + // Input parameter private long jobId = -1; // job which this task belongs to private TUniqueId queryId; @@ -2481,6 +2485,10 @@ public void setHivePartitionUpdateFunc(Consumer> hive this.hivePartitionUpdateFunc = hivePartitionUpdateFunc; } + public void setIcebergCommitDataFunc(Consumer> icebergCommitDataFunc) { + this.icebergCommitDataFunc = icebergCommitDataFunc; + } + // update job progress from BE public void updateFragmentExecStatus(TReportExecStatusParams params) { if (enablePipelineXEngine) { @@ -2531,6 +2539,9 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) { if (params.isSetHivePartitionUpdates() && hivePartitionUpdateFunc != null) { hivePartitionUpdateFunc.accept(params.getHivePartitionUpdates()); } + if (params.isSetIcebergCommitDatas() && icebergCommitDataFunc != null) { + icebergCommitDataFunc.accept(params.getIcebergCommitDatas()); + } Preconditions.checkArgument(params.isSetDetailedReport()); if (ctx.done) { @@ -2596,6 +2607,9 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) { if (params.isSetHivePartitionUpdates() && hivePartitionUpdateFunc != null) { hivePartitionUpdateFunc.accept(params.getHivePartitionUpdates()); } + if (params.isSetIcebergCommitDatas() && icebergCommitDataFunc != null) { + icebergCommitDataFunc.accept(params.getIcebergCommitDatas()); + } if (LOG.isDebugEnabled()) { LOG.debug("Query {} instance {} is marked done", DebugUtil.printId(queryId), DebugUtil.printId(params.getFragmentInstanceId())); @@ -2668,6 +2682,9 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) { if (params.isSetHivePartitionUpdates() && hivePartitionUpdateFunc != null) { hivePartitionUpdateFunc.accept(params.getHivePartitionUpdates()); } + if (params.isSetIcebergCommitDatas() && icebergCommitDataFunc != null) { + icebergCommitDataFunc.accept(params.getIcebergCommitDatas()); + } instancesDoneLatch.markedCountDown(params.getFragmentInstanceId(), -1L); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java new file mode 100644 index 000000000000000..3d6486f939169e4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/IcebergTransactionManager.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.transaction; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.iceberg.IcebergMetadataOps; +import org.apache.doris.datasource.iceberg.IcebergTransaction; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +public class IcebergTransactionManager implements TransactionManager { + + private final Map transactions = new ConcurrentHashMap<>(); + private final IcebergMetadataOps ops; + + public IcebergTransactionManager(IcebergMetadataOps ops) { + this.ops = ops; + } + + @Override + public long begin() { + long id = Env.getCurrentEnv().getNextId(); + IcebergTransaction icebergTransaction = new IcebergTransaction(ops); + transactions.put(id, icebergTransaction); + return id; + } + + @Override + public void commit(long id) throws UserException { + getTransactionWithException(id).commit(); + transactions.remove(id); + } + + @Override + public void rollback(long id) { + getTransactionWithException(id).rollback(); + transactions.remove(id); + } + + @Override + public Transaction getTransaction(long id) { + return getTransactionWithException(id); + } + + public Transaction getTransactionWithException(long id) { + Transaction icebergTransaction = transactions.get(id); + if (icebergTransaction == null) { + throw new RuntimeException("Can't find transaction for " + id); + } + return icebergTransaction; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java index 394494a129d3b55..b8898d9b279e32b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/transaction/TransactionManagerFactory.java @@ -18,6 +18,7 @@ package org.apache.doris.transaction; import org.apache.doris.datasource.hive.HiveMetadataOps; +import org.apache.doris.datasource.iceberg.IcebergMetadataOps; import org.apache.doris.fs.FileSystemProvider; import java.util.concurrent.Executor; @@ -28,4 +29,8 @@ public static TransactionManager createHiveTransactionManager(HiveMetadataOps op FileSystemProvider fileSystemProvider, Executor fileSystemExecutor) { return new HiveTransactionManager(ops, fileSystemProvider, fileSystemExecutor); } + + public static TransactionManager createIcebergTransactionManager(IcebergMetadataOps ops) { + return new IcebergTransactionManager(ops); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java index dc35d38af68241d..48ef7c1f67a9147 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveDDLAndDMLPlanTest.java @@ -82,7 +82,7 @@ public class HiveDDLAndDMLPlanTest extends TestWithFeService { private List checkedHiveCols; private final Set createdDbs = new HashSet<>(); - private final Set createdTables = new HashSet<>(); + private final Set
createdTables = new HashSet<>(); @Override protected void runBeforeAll() throws Exception { @@ -142,7 +142,12 @@ public Database getDatabase(String dbName) { @Mock public boolean tableExists(String dbName, String tblName) { - return createdTables.contains(tblName); + for (Table table : createdTables) { + if (table.getDbName().equals(dbName) && table.getTableName().equals(tblName)) { + return true; + } + } + return false; } @Mock @@ -154,7 +159,7 @@ public List getAllDatabases() { public void createTable(TableMetadata tbl, boolean ignoreIfExists) { if (tbl instanceof HiveTableMetadata) { Table table = HiveUtil.toHiveTable((HiveTableMetadata) tbl); - createdTables.add(table.getTableName()); + createdTables.add(table); if (checkedHiveCols == null) { // if checkedHiveCols is null, skip column check return; @@ -169,6 +174,16 @@ public void createTable(TableMetadata tbl, boolean ignoreIfExists) { } } } + + @Mock + public Table getTable(String dbName, String tblName) { + for (Table createdTable : createdTables) { + if (createdTable.getDbName().equals(dbName) && createdTable.getTableName().equals(tblName)) { + return createdTable; + } + } + return null; + } }; CreateDbStmt createDbStmt = new CreateDbStmt(true, new DbName("hive", mockedDbName), dbProps); Env.getCurrentEnv().createDb(createDbStmt); @@ -213,8 +228,10 @@ public ExternalDatabase getDbNullable(String dbName) { // mock after ThriftHMSCachedClient is mocked @Mock HMSExternalTable getTableNullable(String tableName) { - if (createdTables.contains(tableName)) { - return new HMSExternalTable(0, tableName, mockedDbName, hmsExternalCatalog); + for (Table table : createdTables) { + if (table.getTableName().equals(tableName)) { + return new HMSExternalTable(0, tableName, mockedDbName, hmsExternalCatalog); + } } return null; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java index e441262f12e2dd3..8644ee0ebdc8ec3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HmsCommitTest.java @@ -178,7 +178,6 @@ public void testAppendPartitionForUnPartitionedTable() throws IOException { assertNumRows(3, table); genQueryID(); - System.out.println(DebugUtil.printId(connectContext.queryId())); List pus2 = new ArrayList<>(); pus2.add(createRandomAppend(null)); pus2.add(createRandomAppend(null)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java new file mode 100644 index 000000000000000..10de5427902c163 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -0,0 +1,328 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.common.UserException; +import org.apache.doris.thrift.TFileContent; +import org.apache.doris.thrift.TIcebergCommitData; + +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.CatalogProperties; +import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.catalog.Namespace; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.expressions.Expression; +import org.apache.iceberg.expressions.Expressions; +import org.apache.iceberg.expressions.UnboundPredicate; +import org.apache.iceberg.hadoop.HadoopCatalog; +import org.apache.iceberg.io.CloseableIterable; +import org.apache.iceberg.transforms.Transform; +import org.apache.iceberg.transforms.Transforms; +import org.apache.iceberg.types.Types; +import org.apache.iceberg.util.DateTimeUtil; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +public class IcebergTransactionTest { + + public static String dbName = "db3"; + public static String tbWithPartition = "tbWithPartition"; + public static String tbWithoutPartition = "tbWithoutPartition"; + public static IcebergMetadataOps ops; + public static Schema schema; + + @BeforeClass + public static void beforeClass() throws IOException { + createCatalog(); + createTable(); + } + + public static void createCatalog() throws IOException { + Path warehousePath = Files.createTempDirectory("test_warehouse_"); + String warehouse = "file://" + warehousePath.toAbsolutePath() + "/"; + HadoopCatalog hadoopCatalog = new HadoopCatalog(); + Map props = new HashMap<>(); + props.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse); + hadoopCatalog.setConf(new Configuration()); + hadoopCatalog.initialize("df", props); + ops = new IcebergMetadataOps(null, hadoopCatalog); + } + + public static void createTable() throws IOException { + HadoopCatalog icebergCatalog = (HadoopCatalog) ops.getCatalog(); + icebergCatalog.createNamespace(Namespace.of(dbName)); + schema = new Schema( + Types.NestedField.required(11, "ts1", Types.TimestampType.withoutZone()), + Types.NestedField.required(12, "ts2", Types.TimestampType.withoutZone()), + Types.NestedField.required(13, "ts3", Types.TimestampType.withoutZone()), + Types.NestedField.required(14, "ts4", Types.TimestampType.withoutZone()), + Types.NestedField.required(15, "dt1", Types.DateType.get()), + Types.NestedField.required(16, "dt2", Types.DateType.get()), + Types.NestedField.required(17, "dt3", Types.DateType.get()), + Types.NestedField.required(18, "dt4", Types.DateType.get()), + Types.NestedField.required(19, "str1", Types.StringType.get()), + Types.NestedField.required(20, "str2", Types.StringType.get()), + Types.NestedField.required(21, "int1", Types.IntegerType.get()), + Types.NestedField.required(22, "int2", Types.IntegerType.get()) + ); + + PartitionSpec partitionSpec = PartitionSpec.builderFor(schema) + .year("ts1") + .month("ts2") + .day("ts3") + .hour("ts4") + .year("dt1") + .month("dt2") + .day("dt3") + .identity("dt4") + .identity("str1") + .truncate("str2", 10) + .bucket("int1", 2) + .build(); + icebergCatalog.createTable(TableIdentifier.of(dbName, tbWithPartition), schema, partitionSpec); + icebergCatalog.createTable(TableIdentifier.of(dbName, tbWithoutPartition), schema); + } + + public List createPartitionValues() { + + Instant instant = Instant.parse("2024-12-11T12:34:56.123456Z"); + long ts = DateTimeUtil.microsFromInstant(instant); + int dt = DateTimeUtil.daysFromInstant(instant); + + List partitionValues = new ArrayList<>(); + + // reference: org.apache.iceberg.transforms.Timestamps + partitionValues.add(Integer.valueOf(DateTimeUtil.microsToYears(ts)).toString()); + partitionValues.add(Integer.valueOf(DateTimeUtil.microsToMonths(ts)).toString()); + partitionValues.add("2024-12-11"); + partitionValues.add(Integer.valueOf(DateTimeUtil.microsToHours(ts)).toString()); + + // reference: org.apache.iceberg.transforms.Dates + partitionValues.add(Integer.valueOf(DateTimeUtil.daysToYears(dt)).toString()); + partitionValues.add(Integer.valueOf(DateTimeUtil.daysToMonths(dt)).toString()); + partitionValues.add("2024-12-11"); + + // identity dt4 + partitionValues.add("2024-12-11"); + // identity str1 + partitionValues.add("2024-12-11"); + // truncate str2 + partitionValues.add("2024-12-11"); + // bucket int1 + partitionValues.add("1"); + + return partitionValues; + } + + @Test + public void testPartitionedTable() throws UserException { + List partitionValues = createPartitionValues(); + + List ctdList = new ArrayList<>(); + TIcebergCommitData ctd1 = new TIcebergCommitData(); + ctd1.setFilePath("f1.parquet"); + ctd1.setPartitionValues(partitionValues); + ctd1.setFileContent(TFileContent.DATA); + ctd1.setRowCount(2); + ctd1.setFileSize(2); + + TIcebergCommitData ctd2 = new TIcebergCommitData(); + ctd2.setFilePath("f2.parquet"); + ctd2.setPartitionValues(partitionValues); + ctd2.setFileContent(TFileContent.DATA); + ctd2.setRowCount(4); + ctd2.setFileSize(4); + + ctdList.add(ctd1); + ctdList.add(ctd2); + + IcebergTransaction txn = getTxn(); + txn.updateIcebergCommitData(ctdList); + txn.beginInsert(dbName, tbWithPartition); + txn.finishInsert(); + txn.commit(); + Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithPartition)); + checkSnapshotProperties(table.currentSnapshot().summary(), "6", "2", "6"); + + checkPushDownByPartitionForTs(table, "ts1"); + checkPushDownByPartitionForTs(table, "ts2"); + checkPushDownByPartitionForTs(table, "ts3"); + checkPushDownByPartitionForTs(table, "ts4"); + + checkPushDownByPartitionForDt(table, "dt1"); + checkPushDownByPartitionForDt(table, "dt2"); + checkPushDownByPartitionForDt(table, "dt3"); + checkPushDownByPartitionForDt(table, "dt4"); + + checkPushDownByPartitionForString(table, "str1"); + checkPushDownByPartitionForString(table, "str2"); + + checkPushDownByPartitionForBucketInt(table, "int1"); + } + + public void checkPushDownByPartitionForBucketInt(Table table, String column) { + // (BucketUtil.hash(15) & Integer.MAX_VALUE) % 2 = 0 + Integer i1 = 15; + + UnboundPredicate lessThan = Expressions.lessThan(column, i1); + checkPushDownByPartition(table, lessThan, 2); + // can only filter this case + UnboundPredicate equal = Expressions.equal(column, i1); + checkPushDownByPartition(table, equal, 0); + UnboundPredicate greaterThan = Expressions.greaterThan(column, i1); + checkPushDownByPartition(table, greaterThan, 2); + + // (BucketUtil.hash(25) & Integer.MAX_VALUE) % 2 = 1 + Integer i2 = 25; + + UnboundPredicate lessThan2 = Expressions.lessThan(column, i2); + checkPushDownByPartition(table, lessThan2, 2); + UnboundPredicate equal2 = Expressions.equal(column, i2); + checkPushDownByPartition(table, equal2, 2); + UnboundPredicate greaterThan2 = Expressions.greaterThan(column, i2); + checkPushDownByPartition(table, greaterThan2, 2); + } + + public void checkPushDownByPartitionForString(Table table, String column) { + // Since the string used to create the partition is in date format, the date check can be reused directly + checkPushDownByPartitionForDt(table, column); + } + + public void checkPushDownByPartitionForTs(Table table, String column) { + String lessTs = "2023-12-11T12:34:56.123456"; + String eqTs = "2024-12-11T12:34:56.123456"; + String greaterTs = "2025-12-11T12:34:56.123456"; + + UnboundPredicate lessThan = Expressions.lessThan(column, lessTs); + checkPushDownByPartition(table, lessThan, 0); + UnboundPredicate equal = Expressions.equal(column, eqTs); + checkPushDownByPartition(table, equal, 2); + UnboundPredicate greaterThan = Expressions.greaterThan(column, greaterTs); + checkPushDownByPartition(table, greaterThan, 0); + } + + public void checkPushDownByPartitionForDt(Table table, String column) { + String less = "2023-12-11"; + String eq = "2024-12-11"; + String greater = "2025-12-11"; + + UnboundPredicate lessThan = Expressions.lessThan(column, less); + checkPushDownByPartition(table, lessThan, 0); + UnboundPredicate equal = Expressions.equal(column, eq); + checkPushDownByPartition(table, equal, 2); + UnboundPredicate greaterThan = Expressions.greaterThan(column, greater); + checkPushDownByPartition(table, greaterThan, 0); + } + + public void checkPushDownByPartition(Table table, Expression expr, Integer expectFiles) { + CloseableIterable fileScanTasks = table.newScan().filter(expr).planFiles(); + AtomicReference cnt = new AtomicReference<>(0); + fileScanTasks.forEach(notUse -> cnt.updateAndGet(v -> v + 1)); + Assert.assertEquals(expectFiles, cnt.get()); + } + + @Test + public void testUnPartitionedTable() throws UserException { + ArrayList ctdList = new ArrayList<>(); + TIcebergCommitData ctd1 = new TIcebergCommitData(); + ctd1.setFilePath("f1.parquet"); + ctd1.setFileContent(TFileContent.DATA); + ctd1.setRowCount(2); + ctd1.setFileSize(2); + + TIcebergCommitData ctd2 = new TIcebergCommitData(); + ctd2.setFilePath("f1.parquet"); + ctd2.setFileContent(TFileContent.DATA); + ctd2.setRowCount(4); + ctd2.setFileSize(4); + + ctdList.add(ctd1); + ctdList.add(ctd2); + + IcebergTransaction txn = getTxn(); + txn.updateIcebergCommitData(ctdList); + txn.beginInsert(dbName, tbWithoutPartition); + txn.finishInsert(); + txn.commit(); + + Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + checkSnapshotProperties(table.currentSnapshot().summary(), "6", "2", "6"); + } + + public void checkSnapshotProperties(Map props, + String addRecords, + String addFileCnt, + String addFileSize) { + Assert.assertEquals(addRecords, props.get("added-records")); + Assert.assertEquals(addFileCnt, props.get("added-data-files")); + Assert.assertEquals(addFileSize, props.get("added-files-size")); + } + + public String numToYear(Integer num) { + Transform year = Transforms.year(); + return year.toHumanString(Types.IntegerType.get(), num); + } + + public String numToMonth(Integer num) { + Transform month = Transforms.month(); + return month.toHumanString(Types.IntegerType.get(), num); + } + + public String numToDay(Integer num) { + Transform day = Transforms.day(); + return day.toHumanString(Types.IntegerType.get(), num); + } + + public String numToHour(Integer num) { + Transform hour = Transforms.hour(); + return hour.toHumanString(Types.IntegerType.get(), num); + } + + @Test + public void testTransform() { + Instant instant = Instant.parse("2024-12-11T12:34:56.123456Z"); + long ts = DateTimeUtil.microsFromInstant(instant); + Assert.assertEquals("2024", numToYear(DateTimeUtil.microsToYears(ts))); + Assert.assertEquals("2024-12", numToMonth(DateTimeUtil.microsToMonths(ts))); + Assert.assertEquals("2024-12-11", numToDay(DateTimeUtil.microsToDays(ts))); + Assert.assertEquals("2024-12-11-12", numToHour(DateTimeUtil.microsToHours(ts))); + + int dt = DateTimeUtil.daysFromInstant(instant); + Assert.assertEquals("2024", numToYear(DateTimeUtil.daysToYears(dt))); + Assert.assertEquals("2024-12", numToMonth(DateTimeUtil.daysToMonths(dt))); + Assert.assertEquals("2024-12-11", numToDay(dt)); + } + + public IcebergTransaction getTxn() { + return new IcebergTransaction(ops); + } +} diff --git a/gensrc/thrift/DataSinks.thrift b/gensrc/thrift/DataSinks.thrift index 834c9025cfdc1db..e613ef788b15cc5 100644 --- a/gensrc/thrift/DataSinks.thrift +++ b/gensrc/thrift/DataSinks.thrift @@ -39,6 +39,7 @@ enum TDataSinkType { GROUP_COMMIT_OLAP_TABLE_SINK, // deprecated GROUP_COMMIT_BLOCK_SINK, HIVE_TABLE_SINK, + ICEBERG_TABLE_SINK, } enum TResultSinkType { @@ -355,6 +356,43 @@ struct THivePartitionUpdate { 7: optional list s3_mpu_pending_uploads } +enum TFileContent { + DATA = 0, + POSITION_DELETES = 1, + EQUALITY_DELETES = 2 +} + +struct TIcebergCommitData { + 1: optional string file_path + 2: optional i64 row_count + 3: optional i64 file_size + 4: optional TFileContent file_content + 5: optional list partition_values + 6: optional list referenced_data_files +} + +struct TSortField { + 1: optional i32 source_column_id + 2: optional bool ascending + 3: optional bool null_first +} + +struct TIcebergTableSink { + 1: optional string db_name + 2: optional string tb_name + 3: optional string schema_json + 4: optional map partition_specs_json + 5: optional i32 partition_spec_id + 6: optional list sort_fields + 7: optional PlanNodes.TFileFormatType file_format + 8: optional string output_path + 9: optional map hadoop_config + 10: optional bool overwrite + 11: optional Types.TFileType file_type + 12: optional string original_output_path + 13: optional PlanNodes.TFileCompressType compression_type +} + struct TDataSink { 1: required TDataSinkType type 2: optional TDataStreamSink stream_sink @@ -368,4 +406,5 @@ struct TDataSink { 11: optional TJdbcTableSink jdbc_table_sink 12: optional TMultiCastDataStreamSink multi_cast_stream_sink 13: optional THiveTableSink hive_table_sink + 14: optional TIcebergTableSink iceberg_table_sink } diff --git a/gensrc/thrift/FrontendService.thrift b/gensrc/thrift/FrontendService.thrift index 69045525a5ac97d..aeabd2a22ab6e55 100644 --- a/gensrc/thrift/FrontendService.thrift +++ b/gensrc/thrift/FrontendService.thrift @@ -357,11 +357,11 @@ struct TListTableStatusResult { struct TTableMetadataNameIds { 1: optional string name - 2: optional i64 id + 2: optional i64 id } struct TListTableMetadataNameIdsResult { - 1: optional list tables + 1: optional list tables } // getTableNames returns a list of unqualified table names @@ -484,6 +484,8 @@ struct TReportExecStatusParams { 25: optional TReportWorkloadRuntimeStatusParams report_workload_runtime_status 26: optional list hive_partition_updates + + 27: optional list iceberg_commit_datas } struct TFeResult { From 37f1bf317cebc1cef749e95b07a87a8efd92fcf6 Mon Sep 17 00:00:00 2001 From: Jibing-Li <64681310+Jibing-Li@users.noreply.github.com> Date: Tue, 21 May 2024 22:58:12 +0800 Subject: [PATCH 085/111] [fix](statistics)Disable fetch min/max column stats through HMS, because the value may inaccurate and misleading. (#35124) (#35145) backport #35124 --- .../datasource/hive/HMSExternalTable.java | 65 +++---------------- .../hive/test_hive_statistics_from_hms.groovy | 52 ++++++++------- 2 files changed, 38 insertions(+), 79 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index e5624fb58b5c326..4d3f963aa55b479 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -69,7 +69,6 @@ import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.DateColumnStatsData; -import org.apache.hadoop.hive.metastore.api.Decimal; import org.apache.hadoop.hive.metastore.api.DecimalColumnStatsData; import org.apache.hadoop.hive.metastore.api.DoubleColumnStatsData; import org.apache.hadoop.hive.metastore.api.FieldSchema; @@ -82,9 +81,6 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; -import java.math.BigDecimal; -import java.math.BigInteger; -import java.time.LocalDate; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -697,8 +693,11 @@ private Optional getHiveColumnStats(String colName) { return Optional.empty(); } Map parameters = remoteTable.getParameters(); + if (!parameters.containsKey(NUM_ROWS) || Long.parseLong(parameters.get(NUM_ROWS)) == 0) { + return Optional.empty(); + } ColumnStatisticBuilder columnStatisticBuilder = new ColumnStatisticBuilder(); - double count = parameters.containsKey(NUM_ROWS) ? Double.parseDouble(parameters.get(NUM_ROWS)) : 0; + long count = Long.parseLong(parameters.get(NUM_ROWS)); columnStatisticBuilder.setCount(count); // The tableStats length is at most 1. for (ColumnStatisticsObj tableStat : tableStats) { @@ -719,12 +718,10 @@ private Optional getHiveColumnStats(String colName) { return Optional.of(columnStatisticBuilder.build()); } - private void setStatData(Column col, ColumnStatisticsData data, ColumnStatisticBuilder builder, double count) + private void setStatData(Column col, ColumnStatisticsData data, ColumnStatisticBuilder builder, long count) throws AnalysisException { long ndv = 0; long nulls = 0; - String min = ""; - String max = ""; double colSize = 0; if (!data.isSetStringStats()) { colSize = count * col.getType().getSlotSize(); @@ -734,8 +731,6 @@ private void setStatData(Column col, ColumnStatisticsData data, ColumnStatisticB LongColumnStatsData longStats = data.getLongStats(); ndv = longStats.getNumDVs(); nulls = longStats.getNumNulls(); - min = String.valueOf(longStats.getLowValue()); - max = String.valueOf(longStats.getHighValue()); } else if (data.isSetStringStats()) { StringColumnStatsData stringStats = data.getStringStats(); ndv = stringStats.getNumDVs(); @@ -746,67 +741,23 @@ private void setStatData(Column col, ColumnStatisticsData data, ColumnStatisticB DecimalColumnStatsData decimalStats = data.getDecimalStats(); ndv = decimalStats.getNumDVs(); nulls = decimalStats.getNumNulls(); - if (decimalStats.isSetLowValue()) { - Decimal lowValue = decimalStats.getLowValue(); - if (lowValue != null) { - BigDecimal lowDecimal = new BigDecimal(new BigInteger(lowValue.getUnscaled()), lowValue.getScale()); - min = lowDecimal.toString(); - } - } - if (decimalStats.isSetHighValue()) { - Decimal highValue = decimalStats.getHighValue(); - if (highValue != null) { - BigDecimal highDecimal = - new BigDecimal(new BigInteger(highValue.getUnscaled()), highValue.getScale()); - max = highDecimal.toString(); - } - } } else if (data.isSetDoubleStats()) { DoubleColumnStatsData doubleStats = data.getDoubleStats(); ndv = doubleStats.getNumDVs(); nulls = doubleStats.getNumNulls(); - min = String.valueOf(doubleStats.getLowValue()); - max = String.valueOf(doubleStats.getHighValue()); } else if (data.isSetDateStats()) { DateColumnStatsData dateStats = data.getDateStats(); ndv = dateStats.getNumDVs(); nulls = dateStats.getNumNulls(); - if (dateStats.isSetLowValue()) { - org.apache.hadoop.hive.metastore.api.Date lowValue = dateStats.getLowValue(); - if (lowValue != null) { - LocalDate lowDate = LocalDate.ofEpochDay(lowValue.getDaysSinceEpoch()); - min = lowDate.toString(); - } - } - if (dateStats.isSetHighValue()) { - org.apache.hadoop.hive.metastore.api.Date highValue = dateStats.getHighValue(); - if (highValue != null) { - LocalDate highDate = LocalDate.ofEpochDay(highValue.getDaysSinceEpoch()); - max = highDate.toString(); - } - } } else { - if (LOG.isDebugEnabled()) { - LOG.debug(String.format("Not suitable data type for column %s", col.getName())); - } - throw new RuntimeException("Not supported data type."); + LOG.warn(String.format("Not suitable data type for column %s", col.getName())); } builder.setNdv(ndv); builder.setNumNulls(nulls); builder.setDataSize(colSize); builder.setAvgSizeByte(colSize / count); - if (!min.equals("")) { - builder.setMinValue(StatisticsUtil.convertToDouble(col.getType(), min)); - builder.setMinExpr(StatisticsUtil.readableValue(col.getType(), min)); - } else { - builder.setMinValue(Double.MIN_VALUE); - } - if (!max.equals("")) { - builder.setMaxValue(StatisticsUtil.convertToDouble(col.getType(), max)); - builder.setMaxExpr(StatisticsUtil.readableValue(col.getType(), max)); - } else { - builder.setMaxValue(Double.MAX_VALUE); - } + builder.setMinValue(Double.NEGATIVE_INFINITY); + builder.setMaxValue(Double.POSITIVE_INFINITY); } public void setEventUpdateTime(long updateTime) { diff --git a/regression-test/suites/external_table_p2/hive/test_hive_statistics_from_hms.groovy b/regression-test/suites/external_table_p2/hive/test_hive_statistics_from_hms.groovy index c3c671bb0353408..3a067fa42f9d0e2 100644 --- a/regression-test/suites/external_table_p2/hive/test_hive_statistics_from_hms.groovy +++ b/regression-test/suites/external_table_p2/hive/test_hive_statistics_from_hms.groovy @@ -55,8 +55,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "2.400486E7") assertTrue(result[0][6] == "4.0") - assertTrue(result[0][7] == "\'1992-01-04\'") - assertTrue(result[0][8] == "\'1998-12-31\'") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_tax)""" assertTrue(result.size() == 1) @@ -66,8 +66,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "4.800972E7") assertTrue(result[0][6] == "8.0") - assertTrue(result[0][7] == "0") - assertTrue(result[0][8] == "0.08") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_shipmode)""" assertTrue(result.size() == 1) @@ -77,6 +77,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "2.5717007E7") assertTrue(result[0][6] == "4.285300060071169") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_suppkey)""" assertTrue(result.size() == 1) @@ -86,8 +88,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "2.400486E7") assertTrue(result[0][6] == "4.0") - assertTrue(result[0][7] == "1") - assertTrue(result[0][8] == "7") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_shipdate)""" assertTrue(result.size() == 1) @@ -97,8 +99,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "2.400486E7") assertTrue(result[0][6] == "4.0") - assertTrue(result[0][7] == "\'1992-01-02\'") - assertTrue(result[0][8] == "\'1998-12-01\'") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_commitdate)""" assertTrue(result.size() == 1) @@ -108,8 +110,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "2.400486E7") assertTrue(result[0][6] == "4.0") - assertTrue(result[0][7] == "\'1992-01-31\'") - assertTrue(result[0][8] == "\'1998-10-31\'") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_partkey)""" assertTrue(result.size() == 1) @@ -119,8 +121,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "2.400486E7") assertTrue(result[0][6] == "4.0") - assertTrue(result[0][7] == "1") - assertTrue(result[0][8] == "10000") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_orderkey)""" assertTrue(result.size() == 1) @@ -130,8 +132,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "2.400486E7") assertTrue(result[0][6] == "4.0") - assertTrue(result[0][7] == "1") - assertTrue(result[0][8] == "6000000") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_quantity)""" assertTrue(result.size() == 1) @@ -141,8 +143,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "4.800972E7") assertTrue(result[0][6] == "8.0") - assertTrue(result[0][7] == "1") - assertTrue(result[0][8] == "50") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_linestatus)""" assertTrue(result.size() == 1) @@ -152,6 +154,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "6001215.0") assertTrue(result[0][6] == "1.0") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_comment)""" assertTrue(result.size() == 1) @@ -161,6 +165,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "1.5899739E8") assertTrue(result[0][6] == "26.494199924515286") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_extendedprice)""" assertTrue(result.size() == 1) @@ -170,8 +176,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "4.800972E7") assertTrue(result[0][6] == "8.0") - assertTrue(result[0][7] == "901") - assertTrue(result[0][8] == "104949.5") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_linenumber)""" assertTrue(result.size() == 1) @@ -181,8 +187,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "2.400486E7") assertTrue(result[0][6] == "4.0") - assertTrue(result[0][7] == "1") - assertTrue(result[0][8] == "200000") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_discount)""" assertTrue(result.size() == 1) @@ -192,8 +198,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "4.800972E7") assertTrue(result[0][6] == "8.0") - assertTrue(result[0][7] == "0") - assertTrue(result[0][8] == "0.1") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") result = sql """show column cached stats lineitem (l_shipinstruct)""" assertTrue(result.size() == 1) @@ -203,6 +209,8 @@ suite("test_hive_statistics_from_hms", "p2,external,hive,external_remote,externa assertTrue(result[0][4] == "0.0") assertTrue(result[0][5] == "7.2006178E7") assertTrue(result[0][6] == "11.998599950176756") + assertTrue(result[0][7] == "N/A") + assertTrue(result[0][8] == "N/A") for (int i = 0; i < 10; i++) { result = sql """show table stats lineitem""" From 9fc38fb011a548ec4ff52d7b663ca930ea5e036a Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Tue, 21 May 2024 23:16:55 +0800 Subject: [PATCH 086/111] [branch-2.1](routine-load) fix routine load case error (#35161) * fix routine load case error * update --- .../routine_load/test_routine_load.groovy | 133 +++--------------- 1 file changed, 20 insertions(+), 113 deletions(-) diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy index 03a3d40b786c6d0..c71557f36c7e9c3 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy @@ -278,10 +278,12 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_send_batch_parallelism "select * from ${tableName1} order by k00,k01,k02" + if (i < 5) { + def res = sql "select count(*) from ${tableName1}" + assertEquals(20, res[0][0]) } else { - qt_sql_send_batch_parallelism "select * from ${tableName1} order by k00,k01,k02" + def res = sql "select count(*) from ${tableName1}" + assertEquals(18, res[0][0]) } sql "stop routine load for ${jobs[i]}" @@ -365,10 +367,12 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_desired_concurrent_number "select * from ${tableName1} order by k00,k01,k02" + if (i < 5) { + def res = sql "select count(*) from ${tableName1}" + assertEquals(20, res[0][0]) } else { - qt_sql_desired_concurrent_number "select * from ${tableName1} order by k00,k01,k02" + def res = sql "select count(*) from ${tableName1}" + assertEquals(18, res[0][0]) } sql "stop routine load for ${jobs[i]}" @@ -611,12 +615,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_num_as_string "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_num_as_string "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -692,10 +690,12 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_exec_mem_limit "select * from ${tableName1} order by k00,k01,k02" + if (i < 5) { + def res = sql "select count(*) from ${tableName1}" + assertEquals(20, res[0][0]) } else { - qt_sql_exec_mem_limit "select * from ${tableName1} order by k00,k01,k02" + def res = sql "select count(*) from ${tableName1}" + assertEquals(18, res[0][0]) } sql "stop routine load for ${jobs[i]}" @@ -814,10 +814,12 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_timezone_shanghai "select * from ${tableName1} order by k00,k01,k02" + if (i < 5) { + def res = sql "select count(*) from ${tableName1}" + assertEquals(20, res[0][0]) } else { - qt_sql_timezone_shanghai "select * from ${tableName1} order by k00,k01,k02" + def res = sql "select count(*) from ${tableName1}" + assertEquals(18, res[0][0]) } sql "stop routine load for ${jobs[i]}" @@ -892,12 +894,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_strict_mode "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_strict_mode "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -973,12 +969,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_max_error_number "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_max_error_number "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -1055,12 +1045,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_max_filter_ratio "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_max_filter_ratio "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -1114,11 +1098,6 @@ suite("test_routine_load","p0") { } def tableName1 = "routine_load_" + tableName - if (i <= 3) { - qt_sql_load_to_single_tablet "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_load_to_single_tablet "select * from ${tableName1} order by k00,k01,k02" - } sql "stop routine load for ${jobs[i]}" i++ @@ -1172,12 +1151,6 @@ suite("test_routine_load","p0") { sleep(10000) def tableName1 = "routine_load_" + tableName - if (i <= 3) { - qt_sql_column_separator "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_column_separator "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -1251,11 +1224,6 @@ suite("test_routine_load","p0") { sleep(5000) count++ } - if (i <= 3) { - qt_sql_json "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_json "select * from ${tableName1} order by k00,k01,k02" - } sql "stop routine load for ${jobs[i]}" i++ @@ -1372,11 +1340,6 @@ suite("test_routine_load","p0") { sleep(5000) count++ } - if (i <= 3) { - qt_sql_json_jsonpath "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_json_jsonpath "select * from ${tableName1} order by k00,k01,k02" - } sql "stop routine load for ${jobs[i]}" i++ @@ -1469,12 +1432,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_disable_simdjson_reader "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_disable_simdjson_reader "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -1630,12 +1587,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_multi_table_one_data "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_multi_table_one_data "select * from ${tableName1} order by k00,k01,k02" - } - i++ } } finally { @@ -1709,12 +1660,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_sql_multi_table "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_multi_table "select * from ${tableName1} order by k00,k01,k02" - } - i++ } } finally { @@ -1791,12 +1736,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_show_command "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_show_command "select * from ${tableName1} order by k00,k01,k02" - } - def res = sql "SHOW ROUTINE LOAD TASK WHERE JobName = \"${jobs[i]}\"" log.info("routine load task DataSource: ${res[0][8].toString()}".toString()) def json = parseJson(res[0][8]) @@ -1878,12 +1817,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_pause_and_resume_command "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_pause_and_resume_command "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -1994,12 +1927,6 @@ suite("test_routine_load","p0") { count++ } - if (i <= 3) { - qt_update_command "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_update_command "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -2075,13 +2002,6 @@ suite("test_routine_load","p0") { sleep(5000) count++ } - - if (i <= 3) { - qt_sql_preceding_filter "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_preceding_filter "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -2156,13 +2076,6 @@ suite("test_routine_load","p0") { sleep(5000) count++ } - - if (i <= 3) { - qt_sql_where "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_where "select * from ${tableName1} order by k00,k01,k02" - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -2246,12 +2159,6 @@ suite("test_routine_load","p0") { sleep(5000) count++ } - - if (i <= 3) { - qt_sql_delete "select * from ${tableName1} order by k00,k01,k02" - } else { - qt_sql_delete "select * from ${tableName1} order by k00,k01,k02" - } sql "stop routine load for ${jobs[i]}" i++ From b11f2ad9f0de61ff76e11bdfa2788c56af4624ed Mon Sep 17 00:00:00 2001 From: Yongqiang YANG <98214048+dataroaring@users.noreply.github.com> Date: Mon, 20 May 2024 19:46:23 +0800 Subject: [PATCH 087/111] [fix](cases) fix broker_load & schema_change & table_property cases (#35061) 1. log more info to investigate problem. 2. use def in groovy to avoid gloabl namespace --- .../suites/alter_p2/test_alter_table_property.groovy | 1 + .../suites/load_p2/broker_load/test_broker_load.groovy | 8 +++++--- .../suites/schema_change_p2/test_schema_change.groovy | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/regression-test/suites/alter_p2/test_alter_table_property.groovy b/regression-test/suites/alter_p2/test_alter_table_property.groovy index f3d300aafc27d73..42ee9959d28a9f8 100644 --- a/regression-test/suites/alter_p2/test_alter_table_property.groovy +++ b/regression-test/suites/alter_p2/test_alter_table_property.groovy @@ -39,6 +39,7 @@ suite ("test_alter_table_property") { def queryReplicaCount = { partitionName -> def result = sql "SHOW REPLICA DISTRIBUTION FROM ${tableName} PARTITION ${partitionName}" + logger.info("${result}") int sum = 0 for (row in result) { sum += row[1].toInteger() diff --git a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy index b0e172818db0fea..75aa0a9aa9bc36e 100644 --- a/regression-test/suites/load_p2/broker_load/test_broker_load.groovy +++ b/regression-test/suites/load_p2/broker_load/test_broker_load.groovy @@ -323,7 +323,8 @@ suite("test_broker_load_p2", "p2") { for (String label in uuids) { def max_try_milli_secs = 600000 while (max_try_milli_secs > 0) { - String[][] result = sql """ show load where label="$label" order by createtime desc limit 1; """ + def String[][] result = sql """ show load where label="$label" order by createtime desc limit 1; """ + logger.info("Load status: " + result[0][2] + ", label: $label") if (result[0][2].equals("FINISHED")) { logger.info("Load FINISHED " + label) assertTrue(result[0][6].contains(task_info[i])) @@ -347,7 +348,7 @@ suite("test_broker_load_p2", "p2") { def orc_expect_result = """[[20, 15901, 6025915247311731176, 1373910657, 8863282788606566657], [38, 15901, -9154375582268094750, 1373853561, 4923892366467329038], [38, 15901, -9154375582268094750, 1373853561, 8447995939656287502], [38, 15901, -9154375582268094750, 1373853565, 7451966001310881759], [38, 15901, -9154375582268094750, 1373853565, 7746521994248163870], [38, 15901, -9154375582268094750, 1373853577, 6795654975682437824], [38, 15901, -9154375582268094750, 1373853577, 9009208035649338594], [38, 15901, -9154375582268094750, 1373853608, 6374361939566017108], [38, 15901, -9154375582268094750, 1373853608, 7387298457456465364], [38, 15901, -9154375582268094750, 1373853616, 7463736180224933002]]""" for (String table in tables) { if (table.matches("orc_s3_case[23456789]")) { - String[][] orc_actual_result = sql """select CounterID, EventDate, UserID, EventTime, WatchID from $table order by CounterID, EventDate, UserID, EventTime, WatchID limit 10;""" + def String[][] orc_actual_result = sql """select CounterID, EventDate, UserID, EventTime, WatchID from $table order by CounterID, EventDate, UserID, EventTime, WatchID limit 10;""" assertTrue("$orc_actual_result" == "$orc_expect_result") } } @@ -402,7 +403,8 @@ suite("test_broker_load_p2", "p2") { def max_try_milli_secs = 600000 while (max_try_milli_secs > 0) { - String[][] result = sql """ show load where label="$label_22666" order by createtime desc limit 1; """ + def String[][] result = sql """ show load where label="$label_22666" order by createtime desc limit 1; """ + logger.info("Load status: " + result[0][2] + ", label: $label_22666") if (result[0][2].equals("FINISHED")) { logger.info("Load FINISHED " + label_22666) break; diff --git a/regression-test/suites/schema_change_p2/test_schema_change.groovy b/regression-test/suites/schema_change_p2/test_schema_change.groovy index 7c7435fa18e5480..ccd2efc51622ed7 100644 --- a/regression-test/suites/schema_change_p2/test_schema_change.groovy +++ b/regression-test/suites/schema_change_p2/test_schema_change.groovy @@ -64,7 +64,7 @@ suite("test_schema_change") { ) """ - waitBrokerLoadJob = { String label /* param */ -> + def waitBrokerLoadJob = { String label /* param */ -> // check load state int tryTimes = 20 while (tryTimes-- > 0) { From f38ecd349c32e960ca20c29d5a3efde3cc5dabe6 Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Tue, 21 May 2024 10:53:40 +0800 Subject: [PATCH 088/111] [enhancement](memory) return error if allocate memory failed during add rows method (#35085) * return error when add rows failed * f --------- Co-authored-by: yiguolei --- .../schema_active_queries_scanner.cpp | 2 +- .../schema_backend_active_tasks.cpp | 2 +- .../schema_scanner/schema_routine_scanner.cpp | 2 +- .../schema_workload_groups_scanner.cpp | 2 +- .../schema_workload_sched_policy_scanner.cpp | 2 +- be/src/olap/memtable.cpp | 28 ++++--- be/src/olap/memtable.h | 8 +- be/src/olap/memtable_flush_executor.cpp | 3 +- be/src/olap/memtable_writer.cpp | 2 +- .../partitioned_hash_join_probe_operator.cpp | 4 +- .../partitioned_hash_join_sink_operator.cpp | 13 +++- .../local_exchange/local_exchanger.cpp | 16 ++-- be/src/vec/core/block.cpp | 75 +++++++++++-------- be/src/vec/core/block.h | 7 +- be/src/vec/exec/vaggregation_node.cpp | 4 +- be/src/vec/olap/vcollect_iterator.cpp | 2 +- be/src/vec/sink/vdata_stream_sender.cpp | 2 +- be/src/vec/sink/vrow_distribution.cpp | 2 +- 18 files changed, 100 insertions(+), 76 deletions(-) diff --git a/be/src/exec/schema_scanner/schema_active_queries_scanner.cpp b/be/src/exec/schema_scanner/schema_active_queries_scanner.cpp index 36cb145e3f56361..2115a38a6ebce3a 100644 --- a/be/src/exec/schema_scanner/schema_active_queries_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_active_queries_scanner.cpp @@ -158,7 +158,7 @@ Status SchemaActiveQueriesScanner::get_next_block(vectorized::Block* block, bool int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx); vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block); - mblock.add_rows(_active_query_block.get(), _row_idx, current_batch_rows); + RETURN_IF_ERROR(mblock.add_rows(_active_query_block.get(), _row_idx, current_batch_rows)); _row_idx += current_batch_rows; *eos = _row_idx == _total_rows; diff --git a/be/src/exec/schema_scanner/schema_backend_active_tasks.cpp b/be/src/exec/schema_scanner/schema_backend_active_tasks.cpp index c5f8825c2e41fd0..aa84f0d68c307b8 100644 --- a/be/src/exec/schema_scanner/schema_backend_active_tasks.cpp +++ b/be/src/exec/schema_scanner/schema_backend_active_tasks.cpp @@ -84,7 +84,7 @@ Status SchemaBackendActiveTasksScanner::get_next_block(vectorized::Block* block, int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx); vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block); - mblock.add_rows(_task_stats_block.get(), _row_idx, current_batch_rows); + RETURN_IF_ERROR(mblock.add_rows(_task_stats_block.get(), _row_idx, current_batch_rows)); _row_idx += current_batch_rows; *eos = _row_idx == _total_rows; diff --git a/be/src/exec/schema_scanner/schema_routine_scanner.cpp b/be/src/exec/schema_scanner/schema_routine_scanner.cpp index 7db46ada650bc63..3d55addee6c093b 100644 --- a/be/src/exec/schema_scanner/schema_routine_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_routine_scanner.cpp @@ -162,7 +162,7 @@ Status SchemaRoutinesScanner::get_next_block(vectorized::Block* block, bool* eos int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx); vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block); - mblock.add_rows(_routines_block.get(), _row_idx, current_batch_rows); + RETURN_IF_ERROR(mblock.add_rows(_routines_block.get(), _row_idx, current_batch_rows)); _row_idx += current_batch_rows; *eos = _row_idx == _total_rows; diff --git a/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp b/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp index 55cdfe9cf35906b..def52df531df7b6 100644 --- a/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_workload_groups_scanner.cpp @@ -135,7 +135,7 @@ Status SchemaWorkloadGroupsScanner::get_next_block(vectorized::Block* block, boo int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx); vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block); - mblock.add_rows(_workload_groups_block.get(), _row_idx, current_batch_rows); + RETURN_IF_ERROR(mblock.add_rows(_workload_groups_block.get(), _row_idx, current_batch_rows)); _row_idx += current_batch_rows; *eos = _row_idx == _total_rows; diff --git a/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.cpp b/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.cpp index b1a717228c8446c..035d3bfe217aece 100644 --- a/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.cpp +++ b/be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.cpp @@ -127,7 +127,7 @@ Status SchemaWorkloadSchedulePolicyScanner::get_next_block(vectorized::Block* bl int current_batch_rows = std::min(_block_rows_limit, _total_rows - _row_idx); vectorized::MutableBlock mblock = vectorized::MutableBlock::build_mutable_block(block); - mblock.add_rows(_block.get(), _row_idx, current_batch_rows); + RETURN_IF_ERROR(mblock.add_rows(_block.get(), _row_idx, current_batch_rows)); _row_idx += current_batch_rows; *eos = _row_idx == _total_rows; diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 2676bf7a32e93fd..abd1c0a8314f811 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -178,7 +178,8 @@ int RowInBlockComparator::operator()(const RowInBlock* left, const RowInBlock* r *_pblock, -1); } -void MemTable::insert(const vectorized::Block* input_block, const std::vector& row_idxs) { +Status MemTable::insert(const vectorized::Block* input_block, + const std::vector& row_idxs) { vectorized::Block target_block = *input_block; target_block = input_block->copy_block(_column_offset); if (_is_first_insertion) { @@ -209,7 +210,8 @@ void MemTable::insert(const vectorized::Block* input_block, const std::vector_row_pos, _arena.get()); } } -void MemTable::_put_into_output(vectorized::Block& in_block) { +Status MemTable::_put_into_output(vectorized::Block& in_block) { SCOPED_RAW_TIMER(&_stat.put_into_output_ns); std::vector row_pos_vec; DCHECK(in_block.rows() <= std::numeric_limits::max()); @@ -253,8 +256,8 @@ void MemTable::_put_into_output(vectorized::Block& in_block) { for (int i = 0; i < _row_in_blocks.size(); i++) { row_pos_vec.emplace_back(_row_in_blocks[i]->_row_pos); } - _output_mutable_block.add_rows(&in_block, row_pos_vec.data(), - row_pos_vec.data() + in_block.rows()); + return _output_mutable_block.add_rows(&in_block, row_pos_vec.data(), + row_pos_vec.data() + in_block.rows()); } size_t MemTable::_sort() { @@ -298,7 +301,7 @@ size_t MemTable::_sort() { return same_keys_num; } -void MemTable::_sort_by_cluster_keys() { +Status MemTable::_sort_by_cluster_keys() { SCOPED_RAW_TIMER(&_stat.sort_ns); _stat.sort_times++; // sort all rows @@ -344,8 +347,8 @@ void MemTable::_sort_by_cluster_keys() { for (int i = 0; i < row_in_blocks.size(); i++) { row_pos_vec.emplace_back(row_in_blocks[i]->_row_pos); } - _output_mutable_block.add_rows(&in_block, row_pos_vec.data(), - row_pos_vec.data() + in_block.rows()); + return _output_mutable_block.add_rows(&in_block, row_pos_vec.data(), + row_pos_vec.data() + in_block.rows()); } void MemTable::_sort_one_column(std::vector& row_in_blocks, Tie& tie, @@ -502,27 +505,28 @@ bool MemTable::need_agg() const { return false; } -std::unique_ptr MemTable::to_block() { +Status MemTable::to_block(std::unique_ptr* res) { size_t same_keys_num = _sort(); if (_keys_type == KeysType::DUP_KEYS || same_keys_num == 0) { if (_keys_type == KeysType::DUP_KEYS && _tablet_schema->num_key_columns() == 0) { _output_mutable_block.swap(_input_mutable_block); } else { vectorized::Block in_block = _input_mutable_block.to_block(); - _put_into_output(in_block); + RETURN_IF_ERROR(_put_into_output(in_block)); } } else { _aggregate(); } if (_keys_type == KeysType::UNIQUE_KEYS && _enable_unique_key_mow && !_tablet_schema->cluster_key_idxes().empty()) { - _sort_by_cluster_keys(); + RETURN_IF_ERROR(_sort_by_cluster_keys()); } g_memtable_input_block_allocated_size << -_input_mutable_block.allocated_bytes(); _input_mutable_block.clear(); _insert_mem_tracker->release(_mem_usage); _mem_usage = 0; - return vectorized::Block::create_unique(_output_mutable_block.to_block()); + *res = vectorized::Block::create_unique(_output_mutable_block.to_block()); + return Status::OK(); } } // namespace doris diff --git a/be/src/olap/memtable.h b/be/src/olap/memtable.h index 8362c69222ec21c..d2dfafd972a20dc 100644 --- a/be/src/olap/memtable.h +++ b/be/src/olap/memtable.h @@ -181,7 +181,7 @@ class MemTable { _flush_mem_tracker->consumption(); } // insert tuple from (row_pos) to (row_pos+num_rows) - void insert(const vectorized::Block* block, const std::vector& row_idxs); + Status insert(const vectorized::Block* block, const std::vector& row_idxs); void shrink_memtable_by_agg(); @@ -189,7 +189,7 @@ class MemTable { bool need_agg() const; - std::unique_ptr to_block(); + Status to_block(std::unique_ptr* res); bool empty() const { return _input_mutable_block.rows() == 0; } @@ -244,7 +244,7 @@ class MemTable { //return number of same keys size_t _sort(); - void _sort_by_cluster_keys(); + Status _sort_by_cluster_keys(); void _sort_one_column(std::vector& row_in_blocks, Tie& tie, std::function cmp); template @@ -252,7 +252,7 @@ class MemTable { int row_pos); template void _aggregate(); - void _put_into_output(vectorized::Block& in_block); + Status _put_into_output(vectorized::Block& in_block); bool _is_first_insertion; void _init_agg_functions(const vectorized::Block* block); diff --git a/be/src/olap/memtable_flush_executor.cpp b/be/src/olap/memtable_flush_executor.cpp index ece6930362d99f6..1ebe44aabf22d99 100644 --- a/be/src/olap/memtable_flush_executor.cpp +++ b/be/src/olap/memtable_flush_executor.cpp @@ -141,7 +141,8 @@ Status FlushToken::_do_flush_memtable(MemTable* memtable, int32_t segment_id, in signal::set_signal_task_id(_rowset_writer->load_id()); { SCOPED_CONSUME_MEM_TRACKER(memtable->flush_mem_tracker()); - std::unique_ptr block = memtable->to_block(); + std::unique_ptr block; + RETURN_IF_ERROR(memtable->to_block(&block)); RETURN_IF_ERROR(_rowset_writer->flush_memtable(block.get(), segment_id, flush_size)); } _memtable_stat += memtable->stat(); diff --git a/be/src/olap/memtable_writer.cpp b/be/src/olap/memtable_writer.cpp index c7ad1590a89f6f1..da6af4ffb803b3d 100644 --- a/be/src/olap/memtable_writer.cpp +++ b/be/src/olap/memtable_writer.cpp @@ -109,7 +109,7 @@ Status MemTableWriter::write(const vectorized::Block* block, } _total_received_rows += row_idxs.size(); - _mem_table->insert(block, row_idxs); + RETURN_IF_ERROR(_mem_table->insert(block, row_idxs)); if (UNLIKELY(_mem_table->need_agg() && config::enable_shrink_memory)) { _mem_table->shrink_memtable_by_agg(); diff --git a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp index 21134487c2ed8f1..10fa2effcc00fc7 100644 --- a/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp +++ b/be/src/pipeline/exec/partitioned_hash_join_probe_operator.cpp @@ -602,8 +602,8 @@ Status PartitionedHashJoinProbeOperatorX::push(RuntimeState* state, vectorized:: partitioned_blocks[i] = vectorized::MutableBlock::create_unique(input_block->clone_empty()); } - partitioned_blocks[i]->add_rows(input_block, &(partition_indexes[i][0]), - &(partition_indexes[i][count])); + RETURN_IF_ERROR(partitioned_blocks[i]->add_rows(input_block, &(partition_indexes[i][0]), + &(partition_indexes[i][count]))); if (partitioned_blocks[i]->rows() > 2 * 1024 * 1024 || (eos && partitioned_blocks[i]->rows() > 0)) { diff --git a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp index 97d5d1456047f1f..d253a519b0c7543 100644 --- a/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp +++ b/be/src/pipeline/exec/partitioned_hash_join_sink_operator.cpp @@ -203,7 +203,14 @@ Status PartitionedHashJoinSinkLocalState::_revoke_unpartitioned_block(RuntimeSta { SCOPED_TIMER(_partition_shuffle_timer); - partition_block->add_rows(&build_block, begin, end); + Status st = partition_block->add_rows(&build_block, begin, end); + if (!st.ok()) { + std::unique_lock lock(_spill_lock); + _spill_status = st; + _spill_status_ok = false; + _dependency->set_ready(); + return; + } partitions_indexes[partition_idx].clear(); } @@ -336,8 +343,8 @@ Status PartitionedHashJoinSinkLocalState::_partition_block(RuntimeState* state, partitioned_blocks[i] = vectorized::MutableBlock::create_unique(in_block->clone_empty()); } - partitioned_blocks[i]->add_rows(in_block, &(partition_indexes[i][0]), - &(partition_indexes[i][count])); + RETURN_IF_ERROR(partitioned_blocks[i]->add_rows(in_block, &(partition_indexes[i][0]), + &(partition_indexes[i][count]))); } return Status::OK(); diff --git a/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.cpp b/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.cpp index 71e427f8b25a3ed..ce1f05a22bf5add 100644 --- a/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.cpp +++ b/be/src/pipeline/pipeline_x/local_exchange/local_exchanger.cpp @@ -44,26 +44,28 @@ Status ShuffleExchanger::get_block(RuntimeState* state, vectorized::Block* block PartitionedBlock partitioned_block; std::unique_ptr mutable_block = nullptr; - auto get_data = [&](vectorized::Block* result_block) { + auto get_data = [&](vectorized::Block* result_block) -> Status { do { const auto* offset_start = &(( *std::get<0>(partitioned_block.second))[std::get<1>(partitioned_block.second)]); auto block_wrapper = partitioned_block.first; local_state._shared_state->sub_mem_usage( local_state._channel_id, block_wrapper->data_block.allocated_bytes(), false); - mutable_block->add_rows(&block_wrapper->data_block, offset_start, - offset_start + std::get<2>(partitioned_block.second)); + RETURN_IF_ERROR( + mutable_block->add_rows(&block_wrapper->data_block, offset_start, + offset_start + std::get<2>(partitioned_block.second))); block_wrapper->unref(local_state._shared_state); } while (mutable_block->rows() < state->batch_size() && _data_queue[local_state._channel_id].try_dequeue(partitioned_block)); *result_block = mutable_block->to_block(); + return Status::OK(); }; if (_running_sink_operators == 0) { if (_data_queue[local_state._channel_id].try_dequeue(partitioned_block)) { SCOPED_TIMER(local_state._copy_data_timer); mutable_block = vectorized::MutableBlock::create_unique( partitioned_block.first->data_block.clone_empty()); - get_data(block); + RETURN_IF_ERROR(get_data(block)); } else { COUNTER_UPDATE(local_state._get_block_failed_counter, 1); *eos = true; @@ -72,7 +74,7 @@ Status ShuffleExchanger::get_block(RuntimeState* state, vectorized::Block* block SCOPED_TIMER(local_state._copy_data_timer); mutable_block = vectorized::MutableBlock::create_unique( partitioned_block.first->data_block.clone_empty()); - get_data(block); + RETURN_IF_ERROR(get_data(block)); } else { COUNTER_UPDATE(local_state._get_block_failed_counter, 1); local_state._dependency->block(); @@ -244,7 +246,7 @@ Status BroadcastExchanger::sink(RuntimeState* state, vectorized::Block* in_block LocalExchangeSinkLocalState& local_state) { for (size_t i = 0; i < _num_partitions; i++) { auto mutable_block = vectorized::MutableBlock::create_unique(in_block->clone_empty()); - mutable_block->add_rows(in_block, 0, in_block->rows()); + RETURN_IF_ERROR(mutable_block->add_rows(in_block, 0, in_block->rows())); _data_queue[i].enqueue(mutable_block->to_block()); local_state._shared_state->set_ready_to_read(i); } @@ -335,7 +337,7 @@ Status AdaptivePassthroughExchanger::_split_rows(RuntimeState* state, if (size > 0) { std::unique_ptr mutable_block = vectorized::MutableBlock::create_unique(block->clone_empty()); - mutable_block->add_rows(block, start, size); + RETURN_IF_ERROR(mutable_block->add_rows(block, start, size)); auto new_block = mutable_block->to_block(); local_state._shared_state->add_mem_usage(i, new_block.allocated_bytes()); data_queue[i].enqueue(std::move(new_block)); diff --git a/be/src/vec/core/block.cpp b/be/src/vec/core/block.cpp index 83ecf568d6f1929..466c9b3b559da4f 100644 --- a/be/src/vec/core/block.cpp +++ b/be/src/vec/core/block.cpp @@ -970,44 +970,53 @@ void MutableBlock::add_row(const Block* block, int row) { } } -void MutableBlock::add_rows(const Block* block, const uint32_t* row_begin, - const uint32_t* row_end) { - DCHECK_LE(columns(), block->columns()); - const auto& block_data = block->get_columns_with_type_and_name(); - for (size_t i = 0; i < _columns.size(); ++i) { - DCHECK_EQ(_data_types[i]->get_name(), block_data[i].type->get_name()); - auto& dst = _columns[i]; - const auto& src = *block_data[i].column.get(); - DCHECK_GE(src.size(), row_end - row_begin); - dst->insert_indices_from(src, row_begin, row_end); - } +Status MutableBlock::add_rows(const Block* block, const uint32_t* row_begin, + const uint32_t* row_end) { + RETURN_IF_CATCH_EXCEPTION({ + DCHECK_LE(columns(), block->columns()); + const auto& block_data = block->get_columns_with_type_and_name(); + for (size_t i = 0; i < _columns.size(); ++i) { + DCHECK_EQ(_data_types[i]->get_name(), block_data[i].type->get_name()); + auto& dst = _columns[i]; + const auto& src = *block_data[i].column.get(); + DCHECK_GE(src.size(), row_end - row_begin); + dst->insert_indices_from(src, row_begin, row_end); + } + }); + return Status::OK(); } -void MutableBlock::add_rows(const Block* block, size_t row_begin, size_t length) { - DCHECK_LE(columns(), block->columns()); - const auto& block_data = block->get_columns_with_type_and_name(); - for (size_t i = 0; i < _columns.size(); ++i) { - DCHECK_EQ(_data_types[i]->get_name(), block_data[i].type->get_name()); - auto& dst = _columns[i]; - const auto& src = *block_data[i].column.get(); - dst->insert_range_from(src, row_begin, length); - } +Status MutableBlock::add_rows(const Block* block, size_t row_begin, size_t length) { + RETURN_IF_CATCH_EXCEPTION({ + DCHECK_LE(columns(), block->columns()); + const auto& block_data = block->get_columns_with_type_and_name(); + for (size_t i = 0; i < _columns.size(); ++i) { + DCHECK_EQ(_data_types[i]->get_name(), block_data[i].type->get_name()); + auto& dst = _columns[i]; + const auto& src = *block_data[i].column.get(); + dst->insert_range_from(src, row_begin, length); + } + }); + return Status::OK(); } -void MutableBlock::add_rows(const Block* block, std::vector rows) { - DCHECK_LE(columns(), block->columns()); - const auto& block_data = block->get_columns_with_type_and_name(); - const size_t length = std::ranges::distance(rows); - for (size_t i = 0; i < _columns.size(); ++i) { - DCHECK_EQ(_data_types[i]->get_name(), block_data[i].type->get_name()); - auto& dst = _columns[i]; - const auto& src = *block_data[i].column.get(); - dst->reserve(dst->size() + length); - for (size_t row : rows) { - // we can introduce a new function like `insert_assume_reserved` for IColumn. - dst->insert_from(src, row); +Status MutableBlock::add_rows(const Block* block, std::vector rows) { + RETURN_IF_CATCH_EXCEPTION({ + DCHECK_LE(columns(), block->columns()); + const auto& block_data = block->get_columns_with_type_and_name(); + const size_t length = std::ranges::distance(rows); + for (size_t i = 0; i < _columns.size(); ++i) { + DCHECK_EQ(_data_types[i]->get_name(), block_data[i].type->get_name()); + auto& dst = _columns[i]; + const auto& src = *block_data[i].column.get(); + dst->reserve(dst->size() + length); + for (size_t row : rows) { + // we can introduce a new function like `insert_assume_reserved` for IColumn. + dst->insert_from(src, row); + } } - } + }); + return Status::OK(); } void MutableBlock::erase(const String& name) { diff --git a/be/src/vec/core/block.h b/be/src/vec/core/block.h index 9fea5242d9baff2..89f8e99b66a35df 100644 --- a/be/src/vec/core/block.h +++ b/be/src/vec/core/block.h @@ -603,9 +603,10 @@ class MutableBlock { void swap(MutableBlock&& other) noexcept; void add_row(const Block* block, int row); - void add_rows(const Block* block, const uint32_t* row_begin, const uint32_t* row_end); - void add_rows(const Block* block, size_t row_begin, size_t length); - void add_rows(const Block* block, std::vector rows); + // Batch add row should return error status if allocate memory failed. + Status add_rows(const Block* block, const uint32_t* row_begin, const uint32_t* row_end); + Status add_rows(const Block* block, size_t row_begin, size_t length); + Status add_rows(const Block* block, std::vector rows); /// remove the column with the specified name void erase(const String& name); diff --git a/be/src/vec/exec/vaggregation_node.cpp b/be/src/vec/exec/vaggregation_node.cpp index 67072e2f60f6747..8bfa7e90b4afbec 100644 --- a/be/src/vec/exec/vaggregation_node.cpp +++ b/be/src/vec/exec/vaggregation_node.cpp @@ -1132,7 +1132,7 @@ Status AggregationNode::_spill_hash_table(HashTableCtxType& agg_method, HashTabl for (size_t j = 0; j < partitioned_indices.size(); ++j) { if (partitioned_indices[j] != i) { if (length > 0) { - mutable_block.add_rows(&block, begin, length); + RETURN_IF_ERROR(mutable_block.add_rows(&block, begin, length)); } length = 0; continue; @@ -1145,7 +1145,7 @@ Status AggregationNode::_spill_hash_table(HashTableCtxType& agg_method, HashTabl } if (length > 0) { - mutable_block.add_rows(&block, begin, length); + RETURN_IF_ERROR(mutable_block.add_rows(&block, begin, length)); } CHECK_EQ(mutable_block.rows(), blocks_rows[i]); diff --git a/be/src/vec/olap/vcollect_iterator.cpp b/be/src/vec/olap/vcollect_iterator.cpp index 0a6bafca49c1b70..3ce1869546c470c 100644 --- a/be/src/vec/olap/vcollect_iterator.cpp +++ b/be/src/vec/olap/vcollect_iterator.cpp @@ -377,7 +377,7 @@ Status VCollectIterator::_topn_next(Block* block) { size_t base = mutable_block.rows(); // append block to mutable_block - mutable_block.add_rows(block, 0, rows_to_copy); + RETURN_IF_ERROR(mutable_block.add_rows(block, 0, rows_to_copy)); // insert appended rows pos in mutable_block to sorted_row_pos and sort it for (size_t i = 0; i < rows_to_copy; i++) { sorted_row_pos.insert(base + i); diff --git a/be/src/vec/sink/vdata_stream_sender.cpp b/be/src/vec/sink/vdata_stream_sender.cpp index 69b7054f5005c89..24f92bf2aae0e0c 100644 --- a/be/src/vec/sink/vdata_stream_sender.cpp +++ b/be/src/vec/sink/vdata_stream_sender.cpp @@ -885,7 +885,7 @@ Status BlockSerializer::next_serialized_block(Block* block, PBlock* dest if (!rows->empty()) { SCOPED_TIMER(_parent->split_block_distribute_by_channel_timer()); const auto* begin = rows->data(); - _mutable_block->add_rows(block, begin, begin + rows->size()); + RETURN_IF_ERROR(_mutable_block->add_rows(block, begin, begin + rows->size())); } } else if (!block->empty()) { SCOPED_TIMER(_parent->merge_block_timer()); diff --git a/be/src/vec/sink/vrow_distribution.cpp b/be/src/vec/sink/vrow_distribution.cpp index 3bef7d0117ec517..f740e8f5767861b 100644 --- a/be/src/vec/sink/vrow_distribution.cpp +++ b/be/src/vec/sink/vrow_distribution.cpp @@ -51,7 +51,7 @@ Status VRowDistribution::_save_missing_values( int col_size, Block* block, std::vector filter, const std::vector& col_null_maps) { // de-duplication for new partitions but save all rows. - _batching_block->add_rows(block, filter); + RETURN_IF_ERROR(_batching_block->add_rows(block, filter)); std::vector cur_row_values; for (int row = 0; row < col_strs[0].size(); ++row) { cur_row_values.clear(); From 1e4a83e17bf1fdd3f4753ca0ce86f10f6d3b4890 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Wed, 22 May 2024 01:01:52 +0800 Subject: [PATCH 089/111] fix compile --- be/src/olap/tablet.cpp | 8 ++++---- be/src/olap/tablet.h | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index f0867b4a19de08b..71c45deb3879d56 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -2785,7 +2785,7 @@ Status Tablet::_load_rowset_segments(const RowsetSharedPtr& rowset, return Status::OK(); } -void Tablet::sort_block(vectorized::Block& in_block, vectorized::Block& output_block) { +Status Tablet::sort_block(vectorized::Block& in_block, vectorized::Block& output_block) { vectorized::MutableBlock mutable_input_block = vectorized::MutableBlock::build_mutable_block(&in_block); vectorized::MutableBlock mutable_output_block = @@ -2816,8 +2816,8 @@ void Tablet::sort_block(vectorized::Block& in_block, vectorized::Block& output_b for (int i = 0; i < row_in_blocks.size(); i++) { row_pos_vec.emplace_back(row_in_blocks[i]->_row_pos); } - mutable_output_block.add_rows(&in_block, row_pos_vec.data(), - row_pos_vec.data() + in_block.rows()); + return mutable_output_block.add_rows(&in_block, row_pos_vec.data(), + row_pos_vec.data() + in_block.rows()); } Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, @@ -3001,7 +3001,7 @@ Status Tablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset, RETURN_IF_ERROR(generate_new_block_for_partial_update( rowset_schema, partial_update_info->missing_cids, partial_update_info->update_cids, read_plan_ori, read_plan_update, rsid_to_rowset, &block)); - sort_block(block, ordered_block); + RETURN_IF_ERROR(sort_block(block, ordered_block)); RETURN_IF_ERROR(rowset_writer->flush_single_block(&ordered_block)); } LOG(INFO) << "calc segment delete bitmap, tablet: " << tablet_id() << " rowset: " << rowset_id diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 6eb63648b9a2c8f..f0f152e41f66b8f 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -511,7 +511,7 @@ class Tablet final : public BaseTablet { const std::map>>& location_map); Status all_rs_id(int64_t max_version, RowsetIdUnorderedSet* rowset_ids) const; - void sort_block(vectorized::Block& in_block, vectorized::Block& output_block); + Status sort_block(vectorized::Block& in_block, vectorized::Block& output_block); bool check_all_rowset_segment(); From 9bc2c88384afb71b1cd69484c189e1922241f5d3 Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Tue, 21 May 2024 23:41:11 +0800 Subject: [PATCH 090/111] [enhancement](memory) add exception check in page builder to avoid be oom during flush memtable (#35138) --- .../olap/rowset/segment_v2/binary_plain_page.h | 4 +++- .../rowset/segment_v2/binary_prefix_page.cpp | 17 ++++++++++------- be/src/olap/rowset/segment_v2/bitshuffle_page.h | 4 +++- be/src/olap/rowset/segment_v2/plain_page.h | 4 +++- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/be/src/olap/rowset/segment_v2/binary_plain_page.h b/be/src/olap/rowset/segment_v2/binary_plain_page.h index 0e9836bf0e496e5..7bc5e020a832e14 100644 --- a/be/src/olap/rowset/segment_v2/binary_plain_page.h +++ b/be/src/olap/rowset/segment_v2/binary_plain_page.h @@ -77,7 +77,9 @@ class BinaryPlainPageBuilder : public PageBuilder { } size_t offset = _buffer.size(); _offsets.push_back(offset); - _buffer.append(src->data, src->size); + // This may need a large memory, should return error if could not allocated + // successfully, to avoid BE OOM. + RETURN_IF_CATCH_EXCEPTION(_buffer.append(src->data, src->size)); _last_value_size = src->size; _size_estimate += src->size; diff --git a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp index 183e7bb08534409..9d1ecdb94707782 100644 --- a/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp +++ b/be/src/olap/rowset/segment_v2/binary_prefix_page.cpp @@ -71,13 +71,16 @@ Status BinaryPrefixPageBuilder::add(const uint8_t* vals, size_t* add_count) { } } int non_share_len = entry_len - share_len; - - put_varint32(&_buffer, share_len); - put_varint32(&_buffer, non_share_len); - _buffer.append(entry + share_len, non_share_len); - - _last_entry.clear(); - _last_entry.append(entry, entry_len); + // This may need a large memory, should return error if could not allocated + // successfully, to avoid BE OOM. + RETURN_IF_CATCH_EXCEPTION({ + put_varint32(&_buffer, share_len); + put_varint32(&_buffer, non_share_len); + _buffer.append(entry + share_len, non_share_len); + + _last_entry.clear(); + _last_entry.append(entry, entry_len); + }); ++_count; } diff --git a/be/src/olap/rowset/segment_v2/bitshuffle_page.h b/be/src/olap/rowset/segment_v2/bitshuffle_page.h index 89180c2cd1b9034..0e0f513229462c3 100644 --- a/be/src/olap/rowset/segment_v2/bitshuffle_page.h +++ b/be/src/olap/rowset/segment_v2/bitshuffle_page.h @@ -111,7 +111,9 @@ class BitshufflePageBuilder : public PageBuilder { int to_add = std::min(_remain_element_capacity, *count); int to_add_size = to_add * SIZE_OF_TYPE; size_t orig_size = _data.size(); - _data.resize(orig_size + to_add_size); + // This may need a large memory, should return error if could not allocated + // successfully, to avoid BE OOM. + RETURN_IF_CATCH_EXCEPTION(_data.resize(orig_size + to_add_size)); _count += to_add; _remain_element_capacity -= to_add; // return added number through count diff --git a/be/src/olap/rowset/segment_v2/plain_page.h b/be/src/olap/rowset/segment_v2/plain_page.h index 29cec0e87952572..cb9236ee3157f5a 100644 --- a/be/src/olap/rowset/segment_v2/plain_page.h +++ b/be/src/olap/rowset/segment_v2/plain_page.h @@ -48,7 +48,9 @@ class PlainPageBuilder : public PageBuilder { return Status::OK(); } size_t old_size = _buffer.size(); - _buffer.resize(old_size + *count * SIZE_OF_TYPE); + // This may need a large memory, should return error if could not allocated + // successfully, to avoid BE OOM. + RETURN_IF_CATCH_EXCEPTION(_buffer.resize(old_size + *count * SIZE_OF_TYPE)); memcpy(&_buffer[old_size], vals, *count * SIZE_OF_TYPE); _count += *count; return Status::OK(); From 11971eddb49fe670c3ef11bcb77f407d3b97661d Mon Sep 17 00:00:00 2001 From: yiguolei <676222867@qq.com> Date: Fri, 17 May 2024 22:47:00 +0800 Subject: [PATCH 091/111] [atomicstatus](be) add atomic status to share state between multi thread (#35002) --- be/src/common/status.h | 44 +++++++++++++++++++ .../vec/sink/writer/async_result_writer.cpp | 16 +++---- be/src/vec/sink/writer/async_result_writer.h | 8 ++-- 3 files changed, 53 insertions(+), 15 deletions(-) diff --git a/be/src/common/status.h b/be/src/common/status.h index 39cb205ce896a38..cf9b42a3c69668c 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -542,6 +542,50 @@ class [[nodiscard]] Status { } }; +// There are many thread using status to indicate the cancel state, one thread may update it and +// the other thread will read it. Status is not thread safe, for example, if one thread is update it +// and another thread is call to_string method, it may core, because the _err_msg is an unique ptr and +// it is deconstructed during copy method. +// And also we could not use lock, because we need get status frequently to check if it is cancelled. +// The defaule value is ok. +class AtomicStatus { +public: + AtomicStatus() : error_st_(Status::OK()) {} + + bool ok() const { return error_code_.load() == 0; } + + bool update(const Status& new_status) { + // If new status is normal, or the old status is abnormal, then not need update + if (new_status.ok() || error_code_.load() != 0) { + return false; + } + int16_t expected_error_code = 0; + if (error_code_.compare_exchange_strong(expected_error_code, new_status.code(), + std::memory_order_acq_rel)) { + // lock here for read status, to avoid core during return error_st_ + std::lock_guard l(mutex_); + error_st_ = new_status; + return true; + } else { + return false; + } + } + + // will copy a new status object to avoid concurrency + Status status() { + std::lock_guard l(mutex_); + return error_st_; + } + +private: + std::atomic_int16_t error_code_ = 0; + Status error_st_; + std::mutex mutex_; + + AtomicStatus(const AtomicStatus&) = delete; + void operator=(const AtomicStatus&) = delete; +}; + inline std::ostream& operator<<(std::ostream& ostr, const Status& status) { ostr << '[' << status.code_as_string() << ']'; ostr << status.msg(); diff --git a/be/src/vec/sink/writer/async_result_writer.cpp b/be/src/vec/sink/writer/async_result_writer.cpp index 9a84f374464a10c..4ed878a4634476b 100644 --- a/be/src/vec/sink/writer/async_result_writer.cpp +++ b/be/src/vec/sink/writer/async_result_writer.cpp @@ -55,7 +55,7 @@ Status AsyncResultWriter::sink(Block* block, bool eos) { // if io task failed, just return error status to // end the query if (!_writer_status.ok()) { - return _writer_status; + return _writer_status.status(); } if (_dependency && _is_finished()) { @@ -143,7 +143,7 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* profi auto status = write(*block); if (!status.ok()) [[unlikely]] { std::unique_lock l(_m); - _writer_status = status; + _writer_status.update(status); if (_dependency && _is_finished()) { _dependency->set_ready(); } @@ -172,14 +172,10 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* profi // Should not call finish in lock because it may hang, and it will lock _m too long. // And get_writer_status will also need this lock, it will block pipeline exec thread. Status st = finish(state); - std::lock_guard l(_m); - _writer_status = st; + _writer_status.update(st); } Status st = Status::OK(); - { - std::lock_guard l(_m); - st = _writer_status; - } + { st = _writer_status.status(); } Status close_st = close(st); { @@ -187,7 +183,7 @@ void AsyncResultWriter::process_block(RuntimeState* state, RuntimeProfile* profi // the real reason. std::lock_guard l(_m); if (_writer_status.ok()) { - _writer_status = close_st; + _writer_status.update(close_st); } _writer_thread_closed = true; } @@ -215,7 +211,7 @@ Status AsyncResultWriter::_projection_block(doris::vectorized::Block& input_bloc void AsyncResultWriter::force_close(Status s) { std::lock_guard l(_m); - _writer_status = s; + _writer_status.update(s); if (_dependency && _is_finished()) { _dependency->set_ready(); } diff --git a/be/src/vec/sink/writer/async_result_writer.h b/be/src/vec/sink/writer/async_result_writer.h index 7f9700486da80bd..1fd0fc280f00876 100644 --- a/be/src/vec/sink/writer/async_result_writer.h +++ b/be/src/vec/sink/writer/async_result_writer.h @@ -78,10 +78,7 @@ class AsyncResultWriter : public ResultWriter { // Add the IO thread task process block() to thread pool to dispose the IO Status start_writer(RuntimeState* state, RuntimeProfile* profile); - Status get_writer_status() { - std::lock_guard l(_m); - return _writer_status; - } + Status get_writer_status() { return _writer_status.status(); } protected: Status _projection_block(Block& input_block, Block* output_block); @@ -103,7 +100,8 @@ class AsyncResultWriter : public ResultWriter { std::mutex _m; std::condition_variable _cv; std::deque> _data_queue; - Status _writer_status = Status::OK(); + // Default value is ok + AtomicStatus _writer_status; bool _eos = false; // The writer is not started at the beginning. If prepare failed but not open, the the writer // is not started, so should not pending finish on it. From 2ed6a00fd1b6ec64b28c86a130d730f68eab1700 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Wed, 22 May 2024 09:53:45 +0800 Subject: [PATCH 092/111] [opt](memory) Add GlobalMemoryArbitrator and support ReserveMemory (#34985) (#35070) --- be/src/agent/task_worker_pool.cpp | 3 +- be/src/common/daemon.cpp | 25 ++-- be/src/olap/memtable_memory_limiter.cpp | 22 +-- be/src/olap/olap_server.cpp | 5 +- be/src/olap/rowset/segcompaction.cpp | 3 +- be/src/olap/rowset_builder.cpp | 3 +- .../memory/global_memory_arbitrator.cpp | 34 +++++ .../runtime/memory/global_memory_arbitrator.h | 136 ++++++++++++++++++ be/src/runtime/memory/mem_tracker.h | 8 +- be/src/runtime/memory/mem_tracker_limiter.cpp | 51 +------ be/src/runtime/memory/mem_tracker_limiter.h | 24 +++- .../runtime/memory/thread_mem_tracker_mgr.cpp | 2 + .../runtime/memory/thread_mem_tracker_mgr.h | 101 +++++++++++-- be/src/runtime/thread_context.h | 22 +++ .../runtime/workload_group/workload_group.cpp | 3 +- .../workload_group/workload_group_manager.cpp | 10 +- be/src/util/mem_info.cpp | 53 ++++--- be/src/util/mem_info.h | 32 +---- be/src/vec/common/allocator.cpp | 9 +- 19 files changed, 390 insertions(+), 156 deletions(-) create mode 100644 be/src/runtime/memory/global_memory_arbitrator.cpp create mode 100644 be/src/runtime/memory/global_memory_arbitrator.h diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index 30c83fdb878809f..37ec4bc6da7dc05 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -73,6 +73,7 @@ #include "olap/txn_manager.h" #include "olap/utils.h" #include "runtime/exec_env.h" +#include "runtime/memory/global_memory_arbitrator.h" #include "runtime/snapshot_loader.h" #include "service/backend_options.h" #include "util/debug_points.h" @@ -1563,7 +1564,7 @@ void PublishVersionWorkerPool::publish_version_callback(const TAgentTaskRequest& .error(status); } else { if (!config::disable_auto_compaction && - !MemInfo::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { + !GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { for (auto [tablet_id, _] : succ_tablets) { TabletSharedPtr tablet = _engine.tablet_manager()->get_tablet(tablet_id); if (tablet != nullptr) { diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 8d79e6f21812f72..4787f6036502d07 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -47,6 +47,7 @@ #include "runtime/client_cache.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" +#include "runtime/memory/global_memory_arbitrator.h" #include "runtime/memory/mem_tracker.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/runtime_query_statistics_mgr.h" @@ -191,7 +192,7 @@ void Daemon::memory_maintenance_thread() { // Refresh process memory metrics. doris::PerfCounters::refresh_proc_status(); doris::MemInfo::refresh_proc_meminfo(); - doris::MemInfo::refresh_proc_mem_no_allocator_cache(); + doris::GlobalMemoryArbitrator::refresh_vm_rss_sub_allocator_cache(); // Update and print memory stat when the memory changes by 256M. if (abs(last_print_proc_mem - PerfCounters::get_vm_rss()) > 268435456) { @@ -212,7 +213,7 @@ void Daemon::memory_maintenance_thread() { ExecEnv::GetInstance()->brpc_iobuf_block_memory_tracker()->set_consumption( butil::IOBuf::block_memory()); - LOG(INFO) << MemTrackerLimiter:: + LOG(INFO) << doris::GlobalMemoryArbitrator:: process_mem_log_str(); // print mem log when memory state by 256M } } @@ -229,21 +230,22 @@ void Daemon::memory_gc_thread() { continue; } auto sys_mem_available = doris::MemInfo::sys_mem_available(); - auto proc_mem_no_allocator_cache = doris::MemInfo::proc_mem_no_allocator_cache(); + auto process_memory_usage = doris::GlobalMemoryArbitrator::process_memory_usage(); // GC excess memory for resource groups that not enable overcommit - auto tg_free_mem = doris::MemInfo::tg_not_enable_overcommit_group_gc(); + auto tg_free_mem = doris::MemInfo::tg_disable_overcommit_group_gc(); sys_mem_available += tg_free_mem; - proc_mem_no_allocator_cache -= tg_free_mem; + process_memory_usage -= tg_free_mem; if (memory_full_gc_sleep_time_ms <= 0 && (sys_mem_available < doris::MemInfo::sys_mem_available_low_water_mark() || - proc_mem_no_allocator_cache >= doris::MemInfo::mem_limit())) { + process_memory_usage >= doris::MemInfo::mem_limit())) { // No longer full gc and minor gc during sleep. memory_full_gc_sleep_time_ms = memory_gc_sleep_time_ms; memory_minor_gc_sleep_time_ms = memory_gc_sleep_time_ms; - LOG(INFO) << fmt::format("[MemoryGC] start full GC, {}.", - MemTrackerLimiter::process_limit_exceeded_errmsg_str()); + LOG(INFO) << fmt::format( + "[MemoryGC] start full GC, {}.", + doris::GlobalMemoryArbitrator::process_limit_exceeded_errmsg_str()); doris::MemTrackerLimiter::print_log_process_usage(); if (doris::MemInfo::process_full_gc()) { // If there is not enough memory to be gc, the process memory usage will not be printed in the next continuous gc. @@ -251,11 +253,12 @@ void Daemon::memory_gc_thread() { } } else if (memory_minor_gc_sleep_time_ms <= 0 && (sys_mem_available < doris::MemInfo::sys_mem_available_warning_water_mark() || - proc_mem_no_allocator_cache >= doris::MemInfo::soft_mem_limit())) { + process_memory_usage >= doris::MemInfo::soft_mem_limit())) { // No minor gc during sleep, but full gc is possible. memory_minor_gc_sleep_time_ms = memory_gc_sleep_time_ms; - LOG(INFO) << fmt::format("[MemoryGC] start minor GC, {}.", - MemTrackerLimiter::process_soft_limit_exceeded_errmsg_str()); + LOG(INFO) << fmt::format( + "[MemoryGC] start minor GC, {}.", + doris::GlobalMemoryArbitrator::process_soft_limit_exceeded_errmsg_str()); doris::MemTrackerLimiter::print_log_process_usage(); if (doris::MemInfo::process_minor_gc()) { doris::MemTrackerLimiter::enable_print_log_process_usage(); diff --git a/be/src/olap/memtable_memory_limiter.cpp b/be/src/olap/memtable_memory_limiter.cpp index f0a46aaa4522cf9..dc128137ae43bc6 100644 --- a/be/src/olap/memtable_memory_limiter.cpp +++ b/be/src/olap/memtable_memory_limiter.cpp @@ -88,7 +88,8 @@ int64_t MemTableMemoryLimiter::_avail_mem_lack() { int64_t MemTableMemoryLimiter::_proc_mem_extra() { // reserve a small amount of memory so we do not trigger MinorGC auto reserved_mem = doris::MemInfo::sys_mem_available_low_water_mark(); - auto proc_mem_extra = MemInfo::proc_mem_no_allocator_cache() - MemInfo::soft_mem_limit(); + auto proc_mem_extra = + GlobalMemoryArbitrator::process_memory_usage() - MemInfo::soft_mem_limit(); return proc_mem_extra + reserved_mem; } @@ -222,14 +223,17 @@ void MemTableMemoryLimiter::refresh_mem_tracker() { _last_limit = limit; _log_timer.reset(); - LOG(INFO) << ss.str() << ", process mem: " << PerfCounters::get_vm_rss_str() - << " (without allocator cache: " - << PrettyPrinter::print_bytes(MemInfo::proc_mem_no_allocator_cache()) - << "), load mem: " << PrettyPrinter::print_bytes(_mem_tracker->consumption()) - << ", memtable writers num: " << _writers.size() - << " (active: " << PrettyPrinter::print_bytes(_active_mem_usage) - << ", write: " << PrettyPrinter::print_bytes(_write_mem_usage) - << ", flush: " << PrettyPrinter::print_bytes(_flush_mem_usage) << ")"; + // if not exist load task, this log should not be printed. + if (_mem_usage != 0) { + LOG(INFO) << ss.str() << ", process mem: " << PerfCounters::get_vm_rss_str() + << " (without allocator cache: " + << PrettyPrinter::print_bytes(GlobalMemoryArbitrator::process_memory_usage()) + << "), load mem: " << PrettyPrinter::print_bytes(_mem_tracker->consumption()) + << ", memtable writers num: " << _writers.size() + << " (active: " << PrettyPrinter::print_bytes(_active_mem_usage) + << ", write: " << PrettyPrinter::print_bytes(_write_mem_usage) + << ", flush: " << PrettyPrinter::print_bytes(_flush_mem_usage) << ")"; + } } void MemTableMemoryLimiter::_refresh_mem_tracker() { diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp index 0a8bcedf14e8e67..b667a8049066edf 100644 --- a/be/src/olap/olap_server.cpp +++ b/be/src/olap/olap_server.cpp @@ -70,6 +70,7 @@ #include "olap/task/index_builder.h" #include "runtime/client_cache.h" #include "runtime/memory/cache_manager.h" +#include "runtime/memory/global_memory_arbitrator.h" #include "service/brpc.h" #include "service/point_query_executor.h" #include "util/brpc_client_cache.h" @@ -621,7 +622,7 @@ void StorageEngine::_compaction_tasks_producer_callback() { int64_t interval = config::generate_compaction_tasks_interval_ms; do { if (!config::disable_auto_compaction && - !MemInfo::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { + !GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { _adjust_compaction_thread_num(); bool check_score = false; @@ -1359,7 +1360,7 @@ void StorageEngine::_cold_data_compaction_producer_callback() { while (!_stop_background_threads_latch.wait_for( std::chrono::seconds(config::cold_data_compaction_interval_sec))) { if (config::disable_auto_compaction || - MemInfo::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { + GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { continue; } diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp index 2de881ee13afe63..8fee04ccb80e439 100644 --- a/be/src/olap/rowset/segcompaction.cpp +++ b/be/src/olap/rowset/segcompaction.cpp @@ -56,6 +56,7 @@ #include "olap/storage_engine.h" #include "olap/tablet_reader.h" #include "olap/tablet_schema.h" +#include "runtime/memory/global_memory_arbitrator.h" #include "runtime/thread_context.h" #include "util/debug_points.h" #include "util/mem_info.h" @@ -212,7 +213,7 @@ Status SegcompactionWorker::_create_segment_writer_for_segcompaction( Status SegcompactionWorker::_do_compact_segments(SegCompactionCandidatesSharedPtr segments) { SCOPED_SWITCH_THREAD_MEM_TRACKER_LIMITER(ExecEnv::GetInstance()->segcompaction_mem_tracker()); /* throttle segcompaction task if memory depleted */ - if (MemInfo::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { + if (GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { return Status::Error("skip segcompaction due to memory shortage"); } diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index 2153a9ad1a8e87a..7ff06b39eb0aac0 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -48,6 +48,7 @@ #include "olap/tablet_meta.h" #include "olap/tablet_schema.h" #include "olap/txn_manager.h" +#include "runtime/memory/global_memory_arbitrator.h" #include "util/brpc_client_cache.h" #include "util/mem_info.h" #include "util/ref_count_closure.h" @@ -140,7 +141,7 @@ Status RowsetBuilder::init_mow_context(std::shared_ptr& mow_context) Status RowsetBuilder::check_tablet_version_count() { if (!_tablet->exceed_version_limit(config::max_tablet_version_num - 100) || - MemInfo::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { + GlobalMemoryArbitrator::is_exceed_soft_mem_limit(GB_EXCHANGE_BYTE)) { return Status::OK(); } //trigger compaction diff --git a/be/src/runtime/memory/global_memory_arbitrator.cpp b/be/src/runtime/memory/global_memory_arbitrator.cpp new file mode 100644 index 000000000000000..dc686f7c5ab468d --- /dev/null +++ b/be/src/runtime/memory/global_memory_arbitrator.cpp @@ -0,0 +1,34 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "runtime/memory/global_memory_arbitrator.h" + +#include + +namespace doris { + +bvar::PassiveStatus g_vm_rss_sub_allocator_cache( + "meminfo_vm_rss_sub_allocator_cache", + [](void*) { return GlobalMemoryArbitrator::vm_rss_sub_allocator_cache(); }, nullptr); +bvar::PassiveStatus g_process_memory_usage( + "meminfo_process_memory_usage", + [](void*) { return GlobalMemoryArbitrator::process_memory_usage(); }, nullptr); + +std::atomic GlobalMemoryArbitrator::_s_vm_rss_sub_allocator_cache = -1; +std::atomic GlobalMemoryArbitrator::_s_process_reserved_memory = 0; + +} // namespace doris diff --git a/be/src/runtime/memory/global_memory_arbitrator.h b/be/src/runtime/memory/global_memory_arbitrator.h new file mode 100644 index 000000000000000..b1879cb1a7bf48f --- /dev/null +++ b/be/src/runtime/memory/global_memory_arbitrator.h @@ -0,0 +1,136 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "util/mem_info.h" + +namespace doris { + +class GlobalMemoryArbitrator { +public: + /** jemalloc pdirty is number of pages within unused extents that are potentially + * dirty, and for which madvise() or similar has not been called. + * + * So they will be subtracted from RSS to make accounting more + * accurate, since those pages are not really RSS but a memory + * that can be used at anytime via jemalloc. + */ + static inline void refresh_vm_rss_sub_allocator_cache() { + _s_vm_rss_sub_allocator_cache.store( + PerfCounters::get_vm_rss() - static_cast(MemInfo::allocator_cache_mem()), + std::memory_order_relaxed); + MemInfo::refresh_interval_memory_growth = 0; + } + static inline int64_t vm_rss_sub_allocator_cache() { + return _s_vm_rss_sub_allocator_cache.load(std::memory_order_relaxed); + } + + // If need to use process memory in your execution logic, pls use it. + // equal to real process memory(vm_rss), subtract jemalloc dirty page cache, + // add reserved memory and growth memory since the last vm_rss update. + static inline int64_t process_memory_usage() { + return vm_rss_sub_allocator_cache() + + MemInfo::refresh_interval_memory_growth.load(std::memory_order_relaxed) + + process_reserved_memory(); + } + + static inline bool try_reserve_process_memory(int64_t bytes) { + if (MemInfo::sys_mem_available() - bytes < MemInfo::sys_mem_available_low_water_mark()) { + return false; + } + int64_t old_reserved_mem = _s_process_reserved_memory.load(std::memory_order_relaxed); + int64_t new_reserved_mem = 0; + do { + new_reserved_mem = old_reserved_mem + bytes; + if (UNLIKELY(vm_rss_sub_allocator_cache() + + MemInfo::refresh_interval_memory_growth.load( + std::memory_order_relaxed) + + new_reserved_mem >= + MemInfo::mem_limit())) { + return false; + } + } while (!_s_process_reserved_memory.compare_exchange_weak( + old_reserved_mem, new_reserved_mem, std::memory_order_relaxed)); + return true; + } + + static inline void release_process_reserved_memory(int64_t bytes) { + _s_process_reserved_memory.fetch_sub(bytes, std::memory_order_relaxed); + } + + static inline int64_t process_reserved_memory() { + return _s_process_reserved_memory.load(std::memory_order_relaxed); + } + + static bool is_exceed_soft_mem_limit(int64_t bytes = 0) { + return process_memory_usage() + bytes >= MemInfo::soft_mem_limit() || + MemInfo::sys_mem_available() - bytes < + MemInfo::sys_mem_available_warning_water_mark(); + } + + static bool is_exceed_hard_mem_limit(int64_t bytes = 0) { + // Limit process memory usage using the actual physical memory of the process in `/proc/self/status`. + // This is independent of the consumption value of the mem tracker, which counts the virtual memory + // of the process malloc. + // for fast, expect MemInfo::initialized() to be true. + // + // tcmalloc/jemalloc allocator cache does not participate in the mem check as part of the process physical memory. + // because `new/malloc` will trigger mem hook when using tcmalloc/jemalloc allocator cache, + // but it may not actually alloc physical memory, which is not expected in mem hook fail. + return process_memory_usage() + bytes >= MemInfo::mem_limit() || + MemInfo::sys_mem_available() - bytes < MemInfo::sys_mem_available_low_water_mark(); + } + + static std::string process_mem_log_str() { + return fmt::format( + "os physical memory {}. process memory used {}, limit {}, soft limit {}. sys " + "available memory {}, low water mark {}, warning water mark {}. Refresh interval " + "memory growth {} B", + PrettyPrinter::print(MemInfo::physical_mem(), TUnit::BYTES), + PerfCounters::get_vm_rss_str(), MemInfo::mem_limit_str(), + MemInfo::soft_mem_limit_str(), MemInfo::sys_mem_available_str(), + PrettyPrinter::print(MemInfo::sys_mem_available_low_water_mark(), TUnit::BYTES), + PrettyPrinter::print(MemInfo::sys_mem_available_warning_water_mark(), TUnit::BYTES), + MemInfo::refresh_interval_memory_growth); + } + + static std::string process_limit_exceeded_errmsg_str() { + return fmt::format( + "process memory used {} exceed limit {} or sys available memory {} less than low " + "water mark {}", + PerfCounters::get_vm_rss_str(), MemInfo::mem_limit_str(), + MemInfo::sys_mem_available_str(), + PrettyPrinter::print(MemInfo::sys_mem_available_low_water_mark(), TUnit::BYTES)); + } + + static std::string process_soft_limit_exceeded_errmsg_str() { + return fmt::format( + "process memory used {} exceed soft limit {} or sys available memory {} less than " + "warning water mark {}.", + PerfCounters::get_vm_rss_str(), MemInfo::soft_mem_limit_str(), + MemInfo::sys_mem_available_str(), + PrettyPrinter::print(MemInfo::sys_mem_available_warning_water_mark(), + TUnit::BYTES)); + } + +private: + static std::atomic _s_vm_rss_sub_allocator_cache; + static std::atomic _s_process_reserved_memory; +}; + +} // namespace doris diff --git a/be/src/runtime/memory/mem_tracker.h b/be/src/runtime/memory/mem_tracker.h index de7628c17490bdb..d308d201901e539 100644 --- a/be/src/runtime/memory/mem_tracker.h +++ b/be/src/runtime/memory/mem_tracker.h @@ -70,7 +70,7 @@ class MemTracker { MemCounter() : _current_value(0), _peak_value(0) {} void add(int64_t delta) { - auto value = _current_value.fetch_add(delta, std::memory_order_relaxed) + delta; + int64_t value = _current_value.fetch_add(delta, std::memory_order_relaxed) + delta; update_peak(value); } @@ -79,8 +79,8 @@ class MemTracker { } bool try_add(int64_t delta, int64_t max) { - auto cur_val = _current_value.load(std::memory_order_relaxed); - auto new_val = 0; + int64_t cur_val = _current_value.load(std::memory_order_relaxed); + int64_t new_val = 0; do { new_val = cur_val + delta; if (UNLIKELY(new_val > max)) { @@ -100,7 +100,7 @@ class MemTracker { } void update_peak(int64_t value) { - auto pre_value = _peak_value.load(std::memory_order_relaxed); + int64_t pre_value = _peak_value.load(std::memory_order_relaxed); while (value > pre_value && !_peak_value.compare_exchange_weak( pre_value, value, std::memory_order_relaxed)) { } diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp b/be/src/runtime/memory/mem_tracker_limiter.cpp index 2218bed6959ddc4..b84f7c54957f873 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.cpp +++ b/be/src/runtime/memory/mem_tracker_limiter.cpp @@ -31,6 +31,7 @@ #include "olap/memtable_memory_limiter.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" +#include "runtime/memory/global_memory_arbitrator.h" #include "runtime/thread_context.h" #include "runtime/workload_group/workload_group.h" #include "service/backend_options.h" @@ -306,7 +307,7 @@ void MemTrackerLimiter::print_log_usage(const std::string& msg) { if (_enable_print_log_usage) { _enable_print_log_usage = false; std::string detail = msg; - detail += "\nProcess Memory Summary:\n " + MemTrackerLimiter::process_mem_log_str(); + detail += "\nProcess Memory Summary:\n " + GlobalMemoryArbitrator::process_mem_log_str(); detail += "\nMemory Tracker Summary: " + log_usage(); std::string child_trackers_usage; std::vector snapshots; @@ -324,7 +325,7 @@ void MemTrackerLimiter::print_log_usage(const std::string& msg) { std::string MemTrackerLimiter::log_process_usage_str() { std::string detail; - detail += "\nProcess Memory Summary:\n " + MemTrackerLimiter::process_mem_log_str(); + detail += "\nProcess Memory Summary:\n " + GlobalMemoryArbitrator::process_mem_log_str(); std::vector snapshots; MemTrackerLimiter::make_process_snapshots(&snapshots); MemTrackerLimiter::make_type_snapshots(&snapshots, MemTrackerLimiter::Type::GLOBAL); @@ -355,50 +356,6 @@ void MemTrackerLimiter::print_log_process_usage() { } } -bool MemTrackerLimiter::sys_mem_exceed_limit_check(int64_t bytes) { - // Limit process memory usage using the actual physical memory of the process in `/proc/self/status`. - // This is independent of the consumption value of the mem tracker, which counts the virtual memory - // of the process malloc. - // for fast, expect MemInfo::initialized() to be true. - // - // tcmalloc/jemalloc allocator cache does not participate in the mem check as part of the process physical memory. - // because `new/malloc` will trigger mem hook when using tcmalloc/jemalloc allocator cache, - // but it may not actually alloc physical memory, which is not expected in mem hook fail. - return MemInfo::proc_mem_no_allocator_cache() + bytes >= MemInfo::mem_limit() || - MemInfo::sys_mem_available() < MemInfo::sys_mem_available_low_water_mark(); -} - -std::string MemTrackerLimiter::process_mem_log_str() { - return fmt::format( - "os physical memory {}. process memory used {}, limit {}, soft limit {}. sys " - "available memory {}, low water mark {}, warning water mark {}. Refresh interval " - "memory growth {} B", - PrettyPrinter::print(MemInfo::physical_mem(), TUnit::BYTES), - PerfCounters::get_vm_rss_str(), MemInfo::mem_limit_str(), MemInfo::soft_mem_limit_str(), - MemInfo::sys_mem_available_str(), - PrettyPrinter::print(MemInfo::sys_mem_available_low_water_mark(), TUnit::BYTES), - PrettyPrinter::print(MemInfo::sys_mem_available_warning_water_mark(), TUnit::BYTES), - MemInfo::refresh_interval_memory_growth); -} - -std::string MemTrackerLimiter::process_limit_exceeded_errmsg_str() { - return fmt::format( - "process memory used {} exceed limit {} or sys available memory {} less than low " - "water mark {}", - PerfCounters::get_vm_rss_str(), MemInfo::mem_limit_str(), - MemInfo::sys_mem_available_str(), - PrettyPrinter::print(MemInfo::sys_mem_available_low_water_mark(), TUnit::BYTES)); -} - -std::string MemTrackerLimiter::process_soft_limit_exceeded_errmsg_str() { - return fmt::format( - "process memory used {} exceed soft limit {} or sys available memory {} less than " - "warning water mark {}.", - PerfCounters::get_vm_rss_str(), MemInfo::soft_mem_limit_str(), - MemInfo::sys_mem_available_str(), - PrettyPrinter::print(MemInfo::sys_mem_available_warning_water_mark(), TUnit::BYTES)); -} - std::string MemTrackerLimiter::tracker_limit_exceeded_str() { std::string err_msg = fmt::format( "memory tracker limit exceeded, tracker label:{}, type:{}, limit " @@ -490,7 +447,7 @@ int64_t MemTrackerLimiter::free_top_memory_query( } else if (tracker->consumption() + prepare_free_mem < min_free_mem) { min_pq.emplace(tracker->consumption(), tracker->label()); prepare_free_mem += tracker->consumption(); - } else if (tracker->consumption() > min_pq.top().first) { + } else if (!min_pq.empty() && tracker->consumption() > min_pq.top().first) { min_pq.emplace(tracker->consumption(), tracker->label()); prepare_free_mem += tracker->consumption(); while (prepare_free_mem - min_pq.top().first > min_free_mem) { diff --git a/be/src/runtime/memory/mem_tracker_limiter.h b/be/src/runtime/memory/mem_tracker_limiter.h index 2e510e1f462d2e0..3a891ca3a14bdb9 100644 --- a/be/src/runtime/memory/mem_tracker_limiter.h +++ b/be/src/runtime/memory/mem_tracker_limiter.h @@ -43,7 +43,7 @@ namespace doris { class RuntimeProfile; -constexpr auto MEM_TRACKER_GROUP_NUM = 1000; +constexpr size_t MEM_TRACKER_GROUP_NUM = 1000; struct TrackerLimiterGroup { // Note! in order to enable ExecEnv::mem_tracker_limiter_pool support resize, @@ -129,8 +129,6 @@ class MemTrackerLimiter final : public MemTracker { __builtin_unreachable(); } - static bool sys_mem_exceed_limit_check(int64_t bytes); - void set_consumption() { LOG(FATAL) << "MemTrackerLimiter set_consumption not supported"; } Type type() const { return _type; } int64_t group_num() const { return _group_num; } @@ -138,6 +136,23 @@ class MemTrackerLimiter final : public MemTracker { int64_t limit() const { return _limit; } bool limit_exceeded() const { return _limit >= 0 && _limit < consumption(); } + bool try_consume(int64_t bytes) const { + if (UNLIKELY(bytes == 0)) { + return true; + } + bool st = true; + if (is_overcommit_tracker() && config::enable_query_memory_overcommit) { + st = _consumption->try_add(bytes, _limit); + } else { + _consumption->add(bytes); + } + if (st && _query_statistics) { + _query_statistics->set_max_peak_memory_bytes(_consumption->peak_value()); + _query_statistics->set_current_used_memory_bytes(_consumption->current_value()); + } + return st; + } + Status check_limit(int64_t bytes = 0); bool is_overcommit_tracker() const { return type() == Type::QUERY || type() == Type::LOAD; } @@ -222,9 +237,6 @@ class MemTrackerLimiter final : public MemTracker { return querytid; } - static std::string process_mem_log_str(); - static std::string process_limit_exceeded_errmsg_str(); - static std::string process_soft_limit_exceeded_errmsg_str(); // Log the memory usage when memory limit is exceeded. std::string tracker_limit_exceeded_str(); diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp index bc962b51480b592..766ee6435842120 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.cpp +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.cpp @@ -44,6 +44,7 @@ class AsyncCancelQueryTask : public Runnable { void ThreadMemTrackerMgr::attach_limiter_tracker( const std::shared_ptr& mem_tracker) { DCHECK(mem_tracker); + DCHECK(_reserved_mem == 0); CHECK(init()); flush_untracked_mem(); _limiter_tracker = mem_tracker; @@ -53,6 +54,7 @@ void ThreadMemTrackerMgr::attach_limiter_tracker( void ThreadMemTrackerMgr::detach_limiter_tracker( const std::shared_ptr& old_mem_tracker) { CHECK(init()); + release_reserved(); flush_untracked_mem(); _limiter_tracker = old_mem_tracker; _limiter_tracker_raw = old_mem_tracker.get(); diff --git a/be/src/runtime/memory/thread_mem_tracker_mgr.h b/be/src/runtime/memory/thread_mem_tracker_mgr.h index 40fe6e130325e86..6081b0133465fdb 100644 --- a/be/src/runtime/memory/thread_mem_tracker_mgr.h +++ b/be/src/runtime/memory/thread_mem_tracker_mgr.h @@ -30,6 +30,7 @@ #include "common/config.h" #include "runtime/exec_env.h" +#include "runtime/memory/global_memory_arbitrator.h" #include "runtime/memory/mem_tracker.h" #include "runtime/memory/mem_tracker_limiter.h" #include "util/stack_util.h" @@ -37,6 +38,8 @@ namespace doris { +constexpr size_t SYNC_PROC_RESERVED_INTERVAL_BYTES = (1ULL << 20); // 1M + // Memory Hook is counted in the memory tracker of the current thread. class ThreadMemTrackerMgr { public: @@ -69,14 +72,14 @@ class ThreadMemTrackerMgr { void start_count_scope_mem() { CHECK(init()); - _scope_mem = 0; + _scope_mem = _reserved_mem; // consume in advance _count_scope_mem = true; } int64_t stop_count_scope_mem() { flush_untracked_mem(); _count_scope_mem = false; - return _scope_mem; + return _scope_mem - _reserved_mem; } // Note that, If call the memory allocation operation in Memory Hook, @@ -86,6 +89,9 @@ class ThreadMemTrackerMgr { void consume(int64_t size, int skip_large_memory_check = 0); void flush_untracked_mem(); + bool try_reserve(int64_t size); + void release_reserved(); + bool is_attach_query() { return _query_id != TUniqueId(); } bool is_query_cancelled() const { return _is_query_cancelled; } @@ -123,7 +129,9 @@ class ThreadMemTrackerMgr { bool _init = false; // Cache untracked mem. int64_t _untracked_mem = 0; - int64_t old_untracked_mem = 0; + int64_t _old_untracked_mem = 0; + + int64_t _reserved_mem = 0; bool _count_scope_mem = false; int64_t _scope_mem = 0; @@ -164,16 +172,43 @@ inline bool ThreadMemTrackerMgr::push_consumer_tracker(MemTracker* tracker) { } _consumer_tracker_stack.push_back(tracker); tracker->release(_untracked_mem); + tracker->consume(_reserved_mem); // consume in advance return true; } inline void ThreadMemTrackerMgr::pop_consumer_tracker() { DCHECK(!_consumer_tracker_stack.empty()); _consumer_tracker_stack.back()->consume(_untracked_mem); + _consumer_tracker_stack.back()->release(_reserved_mem); _consumer_tracker_stack.pop_back(); } inline void ThreadMemTrackerMgr::consume(int64_t size, int skip_large_memory_check) { + if (_reserved_mem != 0) { + if (_reserved_mem >= size) { + // only need to subtract _reserved_mem, no need to consume MemTracker, + // every time _reserved_mem is minus the sum of size >= SYNC_PROC_RESERVED_INTERVAL_BYTES, + // subtract size from process global reserved memory, + // because this part of the reserved memory has already been used by BE process. + _reserved_mem -= size; + // store bytes that not synchronized to process reserved memory. + _untracked_mem += size; + if (_untracked_mem >= SYNC_PROC_RESERVED_INTERVAL_BYTES) { + doris::GlobalMemoryArbitrator::release_process_reserved_memory(_untracked_mem); + _untracked_mem = 0; + } + return; + } else { + // reserved memory is insufficient, the remaining _reserved_mem is subtracted from this memory consumed, + // and reset _reserved_mem to 0, and subtract the remaining _reserved_mem from + // process global reserved memory, this means that all reserved memory has been used by BE process. + size -= _reserved_mem; + doris::GlobalMemoryArbitrator::release_process_reserved_memory(_reserved_mem + + _untracked_mem); + _reserved_mem = 0; + _untracked_mem = 0; + } + } _untracked_mem += size; if (!_init && !ExecEnv::ready()) { return; @@ -205,18 +240,64 @@ inline void ThreadMemTrackerMgr::consume(int64_t size, int skip_large_memory_che inline void ThreadMemTrackerMgr::flush_untracked_mem() { // Temporary memory may be allocated during the consumption of the mem tracker, which will lead to entering // the Memory Hook again, so suspend consumption to avoid falling into an infinite loop. - if (!init()) return; + if (_untracked_mem == 0 || !init()) { + return; + } _stop_consume = true; DCHECK(_limiter_tracker_raw); - old_untracked_mem = _untracked_mem; - if (_count_scope_mem) _scope_mem += _untracked_mem; - _limiter_tracker_raw->consume(old_untracked_mem); - for (auto tracker : _consumer_tracker_stack) { - tracker->consume(old_untracked_mem); + _old_untracked_mem = _untracked_mem; + if (_count_scope_mem) { + _scope_mem += _untracked_mem; } - _untracked_mem -= old_untracked_mem; + _limiter_tracker_raw->consume(_old_untracked_mem); + for (auto* tracker : _consumer_tracker_stack) { + tracker->consume(_old_untracked_mem); + } + _untracked_mem -= _old_untracked_mem; _stop_consume = false; } +inline bool ThreadMemTrackerMgr::try_reserve(int64_t size) { + DCHECK(_limiter_tracker_raw); + DCHECK(size >= 0); + CHECK(init()); + // if _reserved_mem not equal to 0, repeat reserve, + // _untracked_mem store bytes that not synchronized to process reserved memory. + if (_reserved_mem == 0) { + flush_untracked_mem(); + } + if (!_limiter_tracker_raw->try_consume(size)) { + return false; + } + if (!doris::GlobalMemoryArbitrator::try_reserve_process_memory(size)) { + _limiter_tracker_raw->release(size); // rollback + return false; + } + if (_count_scope_mem) { + _scope_mem += size; + } + for (auto* tracker : _consumer_tracker_stack) { + tracker->consume(size); + } + _reserved_mem += size; + DCHECK(_untracked_mem == 0); + return true; +} + +inline void ThreadMemTrackerMgr::release_reserved() { + flush_untracked_mem(); + if (_reserved_mem > 0) { + doris::GlobalMemoryArbitrator::release_process_reserved_memory(_reserved_mem); + _limiter_tracker_raw->consume(-_reserved_mem); + if (_count_scope_mem) { + _scope_mem -= _reserved_mem; + } + for (auto* tracker : _consumer_tracker_stack) { + tracker->consume(-_reserved_mem); + } + _reserved_mem = 0; + } +} + } // namespace doris diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h index 84b24e79657fd83..72d3c8111f6c1ca 100644 --- a/be/src/runtime/thread_context.h +++ b/be/src/runtime/thread_context.h @@ -71,6 +71,9 @@ #define SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(mem_tracker) \ auto VARNAME_LINENUM(add_mem_consumer) = doris::AddThreadMemTrackerConsumerByHook(mem_tracker) +#define DEFER_RELEASE_RESERVED() \ + Defer VARNAME_LINENUM(defer) {[&]() { doris::thread_context()->release_reserved_memory(); }}; + #define ORPHAN_TRACKER_CHECK() \ DCHECK(doris::k_doris_exit || !doris::config::enable_memory_orphan_check || \ doris::thread_context()->thread_mem_tracker()->label() != "Orphan") \ @@ -93,6 +96,7 @@ auto VARNAME_LINENUM(scoped_tls_mcbh) = doris::ScopedInitThreadContext() #define SCOPED_CONSUME_MEM_TRACKER_BY_HOOK(mem_tracker) \ auto VARNAME_LINENUM(scoped_tls_cmtbh) = doris::ScopedInitThreadContext() +#define DEFER_RELEASE_RESERVED() (void)0 #define ORPHAN_TRACKER_CHECK() (void)0 #define MEMORY_ORPHAN_CHECK() (void)0 #endif @@ -203,6 +207,24 @@ class ThreadContext { thread_mem_tracker_mgr->consume(size, skip_large_memory_check); } + bool try_reserve_memory(const int64_t size) const { +#ifdef USE_MEM_TRACKER + DCHECK(doris::k_doris_exit || !doris::config::enable_memory_orphan_check || + thread_mem_tracker()->label() != "Orphan") + << doris::memory_orphan_check_msg; +#endif + return thread_mem_tracker_mgr->try_reserve(size); + } + + void release_reserved_memory() const { +#ifdef USE_MEM_TRACKER + DCHECK(doris::k_doris_exit || !doris::config::enable_memory_orphan_check || + thread_mem_tracker()->label() != "Orphan") + << doris::memory_orphan_check_msg; +#endif + thread_mem_tracker_mgr->release_reserved(); + } + int thread_local_handle_count = 0; int skip_memory_check = 0; int skip_large_memory_check = 0; diff --git a/be/src/runtime/workload_group/workload_group.cpp b/be/src/runtime/workload_group/workload_group.cpp index 39e411de72668c8..fd885aaacb186d1 100644 --- a/be/src/runtime/workload_group/workload_group.cpp +++ b/be/src/runtime/workload_group/workload_group.cpp @@ -30,6 +30,7 @@ #include "pipeline/task_queue.h" #include "pipeline/task_scheduler.h" #include "runtime/exec_env.h" +#include "runtime/memory/global_memory_arbitrator.h" #include "runtime/memory/mem_tracker_limiter.h" #include "util/mem_info.h" #include "util/parse_util.h" @@ -171,7 +172,7 @@ int64_t WorkloadGroup::gc_memory(int64_t need_free_mem, RuntimeProfile* profile, MemTracker::print_bytes(_memory_limit), BackendOptions::get_localhost()); } } - std::string process_mem_usage_str = MemTrackerLimiter::process_mem_log_str(); + std::string process_mem_usage_str = GlobalMemoryArbitrator::process_mem_log_str(); auto cancel_top_overcommit_str = [cancel_str, process_mem_usage_str](int64_t mem_consumption, const std::string& label) { return fmt::format( diff --git a/be/src/runtime/workload_group/workload_group_manager.cpp b/be/src/runtime/workload_group/workload_group_manager.cpp index 153e9bab8ce30a0..dc4c73782e48f10 100644 --- a/be/src/runtime/workload_group/workload_group_manager.cpp +++ b/be/src/runtime/workload_group/workload_group_manager.cpp @@ -177,29 +177,29 @@ void WorkloadGroupMgr::refresh_wg_memory_info() { wgs_mem_info[wg_id] = {wg_total_mem_used}; } + // *TODO*, modify to use doris::GlobalMemoryArbitrator::process_memory_usage(). auto proc_vm_rss = PerfCounters::get_vm_rss(); if (all_queries_mem_used <= 0) { return; } - auto process_mem_used = doris::MemInfo::proc_mem_no_allocator_cache(); if (proc_vm_rss < all_queries_mem_used) { all_queries_mem_used = proc_vm_rss; } // process memory used is actually bigger than all_queries_mem_used, // because memory of page cache, allocator cache, segment cache etc. are included - // in process_mem_used. + // in proc_vm_rss. // we count these cache memories equally on workload groups. double ratio = (double)proc_vm_rss / (double)all_queries_mem_used; if (ratio <= 1.25) { - auto sys_mem_available = doris::MemInfo::sys_mem_available(); std::string debug_msg = fmt::format( "\nProcess Memory Summary: process_vm_rss: {}, process mem: {}, sys mem available: " "{}, all quries mem: {}", PrettyPrinter::print(proc_vm_rss, TUnit::BYTES), - PrettyPrinter::print(process_mem_used, TUnit::BYTES), - PrettyPrinter::print(sys_mem_available, TUnit::BYTES), + PrettyPrinter::print(doris::GlobalMemoryArbitrator::process_memory_usage(), + TUnit::BYTES), + doris::MemInfo::sys_mem_available_str(), PrettyPrinter::print(all_queries_mem_used, TUnit::BYTES)); LOG_EVERY_T(INFO, 10) << debug_msg; } diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index ce42d69dccf7b6f..a2cb04049db43c4 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -56,9 +56,6 @@ namespace doris { bvar::PassiveStatus g_sys_mem_avail( "meminfo_sys_mem_avail", [](void*) { return MemInfo::sys_mem_available(); }, nullptr); -bvar::PassiveStatus g_proc_mem_no_allocator_cache( - "meminfo_proc_mem_no_allocator_cache", - [](void*) { return MemInfo::proc_mem_no_allocator_cache(); }, nullptr); bool MemInfo::_s_initialized = false; std::atomic MemInfo::_s_physical_mem = std::numeric_limits::max(); @@ -68,7 +65,6 @@ std::atomic MemInfo::_s_soft_mem_limit = std::numeric_limits:: std::atomic MemInfo::_s_allocator_cache_mem = 0; std::string MemInfo::_s_allocator_cache_mem_str = ""; std::atomic MemInfo::_s_virtual_memory_used = 0; -std::atomic MemInfo::_s_proc_mem_no_allocator_cache = -1; std::atomic MemInfo::refresh_interval_memory_growth = 0; static std::unordered_map _mem_info_bytes; @@ -129,7 +125,7 @@ bool MemInfo::process_minor_gc() { std::string pre_sys_mem_available = MemInfo::sys_mem_available_str(); Defer defer {[&]() { - notify_je_purge_dirty_pages(); + MemInfo::notify_je_purge_dirty_pages(); std::stringstream ss; profile->pretty_print(&ss); LOG(INFO) << fmt::format( @@ -139,16 +135,16 @@ bool MemInfo::process_minor_gc() { }}; freed_mem += CacheManager::instance()->for_each_cache_prune_stale(profile.get()); - notify_je_purge_dirty_pages(); - if (freed_mem > _s_process_minor_gc_size) { + MemInfo::notify_je_purge_dirty_pages(); + if (freed_mem > MemInfo::process_minor_gc_size()) { return true; } if (config::enable_workload_group_memory_gc) { RuntimeProfile* tg_profile = profile->create_child("WorkloadGroup", true, true); - freed_mem += tg_enable_overcommit_group_gc(_s_process_minor_gc_size - freed_mem, tg_profile, - true); - if (freed_mem > _s_process_minor_gc_size) { + freed_mem += tg_enable_overcommit_group_gc(MemInfo::process_minor_gc_size() - freed_mem, + tg_profile, true); + if (freed_mem > MemInfo::process_minor_gc_size()) { return true; } } @@ -160,9 +156,9 @@ bool MemInfo::process_minor_gc() { RuntimeProfile* toq_profile = profile->create_child("FreeTopOvercommitMemoryQuery", true, true); freed_mem += MemTrackerLimiter::free_top_overcommit_query( - _s_process_minor_gc_size - freed_mem, pre_vm_rss, pre_sys_mem_available, + MemInfo::process_minor_gc_size() - freed_mem, pre_vm_rss, pre_sys_mem_available, toq_profile); - if (freed_mem > _s_process_minor_gc_size) { + if (freed_mem > MemInfo::process_minor_gc_size()) { return true; } } @@ -183,7 +179,7 @@ bool MemInfo::process_full_gc() { std::string pre_sys_mem_available = MemInfo::sys_mem_available_str(); Defer defer {[&]() { - notify_je_purge_dirty_pages(); + MemInfo::notify_je_purge_dirty_pages(); std::stringstream ss; profile->pretty_print(&ss); LOG(INFO) << fmt::format( @@ -193,16 +189,16 @@ bool MemInfo::process_full_gc() { }}; freed_mem += CacheManager::instance()->for_each_cache_prune_all(profile.get()); - notify_je_purge_dirty_pages(); - if (freed_mem > _s_process_full_gc_size) { + MemInfo::notify_je_purge_dirty_pages(); + if (freed_mem > MemInfo::process_full_gc_size()) { return true; } if (config::enable_workload_group_memory_gc) { RuntimeProfile* tg_profile = profile->create_child("WorkloadGroup", true, true); - freed_mem += tg_enable_overcommit_group_gc(_s_process_full_gc_size - freed_mem, tg_profile, - false); - if (freed_mem > _s_process_full_gc_size) { + freed_mem += tg_enable_overcommit_group_gc(MemInfo::process_full_gc_size() - freed_mem, + tg_profile, false); + if (freed_mem > MemInfo::process_full_gc_size()) { return true; } } @@ -211,8 +207,9 @@ bool MemInfo::process_full_gc() { "[MemoryGC] before free top memory query in full GC", MemTrackerLimiter::Type::QUERY); RuntimeProfile* tmq_profile = profile->create_child("FreeTopMemoryQuery", true, true); freed_mem += MemTrackerLimiter::free_top_memory_query( - _s_process_full_gc_size - freed_mem, pre_vm_rss, pre_sys_mem_available, tmq_profile); - if (freed_mem > _s_process_full_gc_size) { + MemInfo::process_full_gc_size() - freed_mem, pre_vm_rss, pre_sys_mem_available, + tmq_profile); + if (freed_mem > MemInfo::process_full_gc_size()) { return true; } @@ -223,9 +220,9 @@ bool MemInfo::process_full_gc() { RuntimeProfile* tol_profile = profile->create_child("FreeTopMemoryOvercommitLoad", true, true); freed_mem += MemTrackerLimiter::free_top_overcommit_load( - _s_process_full_gc_size - freed_mem, pre_vm_rss, pre_sys_mem_available, + MemInfo::process_full_gc_size() - freed_mem, pre_vm_rss, pre_sys_mem_available, tol_profile); - if (freed_mem > _s_process_full_gc_size) { + if (freed_mem > MemInfo::process_full_gc_size()) { return true; } } @@ -233,15 +230,13 @@ bool MemInfo::process_full_gc() { VLOG_NOTICE << MemTrackerLimiter::type_detail_usage( "[MemoryGC] before free top memory load in full GC", MemTrackerLimiter::Type::LOAD); RuntimeProfile* tml_profile = profile->create_child("FreeTopMemoryLoad", true, true); - freed_mem += MemTrackerLimiter::free_top_memory_load( - _s_process_full_gc_size - freed_mem, pre_vm_rss, pre_sys_mem_available, tml_profile); - if (freed_mem > _s_process_full_gc_size) { - return true; - } - return false; + freed_mem += + MemTrackerLimiter::free_top_memory_load(MemInfo::process_full_gc_size() - freed_mem, + pre_vm_rss, pre_sys_mem_available, tml_profile); + return freed_mem > MemInfo::process_full_gc_size(); } -int64_t MemInfo::tg_not_enable_overcommit_group_gc() { +int64_t MemInfo::tg_disable_overcommit_group_gc() { MonotonicStopWatch watch; watch.start(); std::vector task_groups; diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index f6c9407d73eb58e..5606ebd45d6b97d 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -92,6 +92,12 @@ class MemInfo { static inline int64_t sys_mem_available_warning_water_mark() { return _s_sys_mem_available_warning_water_mark; } + static inline int64_t process_minor_gc_size() { + return _s_process_minor_gc_size.load(std::memory_order_relaxed); + } + static inline int64_t process_full_gc_size() { + return _s_process_full_gc_size.load(std::memory_order_relaxed); + } static inline int64_t get_tc_metrics(const std::string& name) { #ifndef USE_JEMALLOC @@ -152,30 +158,11 @@ class MemInfo { return _s_allocator_cache_mem.load(std::memory_order_relaxed); } static inline std::string allocator_cache_mem_str() { return _s_allocator_cache_mem_str; } - static inline int64_t proc_mem_no_allocator_cache() { - return _s_proc_mem_no_allocator_cache.load(std::memory_order_relaxed) + - refresh_interval_memory_growth; - } // Tcmalloc property `generic.total_physical_bytes` records the total length of the virtual memory // obtained by the process malloc, not the physical memory actually used by the process in the OS. static void refresh_allocator_mem(); - /** jemalloc pdirty is number of pages within unused extents that are potentially - * dirty, and for which madvise() or similar has not been called. - * - * So they will be subtracted from RSS to make accounting more - * accurate, since those pages are not really RSS but a memory - * that can be used at anytime via jemalloc. - */ - static inline void refresh_proc_mem_no_allocator_cache() { - _s_proc_mem_no_allocator_cache.store( - PerfCounters::get_vm_rss() - static_cast(_s_allocator_cache_mem.load( - std::memory_order_relaxed)), - std::memory_order_relaxed); - refresh_interval_memory_growth = 0; - } - static inline int64_t mem_limit() { DCHECK(_s_initialized); return _s_mem_limit.load(std::memory_order_relaxed); @@ -193,17 +180,13 @@ class MemInfo { return PrettyPrinter::print(_s_soft_mem_limit.load(std::memory_order_relaxed), TUnit::BYTES); } - static bool is_exceed_soft_mem_limit(int64_t bytes = 0) { - return proc_mem_no_allocator_cache() + bytes >= soft_mem_limit() || - sys_mem_available() < sys_mem_available_warning_water_mark(); - } static std::string debug_string(); static bool process_minor_gc(); static bool process_full_gc(); - static int64_t tg_not_enable_overcommit_group_gc(); + static int64_t tg_disable_overcommit_group_gc(); static int64_t tg_enable_overcommit_group_gc(int64_t request_free_memory, RuntimeProfile* profile, bool is_minor_gc); @@ -220,7 +203,6 @@ class MemInfo { static std::atomic _s_allocator_cache_mem; static std::string _s_allocator_cache_mem_str; static std::atomic _s_virtual_memory_used; - static std::atomic _s_proc_mem_no_allocator_cache; static std::atomic _s_sys_mem_available; static int64_t _s_sys_mem_available_low_water_mark; diff --git a/be/src/vec/common/allocator.cpp b/be/src/vec/common/allocator.cpp index a72165f936ca0db..f6758a2dbb9c08a 100644 --- a/be/src/vec/common/allocator.cpp +++ b/be/src/vec/common/allocator.cpp @@ -28,6 +28,7 @@ // Allocator is used by too many files. For compilation speed, put dependencies in `.cpp` as much as possible. #include "runtime/fragment_mgr.h" +#include "runtime/memory/global_memory_arbitrator.h" #include "runtime/memory/mem_tracker_limiter.h" #include "runtime/memory/thread_mem_tracker_mgr.h" #include "runtime/thread_context.h" @@ -46,7 +47,7 @@ void Allocator::sys_memory_check(size_t if (doris::thread_context()->skip_memory_check != 0) { return; } - if (doris::MemTrackerLimiter::sys_mem_exceed_limit_check(size)) { + if (doris::GlobalMemoryArbitrator::is_exceed_hard_mem_limit(size)) { // Only thread attach query, and has not completely waited for thread_wait_gc_max_milliseconds, // will wait for gc, asynchronous cancel or throw bad::alloc. // Otherwise, if the external catch, directly throw bad::alloc. @@ -58,9 +59,9 @@ void Allocator::sys_memory_check(size_t doris::thread_context()->thread_mem_tracker()->peak_consumption(), doris::thread_context()->thread_mem_tracker()->consumption(), doris::thread_context()->thread_mem_tracker_mgr->last_consumer_tracker(), - doris::MemTrackerLimiter::process_limit_exceeded_errmsg_str()); + doris::GlobalMemoryArbitrator::process_limit_exceeded_errmsg_str()); - if (size > 1024l * 1024 * 1024 && !doris::enable_thread_catch_bad_alloc && + if (size > 1024L * 1024 * 1024 && !doris::enable_thread_catch_bad_alloc && !doris::config::disable_memory_gc) { // 1G err_msg += "\nAlloc Stacktrace:\n" + doris::get_stack_trace(); } @@ -83,7 +84,7 @@ void Allocator::sys_memory_check(size_t doris::config::thread_wait_gc_max_milliseconds, err_msg); while (wait_milliseconds < doris::config::thread_wait_gc_max_milliseconds) { std::this_thread::sleep_for(std::chrono::milliseconds(100)); - if (!doris::MemTrackerLimiter::sys_mem_exceed_limit_check(size)) { + if (!doris::GlobalMemoryArbitrator::is_exceed_hard_mem_limit(size)) { doris::MemInfo::refresh_interval_memory_growth += size; break; } From c962c3a0255c39e492678576d79657e04f602b6e Mon Sep 17 00:00:00 2001 From: lihangyu <15605149486@163.com> Date: Tue, 21 May 2024 14:01:59 +0800 Subject: [PATCH 093/111] [regression-test](prepared statment) optimize prepared_stmt.groovy to make test stable (#35087) --- .../prepared_stmt_p0/prepared_stmt.groovy | 28 +++++++++++++------ 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy index bf8959e54738ad7..46f8d7629caf10d 100644 --- a/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy +++ b/regression-test/suites/prepared_stmt_p0/prepared_stmt.groovy @@ -97,15 +97,24 @@ suite("test_prepared_stmt", "nonConcurrent") { ) """ - def insert_stmt = prepareStatement """ INSERT INTO ${tableName} VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?) """ - assertEquals(insert_stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement); - insert_prepared insert_stmt, 1231, 119291.11, "ddd", "laooq", null, "2020-01-01 12:36:38", null, "1022-01-01 11:30:38", "[2022-01-01 11:30:38, 2022-01-01 11:30:38, 2022-01-01 11:30:38]" - insert_prepared insert_stmt, 1232, 12222.99121135, "xxx", "laooq", "2023-01-02", "2020-01-01 12:36:38", 522.762, "2022-01-01 11:30:38", "[2023-01-01 11:30:38, 2023-01-01 11:30:38]" - insert_prepared insert_stmt, 1233, 1.392932911136, "yyy", "laooq", "2024-01-02", "2020-01-01 12:36:38", 52.862, "3022-01-01 11:30:38", "[2024-01-01 11:30:38, 2024-01-01 11:30:38, 2024-01-01 11:30:38]" - insert_prepared insert_stmt, 1234, 12919291.129191137, "xxddd", "laooq", "2025-01-02", "2020-01-01 12:36:38", 552.872, "4022-01-01 11:30:38", "[2025-01-01 11:30:38, 2025-01-01 11:30:38, 2025-01-01 11:30:38]" - insert_prepared insert_stmt, 1235, 991129292901.11138, "dd", null, "2120-01-02", "2020-01-01 12:36:38", 652.692, "5022-01-01 11:30:38", "[]" - insert_prepared insert_stmt, 1236, 100320.11139, "laa ddd", "laooq", "2220-01-02", "2020-01-01 12:36:38", 2.7692, "6022-01-01 11:30:38", "[null]" - insert_prepared insert_stmt, 1237, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", "[2025-01-01 11:30:38]" + // def insert_stmt = prepareStatement """ INSERT INTO ${tableName} VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?) """ + // assertEquals(insert_stmt.class, com.mysql.cj.jdbc.ServerPreparedStatement); + // insert_prepared insert_stmt, 1231, 119291.11, "ddd", "laooq", null, "2020-01-01 12:36:38", null, "1022-01-01 11:30:38", "[2022-01-01 11:30:38, 2022-01-01 11:30:38, 2022-01-01 11:30:38]" + // insert_prepared insert_stmt, 1232, 12222.99121135, "xxx", "laooq", "2023-01-02", "2020-01-01 12:36:38", 522.762, "2022-01-01 11:30:38", "[2023-01-01 11:30:38, 2023-01-01 11:30:38]" + // insert_prepared insert_stmt, 1233, 1.392932911136, "yyy", "laooq", "2024-01-02", "2020-01-01 12:36:38", 52.862, "3022-01-01 11:30:38", "[2024-01-01 11:30:38, 2024-01-01 11:30:38, 2024-01-01 11:30:38]" + // insert_prepared insert_stmt, 1234, 12919291.129191137, "xxddd", "laooq", "2025-01-02", "2020-01-01 12:36:38", 552.872, "4022-01-01 11:30:38", "[2025-01-01 11:30:38, 2025-01-01 11:30:38, 2025-01-01 11:30:38]" + // insert_prepared insert_stmt, 1235, 991129292901.11138, "dd", null, "2120-01-02", "2020-01-01 12:36:38", 652.692, "5022-01-01 11:30:38", "[]" + // insert_prepared insert_stmt, 1236, 100320.11139, "laa ddd", "laooq", "2220-01-02", "2020-01-01 12:36:38", 2.7692, "6022-01-01 11:30:38", "[null]" + // insert_prepared insert_stmt, 1237, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", "[2025-01-01 11:30:38]" + + sql """ INSERT INTO ${tableName} VALUES(1231, 119291.11, "ddd", "laooq", null, "2020-01-01 12:36:38", null, "1022-01-01 11:30:38", "[2022-01-01 11:30:38, 2022-01-01 11:30:38, 2022-01-01 11:30:38]") """ + sql """ INSERT INTO ${tableName} VALUES(1232, 12222.99121135, "xxx", "laooq", "2023-01-02", "2020-01-01 12:36:38", 522.762, "2022-01-01 11:30:38", "[2023-01-01 11:30:38, 2023-01-01 11:30:38]") """ + sql """ INSERT INTO ${tableName} VALUES(1233, 1.392932911136, "yyy", "laooq", "2024-01-02", "2020-01-01 12:36:38", 52.862, "3022-01-01 11:30:38", "[2024-01-01 11:30:38, 2024-01-01 11:30:38, 2024-01-01 11:30:38]") """ + sql """ INSERT INTO ${tableName} VALUES(1234, 12919291.129191137, "xxddd", "laooq", "2025-01-02", "2020-01-01 12:36:38", 552.872, "4022-01-01 11:30:38", "[2025-01-01 11:30:38, 2025-01-01 11:30:38, 2025-01-01 11:30:38]") """ + sql """ INSERT INTO ${tableName} VALUES(1235, 991129292901.11138, "dd", null, "2120-01-02", "2020-01-01 12:36:38", 652.692, "5022-01-01 11:30:38", "[]") """ + sql """ INSERT INTO ${tableName} VALUES(1236, 100320.11139, "laa ddd", "laooq", "2220-01-02", "2020-01-01 12:36:38", 2.7692, "6022-01-01 11:30:38", "[null]") """ + sql """ INSERT INTO ${tableName} VALUES(1237, 120939.11130, "a ddd", "laooq", "2030-01-02", "2020-01-01 12:36:38", 22.822, "7022-01-01 11:30:38", "[2025-01-01 11:30:38]") """ + sql """sync""" qt_sql """select * from ${tableName} order by 1, 2, 3""" qt_sql """select * from ${tableName} order by 1, 2, 3""" @@ -152,6 +161,7 @@ suite("test_prepared_stmt", "nonConcurrent") { sql """insert into mytable1 values(1,1,'user1',10);""" sql """insert into mytable1 values(1,1,'user1',10);""" sql """insert into mytable1 values(1,1,'user1',10);""" + sql "sync" stmt_read = prepareStatement "SELECT *, ? FROM (select *, ? from mytable1 where citycode = ?) AS `SpotfireCustomQuery1` WHERE 1 = 1" stmt_read.setInt(1, 12345) stmt_read.setInt(2, 1234) From 7ae83b60fdfaa633bba6e75732e437075773462d Mon Sep 17 00:00:00 2001 From: xzj7019 <131111794+xzj7019@users.noreply.github.com> Date: Tue, 21 May 2024 14:20:48 +0800 Subject: [PATCH 094/111] [opt](Nereids) opt locality under multi-replica (#34927) Make tablet locality fixed under multi-replica cases. Session variable: set enable_ordered_scan_range_locations = true, default false; 3 replica tpcds 100g: 7% improvement --- .../java/org/apache/doris/qe/Coordinator.java | 72 ++++++++++++++----- .../org/apache/doris/qe/SessionVariable.java | 5 ++ .../org/apache/doris/qe/CoordinatorTest.java | 8 +-- 3 files changed, 65 insertions(+), 20 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index 1c272b1dbb1926b..ba7ea54b0980986 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -144,6 +144,7 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -2264,7 +2265,13 @@ private Map getReplicaNumPerHostForOlapTable() { private void computeScanRangeAssignment() throws Exception { Map assignedBytesPerHost = Maps.newHashMap(); Map replicaNumPerHost = getReplicaNumPerHostForOlapTable(); - Collections.shuffle(scanNodes); + boolean isAllOlapTables = scanNodes.stream().allMatch(e -> e instanceof OlapScanNode); + boolean isEnableOrderedLocations = ConnectContext.get() != null + && ConnectContext.get().getSessionVariable().enableOrderedScanRangeLocations + && isAllOlapTables; + if (isEnableOrderedLocations) { + sortScanNodes(); + } // set scan ranges/locations for scan nodes for (ScanNode scanNode : scanNodes) { if (!(scanNode instanceof ExternalScanNode)) { @@ -2277,6 +2284,9 @@ private void computeScanRangeAssignment() throws Exception { // only analysis olap scan node continue; } + if (isEnableOrderedLocations) { + sortScanRangeLocations(locations); + } Set scanNodeIds = fragmentIdToScanNodeIds.computeIfAbsent(scanNode.getFragmentId(), k -> Sets.newHashSet()); scanNodeIds.add(scanNode.getId().asInt()); @@ -2296,7 +2306,8 @@ private void computeScanRangeAssignment() throws Exception { // A fragment may contain both colocate join and bucket shuffle join // on need both compute scanRange to init basic data for query coordinator if (fragmentContainsColocateJoin) { - computeScanRangeAssignmentByColocate((OlapScanNode) scanNode, assignedBytesPerHost, replicaNumPerHost); + computeScanRangeAssignmentByColocate((OlapScanNode) scanNode, assignedBytesPerHost, + replicaNumPerHost, isEnableOrderedLocations); } if (fragmentContainsBucketShuffleJoin) { bucketShuffleJoinController.computeScanRangeAssignmentByBucket((OlapScanNode) scanNode, @@ -2304,7 +2315,7 @@ private void computeScanRangeAssignment() throws Exception { } if (!(fragmentContainsColocateJoin || fragmentContainsBucketShuffleJoin)) { computeScanRangeAssignmentByScheduler(scanNode, locations, assignment, assignedBytesPerHost, - replicaNumPerHost); + replicaNumPerHost, isEnableOrderedLocations); } } } @@ -2312,7 +2323,7 @@ private void computeScanRangeAssignment() throws Exception { // To ensure the same bucketSeq tablet to the same execHostPort private void computeScanRangeAssignmentByColocate( final OlapScanNode scanNode, Map assignedBytesPerHost, - Map replicaNumPerHost) throws Exception { + Map replicaNumPerHost, boolean isEnableOrderedLocations) throws Exception { if (!fragmentIdToSeqToAddressMap.containsKey(scanNode.getFragmentId())) { fragmentIdToSeqToAddressMap.put(scanNode.getFragmentId(), new HashMap<>()); fragmentIdTobucketSeqToScanRangeMap.put(scanNode.getFragmentId(), new BucketSeqToScanRange()); @@ -2333,7 +2344,8 @@ private void computeScanRangeAssignmentByColocate( List locations = scanNode.bucketSeq2locations.get(bucketSeq); if (!bucketSeqToAddress.containsKey(bucketSeq)) { getExecHostPortForFragmentIDAndBucketSeq(locations.get(0), - scanNode.getFragmentId(), bucketSeq, assignedBytesPerHost, replicaNumPerHost); + scanNode.getFragmentId(), bucketSeq, assignedBytesPerHost, + replicaNumPerHost, isEnableOrderedLocations); } for (TScanRangeLocations location : locations) { @@ -2355,29 +2367,55 @@ private void computeScanRangeAssignmentByColocate( //ensure bucket sequence distribued to every host evenly private void getExecHostPortForFragmentIDAndBucketSeq(TScanRangeLocations seqLocation, PlanFragmentId fragmentId, Integer bucketSeq, Map assignedBytesPerHost, - Map replicaNumPerHost) + Map replicaNumPerHost, boolean isEnableOrderedLocations) throws Exception { Reference backendIdRef = new Reference(); - selectBackendsByRoundRobin(seqLocation, assignedBytesPerHost, replicaNumPerHost, backendIdRef); + selectBackendsByRoundRobin(seqLocation, assignedBytesPerHost, replicaNumPerHost, + backendIdRef, isEnableOrderedLocations); Backend backend = this.idToBackend.get(backendIdRef.getRef()); TNetworkAddress execHostPort = new TNetworkAddress(backend.getHost(), backend.getBePort()); this.addressToBackendID.put(execHostPort, backendIdRef.getRef()); this.fragmentIdToSeqToAddressMap.get(fragmentId).put(bucketSeq, execHostPort); } + private void sortScanNodes() { + Collections.sort(scanNodes, new Comparator() { + @Override + public int compare(ScanNode s1, ScanNode s2) { + return java.lang.Integer.compare(s1.getId().asInt(), s2.getId().asInt()); + } + }); + } + + private void sortScanRangeLocations(List locations) { + Collections.sort(locations, new Comparator() { + @Override + public int compare(TScanRangeLocations o1, TScanRangeLocations o2) { + return org.apache.thrift.TBaseHelper.compareTo( + o1.getScanRange().getPaloScanRange().tablet_id, + o2.getScanRange().getPaloScanRange().tablet_id); + } + }); + } + public TScanRangeLocation selectBackendsByRoundRobin(TScanRangeLocations seqLocation, Map assignedBytesPerHost, Map replicaNumPerHost, - Reference backendIdRef) throws UserException { + Reference backendIdRef, + boolean isEnableOrderedLocations) throws UserException { + List locations = seqLocation.getLocations(); + if (isEnableOrderedLocations) { + Collections.sort(locations); + } if (!Config.enable_local_replica_selection) { - return selectBackendsByRoundRobin(seqLocation.getLocations(), assignedBytesPerHost, replicaNumPerHost, + return selectBackendsByRoundRobin(locations, assignedBytesPerHost, replicaNumPerHost, backendIdRef); } List localLocations = new ArrayList<>(); List nonlocalLocations = new ArrayList<>(); long localBeId = Env.getCurrentSystemInfo().getBackendIdByHost(FrontendOptions.getLocalHostAddress()); - for (final TScanRangeLocation location : seqLocation.getLocations()) { + for (final TScanRangeLocation location : locations) { if (location.backend_id == localBeId) { localLocations.add(location); } else { @@ -2395,14 +2433,15 @@ public TScanRangeLocation selectBackendsByRoundRobin(TScanRangeLocations seqLoca } } - public TScanRangeLocation selectBackendsByRoundRobin(List locations, + public TScanRangeLocation selectBackendsByRoundRobin(List sortedLocations, Map assignedBytesPerHost, Map replicaNumPerHost, Reference backendIdRef) throws UserException { Long minAssignedBytes = Long.MAX_VALUE; Long minReplicaNum = Long.MAX_VALUE; TScanRangeLocation minLocation = null; Long step = 1L; - for (final TScanRangeLocation location : locations) { + + for (final TScanRangeLocation location : sortedLocations) { Long assignedBytes = findOrInsert(assignedBytesPerHost, location.server, 0L); if (assignedBytes < minAssignedBytes || (assignedBytes.equals(minAssignedBytes) && replicaNumPerHost.get(location.server) < minReplicaNum)) { @@ -2411,10 +2450,10 @@ public TScanRangeLocation selectBackendsByRoundRobin(List lo minLocation = location; } } - for (TScanRangeLocation location : locations) { + for (TScanRangeLocation location : sortedLocations) { replicaNumPerHost.put(location.server, replicaNumPerHost.get(location.server) - 1); } - TScanRangeLocation location = SimpleScheduler.getLocation(minLocation, locations, + TScanRangeLocation location = SimpleScheduler.getLocation(minLocation, sortedLocations, this.idToBackend, backendIdRef); assignedBytesPerHost.put(location.server, assignedBytesPerHost.get(location.server) + step); @@ -2426,7 +2465,8 @@ private void computeScanRangeAssignmentByScheduler( final List locations, FragmentScanRangeAssignment assignment, Map assignedBytesPerHost, - Map replicaNumPerHost) throws Exception { + Map replicaNumPerHost, + boolean isEnableOrderedLocations) throws Exception { // Type of locations is List, it could have elements that have same "location" // and we do have this situation for some scan node. // The duplicate "location" will NOT be filtered by FragmentScanRangeAssignment, @@ -2435,7 +2475,7 @@ private void computeScanRangeAssignmentByScheduler( for (TScanRangeLocations scanRangeLocations : locations) { Reference backendIdRef = new Reference(); TScanRangeLocation minLocation = selectBackendsByRoundRobin(scanRangeLocations, - assignedBytesPerHost, replicaNumPerHost, backendIdRef); + assignedBytesPerHost, replicaNumPerHost, backendIdRef, isEnableOrderedLocations); Backend backend = this.idToBackend.get(backendIdRef.getRef()); TNetworkAddress execHostPort = new TNetworkAddress(backend.getHost(), backend.getBePort()); this.addressToBackendID.put(execHostPort, backendIdRef.getRef()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 2af7725e029e997..ac653b82ec9eec8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -420,6 +420,8 @@ public class SessionVariable implements Serializable, Writable { */ public static final String ENABLE_UNIFIED_LOAD = "enable_unified_load"; + public static final String ENABLE_ORDERED_SCAN_RANGE_LOCATIONS = "enable_ordered_scan_range_locations"; + public static final String ENABLE_PARQUET_LAZY_MAT = "enable_parquet_lazy_materialization"; public static final String ENABLE_ORC_LAZY_MAT = "enable_orc_lazy_materialization"; @@ -1468,6 +1470,9 @@ public void setEnableLeftZigZag(boolean enableLeftZigZag) { @VariableMgr.VarAttr(name = ENABLE_CTE_MATERIALIZE) public boolean enableCTEMaterialize = true; + @VariableMgr.VarAttr(name = ENABLE_ORDERED_SCAN_RANGE_LOCATIONS) + public boolean enableOrderedScanRangeLocations = false; + @VariableMgr.VarAttr(name = ENABLE_ANALYZE_COMPLEX_TYPE_COLUMN) public boolean enableAnalyzeComplexTypeColumn = false; diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorTest.java index ca8109e40e3070b..4c38ddd27495fc9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/CoordinatorTest.java @@ -593,7 +593,7 @@ public void testComputeScanRangeAssignmentByScheduler() { locations.add(tScanRangeLocations); locations.add(tScanRangeLocations1); Deencapsulation.invoke(coordinator, "computeScanRangeAssignmentByScheduler", - olapScanNode, locations, assignment, assignedBytesPerHost, replicaNumPerHost); + olapScanNode, locations, assignment, assignedBytesPerHost, replicaNumPerHost, false); for (Map.Entry entry : assignment.entrySet()) { Map> addr = (HashMap>) entry.getValue(); for (Map.Entry item : addr.entrySet()) { @@ -653,11 +653,11 @@ public void testGetExecHostPortForFragmentIDAndBucketSeq() { replicaNumPerHost.put(tScanRangeLocation2.server, 1L); Deencapsulation.invoke(coordinator, "getExecHostPortForFragmentIDAndBucketSeq", tScanRangeLocations, - planFragmentId, 1, assignedBytesPerHost, replicaNumPerHost); + planFragmentId, 1, assignedBytesPerHost, replicaNumPerHost, false); Deencapsulation.invoke(coordinator, "getExecHostPortForFragmentIDAndBucketSeq", tScanRangeLocations, - planFragmentId, 2, assignedBytesPerHost, replicaNumPerHost); + planFragmentId, 2, assignedBytesPerHost, replicaNumPerHost, false); Deencapsulation.invoke(coordinator, "getExecHostPortForFragmentIDAndBucketSeq", tScanRangeLocations, - planFragmentId, 3, assignedBytesPerHost, replicaNumPerHost); + planFragmentId, 3, assignedBytesPerHost, replicaNumPerHost, false); List hosts = new ArrayList<>(); for (Map.Entry item : assignedBytesPerHost.entrySet()) { Assert.assertTrue((Long) item.getValue() == 1); From b96148c9cdeb7f98d402ab2019c9a5591a91b833 Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Tue, 21 May 2024 16:36:45 +0800 Subject: [PATCH 095/111] [Fix](function) fix days/weeks_diff result wrong on BE #35104 select days_diff('2024-01-01 00:00:00', '2023-12-31 23:59:59'); should be 0 but got 1 on BE. --- be/src/vec/runtime/vdatetime_value.h | 13 ++++++++ .../data/correctness_p0/test_date_diff.out | 21 ++++++++++++ .../correctness_p0/test_date_diff.groovy | 32 +++++++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 regression-test/data/correctness_p0/test_date_diff.out create mode 100644 regression-test/suites/correctness_p0/test_date_diff.groovy diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index 2031c782267175d..b1197b611ed0798 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -1405,10 +1406,22 @@ int64_t datetime_diff(const DateV2Value& ts_value1, const DateV2Value& t } case WEEK: { int day = ts_value2.daynr() - ts_value1.daynr(); + int64_t ms_diff = ts_value2.time_part_diff_microsecond(ts_value1); + if (day > 0 && ms_diff < 0) { + day--; + } else if (day < 0 && ms_diff > 0) { + day++; + } return day / 7; } case DAY: { int day = ts_value2.daynr() - ts_value1.daynr(); + int64_t ms_diff = ts_value2.time_part_diff_microsecond(ts_value1); + if (day > 0 && ms_diff < 0) { + day--; + } else if (day < 0 && ms_diff > 0) { + day++; + } return day; } case HOUR: { diff --git a/regression-test/data/correctness_p0/test_date_diff.out b/regression-test/data/correctness_p0/test_date_diff.out new file mode 100644 index 000000000000000..4f2470fe4ee2dae --- /dev/null +++ b/regression-test/data/correctness_p0/test_date_diff.out @@ -0,0 +1,21 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !days -- +0 +0 +1 +2 +3 +4 +5 +6 + +-- !weeks -- +0 +0 +0 +0 +0 +0 +0 +0 + diff --git a/regression-test/suites/correctness_p0/test_date_diff.groovy b/regression-test/suites/correctness_p0/test_date_diff.groovy new file mode 100644 index 000000000000000..91e9cef6eb362fb --- /dev/null +++ b/regression-test/suites/correctness_p0/test_date_diff.groovy @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_date_diff") { + sql """ + create table dt6( + k0 datetime(6) null + ) + DISTRIBUTED BY HASH(`k0`) BUCKETS auto + properties("replication_num" = "1"); + """ + sql """ + insert into dt6 values ("0000-01-01 12:00:00"), ("0000-01-02 12:00:00"), ("0000-01-03 12:00:00"), ("0000-01-04 12:00:00"), + ("0000-01-05 12:00:00"), ("0000-01-06 12:00:00"), ("0000-01-07 12:00:00"), ("0000-01-08 12:00:00"); + """ + qt_days """ select days_diff(k0, '0000-01-01 13:00:00') from dt6 order by k0; """ + qt_weeks """ select weeks_diff(k0, '0000-01-01 13:00:00') from dt6 order by k0; """ +} \ No newline at end of file From e8fb47bec1a1cfc7b07a6ed4eb36283407a4a9fe Mon Sep 17 00:00:00 2001 From: zhiqiang Date: Tue, 21 May 2024 19:22:58 +0800 Subject: [PATCH 096/111] [fix](broker load) Make Config.enable_pipeline_load works as expected for BrokerLoad (#35105) * FIX LOAD PROFILE * FIX --- fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java index ba7ea54b0980986..623237144cabc19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java @@ -385,6 +385,10 @@ public Coordinator(Long jobId, TUniqueId queryId, DescriptorTable descTable, Lis nextInstanceId.setHi(queryId.hi); nextInstanceId.setLo(queryId.lo + 1); this.executionProfile = new ExecutionProfile(queryId, fragments); + // Need to be same with LoadLoadingTask + // https://github.com/apache/doris/blob/bd6f5b6a0e5f1b12744607336123d7f97eb76af9/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java#L155 + this.enablePipelineEngine = Config.enable_pipeline_load; + this.enablePipelineXEngine = Config.enable_pipeline_load; } private void setFromUserProperty(ConnectContext connectContext) { From ced0093d7411cf98455d32f40ba2343874ae02cf Mon Sep 17 00:00:00 2001 From: Jerry Hu Date: Tue, 21 May 2024 19:39:39 +0800 Subject: [PATCH 097/111] [fix](mem_tracker] attach mem tracker in FragmentMgr::apply_filter (#35128) --- be/src/runtime/fragment_mgr.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/be/src/runtime/fragment_mgr.cpp b/be/src/runtime/fragment_mgr.cpp index 3a1d722e8056ce2..427f097cd2e601c 100644 --- a/be/src/runtime/fragment_mgr.cpp +++ b/be/src/runtime/fragment_mgr.cpp @@ -1313,6 +1313,7 @@ Status FragmentMgr::apply_filter(const PPublishFilterRequest* request, std::shared_ptr fragment_executor; std::shared_ptr pip_context; + QueryThreadContext query_thread_context; RuntimeFilterMgr* runtime_filter_mgr = nullptr; if (is_pipeline) { @@ -1326,6 +1327,8 @@ Status FragmentMgr::apply_filter(const PPublishFilterRequest* request, DCHECK(pip_context != nullptr); runtime_filter_mgr = pip_context->get_query_ctx()->runtime_filter_mgr(); + query_thread_context = {pip_context->get_query_ctx()->query_id(), + pip_context->get_query_ctx()->query_mem_tracker}; } else { std::unique_lock lock(_lock); auto iter = _fragment_instance_map.find(tfragment_instance_id); @@ -1338,8 +1341,11 @@ Status FragmentMgr::apply_filter(const PPublishFilterRequest* request, DCHECK(fragment_executor != nullptr); runtime_filter_mgr = fragment_executor->runtime_state()->get_query_ctx()->runtime_filter_mgr(); + query_thread_context = {fragment_executor->get_query_ctx()->query_id(), + fragment_executor->get_query_ctx()->query_mem_tracker}; } + SCOPED_ATTACH_TASK(query_thread_context); return runtime_filter_mgr->update_filter(request, attach_data); } From a8c24d7698f89a1f9ebb63e6e01ff78bbc73a575 Mon Sep 17 00:00:00 2001 From: zclllyybb Date: Tue, 21 May 2024 20:27:09 +0800 Subject: [PATCH 098/111] [Fix](function) fix overflow of date_add function (#35080) fix overflow of date_add function --- be/src/common/status.h | 7 ++ be/src/vec/runtime/vdatetime_value.cpp | 74 +++++++++---------- be/src/vec/runtime/vdatetime_value.h | 43 +++++++++-- .../test_date_function_const.groovy | 5 ++ .../test_date_function.groovy | 11 ++- 5 files changed, 87 insertions(+), 53 deletions(-) diff --git a/be/src/common/status.h b/be/src/common/status.h index cf9b42a3c69668c..6e5a75f7966495b 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -619,6 +619,13 @@ inline std::string Status::to_string_no_stack() const { } \ } while (false) +#define PROPAGATE_FALSE(stmt) \ + do { \ + if (UNLIKELY(!static_cast(stmt))) { \ + return false; \ + } \ + } while (false) + #define THROW_IF_ERROR(stmt) \ do { \ Status _status_ = (stmt); \ diff --git a/be/src/vec/runtime/vdatetime_value.cpp b/be/src/vec/runtime/vdatetime_value.cpp index fcdaedea1839de9..dd0ce3414934016 100644 --- a/be/src/vec/runtime/vdatetime_value.cpp +++ b/be/src/vec/runtime/vdatetime_value.cpp @@ -42,8 +42,6 @@ namespace doris { -static constexpr int s_days_in_month[13] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; - static const char* s_ab_month_name[] = {"", "Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec", nullptr}; @@ -74,7 +72,7 @@ bool VecDateTimeValue::check_range(uint32_t year, uint32_t month, uint32_t day, bool VecDateTimeValue::check_date(uint32_t year, uint32_t month, uint32_t day) { if (month == 2 && day == 29 && doris::is_leap(year)) return false; - if (year > 9999 || month == 0 || month > 12 || day > s_days_in_month[month] || day == 0) { + if (year > 9999 || month == 0 || month > 12 || day > S_DAYS_IN_MONTH[month] || day == 0) { return true; } return false; @@ -520,8 +518,8 @@ bool VecDateTimeValue::get_date_from_daynr(uint64_t daynr) { } } month = 1; - while (days_of_year > s_days_in_month[month]) { - days_of_year -= s_days_in_month[month]; + while (days_of_year > S_DAYS_IN_MONTH[month]) { + days_of_year -= S_DAYS_IN_MONTH[month]; month++; } day = days_of_year + leap_day; @@ -1679,8 +1677,8 @@ bool VecDateTimeValue::date_add_interval(const TimeInterval& interval) { return false; } _month = (months % 12) + 1; - if (_day > s_days_in_month[_month]) { - _day = s_days_in_month[_month]; + if (_day > S_DAYS_IN_MONTH[_month]) { + _day = S_DAYS_IN_MONTH[_month]; if (_month == 2 && doris::is_leap(_year)) { _day++; } @@ -1931,11 +1929,11 @@ bool DateV2Value::is_invalid(uint32_t year, uint32_t month, uint32_t day, uin if (only_time_part) { return false; } - if (year < MIN_YEAR || year > MAX_YEAR) { + if (year > MAX_YEAR) { return true; } if (month == 2 && day == 29 && doris::is_leap(year)) return false; - if (month == 0 || month > 12 || day > s_days_in_month[month] || day == 0) { + if (month == 0 || month > 12 || day > S_DAYS_IN_MONTH[month] || day == 0) { return true; } return false; @@ -1960,7 +1958,7 @@ void DateV2Value::format_datetime(uint32_t* date_val, bool* carry_bits) const date_val[1] += 1; carry_bits[2] = true; } - } else if (date_val[2] == s_days_in_month[date_val[1]] + 1 && carry_bits[3]) { + } else if (date_val[2] == S_DAYS_IN_MONTH[date_val[1]] + 1 && carry_bits[3]) { date_val[2] = 1; date_val[1] += 1; carry_bits[2] = true; @@ -2916,8 +2914,8 @@ bool DateV2Value::get_date_from_daynr(uint64_t daynr) { } } month = 1; - while (days_of_year > s_days_in_month[month]) { - days_of_year -= s_days_in_month[month]; + while (days_of_year > S_DAYS_IN_MONTH[month]) { + days_of_year -= S_DAYS_IN_MONTH[month]; month++; } day = days_of_year + leap_day; @@ -2974,30 +2972,27 @@ bool DateV2Value::date_add_interval(const TimeInterval& interval, DateV2Value to_value.set_time(seconds / 3600, (seconds / 60) % 60, seconds % 60, microseconds); } else if constexpr (unit == YEAR) { // This only change year information - to_value.template set_time_unit(date_v2_value_.year_ + interval.year); - if (to_value.year() > 9999) { - return false; - } + PROPAGATE_FALSE(to_value.template set_time_unit(date_v2_value_.year_ + + interval.year)); if (date_v2_value_.month_ == 2 && date_v2_value_.day_ == 29 && !doris::is_leap(to_value.year())) { - to_value.template set_time_unit(28); + // add year. so if from Leap Year to Equal Year, use last day of Feb(29 to 28) + PROPAGATE_FALSE(to_value.template set_time_unit(28)); } } else if constexpr (unit == QUARTER || unit == MONTH || unit == YEAR_MONTH) { // This will change month and year information, maybe date. int64_t months = date_v2_value_.year_ * 12 + date_v2_value_.month_ - 1 + 12 * interval.year + interval.month; - to_value.template set_time_unit(months / 12); if (months < 0) { return false; } - if (to_value.year() > MAX_YEAR) { - return false; - } - to_value.template set_time_unit((months % 12) + 1); - if (date_v2_value_.day_ > s_days_in_month[to_value.month()]) { - date_v2_value_.day_ = s_days_in_month[to_value.month()]; + PROPAGATE_FALSE(to_value.template set_time_unit(months / 12)); + PROPAGATE_FALSE(to_value.template set_time_unit((months % 12) + 1)); + if (date_v2_value_.day_ > S_DAYS_IN_MONTH[to_value.month()]) { + date_v2_value_.day_ = S_DAYS_IN_MONTH[to_value.month()]; if (to_value.month() == 2 && doris::is_leap(to_value.year())) { - to_value.template set_time_unit(date_v2_value_.day_ + 1); + PROPAGATE_FALSE( + to_value.template set_time_unit(date_v2_value_.day_ + 1)); } } } @@ -3050,30 +3045,27 @@ bool DateV2Value::date_add_interval(const TimeInterval& interval) { } } else if constexpr (unit == YEAR) { // This only change year information - this->template set_time_unit(date_v2_value_.year_ + interval.year); - if (this->year() > 9999) { - return false; - } + PROPAGATE_FALSE( + this->template set_time_unit(date_v2_value_.year_ + interval.year)); if (date_v2_value_.month_ == 2 && date_v2_value_.day_ == 29 && !doris::is_leap(this->year())) { - this->template set_time_unit(28); + // add year. so if from Leap Year to Equal Year, use last day of Feb(29 to 28) + PROPAGATE_FALSE(this->template set_time_unit(28)); } } else if constexpr (unit == QUARTER || unit == MONTH || unit == YEAR_MONTH) { // This will change month and year information, maybe date. int64_t months = date_v2_value_.year_ * 12 + date_v2_value_.month_ - 1 + 12 * interval.year + interval.month; - this->template set_time_unit(months / 12); if (months < 0) { return false; } - if (this->year() > MAX_YEAR) { - return false; - } - this->template set_time_unit((months % 12) + 1); - if (date_v2_value_.day_ > s_days_in_month[this->month()]) { - date_v2_value_.day_ = s_days_in_month[this->month()]; + PROPAGATE_FALSE(this->template set_time_unit(months / 12)); + PROPAGATE_FALSE(this->template set_time_unit((months % 12) + 1)); + if (date_v2_value_.day_ > S_DAYS_IN_MONTH[this->month()]) { + date_v2_value_.day_ = S_DAYS_IN_MONTH[this->month()]; if (this->month() == 2 && doris::is_leap(this->year())) { - this->template set_time_unit(date_v2_value_.day_ + 1); + PROPAGATE_FALSE( + this->template set_time_unit(date_v2_value_.day_ + 1)); } } } @@ -3102,13 +3094,13 @@ bool DateV2Value::date_set_interval(const TimeInterval& interval) { } } else if constexpr (unit == YEAR) { this->set_time(0, 1, 1, 0, 0, 0, 0); - this->template set_time_unit(interval.year); + PROPAGATE_FALSE(this->template set_time_unit(interval.year)); } else if constexpr (unit == MONTH) { // This will change month and year information, maybe date. this->set_time(0, 1, 1, 0, 0, 0, 0); int64_t months = 12 * interval.year + interval.month; - this->template set_time_unit(months / 12); - this->template set_time_unit((months % 12) + 1); + PROPAGATE_FALSE(this->template set_time_unit(months / 12)); + PROPAGATE_FALSE(this->template set_time_unit((months % 12) + 1)); } return true; } diff --git a/be/src/vec/runtime/vdatetime_value.h b/be/src/vec/runtime/vdatetime_value.h index b1197b611ed0798..8fcb45a64400039 100644 --- a/be/src/vec/runtime/vdatetime_value.h +++ b/be/src/vec/runtime/vdatetime_value.h @@ -19,15 +19,13 @@ #include #include -#include -#include #include #include #include +#include #include #include -#include #include #include #include @@ -168,6 +166,8 @@ constexpr int HOUR_PER_DAY = 24; constexpr int64_t SECOND_PER_HOUR = 3600; constexpr int64_t SECOND_PER_MINUTE = 60; +inline constexpr int S_DAYS_IN_MONTH[13] = {0, 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + constexpr size_t const_length(const char* str) { return (str == nullptr || *str == 0) ? 0 : const_length(str + 1) + 1; } @@ -201,7 +201,11 @@ static constexpr uint64_t MAX_DATETIME_V2 = ((uint64_t)MAX_DATE_V2 << TIME_PART_ static constexpr uint64_t MIN_DATETIME_V2 = (uint64_t)MIN_DATE_V2 << TIME_PART_LENGTH; static constexpr uint32_t MAX_YEAR = 9999; -static constexpr uint32_t MIN_YEAR = 0; +static constexpr uint32_t MAX_MONTH = 12; +static constexpr uint32_t MAX_HOUR = 23; +static constexpr uint32_t MAX_MINUTE = 59; +static constexpr uint32_t MAX_SECOND = 59; +static constexpr uint32_t MAX_MICROSECOND = 999999; static constexpr uint32_t DATEV2_YEAR_WIDTH = 23; static constexpr uint32_t DATETIMEV2_YEAR_WIDTH = 18; @@ -1159,31 +1163,58 @@ class DateV2Value { bool get_date_from_daynr(uint64_t); + // should do check template - void set_time_unit(uint32_t val) { + bool set_time_unit(uint32_t val) { + // is uint so need check upper bound only if constexpr (unit == TimeUnit::YEAR) { + if (val > MAX_YEAR) [[unlikely]] { + return false; + } date_v2_value_.year_ = val; } else if constexpr (unit == TimeUnit::MONTH) { + if (val > MAX_MONTH) [[unlikely]] { + return false; + } date_v2_value_.month_ = val; } else if constexpr (unit == TimeUnit::DAY) { + DCHECK(date_v2_value_.month_ <= MAX_MONTH); + DCHECK(date_v2_value_.month_ != 0); + if (val > S_DAYS_IN_MONTH[date_v2_value_.month_] && + !(is_leap(date_v2_value_.year_) && date_v2_value_.month_ == 2 && val == 29)) { + return false; + } date_v2_value_.day_ = val; } else if constexpr (unit == TimeUnit::HOUR) { if constexpr (is_datetime) { + if (val > MAX_HOUR) [[unlikely]] { + return false; + } date_v2_value_.hour_ = val; } } else if constexpr (unit == TimeUnit::MINUTE) { if constexpr (is_datetime) { + if (val > MAX_MINUTE) [[unlikely]] { + return false; + } date_v2_value_.minute_ = val; } } else if constexpr (unit == TimeUnit::SECOND) { if constexpr (is_datetime) { + if (val > MAX_SECOND) [[unlikely]] { + return false; + } date_v2_value_.second_ = val; } - } else if constexpr (unit == TimeUnit::SECOND_MICROSECOND) { + } else if constexpr (unit == TimeUnit::MICROSECOND) { if constexpr (is_datetime) { + if (val > MAX_MICROSECOND) [[unlikely]] { + return false; + } date_v2_value_.microsecond_ = val; } } + return true; } operator int64_t() const { return to_int64(); } diff --git a/regression-test/suites/correctness/test_date_function_const.groovy b/regression-test/suites/correctness/test_date_function_const.groovy index 0caded34f5d3d1c..d1ba4db4e689877 100644 --- a/regression-test/suites/correctness/test_date_function_const.groovy +++ b/regression-test/suites/correctness/test_date_function_const.groovy @@ -58,4 +58,9 @@ suite("test_date_function_const") { sql("""select date_add(CURRENT_DATE(),-2);""") notContains("00:00:00") } + + test { + sql """select date_add("1900-01-01 12:00:00.123456", interval 10000000000 month);""" + exception "Operation months_add 133705200962757184 1410065408 out of range" + } } diff --git a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy index d1e8fcab8052ec3..2f1ef98b4ea3d5c 100644 --- a/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy +++ b/regression-test/suites/nereids_p0/sql_functions/datetime_functions/test_date_function.groovy @@ -600,9 +600,9 @@ suite("test_date_function") { qt_sql """ select seconds_sub(test_time2,1) result from ${tableName}; """ // test last_day for vec - sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ DROP TABLE IF EXISTS test_time_add_sub_function; """ sql """ - CREATE TABLE IF NOT EXISTS ${tableName} ( + CREATE TABLE IF NOT EXISTS test_time_add_sub_function ( birth date, birth1 datev2, birth2 datetime, @@ -612,7 +612,7 @@ suite("test_date_function") { PROPERTIES( "replication_allocation" = "tag.location.default: 1"); """ sql """ - insert into ${tableName} values + insert into test_time_add_sub_function values ('2022-01-01', '2022-01-01', '2022-01-01 00:00:00', '2022-01-01 00:00:00'), ('2000-02-01', '2000-02-01', '2000-02-01 00:00:00', '2000-02-01 00:00:00.123'), ('2022-02-27', '2022-02-27', '2022-02-27 00:00:00', '2022-02-27 00:00:00'), @@ -620,11 +620,10 @@ suite("test_date_function") { qt_sql """ select last_day(birth), last_day(birth1), last_day(birth2), last_day(birth3) - from ${tableName}; + from test_time_add_sub_function; """ - sql """ DROP TABLE IF EXISTS ${tableName}; """ - sql """ DROP TABLE IF EXISTS ${tableName}; """ + sql """ DROP TABLE IF EXISTS test_time_add_sub_function; """ sql """ CREATE TABLE IF NOT EXISTS ${tableName} ( birth date, From af7b16f213ed5ccc25d753b8cd4317d0a1be2be5 Mon Sep 17 00:00:00 2001 From: Xujian Duan <50550370+DarvenDuan@users.noreply.github.com> Date: Tue, 21 May 2024 20:28:02 +0800 Subject: [PATCH 099/111] [optimize](desc) display the correct data type of aggStateType (#34968) If a table column is AGG_STATE type, we can't get the clear defined data type if we use `desc tbl` statement. create table a_table( k1 int null, k2 agg_state generic, k3 agg_state generic ) aggregate key (k1) distributed BY hash(k1) buckets 3 properties("replication_num" = "1"); before optimize: mysql> desc a_table; +-------+------------------------------------------------+------+-------+---------+---------+ | Field | Type | Null | Key | Default | Extra | +-------+------------------------------------------------+------+-------+---------+---------+ | k1 | INT | Yes | true | NULL | | | k2 | org.apache.doris.catalog.AggStateType@239f771c | No | false | NULL | GENERIC | | k3 | org.apache.doris.catalog.AggStateType@2e535f50 | No | false | NULL | GENERIC | +-------+------------------------------------------------+------+-------+---------+---------+ 3 rows in set (0.00 sec) after optimize: mysql> desc a_table; +-------+------------------------------------+------+-------+---------+---------+ | Field | Type | Null | Key | Default | Extra | +-------+------------------------------------+------+-------+---------+---------+ | k1 | INT | Yes | true | NULL | | | k2 | AGG_STATE | No | false | NULL | GENERIC | | k3 | AGG_STATE | No | false | NULL | GENERIC | +-------+------------------------------------+------+-------+---------+---------+ Co-authored-by: duanxujian --- .../src/main/java/org/apache/doris/catalog/AggStateType.java | 5 +++++ .../main/java/org/apache/doris/analysis/DescribeStmt.java | 3 +-- .../org/apache/doris/common/proc/IndexSchemaProcNode.java | 4 +--- .../datatype_p0/agg_state/nereids/test_agg_state_nereids.out | 4 ++++ .../data/datatype_p0/agg_state/test_agg_state.out | 4 ++++ .../agg_state/nereids/test_agg_state_nereids.groovy | 2 ++ .../suites/datatype_p0/agg_state/test_agg_state.groovy | 2 ++ 7 files changed, 19 insertions(+), 5 deletions(-) diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/AggStateType.java b/fe/fe-common/src/main/java/org/apache/doris/catalog/AggStateType.java index e28364103f4b0e0..d2c5b625ca62cf7 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/catalog/AggStateType.java +++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/AggStateType.java @@ -90,6 +90,11 @@ public String toSql(int depth) { return stringBuilder.toString(); } + @Override + public String toString() { + return toSql(); + } + @Override protected String prettyPrint(int lpad) { return Strings.repeat(" ", lpad) + toSql(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java index b0c4ecb2527ceda..45883fd6f46dbf4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java @@ -129,14 +129,13 @@ public void analyze(Analyzer analyzer) throws UserException { for (Column column : columns) { List row = Arrays.asList( column.getName(), - column.getOriginType().toString(), + column.getOriginType().hideVersionForVersionColumn(true), column.isAllowNull() ? "Yes" : "No", ((Boolean) column.isKey()).toString(), column.getDefaultValue() == null ? FeConstants.null_string : column.getDefaultValue(), "NONE" ); - row.set(1, column.getOriginType().hideVersionForVersionColumn(false)); totalRows.add(row); } return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java index 60d1980d28bcc62..0f2fb911d4658b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexSchemaProcNode.java @@ -68,14 +68,12 @@ public static ProcResult createResult(List schema, Set bfColumns String extraStr = StringUtils.join(extras, ","); List rowList = Arrays.asList(column.getDisplayName(), - column.getOriginType().toString(), + column.getOriginType().hideVersionForVersionColumn(true), column.isAllowNull() ? "Yes" : "No", ((Boolean) column.isKey()).toString(), column.getDefaultValue() == null ? FeConstants.null_string : column.getDefaultValue(), extraStr); - - rowList.set(1, column.getOriginType().hideVersionForVersionColumn(false)); result.addRow(rowList); } return result; diff --git a/regression-test/data/datatype_p0/agg_state/nereids/test_agg_state_nereids.out b/regression-test/data/datatype_p0/agg_state/nereids/test_agg_state_nereids.out index 03b46f74a498af5..8cdb1b8a59d4978 100644 --- a/regression-test/data/datatype_p0/agg_state/nereids/test_agg_state_nereids.out +++ b/regression-test/data/datatype_p0/agg_state/nereids/test_agg_state_nereids.out @@ -14,6 +14,10 @@ -- !sum_null -- \N +-- !desc -- +k1 INT Yes true \N +k2 AGG_STATE No false \N GENERIC + -- !length1 -- 1 11 diff --git a/regression-test/data/datatype_p0/agg_state/test_agg_state.out b/regression-test/data/datatype_p0/agg_state/test_agg_state.out index 120a9844f645b7e..0e1e5b6ed91e976 100644 --- a/regression-test/data/datatype_p0/agg_state/test_agg_state.out +++ b/regression-test/data/datatype_p0/agg_state/test_agg_state.out @@ -14,6 +14,10 @@ -- !sum_null -- \N +-- !desc -- +k1 INT Yes true \N +k2 AGG_STATE No false \N GENERIC + -- !length1 -- 1 11 diff --git a/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy b/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy index a302c27405868d5..cc5f9d45ca0d185 100644 --- a/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy +++ b/regression-test/suites/datatype_p0/agg_state/nereids/test_agg_state_nereids.groovy @@ -55,6 +55,8 @@ suite("test_agg_state_nereids") { properties("replication_num" = "1"); """ + qt_desc "desc a_table;" + sql "explain insert into a_table select 1,max_by_state(1,3);" sql "insert into a_table select 1,max_by_state(1,3);" diff --git a/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy b/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy index 78edeaaa3409092..515b74fdc1f5d96 100644 --- a/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy +++ b/regression-test/suites/datatype_p0/agg_state/test_agg_state.groovy @@ -52,6 +52,8 @@ suite("test_agg_state") { properties("replication_num" = "1"); """ + qt_desc "desc a_table;" + sql "insert into a_table select 1,max_by_state(1,3);" sql "insert into a_table select 1,max_by_state(2,2);" sql "insert into a_table values(1,max_by_state(3,1));" From dbf7a76592100f8c7f56769f74900e38d614f015 Mon Sep 17 00:00:00 2001 From: yiguolei Date: Wed, 22 May 2024 10:19:51 +0800 Subject: [PATCH 100/111] Revert "[Chore](rollup) check duplicate column name when create table with rollup (#34827)" This reverts commit 4a8df535537e8eab8fa2ad54934a185e17d4e660. --- .../plans/commands/info/CreateTableInfo.java | 4 ---- .../plans/commands/info/RollupDefinition.java | 14 +---------- .../test_dup_mv_useless.groovy | 23 ------------------- .../nereids_p0/create_table/ddl/table.sql | 2 +- 4 files changed, 2 insertions(+), 41 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 3d4607517d95508..6456e31a5df14cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -460,10 +460,6 @@ public void validate(ConnectContext ctx) { } } } - - for (RollupDefinition rollup : rollups) { - rollup.validate(); - } } else { // mysql, broker and hive do not need key desc if (keysType != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RollupDefinition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RollupDefinition.java index b01e380e9048279..6c3857279f5e341 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RollupDefinition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/RollupDefinition.java @@ -18,15 +18,12 @@ package org.apache.doris.nereids.trees.plans.commands.info; import org.apache.doris.analysis.AddRollupClause; -import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.util.Utils; import com.google.common.collect.Maps; -import com.google.common.collect.Sets; import java.util.List; import java.util.Map; -import java.util.Set; /** * rollup definition @@ -44,16 +41,7 @@ public RollupDefinition(String name, List cols, List dupKeys, Ma this.properties = Maps.newHashMap(properties); } - /** - * check rollup validity - */ - public void validate() throws AnalysisException { - Set colSet = Sets.newHashSet(); - for (String col : cols) { - if (!colSet.add(col)) { - throw new AnalysisException("rollup has duplicate column name " + col); - } - } + public void validate() { } public AddRollupClause translateToCatalogStyle() { diff --git a/regression-test/suites/mv_p0/test_mv_useless/test_dup_mv_useless.groovy b/regression-test/suites/mv_p0/test_mv_useless/test_dup_mv_useless.groovy index d91cafbe93d83a7..7640b6e18064f6d 100644 --- a/regression-test/suites/mv_p0/test_mv_useless/test_dup_mv_useless.groovy +++ b/regression-test/suites/mv_p0/test_mv_useless/test_dup_mv_useless.groovy @@ -48,27 +48,4 @@ suite ("test_dup_mv_useless") { createMV("create materialized view k1_k2_u21 as select k2,k1 from ${testTable} group by k2,k1 order by k2,k1;") createMV("create materialized view k1_k2_sumk3 as select k1,k2,sum(k3) from ${testTable} group by k1,k2;") sql "insert into ${testTable} select 4,4,4;" - - test { - sql """ - create table test_rollup ( - `id` int not null, - `kbool` boolean not null, - `ktint` tinyint(4) not null, - `ksint` smallint(6) not null, - `kint` int(11) not null, - `kbint` bigint(20) not null, - `klint` largeint(40) not null - ) engine=OLAP - duplicate key(id, kbool, ktint) - distributed by random buckets auto - rollup ( - r1 (id, ktint, kbool, ktint, kbint) duplicate key(id) - ) - properties ( - "replication_num"="1" - ); - """ - exception "duplicate column name" - } } diff --git a/regression-test/suites/nereids_p0/create_table/ddl/table.sql b/regression-test/suites/nereids_p0/create_table/ddl/table.sql index c1f244071780791..bcac1168b731a68 100644 --- a/regression-test/suites/nereids_p0/create_table/ddl/table.sql +++ b/regression-test/suites/nereids_p0/create_table/ddl/table.sql @@ -238,7 +238,7 @@ create table test_rollup ( duplicate key(id, kbool, ktint) distributed by random buckets auto rollup ( - r1 (id, ktint, kbool, kbint) duplicate key(id) + r1 (id, ktint, kbool, ktint, kbint) duplicate key(id) ) properties ( "replication_num"="1" From 7ca7458f44e2d822f14af1b25558b0e7b74563b2 Mon Sep 17 00:00:00 2001 From: HHoflittlefish777 <77738092+HHoflittlefish777@users.noreply.github.com> Date: Wed, 22 May 2024 10:38:55 +0800 Subject: [PATCH 101/111] [branch-2.1](routine-load) fix routine load case fail (#35173) * fix routine load case error --- .../routine_load/test_routine_load.groovy | 32 ------------------- 1 file changed, 32 deletions(-) diff --git a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy index c71557f36c7e9c3..ba385076a655b2d 100644 --- a/regression-test/suites/load_p0/routine_load/test_routine_load.groovy +++ b/regression-test/suites/load_p0/routine_load/test_routine_load.groovy @@ -278,14 +278,6 @@ suite("test_routine_load","p0") { count++ } - if (i < 5) { - def res = sql "select count(*) from ${tableName1}" - assertEquals(20, res[0][0]) - } else { - def res = sql "select count(*) from ${tableName1}" - assertEquals(18, res[0][0]) - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -367,14 +359,6 @@ suite("test_routine_load","p0") { count++ } - if (i < 5) { - def res = sql "select count(*) from ${tableName1}" - assertEquals(20, res[0][0]) - } else { - def res = sql "select count(*) from ${tableName1}" - assertEquals(18, res[0][0]) - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -690,14 +674,6 @@ suite("test_routine_load","p0") { count++ } - if (i < 5) { - def res = sql "select count(*) from ${tableName1}" - assertEquals(20, res[0][0]) - } else { - def res = sql "select count(*) from ${tableName1}" - assertEquals(18, res[0][0]) - } - sql "stop routine load for ${jobs[i]}" i++ } @@ -814,14 +790,6 @@ suite("test_routine_load","p0") { count++ } - if (i < 5) { - def res = sql "select count(*) from ${tableName1}" - assertEquals(20, res[0][0]) - } else { - def res = sql "select count(*) from ${tableName1}" - assertEquals(18, res[0][0]) - } - sql "stop routine load for ${jobs[i]}" i++ } From f0b2f5ba368418490407989b460ec3be8e13f1db Mon Sep 17 00:00:00 2001 From: HappenLee Date: Wed, 22 May 2024 10:57:57 +0800 Subject: [PATCH 102/111] [Fix](bug) agg limit contains null values may cause error result (#35180) --- be/src/vec/common/columns_hashing.h | 8 ++++++++ be/src/vec/common/columns_hashing_impl.h | 2 +- be/src/vec/common/hash_table/hash_map_context.h | 2 +- regression-test/data/query_p0/aggregate/aggregate.out | 2 ++ .../suites/query_p0/aggregate/aggregate.groovy | 4 ++++ 5 files changed, 16 insertions(+), 2 deletions(-) diff --git a/be/src/vec/common/columns_hashing.h b/be/src/vec/common/columns_hashing.h index 83f01fdf4b2fc1a..c25c8acdb7b64c8 100644 --- a/be/src/vec/common/columns_hashing.h +++ b/be/src/vec/common/columns_hashing.h @@ -136,6 +136,14 @@ struct HashMethodSingleLowNullableColumn : public SingleColumnMethod { data.lazy_emplace(std::forward(key), it, hash_value, std::forward(f)); return *lookup_result_get_mapped(it); } + + template + ALWAYS_INLINE FindResult find_key_with_hash(Data& data, size_t i, Key key, size_t hash_value) { + if (key_column->is_null_at(i) && data.has_null_key_data()) { + return FindResult {&data.template get_null_key_data(), true}; + } + return Base::find_key_impl(key, hash_value, data); + } }; } // namespace ColumnsHashing diff --git a/be/src/vec/common/columns_hashing_impl.h b/be/src/vec/common/columns_hashing_impl.h index d0ef8a4b014c965..6d8ba4395102e7d 100644 --- a/be/src/vec/common/columns_hashing_impl.h +++ b/be/src/vec/common/columns_hashing_impl.h @@ -89,7 +89,7 @@ class HashMethodBase { } template - ALWAYS_INLINE FindResult find_key_with_hash(Data& data, size_t hash_value, Key key) { + ALWAYS_INLINE FindResult find_key_with_hash(Data& data, size_t i, Key key, size_t hash_value) { return find_key_impl(key, hash_value, data); } diff --git a/be/src/vec/common/hash_table/hash_map_context.h b/be/src/vec/common/hash_table/hash_map_context.h index f52be441dd661f7..0dec7eef88d3e54 100644 --- a/be/src/vec/common/hash_table/hash_map_context.h +++ b/be/src/vec/common/hash_table/hash_map_context.h @@ -125,7 +125,7 @@ struct MethodBaseInner { if constexpr (!is_string_hash_map()) { prefetch(i); } - return state.find_key_with_hash(*hash_table, hash_values[i], keys[i]); + return state.find_key_with_hash(*hash_table, i, keys[i], hash_values[i]); } template diff --git a/regression-test/data/query_p0/aggregate/aggregate.out b/regression-test/data/query_p0/aggregate/aggregate.out index 9f9870797a0b03b..ffd3790499423fa 100644 --- a/regression-test/data/query_p0/aggregate/aggregate.out +++ b/regression-test/data/query_p0/aggregate/aggregate.out @@ -695,3 +695,5 @@ TESTING AGAIN -- !subquery_without_inner_predicate -- 7 +-- !aggregate_limit_contain_null -- +16 \N diff --git a/regression-test/suites/query_p0/aggregate/aggregate.groovy b/regression-test/suites/query_p0/aggregate/aggregate.groovy index 1b0987538cc0b27..b611ff92b0eabaa 100644 --- a/regression-test/suites/query_p0/aggregate/aggregate.groovy +++ b/regression-test/suites/query_p0/aggregate/aggregate.groovy @@ -304,4 +304,8 @@ suite("aggregate") { qt_subquery_without_inner_predicate """ select count(*) from (select t2.c_bigint, t2.c_double, t2.c_string from (select c_bigint, c_double, c_string, c_date,c_timestamp, c_short_decimal from regression_test_query_p0_aggregate.${tableName} where c_bigint > 5000) t2)t1 """ + + qt_aggregate_limit_contain_null """ + select count(), cast(k12 as int) as t from baseall group by t limit 1; + """ } From 9d7c65b4d8120c9170fd5c8342a6844da4fd169f Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Wed, 22 May 2024 11:42:08 +0800 Subject: [PATCH 103/111] [fix](memory) Avoid frequently refresh cgroup memory info (#35083) (#35182) pick #35083 --- be/src/util/mem_info.cpp | 23 +++++++++++++++++++---- be/src/util/mem_info.h | 2 ++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp index a2cb04049db43c4..f01524bb7ea05a0 100644 --- a/be/src/util/mem_info.cpp +++ b/be/src/util/mem_info.cpp @@ -59,6 +59,8 @@ bvar::PassiveStatus g_sys_mem_avail( bool MemInfo::_s_initialized = false; std::atomic MemInfo::_s_physical_mem = std::numeric_limits::max(); +int64_t MemInfo::_s_cgroup_mem_limit = std::numeric_limits::max(); +int64_t MemInfo::_s_cgroup_mem_limit_refresh_wait_times = 0; std::atomic MemInfo::_s_mem_limit = std::numeric_limits::max(); std::atomic MemInfo::_s_soft_mem_limit = std::numeric_limits::max(); @@ -395,10 +397,23 @@ void MemInfo::refresh_proc_meminfo() { int64_t physical_mem = -1; int64_t cgroup_mem_limit = -1; physical_mem = _mem_info_bytes["MemTotal"]; - Status status = CGroupUtil::find_cgroup_mem_limit(&cgroup_mem_limit); - if (status.ok() && cgroup_mem_limit > 0) { + if (_s_cgroup_mem_limit_refresh_wait_times >= 0) { + Status status = CGroupUtil::find_cgroup_mem_limit(&cgroup_mem_limit); + if (status.ok() && cgroup_mem_limit > 0) { + _s_cgroup_mem_limit = cgroup_mem_limit; + _s_cgroup_mem_limit_refresh_wait_times = + -1000; // wait 10s, 1000 * 100ms, avoid too frequently. + } else { + _s_cgroup_mem_limit = std::numeric_limits::max(); + _s_cgroup_mem_limit_refresh_wait_times = + -6000; // find cgroup failed, wait 60s, 6000 * 100ms. + } + } else { + _s_cgroup_mem_limit_refresh_wait_times++; + } + if (_s_cgroup_mem_limit > 0) { // In theory, always cgroup_mem_limit < physical_mem - physical_mem = std::min(physical_mem, cgroup_mem_limit); + physical_mem = std::min(physical_mem, _s_cgroup_mem_limit); } if (physical_mem <= 0) { @@ -438,7 +453,7 @@ void MemInfo::refresh_proc_meminfo() { if (_mem_info_bytes.find("MemAvailable") != _mem_info_bytes.end()) { mem_available = _mem_info_bytes["MemAvailable"]; } - status = CGroupUtil::find_cgroup_mem_usage(&cgroup_mem_usage); + auto status = CGroupUtil::find_cgroup_mem_usage(&cgroup_mem_usage); if (status.ok() && cgroup_mem_usage > 0 && cgroup_mem_limit > 0) { if (mem_available < 0) { mem_available = cgroup_mem_limit - cgroup_mem_usage; diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h index 5606ebd45d6b97d..59030130e016752 100644 --- a/be/src/util/mem_info.h +++ b/be/src/util/mem_info.h @@ -197,6 +197,8 @@ class MemInfo { private: static bool _s_initialized; static std::atomic _s_physical_mem; + static int64_t _s_cgroup_mem_limit; + static int64_t _s_cgroup_mem_limit_refresh_wait_times; static std::atomic _s_mem_limit; static std::atomic _s_soft_mem_limit; From 84f7bfffe2d6f4b7f354afb948065d9aeadf0e73 Mon Sep 17 00:00:00 2001 From: Pxl Date: Mon, 29 Apr 2024 15:26:33 +0800 Subject: [PATCH 104/111] [Bug](bitmap-filter) fix empty bitmap when rf do merge (#34182) fix empty bitmap when rf do merge --- be/src/exprs/runtime_filter.cpp | 3 ++- .../data/query_p0/join/test_bitmap_filter_nereids.out | 11 +++++++++++ .../query_p0/join/test_bitmap_filter_nereids.groovy | 2 ++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/be/src/exprs/runtime_filter.cpp b/be/src/exprs/runtime_filter.cpp index e01c353b804ee6d..e51b3c739f645f5 100644 --- a/be/src/exprs/runtime_filter.cpp +++ b/be/src/exprs/runtime_filter.cpp @@ -543,7 +543,8 @@ class RuntimePredicateWrapper { break; } case RuntimeFilterType::BITMAP_FILTER: { - // do nothing because we assume bitmap filter join always have full data + // use input bitmap directly because we assume bitmap filter join always have full data + _context->bitmap_filter_func = wrapper->_context->bitmap_filter_func; break; } default: diff --git a/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out b/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out index f44900b0e0bb7f2..2b1f654a198d0cc 100644 --- a/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out +++ b/regression-test/data/query_p0/join/test_bitmap_filter_nereids.out @@ -98,3 +98,14 @@ 1 1989 10 1991 +-- !sql1 -- +1 1989 +3 1989 +5 1985 +7 -32767 +9 1991 +10 1991 +11 1989 +12 32767 +13 -32767 +14 255 \ No newline at end of file diff --git a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy index 9557aada40daef0..be18b4fe0ea05fc 100644 --- a/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy +++ b/regression-test/suites/query_p0/join/test_bitmap_filter_nereids.groovy @@ -91,4 +91,6 @@ suite("test_bitmap_filter_nereids") { sql " select k1, k2 from (select 2 k1, 2 k2) t where k1 in (select k2 from bitmap_table_nereids)" notContains "RF000[bitmap]" } + sql "set parallel_pipeline_task_num=6;" + qt_sql1 "select k1, k2 from test_query_db.bigtable where k1 in (select k2 from bitmap_table_nereids) order by k1;" } From c23384ff0759b442c8d540110ee3d2bbf079a0e1 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Wed, 22 May 2024 14:24:22 +0800 Subject: [PATCH 105/111] [fix](decimal) Fix long string casting to decimalv2 (#35121) --- be/src/util/string_parser.hpp | 169 +++++++----------- be/test/vec/data_types/from_string_test.cpp | 2 +- .../serde/data_type_serde_csv_test.cpp | 31 ++-- .../serde/data_type_serde_text_test.cpp | 37 ++-- .../decimalv2/test_decimalv2_load.out | 8 - .../decimalv2/test_decimalv2_overflow2.out | 11 +- .../decimalv2/test_decimalv2_load.groovy | 36 ---- .../decimalv2/test_decimalv2_overflow2.groovy | 18 ++ 8 files changed, 121 insertions(+), 191 deletions(-) diff --git a/be/src/util/string_parser.hpp b/be/src/util/string_parser.hpp index 521d1a6f39c71f8..34bd678c947f9e9 100644 --- a/be/src/util/string_parser.hpp +++ b/be/src/util/string_parser.hpp @@ -594,123 +594,74 @@ T StringParser::string_to_decimal(const char* __restrict s, int len, int type_pr bool found_exponent = false; int8_t exponent = 0; T value = 0; - if constexpr (TYPE_DECIMALV2 == P) { - // decimalv2 do not care type_scale and type_precision,just keep the origin logic - for (int i = 0; i < len; ++i) { - const char& c = s[i]; - if (LIKELY('0' <= c && c <= '9')) { - found_value = true; - // Ignore digits once the type's precision limit is reached. This avoids - // overflowing the underlying storage while handling a string like - // 10000000000e-10 into a DECIMAL(1, 0). Adjustments for ignored digits and - // an exponent will be made later. - if (LIKELY(type_precision > precision)) { - value = (value * 10) + (c - '0'); // Benchmarks are faster with parenthesis... - } else { - *result = StringParser::PARSE_OVERFLOW; - value = is_negative - ? vectorized::min_decimal_value(type_precision) - : vectorized::max_decimal_value(type_precision); - return value; - } - DCHECK(value >= 0); // For some reason //DCHECK_GE doesn't work with __int128. + bool has_round = false; + for (int i = 0; i < len; ++i) { + const char& c = s[i]; + if (LIKELY('0' <= c && c <= '9')) { + found_value = true; + // Ignore digits once the type's precision limit is reached. This avoids + // overflowing the underlying storage while handling a string like + // 10000000000e-10 into a DECIMAL(1, 0). Adjustments for ignored digits and + // an exponent will be made later. + if (LIKELY(type_precision > precision) && !has_round) { + value = (value * 10) + (c - '0'); // Benchmarks are faster with parenthesis... ++precision; scale += found_dot; - } else if (c == '.' && LIKELY(!found_dot)) { - found_dot = 1; - } else if ((c == 'e' || c == 'E') && LIKELY(!found_exponent)) { - found_exponent = true; - exponent = string_to_int_internal(s + i + 1, len - i - 1, result); - if (UNLIKELY(*result != StringParser::PARSE_SUCCESS)) { - if (*result == StringParser::PARSE_OVERFLOW && exponent < 0) { - *result = StringParser::PARSE_UNDERFLOW; - } - return 0; - } - break; - } else { - if (value == 0) { - *result = StringParser::PARSE_FAILURE; - return 0; - } - *result = StringParser::PARSE_SUCCESS; - value *= get_scale_multiplier(type_scale - scale); - - return is_negative ? T(-value) : T(value); - } - } - } else { - // decimalv3 - bool has_round = false; - for (int i = 0; i < len; ++i) { - const char& c = s[i]; - if (LIKELY('0' <= c && c <= '9')) { - found_value = true; - // Ignore digits once the type's precision limit is reached. This avoids - // overflowing the underlying storage while handling a string like - // 10000000000e-10 into a DECIMAL(1, 0). Adjustments for ignored digits and - // an exponent will be made later. - if (LIKELY(type_precision > precision) && !has_round) { - value = (value * 10) + (c - '0'); // Benchmarks are faster with parenthesis... - ++precision; - scale += found_dot; - cur_digit = precision - scale; - } else if (!found_dot && max_digit < (precision - scale)) { - *result = StringParser::PARSE_OVERFLOW; - value = is_negative - ? vectorized::min_decimal_value(type_precision) + cur_digit = precision - scale; + } else if (!found_dot && max_digit < (precision - scale)) { + *result = StringParser::PARSE_OVERFLOW; + value = is_negative ? vectorized::min_decimal_value(type_precision) : vectorized::max_decimal_value(type_precision); - return value; - } else if (found_dot && scale >= type_scale && !has_round) { - // make rounding cases - if (c > '4') { - value += 1; - } - has_round = true; - continue; - } else if (!found_dot) { - ++cur_digit; - } - DCHECK(value >= 0); // For some reason //DCHECK_GE doesn't work with __int128. - } else if (c == '.' && LIKELY(!found_dot)) { - found_dot = 1; - } else if ((c == 'e' || c == 'E') && LIKELY(!found_exponent)) { - found_exponent = true; - exponent = string_to_int_internal(s + i + 1, len - i - 1, result); - if (UNLIKELY(*result != StringParser::PARSE_SUCCESS)) { - if (*result == StringParser::PARSE_OVERFLOW && exponent < 0) { - *result = StringParser::PARSE_UNDERFLOW; - } - return 0; + return value; + } else if (found_dot && scale >= type_scale && !has_round) { + // make rounding cases + if (c > '4') { + value += 1; } - break; - } else { - if (value == 0) { - *result = StringParser::PARSE_FAILURE; - return 0; + has_round = true; + continue; + } else if (!found_dot) { + ++cur_digit; + } + DCHECK(value >= 0); // For some reason //DCHECK_GE doesn't work with __int128. + } else if (c == '.' && LIKELY(!found_dot)) { + found_dot = 1; + } else if ((c == 'e' || c == 'E') && LIKELY(!found_exponent)) { + found_exponent = true; + exponent = string_to_int_internal(s + i + 1, len - i - 1, result); + if (UNLIKELY(*result != StringParser::PARSE_SUCCESS)) { + if (*result == StringParser::PARSE_OVERFLOW && exponent < 0) { + *result = StringParser::PARSE_UNDERFLOW; } - // here to handle - *result = StringParser::PARSE_SUCCESS; - if (type_scale >= scale) { - value *= get_scale_multiplier(type_scale - scale); - // here meet non-valid character, should return the value, keep going to meet - // the E/e character because we make right user-given type_precision - // not max number type_precision - if (!is_numeric_ascii(c)) { - if (cur_digit > type_precision) { - *result = StringParser::PARSE_OVERFLOW; - value = is_negative ? vectorized::min_decimal_value( - type_precision) - : vectorized::max_decimal_value( - type_precision); - return value; - } - return is_negative ? T(-value) : T(value); + return 0; + } + break; + } else { + if (value == 0) { + *result = StringParser::PARSE_FAILURE; + return 0; + } + // here to handle + *result = StringParser::PARSE_SUCCESS; + if (type_scale >= scale) { + value *= get_scale_multiplier(type_scale - scale); + // here meet non-valid character, should return the value, keep going to meet + // the E/e character because we make right user-given type_precision + // not max number type_precision + if (!is_numeric_ascii(c)) { + if (cur_digit > type_precision) { + *result = StringParser::PARSE_OVERFLOW; + value = is_negative + ? vectorized::min_decimal_value(type_precision) + : vectorized::max_decimal_value( + type_precision); + return value; } + return is_negative ? T(-value) : T(value); } - - return is_negative ? T(-value) : T(value); } + + return is_negative ? T(-value) : T(value); } } diff --git a/be/test/vec/data_types/from_string_test.cpp b/be/test/vec/data_types/from_string_test.cpp index c19a5f1706d3a3e..83b65f0fa3ae4ce 100644 --- a/be/test/vec/data_types/from_string_test.cpp +++ b/be/test/vec/data_types/from_string_test.cpp @@ -103,7 +103,7 @@ TEST(FromStringTest, ScalaWrapperFieldVsDataType) { "12345678901234567.012345677", "12345678901234567.012345677", "999999999999999999.999999999"}, {"12345678901234567.012345678", "123456789012345678.012345670", - "12345678901234567.012345678", "", ""}), + "12345678901234567.012345678", "12345678901234567.012345678", ""}), // decimal32 ==> decimal32(9,2) FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DECIMAL32, // (7,2) (6,3) (7,3) (8,1) diff --git a/be/test/vec/data_types/serde/data_type_serde_csv_test.cpp b/be/test/vec/data_types/serde/data_type_serde_csv_test.cpp index 315ecded4900acf..ca730fe7bc9b8b5 100644 --- a/be/test/vec/data_types/serde/data_type_serde_csv_test.cpp +++ b/be/test/vec/data_types/serde/data_type_serde_csv_test.cpp @@ -74,21 +74,22 @@ TEST(CsvSerde, ScalaDataTypeSerdeCsvTest) { FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_STRING, {"doris be better"}, {"doris be better"}), // decimal ==> decimalv2(decimal<128>(27,9)) - FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DECIMAL, - { - // (17, 9)(first 0 will ignore) - "012345678901234567.012345678", - // (18, 8) (automatically fill 0 for scala) - "123456789012345678.01234567", - // (17, 10) (rounding last to make it fit) - "12345678901234567.0123456779", - // (17, 11) (rounding last to make it fit) - "12345678901234567.01234567791", - // (19, 8) (wrong) - "1234567890123456789.01234567", - }, - {"12345678901234567.012345678", "123456789012345678.012345670", - "12345678901234567.012345678", "", ""}), + FieldType_RandStr( + FieldType::OLAP_FIELD_TYPE_DECIMAL, + { + // (17, 9)(first 0 will ignore) + "012345678901234567.012345678", + // (18, 8) (automatically fill 0 for scala) + "123456789012345678.01234567", + // (17, 10) (rounding last to make it fit) + "12345678901234567.0123456779", + // (17, 11) (rounding last to make it fit) + "12345678901234567.01234567791", + // (19, 8) (wrong) + "1234567890123456789.01234567", + }, + {"12345678901234567.012345678", "123456789012345678.012345670", + "12345678901234567.012345678", "12345678901234567.012345678", ""}), // decimal32 ==> decimal32(9,2) (7,2) (6,3) (7,3) (8,1) FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DECIMAL32, {"1234567.12", "123456.123", "1234567.123", "12345679.1"}, diff --git a/be/test/vec/data_types/serde/data_type_serde_text_test.cpp b/be/test/vec/data_types/serde/data_type_serde_text_test.cpp index 7091f3b30fddf82..2e85c2c3dc11c40 100644 --- a/be/test/vec/data_types/serde/data_type_serde_text_test.cpp +++ b/be/test/vec/data_types/serde/data_type_serde_text_test.cpp @@ -74,21 +74,22 @@ TEST(TextSerde, ScalaDataTypeSerdeTextTest) { FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_STRING, {"doris be better"}, {"doris be better"}), // decimal ==> decimalv2(decimal<128>(27,9)) - FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DECIMAL, - { - // (17, 9)(first 0 will ignore) - "012345678901234567.012345678", - // (18, 8) (automatically fill 0 for scala) - "123456789012345678.01234567", - // (17, 10) (rounding last to make it fit) - "12345678901234567.0123456779", - // (17, 11) (rounding last to make it fit) - "12345678901234567.01234567791", - // (19, 8) (wrong) - "1234567890123456789.01234567", - }, - {"12345678901234567.012345678", "123456789012345678.012345670", - "12345678901234567.012345678", "", ""}), + FieldType_RandStr( + FieldType::OLAP_FIELD_TYPE_DECIMAL, + { + // (17, 9)(first 0 will ignore) + "012345678901234567.012345678", + // (18, 8) (automatically fill 0 for scala) + "123456789012345678.01234567", + // (17, 10) (rounding last to make it fit) + "12345678901234567.0123456779", + // (17, 11) (rounding last to make it fit) + "12345678901234567.01234567791", + // (19, 8) (wrong) + "1234567890123456789.01234567", + }, + {"12345678901234567.012345678", "123456789012345678.012345670", + "12345678901234567.012345678", "12345678901234567.012345678", ""}), // decimal32 ==> decimal32(9,2) (7,2) (6,3) (7,3) (8,1) FieldType_RandStr(FieldType::OLAP_FIELD_TYPE_DECIMAL32, {"1234567.12", "123456.123", "1234567.123", "12345679.1"}, @@ -429,13 +430,13 @@ TEST(TextSerde, ComplexTypeSerdeTextTest) { "[\\1234567890123456789.01234567\\]"}, {"[4.000000000, 5.500000000, 6.670000000]", "[12345678901234567.012345678, 123456789012345678.012345670, " - "12345678901234567.012345678, null, null]", + "12345678901234567.012345678, 12345678901234567.012345678, null]", "[null, null, null, null, null]", "[null]"}, {"[4.000000000, 5.500000000, 6.670000000]", "[12345678901234567.012345678, 123456789012345678.012345670, " - "12345678901234567.012345678, null, null]", + "12345678901234567.012345678, 12345678901234567.012345678, null]", "[12345678901234567.012345678, 123456789012345678.012345670, " - "12345678901234567.012345678, null, null]", + "12345678901234567.012345678, 12345678901234567.012345678, null]", "[null]"}), }; // array type diff --git a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_load.out b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_load.out index ae9921a0f37704f..8156a9144aa4c2e 100644 --- a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_load.out +++ b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_load.out @@ -15,11 +15,3 @@ 11.99990 837.43444 --- !decimalv2_insert -- -999999999999999999.999999999 1.000000000 --999999999999999999.999999999 2.000000000 -999999999999999999.999999999 3.000000000 --999999999999999999.999999999 4.000000000 -999999999999999999.999999999 5.000000000 --999999999999999999.999999999 6.000000000 - diff --git a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow2.out b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow2.out index ecce20f1b229c1d..fdd14e48bad2515 100644 --- a/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow2.out +++ b/regression-test/data/datatype_p0/decimalv2/test_decimalv2_overflow2.out @@ -27,10 +27,10 @@ 999999999999999999.999999999 -- !multi_overflow2 -- -999999999999999999.999999999 999999999999999999.999999999000000000 +999999999999999999.999999999 999999999999999999.999999999 -- !multi_overflow3 -- -999999999999999999.999999999 999999999999999999.999999999000000000 +999999999999999999.999999999 999999999999999999.999999999 -- !multi_overflow4 -- 999999999999999999.999999999 1.000000000 999999999999999999.999999999 @@ -39,10 +39,10 @@ 99999999999999999.999999999 0.100000000 999999999999999999.999999990 -- !div_overflow2 -- -999999999999999999.999999990 +999999999999999999.99999999 -- !div_overflow3 -- -99999999999999999.999999999 0.1 999999999999999999.9999999900000 +99999999999999999.999999999 0.1 999999999999999999.999999990 -- !div_overflow4 -- 999999999999999999.999999990 @@ -59,3 +59,6 @@ -- !mod4 -- 0.099999999 +-- !sql -- +2023-12-18T00:00 95357.10 + diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_load.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_load.groovy index 08027c96d1d4a1d..5c065a921a0b290 100644 --- a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_load.groovy +++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_load.groovy @@ -84,42 +84,6 @@ suite("test_decimalv2_load", "nonConcurrent") { select * from ${tableName2} order by 1; """ - sql """ - drop table if exists test_decimalv2_insert; - """ - sql """ - CREATE TABLE `test_decimalv2_insert` ( - `k1` decimalv2(27, 9) null, - `k2` decimalv2(27, 9) null - ) - DISTRIBUTED BY HASH(`k1`) BUCKETS 10 - PROPERTIES ( - "replication_num" = "1" - ); - """ - sql "set enable_insert_strict=true;" - // overflow, max is inserted - sql """ - insert into test_decimalv2_insert values("999999999999999999999999999999",1); - """ - // underflow, min is inserted - sql """ - insert into test_decimalv2_insert values("-999999999999999999999999999999",2); - """ - sql """ - insert into test_decimalv2_insert values("999999999999999999.9999999991",3); - """ - sql """ - insert into test_decimalv2_insert values("-999999999999999999.9999999991",4); - """ - sql """ - insert into test_decimalv2_insert values("999999999999999999.9999999995",5); - """ - sql """ - insert into test_decimalv2_insert values("-999999999999999999.9999999995",6); - """ - qt_decimalv2_insert "select * from test_decimalv2_insert order by 2; " - sql """ admin set frontend config("enable_decimal_conversion" = "true"); """ diff --git a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy index ad6dea6765cd9a5..b183e00243f05a7 100644 --- a/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy +++ b/regression-test/suites/datatype_p0/decimalv2/test_decimalv2_overflow2.groovy @@ -269,6 +269,24 @@ suite("test_decimalv2_overflow2") { """ + sql """ drop TABLE if exists test_table """ + sql """ CREATE TABLE `test_table` ( + `day_date` datetime NULL COMMENT '', + `growth_money` decimalv2(18, 2) NULL COMMENT '' + ) ENGINE=OLAP + UNIQUE KEY(`day_date`) + COMMENT '' + DISTRIBUTED BY HASH(`day_date`) BUCKETS 4 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "is_being_synced" = "false", + "storage_format" = "V2", + "disable_auto_compaction" = "false", + "enable_single_replica_compaction" = "false" + ); """ + sql """ insert into test_table values ('2023-12-18', '95357.100000000000000000000000000000000000')""" + qt_sql """ select * from test_table """ + sql """ drop TABLE if exists test_table """ // TODO // decimalv2 +-*/ integer // integer +-*/ decimalv2 From 15f70c8183d31deabf932d78212a8f71c5f36104 Mon Sep 17 00:00:00 2001 From: feiniaofeiafei <53502832+feiniaofeiafei@users.noreply.github.com> Date: Wed, 22 May 2024 15:17:46 +0800 Subject: [PATCH 106/111] [Feat](planner)create table stmt offer default distribution attribute :random distribution and auto bucket (#35189) Co-authored-by: feiniaofeiafei --- .../doris/analysis/CreateTableStmt.java | 2 +- .../plans/commands/info/CreateTableInfo.java | 2 +- ...test_create_table_without_distribution.out | 19 ++++++++ ...t_create_table_without_distribution.groovy | 43 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 regression-test/data/ddl_p0/test_create_table_without_distribution.out create mode 100644 regression-test/suites/ddl_p0/test_create_table_without_distribution.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java index 40383826f4036fb..179776654667bbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java @@ -516,7 +516,7 @@ public void analyze(Analyzer analyzer) throws UserException { // analyze distribution if (distributionDesc == null) { - throw new AnalysisException("Create olap table should contain distribution desc"); + distributionDesc = new RandomDistributionDesc(FeConstants.default_bucket_num, true); } distributionDesc.analyze(columnSet, columnDefs, keysDesc); if (distributionDesc.type == DistributionInfo.DistributionInfoType.RANDOM) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java index 6456e31a5df14cd..2a7aa291d0632bb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java @@ -210,7 +210,7 @@ public void validate(ConnectContext ctx) { if (engineName.equalsIgnoreCase("olap")) { if (distribution == null) { - throw new AnalysisException("Create olap table should contain distribution desc"); + distribution = new DistributionDescriptor(false, true, FeConstants.default_bucket_num, null); } properties = maybeRewriteByAutoBucket(distribution, properties); } diff --git a/regression-test/data/ddl_p0/test_create_table_without_distribution.out b/regression-test/data/ddl_p0/test_create_table_without_distribution.out new file mode 100644 index 000000000000000..ccb7fe6195f30c5 --- /dev/null +++ b/regression-test/data/ddl_p0/test_create_table_without_distribution.out @@ -0,0 +1,19 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !test_insert -- +1 + +-- !test_select -- +1 2 + +-- !test_show -- +test_create_table_without_distribution CREATE TABLE `test_create_table_without_distribution` (\n `a` INT NULL,\n `b` INT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`a`, `b`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY RANDOM BUCKETS AUTO\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + +-- !test_insert_old_planner -- +1 + +-- !test_select_old_planner -- +1 2 + +-- !test_show_old_planner -- +test_create_table_without_distribution CREATE TABLE `test_create_table_without_distribution` (\n `a` INT NULL,\n `b` INT NULL\n) ENGINE=OLAP\nDUPLICATE KEY(`a`, `b`)\nCOMMENT 'OLAP'\nDISTRIBUTED BY RANDOM BUCKETS AUTO\nPROPERTIES (\n"replication_allocation" = "tag.location.default: 1",\n"min_load_replica_num" = "-1",\n"is_being_synced" = "false",\n"storage_medium" = "hdd",\n"storage_format" = "V2",\n"inverted_index_storage_format" = "V1",\n"light_schema_change" = "true",\n"disable_auto_compaction" = "false",\n"binlog.enable" = "false",\n"binlog.ttl_seconds" = "86400",\n"binlog.max_bytes" = "9223372036854775807",\n"binlog.max_history_nums" = "9223372036854775807",\n"enable_single_replica_compaction" = "false",\n"group_commit_interval_ms" = "10000",\n"group_commit_data_bytes" = "134217728"\n); + diff --git a/regression-test/suites/ddl_p0/test_create_table_without_distribution.groovy b/regression-test/suites/ddl_p0/test_create_table_without_distribution.groovy new file mode 100644 index 000000000000000..704772a0a952b67 --- /dev/null +++ b/regression-test/suites/ddl_p0/test_create_table_without_distribution.groovy @@ -0,0 +1,43 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_create_table_without_distribution") { + sql "SET enable_nereids_planner=true;" + sql "SET enable_fallback_to_original_planner=false;" + multi_sql """ + drop table if exists test_create_table_without_distribution; + create table test_create_table_without_distribution(a int, b int) properties ("replication_num"="1") + """ + qt_test_insert """ + insert into test_create_table_without_distribution values(1,2); + """ + qt_test_select "select * from test_create_table_without_distribution;" + qt_test_show " show create table test_create_table_without_distribution;" + + sql "SET enable_nereids_planner=false;" + multi_sql """ + drop table if exists test_create_table_without_distribution; + create table test_create_table_without_distribution(a int, b int) properties ("replication_num"="1") + """ + qt_test_insert_old_planner """ + insert into test_create_table_without_distribution values(1,2); + """ + qt_test_select_old_planner "select * from test_create_table_without_distribution;" + qt_test_show_old_planner " show create table test_create_table_without_distribution;" + + +} \ No newline at end of file From 30a66a4f9d34023827eba1764d16835bc970b725 Mon Sep 17 00:00:00 2001 From: shuke <37901441+shuke987@users.noreply.github.com> Date: Wed, 22 May 2024 15:58:37 +0800 Subject: [PATCH 107/111] [regression-test](fix) fix case bug #35201 --- .../suites/account_p0/test_nereids_row_policy.groovy | 8 -------- 1 file changed, 8 deletions(-) diff --git a/regression-test/suites/account_p0/test_nereids_row_policy.groovy b/regression-test/suites/account_p0/test_nereids_row_policy.groovy index 408559c8bb7f7c4..59edd884d5e9904 100644 --- a/regression-test/suites/account_p0/test_nereids_row_policy.groovy +++ b/regression-test/suites/account_p0/test_nereids_row_policy.groovy @@ -83,14 +83,6 @@ suite("test_nereids_row_policy") { sql 'sync' - //cloud-mode - if (cloudMode) { - def clusters = sql " SHOW CLUSTERS; " - assertTrue(!clusters.isEmpty()) - def validCluster = clusters[0][0] - sql """GRANT USAGE_PRIV ON CLUSTER ${validCluster} TO ${user}"""; - } - dropPolciy "policy0" dropPolciy "policy1" dropPolciy "policy2" From 9ed4a2023bc4f75cfa3dedbc1dc51da11c6e994a Mon Sep 17 00:00:00 2001 From: morrySnow <101034200+morrySnow@users.noreply.github.com> Date: Wed, 22 May 2024 16:23:20 +0800 Subject: [PATCH 108/111] [fix](Nereids) DatetimeV2 round floor and round ceiling is wrong (#35153) (#35155) pick from master #35153 1. round floor was incorrectly implemented as round 2. round ceiling not really round because use double type when divide --- .../literal/DateTimeV2Literal.java | 6 ++-- .../SimplifyComparisonPredicateSqlTest.java | 4 +-- .../SimplifyComparisonPredicateTest.java | 4 +-- .../literal/DateTimeLiteralTest.java | 32 +++++++++++++++++++ 4 files changed, 39 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java index 2417d92e89addf3..778d74e1e3fcaa3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeV2Literal.java @@ -232,7 +232,7 @@ public DateTimeV2Literal roundCeiling(int newScale) { long newYear = year; if (remain != 0) { newMicroSecond = Double - .valueOf((microSecond + (Math.pow(10, 6 - newScale))) + .valueOf((microSecond + (int) (Math.pow(10, 6 - newScale))) / (int) (Math.pow(10, 6 - newScale)) * (Math.pow(10, 6 - newScale))) .longValue(); } @@ -251,8 +251,8 @@ public DateTimeV2Literal roundCeiling(int newScale) { } public DateTimeV2Literal roundFloor(int newScale) { - // use roundMicroSecond in constructor - return new DateTimeV2Literal(DateTimeV2Type.of(newScale), year, month, day, hour, minute, second, microSecond); + return new DateTimeV2Literal(DateTimeV2Type.of(newScale), year, month, day, hour, minute, second, + microSecond / (int) Math.pow(10, 6 - newScale) * (int) Math.pow(10, 6 - newScale)); } public static Expression fromJavaDateType(LocalDateTime dateTime) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java index 517b482879804ad..07cbf086abae62f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateSqlTest.java @@ -66,7 +66,7 @@ void testSql() { .rewrite() .matches( logicalFilter() - .when(f -> f.getConjuncts().stream().anyMatch(e -> e.toSql().equals("(a <= '2023-06-16 00:00:00')"))) + .when(f -> f.getConjuncts().stream().anyMatch(e -> e.toSql().equals("(a <= '2023-06-15 23:59:59')"))) .when(f -> f.getConjuncts().stream().anyMatch(e -> e.toSql().equals("(b <= 111.11)"))) ); @@ -82,7 +82,7 @@ void testSql() { .rewrite() .matches( logicalFilter() - .when(f -> f.getConjuncts().stream().anyMatch(e -> e.toSql().equals("(a > '2023-06-16 00:00:00')"))) + .when(f -> f.getConjuncts().stream().anyMatch(e -> e.toSql().equals("(a > '2023-06-15 23:59:59')"))) .when(f -> f.getConjuncts().stream().anyMatch(e -> e.toSql().equals("(b > 111.11)"))) ); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java index 1955571231bb230..e85cf38a73796a1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/SimplifyComparisonPredicateTest.java @@ -131,8 +131,8 @@ void testRound() { Expression expression = new GreaterThan(left, right); Expression rewrittenExpression = executor.rewrite(typeCoercion(expression), context); - // right should round to be 2021-01-02 00:00:00.00 - Assertions.assertEquals(new DateTimeLiteral("2021-01-02 00:00:00"), rewrittenExpression.child(1)); + // right should round to be 2021-01-01 23:59:59 + Assertions.assertEquals(new DateTimeLiteral("2021-01-01 23:59:59"), rewrittenExpression.child(1)); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java index 184cf5dbf6342f4..94e456cf9289af3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/DateTimeLiteralTest.java @@ -443,4 +443,36 @@ void testDateTimeV2Scale() { new DateTimeV2Literal(DateTimeV2Type.of(5), "2016-12-31 23:59:59.999999"), new DateTimeV2Literal("2017-01-01 00:00:00.0")); } + + @Test + void testRoundFloor() { + DateTimeV2Literal literal; + literal = new DateTimeV2Literal(DateTimeV2Type.of(6), 2000, 2, 2, 2, 2, 2, 222222); + Assertions.assertEquals(222222, literal.roundFloor(6).microSecond); + Assertions.assertEquals(222220, literal.roundFloor(5).microSecond); + Assertions.assertEquals(222200, literal.roundFloor(4).microSecond); + Assertions.assertEquals(222000, literal.roundFloor(3).microSecond); + Assertions.assertEquals(220000, literal.roundFloor(2).microSecond); + Assertions.assertEquals(200000, literal.roundFloor(1).microSecond); + Assertions.assertEquals(0, literal.roundFloor(0).microSecond); + } + + @Test + void testRoundCeiling() { + DateTimeV2Literal literal; + literal = new DateTimeV2Literal(DateTimeV2Type.of(6), 2000, 12, 31, 23, 59, 59, 888888); + Assertions.assertEquals(888888, literal.roundCeiling(6).microSecond); + Assertions.assertEquals(888890, literal.roundCeiling(5).microSecond); + Assertions.assertEquals(888900, literal.roundCeiling(4).microSecond); + Assertions.assertEquals(889000, literal.roundCeiling(3).microSecond); + Assertions.assertEquals(890000, literal.roundCeiling(2).microSecond); + Assertions.assertEquals(900000, literal.roundCeiling(1).microSecond); + Assertions.assertEquals(0, literal.roundCeiling(0).microSecond); + Assertions.assertEquals(0, literal.roundCeiling(0).second); + Assertions.assertEquals(0, literal.roundCeiling(0).minute); + Assertions.assertEquals(0, literal.roundCeiling(0).hour); + Assertions.assertEquals(1, literal.roundCeiling(0).day); + Assertions.assertEquals(1, literal.roundCeiling(0).month); + Assertions.assertEquals(2001, literal.roundCeiling(0).year); + } } From 72f2d0d449c370d032b7daca67d0248b8717e236 Mon Sep 17 00:00:00 2001 From: Xinyi Zou Date: Wed, 22 May 2024 18:11:14 +0800 Subject: [PATCH 109/111] [fix](memory) Allow flush memtable failed when process exceed memlimit #35150 --- be/src/olap/rowset/beta_rowset_writer.cpp | 1 - be/src/olap/rowset/beta_rowset_writer_v2.cpp | 1 - 2 files changed, 2 deletions(-) diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp index 930a946077119e5..de051eea45e349b 100644 --- a/be/src/olap/rowset/beta_rowset_writer.cpp +++ b/be/src/olap/rowset/beta_rowset_writer.cpp @@ -502,7 +502,6 @@ Status BaseBetaRowsetWriter::flush() { Status BaseBetaRowsetWriter::flush_memtable(vectorized::Block* block, int32_t segment_id, int64_t* flush_size) { - SCOPED_SKIP_MEMORY_CHECK(); if (block->rows() == 0) { return Status::OK(); } diff --git a/be/src/olap/rowset/beta_rowset_writer_v2.cpp b/be/src/olap/rowset/beta_rowset_writer_v2.cpp index 921d80cfc581aed..225ba490a35d852 100644 --- a/be/src/olap/rowset/beta_rowset_writer_v2.cpp +++ b/be/src/olap/rowset/beta_rowset_writer_v2.cpp @@ -93,7 +93,6 @@ Status BetaRowsetWriterV2::add_segment(uint32_t segment_id, const SegmentStatist Status BetaRowsetWriterV2::flush_memtable(vectorized::Block* block, int32_t segment_id, int64_t* flush_size) { - SCOPED_SKIP_MEMORY_CHECK(); if (block->rows() == 0) { return Status::OK(); } From d63c3ae2d417af35c52fa9747f9455b3f34c91ad Mon Sep 17 00:00:00 2001 From: wuwenchi Date: Wed, 22 May 2024 18:13:09 +0800 Subject: [PATCH 110/111] [bugfix](hive)fix testcase for viewfs for 2.1 #35178 --- be/src/io/fs/hdfs_file_system.cpp | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/be/src/io/fs/hdfs_file_system.cpp b/be/src/io/fs/hdfs_file_system.cpp index 1340a3078e28e44..5e0116032cd9fb3 100644 --- a/be/src/io/fs/hdfs_file_system.cpp +++ b/be/src/io/fs/hdfs_file_system.cpp @@ -457,15 +457,22 @@ Status HdfsFileSystemCache::get_connection(const THdfsParams& hdfs_params, uint64 HdfsFileSystemCache::_hdfs_hash_code(const THdfsParams& hdfs_params, const std::string& fs_name) { uint64 hash_code = 0; - hash_code += Fingerprint(fs_name); + // The specified fsname is used first. + // If there is no specified fsname, the default fsname is used + if (!fs_name.empty()) { + hash_code ^= Fingerprint(fs_name); + } else if (hdfs_params.__isset.fs_name) { + hash_code ^= Fingerprint(hdfs_params.fs_name); + } + if (hdfs_params.__isset.user) { - hash_code += Fingerprint(hdfs_params.user); + hash_code ^= Fingerprint(hdfs_params.user); } if (hdfs_params.__isset.hdfs_kerberos_principal) { - hash_code += Fingerprint(hdfs_params.hdfs_kerberos_principal); + hash_code ^= Fingerprint(hdfs_params.hdfs_kerberos_principal); } if (hdfs_params.__isset.hdfs_kerberos_keytab) { - hash_code += Fingerprint(hdfs_params.hdfs_kerberos_keytab); + hash_code ^= Fingerprint(hdfs_params.hdfs_kerberos_keytab); } if (hdfs_params.__isset.hdfs_conf) { std::map conf_map; @@ -473,8 +480,8 @@ uint64 HdfsFileSystemCache::_hdfs_hash_code(const THdfsParams& hdfs_params, conf_map[conf.key] = conf.value; } for (auto& conf : conf_map) { - hash_code += Fingerprint(conf.first); - hash_code += Fingerprint(conf.second); + hash_code ^= Fingerprint(conf.first); + hash_code ^= Fingerprint(conf.second); } } return hash_code; From 05cedfca4e5238bcfe610c1cc573c34e20ed57fd Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Wed, 22 May 2024 18:44:19 +0800 Subject: [PATCH 111/111] [fix](hudi) catch exception when getting hudi partition (#35027) (#35159) bp #35027 --- .../datasource/hive/HiveMetaStoreCache.java | 2 +- .../doris/datasource/hive/HiveUtil.java | 2 +- .../datasource/hudi/source/HudiScanNode.java | 78 +++++++++++-------- 3 files changed, 46 insertions(+), 36 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java index b97284eda9cfcc5..a22e951be400cec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java @@ -355,7 +355,7 @@ private FileCacheValue getFileCache(String location, String inputFormat, RemoteFileSystem fs = Env.getCurrentEnv().getExtMetaCacheMgr().getFsCache().getRemoteFileSystem( new FileSystemCache.FileSystemCacheKey(LocationPath.getFSIdentity( location, bindBrokerName), properties, bindBrokerName)); - result.setSplittable(HiveUtil.isSplittable(fs, inputFormat, location, jobConf)); + result.setSplittable(HiveUtil.isSplittable(fs, inputFormat, location)); // For Tez engine, it may generate subdirectoies for "union" query. // So there may be files and directories in the table directory at the same time. eg: // /user/hive/warehouse/region_tmp_union_all2/000000_0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java index bca04215fc4436a..5ca42dd02453edc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveUtil.java @@ -110,7 +110,7 @@ private HiveUtil() { } public static boolean isSplittable(RemoteFileSystem remoteFileSystem, String inputFormat, - String location, JobConf jobConf) throws UserException { + String location) throws UserException { if (remoteFileSystem instanceof BrokerFileSystem) { return ((BrokerFileSystem) remoteFileSystem).isSplittable(location, inputFormat); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java index 8dd853a48f373b0..61edc333f6c4e3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/source/HudiScanNode.java @@ -77,6 +77,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; public class HudiScanNode extends HiveScanNode { @@ -329,49 +330,58 @@ private List getIncrementalSplits() { private void getPartitionSplits(List partitions, List splits) { Executor executor = Env.getCurrentEnv().getExtMetaCacheMgr().getFileListingExecutor(); CountDownLatch countDownLatch = new CountDownLatch(partitions.size()); + AtomicReference throwable = new AtomicReference<>(); partitions.forEach(partition -> executor.execute(() -> { - String globPath; - String partitionName = ""; - if (partition.isDummyPartition()) { - globPath = hudiClient.getBasePathV2().toString() + "/*"; - } else { - partitionName = FSUtils.getRelativePartitionPath(hudiClient.getBasePathV2(), - new Path(partition.getPath())); - globPath = String.format("%s/%s/*", hudiClient.getBasePathV2().toString(), partitionName); - } - List statuses; try { - statuses = FSUtils.getGlobStatusExcludingMetaFolder(hudiClient.getRawFs(), - new Path(globPath)); - } catch (IOException e) { - throw new RuntimeException("Failed to get hudi file statuses on path: " + globPath, e); - } - HoodieTableFileSystemView fileSystemView = new HoodieTableFileSystemView(hudiClient, - timeline, statuses.toArray(new FileStatus[0])); - - if (isCowOrRoTable) { - fileSystemView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant).forEach(baseFile -> { - noLogsSplitNum.incrementAndGet(); - String filePath = baseFile.getPath(); - long fileSize = baseFile.getFileSize(); - // Need add hdfs host to location - LocationPath locationPath = new LocationPath(filePath, hmsTable.getCatalogProperties()); - Path splitFilePath = locationPath.toStorageLocation(); - splits.add(new FileSplit(splitFilePath, 0, fileSize, fileSize, - new String[0], partition.getPartitionValues())); - }); - } else { - fileSystemView.getLatestMergedFileSlicesBeforeOrOn(partitionName, queryInstant) - .forEach(fileSlice -> splits.add( - generateHudiSplit(fileSlice, partition.getPartitionValues(), queryInstant))); + String globPath; + String partitionName = ""; + if (partition.isDummyPartition()) { + globPath = hudiClient.getBasePathV2().toString() + "/*"; + } else { + partitionName = FSUtils.getRelativePartitionPath(hudiClient.getBasePathV2(), + new Path(partition.getPath())); + globPath = String.format("%s/%s/*", hudiClient.getBasePathV2().toString(), partitionName); + } + List statuses; + try { + statuses = FSUtils.getGlobStatusExcludingMetaFolder(hudiClient.getRawFs(), + new Path(globPath)); + } catch (IOException e) { + throw new RuntimeException("Failed to get hudi file statuses on path: " + globPath, e); + } + HoodieTableFileSystemView fileSystemView = new HoodieTableFileSystemView(hudiClient, + timeline, statuses.toArray(new FileStatus[0])); + + if (isCowOrRoTable) { + fileSystemView.getLatestBaseFilesBeforeOrOn(partitionName, queryInstant).forEach(baseFile -> { + noLogsSplitNum.incrementAndGet(); + String filePath = baseFile.getPath(); + long fileSize = baseFile.getFileSize(); + // Need add hdfs host to location + LocationPath locationPath = new LocationPath(filePath, hmsTable.getCatalogProperties()); + Path splitFilePath = locationPath.toStorageLocation(); + splits.add(new FileSplit(splitFilePath, 0, fileSize, fileSize, + new String[0], partition.getPartitionValues())); + }); + } else { + fileSystemView.getLatestMergedFileSlicesBeforeOrOn(partitionName, queryInstant) + .forEach(fileSlice -> splits.add( + generateHudiSplit(fileSlice, partition.getPartitionValues(), queryInstant))); + } + } catch (Throwable t) { + throwable.set(t); + } finally { + countDownLatch.countDown(); } - countDownLatch.countDown(); })); try { countDownLatch.await(); } catch (InterruptedException e) { throw new RuntimeException(e.getMessage(), e); } + if (throwable.get() != null) { + throw new RuntimeException(throwable.get().getMessage(), throwable.get()); + } } @Override