From 80226fe3ad9f5d86c05d839f6ad68095e57d3ef0 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Thu, 19 Sep 2024 10:39:34 +0800 Subject: [PATCH] [opt](partial update) Forbid partial update on merge-on-write table with sync mv (#40190) doc: https://github.com/apache/doris-website/pull/1103 --- .../doris/analysis/NativeInsertStmt.java | 8 ++ .../plans/commands/insert/InsertUtils.java | 6 + .../doris/planner/StreamLoadPlanner.java | 9 ++ .../test_partial_update_mow_with_sync_mv.csv | 1 + .../delete_p0/test_delete_with_sync_mv.groovy | 4 + ...est_partial_update_mow_with_sync_mv.groovy | 112 ++++++++++++++++++ 6 files changed, 140 insertions(+) create mode 100644 regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_mow_with_sync_mv.csv create mode 100644 regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_mow_with_sync_mv.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java index 132d2a86b22c23..4f3f720fae9726 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java @@ -1365,6 +1365,14 @@ private void trySetPartialUpdate() throws UserException { if (hasEmptyTargetColumns) { return; } + + boolean hasSyncMaterializedView = olapTable.getFullSchema().stream() + .anyMatch(col -> col.isMaterializedViewColumn()); + if (hasSyncMaterializedView) { + throw new UserException("Can't do partial update on merge-on-write Unique table" + + " with sync materialized view."); + } + boolean hasMissingColExceptAutoIncKey = false; for (Column col : olapTable.getFullSchema()) { boolean exists = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 49e7858f6faf65..e0b167b93b6357 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -289,6 +289,12 @@ public static Plan normalizePlan(Plan plan, TableIf table, Optional) unboundLogicalSink).setPartialUpdate(false); } else { + boolean hasSyncMaterializedView = olapTable.getFullSchema().stream() + .anyMatch(col -> col.isMaterializedViewColumn()); + if (hasSyncMaterializedView) { + throw new AnalysisException("Can't do partial update on merge-on-write Unique table" + + " with sync materialized view."); + } boolean hasMissingColExceptAutoIncKey = false; for (Column col : olapTable.getFullSchema()) { Optional insertCol = unboundLogicalSink.getColNames().stream() diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index c302c4e905bb60..acef987a0a3985 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -154,6 +154,15 @@ public TPipelineFragmentParams plan(TUniqueId loadId, int fragmentInstanceIdInde isPartialUpdate = false; } + if (isPartialUpdate) { + boolean hasSyncMaterializedView = destTable.getFullSchema().stream() + .anyMatch(col -> col.isMaterializedViewColumn()); + if (hasSyncMaterializedView) { + throw new DdlException("Can't do partial update on merge-on-write Unique table" + + " with sync materialized view."); + } + } + HashSet partialUpdateInputColumns = new HashSet<>(); if (isPartialUpdate) { for (Column col : destTable.getFullSchema()) { diff --git a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_mow_with_sync_mv.csv b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_mow_with_sync_mv.csv new file mode 100644 index 00000000000000..6e745e19bcad17 --- /dev/null +++ b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_mow_with_sync_mv.csv @@ -0,0 +1 @@ +2,3,2,1,'2023-10-18','k' \ No newline at end of file diff --git a/regression-test/suites/delete_p0/test_delete_with_sync_mv.groovy b/regression-test/suites/delete_p0/test_delete_with_sync_mv.groovy index ad9fdb3f752332..0ddf48d2d4298b 100644 --- a/regression-test/suites/delete_p0/test_delete_with_sync_mv.groovy +++ b/regression-test/suites/delete_p0/test_delete_with_sync_mv.groovy @@ -72,5 +72,9 @@ suite("test_delete_with_sync_mv") { from test_delete_with_sync_mv; """) + explain { + sql """delete from test_delete_with_sync_mv where l_orderkey = 2""" + contains "IS_PARTIAL_UPDATE: false" + } sql """delete from test_delete_with_sync_mv where l_orderkey = 2""" } diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_mow_with_sync_mv.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_mow_with_sync_mv.groovy new file mode 100644 index 00000000000000..3440584326985e --- /dev/null +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_mow_with_sync_mv.groovy @@ -0,0 +1,112 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_partial_update_mow_with_sync_mv") { + + sql """drop table if exists test_partial_update_mow_with_sync_mv""" + + sql """ + CREATE TABLE `test_partial_update_mow_with_sync_mv` ( + `l_orderkey` BIGINT NULL, + `l_linenumber` INT NULL, + `l_partkey` INT NULL, + `l_suppkey` INT NULL, + `l_shipdate` DATE not NULL, + `l_quantity` DECIMAL(15, 2) NULL, + `l_extendedprice` DECIMAL(15, 2) NULL, + `l_discount` DECIMAL(15, 2) NULL, + `l_tax` DECIMAL(15, 2) NULL, + `l_returnflag` VARCHAR(1) NULL, + `l_linestatus` VARCHAR(1) NULL, + `l_commitdate` DATE NULL, + `l_receiptdate` DATE NULL, + `l_shipinstruct` VARCHAR(25) NULL, + `l_shipmode` VARCHAR(10) NULL, + `l_comment` VARCHAR(44) NULL + ) + unique KEY(l_orderkey, l_linenumber, l_partkey, l_suppkey, l_shipdate) + DISTRIBUTED BY HASH(`l_orderkey`) BUCKETS 96 + PROPERTIES ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ); + """ + + sql """ + insert into test_partial_update_mow_with_sync_mv values + (null, 1, 2, 3, '2023-10-17', 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (1, null, 3, 1, '2023-10-17', 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'), + (3, 3, null, 2, '2023-10-19', 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), + (1, 2, 3, null, '2023-10-17', 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (2, 3, 2, 1, '2023-10-18', 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy'), + (3, 1, 1, 2, '2023-10-19', 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx'), + (1, 3, 2, 2, '2023-10-17', 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (null, 1, 2, 3, '2023-10-17', 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (1, null, 3, 1, '2023-10-17', 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy'), + (3, 3, null, 2, '2023-10-19', 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx'), + (1, 2, 3, null, '2023-10-17', 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy'), + (2, 3, 2, 1, '2023-10-18', 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy'), + (3, 1, 1, 2, '2023-10-19', 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx'), + (1, 3, 2, 2, '2023-10-17', 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy') + """ + + createMV (""" + CREATE MATERIALIZED VIEW mv + AS + select l_orderkey, l_linenumber, l_partkey, l_suppkey, l_shipdate, + substring(concat(l_returnflag, l_linestatus), 1) + from test_partial_update_mow_with_sync_mv; + """) + + for (def use_nereids : [true, false]) { + if (use_nereids) { + sql "set enable_nereids_planner=true" + sql "set enable_fallback_to_original_planner=false" + } else { + sql "set enable_nereids_planner=false" + } + + sql "set enable_unique_key_partial_update=true;" + sql "sync;" + + test { + sql """insert into test_partial_update_mow_with_sync_mv(l_orderkey, l_linenumber, l_partkey, l_suppkey, l_shipdate, l_returnflag) values + (2, 3, 2, 1, '2023-10-18', 'k'); """ + exception "Can't do partial update on merge-on-write Unique table with sync materialized view." + } + } + + streamLoad { + table "test_partial_update_mow_with_sync_mv" + set 'column_separator', ',' + set 'format', 'csv' + set 'partial_columns', 'true' + set 'columns', 'l_orderkey, l_linenumber, l_partkey, l_suppkey, l_shipdate, l_returnflag' + + file 'test_partial_update_mow_with_sync_mv.csv' + time 10000 + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("fail", json.Status.toLowerCase()) + assertTrue(json.Message.contains("Can't do partial update on merge-on-write Unique table with sync materialized view.")) + } + } +}