From d97b33749b3d827200eac5b73d38a192647e4945 Mon Sep 17 00:00:00 2001 From: bobhan1 Date: Tue, 19 Dec 2023 23:00:34 +0800 Subject: [PATCH] Revert "[fix](partial update) Fix missing rowsets during doing alignment when flushing memtable due to compaction #28062 (#28468)" This reverts commit 8c8f322574df58e900d9ccef0d680d723d594368. --- be/src/olap/memtable.cpp | 26 ------- be/src/olap/olap_common.h | 9 +-- .../olap/rowset/segment_v2/segment_writer.cpp | 26 ------- .../test_unique_key_mow_rowsets_deleted.out | 19 ----- ...test_unique_key_mow_rowsets_deleted.groovy | 75 ------------------- .../test_partial_update_publish.groovy | 4 +- 6 files changed, 4 insertions(+), 155 deletions(-) delete mode 100644 regression-test/data/fault_injection_p0/test_unique_key_mow_rowsets_deleted.out delete mode 100644 regression-test/suites/fault_injection_p0/test_unique_key_mow_rowsets_deleted.groovy diff --git a/be/src/olap/memtable.cpp b/be/src/olap/memtable.cpp index 578ee4a7857c70e..880609b053f9071 100644 --- a/be/src/olap/memtable.cpp +++ b/be/src/olap/memtable.cpp @@ -44,7 +44,6 @@ #include "runtime/load_channel_mgr.h" #include "runtime/thread_context.h" #include "tablet_meta.h" -#include "util/debug_points.h" #include "util/doris_metrics.h" #include "util/runtime_profile.h" #include "util/stopwatch.hpp" @@ -475,31 +474,6 @@ Status MemTable::_generate_delete_bitmap(int32_t segment_id) { { std::shared_lock meta_rlock(_tablet->get_header_lock()); specified_rowsets = _tablet->get_rowset_by_ids(&_mow_context->rowset_ids); - DBUG_EXECUTE_IF("_append_block_with_partial_content.clear_specified_rowsets", - { specified_rowsets.clear(); }); - if (specified_rowsets.size() != _mow_context->rowset_ids.size()) { - // `get_rowset_by_ids` may fail to find some of the rowsets we request if cumulative compaction delete - // rowsets from `_rs_version_map`(see `Tablet::modify_rowsets` for detials) before we get here. - // Becasue we havn't begun calculation for merge-on-write table, we can safely reset the `_mow_context->rowset_ids` - // to the latest value and re-request the correspoding rowsets. - LOG(INFO) << fmt::format( - "[Memtable Flush] some rowsets have been deleted due to " - "compaction(specified_rowsets.size()={}, but rowset_ids.size()={}), reset " - "rowset_ids to the latest value. tablet_id: {}, cur max_version: {}, " - "transaction_id: {}", - specified_rowsets.size(), _mow_context->rowset_ids.size(), _tablet->tablet_id(), - _mow_context->max_version, _mow_context->txn_id); - Status st {Status::OK()}; - _mow_context->update_rowset_ids_with_lock([&]() { - _mow_context->rowset_ids.clear(); - st = _tablet->all_rs_id(_mow_context->max_version, &_mow_context->rowset_ids); - }); - if (!st.ok()) { - return st; - } - specified_rowsets = _tablet->get_rowset_by_ids(&_mow_context->rowset_ids); - DCHECK(specified_rowsets.size() == _mow_context->rowset_ids.size()); - } } OlapStopWatch watch; RETURN_IF_ERROR(_tablet->calc_delete_bitmap(rowset, segments, specified_rowsets, diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h index 61742cce67a4818..174a15e86f86a9f 100644 --- a/be/src/olap/olap_common.h +++ b/be/src/olap/olap_common.h @@ -468,18 +468,13 @@ using RowsetIdUnorderedSet = std::unordered_set; class DeleteBitmap; // merge on write context struct MowContext { - MowContext(int64_t version, int64_t txnid, RowsetIdUnorderedSet& ids, + MowContext(int64_t version, int64_t txnid, const RowsetIdUnorderedSet& ids, std::shared_ptr db) : max_version(version), txn_id(txnid), rowset_ids(ids), delete_bitmap(db) {} - void update_rowset_ids_with_lock(std::function callback) { - std::lock_guard lock(m); - callback(); - } int64_t max_version; int64_t txn_id; - RowsetIdUnorderedSet& rowset_ids; + const RowsetIdUnorderedSet& rowset_ids; std::shared_ptr delete_bitmap; - std::mutex m; // protection for updating rowset_ids only }; // used in mow partial update diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp b/be/src/olap/rowset/segment_v2/segment_writer.cpp index e58619ec25b247c..7ef33980b5e5937 100644 --- a/be/src/olap/rowset/segment_v2/segment_writer.cpp +++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp @@ -51,7 +51,6 @@ #include "service/point_query_executor.h" #include "util/coding.h" #include "util/crc32c.h" -#include "util/debug_points.h" #include "util/faststring.h" #include "util/key_util.h" #include "vec/columns/column_nullable.h" @@ -402,31 +401,6 @@ Status SegmentWriter::append_block_with_partial_content(const vectorized::Block* { std::shared_lock rlock(_tablet->get_header_lock()); specified_rowsets = _tablet->get_rowset_by_ids(&_mow_context->rowset_ids); - DBUG_EXECUTE_IF("_append_block_with_partial_content.clear_specified_rowsets", - { specified_rowsets.clear(); }); - if (specified_rowsets.size() != _mow_context->rowset_ids.size()) { - // `get_rowset_by_ids` may fail to find some of the rowsets we request if cumulative compaction delete - // rowsets from `_rs_version_map`(see `Tablet::modify_rowsets` for detials) before we get here. - // Becasue we havn't begun calculation for merge-on-write table, we can safely reset the `_mow_context->rowset_ids` - // to the latest value and re-request the correspoding rowsets. - LOG(INFO) << fmt::format( - "[Memtable Flush] some rowsets have been deleted due to " - "compaction(specified_rowsets.size()={}, but rowset_ids.size()={}), reset " - "rowset_ids to the latest value. tablet_id: {}, cur max_version: {}, " - "transaction_id: {}", - specified_rowsets.size(), _mow_context->rowset_ids.size(), _tablet->tablet_id(), - _mow_context->max_version, _mow_context->txn_id); - Status st {Status::OK()}; - _mow_context->update_rowset_ids_with_lock([&]() { - _mow_context->rowset_ids.clear(); - st = _tablet->all_rs_id(_mow_context->max_version, &_mow_context->rowset_ids); - }); - if (!st.ok()) { - return st; - } - specified_rowsets = _tablet->get_rowset_by_ids(&_mow_context->rowset_ids); - DCHECK(specified_rowsets.size() == _mow_context->rowset_ids.size()); - } } std::vector> segment_caches(specified_rowsets.size()); // locate rows in base data diff --git a/regression-test/data/fault_injection_p0/test_unique_key_mow_rowsets_deleted.out b/regression-test/data/fault_injection_p0/test_unique_key_mow_rowsets_deleted.out deleted file mode 100644 index 30662e8e0ba2e3d..000000000000000 --- a/regression-test/data/fault_injection_p0/test_unique_key_mow_rowsets_deleted.out +++ /dev/null @@ -1,19 +0,0 @@ --- This file is automatically generated. You should know what you did if you want to edit this --- !1 -- -1 doris 1000 123 1 -2 doris2 2000 223 1 - --- !2 -- -1 doris 200 123 1 -2 doris2 400 223 1 -4 yixiu 400 \N 4321 - --- !3 -- -1 doris 1000 123 1 -2 doris2 2000 223 1 - --- !4 -- -1 doris333 6666 555 4 -2 doris666 9999 888 7 -3 doris222 1111 987 567 - diff --git a/regression-test/suites/fault_injection_p0/test_unique_key_mow_rowsets_deleted.groovy b/regression-test/suites/fault_injection_p0/test_unique_key_mow_rowsets_deleted.groovy deleted file mode 100644 index a1db433331b3f6f..000000000000000 --- a/regression-test/suites/fault_injection_p0/test_unique_key_mow_rowsets_deleted.groovy +++ /dev/null @@ -1,75 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -import org.codehaus.groovy.runtime.IOGroovyMethods -import org.apache.doris.regression.util.Http - -suite("test_unique_key_mow_rowsets_deleted", "nonConcurrent"){ - - def tableName = "test_unique_key_mow_rowsets_deleted1" - - // 1. requested rowsets have been deleted during partial update - sql """ DROP TABLE IF EXISTS ${tableName} """ - sql """ CREATE TABLE ${tableName} ( - `id` int(11) NOT NULL COMMENT "用户 ID", - `name` varchar(65533) NOT NULL DEFAULT "yixiu" COMMENT "用户姓名", - `score` int(11) NOT NULL COMMENT "用户得分", - `test` int(11) NULL COMMENT "null test", - `dft` int(11) DEFAULT "4321") - UNIQUE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "true"); """ - - sql """insert into ${tableName} values(2, "doris2", 2000, 223, 1),(1, "doris", 1000, 123, 1)""" - qt_1 """ select * from ${tableName} order by id; """ - try { - GetDebugPoint().enableDebugPointForAllBEs("_append_block_with_partial_content.clear_specified_rowsets") - sql "set enable_unique_key_partial_update=true;" - sql "set enable_insert_strict = false;" - sql "sync;" - sql """insert into ${tableName}(id,score) values(2,400),(1,200),(4,400)""" - qt_2 """ select * from ${tableName} order by id; """ - sql "set enable_unique_key_partial_update=false;" - sql "set enable_insert_strict = true;" - sql "sync;" - } finally { - GetDebugPoint().disableDebugPointForAllBEs("_append_block_with_partial_content.clear_specified_rowsets") - } - sql "DROP TABLE IF EXISTS ${tableName};" - - - // 2. requested rowsets have been deleted during row update - tableName = "test_unique_key_mow_rowsets_deleted2" - sql """ DROP TABLE IF EXISTS ${tableName} """ - sql """ CREATE TABLE ${tableName} ( - `id` int(11) NOT NULL COMMENT "用户 ID", - `name` varchar(65533) NOT NULL DEFAULT "yixiu" COMMENT "用户姓名", - `score` int(11) NOT NULL COMMENT "用户得分", - `test` int(11) NULL COMMENT "null test", - `dft` int(11) DEFAULT "4321") - UNIQUE KEY(`id`) DISTRIBUTED BY HASH(`id`) BUCKETS 1 - PROPERTIES("replication_num" = "1", "enable_unique_key_merge_on_write" = "true"); """ - sql """insert into ${tableName} values(2, "doris2", 2000, 223, 1),(1, "doris", 1000, 123, 1)""" - qt_3 """ select * from ${tableName} order by id; """ - try { - GetDebugPoint().enableDebugPointForAllBEs("BetaRowsetWriter::_generate_delete_bitmap.clear_specified_rowsets") - sql """insert into ${tableName} values(2, "doris666", 9999, 888, 7),(1, "doris333", 6666, 555, 4), (3, "doris222", 1111, 987, 567);""" - qt_4 """ select * from ${tableName} order by id; """ - } finally { - GetDebugPoint().disableDebugPointForAllBEs("BetaRowsetWriter::_generate_delete_bitmap.clear_specified_rowsets") - } - sql "DROP TABLE IF EXISTS ${tableName};" -} diff --git a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_publish.groovy b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_publish.groovy index 0e63b87331ce31d..5f83c092a95c822 100644 --- a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_publish.groovy +++ b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_publish.groovy @@ -37,7 +37,7 @@ suite("test_primary_key_partial_update_publish", "p0") { file '10000.csv' time 10000 // limit inflight 10s - } + } streamLoad { table "${tableName}" @@ -68,5 +68,5 @@ suite("test_primary_key_partial_update_publish", "p0") { """ // drop drop - sql """ DROP TABLE IF EXISTS ${tableName} """ + // sql """ DROP TABLE IF EXISTS ${tableName} """ }