[fix](ut) repair segcompaction ut (#38165) (#38225) (#39153)

cherry-pick #38165 and #22928
apache · Aug 21, 2024 · 7e52a1a · 7e52a1a
1 parent 5e73981
commit 7e52a1a
Show file tree

Hide file tree

Showing 5 changed files with 242 additions and 266 deletions.
diff --git a/be/src/olap/rowset/beta_rowset_writer.cpp b/be/src/olap/rowset/beta_rowset_writer.cpp
@@ -458,6 +458,7 @@ Status BetaRowsetWriter::add_rowset_for_linked_schema_change(RowsetSharedPtr row
 Status BetaRowsetWriter::flush() {
     if (_segment_writer != nullptr) {
         RETURN_IF_ERROR(_flush_segment_writer(&_segment_writer));
+        RETURN_IF_ERROR(_segcompaction_if_necessary());
     }
     return Status::OK();
 }

diff --git a/be/src/olap/rowset/segcompaction.cpp b/be/src/olap/rowset/segcompaction.cpp
@@ -66,6 +66,8 @@
 namespace doris {
 using namespace ErrorCode;
 
+SegcompactionWorker::SegcompactionWorker(BetaRowsetWriter* writer) : _writer(writer) {}
+
 Status SegcompactionWorker::_get_segcompaction_reader(
         SegCompactionCandidatesSharedPtr segments, TabletSharedPtr tablet,
         std::shared_ptr<Schema> schema, OlapReaderStatistics* stat,
@@ -151,8 +153,8 @@ Status SegcompactionWorker::_delete_original_segments(uint32_t begin, uint32_t e
 }
 
 Status SegcompactionWorker::_check_correctness(OlapReaderStatistics& reader_stat,
-                                               Merger::Statistics& merger_stat, uint64_t begin,
-                                               uint64_t end) {
+                                               Merger::Statistics& merger_stat, uint32_t begin,
+                                               uint32_t end) {
     uint64_t raw_rows_read = reader_stat.raw_rows_read; /* total rows read before merge */
     uint64_t sum_src_row = 0; /* sum of rows in each involved source segments */
     uint64_t filtered_rows = merger_stat.filtered_rows; /* rows filtered by del conditions */
@@ -192,7 +194,7 @@ Status SegcompactionWorker::_check_correctness(OlapReaderStatistics& reader_stat
 }
 
 Status SegcompactionWorker::_create_segment_writer_for_segcompaction(
-        std::unique_ptr<segment_v2::SegmentWriter>* writer, uint64_t begin, uint64_t end) {
+        std::unique_ptr<segment_v2::SegmentWriter>* writer, uint32_t begin, uint32_t end) {
     return _writer->_do_create_segment_writer(writer, true, begin, end);
 }
 

diff --git a/be/src/olap/rowset/segcompaction.h b/be/src/olap/rowset/segcompaction.h
@@ -48,7 +48,7 @@ class SegcompactionWorker {
     friend class BetaRowsetWriter;
 
 public:
-    SegcompactionWorker(BetaRowsetWriter* writer) { _writer = writer; }
+    SegcompactionWorker(BetaRowsetWriter* writer);
 
     void compact_segments(SegCompactionCandidatesSharedPtr segments);
 
@@ -59,7 +59,7 @@ class SegcompactionWorker {
 
 private:
     Status _create_segment_writer_for_segcompaction(
-            std::unique_ptr<segment_v2::SegmentWriter>* writer, uint64_t begin, uint64_t end);
+            std::unique_ptr<segment_v2::SegmentWriter>* writer, uint32_t begin, uint32_t end);
     Status _get_segcompaction_reader(SegCompactionCandidatesSharedPtr segments,
                                      TabletSharedPtr tablet, std::shared_ptr<Schema> schema,
                                      OlapReaderStatistics* stat,
@@ -70,7 +70,7 @@ class SegcompactionWorker {
                                                                             uint64_t end);
     Status _delete_original_segments(uint32_t begin, uint32_t end);
     Status _check_correctness(OlapReaderStatistics& reader_stat, Merger::Statistics& merger_stat,
-                              uint64_t begin, uint64_t end);
+                              uint32_t begin, uint32_t end);
     Status _do_compact_segments(SegCompactionCandidatesSharedPtr segments);
 
 private:

diff --git a/be/test/CMakeLists.txt b/be/test/CMakeLists.txt
@@ -46,7 +46,6 @@ list(REMOVE_ITEM UT_FILES
     ${CMAKE_CURRENT_SOURCE_DIR}/olap/rowset/segment_v2/frame_of_reference_page_test.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/olap/rowset/segment_v2/plain_page_test.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/olap/rowset/segment_v2/rle_page_test.cpp
-    ${CMAKE_CURRENT_SOURCE_DIR}/olap/segcompaction_test.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/runtime/decimal_value_test.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/runtime/result_buffer_mgr_test.cpp
     ${CMAKE_CURRENT_SOURCE_DIR}/util/decompress_test.cpp