update

Yukang-Lian · Jul 9, 2023 · 169c1ae · 169c1ae
1 parent d918872
commit 169c1ae
Show file tree

Hide file tree

Showing 4 changed files with 81 additions and 1 deletion.
diff --git a/be/src/olap/full_compaction.cpp b/be/src/olap/full_compaction.cpp
@@ -19,12 +19,16 @@
 
 #include <time.h>
 
+#include <memory>
 #include <mutex>
 #include <ostream>
 
 #include "common/config.h"
+#include "common/status.h"
 #include "olap/cumulative_compaction_policy.h"
+#include "olap/olap_common.h"
 #include "olap/olap_define.h"
+#include "olap/tablet_meta.h"
 #include "runtime/thread_context.h"
 #include "util/thread.h"
 #include "util/trace.h"
@@ -102,12 +106,27 @@ Status FullCompaction::pick_rowsets_to_compact() {
 }
 
 Status FullCompaction::modify_rowsets(const Merger::Statistics* stats) {
+    // TODO: calculate publish rowsets delete bitmaps for full compaction.
+
+    std::lock_guard<std::mutex> wrlock_(_tablet->get_rowset_update_lock());
+    std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock());
+    RETURN_IF_ERROR(_tablet->full_compaction_update_delete_bitmap(
+            _output_rowset, pre_rowset_ids(),
+            std::make_shared<DeleteBitmap>(_output_rowset->rowset_meta()->tablet_id()),
+            _output_rs_writer.get()));
     std::vector<RowsetSharedPtr> output_rowsets;
     output_rowsets.push_back(_output_rowset);
-    std::lock_guard<std::shared_mutex> wrlock(_tablet->get_header_lock());
     RETURN_IF_ERROR(_tablet->modify_rowsets(output_rowsets, _input_rowsets, true));
     _tablet->save_meta();
     return Status::OK();
 }
 
+RowsetIdUnorderedSet FullCompaction::pre_rowset_ids() {
+    RowsetIdUnorderedSet pre_rowset_ids {};
+    for (const auto& rowset : _input_rowsets) {
+        pre_rowset_ids.insert(rowset->rowset_id());
+    }
+    return pre_rowset_ids;
+}
+
 } // namespace doris
diff --git a/be/src/olap/full_compaction.h b/be/src/olap/full_compaction.h
@@ -49,6 +49,7 @@ class FullCompaction : public Compaction {
 
 private:
     Status _check_rowset_overlapping(const vector<RowsetSharedPtr>& rowsets);
+    RowsetIdUnorderedSet pre_rowset_ids();
 
     DISALLOW_COPY_AND_ASSIGN(FullCompaction);
 };

diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
@@ -3248,6 +3248,61 @@ Status Tablet::commit_phase_update_delete_bitmap(
     return Status::OK();
 }
 
+Status Tablet::full_compaction_update_delete_bitmap(const RowsetSharedPtr& rowset,
+                                                    RowsetWriter* rowset_writer) {
+    RowsetIdUnorderedSet cur_rowset_ids;
+    RowsetIdUnorderedSet rowset_ids_to_add;
+    DeleteBitmapPtr delete_bitmap =
+            std::make_shared<DeleteBitmap>(rowset->rowset_meta()->rowset_id());
+    int64_t cur_version = _tablet_meta->max_version().second;
+
+    std::vector<segment_v2::SegmentSharedPtr> segments;
+    _load_rowset_segments(rowset, &segments);
+
+    std::shared_lock meta_rlock(_meta_lock);
+    // tablet is under alter process. The delete bitmap will be calculated after conversion.
+    if (tablet_state() == TABLET_NOTREADY &&
+        SchemaChangeHandler::tablet_in_converting(tablet_id())) {
+        LOG(INFO) << "tablet is under alter process, update delete bitmap later, tablet_id="
+                  << tablet_id();
+        return Status::OK();
+    }
+    cur_rowset_ids = all_rs_id(cur_version);
+    for (const auto& id : cur_rowset_ids) {
+        if (id < rowset->rowset_meta()->rowset_id()) {
+            cur_rowset_ids.erase(id);
+        }
+    }
+
+    std::vector<RowsetSharedPtr> specified_rowsets;
+    {
+        std::shared_lock meta_rlock(_meta_lock);
+        specified_rowsets = get_rowset_by_ids(&rowset_ids_to_add);
+    }
+
+    OlapStopWatch watch;
+    RETURN_IF_ERROR(calc_delete_bitmap(rowset, segments, specified_rowsets, delete_bitmap,
+                                       cur_version, rowset_writer));
+    size_t total_rows = std::accumulate(
+            segments.begin(), segments.end(), 0,
+            [](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); });
+    LOG(INFO) << "[Full compaction] construct delete bitmap tablet: " << tablet_id()
+              << ", rowset_ids to add: " << rowset_ids_to_add.size()
+              << ", cur max_version: " << cur_version << ", cost: " << watch.get_elapse_time_us()
+              << "(us), total rows: " << total_rows;
+
+    // update version without write lock, compaction and publish_txn
+    // will update delete bitmap, handle compaction with _rowset_update_lock
+    // and publish_txn runs sequential so no need to lock here
+    for (auto iter = delete_bitmap->delete_bitmap.begin();
+         iter != delete_bitmap->delete_bitmap.end(); ++iter) {
+        _tablet_meta->delete_bitmap().merge(
+                {std::get<0>(iter->first), std::get<1>(iter->first), cur_version}, iter->second);
+    }
+
+    return Status::OK();
+}
+
 Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset,
                                     const RowsetIdUnorderedSet& pre_rowset_ids,
                                     DeleteBitmapPtr delete_bitmap, int64_t txn_id,

diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
@@ -470,6 +470,11 @@ class Tablet : public BaseTablet {
             const std::vector<segment_v2::SegmentSharedPtr>& segments, int64_t txn_id,
             RowsetWriter* rowset_writer = nullptr);
 
+    Status full_compaction_update_delete_bitmap(const RowsetSharedPtr& rowset,
+                                                const RowsetIdUnorderedSet& pre_rowset_ids,
+                                                DeleteBitmapPtr delete_bitmap,
+                                                RowsetWriter* rowset_writer = nullptr);
+
     Status update_delete_bitmap(const RowsetSharedPtr& rowset,
                                 const RowsetIdUnorderedSet& pre_rowset_ids,
                                 DeleteBitmapPtr delete_bitmap, int64_t txn_id,