diff --git a/be/src/olap/full_compaction.cpp b/be/src/olap/full_compaction.cpp index 0f785ce88405f84..3aed7d633eee930 100644 --- a/be/src/olap/full_compaction.cpp +++ b/be/src/olap/full_compaction.cpp @@ -19,12 +19,16 @@ #include +#include #include #include #include "common/config.h" +#include "common/status.h" #include "olap/cumulative_compaction_policy.h" +#include "olap/olap_common.h" #include "olap/olap_define.h" +#include "olap/tablet_meta.h" #include "runtime/thread_context.h" #include "util/thread.h" #include "util/trace.h" @@ -102,12 +106,27 @@ Status FullCompaction::pick_rowsets_to_compact() { } Status FullCompaction::modify_rowsets(const Merger::Statistics* stats) { + // TODO: calculate publish rowsets delete bitmaps for full compaction. + + std::lock_guard wrlock_(_tablet->get_rowset_update_lock()); + std::lock_guard wrlock(_tablet->get_header_lock()); + RETURN_IF_ERROR(_tablet->full_compaction_update_delete_bitmap( + _output_rowset, pre_rowset_ids(), + std::make_shared(_output_rowset->rowset_meta()->tablet_id()), + _output_rs_writer.get())); std::vector output_rowsets; output_rowsets.push_back(_output_rowset); - std::lock_guard wrlock(_tablet->get_header_lock()); RETURN_IF_ERROR(_tablet->modify_rowsets(output_rowsets, _input_rowsets, true)); _tablet->save_meta(); return Status::OK(); } +RowsetIdUnorderedSet FullCompaction::pre_rowset_ids() { + RowsetIdUnorderedSet pre_rowset_ids {}; + for (const auto& rowset : _input_rowsets) { + pre_rowset_ids.insert(rowset->rowset_id()); + } + return pre_rowset_ids; +} + } // namespace doris diff --git a/be/src/olap/full_compaction.h b/be/src/olap/full_compaction.h index 4d73a8d3ec97f00..8e8f043cdc3d4a8 100644 --- a/be/src/olap/full_compaction.h +++ b/be/src/olap/full_compaction.h @@ -49,6 +49,7 @@ class FullCompaction : public Compaction { private: Status _check_rowset_overlapping(const vector& rowsets); + RowsetIdUnorderedSet pre_rowset_ids(); DISALLOW_COPY_AND_ASSIGN(FullCompaction); }; diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp index caa874dd3c04618..e8471b4147b7393 100644 --- a/be/src/olap/tablet.cpp +++ b/be/src/olap/tablet.cpp @@ -3248,6 +3248,61 @@ Status Tablet::commit_phase_update_delete_bitmap( return Status::OK(); } +Status Tablet::full_compaction_update_delete_bitmap(const RowsetSharedPtr& rowset, + RowsetWriter* rowset_writer) { + RowsetIdUnorderedSet cur_rowset_ids; + RowsetIdUnorderedSet rowset_ids_to_add; + DeleteBitmapPtr delete_bitmap = + std::make_shared(rowset->rowset_meta()->rowset_id()); + int64_t cur_version = _tablet_meta->max_version().second; + + std::vector segments; + _load_rowset_segments(rowset, &segments); + + std::shared_lock meta_rlock(_meta_lock); + // tablet is under alter process. The delete bitmap will be calculated after conversion. + if (tablet_state() == TABLET_NOTREADY && + SchemaChangeHandler::tablet_in_converting(tablet_id())) { + LOG(INFO) << "tablet is under alter process, update delete bitmap later, tablet_id=" + << tablet_id(); + return Status::OK(); + } + cur_rowset_ids = all_rs_id(cur_version); + for (const auto& id : cur_rowset_ids) { + if (id < rowset->rowset_meta()->rowset_id()) { + cur_rowset_ids.erase(id); + } + } + + std::vector specified_rowsets; + { + std::shared_lock meta_rlock(_meta_lock); + specified_rowsets = get_rowset_by_ids(&rowset_ids_to_add); + } + + OlapStopWatch watch; + RETURN_IF_ERROR(calc_delete_bitmap(rowset, segments, specified_rowsets, delete_bitmap, + cur_version, rowset_writer)); + size_t total_rows = std::accumulate( + segments.begin(), segments.end(), 0, + [](size_t sum, const segment_v2::SegmentSharedPtr& s) { return sum += s->num_rows(); }); + LOG(INFO) << "[Full compaction] construct delete bitmap tablet: " << tablet_id() + << ", rowset_ids to add: " << rowset_ids_to_add.size() + << ", cur max_version: " << cur_version << ", cost: " << watch.get_elapse_time_us() + << "(us), total rows: " << total_rows; + + // update version without write lock, compaction and publish_txn + // will update delete bitmap, handle compaction with _rowset_update_lock + // and publish_txn runs sequential so no need to lock here + for (auto iter = delete_bitmap->delete_bitmap.begin(); + iter != delete_bitmap->delete_bitmap.end(); ++iter) { + _tablet_meta->delete_bitmap().merge( + {std::get<0>(iter->first), std::get<1>(iter->first), cur_version}, iter->second); + } + + return Status::OK(); +} + Status Tablet::update_delete_bitmap(const RowsetSharedPtr& rowset, const RowsetIdUnorderedSet& pre_rowset_ids, DeleteBitmapPtr delete_bitmap, int64_t txn_id, diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index 8f94f1013604a34..28e7138b966dcd2 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -470,6 +470,11 @@ class Tablet : public BaseTablet { const std::vector& segments, int64_t txn_id, RowsetWriter* rowset_writer = nullptr); + Status full_compaction_update_delete_bitmap(const RowsetSharedPtr& rowset, + const RowsetIdUnorderedSet& pre_rowset_ids, + DeleteBitmapPtr delete_bitmap, + RowsetWriter* rowset_writer = nullptr); + Status update_delete_bitmap(const RowsetSharedPtr& rowset, const RowsetIdUnorderedSet& pre_rowset_ids, DeleteBitmapPtr delete_bitmap, int64_t txn_id,