From cb312cabb22ca4cfa72a8db71e51938e0989aec7 Mon Sep 17 00:00:00 2001 From: Luwei <814383175@qq.com> Date: Tue, 27 Aug 2024 20:10:17 +0800 Subject: [PATCH] [Fix](tablet-meta) limit the data size of tablet meta (#39455) (#39974) pick master #39455 --- be/src/common/config.cpp | 7 +++++++ be/src/common/config.h | 2 ++ be/src/olap/push_handler.cpp | 11 +++++++++++ be/src/olap/rowset_builder.cpp | 10 ++++++++++ be/src/olap/schema_change.cpp | 2 +- be/src/olap/tablet.h | 11 +++++++++++ be/src/olap/tablet_meta.cpp | 21 +++++++++++++++++++++ be/src/olap/tablet_meta.h | 9 +++++++++ 8 files changed, 72 insertions(+), 1 deletion(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index fa270b513d8361..0567793c388f61 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -1270,6 +1270,13 @@ DEFINE_mBool(enable_parquet_page_index, "false"); DEFINE_mBool(ignore_not_found_file_in_external_table, "true"); +// Tablet meta size limit after serialization, 1.5GB +DEFINE_mInt64(tablet_meta_serialize_size_limit, "1610612736"); +// Protobuf supports a maximum of 2GB, so the size of the tablet meta after serialization must be less than 2GB +// 1717986918 = 2GB * 0.8 +DEFINE_Validator(tablet_meta_serialize_size_limit, + [](const int64_t config) -> bool { return config < 1717986918; }); + DEFINE_mInt64(pipeline_task_leakage_detect_period_secs, "60"); // clang-format off diff --git a/be/src/common/config.h b/be/src/common/config.h index b79db0145461b5..605700170b392f 100644 --- a/be/src/common/config.h +++ b/be/src/common/config.h @@ -1360,6 +1360,8 @@ DECLARE_mBool(enable_parquet_page_index); // Default is true, if set to false, the not found file will result in query failure. DECLARE_mBool(ignore_not_found_file_in_external_table); +DECLARE_mInt64(tablet_meta_serialize_size_limit); + DECLARE_mInt64(pipeline_task_leakage_detect_period_secs); #ifdef BE_TEST diff --git a/be/src/olap/push_handler.cpp b/be/src/olap/push_handler.cpp index 0bf3a23f0abe48..4dc85c8a510ba0 100644 --- a/be/src/olap/push_handler.cpp +++ b/be/src/olap/push_handler.cpp @@ -167,6 +167,17 @@ Status PushHandler::_do_streaming_ingestion(TabletSharedPtr tablet, const TPushR "failed to push data. version count: {}, exceed limit: {}, tablet: {}", tablet->version_count(), config::max_tablet_version_num, tablet->tablet_id()); } + + int version_count = tablet->version_count() + tablet->stale_version_count(); + if (tablet->avg_rs_meta_serialize_size() * version_count > + config::tablet_meta_serialize_size_limit) { + return Status::Error( + "failed to init rowset builder. meta serialize size : {}, exceed limit: {}, " + "tablet: {}", + tablet->avg_rs_meta_serialize_size() * version_count, + config::tablet_meta_serialize_size_limit, tablet->tablet_id()); + } + auto tablet_schema = std::make_shared(); tablet_schema->copy_from(*tablet->tablet_schema()); if (!request.columns_desc.empty() && request.columns_desc[0].col_unique_id >= 0) { diff --git a/be/src/olap/rowset_builder.cpp b/be/src/olap/rowset_builder.cpp index 4194d3ae6c3009..52a5bf1118afb6 100644 --- a/be/src/olap/rowset_builder.cpp +++ b/be/src/olap/rowset_builder.cpp @@ -188,6 +188,16 @@ Status RowsetBuilder::init() { RETURN_IF_ERROR(check_tablet_version_count()); } + int version_count = tablet()->version_count() + tablet()->stale_version_count(); + if (tablet()->avg_rs_meta_serialize_size() * version_count > + config::tablet_meta_serialize_size_limit) { + return Status::Error( + "failed to init rowset builder. meta serialize size : {}, exceed limit: {}, " + "tablet: {}", + tablet()->avg_rs_meta_serialize_size() * version_count, + config::tablet_meta_serialize_size_limit, _tablet->tablet_id()); + } + RETURN_IF_ERROR(prepare_txn()); DBUG_EXECUTE_IF("BaseRowsetBuilder::init.check_partial_update_column_num", { diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp index 81e8d022ecc5b7..97d367e78d4397 100644 --- a/be/src/olap/schema_change.cpp +++ b/be/src/olap/schema_change.cpp @@ -878,7 +878,7 @@ Status SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2& } } std::vector empty_vec; - RETURN_IF_ERROR(new_tablet->modify_rowsets(empty_vec, rowsets_to_delete)); + new_tablet->delete_rowsets(rowsets_to_delete, false); // inherit cumulative_layer_point from base_tablet // check if new_tablet.ce_point > base_tablet.ce_point? new_tablet->set_cumulative_layer_point(-1); diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index b4aca7ba3cbf67..e24e5584736577 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -142,6 +142,7 @@ class Tablet final : public BaseTablet { size_t num_rows(); int version_count() const; bool exceed_version_limit(int32_t limit) const override; + int stale_version_count() const; uint64_t segment_count() const; Version max_version() const; Version max_version_unlocked() const; @@ -158,6 +159,7 @@ class Tablet final : public BaseTablet { double bloom_filter_fpp() const; size_t next_unique_id() const; size_t row_size() const; + int64_t avg_rs_meta_serialize_size() const; // operation in rowsets Status add_rowset(RowsetSharedPtr rowset); @@ -827,6 +829,11 @@ inline int Tablet::version_count() const { return _tablet_meta->version_count(); } +inline int Tablet::stale_version_count() const { + std::shared_lock rdlock(_meta_lock); + return _tablet_meta->stale_version_count(); +} + inline Version Tablet::max_version() const { std::shared_lock rdlock(_meta_lock); return _tablet_meta->max_version(); @@ -885,4 +892,8 @@ inline size_t Tablet::row_size() const { return _tablet_meta->tablet_schema()->row_size(); } +inline int64_t Tablet::avg_rs_meta_serialize_size() const { + return _tablet_meta->avg_rs_meta_serialize_size(); +} + } // namespace doris diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp index 4f20cbc01f589b..83a92faf903fbc 100644 --- a/be/src/olap/tablet_meta.cpp +++ b/be/src/olap/tablet_meta.cpp @@ -26,10 +26,12 @@ #include #include +#include #include #include #include "common/config.h" +#include "gutil/integral_types.h" #include "io/fs/file_writer.h" #include "olap/data_dir.h" #include "olap/file_header.h" @@ -504,6 +506,25 @@ void TabletMeta::serialize(string* meta_binary) { << partition_id << " new=" << tablet_meta_pb.DebugString(); }); bool serialize_success = tablet_meta_pb.SerializeToString(meta_binary); + if (!_rs_metas.empty() || !_stale_rs_metas.empty()) { + _avg_rs_meta_serialize_size = + meta_binary->length() / (_rs_metas.size() + _stale_rs_metas.size()); + if (meta_binary->length() > config::tablet_meta_serialize_size_limit || + !serialize_success) { + int64_t origin_meta_size = meta_binary->length(); + int64_t stale_rowsets_num = tablet_meta_pb.stale_rs_metas().size(); + tablet_meta_pb.clear_stale_rs_metas(); + meta_binary->clear(); + serialize_success = tablet_meta_pb.SerializeToString(meta_binary); + LOG(WARNING) << "tablet meta serialization size exceeds limit: " + << config::tablet_meta_serialize_size_limit + << " clean up stale rowsets, tablet id: " << tablet_id() + << " stale rowset num: " << stale_rowsets_num + << " serialization size before clean " << origin_meta_size + << " serialization size after clean " << meta_binary->length(); + } + } + if (!serialize_success) { LOG(FATAL) << "failed to serialize meta " << tablet_id(); } diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h index cd9f14f8612918..77d5554aae2eba 100644 --- a/be/src/olap/tablet_meta.h +++ b/be/src/olap/tablet_meta.h @@ -166,6 +166,7 @@ class TabletMeta { // Remote disk space occupied by tablet. size_t tablet_remote_size() const; size_t version_count() const; + size_t stale_version_count() const; size_t version_count_cross_with_range(const Version& range) const; Version max_version() const; @@ -269,6 +270,8 @@ class TabletMeta { return _time_series_compaction_level_threshold; } + int64_t avg_rs_meta_serialize_size() const { return _avg_rs_meta_serialize_size; } + private: Status _save_meta(DataDir* data_dir); @@ -323,6 +326,8 @@ class TabletMeta { int64_t _time_series_compaction_empty_rowsets_threshold = 0; int64_t _time_series_compaction_level_threshold = 0; + int64_t _avg_rs_meta_serialize_size = 0; + mutable std::shared_mutex _meta_lock; }; @@ -620,6 +625,10 @@ inline size_t TabletMeta::version_count() const { return _rs_metas.size(); } +inline size_t TabletMeta::stale_version_count() const { + return _rs_metas.size(); +} + inline TabletState TabletMeta::tablet_state() const { return _tablet_state; }