From 7b3cb4ad63c8d2f94de758f9e15e27392596d045 Mon Sep 17 00:00:00 2001 From: walter Date: Thu, 30 May 2024 12:14:58 +0800 Subject: [PATCH] [fix](inverted index)Change index_id from int32 to int64 to avoid overflow #35206 (#35573) Co-authored-by: qiye --- be/src/index-tools/index_tool.cpp | 4 ++-- be/src/olap/rowset/rowset_writer_context.h | 1 + be/src/olap/rowset/segment_v2/inverted_index_desc.cpp | 6 +++--- be/src/olap/rowset/segment_v2/inverted_index_desc.h | 6 +++--- be/src/olap/rowset/segment_v2/inverted_index_reader.h | 2 +- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/be/src/index-tools/index_tool.cpp b/be/src/index-tools/index_tool.cpp index 457560fdf05df7..9892d9d5bcbfea 100644 --- a/be/src/index-tools/index_tool.cpp +++ b/be/src/index-tools/index_tool.cpp @@ -51,7 +51,7 @@ DEFINE_string(pred_type, "", "inverted index term query predicate, eq/lt/gt/le/g DEFINE_bool(print_row_id, false, "print row id when query terms"); DEFINE_bool(print_doc_id, false, "print doc id when check terms stats"); // only for debug index compaction -DEFINE_int32(idx_id, -1, "inverted index id"); +DEFINE_int64(idx_id, -1, "inverted index id"); DEFINE_string(src_idx_dirs_file, "", "source segment index files"); DEFINE_string(dest_idx_dirs_file, "", "destination segment index files"); DEFINE_string(dest_seg_num_rows_file, "", "destination segment number of rows"); @@ -276,7 +276,7 @@ int main(int argc, char** argv) { return true; }; - int32_t index_id = FLAGS_idx_id; + int64_t index_id = FLAGS_idx_id; std::string tablet_path = FLAGS_tablet_path; std::string src_index_dirs_string; std::string dest_index_dirs_string; diff --git a/be/src/olap/rowset/rowset_writer_context.h b/be/src/olap/rowset/rowset_writer_context.h index f77108ee413e43..d2fa5998f2ff97 100644 --- a/be/src/olap/rowset/rowset_writer_context.h +++ b/be/src/olap/rowset/rowset_writer_context.h @@ -83,6 +83,7 @@ struct RowsetWriterContext { int64_t newest_write_timestamp; bool enable_unique_key_merge_on_write = false; + // store column_unique_id to skip write inverted index std::set skip_inverted_index; DataWriteType write_type = DataWriteType::TYPE_DEFAULT; std::shared_ptr tablet = nullptr; diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp index 4bdf44ead06e80..ec5efd2553c578 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.cpp @@ -28,13 +28,13 @@ const std::string index_suffix = ".idx"; const std::string index_name_separator = "_"; std::string InvertedIndexDescriptor::get_temporary_index_path(const std::string& segment_path, - uint32_t uuid) { + uint64_t uuid) { return StripSuffixString(segment_path, segment_suffix) + index_name_separator + std::to_string(uuid); } std::string InvertedIndexDescriptor::get_index_file_name(const std::string& segment_path, - uint32_t uuid) { + uint64_t uuid) { return StripSuffixString(segment_path, segment_suffix) + index_name_separator + std::to_string(uuid) + index_suffix; } @@ -53,4 +53,4 @@ std::string InvertedIndexDescriptor::local_inverted_index_path_segcompacted( return fmt::format("{}/{}_{}-{}_{}.idx", tablet_path, rowset_id.to_string(), begin, end, index_id); } -} // namespace doris::segment_v2 \ No newline at end of file +} // namespace doris::segment_v2 diff --git a/be/src/olap/rowset/segment_v2/inverted_index_desc.h b/be/src/olap/rowset/segment_v2/inverted_index_desc.h index 1cf4636d80b171..dad45a3aa47ce8 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_desc.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_desc.h @@ -28,8 +28,8 @@ namespace segment_v2 { class InvertedIndexDescriptor { public: - static std::string get_temporary_index_path(const std::string& segment_path, uint32_t uuid); - static std::string get_index_file_name(const std::string& path, uint32_t uuid); + static std::string get_temporary_index_path(const std::string& segment_path, uint64_t uuid); + static std::string get_index_file_name(const std::string& path, uint64_t uuid); static const std::string get_temporary_null_bitmap_file_name() { return "null_bitmap"; } static const std::string get_temporary_bkd_index_data_file_name() { return "bkd"; } static const std::string get_temporary_bkd_index_meta_file_name() { return "bkd_meta"; } @@ -45,4 +45,4 @@ class InvertedIndexDescriptor { }; } // namespace segment_v2 -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/src/olap/rowset/segment_v2/inverted_index_reader.h b/be/src/olap/rowset/segment_v2/inverted_index_reader.h index 9c8d9dabf79480..905fd12dfb399c 100644 --- a/be/src/olap/rowset/segment_v2/inverted_index_reader.h +++ b/be/src/olap/rowset/segment_v2/inverted_index_reader.h @@ -91,7 +91,7 @@ class InvertedIndexReader : public std::enable_shared_from_this& get_index_properties() const { return _index_meta.properties();