Skip to content

Commit

Permalink
[fix](inverted index) Fix the issue where the inverted index file_cac…
Browse files Browse the repository at this point in the history 
…he was not being counted
  • Loading branch information
@zzzxl1993
zzzxl1993 committed Aug 10, 2024
1 parent 289100b commit 584230f
Show file tree
Hide file tree
Showing 4 changed files with 51 additions and 28 deletions.
3 changes: 1 addition & 2 deletions be/src/olap/rowset/segment_v2/column_reader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,8 +332,7 @@ Status ColumnReader::new_inverted_index_iterator(
{
std::shared_lock<std::shared_mutex> rlock(_load_index_lock);
if (_inverted_index) {
RETURN_IF_ERROR(_inverted_index->new_iterator(read_options.stats,
read_options.runtime_state, iterator));
RETURN_IF_ERROR(_inverted_index->new_iterator(read_options, iterator));
}
}
return Status::OK();
Expand Down
14 changes: 13 additions & 1 deletion be/src/olap/rowset/segment_v2/inverted_index_fs_directory.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
#include "inverted_index_desc.h"
#include "io/fs/file_reader.h"
#include "io/fs/file_writer.h"
#include "olap/iterators.h"
#include "olap/rowset/segment_v2/inverted_index_reader.h"
#include "olap/tablet_schema.h"
#include "util/debug_points.h"
#include "util/slice.h"
Expand Down Expand Up @@ -230,7 +232,17 @@ void DorisFSDirectory::FSIndexInput::readInternal(uint8_t* b, const int32_t len)

Slice result {b, (size_t)len};
size_t bytes_read = 0;
if (!_handle->_reader->read_at(_pos, result, &bytes_read, &_io_ctx).ok()) {

Status res;
if (InvertedIndexIterator::_tls_opts) {
res = _handle->_reader->read_at(_pos, result, &bytes_read,
&InvertedIndexIterator::_tls_opts->io_ctx);
} else {
io::IOContext _io_ctx;
_io_ctx.reader_type = ReaderType::READER_QUERY;
res = _handle->_reader->read_at(_pos, result, &bytes_read, &_io_ctx);
}
if (!res.ok()) {
_CLTHROWA(CL_ERR_IO, "read past EOF");
}
bufferLength = len;
Expand Down
39 changes: 25 additions & 14 deletions be/src/olap/rowset/segment_v2/inverted_index_reader.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@

#include "gutil/integral_types.h"
#include "inverted_index_query_type.h"
#include "olap/iterators.h"
#include "olap/rowset/segment_v2/inverted_index/query/phrase_query.h"

#ifdef __clang__
Expand Down Expand Up @@ -323,9 +324,9 @@ Status InvertedIndexReader::match_index_search(
return Status::OK();
}

Status FullTextIndexReader::new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state,
Status FullTextIndexReader::new_iterator(const StorageReadOptions& opts,
std::unique_ptr<InvertedIndexIterator>* iterator) {
*iterator = InvertedIndexIterator::create_unique(stats, runtime_state, shared_from_this());
*iterator = InvertedIndexIterator::create_unique(opts, shared_from_this());
return Status::OK();
}

Expand Down Expand Up @@ -450,9 +451,8 @@ void FullTextIndexReader::setup_analyzer_use_stopwords(
}

Status StringTypeInvertedIndexReader::new_iterator(
OlapReaderStatistics* stats, RuntimeState* runtime_state,
std::unique_ptr<InvertedIndexIterator>* iterator) {
*iterator = InvertedIndexIterator::create_unique(stats, runtime_state, shared_from_this());
const StorageReadOptions& opts, std::unique_ptr<InvertedIndexIterator>* iterator) {
*iterator = InvertedIndexIterator::create_unique(opts, shared_from_this());
return Status::OK();
}

Expand Down Expand Up @@ -583,9 +583,9 @@ InvertedIndexReaderType StringTypeInvertedIndexReader::type() {
return InvertedIndexReaderType::STRING_TYPE;
}

Status BkdIndexReader::new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state,
Status BkdIndexReader::new_iterator(const StorageReadOptions& opts,
std::unique_ptr<InvertedIndexIterator>* iterator) {
*iterator = InvertedIndexIterator::create_unique(stats, runtime_state, shared_from_this());
*iterator = InvertedIndexIterator::create_unique(opts, shared_from_this());
return Status::OK();
}

Expand Down Expand Up @@ -1212,11 +1212,11 @@ Status InvertedIndexIterator::read_from_inverted_index(
throw CLuceneError(CL_ERR_NullPointer, "bkd index reader is null", false);
}
if (!skip_try && _reader->type() == InvertedIndexReaderType::BKD) {
if (_runtime_state != nullptr &&
_runtime_state->query_options().inverted_index_skip_threshold > 0 &&
_runtime_state->query_options().inverted_index_skip_threshold < 100) {
if (_opts.runtime_state != nullptr &&
_opts.runtime_state->query_options().inverted_index_skip_threshold > 0 &&
_opts.runtime_state->query_options().inverted_index_skip_threshold < 100) {
auto query_bkd_limit_percent =
_runtime_state->query_options().inverted_index_skip_threshold;
_opts.runtime_state->query_options().inverted_index_skip_threshold;
uint32_t hit_count = 0;
RETURN_IF_ERROR(
try_read_from_inverted_index(column_name, query_value, query_type, &hit_count));
Expand All @@ -1228,8 +1228,13 @@ Status InvertedIndexIterator::read_from_inverted_index(
}
}

RETURN_IF_ERROR(
_reader->query(_stats, _runtime_state, column_name, query_value, query_type, bit_map));
Defer defer {[&] { _tls_opts = nullptr; }};
if (_opts.runtime_state && _opts.runtime_state->enable_profile()) {
_tls_opts = &_opts;
}

RETURN_IF_ERROR(_reader->query(_opts.stats, _opts.runtime_state, column_name, query_value,
query_type, bit_map));
return Status::OK();
}

Expand All @@ -1243,11 +1248,17 @@ Status InvertedIndexIterator::try_read_from_inverted_index(const std::string& co
query_type == InvertedIndexQueryType::LESS_EQUAL_QUERY ||
query_type == InvertedIndexQueryType::LESS_THAN_QUERY ||
query_type == InvertedIndexQueryType::EQUAL_QUERY) {
RETURN_IF_ERROR(_reader->try_query(_stats, column_name, query_value, query_type, count));
RETURN_IF_ERROR(
_reader->try_query(_opts.stats, column_name, query_value, query_type, count));
}
return Status::OK();
}

Status InvertedIndexIterator::read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir) {
return _reader->read_null_bitmap(_opts.stats, cache_handle, dir);
}

InvertedIndexReaderType InvertedIndexIterator::get_inverted_index_reader_type() const {
return _reader->type();
}
Expand Down
23 changes: 12 additions & 11 deletions be/src/olap/rowset/segment_v2/inverted_index_reader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,13 +66,15 @@ class KeyCoder;
class TypeInfo;
struct OlapReaderStatistics;
class RuntimeState;
class StorageReadOptions;

namespace segment_v2 {

class InvertedIndexIterator;
class InvertedIndexQueryCacheHandle;
class InvertedIndexFileReader;
struct InvertedIndexQueryInfo;

class InvertedIndexReader : public std::enable_shared_from_this<InvertedIndexReader> {
public:
explicit InvertedIndexReader(
Expand All @@ -83,7 +85,7 @@ class InvertedIndexReader : public std::enable_shared_from_this<InvertedIndexRea
virtual ~InvertedIndexReader() = default;

// create a new column iterator. Client should delete returned iterator
virtual Status new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state,
virtual Status new_iterator(const StorageReadOptions& opts,
std::unique_ptr<InvertedIndexIterator>* iterator) = 0;
virtual Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
const std::string& column_name, const void* query_value,
Expand Down Expand Up @@ -165,7 +167,7 @@ class FullTextIndexReader : public InvertedIndexReader {
: InvertedIndexReader(index_meta, inverted_index_file_reader) {}
~FullTextIndexReader() override = default;

Status new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state,
Status new_iterator(const StorageReadOptions& opts,
std::unique_ptr<InvertedIndexIterator>* iterator) override;
Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
const std::string& column_name, const void* query_value,
Expand Down Expand Up @@ -196,7 +198,7 @@ class StringTypeInvertedIndexReader : public InvertedIndexReader {
: InvertedIndexReader(index_meta, inverted_index_file_reader) {}
~StringTypeInvertedIndexReader() override = default;

Status new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state,
Status new_iterator(const StorageReadOptions& opts,
std::unique_ptr<InvertedIndexIterator>* iterator) override;
Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
const std::string& column_name, const void* query_value,
Expand Down Expand Up @@ -255,7 +257,7 @@ class BkdIndexReader : public InvertedIndexReader {
: InvertedIndexReader(index_meta, inverted_index_file_reader) {}
~BkdIndexReader() override = default;

Status new_iterator(OlapReaderStatistics* stats, RuntimeState* runtime_state,
Status new_iterator(const StorageReadOptions& opts,
std::unique_ptr<InvertedIndexIterator>* iterator) override;

Status query(OlapReaderStatistics* stats, RuntimeState* runtime_state,
Expand Down Expand Up @@ -360,9 +362,9 @@ class InvertedIndexIterator {
ENABLE_FACTORY_CREATOR(InvertedIndexIterator);

public:
InvertedIndexIterator(OlapReaderStatistics* stats, RuntimeState* runtime_state,
InvertedIndexIterator(const StorageReadOptions& opts,
std::shared_ptr<InvertedIndexReader> reader)
: _stats(stats), _runtime_state(runtime_state), _reader(std::move(reader)) {}
: _opts(opts), _reader(std::move(reader)) {}

Status read_from_inverted_index(const std::string& column_name, const void* query_value,
InvertedIndexQueryType query_type, uint32_t segment_num_rows,
Expand All @@ -372,19 +374,18 @@ class InvertedIndexIterator {
InvertedIndexQueryType query_type, uint32_t* count);

Status read_null_bitmap(InvertedIndexQueryCacheHandle* cache_handle,
lucene::store::Directory* dir = nullptr) {
return _reader->read_null_bitmap(_stats, cache_handle, dir);
}
lucene::store::Directory* dir = nullptr);

[[nodiscard]] InvertedIndexReaderType get_inverted_index_reader_type() const;
[[nodiscard]] const std::map<string, string>& get_index_properties() const;
[[nodiscard]] bool has_null() { return _reader->has_null(); };

const InvertedIndexReaderPtr& reader() { return _reader; }

static inline thread_local const StorageReadOptions* _tls_opts = nullptr;

private:
OlapReaderStatistics* _stats = nullptr;
RuntimeState* _runtime_state = nullptr;
const StorageReadOptions& _opts;
std::shared_ptr<InvertedIndexReader> _reader;
};

Expand Down

0 comments on commit 584230f

Please sign in to comment.