Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Fix](Schema Change ) fix schema change fail as internal sorting will no change to run #39979

Open
wants to merge 1 commit into
base: branch-2.0
Choose a base branch
from
Open

[Fix](Schema Change ) fix schema change fail as internal sorting will no change to run #39979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions be/src/common/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,11 @@ DEFINE_mInt64(column_dictionary_key_size_threshold, "0");
DEFINE_mInt64(memory_limitation_per_thread_for_schema_change_bytes, "2147483648");
DEFINE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes, "100000000");

// memory_limitation_per_thread_for_schema_change_internal_sorting_bytes unit bytes
DEFINE_mInt64(memory_limitation_per_thread_for_schema_change_internal_sorting_bytes, "2147483648");
// schema change internal sorting memory limit as a fraction of soft memory limit
DEFINE_Double(schema_change_internal_sorting_mem_limit_frac_per_thread, "0.5");

// the clean interval of file descriptor cache and segment cache
DEFINE_mInt32(cache_clean_interval, "10");
// the clean interval of tablet lookup cache
Expand Down
3 changes: 3 additions & 0 deletions be/src/common/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -316,6 +316,9 @@ DECLARE_mInt64(column_dictionary_key_size_threshold);
DECLARE_mInt64(memory_limitation_per_thread_for_schema_change_bytes);
DECLARE_mInt64(memory_limitation_per_thread_for_storage_migration_bytes);

DECLARE_mInt64(memory_limitation_per_thread_for_schema_change_internal_sorting_bytes);
DECLARE_Double(schema_change_internal_sorting_mem_limit_frac_per_thread);

// the clean interval of file descriptor cache and segment cache
DECLARE_mInt32(cache_clean_interval);
// the clean interval of tablet lookup cache
Expand Down
7 changes: 4 additions & 3 deletions be/src/olap/schema_change.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,9 +239,10 @@ class SchemaChangeHandler {
bool sc_sorting, bool sc_directly) {
if (sc_sorting) {
return std::make_unique<VSchemaChangeWithSorting>(
changer, ExecEnv::GetInstance()
->storage_engine()
->memory_limitation_bytes_per_thread_for_schema_change());
changer,
ExecEnv::GetInstance()
->storage_engine()
->memory_limitation_bytes_per_thread_for_schema_change_internal_sorting());
}

if (sc_directly) {
Expand Down
8 changes: 8 additions & 0 deletions be/src/olap/storage_engine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,14 @@ int64_t StorageEngine::memory_limitation_bytes_per_thread_for_schema_change() co
config::memory_limitation_per_thread_for_schema_change_bytes);
}

int64_t StorageEngine::memory_limitation_bytes_per_thread_for_schema_change_internal_sorting()
const {
return std::min(
static_cast<int64_t>(memory_limitation_bytes_per_thread_for_schema_change() *
config::schema_change_internal_sorting_mem_limit_frac_per_thread),
config::memory_limitation_per_thread_for_schema_change_internal_sorting_bytes);
}

Status StorageEngine::load_data_dirs(const std::vector<DataDir*>& data_dirs) {
std::vector<std::thread> threads;
std::vector<Status> results(data_dirs.size());
Expand Down
2 changes: 2 additions & 0 deletions be/src/olap/storage_engine.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,8 @@ class StorageEngine {

int64_t memory_limitation_bytes_per_thread_for_schema_change() const;

int64_t memory_limitation_bytes_per_thread_for_schema_change_internal_sorting() const;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

warning: function 'memory_limitation_bytes_per_thread_for_schema_change_internal_sorting' should be marked [[nodiscard]] [modernize-use-nodiscard]

Suggested change
int64_t memory_limitation_bytes_per_thread_for_schema_change_internal_sorting() const;
[[nodiscard]] int64_t memory_limitation_bytes_per_thread_for_schema_change_internal_sorting() const;


private:
// Instance should be inited from `static open()`
// MUST NOT be called in other circumstances.
Expand Down
Loading