Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[feature](merge-cloud) Add tablet rebalance run in cloud #32925

Merged
merged 1 commit into from
Apr 1, 2024
Merged

[feature](merge-cloud) Add tablet rebalance run in cloud #32925

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 29 additions & 0 deletions fe/fe-common/src/main/java/org/apache/doris/common/Config.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2635,6 +2635,35 @@ public static boolean isNotCloudMode() {
@ConfField
public static int drop_user_notify_ms_max_times = 86400;

@ConfField(mutable = true)
public static long cloud_tablet_rebalancer_interval_second = 20;

@ConfField(mutable = true)
public static boolean enable_cloud_partition_balance = true;

@ConfField(mutable = true)
public static boolean enable_cloud_table_balance = true;

@ConfField(mutable = true)
public static boolean enable_cloud_global_balance = true;

@ConfField(mutable = true)
public static int cloud_pre_heating_time_limit_sec = 300;

@ConfField(mutable = true)
public static double cloud_rebalance_percent_threshold = 0.05;

@ConfField(mutable = true)
public static long cloud_rebalance_number_threshold = 2;

@ConfField(mutable = true)
public static double cloud_balance_tablet_percent_per_run = 0.05;

@ConfField(mutable = true)
public static int cloud_min_balance_tablet_num_per_run = 2;

@ConfField(mutable = true)
public static boolean cloud_preheating_enabled = true;
//==========================================================================
// end of cloud config
//==========================================================================
Expand Down
28 changes: 15 additions & 13 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1646,24 +1646,26 @@ protected void startMasterOnlyDaemonThreads() {
loadJobScheduler.start();
loadEtlChecker.start();
loadLoadingChecker.start();
// Tablet checker and scheduler
tabletChecker.start();
tabletScheduler.start();
// Colocate tables checker and balancer
ColocateTableCheckerAndBalancer.getInstance().start();
// Publish Version Daemon
publishVersionDaemon.start();
// Start txn cleaner
txnCleaner.start();
if (Config.isNotCloudMode()) {
// Tablet checker and scheduler
tabletChecker.start();
tabletScheduler.start();
// Colocate tables checker and balancer
ColocateTableCheckerAndBalancer.getInstance().start();
// Publish Version Daemon
publishVersionDaemon.start();
// Start txn cleaner
txnCleaner.start();
// Consistency checker
getConsistencyChecker().start();
// Backup handler
getBackupHandler().start();
}
jobManager.start();
// transient task manager
transientTaskManager.start();
// Alter
getAlterInstance().start();
// Consistency checker
getConsistencyChecker().start();
// Backup handler
getBackupHandler().start();
// catalog recycle bin
getRecycleBin().start();
// time printer
Expand Down
15 changes: 15 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/cloud/catalog/CloudEnv.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,15 @@

import org.apache.doris.analysis.ResourceTypeEnum;
import org.apache.doris.catalog.Env;
import org.apache.doris.cloud.datasource.CloudInternalCatalog;
import org.apache.doris.cloud.persist.UpdateCloudReplicaInfo;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.proto.Cloud.NodeInfoPB;
import org.apache.doris.cloud.system.CloudSystemInfoService;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.UserException;
import org.apache.doris.common.io.CountingDataOutputStream;
import org.apache.doris.common.util.HttpURLUtil;
Expand Down Expand Up @@ -57,17 +60,25 @@ public class CloudEnv extends Env {
private CloudInstanceStatusChecker cloudInstanceStatusChecker;
private CloudClusterChecker cloudClusterCheck;

private CloudTabletRebalancer cloudTabletRebalancer;

public CloudEnv(boolean isCheckpointCatalog) {
super(isCheckpointCatalog);
this.cloudClusterCheck = new CloudClusterChecker((CloudSystemInfoService) systemInfo);
this.cloudInstanceStatusChecker = new CloudInstanceStatusChecker((CloudSystemInfoService) systemInfo);
this.cloudTabletRebalancer = new CloudTabletRebalancer((CloudSystemInfoService) systemInfo);
}

public CloudTabletRebalancer getCloudTabletRebalancer() {
return this.cloudTabletRebalancer;
}

@Override
protected void startMasterOnlyDaemonThreads() {
LOG.info("start cloud Master only daemon threads");
super.startMasterOnlyDaemonThreads();
cloudClusterCheck.start();
cloudTabletRebalancer.start();
}

@Override
Expand Down Expand Up @@ -416,4 +427,8 @@ public String analyzeCloudCluster(String name, ConnectContext ctx) throws DdlExc
changeCloudCluster(res[1], ctx);
return res[0];
}

public void replayUpdateCloudReplica(UpdateCloudReplicaInfo info) throws MetaNotFoundException {
((CloudInternalCatalog) getInternalCatalog()).replayUpdateCloudReplica(info);
}
}
Loading
Loading