Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[chore](show replica) show replica print path #30402

Merged
merged 4 commits into from
Feb 1, 2024
Merged

[chore](show replica) show replica print path #30402

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
b3dd179
show replica print path
yujun777 Jan 26, 2024
f060a64
update regression
yujun777 Jan 30, 2024
25c75f4
update
yujun777 Jan 26, 2024
e68100e
update regression
yujun777 Jan 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 9 additions & 6 deletions fe/fe-core/src/main/java/org/apache/doris/common/proc/BackendProcNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
public class BackendProcNode implements ProcNodeInterface {
public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
.add("RootPath").add("DataUsedCapacity").add("OtherUsedCapacity").add("AvailCapacity")
.add("TotalCapacity").add("TotalUsedPct").add("State").add("PathHash")
.add("TotalCapacity").add("TotalUsedPct").add("State").add("PathHash").add("StorageMedium")
.build();

private Backend backend;
Expand All @@ -53,17 +53,19 @@ public ProcResult fetchResult() throws AnalysisException {
List<String> info = Lists.newArrayList();
info.add(entry.getKey());

DiskInfo disk = entry.getValue();

// data used
long dataUsedB = entry.getValue().getDataUsedCapacityB();
long dataUsedB = disk.getDataUsedCapacityB();
Pair<Double, String> dataUsedUnitPair = DebugUtil.getByteUint(dataUsedB);
info.add(DebugUtil.DECIMAL_FORMAT_SCALE_3.format(dataUsedUnitPair.first) + " "
+ dataUsedUnitPair.second);

// avail
long availB = entry.getValue().getAvailableCapacityB();
long availB = disk.getAvailableCapacityB();
Pair<Double, String> availUnitPair = DebugUtil.getByteUint(availB);
// total
long totalB = entry.getValue().getTotalCapacityB();
long totalB = disk.getTotalCapacityB();
Pair<Double, String> totalUnitPair = DebugUtil.getByteUint(totalB);
// other
long otherB = totalB - availB - dataUsedB;
Expand All @@ -82,8 +84,9 @@ public ProcResult fetchResult() throws AnalysisException {
}
info.add(String.format("%.2f", used) + " %");

info.add(entry.getValue().getState().name());
info.add(String.valueOf(entry.getValue().getPathHash()));
info.add(disk.getState().name());
info.add(String.valueOf(disk.getPathHash()));
info.add(disk.getStorageMedium().name());

result.addRow(info);
}
Expand Down
15 changes: 14 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/common/proc/ReplicasProcNode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.common.proc;

import org.apache.doris.catalog.DiskInfo;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Replica;
Expand All @@ -43,7 +44,8 @@ public class ReplicasProcNode implements ProcNodeInterface {
.add("BackendId").add("Version").add("LstSuccessVersion").add("LstFailedVersion").add("LstFailedTime")
.add("SchemaHash").add("LocalDataSize").add("RemoteDataSize").add("RowCount").add("State").add("IsBad")
.add("IsUserDrop")
.add("VersionCount").add("PathHash").add("MetaUrl").add("CompactionStatus").add("CooldownReplicaId")
.add("VersionCount").add("PathHash").add("Path")
.add("MetaUrl").add("CompactionStatus").add("CooldownReplicaId")
.add("CooldownMetaId").add("QueryHits").build();

private long tabletId;
Expand Down Expand Up @@ -84,6 +86,16 @@ public ProcResult fetchResult() {
String metaUrl = String.format("http://" + hostPort + "/api/meta/header/%d", tabletId);
String compactionUrl = String.format("http://" + hostPort + "/api/compaction/show?tablet_id=%d", tabletId);

String path = "";
if (be != null) {
DiskInfo diskInfo = be.getDisks().values().stream()
.filter(disk -> disk.getPathHash() == replica.getPathHash())
.findFirst().orElse(null);
if (diskInfo != null) {
path = diskInfo.getRootPath();
}
}

String cooldownMetaId = "";
if (replica.getCooldownMetaId() != null) {
cooldownMetaId = replica.getCooldownMetaId().toString();
Expand All @@ -107,6 +119,7 @@ public ProcResult fetchResult() {
String.valueOf(replica.isUserDrop()),
String.valueOf(replica.getVersionCount()),
String.valueOf(replica.getPathHash()),
path,
metaUrl,
compactionUrl,
String.valueOf(tablet.getCooldownConf().first),
Expand Down
13 changes: 12 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.doris.common.proc;

import org.apache.doris.catalog.DiskInfo;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.Replica;
Expand Down Expand Up @@ -51,7 +52,8 @@ public class TabletsProcDir implements ProcDirInterface {
.add("LstSuccessVersion").add("LstFailedVersion").add("LstFailedTime")
.add("LocalDataSize").add("RemoteDataSize").add("RowCount").add("State")
.add("LstConsistencyCheckTime").add("CheckVersion")
.add("VersionCount").add("QueryHits").add("PathHash").add("MetaUrl").add("CompactionStatus")
.add("VersionCount").add("QueryHits").add("PathHash").add("Path")
.add("MetaUrl").add("CompactionStatus")
.add("CooldownReplicaId").add("CooldownMetaId").build();

private Table table;
Expand All @@ -68,6 +70,12 @@ public List<List<Comparable>> fetchComparableResult(long version, long backendId
ImmutableMap<Long, Backend> backendMap = Env.getCurrentSystemInfo().getIdToBackend();

List<List<Comparable>> tabletInfos = new ArrayList<List<Comparable>>();
Map<Long, String> pathHashToRoot = new HashMap<>();
for (Backend be : backendMap.values()) {
for (DiskInfo diskInfo : be.getDisks().values()) {
pathHashToRoot.put(diskInfo.getPathHash(), diskInfo.getRootPath());
}
}
table.readLock();
try {
Map<Long, Long> replicaIdToQueryHits = new HashMap<>();
Expand All @@ -80,6 +88,7 @@ public List<List<Comparable>> fetchComparableResult(long version, long backendId
}
replicaIdToQueryHits = QueryStatsUtil.getMergedReplicasStats(replicaIds);
}

// get infos
for (Tablet tablet : index.getTablets()) {
long tabletId = tablet.getId();
Expand Down Expand Up @@ -107,6 +116,7 @@ public List<List<Comparable>> fetchComparableResult(long version, long backendId
tabletInfo.add(-1); // version count
tabletInfo.add(0L); // query hits
tabletInfo.add(-1); // path hash
tabletInfo.add(FeConstants.null_string); // path
tabletInfo.add(FeConstants.null_string); // meta url
tabletInfo.add(FeConstants.null_string); // compaction status
tabletInfo.add(-1); // cooldown replica id
Expand Down Expand Up @@ -140,6 +150,7 @@ public List<List<Comparable>> fetchComparableResult(long version, long backendId
tabletInfo.add(replica.getVersionCount());
tabletInfo.add(replicaIdToQueryHits.getOrDefault(replica.getId(), 0L));
tabletInfo.add(replica.getPathHash());
tabletInfo.add(pathHashToRoot.getOrDefault(replica.getPathHash(), ""));
Backend be = backendMap.get(replica.getBackendId());
String host = (be == null ? Backend.DUMMY_IP : be.getHost());
int port = (be == null ? 0 : be.getHttpPort());
Expand Down
2 changes: 1 addition & 1 deletion fe/fe-core/src/test/java/org/apache/doris/common/proc/BackendProcNodeTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public void testResultNormal() throws AnalysisException {

Assert.assertTrue(result.getRows().size() >= 1);
Assert.assertEquals(Lists.newArrayList("RootPath", "DataUsedCapacity", "OtherUsedCapacity", "AvailCapacity",
"TotalCapacity", "TotalUsedPct", "State", "PathHash"), result.getColumnNames());
"TotalCapacity", "TotalUsedPct", "State", "PathHash", "StorageMedium"), result.getColumnNames());
}

}
7 changes: 6 additions & 1 deletion regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -975,7 +975,12 @@ class Suite implements GroovyInterceptable {
def result = [:]

tablets.each { row ->
def tablet_id = row[0]
def tablet_id
if (row.containsKey("TabletId")) {
tablet_id = row.TabletId
} else {
tablet_id = row[0]
}
if (!result.containsKey(tablet_id)) {
result[tablet_id] = row
}
Expand Down
8 changes: 4 additions & 4 deletions ...ssion-test/suites/compaction/test_base_compaction_with_dup_key_max_file_size_limit.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,9 +150,9 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") {
)
"""

def tablet = (sql """ show tablets from ${tableName}; """)[0]
String tablet_id = tablet[0]
String trigger_backend_id = tablet[2]
def tablet = (sql_return_maparray """ show tablets from ${tableName}; """)[0]
String tablet_id = tablet.TabletId
String trigger_backend_id = tablet.BackendId

// rowsets:
// [0-1] 0
Expand Down Expand Up @@ -224,4 +224,4 @@ suite("test_base_compaction_with_dup_key_max_file_size_limit", "p2") {
assertTrue(rowCount[0][0] != rows)
} finally {
}
}
}
21 changes: 10 additions & 11 deletions regression-test/suites/compaction/test_compacation_with_delete.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,12 +96,12 @@ suite("test_compaction_with_delete") {
DELETE from ${tableName} where cost = '5'
"""
//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
def tablets = sql_return_maparray """ show tablets from ${tableName}; """

// trigger compactions for all tablets in ${tableName}
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
backend_id = tablet[2]
for (def tablet in tablets) {
String tablet_id = tablet.TabletId
backend_id = tablet.BackendId
(code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
Expand All @@ -116,12 +116,12 @@ suite("test_compaction_with_delete") {
}

// wait for all compactions done
for (String[] tablet in tablets) {
for (def tablet in tablets) {
boolean running = true
do {
Thread.sleep(1000)
String tablet_id = tablet[0]
backend_id = tablet[2]
String tablet_id = tablet.TabletId
backend_id = tablet.BackendId
(code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
Expand All @@ -134,10 +134,9 @@ suite("test_compaction_with_delete") {
def replicaNum = get_table_replica_num(tableName)
logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
def compactionStatusUrlIndex = 18
(code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
for (def tablet in tablets) {
String tablet_id = tablet.TabletId
(code, out, err) = curl("GET", tablet.CompactionStatus)
logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
def tabletJson = parseJson(out.trim())
Expand Down
21 changes: 10 additions & 11 deletions regression-test/suites/compaction/test_compaction_agg_keys.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,12 +101,12 @@ suite("test_compaction_agg_keys") {
qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """

//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,QueryHits,VersionCount,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
def tablets = sql_return_maparray """ show tablets from ${tableName}; """

// trigger compactions for all tablets in ${tableName}
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
backend_id = tablet[2]
for (def tablet in tablets) {
String tablet_id = tablet.TabletId
backend_id = tablet.BackendId
(code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
Expand All @@ -121,12 +121,12 @@ suite("test_compaction_agg_keys") {
}

// wait for all compactions done
for (String[] tablet in tablets) {
for (def tablet in tablets) {
boolean running = true
do {
Thread.sleep(1000)
String tablet_id = tablet[0]
backend_id = tablet[2]
String tablet_id = tablet.TabletId
backend_id = tablet.BackendId
(code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
Expand All @@ -140,11 +140,10 @@ suite("test_compaction_agg_keys") {
def replicaNum = get_table_replica_num(tableName)
logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
def compactionStatusUrlIndex = 18
for (def tablet in tablets) {
String tablet_id = tablet.TabletId

(code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
(code, out, err) = curl("GET", tablet.CompactionStatus)
logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)

assertEquals(code, 0)
Expand Down
21 changes: 10 additions & 11 deletions regression-test/suites/compaction/test_compaction_agg_keys_with_delete.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,12 @@ suite("test_compaction_agg_keys_with_delete") {
qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id; """

//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
def tablets = sql_return_maparray """ show tablets from ${tableName}; """

// trigger compactions for all tablets in ${tableName}
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
backend_id = tablet[2]
for (def tablet in tablets) {
String tablet_id = tablet.TabletId
backend_id = tablet.BackendId
(code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
Expand All @@ -132,12 +132,12 @@ suite("test_compaction_agg_keys_with_delete") {
}

// wait for all compactions done
for (String[] tablet in tablets) {
for (def tablet in tablets) {
boolean running = true
do {
Thread.sleep(1000)
String tablet_id = tablet[0]
backend_id = tablet[2]
String tablet_id = tablet.TabletId
backend_id = tablet.BackendId
(code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
Expand All @@ -150,11 +150,10 @@ suite("test_compaction_agg_keys_with_delete") {
def replicaNum = get_table_replica_num(tableName)
logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
def compactionStatusUrlIndex = 18
for (def tablet in tablets) {
String tablet_id = tablet.TabletId

(code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
(code, out, err) = curl("GET", tablet.CompactionStatus)
logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)

Expand Down
21 changes: 10 additions & 11 deletions regression-test/suites/compaction/test_compaction_dup_keys.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,12 +100,12 @@ suite("test_compaction_dup_keys") {
qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id,date,city,age,sex,last_visit_date,last_update_date,last_visit_date_not_null,cost,max_dwell_time,min_dwell_time; """

//TabletId,ReplicaId,BackendId,SchemaHash,Version,LstSuccessVersion,LstFailedVersion,LstFailedTime,LocalDataSize,RemoteDataSize,RowCount,State,LstConsistencyCheckTime,CheckVersion,VersionCount,QueryHits,PathHash,MetaUrl,CompactionStatus
String[][] tablets = sql """ show tablets from ${tableName}; """
def tablets = sql_return_maparray """ show tablets from ${tableName}; """

// trigger compactions for all tablets in ${tableName}
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
backend_id = tablet[2]
for (def tablet in tablets) {
String tablet_id = tablet.TabletId
backend_id = tablet.BackendId
(code, out, err) = be_run_cumulative_compaction(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
logger.info("Run compaction: code=" + code + ", out=" + out + ", err=" + err)

Expand All @@ -121,12 +121,12 @@ suite("test_compaction_dup_keys") {
}

// wait for all compactions done
for (String[] tablet in tablets) {
for (def tablet in tablets) {
boolean running = true
do {
Thread.sleep(1000)
String tablet_id = tablet[0]
backend_id = tablet[2]
String tablet_id = tablet.TabletId
backend_id = tablet.BackendId
(code, out, err) = be_get_compaction_status(backendId_to_backendIP.get(backend_id), backendId_to_backendHttpPort.get(backend_id), tablet_id)
logger.info("Get compaction status: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)
Expand All @@ -139,11 +139,10 @@ suite("test_compaction_dup_keys") {
def replicaNum = get_table_replica_num(tableName)
logger.info("get table replica num: " + replicaNum)
int rowCount = 0
for (String[] tablet in tablets) {
String tablet_id = tablet[0]
def compactionStatusUrlIndex = 18
for (def tablet in tablets) {
String tablet_id = tablet.TabletId

(code, out, err) = curl("GET", tablet[compactionStatusUrlIndex])
(code, out, err) = curl("GET", tablet.CompactionStatus)
logger.info("Show tablets status: code=" + code + ", out=" + out + ", err=" + err)
assertEquals(code, 0)

Expand Down
Loading
Loading