Skip to content

Commit

Permalink
Update placement groups view in monitoring (#603)
Browse files Browse the repository at this point in the history 
* Update placement groups view in monitoring

* fix errors

* review fixes

* fix order

* refactor strategy name

---------

Co-authored-by: Dmitry Razumov <[email protected]>
  • Loading branch information
@dvrazumov
dvrazumov and Dmitry Razumov committed Mar 5, 2024
1 parent 7b1f20d commit 44f5d0e
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 34 deletions.
157 changes: 129 additions & 28 deletions cloud/blockstore/libs/storage/disk_registry/disk_registry_actor_monitoring.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
#include <library/cpp/monlib/service/pages/templates.h>

#include <util/stream/str.h>
#include <util/string/join.h>

namespace NCloud::NBlockStore::NStorage {

Expand Down Expand Up @@ -161,6 +162,45 @@ void DumpDeviceLink(IOutputStream& out, ui64 tabletId, TStringBuf uuid)
<< "</a>";
}

void DumpSquare(IOutputStream& out, const TStringBuf& color)
{
const char* utfBlackSquare = "&#9632";
const char* nonBreakingSpace = "&nbsp";
out << "<font color="
<< color
<< ">"
<< utfBlackSquare
<< nonBreakingSpace
<< "</font>";
}

using DiskInfoArray = ::google::protobuf::RepeatedPtrField<NProto::TPlacementGroupConfig_TDiskInfo>;
auto GetSortedDisksView(
const DiskInfoArray& disks,
const THashSet<TString>& brokenDisks)
{
using TDiskIterator = decltype(disks.begin());

TVector<TDiskIterator> diskIndices(disks.size());
std::iota(diskIndices.begin(), diskIndices.end(), disks.begin());

auto sortByFailureThenByPartition =
[&](const TDiskIterator& d1, const TDiskIterator& d2)
{
auto makeComparableTuple = [&brokenDisks](const TDiskIterator& d){
return std::make_tuple(
!brokenDisks.contains(d->GetDiskId()),
d->GetPlacementPartitionIndex(),
d->GetDiskId()
);
};
return makeComparableTuple(d1) < makeComparableTuple(d2);
};

std::sort(diskIndices.begin(), diskIndices.end(), sortByFailureThenByPartition);
return diskIndices;
}

} // namespace

////////////////////////////////////////////////////////////////////////////////
Expand Down Expand Up @@ -1440,65 +1480,126 @@ void TDiskRegistryActor::RenderPlacementGroupList(
TABLEHEAD() {
TABLER() {
TABLEH() { out << "GroupId"; }
TABLEH() { out << "Strategy"; }
TABLEH() { out << "State"; }
TABLEH() { out << "Stats"; }
TABLEH() { out << "Disks"; }
}
}

for (const auto& x: State->GetPlacementGroups()) {
for (const auto& [groupId, groupInfo]: State->GetPlacementGroups()) {
TABLER() {
TABLED() {
auto it = brokenGroups.find(x.first);
const auto strategy = groupInfo.Config.GetPlacementStrategy();
const bool isPartitionGroup =
strategy == NProto::PLACEMENT_STRATEGY_PARTITION;

if (it != brokenGroups.end()) {
if (it->second.Recently.GetBrokenPartitionCount() == 1) {
out << "<font color=yellow>&#9632;&nbsp;</font>";
}
auto brokenGroupInfo = brokenGroups.FindPtr(groupId);
const size_t brokenPartitionsCount = brokenGroupInfo
? brokenGroupInfo->Recently.GetBrokenPartitionCount()
: 0;

if (it->second.Recently.GetBrokenPartitionCount() > 1) {
out << "<font color=red>&#9632;&nbsp;</font>";
}
TABLED() {
const auto* color = brokenPartitionsCount == 0
? "green"
: (brokenPartitionsCount == 1 ? "orange" : "red");
DumpSquare(out, color);
out << groupId;
}
TABLED() {
TStringBuf name = EPlacementStrategy_Name(strategy);
name.AfterPrefix("PLACEMENT_STRATEGY_", name);
out << name;
}
TABLED() {
size_t totalPartitionsCount = isPartitionGroup
? groupInfo.Config.GetPlacementPartitionCount()
: groupInfo.Config.GetDisks().size();

out << Sprintf("%s: <font color=green>Fine: %zu</font>, ",
isPartitionGroup ? "Partitions" : "Disks",
totalPartitionsCount - brokenPartitionsCount);

if (brokenPartitionsCount > 0) {
out << Sprintf("<font color=%s>Broken: %zu</font>, ",
brokenPartitionsCount == 1 ? "orange" : "red",
brokenPartitionsCount);
}

out << x.first;
out << Sprintf("Total: %zu<br>", totalPartitionsCount);
}
TABLED() {
if (x.second.Full) {
out << "<font color=red>GROUP IS FULL</font>, ";
if (groupInfo.Full) {
out << "<font color=red>GROUP IS FULL</font><br>";
}

out << "BiggestDisk: " << x.second.BiggestDiskId
out << "BiggestDisk: " << groupInfo.BiggestDiskId
<< " ("
<< FormatByteSize(x.second.BiggestDiskSize) << ")";
<< FormatByteSize(groupInfo.BiggestDiskSize) << ")";
}
TABLED() {
TABLE_SORTABLE_CLASS("table table-bordered") {
TABLER() {
TABLED() { out << "ConfigVersion"; }
TABLED() { out << x.second.Config.GetConfigVersion(); }
TABLED()
{
out << groupInfo.Config.GetConfigVersion();
}
}
TABLER() {
TABLED() { out << "Disk count"; }
TABLED()
{
out << groupInfo.Config.GetDisks().size();
}
}

TABLER() {
TABLED() { out << "Disks"; }
TABLED() {
TABLED_ATTRS({{"colspan", "2"}}) {
TABLE_SORTABLE_CLASS("table table-bordered") {
TABLEHEAD() {
TABLER() {
TABLEH() { out << "DiskId"; }
TABLEH() { out << "Racks"; }
if (isPartitionGroup) {
TABLEH()
{
out << "Partition";
}
}
}
}
for (const auto& d: x.second.Config.GetDisks()) {

const auto brokenDisks = brokenGroupInfo
? brokenGroupInfo->Recently.GetBrokenDisks()
: THashSet<TString>();

const auto sortedDisks =
GetSortedDisksView(
groupInfo.Config.GetDisks(),
brokenDisks);

for (const auto& d: sortedDisks) {
TABLER() {
TABLED() { DumpDiskLink(out, TabletID(), d.GetDiskId()); }
TABLED() {
for (ui32 i = 0; i < d.DeviceRacksSize(); ++i) {
const auto& rack = d.GetDeviceRacks(i);
if (i) {
out << ", ";
}

out << rack;
const bool isBroken =
brokenDisks.contains(
d->GetDiskId());
DumpSquare(
out,
isBroken ? "red"
: "green");
DumpDiskLink(
out,
TabletID(),
d->GetDiskId());
}
TABLED() {
out << JoinSeq(
", ",
d->GetDeviceRacks());
}
if (isPartitionGroup) {
TABLED() {
out << d->GetPlacementPartitionIndex();
}
}
}
Expand Down
4 changes: 2 additions & 2 deletions cloud/blockstore/libs/storage/disk_registry/disk_registry_state.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3642,9 +3642,9 @@ THashMap<TString, TBrokenGroupInfo> TDiskRegistryState::GatherBrokenGroupsInfo(
auto res = groups.try_emplace(groupId, pg->Config.GetPlacementStrategy());
TBrokenGroupInfo& info = res.first->second;

info.Total.Increment(disk.PlacementPartitionIndex);
info.Total.Increment(diskId, disk.PlacementPartitionIndex);
if (now - period < disk.StateTs) {
info.Recently.Increment(disk.PlacementPartitionIndex);
info.Recently.Increment(diskId, disk.PlacementPartitionIndex);
}
}

Expand Down
13 changes: 9 additions & 4 deletions cloud/blockstore/libs/storage/disk_registry/disk_registry_state.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -111,9 +111,9 @@ class TBrokenCounter
: Strategy(strategy)
{}

void Increment(ui32 partitionIndex)
void Increment(const TString& diskId, ui32 partitionIndex)
{
++BrokenDiskCount;
BrokenDisks.insert(diskId);
if (Strategy == NProto::EPlacementStrategy::PLACEMENT_STRATEGY_PARTITION) {
BrokenPartitions.insert(partitionIndex);
}
Expand All @@ -123,7 +123,7 @@ class TBrokenCounter
{
switch (Strategy) {
case NProto::EPlacementStrategy::PLACEMENT_STRATEGY_SPREAD:
return BrokenDiskCount;
return BrokenDisks.size();
case NProto::EPlacementStrategy::PLACEMENT_STRATEGY_PARTITION:
return BrokenPartitions.size();
default:
Expand All @@ -135,9 +135,14 @@ class TBrokenCounter
}
}

[[nodiscard]] const THashSet<TString>& GetBrokenDisks() const
{
return BrokenDisks;
}

private:
NProto::EPlacementStrategy Strategy;
ui32 BrokenDiskCount = 0;
THashSet<TString> BrokenDisks;
THashSet<ui32> BrokenPartitions;
};

Expand Down

0 comments on commit 44f5d0e

Please sign in to comment.