Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[Filestore, merge to stable]: issue-1795: LargeDeletionMarkers, issue-1922: making background ops' read/writeblob requests async, issue-1922: avoiding Compaction loops, allow for some actions to ignore auth, add information about backpressure to the monpage, issue-1146: add setters for in-memory state #2020

Closed
wants to merge 8 commits into from
Closed

[Filestore, merge to stable]: issue-1795: LargeDeletionMarkers, issue-1922: making background ops' read/writeblob requests async, issue-1922: avoiding Compaction loops, allow for some actions to ignore auth, add information about backpressure to the monpage, issue-1146: add setters for in-memory state #2020

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
babde3c
[Filestore] allow for some actions to ignore auth (#2009)
debnatkh Sep 12, 2024
b534d97
[Filestore] add information about backpressure to the monpage (#1966)
debnatkh Sep 11, 2024
3f356ee
issue-1146: add setters for in-memory state, temporary get rid of _Ve…
debnatkh Sep 12, 2024
44b1d90
issue-1795: LargeDeletionMarkers (#1907)
qkrorlqr Sep 10, 2024
52c3f51
issue-1922: making background ops' read/writeblob requests async (#1996)
qkrorlqr Sep 10, 2024
8fb1ce2
issue-1922: avoiding Compaction loops by limiting BlobsCount incremen…
qkrorlqr Sep 12, 2024
49a73bb
fix CMakeLists.txt
debnatkh Sep 12, 2024
bfebb2f
fix build
debnatkh Sep 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions cloud/filestore/config/server.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ message TServerConfig
// Unix-socket details.
optional string UnixSocketPath = 17;
optional uint32 UnixSocketBacklog = 18;

// List of actions served by the server without authorization.
repeated string ActionsNoAuth = 19;
}

////////////////////////////////////////////////////////////////////////////////
Expand Down
26 changes: 21 additions & 5 deletions cloud/filestore/config/storage.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ message TStorageConfig

// The size of data (in bytes) in the fresh bytes table that triggers
// flushing.
optional uint32 FlushBytesThreshold = 14;
optional uint64 FlushBytesThreshold = 14;

// Size of allocation unit for HDD drives (in GiB).
optional uint32 AllocationUnitHDD = 15;
Expand Down Expand Up @@ -143,7 +143,7 @@ message TStorageConfig
optional uint32 FlushThresholdForBackpressure = 59;
optional uint32 CleanupThresholdForBackpressure = 60;
optional uint32 CompactionThresholdForBackpressure = 61;
optional uint32 FlushBytesThresholdForBackpressure = 62;
optional uint64 FlushBytesThresholdForBackpressure = 62;

// Threshold for blob size in bytes.
optional uint32 MaxBlobSize = 63;
Expand Down Expand Up @@ -351,11 +351,11 @@ message TStorageConfig
optional bool InMemoryIndexCacheEnabled = 371;
// Capacity of in-memory index cache, in number of entries per each table
optional uint64 InMemoryIndexCacheNodesCapacity = 372;
optional uint64 InMemoryIndexCacheNodesVerCapacity = 373;
reserved 373; // InMemoryIndexCacheNodesVerCapacity
optional uint64 InMemoryIndexCacheNodeAttrsCapacity = 374;
optional uint64 InMemoryIndexCacheNodeAttrsVerCapacity = 375;
reserved 375; // InMemoryIndexCacheNodeAttrsVerCapacity
optional uint64 InMemoryIndexCacheNodeRefsCapacity = 376;
optional uint64 InMemoryIndexCacheNodeRefsVerCapacity = 377;
reserved 377; // InMemoryIndexCacheNodeRefsVerCapacity

// Used to send non-network metrics as network ones to HIVE,
// while we use them for load balancing
Expand All @@ -371,4 +371,20 @@ message TStorageConfig
optional uint32 NodeRegistrationMaxAttempts = 381;
optional uint32 NodeRegistrationTimeout = 382; // in ms
optional uint32 NodeRegistrationErrorTimeout = 383; // in ms

// Max block count per file.
// uint32 is chosen deliberately - using values that exceed 2^32 - 1 or even
// 2^31 (which is 8TiB for a 4KiB block) should be thoroughly tested anyway
optional uint32 MaxFileBlocks = 384;
// Enables the usage of large deletion markers (needed for efficient
// truncate ops on large files).
optional bool LargeDeletionMarkersEnabled = 385;
// Sets max block count per single large deletion marker.
optional uint64 LargeDeletionMarkerBlocks = 386;
// Truncate and allocate ops that exceed this threshold will lead to large
// deletion marker generation.
optional uint64 LargeDeletionMarkersThreshold = 387;
// If the number of blocks marked for deletion via large deletion markers
// exceeds this threshold, Cleanup will be triggered.
optional uint64 LargeDeletionMarkersCleanupThreshold = 388;
}
3 changes: 2 additions & 1 deletion cloud/filestore/libs/daemon/server/bootstrap.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,8 @@ void TBootstrapServer::InitKikimrService()

Service = CreateAuthService(
std::move(Service),
CreateKikimrAuthProvider(ActorSystem));
CreateKikimrAuthProvider(ActorSystem),
Configs->ServerConfig->GetActionsNoAuth());

STORAGE_INFO("AuthService initialized");
}
Expand Down
2 changes: 1 addition & 1 deletion cloud/filestore/libs/endpoint/service_auth.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ class TAuthService final
{
const auto& headers = request->GetHeaders();
const auto& internal = headers.GetInternal();
auto permissions = GetRequestPermissions(*request);
auto permissions = GetRequestPermissions(*request, {});

bool needAuth = AuthProvider->NeedAuth(
internal.GetRequestSource(),
Expand Down
24 changes: 24 additions & 0 deletions cloud/filestore/libs/server/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ constexpr TDuration Seconds(int s)
xxx(Certs, TVector<TCertificate>, {} )\
xxx(UnixSocketPath, TString, {} )\
xxx(UnixSocketBacklog, ui32, 16 )\
\
xxx(ActionsNoAuth, TVector<TString>, {} )\
// FILESTORE_SERVER_CONFIG

#define FILESTORE_SERVER_DECLARE_CONFIG(name, type, value) \
Expand Down Expand Up @@ -71,6 +73,17 @@ TVector<TCertificate> ConvertValue(
return v;
}

template <>
TVector<TString> ConvertValue(
const google::protobuf::RepeatedPtrField<TString>& value)
{
TVector<TString> v;
for (const auto& x : value) {
v.push_back(x);
}
return v;
}

template <typename T>
bool IsEmpty(const T& t)
{
Expand Down Expand Up @@ -105,6 +118,17 @@ void DumpImpl(const TVector<TCertificate>& value, IOutputStream& os)
}
}

template <>
void DumpImpl(const TVector<TString>& value, IOutputStream& os)
{
for (size_t i = 0; i < value.size(); ++i) {
if (i) {
os << ",";
}
os << value[i];
}
}

} // namespace

////////////////////////////////////////////////////////////////////////////////
Expand Down
2 changes: 2 additions & 0 deletions cloud/filestore/libs/server/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ class TServerConfig
TString GetUnixSocketPath() const;
ui32 GetUnixSocketBacklog() const;

TVector<TString> GetActionsNoAuth() const;

const NProto::TServerConfig& GetProto() const
{
return ProtoConfig;
Expand Down
7 changes: 6 additions & 1 deletion cloud/filestore/libs/service/auth_scheme.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,8 @@ TPermissionList GetRequestPermissions(EFileStoreRequest requestType)
}

TPermissionList GetRequestPermissions(
const NProto::TExecuteActionRequest& request)
const NProto::TExecuteActionRequest& request,
const TVector<TString>& actionsNoAuth)
{
TString action = request.GetAction();
action.to_lower();
Expand All @@ -108,6 +109,10 @@ TPermissionList GetRequestPermissions(
return std::pair {name, std::move(lst)};
};

if (!!FindPtr(actionsNoAuth, action)) {
return TPermissionList();
}

static const THashMap<TString, TPermissionList> actions = {
// Get
perms("getstorageconfigfields", CreatePermissionList({EPermission::Get})),
Expand Down
8 changes: 5 additions & 3 deletions cloud/filestore/libs/service/auth_scheme.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,15 @@ namespace NCloud::NFileStore {
TPermissionList GetRequestPermissions(EFileStoreRequest requestType);

template <typename T>
TPermissionList GetRequestPermissions(const T& request)
TPermissionList GetRequestPermissions(
const T& request,
const TVector<TString>& actionsNoAuth)
{
Y_UNUSED(request);
Y_UNUSED(request, actionsNoAuth);
return GetRequestPermissions(GetFileStoreRequest<T>());
}

TPermissionList GetRequestPermissions(
const NProto::TExecuteActionRequest& request);
const NProto::TExecuteActionRequest& request, const TVector<TString>& actionsNoAuth);

} // namespace NCloud::NFileStore
1 change: 0 additions & 1 deletion cloud/filestore/libs/service/filestore.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ constexpr ui32 MaxLink = NProto::E_FS_LIMITS_LINK;
constexpr ui32 MaxName = NProto::E_FS_LIMITS_NAME;
constexpr ui32 MaxPath = NProto::E_FS_LIMITS_PATH;
constexpr ui32 MaxSymlink = NProto::E_FS_LIMITS_SYMLINK;
constexpr ui64 MaxFileBlocks = static_cast<ui32>(NProto::E_FS_LIMITS_FILEBLOCKS);
constexpr ui64 MaxNodes = static_cast<ui32>(NProto::E_FS_LIMITS_INODES);
constexpr ui64 MaxXAttrName = NProto::E_FS_LIMITS_XATTR_NAME;
constexpr ui64 MaxXAttrValue = NProto::E_FS_LIMITS_XATTR_VALUE;
Expand Down
13 changes: 9 additions & 4 deletions cloud/filestore/libs/service/service_auth.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,16 @@ class TAuthService final
private:
const IFileStoreServicePtr Service;
const IAuthProviderPtr AuthProvider;
const TVector<TString> ActionsNoAuth;

public:
TAuthService(
IFileStoreServicePtr service,
IAuthProviderPtr authProvider)
IAuthProviderPtr authProvider,
TVector<TString> actionsNoAuth)
: Service(std::move(service))
, AuthProvider(std::move(authProvider))
, ActionsNoAuth(std::move(actionsNoAuth))
{}

void Start() override
Expand Down Expand Up @@ -78,7 +81,7 @@ class TAuthService final
{
const auto& headers = request->GetHeaders();
const auto& internal = headers.GetInternal();
auto permissions = GetRequestPermissions(*request);
auto permissions = GetRequestPermissions(*request, ActionsNoAuth);

bool needAuth = AuthProvider->NeedAuth(
internal.GetRequestSource(),
Expand Down Expand Up @@ -149,11 +152,13 @@ class TAuthService final

IFileStoreServicePtr CreateAuthService(
IFileStoreServicePtr service,
IAuthProviderPtr authProvider)
IAuthProviderPtr authProvider,
const TVector<TString>& actionsNoAuth)
{
return std::make_shared<TAuthService>(
std::move(service),
std::move(authProvider));
std::move(authProvider),
actionsNoAuth);
}

} // namespace NCloud::NFileStore
5 changes: 4 additions & 1 deletion cloud/filestore/libs/service/service_auth.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,15 @@

#include "public.h"

#include <util/generic/vector.h>

namespace NCloud::NFileStore {

////////////////////////////////////////////////////////////////////////////////

IFileStoreServicePtr CreateAuthService(
IFileStoreServicePtr service,
IAuthProviderPtr authProvider);
IAuthProviderPtr authProvider,
const TVector<TString>& actionsNoAuth);

} // namespace NCloud::NFileStore
6 changes: 4 additions & 2 deletions cloud/filestore/libs/service_kikimr/auth_provider_kikimr_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,8 @@ Y_UNIT_TEST_SUITE(TKikimrAuthProviderTest)

auto service = CreateAuthService(
testService,
CreateKikimrAuthProvider(actorSystem));
CreateKikimrAuthProvider(actorSystem),
{});

// When requiring authorization and failing it, we fail the request.
{
Expand Down Expand Up @@ -185,7 +186,8 @@ Y_UNIT_TEST_SUITE(TKikimrAuthProviderTest)

auto service = CreateAuthService(
std::make_shared<TFileStoreTest>(),
CreateKikimrAuthProvider(actorSystem));
CreateKikimrAuthProvider(actorSystem),
{});

auto request = std::make_shared<NProto::TCreateFileStoreRequest>();
auto& headers = *request->MutableHeaders();
Expand Down
14 changes: 9 additions & 5 deletions cloud/filestore/libs/storage/core/config.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,18 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
xxx(GarbageCompactionThresholdAverage, ui32, 20 )\
xxx(NewCompactionEnabled, bool, false )\
xxx(CollectGarbageThreshold, ui32, 4_MB )\
xxx(FlushBytesThreshold, ui32, 4_MB )\
xxx(FlushBytesThreshold, ui64, 4_MB )\
xxx(MaxDeleteGarbageBlobsPerTx, ui32, 16384 )\
xxx(LoadedCompactionRangesPerTx, ui32, 10 * 1024 * 1024 )\
xxx(MaxBlocksPerTruncateTx, ui32, 0 /*TODO: 32GiB/4KiB*/ )\
xxx(MaxTruncateTxInflight, ui32, 10 )\
\
xxx(MaxFileBlocks, ui32, 300_GB / 4_KB )\
xxx(LargeDeletionMarkersEnabled, bool, false )\
xxx(LargeDeletionMarkerBlocks, ui64, 1_GB / 4_KB )\
xxx(LargeDeletionMarkersThreshold, ui64, 128_GB / 4_KB )\
xxx(LargeDeletionMarkersCleanupThreshold, ui64, 1_TB / 4_KB )\
\
xxx(CompactionRetryTimeout, TDuration, TDuration::Seconds(1) )\
xxx(BlobIndexOpsPriority, \
NProto::EBlobIndexOpsPriority, \
Expand All @@ -56,7 +63,7 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
xxx(FlushThresholdForBackpressure, ui32, 128_MB )\
xxx(CleanupThresholdForBackpressure, ui32, 32768 )\
xxx(CompactionThresholdForBackpressure, ui32, 200 )\
xxx(FlushBytesThresholdForBackpressure, ui32, 128_MB )\
xxx(FlushBytesThresholdForBackpressure, ui64, 128_MB )\
\
xxx(HDDSystemChannelPoolKind, TString, "rot" )\
xxx(HDDLogChannelPoolKind, TString, "rot" )\
Expand Down Expand Up @@ -186,11 +193,8 @@ using TAliases = NProto::TStorageConfig::TFilestoreAliases;
\
xxx(InMemoryIndexCacheEnabled, bool, false )\
xxx(InMemoryIndexCacheNodesCapacity, ui64, 0 )\
xxx(InMemoryIndexCacheNodesVerCapacity, ui64, 0 )\
xxx(InMemoryIndexCacheNodeAttrsCapacity, ui64, 0 )\
xxx(InMemoryIndexCacheNodeAttrsVerCapacity, ui64, 0 )\
xxx(InMemoryIndexCacheNodeRefsCapacity, ui64, 0 )\
xxx(InMemoryIndexCacheNodeRefsVerCapacity, ui64, 0 )\
xxx(NonNetworkMetricsBalancingFactor, ui32, 1_KB )\
\
xxx(AsyncDestroyHandleEnabled, bool, false )\
Expand Down
13 changes: 8 additions & 5 deletions cloud/filestore/libs/storage/core/config.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,14 +82,14 @@ class TStorageConfig
ui32 GetGarbageCompactionThresholdAverage() const;
bool GetNewCompactionEnabled() const;
ui32 GetCollectGarbageThreshold() const;
ui32 GetFlushBytesThreshold() const;
ui64 GetFlushBytesThreshold() const;
ui32 GetMaxDeleteGarbageBlobsPerTx() const;
ui32 GetLoadedCompactionRangesPerTx() const;

ui32 GetFlushThresholdForBackpressure() const;
ui32 GetCleanupThresholdForBackpressure() const;
ui32 GetCompactionThresholdForBackpressure() const;
ui32 GetFlushBytesThresholdForBackpressure() const;
ui64 GetFlushBytesThresholdForBackpressure() const;

TString GetHDDSystemChannelPoolKind() const;
TString GetHDDLogChannelPoolKind() const;
Expand Down Expand Up @@ -227,11 +227,8 @@ class TStorageConfig

bool GetInMemoryIndexCacheEnabled() const;
ui64 GetInMemoryIndexCacheNodesCapacity() const;
ui64 GetInMemoryIndexCacheNodesVerCapacity() const;
ui64 GetInMemoryIndexCacheNodeAttrsCapacity() const;
ui64 GetInMemoryIndexCacheNodeAttrsVerCapacity() const;
ui64 GetInMemoryIndexCacheNodeRefsCapacity() const;
ui64 GetInMemoryIndexCacheNodeRefsVerCapacity() const;

bool GetAsyncDestroyHandleEnabled() const;
TDuration GetAsyncHandleOperationPeriod() const;
Expand Down Expand Up @@ -260,6 +257,12 @@ class TStorageConfig

ui32 GetChannelFreeSpaceThreshold() const;
ui32 GetChannelMinFreeSpace() const;

ui32 GetMaxFileBlocks() const;
bool GetLargeDeletionMarkersEnabled() const;
ui64 GetLargeDeletionMarkerBlocks() const;
ui64 GetLargeDeletionMarkersThreshold() const;
ui64 GetLargeDeletionMarkersCleanupThreshold() const;
};

} // namespace NCloud::NFileStore::NStorage
4 changes: 2 additions & 2 deletions cloud/filestore/libs/storage/tablet/helpers.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,13 +150,13 @@ NProto::TError ValidateXAttrValue(const TString& name, const TString& value)
return {};
}

NProto::TError ValidateRange(TByteRange byteRange)
NProto::TError ValidateRange(TByteRange byteRange, ui32 maxFileBlocks)
{
if (!byteRange.Length) {
return ErrorInvalidArgument();
}

if (byteRange.LastBlock() + 1 > MaxFileBlocks) {
if (byteRange.LastBlock() + 1 > maxFileBlocks) {
return ErrorFileTooBig();
}

Expand Down
2 changes: 1 addition & 1 deletion cloud/filestore/libs/storage/tablet/helpers.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ NProto::ELockType ConvertToImpl(ELockMode source, TTag<NProto::ELockType>);
NProto::TError ValidateNodeName(const TString& name);
NProto::TError ValidateXAttrName(const TString& name);
NProto::TError ValidateXAttrValue(const TString& name, const TString& value);
NProto::TError ValidateRange(TByteRange byteRange);
NProto::TError ValidateRange(TByteRange byteRange, ui32 maxFileBlocks);

////////////////////////////////////////////////////////////////////////////////

Expand Down
2 changes: 2 additions & 0 deletions cloud/filestore/libs/storage/tablet/model/CMakeLists.darwin-x86_64.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,15 @@ target_sources(storage-tablet-model PRIVATE
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/fresh_bytes.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/garbage_queue.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/group_by.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/large_blocks.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/mixed_blocks.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/node_index_cache.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/node_session_stat.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/operation.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/range.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/range_locks.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/read_ahead.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/sparse_segment.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/split_range.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/throttler_logger.cpp
${CMAKE_SOURCE_DIR}/cloud/filestore/libs/storage/tablet/model/throttling_policy.cpp
Expand Down
Loading