use new ydb registration api in filestore + tests (#1661)

* use new ydb registration api in filestore + tests * remove blank lines * remove getting configs from cms * update * remove some includes * moved registration parameters to storage config * update * update * update * update * update --------- Co-authored-by: yegorskii <[email protected]>
ydb-platform · Aug 16, 2024 · 1998c2f · 1998c2f
1 parent ab5d62f
commit 1998c2f
Show file tree

Hide file tree

Showing 11 changed files with 196 additions and 5 deletions.
diff --git a/cloud/filestore/config/storage.proto b/cloud/filestore/config/storage.proto
@@ -3,8 +3,9 @@ syntax = "proto2";
 package NCloud.NFileStore.NProto;
 
 import "cloud/storage/core/protos/authorization_mode.proto";
+import "cloud/storage/core/protos/certificate.proto";
 
-option go_package = "a.yandex-team.ru/cloud/filestore/config";
+option go_package = "github.com/ydb-platform/nbs/cloud/filestore/config";
 
 ////////////////////////////////////////////////////////////////////////////////
 
@@ -308,7 +309,17 @@ message TStorageConfig
 
     // Max number of items to delete during TrimBytes.
     optional uint64 TrimBytesItemCount = 359;
-
+
+    // auth token for node registration via ydb discovery api.
+    optional string NodeRegistrationToken = 360;
+
+    // Node type.
+    optional string NodeType = 361;
+
+    // TLS node registration details.
+    optional string NodeRegistrationRootCertsFile = 362;
+    optional NCloud.NProto.TCertificate NodeRegistrationCert = 363;
+
     // Blob compression experiment params.
     optional uint32 BlobCompressionRate = 364;
     optional string BlobCompressionCodec = 365;
@@ -330,4 +341,4 @@ message TStorageConfig
         repeated TFilestoreAliasEntry Entries = 1;
     }
     optional TFilestoreAliases FilestoreAliases = 368;
-}
+}
diff --git a/cloud/filestore/libs/daemon/common/bootstrap.cpp b/cloud/filestore/libs/daemon/common/bootstrap.cpp
@@ -288,6 +288,8 @@ void TBootstrapCommon::InitActorSystem()
     registerOpts.RegistrationTimeout = Configs->Options->NodeRegistrationTimeout;
     registerOpts.ErrorTimeout = Configs->Options->NodeRegistrationErrorTimeout;
     registerOpts.LoadCmsConfigs = Configs->Options->LoadCmsConfigs;
+    registerOpts.UseNodeBrokerSsl = Configs->Options->UseNodeBrokerSsl,
+    registerOpts.Settings = Configs->GetNodeRegistrationSettings();
 
     auto [nodeId, scopeId, cmsConfig] = RegisterDynamicNode(
         Configs->KikimrConfig,

diff --git a/cloud/filestore/libs/daemon/common/config_initializer.cpp b/cloud/filestore/libs/daemon/common/config_initializer.cpp
@@ -8,6 +8,8 @@
 
 namespace NCloud::NFileStore::NDaemon {
 
+using namespace NCloud::NStorage;
+
 ////////////////////////////////////////////////////////////////////////////////
 
 TConfigInitializerCommon::TConfigInitializerCommon(TOptionsCommonPtr options)
@@ -56,4 +58,22 @@ void TConfigInitializerCommon::InitFeaturesConfig()
         std::move(featuresConfig));
 }
 
+TNodeRegistrationSettings
+    TConfigInitializerCommon::GetNodeRegistrationSettings()
+{
+    TNodeRegistrationSettings settings;
+    settings.MaxAttempts = Options->NodeRegistrationMaxAttempts;
+    settings.RegistrationTimeout = Options->NodeRegistrationTimeout;
+    settings.ErrorTimeout = Options->NodeRegistrationErrorTimeout;
+    settings.PathToGrpcCaFile = StorageConfig->GetNodeRegistrationRootCertsFile();
+    settings.NodeRegistrationToken = StorageConfig->GetNodeRegistrationToken();
+    settings.NodeType = StorageConfig->GetNodeType();
+
+    const auto& cert = StorageConfig->GetNodeRegistrationCert();
+    settings.PathToGrpcCertFile = cert.CertFile;
+    settings.PathToGrpcPrivateKeyFile = cert.CertPrivateKeyFile;
+
+    return settings;
+}
+
 }   // namespace NCloud::NFileStore::NDaemon
diff --git a/cloud/filestore/libs/daemon/common/config_initializer.h b/cloud/filestore/libs/daemon/common/config_initializer.h
@@ -7,6 +7,7 @@
 
 #include <cloud/storage/core/libs/features/features_config.h>
 #include <cloud/storage/core/libs/kikimr/config_initializer.h>
+#include <cloud/storage/core/libs/kikimr/node_registration_settings.h>
 
 namespace NCloud::NFileStore::NDaemon {
 
@@ -26,6 +27,8 @@ struct TConfigInitializerCommon
     void InitDiagnosticsConfig();
     void InitStorageConfig();
     void InitFeaturesConfig();
+
+    NCloud::NStorage::TNodeRegistrationSettings GetNodeRegistrationSettings();
 };
 
 }   // namespace NCloud::NFileStore::NDaemon
diff --git a/cloud/filestore/libs/storage/core/config.cpp b/cloud/filestore/libs/storage/core/config.cpp
@@ -1,9 +1,12 @@
 #include "config.h"
 
+#include <cloud/storage/core/protos/certificate.pb.h>
+
 #include <library/cpp/monlib/service/pages/templates.h>
 
 #include <util/generic/hash.h>
 #include <util/generic/size_literals.h>
+#include <util/generic/vector.h>
 
 #include <google/protobuf/text_format.h>
 
@@ -172,6 +175,10 @@ using TAliasMap = THashMap<TString, TString>;
     xxx(GetNodeAttrBatchEnabled,                        bool,      false      )\
     xxx(AllowFileStoreForceDestroy,                     bool,      false      )\
     xxx(TrimBytesItemCount,                             ui64,      100'000    )\
+    xxx(NodeRegistrationRootCertsFile,   TString,               {}            )\
+    xxx(NodeRegistrationCert,            TCertificate,          {}            )\
+    xxx(NodeRegistrationToken,           TString,               "root@builtin")\
+    xxx(NodeType,                        TString,               {}            )\
     xxx(BlobCompressionRate,             ui32,                  0             )\
     xxx(BlobCompressionCodec,            TString,               "lz4"         )\
                                                                                \

diff --git a/cloud/filestore/libs/storage/core/config.h b/cloud/filestore/libs/storage/core/config.h
@@ -11,6 +11,14 @@ namespace NCloud::NFileStore::NStorage {
 
 ////////////////////////////////////////////////////////////////////////////////
 
+struct TCertificate
+{
+    TString CertFile;
+    TString CertPrivateKeyFile;
+};
+
+////////////////////////////////////////////////////////////////////////////////
+
 class TStorageConfig
 {
 private:
@@ -222,6 +230,11 @@ class TStorageConfig
     void DumpHtml(IOutputStream& out) const;
     void DumpOverridesHtml(IOutputStream& out) const;
 
+    TString GetNodeRegistrationToken() const;
+    TString GetNodeType() const;
+    TString GetNodeRegistrationRootCertsFile() const;
+    TCertificate GetNodeRegistrationCert() const;
+
     ui32 GetBlobCompressionRate() const;
     TString GetBlobCompressionCodec() const;
 

diff --git a/cloud/filestore/tests/python/lib/client.py b/cloud/filestore/tests/python/lib/client.py
@@ -203,6 +203,13 @@ def execute_action(self, action, request):
         os.unlink(request_file.name)
         return res.stdout
 
+    def get_storage_service_config(self, fs_id=None):
+        req = {"FileSystemId": "" if fs_id is None else fs_id}
+
+        resp = self.execute_action("getstorageconfig", req)
+
+        return json.loads(resp)
+
     def __cmd_opts(self, vhost=False):
         opts = [
             "--server-address", "localhost",

diff --git a/cloud/filestore/tests/python/lib/daemon_config.py b/cloud/filestore/tests/python/lib/daemon_config.py
@@ -54,6 +54,7 @@ def __init__(
         restart_flag=None,
         access_service_port=0,
         storage_config=None,
+        use_secure_registration=False,
     ):
         self.__binary_path = binary_path
         self.__working_dir, self.__configs_dir = get_directories()
@@ -78,6 +79,8 @@ def __init__(
         self.__ic_port = self._port_manager.get_port()
         self.__access_service_port = access_service_port
 
+        self.__use_secure_registration = use_secure_registration
+
         if access_service_port:
             self.__app_config.ServerConfig.SecurePort = self._port_manager.get_port()
 
@@ -307,8 +310,6 @@ def generate_command(self):
                 self.__domain,
                 "--ic-port",
                 str(self.__ic_port),
-                "--node-broker",
-                "localhost:" + str(self.__kikimr_port),
                 "--diag-file",
                 self.__config_file_path("diag.txt"),
                 "--domains-file",
@@ -325,7 +326,13 @@ def generate_command(self):
                 self.__config_file_path("dyn_ns.txt"),
                 "--suppress-version-check",
                 "--load-configs-from-cms",
+                "--node-broker",
+                "localhost:{}".format(self.__kikimr_port),
             ]
+
+            if self.__use_secure_registration:
+                command += ["--use-secure-registration"]
+
             if self.__access_service_port:
                 command += [
                     "--auth-file",
@@ -364,6 +371,7 @@ def __init__(
         restart_interval=None,
         access_service_port=0,
         storage_config=None,
+        use_secure_registration=False,
     ):
         super().__init__(
             binary_path,
@@ -379,6 +387,7 @@ def __init__(
             restart_flag=None,
             access_service_port=access_service_port,
             storage_config=storage_config,
+            use_secure_registration=use_secure_registration,
         )
 
 
@@ -395,6 +404,7 @@ def __init__(
         restart_flag,
         access_service_port=0,
         storage_config=None,
+        use_secure_registration=False,
     ):
         super().__init__(
             binary_path,
@@ -410,6 +420,7 @@ def __init__(
             restart_flag=restart_flag,
             access_service_port=access_service_port,
             storage_config=storage_config,
+            use_secure_registration=use_secure_registration,
         )
 
         self.__local_service_port = self._port_manager.get_port()

diff --git a/cloud/filestore/tests/registration/test.py b/cloud/filestore/tests/registration/test.py
@@ -0,0 +1,87 @@
+from cloud.filestore.config.server_pb2 import TServerAppConfig, TKikimrServiceConfig
+from cloud.filestore.config.storage_pb2 import TStorageConfig
+from cloud.filestore.tests.python.lib.server import NfsServer, wait_for_nfs_server
+from cloud.filestore.tests.python.lib.daemon_config import NfsServerConfigGenerator
+
+from contrib.ydb.tests.library.harness.kikimr_cluster import kikimr_cluster_factory
+from contrib.ydb.tests.library.harness.kikimr_config import KikimrConfigGenerator
+
+import yatest.common as yatest_common
+
+
+def setup_and_run_test(is_secure_kikimr, is_secure_filestore):
+    kikimr_binary_path = yatest_common.binary_path("contrib/ydb/apps/ydbd/ydbd")
+
+    configurator = KikimrConfigGenerator(
+        erasure=None,
+        binary_path=kikimr_binary_path,
+        has_cluster_uuid=False,
+        use_in_memory_pdisks=True,
+        grpc_ssl_enable=is_secure_kikimr,
+        dynamic_storage_pools=[
+            dict(name="dynamic_storage_pool:1", kind="hdd", pdisk_user_kind=0),
+            dict(name="dynamic_storage_pool:2", kind="ssd", pdisk_user_kind=0)
+        ])
+
+    nfs_binary_path = yatest_common.binary_path("cloud/filestore/apps/server/filestore-server")
+
+    kikimr_cluster = kikimr_cluster_factory(configurator=configurator)
+    kikimr_cluster.start()
+
+    kikimr_port = list(kikimr_cluster.nodes.values())[0].port
+    kikimr_ssl_port = list(kikimr_cluster.nodes.values())[0].grpc_ssl_port
+
+    server_config = TServerAppConfig()
+    server_config.KikimrServiceConfig.CopyFrom(TKikimrServiceConfig())
+
+    storage_config = TStorageConfig()
+
+    if is_secure_filestore and is_secure_kikimr:
+        storage_config.NodeRegistrationRootCertsFile = configurator.grpc_tls_ca_path
+        storage_config.NodeRegistrationCert.CertFile = configurator.grpc_tls_cert_path
+        storage_config.NodeRegistrationCert.CertPrivateKeyFile = configurator.grpc_tls_key_path
+
+    storage_config.NodeType = "filestore_server"
+
+    domain = configurator.domains_txt.Domain[0].Name
+
+    port = kikimr_port
+    if is_secure_filestore and kikimr_ssl_port is not None:
+        port = kikimr_ssl_port
+
+    nfs_configurator = NfsServerConfigGenerator(
+        binary_path=nfs_binary_path,
+        app_config=server_config,
+        service_type="kikimr",
+        verbose=True,
+        kikimr_port=port,
+        domain=domain,
+        storage_config=storage_config,
+        use_secure_registration=is_secure_filestore
+    )
+    nfs_configurator.generate_configs(configurator.domains_txt, configurator.names_txt)
+
+    nfs_server = NfsServer(configurator=nfs_configurator)
+
+    nfs_server.start()
+
+    try:
+        wait_for_nfs_server(nfs_server, nfs_configurator.port)
+    except RuntimeError:
+        return False
+
+    nfs_server.stop()
+
+    return True
+
+
+def test_registration_non_secure():
+    assert setup_and_run_test(False, False)
+
+
+def test_registration_secure():
+    assert setup_and_run_test(True, True)
+
+
+def test_fail_registration_at_wrong_port():
+    assert not setup_and_run_test(False, True)
diff --git a/cloud/filestore/tests/registration/ya.make b/cloud/filestore/tests/registration/ya.make
@@ -0,0 +1,29 @@
+PY3TEST()
+
+INCLUDE(${ARCADIA_ROOT}/cloud/storage/core/tests/recipes/medium.inc)
+
+TEST_SRCS(test.py)
+
+PEERDIR(
+    cloud/filestore/config
+    cloud/filestore/tests/python/lib
+
+    library/python/testing/yatest_common
+
+    contrib/ydb/tests/library
+
+    contrib/python/requests/py3
+)
+
+DEPENDS(
+    cloud/filestore/apps/client
+    cloud/filestore/apps/server
+    contrib/ydb/apps/ydbd
+)
+
+DATA(
+    arcadia/cloud/filestore/tests/certs/server.crt
+    arcadia/cloud/filrstore/tests/certs/server.key
+)
+
+END()
diff --git a/cloud/filestore/tests/ya.make b/cloud/filestore/tests/ya.make
@@ -13,6 +13,7 @@ RECURSE_FOR_TESTS(
     fs_posix_compliance
     loadtest
     profile_log
+    registration
     service
     xfs_suite
 )