From ade87e8bc9ba8ac9f819c7ac020e3d78f16cb654 Mon Sep 17 00:00:00 2001 From: Bulat Gayazov Date: Thu, 22 Aug 2024 17:13:45 +0000 Subject: [PATCH] Supported SDK work in ydb repo --- cmake/protos_public_headers.txt | 3 +- examples/CMakeLists.txt | 1 + examples/basic_example/basic_example.cpp | 72 +- examples/bulk_upsert_simple/main.cpp | 4 +- examples/pagination/pagination.cpp | 3 +- examples/secondary_index/secondary_index.h | 2 + .../secondary_index_builtin/secondary_index.h | 2 + examples/ttl/ttl.cpp | 41 +- examples/vector_index/vector_index.h | 2 +- .../client/datastreams/datastreams.h | 2 +- .../ydb-cpp-sdk/client/discovery/discovery.h | 8 +- .../ydb-cpp-sdk/client/draft/ydb_scripting.h | 6 +- .../client/federated_topic/federated_topic.h | 2 +- include/ydb-cpp-sdk/client/params/params.h | 9 +- include/ydb-cpp-sdk/client/proto/accessor.h | 20 +- include/ydb-cpp-sdk/client/table/table.h | 2 +- .../ydb-cpp-sdk/client/topic/control_plane.h | 114 +- include/ydb-cpp-sdk/client/topic/errors.h | 2 +- include/ydb-cpp-sdk/client/topic/executor.h | 1 + .../ydb-cpp-sdk/client/topic/read_session.h | 4 +- .../ydb-cpp-sdk/client/topic/retry_policy.h | 2 +- .../ydb-cpp-sdk/client/types/status/status.h | 2 +- .../library/grpc/client/grpc_client_low.h | 6 +- .../library/grpc/client/grpc_common.h | 18 +- .../public => yql_common}/issue/yql_issue.h | 0 .../issue/yql_issue_id.h | 2 +- .../library/{yql => yql_common}/utils/utf8.h | 0 include/ydb-cpp-sdk/type_switcher.h | 13 + src/api/CMakeLists.txt | 1 - src/client/coordination/coordination.cpp | 114 +- src/client/datastreams/datastreams.cpp | 70 +- src/client/discovery/discovery.cpp | 28 +- src/client/draft/ydb_dynamic_config.cpp | 70 +- src/client/draft/ydb_replication.cpp | 8 +- src/client/draft/ydb_scripting.cpp | 22 +- src/client/driver/driver.cpp | 24 +- src/client/export/export.cpp | 32 +- .../impl/federated_topic_impl.h | 2 +- .../impl/federated_write_session.cpp | 14 +- .../impl/federated_write_session.h | 12 +- .../impl/federation_observer.cpp | 12 +- .../impl/federation_observer.h | 2 +- .../ut/CMakeLists.darwin-arm64.txt | 87 - .../ut/CMakeLists.darwin-x86_64.txt | 88 - .../ut/CMakeLists.linux-aarch64.txt | 91 - .../ut/CMakeLists.linux-x86_64.txt | 93 - src/client/federated_topic/ut/CMakeLists.txt | 19 - .../ut/CMakeLists.windows-x86_64.txt | 81 - .../federated_topic/ut/basic_usage_ut.cpp | 592 ----- src/client/federated_topic/ut/fds_mock.h | 112 - src/client/iam/common/iam.cpp | 4 +- src/client/iam_private/iam.cpp | 4 +- src/client/impl/ydb_internal/common/types.h | 2 +- .../db_driver_state/endpoint_pool.h | 2 +- .../ydb_internal/grpc_connections/actions.cpp | 4 +- .../ydb_internal/grpc_connections/actions.h | 2 +- .../grpc_connections/grpc_connections.cpp | 2 +- .../grpc_connections/grpc_connections.h | 6 +- .../ydb_internal/grpc_connections/params.h | 8 +- .../kqp_session_common/CMakeLists.txt | 1 + .../impl/ydb_internal/make_request/make.h | 2 +- .../impl/ydb_internal/plain_status/status.h | 4 +- .../impl/ydb_internal/table_helpers/helpers.h | 2 +- .../impl/ydb_internal/value_helpers/helpers.h | 2 +- src/client/import/import.cpp | 26 +- src/client/monitoring/monitoring.cpp | 4 +- src/client/operation/operation.cpp | 12 +- src/client/params/impl.cpp | 10 +- src/client/params/impl.h | 8 +- src/client/params/params.cpp | 16 +- src/client/persqueue_public/impl/common.h | 2 +- .../persqueue_public/impl/persqueue_impl.h | 26 +- .../persqueue_public/impl/read_session.cpp | 6 +- .../impl/write_session_impl.cpp | 21 +- .../persqueue_public/include/control_plane.h | 2 +- src/client/persqueue_public/persqueue.h | 3 + .../persqueue_public/ut/basic_usage_ut.cpp | 513 +++++ src/client/persqueue_public/ut/common_ut.cpp | 23 + .../ut/compress_executor_ut.cpp | 104 + .../persqueue_public/ut/compression_ut.cpp | 165 ++ .../persqueue_public/ut/read_session_ut.cpp | 1948 +++++++++++++++++ .../persqueue_public/ut/retry_policy_ut.cpp | 416 ++++ .../ut/ut_utils/data_plane_helpers.cpp | 102 + .../ut/ut_utils/data_plane_helpers.h | 49 + .../ut/ut_utils/sdk_test_setup.h | 249 +++ .../ut/ut_utils/test_server.cpp | 13 + .../ut/ut_utils/test_server.h | 171 ++ .../persqueue_public/ut/ut_utils/test_utils.h | 74 + .../persqueue_public/ut/ut_utils/ut_utils.cpp | 33 + .../persqueue_public/ut/ut_utils/ut_utils.h | 468 ++++ src/client/proto/CMakeLists.txt | 1 + src/client/proto/accessor.cpp | 4 +- src/client/query/client.cpp | 30 +- src/client/query/impl/client_session.cpp | 2 +- src/client/query/impl/client_session.h | 2 +- src/client/query/impl/exec_query.cpp | 10 +- src/client/query/query.cpp | 2 +- src/client/query/stats.cpp | 6 +- src/client/rate_limiter/rate_limiter.cpp | 22 +- src/client/result/result.cpp | 4 +- src/client/scheme/scheme.cpp | 22 +- src/client/ss_tasks/task.cpp | 6 +- src/client/table/impl/CMakeLists.txt | 1 + src/client/table/impl/client_session.h | 6 +- src/client/table/impl/data_query.cpp | 4 +- src/client/table/impl/data_query.h | 12 +- src/client/table/impl/readers.h | 2 +- src/client/table/impl/table_client.cpp | 78 +- src/client/table/impl/table_client.h | 18 +- src/client/table/table.cpp | 162 +- src/client/topic/impl/deferred_commit.cpp | 6 +- src/client/topic/impl/event_handlers.cpp | 2 +- src/client/topic/impl/read_session_event.cpp | 26 +- src/client/topic/impl/read_session_impl.h | 4 +- src/client/topic/impl/read_session_impl.ipp | 29 +- src/client/topic/impl/topic.cpp | 84 +- src/client/topic/impl/topic_impl.h | 60 +- src/client/topic/impl/write_session.cpp | 10 +- src/client/topic/impl/write_session.h | 10 +- src/client/topic/impl/write_session_impl.cpp | 57 +- src/client/topic/impl/write_session_impl.h | 44 +- .../topic/ut/CMakeLists.darwin-arm64.txt | 90 - .../topic/ut/CMakeLists.darwin-x86_64.txt | 91 - .../topic/ut/CMakeLists.linux-aarch64.txt | 94 - .../topic/ut/CMakeLists.linux-x86_64.txt | 96 - src/client/topic/ut/CMakeLists.txt | 19 - .../topic/ut/CMakeLists.windows-x86_64.txt | 84 - src/client/topic/ut/basic_usage_ut.cpp | 367 +++- src/client/topic/ut/describe_topic_ut.cpp | 91 +- src/client/topic/ut/local_partition_ut.cpp | 466 +++- src/client/topic/ut/topic_to_table_ut.cpp | 1729 ++++++++++++++- src/client/topic/ut/trace_ut.cpp | 168 ++ .../ut/ut_utils/CMakeLists.darwin-arm64.txt | 28 - .../ut/ut_utils/CMakeLists.darwin-x86_64.txt | 28 - .../ut/ut_utils/CMakeLists.linux-aarch64.txt | 29 - .../ut/ut_utils/CMakeLists.linux-x86_64.txt | 29 - src/client/topic/ut/ut_utils/CMakeLists.txt | 19 - .../ut/ut_utils/CMakeLists.windows-x86_64.txt | 28 - .../topic/ut/ut_utils/managed_executor.cpp | 32 +- .../topic/ut/ut_utils/managed_executor.h | 9 +- .../ut/ut_utils/topic_sdk_test_setup.cpp | 37 +- .../topic/ut/ut_utils/topic_sdk_test_setup.h | 31 +- src/client/topic/ut/ut_utils/trace.cpp | 146 ++ src/client/topic/ut/ut_utils/trace.h | 88 + src/client/types/credentials/login/login.cpp | 2 +- .../oauth2_token_exchange/credentials.cpp | 2 +- .../oauth2_token_exchange/from_file.cpp | 4 +- src/client/types/operation/operation.cpp | 5 +- src/client/value/value.cpp | 129 +- src/client/ymq/ymq.cpp | 14 +- src/library/CMakeLists.txt | 3 +- src/library/json_value/ydb_json_value.cpp | 4 +- src/library/operation_id/operation_id.cpp | 4 +- .../proto_output/CMakeLists.txt | 0 .../proto_output/proto_output.cpp | 4 +- src/library/retry/retry.h | 2 +- src/library/yql/CMakeLists.txt | 3 - .../{yql/public => yql_common}/CMakeLists.txt | 1 + .../decimal/CMakeLists.txt | 0 .../decimal/yql_decimal.cpp | 0 .../decimal/yql_decimal.h | 0 .../decimal/yql_decimal_serialize.cpp | 0 .../decimal/yql_decimal_serialize.h | 2 +- .../decimal/yql_wide_int.h | 0 .../issue/CMakeLists.txt | 1 + src/library/yql_common/issue/out.cpp | 22 + .../issue/protos/CMakeLists.txt | 0 .../issue/protos/issue_message.proto | 0 .../issue/protos/issue_severity.proto | 0 .../public => yql_common}/issue/yql_issue.cpp | 4 +- .../issue/yql_issue_id.cpp | 2 +- .../issue/yql_issue_message.cpp | 52 +- .../issue/yql_issue_message.h | 2 +- .../{yql => yql_common}/utils/CMakeLists.txt | 0 .../{yql => yql_common}/utils/utf8.cpp | 2 +- tests/unit/client/CMakeLists.txt | 33 +- .../{ => coordination}/coordination_ut.cpp | 8 +- .../discovery_mutator_ut.cpp | 2 +- tests/unit/client/{ => driver}/driver_ut.cpp | 7 +- .../client/{ => endpoints}/endpoints_ut.cpp | 2 +- tests/unit/client/{ => params}/params_ut.cpp | 0 tests/unit/client/{ => result}/result_ut.cpp | 13 +- .../ydb_scripting_response_headers_ut.cpp | 7 +- tests/unit/client/{ => value}/value_ut.cpp | 7 +- tests/unit/library/CMakeLists.txt | 2 +- .../{yql => yql_common}/CMakeLists.txt | 0 .../decimal}/yql_decimal_ut.cpp | 4 +- .../decimal}/yql_wide_int_ut.cpp | 2 +- .../issue}/yql_issue_ut.cpp | 14 +- .../{yql => yql_common/utils}/utf8_ut.cpp | 2 +- 190 files changed, 8358 insertions(+), 2960 deletions(-) rename include/ydb-cpp-sdk/library/{yql/public => yql_common}/issue/yql_issue.h (100%) rename include/ydb-cpp-sdk/library/{yql/public => yql_common}/issue/yql_issue_id.h (73%) rename include/ydb-cpp-sdk/library/{yql => yql_common}/utils/utf8.h (100%) create mode 100644 include/ydb-cpp-sdk/type_switcher.h delete mode 100644 src/client/federated_topic/ut/CMakeLists.darwin-arm64.txt delete mode 100644 src/client/federated_topic/ut/CMakeLists.darwin-x86_64.txt delete mode 100644 src/client/federated_topic/ut/CMakeLists.linux-aarch64.txt delete mode 100644 src/client/federated_topic/ut/CMakeLists.linux-x86_64.txt delete mode 100644 src/client/federated_topic/ut/CMakeLists.txt delete mode 100644 src/client/federated_topic/ut/CMakeLists.windows-x86_64.txt delete mode 100644 src/client/federated_topic/ut/basic_usage_ut.cpp delete mode 100644 src/client/federated_topic/ut/fds_mock.h create mode 100644 src/client/persqueue_public/persqueue.h create mode 100644 src/client/persqueue_public/ut/basic_usage_ut.cpp create mode 100644 src/client/persqueue_public/ut/common_ut.cpp create mode 100644 src/client/persqueue_public/ut/compress_executor_ut.cpp create mode 100644 src/client/persqueue_public/ut/compression_ut.cpp create mode 100644 src/client/persqueue_public/ut/read_session_ut.cpp create mode 100644 src/client/persqueue_public/ut/retry_policy_ut.cpp create mode 100644 src/client/persqueue_public/ut/ut_utils/data_plane_helpers.cpp create mode 100644 src/client/persqueue_public/ut/ut_utils/data_plane_helpers.h create mode 100644 src/client/persqueue_public/ut/ut_utils/sdk_test_setup.h create mode 100644 src/client/persqueue_public/ut/ut_utils/test_server.cpp create mode 100644 src/client/persqueue_public/ut/ut_utils/test_server.h create mode 100644 src/client/persqueue_public/ut/ut_utils/test_utils.h create mode 100644 src/client/persqueue_public/ut/ut_utils/ut_utils.cpp create mode 100644 src/client/persqueue_public/ut/ut_utils/ut_utils.h delete mode 100644 src/client/topic/ut/CMakeLists.darwin-arm64.txt delete mode 100644 src/client/topic/ut/CMakeLists.darwin-x86_64.txt delete mode 100644 src/client/topic/ut/CMakeLists.linux-aarch64.txt delete mode 100644 src/client/topic/ut/CMakeLists.linux-x86_64.txt delete mode 100644 src/client/topic/ut/CMakeLists.txt delete mode 100644 src/client/topic/ut/CMakeLists.windows-x86_64.txt create mode 100644 src/client/topic/ut/trace_ut.cpp delete mode 100644 src/client/topic/ut/ut_utils/CMakeLists.darwin-arm64.txt delete mode 100644 src/client/topic/ut/ut_utils/CMakeLists.darwin-x86_64.txt delete mode 100644 src/client/topic/ut/ut_utils/CMakeLists.linux-aarch64.txt delete mode 100644 src/client/topic/ut/ut_utils/CMakeLists.linux-x86_64.txt delete mode 100644 src/client/topic/ut/ut_utils/CMakeLists.txt delete mode 100644 src/client/topic/ut/ut_utils/CMakeLists.windows-x86_64.txt create mode 100644 src/client/topic/ut/ut_utils/trace.cpp create mode 100644 src/client/topic/ut/ut_utils/trace.h rename src/{api => library}/proto_output/CMakeLists.txt (100%) rename src/{api => library}/proto_output/proto_output.cpp (91%) delete mode 100644 src/library/yql/CMakeLists.txt rename src/library/{yql/public => yql_common}/CMakeLists.txt (67%) rename src/library/{yql/public => yql_common}/decimal/CMakeLists.txt (100%) rename src/library/{yql/public => yql_common}/decimal/yql_decimal.cpp (100%) rename src/library/{yql/public => yql_common}/decimal/yql_decimal.h (100%) rename src/library/{yql/public => yql_common}/decimal/yql_decimal_serialize.cpp (100%) rename src/library/{yql/public => yql_common}/decimal/yql_decimal_serialize.h (78%) rename src/library/{yql/public => yql_common}/decimal/yql_wide_int.h (100%) rename src/library/{yql/public => yql_common}/issue/CMakeLists.txt (97%) create mode 100644 src/library/yql_common/issue/out.cpp rename src/library/{yql/public => yql_common}/issue/protos/CMakeLists.txt (100%) rename src/library/{yql/public => yql_common}/issue/protos/issue_message.proto (100%) rename src/library/{yql/public => yql_common}/issue/protos/issue_severity.proto (100%) rename src/library/{yql/public => yql_common}/issue/yql_issue.cpp (98%) rename src/library/{yql/public => yql_common}/issue/yql_issue_id.cpp (82%) rename src/library/{yql/public => yql_common}/issue/yql_issue_message.cpp (79%) rename src/library/{yql/public => yql_common}/issue/yql_issue_message.h (94%) rename src/library/{yql => yql_common}/utils/CMakeLists.txt (100%) rename src/library/{yql => yql_common}/utils/utf8.cpp (99%) rename tests/unit/client/{ => coordination}/coordination_ut.cpp (97%) rename tests/unit/client/{ => discovery_mutator}/discovery_mutator_ut.cpp (96%) rename tests/unit/client/{ => driver}/driver_ut.cpp (96%) rename tests/unit/client/{ => endpoints}/endpoints_ut.cpp (99%) rename tests/unit/client/{ => params}/params_ut.cpp (100%) rename tests/unit/client/{ => result}/result_ut.cpp (93%) rename tests/unit/client/{ => scripting}/ydb_scripting_response_headers_ut.cpp (88%) rename tests/unit/client/{ => value}/value_ut.cpp (99%) rename tests/unit/library/{yql => yql_common}/CMakeLists.txt (100%) rename tests/unit/library/{yql => yql_common/decimal}/yql_decimal_ut.cpp (99%) rename tests/unit/library/{yql => yql_common/decimal}/yql_wide_int_ut.cpp (99%) rename tests/unit/library/{yql => yql_common/issue}/yql_issue_ut.cpp (96%) rename tests/unit/library/{yql => yql_common/utils}/utf8_ut.cpp (98%) diff --git a/cmake/protos_public_headers.txt b/cmake/protos_public_headers.txt index a26d90b2969..5920de5ea17 100644 --- a/cmake/protos_public_headers.txt +++ b/cmake/protos_public_headers.txt @@ -20,4 +20,5 @@ src/api/protos/ydb_export.pb.h src/api/protos/ydb_coordination.pb.h src/api/protos/ydb_status_codes.pb.h src/api/protos/draft/ydb_replication.pb.h -src/library/yql/public/issue/protos/issue_severity.pb.h \ No newline at end of file +src/library/operation_id/protos/operation_id.pb.h +src/library/yql_common/issue/protos/issue_severity.pb.h \ No newline at end of file diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index b143e6eb9a5..97119cff5e8 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -5,3 +5,4 @@ add_subdirectory(secondary_index) add_subdirectory(secondary_index_builtin) add_subdirectory(topic_reader) add_subdirectory(ttl) +add_subdirectory(vector_index) diff --git a/examples/basic_example/basic_example.cpp b/examples/basic_example/basic_example.cpp index 38961951d9b..5c506f39ad0 100644 --- a/examples/basic_example/basic_example.cpp +++ b/examples/basic_example/basic_example.cpp @@ -8,6 +8,8 @@ using namespace NYdb; using namespace NYdb::NTable; +namespace { + class TYdbErrorException : public yexception { public: TYdbErrorException(const TStatus& status) @@ -16,18 +18,34 @@ class TYdbErrorException : public yexception { TStatus Status; }; -static void ThrowOnError(const TStatus& status) { +void ThrowOnError(const TStatus& status) { if (!status.IsSuccess()) { throw TYdbErrorException(status) << status; } } -static void PrintStatus(const TStatus& status) { +void PrintStatus(const TStatus& status) { std::cerr << "Status: " << ToString(status.GetStatus()) << std::endl; std::cerr << status.GetIssues().ToString(); } -static std::string JoinPath(const std::string& basePath, const std::string& path) { +template +std::string OptionalToString(const std::optional& opt) { + if (opt.has_value()) { + return std::to_string(opt.value()); + } + return "(NULL)"; +} + +template <> +std::string OptionalToString(const std::optional& opt) { + if (opt.has_value()) { + return opt.value(); + } + return "(NULL)"; +} + +std::string JoinPath(const std::string& basePath, const std::string& path) { if (basePath.empty()) { return path; } @@ -41,7 +59,7 @@ static std::string JoinPath(const std::string& basePath, const std::string& path /////////////////////////////////////////////////////////////////////////////// //! Creates sample tables with CrateTable API. -static void CreateTables(TTableClient client, const std::string& path) { +void CreateTables(TTableClient client, const std::string& path) { ThrowOnError(client.RetryOperationSync([path](TSession session) { auto seriesDesc = TTableBuilder() .AddNullableColumn("series_id", EPrimitiveType::Uint64) @@ -83,7 +101,7 @@ static void CreateTables(TTableClient client, const std::string& path) { } //! Describe existing table. -static void DescribeTable(TTableClient client, const std::string& path, const std::string& name) { +void DescribeTable(TTableClient client, const std::string& path, const std::string& name) { std::optional desc; ThrowOnError(client.RetryOperationSync([path, name, &desc](TSession session) { @@ -105,7 +123,7 @@ static void DescribeTable(TTableClient client, const std::string& path, const st /////////////////////////////////////////////////////////////////////////////// //! Fills sample tables with data in single parameterized data query. -static TStatus FillTableDataTransaction(TSession session, const std::string& path) { +TStatus FillTableDataTransaction(TSession session, const std::string& path) { auto query = std::format(R"( PRAGMA TablePathPrefix("{}"); @@ -165,7 +183,7 @@ static TStatus FillTableDataTransaction(TSession session, const std::string& pat } //! Shows basic usage of YDB data queries and transactions. -static TStatus SelectSimpleTransaction(TSession session, const std::string& path, +TStatus SelectSimpleTransaction(TSession session, const std::string& path, std::optional& resultSet) { auto query = std::format(R"( @@ -194,7 +212,7 @@ static TStatus SelectSimpleTransaction(TSession session, const std::string& path } //! Shows basic usage of mutating operations. -static TStatus UpsertSimpleTransaction(TSession session, const std::string& path) { +TStatus UpsertSimpleTransaction(TSession session, const std::string& path) { auto query = std::format(R"( --!syntax_v1 PRAGMA TablePathPrefix("{}"); @@ -208,7 +226,7 @@ static TStatus UpsertSimpleTransaction(TSession session, const std::string& path } //! Shows usage of parameters in data queries. -static TStatus SelectWithParamsTransaction(TSession session, const std::string& path, +TStatus SelectWithParamsTransaction(TSession session, const std::string& path, uint64_t seriesId, uint64_t seasonId, std::optional& resultSet) { auto query = std::format(R"( @@ -248,7 +266,7 @@ static TStatus SelectWithParamsTransaction(TSession session, const std::string& } //! Shows usage of prepared queries. -static TStatus PreparedSelectTransaction(TSession session, const std::string& path, +TStatus PreparedSelectTransaction(TSession session, const std::string& path, uint64_t seriesId, uint64_t seasonId, uint64_t episodeId, std::optional& resultSet) { // Once prepared, query data is stored in the session and identified by QueryId. @@ -301,7 +319,7 @@ static TStatus PreparedSelectTransaction(TSession session, const std::string& pa } //! Shows usage of transactions consisting of multiple data queries with client logic between them. -static TStatus MultiStepTransaction(TSession session, const std::string& path, uint64_t seriesId, uint64_t seasonId, +TStatus MultiStepTransaction(TSession session, const std::string& path, uint64_t seriesId, uint64_t seasonId, std::optional& resultSet) { auto query1 = std::format(R"( @@ -394,7 +412,7 @@ static TStatus MultiStepTransaction(TSession session, const std::string& path, u // Show usage of explicit Begin/Commit transaction control calls. // In most cases it's better to use transaction control settings in ExecuteDataQuery calls instead // to avoid additional hops to YDB cluster and allow more efficient execution of queries. -static TStatus ExplicitTclTransaction(TSession session, const std::string& path, const TInstant& airDate) { +TStatus ExplicitTclTransaction(TSession session, const std::string& path, const TInstant& airDate) { auto beginResult = session.BeginTransaction(TTxSettings::SerializableRW()).GetValueSync(); if (!beginResult.IsSuccess()) { return beginResult; @@ -432,6 +450,8 @@ static TStatus ExplicitTclTransaction(TSession session, const std::string& path, return tx.Commit().GetValueSync(); } +} + /////////////////////////////////////////////////////////////////////////////// void SelectSimple(TTableClient client, const std::string& path) { @@ -443,9 +463,9 @@ void SelectSimple(TTableClient client, const std::string& path) { TResultSetParser parser(*resultSet); if (parser.TryNextRow()) { std::cout << "> SelectSimple:" << std::endl << "Series" - << ", Id: " << ToString(parser.ColumnParser("series_id").GetOptionalUint64()) - << ", Title: " << ToString(parser.ColumnParser("title").GetOptionalUtf8()) - << ", Release date: " << ToString(parser.ColumnParser("release_date").GetOptionalString()) + << ", Id: " << OptionalToString(parser.ColumnParser("series_id").GetOptionalUint64()) + << ", Title: " << OptionalToString(parser.ColumnParser("title").GetOptionalUtf8()) + << ", Release date: " << OptionalToString(parser.ColumnParser("release_date").GetOptionalString()) << std::endl; } } @@ -465,8 +485,8 @@ void SelectWithParams(TTableClient client, const std::string& path) { TResultSetParser parser(*resultSet); if (parser.TryNextRow()) { std::cout << "> SelectWithParams:" << std::endl << "Season" - << ", Title: " << ToString(parser.ColumnParser("season_title").GetOptionalUtf8()) - << ", Series title: " << ToString(parser.ColumnParser("series_title").GetOptionalUtf8()) + << ", Title: " << OptionalToString(parser.ColumnParser("season_title").GetOptionalUtf8()) + << ", Series title: " << OptionalToString(parser.ColumnParser("series_title").GetOptionalUtf8()) << std::endl; } } @@ -481,8 +501,8 @@ void PreparedSelect(TTableClient client, const std::string& path, uint32_t serie if (parser.TryNextRow()) { auto airDate = TInstant::Days(*parser.ColumnParser("air_date").GetOptionalUint64()); - std::cout << "> PreparedSelect:" << std::endl << "Episode " << ToString(parser.ColumnParser("episode_id").GetOptionalUint64()) - << ", Title: " << ToString(parser.ColumnParser("title").GetOptionalUtf8()) + std::cout << "> PreparedSelect:" << std::endl << "Episode " << OptionalToString(parser.ColumnParser("episode_id").GetOptionalUint64()) + << ", Title: " << OptionalToString(parser.ColumnParser("title").GetOptionalUtf8()) << ", Air date: " << airDate.FormatLocalTime("%a %b %d, %Y") << std::endl; } @@ -499,9 +519,9 @@ void MultiStep(TTableClient client, const std::string& path) { while (parser.TryNextRow()) { auto airDate = TInstant::Days(*parser.ColumnParser("air_date").GetOptionalUint64()); - std::cout << "Episode " << ToString(parser.ColumnParser("episode_id").GetOptionalUint64()) - << ", Season: " << ToString(parser.ColumnParser("season_id").GetOptionalUint64()) - << ", Title: " << ToString(parser.ColumnParser("title").GetOptionalUtf8()) + std::cout << "Episode " << OptionalToString(parser.ColumnParser("episode_id").GetOptionalUint64()) + << ", Season: " << OptionalToString(parser.ColumnParser("season_id").GetOptionalUint64()) + << ", Title: " << OptionalToString(parser.ColumnParser("title").GetOptionalUtf8()) << ", Air date: " << airDate.FormatLocalTime("%a %b %d, %Y") << std::endl; } @@ -561,10 +581,10 @@ void ScanQuerySelect(TTableClient client, const std::string& path) { TResultSetParser parser(rs); while (parser.TryNextRow()) { std::cout << "Season" - << ", SeriesId: " << ToString(parser.ColumnParser("series_id").GetOptionalUint64()) - << ", SeasonId: " << ToString(parser.ColumnParser("season_id").GetOptionalUint64()) - << ", Title: " << ToString(parser.ColumnParser("title").GetOptionalUtf8()) - << ", Air date: " << ToString(parser.ColumnParser("first_aired").GetOptionalString()) + << ", SeriesId: " << OptionalToString(parser.ColumnParser("series_id").GetOptionalUint64()) + << ", SeasonId: " << OptionalToString(parser.ColumnParser("season_id").GetOptionalUint64()) + << ", Title: " << OptionalToString(parser.ColumnParser("title").GetOptionalUtf8()) + << ", Air date: " << OptionalToString(parser.ColumnParser("first_aired").GetOptionalString()) << std::endl; } } diff --git a/examples/bulk_upsert_simple/main.cpp b/examples/bulk_upsert_simple/main.cpp index 7b194f7f856..f30d1627950 100644 --- a/examples/bulk_upsert_simple/main.cpp +++ b/examples/bulk_upsert_simple/main.cpp @@ -18,8 +18,8 @@ void GetLogBatch(uint64_t logOffset, std::vector& logBatch) { logBatch.clear(); for (size_t i = 0; i < BATCH_SIZE; ++i) { TLogMessage message; - message.App = "App_" + ToString(logOffset % 10); - message.Host = "192.168.0." + ToString(logOffset % 11); + message.App = "App_" + std::to_string(logOffset % 10); + message.Host = "192.168.0." + std::to_string(logOffset % 11); message.Timestamp = TInstant::Now() + TDuration::MilliSeconds(i % 1000); message.HttpCode = 200; message.Message = i % 2 ? "GET / HTTP/1.1" : "GET /images/logo.png HTTP/1.1"; diff --git a/examples/pagination/pagination.cpp b/examples/pagination/pagination.cpp index 63a38285aaa..c6080b9df7f 100644 --- a/examples/pagination/pagination.cpp +++ b/examples/pagination/pagination.cpp @@ -3,6 +3,7 @@ #include #include +#include using namespace NYdb; using namespace NYdb::NTable; @@ -152,7 +153,7 @@ bool SelectPaging(TTableClient client, const std::string& path, uint64_t pageLim do { lastCity = parser.ColumnParser("city").GetOptionalUtf8().value(); lastNumber = parser.ColumnParser("number").GetOptionalUint32().value(); - std::cout << lastCity << ", Школа №" << lastNumber << ", Адрес: " << ToString(parser.ColumnParser("address").GetOptionalUtf8()) << std::endl; + std::cout << lastCity << ", Школа №" << lastNumber << ", Адрес: " << parser.ColumnParser("address").GetOptionalUtf8().value_or("(NULL)") << std::endl; } while (parser.TryNextRow()); return true; } diff --git a/examples/secondary_index/secondary_index.h b/examples/secondary_index/secondary_index.h index b2b1caa1e1e..755ba617fb1 100644 --- a/examples/secondary_index/secondary_index.h +++ b/examples/secondary_index/secondary_index.h @@ -5,6 +5,8 @@ #include +#include + //////////////////////////////////////////////////////////////////////////////// #define TABLE_SERIES "series" diff --git a/examples/secondary_index_builtin/secondary_index.h b/examples/secondary_index_builtin/secondary_index.h index 849b8e7f5e3..f32bf8799de 100644 --- a/examples/secondary_index_builtin/secondary_index.h +++ b/examples/secondary_index_builtin/secondary_index.h @@ -8,6 +8,8 @@ #include +#include + #define TABLE_USERS "users" #define TABLE_SERIES "series" diff --git a/examples/ttl/ttl.cpp b/examples/ttl/ttl.cpp index 6ce6518478f..bc6086ab5db 100644 --- a/examples/ttl/ttl.cpp +++ b/examples/ttl/ttl.cpp @@ -1,6 +1,8 @@ #include "ttl.h" #include "util.h" +#include + using namespace NExample; using namespace NYdb; using namespace NYdb::NTable; @@ -8,8 +10,26 @@ using namespace NYdb::NTable; constexpr uint32_t DOC_TABLE_PARTITION_COUNT = 4; constexpr uint32_t EXPIRATION_QUEUE_COUNT = 4; +namespace { + +template +std::string OptionalToString(const std::optional& opt) { + if (opt.has_value()) { + return std::to_string(opt.value()); + } + return "(NULL)"; +} + +template <> +std::string OptionalToString(const std::optional& opt) { + if (opt.has_value()) { + return opt.value(); + } + return "(NULL)"; +} + //! Creates Documents table and multiple ExpirationQueue tables -static void CreateTables(TTableClient client, const std::string& path) { +void CreateTables(TTableClient client, const std::string& path) { // Documents table stores the contents of web pages. // The table is partitioned by hash(Url) in order to evenly distribute the load. ThrowOnError(client.RetryOperationSync([path](TSession session) { @@ -48,7 +68,7 @@ static void CreateTables(TTableClient client, const std::string& path) { /////////////////////////////////////////////////////////////////////////////// //! Insert or replaces a document. -static TStatus AddDocumentTransaction(TSession session, const std::string& path, +TStatus AddDocumentTransaction(TSession session, const std::string& path, const std::string& url, const std::string& html, uint64_t timestamp) { // Add an entry to a random expiration queue in order to evenly distribute the load @@ -88,7 +108,7 @@ static TStatus AddDocumentTransaction(TSession session, const std::string& path, } //! Reads document contents. -static TStatus ReadDocumentTransaction(TSession session, const std::string& path, +TStatus ReadDocumentTransaction(TSession session, const std::string& path, const std::string& url, std::optional& resultSet) { auto query = std::format(R"( @@ -121,7 +141,7 @@ static TStatus ReadDocumentTransaction(TSession session, const std::string& path } //! Reads a batch of entries from expiration queue -static TStatus ReadExpiredBatchTransaction(TSession session, const std::string& path, const uint32_t queue, +TStatus ReadExpiredBatchTransaction(TSession session, const std::string& path, const uint32_t queue, const uint64_t timestamp, const uint64_t prevTimestamp, const uint64_t prevDocId, std::optional& resultSet) { auto query = std::format(R"( @@ -177,7 +197,7 @@ static TStatus ReadExpiredBatchTransaction(TSession session, const std::string& } //! Deletes an expired document -static TStatus DeleteDocumentWithTimestamp(TSession session, const std::string& path, const uint32_t queue, +TStatus DeleteDocumentWithTimestamp(TSession session, const std::string& path, const uint32_t queue, const uint64_t docId, const uint64_t timestamp) { auto query = std::format(R"( @@ -206,6 +226,9 @@ static TStatus DeleteDocumentWithTimestamp(TSession session, const std::string& return result; } + +} + /////////////////////////////////////////////////////////////////////////////// void AddDocument(TTableClient client, const std::string& path, const std::string& url, @@ -230,10 +253,10 @@ void ReadDocument(TTableClient client, const std::string& path, const std::strin TResultSetParser parser(*resultSet); if (parser.TryNextRow()) { - std::cout << " DocId: " << ToString(parser.ColumnParser("doc_id").GetOptionalUint64()) << std::endl - << " Url: " << ToString(parser.ColumnParser("url").GetOptionalUtf8()) << std::endl - << " Timestamp: " << ToString(parser.ColumnParser("timestamp").GetOptionalUint64()) << std::endl - << " Html: " << ToString(parser.ColumnParser("html").GetOptionalUtf8()) << std::endl; + std::cout << " DocId: " << OptionalToString(parser.ColumnParser("doc_id").GetOptionalUint64()) << std::endl + << " Url: " << OptionalToString(parser.ColumnParser("url").GetOptionalUtf8()) << std::endl + << " Timestamp: " << OptionalToString(parser.ColumnParser("timestamp").GetOptionalUint64()) << std::endl + << " Html: " << OptionalToString(parser.ColumnParser("html").GetOptionalUtf8()) << std::endl; } else { std::cout << " Not found" << std::endl; } diff --git a/examples/vector_index/vector_index.h b/examples/vector_index/vector_index.h index a253e265bc1..532de0fd3b4 100644 --- a/examples/vector_index/vector_index.h +++ b/examples/vector_index/vector_index.h @@ -3,7 +3,7 @@ #include #include -#include +#include enum class ECommand { DropIndex, diff --git a/include/ydb-cpp-sdk/client/datastreams/datastreams.h b/include/ydb-cpp-sdk/client/datastreams/datastreams.h index 15ead26b0c1..65cbb56e3dc 100644 --- a/include/ydb-cpp-sdk/client/datastreams/datastreams.h +++ b/include/ydb-cpp-sdk/client/datastreams/datastreams.h @@ -2,7 +2,7 @@ #include -#include +#include namespace NYdb::NDataStreams::V1 { diff --git a/include/ydb-cpp-sdk/client/discovery/discovery.h b/include/ydb-cpp-sdk/client/discovery/discovery.h index e9563483eef..dd8c86427b8 100644 --- a/include/ydb-cpp-sdk/client/discovery/discovery.h +++ b/include/ydb-cpp-sdk/client/discovery/discovery.h @@ -102,12 +102,12 @@ class TNodeRegistrationResult : public TStatus { public: TNodeRegistrationResult() : TStatus(EStatus::GENERIC_ERROR, NYql::TIssues()) {} TNodeRegistrationResult(TStatus&& status, const Ydb::Discovery::NodeRegistrationResult& proto); - const uint32_t& GetNodeId() const; + uint32_t GetNodeId() const; const std::string& GetDomainPath() const; - const uint64_t& GetExpire() const; - const uint64_t& GetScopeTabletId() const; + uint64_t GetExpire() const; + uint64_t GetScopeTabletId() const; bool HasScopeTabletId() const; - const uint64_t& GetScopePathId() const; + uint64_t GetScopePathId() const; bool HasScopePathId() const; const std::string& GetNodeName() const; bool HasNodeName() const; diff --git a/include/ydb-cpp-sdk/client/draft/ydb_scripting.h b/include/ydb-cpp-sdk/client/draft/ydb_scripting.h index 92fe47cb1b6..58cc92d1b51 100644 --- a/include/ydb-cpp-sdk/client/draft/ydb_scripting.h +++ b/include/ydb-cpp-sdk/client/draft/ydb_scripting.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include namespace NYdb { namespace NScripting { @@ -84,13 +84,13 @@ class TYqlResultPartIterator : public TStatus { class TExplainYqlResult : public TStatus { public: - TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map&& types, std::string&& plan); + TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map&& types, std::string&& plan); std::map GetParameterTypes() const; const std::string& GetPlan() const; private: - ::google::protobuf::Map ParameterTypes_; + ::google::protobuf::Map ParameterTypes_; std::string Plan_; }; diff --git a/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h b/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h index f952fd6c541..30b98b0c806 100644 --- a/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h +++ b/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h @@ -2,7 +2,7 @@ #include -#include +#include #include diff --git a/include/ydb-cpp-sdk/client/params/params.h b/include/ydb-cpp-sdk/client/params/params.h index 6e9969c41e3..d7f72bcc4be 100644 --- a/include/ydb-cpp-sdk/client/params/params.h +++ b/include/ydb-cpp-sdk/client/params/params.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include @@ -50,10 +51,10 @@ class TParams { std::optional GetValue(const std::string& name) const; private: - TParams(::google::protobuf::Map&& protoMap); + TParams(::google::protobuf::Map&& protoMap); - ::google::protobuf::Map* GetProtoMapPtr(); - const ::google::protobuf::Map& GetProtoMap() const; + ::google::protobuf::Map* GetProtoMapPtr(); + const ::google::protobuf::Map& GetProtoMap() const; class TImpl; std::shared_ptr Impl_; @@ -89,7 +90,7 @@ class TParamsBuilder : public TMoveOnly { TParams Build(); private: - TParamsBuilder(const ::google::protobuf::Map& typeInfo); + TParamsBuilder(const ::google::protobuf::Map& typeInfo); class TImpl; std::unique_ptr Impl_; diff --git a/include/ydb-cpp-sdk/client/proto/accessor.h b/include/ydb-cpp-sdk/client/proto/accessor.h index 390f7f502cd..f476b91de80 100644 --- a/include/ydb-cpp-sdk/client/proto/accessor.h +++ b/include/ydb-cpp-sdk/client/proto/accessor.h @@ -1,13 +1,13 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -37,8 +37,8 @@ class TProtoAccessor { static const Ydb::Type& GetProto(const TType& type); static const Ydb::Value& GetProto(const TValue& value); static const Ydb::ResultSet& GetProto(const TResultSet& resultSet); - static const ::google::protobuf::Map& GetProtoMap(const TParams& params); - static ::google::protobuf::Map* GetProtoMapPtr(TParams& params); + static const ::google::protobuf::Map& GetProtoMap(const TParams& params); + static ::google::protobuf::Map* GetProtoMapPtr(TParams& params); static const Ydb::TableStats::QueryStats& GetProto(const NTable::TQueryStats& queryStats); static const Ydb::Table::DescribeTableResult& GetProto(const NTable::TTableDescription& tableDescription); static const Ydb::Topic::DescribeTopicResult& GetProto(const NYdb::NTopic::TTopicDescription& topicDescription); diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index fe5be360caf..245b9f1742c 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -1762,7 +1762,7 @@ class TDataQuery { private: TDataQuery(const TSession& session, const std::string& text, const std::string& id); TDataQuery(const TSession& session, const std::string& text, const std::string& id, - const ::google::protobuf::Map& types); + const ::google::protobuf::Map& types); class TImpl; std::shared_ptr Impl_; diff --git a/include/ydb-cpp-sdk/client/topic/control_plane.h b/include/ydb-cpp-sdk/client/topic/control_plane.h index 8f1e37ed16b..5d1f8feddae 100644 --- a/include/ydb-cpp-sdk/client/topic/control_plane.h +++ b/include/ydb-cpp-sdk/client/topic/control_plane.h @@ -4,7 +4,7 @@ #include -#include +#include #include @@ -28,7 +28,7 @@ enum class EMeteringMode : uint32_t { Unknown = std::numeric_limits::max(), }; -enum class EAutoscalingStrategy: uint32_t { +enum class EAutoPartitioningStrategy: uint32_t { Unspecified = 0, Disabled = 1, ScaleUp = 2, @@ -157,44 +157,44 @@ class TPartitionInfo { struct TAlterPartitioningSettings; struct TAlterTopicSettings; -struct TAutoscalingSettings { -friend struct TAutoscalingSettingsBuilder; +struct TAutoPartitioningSettings { +friend struct TAutoPartitioningSettingsBuilder; public: - TAutoscalingSettings() - : Strategy_(EAutoscalingStrategy::Disabled) - , ThresholdTime_(TDuration::Seconds(0)) - , ScaleDownThresholdPercent_(0) - , ScaleUpThresholdPercent_(0) { - } - TAutoscalingSettings(const Ydb::Topic::AutoscalingSettings& settings); - TAutoscalingSettings(EAutoscalingStrategy strategy, TDuration thresholdTime, uint64_t scaleUpThresholdPercent, uint64_t scaleDownThresholdPercent) +TAutoPartitioningSettings() + : Strategy_(EAutoPartitioningStrategy::Disabled) + , StabilizationWindow_(TDuration::Seconds(0)) + , DownUtilizationPercent_(0) + , UpUtilizationPercent_(0) { + } + TAutoPartitioningSettings(const Ydb::Topic::AutoPartitioningSettings& settings); + TAutoPartitioningSettings(EAutoPartitioningStrategy strategy, TDuration stabilizationWindow, ui64 downUtilizationPercent, ui64 upUtilizationPercent) : Strategy_(strategy) - , ThresholdTime_(thresholdTime) - , ScaleDownThresholdPercent_(scaleDownThresholdPercent) - , ScaleUpThresholdPercent_(scaleUpThresholdPercent) {} - - EAutoscalingStrategy GetStrategy() const; - TDuration GetThresholdTime() const; - uint32_t GetScaleDownThresholdPercent() const; - uint32_t GetScaleUpThresholdPercent() const; + , StabilizationWindow_(stabilizationWindow) + , DownUtilizationPercent_(downUtilizationPercent) + , UpUtilizationPercent_(upUtilizationPercent) {} + + EAutoPartitioningStrategy GetStrategy() const; + TDuration GetStabilizationWindow() const; + ui32 GetDownUtilizationPercent() const; + ui32 GetUpUtilizationPercent() const; private: - EAutoscalingStrategy Strategy_; - TDuration ThresholdTime_; - uint32_t ScaleDownThresholdPercent_; - uint32_t ScaleUpThresholdPercent_; + EAutoPartitioningStrategy Strategy_; + TDuration StabilizationWindow_; + ui32 DownUtilizationPercent_; + ui32 UpUtilizationPercent_; }; -struct TAlterAutoscalingSettings { - using TSelf = TAlterAutoscalingSettings; +struct TAlterAutoPartitioningSettings { + using TSelf = TAlterAutoPartitioningSettings; public: - TAlterAutoscalingSettings(TAlterPartitioningSettings& parent): Parent_(parent) {} + TAlterAutoPartitioningSettings(TAlterPartitioningSettings& parent): Parent_(parent) {} - FLUENT_SETTING_OPTIONAL(EAutoscalingStrategy, Strategy); - FLUENT_SETTING_OPTIONAL(TDuration, ThresholdTime); - FLUENT_SETTING_OPTIONAL(uint64_t, ScaleUpThresholdPercent); - FLUENT_SETTING_OPTIONAL(uint64_t, ScaleDownThresholdPercent); + FLUENT_SETTING_OPTIONAL(EAutoPartitioningStrategy, Strategy); + FLUENT_SETTING_OPTIONAL(TDuration, StabilizationWindow); + FLUENT_SETTING_OPTIONAL(ui64, DownUtilizationPercent); + FLUENT_SETTING_OPTIONAL(ui64, UpUtilizationPercent); - TAlterPartitioningSettings& EndAlterAutoscalingSettings() { return Parent_; }; + TAlterPartitioningSettings& EndAlterAutoPartitioningSettings() { return Parent_; }; private: TAlterPartitioningSettings& Parent_; @@ -204,25 +204,25 @@ class TPartitioningSettings { using TSelf = TPartitioningSettings; friend struct TPartitioningSettingsBuilder; public: - TPartitioningSettings() : MinActivePartitions_(0), MaxActivePartitions_(0), PartitionCountLimit_(0), AutoscalingSettings_(){} + TPartitioningSettings() : MinActivePartitions_(0), MaxActivePartitions_(0), PartitionCountLimit_(0), AutoPartitioningSettings_(){} TPartitioningSettings(const Ydb::Topic::PartitioningSettings& settings); - TPartitioningSettings(uint64_t minActivePartitions, uint64_t maxActivePartitions, TAutoscalingSettings autoscalingSettings = {}) + TPartitioningSettings(ui64 minActivePartitions, ui64 maxActivePartitions, TAutoPartitioningSettings autoscalingSettings = {}) : MinActivePartitions_(minActivePartitions) , MaxActivePartitions_(maxActivePartitions) , PartitionCountLimit_(0) - , AutoscalingSettings_(autoscalingSettings) + , AutoPartitioningSettings_(autoscalingSettings) { } uint64_t GetMinActivePartitions() const; uint64_t GetMaxActivePartitions() const; uint64_t GetPartitionCountLimit() const; - TAutoscalingSettings GetAutoscalingSettings() const; + TAutoPartitioningSettings GetAutoPartitioningSettings() const; private: uint64_t MinActivePartitions_; uint64_t MaxActivePartitions_; uint64_t PartitionCountLimit_; - TAutoscalingSettings AutoscalingSettings_; + TAutoPartitioningSettings AutoPartitioningSettings_; }; struct TAlterTopicSettings; @@ -237,12 +237,12 @@ struct TAlterPartitioningSettings { TAlterTopicSettings& EndAlterTopicPartitioningSettings() { return Parent_; }; - TAlterAutoscalingSettings& BeginAlterAutoscalingSettings() { - AutoscalingSettings_.emplace(*this); - return *AutoscalingSettings_; + TAlterAutoPartitioningSettings& BeginAlterAutoPartitioningSettings() { + AutoPartitioningSettings_.emplace(*this); + return *AutoPartitioningSettings_; } - std::optional AutoscalingSettings_; + std::optional AutoPartitioningSettings_; private: TAlterTopicSettings& Parent_; @@ -558,46 +558,46 @@ struct TCreateTopicSettings : public TOperationRequestSettings #include -#include +#include namespace NYdb::NTopic { diff --git a/include/ydb-cpp-sdk/client/topic/executor.h b/include/ydb-cpp-sdk/client/topic/executor.h index 277ff2c48e5..23484dc79ac 100644 --- a/include/ydb-cpp-sdk/client/topic/executor.h +++ b/include/ydb-cpp-sdk/client/topic/executor.h @@ -5,6 +5,7 @@ #include #include +#include namespace NYdb::NTopic { diff --git a/include/ydb-cpp-sdk/client/topic/read_session.h b/include/ydb-cpp-sdk/client/topic/read_session.h index 249de9b6462..889850417f7 100644 --- a/include/ydb-cpp-sdk/client/topic/read_session.h +++ b/include/ydb-cpp-sdk/client/topic/read_session.h @@ -193,8 +193,8 @@ struct TReadSessionSettings: public TRequestSettings { FLUENT_SETTING_DEFAULT(TDuration, ConnectTimeout, TDuration::Seconds(30)); - //! AutoscalingSupport. - FLUENT_SETTING_DEFAULT(bool, AutoscalingSupport, false); + //! AutoPartitioningSupport. + FLUENT_SETTING_DEFAULT(bool, AutoPartitioningSupport, false); //! Log. FLUENT_SETTING_OPTIONAL(TLog, Log); diff --git a/include/ydb-cpp-sdk/client/topic/retry_policy.h b/include/ydb-cpp-sdk/client/topic/retry_policy.h index 8aef9f3556a..255c3b46811 100644 --- a/include/ydb-cpp-sdk/client/topic/retry_policy.h +++ b/include/ydb-cpp-sdk/client/topic/retry_policy.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include namespace NYdb::NTopic { diff --git a/include/ydb-cpp-sdk/client/types/status/status.h b/include/ydb-cpp-sdk/client/types/status/status.h index 611f43807a3..85b8793ab97 100644 --- a/include/ydb-cpp-sdk/client/types/status/status.h +++ b/include/ydb-cpp-sdk/client/types/status/status.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include diff --git a/include/ydb-cpp-sdk/library/grpc/client/grpc_client_low.h b/include/ydb-cpp-sdk/library/grpc/client/grpc_client_low.h index 180f6a51fe4..6493b95cc57 100644 --- a/include/ydb-cpp-sdk/library/grpc/client/grpc_client_low.h +++ b/include/ydb-cpp-sdk/library/grpc/client/grpc_client_low.h @@ -2,6 +2,8 @@ #include "grpc_common.h" +#include + #include #include #include @@ -203,7 +205,7 @@ class TGRpcRequestProcessorCommon { protected: void ApplyMeta(const TCallMeta& meta) { for (const auto& rec : meta.Aux) { - Context.AddMetadata(rec.first, rec.second); + Context.AddMetadata(NYdb::TStringType{rec.first}, NYdb::TStringType{rec.second}); } if (meta.CallCredentials) { Context.set_credentials(meta.CallCredentials); @@ -529,7 +531,7 @@ class TChannelPool { std::shared_mutex RWMutex_; std::unordered_map Pool_; std::multimap LastUsedQueue_; - TTcpKeepAliveSettings TcpKeepAliveSettings_; + [[maybe_unused]] TTcpKeepAliveSettings TcpKeepAliveSettings_; TDuration ExpireTime_; TDuration UpdateReUseTime_; void EraseFromQueueByTime(const TInstant& lastUseTime, const std::string& channelId); diff --git a/include/ydb-cpp-sdk/library/grpc/client/grpc_common.h b/include/ydb-cpp-sdk/library/grpc/client/grpc_common.h index 9f9d96198f1..58adfeda3e3 100644 --- a/include/ydb-cpp-sdk/library/grpc/client/grpc_common.h +++ b/include/ydb-cpp-sdk/library/grpc/client/grpc_common.h @@ -3,7 +3,9 @@ #include #include +#include #include + #include #include #include @@ -40,7 +42,9 @@ struct TGRpcClientConfig { , MaxMessageSize(maxMessageSize) , MaxInFlight(maxInFlight) , EnableSsl(enableSsl) - , SslCredentials{.pem_root_certs = caCert, .pem_private_key = clientPrivateKey, .pem_cert_chain = clientCert} + , SslCredentials{.pem_root_certs = NYdb::TStringType{caCert}, + .pem_private_key = NYdb::TStringType{clientPrivateKey}, + .pem_cert_chain = NYdb::TStringType{clientCert}} , CompressionAlgoritm(compressionAlgorithm) {} }; @@ -52,11 +56,11 @@ inline std::shared_ptr CreateChannelInterface(const TGRp args.SetCompressionAlgorithm(config.CompressionAlgoritm); for (const auto& kvp: config.StringChannelParams) { - args.SetString(kvp.first, kvp.second); + args.SetString(NYdb::TStringType{kvp.first}, NYdb::TStringType{kvp.second}); } for (const auto& kvp: config.IntChannelParams) { - args.SetInt(kvp.first, kvp.second); + args.SetInt(NYdb::TStringType{kvp.first}, kvp.second); } if (config.MemQuota) { @@ -68,15 +72,15 @@ inline std::shared_ptr CreateChannelInterface(const TGRp args.SetSocketMutator(mutator); } if (!config.LoadBalancingPolicy.empty()) { - args.SetLoadBalancingPolicyName(config.LoadBalancingPolicy); + args.SetLoadBalancingPolicyName(NYdb::TStringType{config.LoadBalancingPolicy}); } if (!config.SslTargetNameOverride.empty()) { - args.SetSslTargetNameOverride(config.SslTargetNameOverride); + args.SetSslTargetNameOverride(NYdb::TStringType{config.SslTargetNameOverride}); } if (config.EnableSsl || !config.SslCredentials.pem_root_certs.empty()) { - return grpc::CreateCustomChannel(config.Locator, grpc::SslCredentials(config.SslCredentials), args); + return grpc::CreateCustomChannel(grpc::string(config.Locator), grpc::SslCredentials(config.SslCredentials), args); } else { - return grpc::CreateCustomChannel(config.Locator, grpc::InsecureChannelCredentials(), args); + return grpc::CreateCustomChannel(grpc::string(config.Locator), grpc::InsecureChannelCredentials(), args); } } diff --git a/include/ydb-cpp-sdk/library/yql/public/issue/yql_issue.h b/include/ydb-cpp-sdk/library/yql_common/issue/yql_issue.h similarity index 100% rename from include/ydb-cpp-sdk/library/yql/public/issue/yql_issue.h rename to include/ydb-cpp-sdk/library/yql_common/issue/yql_issue.h diff --git a/include/ydb-cpp-sdk/library/yql/public/issue/yql_issue_id.h b/include/ydb-cpp-sdk/library/yql_common/issue/yql_issue_id.h similarity index 73% rename from include/ydb-cpp-sdk/library/yql/public/issue/yql_issue_id.h rename to include/ydb-cpp-sdk/library/yql_common/issue/yql_issue_id.h index 930f0b27b0d..5520e4a090f 100644 --- a/include/ydb-cpp-sdk/library/yql/public/issue/yql_issue_id.h +++ b/include/ydb-cpp-sdk/library/yql_common/issue/yql_issue_id.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace NYql { diff --git a/include/ydb-cpp-sdk/library/yql/utils/utf8.h b/include/ydb-cpp-sdk/library/yql_common/utils/utf8.h similarity index 100% rename from include/ydb-cpp-sdk/library/yql/utils/utf8.h rename to include/ydb-cpp-sdk/library/yql_common/utils/utf8.h diff --git a/include/ydb-cpp-sdk/type_switcher.h b/include/ydb-cpp-sdk/type_switcher.h new file mode 100644 index 00000000000..8e90d9474fd --- /dev/null +++ b/include/ydb-cpp-sdk/type_switcher.h @@ -0,0 +1,13 @@ +#pragma once + +#ifdef YDB_SDK_USE_TSTRING +#include +namespace NYdb { +using TStringType = TString; +} +#else +#include +namespace NYdb { +using TStringType = std::string; +} +#endif diff --git a/src/api/CMakeLists.txt b/src/api/CMakeLists.txt index be3c6f80686..a7bc67de9ee 100644 --- a/src/api/CMakeLists.txt +++ b/src/api/CMakeLists.txt @@ -1,4 +1,3 @@ add_subdirectory(client) add_subdirectory(grpc) -add_subdirectory(proto_output) add_subdirectory(protos) diff --git a/src/client/coordination/coordination.cpp b/src/client/coordination/coordination.cpp index a4aa8b45f06..8140712fb09 100644 --- a/src/client/coordination/coordination.cpp +++ b/src/client/coordination/coordination.cpp @@ -5,7 +5,7 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include +#include #include #include @@ -149,7 +149,7 @@ TSemaphoreSession::TSemaphoreSession( { OrderId_ = desc.order_id(); SessionId_ = desc.session_id(); - Timeout_ = desc.timeout_millis() == Max() ? TDuration::Max() : TDuration::MilliSeconds(desc.timeout_millis()); + Timeout_ = desc.timeout_millis() == Max() ? TDuration::Max() : TDuration::MilliSeconds(desc.timeout_millis()); Count_ = desc.count(); Data_ = desc.data(); } @@ -254,7 +254,7 @@ class TSessionContext : public TThrRefBase { struct TSemaphoreOp : public TSimpleRefCount { TInstant SendTimestamp; - ui64 ReqId = 0; + uint64_t ReqId = 0; TResultPromise Promise = NewResultPromise(); const ESemaphoreOpType OpType; @@ -276,9 +276,9 @@ class TSessionContext : public TThrRefBase { , Deadline(Settings.Timeout_.ToDeadLine()) { } - ui64 GetTimeoutMillisLeft() const { + uint64_t GetTimeoutMillisLeft() const { if (Deadline == TInstant::Max()) { - return Max(); + return Max(); } return (Deadline - TInstant::Now()).MilliSeconds(); } @@ -286,10 +286,10 @@ class TSessionContext : public TThrRefBase { void FillRequest(TRequest& req, const std::string& name) const override { auto& inner = *req.mutable_acquire_semaphore(); inner.set_req_id(ReqId); - inner.set_name(name); + inner.set_name(TStringType{name}); inner.set_count(Settings.Count_); inner.set_timeout_millis(GetTimeoutMillisLeft()); - inner.set_data(Settings.Data_); + inner.set_data(TStringType{Settings.Data_}); inner.set_ephemeral(Settings.Ephemeral_); } }; @@ -302,7 +302,7 @@ class TSessionContext : public TThrRefBase { void FillRequest(TRequest& req, const std::string& name) const override { auto& inner = *req.mutable_release_semaphore(); inner.set_req_id(ReqId); - inner.set_name(name); + inner.set_name(TStringType{name}); } }; @@ -310,7 +310,7 @@ class TSessionContext : public TThrRefBase { std::string Name; TIntrusivePtr LastSentOp; TIntrusivePtr LastAckedOp; - std::unordered_map> WaitingOps; + std::unordered_map> WaitingOps; std::deque> OpQueue; bool Restoring = false; @@ -326,7 +326,7 @@ class TSessionContext : public TThrRefBase { TInstant SendTimestamp; virtual ~TSimpleOp() = default; - virtual void FillRequest(TRequest& req, ui64 reqId) const = 0; + virtual void FillRequest(TRequest& req, uint64_t reqId) const = 0; virtual void SetFailure(const TStatus& status) = 0; }; @@ -335,7 +335,7 @@ class TSessionContext : public TThrRefBase { TPingOp() = default; - void FillRequest(TRequest& req, ui64 reqId) const override { + void FillRequest(TRequest& req, uint64_t reqId) const override { auto& inner = *req.mutable_ping(); inner.set_opaque(reqId); } @@ -355,10 +355,10 @@ class TSessionContext : public TThrRefBase { , Settings(settings) {} - void FillRequest(TRequest& req, ui64 reqId) const override { + void FillRequest(TRequest& req, uint64_t reqId) const override { auto& inner = *req.mutable_describe_semaphore(); inner.set_req_id(reqId); - inner.set_name(Name); + inner.set_name(TStringType{Name}); inner.set_include_owners(Settings.IncludeOwners_); inner.set_include_waiters(Settings.IncludeWaiters_); inner.set_watch_data(Settings.WatchData_); @@ -378,22 +378,22 @@ class TSessionContext : public TThrRefBase { struct TCreateSemaphoreOp : public TSimpleOp { const std::string Name; - const ui64 Limit; + const uint64_t Limit; const std::string Data; TResultPromise Promise = NewResultPromise(); - TCreateSemaphoreOp(const std::string& name, ui64 limit, const std::string& data) + TCreateSemaphoreOp(const std::string& name, uint64_t limit, const std::string& data) : Name(name) , Limit(limit) , Data(data) { } - void FillRequest(TRequest& req, ui64 reqId) const override { + void FillRequest(TRequest& req, uint64_t reqId) const override { auto& inner = *req.mutable_create_semaphore(); inner.set_req_id(reqId); - inner.set_name(Name); + inner.set_name(TStringType{Name}); inner.set_limit(Limit); - inner.set_data(Data); + inner.set_data(TStringType{Data}); } void SetFailure(const TStatus& status) override { @@ -411,11 +411,11 @@ class TSessionContext : public TThrRefBase { , Data(data) {} - void FillRequest(TRequest& req, ui64 reqId) const override { + void FillRequest(TRequest& req, uint64_t reqId) const override { auto& inner = *req.mutable_update_semaphore(); inner.set_req_id(reqId); - inner.set_name(Name); - inner.set_data(Data); + inner.set_name(TStringType{Name}); + inner.set_data(TStringType{Data}); } void SetFailure(const TStatus& status) override { @@ -433,10 +433,10 @@ class TSessionContext : public TThrRefBase { , Force(force) {} - void FillRequest(TRequest& req, ui64 reqId) const override { + void FillRequest(TRequest& req, uint64_t reqId) const override { auto& inner = *req.mutable_delete_semaphore(); inner.set_req_id(reqId); - inner.set_name(Name); + inner.set_name(TStringType{Name}); inner.set_force(Force); } @@ -552,7 +552,7 @@ class TSessionContext : public TThrRefBase { return future; } - TAsyncResult DoCreateSemaphore(const std::string& name, ui64 limit, const std::string& data) { + TAsyncResult DoCreateSemaphore(const std::string& name, uint64_t limit, const std::string& data) { std::lock_guard guard(Lock); if (IsClosed()) { return MakeClosedResult(); @@ -779,7 +779,7 @@ class TSessionContext : public TThrRefBase { } bool DoSemaphoreProcessResult( - ui64 reqId, ESemaphoreOpType opType, EStatus status, + uint64_t reqId, ESemaphoreOpType opType, EStatus status, TResultPromise* supersededPromise, TResultPromise* resultPromise) { @@ -850,9 +850,9 @@ class TSessionContext : public TThrRefBase { DoSemaphoreProcessQueue(state); } - ui64 DoSendSimpleOp(std::unique_ptr&& op) { + uint64_t DoSendSimpleOp(std::unique_ptr&& op) { Y_ABORT_UNLESS(IsWriteAllowed()); - ui64 reqId = NextReqId++; + uint64_t reqId = NextReqId++; TRequest req; op->FillRequest(req, reqId); op->SendTimestamp = TInstant::Now(); @@ -862,7 +862,7 @@ class TSessionContext : public TThrRefBase { } template - TOperation* FindSentRequest(ui64 reqId) const { + TOperation* FindSentRequest(uint64_t reqId) const { auto it = SentRequests.find(reqId); if (it == SentRequests.end()) { return nullptr; @@ -1134,8 +1134,8 @@ class TSessionContext : public TThrRefBase { return; } - ui64 seqNo; - ui64 sessionId; + uint64_t seqNo; + uint64_t sessionId; TDuration timeout = Settings_.Timeout_; IQueueClientContextPtr timeoutContext; @@ -1185,11 +1185,11 @@ class TSessionContext : public TThrRefBase { auto* start = req.mutable_session_start(); start->set_seq_no(seqNo); start->set_session_id(sessionId); - start->set_path(Path_); + start->set_path(TStringType{Path_}); start->set_timeout_millis(Settings_.Timeout_ != TDuration::Max() ? - Settings_.Timeout_.MilliSeconds() : Max()); - start->set_description(Settings_.Description_); - start->set_protection_key(ProtectionKey_); + Settings_.Timeout_.MilliSeconds() : Max()); + start->set_description(TStringType{Settings_.Description_}); + start->set_protection_key(TStringType{ProtectionKey_}); processor->Write(std::move(req)); } @@ -1415,7 +1415,7 @@ class TSessionContext : public TThrRefBase { } case TResponse::kPong: { const auto& source = Response->pong(); - const ui64 reqId = source.opaque(); + const uint64_t reqId = source.opaque(); TResultPromise replyPromise; { std::lock_guard guard(Lock); @@ -1539,7 +1539,7 @@ class TSessionContext : public TThrRefBase { } case TResponse::kAcquireSemaphorePending: { const auto& source = Response->acquire_semaphore_pending(); - ui64 reqId = source.req_id(); + uint64_t reqId = source.req_id(); TResultPromise supersededPromise; std::function acceptedCallback; with_lock(Lock) { @@ -1576,7 +1576,7 @@ class TSessionContext : public TThrRefBase { } case TResponse::kAcquireSemaphoreResult: { const auto& source = Response->acquire_semaphore_result(); - const ui64 reqId = source.req_id(); + const uint64_t reqId = source.req_id(); auto plain = MakePlainStatus(source.status(), source.issues()); TResultPromise supersededPromise; TResultPromise resultPromise; @@ -1595,7 +1595,7 @@ class TSessionContext : public TThrRefBase { } case TResponse::kReleaseSemaphoreResult: { const auto& source = Response->release_semaphore_result(); - const ui64 reqId = source.req_id(); + const uint64_t reqId = source.req_id(); auto plain = MakePlainStatus(source.status(), source.issues()); TResultPromise supersededPromise; TResultPromise resultPromise; @@ -1614,7 +1614,7 @@ class TSessionContext : public TThrRefBase { } case TResponse::kDescribeSemaphoreResult: { const auto& source = Response->describe_semaphore_result(); - const ui64 reqId = source.req_id(); + const uint64_t reqId = source.req_id(); auto plain = MakePlainStatus(source.status(), source.issues()); TPromise resultPromise; { @@ -1637,7 +1637,7 @@ class TSessionContext : public TThrRefBase { } case TResponse::kDescribeSemaphoreChanged: { const auto& source = Response->describe_semaphore_changed(); - const ui64 reqId = source.req_id(); + const uint64_t reqId = source.req_id(); std::function callback; bool triggered = false; { @@ -1657,7 +1657,7 @@ class TSessionContext : public TThrRefBase { } case TResponse::kCreateSemaphoreResult: { const auto& source = Response->create_semaphore_result(); - const ui64 reqId = source.req_id(); + const uint64_t reqId = source.req_id(); auto plain = MakePlainStatus(source.status(), source.issues()); TResultPromise resultPromise; { @@ -1675,7 +1675,7 @@ class TSessionContext : public TThrRefBase { } case TResponse::kUpdateSemaphoreResult: { const auto& source = Response->update_semaphore_result(); - const ui64 reqId = source.req_id(); + const uint64_t reqId = source.req_id(); auto plain = MakePlainStatus(source.status(), source.issues()); TResultPromise resultPromise; { @@ -1693,7 +1693,7 @@ class TSessionContext : public TThrRefBase { } case TResponse::kDeleteSemaphoreResult: { const auto& source = Response->delete_semaphore_result(); - const ui64 reqId = source.req_id(); + const uint64_t reqId = source.req_id(); auto plain = MakePlainStatus(source.status(), source.issues()); TResultPromise resultPromise; { @@ -1771,15 +1771,15 @@ class TSessionContext : public TThrRefBase { IQueueClientContextPtr ConnectTimeoutContext; std::unordered_map Semaphores; - std::unordered_map SemaphoreByReqId; + std::unordered_map SemaphoreByReqId; std::deque> PendingRequests; - std::unordered_map> SentRequests; + std::unordered_map> SentRequests; TResultPromise ReconnectPromise; // These are used to manage session timeout IQueueClientContextPtr SessionStartTimeoutContext; IQueueClientContextPtr SessionSelfPingContext; - ui64 SessionSelfPingReqId = 0; + uint64_t SessionSelfPingReqId = 0; // This will be set to true when session is timed out bool SessionTimeout = false; @@ -1798,9 +1798,9 @@ class TSessionContext : public TThrRefBase { IProcessor::TPtr Processor; std::unique_ptr Response; - ui64 SessionSeqNo = 0; - ui64 SessionId = 0; - ui64 NextReqId = 1; + uint64_t SessionSeqNo = 0; + uint64_t SessionId = 0; + uint64_t NextReqId = 1; bool IsStopping = false; }; @@ -1919,7 +1919,7 @@ TAsyncStatus TClient::CreateNode( const TCreateNodeSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); ConvertSettingsToProtoConfig(settings, request.mutable_config()); return Impl_->CreateNode(std::move(request), settings); } @@ -1929,7 +1929,7 @@ TAsyncStatus TClient::AlterNode( const TAlterNodeSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); ConvertSettingsToProtoConfig(settings, request.mutable_config()); return Impl_->AlterNode(std::move(request), settings); } @@ -1939,7 +1939,7 @@ TAsyncStatus TClient::DropNode( const TDropNodeSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); return Impl_->DropNode(std::move(request), settings); } @@ -1948,7 +1948,7 @@ TAsyncDescribeNodeResult TClient::DescribeNode( const TDescribeNodeSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); return Impl_->DescribeNode(std::move(request), settings); } @@ -1964,7 +1964,7 @@ class TSession::TImpl { Context->DoClose(true); } - ui64 GetSessionId() { + uint64_t GetSessionId() { return Context->SessionId; } @@ -2006,7 +2006,7 @@ class TSession::TImpl { return Context->DoDescribeSemaphore(name, settings); } - TAsyncResult CreateSemaphore(const std::string& name, ui64 limit, const std::string& data) { + TAsyncResult CreateSemaphore(const std::string& name, uint64_t limit, const std::string& data) { return Context->DoCreateSemaphore(name, limit, data); } @@ -2028,7 +2028,7 @@ TSession::TSession(TSessionContext* context) : Impl_(std::make_shared(context)) { } -ui64 TSession::GetSessionId() { +uint64_t TSession::GetSessionId() { return Impl_->GetSessionId(); } @@ -2072,7 +2072,7 @@ TAsyncDescribeSemaphoreResult TSession::DescribeSemaphore( TAsyncResult TSession::CreateSemaphore( const std::string& name, - ui64 limit, + uint64_t limit, const std::string& data) { return Impl_->CreateSemaphore(name, limit, data); diff --git a/src/client/datastreams/datastreams.cpp b/src/client/datastreams/datastreams.cpp index 89372a2ed9e..e014026fb1c 100644 --- a/src/client/datastreams/datastreams.cpp +++ b/src/client/datastreams/datastreams.cpp @@ -4,8 +4,8 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include +#include +//#include #include @@ -20,7 +20,7 @@ namespace NYdb::NDataStreams::V1 { auto MakeResultExtractor(NThreading::TPromise promise) { return [promise = std::move(promise)] (google::protobuf::Any *any, TPlainStatus status) mutable { - std::unique_ptr result; + std::unique_ptr result; if (any) { result.reset(new TProtoResult); any->UnpackTo(result.get()); @@ -73,7 +73,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::CreateStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncCreateStream, [&](Ydb::DataStreams::V1::CreateStreamRequest& req) { - req.set_stream_name(path); + req.set_stream_name(TStringType{path}); req.set_shard_count(settings.ShardCount_); if (settings.RetentionStorageMegabytes_.has_value()) { req.set_retention_storage_megabytes(*settings.RetentionStorageMegabytes_); @@ -97,7 +97,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::ListStreamsResponse, Ydb::DataStreams::V1::ListStreamsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreams, [&](Ydb::DataStreams::V1::ListStreamsRequest& req) { - req.set_exclusive_start_stream_name(settings.ExclusiveStartStreamName_); + req.set_exclusive_start_stream_name(TStringType{settings.ExclusiveStartStreamName_}); req.set_limit(settings.Limit_); req.set_recurse(settings.Recurse_); }); @@ -118,12 +118,12 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::ListShardsResponse, Ydb::DataStreams::V1::ListShardsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListShards, [&](Ydb::DataStreams::V1::ListShardsRequest& req) { - req.set_exclusive_start_shard_id(settings.ExclusiveStartShardId_); + req.set_exclusive_start_shard_id(TStringType{settings.ExclusiveStartShardId_}); req.set_max_results(settings.MaxResults_); - req.set_next_token(settings.NextToken_); + req.set_next_token(TStringType{settings.NextToken_}); req.mutable_shard_filter()->CopyFrom(shardFilter); req.set_stream_creation_timestamp(settings.StreamCreationTimestamp_); - req.set_stream_name(path); + req.set_stream_name(TStringType{path}); }); } @@ -133,12 +133,12 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::PutRecordsResponse, Ydb::DataStreams::V1::PutRecordsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecords, [&](Ydb::DataStreams::V1::PutRecordsRequest& req) { - req.set_stream_name(path); + req.set_stream_name(TStringType{path}); for (const auto& record : records) { auto* protoRecord = req.add_records(); - protoRecord->set_partition_key(record.PartitionKey); - protoRecord->set_data(record.Data); - protoRecord->set_explicit_hash_key(record.ExplicitHashDecimal); + protoRecord->set_partition_key(TStringType{record.PartitionKey}); + protoRecord->set_data(TStringType{record.Data}); + protoRecord->set_explicit_hash_key(TStringType{record.ExplicitHashDecimal}); } }); } @@ -149,7 +149,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::GetRecordsResponse, Ydb::DataStreams::V1::GetRecordsResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetRecords, [&](Ydb::DataStreams::V1::GetRecordsRequest& req) { - req.set_shard_iterator(shardIterator); + req.set_shard_iterator(TStringType{shardIterator}); req.set_limit(settings.Limit_); }); } @@ -162,10 +162,10 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::GetShardIteratorResponse, Ydb::DataStreams::V1::GetShardIteratorResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncGetShardIterator, [&](Ydb::DataStreams::V1::GetShardIteratorRequest& req) { - req.set_stream_name(path); - req.set_shard_id(shardId); + req.set_stream_name(TStringType{path}); + req.set_shard_id(TStringType{shardId}); req.set_shard_iterator_type(shardIteratorType); - req.set_starting_sequence_number(settings.StartingSequenceNumber_); + req.set_starting_sequence_number(TStringType{settings.StartingSequenceNumber_}); req.set_timestamp(settings.Timestamp_); }); } @@ -190,7 +190,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::DescribeStreamSummaryResponse, Ydb::DataStreams::V1::DescribeStreamSummaryResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStreamSummary, [&](Ydb::DataStreams::V1::DescribeStreamSummaryRequest& req) { - req.set_stream_name(path); + req.set_stream_name(TStringType{path}); }); } @@ -200,7 +200,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResponse, Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDecreaseStreamRetentionPeriod, [&](Ydb::DataStreams::V1::DecreaseStreamRetentionPeriodRequest& req) { - req.set_stream_name(path); + req.set_stream_name(TStringType{path}); req.set_retention_period_hours(settings.RetentionPeriodHours_); }); @@ -212,7 +212,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResponse, Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncIncreaseStreamRetentionPeriod, [&](Ydb::DataStreams::V1::IncreaseStreamRetentionPeriodRequest& req) { - req.set_stream_name(path); + req.set_stream_name(TStringType{path}); req.set_retention_period_hours(settings.RetentionPeriodHours_); }); @@ -224,7 +224,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::UpdateShardCountResponse, Ydb::DataStreams::V1::UpdateShardCountResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateShardCount, [&](Ydb::DataStreams::V1::UpdateShardCountRequest& req) { - req.set_stream_name(path); + req.set_stream_name(TStringType{path}); req.set_target_shard_count(settings.TargetShardCount_); }); } @@ -235,7 +235,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::UpdateStreamModeResponse, Ydb::DataStreams::V1::UpdateStreamModeResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStreamMode, [&](Ydb::DataStreams::V1::UpdateStreamModeRequest& req) { - req.set_stream_arn(path); + req.set_stream_arn(TStringType{path}); req.mutable_stream_mode_details()->set_stream_mode( settings.StreamMode_ == ESM_PROVISIONED ? Ydb::DataStreams::V1::StreamMode::PROVISIONED @@ -249,8 +249,8 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::RegisterStreamConsumerResponse, Ydb::DataStreams::V1::RegisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncRegisterStreamConsumer, [&](Ydb::DataStreams::V1::RegisterStreamConsumerRequest& req) { - req.set_stream_arn(path); - req.set_consumer_name(consumer_name); + req.set_stream_arn(TStringType{path}); + req.set_consumer_name(TStringType{consumer_name}); }); } @@ -260,8 +260,8 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::DeregisterStreamConsumerResponse, Ydb::DataStreams::V1::DeregisterStreamConsumerResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeregisterStreamConsumer, [&](Ydb::DataStreams::V1::DeregisterStreamConsumerRequest& req) { - req.set_stream_arn(path); - req.set_consumer_name(consumer_name); + req.set_stream_arn(TStringType{path}); + req.set_consumer_name(TStringType{consumer_name}); }); } @@ -277,8 +277,8 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::ListStreamConsumersRequest, Ydb::DataStreams::V1::ListStreamConsumersResponse, Ydb::DataStreams::V1::ListStreamConsumersResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncListStreamConsumers, [&](Ydb::DataStreams::V1::ListStreamConsumersRequest& req) { - req.set_stream_arn(path); - req.set_next_token(settings.NextToken_); + req.set_stream_arn(TStringType{path}); + req.set_next_token(TStringType{settings.NextToken_}); req.set_max_results(settings.MaxResults_); }); } @@ -358,7 +358,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::UpdateStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncUpdateStream, [&](Ydb::DataStreams::V1::UpdateStreamRequest& req) { - req.set_stream_name(streamName); + req.set_stream_name(TStringType{streamName}); req.set_target_shard_count(settings.TargetShardCount_); if (settings.RetentionPeriodHours_.has_value()) { req.set_retention_period_hours(*settings.RetentionPeriodHours_); @@ -382,7 +382,7 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::DeleteStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDeleteStream, [&](Ydb::DataStreams::V1::DeleteStreamRequest& req) { - req.set_stream_name(path); + req.set_stream_name(TStringType{path}); req.set_enforce_consumer_deletion(settings.EnforceConsumerDeletion_); }); } @@ -393,8 +393,8 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::DescribeStreamResponse, Ydb::DataStreams::V1::DescribeStreamResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncDescribeStream, [&](Ydb::DataStreams::V1::DescribeStreamRequest& req) { - req.set_stream_name(path); - req.set_exclusive_start_shard_id(settings.ExclusiveStartShardId_); + req.set_stream_name(TStringType{path}); + req.set_exclusive_start_shard_id(TStringType{settings.ExclusiveStartShardId_}); req.set_limit(settings.Limit_); }); } @@ -405,10 +405,10 @@ namespace NYdb::NDataStreams::V1 { Ydb::DataStreams::V1::PutRecordResponse, Ydb::DataStreams::V1::PutRecordResult>(settings, &Ydb::DataStreams::V1::DataStreamsService::Stub::AsyncPutRecord, [&](Ydb::DataStreams::V1::PutRecordRequest& req) { - req.set_stream_name(path); - req.set_explicit_hash_key(record.ExplicitHashDecimal); - req.set_partition_key(record.PartitionKey); - req.set_data(record.Data); + req.set_stream_name(TStringType{path}); + req.set_explicit_hash_key(TStringType{record.ExplicitHashDecimal}); + req.set_partition_key(TStringType{record.PartitionKey}); + req.set_data(TStringType{record.Data}); }); } diff --git a/src/client/discovery/discovery.cpp b/src/client/discovery/discovery.cpp index 9f3ab7b0742..c7b04032ee5 100644 --- a/src/client/discovery/discovery.cpp +++ b/src/client/discovery/discovery.cpp @@ -95,7 +95,7 @@ TNodeRegistrationResult::TNodeRegistrationResult(TStatus&& status, const Ydb::Di } } -const ui32& TNodeRegistrationResult::GetNodeId() const { +uint32_t TNodeRegistrationResult::GetNodeId() const { return NodeId_; } @@ -103,11 +103,11 @@ const std::string& TNodeRegistrationResult::GetDomainPath() const { return DomainPath_; } -const ui64& TNodeRegistrationResult::GetExpire() const { +uint64_t TNodeRegistrationResult::GetExpire() const { return Expire_; } -const ui64& TNodeRegistrationResult::GetScopeTabletId() const { +uint64_t TNodeRegistrationResult::GetScopeTabletId() const { return ScopeTableId_.value(); } @@ -115,7 +115,7 @@ bool TNodeRegistrationResult::HasScopeTabletId() const { return ScopeTableId_.has_value(); } -const ui64& TNodeRegistrationResult::GetScopePathId() const { +uint64_t TNodeRegistrationResult::GetScopePathId() const { return ScopePathId_.value(); } @@ -143,7 +143,7 @@ class TDiscoveryClient::TImpl : public TClientImplCommonDatabase); + request.set_database(TStringType{DbDriverState_->Database}); auto promise = NThreading::NewPromise(); @@ -199,30 +199,30 @@ class TDiscoveryClient::TImpl : public TClientImplCommonset_data_center(location.DataCenter.value()); + requestLocation->set_data_center(TStringType{location.DataCenter.value()}); } if (location.Module) { - requestLocation->set_module(location.Module.value()); + requestLocation->set_module(TStringType{location.Module.value()}); } if (location.Rack) { - requestLocation->set_rack(location.Rack.value()); + requestLocation->set_rack(TStringType{location.Rack.value()}); } if (location.Unit) { - requestLocation->set_unit(location.Unit.value()); + requestLocation->set_unit(TStringType{location.Unit.value()}); } if (location.DataCenterNum) { diff --git a/src/client/draft/ydb_dynamic_config.cpp b/src/client/draft/ydb_dynamic_config.cpp index 1a30c6bfeac..673e8ef83aa 100644 --- a/src/client/draft/ydb_dynamic_config.cpp +++ b/src/client/draft/ydb_dynamic_config.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include @@ -15,7 +15,7 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon(settings); - request.set_config(config); + request.set_config(TStringType{config}); request.set_dry_run(dryRun); request.set_allow_unknown_fields(allowUnknownFields); @@ -27,7 +27,7 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon(settings); - request.set_config(config); + request.set_config(TStringType{config}); request.set_dry_run(dryRun); request.set_allow_unknown_fields(allowUnknownFields); @@ -37,10 +37,10 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon(settings); - request.mutable_identity()->set_cluster(cluster); + request.mutable_identity()->set_cluster(TStringType{cluster}); request.mutable_identity()->set_version(version); return RunSimple( @@ -51,7 +51,7 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon(settings); - request.set_config(config); + request.set_config(TStringType{config}); return RunSimple( std::move(request), @@ -59,10 +59,10 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon& ids, const TClusterConfigSettings& settings = {}) { + TAsyncStatus RemoveVolatileConfig(const std::string& cluster, uint64_t version, const std::vector& ids, const TClusterConfigSettings& settings = {}) { auto request = MakeOperationRequest(settings); - request.mutable_identity()->set_cluster(cluster); + request.mutable_identity()->set_cluster(TStringType{cluster}); request.mutable_identity()->set_version(version); for (auto& id: ids) { @@ -75,10 +75,10 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon(settings); - request.mutable_identity()->set_cluster(cluster); + request.mutable_identity()->set_cluster(TStringType{cluster}); request.mutable_identity()->set_version(version); request.set_all(true); @@ -88,7 +88,7 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon& ids, const TClusterConfigSettings& settings = {}) { + TAsyncStatus ForceRemoveVolatileConfig(const std::vector& ids, const TClusterConfigSettings& settings = {}) { auto request = MakeOperationRequest(settings); for (auto& id: ids) { @@ -116,7 +116,7 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon(settings); request.set_node_id(nodeId); @@ -152,9 +152,9 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon volatileConfigs; + std::map volatileConfigs; if (Ydb::DynamicConfig::GetConfigResult result; any && any->UnpackTo(&result)) { clusterName = result.identity().cluster(); version = result.identity().version(); @@ -186,7 +186,7 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon volatileConfigs; + std::map volatileConfigs; if (Ydb::DynamicConfig::GetMetadataResult result; any && any->UnpackTo(&result)) { metadata = result.metadata(); for (const auto& config : result.volatile_configs()) { @@ -209,18 +209,18 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon& volatileConfigs, const std::map& labels, const TClusterConfigSettings& settings = {}) { + TAsyncResolveConfigResult ResolveConfig(const std::string& config, const std::map& volatileConfigs, const std::map& labels, const TClusterConfigSettings& settings = {}) { auto request = MakeOperationRequest(settings); - request.set_config(config); + request.set_config(TStringType{config}); for (auto& [id, volatileConfig] : volatileConfigs) { auto* proto = request.add_volatile_configs(); proto->set_id(id); - proto->set_config(volatileConfig); + proto->set_config(TStringType{volatileConfig}); } for (auto& [name, value] : labels) { auto* proto = request.add_labels(); - proto->set_label(name); - proto->set_value(value); + proto->set_label(TStringType{name}); + proto->set_value(TStringType{value}); } auto promise = NThreading::NewPromise(); @@ -246,13 +246,13 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon& volatileConfigs, const TClusterConfigSettings& settings = {}) { + TAsyncResolveConfigResult ResolveConfig(const std::string& config, const std::map& volatileConfigs, const TClusterConfigSettings& settings = {}) { auto request = MakeOperationRequest(settings); - request.set_config(config); + request.set_config(TStringType{config}); for (auto& [id, volatileConfig] : volatileConfigs) { auto* proto = request.add_volatile_configs(); proto->set_id(id); - proto->set_config(volatileConfig); + proto->set_config(TStringType{volatileConfig}); } auto promise = NThreading::NewPromise(); @@ -278,13 +278,13 @@ class TDynamicConfigClient::TImpl : public TClientImplCommon& volatileConfigs, const TClusterConfigSettings& settings = {}) { + TAsyncVerboseResolveConfigResult VerboseResolveConfig(const std::string& config, const std::map& volatileConfigs, const TClusterConfigSettings& settings = {}) { auto request = MakeOperationRequest(settings); - request.set_config(config); + request.set_config(TStringType{config}); for (auto& [id, volatileConfig] : volatileConfigs) { auto* proto = request.add_volatile_configs(); proto->set_id(id); - proto->set_config(volatileConfig); + proto->set_config(TStringType{volatileConfig}); } request.set_verbose_response(true); @@ -360,7 +360,7 @@ TAsyncStatus TDynamicConfigClient::ReplaceConfig( TAsyncStatus TDynamicConfigClient::DropConfig( const std::string& cluster, - ui64 version, + uint64_t version, const TClusterConfigSettings& settings) { return Impl_->DropConfig(cluster, version, settings); } @@ -373,21 +373,21 @@ TAsyncStatus TDynamicConfigClient::AddVolatileConfig( TAsyncStatus TDynamicConfigClient::RemoveVolatileConfig( const std::string& cluster, - ui64 version, - const std::vector& ids, + uint64_t version, + const std::vector& ids, const TClusterConfigSettings& settings) { return Impl_->RemoveVolatileConfig(cluster, version, ids, settings); } TAsyncStatus TDynamicConfigClient::RemoveAllVolatileConfigs( const std::string& cluster, - ui64 version, + uint64_t version, const TClusterConfigSettings& settings) { return Impl_->RemoveAllVolatileConfigs(cluster, version, settings); } TAsyncStatus TDynamicConfigClient::ForceRemoveVolatileConfig( - const std::vector& ids, + const std::vector& ids, const TClusterConfigSettings& settings) { return Impl_->ForceRemoveVolatileConfig(ids, settings); } @@ -405,13 +405,13 @@ TAsyncGetConfigResult TDynamicConfigClient::GetConfig(const TClusterConfigSettin return Impl_->GetConfig(settings); } -TAsyncGetNodeLabelsResult TDynamicConfigClient::GetNodeLabels(ui64 nodeId, const TClusterConfigSettings& settings) { +TAsyncGetNodeLabelsResult TDynamicConfigClient::GetNodeLabels(uint64_t nodeId, const TClusterConfigSettings& settings) { return Impl_->GetNodeLabels(nodeId, settings); } TAsyncResolveConfigResult TDynamicConfigClient::ResolveConfig( const std::string& config, - const std::map& volatileConfigs, + const std::map& volatileConfigs, const std::map& labels, const TClusterConfigSettings& settings) { return Impl_->ResolveConfig(config, volatileConfigs, labels, settings); @@ -419,14 +419,14 @@ TAsyncResolveConfigResult TDynamicConfigClient::ResolveConfig( TAsyncResolveConfigResult TDynamicConfigClient::ResolveConfig( const std::string& config, - const std::map& volatileConfigs, + const std::map& volatileConfigs, const TClusterConfigSettings& settings) { return Impl_->ResolveConfig(config, volatileConfigs, settings); } TAsyncVerboseResolveConfigResult TDynamicConfigClient::VerboseResolveConfig( const std::string& config, - const std::map& volatileConfigs, + const std::map& volatileConfigs, const TClusterConfigSettings& settings) { return Impl_->VerboseResolveConfig(config, volatileConfigs, settings); } diff --git a/src/client/draft/ydb_replication.cpp b/src/client/draft/ydb_replication.cpp index 9e0ace5379f..5aa7926719f 100644 --- a/src/client/draft/ydb_replication.cpp +++ b/src/client/draft/ydb_replication.cpp @@ -4,9 +4,9 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include -#include +#include +#include +#include #include #include @@ -167,7 +167,7 @@ class TReplicationClient::TImpl: public TClientImplCommon(settings); - request.set_path(path); + request.set_path(TStringType{path}); auto promise = NThreading::NewPromise(); diff --git a/src/client/draft/ydb_scripting.cpp b/src/client/draft/ydb_scripting.cpp index 05897386c84..93a8fb923ec 100644 --- a/src/client/draft/ydb_scripting.cpp +++ b/src/client/draft/ydb_scripting.cpp @@ -5,8 +5,8 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include +#include +#include #include namespace NYdb { @@ -123,7 +123,7 @@ TAsyncYqlResultPart TYqlResultPartIterator::ReadNext() { //////////////////////////////////////////////////////////////////////////////// -TExplainYqlResult::TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map&& types, std::string&& plan) +TExplainYqlResult::TExplainYqlResult(TStatus&& status, const ::google::protobuf::Map&& types, std::string&& plan) : TStatus(std::move(status)) , ParameterTypes_(std::move(types)) , Plan_(plan) {} @@ -154,7 +154,7 @@ class TScriptingClient::TImpl : public TClientImplCommon(settings); - request.set_script(script); + request.set_script(TStringType{script}); SetParams(params, &request); request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); request.set_syntax(settings.Syntax_); @@ -200,7 +200,7 @@ class TScriptingClient::TImpl : public TClientImplCommon(settings); - request.set_script(script); + request.set_script(TStringType{script}); SetParams(params, &request); request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); request.set_syntax(settings.Syntax_); @@ -250,7 +250,7 @@ class TScriptingClient::TImpl : public TClientImplCommon(settings); - request.set_script(script); + request.set_script(TStringType{script}); switch (settings.Mode_) { // KIKIMR-10990 @@ -270,7 +270,7 @@ class TScriptingClient::TImpl : public TClientImplCommon types; + ::google::protobuf::Map types; if (any) { Ydb::Scripting::ExplainYqlResult result; any->UnpackTo(&result); @@ -298,14 +298,14 @@ class TScriptingClient::TImpl : public TClientImplCommon - static void SetParams(::google::protobuf::Map* params, TRequest* request) { + static void SetParams(::google::protobuf::Map* params, TRequest* request) { if (params) { request->mutable_parameters()->swap(*params); } } template - static void SetParams(const ::google::protobuf::Map& params, TRequest* request) { + static void SetParams(const ::google::protobuf::Map& params, TRequest* request) { *request->mutable_parameters() = params; } @@ -335,7 +335,7 @@ TAsyncExecuteYqlResult TScriptingClient::ExecuteYqlScript(const std::string &que nullptr, settings); } else { - using TProtoParamsType = const ::google::protobuf::Map; + using TProtoParamsType = const ::google::protobuf::Map; return Impl_->ExecuteYqlScript( query, params.GetProtoMap(), @@ -361,7 +361,7 @@ TAsyncYqlResultPartIterator TScriptingClient::StreamExecuteYqlScript(const std:: if (params.Empty()) { return Impl_->StreamExecuteYqlScript(script, nullptr, settings); } else { - using TProtoParamsType = const ::google::protobuf::Map; + using TProtoParamsType = const ::google::protobuf::Map; return Impl_->StreamExecuteYqlScript(script, params.GetProtoMap(), settings); } } diff --git a/src/client/driver/driver.cpp b/src/client/driver/driver.cpp index 70348e6e340..009852d563b 100644 --- a/src/client/driver/driver.cpp +++ b/src/client/driver/driver.cpp @@ -44,10 +44,10 @@ class TDriverConfig::TImpl : public IConnectionsParams { TDuration GetGRpcKeepAliveTimeout() const override { return GRpcKeepAliveTimeout; } bool GetGRpcKeepAlivePermitWithoutCalls() const override { return GRpcKeepAlivePermitWithoutCalls; } TDuration GetSocketIdleTimeout() const override { return SocketIdleTimeout; } - ui64 GetMemoryQuota() const override { return MemoryQuota; } - ui64 GetMaxInboundMessageSize() const override { return MaxInboundMessageSize; } - ui64 GetMaxOutboundMessageSize() const override { return MaxOutboundMessageSize; } - ui64 GetMaxMessageSize() const override { return MaxMessageSize; } + uint64_t GetMemoryQuota() const override { return MemoryQuota; } + uint64_t GetMaxInboundMessageSize() const override { return MaxInboundMessageSize; } + uint64_t GetMaxOutboundMessageSize() const override { return MaxOutboundMessageSize; } + uint64_t GetMaxMessageSize() const override { return MaxMessageSize; } const TLog& GetLog() const override { return Log; } std::string Endpoint; @@ -71,10 +71,10 @@ class TDriverConfig::TImpl : public IConnectionsParams { TDuration GRpcKeepAliveTimeout; bool GRpcKeepAlivePermitWithoutCalls = false; TDuration SocketIdleTimeout = TDuration::Minutes(6); - ui64 MemoryQuota = 0; - ui64 MaxInboundMessageSize = 0; - ui64 MaxOutboundMessageSize = 0; - ui64 MaxMessageSize = 0; + uint64_t MemoryQuota = 0; + uint64_t MaxInboundMessageSize = 0; + uint64_t MaxOutboundMessageSize = 0; + uint64_t MaxMessageSize = 0; TLog Log; // Null by default. }; @@ -153,7 +153,7 @@ TDriverConfig& TDriverConfig::SetTcpKeepAliveSettings(bool enable, size_t idle, return *this; } -TDriverConfig& TDriverConfig::SetGrpcMemoryQuota(ui64 bytes) { +TDriverConfig& TDriverConfig::SetGrpcMemoryQuota(uint64_t bytes) { Impl_->MemoryQuota = bytes; return *this; } @@ -183,17 +183,17 @@ TDriverConfig& TDriverConfig::SetSocketIdleTimeout(TDuration timeout) { return *this; } -TDriverConfig& TDriverConfig::SetMaxInboundMessageSize(ui64 maxInboundMessageSize) { +TDriverConfig& TDriverConfig::SetMaxInboundMessageSize(uint64_t maxInboundMessageSize) { Impl_->MaxInboundMessageSize = maxInboundMessageSize; return *this; } -TDriverConfig& TDriverConfig::SetMaxOutboundMessageSize(ui64 maxOutboundMessageSize) { +TDriverConfig& TDriverConfig::SetMaxOutboundMessageSize(uint64_t maxOutboundMessageSize) { Impl_->MaxOutboundMessageSize = maxOutboundMessageSize; return *this; } -TDriverConfig& TDriverConfig::SetMaxMessageSize(ui64 maxMessageSize) { +TDriverConfig& TDriverConfig::SetMaxMessageSize(uint64_t maxMessageSize) { Impl_->MaxMessageSize = maxMessageSize; return *this; } diff --git a/src/client/export/export.cpp b/src/client/export/export.cpp index ac82813eba3..c8b5fecc04c 100644 --- a/src/client/export/export.cpp +++ b/src/client/export/export.cpp @@ -4,9 +4,9 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include -#include +#include +#include +#include #include #include @@ -149,18 +149,18 @@ TExportClient::TExportClient(const TDriver& driver, const TCommonClientSettings& TFuture TExportClient::ExportToYt(const TExportToYtSettings& settings) { auto request = MakeOperationRequest(settings); - request.mutable_settings()->set_host(settings.Host_); + request.mutable_settings()->set_host(TStringType{settings.Host_}); request.mutable_settings()->set_port(settings.Port_.value_or(80)); - request.mutable_settings()->set_token(settings.Token_); + request.mutable_settings()->set_token(TStringType{settings.Token_}); for (const auto& item : settings.Item_) { auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); - protoItem.set_source_path(item.Src); - protoItem.set_destination_path(item.Dst); + protoItem.set_source_path(TStringType{item.Src}); + protoItem.set_destination_path(TStringType{item.Dst}); } if (settings.Description_) { - request.mutable_settings()->set_description(settings.Description_.value()); + request.mutable_settings()->set_description(TStringType{settings.Description_.value()}); } if (settings.NumberOfRetries_) { @@ -175,21 +175,21 @@ TFuture TExportClient::ExportToYt(const TExportToYtSettings TFuture TExportClient::ExportToS3(const TExportToS3Settings& settings) { auto request = MakeOperationRequest(settings); - request.mutable_settings()->set_endpoint(settings.Endpoint_); + request.mutable_settings()->set_endpoint(TStringType{settings.Endpoint_}); request.mutable_settings()->set_scheme(TProtoAccessor::GetProto(settings.Scheme_)); request.mutable_settings()->set_storage_class(TProtoAccessor::GetProto(settings.StorageClass_)); - request.mutable_settings()->set_bucket(settings.Bucket_); - request.mutable_settings()->set_access_key(settings.AccessKey_); - request.mutable_settings()->set_secret_key(settings.SecretKey_); + request.mutable_settings()->set_bucket(TStringType{settings.Bucket_}); + request.mutable_settings()->set_access_key(TStringType{settings.AccessKey_}); + request.mutable_settings()->set_secret_key(TStringType{settings.SecretKey_}); for (const auto& item : settings.Item_) { auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); - protoItem.set_source_path(item.Src); - protoItem.set_destination_prefix(item.Dst); + protoItem.set_source_path(TStringType{item.Src}); + protoItem.set_destination_prefix(TStringType{item.Dst}); } if (settings.Description_) { - request.mutable_settings()->set_description(settings.Description_.value()); + request.mutable_settings()->set_description(TStringType{settings.Description_.value()}); } if (settings.NumberOfRetries_) { @@ -197,7 +197,7 @@ TFuture TExportClient::ExportToS3(const TExportToS3Settings } if (settings.Compression_) { - request.mutable_settings()->set_compression(*settings.Compression_); + request.mutable_settings()->set_compression(TStringType{settings.Compression_.value()}); } request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_); diff --git a/src/client/federated_topic/impl/federated_topic_impl.h b/src/client/federated_topic/impl/federated_topic_impl.h index 14991b4bdf5..3121aa275f3 100644 --- a/src/client/federated_topic/impl/federated_topic_impl.h +++ b/src/client/federated_topic/impl/federated_topic_impl.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include diff --git a/src/client/federated_topic/impl/federated_write_session.cpp b/src/client/federated_topic/impl/federated_write_session.cpp index f57819add70..ca2ab80d210 100644 --- a/src/client/federated_topic/impl/federated_write_session.cpp +++ b/src/client/federated_topic/impl/federated_write_session.cpp @@ -168,7 +168,7 @@ std::pair, EStatus> SelectDatabaseByHashImpl( NTopic::TFederatedWriteSessionSettings const& settings, std::vector> const& dbInfos ) { - ui64 totalWeight = 0; + uint64_t totalWeight = 0; std::vector> available; for (const auto& db : dbInfos) { @@ -184,11 +184,11 @@ std::pair, EStatus> SelectDatabaseByHashImpl( std::sort(available.begin(), available.end(), [](auto const& lhs, auto const& rhs) { return lhs->name() < rhs->name(); }); - ui64 hashValue = THash()(settings.Path_); + size_t hashValue = THash()(settings.Path_); hashValue = CombineHashes(hashValue, THash()(settings.ProducerId_)); hashValue %= totalWeight; - ui64 borderWeight = 0; + uint64_t borderWeight = 0; for (auto const& db : available) { borderWeight += db->weight(); if (hashValue < borderWeight) { @@ -322,11 +322,11 @@ std::optional TFederatedWriteSessionImpl::Ge return events.empty() ? std::nullopt : std::optional{std::move(events.front())}; } -NThreading::TFuture TFederatedWriteSessionImpl::GetInitSeqNo() { - return NThreading::MakeFuture(0u); +NThreading::TFuture TFederatedWriteSessionImpl::GetInitSeqNo() { + return NThreading::MakeFuture(0u); } -void TFederatedWriteSessionImpl::Write(NTopic::TContinuationToken&& token, std::string_view data, std::optional seqNo, +void TFederatedWriteSessionImpl::Write(NTopic::TContinuationToken&& token, std::string_view data, std::optional seqNo, std::optional createTimestamp) { NTopic::TWriteMessage message{std::move(data)}; if (seqNo.has_value()) @@ -341,7 +341,7 @@ void TFederatedWriteSessionImpl::Write(NTopic::TContinuationToken&& token, NTopi } void TFederatedWriteSessionImpl::WriteEncoded(NTopic::TContinuationToken&& token, std::string_view data, NTopic::ECodec codec, - ui32 originalSize, std::optional seqNo, std::optional createTimestamp) { + ui32 originalSize, std::optional seqNo, std::optional createTimestamp) { auto message = NTopic::TWriteMessage::CompressedMessage(std::move(data), codec, originalSize); if (seqNo.has_value()) message.SeqNo(*seqNo); diff --git a/src/client/federated_topic/impl/federated_write_session.h b/src/client/federated_topic/impl/federated_write_session.h index 690f2b3835d..af8f1b7a70b 100644 --- a/src/client/federated_topic/impl/federated_write_session.h +++ b/src/client/federated_topic/impl/federated_write_session.h @@ -34,17 +34,17 @@ class TFederatedWriteSessionImpl : public NTopic::TContinuationTokenIssuer, std::optional GetEvent(bool block); std::vector GetEvents(bool block, std::optional maxEventsCount); - NThreading::TFuture GetInitSeqNo(); + NThreading::TFuture GetInitSeqNo(); void Write(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& message); void WriteEncoded(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& params); - void Write(NTopic::TContinuationToken&&, std::string_view, std::optional seqNo = std::nullopt, + void Write(NTopic::TContinuationToken&&, std::string_view, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt); void WriteEncoded(NTopic::TContinuationToken&&, std::string_view, NTopic::ECodec, ui32, - std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt); + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt); bool Close(TDuration timeout); @@ -164,7 +164,7 @@ class TFederatedWriteSession : public NTopic::IWriteSession, std::vector GetEvents(bool block, std::optional maxEventsCount) override { return TryGetImpl()->GetEvents(block, maxEventsCount); } - NThreading::TFuture GetInitSeqNo() override { + NThreading::TFuture GetInitSeqNo() override { return TryGetImpl()->GetInitSeqNo(); } void Write(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& message) override { @@ -173,12 +173,12 @@ class TFederatedWriteSession : public NTopic::IWriteSession, void WriteEncoded(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& params) override { TryGetImpl()->WriteEncoded(std::move(continuationToken), std::move(params)); } - void Write(NTopic::TContinuationToken&& continuationToken, std::string_view data, std::optional seqNo = std::nullopt, + void Write(NTopic::TContinuationToken&& continuationToken, std::string_view data, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override { TryGetImpl()->Write(std::move(continuationToken), data, seqNo, createTimestamp); } void WriteEncoded(NTopic::TContinuationToken&& continuationToken, std::string_view data, NTopic::ECodec codec, ui32 originalSize, - std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override { + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override { TryGetImpl()->WriteEncoded(std::move(continuationToken), data, codec, originalSize, seqNo, createTimestamp); } bool Close(TDuration timeout) override { diff --git a/src/client/federated_topic/impl/federation_observer.cpp b/src/client/federated_topic/impl/federation_observer.cpp index fc3be6455d1..5596795e4c0 100644 --- a/src/client/federated_topic/impl/federation_observer.cpp +++ b/src/client/federated_topic/impl/federation_observer.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include @@ -140,8 +140,8 @@ void TFederatedDbObserverImpl::OnFederationDiscovery(TStatus&& status, Ydb::Fede FederatedDbState->ControlPlaneEndpoint = dbState->DiscoveryEndpoint; // FederatedDbState->SelfLocation = ???; auto db = std::make_shared(); - db->set_path(DbDriverState_->Database); - db->set_endpoint(DbDriverState_->DiscoveryEndpoint); + db->set_path(TStringType{DbDriverState_->Database}); + db->set_endpoint(TStringType{DbDriverState_->DiscoveryEndpoint}); db->set_status(Ydb::FederationDiscovery::DatabaseInfo_Status_AVAILABLE); db->set_weight(100); FederatedDbState->DbInfos.emplace_back(std::move(db)); @@ -151,8 +151,10 @@ void TFederatedDbObserverImpl::OnFederationDiscovery(TStatus&& status, Ydb::Fede if (!FederationDiscoveryRetryState) { FederationDiscoveryRetryState = FederationDiscoveryRetryPolicy->CreateRetryState(); } - std::optional retryDelay = FederationDiscoveryRetryState->GetNextRetryDelay(status.GetStatus()); - if (retryDelay.has_value()) { + + auto retryDelay = FederationDiscoveryRetryState->GetNextRetryDelay(status.GetStatus()); + + if (retryDelay) { ScheduleFederationDiscoveryImpl(*retryDelay); return; } diff --git a/src/client/federated_topic/impl/federation_observer.h b/src/client/federated_topic/impl/federation_observer.h index e35c1d858d5..0d52f3a9b47 100644 --- a/src/client/federated_topic/impl/federation_observer.h +++ b/src/client/federated_topic/impl/federation_observer.h @@ -5,7 +5,7 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include +#include #include #include diff --git a/src/client/federated_topic/ut/CMakeLists.darwin-arm64.txt b/src/client/federated_topic/ut/CMakeLists.darwin-arm64.txt deleted file mode 100644 index b2222286930..00000000000 --- a/src/client/federated_topic/ut/CMakeLists.darwin-arm64.txt +++ /dev/null @@ -1,87 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-public-sdk-client-ydb_federated_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_federated_topic-ut PUBLIC - yutil - cpp-testing-unittest_main - client-ydb_federated_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic - client-ydb_topic-impl - client-ydb_federated_topic-impl -) -target_link_options(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - -Wl,-platform_version,macos,11.0,11.0 - -fPIC - -fPIC - -framework - CoreFoundation -) -target_sources(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic/ut/basic_usage_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_federated_topic-ut - system_allocator -) -vcs_info(ydb-public-sdk-client-ydb_federated_topic-ut) diff --git a/src/client/federated_topic/ut/CMakeLists.darwin-x86_64.txt b/src/client/federated_topic/ut/CMakeLists.darwin-x86_64.txt deleted file mode 100644 index b574a073043..00000000000 --- a/src/client/federated_topic/ut/CMakeLists.darwin-x86_64.txt +++ /dev/null @@ -1,88 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-public-sdk-client-ydb_federated_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_federated_topic-ut PUBLIC - yutil - cpuid_check - cpp-testing-unittest_main - client-ydb_federated_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic - client-ydb_topic-impl - client-ydb_federated_topic-impl -) -target_link_options(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - -Wl,-platform_version,macos,11.0,11.0 - -fPIC - -fPIC - -framework - CoreFoundation -) -target_sources(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic/ut/basic_usage_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_federated_topic-ut - system_allocator -) -vcs_info(ydb-public-sdk-client-ydb_federated_topic-ut) diff --git a/src/client/federated_topic/ut/CMakeLists.linux-aarch64.txt b/src/client/federated_topic/ut/CMakeLists.linux-aarch64.txt deleted file mode 100644 index fd109bda490..00000000000 --- a/src/client/federated_topic/ut/CMakeLists.linux-aarch64.txt +++ /dev/null @@ -1,91 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-public-sdk-client-ydb_federated_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_federated_topic-ut PUBLIC - - yutil - cpp-testing-unittest_main - client-ydb_federated_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic - client-ydb_topic-impl - client-ydb_federated_topic-impl -) -target_link_options(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - -ldl - -lrt - -Wl,--no-as-needed - -fPIC - -fPIC - -lpthread - -lrt - -ldl -) -target_sources(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic/ut/basic_usage_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_federated_topic-ut - cpp-malloc-jemalloc -) -vcs_info(ydb-public-sdk-client-ydb_federated_topic-ut) diff --git a/src/client/federated_topic/ut/CMakeLists.linux-x86_64.txt b/src/client/federated_topic/ut/CMakeLists.linux-x86_64.txt deleted file mode 100644 index 20b83b43304..00000000000 --- a/src/client/federated_topic/ut/CMakeLists.linux-x86_64.txt +++ /dev/null @@ -1,93 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-public-sdk-client-ydb_federated_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_federated_topic-ut PUBLIC - - yutil - cpuid_check - cpp-testing-unittest_main - client-ydb_federated_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic - client-ydb_topic-impl - client-ydb_federated_topic-impl -) -target_link_options(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - -ldl - -lrt - -Wl,--no-as-needed - -fPIC - -fPIC - -lpthread - -lrt - -ldl -) -target_sources(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic/ut/basic_usage_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_federated_topic-ut - cpp-malloc-tcmalloc - libs-tcmalloc-no_percpu_cache -) -vcs_info(ydb-public-sdk-client-ydb_federated_topic-ut) diff --git a/src/client/federated_topic/ut/CMakeLists.txt b/src/client/federated_topic/ut/CMakeLists.txt deleted file mode 100644 index 4fbe9853d99..00000000000 --- a/src/client/federated_topic/ut/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.linux-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") - include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") - include(CMakeLists.darwin-arm64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") - include(CMakeLists.windows-x86_64.txt) -endif() diff --git a/src/client/federated_topic/ut/CMakeLists.windows-x86_64.txt b/src/client/federated_topic/ut/CMakeLists.windows-x86_64.txt deleted file mode 100644 index f3d9ab2023b..00000000000 --- a/src/client/federated_topic/ut/CMakeLists.windows-x86_64.txt +++ /dev/null @@ -1,81 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -add_executable(ydb-public-sdk-client-ydb_federated_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_federated_topic-ut PUBLIC - yutil - cpuid_check - cpp-testing-unittest_main - client-ydb_federated_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic - client-ydb_topic-impl - client-ydb_federated_topic-impl -) -target_sources(ydb-public-sdk-client-ydb_federated_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_federated_topic/ut/basic_usage_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_federated_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_federated_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_federated_topic-ut - system_allocator -) -vcs_info(ydb-public-sdk-client-ydb_federated_topic-ut) diff --git a/src/client/federated_topic/ut/basic_usage_ut.cpp b/src/client/federated_topic/ut/basic_usage_ut.cpp deleted file mode 100644 index 1a29a8900ea..00000000000 --- a/src/client/federated_topic/ut/basic_usage_ut.cpp +++ /dev/null @@ -1,592 +0,0 @@ -#include - -#include - -#include - -#include -#include - -#include -#include - -#include -#include -#include -#include - -#include - -namespace NYdb::NFederatedTopic::NTests { - -Y_UNIT_TEST_SUITE(BasicUsage) { - - Y_UNIT_TEST(GetAllStartPartitionSessions) { - size_t partitionsCount = 5; - - auto setup = std::make_shared( - TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2, partitionsCount); - - setup->Start(true, true); - - TFederationDiscoveryServiceMock fdsMock; - fdsMock.Port = setup->GetGrpcPort(); - - ui16 newServicePort = setup->GetPortManager()->GetPort(4285); - auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); - - std::shared_ptr ReadSession; - - // Create topic client. - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); - cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); - NYdb::TDriver driver(cfg); - NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); - - // Create read session. - NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; - readSettings - .ConsumerName("shared/user") - .MaxMemoryUsageBytes(1_MB) - .AppendTopics(setup->GetTestTopic()); - - ReadSession = topicClient.CreateReadSession(readSettings); - std::cerr << "Session was created" << std::endl; - - ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); - std::optional event = ReadSession->GetEvent(false); - Y_ASSERT(!event); - - auto fdsRequest = fdsMock.GetNextPendingRequest(); - Y_ASSERT(fdsRequest.has_value()); - // TODO check fdsRequest->Req db header - - Ydb::FederationDiscovery::ListFederationDatabasesResponse response; - - auto op = response.mutable_operation(); - op->set_status(Ydb::StatusIds::SUCCESS); - response.mutable_operation()->set_ready(true); - response.mutable_operation()->set_id("12345"); - - Ydb::FederationDiscovery::ListFederationDatabasesResult mockResult; - mockResult.set_control_plane_endpoint("cp.logbroker-federation:2135"); - mockResult.set_self_location("fancy_datacenter"); - auto c1 = mockResult.add_federation_databases(); - c1->set_name("dc1"); - c1->set_path("/Root"); - c1->set_id("account-dc1"); - c1->set_endpoint("localhost:" + ToString(fdsMock.Port)); - c1->set_location("dc1"); - c1->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); - c1->set_weight(1000); - auto c2 = mockResult.add_federation_databases(); - c2->set_name("dc2"); - c2->set_path("/Root"); - c2->set_id("account-dc2"); - c2->set_endpoint("localhost:" + ToString(fdsMock.Port)); - c2->set_location("dc2"); - c2->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); - c2->set_weight(500); - - op->mutable_result()->PackFrom(mockResult); - - fdsRequest->Result.SetValue({std::move(response), grpc::Status::OK}); - - for (size_t i = 0; i < partitionsCount; ++i) { - ReadSession->WaitEvent().Wait(); - // Get event - std::optional event = ReadSession->GetEvent(true/*block - will block if no event received yet*/); - std::cerr << "Got new read session event: " << DebugString(*event) << std::endl; - - auto* startPartitionSessionEvent = std::get_if(&*event); - Y_ASSERT(startPartitionSessionEvent); - startPartitionSessionEvent->Confirm(); - } - - ReadSession->Close(TDuration::MilliSeconds(10)); - } - - Y_UNIT_TEST(WaitEventBlocksBeforeDiscovery) { - auto setup = std::make_shared(TEST_CASE_NAME, false); - setup->Start(true, true); - - TFederationDiscoveryServiceMock fdsMock; - fdsMock.Port = setup->GetGrpcPort(); - - ui16 newServicePort = setup->GetPortManager()->GetPort(4285); - auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); - - std::shared_ptr ReadSession; - - // Create topic client. - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); - cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); - NYdb::TDriver driver(cfg); - NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); - - // Create read session. - NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; - readSettings - .ConsumerName("shared/user") - .MaxMemoryUsageBytes(1_MB) - .AppendTopics(setup->GetTestTopic()); - - std::cerr << "Before ReadSession was created" << std::endl; - ReadSession = topicClient.CreateReadSession(readSettings); - std::cerr << "Session was created" << std::endl; - - auto f = ReadSession->WaitEvent(); - std::cerr << "Returned from WaitEvent" << std::endl; - // observer asyncInit should respect client/session timeouts - UNIT_ASSERT(!f.Wait(TDuration::Seconds(1))); - - std::cerr << "Session blocked successfully" << std::endl; - - UNIT_ASSERT(ReadSession->Close(TDuration::MilliSeconds(10))); - std::cerr << "Session closed gracefully" << std::endl; - } - - Y_UNIT_TEST(RetryDiscoveryWithCancel) { - // TODO register requests in mock, compare time for retries, reschedules - - auto setup = std::make_shared(TEST_CASE_NAME, false); - setup->Start(true, true); - - TFederationDiscoveryServiceMock fdsMock; - fdsMock.Port = setup->GetGrpcPort(); - - ui16 newServicePort = setup->GetPortManager()->GetPort(4285); - auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); - - std::shared_ptr ReadSession; - - // Create topic client. - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); - cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); - NYdb::TDriver driver(cfg); - auto clientSettings = TFederatedTopicClientSettings() - .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( - TDuration::Seconds(10), - TDuration::Seconds(10) - )); - NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); - - bool answerOk = false; - - for (int i = 0; i < 6; ++i) { - std::optional fdsRequest; - do { - Sleep(TDuration::MilliSeconds(50)); - fdsRequest = fdsMock.GetNextPendingRequest(); - - } while (!fdsRequest.has_value()); - - if (answerOk) { - fdsRequest->Result.SetValue(fdsMock.ComposeOkResult()); - } else { - fdsRequest->Result.SetValue({{}, grpc::Status(grpc::StatusCode::UNAVAILABLE, "mock 'unavailable'")}); - } - - answerOk = !answerOk; - } - } - - Y_UNIT_TEST(PropagateSessionClosed) { - auto setup = std::make_shared(TEST_CASE_NAME, false); - setup->Start(true, true); - - TFederationDiscoveryServiceMock fdsMock; - fdsMock.Port = setup->GetGrpcPort(); - - ui16 newServicePort = setup->GetPortManager()->GetPort(4285); - auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); - - std::shared_ptr ReadSession; - - // Create topic client. - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); - cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); - NYdb::TDriver driver(cfg); - auto clientSettings = TFederatedTopicClientSettings() - .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( - TDuration::Seconds(10), - TDuration::Seconds(10) - )); - NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); - - - // Create read session. - NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; - readSettings - .ConsumerName("shared/user") - .MaxMemoryUsageBytes(1_MB) - .AppendTopics(setup->GetTestTopic()); - - ReadSession = topicClient.CreateReadSession(readSettings); - std::cerr << "Session was created" << std::endl; - - Sleep(TDuration::MilliSeconds(50)); - - auto events = ReadSession->GetEvents(false); - UNIT_ASSERT(events.empty()); - - Ydb::FederationDiscovery::ListFederationDatabasesResponse Response; - - auto op = Response.mutable_operation(); - op->set_status(Ydb::StatusIds::SUCCESS); - Response.mutable_operation()->set_ready(true); - Response.mutable_operation()->set_id("12345"); - - Ydb::FederationDiscovery::ListFederationDatabasesResult mockResult; - mockResult.set_control_plane_endpoint("cp.logbroker-federation:2135"); - mockResult.set_self_location("fancy_datacenter"); - auto c1 = mockResult.add_federation_databases(); - c1->set_name("dc1"); - c1->set_path("/Root"); - c1->set_id("account-dc1"); - c1->set_endpoint("localhost:" + ToString(fdsMock.Port)); - c1->set_location("dc1"); - c1->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); - c1->set_weight(1000); - auto c2 = mockResult.add_federation_databases(); - c2->set_name("dc2"); - c2->set_path("/Invalid"); - c2->set_id("account-dc2"); - c2->set_endpoint("localhost:" + ToString(fdsMock.Port)); - c2->set_location("dc2"); - c2->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); - c2->set_weight(500); - - op->mutable_result()->PackFrom(mockResult); - - std::optional fdsRequest; - do { - fdsRequest = fdsMock.GetNextPendingRequest(); - if (!fdsRequest.has_value()) { - Sleep(TDuration::MilliSeconds(50)); - } - } while (!fdsRequest.has_value()); - fdsRequest->Result.SetValue({Response, grpc::Status::OK}); - - NPersQueue::TWriteSessionSettings writeSettings; - writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); - writeSettings.Codec(NPersQueue::ECodec::RAW); - NPersQueue::IExecutor::TPtr executor = new NPersQueue::TSyncExecutor(); - writeSettings.CompressionExecutor(executor); - - auto& client = setup->GetPersQueueClient(); - auto session = client.CreateSimpleBlockingWriteSession(writeSettings); - std::string messageBase = "message----"; - - - ui64 count = 100u; - for (auto i = 0u; i < count; i++) { - auto res = session->Write(messageBase * (200 * 1024) + ToString(i)); - UNIT_ASSERT(res); - - events = ReadSession->GetEvents(true); - UNIT_ASSERT(!events.empty()); - - for (auto& e : events) { - std::cerr << ">>> Got event: " << DebugString(e) << std::endl; - if (auto* dataEvent = std::get_if(&e)) { - dataEvent->Commit(); - } else if (auto* startPartitionSessionEvent = std::get_if(&e)) { - startPartitionSessionEvent->Confirm(); - } else if (auto* stopPartitionSessionEvent = std::get_if(&e)) { - stopPartitionSessionEvent->Confirm(); - } - } - } - - session->Close(); - } - - Y_UNIT_TEST(RecreateObserver) { - // TODO register requests in mock, compare time for retries, reschedules - - auto setup = std::make_shared(TEST_CASE_NAME, false); - setup->Start(true, true); - - TFederationDiscoveryServiceMock fdsMock; - fdsMock.Port = setup->GetGrpcPort(); - - ui16 newServicePort = setup->GetPortManager()->GetPort(4285); - auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); - - std::shared_ptr ReadSession; - - // Create topic client. - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); - cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); - NYdb::TDriver driver(cfg); - auto clientSettings = TFederatedTopicClientSettings() - .RetryPolicy(NTopic::IRetryPolicy::GetNoRetryPolicy()); - NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); - - - // Create read session. - NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; - readSettings - .ConsumerName("shared/user") - .MaxMemoryUsageBytes(1_MB) - .AppendTopics(setup->GetTestTopic()); - - ReadSession = topicClient.CreateReadSession(readSettings); - std::cerr << "Session was created" << std::endl; - - ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); - auto event = ReadSession->GetEvent(false); - UNIT_ASSERT(!event.has_value()); - - - std::optional fdsRequest; - do { - fdsRequest = fdsMock.GetNextPendingRequest(); - if (!fdsRequest.has_value()) { - Sleep(TDuration::MilliSeconds(50)); - } - } while (!fdsRequest.has_value()); - - fdsRequest->Result.SetValue({{}, grpc::Status(grpc::StatusCode::UNAVAILABLE, "mock 'unavailable'")}); - - ReadSession->WaitEvent().Wait(); - event = ReadSession->GetEvent(false); - UNIT_ASSERT(event.has_value()); - std::cerr << ">>> Got event: " << DebugString(*event) << std::endl; - UNIT_ASSERT(std::holds_alternative(*event)); - - auto ReadSession2 = topicClient.CreateReadSession(readSettings); - std::cerr << "Session2 was created" << std::endl; - - ReadSession2->WaitEvent().Wait(TDuration::Seconds(1)); - event = ReadSession2->GetEvent(false); - UNIT_ASSERT(!event.has_value()); - - do { - fdsRequest = fdsMock.GetNextPendingRequest(); - if (!fdsRequest.has_value()) { - Sleep(TDuration::MilliSeconds(50)); - } - } while (!fdsRequest.has_value()); - - fdsRequest->Result.SetValue(fdsMock.ComposeOkResult()); - - event = ReadSession2->GetEvent(true); - UNIT_ASSERT(event.has_value()); - std::cerr << ">>> Got event: " << DebugString(*event) << std::endl; - UNIT_ASSERT(std::holds_alternative(*event)); - - // std::cerr << ">>> Got event: " << DebugString(*event) << std::endl; - // UNIT_ASSERT(std::holds_alternative(*event)); - } - - Y_UNIT_TEST(FallbackToSingleDb) { - auto setup = std::make_shared(TEST_CASE_NAME, false); - - setup->Start(true, true); - - std::shared_ptr ReadSession; - - NYdb::NFederatedTopic::TFederatedTopicClient topicClient(setup->GetDriver()); - - // Create read session. - NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; - readSettings - .ConsumerName("shared/user") - .MaxMemoryUsageBytes(1_MB) - .AppendTopics(setup->GetTestTopic()); - - ReadSession = topicClient.CreateReadSession(readSettings); - std::cerr << "Session was created" << std::endl; - - ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); - std::optional event = ReadSession->GetEvent(false); - Y_ASSERT(event); - std::cerr << "Got new read session event: " << DebugString(*event) << std::endl; - - auto* startPartitionSessionEvent = std::get_if(&*event); - Y_ASSERT(startPartitionSessionEvent); - startPartitionSessionEvent->Confirm(); - - ReadSession->Close(TDuration::MilliSeconds(10)); - } - - Y_UNIT_TEST(SimpleHandlers) { - auto setup = std::make_shared(TEST_CASE_NAME, false); - setup->Start(true, true); - - TFederationDiscoveryServiceMock fdsMock; - fdsMock.Port = setup->GetGrpcPort(); - - ui16 newServicePort = setup->GetPortManager()->GetPort(4285); - auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); - - std::shared_ptr ReadSession; - - // Create topic client. - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); - cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); - NYdb::TDriver driver(cfg); - auto clientSettings = TFederatedTopicClientSettings() - .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( - TDuration::Seconds(10), - TDuration::Seconds(10) - )); - NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); - - ui64 count = 300u; - - std::string messageBase = "message----"; - std::vector sentMessages; - - for (auto i = 0u; i < count; i++) { - // sentMessages.emplace_back(messageBase * (i+1) + ToString(i)); - sentMessages.emplace_back(messageBase * (10 * i + 1)); - } - - NThreading::TPromise checkedPromise = NThreading::NewPromise(); - auto totalReceived = 0u; - - auto f = checkedPromise.GetFuture(); - std::atomic check = 1; - - // Create read session. - NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; - readSettings - .ConsumerName("shared/user") - .MaxMemoryUsageBytes(1_MB) - .AppendTopics(setup->GetTestTopic()); - - readSettings.FederatedEventHandlers_.SimpleDataHandlers([&](TReadSessionEvent::TDataReceivedEvent& ev) mutable { - std::cerr << ">>> event from dataHandler: " << DebugString(ev) << std::endl; - Y_VERIFY_S(check.load() != 0, "check is false"); - auto& messages = ev.GetMessages(); - for (size_t i = 0u; i < messages.size(); ++i) { - auto& message = messages[i]; - UNIT_ASSERT_VALUES_EQUAL(message.GetData(), sentMessages[totalReceived]); - totalReceived++; - } - if (totalReceived == sentMessages.size()) - checkedPromise.SetValue(); - }); - - ReadSession = topicClient.CreateReadSession(readSettings); - std::cerr << ">>> Session was created" << std::endl; - - Sleep(TDuration::MilliSeconds(50)); - - auto events = ReadSession->GetEvents(false); - UNIT_ASSERT(events.empty()); - - std::optional fdsRequest; - do { - fdsRequest = fdsMock.GetNextPendingRequest(); - if (!fdsRequest.has_value()) { - Sleep(TDuration::MilliSeconds(50)); - } - } while (!fdsRequest.has_value()); - - fdsRequest->Result.SetValue(fdsMock.ComposeOkResult()); - - NPersQueue::TWriteSessionSettings writeSettings; - writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); - writeSettings.Codec(NPersQueue::ECodec::RAW); - NPersQueue::IExecutor::TPtr executor = new NPersQueue::TSyncExecutor(); - writeSettings.CompressionExecutor(executor); - - auto& client = setup->GetPersQueueClient(); - auto session = client.CreateSimpleBlockingWriteSession(writeSettings); - - for (auto i = 0u; i < count; i++) { - auto res = session->Write(sentMessages[i]); - UNIT_ASSERT(res); - } - - f.GetValueSync(); - ReadSession->Close(); - check.store(0); - } - - Y_UNIT_TEST(BasicWriteSession) { - auto setup = std::make_shared( - TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2); - - setup->Start(true, true); - - TFederationDiscoveryServiceMock fdsMock; - fdsMock.Port = setup->GetGrpcPort(); - - ui16 newServicePort = setup->GetPortManager()->GetPort(4285); - auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); - - std::shared_ptr WriteSession; - - // Create topic client. - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(TYdbStringBuilder() << "localhost:" << newServicePort); - cfg.SetDatabase("/Root"); - cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); - NYdb::TDriver driver(cfg); - NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); - - // Create write session. - auto writeSettings = NTopic::TWriteSessionSettings() - .Path(setup->GetTestTopic()) - .MessageGroupId("src_id"); - - WriteSession = topicClient.CreateWriteSession(writeSettings); - std::cerr << "Session was created" << std::endl; - - WriteSession->WaitEvent().Wait(TDuration::Seconds(1)); - auto event = WriteSession->GetEvent(false); - Y_ASSERT(event); - std::cerr << "Got new read session event: " << DebugString(*event) << std::endl; - auto* readyToAcceptEvent = std::get_if(&*event); - Y_ASSERT(readyToAcceptEvent); - WriteSession->Write(std::move(readyToAcceptEvent->ContinuationToken), NTopic::TWriteMessage("hello")); - - WriteSession->WaitEvent().Wait(TDuration::Seconds(1)); - event = WriteSession->GetEvent(false); - Y_ASSERT(event); - std::cerr << "Got new read session event: " << DebugString(*event) << std::endl; - - readyToAcceptEvent = std::get_if(&*event); - Y_ASSERT(readyToAcceptEvent); - - std::optional fdsRequest; - do { - fdsRequest = fdsMock.GetNextPendingRequest(); - if (!fdsRequest.has_value()) { - Sleep(TDuration::MilliSeconds(50)); - } - } while (!fdsRequest.has_value()); - - fdsRequest->Result.SetValue(fdsMock.ComposeOkResult()); - - WriteSession->WaitEvent().Wait(TDuration::Seconds(1)); - event = WriteSession->GetEvent(false); - Y_ASSERT(event); - std::cerr << "Got new read session event: " << DebugString(*event) << std::endl; - - auto* acksEvent = std::get_if(&*event); - Y_ASSERT(acksEvent); - - WriteSession->Close(TDuration::MilliSeconds(10)); - } - -} - -} diff --git a/src/client/federated_topic/ut/fds_mock.h b/src/client/federated_topic/ut/fds_mock.h deleted file mode 100644 index 98a70f7c141..00000000000 --- a/src/client/federated_topic/ut/fds_mock.h +++ /dev/null @@ -1,112 +0,0 @@ -#pragma once - -#include -#include - -#include -#include - -namespace NYdb::NFederatedTopic::NTests { - -// ctor gets ---list of response--- optional{response} -// ListFederationDatabases gets 1 element under lock and respond. otherwise -// create 2 queues, for requests and responses -// getrequest() - put request and returns> -// sendresponse() -class TFederationDiscoveryServiceMock: public Ydb::FederationDiscovery::V1::FederationDiscoveryService::Service { -public: - using TRequest = Ydb::FederationDiscovery::ListFederationDatabasesRequest; - using TResponse = Ydb::FederationDiscovery::ListFederationDatabasesResponse; - - struct TGrpcResult { - TResponse Response; - grpc::Status Status; - }; - - struct TManualRequest { - const TRequest* Request; - NThreading::TPromise Result; - }; - -public: - ~TFederationDiscoveryServiceMock() { - while (auto r = GetNextPendingRequest()) { - r->Result.SetValue({}); - } - } - - std::optional GetNextPendingRequest() { - std::optional result; - std::lock_guard guard(Lock); - if (!PendingRequests.empty()) { - result = PendingRequests.front(); - PendingRequests.pop_front(); - } - return result; - } - - virtual grpc::Status ListFederationDatabases(grpc::ServerContext*, - const TRequest* request, - TResponse* response) override { - Y_UNUSED(request); - - auto p = NThreading::NewPromise(); - auto f = p.GetFuture(); - - { - std::lock_guard guard(Lock); - PendingRequests.push_back({request, std::move(p)}); - } - - f.Wait(TDuration::Seconds(35)); - if (f.HasValue()) { - auto result = f.ExtractValueSync(); - std::cerr << ">>> Ready to answer: " << (result.Status.ok() ? "ok" : "err") << std::endl; - *response = std::move(result.Response); - return result.Status; - } - - return grpc::Status(grpc::StatusCode::UNKNOWN, "No response after timeout"); - } - - TGrpcResult ComposeOkResult() { - Ydb::FederationDiscovery::ListFederationDatabasesResponse okResponse; - - auto op = okResponse.mutable_operation(); - op->set_status(Ydb::StatusIds::SUCCESS); - okResponse.mutable_operation()->set_ready(true); - okResponse.mutable_operation()->set_id("12345"); - - Ydb::FederationDiscovery::ListFederationDatabasesResult mockResult; - mockResult.set_control_plane_endpoint("cp.logbroker-federation:2135"); - mockResult.set_self_location("fancy_datacenter"); - auto c1 = mockResult.add_federation_databases(); - c1->set_name("dc1"); - c1->set_path("/Root"); - c1->set_id("account-dc1"); - c1->set_endpoint("localhost:" + ToString(Port)); - c1->set_location("dc1"); - c1->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); - c1->set_weight(1000); - auto c2 = mockResult.add_federation_databases(); - c2->set_name("dc2"); - c2->set_path("/Root"); - c2->set_id("account-dc2"); - c2->set_endpoint("localhost:" + ToString(Port)); - c2->set_location("dc2"); - c2->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); - c2->set_weight(500); - - op->mutable_result()->PackFrom(mockResult); - - return {okResponse, grpc::Status::OK}; - } - - -public: - ui16 Port; - std::deque PendingRequests; - TAdaptiveLock Lock; -}; - -} // namespace NYdb::NFederatedTopic::NTests diff --git a/src/client/iam/common/iam.cpp b/src/client/iam/common/iam.cpp index 4b423545959..ddf1df91f07 100644 --- a/src/client/iam/common/iam.cpp +++ b/src/client/iam/common/iam.cpp @@ -3,8 +3,8 @@ #include #include -#include -#include +#include +#include using namespace NYdbGrpc; using namespace yandex::cloud::iam::v1; diff --git a/src/client/iam_private/iam.cpp b/src/client/iam_private/iam.cpp index a649c36d3ea..2d414567dba 100644 --- a/src/client/iam_private/iam.cpp +++ b/src/client/iam_private/iam.cpp @@ -2,8 +2,8 @@ #include -#include -#include +#include +#include namespace NYdb { diff --git a/src/client/impl/ydb_internal/common/types.h b/src/client/impl/ydb_internal/common/types.h index 4bd7bc35a3b..2191300e212 100644 --- a/src/client/impl/ydb_internal/common/types.h +++ b/src/client/impl/ydb_internal/common/types.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include diff --git a/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.h b/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.h index 638105469d2..847b30dc602 100644 --- a/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.h +++ b/src/client/impl/ydb_internal/db_driver_state/endpoint_pool.h @@ -2,7 +2,7 @@ #include -#include +#include #include #include #include diff --git a/src/client/impl/ydb_internal/grpc_connections/actions.cpp b/src/client/impl/ydb_internal/grpc_connections/actions.cpp index 86f5c8305f1..e20ba4afdef 100644 --- a/src/client/impl/ydb_internal/grpc_connections/actions.cpp +++ b/src/client/impl/ydb_internal/grpc_connections/actions.cpp @@ -2,7 +2,7 @@ #include "actions.h" #include "grpc_connections.h" -#include +#include namespace NYdb { @@ -42,7 +42,7 @@ void TDeferredAction::OnAlarm() { Y_ABORT_UNLESS(Connection_); Ydb::Operations::GetOperationRequest getOperationRequest; - getOperationRequest.set_id(OperationId_); + getOperationRequest.set_id(TStringType{OperationId_}); TRpcRequestSettings settings; settings.PreferredEndpoint = TEndpointKey(Endpoint_, 0); diff --git a/src/client/impl/ydb_internal/grpc_connections/actions.h b/src/client/impl/ydb_internal/grpc_connections/actions.h index 029035068af..756a5ae5a97 100644 --- a/src/client/impl/ydb_internal/grpc_connections/actions.h +++ b/src/client/impl/ydb_internal/grpc_connections/actions.h @@ -2,7 +2,7 @@ #include -#include +#include #include #include #include diff --git a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp index 2028a96111e..d6705e09887 100644 --- a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp +++ b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp @@ -313,7 +313,7 @@ void TGRpcConnectionsImpl::SetGrpcKeepAlive(NYdbGrpc::TGRpcClientConfig& config, TAsyncListEndpointsResult TGRpcConnectionsImpl::GetEndpoints(TDbDriverStatePtr dbState) { Ydb::Discovery::ListEndpointsRequest request; - request.set_database(dbState->Database); + request.set_database(TStringType{dbState->Database}); auto promise = NThreading::NewPromise(); diff --git a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h index d38e82b3f34..0a30f220c3b 100644 --- a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h +++ b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h @@ -6,7 +6,7 @@ #include "actions.h" #include "params.h" -#include +#include #include #include #include @@ -14,7 +14,7 @@ #include #include -#include +#include namespace NYdb { @@ -82,7 +82,7 @@ class TGRpcConnectionsImpl { auto clientConfig = NYdbGrpc::TGRpcClientConfig(dbState->DiscoveryEndpoint); const auto& sslCredentials = dbState->SslCredentials; - clientConfig.SslCredentials = {.pem_root_certs = sslCredentials.CaCert, .pem_private_key = sslCredentials.PrivateKey, .pem_cert_chain = sslCredentials.Cert}; + clientConfig.SslCredentials = {.pem_root_certs = TStringType{sslCredentials.CaCert}, .pem_private_key = TStringType{sslCredentials.PrivateKey}, .pem_cert_chain = TStringType{sslCredentials.Cert}}; clientConfig.EnableSsl = sslCredentials.IsEnabled; clientConfig.MemQuota = MemoryQuota_; diff --git a/src/client/impl/ydb_internal/grpc_connections/params.h b/src/client/impl/ydb_internal/grpc_connections/params.h index b7e496b5f94..5672ec2bf4d 100644 --- a/src/client/impl/ydb_internal/grpc_connections/params.h +++ b/src/client/impl/ydb_internal/grpc_connections/params.h @@ -28,10 +28,10 @@ class IConnectionsParams { virtual bool GetGRpcKeepAlivePermitWithoutCalls() const = 0; virtual TDuration GetSocketIdleTimeout() const = 0; virtual const TLog& GetLog() const = 0; - virtual ui64 GetMemoryQuota() const = 0; - virtual ui64 GetMaxInboundMessageSize() const = 0; - virtual ui64 GetMaxOutboundMessageSize() const = 0; - virtual ui64 GetMaxMessageSize() const = 0; + virtual uint64_t GetMemoryQuota() const = 0; + virtual uint64_t GetMaxInboundMessageSize() const = 0; + virtual uint64_t GetMaxOutboundMessageSize() const = 0; + virtual uint64_t GetMaxMessageSize() const = 0; }; } // namespace NYdb diff --git a/src/client/impl/ydb_internal/kqp_session_common/CMakeLists.txt b/src/client/impl/ydb_internal/kqp_session_common/CMakeLists.txt index ae61c483c19..48eb9fcb9ea 100644 --- a/src/client/impl/ydb_internal/kqp_session_common/CMakeLists.txt +++ b/src/client/impl/ydb_internal/kqp_session_common/CMakeLists.txt @@ -3,6 +3,7 @@ _ydb_sdk_add_library(impl-ydb_internal-kqp_session_common) target_link_libraries(impl-ydb_internal-kqp_session_common PUBLIC yutil threading-future + lib-operation_id-protos client-impl-ydb_endpoints ) diff --git a/src/client/impl/ydb_internal/make_request/make.h b/src/client/impl/ydb_internal/make_request/make.h index ab11ef77d05..1bee7a40893 100644 --- a/src/client/impl/ydb_internal/make_request/make.h +++ b/src/client/impl/ydb_internal/make_request/make.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include diff --git a/src/client/impl/ydb_internal/plain_status/status.h b/src/client/impl/ydb_internal/plain_status/status.h index 03626d94a2b..e1a17cddf1a 100644 --- a/src/client/impl/ydb_internal/plain_status/status.h +++ b/src/client/impl/ydb_internal/plain_status/status.h @@ -5,11 +5,11 @@ #include #include -#include +#include #include -#include +#include namespace NYdb { diff --git a/src/client/impl/ydb_internal/table_helpers/helpers.h b/src/client/impl/ydb_internal/table_helpers/helpers.h index 2749a59dd3a..da444b76864 100644 --- a/src/client/impl/ydb_internal/table_helpers/helpers.h +++ b/src/client/impl/ydb_internal/table_helpers/helpers.h @@ -2,7 +2,7 @@ #include -#include +#include #include namespace NYdb { diff --git a/src/client/impl/ydb_internal/value_helpers/helpers.h b/src/client/impl/ydb_internal/value_helpers/helpers.h index 54cc366e1b4..a2df12e4b98 100644 --- a/src/client/impl/ydb_internal/value_helpers/helpers.h +++ b/src/client/impl/ydb_internal/value_helpers/helpers.h @@ -2,7 +2,7 @@ #include -#include +#include namespace NYdb { diff --git a/src/client/import/import.cpp b/src/client/import/import.cpp index 54cdfd47727..3fa34004823 100644 --- a/src/client/import/import.cpp +++ b/src/client/import/import.cpp @@ -4,9 +4,9 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include -#include +#include +#include +#include #include #include @@ -118,11 +118,11 @@ class TImportClient::TImpl : public TClientImplCommon { TAsyncImportDataResult ImportData(const std::string& table, TData&& data, const TImportYdbDumpDataSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(table); - request.set_data(std::forward(data)); + request.set_path(TStringType{table}); + request.set_data(TStringType{std::forward(data)}); for (const auto& column : settings.Columns_) { - request.mutable_ydb_dump()->add_columns(column); + request.mutable_ydb_dump()->add_columns(TStringType{column}); } return ImportData(std::move(request), settings); @@ -140,20 +140,20 @@ TImportClient::TImportClient(const TDriver& driver, const TCommonClientSettings& TFuture TImportClient::ImportFromS3(const TImportFromS3Settings& settings) { auto request = MakeOperationRequest(settings); - request.mutable_settings()->set_endpoint(settings.Endpoint_); + request.mutable_settings()->set_endpoint(TStringType{settings.Endpoint_}); request.mutable_settings()->set_scheme(TProtoAccessor::GetProto(settings.Scheme_)); - request.mutable_settings()->set_bucket(settings.Bucket_); - request.mutable_settings()->set_access_key(settings.AccessKey_); - request.mutable_settings()->set_secret_key(settings.SecretKey_); + request.mutable_settings()->set_bucket(TStringType{settings.Bucket_}); + request.mutable_settings()->set_access_key(TStringType{settings.AccessKey_}); + request.mutable_settings()->set_secret_key(TStringType{settings.SecretKey_}); for (const auto& item : settings.Item_) { auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); - protoItem.set_source_prefix(item.Src); - protoItem.set_destination_path(item.Dst); + protoItem.set_source_prefix(TStringType{item.Src}); + protoItem.set_destination_path(TStringType{item.Dst}); } if (settings.Description_) { - request.mutable_settings()->set_description(settings.Description_.value()); + request.mutable_settings()->set_description(TStringType{settings.Description_.value()}); } if (settings.NumberOfRetries_) { diff --git a/src/client/monitoring/monitoring.cpp b/src/client/monitoring/monitoring.cpp index 8650dd22cef..51b42a572d1 100644 --- a/src/client/monitoring/monitoring.cpp +++ b/src/client/monitoring/monitoring.cpp @@ -4,8 +4,8 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include +#include +#include #include #include diff --git a/src/client/operation/operation.cpp b/src/client/operation/operation.cpp index e8e922dbe3d..c8cf5231bbe 100644 --- a/src/client/operation/operation.cpp +++ b/src/client/operation/operation.cpp @@ -11,7 +11,7 @@ #include #include -#include +#include #include #include @@ -134,21 +134,21 @@ TOperationClient::TOperationClient(const TDriver& driver, const TCommonClientSet template TFuture TOperationClient::Get(const TOperation::TOperationId& id) { auto request = MakeRequest(); - request.set_id(id.ToString()); + request.set_id(TStringType{id.ToString()}); return Impl_->Get(std::move(request)); } TAsyncStatus TOperationClient::Cancel(const TOperation::TOperationId& id) { auto request = MakeRequest(); - request.set_id(id.ToString()); + request.set_id(TStringType{id.ToString()}); return Impl_->Cancel(std::move(request)); } TAsyncStatus TOperationClient::Forget(const TOperation::TOperationId& id) { auto request = MakeRequest(); - request.set_id(id.ToString()); + request.set_id(TStringType{id.ToString()}); return Impl_->Forget(std::move(request)); } @@ -157,12 +157,12 @@ template TFuture> TOperationClient::List(const std::string& kind, ui64 pageSize, const std::string& pageToken) { auto request = MakeRequest(); - request.set_kind(kind); + request.set_kind(TStringType{kind}); if (pageSize) { request.set_page_size(pageSize); } if (!pageToken.empty()) { - request.set_page_token(pageToken); + request.set_page_token(TStringType{pageToken}); } return Impl_->List(std::move(request)); diff --git a/src/client/params/impl.cpp b/src/client/params/impl.cpp index 2ea5efba483..78d52469dc6 100644 --- a/src/client/params/impl.cpp +++ b/src/client/params/impl.cpp @@ -1,12 +1,12 @@ #include "impl.h" -#include -#include +#include +#include namespace NYdb { -TParams::TImpl::TImpl(::google::protobuf::Map&& paramsMap) { +TParams::TImpl::TImpl(::google::protobuf::Map&& paramsMap) { ParamsMap_.swap(paramsMap); } @@ -36,11 +36,11 @@ std::optional TParams::TImpl::GetValue(const std::string& name) const { return std::optional(); } -::google::protobuf::Map* TParams::TImpl::GetProtoMapPtr() { +::google::protobuf::Map* TParams::TImpl::GetProtoMapPtr() { return &ParamsMap_; } -const ::google::protobuf::Map& TParams::TImpl::GetProtoMap() const { +const ::google::protobuf::Map& TParams::TImpl::GetProtoMap() const { return ParamsMap_; } diff --git a/src/client/params/impl.h b/src/client/params/impl.h index 9c6967701bc..6e4506c4e4e 100644 --- a/src/client/params/impl.h +++ b/src/client/params/impl.h @@ -6,16 +6,16 @@ namespace NYdb { class TParams::TImpl { public: - TImpl(::google::protobuf::Map&& paramsMap); + TImpl(::google::protobuf::Map&& paramsMap); bool Empty() const; std::map GetValues() const; std::optional GetValue(const std::string& name) const; - ::google::protobuf::Map* GetProtoMapPtr(); - const ::google::protobuf::Map& GetProtoMap() const; + ::google::protobuf::Map* GetProtoMapPtr(); + const ::google::protobuf::Map& GetProtoMap() const; private: - ::google::protobuf::Map ParamsMap_; + ::google::protobuf::Map ParamsMap_; }; } // namespace NYdb diff --git a/src/client/params/params.cpp b/src/client/params/params.cpp index 041cf3af77f..00d0d80b7cf 100644 --- a/src/client/params/params.cpp +++ b/src/client/params/params.cpp @@ -2,7 +2,7 @@ #include -#include +#include #include @@ -12,14 +12,14 @@ namespace NYdb { //////////////////////////////////////////////////////////////////////////////// -TParams::TParams(::google::protobuf::Map&& protoMap) +TParams::TParams(::google::protobuf::Map&& protoMap) : Impl_(new TImpl(std::move(protoMap))) {} -::google::protobuf::Map* TParams::GetProtoMapPtr() { +::google::protobuf::Map* TParams::GetProtoMapPtr() { return Impl_->GetProtoMapPtr(); } -const ::google::protobuf::Map& TParams::GetProtoMap() const { +const ::google::protobuf::Map& TParams::GetProtoMap() const { return Impl_->GetProtoMap(); } @@ -41,7 +41,7 @@ class TParamsBuilder::TImpl { public: TImpl() = default; - TImpl(const ::google::protobuf::Map& typeInfo) + TImpl(const ::google::protobuf::Map& typeInfo) : HasTypeInfo_(true) { for (const auto& pair : typeInfo) { @@ -97,7 +97,7 @@ class TParamsBuilder::TImpl { ValueBuildersMap_.clear(); - ::google::protobuf::Map paramsMap; + ::google::protobuf::Map paramsMap; paramsMap.swap(ParamsMap_); return TParams(std::move(paramsMap)); } @@ -123,7 +123,7 @@ class TParamsBuilder::TImpl { private: bool HasTypeInfo_ = false; - ::google::protobuf::Map ParamsMap_; + ::google::protobuf::Map ParamsMap_; std::map ValueBuildersMap_; }; @@ -156,7 +156,7 @@ TParamsBuilder::TParamsBuilder() TParamsBuilder::TParamsBuilder(const std::map& typeInfo) : Impl_(new TImpl(typeInfo)) {} -TParamsBuilder::TParamsBuilder(const ::google::protobuf::Map& typeInfo) +TParamsBuilder::TParamsBuilder(const ::google::protobuf::Map& typeInfo) : Impl_(new TImpl(typeInfo)) {} bool TParamsBuilder::HasTypeInfo() const { diff --git a/src/client/persqueue_public/impl/common.h b/src/client/persqueue_public/impl/common.h index ff957a3153f..3e7d591aa61 100644 --- a/src/client/persqueue_public/impl/common.h +++ b/src/client/persqueue_public/impl/common.h @@ -2,7 +2,7 @@ #include -#include +#include namespace NYdb::NPersQueue { ERetryErrorClass GetRetryErrorClass(EStatus status); diff --git a/src/client/persqueue_public/impl/persqueue_impl.h b/src/client/persqueue_public/impl/persqueue_impl.h index 7e542c45e33..e4f0cbf3844 100644 --- a/src/client/persqueue_public/impl/persqueue_impl.h +++ b/src/client/persqueue_public/impl/persqueue_impl.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include #define INCLUDE_YDB_INTERNAL_H @@ -35,7 +35,7 @@ class TPersQueueClient::TImpl : public TClientImplCommon static void ConvertToProtoReadRule(const TReadRule& readRule, Ydb::PersQueue::V1::TopicSettings::ReadRule& rrProps) { - rrProps.set_consumer_name(readRule.ConsumerName_); + rrProps.set_consumer_name(TStringType{readRule.ConsumerName_}); rrProps.set_important(readRule.Important_); rrProps.set_starting_message_timestamp_ms(readRule.StartingMessageTimestamp_.MilliSeconds()); rrProps.set_version(readRule.Version_); @@ -43,13 +43,13 @@ class TPersQueueClient::TImpl : public TClientImplCommon(codec))); } - rrProps.set_service_type(readRule.ServiceType_); + rrProps.set_service_type(TStringType{readRule.ServiceType_}); } template static TRequest MakePropsCreateOrAlterRequest(const std::string& path, const TSettings& settings) { TRequest request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); Ydb::PersQueue::V1::TopicSettings& props = *request.mutable_settings(); @@ -78,9 +78,9 @@ class TPersQueueClient::TImpl : public TClientImplCommonset_endpoint(settings.RemoteMirrorRule_.value().Endpoint_); - rmr->set_topic_path(settings.RemoteMirrorRule_.value().TopicPath_); - rmr->set_consumer_name(settings.RemoteMirrorRule_.value().ConsumerName_); + rmr->set_endpoint(TStringType{settings.RemoteMirrorRule_.value().Endpoint_}); + rmr->set_topic_path(TStringType{settings.RemoteMirrorRule_.value().TopicPath_}); + rmr->set_consumer_name(TStringType{settings.RemoteMirrorRule_.value().ConsumerName_}); rmr->set_starting_message_timestamp_ms(settings.RemoteMirrorRule_.value().StartingMessageTimestamp_.MilliSeconds()); const auto& credentials = settings.RemoteMirrorRule_.value().Credentials_; switch (credentials.GetMode()) { @@ -104,7 +104,7 @@ class TPersQueueClient::TImpl : public TClientImplCommonset_database(settings.RemoteMirrorRule_.value().Database_); + rmr->set_database(TStringType{settings.RemoteMirrorRule_.value().Database_}); } return request; } @@ -159,7 +159,7 @@ class TPersQueueClient::TImpl : public TClientImplCommon(settings); - request.set_path(path); + request.set_path(TStringType{path}); return RunSimple( std::move(request), @@ -169,7 +169,7 @@ class TPersQueueClient::TImpl : public TClientImplCommon(settings); - request.set_path(path); + request.set_path(TStringType{path}); ConvertToProtoReadRule(settings.ReadRule_, *request.mutable_read_rule()); return RunSimple( std::move(request), @@ -179,9 +179,9 @@ class TPersQueueClient::TImpl : public TClientImplCommon(settings); - request.set_path(path); + request.set_path(TStringType{path}); - request.set_consumer_name(settings.ConsumerName_); + request.set_consumer_name(TStringType{settings.ConsumerName_}); return RunSimple( std::move(request), &Ydb::PersQueue::V1::PersQueueService::Stub::AsyncRemoveReadRule, @@ -191,7 +191,7 @@ class TPersQueueClient::TImpl : public TClientImplCommon(settings); - request.set_path(path); + request.set_path(TStringType{path}); auto promise = NThreading::NewPromise(); diff --git a/src/client/persqueue_public/impl/read_session.cpp b/src/client/persqueue_public/impl/read_session.cpp index 482fd265e2c..dd45ffcc1f0 100644 --- a/src/client/persqueue_public/impl/read_session.cpp +++ b/src/client/persqueue_public/impl/read_session.cpp @@ -14,7 +14,7 @@ namespace NYdb::NPersQueue { static const std::string DRIVER_IS_STOPPING_DESCRIPTION = "Driver is stopping"; -std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, ui64 index) { +std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, uint64_t index) { if (dataReceivedEvent.IsCompressedMessages()) { const auto& msg = dataReceivedEvent.GetCompressedMessages()[index]; return {msg.GetOffset(0), msg.GetOffset(msg.GetBlocksCount() - 1) + 1}; @@ -73,7 +73,7 @@ Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest TReadSession::MakeClus Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest req; for (const TTopicReadSettings& topic : Settings.Topics_) { auto* params = req.add_read_sessions(); - params->set_topic(topic.Path_); + params->set_topic(TStringType{topic.Path_}); params->mutable_all_original(); // set all_original } return req; @@ -230,7 +230,7 @@ void TReadSession::OnClusterDiscovery(const TStatus& status, const Ydb::PersQueu if (!ClusterDiscoveryRetryState) { ClusterDiscoveryRetryState = Settings.RetryPolicy_->CreateRetryState(); } - std::optional retryDelay = ClusterDiscoveryRetryState->GetNextRetryDelay(status.GetStatus()); + auto retryDelay = ClusterDiscoveryRetryState->GetNextRetryDelay(status.GetStatus()); if (retryDelay) { LOG_LAZY(Log, TLOG_INFO, diff --git a/src/client/persqueue_public/impl/write_session_impl.cpp b/src/client/persqueue_public/impl/write_session_impl.cpp index aacbdec3188..54ac517ca0b 100644 --- a/src/client/persqueue_public/impl/write_session_impl.cpp +++ b/src/client/persqueue_public/impl/write_session_impl.cpp @@ -98,11 +98,10 @@ TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStat << ". Description: " << IssuesSingleLineString(status.Issues) ); SessionEstablished = false; - std::optional nextDelay = TDuration::Zero(); if (!RetryState) { RetryState = Settings.RetryPolicy_->CreateRetryState(); } - nextDelay = RetryState->GetNextRetryDelay(status.Status); + auto nextDelay = RetryState->GetNextRetryDelay(status.Status); if (nextDelay) { result.StartDelay = *nextDelay; @@ -153,12 +152,12 @@ void TWriteSessionImpl::DoCdsRequest(TDuration delay) { Ydb::PersQueue::ClusterDiscovery::DiscoverClustersRequest req; auto* params = req.add_write_sessions(); - params->set_topic(Settings.Path_); - params->set_source_id(Settings.MessageGroupId_); + params->set_topic(TStringType{Settings.Path_}); + params->set_source_id(TStringType{Settings.MessageGroupId_}); if (Settings.PartitionGroupId_.has_value()) - params->set_partition_group(*Settings.PartitionGroupId_); + params->set_partition_group(Settings.PartitionGroupId_.value()); if (Settings.PreferredCluster_.has_value()) - params->set_preferred_cluster_name(*Settings.PreferredCluster_); + params->set_preferred_cluster_name(TStringType{Settings.PreferredCluster_.value()}); LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Do schedule cds request after " << delay.MilliSeconds() << " ms\n"); auto cdsRequestCall = [req_=std::move(req), extr=std::move(extractor), connections = std::shared_ptr(Connections), dbState=DbDriverState, settings=Settings]() mutable { @@ -572,13 +571,13 @@ void TWriteSessionImpl::InitImpl() { Ydb::PersQueue::V1::StreamingWriteClientMessage req; auto* init = req.mutable_init_request(); - init->set_topic(Settings.Path_); - init->set_message_group_id(Settings.MessageGroupId_); + init->set_topic(TStringType{Settings.Path_}); + init->set_message_group_id(TStringType{Settings.MessageGroupId_}); if (Settings.PartitionGroupId_) { init->set_partition_group_id(*Settings.PartitionGroupId_); } init->set_max_supported_format_version(0); - init->set_preferred_cluster(PreferredClusterByCDS); + init->set_preferred_cluster(TStringType{PreferredClusterByCDS}); for (const auto& attr : Settings.Meta_.Fields) { (*init->mutable_session_meta())[attr.first] = attr.second; @@ -1167,7 +1166,7 @@ void TWriteSessionImpl::UpdateTokenIfNeededImpl() { if (token == PrevToken) return; UpdateTokenInProgress = true; - updateRequest->set_token(token); + updateRequest->set_token(TStringType{token}); PrevToken = token; LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: updating token"); @@ -1206,7 +1205,7 @@ void TWriteSessionImpl::SendImpl() { writeRequest->add_blocks_message_counts(block.MessageCount); writeRequest->add_blocks_part_numbers(block.PartNumber); writeRequest->add_blocks_uncompressed_sizes(block.OriginalSize); - writeRequest->add_blocks_headers(block.CodecID); + writeRequest->add_blocks_headers(TStringType{block.CodecID}); if (block.Compressed) writeRequest->add_blocks_data(block.Data.data(), block.Data.size()); else { diff --git a/src/client/persqueue_public/include/control_plane.h b/src/client/persqueue_public/include/control_plane.h index c8cc2ec7e35..4dffb0ba843 100644 --- a/src/client/persqueue_public/include/control_plane.h +++ b/src/client/persqueue_public/include/control_plane.h @@ -6,7 +6,7 @@ #include #include -#include +#include #include diff --git a/src/client/persqueue_public/persqueue.h b/src/client/persqueue_public/persqueue.h new file mode 100644 index 00000000000..c4673171ba6 --- /dev/null +++ b/src/client/persqueue_public/persqueue.h @@ -0,0 +1,3 @@ +#pragma once + +#include "include/client.h" diff --git a/src/client/persqueue_public/ut/basic_usage_ut.cpp b/src/client/persqueue_public/ut/basic_usage_ut.cpp new file mode 100644 index 00000000000..f4434fe2105 --- /dev/null +++ b/src/client/persqueue_public/ut/basic_usage_ut.cpp @@ -0,0 +1,513 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using namespace NThreading; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; +using namespace NPersQueue; + +namespace NYdb::NPersQueue::NTests { +// This suite tests basic Logbroker usage scenario: write and read a bunch of messages +Y_UNIT_TEST_SUITE(BasicUsage) { + template + void AssertStreamingMessageCase(const typename TResponse::ResponseCase& expectedCase, const TResponse& actualResponse) { + UNIT_ASSERT_EQUAL_C(expectedCase, actualResponse.GetResponseCase(), "Got unexpected streaming message or error:\n" + actualResponse.DebugString()); + } + + using TWriteCallable = std::function(TString& data, ui64 sequenceNumber, TInstant createdAt)>; + + void WriteAndReadAndCommitRandomMessages(TPersQueueYdbSdkTestSetup* setup, TWriteCallable write, bool disableClusterDiscovery = false) { + auto log = setup->GetLog(); + const TInstant start = TInstant::Now(); + TVector messages; + const ui32 messageCount = 5; + ui32 totalSize = 0; + std::queue> writeFutures; + for (ui32 i = 1; i <= messageCount; ++i) { + const size_t size = (1 + (i%100))*1_KB; + + const auto timestamp = TInstant::MilliSeconds(i); + const auto message = NUnitTest::RandomString(size, std::rand()); + + messages.emplace_back(std::move(message)); + + auto writeFuture = write(messages.back(), i, timestamp); + writeFutures.push(writeFuture); + totalSize += size; + } + + while (!writeFutures.empty()) { + const auto& oldestWriteFuture = writeFutures.front(); + const auto& result = oldestWriteFuture.GetValueSync(); + UNIT_ASSERT_C(result.Ok || result.NoWait, result.ResponseDebugString); + writeFutures.pop(); + } + + log.Write(TLOG_INFO, "All messages are written"); + + std::shared_ptr readSession = setup->GetPersQueueClient().CreateReadSession( + setup->GetReadSessionSettings().DisableClusterDiscovery(disableClusterDiscovery) + ); +// auto isStarted = consumer->Start().ExtractValueSync(); +// AssertStreamingMessageCase(TReadResponse::kInit, isStarted.Response); + + ui32 readMessageCount = 0; + TMaybe committedOffset; + ui32 previousOffset = 0; + bool closed = false; + while ((readMessageCount < messageCount || committedOffset <= previousOffset) && !closed) { + Cerr << "Get event on client\n"; + auto event = *readSession->GetEvent(true); + std::visit(TOverloaded { + [&](TReadSessionEvent::TDataReceivedEvent& event) { + for (auto& message: event.GetMessages()) { + TString sourceId = message.GetMessageGroupId(); + ui32 seqNo = message.GetSeqNo(); + UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo); + ++readMessageCount; + auto offset = message.GetOffset(); + if (readMessageCount == 1) { + UNIT_ASSERT_VALUES_EQUAL(offset, 0); + } else { + UNIT_ASSERT_VALUES_EQUAL(offset, previousOffset + 1); + } + previousOffset = offset; + message.Commit(); + } + }, + [&](TReadSessionEvent::TCommitAcknowledgementEvent& event) { + auto offset = event.GetCommittedOffset(); + log << TLOG_INFO << "Got commit ack with offset " << offset; + if (committedOffset.Defined()) { + UNIT_ASSERT_GT(offset, *committedOffset); + } + committedOffset = offset; + }, + [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) { + //UNIT_FAIL("Test does not support lock sessions yet"); + event.Confirm(); + }, + [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { +// UNIT_FAIL("Test does not support lock sessions yet"); + event.Confirm(); + }, + [&](TReadSessionEvent::TPartitionStreamStatusEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TReadSessionEvent::TPartitionStreamClosedEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TSessionClosedEvent& event) { + log << TLOG_INFO << "Got close event: " << event.DebugString(); + //UNIT_FAIL("Session closed"); + closed = true; + } + + }, event); + log << TLOG_INFO << "Read message count is " << readMessageCount << " (from " << messageCount + << " in total), committed offset count is " + << (committedOffset.Defined() ? * committedOffset : 0) << " (maximum offset: " + << previousOffset << ")"; + } + + UNIT_ASSERT_VALUES_EQUAL(previousOffset + 1, committedOffset); + UNIT_ASSERT_VALUES_EQUAL(readMessageCount, messageCount); + log.Write(TLOG_INFO, Sprintf("Time took to write and read %u messages, %lu [MiB] in total is %lu [s]", + messageCount, (totalSize / 1_MB), (TInstant::Now() - start).Seconds())); + } + + + void SimpleWriteAndValidateData( + TPersQueueYdbSdkTestSetup* setup, TWriteSessionSettings& writeSettings, ui64 count, + TMaybe shouldCaptureData = Nothing() + ) { + std::shared_ptr readSession; + + auto& client = setup->GetPersQueueClient(); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + TString messageBase = "message-"; + TVector sentMessages; + + for (auto i = 0u; i < count; i++) { + sentMessages.emplace_back(messageBase * (i+1) + ToString(i)); + auto res = session->Write(sentMessages.back()); + UNIT_ASSERT(res); + } + { + auto sessionAdapter = TSimpleWriteSessionTestAdapter( + dynamic_cast(session.get())); + if (shouldCaptureData.Defined()) { + TStringBuilder msg; + msg << "Session has captured " << sessionAdapter.GetAcquiredMessagesCount() + << " messages, capturing was expected: " << *shouldCaptureData << Endl; + UNIT_ASSERT_VALUES_EQUAL_C(sessionAdapter.GetAcquiredMessagesCount() > 0, *shouldCaptureData, msg.c_str()); + } + } + session->Close(); + + auto readSettings = setup->GetReadSessionSettings(); + NThreading::TPromise checkedPromise = NThreading::NewPromise(); + auto totalReceived = 0u; + readSettings.EventHandlers_.SimpleDataHandlers([&](NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) { + auto& messages = ev.GetMessages(); + for (size_t i = 0u; i < messages.size(); ++i) { + auto& message = messages[i]; + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), sentMessages[totalReceived]); + totalReceived++; + } + if (totalReceived == sentMessages.size()) + checkedPromise.SetValue(); + }); + readSession = client.CreateReadSession(readSettings); + checkedPromise.GetFuture().GetValueSync(); + readSession->Close(TDuration::MilliSeconds(10)); + } + + Y_UNIT_TEST(MaxByteSizeEqualZero) { + auto setup = std::make_shared(TEST_CASE_NAME); + TPersQueueClient client(setup->GetDriver()); + + auto writeSettings = TWriteSessionSettings() + .Path(setup->GetTestTopic()) + .MessageGroupId("group_id"); + auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); + UNIT_ASSERT(writeSession->Write("message")); + writeSession->Close(); + + auto readSettings = TReadSessionSettings() + .ConsumerName(setup->GetTestConsumer()) + .AppendTopics(setup->GetTestTopic()); + auto readSession = client.CreateReadSession(readSettings); + + auto event = readSession->GetEvent(true); + UNIT_ASSERT(event.Defined()); + + auto& createPartitionStream = std::get(*event); + createPartitionStream.Confirm(); + + UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvent(true, 0), TContractViolation); + UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvents(true, Nothing(), 0), TContractViolation); + + event = readSession->GetEvent(true, 1); + UNIT_ASSERT(event.Defined()); + + auto& dataReceived = std::get(*event); + dataReceived.Commit(); + } + + Y_UNIT_TEST(WriteAndReadSomeMessagesWithAsyncCompression) { + auto setup = std::make_shared(TEST_CASE_NAME); + TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); + SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true); + } + + Y_UNIT_TEST(WriteAndReadSomeMessagesWithSyncCompression) { + auto setup = std::make_shared(TEST_CASE_NAME); + TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); + IExecutor::TPtr executor = CreateSyncExecutor(); + writeSettings.CompressionExecutor(executor); + // LOGBROKER-7189 + //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false); + SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true); + } + + Y_UNIT_TEST(WriteAndReadSomeMessagesWithNoCompression) { + auto setup = std::make_shared(TEST_CASE_NAME); + TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); + writeSettings.Codec(ECodec::RAW); + // LOGBROKER-7189 + //SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, false); + SimpleWriteAndValidateData(setup.get(), writeSettings, 100u, true); + } + + Y_UNIT_TEST(TWriteSession_WriteAndReadAndCommitRandomMessages) { + auto setup = std::make_shared(TEST_CASE_NAME); + auto log = setup->GetLog(); + TYDBClientEventLoop clientEventLoop{setup}; + + TAutoEvent messagesWrittenToBuffer; + auto clientWrite = [&](TString& message, ui64 sequenceNumber, TInstant createdAt) { + auto promise = NewPromise(); + //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber; + clientEventLoop.MessageBuffer.Enqueue(TAcknowledgableMessage{message, sequenceNumber, createdAt, promise}); + messagesWrittenToBuffer.Signal(); + return promise.GetFuture(); + }; + + WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite)); + } + + Y_UNIT_TEST(TWriteSession_WriteAndReadAndCommitRandomMessagesNoClusterDiscovery) { + auto setup = std::make_shared(TEST_CASE_NAME); + auto log = setup->GetLog(); + TYDBClientEventLoop clientEventLoop{setup}; + + TAutoEvent messagesWrittenToBuffer; + auto clientWrite = [&](TString& message, ui64 sequenceNumber, TInstant createdAt) { + auto promise = NewPromise(); + //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber; + clientEventLoop.MessageBuffer.Enqueue(TAcknowledgableMessage{message, sequenceNumber, createdAt, promise}); + messagesWrittenToBuffer.Signal(); + return promise.GetFuture(); + }; + + WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite), true); + } + + Y_UNIT_TEST(TSimpleWriteSession_AutoSeqNo_BasicUsage) { + auto setup = std::make_shared(TEST_CASE_NAME); + auto& client = setup->GetPersQueueClient(); + auto settings = setup->GetWriteSessionSettings(); + auto simpleSession = client.CreateSimpleBlockingWriteSession(settings); + + TString msg = "message"; + auto clientWrite = [&](TString& message, ui64 seqNo, TInstant createdAt) { + auto promise = NewPromise(); + //log << TLOG_INFO << "Enqueue message with sequence number " << sequenceNumber; + simpleSession->Write(message, seqNo, createdAt); + promise.SetValue(TWriteResult{true}); + return promise.GetFuture(); + }; + WriteAndReadAndCommitRandomMessages(setup.get(), std::move(clientWrite)); + auto res = simpleSession->Close(); + UNIT_ASSERT(res); + UNIT_ASSERT(res); + } + + + + Y_UNIT_TEST(TWriteSession_AutoBatching) { + // ToDo: Re-enable once batching takes more than 1 message at once + return; + auto setup = std::make_shared(TEST_CASE_NAME); + auto& client = setup->GetPersQueueClient(); + auto settings = setup->GetWriteSessionSettings(); + size_t batchSize = 100; + settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time. + settings.BatchFlushSizeBytes(batchSize); + auto writer = client.CreateWriteSession(settings); + TString message = "message"; + size_t totalSize = 0, seqNo = 0; + while (totalSize < batchSize) { + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->Write(std::move(continueToken), message, ++seqNo); + totalSize += message.size(); + } + WaitMessagesAcked(writer, 1, seqNo); + } + + Y_UNIT_TEST(TWriteSession_WriteEncoded) { + auto setup = std::make_shared(TEST_CASE_NAME); + auto& client = setup->GetPersQueueClient(); + auto settings = setup->GetWriteSessionSettings(); + size_t batchSize = 100000000; + settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time. + settings.BatchFlushSizeBytes(batchSize); + auto writer = client.CreateWriteSession(settings); + TString message = "message"; + TString packed; + { + TStringOutput so(packed); + TZLibCompress oss(&so, ZLib::GZip, 6); + oss << message; + } + + Cerr << message << " " << packed << "\n"; + + { + auto event = *writer->GetEvent(true); + UNIT_ASSERT(!writer->WaitEvent().Wait(TDuration::Seconds(1))); + auto ev = writer->WaitEvent(); + UNIT_ASSERT(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->Write(std::move(continueToken), message); + UNIT_ASSERT(ev.Wait(TDuration::Seconds(1))); + } + { + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->Write(std::move(continueToken), ""); + } + { + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->WriteEncoded(std::move(continueToken), packed, ECodec::GZIP, message.size()); + } + + ui32 acks = 0, tokens = 0; + while(acks < 4 || tokens < 2) { + auto event = *writer->GetEvent(true); + if (std::holds_alternative(event)) acks += std::get(event).Acks.size(); + if (std::holds_alternative(event)) { + if (tokens == 0) { + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->WriteEncoded(std::move(continueToken), "", ECodec::RAW, 0); + } + ++tokens; + } + Cerr << "GOT EVENT " << acks << " " << tokens << "\n"; + + } + UNIT_ASSERT(!writer->WaitEvent().Wait(TDuration::Seconds(5))); + + UNIT_ASSERT_VALUES_EQUAL(acks, 4); + UNIT_ASSERT_VALUES_EQUAL(tokens, 2); + + std::shared_ptr readSession = setup->GetPersQueueClient().CreateReadSession( + setup->GetReadSessionSettings().DisableClusterDiscovery(true) + ); + ui32 readMessageCount = 0; + while (readMessageCount < 4) { + Cerr << "Get event on client\n"; + auto event = *readSession->GetEvent(true); + std::visit(TOverloaded { + [&](TReadSessionEvent::TDataReceivedEvent& event) { + for (auto& message: event.GetMessages()) { + TString sourceId = message.GetMessageGroupId(); + ui32 seqNo = message.GetSeqNo(); + UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo); + ++readMessageCount; + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), (seqNo % 2) == 1 ? "message" : ""); + } + }, + [&](TReadSessionEvent::TCommitAcknowledgementEvent&) { + UNIT_FAIL("no commits in test"); + }, + [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TPartitionStreamStatusEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TReadSessionEvent::TPartitionStreamClosedEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TSessionClosedEvent&) { + UNIT_FAIL("Session closed"); + } + + }, event); + } + } + + + Y_UNIT_TEST(TWriteSession_BatchingProducesContinueTokens) { + // ToDo: Re-enable once batching takes more than 1 message at once + return; + auto setup = std::make_shared(TEST_CASE_NAME); + auto& client = setup->GetPersQueueClient(); + auto settings = setup->GetWriteSessionSettings(); + auto batchInterval = TDuration::Seconds(10); + settings.BatchFlushInterval(batchInterval); // Batch on size, not on time. + settings.BatchFlushSizeBytes(9999999999); + settings.MaxMemoryUsage(99); + auto writer = client.CreateWriteSession(settings); + TString message = "0123456789"; + auto seqNo = 10u; + + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->Write(std::move(continueToken), message * 5, ++seqNo); + auto eventF = writer->WaitEvent(); + eventF.Wait(batchInterval / 2); + UNIT_ASSERT(!eventF.HasValue()); + while (true) { + event = *writer->GetEvent(true); + if (std::holds_alternative(event)) { + break; + } else { + UNIT_ASSERT(std::holds_alternative(event)); + } + } + writer->Close(); + } + + class TBrokenCredentialsProvider : public ICredentialsProvider { + public: + TBrokenCredentialsProvider() {} + virtual ~TBrokenCredentialsProvider() {} + TStringType GetAuthInfo() const { + ythrow yexception() << "exception during creation"; + return ""; + } + bool IsValid() const { return true; } + }; + + class TBrokenCredentialsProviderFactory : public ICredentialsProviderFactory { + public: + TBrokenCredentialsProviderFactory() {} + + virtual ~TBrokenCredentialsProviderFactory() {} + virtual TCredentialsProviderPtr CreateProvider() const { + return std::make_shared(); + } + + virtual TStringType GetClientIdentity() const { + return "abacaba"; + } + }; + + Y_UNIT_TEST(BrokenCredentialsProvider) { + + std::shared_ptr brokenCredentialsProviderFactory; + brokenCredentialsProviderFactory.reset(new TBrokenCredentialsProviderFactory{}); + auto setup = std::make_shared(TEST_CASE_NAME); + auto client = TPersQueueClient(setup->GetDriver(), TPersQueueClientSettings().CredentialsProviderFactory(brokenCredentialsProviderFactory)); + auto settings = setup->GetReadSessionSettings(); + settings.DisableClusterDiscovery(true) + .RetryPolicy(IRetryPolicy::GetNoRetryPolicy()); + std::shared_ptr readSession = client.CreateReadSession(settings); + + Cerr << "Get event on client\n"; + auto event = *readSession->GetEvent(true); + std::visit(TOverloaded { + [&](TReadSessionEvent::TDataReceivedEvent&) { + UNIT_ASSERT(false); + }, + [&](TReadSessionEvent::TCommitAcknowledgementEvent&) { + UNIT_ASSERT(false); + }, + [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) { + UNIT_ASSERT(false); + event.Confirm(); + }, + [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { + UNIT_ASSERT(false); + event.Confirm(); + }, + [&](TReadSessionEvent::TPartitionStreamStatusEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TReadSessionEvent::TPartitionStreamClosedEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TSessionClosedEvent& event) { + Cerr << "Got close event: " << event.DebugString(); + } + + }, event); + + } +} +} // namespace NYdb::NPersQueue::NTests diff --git a/src/client/persqueue_public/ut/common_ut.cpp b/src/client/persqueue_public/ut/common_ut.cpp new file mode 100644 index 00000000000..f970073f5f2 --- /dev/null +++ b/src/client/persqueue_public/ut/common_ut.cpp @@ -0,0 +1,23 @@ +#include + +#include + +using NYdb::NTopic::ApplyClusterEndpoint; + +Y_UNIT_TEST_SUITE(ApplyClusterEndpointTest) { + Y_UNIT_TEST(NoPorts) { + UNIT_ASSERT_STRINGS_EQUAL(ApplyClusterEndpoint("driver", "cluster"), "cluster"); + UNIT_ASSERT_STRINGS_EQUAL(ApplyClusterEndpoint("driver", "clus:ter"), "clus:ter"); + UNIT_ASSERT_STRINGS_EQUAL(ApplyClusterEndpoint("dri:ver", "cluster"), "cluster"); + } + + Y_UNIT_TEST(PortFromCds) { + UNIT_ASSERT_STRINGS_EQUAL(ApplyClusterEndpoint("driver", "cluster:80"), "cluster:80"); + UNIT_ASSERT_STRINGS_EQUAL(ApplyClusterEndpoint("driver:75", "cluster:80"), "cluster:80"); + } + + Y_UNIT_TEST(PortFromDriver) { + UNIT_ASSERT_STRINGS_EQUAL(ApplyClusterEndpoint("driver:45", "cluster"), "cluster:45"); + UNIT_ASSERT_STRINGS_EQUAL(ApplyClusterEndpoint("driver:75", "cluster:8A0"), "[cluster:8A0]:75"); + } +} diff --git a/src/client/persqueue_public/ut/compress_executor_ut.cpp b/src/client/persqueue_public/ut/compress_executor_ut.cpp new file mode 100644 index 00000000000..9c8b20fa07b --- /dev/null +++ b/src/client/persqueue_public/ut/compress_executor_ut.cpp @@ -0,0 +1,104 @@ +#include + +namespace NYdb::NPersQueue::NTests { + +Y_UNIT_TEST_SUITE(CompressExecutor) { + Y_UNIT_TEST(TestReorderedExecutor) { + auto queue = std::make_shared>(); + TYdbPqWriterTestHelper helper(TEST_CASE_NAME, queue); + + auto f1 = helper.Write(); + auto f2 = helper.Write(); + f1.Wait(TDuration::Seconds(1)); + UNIT_ASSERT(!f1.HasValue()); + UNIT_ASSERT(!f2.HasValue()); + + queue->Enqueue(2); + f2.Wait(TDuration::Seconds(1)); + UNIT_ASSERT(!f1.HasValue()); + UNIT_ASSERT(!f2.HasValue()); + queue->Enqueue(1); + Cerr << "Waiting for writes to complete.\n"; + f1.Wait(); + f2.Wait(); + + } + Y_UNIT_TEST(TestExecutorMemUsage) { + auto queue = std::make_shared>(); + auto setup = std::make_shared(TEST_CASE_NAME); + auto executor = MakeIntrusive(queue); + auto config = setup->GetWriteSessionSettings(); + auto memUsageLimit = 1_KB; + config.MaxMemoryUsage(memUsageLimit); + config.CompressionExecutor(executor); + auto retryPolicy = std::make_shared(); + + config.RetryPolicy(retryPolicy); + + auto writer = setup->GetPersQueueClient().CreateWriteSession(config); + + TMaybe continueToken; + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + continueToken = std::move(std::get(event).ContinuationToken); + + auto waitEventFuture = writer->WaitEvent(); + if (waitEventFuture.HasValue()) { + auto event = *writer->GetEvent(true); + if(std::holds_alternative(event)) { + Y_ABORT("ANother ready to accept!"); + } + if(std::holds_alternative(event)) { + Cerr << "Session closed: " << std::get(event).DebugString() << "\n"; + Y_ABORT(""); + } + } + UNIT_ASSERT(!waitEventFuture.HasValue()); + + TStringBuilder msgBuilder; + while (msgBuilder.size() < 100_KB) { + msgBuilder << "0123456789abcdefghijk"; + } + const ui64 COMPRESSED_SIZE = 305; + TString message = TString(msgBuilder); + ui64 seqNo = 1; + auto doWrite = [&] { + writer->Write(std::move(*continueToken), message, seqNo); + ++seqNo; + }; + doWrite(); + waitEventFuture = writer->WaitEvent(); + waitEventFuture.Wait(TDuration::Seconds(3)); + UNIT_ASSERT(!waitEventFuture.HasValue()); + queue->Enqueue(1); + event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + continueToken = std::move(std::get(event).ContinuationToken); + event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + + Cerr << "===Will now kick tablets\n"; + setup->KickTablets(); + retryPolicy->ExpectBreakDown(); + retryPolicy->WaitForRetriesSync(1); + ui64 currentUsageEstimate = 0; + while (currentUsageEstimate <= memUsageLimit) { + doWrite(); + queue->Enqueue(seqNo - 1); + currentUsageEstimate += COMPRESSED_SIZE; + if (currentUsageEstimate <= memUsageLimit) { + event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + continueToken = std::move(std::get(event).ContinuationToken); + } + } + Cerr << "============ UT - Wait event...\n"; + waitEventFuture = writer->WaitEvent(); + waitEventFuture.Wait(TDuration::Seconds(3)); + UNIT_ASSERT(!waitEventFuture.HasValue()); + + writer = nullptr; + retryPolicy = nullptr; + } +} +}; diff --git a/src/client/persqueue_public/ut/compression_ut.cpp b/src/client/persqueue_public/ut/compression_ut.cpp new file mode 100644 index 00000000000..8d86ba9f645 --- /dev/null +++ b/src/client/persqueue_public/ut/compression_ut.cpp @@ -0,0 +1,165 @@ +#include + +namespace NYdb::NPersQueue::NTests { + +Y_UNIT_TEST_SUITE(Compression) { + TVector GetTestMessages(ECodec codec = ECodec::RAW) { + static const TVector> TEST_MESSAGES = { + { + {ECodec::RAW, "Alice and Bob"}, + {ECodec::GZIP, TString("\x1F\x8B\x08\x0\x0\x0\x0\x0\x0\x3\x73\xCC\xC9\x4C\x4E\x55\x48\xCC\x4B\x51\x70\xCA\x4F\x2\x0\x2C\xE7\x84\x5D\x0D\x0\x0\x0", 33)}, + {ECodec::ZSTD, TString("\x28\xB5\x2F\xFD\x0\x58\x69\x0\x0\x41\x6C\x69\x63\x65\x20\x61\x6E\x64\x20\x42\x6F\x62", 22)} + }, + { + {ECodec::RAW, "Yandex.Cloud"}, + {ECodec::GZIP, TString("\x1F\x8B\x8\x0\x0\x0\x0\x0\x0\x3\x8B\x4C\xCC\x4B\x49\xAD\xD0\x73\xCE\xC9\x2F\x4D\x1\x0\x79\x91\x69\xCA\xC\x0\x0\x0", 32)}, + {ECodec::ZSTD, TString("\x28\xB5\x2F\xFD\x0\x58\x61\x0\x0\x59\x61\x6E\x64\x65\x78\x2E\x43\x6C\x6F\x75\x64", 21)} + } + }; + TVector messages; + for (auto& m : TEST_MESSAGES) { + messages.push_back(m.at(codec)); + } + return messages; + } + + void AlterTopic(TPersQueueYdbSdkTestSetup& setup) { + std::shared_ptr channel; + std::unique_ptr stub; + + { + channel = grpc::CreateChannel("localhost:" + ToString(setup.GetGrpcPort()), grpc::InsecureChannelCredentials()); + stub = Ydb::PersQueue::V1::PersQueueService::NewStub(channel); + } + + Ydb::PersQueue::V1::AlterTopicRequest request; + request.set_path(TStringBuilder() << "/Root/PQ/rt3.dc1--" << setup.GetTestTopic()); + auto props = request.mutable_settings(); + props->set_partitions_count(1); + props->set_supported_format(Ydb::PersQueue::V1::TopicSettings::FORMAT_BASE); + props->set_retention_period_ms(TDuration::Days(1).MilliSeconds()); + props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_RAW); + props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_GZIP); + props->add_supported_codecs(Ydb::PersQueue::V1::CODEC_ZSTD); + auto rr = props->add_read_rules(); + rr->set_consumer_name(setup.GetTestConsumer()); + rr->set_supported_format(Ydb::PersQueue::V1::TopicSettings::Format(1)); + rr->add_supported_codecs(Ydb::PersQueue::V1::CODEC_RAW); + rr->add_supported_codecs(Ydb::PersQueue::V1::CODEC_GZIP); + rr->add_supported_codecs(Ydb::PersQueue::V1::CODEC_ZSTD); + rr->set_version(1); + + Ydb::PersQueue::V1::AlterTopicResponse response; + grpc::ClientContext rcontext; + auto status = stub->AlterTopic(&rcontext, request, &response); + UNIT_ASSERT(status.ok()); + Ydb::PersQueue::V1::AlterTopicResult result; + response.operation().result().UnpackTo(&result); + Cerr << "Alter topic response: " << response << "\nAlter result: " << result << "\n"; + UNIT_ASSERT_VALUES_EQUAL(response.operation().status(), Ydb::StatusIds::SUCCESS); + } + + void Write(TPersQueueYdbSdkTestSetup& setup, const TVector& messages, ECodec codec) { + auto& client = setup.GetPersQueueClient(); + TWriteSessionSettings writeSettings = setup.GetWriteSessionSettings(); + writeSettings.Codec(codec); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + + for (auto& message : messages) { + auto result = session->Write(message); + UNIT_ASSERT(result); + } + session->Close(); + } + + void Read( + TPersQueueYdbSdkTestSetup& setup, + const TVector messages, + const TVector codecs, + bool decompress + ) { + Cerr << Endl << "Start read" << Endl << Endl; + auto readSettings = setup.GetReadSessionSettings(); + readSettings.Decompress(decompress); + NThreading::TPromise checkedPromise = NThreading::NewPromise(); + auto totalReceived = 0u; + readSettings.EventHandlers_.SimpleDataHandlers( + [&](const NYdb::NPersQueue::TReadSessionEvent::TDataReceivedEvent& ev) { + Cerr << Endl << Endl << Endl << "Got messages" << Endl << Endl; + if (decompress) { + UNIT_ASSERT_NO_EXCEPTION(ev.GetMessages()); + UNIT_ASSERT_EXCEPTION(ev.GetCompressedMessages(), yexception); + for (auto& message : ev.GetMessages()) { + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), messages[totalReceived]); + ++totalReceived; + } + } else { + UNIT_ASSERT_EXCEPTION(ev.GetMessages(), yexception); + UNIT_ASSERT_NO_EXCEPTION(ev.GetCompressedMessages()); + for (auto& message : ev.GetCompressedMessages()) { + UNIT_ASSERT_VALUES_EQUAL(message.GetCodec(), codecs[totalReceived]); + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), messages[totalReceived]); + ++totalReceived; + } + } + Cerr << Endl << "totalReceived: " << totalReceived << " wait: " << messages.size() << Endl << Endl; + if (totalReceived == messages.size()) + checkedPromise.SetValue(); + } + ); + auto& client = setup.GetPersQueueClient(); + auto readSession = client.CreateReadSession(readSettings); + checkedPromise.GetFuture().GetValueSync(); + } + + void WriteWithOneCodec(TPersQueueYdbSdkTestSetup& setup, ECodec codec) { + AlterTopic(setup); // add zstd support + + auto messages = GetTestMessages(); + Write(setup, messages, codec); + Read(setup, messages, TVector(messages.size(), ECodec::RAW), true); + Read(setup, GetTestMessages(codec), TVector(messages.size(), codec), false); + } + + Y_UNIT_TEST(WriteRAW) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + WriteWithOneCodec(setup, ECodec::RAW); + } + + Y_UNIT_TEST(WriteGZIP) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + WriteWithOneCodec(setup, ECodec::GZIP); + } + + Y_UNIT_TEST(WriteZSTD) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + WriteWithOneCodec(setup, ECodec::ZSTD); + } + + Y_UNIT_TEST(WriteWithMixedCodecs) { + TPersQueueYdbSdkTestSetup setup(TEST_CASE_NAME); + AlterTopic(setup); // add zstd support + + auto messages = GetTestMessages(); + TVector originalMessages; + TVector targetMessages; + TVector targetCodecs; + + auto addToTarget = [&](ECodec codec) { + originalMessages.insert(originalMessages.end(), messages.begin(), messages.end()); + for (auto& m : GetTestMessages(codec)) { + targetMessages.push_back(m); + targetCodecs.push_back(codec); + } + Write(setup, messages, codec); + }; + + addToTarget(ECodec::RAW); + addToTarget(ECodec::GZIP); + addToTarget(ECodec::ZSTD); + + Read(setup, originalMessages, TVector(originalMessages.size(), ECodec::RAW), true); + Read(setup, targetMessages, targetCodecs, false); + } +} +}; diff --git a/src/client/persqueue_public/ut/read_session_ut.cpp b/src/client/persqueue_public/ut/read_session_ut.cpp new file mode 100644 index 00000000000..ac573bb2e0d --- /dev/null +++ b/src/client/persqueue_public/ut/read_session_ut.cpp @@ -0,0 +1,1948 @@ +#include + +#define INCLUDE_YDB_INTERNAL_H +#include +#undef INCLUDE_YDB_INTERNAL_H + +#include +#include + +#include +#include +#include + +#include + +#include + +using namespace NYdb; +using namespace NYdb::NPersQueue; +using IExecutor = NYdb::NPersQueue::IExecutor; +using namespace ::testing; // Google mock. + +#define UNIT_ASSERT_EVENT_TYPE(event, type) \ + UNIT_ASSERT_C( \ + std::holds_alternative(event), \ + "Real event got: " << DebugString(event)) \ + /**/ + +#define UNIT_ASSERT_NOT_EVENT_TYPE(event, type) \ + UNIT_ASSERT_C( \ + !std::holds_alternative(event), \ + "Real event got: " << DebugString(event)) \ + /**/ + +TString Compress(const TString& sourceData, Ydb::PersQueue::V1::Codec codec = Ydb::PersQueue::V1::CODEC_GZIP) { + if (codec == Ydb::PersQueue::V1::CODEC_RAW || codec == Ydb::PersQueue::V1::CODEC_UNSPECIFIED) { + return sourceData; + } + + TString compressed; + TStringOutput out(compressed); + THolder coder; + switch (codec) { + case Ydb::PersQueue::V1::CODEC_GZIP: + coder = MakeHolder(&out, ZLib::GZip); + break; + case Ydb::PersQueue::V1::CODEC_LZOP: + throw yexception() << "LZO codec is disabled"; + break; + case Ydb::PersQueue::V1::CODEC_ZSTD: + coder = MakeHolder(&out); + break; + default: + UNIT_ASSERT(false); + } + coder->Write(sourceData); + coder->Finish(); + return compressed; +} + +template +struct TMockProcessorFactory : public ISessionConnectionProcessorFactory { + using IFactory = ISessionConnectionProcessorFactory; + + virtual ~TMockProcessorFactory() { + Wait(); + } + + void CreateProcessor( // ISessionConnectionProcessorFactory method. + typename IFactory::TConnectedCallback callback, + const TRpcRequestSettings& requestSettings, + NYdbGrpc::IQueueClientContextPtr connectContext, + TDuration connectTimeout, + NYdbGrpc::IQueueClientContextPtr connectTimeoutContext, + typename IFactory::TConnectTimeoutCallback connectTimeoutCallback, + TDuration connectDelay, + NYdbGrpc::IQueueClientContextPtr connectDelayOperationContext) override + { + UNIT_ASSERT_C(!ConnectedCallback, "Only one connect at a time is expected"); + UNIT_ASSERT_C(!ConnectTimeoutCallback, "Only one connect at a time is expected"); + ConnectedCallback = callback; + ConnectTimeoutCallback = connectTimeoutCallback; + + Y_UNUSED(requestSettings); + UNIT_ASSERT(connectContext); + UNIT_ASSERT(connectTimeout); + UNIT_ASSERT(connectTimeoutContext); + UNIT_ASSERT(connectTimeoutCallback); + UNIT_ASSERT(!connectDelay || connectDelayOperationContext); + + OnCreateProcessor(++CreateCallsCount); + } + + MOCK_METHOD(void, ValidateConnectTimeout, (TDuration), ()); + + // Handler is called in CreateProcessor() method after parameter validation. + MOCK_METHOD(void, OnCreateProcessor, (size_t callNumber)); // 1-based + + // Actions to use in OnCreateProcessor handler: + void CreateProcessor(typename IFactory::IProcessor::TPtr processor) { // Success. + UNIT_ASSERT(ConnectedCallback); + auto cb = std::move(ConnectedCallback); + ConnectedCallback = nullptr; + ConnectTimeoutCallback = nullptr; + with_lock (Lock) { + CallbackFutures.push(std::async(std::launch::async, std::move(cb), TPlainStatus(), processor)); + } + } + + void FailCreation(EStatus status = EStatus::INTERNAL_ERROR, const TString& message = {}) { // Fail. + UNIT_ASSERT(ConnectedCallback); + auto cb = std::move(ConnectedCallback); + ConnectedCallback = nullptr; + ConnectTimeoutCallback = nullptr; + with_lock (Lock) { + CallbackFutures.push(std::async(std::launch::async, std::move(cb), TPlainStatus(status, message), nullptr)); + } + } + + void Timeout() { // Timeout. + UNIT_ASSERT(ConnectTimeoutCallback); + auto cb = std::move(ConnectTimeoutCallback); + ConnectedCallback = nullptr; + ConnectTimeoutCallback = nullptr; + with_lock (Lock) { + CallbackFutures.push(std::async(std::launch::async, std::move(cb), true)); + } + } + + void CreateAndThenTimeout(typename IFactory::IProcessor::TPtr processor) { + UNIT_ASSERT(ConnectedCallback); + UNIT_ASSERT(ConnectTimeoutCallback); + auto cb2 = [cbt = std::move(ConnectTimeoutCallback), cb = std::move(ConnectedCallback), processor]() mutable { + cb(TPlainStatus(), std::move(processor)); + cbt(true); + }; + ConnectedCallback = nullptr; + ConnectTimeoutCallback = nullptr; + with_lock (Lock) { + CallbackFutures.push(std::async(std::launch::async, std::move(cb2))); + } + } + + void FailAndThenTimeout(EStatus status = EStatus::INTERNAL_ERROR, const TString& message = {}) { + UNIT_ASSERT(ConnectedCallback); + UNIT_ASSERT(ConnectTimeoutCallback); + auto cb2 = [cbt = std::move(ConnectTimeoutCallback), cb = std::move(ConnectedCallback), status, message]() mutable { + cb(TPlainStatus(status, message), nullptr); + cbt(true); + }; + ConnectedCallback = nullptr; + ConnectTimeoutCallback = nullptr; + with_lock (Lock) { + CallbackFutures.push(std::async(std::launch::async, std::move(cb2))); + } + } + + void TimeoutAndThenCreate(typename IFactory::IProcessor::TPtr processor) { + UNIT_ASSERT(ConnectedCallback); + UNIT_ASSERT(ConnectTimeoutCallback); + auto cb2 = [cbt = std::move(ConnectTimeoutCallback), cb = std::move(ConnectedCallback), processor]() mutable { + cbt(true); + cb(TPlainStatus(), std::move(processor)); + }; + ConnectedCallback = nullptr; + ConnectTimeoutCallback = nullptr; + with_lock (Lock) { + CallbackFutures.push(std::async(std::launch::async, std::move(cb2))); + } + } + + void Wait() { + std::queue> futuresQueue; + with_lock (Lock) { + CallbackFutures.swap(futuresQueue); + } + while (!futuresQueue.empty()) { + futuresQueue.front().wait(); + futuresQueue.pop(); + } + } + + void Validate() { + UNIT_ASSERT(CallbackFutures.empty()); + ConnectedCallback = nullptr; + ConnectTimeoutCallback = nullptr; + } + + std::atomic CreateCallsCount = 0; + +private: + TAdaptiveLock Lock; + typename IFactory::TConnectedCallback ConnectedCallback; + typename IFactory::TConnectTimeoutCallback ConnectTimeoutCallback; + std::queue> CallbackFutures; +}; + +struct TMockReadSessionProcessor : public TMockProcessorFactory::IProcessor { + // Request to read. + struct TClientReadInfo { + TReadCallback Callback; + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage* Dst; + + operator bool() const { + return Dst != nullptr; + } + }; + + // Response from server. + struct TServerReadInfo { + NYdbGrpc::TGrpcStatus Status; + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage Response; + + TServerReadInfo& Failure(grpc::StatusCode status = grpc::StatusCode::UNAVAILABLE, const TString& message = {}, bool internal = false) { + Status.GRpcStatusCode = status; + Status.InternalError = internal; + Status.Msg = message; + return *this; + } + + TServerReadInfo& InitResponse(const TString& sessionId) { + Response.mutable_init_response()->set_session_id(sessionId); + return *this; + } + + TServerReadInfo& CreatePartitionStream(const TString& topic = "TestTopic", const TString& cluster = "TestCluster", const ui64 partition = 1, const ui64 assignId = 1, ui64 readOffset = 0, ui64 endOffset = 0) { + auto* req = Response.mutable_assigned(); + req->mutable_topic()->set_path(topic); + req->set_cluster(cluster); + req->set_partition(partition); + req->set_assign_id(assignId); + req->set_read_offset(readOffset); + req->set_end_offset(endOffset); + return *this; + } + + TServerReadInfo& ReleasePartitionStream(const TString& topic = "TestTopic", const TString& cluster = "TestCluster", const ui64 partition = 1, const ui64 assignId = 1, ui64 commitOffset = 0, bool forceful = false) { + auto* req = Response.mutable_release(); + req->mutable_topic()->set_path(topic); + req->set_cluster(cluster); + req->set_partition(partition); + req->set_assign_id(assignId); + req->set_commit_offset(commitOffset); + req->set_forceful_release(forceful); + return *this; + } + + TServerReadInfo& ForcefulReleasePartitionStream(const TString& topic = "TestTopic", const TString& cluster = "TestCluster", const ui64 partition = 1, const ui64 assignId = 1, ui64 commitOffset = 0) { + return ReleasePartitionStream(topic, cluster, partition, assignId, commitOffset, true); + } + + // Data helpers. + TServerReadInfo& PartitionData(const ui64 cookie, const TString& topic = "TestTopic", const TString& cluster = "TestCluster", const ui64 partition = 1, const ui64 assignId = 1) { + auto* req = Response.mutable_data_batch()->add_partition_data(); + req->mutable_topic()->set_path(topic); + req->set_cluster(cluster); + req->set_partition(partition); + auto* cookieMsg = req->mutable_cookie(); + cookieMsg->set_assign_id(assignId); + cookieMsg->set_partition_cookie(cookie); + return *this; + } + + TServerReadInfo& Batch(const TString& sourceId, TInstant writeTimestamp = TInstant::MilliSeconds(42), const TString& ip = "::1", const std::vector>& extraFields = {}) { + const int lastPartitionData = Response.data_batch().partition_data_size(); + UNIT_ASSERT(lastPartitionData > 0); + auto* partitionData = Response.mutable_data_batch()->mutable_partition_data(lastPartitionData - 1); + auto* req = partitionData->add_batches(); + req->set_source_id(sourceId); + req->set_write_timestamp_ms(writeTimestamp.MilliSeconds()); + req->set_ip(ip); + for (auto&& [k, v] : extraFields) { + auto* kv = req->add_extra_fields(); + kv->set_key(k); + kv->set_value(v); + } + return *this; + } + + TServerReadInfo& Message(ui64 offset, const TString& data, Ydb::PersQueue::V1::Codec codec, ui64 seqNo = 1, TInstant createTimestamp = TInstant::MilliSeconds(42)) { + const int lastPartitionData = Response.data_batch().partition_data_size(); + UNIT_ASSERT(lastPartitionData > 0); + auto* partitionData = Response.mutable_data_batch()->mutable_partition_data(lastPartitionData - 1); + const int lastBatch = partitionData->batches_size(); + UNIT_ASSERT(lastBatch > 0); + auto* batch = partitionData->mutable_batches(lastBatch - 1); + auto* req = batch->add_message_data(); + req->set_offset(offset); + req->set_seq_no(seqNo); + req->set_create_timestamp_ms(createTimestamp.MilliSeconds()); + req->set_data(data); + req->set_codec(codec); + return *this; + } + + TServerReadInfo& CompressMessage(ui64 offset, const TString& sourceData, Ydb::PersQueue::V1::Codec codec = Ydb::PersQueue::V1::CODEC_GZIP, ui64 seqNo = 1, TInstant createTimestamp = TInstant::MilliSeconds(42)) { + return Message(offset, Compress(sourceData, codec), codec, seqNo, createTimestamp); + } + + TServerReadInfo& BrokenCompressMessage(ui64 offset, const TString& sourceData, Ydb::PersQueue::V1::Codec codec = Ydb::PersQueue::V1::CODEC_GZIP, ui64 seqNo = 1, TInstant createTimestamp = TInstant::MilliSeconds(42)) { + return Message(offset, "broken_header_" + Compress(sourceData, codec), codec, seqNo, createTimestamp); + } + + + TServerReadInfo& PartitionStreamStatus(ui64 committedOffset, ui64 endOffset, TInstant writeWatermark, const TString& topic = "TestTopic", const TString& cluster = "TestCluster", const ui64 partition = 1, const ui64 assignId = 1) { + auto* req = Response.mutable_partition_status(); + req->mutable_topic()->set_path(topic); + req->set_cluster(cluster); + req->set_partition(partition); + req->set_assign_id(assignId); + req->set_committed_offset(committedOffset); + req->set_end_offset(endOffset); + req->set_write_watermark_ms(writeWatermark.MilliSeconds()); + return *this; + } + + TServerReadInfo& CommitAcknowledgement(ui64 cookie, ui64 assignId = 1) { + auto* req = Response.mutable_committed(); + auto* cookieInfo = req->add_cookies(); + cookieInfo->set_partition_cookie(cookie); + cookieInfo->set_assign_id(assignId); + return *this; + } + }; + + ~TMockReadSessionProcessor() { + Wait(); + } + + void Cancel() override { + } + + void ReadInitialMetadata(std::unordered_multimap* metadata, TReadCallback callback) override { + Y_UNUSED(metadata); + Y_UNUSED(callback); + UNIT_ASSERT_C(false, "This method is not expected to be called"); + } + + void Finish(TReadCallback callback) override { + Y_UNUSED(callback); + UNIT_ASSERT_C(false, "This method is not expected to be called"); + } + + void AddFinishedCallback(TReadCallback callback) override { + Y_UNUSED(callback); + UNIT_ASSERT_C(false, "This method is not expected to be called"); + } + + void Read(Ydb::PersQueue::V1::MigrationStreamingReadServerMessage* response, TReadCallback callback) override { + with_lock (Lock) { + UNIT_ASSERT(!ActiveRead); + ActiveRead.Callback = std::move(callback); + ActiveRead.Dst = response; + if (!ReadResponses.empty()) { + StartProcessReadImpl(); + } + } + } + + void Write(Ydb::PersQueue::V1::MigrationStreamingReadClientMessage&& request, TWriteCallback callback) override { + UNIT_ASSERT(!callback); // Read session doesn't set callbacks. + switch (request.request_case()) { + case Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::kInitRequest: + OnInitRequest(request.init_request()); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::kRead: + OnReadRequest(request.read()); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::kStartRead: + OnStartReadRequest(request.start_read()); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::kCommit: + OnCommitRequest(request.commit()); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::kReleased: + OnReleasedRequest(request.released()); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::kStatus: + OnStatusRequest(request.status()); + break; + case Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::REQUEST_NOT_SET: + UNIT_ASSERT_C(false, "Invalid request"); + break; + } + } + + MOCK_METHOD(void, OnInitRequest, (const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::InitRequest&), ()); + MOCK_METHOD(void, OnReadRequest, (const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::Read&), ()); + MOCK_METHOD(void, OnStartReadRequest, (const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::StartRead&), ()); + MOCK_METHOD(void, OnCommitRequest, (const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::Commit&), ()); + MOCK_METHOD(void, OnReleasedRequest, (const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::Released&), ()); + MOCK_METHOD(void, OnStatusRequest, (const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::Status&), ()); + + void Wait() { + std::queue> callbackFutures; + with_lock (Lock) { + CallbackFutures.swap(callbackFutures); + } + + while (!callbackFutures.empty()) { + callbackFutures.front().wait(); + callbackFutures.pop(); + } + } + + void Validate() { + with_lock (Lock) { + UNIT_ASSERT(ReadResponses.empty()); + UNIT_ASSERT(CallbackFutures.empty()); + + ActiveRead = TClientReadInfo{}; + } + } + + + void ProcessRead() { + NYdbGrpc::TGrpcStatus status; + TReadCallback callback; + with_lock (Lock) { + *ActiveRead.Dst = ReadResponses.front().Response; + ActiveRead.Dst = nullptr; + status = std::move(ReadResponses.front().Status); + ReadResponses.pop(); + callback = std::move(ActiveRead.Callback); + } + callback(std::move(status)); + } + + void StartProcessReadImpl() { + CallbackFutures.push(std::async(std::launch::async, &TMockReadSessionProcessor::ProcessRead, this)); + } + + void AddServerResponse(TServerReadInfo result) { + bool hasActiveRead = false; + with_lock (Lock) { + ReadResponses.emplace(std::move(result)); + if (ActiveRead) { + hasActiveRead = true; + } + } + if (hasActiveRead) { + ProcessRead(); + } + } + + + TAdaptiveLock Lock; + TClientReadInfo ActiveRead; + std::queue ReadResponses; + std::queue> CallbackFutures; +}; + +// Class for testing read session impl +// with mocks. +class TReadSessionImplTestSetup { +public: + // Types + using IReadSessionConnectionProcessorFactory = ISessionConnectionProcessorFactory; + using TMockProcessorFactory = ::TMockProcessorFactory; + + + struct TFakeContext : public NYdbGrpc::IQueueClientContext { + IQueueClientContextPtr CreateContext() override { + return std::make_shared(); + } + + grpc::CompletionQueue* CompletionQueue() override { + UNIT_ASSERT_C(false, "This method is not expected to be called"); + return nullptr; + } + + bool IsCancelled() const override { + UNIT_ASSERT_C(false, "This method is not expected to be called"); + return false; + } + + bool Cancel() override { + return false; + } + + void SubscribeCancel(std::function) override { + UNIT_ASSERT_C(false, "This method is not expected to be called"); + } + }; + + // Methods + TReadSessionImplTestSetup(); + ~TReadSessionImplTestSetup() noexcept(false); // Performs extra validation and UNIT_ASSERTs + + TSingleClusterReadSessionImpl* GetSession(); + + std::shared_ptr GetEventsQueue(); + ::IExecutor::TPtr GetDefaultExecutor(); + + void SuccessfulInit(bool flag = true); + TPartitionStream::TPtr CreatePartitionStream(const TString& topic = "TestTopic", const TString& cluster = "TestCluster", ui64 partition = 1, ui64 assignId = 1); + + // Assertions. + void AssertNoEvents(); + +public: + // Members + TReadSessionSettings Settings; + TString ClusterName = "cluster"; + TLog Log = CreateLogBackend("cerr"); + std::shared_ptr EventsQueue; + std::shared_ptr FakeContext = std::make_shared(); + std::shared_ptr MockProcessorFactory = std::make_shared(); + TIntrusivePtr MockProcessor = MakeIntrusive(); + ui64 PartitionIdStart = 1; + ui64 PartitionIdStep = 1; + typename TSingleClusterReadSessionImpl::TPtr Session; + std::shared_ptr> CbContext; + std::shared_ptr ThreadPool; + ::IExecutor::TPtr DefaultExecutor; + std::shared_ptr>> ProvidedCodecs = std::make_shared>>(); +}; + +class TReorderingExecutor : public ::IExecutor { +public: + TReorderingExecutor(size_t cycleCount = 2, ::IExecutor::TPtr executor = CreateThreadPoolExecutor(1)) + : CycleCount(cycleCount) + , Executor(std::move(executor)) + { + } + + bool IsAsync() const override { + return Executor->IsAsync(); + } + + void Post(TFunction&& f) override { + with_lock (Lock) { + Cerr << "Post function" << Endl; + ++TasksAdded; + if (Functions.empty()) { + Functions.reserve(CycleCount); + } + Functions.emplace_back(std::move(f)); + if (Functions.size() == CycleCount) { + Executor->Post([functions = std::move(Functions)]() { + for (auto i = functions.rbegin(), end = functions.rend(); i != end; ++i) { + (*i)(); + } + }); + Functions.clear(); + } + } + } + + void DoStart() override { + Executor->Start(); + } + + size_t GetTasksAdded() { + with_lock (Lock) { + return TasksAdded; + } + } + +private: + TAdaptiveLock Lock; + size_t CycleCount; + size_t TasksAdded = 0; + ::IExecutor::TPtr Executor; + std::vector Functions; +}; + +class TSynchronousExecutor : public ::IExecutor { + bool IsAsync() const override { + return false; + } + + void Post(TFunction&& f) override { + f(); + } + + void DoStart() override { + } +}; + +extern TLogFormatter NYdb::GetPrefixLogFormatter(const TString& prefix); // Defined in ydb.cpp. + +TReadSessionImplTestSetup::TReadSessionImplTestSetup() { + Settings + .AppendTopics({"TestTopic"}) + .ConsumerName("TestConsumer") + .RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetFixedIntervalPolicy(TDuration::MilliSeconds(10))) + .Counters(MakeIntrusive(MakeIntrusive<::NMonitoring::TDynamicCounters>())); + + Log.SetFormatter(GetPrefixLogFormatter("")); + + (*ProvidedCodecs)[ECodec::GZIP] = MakeHolder(); + (*ProvidedCodecs)[ECodec::LZOP] = MakeHolder(); + (*ProvidedCodecs)[ECodec::ZSTD] = MakeHolder(); +} + +TReadSessionImplTestSetup::~TReadSessionImplTestSetup() noexcept(false) { + if (!std::uncaught_exception()) { // Exiting from test successfully. Check additional expectations. + MockProcessorFactory->Wait(); + MockProcessor->Wait(); + + MockProcessorFactory->Validate(); + MockProcessor->Validate(); + } + + CbContext->Cancel(); + Session = nullptr; + + ThreadPool->Stop(); +} + +::IExecutor::TPtr TReadSessionImplTestSetup::GetDefaultExecutor() { + if (!DefaultExecutor) { + ThreadPool = std::make_shared(); + ThreadPool->Start(1); + DefaultExecutor = CreateThreadPoolExecutorAdapter(ThreadPool); + } + return DefaultExecutor; +} + +TSingleClusterReadSessionImpl* TReadSessionImplTestSetup::GetSession() { + if (!Session) { + if (!Settings.DecompressionExecutor_) { + Settings.DecompressionExecutor(GetDefaultExecutor()); + } + if (!Settings.EventHandlers_.HandlersExecutor_) { + Settings.EventHandlers_.HandlersExecutor(GetDefaultExecutor()); + } + CbContext = MakeWithCallbackContext( + Settings, + "db", + "sessionid", + ClusterName, + Log, + MockProcessorFactory, + GetEventsQueue(), + FakeContext, + PartitionIdStart, + PartitionIdStep); + Session = CbContext->TryGet(); + } + return Session.get(); +} + +std::shared_ptr TReadSessionImplTestSetup::GetEventsQueue() { + if (!EventsQueue) { + EventsQueue = std::make_shared(Settings); + } + return EventsQueue; +} + +void TReadSessionImplTestSetup::SuccessfulInit(bool hasInitRequest) { + EXPECT_CALL(*MockProcessorFactory, OnCreateProcessor(1)) + .WillOnce([&](){ MockProcessorFactory->CreateProcessor(MockProcessor); }); + if (hasInitRequest) + EXPECT_CALL(*MockProcessor, OnInitRequest(_)); + MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo().InitResponse("123-session-id-321")); + GetSession()->Start(); + MockProcessorFactory->Wait(); + MockProcessor->Wait(); +} + +TPartitionStream::TPtr TReadSessionImplTestSetup::CreatePartitionStream(const TString& topic, const TString& cluster, ui64 partition, ui64 assignId) { + MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo().CreatePartitionStream(topic, cluster, partition, assignId)); // Callback will be called. + TMaybe event = EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TCreatePartitionStreamEvent); + auto& createEvent = std::get(*event); + auto stream = createEvent.GetPartitionStream(); + UNIT_ASSERT(stream); + createEvent.Confirm(); + return stream; +} + +void TReadSessionImplTestSetup::AssertNoEvents() { + TMaybe event = GetEventsQueue()->GetEvent(false); + UNIT_ASSERT(!event); +} + +using NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup; +Y_UNIT_TEST_SUITE(PersQueueSdkReadSessionTest) { + void ReadSessionImpl(bool close, bool commit, bool explicitlySpecifiedPartitions = false) { + NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup setup("ReadSession"); + setup.WriteToTopic({"message1", "message2"}); + auto settings = setup.GetReadSessionSettings(); + if (explicitlySpecifiedPartitions) { + settings.Topics_[0].AppendPartitionGroupIds(1); + } + std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); + + TDeferredCommit dc; + // Event 1: create partition stream. + { + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TCreatePartitionStreamEvent); + std::get(*event).Confirm(); + Cerr << "create event " << DebugString(*event) << Endl; + } + // Event 2: data. + { + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages().size(), 2); + for (auto& msg : dataEvent.GetMessages()) { + UNIT_ASSERT(msg.GetData() == "message1" || msg.GetData() == "message2"); + } + Cerr << "data event " << DebugString(*event) << Endl; + if (commit) { + dc.Add(dataEvent); + } + } + setup.WriteToTopic({"message3"}); + // Event 3: data. + { + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages().size(), 1); + for (auto& msg : dataEvent.GetMessages()) { + UNIT_ASSERT(msg.GetData() == "message3"); + } + Cerr << "data event " << DebugString(*event) << Endl; + + dataEvent.Commit(); // Commit right now! + } + + dc.Commit(); + + if (close) { + session->Close(TDuration::Seconds(30)); + } + + // Event 4: commit ack. + if (commit) { // (commit && close) branch check is broken with current TReadSession::Close quick fix + TMaybe event = session->GetEvent(!close); // Event is expected to be already in queue if closed. + UNIT_ASSERT(event); + Cerr << "commit ack or close event " << DebugString(*event) << Endl; + UNIT_ASSERT(std::holds_alternative(*event) || std::holds_alternative(*event)); + } + + if (close && !commit) { + TMaybe event = session->GetEvent(false); + UNIT_ASSERT(event); + Cerr << "close event " << DebugString(*event) << Endl; + UNIT_ASSERT(std::holds_alternative(*event)); + UNIT_ASSERT_STRING_CONTAINS(DebugString(*event), "Session was gracefully closed"); + } + } + + Y_UNIT_TEST(ReadSessionWithAbort) { + ReadSessionImpl(false, true); + } + + Y_UNIT_TEST(ReadSessionWithClose) { + ReadSessionImpl(true, true); + } + + Y_UNIT_TEST(ReadSessionWithCloseNotCommitted) { + ReadSessionImpl(true, false); + } + + Y_UNIT_TEST(ReadSessionWithExplicitlySpecifiedPartitions) { + ReadSessionImpl(true, true, true); + } + + Y_UNIT_TEST(SettingsValidation) { + TPersQueueYdbSdkTestSetup setup("SettingsValidation"); + const auto goodSettings = setup.GetReadSessionSettings(); + +#define ASSERT_BAD_VALIDATION \ + std::shared_ptr session = \ + setup.GetPersQueueClient().CreateReadSession(settings); \ + session->WaitEvent().Wait(); \ + TMaybe event = \ + session->GetEvent(true); \ + UNIT_ASSERT(event); \ + Cerr << DebugString(*event) << Endl; \ + UNIT_ASSERT_EVENT_TYPE(*event, TSessionClosedEvent); \ + /**/ + + // No topics to read. + { + auto settings = goodSettings; + settings.Topics_.clear(); + ASSERT_BAD_VALIDATION; + } + + // No consumer name. + { + auto settings = goodSettings; + settings.ConsumerName(""); + ASSERT_BAD_VALIDATION; + } + + // Small max memory usage. + { + auto settings = goodSettings; + settings.MaxMemoryUsageBytes(100); + ASSERT_BAD_VALIDATION; + } + } + + Y_UNIT_TEST(ClosesAfterFailedConnectionToCds) { + TPersQueueYdbSdkTestSetup setup("ClosesAfterFailedConnectionToCds"); + setup.ShutdownGRpc(); + + TReadSessionSettings settings = setup.GetReadSessionSettings(); + // Set policy with max retries == 3. + settings.RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetExponentialBackoffPolicy(TDuration::MilliSeconds(10), TDuration::MilliSeconds(10), TDuration::MilliSeconds(100), 3)); + std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + Cerr << DebugString(*event) << Endl; + UNIT_ASSERT_EVENT_TYPE(*event, TSessionClosedEvent); + } + + Y_UNIT_TEST(SpecifyClustersExplicitly) { + TPersQueueYdbSdkTestSetup setup("SpecifyClustersExplicitly"); + + auto settings = setup.GetReadSessionSettings(); + settings.ReadOriginal({setup.GetLocalCluster()}); + + // Success. + { + std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + Cerr << DebugString(*event) << Endl; + UNIT_ASSERT_NOT_EVENT_TYPE(*event, TSessionClosedEvent); + } + + // Failure: one cluster endpoint is invalid. + { + settings.AppendClusters("unknown_cluster"); + std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + Cerr << DebugString(*event) << Endl; + UNIT_ASSERT_EVENT_TYPE(*event, TSessionClosedEvent); + } + + // Failure: no valid cluster endpoints. + { + settings.ReadOriginal({"unknown_cluster"}); + std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + Cerr << DebugString(*event) << Endl; + UNIT_ASSERT_EVENT_TYPE(*event, TSessionClosedEvent); + } + } + + Y_UNIT_TEST(StopResumeReadingData) { + NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup setup("ReadSession"); + setup.WriteToTopic({"message1"}); + std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(setup.GetReadSessionSettings()); + + // Event 1: create partition stream. + { + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TCreatePartitionStreamEvent); + std::get(*event).Confirm(); + Cerr << DebugString(*event) << Endl; + } + + // Event 2: receive data. + auto GetDataEvent = [&](const TString& content) -> TMaybe { + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages()[0].GetData(), content); + Cerr << DebugString(*event) << Endl; + return event; + }; + + TMaybe dataEvents[2]; + + dataEvents[0] = GetDataEvent("message1"); + + // Stop reading data. + session->StopReadingData(); + + // Write data. + setup.WriteToTopic({"message2"}); + + // Already requested read. + dataEvents[1] = GetDataEvent("message2"); + + // Write data. + setup.WriteToTopic({"message3"}); + + // Commit and check that other events will come. + for (int i = 0; i < 2; ++i) { + std::get(*dataEvents[i]).Commit(); + TMaybe event = session->GetEvent(true); + UNIT_ASSERT(event); + Y_ASSERT(std::holds_alternative(*event)); + Cerr << DebugString(*event) << Endl; + } + + auto eventFuture = session->WaitEvent(); + UNIT_ASSERT_C(!eventFuture.Wait(TDuration::Seconds(1)), DebugString(*session->GetEvent(false))); + + // Resume reading data. + session->ResumeReadingData(); + + // And now we can read. + auto dataEvent3 = GetDataEvent("message3"); + + session->Close(TDuration::Seconds(3)); + } +} + +Y_UNIT_TEST_SUITE(ReadSessionImplTest) { + void SuccessfulInitImpl(bool thenTimeout) { + TReadSessionImplTestSetup setup; + setup.Settings + .MaxTimeLag(TDuration::Seconds(32)) + .StartingMessageTimestamp(TInstant::Seconds(42)); + + setup.Settings.Topics_[0] + .StartingMessageTimestamp(TInstant::Seconds(146)) + .AppendPartitionGroupIds(100) + .AppendPartitionGroupIds(101); + + EXPECT_CALL(*setup.MockProcessorFactory, OnCreateProcessor(_)) + .WillOnce([&](){ + if (thenTimeout) { + setup.MockProcessorFactory->CreateAndThenTimeout(setup.MockProcessor); + } else { + setup.MockProcessorFactory->CreateProcessor(setup.MockProcessor); + } + }); + EXPECT_CALL(*setup.MockProcessor, OnInitRequest(_)) + .WillOnce(Invoke([](const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::InitRequest& req) { + UNIT_ASSERT_STRINGS_EQUAL(req.consumer(), "TestConsumer"); + UNIT_ASSERT_VALUES_EQUAL(req.max_lag_duration_ms(), 32000); + UNIT_ASSERT_VALUES_EQUAL(req.start_from_written_at_ms(), 42000); + UNIT_ASSERT_VALUES_EQUAL(req.topics_read_settings_size(), 1); + UNIT_ASSERT_VALUES_EQUAL(req.topics_read_settings(0).topic(), "TestTopic"); + UNIT_ASSERT_VALUES_EQUAL(req.topics_read_settings(0).start_from_written_at_ms(), 146000); + UNIT_ASSERT_VALUES_EQUAL(req.topics_read_settings(0).partition_group_ids_size(), 2); + UNIT_ASSERT_VALUES_EQUAL(req.topics_read_settings(0).partition_group_ids(0), 100); + UNIT_ASSERT_VALUES_EQUAL(req.topics_read_settings(0).partition_group_ids(1), 101); + })); + setup.GetSession()->Start(); + setup.MockProcessorFactory->Wait(); + + EXPECT_CALL(*setup.MockProcessor, OnReadRequest(_)); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo().InitResponse("session id")); + + setup.AssertNoEvents(); + } + + Y_UNIT_TEST(SuccessfulInit) { + SuccessfulInitImpl(false); + } + + Y_UNIT_TEST(SuccessfulInitAndThenTimeoutCallback) { + SuccessfulInitImpl(true); + } + + void ReconnectOnTmpErrorImpl(bool timeout, bool thenSecondCallback) { + TReadSessionImplTestSetup setup; + EXPECT_CALL(*setup.MockProcessorFactory, OnCreateProcessor(_)) + .WillOnce([&](){ + if (timeout) { + if (thenSecondCallback) { + setup.MockProcessorFactory->TimeoutAndThenCreate(nullptr); + } else { + setup.MockProcessorFactory->Timeout(); + } + } else { + if (thenSecondCallback) { + setup.MockProcessorFactory->FailAndThenTimeout(); + } else { + setup.MockProcessorFactory->FailCreation(); + } + } + }) + .WillOnce([&](){ setup.MockProcessorFactory->CreateProcessor(setup.MockProcessor); }); + EXPECT_CALL(*setup.MockProcessor, OnInitRequest(_)); + setup.GetSession()->Start(); + setup.MockProcessorFactory->Wait(); + + EXPECT_CALL(*setup.MockProcessor, OnReadRequest(_)); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo().InitResponse("session id")); + + setup.AssertNoEvents(); + } + + Y_UNIT_TEST(ReconnectOnTmpError) { + ReconnectOnTmpErrorImpl(false, false); + } + + Y_UNIT_TEST(ReconnectOnTmpErrorAndThenTimeout) { + ReconnectOnTmpErrorImpl(false, true); + } + + Y_UNIT_TEST(ReconnectOnTimeout) { + ReconnectOnTmpErrorImpl(true, false); + } + + Y_UNIT_TEST(ReconnectOnTimeoutAndThenCreate) { + ReconnectOnTmpErrorImpl(true, true); + } + + void StopsRetryAfterFailedAttemptImpl(bool timeout) { + TReadSessionImplTestSetup setup; + setup.Settings.RetryPolicy(NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()); + EXPECT_CALL(*setup.MockProcessorFactory, OnCreateProcessor(_)) + .WillOnce([&]() { + if (timeout) + setup.MockProcessorFactory->Timeout(); + else + setup.MockProcessorFactory->FailCreation(); + }); + + //EXPECT_CALL(*setup.MockErrorHandler, AbortSession(_)); + + setup.GetSession()->Start(); + setup.MockProcessorFactory->Wait(); + UNIT_ASSERT(setup.GetEventsQueue()->IsClosed()); + } + + + Y_UNIT_TEST(StopsRetryAfterFailedAttempt) { + StopsRetryAfterFailedAttemptImpl(false); + } + + Y_UNIT_TEST(StopsRetryAfterTimeout) { + StopsRetryAfterFailedAttemptImpl(true); + } + + Y_UNIT_TEST(UsesOnRetryStateDuringRetries) { + class TTestRetryState : public NYdb::NPersQueue::IRetryPolicy::IRetryState { + TDuration Delay; + + TMaybe GetNextRetryDelay(NYdb::EStatus) override { + Delay += TDuration::Seconds(1); + return Delay; + } + }; + + class TTestRetryPolicy : public NYdb::NPersQueue::IRetryPolicy { + IRetryState::TPtr CreateRetryState() const override { + return IRetryState::TPtr(new TTestRetryState()); + } + }; + + std::shared_ptr state(new TTestRetryState()); + TReadSessionImplTestSetup setup; + ON_CALL(*setup.MockProcessorFactory, ValidateConnectTimeout(_)) + .WillByDefault([state](TDuration timeout) mutable { + UNIT_ASSERT_VALUES_EQUAL(timeout, *state->GetNextRetryDelay(NYdb::EStatus::INTERNAL_ERROR)); + }); + auto failCreation = [&]() { + setup.MockProcessorFactory->FailCreation(); + }; + EXPECT_CALL(*setup.MockProcessorFactory, OnCreateProcessor(_)) + .WillOnce(failCreation) + .WillOnce(failCreation) + .WillOnce(failCreation) + .WillOnce([](){}); // No action. The end of test. + + setup.GetSession()->Start(); + while (setup.MockProcessorFactory->CreateCallsCount < 4) { + Sleep(TDuration::MilliSeconds(10)); + } + setup.MockProcessorFactory->Wait(); + + setup.AssertNoEvents(); + } + + Y_UNIT_TEST(ReconnectsAfterFailure) { + TReadSessionImplTestSetup setup; + setup.SuccessfulInit(); + + auto secondProcessor = MakeIntrusive(); + EXPECT_CALL(*setup.MockProcessorFactory, OnCreateProcessor(2)) + .WillOnce([&](){ setup.MockProcessorFactory->CreateProcessor(secondProcessor); }); + + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo().Failure()); // Callback will be called. + + setup.Session->Abort(); + + setup.AssertNoEvents(); + + setup.MockProcessor->Wait(); + secondProcessor->Wait(); + secondProcessor->Validate(); + + } + + Y_UNIT_TEST(CreatePartitionStream) { + TReadSessionImplTestSetup setup; + setup.SuccessfulInit(); + + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo().CreatePartitionStream()); // Callback will be called. + { + TVector events = setup.EventsQueue->GetEvents(true); + UNIT_ASSERT_VALUES_EQUAL(events.size(), 1); + UNIT_ASSERT_EVENT_TYPE(events[0], TReadSessionEvent::TCreatePartitionStreamEvent); + auto& event = std::get(events[0]); + auto stream = event.GetPartitionStream(); + UNIT_ASSERT(stream); + + EXPECT_CALL(*setup.MockProcessor, OnStartReadRequest(_)) + .WillOnce(Invoke([](const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::StartRead& req) { + UNIT_ASSERT_STRINGS_EQUAL(req.topic().path(), "TestTopic"); + UNIT_ASSERT_STRINGS_EQUAL(req.cluster(), "TestCluster"); + UNIT_ASSERT_VALUES_EQUAL(req.partition(), 1); + UNIT_ASSERT_VALUES_EQUAL(req.assign_id(), 1); + UNIT_ASSERT_VALUES_EQUAL(req.read_offset(), 13); + UNIT_ASSERT_VALUES_EQUAL(req.commit_offset(), 31); + })); + + event.Confirm(13, 31); + } + } + + void DestroyPartitionStreamImpl(bool forceful) { + TReadSessionImplTestSetup setup; + setup.SuccessfulInit(); + + Y_UNUSED(forceful); + TPartitionStream::TPtr stream = setup.CreatePartitionStream(); + + // Send release by server. + { + TMockReadSessionProcessor::TServerReadInfo resp; + if (forceful) { + resp.ForcefulReleasePartitionStream(); + } else { + resp.ReleasePartitionStream(); + } + setup.MockProcessor->AddServerResponse(resp); // Callback will be called. + } + + // Check destroy event. + if (!forceful) { + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDestroyPartitionStreamEvent); + auto& destroyEvent = std::get(*event); + UNIT_ASSERT_EQUAL(destroyEvent.GetPartitionStream(), stream); + + EXPECT_CALL(*setup.MockProcessor, OnReleasedRequest(_)) + .WillOnce(Invoke([](const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::Released& req) { + UNIT_ASSERT_STRINGS_EQUAL(req.topic().path(), "TestTopic"); + UNIT_ASSERT_STRINGS_EQUAL(req.cluster(), "TestCluster"); + UNIT_ASSERT_VALUES_EQUAL(req.partition(), 1); + UNIT_ASSERT_VALUES_EQUAL(req.assign_id(), 1); + })); + + destroyEvent.Confirm(); + } + + // Check closed event. + { + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TPartitionStreamClosedEvent); + auto& closedEvent = std::get(*event); + UNIT_ASSERT_EQUAL(closedEvent.GetPartitionStream(), stream); + + if (forceful) { + UNIT_ASSERT_EQUAL_C(closedEvent.GetReason(), TReadSessionEvent::TPartitionStreamClosedEvent::EReason::Lost, DebugString(*event)); + } else { + UNIT_ASSERT_EQUAL_C(closedEvent.GetReason(), TReadSessionEvent::TPartitionStreamClosedEvent::EReason::DestroyConfirmedByUser, DebugString(*event)); + } + } + } + + Y_UNIT_TEST(ForcefulDestroyPartitionStream) { + DestroyPartitionStreamImpl(true); + } + + Y_UNIT_TEST(DestroyPartitionStreamRequest) { + DestroyPartitionStreamImpl(false); + } + + Y_UNIT_TEST(ProperlyOrdersDecompressedData) { + TReadSessionImplTestSetup setup; + setup.Settings.DecompressionExecutor(new TReorderingExecutor()); + setup.SuccessfulInit(); + TPartitionStream::TPtr stream = setup.CreatePartitionStream(); + for (ui64 i = 1; i <= 2; ++i) { + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(i) + .Batch("src_id") + .CompressMessage(i, TStringBuilder() << "message" << i)); // Callback will be called. + } + + for (ui64 i = 1; i <= 2; ) { + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + auto& dataEvent = std::get(*event); + for (ui32 j = 0; j < dataEvent.GetMessages().size(); ++j, ++i) { + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages()[j].GetData(), TStringBuilder() << "message" << i); + } + } + + setup.AssertNoEvents(); + } + + Y_UNIT_TEST(BrokenCompressedData) { + TReadSessionImplTestSetup setup; + setup.Settings.DecompressionExecutor(new TReorderingExecutor(1)); + setup.SuccessfulInit(); + TPartitionStream::TPtr stream = setup.CreatePartitionStream(); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(1) + .Batch("src_id") + .BrokenCompressMessage(1, "message") + .CompressMessage(2, "message2") + .CompressMessage(3, "message3")); + + // Exception was passed during decompression. + { + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + auto& dataEvent = std::get(*event); + Cerr << dataEvent.DebugString() << "\n"; + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages().size(), 3); + UNIT_ASSERT_EXCEPTION(dataEvent.GetMessages()[0].GetData(), TZLibDecompressorError); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages()[1].GetData(), "message2"); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages()[2].GetData(), "message3"); + } + + setup.AssertNoEvents(); + } + + void DecompressImpl(Ydb::PersQueue::V1::Codec codec, const TString& data = "msg", ::IExecutor::TPtr executor = nullptr) { + TReadSessionImplTestSetup setup; + if (executor) { + setup.Settings.DecompressionExecutor(executor); + } + setup.SuccessfulInit(false); + TPartitionStream::TPtr stream = setup.CreatePartitionStream(); + + EXPECT_CALL(*setup.MockProcessor, OnReadRequest(_)); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(1) + .Batch("src_id") + .CompressMessage(1, data, codec)); + + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + auto& dataEvent = std::get(*event); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages().size(), 1); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages()[0].GetData(), data); + + setup.AssertNoEvents(); + } + + Y_UNIT_TEST(DecompressRaw) { + DecompressImpl(Ydb::PersQueue::V1::CODEC_RAW); + DecompressImpl(Ydb::PersQueue::V1::CODEC_UNSPECIFIED); // The same. + } + + Y_UNIT_TEST(DecompressGzip) { + DecompressImpl(Ydb::PersQueue::V1::CODEC_GZIP); + } + + Y_UNIT_TEST(DecompressZstd) { + DecompressImpl(Ydb::PersQueue::V1::CODEC_ZSTD); + } + + Y_UNIT_TEST(DecompressRawEmptyMessage) { + DecompressImpl(Ydb::PersQueue::V1::CODEC_RAW, ""); + DecompressImpl(Ydb::PersQueue::V1::CODEC_UNSPECIFIED, ""); // The same. + } + + Y_UNIT_TEST(DecompressGzipEmptyMessage) { + DecompressImpl(Ydb::PersQueue::V1::CODEC_GZIP, ""); + } + + Y_UNIT_TEST(DecompressZstdEmptyMessage) { + DecompressImpl(Ydb::PersQueue::V1::CODEC_ZSTD, ""); + } + + Y_UNIT_TEST(DecompressWithSynchronousExecutor) { + DecompressImpl(Ydb::PersQueue::V1::CODEC_ZSTD, "msg", new TSynchronousExecutor()); + } + + TString GenerateMessageData(size_t size) { + TString result; + result.reserve(size); + unsigned char ch = static_cast(size); + while (size--) { + result.append(static_cast(ch)); + do { + ch = ch * 7 + 23; + } while (ch == 0); + } + return result; + } + + void PacksBatchesImpl(size_t serverBatchesCount, size_t messagesInServerBatchCount, size_t messageSize, size_t batchLimit, size_t batches, size_t messagesInBatch, size_t expectedTasks = 0, size_t reorderedCycleSize = 0, size_t memoryLimit = 0) { + if (!expectedTasks) { + expectedTasks = serverBatchesCount; + } + if (!reorderedCycleSize) { + reorderedCycleSize = expectedTasks; + } + TReadSessionImplTestSetup setup; + if (memoryLimit) { + setup.Settings.MaxMemoryUsageBytes(memoryLimit); + } + auto executor = MakeIntrusive(reorderedCycleSize); + setup.Settings.DecompressionExecutor(executor); + setup.SuccessfulInit(); + TPartitionStream::TPtr stream = setup.CreatePartitionStream(); + + const TString messageData = GenerateMessageData(messageSize); + const TString compressedMessageData = Compress(messageData); + Cerr << "Message data size: " << messageData.size() << Endl; + Cerr << "Compressed message data size: " << compressedMessageData.size() << Endl; + ui64 offset = 1; + ui64 seqNo = 42; + THashSet committedCookies; + THashSet committedOffsets; + EXPECT_CALL(*setup.MockProcessor, OnCommitRequest(_)) + .WillRepeatedly(Invoke([&committedCookies, &committedOffsets](const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::Commit& req) { + for (const auto& commit : req.cookies()) { + committedCookies.insert(commit.partition_cookie()); + } + for (const auto& range : req.offset_ranges()) { + Cerr << "GOT RANGE " << range.start_offset() << " " << range.end_offset() << "\n"; + for (ui64 i = range.start_offset(); i < range.end_offset(); ++i) { + committedOffsets.insert(i); + } + } + })); + + for (ui64 i = 1; i <= serverBatchesCount; ++i) { + TMockReadSessionProcessor::TServerReadInfo resp; + resp.PartitionData(i).Batch("src_id", TInstant::Seconds(123), "127.0.0.1", { { "k", "v" }, { "k1", "v1" }}); + for (size_t j = 0; j < messagesInServerBatchCount; ++j) { + resp.Message(offset++, compressedMessageData, Ydb::PersQueue::V1::CODEC_GZIP, seqNo++); + if (j == messagesInServerBatchCount / 2) { // This may lead to empty batch. Client is expected to behave well with empty batch in protobuf. + resp.Batch("src_id_2", TInstant::Seconds(321), "1.0.0.127", { { "v", "k" }, { "v1", "k1" }}); + } + } + setup.MockProcessor->AddServerResponse(resp); + } + + ui64 prevOffset = 0; + ui64 prevSeqNo = 41; + for (size_t i = 0; i < batches; ++i) { + Cerr << "Getting new event" << Endl; + TMaybe event = setup.EventsQueue->GetEvent(true, batchLimit); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + Cerr << DebugString(*event) << Endl; + auto& dataEvent = std::get(*event); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages().size(), messagesInBatch); + for (const auto& message : dataEvent.GetMessages()) { + auto meta = message.GetMeta(); + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), messageData); + UNIT_ASSERT_VALUES_EQUAL(message.GetOffset(), prevOffset + 1); + UNIT_ASSERT_VALUES_EQUAL(message.GetSeqNo(), prevSeqNo + 1); + ++prevOffset; + ++prevSeqNo; + UNIT_ASSERT_VALUES_EQUAL(message.GetCreateTime(), TInstant::MilliSeconds(42)); + if (message.GetMessageGroupId() == "src_id") { + UNIT_ASSERT_VALUES_EQUAL(message.GetWriteTime(), TInstant::Seconds(123)); + UNIT_ASSERT_VALUES_EQUAL(message.GetIp(), "127.0.0.1"); + UNIT_ASSERT_VALUES_EQUAL(meta->Fields.at("k"), "v"); + UNIT_ASSERT_VALUES_EQUAL(meta->Fields.at("k1"), "v1"); + } else if (message.GetMessageGroupId() == "src_id_2") { + UNIT_ASSERT_VALUES_EQUAL(message.GetWriteTime(), TInstant::Seconds(321)); + UNIT_ASSERT_VALUES_EQUAL(message.GetIp(), "1.0.0.127"); + UNIT_ASSERT_VALUES_EQUAL(meta->Fields.at("v"), "k"); + UNIT_ASSERT_VALUES_EQUAL(meta->Fields.at("v1"), "k1"); + } else { + UNIT_ASSERT_C(false, message.GetMessageGroupId()); + } + } + dataEvent.Commit(); + } + if (committedOffsets.empty()) { + UNIT_ASSERT_VALUES_EQUAL(committedCookies.size(), serverBatchesCount); + for (ui64 i = 1; i <= serverBatchesCount; ++i) { + UNIT_ASSERT(committedCookies.contains(i)); + } + } else { + UNIT_ASSERT_VALUES_EQUAL(committedOffsets.size(), batches * messagesInBatch + 1); + for (ui64 i = 0; i <= batches * messagesInBatch; ++i) { + UNIT_ASSERT(committedOffsets.contains(i)); + } + + } + UNIT_ASSERT_VALUES_EQUAL(executor->GetTasksAdded(), expectedTasks); + + setup.AssertNoEvents(); + } + + Y_UNIT_TEST(PacksBatches_BatchABitBiggerThanLimit) { + const size_t serverBatchesCount = 2; + const size_t messagesInServerBatchCount = 4; + const size_t messageSize = 11; + const size_t batchLimit = 20; + const size_t batches = 4; + const size_t messagesInBatch = 2; + PacksBatchesImpl(serverBatchesCount, messagesInServerBatchCount, messageSize, batchLimit, batches, messagesInBatch); + } + + Y_UNIT_TEST(PacksBatches_ExactlyTwoMessagesInBatch) { + const size_t serverBatchesCount = 2; + const size_t messagesInServerBatchCount = 4; + const size_t messageSize = 10; // Exactly two messages in batch. + const size_t batchLimit = 20; + const size_t batches = 4; + const size_t messagesInBatch = 2; + PacksBatchesImpl(serverBatchesCount, messagesInServerBatchCount, messageSize, batchLimit, batches, messagesInBatch); + } + + Y_UNIT_TEST(PacksBatches_BatchesEqualToServerBatches) { + const size_t serverBatchesCount = 2; + const size_t messagesInServerBatchCount = 4; + const size_t messageSize = 10; + const size_t batchLimit = 40; // As in server messages. + const size_t batches = 2; + const size_t messagesInBatch = 4; + PacksBatchesImpl(serverBatchesCount, messagesInServerBatchCount, messageSize, batchLimit, batches, messagesInBatch); + } + + Y_UNIT_TEST(PacksBatches_OneMessageInEveryBatch) { + const size_t serverBatchesCount = 2; + const size_t messagesInServerBatchCount = 4; + const size_t messageSize = 100; + const size_t batchLimit = 90; // One message in every batch. + const size_t batches = 8; + const size_t messagesInBatch = 1; + PacksBatchesImpl(serverBatchesCount, messagesInServerBatchCount, messageSize, batchLimit, batches, messagesInBatch); + } + + Y_UNIT_TEST(PacksBatches_BigBatchDecompressWithTwoBatchTasks) { + const size_t serverBatchesCount = 1; + const size_t messagesInServerBatchCount = 200; // Many messages in order to fit in two 512 KB-tasks packs. + const size_t messageSize = 1000000; + const size_t batchLimit = std::numeric_limits::max(); + const size_t batches = 1; + const size_t messagesInBatch = messagesInServerBatchCount; + const size_t expectedDecompressionTasksCount = 2; + PacksBatchesImpl(serverBatchesCount, messagesInServerBatchCount, messageSize, batchLimit, batches, messagesInBatch, expectedDecompressionTasksCount); + } + + Y_UNIT_TEST(PacksBatches_DecompressesOneMessagePerTime) { + const size_t serverBatchesCount = 1; + const size_t messagesInServerBatchCount = 10; // Many messages in order to fit in two 512 KB-tasks packs. + const size_t messageSize = 1000000; + const size_t batchLimit = std::numeric_limits::max(); + const size_t batches = 1; + const size_t messagesInBatch = 10; + const size_t expectedDecompressionTasksCount = 1; + const size_t reorderedCycleSize = 1; + const size_t memoryLimit = 10; + PacksBatchesImpl(serverBatchesCount, messagesInServerBatchCount, messageSize, batchLimit, batches, messagesInBatch, expectedDecompressionTasksCount, reorderedCycleSize, memoryLimit); + } + + Y_UNIT_TEST(UnpackBigBatchWithTwoPartitions) { + TReadSessionImplTestSetup setup; + + setup.Settings.MaxMemoryUsageBytes(5000); + setup.SuccessfulInit(); + TPartitionStream::TPtr stream1 = setup.CreatePartitionStream("TestTopic", "TestCluster", 1, 1); + TPartitionStream::TPtr stream2 = setup.CreatePartitionStream("TestTopic", "TestCluster", 2, 2); + + + const TString messageData = GenerateMessageData(100); + const TString compressedMessageData = Compress(messageData); + ui64 offset = 1; + ui64 seqNo = 42; + + for (ui64 partition = 1; partition <= 2; ++partition) { + TMockReadSessionProcessor::TServerReadInfo resp; + resp.PartitionData(partition, "TestTopic", "TestCluster", partition, partition) + .Batch("src_id", TInstant::Seconds(123), "127.0.0.1", { { "k", "v" }, { "k1", "v1" }}); + + for (size_t i = 0; i < 50; ++i) { + if (i == 22) { + resp.Batch("src_id2", TInstant::Seconds(123), "127.0.0.1", { { "k", "v" }, { "k1", "v1" }}); + } + resp.Message(offset++, compressedMessageData, Ydb::PersQueue::V1::CODEC_GZIP, seqNo++); + } + setup.MockProcessor->AddServerResponse(resp); + } + + { + TVector events = setup.EventsQueue->GetEvents(true); + UNIT_ASSERT_VALUES_EQUAL(events.size(), 1); + UNIT_ASSERT_EVENT_TYPE(events[0], TReadSessionEvent::TDataReceivedEvent); + TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(events[0]); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages().size(), 50); + UNIT_ASSERT_EQUAL(dataEvent.GetPartitionStream(), stream1); + } + + { + TVector events = setup.EventsQueue->GetEvents(true); + UNIT_ASSERT_VALUES_EQUAL(events.size(), 1); + UNIT_ASSERT_EVENT_TYPE(events[0], TReadSessionEvent::TDataReceivedEvent); + TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(events[0]); + UNIT_ASSERT_VALUES_EQUAL(dataEvent.GetMessages().size(), 50); + UNIT_ASSERT_EQUAL(dataEvent.GetPartitionStream(), stream2); + } + + setup.AssertNoEvents(); + } + + Y_UNIT_TEST(PartitionStreamStatus) { + TReadSessionImplTestSetup setup; + setup.SuccessfulInit(); + TPartitionStream::TPtr stream = setup.CreatePartitionStream(); + EXPECT_CALL(*setup.MockProcessor, OnStatusRequest(_)) + .WillOnce(Invoke([](const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::Status& req) { + UNIT_ASSERT_VALUES_EQUAL(req.topic().path(), "TestTopic"); + UNIT_ASSERT_VALUES_EQUAL(req.cluster(), "TestCluster"); + UNIT_ASSERT_VALUES_EQUAL(req.partition(), 1); + UNIT_ASSERT_VALUES_EQUAL(req.assign_id(), 1); + })); + // Another assign id. + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionStreamStatus(11, 34, TInstant::Seconds(4), "TestTopic", "TestCluster", 1 /*partition*/, 13/*assign id to ignore*/)); + + // Proper assign id. + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo().PartitionStreamStatus(30, 42, TInstant::Seconds(20))); + + stream->RequestStatus(); + + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TPartitionStreamStatusEvent); + auto& statusEvent = std::get(*event); + UNIT_ASSERT_VALUES_EQUAL(statusEvent.GetCommittedOffset(), 30); + UNIT_ASSERT_VALUES_EQUAL(statusEvent.GetEndOffset(), 42); + UNIT_ASSERT_VALUES_EQUAL(statusEvent.GetWriteWatermark(), TInstant::MilliSeconds(20000)); + + setup.AssertNoEvents(); + } + + Y_UNIT_TEST(HoleBetweenOffsets) { + TReadSessionImplTestSetup setup; + setup.Settings.DecompressionExecutor(MakeIntrusive(2ull)); + setup.SuccessfulInit(); + TPartitionStream::TPtr stream = setup.CreatePartitionStream(); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(1) + .Batch("src_id") + .CompressMessage(1, "message1") + .CompressMessage(2, "message2")); + + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(2) + .Batch("src_id") + .CompressMessage(10, "message3") + .CompressMessage(11, "message4")); + + bool has1 = false; + bool has2 = false; + EXPECT_CALL(*setup.MockProcessor, OnCommitRequest(_)) + .WillRepeatedly(Invoke([&](const Ydb::PersQueue::V1::MigrationStreamingReadClientMessage::Commit& req) { + Cerr << "Got commit req " << req << "\n"; + for (const auto& commit : req.cookies()) { + if (commit.partition_cookie() == 1) { + has1 = true; + } else if (commit.partition_cookie() == 2) { + has2 = true; + } else { + UNIT_ASSERT(false); + } + } + for (const auto& range : req.offset_ranges()) { + Cerr << "RANGE " << range.start_offset() << " " << range.end_offset() << "\n"; + if (range.start_offset() == 3 && range.end_offset() == 12) has1 = true; + else if (range.start_offset() == 0 && range.end_offset() == 3) has2 = true; + else UNIT_ASSERT(false); + } + })); + + for (int i = 0; i < 2; ) { + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); + Cerr << "got data event: " << dataEvent.DebugString() << "\n"; + dataEvent.Commit(); + + i += dataEvent.GetMessagesCount(); + } + + UNIT_ASSERT(has1); + UNIT_ASSERT(has2); + + setup.AssertNoEvents(); + } + + Y_UNIT_TEST(CommitOffsetTwiceIsError) { + TReadSessionImplTestSetup setup; + setup.SuccessfulInit(); + TPartitionStream::TPtr stream = setup.CreatePartitionStream(); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(1) + .Batch("src_id") + .CompressMessage(1, "message1")); + + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + TReadSessionEvent::TDataReceivedEvent& dataEvent = std::get(*event); + + // First time. + dataEvent.GetMessages()[0].Commit(); + + UNIT_ASSERT_EXCEPTION(dataEvent.GetMessages()[0].Commit(), NYdb::TContractViolation); + } + + Y_UNIT_TEST(DataReceivedCallbackReal) { + NYdb::NPersQueue::NTests::TPersQueueYdbSdkTestSetup setup("ReadSession"); + auto settings = setup.GetReadSessionSettings(); + + auto calledPromise = NThreading::NewPromise(); + int time = 0; + + settings.EventHandlers_.SimpleDataHandlers([&](TReadSessionEvent::TDataReceivedEvent& event) { + for (auto& message: event.GetMessages()) { + ++time; + Cerr << "GOT MESSAGE: " << message.DebugString(true) << "\n"; + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), TStringBuilder() << "message" << time); + if (time == 3) { + calledPromise.SetValue(); + } + } + UNIT_ASSERT(time <= 3); + }, true); + + std::shared_ptr session = setup.GetPersQueueClient().CreateReadSession(settings); + + UNIT_ASSERT(!calledPromise.GetFuture().Wait(TDuration::Seconds(5))); + + setup.WriteToTopic({"message1"}, false); + setup.WriteToTopic({"message2"}, false); + Sleep(TDuration::Seconds(1)); + setup.WriteToTopic({"message3"}, false); + + calledPromise.GetFuture().Wait(); + Sleep(TDuration::Seconds(10)); + } + + Y_UNIT_TEST(DataReceivedCallback) { + TReadSessionImplTestSetup setup; + setup.Settings.DecompressionExecutor(MakeIntrusive(2ull)); + auto calledPromise = NThreading::NewPromise(); + int time = 0; + setup.Settings.EventHandlers_.DataReceivedHandler([&](TReadSessionEvent::TDataReceivedEvent& event) { + for (ui32 i = 0; i < event.GetMessages().size(); ++i) { + ++time; + UNIT_ASSERT_VALUES_EQUAL(event.GetMessages()[i].GetData(), TStringBuilder() << "message" << time); + + if (time == 2) { + calledPromise.SetValue(); + } + } + }); + setup.SuccessfulInit(); + TPartitionStream::TPtr stream = setup.CreatePartitionStream(); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(1) + .Batch("src_id") + .CompressMessage(1, "message1")); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(2) + .Batch("src_id") + .CompressMessage(2, "message2")); + + // + // when the PartitionStreamClosed arrives the raw messages are deleted + // we give time to process the messages + // + Sleep(TDuration::Seconds(2)); + + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .ForcefulReleasePartitionStream()); + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TPartitionStreamClosedEvent); + calledPromise.GetFuture().Wait(); + } + + Y_UNIT_TEST(PartitionStreamCallbacks) { + TReadSessionImplTestSetup setup; + bool createCalled = false; + setup.Settings.EventHandlers_.CreatePartitionStreamHandler([&](TReadSessionEvent::TCreatePartitionStreamEvent&) { + createCalled = true; + }); + auto destroyCalledPromise = NThreading::NewPromise(); + auto destroyCalled = destroyCalledPromise.GetFuture(); + setup.Settings.EventHandlers_.DestroyPartitionStreamHandler([destroyCalledPromise = std::move(destroyCalledPromise)](TReadSessionEvent::TDestroyPartitionStreamEvent& event) mutable { + destroyCalledPromise.SetValue(std::move(event)); + }); + auto closedCalledPromise = NThreading::NewPromise(); + auto closedCalled = closedCalledPromise.GetFuture(); + setup.Settings.EventHandlers_.PartitionStreamClosedHandler([&](TReadSessionEvent::TPartitionStreamClosedEvent&) { + closedCalledPromise.SetValue(); + }); + + setup.SuccessfulInit(); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .CreatePartitionStream()); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(1) + .Batch("src_id") + .CompressMessage(1, "message1")); + + TMaybe event = setup.EventsQueue->GetEvent(true); + UNIT_ASSERT(event); + UNIT_ASSERT_EVENT_TYPE(*event, TReadSessionEvent::TDataReceivedEvent); + + UNIT_ASSERT(createCalled); + UNIT_ASSERT(!destroyCalled.HasValue()); + UNIT_ASSERT(!closedCalled.HasValue()); + + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .ReleasePartitionStream()); + destroyCalled.Wait(); + UNIT_ASSERT(!closedCalled.HasValue()); + + destroyCalled.ExtractValue().Confirm(); + closedCalled.Wait(); + } + + Y_UNIT_TEST(CommonHandler) { + bool createCalled = false; + bool dataCalled = false; + auto callPromise = NThreading::NewPromise(); + TReadSessionImplTestSetup setup; + setup.Settings.EventHandlers_.CommonHandler([&](TReadSessionEvent::TEvent& event) { + if (std::holds_alternative(event)) { + UNIT_ASSERT(!createCalled); + createCalled = true; + } else if (std::holds_alternative(event)) { + UNIT_ASSERT(!dataCalled); + dataCalled = true; + } else { + UNIT_ASSERT(false); + } + if (createCalled && dataCalled) { + UNIT_ASSERT_C(!callPromise.HasValue(), "Event: " << DebugString(event)); + callPromise.SetValue(); + } + }); + setup.SuccessfulInit(); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .CreatePartitionStream()); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(1) + .Batch("src_id") + .CompressMessage(1, "message1")); + + callPromise.GetFuture().Wait(); + } + + void SimpleDataHandlersImpl(bool withCommit, bool withGracefulRelease) { + TReadSessionImplTestSetup setup; + auto dataReceived = std::make_shared>(NThreading::NewPromise()); + auto dataReceivedFuture = dataReceived->GetFuture(); + std::shared_ptr> dataReceivedEvent = std::make_shared>(); + setup.Settings.EventHandlers_.SimpleDataHandlers([dataReceivedEvent,dataReceived](TReadSessionEvent::TDataReceivedEvent& event) mutable { + *dataReceivedEvent = std::move(event); + dataReceived->SetValue(); + }, withCommit, withGracefulRelease); + setup.SuccessfulInit(); + + auto commitCalled = std::make_shared>(NThreading::NewPromise()); + auto commitCalledFuture = commitCalled->GetFuture(); + + if (withCommit) { + EXPECT_CALL(*setup.MockProcessor, OnCommitRequest(_)) + .WillOnce([commitCalled](){ commitCalled->SetValue(); }); + } + + auto destroyCalled = std::make_shared>(NThreading::NewPromise()); + auto destroyCalledFuture = destroyCalled->GetFuture(); + EXPECT_CALL(*setup.MockProcessor, OnReleasedRequest(_)) + .WillOnce([destroyCalled](){ destroyCalled->SetValue(); }); + + EXPECT_CALL(*setup.MockProcessor, OnStartReadRequest(_)); + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .CreatePartitionStream()); + + UNIT_ASSERT(!dataReceivedFuture.Wait(TDuration::MilliSeconds(100))); + + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .PartitionData(1) + .Batch("src_id") + .CompressMessage(1, "msg") + .Batch("src_id") + .CompressMessage(2, "msg")); + + dataReceivedFuture.Wait(); + UNIT_ASSERT(*dataReceivedEvent); + + UNIT_ASSERT_VALUES_EQUAL((*dataReceivedEvent)->GetMessages().size(), 2); + + if (withCommit) { + commitCalledFuture.Wait(); + } else { + UNIT_ASSERT(!commitCalledFuture.Wait(TDuration::MilliSeconds(100))); + } + + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .ReleasePartitionStream()); + + if (!withCommit && withGracefulRelease) { + UNIT_ASSERT(!destroyCalledFuture.Wait(TDuration::MilliSeconds(100))); + + (*dataReceivedEvent)->Commit(); + UNIT_ASSERT(!destroyCalledFuture.Wait(TDuration::MilliSeconds(100))); + } + + if (withCommit) { + commitCalledFuture.Wait(); + } + if (withCommit || withGracefulRelease) { + setup.MockProcessor->AddServerResponse(TMockReadSessionProcessor::TServerReadInfo() + .CommitAcknowledgement(1)); + } + + destroyCalledFuture.Wait(); + + // Guarantee that all callbacks are finished. + setup.Session->Abort(); + } + + Y_UNIT_TEST(SimpleDataHandlers) { + SimpleDataHandlersImpl(false, false); + } + + Y_UNIT_TEST(SimpleDataHandlersWithCommit) { + SimpleDataHandlersImpl(true, false); + } + + Y_UNIT_TEST(SimpleDataHandlersWithGracefulRelease) { + SimpleDataHandlersImpl(false, true); + } + + Y_UNIT_TEST(SimpleDataHandlersWithGracefulReleaseWithCommit) { + SimpleDataHandlersImpl(true, true); + } + + Y_UNIT_TEST(LOGBROKER_7702) { + using namespace NYdb::NPersQueue; + + using TServiceEvent = + typename NTopic::TAReadSessionEvent::TPartitionStreamStatusEvent; + +#define UNIT_ASSERT_CONTROL_EVENT() \ + { \ + using TExpectedEvent = typename NTopic::TAReadSessionEvent::TPartitionStreamStatusEvent; \ +\ + size_t maxByteSize = std::numeric_limits::max();\ + NTopic::TUserRetrievedEventsInfoAccumulator accumulator; \ +\ + auto event = sessionQueue.GetEventImpl(maxByteSize, accumulator); \ +\ + UNIT_ASSERT(std::holds_alternative(event.GetEvent()));\ + } + +#define UNIT_ASSERT_DATA_EVENT(count) \ + { \ + using TExpectedEvent = typename NTopic::TAReadSessionEvent::TDataReceivedEvent; \ +\ + size_t maxByteSize = std::numeric_limits::max(); \ + NTopic::TUserRetrievedEventsInfoAccumulator accumulator; \ +\ + auto event = sessionQueue.GetEventImpl(maxByteSize, accumulator); \ +\ + UNIT_ASSERT(std::holds_alternative(event.GetEvent())); \ + UNIT_ASSERT_VALUES_EQUAL(std::get(event.GetEvent()).GetMessagesCount(), count); \ + } + + NTopic::TAReadSessionSettings settings; + std::shared_ptr session; + auto cbCtx = std::make_shared>(session); + TReadSessionEventsQueue sessionQueue{settings}; + + auto stream = MakeIntrusive(1ull, + "", + "", + 1ull, + 1ull, + 1ull, + 0ull, + cbCtx); + + NTopic::TPartitionData message; + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage_DataBatch_Batch* batch = + message.mutable_batches()->Add(); + Ydb::PersQueue::V1::MigrationStreamingReadServerMessage_DataBatch_MessageData* messageData = + batch->mutable_message_data()->Add(); + *messageData->mutable_data() = "*"; + + auto data = std::make_shared>(std::move(message), + cbCtx, + false, + 0); + + std::atomic ready = true; + + stream->InsertDataEvent(0, 0, data, ready); + stream->InsertEvent(TServiceEvent{stream, 0, 0, 0, {}}); + stream->InsertDataEvent(0, 0, data, ready); + stream->InsertDataEvent(0, 0, data, ready); + stream->InsertEvent(TServiceEvent{stream, 0, 0, 0, {}}); + stream->InsertEvent(TServiceEvent{stream, 0, 0, 0, {}}); + stream->InsertDataEvent(0, 0, data, ready); + + TDeferredActions actions; + + TPartitionStreamImpl::SignalReadyEvents(stream, + &sessionQueue, + actions); + + UNIT_ASSERT_DATA_EVENT(1); + UNIT_ASSERT_CONTROL_EVENT(); + UNIT_ASSERT_DATA_EVENT(2); + UNIT_ASSERT_CONTROL_EVENT(); + UNIT_ASSERT_CONTROL_EVENT(); + UNIT_ASSERT_DATA_EVENT(1); + +#undef UNIT_ASSERT_CONTROL_EVENT +#undef UNIT_ASSERT_DATA_EVENT + + cbCtx->Cancel(); + } +} diff --git a/src/client/persqueue_public/ut/retry_policy_ut.cpp b/src/client/persqueue_public/ut/retry_policy_ut.cpp new file mode 100644 index 00000000000..ec26f55d292 --- /dev/null +++ b/src/client/persqueue_public/ut/retry_policy_ut.cpp @@ -0,0 +1,416 @@ +#include + +#include +#include + + +using namespace NThreading; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; +using namespace NPersQueue; + +namespace NYdb::NPersQueue::NTests { + +Y_UNIT_TEST_SUITE(RetryPolicy) { + Y_UNIT_TEST(TWriteSession_TestPolicy) { + TYdbPqWriterTestHelper helper(TEST_CASE_NAME); + helper.Write(true); + helper.Policy->Initialize(); // Thus ignoring possible early retries on "cluster initializing" + auto doBreakDown = [&] () { + helper.Policy->ExpectBreakDown(); + NThreading::TPromise retriesPromise = NThreading::NewPromise(); + Cerr << "WAIT for retries...\n"; + helper.Policy->WaitForRetries(30, retriesPromise); + Cerr << "KICK tablets\n"; + helper.Setup->KickTablets(); + + auto f1 = helper.Write(false); + auto f2 = helper.Write(); + + auto retriesFuture = retriesPromise.GetFuture(); + retriesFuture.Wait(); + Cerr << "WAIT for retries done\n"; + + NThreading::TPromise repairPromise = NThreading::NewPromise(); + auto repairFuture = repairPromise.GetFuture(); + helper.Policy->WaitForRepair(repairPromise); + + + Cerr << "ALLOW tablets\n"; + helper.Setup->AllowTablets(); + + Cerr << "WAIT for repair\n"; + repairFuture.Wait(); + Cerr << "REPAIR done\n"; + f1.Wait(); + f2.Wait(); + helper.Write(true); + }; + doBreakDown(); + doBreakDown(); + + } + Y_UNIT_TEST(TWriteSession_TestBrokenPolicy) { + TYdbPqWriterTestHelper helper(TEST_CASE_NAME); + helper.Write(); + helper.Policy->Initialize(); + helper.Policy->ExpectFatalBreakDown(); + helper.EventLoop->AllowStop(); + auto f1 = helper.Write(false); + helper.Setup->KickTablets(); + helper.Write(false); + + helper.EventLoop->WaitForStop(); + UNIT_ASSERT(!f1.HasValue()); + helper.Setup = nullptr; + + }; + + Y_UNIT_TEST(TWriteSession_RetryOnTargetCluster) { + auto setup1 = std::make_shared(TEST_CASE_NAME, false); + SDKTestSetup setup2("RetryOnTargetCluster_Dc2"); + setup1->AddDataCenter("dc2", setup2, false); + setup1->Start(); + auto retryPolicy = std::make_shared(); + auto settings = setup1->GetWriteSessionSettings(); + settings.ClusterDiscoveryMode(EClusterDiscoveryMode::On); + settings.PreferredCluster("dc2"); + settings.AllowFallbackToOtherClusters(false); + settings.RetryPolicy(retryPolicy); + + retryPolicy->Initialize(); + retryPolicy->ExpectBreakDown(); + + auto& client = setup1->GetPersQueueClient(); + Cerr << "=== Create write session \n"; + auto writer = client.CreateWriteSession(settings); + + NThreading::TPromise retriesPromise = NThreading::NewPromise(); + auto retriesFuture = retriesPromise.GetFuture(); + retryPolicy->WaitForRetries(3, retriesPromise); + Cerr << "=== Wait retries\n"; + retriesFuture.Wait(); + + Cerr << "=== Enable dc2\n"; + setup1->EnableDataCenter("dc2"); + + NThreading::TPromise repairPromise = NThreading::NewPromise(); + auto repairFuture = repairPromise.GetFuture(); + retryPolicy->WaitForRepair(repairPromise); + Cerr << "=== Wait for repair\n"; + repairFuture.Wait(); + Cerr << "=== Close writer\n"; + writer->Close(); + } + + Y_UNIT_TEST(TWriteSession_SwitchBackToLocalCluster) { + Cerr << "====Start test\n"; + + auto setup1 = std::make_shared(TEST_CASE_NAME, false); + SDKTestSetup setup2("SwitchBackToLocalCluster", false); + setup2.SetSingleDataCenter("dc2"); + setup2.AddDataCenter("dc1", *setup1, true); + setup1->AddDataCenter("dc2", setup2, true); + setup1->Start(); + setup2.Start(false); + Cerr << "=== Start session 1\n"; + auto helper = MakeHolder("", nullptr, TString(), setup1); + helper->Write(true); + auto retryPolicy = helper->Policy; + retryPolicy->Initialize(); + + auto waitForReconnect = [&](bool enable) { + Cerr << "=== Expect breakdown\n"; + retryPolicy->ExpectBreakDown(); + + NThreading::TPromise retriesPromise = NThreading::NewPromise(); + auto retriesFuture = retriesPromise.GetFuture(); + retryPolicy->WaitForRetries(1, retriesPromise); + + NThreading::TPromise repairPromise = NThreading::NewPromise(); + auto repairFuture = repairPromise.GetFuture(); + retryPolicy->WaitForRepair(repairPromise); + + if (enable) { + Cerr << "===Enabled DC1\n"; + setup1->EnableDataCenter("dc1"); + setup2.EnableDataCenter("dc1"); + } else { + Cerr << "===Disabled DC1\n"; + setup1->DisableDataCenter("dc1"); + setup2.DisableDataCenter("dc1"); + } + Sleep(TDuration::Seconds(5)); + + retriesFuture.Wait(); + repairFuture.Wait(); + }; + Cerr << "===Wait for 1st reconnect\n"; + waitForReconnect(false); + Cerr << "===Wait for 2nd reconnect\n"; + waitForReconnect(true); + } + + Y_UNIT_TEST(TWriteSession_SeqNoShift) { + auto setup1 = std::make_shared(TEST_CASE_NAME, false, TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_TRACE); + SDKTestSetup setup2("SeqNoShift_Dc2", false, TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_TRACE); + setup2.SetSingleDataCenter("dc2"); + setup2.AddDataCenter("dc1", *setup1, true); + setup2.Start(true, false); + setup1->AddDataCenter("dc2", setup2, true); + setup1->Start(true, false); + + TString sourceId1 = SDKTestSetup::GetTestMessageGroupId() + "1"; + TString sourceId2 = SDKTestSetup::GetTestMessageGroupId() + "2"; + auto writer1 = MakeHolder("", nullptr, "dc1", setup1, sourceId1 , true); + auto writer2 = MakeHolder("", nullptr, "dc1", setup1, sourceId2, true); + + auto settings = setup1->GetWriteSessionSettings(); + auto& client = setup1->GetPersQueueClient(); + + //! Fill data in dc1 1 with SeqNo = 1..10 for 2 different SrcId + Cerr << "===Write 10 messages into every writer\n"; + for (auto i = 0; i != 10; i++) { + writer1->Write(true); // 1 + writer2->Write(true); // 1 + } + Cerr << "===Messages were written\n"; + + Cerr << "===Disable dc1\n"; + //! Leave only dc2 available + writer1->Policy->ExpectBreakDown(); + writer2->Policy->ExpectBreakDown(); + setup1->DisableDataCenter("dc1"); + writer1->Policy->WaitForRetriesSync(1); + writer2->Policy->WaitForRetriesSync(1); + + Cerr << "===Recreate writers\n"; + + //! Re-create writers, kill previous sessions. New sessions will connect to dc2. + writer1 = MakeHolder("", nullptr, TString(), setup1, sourceId1, true); + writer2 = MakeHolder("", nullptr, TString(), setup1, sourceId2, true); + + //! Write some data and await confirmation - just to ensure sessions are started. + Cerr << "===Write one message into every writer\n"; + writer1->Write(true); + writer2->Write(true); + Cerr << "===Messages were written\n"; + + //! Leave no available DCs + writer1->Policy->ExpectBreakDown(); + writer2->Policy->ExpectBreakDown(); + writer1->Policy->Initialize(); + writer2->Policy->Initialize(); + + Cerr << "===Disable dc2\n"; + setup1->DisableDataCenter("dc2"); + Cerr << "===Wait for retries after initial dc2 shutdown\n"; + writer1->Policy->WaitForRetriesSync(1); + writer2->Policy->WaitForRetriesSync(1); + + //! Put some data inflight. It cannot be written now, but SeqNo will be assigned. + Cerr << "===Write four async message into every writer\n"; + for (auto i = 0; i != 3; i++) { + writer1->Write(false); + writer2->Write(false); + } + auto f1 = writer1->Write(false); + auto f2 = writer2->Write(false); + + //! Enable DC1. Now writers gonna write collected data to DC1 having LastSeqNo = 10 + //! (because of data written in the very beginning), and inflight data has SeqNo assigned = 2..5, + //! so the SeqNo shift takes place. + Cerr << "===Enable dc2\n"; + setup1->EnableDataCenter("dc1"); + + Cerr << "===Wait for writes to complete\n"; + f1.Wait(); + f2.Wait(); + Cerr << "===Messages were written\n"; + + //! Writer1 is not used any more. + writer1->EventLoop->AllowStop(); + writer1 = nullptr; + + Cerr << "===Writer 1 closed\n"; + writer2->Policy->ExpectBreakDown(); + writer2->Policy->Initialize(); + + //! For the second writer, do switchback to dc2. + Cerr << "===Disable dc1\n"; + setup1->DisableDataCenter("dc1"); + Cerr << "===Wait for retries after dc1 shutdown\n"; + writer2->Policy->WaitForRetriesSync(1); + + //! Put some data inflight again; + Cerr << "===Write four async messages into writer2\n"; + for (auto i = 0; i != 3; i++) { + writer2->Write(false); + } + f2 = writer2->Write(false); + + Cerr << "===Enable dc2\n"; + setup1->EnableDataCenter("dc2"); + + f2.Wait(); + Cerr << "===Messages were written\n"; + + writer2->EventLoop->AllowStop(); + writer2->Policy->ExpectBreakDown(); + writer2 = nullptr; + + Cerr << "===Enable dc1\n"; + setup1->EnableDataCenter("dc1"); + auto CheckSeqNo = [&] (const TString& dcName, ui64 expectedSeqNo) { + settings.PreferredCluster(dcName); + settings.AllowFallbackToOtherClusters(false); + settings.RetryPolicy(nullptr); //switch to default policy; + auto writer = client.CreateWriteSession(settings); + auto seqNo = writer->GetInitSeqNo().GetValueSync(); + UNIT_ASSERT_VALUES_EQUAL(seqNo, expectedSeqNo); + writer->Close(TDuration::Zero()); + }; + + //!check SeqNo in both DC. For writer1 We expect 14 messages in DC1 + //! (10 written initially + 4 written after reconnect) and 1 message in DC2 (only initial message). + settings.MessageGroupId(sourceId1); + Cerr << "===Check SeqNo writer1, dc2\n"; + CheckSeqNo("dc2", 1); + Cerr << "===Check SeqNo writer1, dc1\n"; + CheckSeqNo("dc1", 14); + + //! Check SeqNo for writer 2; Expect to have 6 messages on DC2 with MaxSeqNo = 6; + settings.MessageGroupId(sourceId2); + Cerr << "===Check SeqNo writer2 dc1\n"; + CheckSeqNo("dc1", 14); + //! DC2 has no shift in SeqNo since 5 messages were written to dc 1. + Cerr << "===Check SeqNo writer2 dc2\n"; + CheckSeqNo("dc2", 9); + + + auto readSession = client.CreateReadSession(setup1->GetReadSessionSettings()); + + bool stop = false; + THashMap seqNoByClusterSrc1 = { + {"dc1", 0}, + {"dc2", 0} + }; + auto SeqNoByClusterSrc2 = seqNoByClusterSrc1; + + THashMap MsgCountByClusterSrc1 = { + {"dc1", 14}, + {"dc2", 1} + }; + THashMap MsgCountByClusterSrc2 = { + {"dc1", 14}, + {"dc2", 5} + }; + ui32 clustersPendingSrc1 = 2; + ui32 clustersPendingSrc2 = 2; + + while (!stop && (clustersPendingSrc2 || clustersPendingSrc1)) { + Cerr << "===Get event on client\n"; + auto event = *readSession->GetEvent(true); + std::visit(TOverloaded { + [&](TReadSessionEvent::TDataReceivedEvent& event) { + Cerr << "===Data event\n"; + auto& clusterName = event.GetPartitionStream()->GetCluster(); + for (auto& message: event.GetMessages()) { + TString sourceId = message.GetMessageGroupId(); + ui32 seqNo = message.GetSeqNo(); + if (sourceId == sourceId1) { + UNIT_ASSERT_VALUES_EQUAL(seqNo, seqNoByClusterSrc1[clusterName] + 1); + seqNoByClusterSrc1[clusterName]++; + auto& msgRemaining = MsgCountByClusterSrc1[clusterName]; + UNIT_ASSERT(msgRemaining > 0); + msgRemaining--; + if (!msgRemaining) + clustersPendingSrc1--; + } else { + UNIT_ASSERT_VALUES_EQUAL(sourceId, sourceId2); + auto& prevSeqNo = SeqNoByClusterSrc2[clusterName]; + if (clusterName == "dc1") { + UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 1); + prevSeqNo++; + } else { + UNIT_ASSERT_VALUES_EQUAL(clusterName, "dc2"); + if (prevSeqNo == 0) { + UNIT_ASSERT_VALUES_EQUAL(seqNo, 1); + } else if (prevSeqNo == 1) { + UNIT_ASSERT_VALUES_EQUAL(seqNo, 6); + } else { + UNIT_ASSERT_VALUES_EQUAL(seqNo, prevSeqNo + 1); + } + prevSeqNo = seqNo; + } + auto& msgRemaining = MsgCountByClusterSrc2[clusterName]; + UNIT_ASSERT(msgRemaining > 0); + msgRemaining--; + if (!msgRemaining) + clustersPendingSrc2--; + } + message.Commit(); + } + }, + [&](TReadSessionEvent::TCommitAcknowledgementEvent&) { + }, + [&](TReadSessionEvent::TCreatePartitionStreamEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TDestroyPartitionStreamEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TPartitionStreamStatusEvent&) { + Cerr << "===Status event\n"; + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TReadSessionEvent::TPartitionStreamClosedEvent&) { + Cerr << "===Stream closed event\n"; + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TSessionClosedEvent& event) { + Cerr << "===Got close event: " << event.DebugString(); + stop = true; + } + + }, event); + } + UNIT_ASSERT_VALUES_EQUAL(clustersPendingSrc1 || clustersPendingSrc2, 0); + } + Y_UNIT_TEST(RetryWithBatching) { + auto setup = std::make_shared(TEST_CASE_NAME); + auto retryPolicy = std::make_shared(); + auto settings = setup->GetWriteSessionSettings() + .BatchFlushInterval(TDuration::Seconds(1000)) // Batch on size, not on time. + .BatchFlushSizeBytes(100) + .RetryPolicy(retryPolicy); + auto& client = setup->GetPersQueueClient(); + auto writer = client.CreateWriteSession(settings); + auto event = *writer->GetEvent(true); + Cerr << NYdb::NPersQueue::DebugString(event) << "\n"; + UNIT_ASSERT(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + TString message = "1234567890"; + ui64 seqNo = 0; + setup->KickTablets(); + setup->WaitForTabletsDown(); + + writer->Write(std::move(continueToken), message, ++seqNo); + retryPolicy->ExpectBreakDown(); + retryPolicy->WaitForRetriesSync(3); + while (seqNo < 10) { + auto event = *writer->GetEvent(true); + Cerr << NYdb::NPersQueue::DebugString(event) << "\n"; + UNIT_ASSERT(std::holds_alternative(event)); + writer->Write( + std::move(std::get(event).ContinuationToken), + message, ++seqNo + ); + } + + setup->AllowTablets(); + retryPolicy->WaitForRepairSync(); + WaitMessagesAcked(writer, 1, seqNo); + } +}; +}; //NYdb::NPersQueue::NTests diff --git a/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.cpp b/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.cpp new file mode 100644 index 00000000000..0ef9b8750f3 --- /dev/null +++ b/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.cpp @@ -0,0 +1,102 @@ +#include "data_plane_helpers.h" + +namespace NKikimr::NPersQueueTests { + + using namespace NYdb::NPersQueue; + + std::shared_ptr CreateWriter( + NYdb::TDriver& driver, + const NYdb::NPersQueue::TWriteSessionSettings& settings, + std::shared_ptr creds + ) { + TPersQueueClientSettings clientSettings; + if (creds) clientSettings.CredentialsProviderFactory(creds); + return TPersQueueClient(driver, clientSettings).CreateWriteSession(TWriteSessionSettings(settings).ClusterDiscoveryMode(EClusterDiscoveryMode::Off)); + } + + std::shared_ptr CreateWriter( + NYdb::TDriver& driver, + const TString& topic, + const TString& sourceId, + std::optional partitionGroup, + std::optional codec, + std::optional reconnectOnFailure, + std::shared_ptr creds + ) { + auto settings = TWriteSessionSettings().Path(topic).MessageGroupId(sourceId); + if (partitionGroup) settings.PartitionGroupId(*partitionGroup); + settings.RetryPolicy((reconnectOnFailure && *reconnectOnFailure) ? NYdb::NPersQueue::IRetryPolicy::GetDefaultPolicy() : NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()); + if (codec) { + if (*codec == "raw") + settings.Codec(ECodec::RAW); + if (*codec == "zstd") + settings.Codec(ECodec::ZSTD); + if (*codec == "lzop") + settings.Codec(ECodec::LZOP); + } + return CreateWriter(driver, settings, creds); + } + + std::shared_ptr CreateSimpleWriter( + NYdb::TDriver& driver, + const NYdb::NPersQueue::TWriteSessionSettings& settings + ) { + return TPersQueueClient(driver).CreateSimpleBlockingWriteSession(TWriteSessionSettings(settings).ClusterDiscoveryMode(EClusterDiscoveryMode::Off)); + } + + std::shared_ptr CreateSimpleWriter( + NYdb::TDriver& driver, + const TString& topic, + const TString& sourceId, + std::optional partitionGroup, + std::optional codec, + std::optional reconnectOnFailure, + THashMap sessionMeta + ) { + auto settings = TWriteSessionSettings().Path(topic).MessageGroupId(sourceId); + if (partitionGroup) settings.PartitionGroupId(*partitionGroup); + settings.RetryPolicy((reconnectOnFailure && *reconnectOnFailure) ? NYdb::NPersQueue::IRetryPolicy::GetDefaultPolicy() : NYdb::NPersQueue::IRetryPolicy::GetNoRetryPolicy()); + if (codec) { + if (*codec == "raw") + settings.Codec(ECodec::RAW); + if (*codec == "zstd") + settings.Codec(ECodec::ZSTD); + if (*codec == "lzop") + settings.Codec(ECodec::LZOP); + } + settings.MaxMemoryUsage(1024*1024*1024*1024ll); + settings.Meta_.Fields = sessionMeta; + return CreateSimpleWriter(driver, settings); + } + + std::shared_ptr CreateReader( + NYdb::TDriver& driver, + const NYdb::NPersQueue::TReadSessionSettings& settings, + std::shared_ptr creds + ) { + TPersQueueClientSettings clientSettings; + if (creds) clientSettings.CredentialsProviderFactory(creds); + return TPersQueueClient(driver, clientSettings).CreateReadSession(TReadSessionSettings(settings).DisableClusterDiscovery(true)); + } + + TMaybe GetNextMessageSkipAssignment(std::shared_ptr& reader, TDuration timeout) { + while (true) { + auto future = reader->WaitEvent(); + future.Wait(timeout); + + TMaybe event = reader->GetEvent(false, 1); + if (!event) + return {}; + if (auto dataEvent = std::get_if(&*event)) { + return *dataEvent; + } else if (auto* createPartitionStreamEvent = std::get_if(&*event)) { + createPartitionStreamEvent->Confirm(); + } else if (auto* destroyPartitionStreamEvent = std::get_if(&*event)) { + destroyPartitionStreamEvent->Confirm(); + } else if (auto* closeSessionEvent = std::get_if(&*event)) { + return {}; + } + } + return {}; + } +} diff --git a/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.h b/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.h new file mode 100644 index 00000000000..48f90c03f61 --- /dev/null +++ b/src/client/persqueue_public/ut/ut_utils/data_plane_helpers.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include +#include + +namespace NKikimr::NPersQueueTests { + + std::shared_ptr CreateWriter( + NYdb::TDriver& driver, + const NYdb::NPersQueue::TWriteSessionSettings& settings, + std::shared_ptr creds = nullptr + ); + + std::shared_ptr CreateWriter( + NYdb::TDriver& driver, + const TString& topic, + const TString& sourceId, + std::optional partitionGroup = {}, + std::optional codec = {}, + std::optional reconnectOnFailure = {}, + std::shared_ptr creds = nullptr + ); + + std::shared_ptr CreateSimpleWriter( + NYdb::TDriver& driver, + const NYdb::NPersQueue::TWriteSessionSettings& settings + ); + + std::shared_ptr CreateSimpleWriter( + NYdb::TDriver& driver, + const TString& topic, + const TString& sourceId, + std::optional partitionGroup = {}, + std::optional codec = {}, + std::optional reconnectOnFailure = {}, + THashMap sessionMeta = {} + ); + + std::shared_ptr CreateReader( + NYdb::TDriver& driver, + const NYdb::NPersQueue::TReadSessionSettings& settings, + std::shared_ptr creds = nullptr + + ); + + TMaybe GetNextMessageSkipAssignment(std::shared_ptr& reader, TDuration timeout = TDuration::Max()); + +} diff --git a/src/client/persqueue_public/ut/ut_utils/sdk_test_setup.h b/src/client/persqueue_public/ut/ut_utils/sdk_test_setup.h new file mode 100644 index 00000000000..58e26ab5b30 --- /dev/null +++ b/src/client/persqueue_public/ut/ut_utils/sdk_test_setup.h @@ -0,0 +1,249 @@ +#pragma once +#include "test_server.h" +#include +#include +#include + +#define TEST_CASE_NAME (this->Name_) + +namespace NPersQueue { + +class SDKTestSetup { +protected: + TString TestCaseName; + + THolder NetDataFile; + THashMap DataCenters; + TString LocalDC = "dc1"; + TTestServer Server; + TLog Log = CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG); + size_t TopicPartitionsCount = 1; + +public: + SDKTestSetup(const TString& testCaseName, bool start = true, + const TVector& logServices = TTestServer::LOGGED_SERVICES, + NActors::NLog::EPriority logPriority = NActors::NLog::PRI_DEBUG, + ui32 nodeCount = NKikimr::NPersQueueTests::PQ_DEFAULT_NODE_COUNT, + size_t topicPartitionsCount = 1) + : TestCaseName(testCaseName) + , Server(NKikimr::NPersQueueTests::PQSettings(), false, logServices, logPriority, Nothing()) + , TopicPartitionsCount(topicPartitionsCount) + { + InitOptions(nodeCount); + if (start) { + Start(); + } + } + + void InitOptions(ui32 nodeCount = NKikimr::NPersQueueTests::PQ_DEFAULT_NODE_COUNT) { + Log.SetFormatter([testCaseName = TestCaseName](ELogPriority priority, TStringBuf message) { + return TStringBuilder() << TInstant::Now() << " :" << testCaseName << " " << priority << ": " << message << Endl; + }); + Server.ServerSettings.SetNodeCount(nodeCount); + Server.GrpcServerOptions.SetGRpcShutdownDeadline(TDuration::Max()); + // Default TTestServer value for 'MaxReadCookies' is 10. With this value the tests are flapping with two errors: + // 1. 'got more than 10 unordered cookies to commit 12' + // 2. 'got more than 10 uncommitted reads' + Server.ServerSettings.PQConfig.Clear(); + Server.ServerSettings.PQConfig.SetEnabled(true); + Server.ServerSettings.PQConfig.SetRemoteClusterEnabledDelaySec(1); + Server.ServerSettings.PQConfig.SetCloseClientSessionWithEnabledRemotePreferredClusterDelaySec(1); + Server.ServerSettings.PQClusterDiscoveryConfig.SetEnabled(true); + // NOTE(shmel1k@): KIKIMR-14221 + Server.ServerSettings.PQConfig.SetCheckACL(false); + Server.ServerSettings.PQConfig.SetRequireCredentialsInNewProtocol(false); + Server.ServerSettings.PQConfig.SetClusterTablePath("/Root/PQ/Config/V2/Cluster"); + Server.ServerSettings.PQConfig.SetVersionTablePath("/Root/PQ/Config/V2/Versions"); + Server.ServerSettings.PQConfig.SetTopicsAreFirstClassCitizen(false); + Server.ServerSettings.PQConfig.SetRoot("/Root/PQ"); + Server.ServerSettings.PQConfig.MutableQuotingConfig()->SetEnableQuoting(false); + + SetNetDataViaFile("::1/128\t" + GetLocalCluster()); + + auto seed = TInstant::Now().MicroSeconds(); + // This makes failing randomized tests (for example with NUnitTest::RandomString(size, std::rand()) calls) reproducable + Log << TLOG_INFO << "Random seed for debugging is " << seed; + std::srand(seed); + } + + void Start(bool waitInit = true, bool addBrokenDatacenter = false) { + Server.StartServer(false); + Server.AnnoyingClient->InitRoot(); + if (DataCenters.empty()) { + THashMap dataCenters; + dataCenters.emplace("dc1", NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true}); + if (addBrokenDatacenter) { + dataCenters.emplace("dc2", NKikimr::NPersQueueTests::TPQTestClusterInfo{"dc2.logbroker.yandex.net", false}); + } + Server.AnnoyingClient->InitDCs(dataCenters); + Server.AnnoyingClient->CheckClustersList(Server.CleverServer->GetRuntime(), true, dataCenters); + } else { + Server.AnnoyingClient->InitDCs(DataCenters, LocalDC); + Server.AnnoyingClient->CheckClustersList(Server.CleverServer->GetRuntime(), true, DataCenters); + } + Server.AnnoyingClient->InitSourceIds(); + CreateTopic(GetTestTopic(), GetLocalCluster(), TopicPartitionsCount); + if (waitInit) { + Server.WaitInit(GetTestTopic()); + } + } + + static TString GetTestTopic() { + return "test-topic"; + } + + static TString GetTestConsumer() { + return "shared/user"; + } + + static TString GetTestMessageGroupId() { + return "test-message-group-id"; + } + + TString GetLocalCluster() const { + return LocalDC; + } + + TString GetTestTopicPath() const + { + return Server.ServerSettings.PQConfig.GetRoot() + "/" + ::NPersQueue::BuildFullTopicName(GetTestTopic(), LocalDC); + } + + ui16 GetGrpcPort() const { + return Server.GrpcPort; + } + + TSimpleSharedPtr GetPortManager() { + return Server.PortManager; + } + + std::unique_ptr StartGrpcService(const ui16 port, grpc::Service* service) { + grpc::ServerBuilder builder; + builder.AddListeningPort("[::]:" + ToString(port), grpc::InsecureServerCredentials()).RegisterService(service); + std::unique_ptr grpcServer(builder.BuildAndStart()); + return grpcServer; + } + + NYdbGrpc::TServerOptions& GetGrpcServerOptions() { + return Server.GrpcServerOptions; + } + + void SetNetDataViaFile(const TString& netDataTsv) { + NetDataFile = MakeHolder(); + NetDataFile->Write(netDataTsv.Data(), netDataTsv.Size()); + NetDataFile->FlushData(); + Server.ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); + } + + + TLog& GetLog() { + return Log; + } + + TTestServer& GetServer() { + return Server; + } + + NActors::TTestActorRuntime& GetRuntime() { + return *Server.CleverServer->GetRuntime(); + } + + template + void Start(const THolder& obj) { + auto startFuture = obj->Start(); + const auto& initResponse = startFuture.GetValueSync(); + UNIT_ASSERT_C(!initResponse.Response.HasError(), "Failed to start: " << initResponse.Response); + } + + void WriteToTopic(const TVector& data, bool compress = true) { + + auto client = NYdb::NPersQueue::TPersQueueClient(*(Server.AnnoyingClient->GetDriver())); + NYdb::NPersQueue::TWriteSessionSettings settings; + settings.Path(GetTestTopic()).MessageGroupId(GetTestMessageGroupId()); + if (!compress) settings.Codec(NYdb::NPersQueue::ECodec::RAW); + auto writer = client.CreateSimpleBlockingWriteSession(settings); + + for (const TString& d : data) { + Log << TLOG_INFO << "WriteToTopic: " << d; + auto res = writer->Write(d); + UNIT_ASSERT(res); + } + writer->Close(); + } + + void SetSingleDataCenter(const TString& name = "dc1") { + UNIT_ASSERT( + DataCenters.insert(std::make_pair( + name, + NKikimr::NPersQueueTests::TPQTestClusterInfo{TStringBuilder() << "localhost:" << Server.GrpcPort, true} + )).second + ); + LocalDC = name; + } + + void AddDataCenter(const TString& name, const TString& address, bool enabled = true, bool setSelfAsDc = true) { + if (DataCenters.empty() && setSelfAsDc) { + SetSingleDataCenter(); + } + NKikimr::NPersQueueTests::TPQTestClusterInfo info{ + address, + enabled + }; + UNIT_ASSERT(DataCenters.insert(std::make_pair(name, info)).second); + } + + void AddDataCenter(const TString& name, const SDKTestSetup& cluster, bool enabled = true, bool setSelfAsDc = true) { + AddDataCenter(name, TStringBuilder() << "localhost:" << cluster.Server.GrpcPort, enabled, setSelfAsDc); + } + + void EnableDataCenter(const TString& name) { + auto iter = DataCenters.find(name); + UNIT_ASSERT(iter != DataCenters.end()); + Server.AnnoyingClient->UpdateDcEnabled(name, true); + } + void DisableDataCenter(const TString& name) { + auto iter = DataCenters.find(name); + UNIT_ASSERT(iter != DataCenters.end()); + Server.AnnoyingClient->UpdateDcEnabled(name, false); + } + + void ShutdownGRpc() { + Server.ShutdownGRpc(); + } + + void EnableGRpc() { + Server.EnableGRpc(); + Server.WaitInit(GetTestTopic()); + } + + void KickTablets() { + for (ui32 i = 0; i < Server.CleverServer->StaticNodes() + Server.CleverServer->DynamicNodes(); i++) { + Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, false); + } + for (ui32 i = 0; i < Server.CleverServer->StaticNodes() + Server.CleverServer->DynamicNodes(); i++) { + Server.AnnoyingClient->KickNodeInHive(Server.CleverServer->GetRuntime(), i); + } + } + + void WaitForTabletsDown() { + // After calling KickTablets wait until the tablets are in fact dead. + + auto describeResult = Server.AnnoyingClient->Ls(GetTestTopicPath()); + UNIT_ASSERT_C(describeResult->Record.GetPathDescription().HasPersQueueGroup(), describeResult->Record); + auto persQueueGroup = describeResult->Record.GetPathDescription().GetPersQueueGroup(); + for (const auto& p : persQueueGroup.GetPartitions()) { + Server.AnnoyingClient->WaitForTabletDown(Server.CleverServer->GetRuntime(), p.GetTabletId(), true, TDuration::Max()); + } + } + + void AllowTablets() { + for (ui32 i = 0; i < Server.CleverServer->StaticNodes() + Server.CleverServer->DynamicNodes(); i++) { + Server.AnnoyingClient->MarkNodeInHive(Server.CleverServer->GetRuntime(), i, true); + } + } + + void CreateTopic(const TString& topic, const TString& cluster, size_t partitionsCount = 1) { + Server.AnnoyingClient->CreateTopic(BuildFullTopicName(topic, cluster), partitionsCount); + } +}; +} diff --git a/src/client/persqueue_public/ut/ut_utils/test_server.cpp b/src/client/persqueue_public/ut/ut_utils/test_server.cpp new file mode 100644 index 00000000000..73ec953b017 --- /dev/null +++ b/src/client/persqueue_public/ut/ut_utils/test_server.cpp @@ -0,0 +1,13 @@ +#include "test_server.h" + +namespace NPersQueue { + +const TVector TTestServer::LOGGED_SERVICES = { + NKikimrServices::PQ_READ_PROXY, + NKikimrServices::PQ_WRITE_PROXY, + NKikimrServices::PQ_PARTITION_CHOOSER, + NKikimrServices::PERSQUEUE, + NKikimrServices::PERSQUEUE_READ_BALANCER +}; + +} // namespace NPersQueue diff --git a/src/client/persqueue_public/ut/ut_utils/test_server.h b/src/client/persqueue_public/ut/ut_utils/test_server.h new file mode 100644 index 00000000000..4e5600e8f1f --- /dev/null +++ b/src/client/persqueue_public/ut/ut_utils/test_server.h @@ -0,0 +1,171 @@ +#pragma once +#include +#include + +#include + +#include +#include + +#include + +namespace NPersQueue { + +static constexpr int DEBUG_LOG_LEVEL = 7; + +class TTestServer { +public: + TTestServer(const NKikimr::Tests::TServerSettings& settings, + bool start = true, + const TVector& logServices = TTestServer::LOGGED_SERVICES, + NActors::NLog::EPriority logPriority = NActors::NLog::PRI_DEBUG, + TMaybe> portManager = Nothing()) + : PortManager(portManager.GetOrElse(MakeSimpleShared())) + , Port(PortManager->GetPort(2134)) + , GrpcPort(PortManager->GetPort(2135)) + , ServerSettings(settings) + , GrpcServerOptions(NYdbGrpc::TServerOptions().SetHost("[::1]").SetPort(GrpcPort)) + { + auto loggerInitializer = [logServices, logPriority](NActors::TTestActorRuntime& runtime) { + for (auto s : logServices) + runtime.SetLogPriority(s, logPriority); + }; + ServerSettings.SetLoggerInitializer(loggerInitializer); + + ServerSettings.Port = Port; + ServerSettings.SetGrpcPort(GrpcPort); + + if (start) + StartServer(); + } + + TTestServer(bool start = true) + : TTestServer(NKikimr::NPersQueueTests::PQSettings(), start) + { + } + + void StartServer(bool doClientInit = true, TMaybe databaseName = Nothing()) { + Log.SetFormatter([](ELogPriority priority, TStringBuf message) { + return TStringBuilder() << TInstant::Now() << " " << priority << ": " << message << Endl; + }); + + PrepareNetDataFile(); + + CleverServer = MakeHolder(ServerSettings); + CleverServer->EnableGRpc(GrpcServerOptions); + + Log << TLOG_INFO << "TTestServer started on Port " << Port << " GrpcPort " << GrpcPort; + + AnnoyingClient = MakeHolder(ServerSettings, GrpcPort, databaseName); + if (doClientInit) { + AnnoyingClient->FullInit(); + AnnoyingClient->CheckClustersList(CleverServer->GetRuntime()); + } + } + + void ShutdownGRpc() { + CleverServer->ShutdownGRpc(); + } + + void EnableGRpc() { + CleverServer->EnableGRpc(GrpcServerOptions); + } + + void ShutdownServer() { + CleverServer = nullptr; + } + + void RestartServer() { + ShutdownServer(); + StartServer(); + } + + auto GetRuntime() { + return CleverServer->GetRuntime(); + } + + void EnableLogs(const TVector& services = LOGGED_SERVICES, + NActors::NLog::EPriority prio = NActors::NLog::PRI_DEBUG) { + Y_ABORT_UNLESS(CleverServer != nullptr, "Start server before enabling logs"); + for (auto s : services) { + CleverServer->GetRuntime()->SetLogPriority(s, prio); + } + } + + void WaitInit(const TString& topic) { + AnnoyingClient->WaitTopicInit(topic); + } + + bool PrepareNetDataFile(const TString& content = "::1/128\tdc1") { + if (NetDataFile) + return false; + NetDataFile = MakeHolder(); + NetDataFile->Write(content.Data(), content.Size()); + NetDataFile->FlushData(); + ServerSettings.NetClassifierConfig.SetNetDataFilePath(NetDataFile->Name()); + return true; + } + + void UpdateDC(const TString& name, bool local, bool enabled) { + AnnoyingClient->UpdateDC(name, local, enabled); + } + + const NYdb::TDriver& GetDriver() const { + return CleverServer->GetDriver(); + } + + void KillTopicPqrbTablet(const TString& topicPath) { + KillTopicTablets(topicPath, true, false); + } + + void KillTopicPqTablets(const TString& topicPath) { + KillTopicTablets(topicPath, false, true); + } + +private: + void KillTopicTablets(const TString& topicPath, bool killPqrb, bool killPq) { + auto describeResult = AnnoyingClient->Ls(topicPath); + UNIT_ASSERT_C(describeResult->Record.GetPathDescription().HasPersQueueGroup(), describeResult->Record); + auto persQueueGroup = describeResult->Record.GetPathDescription().GetPersQueueGroup(); + + if (killPqrb) + { + Log << TLOG_INFO << "Kill PQRB tablet " << persQueueGroup.GetBalancerTabletID(); + AnnoyingClient->KillTablet(*CleverServer, persQueueGroup.GetBalancerTabletID()); + } + + if (killPq) + { + THashSet restartedTablets; + for (const auto& p : persQueueGroup.GetPartitions()) { + if (restartedTablets.insert(p.GetTabletId()).second) { + Log << TLOG_INFO << "Kill PQ tablet " << p.GetTabletId(); + AnnoyingClient->KillTablet(*CleverServer, p.GetTabletId()); + } + } + } + + CleverServer->GetRuntime()->DispatchEvents(); + } + +public: + TString TestCaseName; + + TSimpleSharedPtr PortManager; + ui16 Port; + ui16 GrpcPort; + + THolder CleverServer; + NKikimr::Tests::TServerSettings ServerSettings; + NYdbGrpc::TServerOptions GrpcServerOptions; + THolder NetDataFile; + + TLog Log = CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG); + + THolder AnnoyingClient; + + + static const TVector LOGGED_SERVICES; +}; + +} // namespace NPersQueue diff --git a/src/client/persqueue_public/ut/ut_utils/test_utils.h b/src/client/persqueue_public/ut/ut_utils/test_utils.h new file mode 100644 index 00000000000..f5841d62773 --- /dev/null +++ b/src/client/persqueue_public/ut/ut_utils/test_utils.h @@ -0,0 +1,74 @@ +#pragma once +#include +#include +#include +#include +#include + +#include "sdk_test_setup.h" + +namespace NPersQueue { + +using namespace NThreading; +using namespace NYdb::NPersQueue; +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; + + +struct TWriteResult { + bool Ok = false; + // No acknowledgement is expected from a writer under test + bool NoWait = false; + TString ResponseDebugString = TString(); +}; + +struct TAcknowledgableMessage { + TString Value; + ui64 SequenceNumber; + TInstant CreatedAt; + TPromise AckPromise; +}; + +class IClientEventLoop { +protected: + std::atomic_bool MayStop; + std::atomic_bool MustStop; + bool Stopped = false; + std::unique_ptr Thread; + TLog Log; + +public: + IClientEventLoop() + : MayStop() + , MustStop() + , MessageBuffer() + {} + + void AllowStop() { + MayStop = true; + } + + void WaitForStop() { + if (!Stopped) { + Log << TLOG_INFO << "Wait for writer to die on itself"; + Thread->Join(); + Log << TLOG_INFO << "Client write event loop stopped"; + } + Stopped = true; + } + + virtual ~IClientEventLoop() { + MustStop = true; + if (!Stopped) { + Log << TLOG_INFO << "Wait for client write event loop to stop"; + Thread->Join(); + Log << TLOG_INFO << "Client write event loop stopped"; + } + Stopped = true; + } + + TManyOneQueue MessageBuffer; + +}; + +} // namespace NPersQueue diff --git a/src/client/persqueue_public/ut/ut_utils/ut_utils.cpp b/src/client/persqueue_public/ut/ut_utils/ut_utils.cpp new file mode 100644 index 00000000000..21b2be71948 --- /dev/null +++ b/src/client/persqueue_public/ut/ut_utils/ut_utils.cpp @@ -0,0 +1,33 @@ +#include "ut_utils.h" + +namespace NYdb::NPersQueue::NTests { + +void WaitMessagesAcked(std::shared_ptr writer, ui64 startSeqNo, ui64 endSeqNo) { + THashSet ackedSeqNo; + while (ackedSeqNo.size() < endSeqNo - startSeqNo + 1) { + auto event = *writer->GetEvent(true); + if (std::holds_alternative(event)) { + continue; + } else { + UNIT_ASSERT(std::holds_alternative(event)); + for (auto& ack : std::get(event).Acks) { + UNIT_ASSERT(!ackedSeqNo.contains(ack.SeqNo)); + UNIT_ASSERT(ack.SeqNo >= startSeqNo && ack.SeqNo <= endSeqNo); + ackedSeqNo.insert(ack.SeqNo); + } + } + } +} + +TSimpleWriteSessionTestAdapter::TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session) + : Session(session) +{} + +ui64 TSimpleWriteSessionTestAdapter::GetAcquiredMessagesCount() const { + if (Session->Writer) + return Session->Writer->TryGetImpl()->MessagesAcquired; + else + return 0; +} + +} diff --git a/src/client/persqueue_public/ut/ut_utils/ut_utils.h b/src/client/persqueue_public/ut/ut_utils/ut_utils.h new file mode 100644 index 00000000000..7da4f7b7f3e --- /dev/null +++ b/src/client/persqueue_public/ut/ut_utils/ut_utils.h @@ -0,0 +1,468 @@ +#pragma once + +#include +#include +#include +#include + +#include + +using namespace NKikimr; +using namespace NKikimr::NPersQueueTests; + +using NYdb::NTopic::IAsyncExecutor; + +namespace NYdb::NPersQueue::NTests { + +class TPersQueueYdbSdkTestSetup : public ::NPersQueue::SDKTestSetup { + THolder Driver; + THolder PersQueueClient; + + TAdaptiveLock Lock; +public: + TPersQueueYdbSdkTestSetup(const TString& testCaseName, bool start = true, + const TVector& logServices = ::NPersQueue::TTestServer::LOGGED_SERVICES, + NActors::NLog::EPriority logPriority = NActors::NLog::PRI_DEBUG, + ui32 nodeCount = NKikimr::NPersQueueTests::PQ_DEFAULT_NODE_COUNT, + size_t topicPartitionsCount = 1) + : SDKTestSetup(testCaseName, start, logServices, logPriority, nodeCount, topicPartitionsCount) + { + } + + ~TPersQueueYdbSdkTestSetup() { + if (PersQueueClient) { + PersQueueClient = nullptr; + } + + if (Driver) { + Driver->Stop(true); + Driver = nullptr; + } + } + + NYdb::TDriver& GetDriver() { + if (!Driver) { + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << Server.GrpcPort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + Driver = MakeHolder(cfg); + } + return *Driver; + } + + NYdb::NPersQueue::TPersQueueClient& GetPersQueueClient() { + with_lock(Lock) { + if (!PersQueueClient) { + PersQueueClient = MakeHolder(GetDriver()); + } + return *PersQueueClient; + } + } + + NYdb::NPersQueue::TReadSessionSettings GetReadSessionSettings() { + NYdb::NPersQueue::TReadSessionSettings settings; + settings + .ConsumerName(GetTestConsumer()) + .AppendTopics(GetTestTopic()); + return settings; + } + + NYdb::NPersQueue::TWriteSessionSettings GetWriteSessionSettings() { + TWriteSessionSettings settings; + settings + .Path(GetTestTopic()) + .MessageGroupId(GetTestMessageGroupId()) + .ClusterDiscoveryMode(EClusterDiscoveryMode::On); + return settings; + } +}; + +struct TYDBClientEventLoop : public ::NPersQueue::IClientEventLoop { +public: + std::shared_ptr Setup; + using TAcksCallback = std::function&)>; + + TYDBClientEventLoop( + std::shared_ptr setup, + IRetryPolicy::TPtr retryPolicy = nullptr, + IExecutor::TPtr compressExecutor = nullptr, + const TString& preferredCluster = TString(), + const TString& sourceId = TString(), + bool autoSeqNo = false + ) + : IClientEventLoop() + , Setup(setup) + , AutoSeqNo(autoSeqNo) + { + Log = Setup->GetLog(); + Thread = std::make_unique([setup, retryPolicy, compressExecutor, preferredCluster, sourceId, this]() { + auto writerConfig = Setup->GetWriteSessionSettings(); + writerConfig.MaxMemoryUsage(100_MB); + if (!sourceId.empty()) { + writerConfig.MessageGroupId(sourceId); + } + if (retryPolicy != nullptr) + writerConfig.RetryPolicy(retryPolicy); + if (compressExecutor != nullptr) + writerConfig.CompressionExecutor(compressExecutor); + if (preferredCluster) + writerConfig.PreferredCluster(preferredCluster); + auto writer = setup->GetPersQueueClient().CreateWriteSession(writerConfig); + + TMaybe continueToken; + NThreading::TFuture waitEventFuture = writer->WaitEvent(); + THashMap> ackPromiseBySequenceNumber; + TDeque> ackPromiseQueue; + while (!MustStop) { + if (!continueToken) { + Log << TLOG_INFO << "Wait for writer event"; + waitEventFuture.Wait(); + } + + bool closed = false; + while (waitEventFuture.HasValue() && !closed) { + TWriteSessionEvent::TEvent event = *writer->GetEvent(true); + waitEventFuture = writer->WaitEvent(); + std::visit(TOverloaded { + [&](const TWriteSessionEvent::TAcksEvent& event) { + for (const auto& ack : event.Acks) { + if (AutoSeqNo) { + UNIT_ASSERT(!ackPromiseQueue.empty()); + ackPromiseQueue.front().SetValue({true, false}); + ackPromiseQueue.pop_front(); + } else { + UNIT_ASSERT(ackPromiseBySequenceNumber.contains(ack.SeqNo)); + ackPromiseBySequenceNumber[ack.SeqNo].SetValue({true, false}); + ackPromiseBySequenceNumber.erase(ack.SeqNo); + } + } + }, + [&](TWriteSessionEvent::TReadyToAcceptEvent& event) { + Log << TLOG_INFO << "Got new continue token"; + continueToken = std::move(event.ContinuationToken); + }, + [&](const TSessionClosedEvent& event) { + Log << TLOG_INFO << "Got close event: " << event.DebugString() << Endl; + if (!MayStop) { + UNIT_ASSERT(MustStop); + UNIT_ASSERT(MessageBuffer.IsEmpty()); + UNIT_ASSERT(ackPromiseBySequenceNumber.empty()); + UNIT_ASSERT(ackPromiseQueue.empty()); + } else { + MustStop = true; + closed = true; + } + } + }, event); + } + + if (continueToken && !MessageBuffer.IsEmpty()) { + ::NPersQueue::TAcknowledgableMessage acknowledgeableMessage; + Y_ABORT_UNLESS(MessageBuffer.Dequeue(acknowledgeableMessage)); + if (AutoSeqNo) { + ackPromiseQueue.emplace_back(acknowledgeableMessage.AckPromise); + } else { + ackPromiseBySequenceNumber.emplace(acknowledgeableMessage.SequenceNumber, + acknowledgeableMessage.AckPromise); + } + Y_ABORT_UNLESS(continueToken); + + TMaybe seqNo = Nothing(); + if (!AutoSeqNo) { + seqNo = acknowledgeableMessage.SequenceNumber; + Log << TLOG_INFO << "[" << sourceId << "] Write messages with sequence numbers " + << acknowledgeableMessage.SequenceNumber; + } + writer->Write( + std::move(*continueToken), + std::move(acknowledgeableMessage.Value), + seqNo, + acknowledgeableMessage.CreatedAt + ); + continueToken = Nothing(); + } + } + Log << TLOG_DEBUG << "Close writer (stop)"; + writer->Close(TDuration::Zero()); + writer = nullptr; + Log << TLOG_DEBUG << "Writer closed"; + }); + Thread->Start(); + } + +private: + bool AutoSeqNo; +}; + +struct TYdbPqTestRetryState : NYdb::NPersQueue::IRetryPolicy::IRetryState { + TYdbPqTestRetryState( + std::function retryCallback, std::function destroyCallback, const TDuration& delay + ) + : RetryDone(retryCallback) + , DestroyDone(destroyCallback) + , Delay(delay) + {} + + TMaybe GetNextRetryDelay(NYdb::EStatus) override { + Cerr << "Test retry state: get retry delay\n"; + RetryDone(); + return Delay; + } + std::function RetryDone; + std::function DestroyDone; + TDuration Delay; + + ~TYdbPqTestRetryState() { + DestroyDone(); + } +}; +struct TYdbPqNoRetryState : NYdb::NPersQueue::IRetryPolicy::IRetryState { + TAtomic DelayCalled = 0; + TMaybe GetNextRetryDelay(NYdb::EStatus) override { + auto res = AtomicSwap(&DelayCalled, 0); + UNIT_ASSERT(!res); + return Nothing(); + } +}; + +struct TYdbPqTestRetryPolicy : IRetryPolicy { + TYdbPqTestRetryPolicy(const TDuration& delay = TDuration::MilliSeconds(2000)) + : Delay(delay) + { + Cerr << "====TYdbPqTestRetryPolicy()\n"; + } + + IRetryState::TPtr CreateRetryState() const override { + Cerr << "====CreateRetryState\n"; + if (AtomicSwap(&OnFatalBreakDown, 0)) { + return std::make_unique(); + } + if (AtomicGet(Initialized_)) + { + Cerr << "====CreateRetryState Initialized\n"; + auto res = AtomicSwap(&OnBreakDown, 0); + UNIT_ASSERT(res); + for (size_t i = 0; i < 100; i++) { + if (AtomicGet(CurrentRetries) == 0) + break; + Sleep(TDuration::MilliSeconds(100)); + } + UNIT_ASSERT(AtomicGet(CurrentRetries) == 0); + } + auto retryCb = [this]() mutable {this->RetryDone();}; + auto destroyCb = [this]() mutable {this->StateDestroyed();}; + return std::make_unique(retryCb, destroyCb, Delay); + } + + void RetryDone() const { + AtomicAdd(CurrentRetries, 1); + auto expected = AtomicGet(RetriesExpected); + if (expected > 0 && AtomicGet(CurrentRetries) >= expected) { + with_lock(Lock) { + RetryPromise.SetValue(); + } + AtomicSet(RetriesExpected, 0); + } + } + void StateDestroyed() const { + auto expected = AtomicSwap(&RepairExpected, 0); + if (expected) { + with_lock(Lock) { + RepairPromise.SetValue(); + } + } + } + void ExpectBreakDown() { + // Either TYdbPqTestRetryPolicy() or Initialize() should be called beforehand in order to set OnBreakDown=0 + Cerr << "====ExpectBreakDown\n"; + for (size_t i = 0; i < 100; i++) { + if (AtomicGet(OnBreakDown) == 0) + break; + Sleep(TDuration::MilliSeconds(100)); + } + UNIT_ASSERT(AtomicGet(OnBreakDown) == 0); + AtomicSet(CurrentRetries, 0); + AtomicSet(OnBreakDown, 1); + } + void ExpectFatalBreakDown() { + AtomicSet(OnFatalBreakDown, 1); + } + + void WaitForRetries(ui64 retryCount, NThreading::TPromise& promise) { + AtomicSet(RetriesExpected, retryCount); + with_lock(Lock) { + RetryPromise = promise; + } + } + void WaitForRetriesSync(ui64 retryCount) { + NThreading::TPromise retriesPromise = NThreading::NewPromise(); + auto retriesFuture = retriesPromise.GetFuture(); + WaitForRetries(retryCount, retriesPromise); + retriesFuture.Wait(); + } + + void WaitForRepair(NThreading::TPromise& promise) { + AtomicSet(RepairExpected, 1 ); + with_lock(Lock) { + RepairPromise = promise; + } + } + + void WaitForRepairSync() { + NThreading::TPromise repairPromise = NThreading::NewPromise(); + auto repairFuture = repairPromise.GetFuture(); + WaitForRepair(repairPromise); + repairFuture.Wait(); + } + + void Initialize() { + AtomicSet(Initialized_, 1); + AtomicSet(CurrentRetries, 0); + } +private: + TDuration Delay; + mutable TAtomic CurrentRetries = 0; + mutable TAtomic Initialized_ = 0; + mutable TAtomic OnBreakDown = 0; + mutable TAtomic OnFatalBreakDown = 0; + mutable NThreading::TPromise RetryPromise; + mutable NThreading::TPromise RepairPromise; + mutable TAtomic RetriesExpected = 0; + mutable TAtomic RepairExpected = 0; + mutable TAdaptiveLock Lock; +}; + +class TYdbPqTestExecutor : public IAsyncExecutor { +public: + TYdbPqTestExecutor(std::shared_ptr> idsQueue) + : Stop() + , ExecIdsQueue(idsQueue) + , Thread([idsQueue, this]() { + while(!Stop) { + TFunction f; + while (TasksQueue.Dequeue(&f)) { + ++CurrentTaskId; + Cerr << "Enqueue task with id " << CurrentTaskId << Endl; + Tasks[CurrentTaskId] = f; + } + ui64 id = 0; + while (ExecIdsQueue->Dequeue(&id)) { + ExecIds.push(id); + Cerr << "Got ok to execute task with id " << id << Endl; + + } + while (!ExecIds.empty()) { + auto id = ExecIds.front(); + auto iter = Tasks.find(id); + if (iter == Tasks.end()) + break; + Cerr << "Executing compression of " << id << Endl; + ExecIds.pop(); + try { + (iter->second)(); + } catch (...) { + Cerr << "Failed on compression call: " << CurrentExceptionMessage() << Endl; + Y_ABORT(); + } + Cerr << "Compression of " << id << " Done\n"; + Tasks.erase(iter); + } + + } + }) + { + } + ~TYdbPqTestExecutor() { + Stop = true; + Thread.Join(); + } + void PostImpl(TVector&& fs) override { + for (auto& f : fs) { + TasksQueue.Enqueue(std::move(f)); + } + } + + void PostImpl(TFunction&& f) override { + TasksQueue.Enqueue(std::move(f)); + } + + void DoStart() override { + Thread.Start(); + } + +private: + std::atomic_bool Stop; + TLockFreeQueue TasksQueue; + std::shared_ptr> ExecIdsQueue; + THashMap Tasks; + TQueue ExecIds; + ui64 CurrentTaskId = 0; + TThread Thread; + +}; + + +struct TYdbPqWriterTestHelper { + std::shared_ptr Setup; + std::shared_ptr Policy; + std::unique_ptr EventLoop; + TIntrusivePtr CompressExecutor; + + TAutoEvent MessagesWrittenToBuffer; + ui64 SeqNo = 1; + TString Message = "message"; +public: + TYdbPqWriterTestHelper( + const TString& name, + std::shared_ptr> executorQueue = nullptr, + const TString& preferredCluster = TString(), + std::shared_ptr setup = nullptr, + const TString& sourceId = TString(), + bool autoSeqNo = false + ) + : Setup(setup ? setup : std::make_shared(name)) + , Policy(std::make_shared()) + { + if (executorQueue) + CompressExecutor = MakeIntrusive(executorQueue); + EventLoop = std::make_unique(Setup, Policy, CompressExecutor, preferredCluster, sourceId, + autoSeqNo); + } + + NThreading::TFuture<::NPersQueue::TWriteResult> Write(bool doWait = false, const TString& message = TString()) { + //auto f = ClientWrite(Message, SeqNo, TInstant::Now()); + auto promise = NThreading::NewPromise<::NPersQueue::TWriteResult>(); + auto log = Setup->GetLog(); + log << TLOG_INFO << "Enqueue message with sequence number " << SeqNo; + EventLoop->MessageBuffer.Enqueue(::NPersQueue::TAcknowledgableMessage{ + message.Empty() ? Message : message, + SeqNo, TInstant::Now(), promise + }); + MessagesWrittenToBuffer.Signal(); + auto f = promise.GetFuture(); + ++SeqNo; + if (doWait) + f.Wait(); + return f; + } + ~TYdbPqWriterTestHelper() { + EventLoop = nullptr; + Setup = nullptr; + CompressExecutor = nullptr; + Policy = nullptr; + } +}; + +class TSimpleWriteSessionTestAdapter { +public: + TSimpleWriteSessionTestAdapter(TSimpleBlockingWriteSession* session); + ui64 GetAcquiredMessagesCount() const; + +private: + TSimpleBlockingWriteSession* Session; +}; + + +void WaitMessagesAcked(std::shared_ptr writer, ui64 startSeqNo, ui64 endSeqNo); +} // namespace NYdb::NPersQueue::NTests diff --git a/src/client/proto/CMakeLists.txt b/src/client/proto/CMakeLists.txt index 7a86074b957..b5dcd5e6548 100644 --- a/src/client/proto/CMakeLists.txt +++ b/src/client/proto/CMakeLists.txt @@ -5,6 +5,7 @@ target_link_libraries(client-ydb_proto PUBLIC api-grpc api-grpc-draft api-protos + lib-operation_id-protos client-ydb_params client-ydb_value yql-public-issue-protos diff --git a/src/client/proto/accessor.cpp b/src/client/proto/accessor.cpp index 22f126dd706..7ec24b47155 100644 --- a/src/client/proto/accessor.cpp +++ b/src/client/proto/accessor.cpp @@ -23,11 +23,11 @@ typename TProtoSettings::Scheme TProtoAccessor::GetProto(ES3Scheme value) { } } -const ::google::protobuf::Map& TProtoAccessor::GetProtoMap(const TParams& params) { +const ::google::protobuf::Map& TProtoAccessor::GetProtoMap(const TParams& params) { return params.GetProtoMap(); } -::google::protobuf::Map* TProtoAccessor::GetProtoMapPtr(TParams& params) { +::google::protobuf::Map* TProtoAccessor::GetProtoMapPtr(TParams& params) { return params.GetProtoMapPtr(); } diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 8e044c36af3..8f897a77254 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -17,7 +17,7 @@ #include #include -#include +#include namespace NYdb::NQuery { @@ -90,7 +90,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public request.set_exec_mode(::Ydb::Query::ExecMode(settings.ExecMode_)); request.set_stats_mode(::Ydb::Query::StatsMode(settings.StatsMode_)); request.mutable_script_content()->set_syntax(::Ydb::Query::Syntax(settings.Syntax_)); - request.mutable_script_content()->set_text(script); + request.mutable_script_content()->set_text(TStringType{script}); SetDuration(settings.ResultsTtl_, *request.mutable_results_ttl()); if (params) { @@ -128,7 +128,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public TAsyncFetchScriptResultsResult FetchScriptResults(const NKikimr::NOperationId::TOperationId& operationId, int64_t resultSetIndex, const TFetchScriptResultsSettings& settings) { auto request = MakeRequest(); - request.set_operation_id(operationId.ToString()); + request.set_operation_id(TStringType{operationId.ToString()}); request.set_result_set_index(resultSetIndex); return FetchScriptResultsImpl(std::move(request), settings); } @@ -136,8 +136,8 @@ class TQueryClient::TImpl: public TClientImplCommon, public TAsyncStatus RollbackTransaction(const std::string& txId, const NYdb::NQuery::TRollbackTxSettings& settings, const TSession& session) { using namespace Ydb::Query; auto request = MakeRequest(); - request.set_session_id(session.GetId()); - request.set_tx_id(txId); + request.set_session_id(TStringType{session.GetId()}); + request.set_tx_id(TStringType{txId}); auto promise = NThreading::NewPromise(); @@ -172,8 +172,8 @@ class TQueryClient::TImpl: public TClientImplCommon, public TAsyncCommitTransactionResult CommitTransaction(const std::string& txId, const NYdb::NQuery::TCommitTxSettings& settings, const TSession& session) { using namespace Ydb::Query; auto request = MakeRequest(); - request.set_session_id(session.GetId()); - request.set_tx_id(txId); + request.set_session_id(TStringType{session.GetId()}); + request.set_tx_id(TStringType{txId}); auto promise = NThreading::NewPromise(); @@ -211,7 +211,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public { using namespace Ydb::Query; auto request = MakeRequest(); - request.set_session_id(session.GetId()); + request.set_session_id(TStringType{session.GetId()}); SetTxSettings(txSettings, request.mutable_tx_settings()); auto promise = NThreading::NewPromise(); @@ -250,7 +250,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public TAsyncFetchScriptResultsResult FetchScriptResultsImpl(Ydb::Query::FetchScriptResultsRequest&& request, const TFetchScriptResultsSettings& settings) { using namespace Ydb::Query; if (!settings.FetchToken_.empty()) { - request.set_fetch_token(settings.FetchToken_); + request.set_fetch_token(TStringType{settings.FetchToken_}); } request.set_rows_limit(settings.RowsLimit_); @@ -457,15 +457,15 @@ class TQueryClient::TImpl: public TClientImplCommon, public return future; } - i64 GetActiveSessionCount() const { + int64_t GetActiveSessionCount() const { return SessionPool_.GetActiveSessions(); } - i64 GetActiveSessionsLimit() const { + int64_t GetActiveSessionsLimit() const { return SessionPool_.GetActiveSessionsLimit(); } - i64 GetCurrentPoolSize() const { + int64_t GetCurrentPoolSize() const { return SessionPool_.GetCurrentPoolSize(); } @@ -564,15 +564,15 @@ TAsyncCreateSessionResult TQueryClient::GetSession(const TCreateSessionSettings& return Impl_->GetSession(settings); } -i64 TQueryClient::GetActiveSessionCount() const { +int64_t TQueryClient::GetActiveSessionCount() const { return Impl_->GetActiveSessionCount(); } -i64 TQueryClient::GetActiveSessionsLimit() const { +int64_t TQueryClient::GetActiveSessionsLimit() const { return Impl_->GetActiveSessionsLimit(); } -i64 TQueryClient::GetCurrentPoolSize() const { +int64_t TQueryClient::GetCurrentPoolSize() const { return Impl_->GetCurrentPoolSize(); } diff --git a/src/client/query/impl/client_session.cpp b/src/client/query/impl/client_session.cpp index f471d37596d..4d37337dfd8 100644 --- a/src/client/query/impl/client_session.cpp +++ b/src/client/query/impl/client_session.cpp @@ -4,7 +4,7 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include +#include namespace NYdb::NQuery { diff --git a/src/client/query/impl/client_session.h b/src/client/query/impl/client_session.h index 4400c30d3b3..d4f31ef4adb 100644 --- a/src/client/query/impl/client_session.h +++ b/src/client/query/impl/client_session.h @@ -3,7 +3,7 @@ #include #include -#include +#include namespace NYdb::NQuery { diff --git a/src/client/query/impl/exec_query.cpp b/src/client/query/impl/exec_query.cpp index 827d207200f..bab443947c7 100644 --- a/src/client/query/impl/exec_query.cpp +++ b/src/client/query/impl/exec_query.cpp @@ -9,7 +9,7 @@ #include -#include +#include namespace NYdb::NQuery { @@ -206,16 +206,16 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { TFuture> StreamExecuteQueryImpl( const std::shared_ptr& connections, const TDbDriverStatePtr& driverState, - const std::string& query, const TTxControl& txControl, const ::google::protobuf::Map* params, + const std::string& query, const TTxControl& txControl, const ::google::protobuf::Map* params, const TExecuteQuerySettings& settings, const std::optional& session) { auto request = MakeRequest(); request.set_exec_mode(::Ydb::Query::ExecMode(settings.ExecMode_)); request.set_stats_mode(::Ydb::Query::StatsMode(settings.StatsMode_)); - request.mutable_query_content()->set_text(query); + request.mutable_query_content()->set_text(TStringType{query}); request.mutable_query_content()->set_syntax(::Ydb::Query::Syntax(settings.Syntax_)); if (session.has_value()) { - request.set_session_id(session->GetId()); + request.set_session_id(TStringType{session->GetId()}); } else if ((txControl.TxSettings_.has_value() && !txControl.CommitTx_) || txControl.TxId_.has_value()) { throw TContractViolation("Interactive tx must use explisit session"); } @@ -232,7 +232,7 @@ TFuture> StreamExecuteQueryIm auto requestTxControl = request.mutable_tx_control(); requestTxControl->set_commit_tx(txControl.CommitTx_); if (txControl.TxId_) { - requestTxControl->set_tx_id(*txControl.TxId_); + requestTxControl->set_tx_id(TStringType{txControl.TxId_.value()}); } else { Y_ASSERT(txControl.TxSettings_); SetTxSettings(*txControl.TxSettings_, requestTxControl->mutable_begin_tx()); diff --git a/src/client/query/query.cpp b/src/client/query/query.cpp index 971d33014c0..31abc4ddb7e 100644 --- a/src/client/query/query.cpp +++ b/src/client/query/query.cpp @@ -1,6 +1,6 @@ #include -#include +#include namespace NYdb::NQuery { diff --git a/src/client/query/stats.cpp b/src/client/query/stats.cpp index ca28cadf4d2..181c92c7a7e 100644 --- a/src/client/query/stats.cpp +++ b/src/client/query/stats.cpp @@ -1,6 +1,8 @@ #include -#include +#include + +#include #include @@ -31,7 +33,7 @@ std::string TExecStats::ToString(bool withPlan) const { proto.clear_query_ast(); } - std::string res; + TStringType res; ::google::protobuf::TextFormat::PrintToString(proto, &res); return res; } diff --git a/src/client/rate_limiter/rate_limiter.cpp b/src/client/rate_limiter/rate_limiter.cpp index d34afe9d288..6059b27b0ae 100644 --- a/src/client/rate_limiter/rate_limiter.cpp +++ b/src/client/rate_limiter/rate_limiter.cpp @@ -4,7 +4,7 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include +#include #include namespace NYdb::NRateLimiter { @@ -50,10 +50,10 @@ class TRateLimiterClient::TImpl : public TClientImplCommon static TRequest MakePropsCreateOrAlterRequest(const std::string& coordinationNodePath, const std::string& resourcePath, const TSettings& settings) { TRequest request = MakeOperationRequest(settings); - request.set_coordination_node_path(coordinationNodePath); + request.set_coordination_node_path(TStringType{coordinationNodePath}); Ydb::RateLimiter::Resource& resource = *request.mutable_resource(); - resource.set_resource_path(resourcePath); + resource.set_resource_path(TStringType{resourcePath}); Ydb::RateLimiter::HierarchicalDrrSettings& hdrr = *resource.mutable_hierarchical_drr(); if (settings.MaxUnitsPerSecond_) { @@ -92,8 +92,8 @@ class TRateLimiterClient::TImpl : public TClientImplCommon(settings); - request.set_coordination_node_path(coordinationNodePath); - request.set_resource_path(resourcePath); + request.set_coordination_node_path(TStringType{coordinationNodePath}); + request.set_resource_path(TStringType{resourcePath}); return RunSimple( std::move(request), @@ -103,8 +103,8 @@ class TRateLimiterClient::TImpl : public TClientImplCommon(settings); - request.set_coordination_node_path(coordinationNodePath); - request.set_resource_path(resourcePath); + request.set_coordination_node_path(TStringType{coordinationNodePath}); + request.set_resource_path(TStringType{resourcePath}); request.set_recursive(settings.Recursive_); auto promise = NThreading::NewPromise(); @@ -138,8 +138,8 @@ class TRateLimiterClient::TImpl : public TClientImplCommon(settings); - request.set_coordination_node_path(coordinationNodePath); - request.set_resource_path(resourcePath); + request.set_coordination_node_path(TStringType{coordinationNodePath}); + request.set_resource_path(TStringType{resourcePath}); auto promise = NThreading::NewPromise(); @@ -167,8 +167,8 @@ class TRateLimiterClient::TImpl : public TClientImplCommon(settings); - request.set_coordination_node_path(coordinationNodePath); - request.set_resource_path(resourcePath); + request.set_coordination_node_path(TStringType{coordinationNodePath}); + request.set_resource_path(TStringType{resourcePath}); if (settings.IsUsedAmount_) { request.set_used(settings.Amount_.value()); diff --git a/src/client/result/result.cpp b/src/client/result/result.cpp index 4f52c549325..a2b6fb55568 100644 --- a/src/client/result/result.cpp +++ b/src/client/result/result.cpp @@ -2,8 +2,8 @@ #include -#include -#include +#include +#include #include diff --git a/src/client/scheme/scheme.cpp b/src/client/scheme/scheme.cpp index 14c27cfc50d..07c1b384a3f 100644 --- a/src/client/scheme/scheme.cpp +++ b/src/client/scheme/scheme.cpp @@ -5,8 +5,8 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include +#include +#include #include #include @@ -17,7 +17,7 @@ namespace NScheme { using namespace NThreading; using namespace Ydb::Scheme; -TVirtualTimestamp::TVirtualTimestamp(ui64 planStep, ui64 txId) +TVirtualTimestamp::TVirtualTimestamp(uint64_t planStep, uint64_t txId) : PlanStep(planStep) , TxId(txId) {} @@ -127,7 +127,7 @@ class TSchemeClient::TImpl : public TClientImplCommon { TAsyncStatus MakeDirectory(const std::string& path, const TMakeDirectorySettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); return RunSimple( std::move(request), @@ -137,7 +137,7 @@ class TSchemeClient::TImpl : public TClientImplCommon { TAsyncStatus RemoveDirectory(const std::string& path, const TRemoveDirectorySettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); return RunSimple( std::move(request), @@ -147,7 +147,7 @@ class TSchemeClient::TImpl : public TClientImplCommon { TAsyncDescribePathResult DescribePath(const std::string& path, const TDescribePathSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); auto promise = NThreading::NewPromise(); @@ -174,7 +174,7 @@ class TSchemeClient::TImpl : public TClientImplCommon { TAsyncListDirectoryResult ListDirectory(const std::string& path, const TListDirectorySettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); auto promise = NThreading::NewPromise(); @@ -207,15 +207,15 @@ class TSchemeClient::TImpl : public TClientImplCommon { } void PermissionsToRequest(const TPermissions& permissions, Permissions* to) { - to->set_subject(permissions.Subject); + to->set_subject(TStringType{permissions.Subject}); for (const auto& perm : permissions.PermissionNames) { - to->add_permission_names(perm); + to->add_permission_names(TStringType{perm}); } } TAsyncStatus ModifyPermissions(const std::string& path, const TModifyPermissionsSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); if (settings.ClearAcl_) { request.set_clear_permissions(true); } @@ -224,7 +224,7 @@ class TSchemeClient::TImpl : public TClientImplCommon { auto protoAction = request.add_actions(); switch (action.first) { case EModifyPermissionsAction::Chown: { - protoAction->set_change_owner(action.second.Subject); + protoAction->set_change_owner(TStringType{action.second.Subject}); } break; case EModifyPermissionsAction::Grant: { diff --git a/src/client/ss_tasks/task.cpp b/src/client/ss_tasks/task.cpp index 39216d228ef..3dcf6d23793 100644 --- a/src/client/ss_tasks/task.cpp +++ b/src/client/ss_tasks/task.cpp @@ -1,7 +1,7 @@ #include "task.h" -#include -#include -#include +#include +#include +#include #include #include diff --git a/src/client/table/impl/CMakeLists.txt b/src/client/table/impl/CMakeLists.txt index f434bcf725d..9cb4ba95479 100644 --- a/src/client/table/impl/CMakeLists.txt +++ b/src/client/table/impl/CMakeLists.txt @@ -6,6 +6,7 @@ target_link_libraries(client-ydb_table-impl threading-future api-protos api-grpc + lib-operation_id-protos client-impl-ydb_endpoints impl-ydb_internal-session_pool client-ydb_table-query_stats diff --git a/src/client/table/impl/client_session.h b/src/client/table/impl/client_session.h index eb13c68c303..007ad020e0c 100644 --- a/src/client/table/impl/client_session.h +++ b/src/client/table/impl/client_session.h @@ -5,7 +5,7 @@ #include #include -#include +#include #include @@ -32,12 +32,12 @@ class TSession::TImpl : public TKqpSessionCommon { public: struct TDataQueryInfo { std::string QueryId; - ::google::protobuf::Map ParameterTypes; + ::google::protobuf::Map ParameterTypes; TDataQueryInfo() {} TDataQueryInfo(const std::string& queryId, - const ::google::protobuf::Map& parameterTypes) + const ::google::protobuf::Map& parameterTypes) : QueryId(queryId) , ParameterTypes(parameterTypes) {} }; diff --git a/src/client/table/impl/data_query.cpp b/src/client/table/impl/data_query.cpp index 7ef703f3068..7d9f87247ca 100644 --- a/src/client/table/impl/data_query.cpp +++ b/src/client/table/impl/data_query.cpp @@ -33,7 +33,7 @@ TDataQuery::TImpl::TImpl(const TSession& session, const std::string& text, bool {} TDataQuery::TImpl::TImpl(const TSession& session, const std::string& text, bool keepText, const std::string& id, bool allowMigration, - const ::google::protobuf::Map& types) + const ::google::protobuf::Map& types) : Session_(session) , Id_(id) , ParameterTypes_(types) @@ -45,7 +45,7 @@ const std::string& TDataQuery::TImpl::GetId() const { return Id_; } -const ::google::protobuf::Map& TDataQuery::TImpl::GetParameterTypes() const { +const ::google::protobuf::Map& TDataQuery::TImpl::GetParameterTypes() const { return ParameterTypes_; } diff --git a/src/client/table/impl/data_query.h b/src/client/table/impl/data_query.h index c985f69422f..a3c1b0bcd77 100644 --- a/src/client/table/impl/data_query.h +++ b/src/client/table/impl/data_query.h @@ -1,7 +1,11 @@ #pragma once +#include #include -#include + +#include + +#include namespace NYdb { namespace NTable { @@ -17,17 +21,17 @@ class TDataQuery::TImpl { TImpl(const TSession& session, const std::string& text, bool keepText, const std::string& id, bool allowMigration); TImpl(const TSession& session, const std::string& text, bool keepText, const std::string& id, bool allowMigration, - const ::google::protobuf::Map& types); + const ::google::protobuf::Map& types); const std::string& GetId() const; - const ::google::protobuf::Map& GetParameterTypes() const; + const ::google::protobuf::Map& GetParameterTypes() const; const std::string& GetTextHash() const; const std::optional& GetText() const; private: NTable::TSession Session_; std::string Id_; - ::google::protobuf::Map ParameterTypes_; + ::google::protobuf::Map ParameterTypes_; std::string TextHash_; std::optional Text_; }; diff --git a/src/client/table/impl/readers.h b/src/client/table/impl/readers.h index cba8c674b1e..9c44b4a45d8 100644 --- a/src/client/table/impl/readers.h +++ b/src/client/table/impl/readers.h @@ -2,7 +2,7 @@ #include -#include +#include #include #include diff --git a/src/client/table/impl/table_client.cpp b/src/client/table/impl/table_client.cpp index 1b94e903089..68734dd9289 100644 --- a/src/client/table/impl/table_client.cpp +++ b/src/client/table/impl/table_client.cpp @@ -402,7 +402,7 @@ TAsyncCreateSessionResult TTableClient::TImpl::CreateSession(const TCreateSessio TAsyncKeepAliveResult TTableClient::TImpl::KeepAlive(const TSession::TImpl* session, const TKeepAliveSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(session->GetId()); + request.set_session_id(TStringType{session->GetId()}); auto keepAliveResultPromise = NewPromise(); auto self = shared_from_this(); @@ -472,9 +472,9 @@ TFuture TTableClient::TImpl::CopyTable(const std::string& sessionId, co const TCopyTableSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(sessionId); - request.set_source_path(src); - request.set_destination_path(dst); + request.set_session_id(TStringType{sessionId}); + request.set_source_path(TStringType{src}); + request.set_destination_path(TStringType{dst}); return RunSimple( std::move(request), @@ -500,8 +500,8 @@ TFuture TTableClient::TImpl::RenameTables(Ydb::Table::RenameTablesReque TFuture TTableClient::TImpl::DropTable(const std::string& sessionId, const std::string& path, const TDropTableSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(sessionId); - request.set_path(path); + request.set_session_id(TStringType{sessionId}); + request.set_path(TStringType{path}); return RunSimple( std::move(request), @@ -511,8 +511,8 @@ TFuture TTableClient::TImpl::DropTable(const std::string& sessionId, co TAsyncDescribeTableResult TTableClient::TImpl::DescribeTable(const std::string& sessionId, const std::string& path, const TDescribeTableSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(sessionId); - request.set_path(path); + request.set_session_id(TStringType{sessionId}); + request.set_path(TStringType{path}); if (settings.WithKeyShardBoundary_) { request.set_include_shard_key_bounds(true); } @@ -553,8 +553,8 @@ TAsyncPrepareQueryResult TTableClient::TImpl::PrepareDataQuery(const TSession& s const TPrepareDataQuerySettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(session.GetId()); - request.set_yql_text(query); + request.set_session_id(TStringType{session.GetId()}); + request.set_yql_text(TStringType{query}); auto promise = NewPromise(); @@ -569,7 +569,7 @@ TAsyncPrepareQueryResult TTableClient::TImpl::PrepareDataQuery(const TSession& s any->UnpackTo(&result); if (status.Ok()) { - dataQuery = TDataQuery(*sessionPtr, query, result.query_id(), result.parameters_types()); + dataQuery = TDataQuery(*sessionPtr, query, std::string{result.query_id()}, result.parameters_types()); sessionPtr->SessionImpl_->AddQueryToCache(dataQuery); } } @@ -598,8 +598,8 @@ TAsyncStatus TTableClient::TImpl::ExecuteSchemeQuery(const std::string& sessionI const TExecSchemeQuerySettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(sessionId); - request.set_yql_text(query); + request.set_session_id(TStringType{sessionId}); + request.set_yql_text(TStringType{query}); return RunSimple( std::move(request), @@ -611,7 +611,7 @@ TAsyncBeginTransactionResult TTableClient::TImpl::BeginTransaction(const TSessio const TBeginTxSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(session.GetId()); + request.set_session_id(TStringType{session.GetId()}); SetTxSettings(txSettings, request.mutable_tx_settings()); auto promise = NewPromise(); @@ -646,8 +646,8 @@ TAsyncCommitTransactionResult TTableClient::TImpl::CommitTransaction(const TSess const TCommitTxSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(session.GetId()); - request.set_tx_id(tx.GetId()); + request.set_session_id(TStringType{session.GetId()}); + request.set_tx_id(TStringType{tx.GetId()}); request.set_collect_stats(GetStatsCollectionMode(settings.CollectQueryStats_)); auto promise = NewPromise(); @@ -684,8 +684,8 @@ TAsyncStatus TTableClient::TImpl::RollbackTransaction(const TSession& session, c const TRollbackTxSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(session.GetId()); - request.set_tx_id(tx.GetId()); + request.set_session_id(TStringType{session.GetId()}); + request.set_tx_id(TStringType{tx.GetId()}); return RunSimple( std::move(request), @@ -698,8 +698,8 @@ TAsyncExplainDataQueryResult TTableClient::TImpl::ExplainDataQuery(const TSessio const TExplainDataQuerySettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(session.GetId()); - request.set_yql_text(query); + request.set_session_id(TStringType{session.GetId()}); + request.set_yql_text(TStringType{query}); request.set_collect_full_diagnostics(settings.WithCollectFullDiagnostics_); auto promise = NewPromise(); @@ -744,14 +744,14 @@ NThreading::TFuture(); - request.set_session_id(sessionId); - request.set_path(path); + request.set_session_id(TStringType{sessionId}); + request.set_path(TStringType{path}); request.set_ordered(settings.Ordered_); if (settings.RowLimit_) { request.set_row_limit(settings.RowLimit_.value()); } for (const auto& col : settings.Columns_) { - request.add_columns(col); + request.add_columns(TStringType{col}); } if (settings.UseSnapshot_) { request.set_use_snapshot( @@ -810,7 +810,7 @@ NThreading::TFuture& columns, const TReadRowsSettings& settings) { auto request = MakeRequest(); - request.set_path(path); + request.set_path(TStringType{path}); auto* protoKeys = request.mutable_keys(); *protoKeys->mutable_type() = TProtoAccessor::GetProto(keys.GetType()); *protoKeys->mutable_value() = TProtoAccessor::GetProto(keys); @@ -849,7 +849,7 @@ TAsyncReadRowsResult TTableClient::TImpl::ReadRows(const std::string& path, TVal TAsyncStatus TTableClient::TImpl::Close(const TKqpSessionCommon* sessionImpl, const TCloseSessionSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(sessionImpl->GetId()); + request.set_session_id(TStringType{sessionImpl->GetId()}); return RunSimple( std::move(request), &Ydb::Table::V1::TableService::Stub::AsyncDeleteSession, @@ -928,7 +928,7 @@ void TTableClient::TImpl::SetStatCollector(const NSdkStats::TStatCollector::TCli TAsyncBulkUpsertResult TTableClient::TImpl::BulkUpsert(const std::string& table, TValue&& rows, const TBulkUpsertSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_table(table); + request.set_table(TStringType{table}); *request.mutable_rows()->mutable_type() = TProtoAccessor::GetProto(rows.GetType()); *request.mutable_rows()->mutable_value() = rows.GetProto(); @@ -956,20 +956,20 @@ TAsyncBulkUpsertResult TTableClient::TImpl::BulkUpsert(const std::string& table, const std::string& data, const std::string& schema, const TBulkUpsertSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_table(table); + request.set_table(TStringType{table}); if (format == EDataFormat::ApacheArrow) { - request.mutable_arrow_batch_settings()->set_schema(schema); + request.mutable_arrow_batch_settings()->set_schema(TStringType{schema}); } else if (format == EDataFormat::CSV) { auto* csv_settings = request.mutable_csv_settings(); const auto& format_settings = settings.FormatSettings_; if (!format_settings.empty()) { - bool ok = csv_settings->ParseFromString(format_settings); + bool ok = csv_settings->ParseFromString(TStringType{format_settings}); if (!ok) { return {}; } } } - request.set_data(data); + request.set_data(TStringType{data}); auto promise = NewPromise(); @@ -992,11 +992,11 @@ TAsyncBulkUpsertResult TTableClient::TImpl::BulkUpsert(const std::string& table, } TFuture> TTableClient::TImpl::StreamExecuteScanQueryInternal(const std::string& query, - const ::google::protobuf::Map* params, + const ::google::protobuf::Map* params, const TStreamExecScanQuerySettings& settings) { auto request = MakeRequest(); - request.mutable_query()->set_yql_text(query); + request.mutable_query()->set_yql_text(TStringType{query}); if (params) { *request.mutable_parameters() = *params; @@ -1031,7 +1031,7 @@ TFuture> TT } TAsyncScanQueryPartIterator TTableClient::TImpl::StreamExecuteScanQuery(const std::string& query, - const ::google::protobuf::Map* params, + const ::google::protobuf::Map* params, const TStreamExecScanQuerySettings& settings) { auto promise = NewPromise(); @@ -1059,7 +1059,7 @@ TAsyncScanQueryPartIterator TTableClient::TImpl::StreamExecuteScanQuery(const st void TTableClient::TImpl::SetParams( - ::google::protobuf::Map* params, + ::google::protobuf::Map* params, Ydb::Table::ExecuteDataQueryRequest* request) { if (params) { @@ -1068,14 +1068,14 @@ void TTableClient::TImpl::SetParams( } void TTableClient::TImpl::SetParams( - const ::google::protobuf::Map& params, + const ::google::protobuf::Map& params, Ydb::Table::ExecuteDataQueryRequest* request) { *request->mutable_parameters() = params; } void TTableClient::TImpl::CollectParams( - ::google::protobuf::Map* params, + ::google::protobuf::Map* params, NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> histgoram) { @@ -1089,7 +1089,7 @@ void TTableClient::TImpl::CollectParams( } void TTableClient::TImpl::CollectParams( - const ::google::protobuf::Map& params, + const ::google::protobuf::Map& params, NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> histgoram) { @@ -1132,11 +1132,11 @@ void TTableClient::TImpl::SetTxSettings(const TTxSettings& txSettings, Ydb::Tabl } void TTableClient::TImpl::SetQuery(const std::string& queryText, Ydb::Table::Query* query) { - query->set_yql_text(queryText); + query->set_yql_text(TStringType{queryText}); } void TTableClient::TImpl::SetQuery(const TDataQuery& queryData, Ydb::Table::Query* query) { - query->set_id(queryData.GetId()); + query->set_id(TStringType{queryData.GetId()}); } void TTableClient::TImpl::SetQueryCachePolicy(const std::string&, const TExecDataQuerySettings& settings, diff --git a/src/client/table/impl/table_client.h b/src/client/table/impl/table_client.h index bfe168858fc..167449a2d60 100644 --- a/src/client/table/impl/table_client.h +++ b/src/client/table/impl/table_client.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include "client_session.h" #include "data_query.h" @@ -139,10 +139,10 @@ class TTableClient::TImpl: public TClientImplCommon, public const std::string& data, const std::string& schema, const TBulkUpsertSettings& settings); TFuture> StreamExecuteScanQueryInternal(const std::string& query, - const ::google::protobuf::Map* params, + const ::google::protobuf::Map* params, const TStreamExecScanQuerySettings& settings); TAsyncScanQueryPartIterator StreamExecuteScanQuery(const std::string& query, - const ::google::protobuf::Map* params, + const ::google::protobuf::Map* params, const TStreamExecScanQuerySettings& settings); void CollectRetryStatAsync(EStatus status); void CollectRetryStatSync(EStatus status); @@ -152,19 +152,19 @@ class TTableClient::TImpl: public TClientImplCommon, public private: static void SetParams( - ::google::protobuf::Map* params, + ::google::protobuf::Map* params, Ydb::Table::ExecuteDataQueryRequest* request); static void SetParams( - const ::google::protobuf::Map& params, + const ::google::protobuf::Map& params, Ydb::Table::ExecuteDataQueryRequest* request); static void CollectParams( - ::google::protobuf::Map* params, + ::google::protobuf::Map* params, NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> histgoram); static void CollectParams( - const ::google::protobuf::Map& params, + const ::google::protobuf::Map& params, NSdkStats::TAtomicHistogram<::NMonitoring::THistogram> histgoram); static void CollectQuerySize(const std::string& query, NSdkStats::TAtomicHistogram<::NMonitoring::THistogram>& querySizeHistogram); @@ -177,11 +177,11 @@ class TTableClient::TImpl: public TClientImplCommon, public const TExecDataQuerySettings& settings, bool fromCache ) { auto request = MakeOperationRequest(settings); - request.set_session_id(session.GetId()); + request.set_session_id(TStringType{session.GetId()}); auto txControlProto = request.mutable_tx_control(); txControlProto->set_commit_tx(txControl.CommitTx_); if (txControl.TxId_) { - txControlProto->set_tx_id(*txControl.TxId_); + txControlProto->set_tx_id(TStringType{txControl.TxId_.value()}); } else { SetTxSettings(txControl.BeginTx_, txControlProto->mutable_begin_tx()); } diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index f472313620b..03f5a3bc0d8 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -9,8 +9,8 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include +#include +#include #include #include #include @@ -217,37 +217,37 @@ std::optional TPartitioningSettings::GetPartitioningByLoad() const { } } -ui64 TPartitioningSettings::GetPartitionSizeMb() const { +uint64_t TPartitioningSettings::GetPartitionSizeMb() const { return GetProto().partition_size_mb(); } -ui64 TPartitioningSettings::GetMinPartitionsCount() const { +uint64_t TPartitioningSettings::GetMinPartitionsCount() const { return GetProto().min_partitions_count(); } -ui64 TPartitioningSettings::GetMaxPartitionsCount() const { +uint64_t TPartitioningSettings::GetMaxPartitionsCount() const { return GetProto().max_partitions_count(); } //////////////////////////////////////////////////////////////////////////////// struct TTableStats { - ui64 Rows = 0; - ui64 Size = 0; - ui64 Partitions = 0; + uint64_t Rows = 0; + uint64_t Size = 0; + uint64_t Partitions = 0; TInstant ModificationTime; TInstant CreationTime; }; static TInstant ProtobufTimestampToTInstant(const google::protobuf::Timestamp& timestamp) { - ui64 lastModificationUs = timestamp.seconds() * 1000000; + uint64_t lastModificationUs = timestamp.seconds() * 1000000; lastModificationUs += timestamp.nanos() / 1000; return TInstant::MicroSeconds(lastModificationUs); } static void SerializeTo(const TRenameIndex& rename, Ydb::Table::RenameIndexItem& proto) { - proto.set_source_name(rename.SourceName_); - proto.set_destination_name(rename.DestinationName_); + proto.set_source_name(TStringType{rename.SourceName_}); + proto.set_destination_name(TStringType{rename.DestinationName_}); proto.set_replace_destination(rename.ReplaceDestination_); } @@ -493,7 +493,7 @@ class TTableDescription::TImpl { CompactionPolicy_ = name; } - void SetUniformPartitions(ui64 partitionsCount) { + void SetUniformPartitions(uint64_t partitionsCount) { UniformPartitions_ = partitionsCount; } @@ -510,7 +510,7 @@ class TTableDescription::TImpl { KeyBloomFilter_ = enabled; } - void SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { + void SetReadReplicasSettings(TReadReplicasSettings::EMode mode, uint64_t readReplicasCount) { ReadReplicasSettings_ = TReadReplicasSettings(mode, readReplicasCount); } @@ -590,7 +590,7 @@ class TTableDescription::TImpl { return CompactionPolicy_; } - const std::optional& GetUniformPartitions() const { + const std::optional& GetUniformPartitions() const { return UniformPartitions_; } @@ -632,7 +632,7 @@ class TTableDescription::TImpl { std::vector ColumnFamilies_; std::unordered_map Attributes_; std::string CompactionPolicy_; - std::optional UniformPartitions_; + std::optional UniformPartitions_; std::optional PartitionAtKeys_; TPartitioningSettings PartitioningSettings_; std::optional KeyBloomFilter_; @@ -793,7 +793,7 @@ void TTableDescription::SetCompactionPolicy(const std::string& name) { Impl_->SetCompactionPolicy(name); } -void TTableDescription::SetUniformPartitions(ui64 partitionsCount) { +void TTableDescription::SetUniformPartitions(uint64_t partitionsCount) { Impl_->SetUniformPartitions(partitionsCount); } @@ -809,7 +809,7 @@ void TTableDescription::SetKeyBloomFilter(bool enabled) { Impl_->SetKeyBloomFilter(enabled); } -void TTableDescription::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { +void TTableDescription::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, uint64_t readReplicasCount) { Impl_->SetReadReplicasSettings(mode, readReplicasCount); } @@ -829,15 +829,15 @@ TInstant TTableDescription::GetCreationTime() const { return Impl_->GetTableStats().CreationTime; } -ui64 TTableDescription::GetTableSize() const { +uint64_t TTableDescription::GetTableSize() const { return Impl_->GetTableStats().Size; } -ui64 TTableDescription::GetTableRows() const { +uint64_t TTableDescription::GetTableRows() const { return Impl_->GetTableStats().Rows; } -ui64 TTableDescription::GetPartitionsCount() const { +uint64_t TTableDescription::GetPartitionsCount() const { return Impl_->GetTableStats().Partitions; } @@ -872,16 +872,16 @@ const Ydb::Table::DescribeTableResult& TTableDescription::GetProto() const { void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) const { for (const auto& column : Impl_->GetColumns()) { auto& protoColumn = *request.add_columns(); - protoColumn.set_name(column.Name); + protoColumn.set_name(TStringType{column.Name}); protoColumn.mutable_type()->CopyFrom(TProtoAccessor::GetProto(column.Type)); - protoColumn.set_family(column.Family); + protoColumn.set_family(TStringType{column.Family}); if (column.NotNull.has_value()) { protoColumn.set_not_null(column.NotNull.value()); } } for (const auto& pk : Impl_->GetPrimaryKeyColumns()) { - request.add_primary_key(pk); + request.add_primary_key(TStringType{pk}); } for (const auto& index : Impl_->GetIndexDescriptions()) { @@ -893,7 +893,7 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con } if (const auto& tiering = Impl_->GetTiering()) { - request.set_tiering(*tiering); + request.set_tiering(TStringType{tiering.value()}); } if (Impl_->GetStoreType() == EStoreType::Column) { @@ -914,7 +914,7 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con } if (!Impl_->GetCompactionPolicy().empty()) { - request.set_compaction_policy(Impl_->GetCompactionPolicy()); + request.set_compaction_policy(TStringType{Impl_->GetCompactionPolicy()}); } if (const auto& uniformPartitions = Impl_->GetUniformPartitions()) { @@ -972,17 +972,17 @@ TStorageSettingsBuilder::TStorageSettingsBuilder() TStorageSettingsBuilder::~TStorageSettingsBuilder() { } TStorageSettingsBuilder& TStorageSettingsBuilder::SetTabletCommitLog0(const std::string& media) { - Impl_->Proto.mutable_tablet_commit_log0()->set_media(media); + Impl_->Proto.mutable_tablet_commit_log0()->set_media(TStringType{media}); return *this; } TStorageSettingsBuilder& TStorageSettingsBuilder::SetTabletCommitLog1(const std::string& media) { - Impl_->Proto.mutable_tablet_commit_log1()->set_media(media); + Impl_->Proto.mutable_tablet_commit_log1()->set_media(TStringType{media}); return *this; } TStorageSettingsBuilder& TStorageSettingsBuilder::SetExternal(const std::string& media) { - Impl_->Proto.mutable_external()->set_media(media); + Impl_->Proto.mutable_external()->set_media(TStringType{media}); return *this; } @@ -1021,17 +1021,17 @@ TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitioningByLoa return *this; } -TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitionSizeMb(ui64 sizeMb) { +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetPartitionSizeMb(uint64_t sizeMb) { Impl_->Proto.set_partition_size_mb(sizeMb); return *this; } -TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMinPartitionsCount(ui64 count) { +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMinPartitionsCount(uint64_t count) { Impl_->Proto.set_min_partitions_count(count); return *this; } -TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMaxPartitionsCount(ui64 count) { +TPartitioningSettingsBuilder& TPartitioningSettingsBuilder::SetMaxPartitionsCount(uint64_t count) { Impl_->Proto.set_max_partitions_count(count); return *this; } @@ -1050,13 +1050,13 @@ class TColumnFamilyBuilder::TImpl { TColumnFamilyBuilder::TColumnFamilyBuilder(const std::string& name) : Impl_(new TImpl) { - Impl_->Proto.set_name(name); + Impl_->Proto.set_name(TStringType{name}); } TColumnFamilyBuilder::~TColumnFamilyBuilder() { } TColumnFamilyBuilder& TColumnFamilyBuilder::SetData(const std::string& media) { - Impl_->Proto.mutable_data()->set_media(media); + Impl_->Proto.mutable_data()->set_media(TStringType{media}); return *this; } @@ -1257,7 +1257,7 @@ TTableBuilder& TTableBuilder::SetCompactionPolicy(const std::string& name) { return *this; } -TTableBuilder& TTableBuilder::SetUniformPartitions(ui64 partitionsCount) { +TTableBuilder& TTableBuilder::SetUniformPartitions(uint64_t partitionsCount) { TableDescription_.SetUniformPartitions(partitionsCount); return *this; } @@ -1277,7 +1277,7 @@ TTableBuilder& TTableBuilder::SetKeyBloomFilter(bool enabled) { return *this; } -TTableBuilder& TTableBuilder::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, ui64 readReplicasCount) { +TTableBuilder& TTableBuilder::SetReadReplicasSettings(TReadReplicasSettings::EMode mode, uint64_t readReplicasCount) { TableDescription_.SetReadReplicasSettings(mode, readReplicasCount); return *this; } @@ -1373,15 +1373,15 @@ TAsyncCreateSessionResult TTableClient::GetSession(const TCreateSessionSettings& return Impl_->GetSession(settings); } -i64 TTableClient::GetActiveSessionCount() const { +int64_t TTableClient::GetActiveSessionCount() const { return Impl_->GetActiveSessionCount(); } -i64 TTableClient::GetActiveSessionsLimit() const { +int64_t TTableClient::GetActiveSessionsLimit() const { return Impl_->GetActiveSessionsLimit(); } -i64 TTableClient::GetCurrentPoolSize() const { +int64_t TTableClient::GetCurrentPoolSize() const { return Impl_->GetCurrentPoolSize(); } @@ -1457,18 +1457,18 @@ TAsyncScanQueryPartIterator TTableClient::StreamExecuteScanQuery(const std::stri static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settings, Ydb::Table::TableProfile* proto) { if (settings.PresetName_) { - proto->set_preset_name(settings.PresetName_.value()); + proto->set_preset_name(TStringType{settings.PresetName_.value()}); } if (settings.ExecutionPolicy_) { - proto->mutable_execution_policy()->set_preset_name(settings.ExecutionPolicy_.value()); + proto->mutable_execution_policy()->set_preset_name(TStringType{settings.ExecutionPolicy_.value()}); } if (settings.CompactionPolicy_) { - proto->mutable_compaction_policy()->set_preset_name(settings.CompactionPolicy_.value()); + proto->mutable_compaction_policy()->set_preset_name(TStringType{settings.CompactionPolicy_.value()}); } if (settings.PartitioningPolicy_) { const auto& policy = settings.PartitioningPolicy_.value(); if (policy.PresetName_) { - proto->mutable_partitioning_policy()->set_preset_name(policy.PresetName_.value()); + proto->mutable_partitioning_policy()->set_preset_name(TStringType{policy.PresetName_.value()}); } if (policy.AutoPartitioning_) { proto->mutable_partitioning_policy()->set_auto_partitioning(static_cast(policy.AutoPartitioning_.value())); @@ -1488,30 +1488,30 @@ static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settin if (settings.StoragePolicy_) { const auto& policy = settings.StoragePolicy_.value(); if (policy.PresetName_) { - proto->mutable_storage_policy()->set_preset_name(policy.PresetName_.value()); + proto->mutable_storage_policy()->set_preset_name(TStringType{policy.PresetName_.value()}); } if (policy.SysLog_) { - proto->mutable_storage_policy()->mutable_syslog()->set_media(policy.SysLog_.value()); + proto->mutable_storage_policy()->mutable_syslog()->set_media(TStringType{policy.SysLog_.value()}); } if (policy.Log_) { - proto->mutable_storage_policy()->mutable_log()->set_media(policy.Log_.value()); + proto->mutable_storage_policy()->mutable_log()->set_media(TStringType{policy.Log_.value()}); } if (policy.Data_) { - proto->mutable_storage_policy()->mutable_data()->set_media(policy.Data_.value()); + proto->mutable_storage_policy()->mutable_data()->set_media(TStringType{policy.Data_.value()}); } if (policy.External_) { - proto->mutable_storage_policy()->mutable_external()->set_media(policy.External_.value()); + proto->mutable_storage_policy()->mutable_external()->set_media(TStringType{policy.External_.value()}); } for (const auto& familyPolicy : policy.ColumnFamilies_) { auto* familyProto = proto->mutable_storage_policy()->add_column_families(); if (familyPolicy.Name_) { - familyProto->set_name(familyPolicy.Name_.value()); + familyProto->set_name(TStringType{familyPolicy.Name_.value()}); } if (familyPolicy.Data_) { - familyProto->mutable_data()->set_media(familyPolicy.Data_.value()); + familyProto->mutable_data()->set_media(TStringType{familyPolicy.Data_.value()}); } if (familyPolicy.External_) { - familyProto->mutable_external()->set_media(familyPolicy.External_.value()); + familyProto->mutable_external()->set_media(TStringType{familyPolicy.External_.value()}); } if (familyPolicy.KeepInMemory_) { familyProto->set_keep_in_memory( @@ -1530,7 +1530,7 @@ static void ConvertCreateTableSettingsToProto(const TCreateTableSettings& settin if (settings.ReplicationPolicy_) { const auto& policy = settings.ReplicationPolicy_.value(); if (policy.PresetName_) { - proto->mutable_replication_policy()->set_preset_name(policy.PresetName_.value()); + proto->mutable_replication_policy()->set_preset_name(TStringType{policy.PresetName_.value()}); } if (policy.ReplicasCount_) { proto->mutable_replication_policy()->set_replicas_count(policy.ReplicasCount_.value()); @@ -1578,8 +1578,8 @@ TFuture TSession::CreateTable(const std::string& path, TTableDescriptio const TCreateTableSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(SessionImpl_->GetId()); - request.set_path(path); + request.set_session_id(TStringType{SessionImpl_->GetId()}); + request.set_path(TStringType{path}); tableDesc.SerializeTo(request); @@ -1604,24 +1604,24 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( const std::string& path, const TAlterTableSettings& settings, const std::string& sessionId) { auto request = MakeOperationRequest(settings); - request.set_session_id(sessionId); - request.set_path(path); + request.set_session_id(TStringType{sessionId}); + request.set_path(TStringType{path}); for (const auto& column : settings.AddColumns_) { auto& protoColumn = *request.add_add_columns(); - protoColumn.set_name(column.Name); + protoColumn.set_name(TStringType{column.Name}); protoColumn.mutable_type()->CopyFrom(TProtoAccessor::GetProto(column.Type)); - protoColumn.set_family(column.Family); + protoColumn.set_family(TStringType{column.Family}); } for (const auto& columnName : settings.DropColumns_) { - request.add_drop_columns(columnName); + request.add_drop_columns(TStringType{columnName}); } for (const auto& alter : settings.AlterColumns_) { auto& protoAlter = *request.add_alter_columns(); - protoAlter.set_name(alter.Name); - protoAlter.set_family(alter.Family); + protoAlter.set_name(TStringType{alter.Name}); + protoAlter.set_family(TStringType{alter.Family}); } for (const auto& addIndex : settings.AddIndexes_) { @@ -1629,7 +1629,7 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( } for (const auto& name : settings.DropIndexes_) { - request.add_drop_indexes(name); + request.add_drop_indexes(TStringType{name}); } for (const auto& rename : settings.RenameIndexes_) { @@ -1641,7 +1641,7 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( } for (const auto& name : settings.DropChangefeeds_) { - request.add_drop_changefeeds(name); + request.add_drop_changefeeds(TStringType{name}); } if (settings.AlterStorageSettings_) { @@ -1672,7 +1672,7 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( } if (!settings.SetCompactionPolicy_.empty()) { - request.set_set_compaction_policy(settings.SetCompactionPolicy_); + request.set_set_compaction_policy(TStringType{settings.SetCompactionPolicy_}); } if (settings.AlterPartitioningSettings_) { @@ -1725,12 +1725,12 @@ TAsyncOperation TSession::AlterTableLong(const std::string& path, const TAlterTa TAsyncStatus TSession::RenameTables(const std::vector& renameItems, const TRenameTablesSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(SessionImpl_->GetId()); + request.set_session_id(TStringType{SessionImpl_->GetId()}); for (const auto& item: renameItems) { auto add = request.add_tables(); - add->set_source_path(item.SourcePath()); - add->set_destination_path(item.DestinationPath()); + add->set_source_path(TStringType{item.SourcePath()}); + add->set_destination_path(TStringType{item.DestinationPath()}); add->set_replace_destination(item.ReplaceDestination()); } @@ -1743,12 +1743,12 @@ TAsyncStatus TSession::RenameTables(const std::vector& renameItems, TAsyncStatus TSession::CopyTables(const std::vector& copyItems, const TCopyTablesSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_session_id(SessionImpl_->GetId()); + request.set_session_id(TStringType{SessionImpl_->GetId()}); for (const auto& item: copyItems) { auto add = request.add_tables(); - add->set_source_path(item.SourcePath()); - add->set_destination_path(item.DestinationPath()); + add->set_source_path(TStringType{item.SourcePath()}); + add->set_destination_path(TStringType{item.DestinationPath()}); add->set_omit_indexes(item.OmitIndexes()); } @@ -1795,7 +1795,7 @@ TAsyncDataQueryResult TSession::ExecuteDataQuery(const std::string& query, const nullptr, settings); } else { - using TProtoParamsType = const ::google::protobuf::Map; + using TProtoParamsType = const ::google::protobuf::Map; return Client_->ExecuteDataQuery( *this, query, @@ -1936,7 +1936,7 @@ TDataQuery::TDataQuery(const TSession& session, const std::string& text, const s {} TDataQuery::TDataQuery(const TSession& session, const std::string& text, const std::string& id, - const ::google::protobuf::Map& types) + const ::google::protobuf::Map& types) : Impl_(new TImpl(session, text, session.Client_->Settings_.KeepDataQueryText_, id, session.Client_->Settings_.AllowRequestMigration_, types)) {} @@ -1984,7 +1984,7 @@ TAsyncDataQueryResult TDataQuery::Execute(const TTxControl& txControl, const TPa settings, false); } else { - using TProtoParamsType = const ::google::protobuf::Map; + using TProtoParamsType = const ::google::protobuf::Map; return Impl_->Session_.Client_->ExecuteDataQuery( Impl_->Session_, *this, @@ -2240,7 +2240,7 @@ const std::vector& TIndexDescription::GetDataColumns() const { return DataColumns_; } -ui64 TIndexDescription::GetSizeBytes() const { +uint64_t TIndexDescription::GetSizeBytes() const { return SizeBytes; } @@ -2277,9 +2277,9 @@ TIndexDescription TIndexDescription::FromProto(const TProto& proto) { } void TIndexDescription::SerializeTo(Ydb::Table::TableIndex& proto) const { - proto.set_name(IndexName_); + proto.set_name(TStringType{IndexName_}); for (const auto& indexCol : IndexColumns_) { - proto.add_index_columns(indexCol); + proto.add_index_columns(TStringType{indexCol}); } *proto.mutable_data_columns() = {DataColumns_.begin(), DataColumns_.end()}; @@ -2498,10 +2498,10 @@ TChangefeedDescription TChangefeedDescription::FromProto(const TProto& proto) { } void TChangefeedDescription::SerializeTo(Ydb::Table::Changefeed& proto) const { - proto.set_name(Name_); + proto.set_name(TStringType{Name_}); proto.set_virtual_timestamps(VirtualTimestamps_); proto.set_initial_scan(InitialScan_); - proto.set_aws_region(AwsRegion_); + proto.set_aws_region(TStringType{AwsRegion_}); switch (Mode_) { case EChangefeedMode::KeysOnly: @@ -2599,7 +2599,7 @@ TDateTypeColumnModeSettings::TDateTypeColumnModeSettings(const std::string& colu {} void TDateTypeColumnModeSettings::SerializeTo(Ydb::Table::DateTypeColumnModeSettings& proto) const { - proto.set_column_name(ColumnName_); + proto.set_column_name(TStringType{ColumnName_}); proto.set_expire_after_seconds(ExpireAfter_.Seconds()); } @@ -2618,7 +2618,7 @@ TValueSinceUnixEpochModeSettings::TValueSinceUnixEpochModeSettings(const std::st {} void TValueSinceUnixEpochModeSettings::SerializeTo(Ydb::Table::ValueSinceUnixEpochModeSettings& proto) const { - proto.set_column_name(ColumnName_); + proto.set_column_name(TStringType{ColumnName_}); proto.set_column_unit(TProtoAccessor::GetProto(ColumnUnit_)); proto.set_expire_after_seconds(ExpireAfter_.Seconds()); } @@ -2828,7 +2828,7 @@ const std::optional& TAlterTableSettings::GetAlterTtlSettings //////////////////////////////////////////////////////////////////////////////// -TReadReplicasSettings::TReadReplicasSettings(EMode mode, ui64 readReplicasCount) +TReadReplicasSettings::TReadReplicasSettings(EMode mode, uint64_t readReplicasCount) : Mode_(mode) , ReadReplicasCount_(readReplicasCount) {} @@ -2837,7 +2837,7 @@ TReadReplicasSettings::EMode TReadReplicasSettings::GetMode() const { return Mode_; } -ui64 TReadReplicasSettings::GetReadReplicasCount() const { +uint64_t TReadReplicasSettings::GetReadReplicasCount() const { return ReadReplicasCount_; } diff --git a/src/client/topic/impl/deferred_commit.cpp b/src/client/topic/impl/deferred_commit.cpp index f6d2bb19140..128e56cf089 100644 --- a/src/client/topic/impl/deferred_commit.cpp +++ b/src/client/topic/impl/deferred_commit.cpp @@ -6,7 +6,7 @@ namespace NYdb::NTopic { -std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, ui64 index); +std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, uint64_t index); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // NTopic::TDeferredCommit @@ -46,11 +46,11 @@ TDeferredCommit::~TDeferredCommit() { } \ Impl -void TDeferredCommit::Add(const TPartitionSession::TPtr& partitionStream, ui64 startOffset, ui64 endOffset) { +void TDeferredCommit::Add(const TPartitionSession::TPtr& partitionStream, uint64_t startOffset, uint64_t endOffset) { GET_IMPL()->Add(partitionStream, startOffset, endOffset); } -void TDeferredCommit::Add(const TPartitionSession::TPtr& partitionStream, ui64 offset) { +void TDeferredCommit::Add(const TPartitionSession::TPtr& partitionStream, uint64_t offset) { GET_IMPL()->Add(partitionStream, offset); } diff --git a/src/client/topic/impl/event_handlers.cpp b/src/client/topic/impl/event_handlers.cpp index 39afbb842eb..259a7ed707e 100644 --- a/src/client/topic/impl/event_handlers.cpp +++ b/src/client/topic/impl/event_handlers.cpp @@ -4,7 +4,7 @@ namespace NYdb::NTopic { -std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, ui64 index); +std::pair GetMessageOffsetRange(const TReadSessionEvent::TDataReceivedEvent& dataReceivedEvent, uint64_t index); //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // NTopic::TReadSessionEventHandlers diff --git a/src/client/topic/impl/read_session_event.cpp b/src/client/topic/impl/read_session_event.cpp index 79c8475c1fa..f97960f66af 100644 --- a/src/client/topic/impl/read_session_event.cpp +++ b/src/client/topic/impl/read_session_event.cpp @@ -23,7 +23,7 @@ using TPartitionSessionClosedEvent = TReadSessionEvent::TPartitionSessionClosedE //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Helpers -std::pair GetMessageOffsetRange(const TDataReceivedEvent& dataReceivedEvent, ui64 index) { +std::pair GetMessageOffsetRange(const TDataReceivedEvent& dataReceivedEvent, uint64_t index) { if (dataReceivedEvent.HasCompressedMessages()) { const auto& msg = dataReceivedEvent.GetCompressedMessages()[index]; return {msg.GetOffset(), msg.GetOffset() + 1}; @@ -36,14 +36,14 @@ std::pair GetMessageOffsetRange(const TDataReceivedEvent& dataReceiv // NTopic::TReadSessionEvent::TDataReceivedEvent::TMessageInformation TMessageInformation::TMessageInformation( - ui64 offset, + uint64_t offset, std::string producerId, - ui64 seqNo, + uint64_t seqNo, TInstant createTime, TInstant writeTime, TWriteSessionMeta::TPtr meta, TMessageMeta::TPtr messageMeta, - ui64 uncompressedSize, + uint64_t uncompressedSize, std::string messageGroupId ) : Offset(offset) @@ -88,7 +88,7 @@ const std::string& TMessageBase::GetData() const { return Data; } -ui64 TMessageBase::GetOffset() const { +uint64_t TMessageBase::GetOffset() const { return Information.Offset; } @@ -100,7 +100,7 @@ const std::string& TMessageBase::GetMessageGroupId() const { return Information.MessageGroupId; } -ui64 TMessageBase::GetSeqNo() const { +uint64_t TMessageBase::GetSeqNo() const { return Information.SeqNo; } @@ -209,7 +209,7 @@ ECodec TCompressedMessage::GetCodec() const { return Codec; } -ui64 TCompressedMessage::GetUncompressedSize() const { +uint64_t TCompressedMessage::GetUncompressedSize() const { return Information.UncompressedSize; } @@ -276,7 +276,7 @@ void TPrintable::DebugString(TStringBuilder& ret, bool print // NTopic::TReadSessionEvent::TCommitOffsetAcknowledgementEvent TCommitOffsetAcknowledgementEvent::TCommitOffsetAcknowledgementEvent(TPartitionSession::TPtr partitionSession, - ui64 committedOffset) + uint64_t committedOffset) : TPartitionSessionAccessor(std::move(partitionSession)) , CommittedOffset(committedOffset) { } @@ -293,14 +293,14 @@ void TPrintable::DebugString(TStringBuilder& //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // NTopic::TReadSessionEvent::TStartPartitionSessionEvent -TStartPartitionSessionEvent::TStartPartitionSessionEvent(TPartitionSession::TPtr partitionSession, ui64 committedOffset, - ui64 endOffset) +TStartPartitionSessionEvent::TStartPartitionSessionEvent(TPartitionSession::TPtr partitionSession, uint64_t committedOffset, + uint64_t endOffset) : TPartitionSessionAccessor(std::move(partitionSession)) , CommittedOffset(committedOffset) , EndOffset(endOffset) { } -void TStartPartitionSessionEvent::Confirm(std::optional readOffset, std::optional commitOffset) { +void TStartPartitionSessionEvent::Confirm(std::optional readOffset, std::optional commitOffset) { if (PartitionSession) { static_cast*>(PartitionSession.Get()) ->ConfirmCreate(readOffset, commitOffset); @@ -320,7 +320,7 @@ void TPrintable::DebugString(TStringBuilder& ret, b //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // NTopic::TReadSessionEvent::TStopPartitionSessionEvent -TStopPartitionSessionEvent::TStopPartitionSessionEvent(TPartitionSession::TPtr partitionSession, ui64 committedOffset) +TStopPartitionSessionEvent::TStopPartitionSessionEvent(TPartitionSession::TPtr partitionSession, uint64_t committedOffset) : TPartitionSessionAccessor(std::move(partitionSession)) , CommittedOffset(committedOffset) { } @@ -382,7 +382,7 @@ void TPrintable::DebugString(TStringBuilder& ret, boo // NTopic::TReadSessionEvent::TPartitionSessionStatusEvent TPartitionSessionStatusEvent::TPartitionSessionStatusEvent(TPartitionSession::TPtr partitionSession, - ui64 committedOffset, ui64 readOffset, ui64 endOffset, + uint64_t committedOffset, uint64_t readOffset, uint64_t endOffset, TInstant writeTimeHighWatermark) : TPartitionSessionAccessor(std::move(partitionSession)) , CommittedOffset(committedOffset) diff --git a/src/client/topic/impl/read_session_impl.h b/src/client/topic/impl/read_session_impl.h index b7464cae4c0..9679154c71f 100644 --- a/src/client/topic/impl/read_session_impl.h +++ b/src/client/topic/impl/read_session_impl.h @@ -13,8 +13,8 @@ #include -#include -#include +#include +#include #include diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index 888a0737de6..fb070e3b7a7 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -302,7 +302,8 @@ bool TSingleClusterReadSessionImpl::Reconnect(const TPlain RetryState = Settings.RetryPolicy_->CreateRetryState(); } if (!status.Ok()) { - std::optional nextDelay = RetryState->GetNextRetryDelay(status.Status); + auto nextDelay = RetryState->GetNextRetryDelay(status.Status); + if (!nextDelay) { return false; } @@ -459,7 +460,7 @@ inline void TSingleClusterReadSessionImpl::InitImpl(TDeferredActions init.set_ranges_mode(GetRangesMode()); for (const NPersQueue::TTopicReadSettings& topic : Settings.Topics_) { auto* topicSettings = init.add_topics_read_settings(); - topicSettings->set_topic(topic.Path_); + topicSettings->set_topic(TStringType{topic.Path_}); if (topic.StartingMessageTimestamp_) { topicSettings->set_start_from_written_at_ms(topic.StartingMessageTimestamp_->MilliSeconds()); } @@ -467,7 +468,7 @@ inline void TSingleClusterReadSessionImpl::InitImpl(TDeferredActions topicSettings->add_partition_group_ids(groupId); } } - init.set_consumer(Settings.ConsumerName_); + init.set_consumer(TStringType{Settings.ConsumerName_}); init.set_read_only_original(Settings.ReadOnlyOriginal_); init.mutable_read_params()->set_max_read_size(Settings.MaxMemoryUsageBytes_); if (Settings.MaxTimeLag_) { @@ -488,12 +489,12 @@ inline void TSingleClusterReadSessionImpl::InitImpl(TDeferredActions req; auto& init = *req.mutable_init_request(); - init.set_consumer(Settings.ConsumerName_); - init.set_autoscaling_support(Settings.AutoscalingSupport_); + init.set_consumer(TStringType{Settings.ConsumerName_}); + init.set_auto_partitioning_support(Settings.AutoPartitioningSupport_); for (const TTopicReadSettings& topic : Settings.Topics_) { auto* topicSettings = init.add_topics_read_settings(); - topicSettings->set_path(topic.Path_); + topicSettings->set_path(TStringType{topic.Path_}); for (ui64 partitionId : topic.PartitionIds_) { topicSettings->add_partition_ids(partitionId); } @@ -609,8 +610,8 @@ void TSingleClusterReadSessionImpl::ConfirmPartitionStream if constexpr (UseMigrationProtocol) { auto& startRead = *req.mutable_start_read(); - startRead.mutable_topic()->set_path(partitionStream->GetTopicPath()); - startRead.set_cluster(partitionStream->GetCluster()); + startRead.mutable_topic()->set_path(TStringType{partitionStream->GetTopicPath()}); + startRead.set_cluster(TStringType{partitionStream->GetCluster()}); startRead.set_partition(partitionStream->GetPartitionId()); startRead.set_assign_id(partitionStream->GetAssignId()); if (readOffset) { @@ -681,8 +682,8 @@ void TSingleClusterReadSessionImpl::ConfirmPartitionStream if constexpr (UseMigrationProtocol) { auto& released = *req.mutable_released(); - released.mutable_topic()->set_path(partitionStream->GetTopicPath()); - released.set_cluster(partitionStream->GetCluster()); + released.mutable_topic()->set_path(TStringType{partitionStream->GetTopicPath()}); + released.set_cluster(TStringType{partitionStream->GetCluster()}); released.set_partition(partitionStream->GetPartitionId()); released.set_assign_id(partitionStream->GetAssignId()); } else { @@ -754,8 +755,8 @@ void TSingleClusterReadSessionImpl::RequestPartitionStream if constexpr (UseMigrationProtocol) { auto& status = *req.mutable_status(); - status.mutable_topic()->set_path(partitionStream->GetTopicPath()); - status.set_cluster(partitionStream->GetCluster()); + status.mutable_topic()->set_path(TStringType{partitionStream->GetTopicPath()}); + status.set_cluster(TStringType{partitionStream->GetCluster()}); status.set_partition(partitionStream->GetPartitionId()); status.set_assign_id(partitionStream->GetAssignId()); } else { @@ -2689,7 +2690,7 @@ void TDataDecompressionInfo::TDecompressionTask::operator( ) { const ICodec* codecImpl = TCodecMap::GetTheCodecMap().GetOrThrow(static_cast(data.codec())); std::string decompressed = codecImpl->Decompress(data.data()); - data.set_data(decompressed); + data.set_data(TStringType{decompressed}); data.set_codec(Ydb::PersQueue::V1::CODEC_RAW); } } else { @@ -2699,7 +2700,7 @@ void TDataDecompressionInfo::TDecompressionTask::operator( ) { const ICodec* codecImpl = TCodecMap::GetTheCodecMap().GetOrThrow(static_cast(batch.codec())); std::string decompressed = codecImpl->Decompress(data.data()); - data.set_data(decompressed); + data.set_data(TStringType{decompressed}); } } diff --git a/src/client/topic/impl/topic.cpp b/src/client/topic/impl/topic.cpp index f511f8137d4..a5b31dd5ea7 100644 --- a/src/client/topic/impl/topic.cpp +++ b/src/client/topic/impl/topic.cpp @@ -14,8 +14,8 @@ namespace NYdb::NTopic { class TCommonCodecsProvider { public: TCommonCodecsProvider() { - TCodecMap::GetTheCodecMap().Set((ui32)ECodec::GZIP, std::make_unique()); - TCodecMap::GetTheCodecMap().Set((ui32)ECodec::ZSTD, std::make_unique()); + TCodecMap::GetTheCodecMap().Set((uint32_t)ECodec::GZIP, std::make_unique()); + TCodecMap::GetTheCodecMap().Set((uint32_t)ECodec::ZSTD, std::make_unique()); } }; TCommonCodecsProvider COMMON_CODECS_PROVIDER; @@ -54,7 +54,7 @@ TTopicDescription::TTopicDescription(Ydb::Topic::DescribeTopicResult&& result) : Proto_(std::move(result)) , PartitioningSettings_(Proto_.partitioning_settings()) , RetentionPeriod_(TDuration::Seconds(Proto_.retention_period().seconds())) - , RetentionStorageMb_(Proto_.retention_storage_mb() > 0 ? std::optional(Proto_.retention_storage_mb()) : std::nullopt) + , RetentionStorageMb_(Proto_.retention_storage_mb() > 0 ? std::optional(Proto_.retention_storage_mb()) : std::nullopt) , PartitionWriteSpeedBytesPerSecond_(Proto_.partition_write_speed_bytes_per_second()) , PartitionWriteBurstBytes_(Proto_.partition_write_burst_bytes()) , MeteringMode_(TProtoAccessor::FromProto(Proto_.metering_mode())) @@ -131,7 +131,7 @@ const TPartitioningSettings& TTopicDescription::GetPartitioningSettings() const return PartitioningSettings_; } -ui32 TTopicDescription::GetTotalPartitionsCount() const { +uint32_t TTopicDescription::GetTotalPartitionsCount() const { return Partitions_.size(); } @@ -159,15 +159,15 @@ const TDuration& TTopicDescription::GetRetentionPeriod() const { return RetentionPeriod_; } -std::optional TTopicDescription::GetRetentionStorageMb() const { +std::optional TTopicDescription::GetRetentionStorageMb() const { return RetentionStorageMb_; } -ui64 TTopicDescription::GetPartitionWriteSpeedBytesPerSecond() const { +uint64_t TTopicDescription::GetPartitionWriteSpeedBytesPerSecond() const { return PartitionWriteSpeedBytesPerSecond_; } -ui64 TTopicDescription::GetPartitionWriteBurstBytes() const { +uint64_t TTopicDescription::GetPartitionWriteBurstBytes() const { return PartitionWriteBurstBytes_; } @@ -224,46 +224,46 @@ TPartitioningSettings::TPartitioningSettings(const Ydb::Topic::PartitioningSetti : MinActivePartitions_(settings.min_active_partitions()) , MaxActivePartitions_(settings.max_active_partitions()) , PartitionCountLimit_(settings.partition_count_limit()) - , AutoscalingSettings_(settings.autoscaling_settings()) + , AutoPartitioningSettings_(settings.auto_partitioning_settings()) {} -ui64 TPartitioningSettings::GetMinActivePartitions() const { +uint64_t TPartitioningSettings::GetMinActivePartitions() const { return MinActivePartitions_; } -ui64 TPartitioningSettings::GetMaxActivePartitions() const { +uint64_t TPartitioningSettings::GetMaxActivePartitions() const { return MaxActivePartitions_; } -ui64 TPartitioningSettings::GetPartitionCountLimit() const { +uint64_t TPartitioningSettings::GetPartitionCountLimit() const { return PartitionCountLimit_; } -TAutoscalingSettings TPartitioningSettings::GetAutoscalingSettings() const { - return AutoscalingSettings_; +TAutoPartitioningSettings TPartitioningSettings::GetAutoPartitioningSettings() const { + return AutoPartitioningSettings_; } -TAutoscalingSettings::TAutoscalingSettings(const Ydb::Topic::AutoscalingSettings& settings) - : Strategy_(static_cast(settings.strategy())) - , ThresholdTime_(TDuration::Seconds(settings.partition_write_speed().threshold_time().seconds())) - , ScaleDownThresholdPercent_(settings.partition_write_speed().scale_down_threshold_percent()) - , ScaleUpThresholdPercent_(settings.partition_write_speed().scale_up_threshold_percent()) +TAutoPartitioningSettings::TAutoPartitioningSettings(const Ydb::Topic::AutoPartitioningSettings& settings) + : Strategy_(static_cast(settings.strategy())) + , StabilizationWindow_(TDuration::Seconds(settings.partition_write_speed().stabilization_window().seconds())) + , DownUtilizationPercent_(settings.partition_write_speed().down_utilization_percent()) + , UpUtilizationPercent_(settings.partition_write_speed().up_utilization_percent()) {} -EAutoscalingStrategy TAutoscalingSettings::GetStrategy() const { +EAutoPartitioningStrategy TAutoPartitioningSettings::GetStrategy() const { return Strategy_; } -TDuration TAutoscalingSettings::GetThresholdTime() const { - return ThresholdTime_; +TDuration TAutoPartitioningSettings::GetStabilizationWindow() const { + return StabilizationWindow_; } -ui32 TAutoscalingSettings::GetScaleUpThresholdPercent() const { - return ScaleUpThresholdPercent_; +uint32_t TAutoPartitioningSettings::GetUpUtilizationPercent() const { + return UpUtilizationPercent_; } -ui32 TAutoscalingSettings::GetScaleDownThresholdPercent() const { - return ScaleDownThresholdPercent_; +uint32_t TAutoPartitioningSettings::GetDownUtilizationPercent() const { + return DownUtilizationPercent_; } TTopicStats::TTopicStats(const Ydb::Topic::DescribeTopicResult::TopicStats& topicStats) @@ -276,7 +276,7 @@ TTopicStats::TTopicStats(const Ydb::Topic::DescribeTopicResult::TopicStats& topi { } -ui64 TTopicStats::GetStoreSizeBytes() const { +uint64_t TTopicStats::GetStoreSizeBytes() const { return StoreSizeBytes_; } @@ -288,15 +288,15 @@ TDuration TTopicStats::GetMaxWriteTimeLag() const { return MaxWriteTimeLag_; } -ui64 TTopicStats::GetBytesWrittenPerMinute() const { +uint64_t TTopicStats::GetBytesWrittenPerMinute() const { return BytesWrittenPerMinute_; } -ui64 TTopicStats::GetBytesWrittenPerHour() const { +uint64_t TTopicStats::GetBytesWrittenPerHour() const { return BytesWrittenPerHour_; } -ui64 TTopicStats::GetBytesWrittenPerDay() const { +uint64_t TTopicStats::GetBytesWrittenPerDay() const { return BytesWrittenPerDay_; } @@ -313,15 +313,15 @@ TPartitionStats::TPartitionStats(const Ydb::Topic::PartitionStats& partitionStat {} -ui64 TPartitionStats::GetStartOffset() const { +uint64_t TPartitionStats::GetStartOffset() const { return StartOffset_; } -ui64 TPartitionStats::GetEndOffset() const { +uint64_t TPartitionStats::GetEndOffset() const { return EndOffset_; } -ui64 TPartitionStats::GetStoreSizeBytes() const { +uint64_t TPartitionStats::GetStoreSizeBytes() const { return StoreSizeBytes_; } @@ -333,15 +333,15 @@ TDuration TPartitionStats::GetMaxWriteTimeLag() const { return MaxWriteTimeLag_; } -ui64 TPartitionStats::GetBytesWrittenPerMinute() const { +uint64_t TPartitionStats::GetBytesWrittenPerMinute() const { return BytesWrittenPerMinute_; } -ui64 TPartitionStats::GetBytesWrittenPerHour() const { +uint64_t TPartitionStats::GetBytesWrittenPerHour() const { return BytesWrittenPerHour_; } -ui64 TPartitionStats::GetBytesWrittenPerDay() const { +uint64_t TPartitionStats::GetBytesWrittenPerDay() const { return BytesWrittenPerDay_; } @@ -353,11 +353,11 @@ TPartitionConsumerStats::TPartitionConsumerStats(const Ydb::Topic::DescribeConsu , ReadSessionId_(partitionStats.read_session_id()) {} -ui64 TPartitionConsumerStats::GetCommittedOffset() const { +uint64_t TPartitionConsumerStats::GetCommittedOffset() const { return CommittedOffset_; } -ui64 TPartitionConsumerStats::GetLastReadOffset() const { +uint64_t TPartitionConsumerStats::GetLastReadOffset() const { return LastReadOffset_; } @@ -375,11 +375,11 @@ TPartitionLocation::TPartitionLocation(const Ydb::Topic::PartitionLocation& part { } -i32 TPartitionLocation::GetNodeId() const { +int32_t TPartitionLocation::GetNodeId() const { return NodeId_; } -i64 TPartitionLocation::GetGeneration() const { +int64_t TPartitionLocation::GetGeneration() const { return Generation_; } @@ -441,7 +441,7 @@ bool TPartitionInfo::GetActive() const { return Active_; } -ui64 TPartitionInfo::GetPartitionId() const { +uint64_t TPartitionInfo::GetPartitionId() const { return PartitionId_; } @@ -473,7 +473,7 @@ TAsyncDescribeConsumerResult TTopicClient::DescribeConsumer(const std::string& p return Impl_->DescribeConsumer(path, consumer, settings); } -TAsyncDescribePartitionResult TTopicClient::DescribePartition(const std::string& path, i64 partitionId, const TDescribePartitionSettings& settings) { +TAsyncDescribePartitionResult TTopicClient::DescribePartition(const std::string& path, int64_t partitionId, const TDescribePartitionSettings& settings) { return Impl_->DescribePartition(path, partitionId, settings); } @@ -490,7 +490,7 @@ std::shared_ptr TTopicClient::CreateWriteSession(const TWriteSess return Impl_->CreateWriteSession(settings); } -TAsyncStatus TTopicClient::CommitOffset(const std::string& path, ui64 partitionId, const std::string& consumerName, ui64 offset, +TAsyncStatus TTopicClient::CommitOffset(const std::string& path, uint64_t partitionId, const std::string& consumerName, uint64_t offset, const TCommitOffsetSettings& settings) { return Impl_->CommitOffset(path, partitionId, consumerName, offset, settings); } diff --git a/src/client/topic/impl/topic_impl.h b/src/client/topic/impl/topic_impl.h index aee93ac6e38..5c4e594ce01 100644 --- a/src/client/topic/impl/topic_impl.h +++ b/src/client/topic/impl/topic_impl.h @@ -9,7 +9,7 @@ #include #include -#include +#include namespace NYdb::NTopic { struct TOffsetsRange { @@ -42,7 +42,7 @@ class TTopicClient::TImpl : public TClientImplCommon { template static void ConvertConsumerToProto(const TConsumerSettings& settings, Ydb::Topic::Consumer& consumerProto) { - consumerProto.set_name(settings.ConsumerName_); + consumerProto.set_name(TStringType{settings.ConsumerName_}); consumerProto.set_important(settings.Important_); consumerProto.mutable_read_from()->set_seconds(settings.ReadFrom_.Seconds()); @@ -55,7 +55,7 @@ class TTopicClient::TImpl : public TClientImplCommon { } static void ConvertAlterConsumerToProto(const TAlterConsumerSettings& settings, Ydb::Topic::AlterConsumer& consumerProto) { - consumerProto.set_name(settings.ConsumerName_); + consumerProto.set_name(TStringType{settings.ConsumerName_}); if (settings.SetImportant_) consumerProto.set_set_important(*settings.SetImportant_); if (settings.SetReadFrom_) @@ -75,15 +75,15 @@ class TTopicClient::TImpl : public TClientImplCommon { static Ydb::Topic::CreateTopicRequest MakePropsCreateRequest(const std::string& path, const TCreateTopicSettings& settings) { Ydb::Topic::CreateTopicRequest request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); request.mutable_partitioning_settings()->set_min_active_partitions(settings.PartitioningSettings_.GetMinActivePartitions()); request.mutable_partitioning_settings()->set_partition_count_limit(settings.PartitioningSettings_.GetPartitionCountLimit()); request.mutable_partitioning_settings()->set_max_active_partitions(settings.PartitioningSettings_.GetMaxActivePartitions()); - request.mutable_partitioning_settings()->mutable_autoscaling_settings()->set_strategy(static_cast(settings.PartitioningSettings_.GetAutoscalingSettings().GetStrategy())); - request.mutable_partitioning_settings()->mutable_autoscaling_settings()->mutable_partition_write_speed()->mutable_threshold_time()->set_seconds(settings.PartitioningSettings_.GetAutoscalingSettings().GetThresholdTime().Seconds()); - request.mutable_partitioning_settings()->mutable_autoscaling_settings()->mutable_partition_write_speed()->set_scale_up_threshold_percent(settings.PartitioningSettings_.GetAutoscalingSettings().GetScaleUpThresholdPercent()); - request.mutable_partitioning_settings()->mutable_autoscaling_settings()->mutable_partition_write_speed()->set_scale_down_threshold_percent(settings.PartitioningSettings_.GetAutoscalingSettings().GetScaleDownThresholdPercent()); + request.mutable_partitioning_settings()->mutable_auto_partitioning_settings()->set_strategy(static_cast(settings.PartitioningSettings_.GetAutoPartitioningSettings().GetStrategy())); + request.mutable_partitioning_settings()->mutable_auto_partitioning_settings()->mutable_partition_write_speed()->mutable_stabilization_window()->set_seconds(settings.PartitioningSettings_.GetAutoPartitioningSettings().GetStabilizationWindow().Seconds()); + request.mutable_partitioning_settings()->mutable_auto_partitioning_settings()->mutable_partition_write_speed()->set_up_utilization_percent(settings.PartitioningSettings_.GetAutoPartitioningSettings().GetUpUtilizationPercent()); + request.mutable_partitioning_settings()->mutable_auto_partitioning_settings()->mutable_partition_write_speed()->set_down_utilization_percent(settings.PartitioningSettings_.GetAutoPartitioningSettings().GetDownUtilizationPercent()); request.mutable_retention_period()->set_seconds(settings.RetentionPeriod_.Seconds()); @@ -119,7 +119,7 @@ class TTopicClient::TImpl : public TClientImplCommon { static Ydb::Topic::AlterTopicRequest MakePropsAlterRequest(const std::string& path, const TAlterTopicSettings& settings) { Ydb::Topic::AlterTopicRequest request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); if (settings.AlterPartitioningSettings_) { if (settings.AlterPartitioningSettings_->MinActivePartitions_) { @@ -128,18 +128,18 @@ class TTopicClient::TImpl : public TClientImplCommon { if (settings.AlterPartitioningSettings_->MaxActivePartitions_) { request.mutable_alter_partitioning_settings()->set_set_max_active_partitions(*settings.AlterPartitioningSettings_->MaxActivePartitions_); } - if (settings.AlterPartitioningSettings_->AutoscalingSettings_) { - if (settings.AlterPartitioningSettings_->AutoscalingSettings_->Strategy_) { - request.mutable_alter_partitioning_settings()->mutable_alter_autoscaling_settings()->set_set_strategy(static_cast(*settings.AlterPartitioningSettings_->AutoscalingSettings_->Strategy_)); + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_) { + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_->Strategy_) { + request.mutable_alter_partitioning_settings()->mutable_alter_auto_partitioning_settings()->set_set_strategy(static_cast(*settings.AlterPartitioningSettings_->AutoPartitioningSettings_->Strategy_)); } - if (settings.AlterPartitioningSettings_->AutoscalingSettings_->ScaleDownThresholdPercent_) { - request.mutable_alter_partitioning_settings()->mutable_alter_autoscaling_settings()->mutable_set_partition_write_speed()->set_set_scale_down_threshold_percent(*settings.AlterPartitioningSettings_->AutoscalingSettings_->ScaleDownThresholdPercent_); + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_->DownUtilizationPercent_) { + request.mutable_alter_partitioning_settings()->mutable_alter_auto_partitioning_settings()->mutable_set_partition_write_speed()->set_set_down_utilization_percent(*settings.AlterPartitioningSettings_->AutoPartitioningSettings_->DownUtilizationPercent_); } - if (settings.AlterPartitioningSettings_->AutoscalingSettings_->ScaleUpThresholdPercent_) { - request.mutable_alter_partitioning_settings()->mutable_alter_autoscaling_settings()->mutable_set_partition_write_speed()->set_set_scale_up_threshold_percent(*settings.AlterPartitioningSettings_->AutoscalingSettings_->ScaleUpThresholdPercent_); + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_->UpUtilizationPercent_) { + request.mutable_alter_partitioning_settings()->mutable_alter_auto_partitioning_settings()->mutable_set_partition_write_speed()->set_set_up_utilization_percent(*settings.AlterPartitioningSettings_->AutoPartitioningSettings_->UpUtilizationPercent_); } - if (settings.AlterPartitioningSettings_->AutoscalingSettings_->ThresholdTime_) { - request.mutable_alter_partitioning_settings()->mutable_alter_autoscaling_settings()->mutable_set_partition_write_speed()->mutable_set_threshold_time()->set_seconds(settings.AlterPartitioningSettings_->AutoscalingSettings_->ThresholdTime_->Seconds()); + if (settings.AlterPartitioningSettings_->AutoPartitioningSettings_->StabilizationWindow_) { + request.mutable_alter_partitioning_settings()->mutable_alter_auto_partitioning_settings()->mutable_set_partition_write_speed()->mutable_set_stabilization_window()->set_seconds(settings.AlterPartitioningSettings_->AutoPartitioningSettings_->StabilizationWindow_->Seconds()); } } } @@ -174,7 +174,7 @@ class TTopicClient::TImpl : public TClientImplCommon { } for (const auto& consumer : settings.DropConsumers_) { - request.add_drop_consumers(consumer); + request.add_drop_consumers(TStringType{consumer}); } for (const auto& consumer : settings.AlterConsumers_) { @@ -198,7 +198,7 @@ class TTopicClient::TImpl : public TClientImplCommon { TAsyncStatus DropTopic(const std::string& path, const TDropTopicSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); return RunSimple( std::move(request), @@ -208,7 +208,7 @@ class TTopicClient::TImpl : public TClientImplCommon { TAsyncDescribeTopicResult DescribeTopic(const std::string& path, const TDescribeTopicSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); if (settings.IncludeStats_) { request.set_include_stats(true); @@ -244,8 +244,8 @@ class TTopicClient::TImpl : public TClientImplCommon { TAsyncDescribeConsumerResult DescribeConsumer(const std::string& path, const std::string& consumer, const TDescribeConsumerSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); - request.set_consumer(consumer); + request.set_path(TStringType{path}); + request.set_consumer(TStringType{consumer}); if (settings.IncludeStats_) { request.set_include_stats(true); @@ -281,7 +281,7 @@ class TTopicClient::TImpl : public TClientImplCommon { TAsyncDescribePartitionResult DescribePartition(const std::string& path, i64 partitionId, const TDescribePartitionSettings& settings) { auto request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); request.set_partition_id(partitionId); if (settings.IncludeStats_) { @@ -318,9 +318,9 @@ class TTopicClient::TImpl : public TClientImplCommon { TAsyncStatus CommitOffset(const std::string& path, ui64 partitionId, const std::string& consumerName, ui64 offset, const TCommitOffsetSettings& settings) { Ydb::Topic::CommitOffsetRequest request = MakeOperationRequest(settings); - request.set_path(path); + request.set_path(TStringType{path}); request.set_partition_id(partitionId); - request.set_consumer(consumerName); + request.set_consumer(TStringType{consumerName}); request.set_offset(offset); return RunSimple( @@ -336,12 +336,12 @@ class TTopicClient::TImpl : public TClientImplCommon { { auto request = MakeOperationRequest(settings); - request.mutable_tx()->set_id(tx.GetId()); - request.mutable_tx()->set_session(tx.GetSession().GetId()); + request.mutable_tx()->set_id(TStringType{tx.GetId()}); + request.mutable_tx()->set_session(TStringType{tx.GetSession().GetId()}); for (auto& t : topics) { auto* topic = request.mutable_topics()->Add(); - topic->set_path(t.Path); + topic->set_path(TStringType{t.Path}); for (auto& p : t.Partitions) { auto* partition = topic->mutable_partitions()->Add(); @@ -355,7 +355,7 @@ class TTopicClient::TImpl : public TClientImplCommon { } } - request.set_consumer(consumerName); + request.set_consumer(TStringType{consumerName}); return RunSimple( std::move(request), diff --git a/src/client/topic/impl/write_session.cpp b/src/client/topic/impl/write_session.cpp index 99635d71cc7..c741c477732 100644 --- a/src/client/topic/impl/write_session.cpp +++ b/src/client/topic/impl/write_session.cpp @@ -19,7 +19,7 @@ void TWriteSession::Start(const TDuration& delay) { TryGetImpl()->Start(delay); } -NThreading::TFuture TWriteSession::GetInitSeqNo() { +NThreading::TFuture TWriteSession::GetInitSeqNo() { return TryGetImpl()->GetInitSeqNo(); } @@ -36,7 +36,7 @@ NThreading::TFuture TWriteSession::WaitEvent() { } void TWriteSession::WriteEncoded(TContinuationToken&& token, std::string_view data, ECodec codec, ui32 originalSize, - std::optional seqNo, std::optional createTimestamp) { + std::optional seqNo, std::optional createTimestamp) { auto message = TWriteMessage::CompressedMessage(std::move(data), codec, originalSize); if (seqNo.has_value()) message.SeqNo(*seqNo); @@ -50,7 +50,7 @@ void TWriteSession::WriteEncoded(TContinuationToken&& token, TWriteMessage&& mes TryGetImpl()->WriteInternal(std::move(token), std::move(message)); } -void TWriteSession::Write(TContinuationToken&& token, std::string_view data, std::optional seqNo, +void TWriteSession::Write(TContinuationToken&& token, std::string_view data, std::optional seqNo, std::optional createTimestamp) { TWriteMessage message{std::move(data)}; if (seqNo.has_value()) @@ -102,12 +102,12 @@ TSimpleBlockingWriteSession::TSimpleBlockingWriteSession( Writer->Start(TDuration::Max()); } -ui64 TSimpleBlockingWriteSession::GetInitSeqNo() { +uint64_t TSimpleBlockingWriteSession::GetInitSeqNo() { return Writer->GetInitSeqNo().GetValueSync(); } bool TSimpleBlockingWriteSession::Write( - std::string_view data, std::optional seqNo, std::optional createTimestamp, const TDuration& blockTimeout + std::string_view data, std::optional seqNo, std::optional createTimestamp, const TDuration& blockTimeout ) { auto message = TWriteMessage(std::move(data)) .SeqNo(seqNo) diff --git a/src/client/topic/impl/write_session.h b/src/client/topic/impl/write_session.h index a43d8e794a8..1922c43068a 100644 --- a/src/client/topic/impl/write_session.h +++ b/src/client/topic/impl/write_session.h @@ -28,13 +28,13 @@ class TWriteSession : public IWriteSession, std::optional GetEvent(bool block = false) override; std::vector GetEvents(bool block = false, std::optional maxEventsCount = std::nullopt) override; - NThreading::TFuture GetInitSeqNo() override; + NThreading::TFuture GetInitSeqNo() override; void Write(TContinuationToken&& continuationToken, std::string_view data, - std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override; + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override; void WriteEncoded(TContinuationToken&& continuationToken, std::string_view data, ECodec codec, ui32 originalSize, - std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override; + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override; void Write(TContinuationToken&& continuationToken, TWriteMessage&& message) override; @@ -64,12 +64,12 @@ class TSimpleBlockingWriteSession : public ISimpleBlockingWriteSession { std::shared_ptr connections, TDbDriverStatePtr dbDriverState); - bool Write(std::string_view data, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt, + bool Write(std::string_view data, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt, const TDuration& blockTimeout = TDuration::Max()) override; bool Write(TWriteMessage&& message, const TDuration& blockTimeout = TDuration::Max()) override; - ui64 GetInitSeqNo() override; + uint64_t GetInitSeqNo() override; bool Close(TDuration closeTimeout = TDuration::Max()) override; bool IsAlive() const override; diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index 9a2a7d83876..39654875b34 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -17,7 +17,7 @@ namespace NYdb::NTopic { const TDuration UPDATE_TOKEN_PERIOD = TDuration::Hours(1); // Error code from file ydb/public/api/protos/persqueue_error_codes_v1.proto -const ui64 WRITE_ERROR_PARTITION_INACTIVE = 500029; +const uint64_t WRITE_ERROR_PARTITION_INACTIVE = 500029; namespace { @@ -68,7 +68,7 @@ TWriteSessionImpl::TWriteSessionImpl( , Connections(std::move(connections)) , DbDriverState(std::move(dbDriverState)) , PrevToken(DbDriverState->CredentialsProvider ? DbDriverState->CredentialsProvider->GetAuthInfo() : "") - , InitSeqNoPromise(NThreading::NewPromise()) + , InitSeqNoPromise(NThreading::NewPromise()) , WakeupInterval( Settings.BatchFlushInterval_.value_or(TDuration::Zero()) ? std::min(Settings.BatchFlushInterval_.value_or(TDuration::Seconds(1)) / 5, TDuration::MilliSeconds(100)) @@ -184,13 +184,12 @@ TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStat PreferredPartitionLocation = {}; } - std::optional nextDelay = TDuration::Zero(); if (!RetryState) { RetryState = Settings.RetryPolicy_->CreateRetryState(); } - nextDelay = RetryState->GetNextRetryDelay(status.Status); + auto nextDelay = RetryState->GetNextRetryDelay(status.Status); - if (nextDelay.has_value()) { + if (nextDelay) { result.StartDelay = *nextDelay; result.DoRestart = true; LOG_LAZY(DbDriverState->Log, TLOG_WARNING, LogPrefix() << "Write session will restart in " << result.StartDelay); @@ -354,7 +353,7 @@ void TWriteSessionImpl::OnDescribePartition(const TStatus& status, const Ydb::To Connect(TDuration::Zero()); } -std::optional TWriteSessionImpl::GetPreferredEndpointImpl(ui32 partitionId, ui64 partitionNodeId) { +std::optional TWriteSessionImpl::GetPreferredEndpointImpl(ui32 partitionId, uint64_t partitionNodeId) { Y_ABORT_UNLESS(Lock.IsLocked()); TEndpointKey preferredEndpoint{"", partitionNodeId}; @@ -420,7 +419,7 @@ void TWriteSessionImpl::InitWriter() { // No Lock, very initial start - no race } // Client method -NThreading::TFuture TWriteSessionImpl::GetInitSeqNo() { +NThreading::TFuture TWriteSessionImpl::GetInitSeqNo() { if (!Settings.DeduplicationEnabled_.value_or(true)) { LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "GetInitSeqNo called with deduplication disabled"); ThrowFatalError("Cannot call GetInitSeqNo when deduplication is disabled"); @@ -450,24 +449,24 @@ std::vector TWriteSessionImpl::GetEvents(bool block, return EventsQueue->GetEvents(block, maxEventsCount); } -ui64 TWriteSessionImpl::GetIdImpl(ui64 seqNo) { +uint64_t TWriteSessionImpl::GetIdImpl(uint64_t seqNo) { Y_ABORT_UNLESS(AutoSeqNoMode.has_value()); Y_ABORT_UNLESS(!*AutoSeqNoMode || InitSeqNo.has_value() && seqNo > *InitSeqNo); return *AutoSeqNoMode ? seqNo - *InitSeqNo : seqNo; } -ui64 TWriteSessionImpl::GetSeqNoImpl(ui64 id) { +uint64_t TWriteSessionImpl::GetSeqNoImpl(uint64_t id) { Y_ABORT_UNLESS(AutoSeqNoMode.has_value()); Y_ABORT_UNLESS(InitSeqNo.has_value()); return *AutoSeqNoMode ? id + *InitSeqNo : id; } -ui64 TWriteSessionImpl::GetNextIdImpl(const std::optional& seqNo) { +uint64_t TWriteSessionImpl::GetNextIdImpl(const std::optional& seqNo) { Y_ABORT_UNLESS(Lock.IsLocked()); - ui64 id = ++NextId; + uint64_t id = ++NextId; if (!AutoSeqNoMode.has_value()) { AutoSeqNoMode = !seqNo.has_value(); } @@ -725,8 +724,8 @@ void TWriteSessionImpl::InitImpl() { TClientMessage req; auto* init = req.mutable_init_request(); - init->set_path(Settings.Path_); - init->set_producer_id(Settings.ProducerId_); + init->set_path(TStringType{Settings.Path_}); + init->set_producer_id(TStringType{Settings.ProducerId_}); if (Settings.DirectWriteToPartition_ && (Settings.PartitionId_.has_value() || DirectWriteToPartitionId.has_value())) { auto partition_id = Settings.PartitionId_.has_value() ? *Settings.PartitionId_ : *DirectWriteToPartitionId; @@ -738,7 +737,7 @@ void TWriteSessionImpl::InitImpl() { init->set_partition_id(*Settings.PartitionId_); LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: write to partition: " << *Settings.PartitionId_); } else { - init->set_message_group_id(Settings.MessageGroupId_); + init->set_message_group_id(TStringType{Settings.MessageGroupId_}); LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: write to message_group: " << Settings.MessageGroupId_); } @@ -778,7 +777,7 @@ void TWriteSessionImpl::WriteToProcessorImpl(TWriteSessionImpl::TClientMessage&& void TWriteSessionImpl::ReadFromProcessor() { Y_ASSERT(Processor); IProcessor::TPtr prc; - ui64 generation; + uint64_t generation; std::function callback; { std::lock_guard guard(Lock); @@ -943,7 +942,7 @@ TWriteSessionImpl::TProcessSrvMessageResult TWriteSessionImpl::ProcessServerMess } PartitionId = initResponse.partition_id(); - ui64 newLastSeqNo = initResponse.last_seq_no(); + uint64_t newLastSeqNo = initResponse.last_seq_no(); if (!Settings.DeduplicationEnabled_.value_or(true)) { newLastSeqNo = 0; } @@ -995,7 +994,7 @@ TWriteSessionImpl::TProcessSrvMessageResult TWriteSessionImpl::ProcessServerMess for (size_t messageIndex = 0, endIndex = batchWriteResponse.acks_size(); messageIndex != endIndex; ++messageIndex) { // TODO: Fill writer statistics auto ack = batchWriteResponse.acks(messageIndex); - ui64 sequenceNumber = ack.seq_no(); + uint64_t sequenceNumber = ack.seq_no(); Y_ABORT_UNLESS(ack.has_written() || ack.has_skipped()); auto msgWriteStatus = ack.has_written() @@ -1004,7 +1003,7 @@ TWriteSessionImpl::TProcessSrvMessageResult TWriteSessionImpl::ProcessServerMess ? TWriteSessionEvent::TWriteAck::EES_ALREADY_WRITTEN : TWriteSessionEvent::TWriteAck::EES_DISCARDED); - ui64 offset = ack.has_written() ? ack.written().offset() : 0; + uint64_t offset = ack.has_written() ? ack.written().offset() : 0; acksEvent.Acks.push_back(TWriteSessionEvent::TWriteAck{ GetIdImpl(sequenceNumber), @@ -1034,13 +1033,13 @@ TWriteSessionImpl::TProcessSrvMessageResult TWriteSessionImpl::ProcessServerMess return result; } -bool TWriteSessionImpl::CleanupOnAcknowledged(ui64 id) { +bool TWriteSessionImpl::CleanupOnAcknowledged(uint64_t id) { bool result = false; LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session: acknoledged message " << id); UpdateTimedCountersImpl(); const auto& sentFront = SentOriginalMessages.front(); - ui64 size = 0; - ui64 compressedSize = 0; + uint64_t size = 0; + uint64_t compressedSize = 0; if(!SentPackedMessage.empty() && SentPackedMessage.front().Offset == id) { auto memoryUsage = OnMemoryUsageChangedImpl(-SentPackedMessage.front().Data.size()); result = memoryUsage.NowOk && !memoryUsage.WasOk; @@ -1187,7 +1186,7 @@ void TWriteSessionImpl::ResetForRetryImpl() { PackedMessagesToSend.emplace(std::move(SentPackedMessage.front())); SentPackedMessage.pop(); } - ui64 minId = PackedMessagesToSend.empty() ? NextId + 1 : PackedMessagesToSend.top().Offset; + uint64_t minId = PackedMessagesToSend.empty() ? NextId + 1 : PackedMessagesToSend.top().Offset; std::queue freshOriginalMessagesToSend; OriginalMessagesToSend.swap(freshOriginalMessagesToSend); while (!SentOriginalMessages.empty()) { @@ -1210,7 +1209,7 @@ void TWriteSessionImpl::FlushWriteIfRequiredImpl() { Y_ABORT_UNLESS(Lock.IsLocked()); if (!CurrentBatch.Empty() && !CurrentBatch.FlushRequested) { - MessagesAcquired += static_cast(CurrentBatch.Acquire()); + MessagesAcquired += static_cast(CurrentBatch.Acquire()); if (TInstant::Now() - CurrentBatch.StartedAt >= Settings.BatchFlushInterval_.value_or(TDuration::Zero()) || CurrentBatch.CurrentSize >= Settings.BatchFlushSizeBytes_.value_or(0) || CurrentBatch.CurrentSize >= MaxBlockSize @@ -1236,7 +1235,7 @@ size_t TWriteSessionImpl::WriteBatchImpl() { const bool skipCompression = Settings.Codec_ == ECodec::RAW || CurrentBatch.HasCodec(); if (!skipCompression && Settings.CompressionExecutor_->IsAsync()) { - MessagesAcquired += static_cast(CurrentBatch.Acquire()); + MessagesAcquired += static_cast(CurrentBatch.Acquire()); } size_t size = 0; @@ -1336,7 +1335,7 @@ void TWriteSessionImpl::UpdateTokenIfNeededImpl() { PrevToken = token; TClientMessage clientMessage; - clientMessage.mutable_update_token_request()->set_token(token); + clientMessage.mutable_update_token_request()->set_token(TStringType{token}); Processor->Write(std::move(clientMessage)); } @@ -1382,8 +1381,8 @@ void TWriteSessionImpl::SendImpl() { auto* msgData = writeRequest->add_messages(); if (message.Tx) { - writeRequest->mutable_tx()->set_id(message.Tx->GetId()); - writeRequest->mutable_tx()->set_session(message.Tx->GetSession().GetId()); + writeRequest->mutable_tx()->set_id(TStringType{message.Tx->GetId()}); + writeRequest->mutable_tx()->set_session(TStringType{message.Tx->GetSession().GetId()}); } msgData->set_seq_no(GetSeqNoImpl(message.Id)); @@ -1391,8 +1390,8 @@ void TWriteSessionImpl::SendImpl() { for (auto& [k, v] : message.MessageMeta) { auto* pair = msgData->add_metadata_items(); - pair->set_key(k); - pair->set_value(v); + pair->set_key(TStringType{k}); + pair->set_value(TStringType{v}); } SentOriginalMessages.emplace(std::move(message)); OriginalMessagesToSend.pop(); diff --git a/src/client/topic/impl/write_session_impl.h b/src/client/topic/impl/write_session_impl.h index 57446f11b6d..d8d78a487e3 100644 --- a/src/client/topic/impl/write_session_impl.h +++ b/src/client/topic/impl/write_session_impl.h @@ -156,7 +156,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, using IProcessor = IWriteSessionConnectionProcessorFactory::IProcessor; struct TMessage { - ui64 Id; + uint64_t Id; TInstant CreatedAt; std::string_view DataRef; std::optional Codec; @@ -164,7 +164,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, std::vector> MessageMeta; const NTable::TTransaction* Tx; - TMessage(ui64 id, const TInstant& createdAt, std::string_view data, std::optional codec = {}, + TMessage(uint64_t id, const TInstant& createdAt, std::string_view data, std::optional codec = {}, ui32 originalSize = 0, const std::vector>& messageMeta = {}, const NTable::TTransaction* tx = nullptr) : Id(id) @@ -180,12 +180,12 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, struct TMessageBatch { TBuffer Data; std::vector Messages; - ui64 CurrentSize = 0; + uint64_t CurrentSize = 0; TInstant StartedAt = TInstant::Zero(); bool Acquired = false; bool FlushRequested = false; - void Add(ui64 id, const TInstant& createdAt, std::string_view data, std::optional codec, ui32 originalSize, + void Add(uint64_t id, const TInstant& createdAt, std::string_view data, std::optional codec, ui32 originalSize, const std::vector>& messageMeta, const NTable::TTransaction* tx) { if (StartedAt == TInstant::Zero()) @@ -257,13 +257,13 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, }; struct TOriginalMessage { - ui64 Id; + uint64_t Id; TInstant CreatedAt; size_t Size; std::vector> MessageMeta; const NTable::TTransaction* Tx; - TOriginalMessage(const ui64 id, const TInstant createdAt, const size_t size, + TOriginalMessage(const uint64_t id, const TInstant createdAt, const size_t size, const NTable::TTransaction* tx) : Id(id) , CreatedAt(createdAt) @@ -271,7 +271,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, , Tx(tx) {} - TOriginalMessage(const ui64 id, const TInstant createdAt, const size_t size, + TOriginalMessage(const uint64_t id, const TInstant createdAt, const size_t size, std::vector>&& messageMeta, const NTable::TTransaction* tx) : Id(id) @@ -297,7 +297,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, }; struct TProcessSrvMessageResult { THandleResult HandleResult; - std::optional InitSeqNo; + std::optional InitSeqNo; std::vector Events; bool Ok = true; }; @@ -318,11 +318,11 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, std::optional GetEvent(bool block = false); std::vector GetEvents(bool block = false, std::optional maxEventsCount = std::nullopt); - NThreading::TFuture GetInitSeqNo(); + NThreading::TFuture GetInitSeqNo(); void Write(TContinuationToken&& continuationToken, TWriteMessage&& message); - void Write(TContinuationToken&&, std::string_view, std::optional seqNo = std::nullopt, + void Write(TContinuationToken&&, std::string_view, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) { Y_UNUSED(seqNo); Y_UNUSED(createTimestamp); @@ -332,7 +332,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, void WriteEncoded(TContinuationToken&& continuationToken, TWriteMessage&& message); void WriteEncoded(TContinuationToken&&, std::string_view, ECodec, ui32, - std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) { + std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) { Y_UNUSED(seqNo); Y_UNUSED(createTimestamp); Y_ABORT("Do not use this method"); @@ -385,11 +385,11 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, //std::string GetDebugIdentity() const; TClientMessage GetInitClientMessage(); - bool CleanupOnAcknowledged(ui64 id); + bool CleanupOnAcknowledged(uint64_t id); bool IsReadyToSendNextImpl() const; - ui64 GetNextIdImpl(const std::optional& seqNo); - ui64 GetSeqNoImpl(ui64 id); - ui64 GetIdImpl(ui64 seqNo); + uint64_t GetNextIdImpl(const std::optional& seqNo); + uint64_t GetSeqNoImpl(uint64_t id); + uint64_t GetIdImpl(uint64_t seqNo); void SendImpl(); void AbortImpl(); void CloseImpl(EStatus statusCode, NYql::TIssues&& issues); @@ -407,7 +407,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, void ConnectToPreferredPartitionLocation(const TDuration& delay); void OnDescribePartition(const TStatus& status, const Ydb::Topic::DescribePartitionResult& proto, const NYdbGrpc::IQueueClientContextPtr& describePartitionContext); - std::optional GetPreferredEndpointImpl(ui32 partitionId, ui64 partitionNodeId); + std::optional GetPreferredEndpointImpl(ui32 partitionId, uint64_t partitionNodeId); bool TxIsChanged(const Ydb::Topic::StreamWriteMessage_WriteRequest* writeRequest) const; @@ -459,13 +459,13 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, bool SessionEstablished = false; ui32 PartitionId = 0; TPartitionLocation PreferredPartitionLocation = {}; - ui64 NextId = 0; - ui64 MinUnsentId = 1; - std::optional InitSeqNo; + uint64_t NextId = 0; + uint64_t MinUnsentId = 1; + std::optional InitSeqNo; std::optional AutoSeqNoMode; bool ValidateSeqNoMode = false; - NThreading::TPromise InitSeqNoPromise; + NThreading::TPromise InitSeqNoPromise; bool InitSeqNoSetDone = false; TInstant SessionStartedTs; TInstant LastCountersUpdateTs = TInstant::Zero(); @@ -474,9 +474,9 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, TDuration WakeupInterval; // Set by the write session, if Settings.DirectWriteToPartition is true and Settings.PartitionId is unset. Otherwise ignored. - std::optional DirectWriteToPartitionId; + std::optional DirectWriteToPartitionId; protected: - ui64 MessagesAcquired = 0; + uint64_t MessagesAcquired = 0; }; } // namespace NYdb::NTopic diff --git a/src/client/topic/ut/CMakeLists.darwin-arm64.txt b/src/client/topic/ut/CMakeLists.darwin-arm64.txt deleted file mode 100644 index ad068587c32..00000000000 --- a/src/client/topic/ut/CMakeLists.darwin-arm64.txt +++ /dev/null @@ -1,90 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -add_subdirectory(ut_utils) - -add_executable(ydb-public-sdk-client-ydb_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_topic-ut PUBLIC - yutil - cpp-testing-unittest_main - client-ydb_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic-impl - ydb_topic-ut-ut_utils -) -target_link_options(ydb-public-sdk-client-ydb_topic-ut PRIVATE - -Wl,-platform_version,macos,11.0,11.0 - -fPIC - -fPIC - -framework - CoreFoundation -) -target_sources(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/basic_usage_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/describe_topic_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/local_partition_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/topic_to_table_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_topic-ut - system_allocator -) -vcs_info(ydb-public-sdk-client-ydb_topic-ut) diff --git a/src/client/topic/ut/CMakeLists.darwin-x86_64.txt b/src/client/topic/ut/CMakeLists.darwin-x86_64.txt deleted file mode 100644 index bec0c52cd03..00000000000 --- a/src/client/topic/ut/CMakeLists.darwin-x86_64.txt +++ /dev/null @@ -1,91 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -add_subdirectory(ut_utils) - -add_executable(ydb-public-sdk-client-ydb_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_topic-ut PUBLIC - yutil - cpuid_check - cpp-testing-unittest_main - client-ydb_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic-impl - ydb_topic-ut-ut_utils -) -target_link_options(ydb-public-sdk-client-ydb_topic-ut PRIVATE - -Wl,-platform_version,macos,11.0,11.0 - -fPIC - -fPIC - -framework - CoreFoundation -) -target_sources(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/basic_usage_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/describe_topic_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/local_partition_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/topic_to_table_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_topic-ut - system_allocator -) -vcs_info(ydb-public-sdk-client-ydb_topic-ut) diff --git a/src/client/topic/ut/CMakeLists.linux-aarch64.txt b/src/client/topic/ut/CMakeLists.linux-aarch64.txt deleted file mode 100644 index 7f66a57557d..00000000000 --- a/src/client/topic/ut/CMakeLists.linux-aarch64.txt +++ /dev/null @@ -1,94 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -add_subdirectory(ut_utils) - -add_executable(ydb-public-sdk-client-ydb_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_topic-ut PUBLIC - - yutil - cpp-testing-unittest_main - client-ydb_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic-impl - ydb_topic-ut-ut_utils -) -target_link_options(ydb-public-sdk-client-ydb_topic-ut PRIVATE - -ldl - -lrt - -Wl,--no-as-needed - -fPIC - -fPIC - -lpthread - -lrt - -ldl -) -target_sources(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/basic_usage_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/describe_topic_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/local_partition_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/topic_to_table_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_topic-ut - cpp-malloc-jemalloc -) -vcs_info(ydb-public-sdk-client-ydb_topic-ut) diff --git a/src/client/topic/ut/CMakeLists.linux-x86_64.txt b/src/client/topic/ut/CMakeLists.linux-x86_64.txt deleted file mode 100644 index a9fff6c7b7f..00000000000 --- a/src/client/topic/ut/CMakeLists.linux-x86_64.txt +++ /dev/null @@ -1,96 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -add_subdirectory(ut_utils) - -add_executable(ydb-public-sdk-client-ydb_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_topic-ut PUBLIC - - yutil - cpuid_check - cpp-testing-unittest_main - client-ydb_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic-impl - ydb_topic-ut-ut_utils -) -target_link_options(ydb-public-sdk-client-ydb_topic-ut PRIVATE - -ldl - -lrt - -Wl,--no-as-needed - -fPIC - -fPIC - -lpthread - -lrt - -ldl -) -target_sources(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/basic_usage_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/describe_topic_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/local_partition_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/topic_to_table_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_topic-ut - cpp-malloc-tcmalloc - libs-tcmalloc-no_percpu_cache -) -vcs_info(ydb-public-sdk-client-ydb_topic-ut) diff --git a/src/client/topic/ut/CMakeLists.txt b/src/client/topic/ut/CMakeLists.txt deleted file mode 100644 index 4fbe9853d99..00000000000 --- a/src/client/topic/ut/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.linux-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") - include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") - include(CMakeLists.darwin-arm64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") - include(CMakeLists.windows-x86_64.txt) -endif() diff --git a/src/client/topic/ut/CMakeLists.windows-x86_64.txt b/src/client/topic/ut/CMakeLists.windows-x86_64.txt deleted file mode 100644 index f2cbf272fe4..00000000000 --- a/src/client/topic/ut/CMakeLists.windows-x86_64.txt +++ /dev/null @@ -1,84 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -add_subdirectory(ut_utils) - -add_executable(ydb-public-sdk-client-ydb_topic-ut) -target_compile_options(ydb-public-sdk-client-ydb_topic-ut PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_include_directories(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic -) -target_link_libraries(ydb-public-sdk-client-ydb_topic-ut PUBLIC - yutil - cpuid_check - cpp-testing-unittest_main - client-ydb_topic - cpp-testing-gmock_in_unittest - core-testlib-default - public-lib-json_value - public-lib-yson_value - client-ydb_driver - cpp-client-ydb_persqueue_core - client-ydb_persqueue_core-impl - ydb_persqueue_core-ut-ut_utils - client-ydb_topic-codecs - client-ydb_topic-impl - ydb_topic-ut-ut_utils -) -target_sources(ydb-public-sdk-client-ydb_topic-ut PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/basic_usage_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/describe_topic_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/local_partition_ut.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/topic_to_table_ut.cpp -) -set_property( - TARGET - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - SPLIT_FACTOR - 10 -) -add_yunittest( - NAME - ydb-public-sdk-client-ydb_topic-ut - TEST_TARGET - ydb-public-sdk-client-ydb_topic-ut - TEST_ARG - --print-before-suite - --print-before-test - --fork-tests - --print-times - --show-fails -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - LABELS - MEDIUM -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - PROCESSORS - 1 -) -set_yunittest_property( - TEST - ydb-public-sdk-client-ydb_topic-ut - PROPERTY - TIMEOUT - 600 -) -target_allocator(ydb-public-sdk-client-ydb_topic-ut - system_allocator -) -vcs_info(ydb-public-sdk-client-ydb_topic-ut) diff --git a/src/client/topic/ut/basic_usage_ut.cpp b/src/client/topic/ut/basic_usage_ut.cpp index 84b76948915..d3b935adb9a 100644 --- a/src/client/topic/ut/basic_usage_ut.cpp +++ b/src/client/topic/ut/basic_usage_ut.cpp @@ -1,19 +1,20 @@ #include "ut_utils/managed_executor.h" #include "ut_utils/topic_sdk_test_setup.h" -#include +#include -#include +#include + +#include -#include - -#include -#include -#include +#include +#include +#include +#include #include #include -#include -#include +#include +#include #include @@ -71,20 +72,20 @@ void WriteAndReadToEndWithRestarts(TReadSessionSettings readSettings, TWriteSess NThreading::TPromise checkedPromise = NThreading::NewPromise(); - std::atomic lastOffset = 0u; + TAtomic lastOffset = 0u; auto f = checkedPromise.GetFuture(); readSettings.EventHandlers_.SimpleDataHandlers( [&] (TReadSessionEvent::TDataReceivedEvent& ev) mutable { - lastOffset.store(ev.GetMessages().back().GetOffset()); - std::cerr << ">>> TEST: last offset = " << lastOffset << std::endl; + AtomicSet(lastOffset, ev.GetMessages().back().GetOffset()); + Cerr << ">>> TEST: last offset = " << lastOffset << Endl; }); ReadSession = topicClient.CreateReadSession(readSettings); ui32 i = 0; - while (lastOffset.load() + 1 < count) { + while (AtomicGet(lastOffset) + 1 < count) { RunTasks(decompressor, {i++}); } @@ -143,24 +144,24 @@ Y_UNIT_TEST_SUITE(BasicUsage) { .Path(TEST_TOPIC) .ProducerId(TEST_MESSAGE_GROUP_ID) .MessageGroupId(TEST_MESSAGE_GROUP_ID); - std::cerr << ">>> open write session " << i << std::endl; + Cerr << ">>> open write session " << i << Endl; auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); UNIT_ASSERT(writeSession->Write("message_using_MessageGroupId")); - std::cerr << ">>> write session " << i << " message written" << std::endl; + Cerr << ">>> write session " << i << " message written" << Endl; writeSession->Close(); - std::cerr << ">>> write session " << i << " closed" << std::endl; + Cerr << ">>> write session " << i << " closed" << Endl; } { auto writeSettings = TWriteSessionSettings() .Path(TEST_TOPIC) .ProducerId(TEST_MESSAGE_GROUP_ID) .PartitionId(0); - std::cerr << ">>> open write session 100" << std::endl; + Cerr << ">>> open write session 100" << Endl; auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); UNIT_ASSERT(writeSession->Write("message_using_PartitionId")); - std::cerr << ">>> write session 100 message written" << std::endl; + Cerr << ">>> write session 100 message written" << Endl; writeSession->Close(); - std::cerr << ">>> write session 100 closed" << std::endl; + Cerr << ">>> write session 100 closed" << Endl; } { @@ -255,16 +256,16 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NPersQueue::TWriteSessionSettings writeSettings; writeSettings.Path(setup->GetTestTopic()).MessageGroupId(TEST_MESSAGE_GROUP_ID); writeSettings.Codec(NPersQueue::ECodec::RAW); - NPersQueue::IExecutor::TPtr executor = new NPersQueue::TSyncExecutor(); + IExecutor::TPtr executor = new TSyncExecutor(); writeSettings.CompressionExecutor(executor); ui64 count = 100u; - std::optional shouldCaptureData = {true}; + TMaybe shouldCaptureData = {true}; auto& client = setup->GetPersQueueClient(); auto session = client.CreateSimpleBlockingWriteSession(writeSettings); - std::string messageBase = "message----"; - std::vector sentMessages; + TString messageBase = "message----"; + TVector sentMessages; for (auto i = 0u; i < count; i++) { // sentMessages.emplace_back(messageBase * (i+1) + ToString(i)); @@ -276,9 +277,9 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto sessionAdapter = NPersQueue::NTests::TSimpleWriteSessionTestAdapter( dynamic_cast(session.get())); if (shouldCaptureData.Defined()) { - TYdbStringBuilder msg; + TStringBuilder msg; msg << "Session has captured " << sessionAdapter.GetAcquiredMessagesCount() - << " messages, capturing was expected: " << *shouldCaptureData << std::endl; + << " messages, capturing was expected: " << *shouldCaptureData << Endl; UNIT_ASSERT_VALUES_EQUAL_C(sessionAdapter.GetAcquiredMessagesCount() > 0, *shouldCaptureData, msg.c_str()); } } @@ -296,18 +297,18 @@ Y_UNIT_TEST_SUITE(BasicUsage) { .MaxMemoryUsageBytes(1_MB) .AppendTopics(setup->GetTestTopic()); - std::cerr << "Session was created" << std::endl; + Cerr << "Session was created" << Endl; NThreading::TPromise checkedPromise = NThreading::NewPromise(); auto totalReceived = 0u; auto f = checkedPromise.GetFuture(); - std::atomic check = 1; + TAtomic check = 1; readSettings.EventHandlers_.SimpleDataHandlers( // [checkedPromise = std::move(checkedPromise), &check, &sentMessages, &totalReceived] [&] (TReadSessionEvent::TDataReceivedEvent& ev) mutable { - Y_VERIFY_S(check.load() != 0, "check is false"); + Y_VERIFY_S(AtomicGet(check) != 0, "check is false"); auto& messages = ev.GetMessages(); for (size_t i = 0u; i < messages.size(); ++i) { auto& message = messages[i]; @@ -322,7 +323,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { f.GetValueSync(); ReadSession->Close(TDuration::MilliSeconds(10)); - check.store(0); + AtomicSet(check, 0); auto status = topicClient.CommitOffset(setup->GetTestTopic(), 0, setup->GetTestConsumer(), 50); UNIT_ASSERT(status.GetValueSync().IsSuccess()); @@ -405,7 +406,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { writeSession->Close(TDuration::Seconds(10)); } promiseToWrite.SetValue(); - std::cerr << ">>>TEST: write promise set " << std::endl; + Cerr << ">>>TEST: write promise set " << Endl; { NThreading::TPromise promise = NThreading::NewPromise(); @@ -415,7 +416,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { [promise = std::move(promise)](TReadSessionEvent::TDataReceivedEvent& ev) mutable { ev.Commit(); promise.SetValue(); - std::cerr << ">>>TEST: get read event " << std::endl; + Cerr << ">>>TEST: get read event " << Endl; }); auto readSession = topicClient.CreateReadSession(readSettings); @@ -423,7 +424,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { readSession->Close(TDuration::Seconds(10)); } promiseToRead.SetValue(); - std::cerr << ">>>TEST: read promise set " << std::endl; + Cerr << ">>>TEST: read promise set " << Endl; }); @@ -444,31 +445,31 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto RunTasks = [&](auto e, const std::vector& tasks) { size_t n = tasks.size(); - std::cerr << ">>>TEST in RunTasks: before WaitPlannedTasks" << std::endl; + Cerr << ">>>TEST in RunTasks: before WaitPlannedTasks" << Endl; WaitPlannedTasks(e, n); - std::cerr << ">>>TEST in RunTasks: before WaitExecutedTasks" << std::endl; + Cerr << ">>>TEST in RunTasks: before WaitExecutedTasks" << Endl; size_t completed = e->GetExecutedCount(); e->StartFuncs(tasks); WaitExecutedTasks(e, completed + n); }; UNIT_ASSERT(!futureWrite.HasValue()); - std::cerr << ">>>TEST: future write has no value " << std::endl; + Cerr << ">>>TEST: future write has no value " << Endl; RunTasks(stepByStepExecutor, {0}); futureWrite.GetValueSync(); UNIT_ASSERT(futureWrite.HasValue()); - std::cerr << ">>>TEST: future write has value " << std::endl; + Cerr << ">>>TEST: future write has value " << Endl; UNIT_ASSERT(!futureRead.HasValue()); - std::cerr << ">>>TEST: future read has no value " << std::endl; + Cerr << ">>>TEST: future read has no value " << Endl; RunTasks(stepByStepExecutor, {1}); futureRead.GetValueSync(); UNIT_ASSERT(futureRead.HasValue()); - std::cerr << ">>>TEST: future read has value " << std::endl; + Cerr << ">>>TEST: future read has value " << Endl; f.get(); - std::cerr << ">>> TEST: gracefully closed" << std::endl; + Cerr << ">>> TEST: gracefully closed" << Endl; } Y_UNIT_TEST(SessionNotDestroyedWhileUserEventHandlingInFlight) { @@ -516,7 +517,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { // { // std::shared_ptr token; // writeSettings.EventHandlers_.CommonHandler([token](TWriteSessionEvent::TEvent& event){ - // std::cerr << ">>>TEST: in CommonHandler " << std::endl; + // Cerr << ">>>TEST: in CommonHandler " << Endl; // if (std::holds_alternative(event)) { // *token = std::move(std::get(event).ContinuationToken); @@ -531,7 +532,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { // writeSession->Close(TDuration::Seconds(10)); // } // promiseToWrite.SetValue(); - // std::cerr << ">>>TEST: write promise set " << std::endl; + // Cerr << ">>>TEST: write promise set " << Endl; { NThreading::TPromise promise = NThreading::NewPromise(); @@ -539,7 +540,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { readSettings.EventHandlers_.SimpleDataHandlers( [promise = std::move(promise)](TReadSessionEvent::TDataReceivedEvent& ev) mutable { - std::cerr << ">>>TEST: in SimpleDataHandlers " << std::endl; + Cerr << ">>>TEST: in SimpleDataHandlers " << Endl; ev.Commit(); promise.SetValue(); }); @@ -549,7 +550,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { readSession->Close(TDuration::Seconds(10)); } promiseToRead.SetValue(); - std::cerr << ">>>TEST: read promise set " << std::endl; + Cerr << ">>>TEST: read promise set " << Endl; }); @@ -570,9 +571,9 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto RunTasks = [&](auto e, const std::vector& tasks) { size_t n = tasks.size(); - std::cerr << ">>>TEST in RunTasks: before WaitPlannedTasks" << std::endl; + Cerr << ">>>TEST in RunTasks: before WaitPlannedTasks" << Endl; WaitPlannedTasks(e, n); - std::cerr << ">>>TEST in RunTasks: before WaitExecutedTasks" << std::endl; + Cerr << ">>>TEST in RunTasks: before WaitExecutedTasks" << Endl; size_t completed = e->GetExecutedCount(); e->StartFuncs(tasks); WaitExecutedTasks(e, completed + n); @@ -580,25 +581,25 @@ Y_UNIT_TEST_SUITE(BasicUsage) { // RunTasks(stepByStepExecutor, {0}); // UNIT_ASSERT(!futureWrite.HasValue()); - // std::cerr << ">>>TEST: future write has no value " << std::endl; + // Cerr << ">>>TEST: future write has no value " << Endl; // RunTasks(stepByStepExecutor, {1}); // futureWrite.GetValueSync(); // UNIT_ASSERT(futureWrite.HasValue()); - // std::cerr << ">>>TEST: future write has value " << std::endl; + // Cerr << ">>>TEST: future write has value " << Endl; UNIT_ASSERT(!futureRead.HasValue()); - std::cerr << ">>>TEST: future read has no value " << std::endl; + Cerr << ">>>TEST: future read has no value " << Endl; // 0: TStartPartitionSessionEvent RunTasks(stepByStepExecutor, {0}); // 1: TDataReceivedEvent RunTasks(stepByStepExecutor, {1}); futureRead.GetValueSync(); UNIT_ASSERT(futureRead.HasValue()); - std::cerr << ">>>TEST: future read has value " << std::endl; + Cerr << ">>>TEST: future read has value " << Endl; f.get(); - std::cerr << ">>> TEST: gracefully closed" << std::endl; + Cerr << ">>> TEST: gracefully closed" << Endl; } Y_UNIT_TEST(ReadSessionCorrectClose) { @@ -608,7 +609,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { NPersQueue::TWriteSessionSettings writeSettings; writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); writeSettings.Codec(NPersQueue::ECodec::RAW); - NPersQueue::IExecutor::TPtr executor = new NPersQueue::TSyncExecutor(); + IExecutor::TPtr executor = new TSyncExecutor(); writeSettings.CompressionExecutor(executor); auto& client = setup->GetPersQueueClient(); @@ -640,11 +641,11 @@ Y_UNIT_TEST_SUITE(BasicUsage) { readSettings.EventHandlers_.SimpleDataHandlers( [] (NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent& ev) mutable { - std::cerr << ">>> Got TDataReceivedEvent" << std::endl; + Cerr << ">>> Got TDataReceivedEvent" << Endl; ev.Commit(); }); - std::cerr << ">>> TEST: Create session" << std::endl; + Cerr << ">>> TEST: Create session" << Endl; ReadSession = topicClient.CreateReadSession(readSettings); @@ -652,7 +653,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { ReadSession->Close(); ReadSession = nullptr; - std::cerr << ">>> TEST: Session gracefully closed" << std::endl; + Cerr << ">>> TEST: Session gracefully closed" << Endl; Sleep(TDuration::Seconds(5)); @@ -675,7 +676,7 @@ Y_UNIT_TEST_SUITE(BasicUsage) { auto client = setup.MakeClient(); auto session = client.CreateSimpleBlockingWriteSession(writeSettings); - std::string messageBase = "message----"; + TString messageBase = "message----"; for (auto i = 0u; i < count; i++) { auto res = session->Write(messageBase); @@ -698,8 +699,260 @@ Y_UNIT_TEST_SUITE(BasicUsage) { } + Y_UNIT_TEST(TWriteSession_WriteEncoded) { + // This test was adapted from ydb_persqueue tests. + // It writes 4 messages: 2 with default codec, 1 with explicitly set GZIP codec, 1 with RAW codec. + // The last message MUST be sent in a separate WriteRequest, as it has a codec field applied for all messages in the request. + // This separation currently happens in TWriteSessionImpl::SendImpl method. + auto setup = std::make_shared(TEST_CASE_NAME); + auto client = setup->MakeClient(); + auto settings = TWriteSessionSettings() + .Path(TEST_TOPIC) + .MessageGroupId(TEST_MESSAGE_GROUP_ID); -} + size_t batchSize = 100000000; + settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time. + settings.BatchFlushSizeBytes(batchSize); + auto writer = client.CreateWriteSession(settings); + TString message = "message"; + TString packed; + { + TStringOutput so(packed); + TZLibCompress oss(&so, ZLib::GZip, 6); + oss << message; + } -} + Cerr << message << " " << packed << "\n"; + + { + auto event = *writer->GetEvent(true); + UNIT_ASSERT(!writer->WaitEvent().Wait(TDuration::Seconds(1))); + auto ev = writer->WaitEvent(); + UNIT_ASSERT(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->Write(std::move(continueToken), message); + UNIT_ASSERT(ev.Wait(TDuration::Seconds(1))); + } + { + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->Write(std::move(continueToken), ""); + } + { + auto event = *writer->GetEvent(true); + UNIT_ASSERT(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->WriteEncoded(std::move(continueToken), packed, ECodec::GZIP, message.size()); + } + + ui32 acks = 0, tokens = 0; + while(acks < 4 || tokens < 2) { + auto event = *writer->GetEvent(true); + if (std::holds_alternative(event)) { + acks += std::get(event).Acks.size(); + } + if (std::holds_alternative(event)) { + if (tokens == 0) { + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->WriteEncoded(std::move(continueToken), "", ECodec::RAW, 0); + } + ++tokens; + } + Cerr << "GOT EVENT " << acks << " " << tokens << "\n"; + } + UNIT_ASSERT(!writer->WaitEvent().Wait(TDuration::Seconds(5))); + + UNIT_ASSERT_VALUES_EQUAL(acks, 4); + UNIT_ASSERT_VALUES_EQUAL(tokens, 2); + + auto readSettings = TReadSessionSettings() + .ConsumerName(TEST_CONSUMER) + .AppendTopics(TEST_TOPIC); + std::shared_ptr readSession = client.CreateReadSession(readSettings); + ui32 readMessageCount = 0; + while (readMessageCount < 4) { + Cerr << "Get event on client\n"; + auto event = *readSession->GetEvent(true); + std::visit(TOverloaded { + [&](TReadSessionEvent::TDataReceivedEvent& event) { + for (auto& message: event.GetMessages()) { + TString sourceId = message.GetMessageGroupId(); + ui32 seqNo = message.GetSeqNo(); + UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo); + ++readMessageCount; + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), (seqNo % 2) == 1 ? "message" : ""); + } + }, + [&](TReadSessionEvent::TCommitOffsetAcknowledgementEvent&) { + UNIT_FAIL("no commits in test"); + }, + [&](TReadSessionEvent::TStartPartitionSessionEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TStopPartitionSessionEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TEndPartitionSessionEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TPartitionSessionStatusEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TReadSessionEvent::TPartitionSessionClosedEvent&) { + UNIT_FAIL("Test does not support lock sessions yet"); + }, + [&](TSessionClosedEvent&) { + UNIT_FAIL("Session closed"); + } + + }, event); + } + } +} // Y_UNIT_TEST_SUITE(BasicUsage) + +Y_UNIT_TEST_SUITE(TSettingsValidation) { + enum class EExpectedTestResult { + SUCCESS, + FAIL_ON_SDK, + FAIL_ON_RPC + }; + + Y_UNIT_TEST(TestDifferentDedupParams) { + TTopicSdkTestSetup setup(TEST_CASE_NAME); + setup.GetServer().EnableLogs({ + NKikimrServices::PERSQUEUE, NKikimrServices::PERSQUEUE_READ_BALANCER, NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_PARTITION_CHOOSER}, + NActors::NLog::PRI_ERROR); + + + auto client = setup.MakeClient(); + ui64 producerIndex = 0u; + auto runTest = [&](TString producer, TString msgGroup, const TMaybe& useDedup, bool useSeqNo, EExpectedTestResult result) ->bool + { + TWriteSessionSettings writeSettings; + writeSettings.Path(setup.GetTopicPath()).Codec(NTopic::ECodec::RAW); + TString useDedupStr = useDedup.Defined() ? ToString(*useDedup) : ""; + if (producer) { + producer += ToString(producerIndex); + } + if (!msgGroup.empty()) { + msgGroup += ToString(producerIndex); + } + writeSettings.ProducerId(producer).MessageGroupId(msgGroup); + producerIndex++; + Cerr.Flush(); + Sleep(TDuration::MilliSeconds(250)); + Cerr << "=== === START TEST. Producer = '" << producer << "', MsgGroup = '" << msgGroup << "', useDedup: " + << useDedupStr << ", manual SeqNo: " << useSeqNo << Endl; + + try { + if (useDedup.Defined()) { + writeSettings.DeduplicationEnabled(useDedup); + } + auto session = client.CreateWriteSession(writeSettings); + TMaybe token; + ui64 seqNo = 1u; + ui64 written = 0; + while (written < 10) { + auto event = session->GetEvent(true); + if (std::holds_alternative(event.GetRef())) { + auto closed = std::get(*event); + Cerr << "Session failed with error: " << closed.DebugString() << Endl; + UNIT_ASSERT(result == EExpectedTestResult::FAIL_ON_RPC); + return false; + } else if (std::holds_alternative(event.GetRef())) { + token = std::move(std::get(*event).ContinuationToken); + if (useSeqNo) { + session->Write(std::move(*token), "data", seqNo++); + } else { + session->Write(std::move(*token), "data"); + } + continue; + } else { + UNIT_ASSERT(std::holds_alternative(*event)); + const auto& acks = std::get(*event); + for (const auto& ack : acks.Acks) { + UNIT_ASSERT(ack.State == TWriteSessionEvent::TWriteAck::EES_WRITTEN); + } + written += acks.Acks.size(); + } + } + } catch(const NYdb::TContractViolation& ex) { + Cerr << "Test fails on contract validation: " << ex.what() << Endl; + UNIT_ASSERT(result == EExpectedTestResult::FAIL_ON_SDK); + return false; + } + Cerr << "=== === END TEST (supposed ok)=== ===\n\n"; + UNIT_ASSERT(result == EExpectedTestResult::SUCCESS); + return true; + }; + // Normal scenarios: + // Most common: + TVector producers = {"producer", ""}; + TVector> messageGroup = {Nothing(), "producer", "messageGroup", ""}; + TVector> useDedupVariants = {Nothing(), true, false}; + TVector manSeqNoVariants = {true, false}; + runTest("producer", {}, {}, false, EExpectedTestResult::SUCCESS); + runTest("producer", {}, {}, true, EExpectedTestResult::SUCCESS); + // Enable dedup (doesnt take affect anything as it is enabled anyway) + runTest("producer", {}, true, true, EExpectedTestResult::SUCCESS); + runTest("producer", {}, true, false, EExpectedTestResult::SUCCESS); + + //No producer, do dedup + runTest({}, {}, {}, false, EExpectedTestResult::SUCCESS); + // manual seqNo with no-dedup - error + runTest({}, {}, {}, true, EExpectedTestResult::FAIL_ON_SDK); + // No producer but do enable dedup + runTest({}, {}, true, true, EExpectedTestResult::SUCCESS); + runTest({}, {}, true, false, EExpectedTestResult::SUCCESS); + + // MsgGroup = producer with explicit dedup enabling or not + runTest("producer", "producer", {}, false, EExpectedTestResult::SUCCESS); + runTest("producer", "producer", {}, true, EExpectedTestResult::SUCCESS); + runTest("producer", "producer", true, true, EExpectedTestResult::SUCCESS); + runTest("producer", "producer", true, false, EExpectedTestResult::SUCCESS); + + //Bad scenarios + // MsgGroup != producer, triggers error + runTest("producer", "msgGroup", {}, false, EExpectedTestResult::FAIL_ON_SDK); + runTest("producer", "msgGroup", {}, true, EExpectedTestResult::FAIL_ON_SDK); + runTest("producer", "msgGroup", true, true, EExpectedTestResult::FAIL_ON_SDK); + runTest("producer", "msgGroup", true, false, EExpectedTestResult::FAIL_ON_SDK); + + //Set producer or msgGroupId but disnable dedup: + runTest("producer", {}, false, true, EExpectedTestResult::FAIL_ON_SDK); + runTest("producer", {}, false, false, EExpectedTestResult::FAIL_ON_SDK); + runTest({}, "msgGroup", false, true, EExpectedTestResult::FAIL_ON_SDK); + runTest({}, "msgGroup", false, false, EExpectedTestResult::FAIL_ON_SDK); + + //Use msgGroupId as producerId, enable dedup + runTest({}, "msgGroup", true, true, EExpectedTestResult::SUCCESS); + runTest({}, "msgGroup", true, false, EExpectedTestResult::SUCCESS); + + + //Specify msg groupId and don't specify deduplication. Should work with dedup enable + runTest({}, "msgGroup", {}, true, EExpectedTestResult::SUCCESS); + runTest({}, "msgGroup", {}, false, EExpectedTestResult::SUCCESS); + } + + Y_UNIT_TEST(ValidateSettingsFailOnStart) { + TTopicSdkTestSetup setup(TEST_CASE_NAME); + TTopicClient client = setup.MakeClient(); + + auto readSettings = TReadSessionSettings() + .ConsumerName(TEST_CONSUMER) + .MaxMemoryUsageBytes(0) + .AppendTopics(TEST_TOPIC); + + auto readSession = client.CreateReadSession(readSettings); + auto event = readSession->GetEvent(true); + UNIT_ASSERT(event.Defined()); + + auto& closeEvent = std::get(*event); + UNIT_ASSERT(closeEvent.DebugString().Contains("Too small max memory usage")); + } + +} // Y_UNIT_TEST_SUITE(TSettingsValidation) + +} // namespace diff --git a/src/client/topic/ut/describe_topic_ut.cpp b/src/client/topic/ut/describe_topic_ut.cpp index ff9bfc46b53..5124949e9a2 100644 --- a/src/client/topic/ut/describe_topic_ut.cpp +++ b/src/client/topic/ut/describe_topic_ut.cpp @@ -1,16 +1,17 @@ #include "ut_utils/topic_sdk_test_setup.h" -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include - -#include +#include +#include namespace NYdb::NTopic::NTests { @@ -295,7 +296,7 @@ namespace NYdb::NTopic::NTests { // Event 1: start partition session { - std::optional event = readSession->GetEvent(true); + TMaybe event = readSession->GetEvent(true); UNIT_ASSERT(event); auto startPartitionSession = std::get_if(event.Get()); UNIT_ASSERT_C(startPartitionSession, DebugString(*event)); @@ -305,7 +306,7 @@ namespace NYdb::NTopic::NTests { // Event 2: data received { - std::optional event = readSession->GetEvent(true); + TMaybe event = readSession->GetEvent(true); UNIT_ASSERT(event); auto dataReceived = std::get_if(event.Get()); UNIT_ASSERT_C(dataReceived, DebugString(*event)); @@ -315,7 +316,7 @@ namespace NYdb::NTopic::NTests { // Event 3: commit acknowledgement { - std::optional event = readSession->GetEvent(true); + TMaybe event = readSession->GetEvent(true); UNIT_ASSERT(event); auto commitOffsetAck = std::get_if(event.Get()); @@ -344,5 +345,73 @@ namespace NYdb::NTopic::NTests { DescribeConsumer(setup, client, false, false, true, true); DescribePartition(setup, client, false, false, true, true); } + + TDescribePartitionResult RunPermissionTest(TTopicSdkTestSetup& setup, int userId, bool existingTopic, bool allowUpdateRow, bool allowDescribeSchema) { + TString authToken = TStringBuilder() << "x-user-" << userId << "@builtin"; + Cerr << std::format("=== existingTopic={} allowUpdateRow={} allowDescribeSchema={} authToken={}\n", + existingTopic, allowUpdateRow, allowDescribeSchema, std::string(authToken)); + + auto driverConfig = setup.MakeDriverConfig().SetAuthToken(authToken); + auto client = TTopicClient(TDriver(driverConfig)); + auto settings = TDescribePartitionSettings().IncludeLocation(true); + i64 testPartitionId = 0; + + NACLib::TDiffACL acl; + if (allowDescribeSchema) { + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::DescribeSchema, authToken); + } + if (allowUpdateRow) { + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, authToken); + } + setup.GetServer().AnnoyingClient->ModifyACL("/Root", TEST_TOPIC, acl.SerializeAsString()); + + return client.DescribePartition(existingTopic ? TEST_TOPIC : "bad-topic", testPartitionId, settings).GetValueSync(); + } + + Y_UNIT_TEST(DescribePartitionPermissions) { + TTopicSdkTestSetup setup(TEST_CASE_NAME); + setup.GetServer().EnableLogs({NKikimrServices::TX_PROXY_SCHEME_CACHE, NKikimrServices::SCHEME_BOARD_SUBSCRIBER}, NActors::NLog::PRI_TRACE); + + int userId = 0; + + struct Expectation { + bool existingTopic; + bool allowUpdateRow; + bool allowDescribeSchema; + EStatus status; + NYql::TIssueCode issueCode; + }; + + std::vector expectations{ + {0, 0, 0, EStatus::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ACCESS_DENIED}, + {0, 0, 1, EStatus::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ACCESS_DENIED}, + {0, 1, 0, EStatus::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ACCESS_DENIED}, + {0, 1, 1, EStatus::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ACCESS_DENIED}, + {1, 0, 0, EStatus::SCHEME_ERROR, Ydb::PersQueue::ErrorCode::ACCESS_DENIED}, + {1, 0, 1, EStatus::SUCCESS, 0}, + {1, 1, 0, EStatus::SUCCESS, 0}, + {1, 1, 1, EStatus::SUCCESS, 0}, + }; + + for (auto [existing, update, describe, status, issue] : expectations) { + auto result = RunPermissionTest(setup, userId++, existing, update, describe); + auto resultStatus = result.GetStatus(); + auto line = TStringBuilder() << "=== status=" << resultStatus; + NYql::TIssueCode resultIssue = 0; + if (!result.GetIssues().Empty()) { + resultIssue = result.GetIssues().begin()->GetCode(); + line << " issueCode=" << resultIssue; + } + Cerr << (line << " issues=" << result.GetIssues().ToOneLineString() << Endl); + + UNIT_ASSERT_EQUAL(resultStatus, status); + UNIT_ASSERT_EQUAL(resultIssue, issue); + if (resultStatus == EStatus::SUCCESS) { + auto& p = result.GetPartitionDescription().GetPartition(); + UNIT_ASSERT(p.GetActive()); + UNIT_ASSERT(p.GetPartitionLocation().Defined()); + } + } + } } } diff --git a/src/client/topic/ut/local_partition_ut.cpp b/src/client/topic/ut/local_partition_ut.cpp index 30a168b503d..841d0d2233a 100644 --- a/src/client/topic/ut/local_partition_ut.cpp +++ b/src/client/topic/ut/local_partition_ut.cpp @@ -1,18 +1,25 @@ #include "ut_utils/topic_sdk_test_setup.h" -#include +#include -#include +#include -#include +#include +#include +#include -#include -#include +#include + +#include +#include #include #include -#include +#include +#include + +#include using namespace NYdb; using namespace NYdb::NPersQueue::NTests; @@ -20,13 +27,25 @@ using namespace NYdb::NPersQueue::NTests; namespace NYdb::NTopic::NTests { Y_UNIT_TEST_SUITE(LocalPartition) { - std::shared_ptr CreateSetup(const std::string& testCaseName, ui32 nodeCount = 1) { + std::shared_ptr CreateSetup(const TString& testCaseName, ui32 nodeCount = 1, bool createTopic = true) { NKikimr::Tests::TServerSettings settings = TTopicSdkTestSetup::MakeServerSettings(); settings.SetNodeCount(nodeCount); - return std::make_shared(testCaseName, settings); + return std::make_shared(testCaseName, settings, createTopic); + } + + TTopicSdkTestSetup CreateSetupForSplitMerge(const TString& testCaseName) { + NKikimrConfig::TFeatureFlags ff; + ff.SetEnableTopicSplitMerge(true); + ff.SetEnablePQConfigTransactionsAtSchemeShard(true); + auto settings = TTopicSdkTestSetup::MakeServerSettings(); + settings.SetFeatureFlags(ff); + auto setup = TTopicSdkTestSetup(testCaseName, settings, false); + setup.GetRuntime().GetAppData().PQConfig.SetTopicsAreFirstClassCitizen(true); + setup.GetRuntime().GetAppData().PQConfig.SetUseSrcIdMetaMappingInFirstClass(true); + return setup; } - NYdb::TDriverConfig CreateConfig(const TTopicSdkTestSetup& setup, std::string discoveryAddr) + NYdb::TDriverConfig CreateConfig(const TTopicSdkTestSetup& setup, TString discoveryAddr) { NYdb::TDriverConfig config = setup.MakeDriverConfig(); config.SetEndpoint(discoveryAddr); @@ -51,7 +70,7 @@ namespace NYdb::NTopic::NTests { void WriteMessage(TTopicClient& client) { - std::cerr << "=== Write message" << std::endl; + Cerr << "=== Write message" << Endl; auto writeSession = client.CreateSimpleBlockingWriteSession(CreateWriteSessionSettings()); UNIT_ASSERT(writeSession->Write("message")); @@ -60,11 +79,11 @@ namespace NYdb::NTopic::NTests { void ReadMessage(TTopicClient& client, ui64 expectedCommitedOffset = 1) { - std::cerr << "=== Read message" << std::endl; + Cerr << "=== Read message" << Endl; auto readSession = client.CreateReadSession(CreateReadSessionSettings()); - std::optional event = readSession->GetEvent(true); + TMaybe event = readSession->GetEvent(true); UNIT_ASSERT(event); auto startPartitionSession = std::get_if(event.Get()); UNIT_ASSERT_C(startPartitionSession, DebugString(*event)); @@ -90,7 +109,7 @@ namespace NYdb::NTopic::NTests { } template - std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { + std::unique_ptr StartGrpcServer(const TString& address, TService& service) { grpc::ServerBuilder builder; builder.AddListeningPort(address, grpc::InsecureServerCredentials()); builder.RegisterService(&service); @@ -102,35 +121,35 @@ namespace NYdb::NTopic::NTests { TMockDiscoveryService() { ui16 discoveryPort = TPortManager().GetPort(); - DiscoveryAddr = TYdbStringBuilder() << "0.0.0.0:" << discoveryPort; - std::cerr << "==== TMockDiscovery server started on port " << discoveryPort << std::endl; + DiscoveryAddr = TStringBuilder() << "0.0.0.0:" << discoveryPort; + Cerr << "==== TMockDiscovery server started on port " << discoveryPort << Endl; Server = ::NYdb::NTopic::NTests::NTestSuiteLocalPartition::StartGrpcServer(DiscoveryAddr, *this); } void SetGoodEndpoints(TTopicSdkTestSetup& setup) { - std::cerr << "=== TMockDiscovery set good endpoint nodes " << std::endl; - SetEndpoints(setup.GetRuntime().GetNodeId(0), setup.GetRuntime().GetNodeCount(), setup.GetServer().GrpcPort); + std::lock_guard lock(Lock); + Cerr << "=== TMockDiscovery set good endpoint nodes " << Endl; + SetEndpointsLocked(setup.GetRuntime().GetNodeId(0), setup.GetRuntime().GetNodeCount(), setup.GetServer().GrpcPort); } - void SetEndpoints(ui32 firstNodeId, ui32 nodeCount, ui16 port) + // Call this method only after locking the Lock. + void SetEndpointsLocked(ui32 firstNodeId, ui32 nodeCount, ui16 port) { - std::lock_guard lock(Lock); - - std::cerr << "==== TMockDiscovery add endpoints, firstNodeId " << firstNodeId << ", nodeCount " << nodeCount << ", port " << port << std::endl; + Cerr << "==== TMockDiscovery add endpoints, firstNodeId " << firstNodeId << ", nodeCount " << nodeCount << ", port " << port << Endl; MockResults.clear_endpoints(); if (nodeCount > 0) { Ydb::Discovery::EndpointInfo* endpoint = MockResults.add_endpoints(); - endpoint->set_address(TYdbStringBuilder() << "localhost"); + endpoint->set_address(TStringBuilder() << "localhost"); endpoint->set_port(port); endpoint->set_node_id(firstNodeId); } if (nodeCount > 1) { Ydb::Discovery::EndpointInfo* endpoint = MockResults.add_endpoints(); - endpoint->set_address(TYdbStringBuilder() << "ip6-localhost"); // name should be different + endpoint->set_address(TStringBuilder() << "ip6-localhost"); // name should be different endpoint->set_port(port); endpoint->set_node_id(firstNodeId + 1); } @@ -139,14 +158,19 @@ namespace NYdb::NTopic::NTests { } } - grpc::Status ListEndpoints(grpc::ServerContext* context, const Ydb::Discovery::ListEndpointsRequest* request, Ydb::Discovery::ListEndpointsResponse* response) override { + void SetEndpoints(ui32 firstNodeId, ui32 nodeCount, ui16 port) + { std::lock_guard lock(Lock); + SetEndpointsLocked(firstNodeId, nodeCount, port); + } + grpc::Status ListEndpoints(grpc::ServerContext* context, const Ydb::Discovery::ListEndpointsRequest* request, Ydb::Discovery::ListEndpointsResponse* response) override { + std::lock_guard lock(Lock); UNIT_ASSERT(context); if (Delay) { - std::cerr << "==== Delay " << Delay << " before ListEndpoints request" << std::endl; + Cerr << "==== Delay " << Delay << " before ListEndpoints request" << Endl; TInstant start = TInstant::Now(); while (start + Delay < TInstant::Now()) { @@ -156,18 +180,19 @@ namespace NYdb::NTopic::NTests { } } - std::cerr << "==== ListEndpoints request: " << request->ShortDebugString() << std::endl; + Cerr << " ==== ListEndpoints request: " << request->ShortDebugString() << Endl; auto* op = response->mutable_operation(); op->set_ready(true); op->set_status(Ydb::StatusIds::SUCCESS); op->mutable_result()->PackFrom(MockResults); - std::cerr << "==== ListEndpoints response: " << response->ShortDebugString() << std::endl; + Cerr << "==== ListEndpoints response: " << response->ShortDebugString() << Endl; + return grpc::Status::OK; } - std::string GetDiscoveryAddr() const { + TString GetDiscoveryAddr() const { return DiscoveryAddr; } @@ -177,14 +202,14 @@ namespace NYdb::NTopic::NTests { private: Ydb::Discovery::ListEndpointsResult MockResults; - std::string DiscoveryAddr = 0; + TString DiscoveryAddr = 0; std::unique_ptr Server; TAdaptiveLock Lock; TDuration Delay = {}; }; - auto Start(std::string testCaseName, std::shared_ptr mockDiscoveryService = {}) + auto Start(TString testCaseName, std::shared_ptr mockDiscoveryService = {}) { struct Result { std::shared_ptr Setup; @@ -218,7 +243,7 @@ namespace NYdb::NTopic::NTests { auto [setup, client, discovery] = Start(TEST_CASE_NAME); for (size_t i = 1; i <= 10; ++i) { - std::cerr << "=== Restart attempt " << i << std::endl; + Cerr << "=== Restart attempt " << i << Endl; setup->GetServer().KillTopicPqTablets(setup->GetTopicPath()); WriteMessage(*client); ReadMessage(*client, i); @@ -242,23 +267,23 @@ namespace NYdb::NTopic::NTests { retryPolicy->Initialize(); retryPolicy->ExpectBreakDown(); - std::cerr << "=== Create write session\n"; + Cerr << "=== Create write session\n"; TTopicClient client(TDriver(CreateConfig(*setup, discovery.GetDiscoveryAddr()))); auto writeSession = client.CreateWriteSession(writeSettings); - std::cerr << "=== Wait for retries\n"; + Cerr << "=== Wait for retries\n"; retryPolicy->WaitForRetriesSync(3); - std::cerr << "=== Alter partition count\n"; + Cerr << "=== Alter partition count\n"; TAlterTopicSettings alterSettings; alterSettings.AlterPartitioningSettings(2, 2); auto alterResult = client.AlterTopic(setup->GetTopicPath(), alterSettings).GetValueSync(); UNIT_ASSERT_VALUES_EQUAL_C(alterResult.GetStatus(), NYdb::EStatus::SUCCESS, alterResult.GetIssues().ToString()); - std::cerr << "=== Wait for repair\n"; + Cerr << "=== Wait for repair\n"; retryPolicy->WaitForRepairSync(); - std::cerr << "=== Close write session\n"; + Cerr << "=== Close write session\n"; writeSession->Close(); } @@ -276,19 +301,19 @@ namespace NYdb::NTopic::NTests { retryPolicy->Initialize(); retryPolicy->ExpectBreakDown(); - std::cerr << "=== Create write session\n"; + Cerr << "=== Create write session\n"; TTopicClient client(TDriver(CreateConfig(*setup, discovery.GetDiscoveryAddr()))); auto writeSession = client.CreateWriteSession(writeSettings); - std::cerr << "=== Wait for retries\n"; + Cerr << "=== Wait for retries\n"; retryPolicy->WaitForRetriesSync(3); discovery.SetGoodEndpoints(*setup); - std::cerr << "=== Wait for repair\n"; + Cerr << "=== Wait for repair\n"; retryPolicy->WaitForRepairSync(); - std::cerr << "=== Close write session\n"; + Cerr << "=== Close write session\n"; writeSession->Close(); } @@ -306,19 +331,19 @@ namespace NYdb::NTopic::NTests { retryPolicy->Initialize(); retryPolicy->ExpectBreakDown(); - std::cerr << "=== Create write session\n"; + Cerr << "=== Create write session\n"; TTopicClient client(TDriver(CreateConfig(*setup, discovery.GetDiscoveryAddr()))); auto writeSession = client.CreateWriteSession(writeSettings); - std::cerr << "=== Wait for retries\n"; + Cerr << "=== Wait for retries\n"; retryPolicy->WaitForRetriesSync(3); discovery.SetGoodEndpoints(*setup); - std::cerr << "=== Wait for repair\n"; + Cerr << "=== Wait for repair\n"; retryPolicy->WaitForRepairSync(); - std::cerr << "=== Close write session\n"; + Cerr << "=== Close write session\n"; writeSession->Close(); } @@ -336,11 +361,11 @@ namespace NYdb::NTopic::NTests { retryPolicy->Initialize(); retryPolicy->ExpectBreakDown(); - std::cerr << "=== Create write session\n"; + Cerr << "=== Create write session\n"; TTopicClient client(TDriver(CreateConfig(*setup, discovery.GetDiscoveryAddr()))); auto writeSession = client.CreateWriteSession(writeSettings); - std::cerr << "=== Close write session\n"; + Cerr << "=== Close write session\n"; writeSession->Close(); } @@ -351,12 +376,355 @@ namespace NYdb::NTopic::NTests { discovery.SetGoodEndpoints(*setup); discovery.SetDelay(TDuration::Days(1)); - std::cerr << "=== Create write session\n"; + Cerr << "=== Create write session\n"; TTopicClient client(TDriver(CreateConfig(*setup, discovery.GetDiscoveryAddr()))); auto writeSession = client.CreateWriteSession(CreateWriteSessionSettings()); - std::cerr << "=== Close write session\n"; + Cerr << "=== Close write session\n"; + writeSession->Close(); + } + + Y_UNIT_TEST(WithoutPartition) { + // Direct write without partition: happy way. + auto setup = CreateSetup(TEST_CASE_NAME); + TMockDiscoveryService discovery; + discovery.SetGoodEndpoints(*setup); + auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); + auto* tracingBackend = new TTracingBackend(); + driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + TDriver driver(driverConfig); + TTopicClient client(driver); + auto sessionSettings = TWriteSessionSettings() + .Path(TEST_TOPIC) + .ProducerId(TEST_MESSAGE_GROUP_ID) + .MessageGroupId(TEST_MESSAGE_GROUP_ID) + .DirectWriteToPartition(true); + auto writeSession = client.CreateSimpleBlockingWriteSession(sessionSettings); + UNIT_ASSERT(writeSession->Write("message")); + writeSession->Close(); + + auto node0_id = std::to_string(setup->GetRuntime().GetNodeId(0)); + TExpectedTrace expected{ + "InitRequest !partition_id !partition_with_generation", + "InitResponse partition_id=0 session_id", + "DescribePartitionRequest partition_id=0", + std::format("DescribePartitionResponse partition_id=0 pl_generation=1 pl_node_id={}", node0_id), + std::format("PreferredPartitionLocation Generation=1 NodeId={}", node0_id), + "InitRequest !partition_id pwg_partition_id=0 pwg_generation=1", + "InitResponse partition_id=0 session_id", + }; + auto const events = tracingBackend->GetEvents(); + UNIT_ASSERT(expected.Matches(events)); + } + + Y_UNIT_TEST(WithoutPartitionWithRestart) { + // Direct write without partition: with tablet restart. + auto setup = CreateSetup(TEST_CASE_NAME); + TMockDiscoveryService discovery; + discovery.SetGoodEndpoints(*setup); + auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); + auto* tracingBackend = new TTracingBackend(); + driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + TDriver driver(driverConfig); + TTopicClient client(driver); + auto retryPolicy = std::make_shared(); + auto sessionSettings = TWriteSessionSettings() + .Path(TEST_TOPIC) + .MessageGroupId(TEST_MESSAGE_GROUP_ID) + .DirectWriteToPartition(true) + .RetryPolicy(retryPolicy); + + retryPolicy->Initialize(); + retryPolicy->ExpectBreakDown(); + auto writeSession = client.CreateSimpleBlockingWriteSession(sessionSettings); + UNIT_ASSERT(writeSession->Write("message")); + setup->GetServer().KillTopicPqTablets(setup->GetTopicPath()); + retryPolicy->WaitForRepairSync(); writeSession->Close(); + + auto node0_id = std::to_string(setup->GetRuntime().GetNodeId(0)); + TExpectedTrace expected{ + "InitRequest !partition_id !partition_with_generation", + "InitResponse partition_id=0 session_id", + "DescribePartitionRequest partition_id=0", + std::format("DescribePartitionResponse partition_id=0 pl_generation=1 pl_node_id={}", node0_id), + std::format("PreferredPartitionLocation Generation=1 NodeId={}", node0_id), + "InitRequest !partition_id pwg_partition_id=0 pwg_generation=1", + "InitResponse partition_id=0 session_id", + "Error status=UNAVAILABLE", + + // The tablet has been killed, find out the partition node the tablet ends up. + "DescribePartitionRequest partition_id=0", + std::format("DescribePartitionResponse partition_id=0 pl_generation=2 pl_node_id={}", node0_id), + std::format("PreferredPartitionLocation Generation=2 NodeId={}", node0_id), + "InitRequest !partition_id pwg_partition_id=0 pwg_generation=2", + "InitResponse partition_id=0 session_id", + }; + auto const events = tracingBackend->GetEvents(); + UNIT_ASSERT(expected.Matches(events)); + } + + Y_UNIT_TEST(WithoutPartitionUnknownEndpoint) { + // Direct write without partition: with unknown endpoint. + // Create 2 nodes. Mark the node 1 as down, so the topic partition will be assigned to the node 2. + // Our write session then gets assigned to the partition which lives on the node 2. + // But at first we only add node 1 to the discovery service. + auto setup = CreateSetup(TEST_CASE_NAME, 2, /* createTopic = */ false); + setup->GetServer().AnnoyingClient->MarkNodeInHive(setup->GetServer().GetRuntime(), 0, false); + setup->CreateTopic(TEST_TOPIC, TEST_CONSUMER, 1); + setup->GetServer().AnnoyingClient->MarkNodeInHive(setup->GetServer().GetRuntime(), 0, true); + TMockDiscoveryService discovery; + discovery.SetEndpoints(setup->GetRuntime().GetNodeId(0), 1, setup->GetServer().GrpcPort); + auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); + auto* tracingBackend = new TTracingBackend(); + driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + TDriver driver(driverConfig); + TTopicClient client(driver); + auto retryPolicy = std::make_shared(); + auto sessionSettings = TWriteSessionSettings() + .Path(TEST_TOPIC) + .MessageGroupId(TEST_MESSAGE_GROUP_ID) + .DirectWriteToPartition(true) + .RetryPolicy(retryPolicy); + retryPolicy->Initialize(); + retryPolicy->ExpectBreakDown(); + auto writeSession = client.CreateSimpleBlockingWriteSession(sessionSettings); + + retryPolicy->WaitForRetriesSync(1); + discovery.SetGoodEndpoints(*setup); + tracingBackend->WaitForEvent("InitRequest").GetValueSync(); + UNIT_ASSERT(writeSession->Close()); + + auto node1_id = std::to_string(setup->GetRuntime().GetNodeId(1)); + TExpectedTrace expected{ + "InitRequest !partition_id !partition_with_generation", + "InitResponse partition_id=0", + "DescribePartitionRequest partition_id=0", + std::format("DescribePartitionResponse partition_id=0 pl_generation=1 pl_node_id={}", node1_id), + "Error status=UNAVAILABLE", + "DescribePartitionRequest partition_id=0", + std::format("DescribePartitionResponse partition_id=0 pl_generation=1 pl_node_id={}", node1_id), + std::format("PreferredPartitionLocation Generation=1 NodeId={}", node1_id), + "InitRequest !partition_id pwg_partition_id=0 pwg_generation=1", + + // At this point we get an UNAVAILABLE error, as the InitRequest is made for the node 2, but received and processed by node 1. + // It's expected, as our test server starts only one gRPC proxy and all requests go through the node 1. Error looks like this: + // TPartitionWriter (partition=0) received TEvClientConnected with wrong NodeId. Expected: 1, received 2 + }; + auto const events((tracingBackend)->GetEvents()); + UNIT_ASSERT(expected.Matches(events)); + } + + Y_UNIT_TEST(WithoutPartitionDeadNode) { + // This test emulates a situation, when InitResponse directs us to an inaccessible node. + auto setup = CreateSetup(TEST_CASE_NAME); + TMockDiscoveryService discovery; + discovery.SetEndpoints(setup->GetRuntime().GetNodeId(0), 1, 0); + auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); + auto* tracingBackend = new TTracingBackend(); + driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + TDriver driver(driverConfig); + TTopicClient client(driver); + auto retryPolicy = std::make_shared(); + auto sessionSettings = TWriteSessionSettings() + .Path(TEST_TOPIC) + .MessageGroupId(TEST_MESSAGE_GROUP_ID) + .DirectWriteToPartition(true) + .PartitionId(0) + .RetryPolicy(retryPolicy); + retryPolicy->Initialize(); + retryPolicy->ExpectBreakDown(); + auto writeSession = client.CreateSimpleBlockingWriteSession(sessionSettings); + + retryPolicy->WaitForRetriesSync(1); + discovery.SetGoodEndpoints(*setup); + retryPolicy->WaitForRepairSync(); + UNIT_ASSERT(writeSession->Close()); + + auto node0_id = std::to_string(setup->GetRuntime().GetNodeId(0)); + TExpectedTrace expected{ + "DescribePartitionRequest partition_id=0", + "Error status=TRANSPORT_UNAVAILABLE", + "DescribePartitionRequest partition_id=0", + std::format("DescribePartitionResponse partition_id=0 pl_generation=1 pl_node_id={}", node0_id), + std::format("PreferredPartitionLocation Generation=1 NodeId={}", node0_id), + "InitRequest !partition_id pwg_partition_id=0 pwg_generation=1", + "InitResponse partition_id=0", + }; + auto const events = tracingBackend->GetEvents(); + UNIT_ASSERT(expected.Matches(events)); + } + + Y_UNIT_TEST(WithoutPartitionPartitionRelocation) { + // This test emulates partition relocation from one node to another. + auto setup = CreateSetup(TEST_CASE_NAME, 2, /* createTopic = */ false); + // Make the node 1 unavailable. + setup->GetServer().AnnoyingClient->MarkNodeInHive(setup->GetServer().GetRuntime(), 0, false); + setup->CreateTopic(TEST_TOPIC, TEST_CONSUMER, 2); + + TMockDiscoveryService discovery; + discovery.SetGoodEndpoints(*setup); + auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()); + auto* tracingBackend = new TTracingBackend(); + driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + TDriver driver(driverConfig); + TTopicClient client(driver); + auto retryPolicy = std::make_shared(); + auto sessionSettings = TWriteSessionSettings() + .Path(TEST_TOPIC) + .MessageGroupId(TEST_MESSAGE_GROUP_ID) + .DirectWriteToPartition(true) + .RetryPolicy(retryPolicy); + + retryPolicy->Initialize(); + retryPolicy->ExpectBreakDown(); + + // Connect to node 2. + auto writeSession = client.CreateSimpleBlockingWriteSession(sessionSettings); + retryPolicy->WaitForRetriesSync(2); + // Make the node 2 unavailable. + setup->GetServer().AnnoyingClient->MarkNodeInHive(setup->GetServer().GetRuntime(), 1, false); + setup->GetServer().AnnoyingClient->KickNodeInHive(setup->GetServer().GetRuntime(), 1); + // Bring back the node 1. + setup->GetServer().AnnoyingClient->MarkNodeInHive(setup->GetServer().GetRuntime(), 0, true); + retryPolicy->WaitForRepairSync(); + writeSession->Close(); + + auto node0_id = std::to_string(setup->GetRuntime().GetNodeId(0)); + auto node1_id = std::to_string(setup->GetRuntime().GetNodeId(1)); + TExpectedTrace expected{ + "InitRequest !partition_id !partition_with_generation", + "InitResponse partition_id=1 session_id", + "DescribePartitionRequest partition_id=1", + std::format("DescribePartitionResponse partition_id=1 pl_generation=1 pl_node_id={}", node1_id), + std::format("PreferredPartitionLocation Generation=1 NodeId={}", node1_id), + "InitRequest !partition_id pwg_partition_id=1 pwg_generation=1", + "Error status=UNAVAILABLE", + "DescribePartitionRequest partition_id=1", + std::format("DescribePartitionResponse partition_id=1 pl_generation=2 pl_node_id={}", node0_id), + std::format("PreferredPartitionLocation Generation=2 NodeId={}", node0_id), + "InitRequest !partition_id pwg_partition_id=1 pwg_generation=2", + "InitResponse partition_id=1", + }; + auto const events = tracingBackend->GetEvents(); + UNIT_ASSERT(expected.Matches(events)); + } + + Y_UNIT_TEST(DirectWriteWithoutDescribeResourcesPermission) { + // The DirectWrite option makes the write session send a DescribePartitionRequest to locate the partition's node. + // Previously, it required DescribeSchema (DescribeResources) permission. However, this permission is too broad + // to be granted to anyone who needs the DirectWrite option. The DescribePartitionRequest should work when either + // UpdateRow (WriteTopic) or DescribeSchema permission is granted. + // + // In this test, we don't grant DescribeSchema permission and check that direct write works anyway. + + auto setup = CreateSetup(TEST_CASE_NAME); + auto authToken = "x-user-x@builtin"; + + { + // Allow UpdateRow only, no DescribeSchema permission. + NACLib::TDiffACL acl; + acl.AddAccess(NACLib::EAccessType::Allow, NACLib::UpdateRow, authToken); + setup->GetServer().AnnoyingClient->ModifyACL("/Root", TEST_TOPIC, acl.SerializeAsString()); + } + + TMockDiscoveryService discovery; + discovery.SetGoodEndpoints(*setup); + auto* tracingBackend = new TTracingBackend(); + auto driverConfig = CreateConfig(*setup, discovery.GetDiscoveryAddr()) + .SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})) + .SetAuthToken(authToken); + TDriver driver(driverConfig); + TTopicClient client(driver); + + auto sessionSettings = TWriteSessionSettings() + .Path(TEST_TOPIC) + .ProducerId(TEST_MESSAGE_GROUP_ID) + .MessageGroupId(TEST_MESSAGE_GROUP_ID) + .DirectWriteToPartition(true); + + auto writeSession = client.CreateSimpleBlockingWriteSession(sessionSettings); + UNIT_ASSERT(writeSession->Write("message")); + writeSession->Close(); + + TExpectedTrace expected{ + "InitRequest", + "InitResponse partition_id=0", + "DescribePartitionRequest partition_id=0", + "DescribePartitionResponse partition_id=0 pl_generation=1", + "PreferredPartitionLocation Generation=1", + "InitRequest pwg_partition_id=0 pwg_generation=1", + "InitResponse partition_id=0", + }; + auto const events = tracingBackend->GetEvents(); + UNIT_ASSERT(expected.Matches(events)); + } + + Y_UNIT_TEST(WithoutPartitionWithSplit) { + auto setup = CreateSetupForSplitMerge(TEST_CASE_NAME); + setup.CreateTopic(TEST_TOPIC, TEST_CONSUMER, 1, 100); + + TMockDiscoveryService discovery; + discovery.SetGoodEndpoints(setup); + auto driverConfig = CreateConfig(setup, discovery.GetDiscoveryAddr()); + auto* tracingBackend = new TTracingBackend(); + driverConfig.SetLog(CreateCompositeLogBackend({new TStreamLogBackend(&Cerr), tracingBackend})); + TDriver driver(driverConfig); + TTopicClient client(driver); + auto writeSettings = TWriteSessionSettings() + .Path(TEST_TOPIC) + .ProducerId(TEST_MESSAGE_GROUP_ID) + .MessageGroupId(TEST_MESSAGE_GROUP_ID) + .DirectWriteToPartition(true); + auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); + TTestReadSession ReadSession("Session-0", client, 2); + + UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); + + ui64 txId = 1006; + SplitPartition(setup, ++txId, 0, "a"); + + UNIT_ASSERT(writeSession->Write(Msg("message_1.2", 3))); + + ReadSession.WaitAllMessages(); + + for (const auto& info : ReadSession.Impl->ReceivedMessages) { + if (info.Data == "message_1.1") { + UNIT_ASSERT_EQUAL(0, info.PartitionId); + UNIT_ASSERT_EQUAL(2, info.SeqNo); + } else if (info.Data == "message_1.2") { + UNIT_ASSERT(1 == info.PartitionId || 2 == info.PartitionId); + UNIT_ASSERT_EQUAL(3, info.SeqNo); + } else { + UNIT_ASSERT_C(false, "Unexpected message: " << info.Data); + } + } + + writeSession->Close(TDuration::Seconds(1)); + + auto node0_id = std::to_string(setup.GetRuntime().GetNodeId(0)); + TExpectedTrace expected{ + "InitRequest !partition_id !partition_with_generation", + "InitResponse partition_id=0", + "DescribePartitionRequest partition_id=0", + std::format("DescribePartitionResponse partition_id=0 pl_generation=1 pl_node_id={}", node0_id), + std::format("PreferredPartitionLocation Generation=1 NodeId={}", node0_id), + "InitRequest !partition_id pwg_partition_id=0 pwg_generation=1", + "InitResponse partition_id=0", + "Error status=OVERLOADED", + "ClearDirectWriteToPartitionId", + "InitRequest !partition_id !partition_with_generation", + "InitResponse partition_id=2", + "DescribePartitionRequest partition_id=2", + std::format("DescribePartitionResponse partition_id=2 pl_generation=1 pl_node_id={}", node0_id), + std::format("PreferredPartitionLocation Generation=1 NodeId={}", node0_id), + "InitRequest !partition_id pwg_partition_id=2 pwg_generation=1", + "InitResponse partition_id=2", + }; + auto const events = tracingBackend->GetEvents(); + UNIT_ASSERT(expected.Matches(events)); + + ReadSession.Close(); } } -} \ No newline at end of file +} diff --git a/src/client/topic/ut/topic_to_table_ut.cpp b/src/client/topic/ut/topic_to_table_ut.cpp index aea22d6386e..c8a883cf795 100644 --- a/src/client/topic/ut/topic_to_table_ut.cpp +++ b/src/client/topic/ut/topic_to_table_ut.cpp @@ -1,27 +1,53 @@ #include "ut_utils/topic_sdk_test_setup.h" -#include -#include -#include +#include +#include +#include +#include +#include +#include -#include +#include + +#include #include namespace NYdb::NTopic::NTests { +const auto TEST_MESSAGE_GROUP_ID_1 = TEST_MESSAGE_GROUP_ID + "_1"; +const auto TEST_MESSAGE_GROUP_ID_2 = TEST_MESSAGE_GROUP_ID + "_2"; + Y_UNIT_TEST_SUITE(TxUsage) { class TFixture : public NUnitTest::TBaseFixture { protected: + using TTopicReadSession = NTopic::IReadSession; + using TTopicReadSessionPtr = std::shared_ptr; + using TTopicWriteSession = NTopic::IWriteSession; + using TTopicWriteSessionPtr = std::shared_ptr; + + struct TTopicWriteSessionContext { + TTopicWriteSessionPtr Session; + TMaybe ContinuationToken; + size_t WriteCount = 0; + size_t WrittenAckCount = 0; + size_t WrittenInTxAckCount = 0; + + void WaitForContinuationToken(); + void Write(const TString& message, NTable::TTransaction* tx = nullptr); + + size_t AckCount() const { return WrittenAckCount + WrittenInTxAckCount; } + + void WaitForEvent(); + }; + void SetUp(NUnitTest::TTestContext&) override; - NTable::TSession CreateSession(); + NTable::TSession CreateTableSession(); NTable::TTransaction BeginTx(NTable::TSession& session); - void CommitTx(NTable::TTransaction& tx, EStatus status); - - using TTopicReadSession = NTopic::IReadSession; - using TTopicReadSessionPtr = std::shared_ptr; + void CommitTx(NTable::TTransaction& tx, EStatus status = EStatus::SUCCESS); + void RollbackTx(NTable::TTransaction& tx, EStatus status = EStatus::SUCCESS); TTopicReadSessionPtr CreateReader(); @@ -30,21 +56,141 @@ class TFixture : public NUnitTest::TBaseFixture { void ReadMessage(TTopicReadSessionPtr reader, NTable::TTransaction& tx, ui64 offset); - void WriteMessage(const std::string& data); + void WriteMessage(const TString& message); + void WriteMessages(const TVector& messages, + const TString& topic, const TString& groupId, + NTable::TTransaction& tx); + + void CreateTopic(const TString& path = TEST_TOPIC, + const TString& consumer = TEST_CONSUMER, + size_t partitionCount = 1, + std::optional maxPartitionCount = std::nullopt); + + void WriteToTopicWithInvalidTxId(bool invalidTxId); + + TTopicWriteSessionPtr CreateTopicWriteSession(const TString& topicPath, + const TString& messageGroupId, + TMaybe partitionId); + TTopicWriteSessionContext& GetTopicWriteSession(const TString& topicPath, + const TString& messageGroupId, + TMaybe partitionId); + + TTopicReadSessionPtr CreateTopicReadSession(const TString& topicPath, + const TString& consumerName, + TMaybe partitionId); + TTopicReadSessionPtr GetTopicReadSession(const TString& topicPath, + const TString& consumerName, + TMaybe partitionId); + + void WriteToTopic(const TString& topicPath, + const TString& messageGroupId, + const TString& message, + NTable::TTransaction* tx = nullptr, + TMaybe partitionId = Nothing()); + TVector ReadFromTopic(const TString& topicPath, + const TString& consumerName, + const TDuration& duration, + NTable::TTransaction* tx = nullptr, + TMaybe partitionId = Nothing()); + void WaitForAcks(const TString& topicPath, + const TString& messageGroupId, + size_t writtenInTxCount = Max()); + void WaitForSessionClose(const TString& topicPath, + const TString& messageGroupId, + NYdb::EStatus status); + void CloseTopicWriteSession(const TString& topicPath, + const TString& messageGroupId); + + enum EEndOfTransaction { + Commit, + Rollback, + CloseTableSession + }; + + struct TTransactionCompletionTestDescription { + TVector Topics; + EEndOfTransaction EndOfTransaction = Commit; + }; + + void TestTheCompletionOfATransaction(const TTransactionCompletionTestDescription& d); + void RestartLongTxService(); + void RestartPQTablet(const TString& topicPath, ui32 partition); + void DumpPQTabletKeys(const TString& topicName, ui32 partition); + + void DeleteSupportivePartition(const TString& topicName, + ui32 partition); + + struct TTableRecord { + TTableRecord() = default; + TTableRecord(const TString& key, const TString& value); + + TString Key; + TString Value; + }; + + TVector MakeTableRecords(); + TString MakeJsonDoc(const TVector& records); + + void CreateTable(const TString& path); + void WriteToTable(const TString& tablePath, + const TVector& records, + NTable::TTransaction* tx); + size_t GetTableRecordsCount(const TString& tablePath); + + enum ERestartPQTabletMode { + ERestartNo, + ERestartBeforeCommit, + ERestartAfterCommit, + }; + + struct TTestTxWithBigBlobsParams { + size_t OldHeadCount = 0; + size_t BigBlobsCount = 2; + size_t NewHeadCount = 0; + ERestartPQTabletMode RestartMode = ERestartNo; + }; + + void TestTxWithBigBlobs(const TTestTxWithBigBlobsParams& params); -protected: const TDriver& GetDriver() const; + void CheckTabletKeys(const TString& topicName); + void DumpPQTabletKeys(const TString& topicName); + private: template E ReadEvent(TTopicReadSessionPtr reader, NTable::TTransaction& tx); template E ReadEvent(TTopicReadSessionPtr reader); + ui64 GetTopicTabletId(const TActorId& actorId, + const TString& topicPath, + ui32 partition); + TVector GetTabletKeys(const TActorId& actorId, + ui64 tabletId); + NPQ::TWriteId GetTransactionWriteId(const TActorId& actorId, + ui64 tabletId); + void SendLongTxLockStatus(const TActorId& actorId, + ui64 tabletId, + const NPQ::TWriteId& writeId, + NKikimrLongTxService::TEvLockStatus::EStatus status); + void WaitForTheTabletToDeleteTheWriteInfo(const TActorId& actorId, + ui64 tabletId, + const NPQ::TWriteId& writeId); + std::unique_ptr Setup; std::unique_ptr Driver; + + THashMap, TTopicWriteSessionContext> TopicWriteSessions; + THashMap TopicReadSessions; }; +TFixture::TTableRecord::TTableRecord(const TString& key, const TString& value) : + Key(key), + Value(value) +{ +} + void TFixture::SetUp(NUnitTest::TTestContext&) { NKikimr::Tests::TServerSettings settings = TTopicSdkTestSetup::MakeServerSettings(); @@ -54,23 +200,31 @@ void TFixture::SetUp(NUnitTest::TTestContext&) Driver = std::make_unique(Setup->MakeDriver()); } -NTable::TSession TFixture::CreateSession() +NTable::TSession TFixture::CreateTableSession() { NTable::TTableClient client(GetDriver()); auto result = client.CreateSession().ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); return result.GetSession(); } NTable::TTransaction TFixture::BeginTx(NTable::TSession& session) { auto result = session.BeginTransaction().ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); return result.GetTransaction(); } void TFixture::CommitTx(NTable::TTransaction& tx, EStatus status) { auto result = tx.Commit().ExtractValueSync(); - UNIT_ASSERT_VALUES_EQUAL(result.GetStatus(), status); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, result.GetIssues().ToString()); +} + +void TFixture::RollbackTx(NTable::TTransaction& tx, EStatus status) +{ + auto result = tx.Rollback().ExtractValueSync(); + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, result.GetIssues().ToString()); } auto TFixture::CreateReader() -> TTopicReadSessionPtr @@ -131,7 +285,7 @@ E TFixture::ReadEvent(TTopicReadSessionPtr reader) return *ev; } -void TFixture::WriteMessage(const std::string& data) +void TFixture::WriteMessage(const TString& message) { NTopic::TWriteSessionSettings options; options.Path(TEST_TOPIC); @@ -139,20 +293,91 @@ void TFixture::WriteMessage(const std::string& data) NTopic::TTopicClient client(GetDriver()); auto session = client.CreateSimpleBlockingWriteSession(options); - UNIT_ASSERT(session->Write(data)); + UNIT_ASSERT(session->Write(message)); session->Close(); } +void TFixture::WriteMessages(const TVector& messages, + const TString& topic, const TString& groupId, + NTable::TTransaction& tx) +{ + NTopic::TWriteSessionSettings options; + options.Path(topic); + options.MessageGroupId(groupId); + + NTopic::TTopicClient client(GetDriver()); + auto session = client.CreateSimpleBlockingWriteSession(options); + + for (auto& message : messages) { + NTopic::TWriteMessage params(message); + params.Tx(tx); + UNIT_ASSERT(session->Write(std::move(params))); + } + + UNIT_ASSERT(session->Close()); +} + +void TFixture::CreateTopic(const TString& path, + const TString& consumer, + size_t partitionCount, + std::optional maxPartitionCount) + +{ + Setup->CreateTopic(path, consumer, partitionCount, maxPartitionCount); +} + const TDriver& TFixture::GetDriver() const { return *Driver; } +void TFixture::WriteToTopicWithInvalidTxId(bool invalidTxId) +{ + auto tableSession = CreateTableSession(); + auto tx = BeginTx(tableSession); + + NTopic::TWriteSessionSettings options; + options.Path(TEST_TOPIC); + options.MessageGroupId(TEST_MESSAGE_GROUP_ID); + + NTopic::TTopicClient client(GetDriver()); + auto writeSession = client.CreateWriteSession(options); + + auto event = writeSession->GetEvent(true); + UNIT_ASSERT(event.Defined() && std::holds_alternative(event.GetRef())); + auto token = std::move(std::get(event.GetRef()).ContinuationToken); + + NTopic::TWriteMessage params("message"); + params.Tx(tx); + + if (invalidTxId) { + CommitTx(tx, EStatus::SUCCESS); + } else { + auto result = tableSession.Close().ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } + + writeSession->Write(std::move(token), std::move(params)); + + while (true) { + event = writeSession->GetEvent(true); + UNIT_ASSERT(event.Defined()); + auto& v = event.GetRef(); + if (auto e = std::get_if(&v); e) { + UNIT_ASSERT(false); + } else if (auto e = std::get_if(&v); e) { + ; + } else if (auto e = std::get_if(&v); e) { + break; + } + } +} + Y_UNIT_TEST_F(SessionAbort, TFixture) { { auto reader = CreateReader(); - auto session = CreateSession(); + auto session = CreateTableSession(); auto tx = BeginTx(session); StartPartitionSession(reader, tx, 0); @@ -165,7 +390,7 @@ Y_UNIT_TEST_F(SessionAbort, TFixture) } { - auto session = CreateSession(); + auto session = CreateTableSession(); auto tx = BeginTx(session); auto reader = CreateReader(); @@ -187,7 +412,7 @@ Y_UNIT_TEST_F(TwoSessionOneConsumer, TFixture) { WriteMessage("message #0"); - auto session1 = CreateSession(); + auto session1 = CreateTableSession(); auto tx1 = BeginTx(session1); { @@ -198,7 +423,7 @@ Y_UNIT_TEST_F(TwoSessionOneConsumer, TFixture) ReadMessage(reader, tx1, 0); } - auto session2 = CreateSession(); + auto session2 = CreateTableSession(); auto tx2 = BeginTx(session2); { @@ -213,32 +438,1464 @@ Y_UNIT_TEST_F(TwoSessionOneConsumer, TFixture) CommitTx(tx1, EStatus::ABORTED); } -Y_UNIT_TEST_F(WriteToTopic, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Invalid_Session, TFixture) +{ + WriteToTopicWithInvalidTxId(false); +} + +Y_UNIT_TEST_F(WriteToTopic_Invalid_Tx, TFixture) +{ + WriteToTopicWithInvalidTxId(true); +} + +Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession, TFixture) +{ + TString topicPath[2] = { + TEST_TOPIC, + TEST_TOPIC + "_2" + }; + + CreateTopic(topicPath[1]); + + auto createWriteSession = [](NTopic::TTopicClient& client, const TString& topicPath) { + NTopic::TWriteSessionSettings options; + options.Path(topicPath); + options.MessageGroupId(TEST_MESSAGE_GROUP_ID); + + return client.CreateWriteSession(options); + }; + + auto writeMessage = [](auto& ws, const TString& message, auto& tx) { + NTopic::TWriteMessage params(message); + params.Tx(tx); + + auto event = ws->GetEvent(true); + UNIT_ASSERT(event.Defined() && std::holds_alternative(event.GetRef())); + auto token = std::move(std::get(event.GetRef()).ContinuationToken); + + ws->Write(std::move(token), std::move(params)); + }; + + auto tableSession = CreateTableSession(); + auto tx = BeginTx(tableSession); + + NTopic::TTopicClient client(GetDriver()); + + auto ws0 = createWriteSession(client, topicPath[0]); + auto ws1 = createWriteSession(client, topicPath[1]); + + writeMessage(ws0, "message-1", tx); + writeMessage(ws1, "message-2", tx); + + size_t acks = 0; + + while (acks < 2) { + auto event = ws0->GetEvent(false); + if (!event) { + event = ws1->GetEvent(false); + if (!event) { + Sleep(TDuration::MilliSeconds(10)); + continue; + } + } + + auto& v = event.GetRef(); + if (auto e = std::get_if(&v); e) { + ++acks; + } else if (auto e = std::get_if(&v); e) { + ; + } else if (auto e = std::get_if(&v); e) { + break; + } + } + + UNIT_ASSERT_VALUES_EQUAL(acks, 2); +} + +auto TFixture::CreateTopicWriteSession(const TString& topicPath, + const TString& messageGroupId, + TMaybe partitionId) -> TTopicWriteSessionPtr { + NTopic::TTopicClient client(GetDriver()); NTopic::TWriteSessionSettings options; - options.Path(TEST_TOPIC); - options.MessageGroupId(TEST_MESSAGE_GROUP_ID); + options.Path(topicPath); + options.ProducerId(messageGroupId); + options.MessageGroupId(messageGroupId); + options.PartitionId(partitionId); + options.Codec(ECodec::RAW); + return client.CreateWriteSession(options); +} + +auto TFixture::GetTopicWriteSession(const TString& topicPath, + const TString& messageGroupId, + TMaybe partitionId) -> TTopicWriteSessionContext& +{ + std::pair key(topicPath, messageGroupId); + auto i = TopicWriteSessions.find(key); + + if (i == TopicWriteSessions.end()) { + TTopicWriteSessionContext context; + context.Session = CreateTopicWriteSession(topicPath, messageGroupId, partitionId); + + TopicWriteSessions.emplace(key, std::move(context)); + + i = TopicWriteSessions.find(key); + } + + return i->second; +} + +NTopic::TTopicReadSettings MakeTopicReadSettings(const TString& topicPath, + TMaybe partitionId) +{ + TTopicReadSettings options; + options.Path(topicPath); + if (partitionId.Defined()) { + options.AppendPartitionIds(*partitionId); + } + return options; +} + +NTopic::TReadSessionSettings MakeTopicReadSessionSettings(const TString& topicPath, + const TString& consumerName, + TMaybe partitionId) +{ + NTopic::TReadSessionSettings options; + options.AppendTopics(MakeTopicReadSettings(topicPath, partitionId)); + options.ConsumerName(consumerName); + return options; +} + +auto TFixture::CreateTopicReadSession(const TString& topicPath, + const TString& consumerName, + TMaybe partitionId) -> TTopicReadSessionPtr +{ + NTopic::TTopicClient client(GetDriver()); + return client.CreateReadSession(MakeTopicReadSessionSettings(topicPath, + consumerName, + partitionId)); +} + +auto TFixture::GetTopicReadSession(const TString& topicPath, + const TString& consumerName, + TMaybe partitionId) -> TTopicReadSessionPtr +{ + TTopicReadSessionPtr session; + + if (auto i = TopicReadSessions.find(topicPath); i == TopicReadSessions.end()) { + session = CreateTopicReadSession(topicPath, consumerName, partitionId); + auto event = ReadEvent(session); + event.Confirm(); + TopicReadSessions.emplace(topicPath, session); + } else { + session = i->second; + } - auto session = CreateSession(); - auto tx = BeginTx(session); + return session; +} - auto writeMessages = [&](const std::vector& messages) { - NTopic::TTopicClient client(GetDriver()); - auto session = client.CreateSimpleBlockingWriteSession(options); +void TFixture::TTopicWriteSessionContext::WaitForContinuationToken() +{ + while (!ContinuationToken.Defined()) { + WaitForEvent(); + } +} - for (auto& message : messages) { - NTopic::TWriteMessage params(message); - params.Tx(tx); - UNIT_ASSERT(session->Write(std::move(params))); +void TFixture::TTopicWriteSessionContext::WaitForEvent() +{ + Session->WaitEvent().Wait(); + for (auto& event : Session->GetEvents()) { + if (auto* e = std::get_if(&event)) { + ContinuationToken = std::move(e->ContinuationToken); + } else if (auto* e = std::get_if(&event)) { + for (auto& ack : e->Acks) { + switch (ack.State) { + case NTopic::TWriteSessionEvent::TWriteAck::EES_WRITTEN: + ++WrittenAckCount; + break; + case NTopic::TWriteSessionEvent::TWriteAck::EES_WRITTEN_IN_TX: + ++WrittenInTxAckCount; + break; + default: + break; + } + } + } else if (auto* e = std::get_if(&event)) { + UNIT_FAIL(""); } + } +} - UNIT_ASSERT(session->Close()); - }; +void TFixture::TTopicWriteSessionContext::Write(const TString& message, NTable::TTransaction* tx) +{ + NTopic::TWriteMessage params(message); - writeMessages({"a", "bb", "ccc", "dddd"}); - writeMessages({"eeeee", "ffffff", "ggggggg"}); + if (tx) { + params.Tx(*tx); + } - CommitTx(tx, EStatus::ABORTED); + Session->Write(std::move(*ContinuationToken), + std::move(params)); + + ++WriteCount; + ContinuationToken = Nothing(); +} + +void TFixture::CloseTopicWriteSession(const TString& topicPath, + const TString& messageGroupId) +{ + std::pair key(topicPath, messageGroupId); + auto i = TopicWriteSessions.find(key); + + UNIT_ASSERT(i != TopicWriteSessions.end()); + + TTopicWriteSessionContext& context = i->second; + + context.Session->Close(); + TopicWriteSessions.erase(key); +} + +void TFixture::WriteToTopic(const TString& topicPath, + const TString& messageGroupId, + const TString& message, + NTable::TTransaction* tx, + TMaybe partitionId) +{ + TTopicWriteSessionContext& context = GetTopicWriteSession(topicPath, messageGroupId, partitionId); + context.WaitForContinuationToken(); + UNIT_ASSERT(context.ContinuationToken.Defined()); + context.Write(message, tx); +} + +TVector TFixture::ReadFromTopic(const TString& topicPath, + const TString& consumerName, + const TDuration& duration, + NTable::TTransaction* tx, + TMaybe partitionId) +{ + TVector messages; + + TInstant end = TInstant::Now() + duration; + TDuration remain = duration; + + auto session = GetTopicReadSession(topicPath, consumerName, partitionId); + + while (TInstant::Now() < end) { + if (!session->WaitEvent().Wait(remain)) { + return messages; + } + + NTopic::TReadSessionGetEventSettings settings; + if (tx) { + settings.Tx(*tx); + } + + for (auto& event : session->GetEvents(settings)) { + if (auto* e = std::get_if(&event)) { + Cerr << e->HasCompressedMessages() << " " << e->GetMessagesCount() << Endl; + for (auto& m : e->GetMessages()) { + messages.push_back(m.GetData()); + } + + if (!tx) { + e->Commit(); + } + } + } + + remain = end - TInstant::Now(); + } + + return messages; +} + +void TFixture::WaitForAcks(const TString& topicPath, const TString& messageGroupId, size_t writtenInTxCount) +{ + std::pair key(topicPath, messageGroupId); + auto i = TopicWriteSessions.find(key); + UNIT_ASSERT(i != TopicWriteSessions.end()); + + auto& context = i->second; + + UNIT_ASSERT(context.AckCount() <= context.WriteCount); + + while (context.AckCount() < context.WriteCount) { + context.WaitForEvent(); + } + + UNIT_ASSERT((context.WrittenAckCount + context.WrittenInTxAckCount) == context.WriteCount); + + if (writtenInTxCount != Max()) { + UNIT_ASSERT_VALUES_EQUAL(context.WrittenInTxAckCount, writtenInTxCount); + } +} + +void TFixture::WaitForSessionClose(const TString& topicPath, + const TString& messageGroupId, + NYdb::EStatus status) +{ + std::pair key(topicPath, messageGroupId); + auto i = TopicWriteSessions.find(key); + UNIT_ASSERT(i != TopicWriteSessions.end()); + + auto& context = i->second; + + UNIT_ASSERT(context.AckCount() <= context.WriteCount); + + for(bool stop = false; !stop; ) { + context.Session->WaitEvent().Wait(); + for (auto& event : context.Session->GetEvents()) { + if (auto* e = std::get_if(&event)) { + context.ContinuationToken = std::move(e->ContinuationToken); + } else if (auto* e = std::get_if(&event)) { + for (auto& ack : e->Acks) { + switch (ack.State) { + case NTopic::TWriteSessionEvent::TWriteAck::EES_WRITTEN: + ++context.WrittenAckCount; + break; + case NTopic::TWriteSessionEvent::TWriteAck::EES_WRITTEN_IN_TX: + ++context.WrittenInTxAckCount; + break; + default: + break; + } + } + } else if (auto* e = std::get_if(&event)) { + UNIT_ASSERT_VALUES_EQUAL(e->GetStatus(), status); + UNIT_ASSERT_GT(e->GetIssues().Size(), 0); + stop = true; + } + } + } + + UNIT_ASSERT(context.AckCount() <= context.WriteCount); +} + +ui64 TFixture::GetTopicTabletId(const TActorId& actorId, const TString& topicPath, ui32 partition) +{ + auto navigate = std::make_unique(); + navigate->DatabaseName = "/Root"; + + NSchemeCache::TSchemeCacheNavigate::TEntry entry; + entry.Path = SplitPath(topicPath); + entry.SyncVersion = true; + entry.ShowPrivatePath = true; + entry.Operation = NSchemeCache::TSchemeCacheNavigate::OpList; + + navigate->ResultSet.push_back(std::move(entry)); + //navigate->UserToken = "root@builtin"; + navigate->Cookie = 12345; + + auto& runtime = Setup->GetRuntime(); + + runtime.Send(MakeSchemeCacheID(), actorId, + new TEvTxProxySchemeCache::TEvNavigateKeySet(navigate.release()), + 0, + true); + auto response = runtime.GrabEdgeEvent(); + + UNIT_ASSERT_VALUES_EQUAL(response->Request->Cookie, 12345); + UNIT_ASSERT_VALUES_EQUAL(response->Request->ErrorCount, 0); + + auto& front = response->Request->ResultSet.front(); + UNIT_ASSERT(front.PQGroupInfo); + UNIT_ASSERT_GT(front.PQGroupInfo->Description.PartitionsSize(), 0); + UNIT_ASSERT_LT(partition, front.PQGroupInfo->Description.PartitionsSize()); + + for (size_t i = 0; i < front.PQGroupInfo->Description.PartitionsSize(); ++i) { + auto& p = front.PQGroupInfo->Description.GetPartitions(partition); + if (p.GetPartitionId() == partition) { + return p.GetTabletId(); + } + } + + UNIT_FAIL("unknown partition"); + + return Max(); +} + +TVector TFixture::GetTabletKeys(const TActorId& actorId, ui64 tabletId) +{ + using TEvKeyValue = NKikimr::TEvKeyValue; + + auto request = std::make_unique(); + request->Record.SetCookie(12345); + + auto cmd = request->Record.AddCmdReadRange(); + TString from(1, '\x00'); + TString to(1, '\xFF'); + auto range = cmd->MutableRange(); + range->SetFrom(from); + range->SetIncludeFrom(true); + range->SetTo(to); + range->SetIncludeTo(true); + + auto& runtime = Setup->GetRuntime(); + + runtime.SendToPipe(tabletId, actorId, request.release()); + auto response = runtime.GrabEdgeEvent(); + + UNIT_ASSERT(response->Record.HasCookie()); + UNIT_ASSERT_VALUES_EQUAL(response->Record.GetCookie(), 12345); + UNIT_ASSERT_VALUES_EQUAL(response->Record.ReadRangeResultSize(), 1); + + TVector keys; + + auto& result = response->Record.GetReadRangeResult(0); + for (size_t i = 0; i < result.PairSize(); ++i) { + auto& kv = result.GetPair(i); + keys.emplace_back(kv.GetKey()); + } + + return keys; +} + +void TFixture::RestartLongTxService() +{ + auto& runtime = Setup->GetRuntime(); + TActorId edge = runtime.AllocateEdgeActor(); + + for (ui32 node = 0; node < runtime.GetNodeCount(); ++node) { + runtime.Send(NKikimr::NLongTxService::MakeLongTxServiceID(runtime.GetNodeId(node)), edge, + new TEvents::TEvPoison(), + 0, + true); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_1, TFixture) +{ + CreateTopic("topic_A"); + CreateTopic("topic_B"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #4", &tx); + + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #5", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #6", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #7", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #8", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #9", &tx); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); + } + + { + auto messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); + } + + CommitTx(tx, EStatus::SUCCESS); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages[3], "message #4"); + } + + { + auto messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 5); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #5"); + UNIT_ASSERT_VALUES_EQUAL(messages[4], "message #9"); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_2, TFixture) +{ + CreateTopic("topic_A"); + CreateTopic("topic_B"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #3", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #4", &tx); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, "message #5"); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_2, "message #6"); + + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_1, "message #7", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_1, "message #8", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_1, "message #9", &tx); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_1); + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID_1); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_2); + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID_2); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #5"); + } + + { + auto messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #6"); + } + + CommitTx(tx, EStatus::SUCCESS); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages[3], "message #4"); + } + + { + auto messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 3); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #7"); + UNIT_ASSERT_VALUES_EQUAL(messages[2], "message #9"); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_3, TFixture) +{ + CreateTopic("topic_A"); + CreateTopic("topic_B"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3"); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #3"); + } + + CommitTx(tx, EStatus::ABORTED); + + tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx, EStatus::SUCCESS); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_4, TFixture) +{ + CreateTopic("topic_A"); + CreateTopic("topic_B"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx_1 = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx_1); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", &tx_1); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID); + + NTable::TTransaction tx_2 = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", &tx_2); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #4", &tx_2); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); + } + + { + auto messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); + } + + CommitTx(tx_2, EStatus::SUCCESS); + CommitTx(tx_1, EStatus::ABORTED); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #3"); + } + + { + auto messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #4"); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_5, TFixture) +{ + CreateTopic("topic_A"); + CreateTopic("topic_B"); + + NTable::TSession tableSession = CreateTableSession(); + + { + NTable::TTransaction tx_1 = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx_1); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", &tx_1); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx_1, EStatus::SUCCESS); + } + + { + NTable::TTransaction tx_2 = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", &tx_2); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #4", &tx_2); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx_2, EStatus::SUCCESS); + } + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #3"); + } + + { + auto messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #2"); + UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #4"); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_6, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); + } + + CommitTx(tx, EStatus::SUCCESS); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #2"); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_7, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #2", &tx); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, "message #3"); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, "message #4"); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #5", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #6", &tx); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_1); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_2); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #3"); + UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #4"); + } + + CommitTx(tx, EStatus::SUCCESS); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 4); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages[3], "message #6"); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_8, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2"); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #2"); + } + + CommitTx(tx, EStatus::ABORTED); + + tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx, EStatus::SUCCESS); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_9, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx_1 = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx_1); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + NTable::TTransaction tx_2 = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx_2); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); + } + + CommitTx(tx_2, EStatus::SUCCESS); + CommitTx(tx_1, EStatus::ABORTED); + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #2"); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_10, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + + { + NTable::TTransaction tx_1 = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx_1); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx_1, EStatus::SUCCESS); + } + + { + NTable::TTransaction tx_2 = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx_2); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx_2, EStatus::SUCCESS); + } + + { + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #2"); + } +} + +NPQ::TWriteId TFixture::GetTransactionWriteId(const TActorId& actorId, + ui64 tabletId) +{ + using TEvKeyValue = NKikimr::TEvKeyValue; + + auto request = std::make_unique(); + request->Record.SetCookie(12345); + request->Record.AddCmdRead()->SetKey("_txinfo"); + + auto& runtime = Setup->GetRuntime(); + + runtime.SendToPipe(tabletId, actorId, request.release()); + auto response = runtime.GrabEdgeEvent(); + + UNIT_ASSERT(response->Record.HasCookie()); + UNIT_ASSERT_VALUES_EQUAL(response->Record.GetCookie(), 12345); + UNIT_ASSERT_VALUES_EQUAL(response->Record.ReadResultSize(), 1); + + auto& read = response->Record.GetReadResult(0); + + NKikimrPQ::TTabletTxInfo info; + UNIT_ASSERT(info.ParseFromString(read.GetValue())); + + UNIT_ASSERT_VALUES_EQUAL(info.TxWritesSize(), 1); + + auto& writeInfo = info.GetTxWrites(0); + UNIT_ASSERT(writeInfo.HasWriteId()); + + return NPQ::GetWriteId(writeInfo); +} + +void TFixture::SendLongTxLockStatus(const TActorId& actorId, + ui64 tabletId, + const NPQ::TWriteId& writeId, + NKikimrLongTxService::TEvLockStatus::EStatus status) +{ + auto event = + std::make_unique(writeId.KeyId, writeId.NodeId, + status); + auto& runtime = Setup->GetRuntime(); + runtime.SendToPipe(tabletId, actorId, event.release()); +} + +void TFixture::WaitForTheTabletToDeleteTheWriteInfo(const TActorId& actorId, + ui64 tabletId, + const NPQ::TWriteId& writeId) +{ + while (true) { + using TEvKeyValue = NKikimr::TEvKeyValue; + + auto request = std::make_unique(); + request->Record.SetCookie(12345); + request->Record.AddCmdRead()->SetKey("_txinfo"); + + auto& runtime = Setup->GetRuntime(); + + runtime.SendToPipe(tabletId, actorId, request.release()); + auto response = runtime.GrabEdgeEvent(); + + UNIT_ASSERT(response->Record.HasCookie()); + UNIT_ASSERT_VALUES_EQUAL(response->Record.GetCookie(), 12345); + UNIT_ASSERT_VALUES_EQUAL(response->Record.ReadResultSize(), 1); + + auto& read = response->Record.GetReadResult(0); + + NKikimrPQ::TTabletTxInfo info; + UNIT_ASSERT(info.ParseFromString(read.GetValue())); + + bool found = false; + + for (size_t i = 0; i < info.TxWritesSize(); ++i) { + auto& writeInfo = info.GetTxWrites(i); + UNIT_ASSERT(writeInfo.HasWriteId()); + if (NPQ::GetWriteId(writeInfo) == writeId) { + found = true; + break; + } + } + + if (!found) { + break; + } + + Sleep(TDuration::MilliSeconds(100)); + } +} + +void TFixture::RestartPQTablet(const TString& topicName, ui32 partition) +{ + auto& runtime = Setup->GetRuntime(); + TActorId edge = runtime.AllocateEdgeActor(); + ui64 tabletId = GetTopicTabletId(edge, "/Root/" + topicName, partition); + runtime.SendToPipe(tabletId, edge, new TEvents::TEvPoison()); + + Sleep(TDuration::Seconds(2)); +} + +void TFixture::DeleteSupportivePartition(const TString& topicName, ui32 partition) +{ + auto& runtime = Setup->GetRuntime(); + TActorId edge = runtime.AllocateEdgeActor(); + ui64 tabletId = GetTopicTabletId(edge, "/Root/" + topicName, partition); + NPQ::TWriteId writeId = GetTransactionWriteId(edge, tabletId); + + SendLongTxLockStatus(edge, tabletId, writeId, NKikimrLongTxService::TEvLockStatus::STATUS_NOT_FOUND); + + WaitForTheTabletToDeleteTheWriteInfo(edge, tabletId, writeId); +} + +void TFixture::CheckTabletKeys(const TString& topicName) +{ + auto& runtime = Setup->GetRuntime(); + TActorId edge = runtime.AllocateEdgeActor(); + ui64 tabletId = GetTopicTabletId(edge, "/Root/" + topicName, 0); + + const THashSet types { + NPQ::TKeyPrefix::TypeInfo, + NPQ::TKeyPrefix::TypeData, + NPQ::TKeyPrefix::TypeTmpData, + NPQ::TKeyPrefix::TypeMeta, + NPQ::TKeyPrefix::TypeTxMeta, + }; + + bool found; + TVector keys; + for (size_t i = 0; i < 20; ++i) { + keys = GetTabletKeys(edge, tabletId); + + found = false; + for (const auto& key : keys) { + UNIT_ASSERT_GT(key.size(), 0); + if (key[0] == '_') { + continue; + } + + if (types.contains(key[0])) { + found = false; + break; + } + } + + if (!found) { + break; + } + + Sleep(TDuration::MilliSeconds(100)); + } + + if (found) { + Cerr << "keys for tablet " << tabletId << ":" << Endl; + for (const auto& k : keys) { + Cerr << k << Endl; + } + Cerr << "=============" << Endl; + + UNIT_FAIL("unexpected keys for tablet " << tabletId); + } +} + +void TFixture::DumpPQTabletKeys(const TString& topicName) +{ + auto& runtime = Setup->GetRuntime(); + TActorId edge = runtime.AllocateEdgeActor(); + ui64 tabletId = GetTopicTabletId(edge, "/Root/" + topicName, 0); + auto keys = GetTabletKeys(edge, tabletId); + for (const auto& key : keys) { + Cerr << key << Endl; + } +} + +void TFixture::TestTheCompletionOfATransaction(const TTransactionCompletionTestDescription& d) +{ + for (auto& topic : d.Topics) { + CreateTopic(topic); + } + + { + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + for (auto& topic : d.Topics) { + WriteToTopic(topic, TEST_MESSAGE_GROUP_ID, "message", &tx); + WaitForAcks(topic, TEST_MESSAGE_GROUP_ID); + } + + switch (d.EndOfTransaction) { + case Commit: + CommitTx(tx, EStatus::SUCCESS); + break; + case Rollback: + RollbackTx(tx, EStatus::SUCCESS); + break; + case CloseTableSession: + break; + } + } + + Sleep(TDuration::Seconds(5)); + + for (auto& topic : d.Topics) { + CheckTabletKeys(topic); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_11, TFixture) +{ + for (auto endOfTransaction : {Commit, Rollback, CloseTableSession}) { + TestTheCompletionOfATransaction({.Topics={"topic_A"}, .EndOfTransaction = endOfTransaction}); + TestTheCompletionOfATransaction({.Topics={"topic_A", "topic_B"}, .EndOfTransaction = endOfTransaction}); + } +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_12, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + DeleteSupportivePartition("topic_A", 0); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WaitForSessionClose("topic_A", TEST_MESSAGE_GROUP_ID, NYdb::EStatus::PRECONDITION_FAILED); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_13, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message", &tx); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + DeleteSupportivePartition("topic_A", 0); + + CommitTx(tx, EStatus::ABORTED); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_14, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + DeleteSupportivePartition("topic_A", 0); + + CloseTopicWriteSession("topic_A", TEST_MESSAGE_GROUP_ID); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx, EStatus::ABORTED); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_15, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #1", &tx); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_1); + CloseTopicWriteSession("topic_A", TEST_MESSAGE_GROUP_ID_1); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, "message #2", &tx); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_2); + CloseTopicWriteSession("topic_A", TEST_MESSAGE_GROUP_ID_2); + + CommitTx(tx, EStatus::SUCCESS); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_16, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + RestartPQTablet("topic_A", 0); + + CommitTx(tx, EStatus::SUCCESS); + + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); + UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #2"); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_17, TFixture) +{ + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(22'000'000, 'x')); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(100, 'x')); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(200, 'x')); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(300, 'x')); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(10'000'000, 'x')); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString( 6'000'000, 'x'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(20'000'000, 'x'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString( 7'000'000, 'x'), &tx); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx, EStatus::SUCCESS); + + //RestartPQTablet("topic_A", 0); + + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 8); + UNIT_ASSERT_VALUES_EQUAL(messages[0].size(), 22'000'000); + UNIT_ASSERT_VALUES_EQUAL(messages[1].size(), 100); + UNIT_ASSERT_VALUES_EQUAL(messages[2].size(), 200); + UNIT_ASSERT_VALUES_EQUAL(messages[3].size(), 300); + UNIT_ASSERT_VALUES_EQUAL(messages[4].size(), 10'000'000); + UNIT_ASSERT_VALUES_EQUAL(messages[5].size(), 6'000'000); + UNIT_ASSERT_VALUES_EQUAL(messages[6].size(), 20'000'000); + UNIT_ASSERT_VALUES_EQUAL(messages[7].size(), 7'000'000); +} + +void TFixture::TestTxWithBigBlobs(const TTestTxWithBigBlobsParams& params) +{ + size_t oldHeadMsgCount = 0; + size_t bigBlobMsgCount = 0; + size_t newHeadMsgCount = 0; + + CreateTopic("topic_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + for (size_t i = 0; i < params.OldHeadCount; ++i) { + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(100'000, 'x')); + ++oldHeadMsgCount; + } + + for (size_t i = 0; i < params.BigBlobsCount; ++i) { + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(7'900'000, 'x'), &tx); + ++bigBlobMsgCount; + } + + for (size_t i = 0; i < params.NewHeadCount; ++i) { + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(100'000, 'x'), &tx); + ++newHeadMsgCount; + } + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + if (params.RestartMode == ERestartBeforeCommit) { + RestartPQTablet("topic_A", 0); + } + + CommitTx(tx, EStatus::SUCCESS); + + if (params.RestartMode == ERestartAfterCommit) { + RestartPQTablet("topic_A", 0); + } + + TVector messages; + for (size_t i = 0; (i < 10) && (messages.size() < (oldHeadMsgCount + bigBlobMsgCount + newHeadMsgCount)); ++i) { + auto block = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + for (auto& m : block) { + messages.push_back(std::move(m)); + } + } + + UNIT_ASSERT_VALUES_EQUAL(messages.size(), oldHeadMsgCount + bigBlobMsgCount + newHeadMsgCount); + + size_t start = 0; + + for (size_t i = 0; i < oldHeadMsgCount; ++i) { + UNIT_ASSERT_VALUES_EQUAL(messages[start + i].size(), 100'000); + } + start += oldHeadMsgCount; + + for (size_t i = 0; i < bigBlobMsgCount; ++i) { + UNIT_ASSERT_VALUES_EQUAL(messages[start + i].size(), 7'900'000); + } + start += bigBlobMsgCount; + + for (size_t i = 0; i < newHeadMsgCount; ++i) { + UNIT_ASSERT_VALUES_EQUAL(messages[start + i].size(), 100'000); + } +} + +#define Y_UNIT_TEST_WITH_REBOOTS(name, oldHeadCount, bigBlobsCount, newHeadCount) \ +Y_UNIT_TEST_F(name##_RestartNo, TFixture) { \ + TestTxWithBigBlobs({.OldHeadCount = oldHeadCount, .BigBlobsCount = bigBlobsCount, .NewHeadCount = newHeadCount, .RestartMode = ERestartNo}); \ +} \ +Y_UNIT_TEST_F(name##_RestartBeforeCommit, TFixture) { \ + TestTxWithBigBlobs({.OldHeadCount = oldHeadCount, .BigBlobsCount = bigBlobsCount, .NewHeadCount = newHeadCount, .RestartMode = ERestartBeforeCommit}); \ +} \ +Y_UNIT_TEST_F(name##_RestartAfterCommit, TFixture) { \ + TestTxWithBigBlobs({.OldHeadCount = oldHeadCount, .BigBlobsCount = bigBlobsCount, .NewHeadCount = newHeadCount, .RestartMode = ERestartAfterCommit}); \ +} + +Y_UNIT_TEST_WITH_REBOOTS(WriteToTopic_Demo_18, 10, 2, 10); +Y_UNIT_TEST_WITH_REBOOTS(WriteToTopic_Demo_19, 10, 0, 10); +Y_UNIT_TEST_WITH_REBOOTS(WriteToTopic_Demo_20, 10, 2, 0); + +Y_UNIT_TEST_WITH_REBOOTS(WriteToTopic_Demo_21, 0, 2, 10); +Y_UNIT_TEST_WITH_REBOOTS(WriteToTopic_Demo_22, 0, 0, 10); +Y_UNIT_TEST_WITH_REBOOTS(WriteToTopic_Demo_23, 0, 2, 0); + +void TFixture::CreateTable(const TString& tablePath) +{ + UNIT_ASSERT(!tablePath.empty()); + + TString path = (tablePath[0] != '/') ? ("/Root/" + tablePath) : tablePath; + + NTable::TSession session = CreateTableSession(); + auto desc = NTable::TTableBuilder() + .AddNonNullableColumn("key", EPrimitiveType::Utf8) + .AddNonNullableColumn("value", EPrimitiveType::Utf8) + .SetPrimaryKeyColumn("key") + .Build(); + auto result = session.CreateTable(path, std::move(desc)).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); +} + +auto TFixture::MakeTableRecords() -> TVector +{ + TVector records; + records.emplace_back("key-1", "value-1"); + records.emplace_back("key-2", "value-2"); + records.emplace_back("key-3", "value-3"); + records.emplace_back("key-4", "value-4"); + return records; +} + +auto TFixture::MakeJsonDoc(const TVector& records) -> TString +{ + auto makeJsonObject = [](const TTableRecord& r) { + return Sprintf(R"({"key":"%s", "value":"%s"})", + r.Key.data(), + r.Value.data()); + }; + + if (records.empty()) { + return "[]"; + } + + TString s = "["; + + s += makeJsonObject(records.front()); + for (auto i = records.begin() + 1; i != records.end(); ++i) { + s += ", "; + s += makeJsonObject(*i); + } + s += "]"; + + return s; +} + +void TFixture::WriteToTable(const TString& tablePath, + const TVector& records, + NTable::TTransaction* tx) +{ + TString query = Sprintf("DECLARE $key AS Utf8;" + "DECLARE $value AS Utf8;" + "UPSERT INTO `%s` (key, value) VALUES ($key, $value);", + tablePath.data()); + NTable::TSession session = tx->GetSession(); + + for (const auto& r : records) { + auto params = session.GetParamsBuilder() + .AddParam("$key").Utf8(r.Key).Build() + .AddParam("$value").Utf8(r.Value).Build() + .Build(); + auto result = session.ExecuteDataQuery(query, + NYdb::NTable::TTxControl::Tx(*tx), + params).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + } +} + +size_t TFixture::GetTableRecordsCount(const TString& tablePath) +{ + TString query = Sprintf(R"(SELECT COUNT(*) FROM `%s`)", + tablePath.data()); + NTable::TSession session = CreateTableSession(); + NTable::TTransaction tx = BeginTx(session); + + auto result = session.ExecuteDataQuery(query, + NYdb::NTable::TTxControl::Tx(tx).CommitTx(true)).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + NYdb::TResultSetParser parser(result.GetResultSet(0)); + UNIT_ASSERT(parser.TryNextRow()); + + return parser.ColumnParser(0).GetUint64(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_24, TFixture) +{ + // + // the test verifies a transaction in which data is written to a topic and to a table + // + CreateTopic("topic_A"); + CreateTable("/Root/table_A"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + auto records = MakeTableRecords(); + WriteToTable("table_A", records, &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), &tx); + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx, EStatus::SUCCESS); + + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + UNIT_ASSERT_VALUES_EQUAL(messages[0], MakeJsonDoc(records)); + + UNIT_ASSERT_VALUES_EQUAL(GetTableRecordsCount("table_A"), records.size()); + + CheckTabletKeys("topic_A"); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_25, TFixture) +{ + // + // the test verifies a transaction in which data is read from one topic and written to another + // + CreateTopic("topic_A"); + CreateTopic("topic_B"); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1"); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2"); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3"); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), &tx); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 3); + + for (const auto& m : messages) { + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, m, &tx); + } + + WaitForAcks("topic_B", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx, EStatus::SUCCESS); + + messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2)); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 3); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_26, TFixture) +{ + // + // the test verifies a transaction in which data is read from a partition of one topic and written to + // another partition of this topic + // + const ui32 PARTITION_0 = 0; + const ui32 PARTITION_1 = 1; + + CreateTopic("topic_A", TEST_CONSUMER, 2); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", nullptr, PARTITION_0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", nullptr, PARTITION_0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", nullptr, PARTITION_0); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), &tx, PARTITION_0); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 3); + + for (const auto& m : messages) { + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, m, &tx, PARTITION_1); + } + + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); + + CommitTx(tx, EStatus::SUCCESS); + + messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), nullptr, PARTITION_1); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 3); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_27, TFixture) +{ + CreateTopic("topic_A", TEST_CONSUMER); + CreateTopic("topic_B", TEST_CONSUMER); + CreateTopic("topic_C", TEST_CONSUMER); + + for (size_t i = 0, writtenInTx = 0; i < 2; ++i) { + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", nullptr, 0); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", nullptr, 0); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), &tx, 0); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + WriteToTopic("topic_C", TEST_MESSAGE_GROUP_ID, messages[0], &tx, 0); + ++writtenInTx; + WaitForAcks("topic_C", TEST_MESSAGE_GROUP_ID, writtenInTx); + + messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2), &tx, 0); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); + WriteToTopic("topic_C", TEST_MESSAGE_GROUP_ID, messages[0], &tx, 0); + ++writtenInTx; + WaitForAcks("topic_C", TEST_MESSAGE_GROUP_ID, writtenInTx); + + CommitTx(tx, EStatus::SUCCESS); + + messages = ReadFromTopic("topic_C", TEST_CONSUMER, TDuration::Seconds(2), nullptr, 0); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); + + DumpPQTabletKeys("topic_A"); + DumpPQTabletKeys("topic_B"); + DumpPQTabletKeys("topic_C"); + } } } diff --git a/src/client/topic/ut/trace_ut.cpp b/src/client/topic/ut/trace_ut.cpp new file mode 100644 index 00000000000..133bfb3e65c --- /dev/null +++ b/src/client/topic/ut/trace_ut.cpp @@ -0,0 +1,168 @@ +#include + +#include + +namespace NYdb::NTopic::NTests { + + Y_UNIT_TEST_SUITE(Trace) { + + Y_UNIT_TEST(SkipSpaces) { + UNIT_ASSERT_STRINGS_EQUAL(SkipSpaces(""), ""); + UNIT_ASSERT_STRINGS_EQUAL(SkipSpaces(" "), ""); + UNIT_ASSERT_STRINGS_EQUAL(SkipSpaces(" a"), "a"); + UNIT_ASSERT_STRINGS_EQUAL(SkipSpaces(" a "), "a "); + } + + Y_UNIT_TEST(NextToken) { + UNIT_ASSERT_STRINGS_EQUAL(NextToken(""), ""); + UNIT_ASSERT_STRINGS_EQUAL(NextToken(" "), ""); + UNIT_ASSERT_STRINGS_EQUAL(NextToken("a"), "a"); + UNIT_ASSERT_STRINGS_EQUAL(NextToken(" a"), "a"); + UNIT_ASSERT_STRINGS_EQUAL(NextToken(" a "), "a"); + TStringBuf b("a=1"); + UNIT_ASSERT_STRINGS_EQUAL(NextToken(b, '='), "a"); + UNIT_ASSERT_STRINGS_EQUAL(b, "1"); + } + + Y_UNIT_TEST(TTraceEvent) { + UNIT_ASSERT_TEST_FAILS(TTraceEvent::FromString("")); + TString const eventName("init"); + { + TString s(eventName); + auto ev = TTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT(ev.KeyValues.empty()); + } + { + TString s(eventName + " a"); + auto ev = TTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 1); + UNIT_ASSERT(ev.KeyValues.at("a").empty()); + } + { + TString s(eventName + " a b"); + auto ev = TTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 2); + UNIT_ASSERT(ev.KeyValues.at("a").empty()); + UNIT_ASSERT(ev.KeyValues.at("b").empty()); + } + { + TString s(eventName + " ="); + auto ev = TTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 1); + UNIT_ASSERT(ev.KeyValues.at("").empty()); + } + { + TString s(eventName + " a=1 b"); + auto ev = TTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(ev.KeyValues.at("a"), "1"); + UNIT_ASSERT(ev.KeyValues.at("b").empty()); + } + { + TString s(eventName + " a b=2"); + auto ev = TTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 2); + UNIT_ASSERT(ev.KeyValues.at("a").empty()); + UNIT_ASSERT_STRINGS_EQUAL(ev.KeyValues.at("b"), "2"); + } + { + TString s(eventName + " a=1 b=2"); + auto ev = TTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(ev.KeyValues.at("a"), "1"); + UNIT_ASSERT_STRINGS_EQUAL(ev.KeyValues.at("b"), "2"); + } + { + TExpectedTraceEvent expected = {eventName, {{"a", {"1"}}}, {"d"}}; + UNIT_ASSERT(!expected.Matches({"", {}})); + UNIT_ASSERT(!expected.Matches({eventName, {}})); + UNIT_ASSERT(!expected.Matches({eventName, {{"a", ""}}})); + UNIT_ASSERT(!expected.Matches({eventName, {{"a", "0"}}})); + UNIT_ASSERT(!expected.Matches({eventName, {{"c", "1"}}})); + UNIT_ASSERT(expected.Matches({eventName, {{"a", "1"}}})); + UNIT_ASSERT(expected.Matches({eventName, {{"a", "1"}, {"b", "2"}}})); + UNIT_ASSERT(!expected.Matches({eventName, {{"a", "1"}, {"d", "4"}}})); // The "d" should NOT appear in the event. + } + } + + Y_UNIT_TEST(TExpectedTraceEvent) { + UNIT_ASSERT_TEST_FAILS(TExpectedTraceEvent::FromString("")); + TString const eventName("init"); + { + TString s(eventName); + auto ev = TExpectedTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT(ev.KeyValues.empty()); + } + { + TString s(eventName + " a"); + auto ev = TExpectedTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 1); + UNIT_ASSERT(ev.KeyValues.at("a").empty()); + } + { + TString s(eventName + " a b"); + auto ev = TExpectedTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 2); + UNIT_ASSERT(ev.KeyValues.at("a").empty()); + UNIT_ASSERT(ev.KeyValues.at("b").empty()); + } + { + TString s(eventName + " ="); + auto ev = TExpectedTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 1); + UNIT_ASSERT(ev.KeyValues.at("").empty()); + } + { + TString s(eventName + " a=1 b"); + auto ev = TExpectedTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(ev.KeyValues.at("a"), "1"); + UNIT_ASSERT(ev.KeyValues.at("b").empty()); + } + { + TString s(eventName + " a b=2"); + auto ev = TExpectedTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 2); + UNIT_ASSERT(ev.KeyValues.at("a").empty()); + UNIT_ASSERT_STRINGS_EQUAL(ev.KeyValues.at("b"), "2"); + } + { + TString s(eventName + " a=1 b=2"); + auto ev = TExpectedTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT_EQUAL(ev.KeyValues.size(), 2); + UNIT_ASSERT_STRINGS_EQUAL(ev.KeyValues.at("a"), "1"); + UNIT_ASSERT_STRINGS_EQUAL(ev.KeyValues.at("b"), "2"); + } + { + TString s(eventName + " !a"); + auto ev = TExpectedTraceEvent::FromString(s); + UNIT_ASSERT_STRINGS_EQUAL(ev.Event, eventName); + UNIT_ASSERT(ev.KeyValues.empty()); + UNIT_ASSERT_EQUAL(ev.DeniedKeys.size(), 1); + UNIT_ASSERT_STRINGS_EQUAL(ev.DeniedKeys[0], "a"); + } + } + + Y_UNIT_TEST(TExpectedTrace) { + TExpectedTrace expected{"A", "B"}; + TVector events{{"X", {}}, {"A", {}}, {"X", {}}, {"B", {}}, {"X", {}}}; + UNIT_ASSERT(expected.Matches(events)); + expected = {"A", "B", "C"}; + UNIT_ASSERT(!expected.Matches(events)); + } + } +} diff --git a/src/client/topic/ut/ut_utils/CMakeLists.darwin-arm64.txt b/src/client/topic/ut/ut_utils/CMakeLists.darwin-arm64.txt deleted file mode 100644 index 259b37879c7..00000000000 --- a/src/client/topic/ut/ut_utils/CMakeLists.darwin-arm64.txt +++ /dev/null @@ -1,28 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -_ydb_sdk_add_library(ydb_topic-ut-ut_utils) -target_compile_options(ydb_topic-ut-ut_utils PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_link_libraries(ydb_topic-ut-ut_utils PUBLIC - yutil - library-grpc-server - cpp-testing-unittest - threading-chunk_queue - core-testlib-default - persqueue-topic_parser_public - client-ydb_driver - client-ydb_topic - client-ydb_table -) -target_sources(ydb_topic-ut-ut_utils PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/managed_executor.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp -) diff --git a/src/client/topic/ut/ut_utils/CMakeLists.darwin-x86_64.txt b/src/client/topic/ut/ut_utils/CMakeLists.darwin-x86_64.txt deleted file mode 100644 index 259b37879c7..00000000000 --- a/src/client/topic/ut/ut_utils/CMakeLists.darwin-x86_64.txt +++ /dev/null @@ -1,28 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -_ydb_sdk_add_library(ydb_topic-ut-ut_utils) -target_compile_options(ydb_topic-ut-ut_utils PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_link_libraries(ydb_topic-ut-ut_utils PUBLIC - yutil - library-grpc-server - cpp-testing-unittest - threading-chunk_queue - core-testlib-default - persqueue-topic_parser_public - client-ydb_driver - client-ydb_topic - client-ydb_table -) -target_sources(ydb_topic-ut-ut_utils PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/managed_executor.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp -) diff --git a/src/client/topic/ut/ut_utils/CMakeLists.linux-aarch64.txt b/src/client/topic/ut/ut_utils/CMakeLists.linux-aarch64.txt deleted file mode 100644 index 2f5563a660a..00000000000 --- a/src/client/topic/ut/ut_utils/CMakeLists.linux-aarch64.txt +++ /dev/null @@ -1,29 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -_ydb_sdk_add_library(ydb_topic-ut-ut_utils) -target_compile_options(ydb_topic-ut-ut_utils PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_link_libraries(ydb_topic-ut-ut_utils PUBLIC - - yutil - library-grpc-server - cpp-testing-unittest - threading-chunk_queue - core-testlib-default - persqueue-topic_parser_public - client-ydb_driver - client-ydb_topic - client-ydb_table -) -target_sources(ydb_topic-ut-ut_utils PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/managed_executor.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp -) diff --git a/src/client/topic/ut/ut_utils/CMakeLists.linux-x86_64.txt b/src/client/topic/ut/ut_utils/CMakeLists.linux-x86_64.txt deleted file mode 100644 index 2f5563a660a..00000000000 --- a/src/client/topic/ut/ut_utils/CMakeLists.linux-x86_64.txt +++ /dev/null @@ -1,29 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -_ydb_sdk_add_library(ydb_topic-ut-ut_utils) -target_compile_options(ydb_topic-ut-ut_utils PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_link_libraries(ydb_topic-ut-ut_utils PUBLIC - - yutil - library-grpc-server - cpp-testing-unittest - threading-chunk_queue - core-testlib-default - persqueue-topic_parser_public - client-ydb_driver - client-ydb_topic - client-ydb_table -) -target_sources(ydb_topic-ut-ut_utils PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/managed_executor.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp -) diff --git a/src/client/topic/ut/ut_utils/CMakeLists.txt b/src/client/topic/ut/ut_utils/CMakeLists.txt deleted file mode 100644 index 4fbe9853d99..00000000000 --- a/src/client/topic/ut/ut_utils/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - -if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.linux-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64") - include(CMakeLists.linux-aarch64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64") - include(CMakeLists.darwin-x86_64.txt) -elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "arm64") - include(CMakeLists.darwin-arm64.txt) -elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64") - include(CMakeLists.windows-x86_64.txt) -endif() diff --git a/src/client/topic/ut/ut_utils/CMakeLists.windows-x86_64.txt b/src/client/topic/ut/ut_utils/CMakeLists.windows-x86_64.txt deleted file mode 100644 index 259b37879c7..00000000000 --- a/src/client/topic/ut/ut_utils/CMakeLists.windows-x86_64.txt +++ /dev/null @@ -1,28 +0,0 @@ - -# This file was generated by the build system used internally in the Yandex monorepo. -# Only simple modifications are allowed (adding source-files to targets, adding simple properties -# like target_include_directories). These modifications will be ported to original -# ya.make files by maintainers. Any complex modifications which can't be ported back to the -# original buildsystem will not be accepted. - - - -_ydb_sdk_add_library(ydb_topic-ut-ut_utils) -target_compile_options(ydb_topic-ut-ut_utils PRIVATE - -DUSE_CURRENT_UDF_ABI_VERSION -) -target_link_libraries(ydb_topic-ut-ut_utils PUBLIC - yutil - library-grpc-server - cpp-testing-unittest - threading-chunk_queue - core-testlib-default - persqueue-topic_parser_public - client-ydb_driver - client-ydb_topic - client-ydb_table -) -target_sources(ydb_topic-ut-ut_utils PRIVATE - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/managed_executor.cpp - ${YDB_SDK_SOURCE_DIR}/client/ydb_topic/ut/ut_utils/topic_sdk_test_setup.cpp -) diff --git a/src/client/topic/ut/ut_utils/managed_executor.cpp b/src/client/topic/ut/ut_utils/managed_executor.cpp index 63b5f910082..e6b21741f15 100644 --- a/src/client/topic/ut/ut_utils/managed_executor.cpp +++ b/src/client/topic/ut/ut_utils/managed_executor.cpp @@ -14,9 +14,10 @@ bool TManagedExecutor::IsAsync() const void TManagedExecutor::Post(TFunction &&f) { - std::lock_guard guard(Mutex); - Funcs.push_back(std::move(f)); - ++Planned; + with_lock (Mutex) { + Funcs.push_back(std::move(f)); + ++Planned; + } } void TManagedExecutor::DoStart() @@ -45,19 +46,21 @@ void TManagedExecutor::RunTask(TFunction&& func) void TManagedExecutor::StartFuncs(const std::vector& indicies) { - std::lock_guard guard(Mutex); - for (auto index : indicies) { - Y_ABORT_UNLESS(index < Funcs.size()); - Y_ABORT_UNLESS(Funcs[index]); + with_lock (Mutex) { + for (auto index : indicies) { + Y_ABORT_UNLESS(index < Funcs.size()); + Y_ABORT_UNLESS(Funcs[index]); - RunTask(std::move(Funcs[index])); + RunTask(std::move(Funcs[index])); + } } } size_t TManagedExecutor::GetFuncsCount() const { - std::lock_guard guard(Mutex); - return Funcs.size(); + with_lock (Mutex) { + return Funcs.size(); + } } size_t TManagedExecutor::GetPlannedCount() const @@ -77,10 +80,11 @@ size_t TManagedExecutor::GetExecutedCount() const void TManagedExecutor::RunAllTasks() { - std::lock_guard guard(Mutex); - for (auto& func : Funcs) { - if (func) { - RunTask(std::move(func)); + with_lock (Mutex) { + for (auto& func : Funcs) { + if (func) { + RunTask(std::move(func)); + } } } } diff --git a/src/client/topic/ut/ut_utils/managed_executor.h b/src/client/topic/ut/ut_utils/managed_executor.h index 7fd21398de0..02f6d521d3d 100644 --- a/src/client/topic/ut/ut_utils/managed_executor.h +++ b/src/client/topic/ut/ut_utils/managed_executor.h @@ -1,10 +1,11 @@ #pragma once -#include -#include +#include +#include + +#include #include -#include namespace NYdb::NTopic::NTests { @@ -34,7 +35,7 @@ class TManagedExecutor : public IExecutor { void RunTask(TFunction&& func); TExecutorPtr Executor; - std::mutex Mutex; + TMutex Mutex; std::vector Funcs; std::atomic Planned = 0; std::atomic Running = 0; diff --git a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp index af40290c247..806fe50b289 100644 --- a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp +++ b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.cpp @@ -1,16 +1,14 @@ #include "topic_sdk_test_setup.h" -#include - using namespace NYdb; using namespace NYdb::NTopic; using namespace NYdb::NTopic::NTests; -TTopicSdkTestSetup::TTopicSdkTestSetup(const std::string& testCaseName, const NKikimr::Tests::TServerSettings& settings, bool createTopic) +TTopicSdkTestSetup::TTopicSdkTestSetup(const TString& testCaseName, const NKikimr::Tests::TServerSettings& settings, bool createTopic) : Database("/Root") , Server(settings, false) { - Log.SetFormatter([testCaseName](ELogPriority priority, std::string_view message) { + Log.SetFormatter([testCaseName](ELogPriority priority, TStringBuf message) { return TStringBuilder() << TInstant::Now() << " :" << testCaseName << " " << priority << ": " << message << Endl; }); @@ -24,14 +22,27 @@ TTopicSdkTestSetup::TTopicSdkTestSetup(const std::string& testCaseName, const NK } } -void TTopicSdkTestSetup::CreateTopic(const std::string& path, const std::string& consumer, size_t partitionCount) +void TTopicSdkTestSetup::CreateTopicWithAutoscale(const TString& path, const TString& consumer, size_t partitionCount, size_t maxPartitionCount) { + CreateTopic(path, consumer, partitionCount, maxPartitionCount); +} + +void TTopicSdkTestSetup::CreateTopic(const TString& path, const TString& consumer, size_t partitionCount, std::optional maxPartitionCount) { TTopicClient client(MakeDriver()); TCreateTopicSettings topics; - TPartitioningSettings partitions(partitionCount, partitionCount); + topics + .BeginConfigurePartitioningSettings() + .MinActivePartitions(partitionCount) + .MaxActivePartitions(maxPartitionCount.value_or(partitionCount)); + + if (maxPartitionCount.has_value() && maxPartitionCount.value() > partitionCount) { + topics + .BeginConfigurePartitioningSettings() + .BeginConfigureAutoPartitioningSettings() + .Strategy(EAutoPartitioningStrategy::ScaleUp); + } - topics.PartitioningSettings(partitions); TConsumerSettings consumers(topics, consumer); topics.AppendConsumers(consumers); @@ -41,19 +52,19 @@ void TTopicSdkTestSetup::CreateTopic(const std::string& path, const std::string& Server.WaitInit(path); } -std::string TTopicSdkTestSetup::GetEndpoint() const { +TString TTopicSdkTestSetup::GetEndpoint() const { return "localhost:" + ToString(Server.GrpcPort); } -std::string TTopicSdkTestSetup::GetTopicPath(const std::string& name) const { +TString TTopicSdkTestSetup::GetTopicPath(const TString& name) const { return GetTopicParent() + "/" + name; } -std::string TTopicSdkTestSetup::GetTopicParent() const { +TString TTopicSdkTestSetup::GetTopicParent() const { return GetDatabase(); } -std::string TTopicSdkTestSetup::GetDatabase() const { +TString TTopicSdkTestSetup::GetDatabase() const { return Database; } @@ -75,7 +86,7 @@ TDriverConfig TTopicSdkTestSetup::MakeDriverConfig() const config.SetEndpoint(GetEndpoint()); config.SetDatabase(GetDatabase()); config.SetAuthToken("root@builtin"); - config.SetLog(std::make_unique(&std::cerr)); + config.SetLog(MakeHolder(&Cerr)); return config; } @@ -112,4 +123,4 @@ TDriver TTopicSdkTestSetup::MakeDriver(const TDriverConfig& config) const TTopicClient TTopicSdkTestSetup::MakeClient() const { return TTopicClient(MakeDriver()); -} \ No newline at end of file +} diff --git a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h index e6fa568a64d..0d188cdcd5f 100644 --- a/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h +++ b/src/client/topic/ut/ut_utils/topic_sdk_test_setup.h @@ -1,27 +1,30 @@ -#pragma once +#pragma once -#include +#include -#include +#include namespace NYdb::NTopic::NTests { #define TEST_CASE_NAME (this->Name_) -inline static const std::string TEST_TOPIC = "test-topic"; -inline static const std::string TEST_CONSUMER = "test-consumer"; -inline static const std::string TEST_MESSAGE_GROUP_ID = "test-message_group_id"; +inline static const TString TEST_TOPIC = "test-topic"; +inline static const TString TEST_CONSUMER = "test-consumer"; +inline static const TString TEST_MESSAGE_GROUP_ID = "test-message_group_id"; class TTopicSdkTestSetup { public: - TTopicSdkTestSetup(const std::string& testCaseName, const NKikimr::Tests::TServerSettings& settings = MakeServerSettings(), bool createTopic = true); + TTopicSdkTestSetup(const TString& testCaseName, const NKikimr::Tests::TServerSettings& settings = MakeServerSettings(), bool createTopic = true); - void CreateTopic(const std::string& path = TEST_TOPIC, const std::string& consumer = TEST_CONSUMER, size_t partitionCount = 1); + void CreateTopic(const TString& path = TEST_TOPIC, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, + std::optional maxPartitionCount = std::nullopt); + void CreateTopicWithAutoscale(const TString& path = TEST_TOPIC, const TString& consumer = TEST_CONSUMER, size_t partitionCount = 1, + size_t maxPartitionCount = 100); - std::string GetEndpoint() const; - std::string GetTopicPath(const std::string& name = TEST_TOPIC) const; - std::string GetTopicParent() const; - std::string GetDatabase() const; + TString GetEndpoint() const; + TString GetTopicPath(const TString& name = TEST_TOPIC) const; + TString GetTopicParent() const; + TString GetDatabase() const; ::NPersQueue::TTestServer& GetServer(); NActors::TTestActorRuntime& GetRuntime(); @@ -35,10 +38,10 @@ class TTopicSdkTestSetup { TDriverConfig MakeDriverConfig() const; static NKikimr::Tests::TServerSettings MakeServerSettings(); private: - std::string Database; + TString Database; ::NPersQueue::TTestServer Server; TLog Log = CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG); }; -} \ No newline at end of file +} diff --git a/src/client/topic/ut/ut_utils/trace.cpp b/src/client/topic/ut/ut_utils/trace.cpp new file mode 100644 index 00000000000..a2bebdb1b83 --- /dev/null +++ b/src/client/topic/ut/ut_utils/trace.cpp @@ -0,0 +1,146 @@ +#include +#include + +#include "library/cpp/testing/unittest/registar.h" + + +namespace NYdb::NTopic::NTests { + +TStringBuf SkipSpaces(TStringBuf& b) { + while (b.starts_with(' ')) b.Skip(1); + return b; +} +TStringBuf SkipSpaces(TStringBuf&& b) { + return SkipSpaces(b); +} + +TStringBuf NextToken(TStringBuf& b, TStringBuf::TChar const delim) { + SkipSpaces(b); + return b.NextTok(delim); +} +TStringBuf NextToken(TStringBuf&& b, TStringBuf::TChar const delim) { + return NextToken(b, delim); +} + +TTraceEvent TTraceEvent::FromString(TString const& s) { + TStringBuf b(s); + TString const event(NextToken(b)); + UNIT_ASSERT_C(!event.empty(), "Wrong tracing log (format), event token not found"); + THashMap keyValues; + for (TStringBuf pair, key; ; ) { + pair = NextToken(b); + if (pair.empty()) break; + key = pair.NextTok('='); + keyValues.emplace(key, pair); + } + return {event, std::move(keyValues)}; +} + +bool TExpectedTraceEvent::Matches(TTraceEvent const& event) const { + if (Event != event.Event) { + return false; + } + for (auto const& k : DeniedKeys) { + auto it = event.KeyValues.find(k); + if (it != end(event.KeyValues)) { + return false; + } + } + for (auto const& [expectedKey, expectedValue] : KeyValues) { + auto it = event.KeyValues.find(expectedKey); + if (it == end(event.KeyValues) || !(expectedValue.empty() || expectedValue == it->second)) { + return false; + } + } + return true; +} + +TExpectedTraceEvent TExpectedTraceEvent::FromString(TString const& s) { + TStringBuf b(s); + TString const event(NextToken(b)); + UNIT_ASSERT_C(!event.empty(), "Wrong tracing log format, event token not found"); + THashMap keyValues; + TVector missingKeys; + for (TStringBuf pair, key; ; ) { + pair = NextToken(b); + if (pair.empty()) break; + key = pair.NextTok('='); + if (key.starts_with('!')) { + key.remove_prefix(1); + missingKeys.emplace_back(key); + } else { + keyValues.emplace(key, pair); + } + } + return {event, std::move(keyValues), std::move(missingKeys)}; +} + +TExpectedTrace::TExpectedTrace(std::initializer_list expected) { + Expected.reserve(expected.size()); + for (auto const& e : expected) { + Expected.emplace_back(TExpectedTraceEvent::FromString(e)); + } +} + +bool TExpectedTrace::Matches(std::span events) const { + std::span expected(Expected); + while (!expected.empty() && expected.size() <= events.size()) { + if (expected[0].Matches(events[0])) { + expected = expected.subspan(1); + } + events = events.subspan(1); + } + return expected.empty(); +} + +TStringBuf SkipPrefixLog(TStringBuf& b) { + NextToken(b); // Skip the timestamp + NextToken(b); // Skip the log level + NextToken(b); // Skip database path + SkipSpaces(b); + return b; +} + +bool CleanTraceEventBuf(TStringBuf& b, TStringBuf traceEventMarker) { + SkipPrefixLog(b); + if (!b.starts_with(traceEventMarker)) { + return false; + } + while (b.ends_with('\n')) b.remove_suffix(1); + b.Skip(traceEventMarker.size()); + return true; +} + +void TTracingBackend::WriteData(const TLogRecord& rec) { + std::lock_guard lg(Lock); + if (rec.Priority != TLOG_RESOURCES) { + return; + } + if (TStringBuf b(rec.Data, rec.Len); CleanTraceEventBuf(b, TraceEventMarker)) { + TString s(b); + TTraceEvent e(EventParser(s)); + if (EventPromise.Initialized() && !EventPromise.HasValue() && ExpectedEvent.Matches(e)) { + EventPromise.SetValue(); + } + Log.emplace_back(std::move(s)); + Events.emplace_back(std::move(e)); + } +} + +TVector TTracingBackend::GetLog() const { + std::lock_guard lg(Lock); + return Log; +} + +TVector TTracingBackend::GetEvents() const { + std::lock_guard lg(Lock); + return Events; +} + +NThreading::TFuture TTracingBackend::WaitForEvent(TString const& eventName) { + EventPromise = NThreading::NewPromise(); + ExpectedEvent = {eventName, {}, {}}; + return EventPromise.GetFuture(); +} + +} diff --git a/src/client/topic/ut/ut_utils/trace.h b/src/client/topic/ut/ut_utils/trace.h new file mode 100644 index 00000000000..fe45738e1d6 --- /dev/null +++ b/src/client/topic/ut/ut_utils/trace.h @@ -0,0 +1,88 @@ +#pragma once + +#include "ydb/public/sdk/cpp/src/client/topic/common/trace_lazy.h" + +#include "library/cpp/logger/backend.h" +#include "library/cpp/logger/record.h" +#include "library/cpp/threading/future/core/future.h" +#include "util/generic/hash.h" + +#include + + +namespace NYdb::NTopic::NTests { + +TStringBuf SkipSpaces(TStringBuf& b); +TStringBuf SkipSpaces(TStringBuf&& b); + +TStringBuf NextToken(TStringBuf& b, TStringBuf::TChar const delim = ' '); +TStringBuf NextToken(TStringBuf&& b, TStringBuf::TChar const delim = ' '); + +struct TTraceEvent { + TString Event; + THashMap KeyValues; + + // Expects "event [key[=value]]*" kind of string. No spaces around the = sign. Values are optional. + static TTraceEvent FromString(TString const& s); +}; + +struct TExpectedTraceEvent { + TString Event; + THashMap KeyValues; // These key-value pairs should be in the event as is. + TVector DeniedKeys; // These keys should NOT appear in the event. + + bool Matches(TTraceEvent const& event) const; + + // Expects "event [[!]key[=value]]*" kind of string. No spaces around the = sign. Values are optional. + // The bang symbol right before a key denies the key in events. + static TExpectedTraceEvent FromString(TString const& s); +}; + +struct TExpectedTrace { + TVector Expected; + + TExpectedTrace(std::initializer_list expected) : Expected(expected) {} + TExpectedTrace(std::initializer_list expected); + + // Check if the Expected events are a subsequence of the events. + bool Matches(std::span events) const; +}; + +// The log formatter is configured in TDriverConfig::SetDatabase using the GetPrefixLogFormatter function. +// SkipPrefixLog skips the prefix that currently looks like "2024-02-13T07:51:07.979754Z :INFO: [/Root]". +// It'd be better if TTracingBackend received strings before formatter does, but I don't know how to do it. +TStringBuf SkipPrefixLog(TStringBuf& b); + +// If possible, transforms the buffer to the form of "event [key[=value]]*". +bool CleanTraceEventBuf(TStringBuf& b, TStringBuf traceEventMarker); + +// Log backend for tracing events. Expects "... " kind of strings. +// The "" substring is parsed by the Parser function that returns a TTraceEvent object. +// To get the trace, call GetEvents method. +class TTracingBackend : public TLogBackend { +public: + TTracingBackend(const TString& traceEventMarker = TString{TRACE_EVENT_MARKER}, std::function parser = TTraceEvent::FromString) + : TraceEventMarker(traceEventMarker) + , EventParser(parser) {} + + // Only logs strings on TRACE log level that start with the TraceEventMarker (after we strip the log prefix). + void WriteData(const TLogRecord& rec) override; + void ReopenLog() override {} + ELogPriority FiltrationLevel() const override { return TLOG_RESOURCES; } +public: + TVector GetLog() const; + TVector GetEvents() const; + + // Returns a feature that is fulfilled when an event with a particular name gets logged. + NThreading::TFuture WaitForEvent(TString const& eventName); +private: + mutable TAdaptiveLock Lock; + TString TraceEventMarker; // Log strings that start with this marker are parsed and stored. + std::function EventParser; + TVector Log; // Stores clean strings from log records that contained TraceEventMarker. + TVector Events; // Stores parsed trace events. + NThreading::TPromise EventPromise; + TExpectedTraceEvent ExpectedEvent; +}; + +} diff --git a/src/client/types/credentials/login/login.cpp b/src/client/types/credentials/login/login.cpp index c66ec3052b9..013b4b52b76 100644 --- a/src/client/types/credentials/login/login.cpp +++ b/src/client/types/credentials/login/login.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include diff --git a/src/client/types/credentials/oauth2_token_exchange/credentials.cpp b/src/client/types/credentials/oauth2_token_exchange/credentials.cpp index a780fa8e68a..f363dbafa86 100644 --- a/src/client/types/credentials/oauth2_token_exchange/credentials.cpp +++ b/src/client/types/credentials/oauth2_token_exchange/credentials.cpp @@ -5,7 +5,7 @@ #include #include #include -#include +#include #include #include diff --git a/src/client/types/credentials/oauth2_token_exchange/from_file.cpp b/src/client/types/credentials/oauth2_token_exchange/from_file.cpp index b3084eca102..5586a114be3 100644 --- a/src/client/types/credentials/oauth2_token_exchange/from_file.cpp +++ b/src/client/types/credentials/oauth2_token_exchange/from_file.cpp @@ -2,6 +2,8 @@ #include #include +#include + #include #include @@ -248,7 +250,7 @@ TOauth2TokenExchangeParams ReadOauth2ConfigJson(const std::string& configJson, c } TOauth2TokenExchangeParams ReadOauth2ConfigFile(const std::string& configFilePath, const std::string& tokenEndpoint) { - return ReadOauth2ConfigJson(TFileInput(configFilePath).ReadAll(), tokenEndpoint); + return ReadOauth2ConfigJson(TFileInput(TStringType{configFilePath}).ReadAll(), tokenEndpoint); } } // namespace diff --git a/src/client/types/operation/operation.cpp b/src/client/types/operation/operation.cpp index 0b73a2bc065..c09ebf6cd68 100644 --- a/src/client/types/operation/operation.cpp +++ b/src/client/types/operation/operation.cpp @@ -1,10 +1,11 @@ #include #include +#include #include -#include +#include namespace NYdb { @@ -81,7 +82,7 @@ void TOperation::Out(IOutputStream& o) const { std::string TOperation::ToJsonString() const { using namespace google::protobuf::util; - std::string json; + TStringType json; auto status = MessageToJsonString(GetProto(), &json, JsonPrintOptions()); Y_ABORT_UNLESS(status.ok()); return json; diff --git a/src/client/value/value.cpp b/src/client/value/value.cpp index 128db0bcf36..780bb533632 100644 --- a/src/client/value/value.cpp +++ b/src/client/value/value.cpp @@ -4,14 +4,15 @@ #include #undef INCLUDE_YDB_INTERNAL_H +#include #include #include -#include +#include #include -#include +#include #include #include @@ -617,8 +618,8 @@ class TTypeBuilder::TImpl { void Pg(const TPgType& pgType) { auto& pg = *GetProto().mutable_pg_type(); - pg.set_type_name(pgType.TypeName); - pg.set_type_modifier(pgType.TypeModifier); + pg.set_type_name(TStringType{pgType.TypeName}); + pg.set_type_modifier(TStringType{pgType.TypeModifier}); } void BeginOptional() { @@ -658,7 +659,7 @@ class TTypeBuilder::TImpl { CheckPreviousKind(ETypeKind::Struct, "AddMember"); PopPosition(); auto member = GetProto().mutable_struct_type()->add_members(); - member->set_name(memberName); + member->set_name(TStringType{memberName}); AddPosition(member->mutable_type()); } @@ -732,7 +733,7 @@ class TTypeBuilder::TImpl { } void BeginTagged(const std::string& tag) { - GetProto().mutable_tagged_type()->set_tag(tag); + GetProto().mutable_tagged_type()->set_tag(TStringType{tag}); AddPosition(GetProto().mutable_tagged_type()->mutable_type()); } @@ -742,7 +743,7 @@ class TTypeBuilder::TImpl { void Tagged(const std::string& tag, const TType& itemType) { auto taggedType = GetProto().mutable_tagged_type(); - taggedType->set_tag(tag); + taggedType->set_tag(TStringType{tag}); taggedType->mutable_type()->CopyFrom(itemType.GetProto()); } @@ -960,8 +961,8 @@ TDecimalValue::TDecimalValue(const std::string& decimalString, ui8 precision, ui NYql::NDecimal::TInt128 val = NYql::NDecimal::FromString(decimalString, precision, scale); static_assert(sizeof(val) == 16, "wrong TInt128 size"); char* buf = reinterpret_cast(&val); - Low_ = *(ui64*)buf; - Hi_ = *(i64*)(buf + 8); + Low_ = *(uint64_t*)buf; + Hi_ = *(int64_t*)(buf + 8); } std::string TDecimalValue::ToString() const { @@ -1006,7 +1007,7 @@ bool TPgValue::IsText() const { //////////////////////////////////////////////////////////////////////////////// -TUuidValue::TUuidValue(ui64 low_128, ui64 high_128) { +TUuidValue::TUuidValue(uint64_t low_128, uint64_t high_128) { Buf_.Halfs[0] = low_128; Buf_.Halfs[1] = high_128; } @@ -1155,12 +1156,12 @@ class TValueParser::TImpl { return GetProto().uint32_value(); } - i64 GetInt64() const { + int64_t GetInt64() const { CheckPrimitive(NYdb::EPrimitiveType::Int64); return GetProto().int64_value(); } - ui64 GetUint64() const { + uint64_t GetUint64() const { CheckPrimitive(NYdb::EPrimitiveType::Uint64); return GetProto().uint64_value(); } @@ -1190,7 +1191,7 @@ class TValueParser::TImpl { return TInstant::MicroSeconds(GetProto().uint64_value()); } - i64 GetInterval() const { + int64_t GetInterval() const { CheckPrimitive(NYdb::EPrimitiveType::Interval); return GetProto().int64_value(); } @@ -1200,17 +1201,17 @@ class TValueParser::TImpl { return GetProto().int32_value(); } - i64 GetDatetime64() const { + int64_t GetDatetime64() const { CheckPrimitive(NYdb::EPrimitiveType::Datetime64); return GetProto().int64_value(); } - i64 GetTimestamp64() const { + int64_t GetTimestamp64() const { CheckPrimitive(NYdb::EPrimitiveType::Timestamp64); return GetProto().int64_value(); } - i64 GetInterval64() const { + int64_t GetInterval64() const { CheckPrimitive(NYdb::EPrimitiveType::Interval64); return GetProto().int64_value(); } @@ -1691,11 +1692,11 @@ ui32 TValueParser::GetUint32() const { return Impl_->GetUint32(); } -i64 TValueParser::GetInt64() const { +int64_t TValueParser::GetInt64() const { return Impl_->GetInt64(); } -ui64 TValueParser::GetUint64() const { +uint64_t TValueParser::GetUint64() const { return Impl_->GetUint64(); } @@ -1719,7 +1720,7 @@ TInstant TValueParser::GetTimestamp() const { return Impl_->GetTimestamp(); } -i64 TValueParser::GetInterval() const { +int64_t TValueParser::GetInterval() const { return Impl_->GetInterval(); } @@ -1727,15 +1728,15 @@ i32 TValueParser::GetDate32() const { return Impl_->GetDate32(); } -i64 TValueParser::GetDatetime64() const { +int64_t TValueParser::GetDatetime64() const { return Impl_->GetDatetime64(); } -i64 TValueParser::GetTimestamp64() const { +int64_t TValueParser::GetTimestamp64() const { return Impl_->GetTimestamp64(); } -i64 TValueParser::GetInterval64() const { +int64_t TValueParser::GetInterval64() const { return Impl_->GetInterval64(); } @@ -1823,12 +1824,12 @@ std::optional TValueParser::GetOptionalUint32() const { RET_OPT_VALUE(ui32, Uint32); } -std::optional TValueParser::GetOptionalInt64() const { - RET_OPT_VALUE(i64, Int64); +std::optional TValueParser::GetOptionalInt64() const { + RET_OPT_VALUE(int64_t, Int64); } -std::optional TValueParser::GetOptionalUint64() const { - RET_OPT_VALUE(ui64, Uint64); +std::optional TValueParser::GetOptionalUint64() const { + RET_OPT_VALUE(uint64_t, Uint64); } std::optional TValueParser::GetOptionalFloat() const { @@ -1851,24 +1852,24 @@ std::optional TValueParser::GetOptionalTimestamp() const { RET_OPT_VALUE(TInstant, Timestamp); } -std::optional TValueParser::GetOptionalInterval() const { - RET_OPT_VALUE(i64, Interval); +std::optional TValueParser::GetOptionalInterval() const { + RET_OPT_VALUE(int64_t, Interval); } std::optional TValueParser::GetOptionalDate32() const { - RET_OPT_VALUE(i64, Date32); + RET_OPT_VALUE(int64_t, Date32); } -std::optional TValueParser::GetOptionalDatetime64() const { - RET_OPT_VALUE(i64, Datetime64); +std::optional TValueParser::GetOptionalDatetime64() const { + RET_OPT_VALUE(int64_t, Datetime64); } -std::optional TValueParser::GetOptionalTimestamp64() const { - RET_OPT_VALUE(i64, Timestamp64); +std::optional TValueParser::GetOptionalTimestamp64() const { + RET_OPT_VALUE(int64_t, Timestamp64); } -std::optional TValueParser::GetOptionalInterval64() const { - RET_OPT_VALUE(i64, Interval64); +std::optional TValueParser::GetOptionalInterval64() const { + RET_OPT_VALUE(int64_t, Interval64); } std::optional TValueParser::GetOptionalTzDate() const { @@ -2104,12 +2105,12 @@ class TValueBuilderImpl { GetValue().set_uint32_value(value); } - void Int64(i64 value) { + void Int64(int64_t value) { FillPrimitiveType(EPrimitiveType::Int64); GetValue().set_int64_value(value); } - void Uint64(ui64 value) { + void Uint64(uint64_t value) { FillPrimitiveType(EPrimitiveType::Uint64); GetValue().set_uint64_value(value); } @@ -2139,7 +2140,7 @@ class TValueBuilderImpl { GetValue().set_uint64_value(value.MicroSeconds()); } - void Interval(i64 value) { + void Interval(int64_t value) { FillPrimitiveType(EPrimitiveType::Interval); GetValue().set_int64_value(value); } @@ -2149,54 +2150,54 @@ class TValueBuilderImpl { GetValue().set_int32_value(value); } - void Datetime64(const i64 value) { + void Datetime64(const int64_t value) { FillPrimitiveType(EPrimitiveType::Datetime64); GetValue().set_int64_value(value); } - void Timestamp64(const i64 value) { + void Timestamp64(const int64_t value) { FillPrimitiveType(EPrimitiveType::Timestamp64); GetValue().set_int64_value(value); } - void Interval64(const i64 value) { + void Interval64(const int64_t value) { FillPrimitiveType(EPrimitiveType::Interval64); GetValue().set_int64_value(value); } void TzDate(const std::string& value) { FillPrimitiveType(EPrimitiveType::TzDate); - GetValue().set_text_value(value); + GetValue().set_text_value(TStringType{value}); } void TzDatetime(const std::string& value) { FillPrimitiveType(EPrimitiveType::TzDatetime); - GetValue().set_text_value(value); + GetValue().set_text_value(TStringType{value}); } void TzTimestamp(const std::string& value) { FillPrimitiveType(EPrimitiveType::TzTimestamp); - GetValue().set_text_value(value); + GetValue().set_text_value(TStringType{value}); } void String(const std::string& value) { FillPrimitiveType(EPrimitiveType::String); - GetValue().set_bytes_value(value); + GetValue().set_bytes_value(TStringType{value}); } void Utf8(const std::string& value) { FillPrimitiveType(EPrimitiveType::Utf8); - GetValue().set_text_value(value); + GetValue().set_text_value(TStringType{value}); } void Yson(const std::string& value) { FillPrimitiveType(EPrimitiveType::Yson); - GetValue().set_bytes_value(value); + GetValue().set_bytes_value(TStringType{value}); } void Json(const std::string& value) { FillPrimitiveType(EPrimitiveType::Json); - GetValue().set_text_value(value); + GetValue().set_text_value(TStringType{value}); } void Uuid(const TUuidValue& value) { @@ -2207,12 +2208,12 @@ class TValueBuilderImpl { void JsonDocument(const std::string& value) { FillPrimitiveType(EPrimitiveType::JsonDocument); - GetValue().set_text_value(value); + GetValue().set_text_value(TStringType{value}); } void DyNumber(const std::string& value) { FillPrimitiveType(EPrimitiveType::DyNumber); - GetValue().set_text_value(value); + GetValue().set_text_value(TStringType{value}); } void Decimal(const TDecimalValue& value) { @@ -2226,9 +2227,9 @@ class TValueBuilderImpl { if (value.IsNull()) { GetValue().set_null_flag_value(::google::protobuf::NULL_VALUE); } else if (value.IsText()) { - GetValue().set_text_value(value.Content_); + GetValue().set_text_value(TStringType{value.Content_}); } else { - GetValue().set_bytes_value(value.Content_); + GetValue().set_bytes_value(TStringType{value.Content_}); } } @@ -2854,13 +2855,13 @@ TDerived& TValueBuilderBase::Uint32(ui32 value) { } template -TDerived& TValueBuilderBase::Int64(i64 value) { +TDerived& TValueBuilderBase::Int64(int64_t value) { Impl_->Int64(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Uint64(ui64 value) { +TDerived& TValueBuilderBase::Uint64(uint64_t value) { Impl_->Uint64(value); return static_cast(*this); } @@ -2896,7 +2897,7 @@ TDerived& TValueBuilderBase::Timestamp(const TInstant& value) { } template -TDerived& TValueBuilderBase::Interval(i64 value) { +TDerived& TValueBuilderBase::Interval(int64_t value) { Impl_->Interval(value); return static_cast(*this); } @@ -2908,19 +2909,19 @@ TDerived& TValueBuilderBase::Date32(const i32 value) { } template -TDerived& TValueBuilderBase::Datetime64(const i64 value) { +TDerived& TValueBuilderBase::Datetime64(const int64_t value) { Impl_->Datetime64(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Timestamp64(const i64 value) { +TDerived& TValueBuilderBase::Timestamp64(const int64_t value) { Impl_->Timestamp64(value); return static_cast(*this); } template -TDerived& TValueBuilderBase::Interval64(const i64 value) { +TDerived& TValueBuilderBase::Interval64(const int64_t value) { Impl_->Interval64(value); return static_cast(*this); } @@ -3049,12 +3050,12 @@ TDerived& TValueBuilderBase::OptionalUint32(const std::optional& } template -TDerived& TValueBuilderBase::OptionalInt64(const std::optional& value) { +TDerived& TValueBuilderBase::OptionalInt64(const std::optional& value) { SET_OPT_VALUE_MAYBE(Int64); } template -TDerived& TValueBuilderBase::OptionalUint64(const std::optional& value) { +TDerived& TValueBuilderBase::OptionalUint64(const std::optional& value) { SET_OPT_VALUE_MAYBE(Uint64); } @@ -3084,7 +3085,7 @@ TDerived& TValueBuilderBase::OptionalTimestamp(const std::optional -TDerived& TValueBuilderBase::OptionalInterval(const std::optional& value) { +TDerived& TValueBuilderBase::OptionalInterval(const std::optional& value) { SET_OPT_VALUE_MAYBE(Interval); } @@ -3094,17 +3095,17 @@ TDerived& TValueBuilderBase::OptionalDate32(const std::optional& } template -TDerived& TValueBuilderBase::OptionalDatetime64(const std::optional& value) { +TDerived& TValueBuilderBase::OptionalDatetime64(const std::optional& value) { SET_OPT_VALUE_MAYBE(Datetime64); } template -TDerived& TValueBuilderBase::OptionalTimestamp64(const std::optional& value) { +TDerived& TValueBuilderBase::OptionalTimestamp64(const std::optional& value) { SET_OPT_VALUE_MAYBE(Timestamp64); } template -TDerived& TValueBuilderBase::OptionalInterval64(const std::optional& value) { +TDerived& TValueBuilderBase::OptionalInterval64(const std::optional& value) { SET_OPT_VALUE_MAYBE(Interval64); } diff --git a/src/client/ymq/ymq.cpp b/src/client/ymq/ymq.cpp index 10d0ca8d49b..a2cefdaa158 100644 --- a/src/client/ymq/ymq.cpp +++ b/src/client/ymq/ymq.cpp @@ -4,8 +4,10 @@ #include #undef INCLUDE_YDB_INTERNAL_H -#include -#include +#include +#include + +#include #include @@ -68,7 +70,7 @@ namespace NYdb::Ymq::V1 { Ydb::Ymq::V1::GetQueueUrlResult>(settings, &Ydb::Ymq::V1::YmqService::Stub::AsyncGetQueueUrl, [&](Ydb::Ymq::V1::GetQueueUrlRequest& req) { - req.set_queue_name(queueName); + req.set_queue_name(TStringType{queueName}); } ); } @@ -80,7 +82,7 @@ namespace NYdb::Ymq::V1 { Ydb::Ymq::V1::CreateQueueResult>(settings, &Ydb::Ymq::V1::YmqService::Stub::AsyncCreateQueue, [&](Ydb::Ymq::V1::CreateQueueRequest& req) { - req.set_queue_name(queueName); + req.set_queue_name(TStringType{queueName}); } ); } @@ -92,8 +94,8 @@ namespace NYdb::Ymq::V1 { Ydb::Ymq::V1::SendMessageResult>(settings, &Ydb::Ymq::V1::YmqService::Stub::AsyncSendMessage, [&](Ydb::Ymq::V1::SendMessageRequest& req) { - req.set_queue_url(queueUrl); - req.set_message_body(body); + req.set_queue_url(TStringType{queueUrl}); + req.set_message_body(TStringType{body}); } ); } diff --git a/src/library/CMakeLists.txt b/src/library/CMakeLists.txt index 7ef8fa22d15..96fed21fca8 100644 --- a/src/library/CMakeLists.txt +++ b/src/library/CMakeLists.txt @@ -5,10 +5,11 @@ add_subdirectory(malloc/api) add_subdirectory(operation_id) add_subdirectory(persqueue/obfuscate) add_subdirectory(persqueue/topic_parser_public) +add_subdirectory(proto_output) add_subdirectory(retry) add_subdirectory(string_utils/base64) add_subdirectory(string_utils/helpers) add_subdirectory(string_utils/misc) add_subdirectory(uuid) -add_subdirectory(yql) +add_subdirectory(yql_common) add_subdirectory(yson_value) diff --git a/src/library/json_value/ydb_json_value.cpp b/src/library/json_value/ydb_json_value.cpp index 31eb8fffb9b..0797a4ea859 100644 --- a/src/library/json_value/ydb_json_value.cpp +++ b/src/library/json_value/ydb_json_value.cpp @@ -1,10 +1,12 @@ #include -#include +#include #include #include +#include + namespace NYdb { namespace { diff --git a/src/library/operation_id/operation_id.cpp b/src/library/operation_id/operation_id.cpp index e387ce029ef..146e45b4fec 100644 --- a/src/library/operation_id/operation_id.cpp +++ b/src/library/operation_id/operation_id.cpp @@ -1,7 +1,5 @@ #include -#include - #include #include @@ -67,7 +65,7 @@ std::string TOperationId::ToString() const { res << "?"; } - for (int i = 0; i < Data.size(); i++) { + for (size_t i = 0; i < Data.size(); ++i) { TUri::ReEncode(res, Data[i]->Key); res << "="; TUri::ReEncode(res, Data[i]->Value); diff --git a/src/api/proto_output/CMakeLists.txt b/src/library/proto_output/CMakeLists.txt similarity index 100% rename from src/api/proto_output/CMakeLists.txt rename to src/library/proto_output/CMakeLists.txt diff --git a/src/api/proto_output/proto_output.cpp b/src/library/proto_output/proto_output.cpp similarity index 91% rename from src/api/proto_output/proto_output.cpp rename to src/library/proto_output/proto_output.cpp index a8c4cf3eb09..80863fefa6d 100644 --- a/src/api/proto_output/proto_output.cpp +++ b/src/library/proto_output/proto_output.cpp @@ -1,7 +1,7 @@ #include -#include -#include +#include +#include #include diff --git a/src/library/retry/retry.h b/src/library/retry/retry.h index 24b0440a1d1..24ff8eb4e3d 100644 --- a/src/library/retry/retry.h +++ b/src/library/retry/retry.h @@ -3,7 +3,7 @@ #include #include "utils.h" -#include +#include #include #include diff --git a/src/library/yql/CMakeLists.txt b/src/library/yql/CMakeLists.txt deleted file mode 100644 index 9a269f91a10..00000000000 --- a/src/library/yql/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ - -add_subdirectory(public) -add_subdirectory(utils) diff --git a/src/library/yql/public/CMakeLists.txt b/src/library/yql_common/CMakeLists.txt similarity index 67% rename from src/library/yql/public/CMakeLists.txt rename to src/library/yql_common/CMakeLists.txt index 91375e36aa8..a59772ad800 100644 --- a/src/library/yql/public/CMakeLists.txt +++ b/src/library/yql_common/CMakeLists.txt @@ -1,2 +1,3 @@ add_subdirectory(decimal) add_subdirectory(issue) +add_subdirectory(utils) diff --git a/src/library/yql/public/decimal/CMakeLists.txt b/src/library/yql_common/decimal/CMakeLists.txt similarity index 100% rename from src/library/yql/public/decimal/CMakeLists.txt rename to src/library/yql_common/decimal/CMakeLists.txt diff --git a/src/library/yql/public/decimal/yql_decimal.cpp b/src/library/yql_common/decimal/yql_decimal.cpp similarity index 100% rename from src/library/yql/public/decimal/yql_decimal.cpp rename to src/library/yql_common/decimal/yql_decimal.cpp diff --git a/src/library/yql/public/decimal/yql_decimal.h b/src/library/yql_common/decimal/yql_decimal.h similarity index 100% rename from src/library/yql/public/decimal/yql_decimal.h rename to src/library/yql_common/decimal/yql_decimal.h diff --git a/src/library/yql/public/decimal/yql_decimal_serialize.cpp b/src/library/yql_common/decimal/yql_decimal_serialize.cpp similarity index 100% rename from src/library/yql/public/decimal/yql_decimal_serialize.cpp rename to src/library/yql_common/decimal/yql_decimal_serialize.cpp diff --git a/src/library/yql/public/decimal/yql_decimal_serialize.h b/src/library/yql_common/decimal/yql_decimal_serialize.h similarity index 78% rename from src/library/yql/public/decimal/yql_decimal_serialize.h rename to src/library/yql_common/decimal/yql_decimal_serialize.h index b21f0babf45..2f540ac1e66 100644 --- a/src/library/yql/public/decimal/yql_decimal_serialize.h +++ b/src/library/yql_common/decimal/yql_decimal_serialize.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "yql_decimal.h" namespace NYql { namespace NDecimal { diff --git a/src/library/yql/public/decimal/yql_wide_int.h b/src/library/yql_common/decimal/yql_wide_int.h similarity index 100% rename from src/library/yql/public/decimal/yql_wide_int.h rename to src/library/yql_common/decimal/yql_wide_int.h diff --git a/src/library/yql/public/issue/CMakeLists.txt b/src/library/yql_common/issue/CMakeLists.txt similarity index 97% rename from src/library/yql/public/issue/CMakeLists.txt rename to src/library/yql_common/issue/CMakeLists.txt index 7fcdfa87876..63ab398c011 100644 --- a/src/library/yql/public/issue/CMakeLists.txt +++ b/src/library/yql_common/issue/CMakeLists.txt @@ -18,6 +18,7 @@ target_sources(yql-public-issue PRIVATE yql_issue_id.cpp yql_issue_message.cpp yql_issue.cpp + out.cpp ) _ydb_sdk_install_targets(TARGETS yql-public-issue) diff --git a/src/library/yql_common/issue/out.cpp b/src/library/yql_common/issue/out.cpp new file mode 100644 index 00000000000..daba5eb0625 --- /dev/null +++ b/src/library/yql_common/issue/out.cpp @@ -0,0 +1,22 @@ +#include + +#include + +#include + +Y_DECLARE_OUT_SPEC(, NYql::NIssue::NProto::IssueMessage, stream, value) { + google::protobuf::TextFormat::Printer printer; + printer.SetSingleLineMode(true); + printer.SetUseUtf8StringEscaping(true); + + NYdb::TStringType str; + printer.PrintToString(value, &str); + + // Copied from text_format.h + // Single line mode currently might have an extra space at the end. + if (str.size() > 0 && str[str.size() - 1] == ' ') { + str.resize(str.size() - 1); + } + + stream << "{ " << str << " }"; +} diff --git a/src/library/yql/public/issue/protos/CMakeLists.txt b/src/library/yql_common/issue/protos/CMakeLists.txt similarity index 100% rename from src/library/yql/public/issue/protos/CMakeLists.txt rename to src/library/yql_common/issue/protos/CMakeLists.txt diff --git a/src/library/yql/public/issue/protos/issue_message.proto b/src/library/yql_common/issue/protos/issue_message.proto similarity index 100% rename from src/library/yql/public/issue/protos/issue_message.proto rename to src/library/yql_common/issue/protos/issue_message.proto diff --git a/src/library/yql/public/issue/protos/issue_severity.proto b/src/library/yql_common/issue/protos/issue_severity.proto similarity index 100% rename from src/library/yql/public/issue/protos/issue_severity.proto rename to src/library/yql_common/issue/protos/issue_severity.proto diff --git a/src/library/yql/public/issue/yql_issue.cpp b/src/library/yql_common/issue/yql_issue.cpp similarity index 98% rename from src/library/yql/public/issue/yql_issue.cpp rename to src/library/yql_common/issue/yql_issue.cpp index 7cfc6ac4afb..de3274fc27b 100644 --- a/src/library/yql/public/issue/yql_issue.cpp +++ b/src/library/yql_common/issue/yql_issue.cpp @@ -1,6 +1,6 @@ -#include +#include -#include +#include #include #include diff --git a/src/library/yql/public/issue/yql_issue_id.cpp b/src/library/yql_common/issue/yql_issue_id.cpp similarity index 82% rename from src/library/yql/public/issue/yql_issue_id.cpp rename to src/library/yql_common/issue/yql_issue_id.cpp index 9cc12eac6e4..9677234e8e8 100644 --- a/src/library/yql/public/issue/yql_issue_id.cpp +++ b/src/library/yql_common/issue/yql_issue_id.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/src/library/yql/public/issue/yql_issue_message.cpp b/src/library/yql_common/issue/yql_issue_message.cpp similarity index 79% rename from src/library/yql/public/issue/yql_issue_message.cpp rename to src/library/yql_common/issue/yql_issue_message.cpp index 22753521ef5..512cd825749 100644 --- a/src/library/yql/public/issue/yql_issue_message.cpp +++ b/src/library/yql_common/issue/yql_issue_message.cpp @@ -1,13 +1,15 @@ #include "yql_issue_message.h" -#include -#include +#include + +#include +#include #include #include #include -#include +#include namespace NYql { @@ -67,14 +69,14 @@ void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) { auto& position = *message.mutable_position(); position.set_row(issue.Position.Row); position.set_column(issue.Position.Column); - position.set_file(issue.Position.File); + position.set_file(NYdb::TStringType{issue.Position.File}); } if (issue.EndPosition) { auto& endPosition = *message.mutable_end_position(); endPosition.set_row(issue.EndPosition.Row); endPosition.set_column(issue.EndPosition.Column); } - message.set_message(issue.GetMessage()); + message.set_message(NYdb::TStringType{issue.GetMessage()}); message.set_issue_code(issue.GetCode()); message.set_severity(issue.GetSeverity()); @@ -123,16 +125,16 @@ NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) { } std::string IssueToBinaryMessage(const TIssue& issue) { - std::string result; + NYdb::TStringType result; Ydb::Issue::IssueMessage protobuf; IssueToMessage(issue, &protobuf); - protobuf.SerializeToString(&result); + Y_UNUSED(protobuf.SerializeToString(&result)); return result; } TIssue IssueFromBinaryMessage(const std::string& binaryMessage) { Ydb::Issue::IssueMessage protobuf; - if (!protobuf.ParseFromString(binaryMessage)) { + if (!protobuf.ParseFromString(NYdb::TStringType{binaryMessage})) { ythrow yexception() << "unable to parse binary string as issue protobuf"; } return IssueFromMessage(protobuf); @@ -140,40 +142,6 @@ TIssue IssueFromBinaryMessage(const std::string& binaryMessage) { } -Y_DECLARE_OUT_SPEC(, Ydb::Issue::IssueMessage, stream, value) { - google::protobuf::TextFormat::Printer printer; - printer.SetSingleLineMode(true); - printer.SetUseUtf8StringEscaping(true); - - std::string str; - printer.PrintToString(value, &str); - - // Copied from text_format.h - // Single line mode currently might have an extra space at the end. - if (str.size() > 0 && str[str.size() - 1] == ' ') { - str.resize(str.size() - 1); - } - - stream << "{ " << str << " }"; -} - -Y_DECLARE_OUT_SPEC(, NYql::NIssue::NProto::IssueMessage, stream, value) { - google::protobuf::TextFormat::Printer printer; - printer.SetSingleLineMode(true); - printer.SetUseUtf8StringEscaping(true); - - std::string str; - printer.PrintToString(value, &str); - - // Copied from text_format.h - // Single line mode currently might have an extra space at the end. - if (str.size() > 0 && str[str.size() - 1] == ' ') { - str.resize(str.size() - 1); - } - - stream << "{ " << str << " }"; -} - Y_DECLARE_OUT_SPEC(, google::protobuf::RepeatedPtrField, stream, issues) { stream << JoinSeq("", issues); } diff --git a/src/library/yql/public/issue/yql_issue_message.h b/src/library/yql_common/issue/yql_issue_message.h similarity index 94% rename from src/library/yql/public/issue/yql_issue_message.h rename to src/library/yql_common/issue/yql_issue_message.h index 7ef7438c054..a8d84015362 100644 --- a/src/library/yql/public/issue/yql_issue_message.h +++ b/src/library/yql_common/issue/yql_issue_message.h @@ -1,6 +1,6 @@ #pragma once -#include +#include #include diff --git a/src/library/yql/utils/CMakeLists.txt b/src/library/yql_common/utils/CMakeLists.txt similarity index 100% rename from src/library/yql/utils/CMakeLists.txt rename to src/library/yql_common/utils/CMakeLists.txt diff --git a/src/library/yql/utils/utf8.cpp b/src/library/yql_common/utils/utf8.cpp similarity index 99% rename from src/library/yql/utils/utf8.cpp rename to src/library/yql_common/utils/utf8.cpp index a884cc4e1d6..d32e9e1b87e 100644 --- a/src/library/yql/utils/utf8.cpp +++ b/src/library/yql_common/utils/utf8.cpp @@ -1,4 +1,4 @@ -#include +#include #include diff --git a/tests/unit/client/CMakeLists.txt b/tests/unit/client/CMakeLists.txt index 2257d18e507..c3206d86685 100644 --- a/tests/unit/client/CMakeLists.txt +++ b/tests/unit/client/CMakeLists.txt @@ -1,8 +1,6 @@ -add_subdirectory(oauth2_token_exchange) - add_ydb_test(NAME client-ydb_coordination_ut SOURCES - coordination_ut.cpp + coordination/coordination_ut.cpp LINK_LIBRARIES yutil cpp-testing-unittest_main @@ -14,7 +12,7 @@ add_ydb_test(NAME client-ydb_coordination_ut add_ydb_test(NAME client-extensions-discovery_mutator_ut SOURCES - discovery_mutator_ut.cpp + discovery_mutator/discovery_mutator_ut.cpp LINK_LIBRARIES yutil cpp-testing-unittest_main @@ -26,7 +24,7 @@ add_ydb_test(NAME client-extensions-discovery_mutator_ut add_ydb_test(NAME client-ydb_driver_ut SOURCES - driver_ut.cpp + driver/driver_ut.cpp LINK_LIBRARIES yutil cpp-testing-unittest_main @@ -40,7 +38,7 @@ add_ydb_test(NAME client-impl-ydb_endpoints_ut INCLUDE_DIRS ${YDB_SDK_SOURCE_DIR}/src/client/impl/ydb_endpoints SOURCES - endpoints_ut.cpp + endpoints/endpoints_ut.cpp LINK_LIBRARIES yutil cpp-testing-unittest_main @@ -49,9 +47,24 @@ add_ydb_test(NAME client-impl-ydb_endpoints_ut unit ) +add_ydb_test(NAME client-oauth2_ut + SOURCES + oauth2_token_exchange/credentials_ut.cpp + oauth2_token_exchange/jwt_token_source_ut.cpp + LINK_LIBRARIES + yutil + cpp-testing-unittest_main + http-server + json + string_utils-base64 + client-ydb_types-credentials-oauth2 + LABELS + unit +) + add_ydb_test(NAME client-ydb_params_ut SOURCES - params_ut.cpp + params/params_ut.cpp LINK_LIBRARIES yutil cpp-testing-unittest_main @@ -63,7 +76,7 @@ add_ydb_test(NAME client-ydb_params_ut add_ydb_test(NAME client-ydb_result_ut SOURCES - result_ut.cpp + result/result_ut.cpp LINK_LIBRARIES yutil cpp-testing-unittest_main @@ -75,7 +88,7 @@ add_ydb_test(NAME client-ydb_result_ut add_ydb_test(NAME client-ydb_value_ut SOURCES - value_ut.cpp + value/value_ut.cpp LINK_LIBRARIES yutil cpp-testing-unittest_main @@ -89,7 +102,7 @@ add_ydb_test(NAME client-ydb_value_ut add_ydb_test(NAME client-draft_ut SOURCES - ydb_scripting_response_headers_ut.cpp + scripting/ydb_scripting_response_headers_ut.cpp LINK_LIBRARIES yutil cpp-testing-unittest_main diff --git a/tests/unit/client/coordination_ut.cpp b/tests/unit/client/coordination/coordination_ut.cpp similarity index 97% rename from tests/unit/client/coordination_ut.cpp rename to tests/unit/client/coordination/coordination_ut.cpp index 6b05c422e26..6867d679838 100644 --- a/tests/unit/client/coordination_ut.cpp +++ b/tests/unit/client/coordination/coordination_ut.cpp @@ -1,7 +1,9 @@ #include -#include -#include +#include + +#include +#include #include #include @@ -100,7 +102,7 @@ namespace { template std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { grpc::ServerBuilder builder; - builder.AddListeningPort(address, grpc::InsecureServerCredentials()); + builder.AddListeningPort(TStringType{address}, grpc::InsecureServerCredentials()); builder.RegisterService(&service); return builder.BuildAndStart(); } diff --git a/tests/unit/client/discovery_mutator_ut.cpp b/tests/unit/client/discovery_mutator/discovery_mutator_ut.cpp similarity index 96% rename from tests/unit/client/discovery_mutator_ut.cpp rename to tests/unit/client/discovery_mutator/discovery_mutator_ut.cpp index 993eb390842..12061641d77 100644 --- a/tests/unit/client/discovery_mutator_ut.cpp +++ b/tests/unit/client/discovery_mutator/discovery_mutator_ut.cpp @@ -1,6 +1,6 @@ #include -#include +#include #include #include diff --git a/tests/unit/client/driver_ut.cpp b/tests/unit/client/driver/driver_ut.cpp similarity index 96% rename from tests/unit/client/driver_ut.cpp rename to tests/unit/client/driver/driver_ut.cpp index 5169564deec..803ed512e01 100644 --- a/tests/unit/client/driver_ut.cpp +++ b/tests/unit/client/driver/driver_ut.cpp @@ -1,8 +1,9 @@ #include #include +#include -#include -#include +#include +#include #include #include @@ -71,7 +72,7 @@ namespace { template std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { grpc::ServerBuilder builder; - builder.AddListeningPort(address, grpc::InsecureServerCredentials()); + builder.AddListeningPort(TStringType{address}, grpc::InsecureServerCredentials()); builder.RegisterService(&service); return builder.BuildAndStart(); } diff --git a/tests/unit/client/endpoints_ut.cpp b/tests/unit/client/endpoints/endpoints_ut.cpp similarity index 99% rename from tests/unit/client/endpoints_ut.cpp rename to tests/unit/client/endpoints/endpoints_ut.cpp index a73a82296e7..da911f04bc8 100644 --- a/tests/unit/client/endpoints_ut.cpp +++ b/tests/unit/client/endpoints/endpoints_ut.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/tests/unit/client/params_ut.cpp b/tests/unit/client/params/params_ut.cpp similarity index 100% rename from tests/unit/client/params_ut.cpp rename to tests/unit/client/params/params_ut.cpp diff --git a/tests/unit/client/result_ut.cpp b/tests/unit/client/result/result_ut.cpp similarity index 93% rename from tests/unit/client/result_ut.cpp rename to tests/unit/client/result/result_ut.cpp index 73307c885b5..5fef90a4832 100644 --- a/tests/unit/client/result_ut.cpp +++ b/tests/unit/client/result/result_ut.cpp @@ -1,7 +1,8 @@ #include - -#include #include +#include + +#include #include #include @@ -50,7 +51,7 @@ Y_UNIT_TEST_SUITE(CppGrpcClientResultSetTest) { " }\n" "}\n"; Ydb::ResultSet rsProto; - google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); + google::protobuf::TextFormat::ParseFromString(TStringType{resultSetString}, &rsProto); NYdb::TResultSet rs(std::move(rsProto)); NYdb::TResultSetParser rsParser(rs); @@ -112,7 +113,7 @@ Y_UNIT_TEST_SUITE(CppGrpcClientResultSetTest) { " }\n" "}\n"; Ydb::ResultSet rsProto; - google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); + google::protobuf::TextFormat::ParseFromString(TStringType{resultSetString}, &rsProto); NYdb::TResultSet rs(std::move(rsProto)); NYdb::TResultSetParser rsParser(rs); @@ -173,7 +174,7 @@ Y_UNIT_TEST_SUITE(CppGrpcClientResultSetTest) { " }\n" "}\n"; Ydb::ResultSet rsProto; - google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); + google::protobuf::TextFormat::ParseFromString(TStringType{resultSetString}, &rsProto); NYdb::TResultSet rs(std::move(rsProto)); NYdb::TResultSetParser rsParser(rs); @@ -224,7 +225,7 @@ Y_UNIT_TEST_SUITE(CppGrpcClientResultSetTest) { " }\n" "}\n"; Ydb::ResultSet rsProto; - google::protobuf::TextFormat::ParseFromString(resultSetString, &rsProto); + google::protobuf::TextFormat::ParseFromString(TStringType{resultSetString}, &rsProto); NYdb::TResultSet rs(std::move(rsProto)); NYdb::TResultSetParser rsParser(rs); diff --git a/tests/unit/client/ydb_scripting_response_headers_ut.cpp b/tests/unit/client/scripting/ydb_scripting_response_headers_ut.cpp similarity index 88% rename from tests/unit/client/ydb_scripting_response_headers_ut.cpp rename to tests/unit/client/scripting/ydb_scripting_response_headers_ut.cpp index aab2958e436..36a2099cb0e 100644 --- a/tests/unit/client/ydb_scripting_response_headers_ut.cpp +++ b/tests/unit/client/scripting/ydb_scripting_response_headers_ut.cpp @@ -1,7 +1,8 @@ #include +#include -#include -#include +#include +#include #include #include @@ -18,7 +19,7 @@ namespace { template std::unique_ptr StartGrpcServer(const std::string& address, TService& service) { grpc::ServerBuilder builder; - builder.AddListeningPort(address, grpc::InsecureServerCredentials()); + builder.AddListeningPort(TStringType{address}, grpc::InsecureServerCredentials()); builder.RegisterService(&service); return builder.BuildAndStart(); } diff --git a/tests/unit/client/value_ut.cpp b/tests/unit/client/value/value_ut.cpp similarity index 99% rename from tests/unit/client/value_ut.cpp rename to tests/unit/client/value/value_ut.cpp index 1f79df590b6..41d26ec3f75 100644 --- a/tests/unit/client/value_ut.cpp +++ b/tests/unit/client/value/value_ut.cpp @@ -1,6 +1,7 @@ #include +#include -#include +#include #include #include #include @@ -781,7 +782,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { " null_flag_value: NULL_VALUE\n" "}\n"; - std::string protoValueStr; + TStringType protoValueStr; google::protobuf::TextFormat::PrintToString(value.GetProto(), &protoValueStr); UNIT_ASSERT_NO_DIFF(protoValueStr, expectedProtoValueStr); } @@ -843,7 +844,7 @@ Y_UNIT_TEST_SUITE(YdbValue) { " }\n" "}\n"; - std::string protoValueStr; + TStringType protoValueStr; google::protobuf::TextFormat::PrintToString(value.GetProto(), &protoValueStr); UNIT_ASSERT_NO_DIFF(protoValueStr, expectedProtoValueStr); } diff --git a/tests/unit/library/CMakeLists.txt b/tests/unit/library/CMakeLists.txt index 1313ac65f6e..434ab841a60 100644 --- a/tests/unit/library/CMakeLists.txt +++ b/tests/unit/library/CMakeLists.txt @@ -1,3 +1,3 @@ add_subdirectory(json_value) add_subdirectory(operation_id) -add_subdirectory(yql) +add_subdirectory(yql_common) diff --git a/tests/unit/library/yql/CMakeLists.txt b/tests/unit/library/yql_common/CMakeLists.txt similarity index 100% rename from tests/unit/library/yql/CMakeLists.txt rename to tests/unit/library/yql_common/CMakeLists.txt diff --git a/tests/unit/library/yql/yql_decimal_ut.cpp b/tests/unit/library/yql_common/decimal/yql_decimal_ut.cpp similarity index 99% rename from tests/unit/library/yql/yql_decimal_ut.cpp rename to tests/unit/library/yql_common/decimal/yql_decimal_ut.cpp index 937e9ff61e5..ec08e5d0553 100644 --- a/tests/unit/library/yql/yql_decimal_ut.cpp +++ b/tests/unit/library/yql_common/decimal/yql_decimal_ut.cpp @@ -1,5 +1,5 @@ -#include -#include +#include +#include #include diff --git a/tests/unit/library/yql/yql_wide_int_ut.cpp b/tests/unit/library/yql_common/decimal/yql_wide_int_ut.cpp similarity index 99% rename from tests/unit/library/yql/yql_wide_int_ut.cpp rename to tests/unit/library/yql_common/decimal/yql_wide_int_ut.cpp index 31a73d09381..9481d4cce3f 100644 --- a/tests/unit/library/yql/yql_wide_int_ut.cpp +++ b/tests/unit/library/yql_common/decimal/yql_wide_int_ut.cpp @@ -1,4 +1,4 @@ -#include +#include #include namespace NYql { diff --git a/tests/unit/library/yql/yql_issue_ut.cpp b/tests/unit/library/yql_common/issue/yql_issue_ut.cpp similarity index 96% rename from tests/unit/library/yql/yql_issue_ut.cpp rename to tests/unit/library/yql_common/issue/yql_issue_ut.cpp index 794e899c7e3..3f078ec75f1 100644 --- a/tests/unit/library/yql/yql_issue_ut.cpp +++ b/tests/unit/library/yql_common/issue/yql_issue_ut.cpp @@ -1,10 +1,12 @@ -#include -#include +#include +#include #include -#include -#include -#include +#include +#include +#include + +#include #include @@ -194,7 +196,7 @@ Y_UNIT_TEST_SUITE(ToMessage) { Ydb::Issue::IssueMessage msg; IssueToMessage(issue, &msg); - std::string serialized; + NYdb::TStringType serialized; UNIT_ASSERT(msg.SerializeToString(&serialized)); Ydb::Issue::IssueMessage msg2; UNIT_ASSERT(msg2.ParseFromString(serialized)); diff --git a/tests/unit/library/yql/utf8_ut.cpp b/tests/unit/library/yql_common/utils/utf8_ut.cpp similarity index 98% rename from tests/unit/library/yql/utf8_ut.cpp rename to tests/unit/library/yql_common/utils/utf8_ut.cpp index 2017821ee5d..d4ce27e6746 100644 --- a/tests/unit/library/yql/utf8_ut.cpp +++ b/tests/unit/library/yql_common/utils/utf8_ut.cpp @@ -1,4 +1,4 @@ -#include +#include #include