From d1dc15b9617e044891cc41f70ee098542787336a Mon Sep 17 00:00:00 2001 From: XmasApple <86735308+XmasApple@users.noreply.github.com> Date: Tue, 3 Oct 2023 16:28:53 +0300 Subject: [PATCH 01/26] Add auth static provider (user:password) (#42) * Add static authorization * Add automatic user creation in tests * Update Ydb.Proto dependency, * Add more testcases, add throw of InvalidCredentialsException * Add more verbosity in tests --- .github/workflows/tests.yml | 2 +- ...alsProvider.cs => ICredentialsProvider.cs} | 0 src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs | 6 + .../src/Auth/StaticCredentialsProvider.cs | 210 ++++++++++++++++++ src/Ydb.Sdk/src/Driver.cs | 7 + src/Ydb.Sdk/src/Services/Auth/AuthClient.cs | 10 + src/Ydb.Sdk/src/Services/Auth/Login.cs | 75 +++++++ src/Ydb.Sdk/src/Ydb.Sdk.csproj | 3 +- src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs | 122 ++++++++++ .../Value.Tests.csproj => Tests.csproj} | 7 +- src/Ydb.Sdk/tests/{Value => }/Utils.cs | 30 ++- src/Ydb.Sdk/tests/Value/TestBasicUnit.cs | 3 +- .../tests/Value/TestBasicsIntegration.cs | 4 +- src/YdbSdk.sln | 12 +- 14 files changed, 474 insertions(+), 17 deletions(-) rename src/Ydb.Sdk/src/Auth/{CredentialsProvider.cs => ICredentialsProvider.cs} (100%) create mode 100644 src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs create mode 100644 src/Ydb.Sdk/src/Auth/StaticCredentialsProvider.cs create mode 100644 src/Ydb.Sdk/src/Services/Auth/AuthClient.cs create mode 100644 src/Ydb.Sdk/src/Services/Auth/Login.cs create mode 100644 src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs rename src/Ydb.Sdk/tests/{Value/Value.Tests.csproj => Tests.csproj} (83%) rename src/Ydb.Sdk/tests/{Value => }/Utils.cs (58%) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 536b34f5..a3e60cb8 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -67,4 +67,4 @@ jobs: - name: Integration test run: | cd src - dotnet test --filter "Category=Integration" -f ${{ matrix.dotnet-target-framework }} + dotnet test --filter "Category=Integration" -f ${{ matrix.dotnet-target-framework }} -l "console;verbosity=normal" diff --git a/src/Ydb.Sdk/src/Auth/CredentialsProvider.cs b/src/Ydb.Sdk/src/Auth/ICredentialsProvider.cs similarity index 100% rename from src/Ydb.Sdk/src/Auth/CredentialsProvider.cs rename to src/Ydb.Sdk/src/Auth/ICredentialsProvider.cs diff --git a/src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs b/src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs new file mode 100644 index 00000000..a7b99d8e --- /dev/null +++ b/src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs @@ -0,0 +1,6 @@ +namespace Ydb.Sdk.Auth; + +public interface IUseDriverConfig +{ + public Task ProvideConfig(DriverConfig driverConfig); +} \ No newline at end of file diff --git a/src/Ydb.Sdk/src/Auth/StaticCredentialsProvider.cs b/src/Ydb.Sdk/src/Auth/StaticCredentialsProvider.cs new file mode 100644 index 00000000..6bfc06e1 --- /dev/null +++ b/src/Ydb.Sdk/src/Auth/StaticCredentialsProvider.cs @@ -0,0 +1,210 @@ +using System.IdentityModel.Tokens.Jwt; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Ydb.Sdk.Services.Auth; + +namespace Ydb.Sdk.Auth; + +public class StaticCredentialsProvider : ICredentialsProvider, IUseDriverConfig +{ + private readonly ILogger _logger; + + private readonly string _user; + private readonly string? _password; + + private Driver? _driver; + + public int MaxRetries = 5; + + private readonly object _lock = new(); + + private volatile TokenData? _token; + private volatile Task? _refreshTask; + + public float RefreshRatio = .1f; + + /// + /// + /// + /// User of the database + /// Password of the user. If user has no password use null + /// + public StaticCredentialsProvider(string user, string? password, ILoggerFactory? loggerFactory = null) + { + _user = user; + _password = password; + loggerFactory ??= NullLoggerFactory.Instance; + _logger = loggerFactory.CreateLogger(); + } + + private async Task Initialize() + { + _token = await ReceiveToken(); + } + + public string GetAuthInfo() + { + var token = _token; + + if (token is null) + { + lock (_lock) + { + if (_token is not null) return _token.Token; + _logger.LogWarning( + "Blocking for initial token acquirement, please use explicit Initialize async method."); + + Initialize().Wait(); + + return _token!.Token; + } + } + + if (token.IsExpired()) + { + lock (_lock) + { + if (!_token!.IsExpired()) return _token.Token; + _logger.LogWarning("Blocking on expired token."); + + _token = ReceiveToken().Result; + + return _token.Token; + } + } + + if (!token.IsRefreshNeeded() || _refreshTask is not null) return _token!.Token; + lock (_lock) + { + if (!_token!.IsRefreshNeeded() || _refreshTask is not null) return _token!.Token; + _logger.LogInformation("Refreshing token."); + + _refreshTask = Task.Run(RefreshToken); + } + + return _token!.Token; + } + + private async Task RefreshToken() + { + var token = await ReceiveToken(); + + lock (_lock) + { + _token = token; + _refreshTask = null; + } + } + + private async Task ReceiveToken() + { + var retryAttempt = 0; + while (true) + { + try + { + _logger.LogTrace($"Attempting to receive token, attempt: {retryAttempt}"); + + var token = await FetchToken(); + + _logger.LogInformation($"Received token, expires at: {token.ExpiresAt}"); + + return token; + } + catch (InvalidCredentialsException e) + { + _logger.LogWarning($"Invalid credentials, {e}"); + throw; + } + catch (Exception e) + { + _logger.LogDebug($"Failed to fetch token, {e}"); + + if (retryAttempt >= MaxRetries) + { + _logger.LogWarning($"Can't fetch token, {e}"); + throw; + } + + await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, retryAttempt))); + _logger.LogInformation($"Failed to fetch token, attempt {retryAttempt}"); + ++retryAttempt; + } + } + } + + private async Task FetchToken() + { + if (_driver is null) + { + _logger.LogError("Driver in for static auth not provided"); + throw new NullReferenceException(); + } + + var client = new AuthClient(_driver); + var loginResponse = await client.Login(_user, _password); + if (loginResponse.Status.StatusCode == StatusCode.Unauthorized) + { + throw new InvalidCredentialsException(Issue.IssuesToString(loginResponse.Status.Issues)); + } + + loginResponse.Status.EnsureSuccess(); + var token = loginResponse.Result.Token; + var jwt = new JwtSecurityToken(token); + return new TokenData(token, jwt.ValidTo, RefreshRatio); + } + + public async Task ProvideConfig(DriverConfig driverConfig) + { + _driver = await Driver.CreateInitialized( + new DriverConfig( + driverConfig.Endpoint, + driverConfig.Database, + new AnonymousProvider(), + driverConfig.DefaultTransportTimeout, + driverConfig.DefaultStreamingTransportTimeout, + driverConfig.CustomServerCertificate)); + + await Initialize(); + } + + private class TokenData + { + public TokenData(string token, DateTime expiresAt, float refreshInterval) + { + var now = DateTime.UtcNow; + + Token = token; + ExpiresAt = expiresAt; + + if (expiresAt <= now) + { + RefreshAt = expiresAt; + } + else + { + RefreshAt = now + (expiresAt - now) * refreshInterval; + + if (RefreshAt < now) + { + RefreshAt = expiresAt; + } + } + } + + public string Token { get; } + public DateTime ExpiresAt { get; } + + private DateTime RefreshAt { get; } + + public bool IsExpired() + { + return DateTime.UtcNow >= ExpiresAt; + } + + public bool IsRefreshNeeded() + { + return DateTime.UtcNow >= RefreshAt; + } + } +} \ No newline at end of file diff --git a/src/Ydb.Sdk/src/Driver.cs b/src/Ydb.Sdk/src/Driver.cs index 59214d1a..f818d30f 100644 --- a/src/Ydb.Sdk/src/Driver.cs +++ b/src/Ydb.Sdk/src/Driver.cs @@ -4,6 +4,7 @@ using Microsoft.Extensions.Logging.Abstractions; using Ydb.Discovery; using Ydb.Discovery.V1; +using Ydb.Sdk.Auth; namespace Ydb.Sdk; @@ -72,6 +73,12 @@ public ValueTask DisposeAsync() public async Task Initialize() { + if (_config.Credentials is IUseDriverConfig useDriverConfig) + { + await useDriverConfig.ProvideConfig(_config); + _logger.LogInformation("DriverConfig provided to IUseDriverConfig interface"); + } + _logger.LogInformation("Started initial endpoint discovery"); try diff --git a/src/Ydb.Sdk/src/Services/Auth/AuthClient.cs b/src/Ydb.Sdk/src/Services/Auth/AuthClient.cs new file mode 100644 index 00000000..f4325648 --- /dev/null +++ b/src/Ydb.Sdk/src/Services/Auth/AuthClient.cs @@ -0,0 +1,10 @@ +using Ydb.Sdk.Client; + +namespace Ydb.Sdk.Services.Auth; + +public partial class AuthClient : ClientBase +{ + public AuthClient(Driver driver) : base(driver) + { + } +} \ No newline at end of file diff --git a/src/Ydb.Sdk/src/Services/Auth/Login.cs b/src/Ydb.Sdk/src/Services/Auth/Login.cs new file mode 100644 index 00000000..c0b5e8f7 --- /dev/null +++ b/src/Ydb.Sdk/src/Services/Auth/Login.cs @@ -0,0 +1,75 @@ +using Ydb.Auth; +using Ydb.Auth.V1; +using Ydb.Sdk.Client; + +namespace Ydb.Sdk.Services.Auth; + +public class LoginSettings : OperationRequestSettings +{ +} + +public class LoginResponse : ResponseWithResultBase +{ + internal LoginResponse(Status status, ResultData? result = null) + : base(status, result) + { + } + + public class ResultData + { + public string Token { get; } + + internal ResultData(string token) + { + Token = token; + } + + + internal static ResultData FromProto(LoginResult resultProto) + { + var token = resultProto.Token; + return new ResultData(token); + } + } +} + +public partial class AuthClient +{ + public async Task Login(string user, string? password, LoginSettings? settings = null) + { + settings ??= new LoginSettings(); + var request = new LoginRequest + { + OperationParams = MakeOperationParams(settings), + User = user + }; + if (password is not null) + { + request.Password = password; + } + + try + { + var response = await Driver.UnaryCall( + method: AuthService.LoginMethod, + request: request, + settings: settings + ); + + var status = UnpackOperation(response.Data.Operation, out LoginResult? resultProto); + + LoginResponse.ResultData? result = null; + + if (status.IsSuccess && resultProto is not null) + { + result = LoginResponse.ResultData.FromProto(resultProto); + } + + return new LoginResponse(status, result); + } + catch (Driver.TransportException e) + { + return new LoginResponse(e.Status); + } + } +} \ No newline at end of file diff --git a/src/Ydb.Sdk/src/Ydb.Sdk.csproj b/src/Ydb.Sdk/src/Ydb.Sdk.csproj index d1980b7d..99dec4b8 100644 --- a/src/Ydb.Sdk/src/Ydb.Sdk.csproj +++ b/src/Ydb.Sdk/src/Ydb.Sdk.csproj @@ -26,8 +26,9 @@ - + + diff --git a/src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs b/src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs new file mode 100644 index 00000000..cb203da3 --- /dev/null +++ b/src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs @@ -0,0 +1,122 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; +using Xunit.Abstractions; +using Ydb.Sdk.Auth; +using Ydb.Sdk.Services.Table; + +namespace Ydb.Sdk.Tests.Auth; + +[Trait("Category", "Integration")] +public class TestStaticAuth : IDisposable +{ + // ReSharper disable once NotAccessedField.Local + private readonly ITestOutputHelper _output; + private readonly ILoggerFactory? _loggerFactory; + private readonly ILogger _logger; + + public TestStaticAuth(ITestOutputHelper output) + { + _output = output; + _loggerFactory = Utils.GetLoggerFactory() ?? NullLoggerFactory.Instance; + _logger = _loggerFactory.CreateLogger(); + } + + public void Dispose() + { + GC.SuppressFinalize(this); + } + + private async Task CreateUser(TableClient tableClient, string user, string? password) + { + var query = password is null ? $"CREATE USER {user}" : $"CREATE USER {user} PASSWORD '{password}'"; + + await Utils.ExecuteSchemeQuery(tableClient, $"{query}"); + _logger.LogInformation($"User {user} created successfully"); + } + + private async Task DropUser(TableClient tableClient, string user) + { + await Utils.ExecuteSchemeQuery(tableClient, $"DROP USER {user}"); + _logger.LogInformation($"User {user} dropped successfully"); + } + + private async Task Authorize(string user, string? password, int maxRetries) + { + var driverConfig = new DriverConfig( + endpoint: "grpc://localhost:2136", + database: "/local", + new StaticCredentialsProvider(user, password, _loggerFactory) { MaxRetries = maxRetries } + ); + + _logger.LogInformation($"DriverConfig for {user} created"); + + await using var driver = await Driver.CreateInitialized(driverConfig, _loggerFactory); + _logger.LogInformation($"Driver for {user} created and initialized"); + using var tableClient = new TableClient(driver); + + var response = await Utils.ExecuteDataQuery(tableClient, "SELECT 1"); + var row = response.Result.ResultSets[0].Rows[0]; + Assert.Equal(1, row[0].GetInt32()); + } + + private async Task CheckAuth(string? passwordCreate, string? passwordAuth, int maxRetries = 5) + { + _logger.LogInformation("Creating anon driver"); + var anonDriverConfig = new DriverConfig( + endpoint: "grpc://localhost:2136", + database: "/local" + ); + + await using var anonDriver = await Driver.CreateInitialized(anonDriverConfig); + _logger.LogInformation("Anon driver created"); + + using var anonTableClient = new TableClient(anonDriver); + + var user = $"test{Guid.NewGuid():n}"; + + await CreateUser(anonTableClient, user, passwordCreate); + + try + { + await Authorize(user, passwordAuth, maxRetries); + } + finally + { + await DropUser(anonTableClient, user); + } + } + + + [Fact] + public async Task GoodAuth() + { + await CheckAuth("test_password", "test_password"); + } + + [Fact] + public async Task NoPasswordAuth() + { + await CheckAuth(null, null); + } + + [Fact] + public async Task WrongPassword() + { + await Assert.ThrowsAsync( + async () => await CheckAuth("good_password", "wrong_password", maxRetries: 1)); + } + + [Fact] + public async Task NotExistAuth() + { + var driverConfig = new DriverConfig( + endpoint: "grpc://localhost:2136", + database: "/local", + new StaticCredentialsProvider("notexists", "nopass", _loggerFactory) { MaxRetries = 1 } + ); + + await Assert.ThrowsAsync( + async () => await Driver.CreateInitialized(driverConfig, _loggerFactory)); + } +} \ No newline at end of file diff --git a/src/Ydb.Sdk/tests/Value/Value.Tests.csproj b/src/Ydb.Sdk/tests/Tests.csproj similarity index 83% rename from src/Ydb.Sdk/tests/Value/Value.Tests.csproj rename to src/Ydb.Sdk/tests/Tests.csproj index 6c6bdab0..f238f217 100644 --- a/src/Ydb.Sdk/tests/Value/Value.Tests.csproj +++ b/src/Ydb.Sdk/tests/Tests.csproj @@ -3,8 +3,8 @@ net6.0;net7.0; enable - Ydb.Sdk.Value.Tests - Ydb.Sdk.Value.Tests + Ydb.Sdk.Tests + Ydb.Sdk.Tests false enable @@ -16,6 +16,7 @@ + @@ -35,7 +36,7 @@ - + diff --git a/src/Ydb.Sdk/tests/Value/Utils.cs b/src/Ydb.Sdk/tests/Utils.cs similarity index 58% rename from src/Ydb.Sdk/tests/Value/Utils.cs rename to src/Ydb.Sdk/tests/Utils.cs index 3ead6dfc..dacd9438 100644 --- a/src/Ydb.Sdk/tests/Value/Utils.cs +++ b/src/Ydb.Sdk/tests/Utils.cs @@ -1,8 +1,11 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Ydb.Sdk.Services.Table; +using Ydb.Sdk.Value; -namespace Ydb.Sdk.Value.Tests; +namespace Ydb.Sdk.Tests; -public class Utils +public static class Utils { public static async Task ExecuteDataQuery(TableClient tableClient, string query, Dictionary? parameters = null) @@ -22,13 +25,32 @@ public static async Task ExecuteDataQuery(TableClient return (ExecuteDataQueryResponse)response; } - public static async Task ExecuteSchemeQuery(TableClient tableClient, string query) + public static async Task ExecuteSchemeQuery( + TableClient tableClient, string query, bool ensureSuccess = true) { var response = await tableClient.SessionExec( async session => await session.ExecuteSchemeQuery(query: query)); - response.Status.EnsureSuccess(); + if (ensureSuccess) + { + response.Status.EnsureSuccess(); + } + return (ExecuteSchemeQueryResponse)response; } + + + internal static ServiceProvider GetServiceProvider() + { + return new ServiceCollection() + .AddLogging(configure => configure.AddConsole().SetMinimumLevel(LogLevel.Information)) + .BuildServiceProvider(); + } + + internal static ILoggerFactory? GetLoggerFactory() + { + var serviceProvider = GetServiceProvider(); + return serviceProvider.GetService(); + } } \ No newline at end of file diff --git a/src/Ydb.Sdk/tests/Value/TestBasicUnit.cs b/src/Ydb.Sdk/tests/Value/TestBasicUnit.cs index 6afa0e7d..e101b395 100644 --- a/src/Ydb.Sdk/tests/Value/TestBasicUnit.cs +++ b/src/Ydb.Sdk/tests/Value/TestBasicUnit.cs @@ -1,8 +1,9 @@ using System.Text; using Xunit; using Xunit.Abstractions; +using Ydb.Sdk.Value; -namespace Ydb.Sdk.Value.Tests; +namespace Ydb.Sdk.Tests.Value; [Trait("Category", "Unit")] public class TestBasicUnit diff --git a/src/Ydb.Sdk/tests/Value/TestBasicsIntegration.cs b/src/Ydb.Sdk/tests/Value/TestBasicsIntegration.cs index 8009d521..4b6bba90 100644 --- a/src/Ydb.Sdk/tests/Value/TestBasicsIntegration.cs +++ b/src/Ydb.Sdk/tests/Value/TestBasicsIntegration.cs @@ -2,8 +2,9 @@ using Xunit; using Xunit.Abstractions; using Ydb.Sdk.Services.Table; +using Ydb.Sdk.Value; -namespace Ydb.Sdk.Value.Tests; +namespace Ydb.Sdk.Tests.Value; [Trait("Category", "Integration")] public class TestBasicIntegration : IDisposable @@ -30,6 +31,7 @@ public TestBasicIntegration(ITestOutputHelper output) public void Dispose() { + _tableClient.Dispose(); _driver.Dispose(); GC.SuppressFinalize(this); } diff --git a/src/YdbSdk.sln b/src/YdbSdk.sln index 3248b084..718f2ba3 100644 --- a/src/YdbSdk.sln +++ b/src/YdbSdk.sln @@ -11,7 +11,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{316B82EF EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Ydb.Sdk", "Ydb.Sdk\src\Ydb.Sdk.csproj", "{C91FA8B1-713B-40F4-B07A-EB6CD4106392}" EndProject -Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Value.Tests", "Ydb.Sdk\tests\Value\Value.Tests.csproj", "{37345AE2-D477-4998-9123-84C51290672A}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Ydb.Sdk\tests\Tests.csproj", "{A27FD249-6ACB-4392-B00F-CD08FB727C98}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -23,10 +23,10 @@ Global {C91FA8B1-713B-40F4-B07A-EB6CD4106392}.Debug|Any CPU.Build.0 = Debug|Any CPU {C91FA8B1-713B-40F4-B07A-EB6CD4106392}.Release|Any CPU.ActiveCfg = Release|Any CPU {C91FA8B1-713B-40F4-B07A-EB6CD4106392}.Release|Any CPU.Build.0 = Release|Any CPU - {37345AE2-D477-4998-9123-84C51290672A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {37345AE2-D477-4998-9123-84C51290672A}.Debug|Any CPU.Build.0 = Debug|Any CPU - {37345AE2-D477-4998-9123-84C51290672A}.Release|Any CPU.ActiveCfg = Release|Any CPU - {37345AE2-D477-4998-9123-84C51290672A}.Release|Any CPU.Build.0 = Release|Any CPU + {A27FD249-6ACB-4392-B00F-CD08FB727C98}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {A27FD249-6ACB-4392-B00F-CD08FB727C98}.Debug|Any CPU.Build.0 = Debug|Any CPU + {A27FD249-6ACB-4392-B00F-CD08FB727C98}.Release|Any CPU.ActiveCfg = Release|Any CPU + {A27FD249-6ACB-4392-B00F-CD08FB727C98}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -35,7 +35,7 @@ Global {E21B559D-5E8D-47AE-950E-03435F3066DF} = {34D81B90-76BA-430B-B3B1-B830B7206134} {316B82EF-019D-4267-95A9-5E243086B240} = {34D81B90-76BA-430B-B3B1-B830B7206134} {C91FA8B1-713B-40F4-B07A-EB6CD4106392} = {E21B559D-5E8D-47AE-950E-03435F3066DF} - {37345AE2-D477-4998-9123-84C51290672A} = {316B82EF-019D-4267-95A9-5E243086B240} + {A27FD249-6ACB-4392-B00F-CD08FB727C98} = {316B82EF-019D-4267-95A9-5E243086B240} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {0AB27123-0C66-4E43-A75F-D9EAB9ED0849} From 3eb0bc008bea52d658951841978eb8c68c022303 Mon Sep 17 00:00:00 2001 From: XmasApple <86735308+XmasApple@users.noreply.github.com> Date: Sun, 8 Oct 2023 17:45:27 +0300 Subject: [PATCH 02/26] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3746ab9..fad8adde 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +- Add static auth ## v0.1.1 - Add static code analysis - Add CodeQL analysis From a244fc2b5dd6cedfa30a99d8cf477699aa4d5907 Mon Sep 17 00:00:00 2001 From: XmasApple Date: Sun, 8 Oct 2023 18:05:56 +0300 Subject: [PATCH 03/26] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 293393ca..11cee09a 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ await driver.Initialize(); // Make sure to await driver initialization YDB SDK provides two standard ways for authentication: 1) `Ydb.Sdk.Auth.AnonymousProvider`. Anonymous YDB access, mainly for tests purposes. 2) `Ydb.Sdk.Auth.TokenProvider`. Token authentication for OAuth-like tokens. +3) `Ydb.Sdk.Auth.StaticCredentialsProvider`. Username and password based authentication. For Yandex.Cloud specific authentication methods, consider using **[ydb-dotnet-yc](https://github.com/ydb-platform/ydb-dotnet-yc)**. From 3ab962fd777e0efc8eafab2e43ecaddfca08096f Mon Sep 17 00:00:00 2001 From: XmasApple Date: Sun, 8 Oct 2023 18:10:21 +0300 Subject: [PATCH 04/26] Fastfix --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 11cee09a..6fb188e0 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ await driver.Initialize(); // Make sure to await driver initialization ``` ### Credentials -YDB SDK provides two standard ways for authentication: +YDB SDK provides several standard ways for authentication: 1) `Ydb.Sdk.Auth.AnonymousProvider`. Anonymous YDB access, mainly for tests purposes. 2) `Ydb.Sdk.Auth.TokenProvider`. Token authentication for OAuth-like tokens. 3) `Ydb.Sdk.Auth.StaticCredentialsProvider`. Username and password based authentication. From b79aa35bbef29d6aa348ac5b12360188c8235935 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Fri, 13 Oct 2023 09:21:44 +0300 Subject: [PATCH 05/26] Update Version.cs --- src/Ydb.Sdk/src/Version.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Ydb.Sdk/src/Version.cs b/src/Ydb.Sdk/src/Version.cs index e057895b..93a1d4ea 100644 --- a/src/Ydb.Sdk/src/Version.cs +++ b/src/Ydb.Sdk/src/Version.cs @@ -4,5 +4,5 @@ public static class Verison { public const uint Major = 0; public const uint Minor = 1; - public const uint Patch = 1; -} \ No newline at end of file + public const uint Patch = 2; +} From d12dc91668e72326bdb8f299f3cd5ea11b5c96c7 Mon Sep 17 00:00:00 2001 From: robot Date: Fri, 13 Oct 2023 06:24:27 +0000 Subject: [PATCH 06/26] Release v0.1.3 --- CHANGELOG.md | 1 + src/Ydb.Sdk/src/Version.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fad8adde..82d45e5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +## v0.1.3 - Add static auth ## v0.1.1 - Add static code analysis diff --git a/src/Ydb.Sdk/src/Version.cs b/src/Ydb.Sdk/src/Version.cs index 93a1d4ea..55186568 100644 --- a/src/Ydb.Sdk/src/Version.cs +++ b/src/Ydb.Sdk/src/Version.cs @@ -4,5 +4,5 @@ public static class Verison { public const uint Major = 0; public const uint Minor = 1; - public const uint Patch = 2; + public const uint Patch = 3; } From cc6f977e32222e1d6e7c5b3546664db8f89cf75f Mon Sep 17 00:00:00 2001 From: XmasApple Date: Mon, 16 Oct 2023 18:58:10 +0300 Subject: [PATCH 07/26] Add examples --- examples/README.md | 35 +++ examples/src/BasicExample/BasicExample.cs | 81 +++++++ examples/src/BasicExample/BasicExample.csproj | 28 +++ examples/src/BasicExample/DataQuery.cs | 88 ++++++++ examples/src/BasicExample/FillData.cs | 201 ++++++++++++++++++ examples/src/BasicExample/InteractiveTx.cs | 75 +++++++ examples/src/BasicExample/Program.cs | 59 +++++ examples/src/BasicExample/ReadTable.cs | 35 +++ examples/src/BasicExample/ScanQuery.cs | 48 +++++ examples/src/BasicExample/SchemeQuery.cs | 43 ++++ examples/src/Common/AuthUtils.cs | 67 ++++++ examples/src/Common/Common.csproj | 23 ++ examples/src/Common/TableExampleBase.cs | 20 ++ examples/src/YdbExamples.sln | 31 +++ examples/src/YdbExamples.sln.DotSettings | 69 ++++++ 15 files changed, 903 insertions(+) create mode 100644 examples/README.md create mode 100644 examples/src/BasicExample/BasicExample.cs create mode 100644 examples/src/BasicExample/BasicExample.csproj create mode 100644 examples/src/BasicExample/DataQuery.cs create mode 100644 examples/src/BasicExample/FillData.cs create mode 100644 examples/src/BasicExample/InteractiveTx.cs create mode 100644 examples/src/BasicExample/Program.cs create mode 100644 examples/src/BasicExample/ReadTable.cs create mode 100644 examples/src/BasicExample/ScanQuery.cs create mode 100644 examples/src/BasicExample/SchemeQuery.cs create mode 100644 examples/src/Common/AuthUtils.cs create mode 100644 examples/src/Common/Common.csproj create mode 100644 examples/src/Common/TableExampleBase.cs create mode 100644 examples/src/YdbExamples.sln create mode 100644 examples/src/YdbExamples.sln.DotSettings diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..b1ff27e6 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,35 @@ +# YDB .NET SDK Examples + +## Prerequisites +.NET 6 + +## Running examples + +1. Clone repository + ```bash + git clone https://github.com/ydb-platform/ydb-dotnet-sdk.git + ``` + +2. Build solution + ```bash + cd ydb-dotnet-sdk/examples/src + dotnet build + ``` + +3. Run example + ```bash + cd + dotnet run -e -d + ``` + +## Provided examples + +### BasicExample +Demonstrates basic operations with YDB, including: +* Driver initialization +* Table client initialization +* Table creation via SchemeQuery (DDL) +* Data queries (OLTP) & transactions (read, modify) +* Interactive transactions +* ReadTable for streaming read of table contents +* ScanQuery for streaming wide queries (OLAP) diff --git a/examples/src/BasicExample/BasicExample.cs b/examples/src/BasicExample/BasicExample.cs new file mode 100644 index 00000000..4bb2607d --- /dev/null +++ b/examples/src/BasicExample/BasicExample.cs @@ -0,0 +1,81 @@ +using System; +using System.Security.Cryptography.X509Certificates; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Ydb.Sdk.Auth; +using Ydb.Sdk.Services.Table; + +namespace Ydb.Sdk.Examples; + +internal partial class BasicExample : TableExampleBase +{ + private BasicExample(TableClient client, string database, string path) + : base(client, database, path) + { + } + + public static async Task Run( + string endpoint, + string database, + ICredentialsProvider credentialsProvider, + X509Certificate? customServerCertificate, + string path, + ILoggerFactory loggerFactory) + { + var config = new DriverConfig( + endpoint: endpoint, + database: database, + credentials: credentialsProvider, + customServerCertificate: customServerCertificate + ); + + await using var driver = await Driver.CreateInitialized( + config: config, + loggerFactory: loggerFactory + ); + + using var tableClient = new TableClient(driver, new TableClientConfig()); + + var example = new BasicExample(tableClient, database, path); + + await example.SchemeQuery(); + await example.FillData(); + await example.SimpleSelect(1); + await example.SimpleUpsert(10, "Coming soon", DateTime.UtcNow); + await example.SimpleSelect(10); + await example.InteractiveTx(); + await example.ReadTable(); + await example.ScanQuery(DateTime.Parse("2007-01-01")); + } + + private static ExecuteDataQuerySettings DefaultDataQuerySettings => + new() + { + // Indicates that client is no longer interested in the result of operation after the + // specified duration starting from the moment when operation arrives at the server. + // Status code TIMEOUT will be returned from server in case when operation result in + // not available in the specified time period. This status code doesn't indicate the result + // of operation, it might be completed or cancelled. + OperationTimeout = TimeSpan.FromSeconds(1), + + // Transport timeout from the moment operation was sent to server. It is useful in case + // of possible network issues, to that query doesn't hang forever. + // It is recommended to set this value to a larger value than OperationTimeout to give + // server some time to issue a response. + TransportTimeout = TimeSpan.FromSeconds(5), + + // Keep query compilation result in query cache or not. Should be false for ad-hoc queries, + // and true (default) for high-RPS queries. + KeepInQueryCache = false + }; + + private ExecuteDataQuerySettings DefaultCachedDataQuerySettings + { + get + { + var settings = DefaultDataQuerySettings; + settings.KeepInQueryCache = true; + return settings; + } + } +} \ No newline at end of file diff --git a/examples/src/BasicExample/BasicExample.csproj b/examples/src/BasicExample/BasicExample.csproj new file mode 100644 index 00000000..bc6690be --- /dev/null +++ b/examples/src/BasicExample/BasicExample.csproj @@ -0,0 +1,28 @@ + + + + + Exe + net6.0 + enable + Ydb.Sdk.Examples.BasicExample + Ydb.Sdk.Examples + + + + git + https://github.com/ydb-platform/ydb-dotnet-examples + https://github.com/ydb-platform/ydb-dotnet-examples + YANDEX LLC + + + + + + + + + + + + diff --git a/examples/src/BasicExample/DataQuery.cs b/examples/src/BasicExample/DataQuery.cs new file mode 100644 index 00000000..f27406ac --- /dev/null +++ b/examples/src/BasicExample/DataQuery.cs @@ -0,0 +1,88 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Ydb.Sdk.Services.Table; +using Ydb.Sdk.Value; + +namespace Ydb.Sdk.Examples; + +internal partial class BasicExample +{ + private async Task SimpleSelect(ulong id) + { + var response = await Client.SessionExec(async session => + { + var query = @$" + PRAGMA TablePathPrefix('{BasePath}'); + + DECLARE $id AS Uint64; + + SELECT + series_id, + title, + release_date + FROM series + WHERE series_id = $id; + "; + + return await session.ExecuteDataQuery( + query: query, + txControl: TxControl.BeginSerializableRW().Commit(), + parameters: new Dictionary + { + { "$id", YdbValue.MakeUint64(id) } + }, + settings: DefaultCachedDataQuerySettings + ); + }); + + response.Status.EnsureSuccess(); + + var queryResponse = (ExecuteDataQueryResponse)response; + var resultSet = queryResponse.Result.ResultSets[0]; + + Console.WriteLine($"> SimpleSelect, " + + $"columns: {resultSet.Columns.Count}, " + + $"rows: {resultSet.Rows.Count}, " + + $"truncated: {resultSet.Truncated}"); + + foreach (var row in resultSet.Rows) + { + Console.WriteLine($"> Series, " + + $"series_id: {(ulong?)row["series_id"]}, " + + $"title: {(string?)row["title"]}, " + + $"release_date: {(DateTime?)row["release_date"]}"); + } + } + + private async Task SimpleUpsert(ulong id, string title, DateTime date) + { + var response = await Client.SessionExec(async session => + { + var query = @$" + PRAGMA TablePathPrefix('{BasePath}'); + + DECLARE $id AS Uint64; + DECLARE $title AS Utf8; + DECLARE $release_date AS Date; + + UPSERT INTO series (series_id, title, release_date) VALUES + ($id, $title, $release_date); + "; + + return await session.ExecuteDataQuery( + query: query, + txControl: TxControl.BeginSerializableRW().Commit(), + parameters: new Dictionary + { + { "$id", YdbValue.MakeUint64(id) }, + { "$title", YdbValue.MakeUtf8(title) }, + { "$release_date", YdbValue.MakeDate(date) } + }, + settings: DefaultCachedDataQuerySettings + ); + }); + + response.Status.EnsureSuccess(); + } +} \ No newline at end of file diff --git a/examples/src/BasicExample/FillData.cs b/examples/src/BasicExample/FillData.cs new file mode 100644 index 00000000..75949fe5 --- /dev/null +++ b/examples/src/BasicExample/FillData.cs @@ -0,0 +1,201 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Ydb.Sdk.Services.Table; +using Ydb.Sdk.Value; + +namespace Ydb.Sdk.Examples; + +internal partial class BasicExample +{ + // Fill sample tables with initial data. + private async Task FillData() + { + var response = await Client.SessionExec(async session => + { + var query = @$" + PRAGMA TablePathPrefix('{BasePath}'); + + DECLARE $seriesData AS List>; + + DECLARE $seasonsData AS List>; + + DECLARE $episodesData AS List>; + + REPLACE INTO series + SELECT * FROM AS_TABLE($seriesData); + + REPLACE INTO seasons + SELECT * FROM AS_TABLE($seasonsData); + + REPLACE INTO episodes + SELECT * FROM AS_TABLE($episodesData); + "; + + return await session.ExecuteDataQuery( + query: query, + txControl: TxControl.BeginSerializableRW().Commit(), + parameters: GetDataParams(), + settings: DefaultDataQuerySettings + ); + }); + + response.Status.EnsureSuccess(); + } + + internal record Series(int SeriesId, string Title, DateTime ReleaseDate, string Info); + + internal record Season(int SeriesId, int SeasonId, string Title, DateTime FirstAired, DateTime LastAired); + + internal record Episode(int SeriesId, int SeasonId, int EpisodeId, string Title, DateTime AirDate); + + private static Dictionary GetDataParams() + { + var series = new Series[] + { + new(SeriesId: 1, Title: "IT Crowd", ReleaseDate: DateTime.Parse("2006-02-03"), + Info: "The IT Crowd is a British sitcom produced by Channel 4, written by Graham Linehan, " + + "produced by Ash Atalla and starring Chris O'Dowd, Richard Ayoade, Katherine Parkinson, " + + "and Matt Berry."), + new(SeriesId: 2, Title: "Silicon Valley", ReleaseDate: DateTime.Parse("2014-04-06"), + Info: "Silicon Valley is an American comedy television series created by Mike Judge, " + + "John Altschuler and Dave Krinsky. The series focuses on five young men who founded " + + "a startup company in Silicon Valley.") + }; + + var seasons = new Season[] + { + new(1, 1, "Season 1", DateTime.Parse("2006-02-03"), DateTime.Parse("2006-03-03")), + new(1, 2, "Season 2", DateTime.Parse("2007-08-24"), DateTime.Parse("2007-09-28")), + new(1, 3, "Season 3", DateTime.Parse("2008-11-21"), DateTime.Parse("2008-12-26")), + new(1, 4, "Season 4", DateTime.Parse("2010-06-25"), DateTime.Parse("2010-07-30")), + new(2, 1, "Season 1", DateTime.Parse("2014-04-06"), DateTime.Parse("2014-06-01")), + new(2, 2, "Season 2", DateTime.Parse("2015-04-12"), DateTime.Parse("2015-06-14")), + new(2, 3, "Season 3", DateTime.Parse("2016-04-24"), DateTime.Parse("2016-06-26")), + new(2, 4, "Season 4", DateTime.Parse("2017-04-23"), DateTime.Parse("2017-06-25")), + new(2, 5, "Season 5", DateTime.Parse("2018-03-25"), DateTime.Parse("2018-05-13")), + }; + + var episodes = new Episode[] + { + new(1, 1, 1, "Yesterday's Jam", DateTime.Parse("2006-02-03")), + new(1, 1, 2, "Calamity Jen", DateTime.Parse("2006-02-03")), + new(1, 1, 3, "Fifty-Fifty", DateTime.Parse("2006-02-10")), + new(1, 1, 4, "The Red Door", DateTime.Parse("2006-02-17")), + new(1, 1, 5, "The Haunting of Bill Crouse", DateTime.Parse("2006-02-24")), + new(1, 1, 6, "Aunt Irma Visits", DateTime.Parse("2006-03-03")), + new(1, 2, 1, "The Work Outing", DateTime.Parse("2006-08-24")), + new(1, 2, 2, "Return of the Golden Child", DateTime.Parse("2007-08-31")), + new(1, 2, 3, "Moss and the German", DateTime.Parse("2007-09-07")), + new(1, 2, 4, "The Dinner Party", DateTime.Parse("2007-09-14")), + new(1, 2, 5, "Smoke and Mirrors", DateTime.Parse("2007-09-21")), + new(1, 2, 6, "Men Without Women", DateTime.Parse("2007-09-28")), + new(1, 3, 1, "From Hell", DateTime.Parse("2008-11-21")), + new(1, 3, 2, "Are We Not Men?", DateTime.Parse("2008-11-28")), + new(1, 3, 3, "Tramps Like Us", DateTime.Parse("2008-12-05")), + new(1, 3, 4, "The Speech", DateTime.Parse("2008-12-12")), + new(1, 3, 5, "Friendface", DateTime.Parse("2008-12-19")), + new(1, 3, 6, "Calendar Geeks", DateTime.Parse("2008-12-26")), + new(1, 4, 1, "Jen The Fredo", DateTime.Parse("2010-06-25")), + new(1, 4, 2, "The Final Countdown", DateTime.Parse("2010-07-02")), + new(1, 4, 3, "Something Happened", DateTime.Parse("2010-07-09")), + new(1, 4, 4, "Italian For Beginners", DateTime.Parse("2010-07-16")), + new(1, 4, 5, "Bad Boys", DateTime.Parse("2010-07-23")), + new(1, 4, 6, "Reynholm vs Reynholm", DateTime.Parse("2010-07-30")), + new(2, 1, 1, "Minimum Viable Product", DateTime.Parse("2014-04-06")), + new(2, 1, 2, "The Cap Table", DateTime.Parse("2014-04-13")), + new(2, 1, 3, "Articles of Incorporation", DateTime.Parse("2014-04-20")), + new(2, 1, 4, "Fiduciary Duties", DateTime.Parse("2014-04-27")), + new(2, 1, 5, "Signaling Risk", DateTime.Parse("2014-05-04")), + new(2, 1, 6, "Third Party Insourcing", DateTime.Parse("2014-05-11")), + new(2, 1, 7, "Proof of Concept", DateTime.Parse("2014-05-18")), + new(2, 1, 8, "Optimal Tip-to-Tip Efficiency", DateTime.Parse("2014-06-01")), + new(2, 2, 1, "Sand Hill Shuffle", DateTime.Parse("2015-04-12")), + new(2, 2, 2, "Runaway Devaluation", DateTime.Parse("2015-04-19")), + new(2, 2, 3, "Bad Money", DateTime.Parse("2015-04-26")), + new(2, 2, 4, "The Lady", DateTime.Parse("2015-05-03")), + new(2, 2, 5, "Server Space", DateTime.Parse("2015-05-10")), + new(2, 2, 6, "Homicide", DateTime.Parse("2015-05-17")), + new(2, 2, 7, "Adult Content", DateTime.Parse("2015-05-24")), + new(2, 2, 8, "White Hat/Black Hat", DateTime.Parse("2015-05-31")), + new(2, 2, 9, "Binding Arbitration", DateTime.Parse("2015-06-07")), + new(2, 2, 10, "Two Days of the Condor", DateTime.Parse("2015-06-14")), + new(2, 3, 1, "Founder Friendly", DateTime.Parse("2016-04-24")), + new(2, 3, 2, "Two in the Box", DateTime.Parse("2016-05-01")), + new(2, 3, 3, "Meinertzhagen's Haversack", DateTime.Parse("2016-05-08")), + new(2, 3, 4, "Maleant Data Systems Solutions", DateTime.Parse("2016-05-15")), + new(2, 3, 5, "The Empty Chair", DateTime.Parse("2016-05-22")), + new(2, 3, 6, "Bachmanity Insanity", DateTime.Parse("2016-05-29")), + new(2, 3, 7, "To Build a Better Beta", DateTime.Parse("2016-06-05")), + new(2, 3, 8, "Bachman's Earnings Over-Ride", DateTime.Parse("2016-06-12")), + new(2, 3, 9, "Daily Active Users", DateTime.Parse("2016-06-19")), + new(2, 3, 10, "The Uptick", DateTime.Parse("2016-06-26")), + new(2, 4, 1, "Success Failure", DateTime.Parse("2017-04-23")), + new(2, 4, 2, "Terms of Service", DateTime.Parse("2017-04-30")), + new(2, 4, 3, "Intellectual Property", DateTime.Parse("2017-05-07")), + new(2, 4, 4, "Teambuilding Exercise", DateTime.Parse("2017-05-14")), + new(2, 4, 5, "The Blood Boy", DateTime.Parse("2017-05-21")), + new(2, 4, 6, "Customer Service", DateTime.Parse("2017-05-28")), + new(2, 4, 7, "The Patent Troll", DateTime.Parse("2017-06-04")), + new(2, 4, 8, "The Keenan Vortex", DateTime.Parse("2017-06-11")), + new(2, 4, 9, "Hooli-Con", DateTime.Parse("2017-06-18")), + new(2, 4, 10, "Server Error", DateTime.Parse("2017-06-25")), + new(2, 5, 1, "Grow Fast or Die Slow", DateTime.Parse("2018-03-25")), + new(2, 5, 2, "Reorientation", DateTime.Parse("2018-04-01")), + new(2, 5, 3, "Chief Operating Officer", DateTime.Parse("2018-04-08")), + new(2, 5, 4, "Tech Evangelist", DateTime.Parse("2018-04-15")), + new(2, 5, 5, "Facial Recognition", DateTime.Parse("2018-04-22")), + new(2, 5, 6, "Artificial Emotional Intelligence", DateTime.Parse("2018-04-29")), + new(2, 5, 7, "Initial Coin Offering", DateTime.Parse("2018-05-06")), + new(2, 5, 8, "Fifty-One Percent", DateTime.Parse("2018-05-13")), + }; + + var seriesData = series.Select(s => YdbValue.MakeStruct(new Dictionary + { + { "series_id", YdbValue.MakeUint64((ulong)s.SeriesId) }, + { "title", YdbValue.MakeUtf8(s.Title) }, + { "series_info", YdbValue.MakeUtf8(s.Info) }, + { "release_date", YdbValue.MakeDate(s.ReleaseDate) } + })).ToList(); + + var seasonsData = seasons.Select(s => YdbValue.MakeStruct(new Dictionary + { + { "series_id", YdbValue.MakeUint64((ulong)s.SeriesId) }, + { "season_id", YdbValue.MakeUint64((ulong)s.SeasonId) }, + { "title", YdbValue.MakeUtf8(s.Title) }, + { "first_aired", YdbValue.MakeDate(s.FirstAired) }, + { "last_aired", YdbValue.MakeDate(s.LastAired) } + })).ToList(); + + var episodesData = episodes.Select(e => YdbValue.MakeStruct(new Dictionary + { + { "series_id", YdbValue.MakeUint64((ulong)e.SeriesId) }, + { "season_id", YdbValue.MakeUint64((ulong)e.SeasonId) }, + { "episode_id", YdbValue.MakeUint64((ulong)e.EpisodeId) }, + { "title", YdbValue.MakeUtf8(e.Title) }, + { "air_date", YdbValue.MakeDate(e.AirDate) } + })).ToList(); + + return new Dictionary + { + { "$seriesData", YdbValue.MakeList(seriesData) }, + { "$seasonsData", YdbValue.MakeList(seasonsData) }, + { "$episodesData", YdbValue.MakeList(episodesData) } + }; + } +} \ No newline at end of file diff --git a/examples/src/BasicExample/InteractiveTx.cs b/examples/src/BasicExample/InteractiveTx.cs new file mode 100644 index 00000000..cfeca9ff --- /dev/null +++ b/examples/src/BasicExample/InteractiveTx.cs @@ -0,0 +1,75 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Ydb.Sdk.Services.Table; +using Ydb.Sdk.Value; + +namespace Ydb.Sdk.Examples; + +internal partial class BasicExample +{ + private async Task InteractiveTx() + { + var execResponse = await Client.SessionExec(async session => + { + var query1 = @$" + PRAGMA TablePathPrefix('{BasePath}'); + + DECLARE $series_id AS Uint64; + DECLARE $season_id AS Uint64; + + SELECT first_aired FROM seasons + WHERE series_id = $series_id AND season_id = $season_id; + "; + + // Execute first query (no transaction commit) + var response = await session.ExecuteDataQuery( + query: query1, + txControl: TxControl.BeginSerializableRW(), + parameters: new Dictionary + { + { "$series_id", YdbValue.MakeUint64(1) }, + { "$season_id", YdbValue.MakeUint64(3) } + }, + settings: DefaultCachedDataQuerySettings + ); + + if (!response.Status.IsSuccess || response.Tx is null) + { + return response; + } + + // Perform some client logic + var firstAired = (DateTime?)response.Result.ResultSets[0].Rows[0]["first_aired"]; + var newAired = firstAired!.Value.AddDays(2); + + var query2 = @$" + PRAGMA TablePathPrefix('{BasePath}'); + + DECLARE $series_id AS Uint64; + DECLARE $season_id AS Uint64; + DECLARE $air_date AS Date; + + UPSERT INTO seasons (series_id, season_id, first_aired) VALUES + ($series_id, $season_id, $air_date); + "; + + // Execute second query and commit transaction. + response = await session.ExecuteDataQuery( + query: query2, + TxControl.Tx(response.Tx).Commit(), + parameters: new Dictionary + { + { "$series_id", YdbValue.MakeUint64(1) }, + { "$season_id", YdbValue.MakeUint64(3) }, + { "$air_date", YdbValue.MakeDate(newAired) } + }, + settings: DefaultCachedDataQuerySettings + ); + + return response; + }); + + execResponse.Status.EnsureSuccess(); + } +} \ No newline at end of file diff --git a/examples/src/BasicExample/Program.cs b/examples/src/BasicExample/Program.cs new file mode 100644 index 00000000..e3e5b38d --- /dev/null +++ b/examples/src/BasicExample/Program.cs @@ -0,0 +1,59 @@ +using System; +using System.Threading.Tasks; +using CommandLine; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Ydb.Sdk.Examples; + +internal class CmdOptions +{ + [Option('e', "endpoint", Required = true, HelpText = "Database endpoint")] + public string Endpoint { get; set; } = ""; + + [Option('d', "database", Required = true, HelpText = "Database name")] + public string Database { get; set; } = ""; + + [Option('p', "path", HelpText = "Base path for tables")] + public string Path { get; set; } = "ydb-dotnet-basic"; + + [Option("anonymous", Required = false, HelpText = "Fallback anonymous")] + public bool FallbackAnonymous { get; set; } = false; +} + +internal static class Program +{ + private static ServiceProvider GetServiceProvider() + { + return new ServiceCollection() + .AddLogging(configure => configure.AddConsole().SetMinimumLevel(LogLevel.Information)) + .BuildServiceProvider(); + } + + private static async Task Run(CmdOptions cmdOptions) + { + await using var serviceProvider = GetServiceProvider(); + var loggerFactory = serviceProvider.GetService(); + + loggerFactory ??= NullLoggerFactory.Instance; + + await BasicExample.Run( + endpoint: cmdOptions.Endpoint, + database: cmdOptions.Database, + credentialsProvider: await AuthUtils.MakeCredentialsFromEnv( + fallbackAnonymous: cmdOptions.FallbackAnonymous, + loggerFactory: loggerFactory), + customServerCertificate: AuthUtils.GetCustomServerCertificate(), + path: cmdOptions.Path, + loggerFactory: loggerFactory + ); + } + + private static async Task Main(string[] args) + { + AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true); + + await Parser.Default.ParseArguments(args).WithParsedAsync(Run); + } +} \ No newline at end of file diff --git a/examples/src/BasicExample/ReadTable.cs b/examples/src/BasicExample/ReadTable.cs new file mode 100644 index 00000000..c74e4f09 --- /dev/null +++ b/examples/src/BasicExample/ReadTable.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Ydb.Sdk.Services.Table; + +namespace Ydb.Sdk.Examples; + +internal partial class BasicExample +{ + private async Task ReadTable() + { + var readStream = Client.ReadTable( + FullTablePath("seasons"), + new ReadTableSettings + { + Columns = new List { "series_id", "season_id", "first_aired" }, + RowLimit = 5, + Ordered = true + }); + + while (await readStream.Next()) + { + readStream.Response.EnsureSuccess(); + var resultSet = readStream.Response.Result.ResultSet; + + foreach (var row in resultSet.Rows) + { + Console.WriteLine($"> ReadTable seasons, " + + $"series_id: {(ulong?)row["series_id"]}, " + + $"season_id: {(ulong?)row["season_id"]}, " + + $"first_aired: {(DateTime?)row["first_aired"]}"); + } + } + } +} \ No newline at end of file diff --git a/examples/src/BasicExample/ScanQuery.cs b/examples/src/BasicExample/ScanQuery.cs new file mode 100644 index 00000000..22ca3fee --- /dev/null +++ b/examples/src/BasicExample/ScanQuery.cs @@ -0,0 +1,48 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Ydb.Sdk.Value; + +namespace Ydb.Sdk.Examples; + +internal partial class BasicExample +{ + private async Task ScanQuery(DateTime airFrom) + { + var query = @$" + PRAGMA TablePathPrefix('{BasePath}'); + + DECLARE $air_from AS Date; + + SELECT series_id, season_id, COUNT(*) AS episodes_count + FROM episodes + WHERE air_date >= $air_from + GROUP BY series_id, season_id + ORDER BY series_id, season_id; + "; + + var scanStream = Client.ExecuteScanQuery( + query, + new Dictionary + { + { "$air_from", YdbValue.MakeDate(airFrom) } + }); + + while (await scanStream.Next()) + { + scanStream.Response.EnsureSuccess(); + + var resultSet = scanStream.Response.Result.ResultSetPart; + if (resultSet != null) + { + foreach (var row in resultSet.Rows) + { + Console.WriteLine($"> ScanQuery, " + + $"series_id: {(ulong?)row["series_id"]}, " + + $"season_id: {(ulong?)row["season_id"]}, " + + $"episodes_count: {(ulong)row["episodes_count"]}"); + } + } + } + } +} \ No newline at end of file diff --git a/examples/src/BasicExample/SchemeQuery.cs b/examples/src/BasicExample/SchemeQuery.cs new file mode 100644 index 00000000..216ef271 --- /dev/null +++ b/examples/src/BasicExample/SchemeQuery.cs @@ -0,0 +1,43 @@ +using System.Threading.Tasks; + +namespace Ydb.Sdk.Examples; + +internal partial class BasicExample +{ + // Execute Scheme (DDL) query to create sample tables. + private async Task SchemeQuery() + { + var response = await Client.SessionExec(async session => + await session.ExecuteSchemeQuery(@$" + PRAGMA TablePathPrefix('{BasePath}'); + + CREATE TABLE series ( + series_id Uint64, + title Utf8, + series_info Utf8, + release_date Date, + PRIMARY KEY (series_id) + ); + + CREATE TABLE seasons ( + series_id Uint64, + season_id Uint64, + title Utf8, + first_aired Date, + last_aired Date, + PRIMARY KEY (series_id, season_id) + ); + + CREATE TABLE episodes ( + series_id Uint64, + season_id Uint64, + episode_id Uint64, + title Utf8, + air_date Date, + PRIMARY KEY (series_id, season_id, episode_id) + ); + ")); + + response.Status.EnsureSuccess(); + } +} \ No newline at end of file diff --git a/examples/src/Common/AuthUtils.cs b/examples/src/Common/AuthUtils.cs new file mode 100644 index 00000000..5a9824d5 --- /dev/null +++ b/examples/src/Common/AuthUtils.cs @@ -0,0 +1,67 @@ +using System; +using System.Security.Cryptography.X509Certificates; +using System.Threading.Tasks; +using Microsoft.Extensions.Logging; +using Ydb.Sdk.Auth; +using Ydb.Sdk.Yc; + +namespace Ydb.Sdk.Examples; + +public static class AuthUtils +{ + public static async Task MakeCredentialsFromEnv( + bool fallbackAnonymous = false, + ILoggerFactory? loggerFactory = null) + { + var saFileValue = Environment.GetEnvironmentVariable("YDB_SERVICE_ACCOUNT_KEY_FILE_CREDENTIALS"); + if (!string.IsNullOrEmpty(saFileValue)) + { + var saProvider = new ServiceAccountProvider( + saFilePath: saFileValue, + loggerFactory: loggerFactory); + await saProvider.Initialize(); + return saProvider; + } + + var anonymousValue = Environment.GetEnvironmentVariable("YDB_ANONYMOUS_CREDENTIALS"); + if (anonymousValue != null && IsTrueValue(anonymousValue)) + { + return new AnonymousProvider(); + } + + var metadataValue = Environment.GetEnvironmentVariable("YDB_METADATA_CREDENTIALS"); + if (metadataValue != null && IsTrueValue(metadataValue)) + { + var metadataProvider = new MetadataProvider( + loggerFactory: loggerFactory); + await metadataProvider.Initialize(); + return metadataProvider; + } + + var tokenValue = Environment.GetEnvironmentVariable("YDB_ACCESS_TOKEN_CREDENTIALS"); + if (!string.IsNullOrEmpty(tokenValue)) + { + return new TokenProvider(tokenValue); + } + + if (fallbackAnonymous) + { + return new AnonymousProvider(); + } + + throw new InvalidOperationException("Failed to parse credentials from environmet, no valid options found."); + } + + public static X509Certificate GetCustomServerCertificate() + { + return YcCerts.GetDefaultServerCertificate(); + } + + private static bool IsTrueValue(string value) + { + return + value == "1" || + value.ToLower() == "yes" || + value.ToLower() == "true"; + } +} \ No newline at end of file diff --git a/examples/src/Common/Common.csproj b/examples/src/Common/Common.csproj new file mode 100644 index 00000000..7c6e5df5 --- /dev/null +++ b/examples/src/Common/Common.csproj @@ -0,0 +1,23 @@ + + + + net6.0 + Ydb.Sdk.Examples.Common + Ydb.Sdk.Examples + enable + + + + git + https://github.com/ydb-platform/ydb-dotnet-examples + https://github.com/ydb-platform/ydb-dotnet-examples + YANDEX LLC + + + + + + + + + diff --git a/examples/src/Common/TableExampleBase.cs b/examples/src/Common/TableExampleBase.cs new file mode 100644 index 00000000..5e12f1c3 --- /dev/null +++ b/examples/src/Common/TableExampleBase.cs @@ -0,0 +1,20 @@ +using Ydb.Sdk.Services.Table; + +namespace Ydb.Sdk.Examples; + +public class TableExampleBase +{ + protected TableClient Client { get; } + protected string BasePath { get; } + + protected TableExampleBase(TableClient client, string database, string path) + { + Client = client; + BasePath = string.Join('/', database, path); + } + + protected string FullTablePath(string table) + { + return string.Join('/', BasePath, table); + } +} \ No newline at end of file diff --git a/examples/src/YdbExamples.sln b/examples/src/YdbExamples.sln new file mode 100644 index 00000000..926c9a52 --- /dev/null +++ b/examples/src/YdbExamples.sln @@ -0,0 +1,31 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.31205.134 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Common", "Common\Common.csproj", "{59F508A9-5EE0-4A6C-9580-89FC8C6CD4CE}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BasicExample", "BasicExample\BasicExample.csproj", "{9DAD5FF3-B7C2-4A9E-B4B2-A0FBD6097727}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {59F508A9-5EE0-4A6C-9580-89FC8C6CD4CE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {59F508A9-5EE0-4A6C-9580-89FC8C6CD4CE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {59F508A9-5EE0-4A6C-9580-89FC8C6CD4CE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {59F508A9-5EE0-4A6C-9580-89FC8C6CD4CE}.Release|Any CPU.Build.0 = Release|Any CPU + {9DAD5FF3-B7C2-4A9E-B4B2-A0FBD6097727}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {9DAD5FF3-B7C2-4A9E-B4B2-A0FBD6097727}.Debug|Any CPU.Build.0 = Debug|Any CPU + {9DAD5FF3-B7C2-4A9E-B4B2-A0FBD6097727}.Release|Any CPU.ActiveCfg = Release|Any CPU + {9DAD5FF3-B7C2-4A9E-B4B2-A0FBD6097727}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {1E75A264-F26E-469D-AD5C-FBBA4C34F249} + EndGlobalSection +EndGlobal diff --git a/examples/src/YdbExamples.sln.DotSettings b/examples/src/YdbExamples.sln.DotSettings new file mode 100644 index 00000000..0625ed27 --- /dev/null +++ b/examples/src/YdbExamples.sln.DotSettings @@ -0,0 +1,69 @@ + + <?xml version="1.0" encoding="utf-16"?><Profile name="Custom Cleanup"><CppCodeStyleCleanupDescriptor ArrangeBraces="True" ArrangeAuto="True" ArrangeFunctionDeclarations="True" ArrangeNestedNamespaces="True" ArrangeTypeAliases="True" ArrangeCVQualifiers="True" ArrangeSlashesInIncludeDirectives="True" ArrangeOverridingFunctions="True" SortIncludeDirectives="True" SortMemberInitializers="True" /><CppReformatCode>True</CppReformatCode><CSCodeStyleAttributes ArrangeVarStyle="True" ArrangeTypeAccessModifier="True" ArrangeTypeMemberAccessModifier="True" SortModifiers="True" AddMissingParentheses="True" ArrangeAttributes="True" ArrangeCodeBodyStyle="True" ArrangeTrailingCommas="True" ArrangeObjectCreation="True" ArrangeDefaultValue="True" RemoveRedundantParentheses="True" ArrangeNamespaces="True" /><FSReformatCode>True</FSReformatCode><ShaderLabReformatCode>True</ShaderLabReformatCode><Xaml.RedundantFreezeAttribute>True</Xaml.RedundantFreezeAttribute><Xaml.RemoveRedundantModifiersAttribute>True</Xaml.RemoveRedundantModifiersAttribute><Xaml.RemoveRedundantNameAttribute>True</Xaml.RemoveRedundantNameAttribute><Xaml.RemoveRedundantResource>True</Xaml.RemoveRedundantResource><Xaml.RemoveRedundantCollectionProperty>True</Xaml.RemoveRedundantCollectionProperty><Xaml.RemoveRedundantAttachedPropertySetter>True</Xaml.RemoveRedundantAttachedPropertySetter><Xaml.RemoveRedundantStyledValue>True</Xaml.RemoveRedundantStyledValue><Xaml.RemoveRedundantNamespaceAlias>True</Xaml.RemoveRedundantNamespaceAlias><Xaml.RemoveForbiddenResourceName>True</Xaml.RemoveForbiddenResourceName><Xaml.RemoveRedundantGridDefinitionsAttribute>True</Xaml.RemoveRedundantGridDefinitionsAttribute><Xaml.RemoveRedundantUpdateSourceTriggerAttribute>True</Xaml.RemoveRedundantUpdateSourceTriggerAttribute><Xaml.RemoveRedundantBindingModeAttribute>True</Xaml.RemoveRedundantBindingModeAttribute><Xaml.RemoveRedundantGridSpanAttribut>True</Xaml.RemoveRedundantGridSpanAttribut><XMLReformatCode>True</XMLReformatCode><CSArrangeQualifiers>True</CSArrangeQualifiers><CSFixBuiltinTypeReferences>True</CSFixBuiltinTypeReferences><HtmlReformatCode>True</HtmlReformatCode><VBReformatCode>True</VBReformatCode><CSReformatCode>True</CSReformatCode><FormatAttributeQuoteDescriptor>True</FormatAttributeQuoteDescriptor><IDEA_SETTINGS>&lt;profile version="1.0"&gt; + &lt;option name="myName" value="Custom Cleanup" /&gt; + &lt;inspection_tool class="ES6ShorthandObjectProperty" enabled="false" level="INFORMATION" enabled_by_default="false" /&gt; + &lt;inspection_tool class="JSArrowFunctionBracesCanBeRemoved" enabled="false" level="INFORMATION" enabled_by_default="false" /&gt; + &lt;inspection_tool class="JSPrimitiveTypeWrapperUsage" enabled="false" level="WARNING" enabled_by_default="false" /&gt; + &lt;inspection_tool class="JSRemoveUnnecessaryParentheses" enabled="false" level="INFORMATION" enabled_by_default="false" /&gt; + &lt;inspection_tool class="JSUnnecessarySemicolon" enabled="false" level="WARNING" enabled_by_default="false" /&gt; + &lt;inspection_tool class="TypeScriptExplicitMemberType" enabled="false" level="INFORMATION" enabled_by_default="false" /&gt; + &lt;inspection_tool class="UnnecessaryContinueJS" enabled="false" level="WARNING" enabled_by_default="false" /&gt; + &lt;inspection_tool class="UnnecessaryLabelJS" enabled="false" level="WARNING" enabled_by_default="false" /&gt; + &lt;inspection_tool class="UnnecessaryLabelOnBreakStatementJS" enabled="false" level="WARNING" enabled_by_default="false" /&gt; + &lt;inspection_tool class="UnnecessaryLabelOnContinueStatementJS" enabled="false" level="WARNING" enabled_by_default="false" /&gt; + &lt;inspection_tool class="UnnecessaryReturnJS" enabled="false" level="WARNING" enabled_by_default="false" /&gt; + &lt;inspection_tool class="WrongPropertyKeyValueDelimiter" enabled="false" level="WEAK WARNING" enabled_by_default="false" /&gt; +&lt;/profile&gt;</IDEA_SETTINGS><RIDER_SETTINGS>&lt;profile&gt; + &lt;Language id="CSS"&gt; + &lt;Rearrange&gt;false&lt;/Rearrange&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="EditorConfig"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="HTML"&gt; + &lt;Rearrange&gt;false&lt;/Rearrange&gt; + &lt;OptimizeImports&gt;false&lt;/OptimizeImports&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="HTTP Request"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="Handlebars"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="Ini"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="JSON"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="Jade"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="JavaScript"&gt; + &lt;Rearrange&gt;false&lt;/Rearrange&gt; + &lt;OptimizeImports&gt;false&lt;/OptimizeImports&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="Markdown"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="Properties"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="RELAX-NG"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="SQL"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="XML"&gt; + &lt;Rearrange&gt;false&lt;/Rearrange&gt; + &lt;OptimizeImports&gt;false&lt;/OptimizeImports&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; + &lt;Language id="yaml"&gt; + &lt;Reformat&gt;true&lt;/Reformat&gt; + &lt;/Language&gt; +&lt;/profile&gt;</RIDER_SETTINGS><CSUseAutoProperty>True</CSUseAutoProperty><CSMakeFieldReadonly>True</CSMakeFieldReadonly><CSMakeAutoPropertyGetOnly>True</CSMakeAutoPropertyGetOnly><RemoveCodeRedundancies>True</RemoveCodeRedundancies><CSOptimizeUsings><OptimizeUsings>True</OptimizeUsings></CSOptimizeUsings><CSShortenReferences>True</CSShortenReferences></Profile> \ No newline at end of file From 0cc617079d8ce63eb03175c34b5f703ebc8c32a1 Mon Sep 17 00:00:00 2001 From: XmasApple Date: Mon, 16 Oct 2023 19:08:55 +0300 Subject: [PATCH 08/26] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d45e5a..7e3a7eae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +- Move examples from [separated repo](https://github.com/ydb-platform/ydb-dotnet-examples) +- Adjust .NET and SDK versions + ## v0.1.3 - Add static auth ## v0.1.1 From 9805dde44a3940e43b99e2a0613bc8664c0be5b4 Mon Sep 17 00:00:00 2001 From: XmasApple Date: Tue, 17 Oct 2023 13:49:38 +0300 Subject: [PATCH 09/26] add exception on truncated result --- CHANGELOG.md | 2 ++ src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d45e5a..2d4b179c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +- Add exception throwing when results truncated + ## v0.1.3 - Add static auth ## v0.1.1 diff --git a/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs b/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs index 7ecb7f1d..eabf5afe 100644 --- a/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs +++ b/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs @@ -8,6 +8,7 @@ namespace Ydb.Sdk.Services.Table; public class ExecuteDataQuerySettings : OperationRequestSettings { public bool KeepInQueryCache { get; set; } = true; + public bool AllowTruncated { get; set; } = false; } public class ExecuteDataQueryResponse : ResponseWithResultBase @@ -103,6 +104,10 @@ public async Task ExecuteDataQuery( if (status.IsSuccess && resultProto != null) { result = ExecuteDataQueryResponse.ResultData.FromProto(resultProto); + if (!settings.AllowTruncated && result.ResultSets.Any(set => set.Truncated)) + { + throw new TruncateException(); + } } return new ExecuteDataQueryResponse(status, txState, tx, result); @@ -120,4 +125,8 @@ public async Task ExecuteDataQuery( { return await ExecuteDataQuery(query, txControl, new Dictionary(), settings); } +} + +public class TruncateException : Exception +{ } \ No newline at end of file From 537aa49e103a876c0dac6d0f28c1216c60f018d1 Mon Sep 17 00:00:00 2001 From: Timofey Koolin Date: Wed, 18 Oct 2023 14:42:55 +0300 Subject: [PATCH 10/26] remove newline from Version.cs From 8d457d0eeb8842c0c5e3ca3da6380511d5f617c7 Mon Sep 17 00:00:00 2001 From: XmasApple Date: Wed, 18 Oct 2023 16:03:31 +0300 Subject: [PATCH 11/26] Revert "Update CHANGELOG.md" This reverts commit 0cc617079d8ce63eb03175c34b5f703ebc8c32a1. --- CHANGELOG.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e3a7eae..82d45e5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,3 @@ -- Move examples from [separated repo](https://github.com/ydb-platform/ydb-dotnet-examples) -- Adjust .NET and SDK versions - ## v0.1.3 - Add static auth ## v0.1.1 From a2f2741ba61945e07182fafeb7aaed917137a38c Mon Sep 17 00:00:00 2001 From: XmasApple Date: Thu, 19 Oct 2023 13:52:27 +0300 Subject: [PATCH 12/26] lint: add line feed at file end --- CHANGELOG.md | 2 ++ src/Ydb.Sdk/src/Auth/Anonymous.cs | 2 +- src/Ydb.Sdk/src/Auth/Auth.cs | 2 +- src/Ydb.Sdk/src/Auth/ICredentialsProvider.cs | 2 +- src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs | 2 +- src/Ydb.Sdk/src/Auth/StaticCredentialsProvider.cs | 2 +- src/Ydb.Sdk/src/Auth/Token.cs | 2 +- src/Ydb.Sdk/src/ChannelsCache.cs | 2 +- src/Ydb.Sdk/src/Client/ClientBase.cs | 2 +- src/Ydb.Sdk/src/Client/ClientOperation.cs | 2 +- src/Ydb.Sdk/src/Client/Response.cs | 2 +- src/Ydb.Sdk/src/Driver.cs | 2 +- src/Ydb.Sdk/src/DriverConfig.cs | 2 +- src/Ydb.Sdk/src/Metadata.cs | 2 +- src/Ydb.Sdk/src/RequestSettings.cs | 2 +- src/Ydb.Sdk/src/Services/Auth/AuthClient.cs | 2 +- src/Ydb.Sdk/src/Services/Auth/Login.cs | 2 +- src/Ydb.Sdk/src/Services/Operations/GetOperation.cs | 2 +- src/Ydb.Sdk/src/Services/Operations/OperationsClient.cs | 2 +- src/Ydb.Sdk/src/Services/Operations/Poll.cs | 2 +- src/Ydb.Sdk/src/Services/Scheme/SchemeClient.cs | 2 +- src/Ydb.Sdk/src/Services/Table/AlterTable.cs | 2 +- src/Ydb.Sdk/src/Services/Table/CreateSession.cs | 2 +- src/Ydb.Sdk/src/Services/Table/DeleteSession.cs | 2 +- src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs | 2 +- src/Ydb.Sdk/src/Services/Table/ExecuteScanQuery.cs | 2 +- src/Ydb.Sdk/src/Services/Table/ExecuteSchemeQuery.cs | 2 +- src/Ydb.Sdk/src/Services/Table/KeepAlive.cs | 2 +- src/Ydb.Sdk/src/Services/Table/ReadTable.cs | 2 +- src/Ydb.Sdk/src/Services/Table/Retry.cs | 2 +- src/Ydb.Sdk/src/Services/Table/Session.cs | 2 +- src/Ydb.Sdk/src/Services/Table/SessionPool.cs | 2 +- src/Ydb.Sdk/src/Services/Table/TableClient.cs | 2 +- src/Ydb.Sdk/src/Services/Table/Transaction.cs | 2 +- src/Ydb.Sdk/src/Status.cs | 2 +- src/Ydb.Sdk/src/Value/ResultSet.cs | 2 +- src/Ydb.Sdk/src/Value/YdbValue.cs | 2 +- src/Ydb.Sdk/src/Value/YdbValueBuilder.cs | 2 +- src/Ydb.Sdk/src/Value/YdbValueCast.cs | 2 +- src/Ydb.Sdk/src/Value/YdbValueParser.cs | 2 +- src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs | 2 +- src/Ydb.Sdk/tests/Utils.cs | 2 +- src/Ydb.Sdk/tests/Value/TestBasicUnit.cs | 2 +- src/Ydb.Sdk/tests/Value/TestBasicsIntegration.cs | 2 +- src/YdbSdk.sln.DotSettings | 3 ++- 45 files changed, 47 insertions(+), 44 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 82d45e5a..33760f8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,5 @@ +- lint: add line feed at file end + ## v0.1.3 - Add static auth ## v0.1.1 diff --git a/src/Ydb.Sdk/src/Auth/Anonymous.cs b/src/Ydb.Sdk/src/Auth/Anonymous.cs index 445788c1..fbde0a1f 100644 --- a/src/Ydb.Sdk/src/Auth/Anonymous.cs +++ b/src/Ydb.Sdk/src/Auth/Anonymous.cs @@ -6,4 +6,4 @@ public class AnonymousProvider : ICredentialsProvider { return null; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Auth/Auth.cs b/src/Ydb.Sdk/src/Auth/Auth.cs index 45d81cca..ed5e8a8b 100644 --- a/src/Ydb.Sdk/src/Auth/Auth.cs +++ b/src/Ydb.Sdk/src/Auth/Auth.cs @@ -6,4 +6,4 @@ public InvalidCredentialsException(string message) : base(message) { } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Auth/ICredentialsProvider.cs b/src/Ydb.Sdk/src/Auth/ICredentialsProvider.cs index 92156d65..96802873 100644 --- a/src/Ydb.Sdk/src/Auth/ICredentialsProvider.cs +++ b/src/Ydb.Sdk/src/Auth/ICredentialsProvider.cs @@ -3,4 +3,4 @@ public interface ICredentialsProvider { string? GetAuthInfo(); -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs b/src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs index a7b99d8e..6eb3b683 100644 --- a/src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs +++ b/src/Ydb.Sdk/src/Auth/IUseDriverConfig.cs @@ -3,4 +3,4 @@ namespace Ydb.Sdk.Auth; public interface IUseDriverConfig { public Task ProvideConfig(DriverConfig driverConfig); -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Auth/StaticCredentialsProvider.cs b/src/Ydb.Sdk/src/Auth/StaticCredentialsProvider.cs index 6bfc06e1..5a21b617 100644 --- a/src/Ydb.Sdk/src/Auth/StaticCredentialsProvider.cs +++ b/src/Ydb.Sdk/src/Auth/StaticCredentialsProvider.cs @@ -207,4 +207,4 @@ public bool IsRefreshNeeded() return DateTime.UtcNow >= RefreshAt; } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Auth/Token.cs b/src/Ydb.Sdk/src/Auth/Token.cs index 873abe90..03b71af9 100644 --- a/src/Ydb.Sdk/src/Auth/Token.cs +++ b/src/Ydb.Sdk/src/Auth/Token.cs @@ -13,4 +13,4 @@ public string GetAuthInfo() { return _token; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/ChannelsCache.cs b/src/Ydb.Sdk/src/ChannelsCache.cs index 08b86adb..d5bffb85 100644 --- a/src/Ydb.Sdk/src/ChannelsCache.cs +++ b/src/Ydb.Sdk/src/ChannelsCache.cs @@ -291,4 +291,4 @@ public EndpointsData(ChannelsData active, ChannelsData passive) Passive = passive; } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Client/ClientBase.cs b/src/Ydb.Sdk/src/Client/ClientBase.cs index f9c3b450..ed529e86 100644 --- a/src/Ydb.Sdk/src/Client/ClientBase.cs +++ b/src/Ydb.Sdk/src/Client/ClientBase.cs @@ -50,4 +50,4 @@ protected internal static OperationParams MakeOperationParams(OperationRequestSe return opParams; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Client/ClientOperation.cs b/src/Ydb.Sdk/src/Client/ClientOperation.cs index ff8beff7..1f33792d 100644 --- a/src/Ydb.Sdk/src/Client/ClientOperation.cs +++ b/src/Ydb.Sdk/src/Client/ClientOperation.cs @@ -108,4 +108,4 @@ public OperationNotReadyException(string operationId) } public string OperationId { get; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Client/Response.cs b/src/Ydb.Sdk/src/Client/Response.cs index 82642181..0b77974b 100644 --- a/src/Ydb.Sdk/src/Client/Response.cs +++ b/src/Ydb.Sdk/src/Client/Response.cs @@ -194,4 +194,4 @@ protected OperationResponse(Status status) protected abstract TResult UnpackResult(ClientOperation operation); protected abstract TMetadata UnpackMetadata(ClientOperation operation); -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Driver.cs b/src/Ydb.Sdk/src/Driver.cs index f818d30f..ad505d59 100644 --- a/src/Ydb.Sdk/src/Driver.cs +++ b/src/Ydb.Sdk/src/Driver.cs @@ -375,4 +375,4 @@ internal TransportException(RpcException e) public Status Status { get; } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/DriverConfig.cs b/src/Ydb.Sdk/src/DriverConfig.cs index a2d72d17..eff32b20 100644 --- a/src/Ydb.Sdk/src/DriverConfig.cs +++ b/src/Ydb.Sdk/src/DriverConfig.cs @@ -60,4 +60,4 @@ private static string FormatEndpoint(string endpoint) return $"https://{endpoint}"; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Metadata.cs b/src/Ydb.Sdk/src/Metadata.cs index 6b820ea2..b49b8600 100644 --- a/src/Ydb.Sdk/src/Metadata.cs +++ b/src/Ydb.Sdk/src/Metadata.cs @@ -7,4 +7,4 @@ internal static class Metadata public const string RpcRequestTypeHeader = "x-ydb-request-type"; public const string RpcTraceIdHeader = "x-ydb-trace-id"; public const string RpcSdkInfoHeader = "x-ydb-sdk-build-info"; -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/RequestSettings.cs b/src/Ydb.Sdk/src/RequestSettings.cs index f39681b1..043f5e3e 100644 --- a/src/Ydb.Sdk/src/RequestSettings.cs +++ b/src/Ydb.Sdk/src/RequestSettings.cs @@ -10,4 +10,4 @@ public class RequestSettings public class OperationRequestSettings : RequestSettings { public TimeSpan? OperationTimeout { get; set; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Auth/AuthClient.cs b/src/Ydb.Sdk/src/Services/Auth/AuthClient.cs index f4325648..544627e2 100644 --- a/src/Ydb.Sdk/src/Services/Auth/AuthClient.cs +++ b/src/Ydb.Sdk/src/Services/Auth/AuthClient.cs @@ -7,4 +7,4 @@ public partial class AuthClient : ClientBase public AuthClient(Driver driver) : base(driver) { } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Auth/Login.cs b/src/Ydb.Sdk/src/Services/Auth/Login.cs index c0b5e8f7..1ccb7a4a 100644 --- a/src/Ydb.Sdk/src/Services/Auth/Login.cs +++ b/src/Ydb.Sdk/src/Services/Auth/Login.cs @@ -72,4 +72,4 @@ public async Task Login(string user, string? password, LoginSetti return new LoginResponse(e.Status); } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Operations/GetOperation.cs b/src/Ydb.Sdk/src/Services/Operations/GetOperation.cs index 56266cf2..34c462cf 100644 --- a/src/Ydb.Sdk/src/Services/Operations/GetOperation.cs +++ b/src/Ydb.Sdk/src/Services/Operations/GetOperation.cs @@ -29,4 +29,4 @@ public async Task GetOperation(string id, RequestSettings? sett return new ClientOperation(e.Status); } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Operations/OperationsClient.cs b/src/Ydb.Sdk/src/Services/Operations/OperationsClient.cs index 5d9cb8f5..1c623528 100644 --- a/src/Ydb.Sdk/src/Services/Operations/OperationsClient.cs +++ b/src/Ydb.Sdk/src/Services/Operations/OperationsClient.cs @@ -8,4 +8,4 @@ public OperationsClient(Driver driver) { _driver = driver; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Operations/Poll.cs b/src/Ydb.Sdk/src/Services/Operations/Poll.cs index 1d476ce6..a6b64fe3 100644 --- a/src/Ydb.Sdk/src/Services/Operations/Poll.cs +++ b/src/Ydb.Sdk/src/Services/Operations/Poll.cs @@ -22,4 +22,4 @@ public async Task PollReady( await Task.Delay(delay.Value, cancellationToken); } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Scheme/SchemeClient.cs b/src/Ydb.Sdk/src/Services/Scheme/SchemeClient.cs index a500e382..3b7cb3a2 100644 --- a/src/Ydb.Sdk/src/Services/Scheme/SchemeClient.cs +++ b/src/Ydb.Sdk/src/Services/Scheme/SchemeClient.cs @@ -157,4 +157,4 @@ public async Task ListDirectory(string path, ListDirector return new ListDirectoryResponse(e.Status); } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/AlterTable.cs b/src/Ydb.Sdk/src/Services/Table/AlterTable.cs index 200bef52..13386117 100644 --- a/src/Ydb.Sdk/src/Services/Table/AlterTable.cs +++ b/src/Ydb.Sdk/src/Services/Table/AlterTable.cs @@ -120,4 +120,4 @@ public async Task AddIndex(string tablePath, AddIndexSettin return new AlterTableOperation(new OperationsClient(Driver), e.Status); } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/CreateSession.cs b/src/Ydb.Sdk/src/Services/Table/CreateSession.cs index 7699d934..1b5f640c 100644 --- a/src/Ydb.Sdk/src/Services/Table/CreateSession.cs +++ b/src/Ydb.Sdk/src/Services/Table/CreateSession.cs @@ -73,4 +73,4 @@ public async Task CreateSession(CreateSessionSettings? se return new CreateSessionResponse(e.Status); } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/DeleteSession.cs b/src/Ydb.Sdk/src/Services/Table/DeleteSession.cs index 7bf761d0..af626def 100644 --- a/src/Ydb.Sdk/src/Services/Table/DeleteSession.cs +++ b/src/Ydb.Sdk/src/Services/Table/DeleteSession.cs @@ -44,4 +44,4 @@ public async Task DeleteSession(string sessionId, DeleteS return new DeleteSessionResponse(e.Status); } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs b/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs index 7ecb7f1d..ec5dc393 100644 --- a/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs +++ b/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs @@ -120,4 +120,4 @@ public async Task ExecuteDataQuery( { return await ExecuteDataQuery(query, txControl, new Dictionary(), settings); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/ExecuteScanQuery.cs b/src/Ydb.Sdk/src/Services/Table/ExecuteScanQuery.cs index ac00888e..5abb41c9 100644 --- a/src/Ydb.Sdk/src/Services/Table/ExecuteScanQuery.cs +++ b/src/Ydb.Sdk/src/Services/Table/ExecuteScanQuery.cs @@ -90,4 +90,4 @@ public ExecuteScanQueryStream ExecuteScanQuery( { return ExecuteScanQuery(query, new Dictionary(), settings); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/ExecuteSchemeQuery.cs b/src/Ydb.Sdk/src/Services/Table/ExecuteSchemeQuery.cs index 0bf08691..110de516 100644 --- a/src/Ydb.Sdk/src/Services/Table/ExecuteSchemeQuery.cs +++ b/src/Ydb.Sdk/src/Services/Table/ExecuteSchemeQuery.cs @@ -49,4 +49,4 @@ public async Task ExecuteSchemeQuery( return new ExecuteSchemeQueryResponse(e.Status); } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/KeepAlive.cs b/src/Ydb.Sdk/src/Services/Table/KeepAlive.cs index 0bc4e2bd..716b7f90 100644 --- a/src/Ydb.Sdk/src/Services/Table/KeepAlive.cs +++ b/src/Ydb.Sdk/src/Services/Table/KeepAlive.cs @@ -86,4 +86,4 @@ internal async Task KeepAlive(string sessionId, KeepAliveSett return new KeepAliveResponse(e.Status); } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/ReadTable.cs b/src/Ydb.Sdk/src/Services/Table/ReadTable.cs index add041d9..957d5e1a 100644 --- a/src/Ydb.Sdk/src/Services/Table/ReadTable.cs +++ b/src/Ydb.Sdk/src/Services/Table/ReadTable.cs @@ -82,4 +82,4 @@ public ReadTableStream ReadTable(string tablePath, ReadTableSettings? settings = return new ReadTableStream(streamIterator); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/Retry.cs b/src/Ydb.Sdk/src/Services/Table/Retry.cs index 40377aa1..04e070a9 100644 --- a/src/Ydb.Sdk/src/Services/Table/Retry.cs +++ b/src/Ydb.Sdk/src/Services/Table/Retry.cs @@ -135,4 +135,4 @@ public async Task SessionExec( return response; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/Session.cs b/src/Ydb.Sdk/src/Services/Table/Session.cs index 0e12cf92..ffee604c 100644 --- a/src/Ydb.Sdk/src/Services/Table/Session.cs +++ b/src/Ydb.Sdk/src/Services/Table/Session.cs @@ -91,4 +91,4 @@ protected virtual void Dispose(bool disposing) preferredEndpoint: Endpoint ); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs index c0ef6281..cf4c0115 100644 --- a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs +++ b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs @@ -301,4 +301,4 @@ public SessionState(Session session) public Session Session { get; } public DateTime LastAccessTime { get; set; } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/TableClient.cs b/src/Ydb.Sdk/src/Services/Table/TableClient.cs index 3ea510ad..7b0d4542 100644 --- a/src/Ydb.Sdk/src/Services/Table/TableClient.cs +++ b/src/Ydb.Sdk/src/Services/Table/TableClient.cs @@ -53,4 +53,4 @@ private void Dispose(bool disposing) _disposed = true; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Services/Table/Transaction.cs b/src/Ydb.Sdk/src/Services/Table/Transaction.cs index 0f667538..9628b5e3 100644 --- a/src/Ydb.Sdk/src/Services/Table/Transaction.cs +++ b/src/Ydb.Sdk/src/Services/Table/Transaction.cs @@ -93,4 +93,4 @@ internal TransactionControl ToProto() { return _proto.Clone(); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Status.cs b/src/Ydb.Sdk/src/Status.cs index 728e60f5..e4074b7c 100644 --- a/src/Ydb.Sdk/src/Status.cs +++ b/src/Ydb.Sdk/src/Status.cs @@ -242,4 +242,4 @@ public StatusUnsuccessfulException(Status status) } public Status Status { get; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Value/ResultSet.cs b/src/Ydb.Sdk/src/Value/ResultSet.cs index c67efd24..33f39851 100644 --- a/src/Ydb.Sdk/src/Value/ResultSet.cs +++ b/src/Ydb.Sdk/src/Value/ResultSet.cs @@ -130,4 +130,4 @@ internal Row(Ydb.Value row, IReadOnlyList columns, IReadOnlyDictionary this[_columnsMap[columnName]]; } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Value/YdbValue.cs b/src/Ydb.Sdk/src/Value/YdbValue.cs index 9dbcb686..ef34cff6 100644 --- a/src/Ydb.Sdk/src/Value/YdbValue.cs +++ b/src/Ydb.Sdk/src/Value/YdbValue.cs @@ -113,4 +113,4 @@ private static YdbTypeId GetYdbTypeId(Type protoType) return YdbTypeId.Unknown; } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Value/YdbValueBuilder.cs b/src/Ydb.Sdk/src/Value/YdbValueBuilder.cs index 075cbfac..57869375 100644 --- a/src/Ydb.Sdk/src/Value/YdbValueBuilder.cs +++ b/src/Ydb.Sdk/src/Value/YdbValueBuilder.cs @@ -548,4 +548,4 @@ public static YdbValue MakeOptionalDecimal(decimal? value) { return MakeOptionalOf(value, YdbTypeId.DecimalType, MakeDecimal); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Value/YdbValueCast.cs b/src/Ydb.Sdk/src/Value/YdbValueCast.cs index 859a7271..9260a418 100644 --- a/src/Ydb.Sdk/src/Value/YdbValueCast.cs +++ b/src/Ydb.Sdk/src/Value/YdbValueCast.cs @@ -326,4 +326,4 @@ public static explicit operator YdbValue(decimal? value) { return MakeOptionalDecimal(value); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/src/Value/YdbValueParser.cs b/src/Ydb.Sdk/src/Value/YdbValueParser.cs index b60cb1d3..67323780 100644 --- a/src/Ydb.Sdk/src/Value/YdbValueParser.cs +++ b/src/Ydb.Sdk/src/Value/YdbValueParser.cs @@ -324,4 +324,4 @@ internal InvalidTypeException(string expectedType, string actualType) { } } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs b/src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs index cb203da3..9b76eeb5 100644 --- a/src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs +++ b/src/Ydb.Sdk/tests/Auth/TestStaticAuth.cs @@ -119,4 +119,4 @@ public async Task NotExistAuth() await Assert.ThrowsAsync( async () => await Driver.CreateInitialized(driverConfig, _loggerFactory)); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/tests/Utils.cs b/src/Ydb.Sdk/tests/Utils.cs index dacd9438..76dedba4 100644 --- a/src/Ydb.Sdk/tests/Utils.cs +++ b/src/Ydb.Sdk/tests/Utils.cs @@ -53,4 +53,4 @@ internal static ServiceProvider GetServiceProvider() var serviceProvider = GetServiceProvider(); return serviceProvider.GetService(); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/tests/Value/TestBasicUnit.cs b/src/Ydb.Sdk/tests/Value/TestBasicUnit.cs index e101b395..5072580e 100644 --- a/src/Ydb.Sdk/tests/Value/TestBasicUnit.cs +++ b/src/Ydb.Sdk/tests/Value/TestBasicUnit.cs @@ -368,4 +368,4 @@ public void StructType() Assert.Equal(10, (long)s["Member1"]); Assert.Equal("ten", (string)s["Member2"]!); } -} \ No newline at end of file +} diff --git a/src/Ydb.Sdk/tests/Value/TestBasicsIntegration.cs b/src/Ydb.Sdk/tests/Value/TestBasicsIntegration.cs index 4b6bba90..97f998ea 100644 --- a/src/Ydb.Sdk/tests/Value/TestBasicsIntegration.cs +++ b/src/Ydb.Sdk/tests/Value/TestBasicsIntegration.cs @@ -399,4 +399,4 @@ public async Task StructType() Assert.Equal(10, foo2["bar1"].GetInt64()); Assert.Equal("ten", foo2["bar2"].GetUtf8()); } -} \ No newline at end of file +} diff --git a/src/YdbSdk.sln.DotSettings b/src/YdbSdk.sln.DotSettings index 0625ed27..67175064 100644 --- a/src/YdbSdk.sln.DotSettings +++ b/src/YdbSdk.sln.DotSettings @@ -66,4 +66,5 @@ &lt;Language id="yaml"&gt; &lt;Reformat&gt;true&lt;/Reformat&gt; &lt;/Language&gt; -&lt;/profile&gt;</RIDER_SETTINGS><CSUseAutoProperty>True</CSUseAutoProperty><CSMakeFieldReadonly>True</CSMakeFieldReadonly><CSMakeAutoPropertyGetOnly>True</CSMakeAutoPropertyGetOnly><RemoveCodeRedundancies>True</RemoveCodeRedundancies><CSOptimizeUsings><OptimizeUsings>True</OptimizeUsings></CSOptimizeUsings><CSShortenReferences>True</CSShortenReferences></Profile> \ No newline at end of file +&lt;/profile&gt;</RIDER_SETTINGS><CSUseAutoProperty>True</CSUseAutoProperty><CSMakeFieldReadonly>True</CSMakeFieldReadonly><CSMakeAutoPropertyGetOnly>True</CSMakeAutoPropertyGetOnly><RemoveCodeRedundancies>True</RemoveCodeRedundancies><CSOptimizeUsings><OptimizeUsings>True</OptimizeUsings></CSOptimizeUsings><CSShortenReferences>True</CSShortenReferences></Profile> + True \ No newline at end of file From 8b26fbbfe528c1a00ffd54424af01a6bb97e1646 Mon Sep 17 00:00:00 2001 From: XmasApple Date: Thu, 19 Oct 2023 14:07:05 +0300 Subject: [PATCH 13/26] fix .DotSettings file --- src/YdbSdk.sln.DotSettings | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/YdbSdk.sln.DotSettings b/src/YdbSdk.sln.DotSettings index 67175064..37d30088 100644 --- a/src/YdbSdk.sln.DotSettings +++ b/src/YdbSdk.sln.DotSettings @@ -1,4 +1,7 @@ - + <?xml version="1.0" encoding="utf-16"?><Profile name="Custom Cleanup"><CppCodeStyleCleanupDescriptor ArrangeBraces="True" ArrangeAuto="True" ArrangeFunctionDeclarations="True" ArrangeNestedNamespaces="True" ArrangeTypeAliases="True" ArrangeCVQualifiers="True" ArrangeSlashesInIncludeDirectives="True" ArrangeOverridingFunctions="True" SortIncludeDirectives="True" SortMemberInitializers="True" /><CppReformatCode>True</CppReformatCode><CSCodeStyleAttributes ArrangeVarStyle="True" ArrangeTypeAccessModifier="True" ArrangeTypeMemberAccessModifier="True" SortModifiers="True" AddMissingParentheses="True" ArrangeAttributes="True" ArrangeCodeBodyStyle="True" ArrangeTrailingCommas="True" ArrangeObjectCreation="True" ArrangeDefaultValue="True" RemoveRedundantParentheses="True" ArrangeNamespaces="True" /><FSReformatCode>True</FSReformatCode><ShaderLabReformatCode>True</ShaderLabReformatCode><Xaml.RedundantFreezeAttribute>True</Xaml.RedundantFreezeAttribute><Xaml.RemoveRedundantModifiersAttribute>True</Xaml.RemoveRedundantModifiersAttribute><Xaml.RemoveRedundantNameAttribute>True</Xaml.RemoveRedundantNameAttribute><Xaml.RemoveRedundantResource>True</Xaml.RemoveRedundantResource><Xaml.RemoveRedundantCollectionProperty>True</Xaml.RemoveRedundantCollectionProperty><Xaml.RemoveRedundantAttachedPropertySetter>True</Xaml.RemoveRedundantAttachedPropertySetter><Xaml.RemoveRedundantStyledValue>True</Xaml.RemoveRedundantStyledValue><Xaml.RemoveRedundantNamespaceAlias>True</Xaml.RemoveRedundantNamespaceAlias><Xaml.RemoveForbiddenResourceName>True</Xaml.RemoveForbiddenResourceName><Xaml.RemoveRedundantGridDefinitionsAttribute>True</Xaml.RemoveRedundantGridDefinitionsAttribute><Xaml.RemoveRedundantUpdateSourceTriggerAttribute>True</Xaml.RemoveRedundantUpdateSourceTriggerAttribute><Xaml.RemoveRedundantBindingModeAttribute>True</Xaml.RemoveRedundantBindingModeAttribute><Xaml.RemoveRedundantGridSpanAttribut>True</Xaml.RemoveRedundantGridSpanAttribut><XMLReformatCode>True</XMLReformatCode><CSArrangeQualifiers>True</CSArrangeQualifiers><CSFixBuiltinTypeReferences>True</CSFixBuiltinTypeReferences><HtmlReformatCode>True</HtmlReformatCode><VBReformatCode>True</VBReformatCode><CSReformatCode>True</CSReformatCode><FormatAttributeQuoteDescriptor>True</FormatAttributeQuoteDescriptor><IDEA_SETTINGS>&lt;profile version="1.0"&gt; &lt;option name="myName" value="Custom Cleanup" /&gt; &lt;inspection_tool class="ES6ShorthandObjectProperty" enabled="false" level="INFORMATION" enabled_by_default="false" /&gt; @@ -67,4 +70,4 @@ &lt;Reformat&gt;true&lt;/Reformat&gt; &lt;/Language&gt; &lt;/profile&gt;</RIDER_SETTINGS><CSUseAutoProperty>True</CSUseAutoProperty><CSMakeFieldReadonly>True</CSMakeFieldReadonly><CSMakeAutoPropertyGetOnly>True</CSMakeAutoPropertyGetOnly><RemoveCodeRedundancies>True</RemoveCodeRedundancies><CSOptimizeUsings><OptimizeUsings>True</OptimizeUsings></CSOptimizeUsings><CSShortenReferences>True</CSShortenReferences></Profile> - True \ No newline at end of file + True \ No newline at end of file From 32f31e592d31e3374cabdc8226dadf3e73795e2f Mon Sep 17 00:00:00 2001 From: XmasApple Date: Thu, 19 Oct 2023 15:30:07 +0300 Subject: [PATCH 14/26] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6fb188e0..e7498983 100644 --- a/README.md +++ b/README.md @@ -92,4 +92,4 @@ foreach (var row in resultSet.Rows) ## Examples -See **[ydb-dotnet-examples](https://github.com/ydb-platform/ydb-dotnet-examples)**. +See **[examples folder](https://github.com/ydb-platform/ydb-dotnet-sdk/tree/main/examples)** From 75dca07e74e06d2747bc4cf11a50ed6c5c157a5d Mon Sep 17 00:00:00 2001 From: XmasApple Date: Thu, 19 Oct 2023 18:03:04 +0300 Subject: [PATCH 15/26] Add test for TruncateException --- src/Ydb.Sdk/tests/Table/Truncated.cs | 79 ++++++++++++++++++++++++++++ src/Ydb.Sdk/tests/Utils.cs | 8 +++ 2 files changed, 87 insertions(+) create mode 100644 src/Ydb.Sdk/tests/Table/Truncated.cs diff --git a/src/Ydb.Sdk/tests/Table/Truncated.cs b/src/Ydb.Sdk/tests/Table/Truncated.cs new file mode 100644 index 00000000..bb4eea95 --- /dev/null +++ b/src/Ydb.Sdk/tests/Table/Truncated.cs @@ -0,0 +1,79 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; +using Ydb.Sdk.Services.Table; +using Ydb.Sdk.Value; + +namespace Ydb.Sdk.Tests.Table; + +[Trait("Category", "Integration")] +public class Truncated +{ + private readonly ILoggerFactory _loggerFactory; + + private readonly DriverConfig _driverConfig = new( + endpoint: "grpc://localhost:2136", + database: "/local" + ); + + public Truncated() + { + _loggerFactory = Utils.GetLoggerFactory() ?? NullLoggerFactory.Instance; + } + + + [Fact] + public async Task NotAllowTruncated() + { + await using var driver = await Driver.CreateInitialized(_driverConfig, _loggerFactory); + using var tableClient = new TableClient(driver); + + + const string query = "SELECT * FROM AS_TABLE($data)"; + var parameters = new Dictionary { { "$data", MakeData(1001) } }; + + await Assert.ThrowsAsync(async () => await tableClient.SessionExec(async session => + await session.ExecuteDataQuery( + query: query, + parameters: parameters, + txControl: TxControl.BeginSerializableRW().Commit() + ) + )); + } + + [Fact] + public async Task AllowTruncated() + { + await using var driver = await Driver.CreateInitialized(_driverConfig, _loggerFactory); + using var tableClient = new TableClient(driver); + + + const string query = "SELECT * FROM AS_TABLE($data)"; + var parameters = new Dictionary { { "$data", MakeData(1001) } }; + var settings = new ExecuteDataQuerySettings { AllowTruncated = true }; + + var response = await tableClient.SessionExec(async session => + await session.ExecuteDataQuery( + query: query, + parameters: parameters, + txControl: TxControl.BeginSerializableRW().Commit(), + settings: settings) + ); + + Assert.True(response.Status.IsSuccess); + } + + private static YdbValue MakeData(int n) + { + return YdbValue.MakeList( + Enumerable.Range(0, n) + .Select(i => YdbValue.MakeStruct( + new Dictionary + { + { "id", (YdbValue)i } + } + )) + .ToList() + ); + } +} \ No newline at end of file diff --git a/src/Ydb.Sdk/tests/Utils.cs b/src/Ydb.Sdk/tests/Utils.cs index dacd9438..f76b1fb6 100644 --- a/src/Ydb.Sdk/tests/Utils.cs +++ b/src/Ydb.Sdk/tests/Utils.cs @@ -53,4 +53,12 @@ internal static ServiceProvider GetServiceProvider() var serviceProvider = GetServiceProvider(); return serviceProvider.GetService(); } + + internal static async Task CreateSimpleTable( + TableClient tableClient, string tableName, string columnName = "key") + { + return await ExecuteSchemeQuery( + tableClient, + query: $"CREATE TABLE {tableName} ({columnName} Uint64, PRIMARY KEY ({columnName}))"); + } } \ No newline at end of file From 839410b7fd06c68dda9dceec17e6019c5415af38 Mon Sep 17 00:00:00 2001 From: XmasApple Date: Thu, 19 Oct 2023 18:13:52 +0300 Subject: [PATCH 16/26] remove redundant value initialization --- src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs b/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs index eabf5afe..c5360691 100644 --- a/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs +++ b/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs @@ -8,7 +8,7 @@ namespace Ydb.Sdk.Services.Table; public class ExecuteDataQuerySettings : OperationRequestSettings { public bool KeepInQueryCache { get; set; } = true; - public bool AllowTruncated { get; set; } = false; + public bool AllowTruncated { get; set; } } public class ExecuteDataQueryResponse : ResponseWithResultBase From 6e0b6d40aa32b0f6e594a8b03cf2442068c7aa3d Mon Sep 17 00:00:00 2001 From: XmasApple Date: Fri, 20 Oct 2023 20:33:45 +0300 Subject: [PATCH 17/26] Fix styles --- .github/workflows/lint.yml | 2 +- src/Ydb.Sdk/tests/Table/Truncated.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 939872ba..7f192afd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -23,7 +23,7 @@ jobs: - name: Restore run: dotnet restore ./src/YdbSdk.sln - name: Install ReSharper - run: dotnet tool install -g JetBrains.ReSharper.GlobalTools + run: dotnet tool install -g JetBrains.ReSharper.GlobalTools --version 2023.2.1 - name: format all files with auto-formatter run: bash ./.github/scripts/format-all-dotnet-code.sh ./src/ YdbSdk.sln "Custom Cleanup" - name: Check repository diff diff --git a/src/Ydb.Sdk/tests/Table/Truncated.cs b/src/Ydb.Sdk/tests/Table/Truncated.cs index bb4eea95..bba8b5d5 100644 --- a/src/Ydb.Sdk/tests/Table/Truncated.cs +++ b/src/Ydb.Sdk/tests/Table/Truncated.cs @@ -76,4 +76,4 @@ private static YdbValue MakeData(int n) .ToList() ); } -} \ No newline at end of file +} From 3d74048281f20e6d4ac5f1386f3f866ce69393fc Mon Sep 17 00:00:00 2001 From: XmasApple Date: Fri, 20 Oct 2023 20:40:16 +0300 Subject: [PATCH 18/26] fix .DotSettings file --- src/YdbSdk.sln.DotSettings | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/YdbSdk.sln.DotSettings b/src/YdbSdk.sln.DotSettings index 37d30088..84941aaf 100644 --- a/src/YdbSdk.sln.DotSettings +++ b/src/YdbSdk.sln.DotSettings @@ -1,7 +1,4 @@ - + <?xml version="1.0" encoding="utf-16"?><Profile name="Custom Cleanup"><CppCodeStyleCleanupDescriptor ArrangeBraces="True" ArrangeAuto="True" ArrangeFunctionDeclarations="True" ArrangeNestedNamespaces="True" ArrangeTypeAliases="True" ArrangeCVQualifiers="True" ArrangeSlashesInIncludeDirectives="True" ArrangeOverridingFunctions="True" SortIncludeDirectives="True" SortMemberInitializers="True" /><CppReformatCode>True</CppReformatCode><CSCodeStyleAttributes ArrangeVarStyle="True" ArrangeTypeAccessModifier="True" ArrangeTypeMemberAccessModifier="True" SortModifiers="True" AddMissingParentheses="True" ArrangeAttributes="True" ArrangeCodeBodyStyle="True" ArrangeTrailingCommas="True" ArrangeObjectCreation="True" ArrangeDefaultValue="True" RemoveRedundantParentheses="True" ArrangeNamespaces="True" /><FSReformatCode>True</FSReformatCode><ShaderLabReformatCode>True</ShaderLabReformatCode><Xaml.RedundantFreezeAttribute>True</Xaml.RedundantFreezeAttribute><Xaml.RemoveRedundantModifiersAttribute>True</Xaml.RemoveRedundantModifiersAttribute><Xaml.RemoveRedundantNameAttribute>True</Xaml.RemoveRedundantNameAttribute><Xaml.RemoveRedundantResource>True</Xaml.RemoveRedundantResource><Xaml.RemoveRedundantCollectionProperty>True</Xaml.RemoveRedundantCollectionProperty><Xaml.RemoveRedundantAttachedPropertySetter>True</Xaml.RemoveRedundantAttachedPropertySetter><Xaml.RemoveRedundantStyledValue>True</Xaml.RemoveRedundantStyledValue><Xaml.RemoveRedundantNamespaceAlias>True</Xaml.RemoveRedundantNamespaceAlias><Xaml.RemoveForbiddenResourceName>True</Xaml.RemoveForbiddenResourceName><Xaml.RemoveRedundantGridDefinitionsAttribute>True</Xaml.RemoveRedundantGridDefinitionsAttribute><Xaml.RemoveRedundantUpdateSourceTriggerAttribute>True</Xaml.RemoveRedundantUpdateSourceTriggerAttribute><Xaml.RemoveRedundantBindingModeAttribute>True</Xaml.RemoveRedundantBindingModeAttribute><Xaml.RemoveRedundantGridSpanAttribut>True</Xaml.RemoveRedundantGridSpanAttribut><XMLReformatCode>True</XMLReformatCode><CSArrangeQualifiers>True</CSArrangeQualifiers><CSFixBuiltinTypeReferences>True</CSFixBuiltinTypeReferences><HtmlReformatCode>True</HtmlReformatCode><VBReformatCode>True</VBReformatCode><CSReformatCode>True</CSReformatCode><FormatAttributeQuoteDescriptor>True</FormatAttributeQuoteDescriptor><IDEA_SETTINGS>&lt;profile version="1.0"&gt; &lt;option name="myName" value="Custom Cleanup" /&gt; &lt;inspection_tool class="ES6ShorthandObjectProperty" enabled="false" level="INFORMATION" enabled_by_default="false" /&gt; @@ -70,4 +67,8 @@ &lt;Reformat&gt;true&lt;/Reformat&gt; &lt;/Language&gt; &lt;/profile&gt;</RIDER_SETTINGS><CSUseAutoProperty>True</CSUseAutoProperty><CSMakeFieldReadonly>True</CSMakeFieldReadonly><CSMakeAutoPropertyGetOnly>True</CSMakeAutoPropertyGetOnly><RemoveCodeRedundancies>True</RemoveCodeRedundancies><CSOptimizeUsings><OptimizeUsings>True</OptimizeUsings></CSOptimizeUsings><CSShortenReferences>True</CSShortenReferences></Profile> - True \ No newline at end of file + True + True + True + True + True \ No newline at end of file From 19db82d93b9273edea30273d0e2d6b7fe6101a7e Mon Sep 17 00:00:00 2001 From: robot Date: Thu, 26 Oct 2023 11:27:33 +0000 Subject: [PATCH 19/26] Release v0.1.4 --- CHANGELOG.md | 1 + src/Ydb.Sdk/src/Version.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4dfdd105..c7effa80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +## v0.1.4 - Add exception throwing when results truncated - lint: add line feed at file end diff --git a/src/Ydb.Sdk/src/Version.cs b/src/Ydb.Sdk/src/Version.cs index 55186568..c9154a84 100644 --- a/src/Ydb.Sdk/src/Version.cs +++ b/src/Ydb.Sdk/src/Version.cs @@ -4,5 +4,5 @@ public static class Verison { public const uint Major = 0; public const uint Minor = 1; - public const uint Patch = 3; + public const uint Patch = 4; } From b0b0dc22c1199224ea16b0265a0b01bc05dc0d22 Mon Sep 17 00:00:00 2001 From: XmasApple <86735308+XmasApple@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:39:46 +0300 Subject: [PATCH 20/26] Graceful session shutdown support (#47) * Graceful session shutdown support --- src/Ydb.Sdk/src/Driver.cs | 17 ++-- src/Ydb.Sdk/src/Metadata.cs | 3 + .../src/Services/Table/ExecuteDataQuery.cs | 3 +- src/Ydb.Sdk/src/Services/Table/Session.cs | 25 ++++-- .../tests/Table/TestGracefulShutdown.cs | 82 +++++++++++++++++++ 5 files changed, 118 insertions(+), 12 deletions(-) create mode 100644 src/Ydb.Sdk/tests/Table/TestGracefulShutdown.cs diff --git a/src/Ydb.Sdk/src/Driver.cs b/src/Ydb.Sdk/src/Driver.cs index ad505d59..a9a04344 100644 --- a/src/Ydb.Sdk/src/Driver.cs +++ b/src/Ydb.Sdk/src/Driver.cs @@ -117,15 +117,19 @@ internal async Task> UnaryCall( try { - var data = await callInvoker.AsyncUnaryCall( + using var call = callInvoker.AsyncUnaryCall( method: method, host: null, options: GetCallOptions(settings, false), request: request); + var data = await call.ResponseAsync; + var trailers = call.GetTrailers(); + return new UnaryResponse( data: data, - usedEndpoint: endpoint); + usedEndpoint: endpoint, + trailers: trailers); } catch (RpcException e) { @@ -312,17 +316,20 @@ private static Status ConvertStatus(Grpc.Core.Status rpcStatus) internal sealed class UnaryResponse { - internal UnaryResponse( - TResponse data, - string usedEndpoint) + internal UnaryResponse(TResponse data, + string usedEndpoint, + Grpc.Core.Metadata? trailers) { Data = data; UsedEndpoint = usedEndpoint; + Trailers = trailers; } public TResponse Data { get; } public string UsedEndpoint { get; } + + public Grpc.Core.Metadata? Trailers { get; } } internal sealed class StreamIterator diff --git a/src/Ydb.Sdk/src/Metadata.cs b/src/Ydb.Sdk/src/Metadata.cs index b49b8600..0b30c802 100644 --- a/src/Ydb.Sdk/src/Metadata.cs +++ b/src/Ydb.Sdk/src/Metadata.cs @@ -7,4 +7,7 @@ internal static class Metadata public const string RpcRequestTypeHeader = "x-ydb-request-type"; public const string RpcTraceIdHeader = "x-ydb-trace-id"; public const string RpcSdkInfoHeader = "x-ydb-sdk-build-info"; + public const string RpcServerHintsHeader = "x-ydb-server-hints"; + + public const string GracefulShutdownHint = "session-close"; } diff --git a/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs b/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs index b0b5f380..a01681a5 100644 --- a/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs +++ b/src/Ydb.Sdk/src/Services/Table/ExecuteDataQuery.cs @@ -85,8 +85,7 @@ public async Task ExecuteDataQuery( request: request, settings: settings); - ExecuteQueryResult? resultProto; - var status = UnpackOperation(response.Data.Operation, out resultProto); + var status = UnpackOperation(response.Data.Operation, out ExecuteQueryResult? resultProto); OnResponseStatus(status); var txState = TransactionState.Unknown; diff --git a/src/Ydb.Sdk/src/Services/Table/Session.cs b/src/Ydb.Sdk/src/Services/Table/Session.cs index ffee604c..2e3b1706 100644 --- a/src/Ydb.Sdk/src/Services/Table/Session.cs +++ b/src/Ydb.Sdk/src/Services/Table/Session.cs @@ -35,11 +35,24 @@ private void CheckSession() private void OnResponseStatus(Status status) { - if (status.StatusCode == StatusCode.BadSession || status.StatusCode == StatusCode.SessionBusy) + if (status.StatusCode is StatusCode.BadSession or StatusCode.SessionBusy) { - if (_sessionPool != null) + _sessionPool?.InvalidateSession(Id); + } + } + + private void OnResponseTrailers(Grpc.Core.Metadata? trailers) + { + if (trailers is null) + { + return; + } + + foreach (var hint in trailers.GetAll(Metadata.RpcServerHintsHeader)) + { + if (hint.Value == Metadata.GracefulShutdownHint) { - _sessionPool.InvalidateSession(Id); + _sessionPool?.InvalidateSession(Id); } } } @@ -77,18 +90,20 @@ protected virtual void Dispose(bool disposing) _disposed = true; } - internal Task> UnaryCall( + internal async Task> UnaryCall( Method method, TRequest request, RequestSettings settings) where TRequest : class where TResponse : class { - return Driver.UnaryCall( + var response = await Driver.UnaryCall( method: method, request: request, settings: settings, preferredEndpoint: Endpoint ); + OnResponseTrailers(response.Trailers); + return response; } } diff --git a/src/Ydb.Sdk/tests/Table/TestGracefulShutdown.cs b/src/Ydb.Sdk/tests/Table/TestGracefulShutdown.cs new file mode 100644 index 00000000..ccd9950d --- /dev/null +++ b/src/Ydb.Sdk/tests/Table/TestGracefulShutdown.cs @@ -0,0 +1,82 @@ +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; +using Ydb.Sdk.Services.Table; + +namespace Ydb.Sdk.Tests.Table; + +[Trait("Category", "Integration")] +public class TestGracefulShutdown +{ + private readonly ILoggerFactory _loggerFactory; + + private readonly DriverConfig _driverConfig = new( + endpoint: "grpc://localhost:2136", + database: "/local" + ); + + private const string ShutdownUrl = "http://localhost:8765/actors/kqp_proxy?force_shutdown=all"; + + public TestGracefulShutdown() + { + _loggerFactory = Utils.GetLoggerFactory() ?? NullLoggerFactory.Instance; + } + + [Fact] + public async Task Test() + { + await using var driver = await Driver.CreateInitialized(_driverConfig, _loggerFactory); + using var tableClient = new TableClient(driver); + + + var session1 = ""; + await tableClient.SessionExec( + async session => + { + session1 = session.Id; + return await session.ExecuteDataQuery("SELECT 1", TxControl.BeginSerializableRW().Commit()); + } + ); + + var session2 = ""; + await tableClient.SessionExec( + async session => + { + session2 = session.Id; + return await session.ExecuteDataQuery("SELECT 1", TxControl.BeginSerializableRW().Commit()); + } + ); + + // control check + Assert.NotEqual("", session1); + Assert.Equal(session1, session2); + + // SHUTDOWN + using var httpClient = new HttpClient(); + await httpClient.GetAsync(ShutdownUrl); + + // new session + var session3 = ""; + await tableClient.SessionExec( + async session => + { + session3 = session.Id; + return await session.ExecuteDataQuery("SELECT 1", TxControl.BeginSerializableRW().Commit()); + } + ); + + Assert.Equal(session2, session3); + + var session4 = ""; + await tableClient.SessionExec( + async session => + { + session4 = session.Id; + return await session.ExecuteDataQuery("SELECT 1", TxControl.BeginSerializableRW().Commit()); + } + ); + + Assert.NotEqual("", session3); + Assert.NotEqual(session3, session4); + } +} From 84b25c76a831def3e4cc35c72ce4f8601238740f Mon Sep 17 00:00:00 2001 From: XmasApple <86735308+XmasApple@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:33:06 +0300 Subject: [PATCH 21/26] Add linter for examples (#55) * Add linter for examples --- .github/workflows/lint.yml | 22 +++++++++++++++------- examples/src/BasicExample/FillData.cs | 4 ++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7f192afd..c9b629db 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -8,10 +8,15 @@ on: jobs: autoformatter: + strategy: + matrix: + source-dir: ["./src/", "./examples/src/"] + include: + - source-dir: "./src/" + solutionFile: "YdbSdk.sln" + - source-dir: "./examples/src/" + solutionFile: "YdbExamples.sln" name: autoformat check - concurrency: - group: lint-autoformat-${{ github.ref }} - cancel-in-progress: true runs-on: ubuntu-latest steps: - name: Checkout @@ -21,15 +26,18 @@ jobs: with: dotnet-version: '6.0.x' - name: Restore - run: dotnet restore ./src/YdbSdk.sln + run: dotnet restore ${{ matrix.source-dir }}${{ matrix.solutionFile }} - name: Install ReSharper run: dotnet tool install -g JetBrains.ReSharper.GlobalTools --version 2023.2.1 - name: format all files with auto-formatter - run: bash ./.github/scripts/format-all-dotnet-code.sh ./src/ YdbSdk.sln "Custom Cleanup" + run: bash ./.github/scripts/format-all-dotnet-code.sh ${{ matrix.source-dir }} ${{ matrix.solutionFile }} "Custom Cleanup" - name: Check repository diff run: bash ./.github/scripts/check-work-copy-equals-to-committed.sh "auto-format broken" inspection: + strategy: + matrix: + solutionPath: ["./src/YdbSdk.sln", "./examples/src/YdbExamples.sln"] runs-on: ubuntu-latest name: Inspection steps: @@ -40,11 +48,11 @@ jobs: with: dotnet-version: '6.0.x' - name: Restore - run: dotnet restore ./src/YdbSdk.sln + run: dotnet restore ${{ matrix.solutionPath }} - name: Inspect code uses: muno92/resharper_inspectcode@v1 with: - solutionPath: ./src/YdbSdk.sln + solutionPath: ${{ matrix.solutionPath }} version: 2023.2.1 include: | **.cs diff --git a/examples/src/BasicExample/FillData.cs b/examples/src/BasicExample/FillData.cs index 75949fe5..b4f2fb48 100644 --- a/examples/src/BasicExample/FillData.cs +++ b/examples/src/BasicExample/FillData.cs @@ -88,7 +88,7 @@ private static Dictionary GetDataParams() new(2, 2, "Season 2", DateTime.Parse("2015-04-12"), DateTime.Parse("2015-06-14")), new(2, 3, "Season 3", DateTime.Parse("2016-04-24"), DateTime.Parse("2016-06-26")), new(2, 4, "Season 4", DateTime.Parse("2017-04-23"), DateTime.Parse("2017-06-25")), - new(2, 5, "Season 5", DateTime.Parse("2018-03-25"), DateTime.Parse("2018-05-13")), + new(2, 5, "Season 5", DateTime.Parse("2018-03-25"), DateTime.Parse("2018-05-13")) }; var episodes = new Episode[] @@ -162,7 +162,7 @@ private static Dictionary GetDataParams() new(2, 5, 5, "Facial Recognition", DateTime.Parse("2018-04-22")), new(2, 5, 6, "Artificial Emotional Intelligence", DateTime.Parse("2018-04-29")), new(2, 5, 7, "Initial Coin Offering", DateTime.Parse("2018-05-06")), - new(2, 5, 8, "Fifty-One Percent", DateTime.Parse("2018-05-13")), + new(2, 5, 8, "Fifty-One Percent", DateTime.Parse("2018-05-13")) }; var seriesData = series.Select(s => YdbValue.MakeStruct(new Dictionary From e5dec2e33744e118248928bc74d2e225e2b6cc1d Mon Sep 17 00:00:00 2001 From: XmasApple <86735308+XmasApple@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:53:17 +0300 Subject: [PATCH 22/26] Fix transport error on delete session (#56) Fix transport error on delete session --- CHANGELOG.md | 3 +++ src/Ydb.Sdk/src/Services/Table/CreateSession.cs | 3 +-- src/Ydb.Sdk/src/Services/Table/Session.cs | 3 ++- src/Ydb.Sdk/src/Services/Table/SessionPool.cs | 13 ++++++++----- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7effa80..24e8a177 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +- Fix timeout error on create session +- Fix transport error on delete session + ## v0.1.4 - Add exception throwing when results truncated - lint: add line feed at file end diff --git a/src/Ydb.Sdk/src/Services/Table/CreateSession.cs b/src/Ydb.Sdk/src/Services/Table/CreateSession.cs index 1b5f640c..e1df62a6 100644 --- a/src/Ydb.Sdk/src/Services/Table/CreateSession.cs +++ b/src/Ydb.Sdk/src/Services/Table/CreateSession.cs @@ -57,8 +57,7 @@ public async Task CreateSession(CreateSessionSettings? se request: request, settings: settings); - CreateSessionResult? resultProto; - var status = UnpackOperation(response.Data.Operation, out resultProto); + var status = UnpackOperation(response.Data.Operation, out CreateSessionResult? resultProto); CreateSessionResponse.ResultData? result = null; if (status.IsSuccess && resultProto != null) diff --git a/src/Ydb.Sdk/src/Services/Table/Session.cs b/src/Ydb.Sdk/src/Services/Table/Session.cs index 2e3b1706..bb70201c 100644 --- a/src/Ydb.Sdk/src/Services/Table/Session.cs +++ b/src/Ydb.Sdk/src/Services/Table/Session.cs @@ -76,10 +76,11 @@ protected virtual void Dispose(bool disposing) _logger.LogTrace($"Closing detached session on dispose: {Id}"); var client = new TableClient(Driver, new NoPool()); - _ = client.DeleteSession(Id, new DeleteSessionSettings + var task = client.DeleteSession(Id, new DeleteSessionSettings { TransportTimeout = DeleteSessionTimeout }); + task.Wait(); } else { diff --git a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs index cf4c0115..15b769f6 100644 --- a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs +++ b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs @@ -90,8 +90,7 @@ public async Task GetSession() { var sessionId = _idleSessions.Pop(); - SessionState? sessionState; - if (!_sessions.TryGetValue(sessionId, out sessionState)) + if (!_sessions.TryGetValue(sessionId, out var sessionState)) { continue; } @@ -152,8 +151,7 @@ internal void ReturnSession(string id) { lock (_lock) { - SessionState? oldSession; - if (_sessions.TryGetValue(id, out oldSession)) + if (_sessions.TryGetValue(id, out var oldSession)) { var session = new Session( driver: _driver, @@ -275,15 +273,20 @@ private void Dispose(bool disposing) if (disposing) { + var tasks = new Task[_sessions.Count]; + var i = 0; foreach (var state in _sessions.Values) { _logger.LogTrace($"Closing session on session pool dispose: {state.Session.Id}"); - _ = _client.DeleteSession(state.Session.Id, new DeleteSessionSettings + var task = _client.DeleteSession(state.Session.Id, new DeleteSessionSettings { TransportTimeout = Session.DeleteSessionTimeout }); + tasks[i++] = task; } + + Task.WaitAll(tasks); } _disposed = true; From 5f82083f8231394ca1adb2a595509021f26b0b42 Mon Sep 17 00:00:00 2001 From: robot Date: Tue, 21 Nov 2023 13:01:13 +0000 Subject: [PATCH 23/26] Release v0.1.5 --- CHANGELOG.md | 1 + src/Ydb.Sdk/src/Version.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 24e8a177..5f25bbaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ +## v0.1.5 - Fix timeout error on create session - Fix transport error on delete session diff --git a/src/Ydb.Sdk/src/Version.cs b/src/Ydb.Sdk/src/Version.cs index c9154a84..0f0dfac8 100644 --- a/src/Ydb.Sdk/src/Version.cs +++ b/src/Ydb.Sdk/src/Version.cs @@ -4,5 +4,5 @@ public static class Verison { public const uint Major = 0; public const uint Minor = 1; - public const uint Patch = 4; + public const uint Patch = 5; } From 56798890d0735e83124bea57fbb59316ee9e2708 Mon Sep 17 00:00:00 2001 From: XmasApple Date: Thu, 28 Sep 2023 17:19:54 +0300 Subject: [PATCH 24/26] Make SessionPoolConfig properties public --- src/Ydb.Sdk/src/Services/Table/SessionPool.cs | 20 +++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs index cf4c0115..111b2a8d 100644 --- a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs +++ b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs @@ -6,17 +6,25 @@ namespace Ydb.Sdk.Services.Table; public class SessionPoolConfig { public SessionPoolConfig( - uint? sizeLimit = null) + uint? sizeLimit = null, + TimeSpan? keepAliveIdleThreshold = null, + TimeSpan? periodicCheckInterval = null, + TimeSpan? keepAliveTimeout = null, + TimeSpan? createSessionTimeout = null + ) { SizeLimit = sizeLimit ?? 100; + KeepAliveIdleThreshold = keepAliveIdleThreshold ?? TimeSpan.FromMinutes(5); + PeriodicCheckInterval = periodicCheckInterval ?? TimeSpan.FromSeconds(10); + KeepAliveTimeout = keepAliveTimeout ?? TimeSpan.FromSeconds(1); + CreateSessionTimeout = createSessionTimeout ?? TimeSpan.FromSeconds(1); } public uint SizeLimit { get; } - - internal TimeSpan KeepAliveIdleThreshold { get; } = TimeSpan.FromMinutes(5); - internal TimeSpan PeriodicCheckInterval { get; } = TimeSpan.FromSeconds(10); - internal TimeSpan KeepAliveTimeout { get; } = TimeSpan.FromSeconds(1); - internal TimeSpan CreateSessionTimeout { get; } = TimeSpan.FromSeconds(1); + public TimeSpan KeepAliveIdleThreshold { get; } + public TimeSpan PeriodicCheckInterval { get; } + public TimeSpan KeepAliveTimeout { get; } + public TimeSpan CreateSessionTimeout { get; } } internal class GetSessionResponse : ResponseWithResultBase, IDisposable From 1264518aa71ff4bd8aca6cffe925d0c4c2499b84 Mon Sep 17 00:00:00 2001 From: XmasApple Date: Thu, 28 Sep 2023 17:26:29 +0300 Subject: [PATCH 25/26] Add retry to GetSession in SessionPool.cs --- src/Ydb.Sdk/src/Services/Table/SessionPool.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs index 111b2a8d..2edf7fd1 100644 --- a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs +++ b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs @@ -91,6 +91,20 @@ public SessionPool(Driver driver, SessionPoolConfig config) } public async Task GetSession() + { + const int maxAttempts = 100; + + GetSessionResponse getSessionResponse = null!; + for (var attempt = 0; attempt < maxAttempts; attempt++) + { + getSessionResponse = await GetSessionAttempt(); + } + + _logger.LogError($"Failed to get session from pool or create it (attempts: {maxAttempts})"); + return getSessionResponse; + } + + private async Task GetSessionAttempt() { lock (_lock) { From 808410d72148002dc288b7ce182595e9f38844c4 Mon Sep 17 00:00:00 2001 From: XmasApple Date: Thu, 28 Sep 2023 17:35:44 +0300 Subject: [PATCH 26/26] Add leave condition to getSession retryer --- src/Ydb.Sdk/src/Services/Table/SessionPool.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs index 2edf7fd1..667fc112 100644 --- a/src/Ydb.Sdk/src/Services/Table/SessionPool.cs +++ b/src/Ydb.Sdk/src/Services/Table/SessionPool.cs @@ -98,6 +98,7 @@ public async Task GetSession() for (var attempt = 0; attempt < maxAttempts; attempt++) { getSessionResponse = await GetSessionAttempt(); + if (getSessionResponse.Status.IsSuccess) return getSessionResponse; } _logger.LogError($"Failed to get session from pool or create it (attempts: {maxAttempts})");