From 2a058ea39a1384b2d3f30c3de386b344e3ac0f7a Mon Sep 17 00:00:00 2001 From: Bradley Grainger Date: Sat, 28 Oct 2023 20:17:10 -0700 Subject: [PATCH] Move metrics to MetricsReporter class. Make min/max connections metrics an ObservableUpDownCounter, as they are effectively static metrics for a pool. Signed-off-by: Bradley Grainger --- src/MySqlConnector/Core/ConnectionPool.cs | 69 ++++++++----------- src/MySqlConnector/Core/MetricsReporter.cs | 57 +++++++++++++++ src/MySqlConnector/Core/ServerSession.cs | 4 +- src/MySqlConnector/MySqlConnection.cs | 4 +- .../Metrics/MetricsTestsBase.cs | 8 +++ 5 files changed, 96 insertions(+), 46 deletions(-) create mode 100644 src/MySqlConnector/Core/MetricsReporter.cs diff --git a/src/MySqlConnector/Core/ConnectionPool.cs b/src/MySqlConnector/Core/ConnectionPool.cs index bf020186b..674a01300 100644 --- a/src/MySqlConnector/Core/ConnectionPool.cs +++ b/src/MySqlConnector/Core/ConnectionPool.cs @@ -20,8 +20,6 @@ internal sealed class ConnectionPool : IDisposable public SslProtocols SslProtocols { get; set; } - public void AddPendingRequestCount(int delta) => s_pendingRequestsCounter.Add(delta, PoolNameTagList); - public async ValueTask GetSessionAsync(MySqlConnection connection, int startTickCount, int timeoutMilliseconds, Activity? activity, IOBehavior ioBehavior, CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); @@ -53,14 +51,14 @@ public async ValueTask GetSessionAsync(MySqlConnection connection { if (m_sessions.Count > 0) { - // NOTE: s_connectionsUsageCounter updated outside lock below + // NOTE: MetricsReporter updated outside lock below session = m_sessions.First!.Value; m_sessions.RemoveFirst(); } } if (session is not null) { - s_connectionsUsageCounter.Add(-1, IdleStateTagList); + MetricsReporter.RemoveIdle(this); Log.FoundExistingSession(m_logger, Id); bool reuseSession; @@ -101,12 +99,12 @@ public async ValueTask GetSessionAsync(MySqlConnection connection m_leasedSessions.Add(session.Id, session); leasedSessionsCountPooled = m_leasedSessions.Count; } - s_connectionsUsageCounter.Add(1, UsedStateTagList); + MetricsReporter.AddUsed(this); ActivitySourceHelper.CopyTags(session.ActivityTags, activity); Log.ReturningPooledSession(m_logger, Id, session.Id, leasedSessionsCountPooled); session.LastLeasedTicks = unchecked((uint) Environment.TickCount); - s_waitTimeHistory.Record(unchecked(session.LastLeasedTicks - (uint) startTickCount), PoolNameTagList); + MetricsReporter.RecordWaitTime(this, unchecked(session.LastLeasedTicks - (uint) startTickCount)); return session; } } @@ -121,11 +119,11 @@ public async ValueTask GetSessionAsync(MySqlConnection connection m_leasedSessions.Add(session.Id, session); leasedSessionsCountNew = m_leasedSessions.Count; } - s_connectionsUsageCounter.Add(1, UsedStateTagList); + MetricsReporter.AddUsed(this); Log.ReturningNewSession(m_logger, Id, session.Id, leasedSessionsCountNew); session.LastLeasedTicks = unchecked((uint) Environment.TickCount); - s_createTimeHistory.Record(unchecked(session.LastLeasedTicks - (uint) startTickCount), PoolNameTagList); + MetricsReporter.RecordCreateTime(this, unchecked(session.LastLeasedTicks - (uint) startTickCount)); return session; } catch (Exception ex) @@ -177,14 +175,14 @@ public async ValueTask ReturnAsync(IOBehavior ioBehavior, ServerSession session) { lock (m_leasedSessions) m_leasedSessions.Remove(session.Id); - s_connectionsUsageCounter.Add(-1, UsedStateTagList); + MetricsReporter.RemoveUsed(this); session.OwningConnection = null; var sessionHealth = GetSessionHealth(session); if (sessionHealth == 0) { lock (m_sessions) m_sessions.AddFirst(session); - s_connectionsUsageCounter.Add(1, IdleStateTagList); + MetricsReporter.AddIdle(this); } else { @@ -239,6 +237,8 @@ public async Task ReapAsync(IOBehavior ioBehavior, CancellationToken cancellatio public void Dispose() { Log.DisposingConnectionPool(m_logger, Id); + lock (s_allPools) + s_allPools.Remove(this); #if NET6_0_OR_GREATER m_dnsCheckTimer?.Dispose(); m_dnsCheckTimer = null; @@ -258,10 +258,6 @@ public void Dispose() reaperWaitHandle.WaitOne(); } #endif - - s_minIdleConnectionsCounter.Add(-ConnectionSettings.MinimumPoolSize, PoolNameTagList); - s_maxIdleConnectionsCounter.Add(-ConnectionSettings.MaximumPoolSize, PoolNameTagList); - s_maxConnectionsCounter.Add(-ConnectionSettings.MaximumPoolSize, PoolNameTagList); } /// @@ -345,14 +341,14 @@ private async Task CleanPoolAsync(IOBehavior ioBehavior, Func 0) { - // NOTE: s_connectionsUsageCounter updated outside lock below + // NOTE: MetricsReporter updated outside lock below session = m_sessions.Last!.Value; m_sessions.RemoveLast(); } } if (session is null) return; - s_connectionsUsageCounter.Add(-1, IdleStateTagList); + MetricsReporter.RemoveIdle(this); if (shouldCleanFn(session)) { @@ -365,7 +361,7 @@ private async Task CleanPoolAsync(IOBehavior ioBehavior, Func ConnectSessionAsync(MySqlConnection conne else if (pool != newPool) { Log.CreatedPoolWillNotBeUsed(newPool.m_logger, newPool.Id); + newPool.Dispose(); } return pool; @@ -576,10 +573,10 @@ private async ValueTask ConnectSessionAsync(MySqlConnection conne public static async Task ClearPoolsAsync(IOBehavior ioBehavior, CancellationToken cancellationToken) { - foreach (var pool in GetAllPools()) + foreach (var pool in GetCachedPools()) await pool.ClearAsync(ioBehavior, cancellationToken).ConfigureAwait(false); - static List GetAllPools() + static List GetCachedPools() { var pools = new List(s_pools.Count); var uniquePools = new HashSet(); @@ -626,12 +623,9 @@ private ConnectionPool(MySqlConnectorLoggingConfiguration loggingConfiguration, new("state", "used"), ]; - // set pool size counters - s_minIdleConnectionsCounter.Add(ConnectionSettings.MinimumPoolSize, PoolNameTagList); - s_maxIdleConnectionsCounter.Add(ConnectionSettings.MaximumPoolSize, PoolNameTagList); - s_maxConnectionsCounter.Add(ConnectionSettings.MaximumPoolSize, PoolNameTagList); - Id = Interlocked.Increment(ref s_poolId); + lock (s_allPools) + s_allPools.Add(this); Log.CreatingNewConnectionPool(m_logger, Id, connectionString); } @@ -779,12 +773,18 @@ private void AdjustHostConnectionCount(ServerSession session, int delta) } // Provides a slice of m_stateTagList that contains either the 'idle' or 'used' state tag along with the pool name. - private ReadOnlySpan> IdleStateTagList => m_stateTagList.AsSpan(0, 2); - private ReadOnlySpan> UsedStateTagList => m_stateTagList.AsSpan(1, 2); + public ReadOnlySpan> IdleStateTagList => m_stateTagList.AsSpan(0, 2); + public ReadOnlySpan> UsedStateTagList => m_stateTagList.AsSpan(1, 2); // A slice of m_stateTagList that contains only the pool name tag. public ReadOnlySpan> PoolNameTagList => m_stateTagList.AsSpan(1, 1); + public static List GetAllPools() + { + lock (s_allPools) + return new(s_allPools); + } + private sealed class LeastConnectionsLoadBalancer(Dictionary hostSessions) : ILoadBalancer { public IReadOnlyList LoadBalance(IReadOnlyList hosts) @@ -809,21 +809,8 @@ static ConnectionPool() private static void OnAppDomainShutDown(object? sender, EventArgs e) => ClearPoolsAsync(IOBehavior.Synchronous, CancellationToken.None).GetAwaiter().GetResult(); - private static readonly UpDownCounter s_connectionsUsageCounter = ActivitySourceHelper.Meter.CreateUpDownCounter("db.client.connections.usage", - unit: "{connection}", description: "The number of connections that are currently in the state described by the state tag."); - private static readonly UpDownCounter s_maxIdleConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter("db.client.connections.idle.max", - unit: "{connection}", description: "The maximum number of idle open connections allowed."); - private static readonly UpDownCounter s_minIdleConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter("db.client.connections.idle.min", - unit: "{connection}", description: "The minimum number of idle open connections allowed."); - private static readonly UpDownCounter s_maxConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter("db.client.connections.max", - unit: "{connection}", description: "The maximum number of open connections allowed."); - private static readonly UpDownCounter s_pendingRequestsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter("db.client.connections.pending_requests", - unit: "{request}", description: "The number of pending requests for an open connection, cumulative for the entire pool."); - private static readonly Histogram s_createTimeHistory = ActivitySourceHelper.Meter.CreateHistogram("db.client.connections.create_time", - unit: "ms", description: "The time it took to create a new connection."); - private static readonly Histogram s_waitTimeHistory = ActivitySourceHelper.Meter.CreateHistogram("db.client.connections.wait_time", - unit: "ms", description: "The time it took to obtain an open connection from the pool."); private static readonly ConcurrentDictionary s_pools = new(); + private static readonly List s_allPools = new(); private static readonly Action s_createdNewSession = LoggerMessage.Define( LogLevel.Debug, new EventId(EventIds.PoolCreatedNewSession, nameof(EventIds.PoolCreatedNewSession)), "Pool {PoolId} has no pooled session available; created new session {SessionId}"); diff --git a/src/MySqlConnector/Core/MetricsReporter.cs b/src/MySqlConnector/Core/MetricsReporter.cs new file mode 100644 index 000000000..0336b0f5c --- /dev/null +++ b/src/MySqlConnector/Core/MetricsReporter.cs @@ -0,0 +1,57 @@ +using System.Diagnostics.Metrics; +using MySqlConnector.Utilities; + +namespace MySqlConnector.Core; + +internal static class MetricsReporter +{ + public static void AddIdle(ConnectionPool pool) => s_connectionsUsageCounter.Add(1, pool.IdleStateTagList); + public static void RemoveIdle(ConnectionPool pool) => s_connectionsUsageCounter.Add(-1, pool.IdleStateTagList); + public static void AddUsed(ConnectionPool pool) => s_connectionsUsageCounter.Add(1, pool.UsedStateTagList); + public static void RemoveUsed(ConnectionPool pool) => s_connectionsUsageCounter.Add(-1, pool.UsedStateTagList); + public static void RecordCreateTime(ConnectionPool pool, uint ticks) => s_createTimeHistory.Record(ticks, pool.PoolNameTagList); + public static void RecordUseTime(ConnectionPool pool, uint ticks) => s_useTimeHistory.Record(ticks, pool.PoolNameTagList); + public static void RecordWaitTime(ConnectionPool pool, uint ticks) => s_waitTimeHistory.Record(ticks, pool.PoolNameTagList); + + public static void AddPendingRequest(ConnectionPool? pool) + { + if (pool is not null) + s_pendingRequestsCounter.Add(1, pool.PoolNameTagList); + } + + public static void RemovePendingRequest(ConnectionPool? pool) + { + if (pool is not null) + s_pendingRequestsCounter.Add(-1, pool.PoolNameTagList); + } + + static MetricsReporter() + { + ActivitySourceHelper.Meter.CreateObservableUpDownCounter("db.client.connections.idle.max", + observeValues: GetMaximumConnections, unit: "{connection}", + description: "The maximum number of idle open connections allowed; this corresponds to MaximumPoolSize in the connection string."); + ActivitySourceHelper.Meter.CreateObservableUpDownCounter("db.client.connections.idle.min", + observeValues: GetMinimumConnections, unit: "{connection}", + description: "The minimum number of idle open connections allowed; this corresponds to MinimumPoolSize in the connection string."); + ActivitySourceHelper.Meter.CreateObservableUpDownCounter("db.client.connections.max", + observeValues: GetMaximumConnections, unit: "{connection}", + description: "The maximum number of open connections allowed; this corresponds to MaximumPoolSize in the connection string."); + + static IEnumerable> GetMaximumConnections() => + ConnectionPool.GetAllPools().Select(x => new Measurement(x.ConnectionSettings.MaximumPoolSize, x.PoolNameTagList)); + + static IEnumerable> GetMinimumConnections() => + ConnectionPool.GetAllPools().Select(x => new Measurement(x.ConnectionSettings.MinimumPoolSize, x.PoolNameTagList)); + } + + private static readonly UpDownCounter s_connectionsUsageCounter = ActivitySourceHelper.Meter.CreateUpDownCounter("db.client.connections.usage", + unit: "{connection}", description: "The number of connections that are currently in the state described by the state tag."); + private static readonly UpDownCounter s_pendingRequestsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter("db.client.connections.pending_requests", + unit: "{request}", description: "The number of pending requests for an open connection, cumulative for the entire pool."); + private static readonly Histogram s_createTimeHistory = ActivitySourceHelper.Meter.CreateHistogram("db.client.connections.create_time", + unit: "ms", description: "The time it took to create a new connection."); + private static readonly Histogram s_useTimeHistory = ActivitySourceHelper.Meter.CreateHistogram("db.client.connections.use_time", + unit: "ms", description: "The time between borrowing a connection and returning it to the pool."); + private static readonly Histogram s_waitTimeHistory = ActivitySourceHelper.Meter.CreateHistogram("db.client.connections.wait_time", + unit: "ms", description: "The time it took to obtain an open connection from the pool."); +} diff --git a/src/MySqlConnector/Core/ServerSession.cs b/src/MySqlConnector/Core/ServerSession.cs index 1e8788e86..9e08c3003 100644 --- a/src/MySqlConnector/Core/ServerSession.cs +++ b/src/MySqlConnector/Core/ServerSession.cs @@ -79,7 +79,7 @@ public ValueTask ReturnToPoolAsync(IOBehavior ioBehavior, MySqlConnection? ownin LastReturnedTicks = unchecked((uint) Environment.TickCount); if (Pool is null) return default; - s_useTimeHistory.Record(unchecked(LastReturnedTicks - LastLeasedTicks), Pool.PoolNameTagList); + MetricsReporter.RecordUseTime(Pool, unchecked(LastReturnedTicks - LastLeasedTicks)); LastLeasedTicks = 0; return Pool.ReturnAsync(ioBehavior, this); } @@ -1916,8 +1916,6 @@ protected override void OnStatementBegin(int index) [LoggerMessage(EventIds.ExpectedSessionState6, LogLevel.Error, "Session {SessionId} should have state {ExpectedState1} or {ExpectedState2} or {ExpectedState3} or {ExpectedState4} or {ExpectedState5} or {ExpectedState6} but was {SessionState}")] private static partial void ExpectedSessionState6(ILogger logger, string sessionId, State expectedState1, State expectedState2, State expectedState3, State expectedState4, State expectedState5, State expectedState6, State sessionState); - private static readonly Histogram s_useTimeHistory = ActivitySourceHelper.Meter.CreateHistogram("db.client.connections.use_time", - unit: "ms", description: "The time between borrowing a connection and returning it to the pool."); private static readonly PayloadData s_setNamesUtf8NoAttributesPayload = QueryPayload.Create(false, "SET NAMES utf8;"u8); private static readonly PayloadData s_setNamesUtf8mb4NoAttributesPayload = QueryPayload.Create(false, "SET NAMES utf8mb4;"u8); private static readonly PayloadData s_setNamesUtf8WithAttributesPayload = QueryPayload.Create(true, "SET NAMES utf8;"u8); diff --git a/src/MySqlConnector/MySqlConnection.cs b/src/MySqlConnector/MySqlConnection.cs index 81d13a900..dfe3a0c8b 100644 --- a/src/MySqlConnector/MySqlConnection.cs +++ b/src/MySqlConnector/MySqlConnection.cs @@ -896,7 +896,7 @@ internal void FinishQuerying(bool hasWarnings) private async ValueTask CreateSessionAsync(ConnectionPool? pool, int startTickCount, Activity? activity, IOBehavior? ioBehavior, CancellationToken cancellationToken) { - pool?.AddPendingRequestCount(1); + MetricsReporter.AddPendingRequest(pool); var connectionSettings = GetInitializedConnectionSettings(); var actualIOBehavior = ioBehavior ?? (connectionSettings.ForceSynchronous ? IOBehavior.Synchronous : IOBehavior.Asynchronous); @@ -950,7 +950,7 @@ private async ValueTask CreateSessionAsync(ConnectionPool? pool, } finally { - pool?.AddPendingRequestCount(-1); + MetricsReporter.RemovePendingRequest(pool); linkedSource?.Dispose(); timeoutSource?.Dispose(); } diff --git a/tests/MySqlConnector.Tests/Metrics/MetricsTestsBase.cs b/tests/MySqlConnector.Tests/Metrics/MetricsTestsBase.cs index 3d2549e1e..7e4291124 100644 --- a/tests/MySqlConnector.Tests/Metrics/MetricsTestsBase.cs +++ b/tests/MySqlConnector.Tests/Metrics/MetricsTestsBase.cs @@ -56,6 +56,14 @@ protected MySqlConnectionStringBuilder CreateConnectionStringBuilder() => protected void AssertMeasurement(string name, int expected) { + // clear cached measurements from observable counters + lock (m_measurements) + { + m_measurements.Remove("db.client.connections.idle.max"); + m_measurements.Remove("db.client.connections.idle.min"); + m_measurements.Remove("db.client.connections.max"); + } + m_meterListener.RecordObservableInstruments(); lock (m_measurements) Assert.Equal(expected, m_measurements.GetValueOrDefault(name)); }