Move metrics to MetricsReporter class.

Make min/max connections metrics an ObservableUpDownCounter, as they are effectively static metrics for a pool. Signed-off-by: Bradley Grainger <[email protected]>
mysql-net · Oct 29, 2023 · 2a058ea · 2a058ea
1 parent 2f31600
commit 2a058ea
Show file tree

Hide file tree

Showing 5 changed files with 96 additions and 46 deletions.
diff --git a/src/MySqlConnector/Core/ConnectionPool.cs b/src/MySqlConnector/Core/ConnectionPool.cs
@@ -20,8 +20,6 @@ internal sealed class ConnectionPool : IDisposable
 
 	public SslProtocols SslProtocols { get; set; }
 
-	public void AddPendingRequestCount(int delta) => s_pendingRequestsCounter.Add(delta, PoolNameTagList);
-
 	public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection, int startTickCount, int timeoutMilliseconds, Activity? activity, IOBehavior ioBehavior, CancellationToken cancellationToken)
 	{
 		cancellationToken.ThrowIfCancellationRequested();
@@ -53,14 +51,14 @@ public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection
 			{
 				if (m_sessions.Count > 0)
 				{
-					// NOTE: s_connectionsUsageCounter updated outside lock below
+					// NOTE: MetricsReporter updated outside lock below
 					session = m_sessions.First!.Value;
 					m_sessions.RemoveFirst();
 				}
 			}
 			if (session is not null)
 			{
-				s_connectionsUsageCounter.Add(-1, IdleStateTagList);
+				MetricsReporter.RemoveIdle(this);
 				Log.FoundExistingSession(m_logger, Id);
 				bool reuseSession;
 
@@ -101,12 +99,12 @@ public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection
 						m_leasedSessions.Add(session.Id, session);
 						leasedSessionsCountPooled = m_leasedSessions.Count;
 					}
-					s_connectionsUsageCounter.Add(1, UsedStateTagList);
+					MetricsReporter.AddUsed(this);
 					ActivitySourceHelper.CopyTags(session.ActivityTags, activity);
 					Log.ReturningPooledSession(m_logger, Id, session.Id, leasedSessionsCountPooled);
 
 					session.LastLeasedTicks = unchecked((uint) Environment.TickCount);
-					s_waitTimeHistory.Record(unchecked(session.LastLeasedTicks - (uint) startTickCount), PoolNameTagList);
+					MetricsReporter.RecordWaitTime(this, unchecked(session.LastLeasedTicks - (uint) startTickCount));
 					return session;
 				}
 			}
@@ -121,11 +119,11 @@ public async ValueTask<ServerSession> GetSessionAsync(MySqlConnection connection
 				m_leasedSessions.Add(session.Id, session);
 				leasedSessionsCountNew = m_leasedSessions.Count;
 			}
-			s_connectionsUsageCounter.Add(1, UsedStateTagList);
+			MetricsReporter.AddUsed(this);
 			Log.ReturningNewSession(m_logger, Id, session.Id, leasedSessionsCountNew);
 
 			session.LastLeasedTicks = unchecked((uint) Environment.TickCount);
-			s_createTimeHistory.Record(unchecked(session.LastLeasedTicks - (uint) startTickCount), PoolNameTagList);
+			MetricsReporter.RecordCreateTime(this, unchecked(session.LastLeasedTicks - (uint) startTickCount));
 			return session;
 		}
 		catch (Exception ex)
@@ -177,14 +175,14 @@ public async ValueTask ReturnAsync(IOBehavior ioBehavior, ServerSession session)
 		{
 			lock (m_leasedSessions)
 				m_leasedSessions.Remove(session.Id);
-			s_connectionsUsageCounter.Add(-1, UsedStateTagList);
+			MetricsReporter.RemoveUsed(this);
 			session.OwningConnection = null;
 			var sessionHealth = GetSessionHealth(session);
 			if (sessionHealth == 0)
 			{
 				lock (m_sessions)
 					m_sessions.AddFirst(session);
-				s_connectionsUsageCounter.Add(1, IdleStateTagList);
+				MetricsReporter.AddIdle(this);
 			}
 			else
 			{
@@ -239,6 +237,8 @@ public async Task ReapAsync(IOBehavior ioBehavior, CancellationToken cancellatio
 	public void Dispose()
 	{
 		Log.DisposingConnectionPool(m_logger, Id);
+		lock (s_allPools)
+			s_allPools.Remove(this);
 #if NET6_0_OR_GREATER
 		m_dnsCheckTimer?.Dispose();
 		m_dnsCheckTimer = null;
@@ -258,10 +258,6 @@ public void Dispose()
 			reaperWaitHandle.WaitOne();
 		}
 #endif
-
-		s_minIdleConnectionsCounter.Add(-ConnectionSettings.MinimumPoolSize, PoolNameTagList);
-		s_maxIdleConnectionsCounter.Add(-ConnectionSettings.MaximumPoolSize, PoolNameTagList);
-		s_maxConnectionsCounter.Add(-ConnectionSettings.MaximumPoolSize, PoolNameTagList);
 	}
 
 	/// <summary>
@@ -345,14 +341,14 @@ private async Task CleanPoolAsync(IOBehavior ioBehavior, Func<ServerSession, boo
 					{
 						if (m_sessions.Count > 0)
 						{
-							// NOTE: s_connectionsUsageCounter updated outside lock below
+							// NOTE: MetricsReporter updated outside lock below
 							session = m_sessions.Last!.Value;
 							m_sessions.RemoveLast();
 						}
 					}
 					if (session is null)
 						return;
-					s_connectionsUsageCounter.Add(-1, IdleStateTagList);
+					MetricsReporter.RemoveIdle(this);
 
 					if (shouldCleanFn(session))
 					{
@@ -365,7 +361,7 @@ private async Task CleanPoolAsync(IOBehavior ioBehavior, Func<ServerSession, boo
 						// session should not be cleaned; put it back in the queue and stop iterating
 						lock (m_sessions)
 							m_sessions.AddLast(session);
-						s_connectionsUsageCounter.Add(1, IdleStateTagList);
+						MetricsReporter.AddIdle(this);
 						return;
 					}
 				}
@@ -411,7 +407,7 @@ private async Task CreateMinimumPooledSessions(MySqlConnection connection, IOBeh
 				AdjustHostConnectionCount(session, 1);
 				lock (m_sessions)
 					m_sessions.AddFirst(session);
-				s_connectionsUsageCounter.Add(1, IdleStateTagList);
+				MetricsReporter.AddIdle(this);
 			}
 			finally
 			{
@@ -569,17 +565,18 @@ private async ValueTask<ServerSession> ConnectSessionAsync(MySqlConnection conne
 		else if (pool != newPool)
 		{
 			Log.CreatedPoolWillNotBeUsed(newPool.m_logger, newPool.Id);
+			newPool.Dispose();
 		}
 
 		return pool;
 	}
 
 	public static async Task ClearPoolsAsync(IOBehavior ioBehavior, CancellationToken cancellationToken)
 	{
-		foreach (var pool in GetAllPools())
+		foreach (var pool in GetCachedPools())
 			await pool.ClearAsync(ioBehavior, cancellationToken).ConfigureAwait(false);
 
-		static List<ConnectionPool> GetAllPools()
+		static List<ConnectionPool> GetCachedPools()
 		{
 			var pools = new List<ConnectionPool>(s_pools.Count);
 			var uniquePools = new HashSet<ConnectionPool>();
@@ -626,12 +623,9 @@ private ConnectionPool(MySqlConnectorLoggingConfiguration loggingConfiguration,
 			new("state", "used"),
 		];
 
-		// set pool size counters
-		s_minIdleConnectionsCounter.Add(ConnectionSettings.MinimumPoolSize, PoolNameTagList);
-		s_maxIdleConnectionsCounter.Add(ConnectionSettings.MaximumPoolSize, PoolNameTagList);
-		s_maxConnectionsCounter.Add(ConnectionSettings.MaximumPoolSize, PoolNameTagList);
-
 		Id = Interlocked.Increment(ref s_poolId);
+		lock (s_allPools)
+			s_allPools.Add(this);
 		Log.CreatingNewConnectionPool(m_logger, Id, connectionString);
 	}
 
@@ -779,12 +773,18 @@ private void AdjustHostConnectionCount(ServerSession session, int delta)
 	}
 
 	// Provides a slice of m_stateTagList that contains either the 'idle' or 'used' state tag along with the pool name.
-	private ReadOnlySpan<KeyValuePair<string, object?>> IdleStateTagList => m_stateTagList.AsSpan(0, 2);
-	private ReadOnlySpan<KeyValuePair<string, object?>> UsedStateTagList => m_stateTagList.AsSpan(1, 2);
+	public ReadOnlySpan<KeyValuePair<string, object?>> IdleStateTagList => m_stateTagList.AsSpan(0, 2);
+	public ReadOnlySpan<KeyValuePair<string, object?>> UsedStateTagList => m_stateTagList.AsSpan(1, 2);
 
 	// A slice of m_stateTagList that contains only the pool name tag.
 	public ReadOnlySpan<KeyValuePair<string, object?>> PoolNameTagList => m_stateTagList.AsSpan(1, 1);
 
+	public static List<ConnectionPool> GetAllPools()
+	{
+		lock (s_allPools)
+			return new(s_allPools);
+	}
+
 	private sealed class LeastConnectionsLoadBalancer(Dictionary<string, int> hostSessions) : ILoadBalancer
 	{
 		public IReadOnlyList<string> LoadBalance(IReadOnlyList<string> hosts)
@@ -809,21 +809,8 @@ static ConnectionPool()
 	private static void OnAppDomainShutDown(object? sender, EventArgs e) =>
 		ClearPoolsAsync(IOBehavior.Synchronous, CancellationToken.None).GetAwaiter().GetResult();
 
-	private static readonly UpDownCounter<int> s_connectionsUsageCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.usage",
-		unit: "{connection}", description: "The number of connections that are currently in the state described by the state tag.");
-	private static readonly UpDownCounter<int> s_maxIdleConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.idle.max",
-		unit: "{connection}", description: "The maximum number of idle open connections allowed.");
-	private static readonly UpDownCounter<int> s_minIdleConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.idle.min",
-		unit: "{connection}", description: "The minimum number of idle open connections allowed.");
-	private static readonly UpDownCounter<int> s_maxConnectionsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.max",
-		unit: "{connection}", description: "The maximum number of open connections allowed.");
-	private static readonly UpDownCounter<int> s_pendingRequestsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.pending_requests",
-		unit: "{request}", description: "The number of pending requests for an open connection, cumulative for the entire pool.");
-	private static readonly Histogram<float> s_createTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.create_time",
-		unit: "ms", description: "The time it took to create a new connection.");
-	private static readonly Histogram<float> s_waitTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.wait_time",
-		unit: "ms", description: "The time it took to obtain an open connection from the pool.");
 	private static readonly ConcurrentDictionary<string, ConnectionPool?> s_pools = new();
+	private static readonly List<ConnectionPool> s_allPools = new();
 	private static readonly Action<ILogger, int, string, Exception?> s_createdNewSession = LoggerMessage.Define<int, string>(
 		LogLevel.Debug, new EventId(EventIds.PoolCreatedNewSession, nameof(EventIds.PoolCreatedNewSession)),
 		"Pool {PoolId} has no pooled session available; created new session {SessionId}");

diff --git a/src/MySqlConnector/Core/MetricsReporter.cs b/src/MySqlConnector/Core/MetricsReporter.cs
@@ -0,0 +1,57 @@
+using System.Diagnostics.Metrics;
+using MySqlConnector.Utilities;
+
+namespace MySqlConnector.Core;
+
+internal static class MetricsReporter
+{
+	public static void AddIdle(ConnectionPool pool) => s_connectionsUsageCounter.Add(1, pool.IdleStateTagList);
+	public static void RemoveIdle(ConnectionPool pool) => s_connectionsUsageCounter.Add(-1, pool.IdleStateTagList);
+	public static void AddUsed(ConnectionPool pool) => s_connectionsUsageCounter.Add(1, pool.UsedStateTagList);
+	public static void RemoveUsed(ConnectionPool pool) => s_connectionsUsageCounter.Add(-1, pool.UsedStateTagList);
+	public static void RecordCreateTime(ConnectionPool pool, uint ticks) => s_createTimeHistory.Record(ticks, pool.PoolNameTagList);
+	public static void RecordUseTime(ConnectionPool pool, uint ticks) => s_useTimeHistory.Record(ticks, pool.PoolNameTagList);
+	public static void RecordWaitTime(ConnectionPool pool, uint ticks) => s_waitTimeHistory.Record(ticks, pool.PoolNameTagList);
+
+	public static void AddPendingRequest(ConnectionPool? pool)
+	{
+		if (pool is not null)
+			s_pendingRequestsCounter.Add(1, pool.PoolNameTagList);
+	}
+
+	public static void RemovePendingRequest(ConnectionPool? pool)
+	{
+		if (pool is not null)
+			s_pendingRequestsCounter.Add(-1, pool.PoolNameTagList);
+	}
+
+	static MetricsReporter()
+	{
+		ActivitySourceHelper.Meter.CreateObservableUpDownCounter<int>("db.client.connections.idle.max",
+			observeValues: GetMaximumConnections, unit: "{connection}",
+			description: "The maximum number of idle open connections allowed; this corresponds to MaximumPoolSize in the connection string.");
+		ActivitySourceHelper.Meter.CreateObservableUpDownCounter<int>("db.client.connections.idle.min",
+			observeValues: GetMinimumConnections, unit: "{connection}",
+			description: "The minimum number of idle open connections allowed; this corresponds to MinimumPoolSize in the connection string.");
+		ActivitySourceHelper.Meter.CreateObservableUpDownCounter<int>("db.client.connections.max",
+			observeValues: GetMaximumConnections, unit: "{connection}",
+			description: "The maximum number of open connections allowed; this corresponds to MaximumPoolSize in the connection string.");
+
+		static IEnumerable<Measurement<int>> GetMaximumConnections() =>
+			ConnectionPool.GetAllPools().Select(x => new Measurement<int>(x.ConnectionSettings.MaximumPoolSize, x.PoolNameTagList));
+
+		static IEnumerable<Measurement<int>> GetMinimumConnections() =>
+			ConnectionPool.GetAllPools().Select(x => new Measurement<int>(x.ConnectionSettings.MinimumPoolSize, x.PoolNameTagList));
+	}
+
+	private static readonly UpDownCounter<int> s_connectionsUsageCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.usage",
+			unit: "{connection}", description: "The number of connections that are currently in the state described by the state tag.");
+	private static readonly UpDownCounter<int> s_pendingRequestsCounter = ActivitySourceHelper.Meter.CreateUpDownCounter<int>("db.client.connections.pending_requests",
+			unit: "{request}", description: "The number of pending requests for an open connection, cumulative for the entire pool.");
+	private static readonly Histogram<float> s_createTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.create_time",
+			unit: "ms", description: "The time it took to create a new connection.");
+	private static readonly Histogram<float> s_useTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.use_time",
+		unit: "ms", description: "The time between borrowing a connection and returning it to the pool.");
+	private static readonly Histogram<float> s_waitTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.wait_time",
+			unit: "ms", description: "The time it took to obtain an open connection from the pool.");
+}
diff --git a/src/MySqlConnector/Core/ServerSession.cs b/src/MySqlConnector/Core/ServerSession.cs
@@ -79,7 +79,7 @@ public ValueTask ReturnToPoolAsync(IOBehavior ioBehavior, MySqlConnection? ownin
 		LastReturnedTicks = unchecked((uint) Environment.TickCount);
 		if (Pool is null)
 			return default;
-		s_useTimeHistory.Record(unchecked(LastReturnedTicks - LastLeasedTicks), Pool.PoolNameTagList);
+		MetricsReporter.RecordUseTime(Pool, unchecked(LastReturnedTicks - LastLeasedTicks));
 		LastLeasedTicks = 0;
 		return Pool.ReturnAsync(ioBehavior, this);
 	}
@@ -1916,8 +1916,6 @@ protected override void OnStatementBegin(int index)
 	[LoggerMessage(EventIds.ExpectedSessionState6, LogLevel.Error, "Session {SessionId} should have state {ExpectedState1} or {ExpectedState2} or {ExpectedState3} or {ExpectedState4} or {ExpectedState5} or {ExpectedState6} but was {SessionState}")]
 	private static partial void ExpectedSessionState6(ILogger logger, string sessionId, State expectedState1, State expectedState2, State expectedState3, State expectedState4, State expectedState5, State expectedState6, State sessionState);
 
-	private static readonly Histogram<float> s_useTimeHistory = ActivitySourceHelper.Meter.CreateHistogram<float>("db.client.connections.use_time",
-		unit: "ms", description: "The time between borrowing a connection and returning it to the pool.");
 	private static readonly PayloadData s_setNamesUtf8NoAttributesPayload = QueryPayload.Create(false, "SET NAMES utf8;"u8);
 	private static readonly PayloadData s_setNamesUtf8mb4NoAttributesPayload = QueryPayload.Create(false, "SET NAMES utf8mb4;"u8);
 	private static readonly PayloadData s_setNamesUtf8WithAttributesPayload = QueryPayload.Create(true, "SET NAMES utf8;"u8);

diff --git a/src/MySqlConnector/MySqlConnection.cs b/src/MySqlConnector/MySqlConnection.cs
@@ -896,7 +896,7 @@ internal void FinishQuerying(bool hasWarnings)
 
 	private async ValueTask<ServerSession> CreateSessionAsync(ConnectionPool? pool, int startTickCount, Activity? activity, IOBehavior? ioBehavior, CancellationToken cancellationToken)
 	{
-		pool?.AddPendingRequestCount(1);
+		MetricsReporter.AddPendingRequest(pool);
 		var connectionSettings = GetInitializedConnectionSettings();
 		var actualIOBehavior = ioBehavior ?? (connectionSettings.ForceSynchronous ? IOBehavior.Synchronous : IOBehavior.Asynchronous);
 
@@ -950,7 +950,7 @@ private async ValueTask<ServerSession> CreateSessionAsync(ConnectionPool? pool,
 		}
 		finally
 		{
-			pool?.AddPendingRequestCount(-1);
+			MetricsReporter.RemovePendingRequest(pool);
 			linkedSource?.Dispose();
 			timeoutSource?.Dispose();
 		}

diff --git a/tests/MySqlConnector.Tests/Metrics/MetricsTestsBase.cs b/tests/MySqlConnector.Tests/Metrics/MetricsTestsBase.cs
@@ -56,6 +56,14 @@ protected MySqlConnectionStringBuilder CreateConnectionStringBuilder() =>
 
 	protected void AssertMeasurement(string name, int expected)
 	{
+		// clear cached measurements from observable counters
+		lock (m_measurements)
+		{
+			m_measurements.Remove("db.client.connections.idle.max");
+			m_measurements.Remove("db.client.connections.idle.min");
+			m_measurements.Remove("db.client.connections.max");
+		}
+		m_meterListener.RecordObservableInstruments();
 		lock (m_measurements)
 			Assert.Equal(expected, m_measurements.GetValueOrDefault(name));
 	}