From feb2215e1e6d6c3bace925e5326bc5df39c69a0a Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Tue, 28 Jan 2025 22:08:35 -0800 Subject: [PATCH 01/60] Update the transport manifest --- src/ServiceControl.Transports.RabbitMQ/transport.manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/transport.manifest b/src/ServiceControl.Transports.RabbitMQ/transport.manifest index 2442343847..100386fa6a 100644 --- a/src/ServiceControl.Transports.RabbitMQ/transport.manifest +++ b/src/ServiceControl.Transports.RabbitMQ/transport.manifest @@ -27,7 +27,7 @@ "DisplayName": "RabbitMQ - Conventional routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumConventionalRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;DisableManagementApi=;ManagementUri=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumConventialRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" @@ -38,7 +38,7 @@ "DisplayName": "RabbitMQ - Direct routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumDirectRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;DisableManagementApi=;ManagementUri=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumDirectRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" From b7f1039dbd3bdbdf5ac3f2bc86904c59de485803 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Wed, 29 Jan 2025 12:20:08 -0800 Subject: [PATCH 02/60] Use alpha version of RabbitMQ transport --- src/Directory.Packages.props | 3 ++- .../QueueLengthProvider.cs | 16 ++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index cfe0a35952..83a17a6035 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -38,7 +38,8 @@ - + + diff --git a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs index e7e97025c7..056cf8f7cd 100644 --- a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs +++ b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs @@ -75,13 +75,13 @@ async Task FetchQueueLengths(CancellationToken cancellationToken) { foreach (var endpointQueuePair in endpointQueues) { - await queryExecutor.Execute(m => + await queryExecutor.Execute(async m => { var queueName = endpointQueuePair.Value; try { - var size = (int)m.MessageCount(queueName); + var size = (int)await m.MessageCountAsync(queueName, cancellationToken).ConfigureAwait(false); sizes.AddOrUpdate(queueName, _ => size, (_, __) => size); } @@ -120,7 +120,7 @@ public void Initialize() null); // value would come from config API in actual transport } - public async Task Execute(Action action, CancellationToken cancellationToken = default) + public async Task Execute(Action action, CancellationToken cancellationToken = default) { try { @@ -132,14 +132,14 @@ public async Task Execute(Action action, CancellationToken cancellationT await Task.Delay(ReconnectionDelay, cancellationToken); } - if (model == null || model.IsClosed) + if (channel == null || channel.IsClosed) { - model?.Dispose(); + channel?.Dispose(); - model = connection.CreateModel(); + channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken).ConfigureAwait(false); } - action(model); + action(channel); } catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { @@ -154,7 +154,7 @@ public async Task Execute(Action action, CancellationToken cancellationT public void Dispose() => connection?.Dispose(); IConnection connection; - IModel model; + IChannel channel; ConnectionFactory connectionFactory; static readonly TimeSpan ReconnectionDelay = TimeSpan.FromSeconds(5); From e1d3e7400af3c1d46d839e5fafcb8bdb93f9344f Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Wed, 29 Jan 2025 12:21:19 -0800 Subject: [PATCH 03/60] Update RabbitMQ manifest example --- src/ServiceControl.Transports.RabbitMQ/transport.manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/transport.manifest b/src/ServiceControl.Transports.RabbitMQ/transport.manifest index 100386fa6a..3bb7a14df6 100644 --- a/src/ServiceControl.Transports.RabbitMQ/transport.manifest +++ b/src/ServiceControl.Transports.RabbitMQ/transport.manifest @@ -27,7 +27,7 @@ "DisplayName": "RabbitMQ - Conventional routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumConventionalRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;DisableManagementApi=;ManagementUri=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;DisableManagementApi=;ManagementApiUrl=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumConventialRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" @@ -38,7 +38,7 @@ "DisplayName": "RabbitMQ - Direct routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumDirectRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;DisableManagementApi=;ManagementUri=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;DisableManagementApi=;ManagementApiUrl=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumDirectRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" From bc70e3955f479660529814c2f2adee8f6a7f5376 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Wed, 29 Jan 2025 12:21:46 -0800 Subject: [PATCH 04/60] Check connection string to disable management API --- .../TransportConfigurationExtensions.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs b/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs index ee10f15afb..9f638e8547 100644 --- a/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs +++ b/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs @@ -20,6 +20,12 @@ public static void ApplyConnectionString(this TransportExtensions Date: Wed, 29 Jan 2025 17:39:21 -0800 Subject: [PATCH 05/60] Change lock to semaphoreSlim --- .../ConnectionFactory.cs | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/ConnectionFactory.cs b/src/ServiceControl.Transports.RabbitMQ/ConnectionFactory.cs index e52f7e1de3..fce580c92c 100644 --- a/src/ServiceControl.Transports.RabbitMQ/ConnectionFactory.cs +++ b/src/ServiceControl.Transports.RabbitMQ/ConnectionFactory.cs @@ -4,13 +4,15 @@ using System.Net.Security; using System.Security.Authentication; using System.Security.Cryptography.X509Certificates; + using System.Threading; + using System.Threading.Tasks; using global::RabbitMQ.Client; class ConnectionFactory { readonly string endpointName; readonly global::RabbitMQ.Client.ConnectionFactory connectionFactory; - readonly object lockObject = new object(); + readonly SemaphoreSlim semaphoreSlim = new(1, 1); public ConnectionFactory(string endpointName, ConnectionConfiguration connectionConfiguration, X509Certificate2Collection clientCertificateCollection, bool disableRemoteCertificateValidation, @@ -76,21 +78,26 @@ public ConnectionFactory(string endpointName, ConnectionConfiguration connection } } - public IConnection CreatePublishConnection() => CreateConnection($"{endpointName} Publish", false); + public async Task CreatePublishConnection(CancellationToken cancellationToken) => await CreateConnection($"{endpointName} Publish", false, cancellationToken); - public IConnection CreateAdministrationConnection() => CreateConnection($"{endpointName} Administration", false); + public Task CreateAdministrationConnection(CancellationToken cancellationToken) => CreateConnection($"{endpointName} Administration", false, cancellationToken); - public IConnection CreateConnection(string connectionName, bool automaticRecoveryEnabled = true) + public async Task CreateConnection(string connectionName, bool automaticRecoveryEnabled = true, CancellationToken cancellationToken = default) { - lock (lockObject) + await semaphoreSlim.WaitAsync(cancellationToken).ConfigureAwait(false); + try { connectionFactory.AutomaticRecoveryEnabled = automaticRecoveryEnabled; connectionFactory.ClientProperties["connected"] = DateTime.UtcNow.ToString("G"); - var connection = connectionFactory.CreateConnection(connectionName); + var connection = await connectionFactory.CreateConnectionAsync(connectionName, cancellationToken); return connection; } + finally + { + _ = semaphoreSlim.Release(); + } } } } From af318ebe6415ec96b97b4ff7511d3b2c7af55fa0 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Wed, 29 Jan 2025 17:40:05 -0800 Subject: [PATCH 06/60] Update async/await for createConnection --- src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs index 056cf8f7cd..abdf41c37e 100644 --- a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs +++ b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs @@ -124,7 +124,7 @@ public async Task Execute(Action action, CancellationToken cancellatio { try { - connection ??= connectionFactory.CreateConnection("queue length monitor"); + connection ??= await connectionFactory.CreateConnection("queue length monitor", cancellationToken: cancellationToken); //Connection implements reconnection logic while (!connection.IsOpen) From 87a10c8da2e6d14523e29edec7f4106fe1b3d2a1 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Wed, 29 Jan 2025 17:40:32 -0800 Subject: [PATCH 07/60] Cleanup transport method call --- .../TransportConfigurationExtensions.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs b/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs index 9f638e8547..88cbec4ded 100644 --- a/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs +++ b/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs @@ -20,7 +20,6 @@ public static void ApplyConnectionString(this TransportExtensions Date: Sat, 1 Feb 2025 20:41:49 -0800 Subject: [PATCH 08/60] Update RabbitMQQuery with managementApiUrl for throughput --- .../ConnectionConfiguration.cs | 2 +- .../RabbitMQQuery.cs | 112 +++++++++++++----- 2 files changed, 81 insertions(+), 33 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/ConnectionConfiguration.cs b/src/ServiceControl.Transports.RabbitMQ/ConnectionConfiguration.cs index 0f2cb882b5..e6ea32a5c7 100644 --- a/src/ServiceControl.Transports.RabbitMQ/ConnectionConfiguration.cs +++ b/src/ServiceControl.Transports.RabbitMQ/ConnectionConfiguration.cs @@ -179,7 +179,7 @@ static Dictionary ParseAmqpConnectionString(string connectionStr return dictionary; } - static Dictionary ParseNServiceBusConnectionString(string connectionString, StringBuilder invalidOptionsMessage) + internal static Dictionary ParseNServiceBusConnectionString(string connectionString, StringBuilder invalidOptionsMessage) { var dictionary = new DbConnectionStringBuilder { ConnectionString = connectionString } .OfType>() diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 9bacec74c3..65b258a4a9 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -9,6 +9,7 @@ namespace ServiceControl.Transports.RabbitMQ; using System.Net.Http; using System.Net.Http.Json; using System.Runtime.CompilerServices; +using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; @@ -29,6 +30,7 @@ public class RabbitMQQuery : BrokerThroughputQuery readonly ILogger logger; readonly TimeProvider timeProvider; readonly ConnectionConfiguration connectionConfiguration; + readonly string connectionString; public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, @@ -36,64 +38,110 @@ public RabbitMQQuery(ILogger logger, { this.logger = logger; this.timeProvider = timeProvider; + connectionString = transportSettings.ConnectionString; - connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); + connectionConfiguration = ConnectionConfiguration.Create(connectionString, string.Empty); } protected override void InitializeCore(ReadOnlyDictionary settings) { - if (!settings.TryGetValue(RabbitMQSettings.UserName, out string? username) || - string.IsNullOrEmpty(username)) + var mangementApiUrl = GetManagementApiUrl(); + + var userName = GetSettingsValue(settings, RabbitMQSettings.UserName, mangementApiUrl.UserName); + var password = GetSettingsValue(settings, RabbitMQSettings.Password, mangementApiUrl.Password); + var apiUrl = GetSettingsValue(settings, RabbitMQSettings.API, mangementApiUrl.Uri.AbsoluteUri); + + if (userName != mangementApiUrl.UserName) { - logger.LogInformation("Using username from connectionstring"); - username = connectionConfiguration.UserName; - Diagnostics.AppendLine( - $"Username not set, defaulted to using \"{username}\" username from the ConnectionString used by instance"); + _ = Diagnostics.AppendLine($"UserName in settings is different from Management API URL: {userName} != {managementUserName}"); } - else + + if (password != mangementApiUrl.Password) { - Diagnostics.AppendLine($"Username set to \"{username}\""); + _ = Diagnostics.AppendLine($"Password in settings is different from Management API URL."); } - if (!settings.TryGetValue(RabbitMQSettings.Password, out string? password) || - string.IsNullOrEmpty(password)) + if (apiUrl != mangementApiUrl.Uri.AbsoluteUri) { - logger.LogInformation("Using password from connectionstring"); - password = connectionConfiguration.Password; - Diagnostics.AppendLine( - "Password not set, defaulted to using password from the ConnectionString used by instance"); + _ = Diagnostics.AppendLine($"API URL in settings is different from Management API URL: {apiUrl} != {mangementApiUrl.Uri.AbsoluteUri}"); } - else + + if (!Uri.TryCreate(apiUrl, UriKind.Absolute, out _)) { - Diagnostics.AppendLine("Password set"); + InitialiseErrors.Add("API url configured is invalid"); } - var defaultCredential = new NetworkCredential(username, password); + var defaultCredential = new NetworkCredential(userName, password); + + if (InitialiseErrors.Count == 0) + { + // ideally we would use the HttpClientFactory, but it would be a bit more involved to set that up + // so for now we are using a virtual method that can be overriden in tests + // https://github.com/Particular/ServiceControl/issues/4493 + httpClient = CreateHttpClient(defaultCredential, apiUrl); + } + } - if (!settings.TryGetValue(RabbitMQSettings.API, out string? apiUrl) || - string.IsNullOrEmpty(apiUrl)) + string GetSettingsValue(ReadOnlyDictionary settings, string key, string defaultValue) + { + if (!settings.TryGetValue(key, out string? value) || + string.IsNullOrEmpty(value)) { - apiUrl = - $"{(connectionConfiguration.UseTls ? $"https://{connectionConfiguration.Host}:15671" : $"http://{connectionConfiguration.Host}:15672")}"; - Diagnostics.AppendLine( - $"RabbitMQ API Url not set, defaulted to using \"{apiUrl}\" from the ConnectionString used by instance"); + logger.LogInformation($"Using {key} from connection string"); + value = defaultValue; + _ = Diagnostics.AppendLine( + $"{key} not set, defaulted to using {key} from the ConnectionString used by instance"); } else { - Diagnostics.AppendLine($"RabbitMQ API Url set to \"{apiUrl}\""); - if (!Uri.TryCreate(apiUrl, UriKind.Absolute, out _)) + if (key == RabbitMQSettings.Password) { - InitialiseErrors.Add("API url configured is invalid"); + _ = Diagnostics.AppendLine($"{key} set."); } + + _ = Diagnostics.AppendLine($"{key} set to {value}."); } - if (InitialiseErrors.Count == 0) + return value; + } + + UriBuilder GetManagementApiUrl() + { + var dictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(connectionString, new StringBuilder()); + UriBuilder uriBuilder; + + var managementApiUrl = GetValue(dictionary, "ManagementApiUrl", ""); + + if (string.IsNullOrEmpty(managementApiUrl)) { - // ideally we would use the HttpClientFactory, but it would be a bit more involved to set that up - // so for now we are using a virtual method that can be overriden in tests - // https://github.com/Particular/ServiceControl/issues/4493 - httpClient = CreateHttpClient(defaultCredential, apiUrl); + _ = Diagnostics.AppendLine("ManagementApiUrl is empty. Setting credentials from the connectionString used by the instance."); + uriBuilder = new UriBuilder() + { + Scheme = connectionConfiguration.UseTls ? "https" : "http", + Host = connectionConfiguration.Host, + }; } + else + { + uriBuilder = new UriBuilder(managementApiUrl); + } + + uriBuilder.UserName = string.IsNullOrEmpty(uriBuilder.UserName) ? connectionConfiguration.UserName : uriBuilder.UserName; + uriBuilder.Password = string.IsNullOrEmpty(uriBuilder.Password) ? connectionConfiguration.Password : uriBuilder.Password; + + // Check if port was defined in the connection string or if it's the default value from UriBuilder + var isPortProvided = uriBuilder.Port != -1 + && (!((uriBuilder.Port == 80 || uriBuilder.Port == 443) + && !managementApiUrl.Contains($":{uriBuilder.Port}"))); + + uriBuilder.Port = isPortProvided ? uriBuilder.Port : uriBuilder.Scheme == "https" ? 15671 : 15672; + + return uriBuilder; + } + + static string GetValue(Dictionary dictionary, string key, string defaultValue) + { + return dictionary.TryGetValue(key, out var value) ? value : defaultValue; } protected virtual HttpClient CreateHttpClient(NetworkCredential defaultCredential, string apiUrl) => From 850d3706f7a6ed7dc00e25f05dad2fca24606b51 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Sat, 1 Feb 2025 20:42:44 -0800 Subject: [PATCH 09/60] Add functionality for RabbitMQ connection string management options --- ...QConventionalRoutingTransportCustomization.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index bc8db946f3..979d19e2d8 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -1,7 +1,9 @@ namespace ServiceControl.Transports.RabbitMQ { using System; + using System.Collections.Generic; using System.Linq; + using System.Text; using BrokerThroughput; using Microsoft.Extensions.DependencyInjection; using NServiceBus; @@ -22,8 +24,17 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport throw new InvalidOperationException("Connection string not configured"); } + var connectionStringDictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); + var disableManagementApi = GetValue(connectionStringDictionary, "DisableManagementApi", "false"); + if (!disableManagementApi.Equals("true", StringComparison.OrdinalIgnoreCase) && !disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException("The value for 'DisableManagementApi' must be either 'true' or 'false'"); + } + var transport = new RabbitMQTransport(RoutingTopology.Conventional(queueType), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; + transport.ManagementApiUrl = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); + transport.UseManagementApi = disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase); return transport; } @@ -39,5 +50,10 @@ protected sealed override void AddTransportForMonitoringCore(IServiceCollection services.AddSingleton(); services.AddHostedService(provider => provider.GetRequiredService()); } + + static string GetValue(Dictionary dictionary, string key, string defaultValue) + { + return dictionary.TryGetValue(key, out var value) ? value : defaultValue; + } } } \ No newline at end of file From b2e595c03d4a1554008a7a4920ceb2cf99f49f57 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Sat, 1 Feb 2025 22:00:28 -0800 Subject: [PATCH 10/60] Fix diagnostic log text --- src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 65b258a4a9..f0bb039a8b 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -53,7 +53,7 @@ protected override void InitializeCore(ReadOnlyDictionary settin if (userName != mangementApiUrl.UserName) { - _ = Diagnostics.AppendLine($"UserName in settings is different from Management API URL: {userName} != {managementUserName}"); + _ = Diagnostics.AppendLine($"UserName in settings is different from Management API URL: {userName} != {mangementApiUrl.UserName}"); } if (password != mangementApiUrl.Password) From 7c4e12072aa2e28f5b40d680189a6e17754e0d30 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Sat, 1 Feb 2025 22:01:35 -0800 Subject: [PATCH 11/60] Add functionality for management options for direct routing --- ...abbitMQDirectRoutingTransportCustomization.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index c4e09725dd..27a0379e92 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -1,7 +1,9 @@ namespace ServiceControl.Transports.RabbitMQ { using System; + using System.Collections.Generic; using System.Linq; + using System.Text; using BrokerThroughput; using Microsoft.Extensions.DependencyInjection; using NServiceBus; @@ -24,8 +26,17 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport throw new InvalidOperationException("Connection string not configured"); } + var connectionStringDictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); + var disableManagementApi = GetValue(connectionStringDictionary, "DisableManagementApi", "false"); + if (!disableManagementApi.Equals("true", StringComparison.OrdinalIgnoreCase) && !disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase)) + { + throw new ArgumentException("The value for 'DisableManagementApi' must be either 'true' or 'false'"); + } + var transport = new RabbitMQTransport(RoutingTopology.Direct(queueType, routingKeyConvention: type => type.FullName.Replace(".", "-")), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; + transport.ManagementApiUrl = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); + transport.UseManagementApi = disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase); return transport; } @@ -41,5 +52,10 @@ protected sealed override void AddTransportForMonitoringCore(IServiceCollection services.AddSingleton(); services.AddHostedService(provider => provider.GetRequiredService()); } + + static string GetValue(Dictionary dictionary, string key, string defaultValue) + { + return dictionary.TryGetValue(key, out var value) ? value : defaultValue; + } } } \ No newline at end of file From 2e7dee7ebfb150c2802a318fe1e870466ce98717 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Mon, 3 Feb 2025 22:01:11 -0800 Subject: [PATCH 12/60] Allow RabbitMQQuery to access the transport --- .../IRabbitMQTransportExtentions.cs | 9 +++++++ ...nventionalRoutingTransportCustomization.cs | 23 +++++++++++------ ...itMQDirectRoutingTransportCustomization.cs | 25 +++++++++++-------- .../RabbitMQQuery.cs | 23 +++++++++++++---- .../RabbitMQQueryTests.cs | 2 +- .../RabbitMQQueryTests.cs | 2 +- .../RabbitMQQuery_ResponseParsing_Tests.cs | 9 ++++--- 7 files changed, 65 insertions(+), 28 deletions(-) create mode 100644 src/ServiceControl.Transports.RabbitMQ/IRabbitMQTransportExtentions.cs diff --git a/src/ServiceControl.Transports.RabbitMQ/IRabbitMQTransportExtentions.cs b/src/ServiceControl.Transports.RabbitMQ/IRabbitMQTransportExtentions.cs new file mode 100644 index 0000000000..8eb09e50bc --- /dev/null +++ b/src/ServiceControl.Transports.RabbitMQ/IRabbitMQTransportExtentions.cs @@ -0,0 +1,9 @@ +namespace ServiceControl.Transports.RabbitMQ +{ + using NServiceBus; + + public interface IRabbitMQTransportExtensions + { + RabbitMQTransport GetTransport(); + } +} diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index 979d19e2d8..5238e51049 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -9,8 +9,11 @@ using NServiceBus; public abstract class RabbitMQConventionalRoutingTransportCustomization(QueueType queueType) - : TransportCustomization + : TransportCustomization, IRabbitMQTransportExtensions { + + RabbitMQTransport rabbitMQTransport; + protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } @@ -36,14 +39,13 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport transport.ManagementApiUrl = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); transport.UseManagementApi = disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase); + rabbitMQTransport = transport; + return transport; } - protected override void AddTransportForPrimaryCore(IServiceCollection services, - TransportSettings transportSettings) - { - services.AddSingleton(); - } + protected override void AddTransportForPrimaryCore(IServiceCollection services, TransportSettings transportSettings) + => services.AddSingleton(); protected sealed override void AddTransportForMonitoringCore(IServiceCollection services, TransportSettings transportSettings) { @@ -52,8 +54,15 @@ protected sealed override void AddTransportForMonitoringCore(IServiceCollection } static string GetValue(Dictionary dictionary, string key, string defaultValue) + => dictionary.TryGetValue(key, out var value) ? value : defaultValue; + + RabbitMQTransport IRabbitMQTransportExtensions.GetTransport() { - return dictionary.TryGetValue(key, out var value) ? value : defaultValue; + if (rabbitMQTransport == null) + { + throw new InvalidOperationException("Transport instance has not been created yet. Make sure CreateTransport() is called before accessing the transport."); + }; + return rabbitMQTransport; } } } \ No newline at end of file diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 27a0379e92..4fede6937f 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -8,11 +8,10 @@ using Microsoft.Extensions.DependencyInjection; using NServiceBus; - public abstract class RabbitMQDirectRoutingTransportCustomization : TransportCustomization + public abstract class RabbitMQDirectRoutingTransportCustomization(QueueType queueType) + : TransportCustomization, IRabbitMQTransportExtensions { - readonly QueueType queueType; - - protected RabbitMQDirectRoutingTransportCustomization(QueueType queueType) => this.queueType = queueType; + RabbitMQTransport rabbitMQTransport; protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } @@ -38,14 +37,13 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport transport.ManagementApiUrl = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); transport.UseManagementApi = disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase); + rabbitMQTransport = transport; + return transport; } - protected override void AddTransportForPrimaryCore(IServiceCollection services, - TransportSettings transportSettings) - { - services.AddSingleton(); - } + protected override void AddTransportForPrimaryCore(IServiceCollection services, TransportSettings transportSettings) + => services.AddSingleton(); protected sealed override void AddTransportForMonitoringCore(IServiceCollection services, TransportSettings transportSettings) { @@ -54,8 +52,15 @@ protected sealed override void AddTransportForMonitoringCore(IServiceCollection } static string GetValue(Dictionary dictionary, string key, string defaultValue) + => dictionary.TryGetValue(key, out var value) ? value : defaultValue; + + RabbitMQTransport IRabbitMQTransportExtensions.GetTransport() { - return dictionary.TryGetValue(key, out var value) ? value : defaultValue; + if (rabbitMQTransport == null) + { + throw new InvalidOperationException("Transport instance has not been created yet. Make sure CreateTransport() is called before accessing the transport."); + }; + return rabbitMQTransport; } } } \ No newline at end of file diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index f0bb039a8b..7cdabacb22 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -16,6 +16,7 @@ namespace ServiceControl.Transports.RabbitMQ; using System.Threading.Tasks; using System.Web; using Microsoft.Extensions.Logging; +using NServiceBus; using Polly; using Polly.Retry; using ServiceControl.Transports.BrokerThroughput; @@ -30,23 +31,35 @@ public class RabbitMQQuery : BrokerThroughputQuery readonly ILogger logger; readonly TimeProvider timeProvider; readonly ConnectionConfiguration connectionConfiguration; - readonly string connectionString; + readonly TransportSettings transportSettings; + readonly RabbitMQTransport rabbitMQTransport; public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, - TransportSettings transportSettings) : base(logger, "RabbitMQ") + TransportSettings transportSettings, + ITransportCustomization transportCustomization) : base(logger, "RabbitMQ") { this.logger = logger; this.timeProvider = timeProvider; - connectionString = transportSettings.ConnectionString; + this.transportSettings = transportSettings; + if (transportCustomization is IRabbitMQTransportExtensions rabbitMQTransportCustomization) + { + rabbitMQTransport = rabbitMQTransportCustomization.GetTransport(); + } + else + { + throw new InvalidOperationException($"Expected a RabbitMQTransport but received {transportCustomization.GetType().Name}."); + } - connectionConfiguration = ConnectionConfiguration.Create(connectionString, string.Empty); + connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); } protected override void InitializeCore(ReadOnlyDictionary settings) { var mangementApiUrl = GetManagementApiUrl(); + // The licensing component configurations take precedence over the management API connection string configuration options + // https://docs.particular.net/servicecontrol/servicecontrol-instances/configuration#usage-reporting-when-using-the-rabbitmq-transport var userName = GetSettingsValue(settings, RabbitMQSettings.UserName, mangementApiUrl.UserName); var password = GetSettingsValue(settings, RabbitMQSettings.Password, mangementApiUrl.Password); var apiUrl = GetSettingsValue(settings, RabbitMQSettings.API, mangementApiUrl.Uri.AbsoluteUri); @@ -107,7 +120,7 @@ string GetSettingsValue(ReadOnlyDictionary settings, string key, UriBuilder GetManagementApiUrl() { - var dictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(connectionString, new StringBuilder()); + var dictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); UriBuilder uriBuilder; var managementApiUrl = GetValue(dictionary, "ManagementApiUrl", ""); diff --git a/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs b/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs index 062d498524..ac38788349 100644 --- a/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs +++ b/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs @@ -28,7 +28,7 @@ public async Task GetQueueNames_FindsQueues() MaxConcurrency = 1, EndpointName = Guid.NewGuid().ToString("N") }; - var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings); + var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); string[] additionalQueues = Enumerable.Range(1, 10).Select(i => $"myqueue{i}").ToArray(); await configuration.TransportCustomization.ProvisionQueues(transportSettings, additionalQueues); diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs index 91b3a029ce..8cd6a4efea 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs +++ b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs @@ -32,7 +32,7 @@ public void Initialise() MaxConcurrency = 1, EndpointName = Guid.NewGuid().ToString("N") }; - query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings); + query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); } [Test] diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQuery_ResponseParsing_Tests.cs b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQuery_ResponseParsing_Tests.cs index 9e706832b9..fd7aace0da 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQuery_ResponseParsing_Tests.cs +++ b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQuery_ResponseParsing_Tests.cs @@ -14,7 +14,7 @@ namespace ServiceControl.Transport.Tests; using System.Net; [TestFixture] -class RabbitMQQuery_ResponseParsing_Tests +class RabbitMQQuery_ResponseParsing_Tests : TransportTestFixture { FakeTimeProvider provider; TransportSettings transportSettings; @@ -35,7 +35,7 @@ public void SetUp() httpHandler = new FakeHttpHandler(); var httpClient = new HttpClient(httpHandler) { BaseAddress = new Uri("http://localhost:15672") }; - rabbitMQQuery = new TestableRabbitMQQuery(provider, transportSettings, httpClient); + rabbitMQQuery = new TestableRabbitMQQuery(provider, transportSettings, httpClient, configuration.TransportCustomization); rabbitMQQuery.Initialize(ReadOnlyDictionary.Empty); } @@ -131,8 +131,9 @@ public async Task Should_fetch_queue_details_in_old_format() sealed class TestableRabbitMQQuery( TimeProvider timeProvider, TransportSettings transportSettings, - HttpClient customHttpClient) - : RabbitMQQuery(NullLogger.Instance, timeProvider, transportSettings) + HttpClient customHttpClient, + ITransportCustomization transportCustomization) + : RabbitMQQuery(NullLogger.Instance, timeProvider, transportSettings, transportCustomization) { protected override HttpClient CreateHttpClient(NetworkCredential defaultCredential, string apiUrl) => customHttpClient; } From 18e3c870270e07c6b285f9312e7f2e8b935a02b9 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Tue, 4 Feb 2025 00:31:42 -0800 Subject: [PATCH 13/60] Update alpha version --- src/Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 83a17a6035..6698b7a8f3 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -38,7 +38,7 @@ - + From 2dcb6233bbf92998a52d939ecdf334f4e59ceffd Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Tue, 4 Feb 2025 00:33:02 -0800 Subject: [PATCH 14/60] Update transport creation with new API surface --- ...nventionalRoutingTransportCustomization.cs | 32 ++++++++++++++++--- ...itMQDirectRoutingTransportCustomization.cs | 32 ++++++++++++++++--- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index 5238e51049..1709a64659 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -27,20 +27,42 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport throw new InvalidOperationException("Connection string not configured"); } + var connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); var connectionStringDictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); - var disableManagementApi = GetValue(connectionStringDictionary, "DisableManagementApi", "false"); - if (!disableManagementApi.Equals("true", StringComparison.OrdinalIgnoreCase) && !disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase)) + + var disableManagementApiString = GetValue(connectionStringDictionary, "DisableManagementApi", "false"); + if (!bool.TryParse(disableManagementApiString, out var disableManagementApi)) { throw new ArgumentException("The value for 'DisableManagementApi' must be either 'true' or 'false'"); } var transport = new RabbitMQTransport(RoutingTopology.Conventional(queueType), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; - transport.ManagementApiUrl = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); - transport.UseManagementApi = disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase); + transport.UseManagementApi = !disableManagementApi; - rabbitMQTransport = transport; + if (!transport.UseManagementApi) + { + rabbitMQTransport = transport; + return transport; + } + + var url = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); + var username = GetValue(connectionStringDictionary, "ManagementApiUserName", connectionConfiguration.UserName); + var password = GetValue(connectionStringDictionary, "ManagementApiPassword", connectionConfiguration.Password); + if (!string.IsNullOrEmpty(url)) + { + if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) + { + transport.ManagementApiConfiguration = new ManagementApiConfiguration(url, username, password); + } + else + { + transport.ManagementApiConfiguration = new ManagementApiConfiguration(url); + } + } + + rabbitMQTransport = transport; return transport; } diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 4fede6937f..4a34917f81 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -25,20 +25,42 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport throw new InvalidOperationException("Connection string not configured"); } + var connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); var connectionStringDictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); - var disableManagementApi = GetValue(connectionStringDictionary, "DisableManagementApi", "false"); - if (!disableManagementApi.Equals("true", StringComparison.OrdinalIgnoreCase) && !disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase)) + + var disableManagementApiString = GetValue(connectionStringDictionary, "DisableManagementApi", "false"); + if (!bool.TryParse(disableManagementApiString, out var disableManagementApi)) { throw new ArgumentException("The value for 'DisableManagementApi' must be either 'true' or 'false'"); } var transport = new RabbitMQTransport(RoutingTopology.Direct(queueType, routingKeyConvention: type => type.FullName.Replace(".", "-")), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; - transport.ManagementApiUrl = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); - transport.UseManagementApi = disableManagementApi.Equals("false", StringComparison.OrdinalIgnoreCase); + transport.UseManagementApi = !disableManagementApi; - rabbitMQTransport = transport; + if (!transport.UseManagementApi) + { + rabbitMQTransport = transport; + return transport; + } + + var url = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); + var username = GetValue(connectionStringDictionary, "ManagementApiUserName", connectionConfiguration.UserName); + var password = GetValue(connectionStringDictionary, "ManagementApiPassword", connectionConfiguration.Password); + if (!string.IsNullOrEmpty(url)) + { + if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) + { + transport.ManagementApiConfiguration = new ManagementApiConfiguration(url, username, password); + } + else + { + transport.ManagementApiConfiguration = new ManagementApiConfiguration(url); + } + } + + rabbitMQTransport = transport; return transport; } From f8939c0f2656fb5174b6515a6307f559cc6e4b66 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Tue, 4 Feb 2025 00:36:04 -0800 Subject: [PATCH 15/60] Remove connection string parsing --- .../RabbitMQQuery.cs | 61 +++---------------- 1 file changed, 9 insertions(+), 52 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 7cdabacb22..1252c11a30 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -9,7 +9,6 @@ namespace ServiceControl.Transports.RabbitMQ; using System.Net.Http; using System.Net.Http.Json; using System.Runtime.CompilerServices; -using System.Text; using System.Text.Json; using System.Text.Json.Nodes; using System.Threading; @@ -31,7 +30,6 @@ public class RabbitMQQuery : BrokerThroughputQuery readonly ILogger logger; readonly TimeProvider timeProvider; readonly ConnectionConfiguration connectionConfiguration; - readonly TransportSettings transportSettings; readonly RabbitMQTransport rabbitMQTransport; public RabbitMQQuery(ILogger logger, @@ -41,10 +39,10 @@ public RabbitMQQuery(ILogger logger, { this.logger = logger; this.timeProvider = timeProvider; - this.transportSettings = transportSettings; if (transportCustomization is IRabbitMQTransportExtensions rabbitMQTransportCustomization) { rabbitMQTransport = rabbitMQTransportCustomization.GetTransport(); + _ = rabbitMQTransport; } else { @@ -56,27 +54,25 @@ public RabbitMQQuery(ILogger logger, protected override void InitializeCore(ReadOnlyDictionary settings) { - var mangementApiUrl = GetManagementApiUrl(); - // The licensing component configurations take precedence over the management API connection string configuration options // https://docs.particular.net/servicecontrol/servicecontrol-instances/configuration#usage-reporting-when-using-the-rabbitmq-transport - var userName = GetSettingsValue(settings, RabbitMQSettings.UserName, mangementApiUrl.UserName); - var password = GetSettingsValue(settings, RabbitMQSettings.Password, mangementApiUrl.Password); - var apiUrl = GetSettingsValue(settings, RabbitMQSettings.API, mangementApiUrl.Uri.AbsoluteUri); + var userName = GetSettingsValue(settings, RabbitMQSettings.UserName, rabbitMQTransport.ManagementApiConfiguration.UserName); + var password = GetSettingsValue(settings, RabbitMQSettings.Password, rabbitMQTransport.ManagementApiConfiguration.Password); + var apiUrl = GetSettingsValue(settings, RabbitMQSettings.API, rabbitMQTransport.ManagementApiConfiguration.Url); - if (userName != mangementApiUrl.UserName) + if (userName != rabbitMQTransport.ManagementApiConfiguration.UserName) { - _ = Diagnostics.AppendLine($"UserName in settings is different from Management API URL: {userName} != {mangementApiUrl.UserName}"); + _ = Diagnostics.AppendLine($"UserName in settings is different from Management API URL: {userName} != {rabbitMQTransport.ManagementApiConfiguration.UserName}"); } - if (password != mangementApiUrl.Password) + if (password != rabbitMQTransport.ManagementApiConfiguration.Password) { _ = Diagnostics.AppendLine($"Password in settings is different from Management API URL."); } - if (apiUrl != mangementApiUrl.Uri.AbsoluteUri) + if (apiUrl != rabbitMQTransport.ManagementApiConfiguration.Url) { - _ = Diagnostics.AppendLine($"API URL in settings is different from Management API URL: {apiUrl} != {mangementApiUrl.Uri.AbsoluteUri}"); + _ = Diagnostics.AppendLine($"API URL in settings is different from Management API URL: {apiUrl} != {rabbitMQTransport.ManagementApiConfiguration.Url}"); } if (!Uri.TryCreate(apiUrl, UriKind.Absolute, out _)) @@ -118,45 +114,6 @@ string GetSettingsValue(ReadOnlyDictionary settings, string key, return value; } - UriBuilder GetManagementApiUrl() - { - var dictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); - UriBuilder uriBuilder; - - var managementApiUrl = GetValue(dictionary, "ManagementApiUrl", ""); - - if (string.IsNullOrEmpty(managementApiUrl)) - { - _ = Diagnostics.AppendLine("ManagementApiUrl is empty. Setting credentials from the connectionString used by the instance."); - uriBuilder = new UriBuilder() - { - Scheme = connectionConfiguration.UseTls ? "https" : "http", - Host = connectionConfiguration.Host, - }; - } - else - { - uriBuilder = new UriBuilder(managementApiUrl); - } - - uriBuilder.UserName = string.IsNullOrEmpty(uriBuilder.UserName) ? connectionConfiguration.UserName : uriBuilder.UserName; - uriBuilder.Password = string.IsNullOrEmpty(uriBuilder.Password) ? connectionConfiguration.Password : uriBuilder.Password; - - // Check if port was defined in the connection string or if it's the default value from UriBuilder - var isPortProvided = uriBuilder.Port != -1 - && (!((uriBuilder.Port == 80 || uriBuilder.Port == 443) - && !managementApiUrl.Contains($":{uriBuilder.Port}"))); - - uriBuilder.Port = isPortProvided ? uriBuilder.Port : uriBuilder.Scheme == "https" ? 15671 : 15672; - - return uriBuilder; - } - - static string GetValue(Dictionary dictionary, string key, string defaultValue) - { - return dictionary.TryGetValue(key, out var value) ? value : defaultValue; - } - protected virtual HttpClient CreateHttpClient(NetworkCredential defaultCredential, string apiUrl) => new(new SocketsHttpHandler { From 3f079f425e0d4136fde4173a341502b66b4e87af Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Thu, 6 Feb 2025 21:30:59 -0800 Subject: [PATCH 16/60] Enable strong naming --- .reposync.yml | 3 +-- src/NServiceBus.snk | Bin 0 -> 596 bytes .../ServiceControl.Transports.RabbitMQ.csproj | 2 ++ .../ServiceControl.Transports.csproj | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 src/NServiceBus.snk diff --git a/.reposync.yml b/.reposync.yml index 05ec9e3185..8b13789179 100644 --- a/.reposync.yml +++ b/.reposync.yml @@ -1,2 +1 @@ -exclusions: -- src/NServiceBus.snk + diff --git a/src/NServiceBus.snk b/src/NServiceBus.snk new file mode 100644 index 0000000000000000000000000000000000000000..6fa7ddec107b33bd6da190c08275d6c10af3be7d GIT binary patch literal 596 zcmV-a0;~N80ssI2Bme+XQ$aES1ONa50098q>1E~@E&-azEG9Ys2^;tEH|(KljtaE-$XkpB@Ownw@ zb+wC<19hxmP<9uABWXNrPZKd_8e;3Bj3kqWg;qCs4nOtsI&FnV0P3cIKJgVr(JeRIRcl*d;SswO<|nyVEFU@+C20$L@E~%;cCH zxeB1g4xZq;@C<1|dd0gnD6E^s6Kw}I6O7qzHY@GWLw3@VJ zlE6iHNPJy}#^!Lt3$iiqH77>+v^OolAVgl1Rs^i%54KcUycsmu#O+>8625DxPD<}| zeEmEMT%o(BV;EXSAMbx`zdJ|(lv(Vd=FDi`ajh&TSUl=wi@aYShFOEDgY}ZN<^#oY#T*~SFhB|#dQ6Q}V literal 0 HcmV?d00001 diff --git a/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj b/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj index b04877eea0..3450935934 100644 --- a/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj +++ b/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj @@ -2,6 +2,8 @@ net8.0 + true + ..\NServiceBus.snk true diff --git a/src/ServiceControl.Transports/ServiceControl.Transports.csproj b/src/ServiceControl.Transports/ServiceControl.Transports.csproj index 6bd8eeea61..435491c160 100644 --- a/src/ServiceControl.Transports/ServiceControl.Transports.csproj +++ b/src/ServiceControl.Transports/ServiceControl.Transports.csproj @@ -2,6 +2,8 @@ net8.0 + true + ..\NServiceBus.snk From 6edd1961dec813af0682405a415da9cfd2832662 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Mon, 10 Feb 2025 21:26:03 -0800 Subject: [PATCH 17/60] Change connection string option to ValidateDeliveryLimit --- ...nventionalRoutingTransportCustomization.cs | 28 ++++++------------- ...itMQDirectRoutingTransportCustomization.cs | 28 ++++++------------- .../transport.manifest | 4 +-- 3 files changed, 18 insertions(+), 42 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index 1709a64659..5f42b65108 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -30,36 +30,24 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport var connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); var connectionStringDictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); - var disableManagementApiString = GetValue(connectionStringDictionary, "DisableManagementApi", "false"); - if (!bool.TryParse(disableManagementApiString, out var disableManagementApi)) + var ValidateDeliveryLimitsString = GetValue(connectionStringDictionary, "ValidateDeliveryLimit", "false"); + if (!bool.TryParse(ValidateDeliveryLimitsString, out var validateDeliveryLimits)) { - throw new ArgumentException("The value for 'DisableManagementApi' must be either 'true' or 'false'"); + throw new ArgumentException("The value for 'ValidateDeliveryLimit' must be either 'true' or 'false'"); } var transport = new RabbitMQTransport(RoutingTopology.Conventional(queueType), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; - transport.UseManagementApi = !disableManagementApi; - - if (!transport.UseManagementApi) - { - rabbitMQTransport = transport; - return transport; - } + transport.ValidateDeliveryLimits = validateDeliveryLimits; var url = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); - var username = GetValue(connectionStringDictionary, "ManagementApiUserName", connectionConfiguration.UserName); - var password = GetValue(connectionStringDictionary, "ManagementApiPassword", connectionConfiguration.Password); if (!string.IsNullOrEmpty(url)) { - if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) - { - transport.ManagementApiConfiguration = new ManagementApiConfiguration(url, username, password); - } - else - { - transport.ManagementApiConfiguration = new ManagementApiConfiguration(url); - } + var username = GetValue(connectionStringDictionary, "ManagementApiUserName", connectionConfiguration.UserName); + var password = GetValue(connectionStringDictionary, "ManagementApiPassword", connectionConfiguration.Password); + transport.ManagementApiConfiguration = !string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password) + ? new ManagementApiConfiguration(url, username, password) : new ManagementApiConfiguration(url); } rabbitMQTransport = transport; diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 4a34917f81..5b8c9cd8b2 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -28,36 +28,24 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport var connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); var connectionStringDictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); - var disableManagementApiString = GetValue(connectionStringDictionary, "DisableManagementApi", "false"); - if (!bool.TryParse(disableManagementApiString, out var disableManagementApi)) + var ValidateDeliveryLimitsString = GetValue(connectionStringDictionary, "ValidateDeliveryLimits", "true"); + if (!bool.TryParse(ValidateDeliveryLimitsString, out var validateDeliveryLimits)) { - throw new ArgumentException("The value for 'DisableManagementApi' must be either 'true' or 'false'"); + throw new ArgumentException("The value for 'ValidateDeliveryLimit' must be either 'true' or 'false'"); } var transport = new RabbitMQTransport(RoutingTopology.Direct(queueType, routingKeyConvention: type => type.FullName.Replace(".", "-")), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; - transport.UseManagementApi = !disableManagementApi; - - if (!transport.UseManagementApi) - { - rabbitMQTransport = transport; - return transport; - } + transport.ValidateDeliveryLimits = validateDeliveryLimits; var url = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); - var username = GetValue(connectionStringDictionary, "ManagementApiUserName", connectionConfiguration.UserName); - var password = GetValue(connectionStringDictionary, "ManagementApiPassword", connectionConfiguration.Password); if (!string.IsNullOrEmpty(url)) { - if (!string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password)) - { - transport.ManagementApiConfiguration = new ManagementApiConfiguration(url, username, password); - } - else - { - transport.ManagementApiConfiguration = new ManagementApiConfiguration(url); - } + var username = GetValue(connectionStringDictionary, "ManagementApiUserName", connectionConfiguration.UserName); + var password = GetValue(connectionStringDictionary, "ManagementApiPassword", connectionConfiguration.Password); + transport.ManagementApiConfiguration = !string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password) + ? new ManagementApiConfiguration(url, username, password) : new ManagementApiConfiguration(url); } rabbitMQTransport = transport; diff --git a/src/ServiceControl.Transports.RabbitMQ/transport.manifest b/src/ServiceControl.Transports.RabbitMQ/transport.manifest index 3bb7a14df6..a7aa1ef04b 100644 --- a/src/ServiceControl.Transports.RabbitMQ/transport.manifest +++ b/src/ServiceControl.Transports.RabbitMQ/transport.manifest @@ -27,7 +27,7 @@ "DisplayName": "RabbitMQ - Conventional routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumConventionalRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;DisableManagementApi=;ManagementApiUrl=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=;ManagementApiUrl=;ManagementApiPassword=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumConventialRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" @@ -38,7 +38,7 @@ "DisplayName": "RabbitMQ - Direct routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumDirectRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;DisableManagementApi=;ManagementApiUrl=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=ManagementApiUrl=;ManagementApiPassword=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumDirectRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" From 316d6ab365e4d9d530555991b07a9387387caa2f Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Tue, 11 Feb 2025 17:57:09 -0800 Subject: [PATCH 18/60] Make RabbitMQ transport internals visible to RabbitMQ transport tests --- .../ServiceControl.Transports.RabbitMQ.csproj | 4 ++++ ...ontrol.Transports.RabbitMQQuorumDirectRouting.Tests.csproj | 2 ++ src/TestHelper/TestHelper.csproj | 2 ++ 3 files changed, 8 insertions(+) diff --git a/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj b/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj index 3450935934..f8c635dca9 100644 --- a/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj +++ b/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj @@ -25,4 +25,8 @@ + + + + \ No newline at end of file diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj index 6cb4aacd7d..99947dda0a 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj +++ b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj @@ -2,6 +2,8 @@ net8.0 + true + ..\NServiceBus.snk diff --git a/src/TestHelper/TestHelper.csproj b/src/TestHelper/TestHelper.csproj index a310aef090..870af44c6a 100644 --- a/src/TestHelper/TestHelper.csproj +++ b/src/TestHelper/TestHelper.csproj @@ -2,6 +2,8 @@ net8.0 + true + ..\NServiceBus.snk From 70e062ba1ae3ef3473e88f321ae0ed2ae16a6ce5 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Tue, 11 Feb 2025 17:57:59 -0800 Subject: [PATCH 19/60] Remove disableManagementApi --- .../TransportConfigurationExtensions.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs b/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs index 88cbec4ded..ee10f15afb 100644 --- a/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs +++ b/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs @@ -21,11 +21,6 @@ public static void ApplyConnectionString(this TransportExtensions Date: Tue, 11 Feb 2025 18:55:53 -0800 Subject: [PATCH 20/60] Update RabbitMQQuery to use the transport management client --- .../RabbitMQBrokerQueueDetails.cs | 17 +- .../RabbitMQQuery.cs | 213 ++++++++---------- 2 files changed, 94 insertions(+), 136 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs index 742efa89a1..686778637d 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs @@ -2,18 +2,18 @@ namespace ServiceControl.Transports.RabbitMQ; using System.Collections.Generic; -using System.Text.Json; using ServiceControl.Transports.BrokerThroughput; +using NServiceBus.Transport.RabbitMQ.ManagementApi; -public class RabbitMQBrokerQueueDetails(JsonElement token) : IBrokerQueue +class RabbitMQBrokerQueueDetails(Queue queue) : IBrokerQueue { - public string QueueName { get; } = token.GetProperty("name").GetString()!; + public string QueueName { get; } = queue.Name; public string SanitizedName => QueueName; public string Scope => VHost; - public string VHost { get; } = token.GetProperty("vhost").GetString()!; + public string VHost { get; } = queue.Vhost; public List EndpointIndicators { get; } = []; - long? AckedMessages { get; set; } = FromToken(token); - long Baseline { get; set; } = FromToken(token) ?? 0; + long? AckedMessages { get; set; } = queue.MessageStats?.Ack; + long Baseline { get; set; } = queue.MessageStats?.Ack ?? 0; public long CalculateThroughputFrom(RabbitMQBrokerQueueDetails newReading) { @@ -32,9 +32,4 @@ public long CalculateThroughputFrom(RabbitMQBrokerQueueDetails newReading) return newlyAckedMessages; } - - static long? FromToken(JsonElement jsonElement) => - jsonElement.TryGetProperty("message_stats", out var stats) && stats.TryGetProperty("ack", out var val) - ? val.GetInt64() - : null; } \ No newline at end of file diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 1252c11a30..c5de9c1914 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -7,10 +7,7 @@ namespace ServiceControl.Transports.RabbitMQ; using System.Linq; using System.Net; using System.Net.Http; -using System.Net.Http.Json; using System.Runtime.CompilerServices; -using System.Text.Json; -using System.Text.Json.Nodes; using System.Threading; using System.Threading.Tasks; using System.Web; @@ -19,6 +16,7 @@ namespace ServiceControl.Transports.RabbitMQ; using Polly; using Polly.Retry; using ServiceControl.Transports.BrokerThroughput; +using NServiceBus.Transport.RabbitMQ.ManagementApi; public class RabbitMQQuery : BrokerThroughputQuery { @@ -29,7 +27,6 @@ public class RabbitMQQuery : BrokerThroughputQuery .Build(); readonly ILogger logger; readonly TimeProvider timeProvider; - readonly ConnectionConfiguration connectionConfiguration; readonly RabbitMQTransport rabbitMQTransport; public RabbitMQQuery(ILogger logger, @@ -39,81 +36,38 @@ public RabbitMQQuery(ILogger logger, { this.logger = logger; this.timeProvider = timeProvider; - if (transportCustomization is IRabbitMQTransportExtensions rabbitMQTransportCustomization) - { - rabbitMQTransport = rabbitMQTransportCustomization.GetTransport(); - _ = rabbitMQTransport; - } - else - { - throw new InvalidOperationException($"Expected a RabbitMQTransport but received {transportCustomization.GetType().Name}."); - } - - connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); + rabbitMQTransport = GetRabbitMQTransport(transportCustomization); } protected override void InitializeCore(ReadOnlyDictionary settings) { - // The licensing component configurations take precedence over the management API connection string configuration options - // https://docs.particular.net/servicecontrol/servicecontrol-instances/configuration#usage-reporting-when-using-the-rabbitmq-transport - var userName = GetSettingsValue(settings, RabbitMQSettings.UserName, rabbitMQTransport.ManagementApiConfiguration.UserName); - var password = GetSettingsValue(settings, RabbitMQSettings.Password, rabbitMQTransport.ManagementApiConfiguration.Password); - var apiUrl = GetSettingsValue(settings, RabbitMQSettings.API, rabbitMQTransport.ManagementApiConfiguration.Url); - - if (userName != rabbitMQTransport.ManagementApiConfiguration.UserName) - { - _ = Diagnostics.AppendLine($"UserName in settings is different from Management API URL: {userName} != {rabbitMQTransport.ManagementApiConfiguration.UserName}"); - } - - if (password != rabbitMQTransport.ManagementApiConfiguration.Password) - { - _ = Diagnostics.AppendLine($"Password in settings is different from Management API URL."); - } - - if (apiUrl != rabbitMQTransport.ManagementApiConfiguration.Url) - { - _ = Diagnostics.AppendLine($"API URL in settings is different from Management API URL: {apiUrl} != {rabbitMQTransport.ManagementApiConfiguration.Url}"); - } + //// TODO: Update documentation + //// https://docs.particular.net/servicecontrol/servicecontrol-instances/configuration#usage-reporting-when-using-the-rabbitmq-transport + CheckLegacySettings(settings, RabbitMQSettings.UserName); + CheckLegacySettings(settings, RabbitMQSettings.Password); + CheckLegacySettings(settings, RabbitMQSettings.API); + } - if (!Uri.TryCreate(apiUrl, UriKind.Absolute, out _)) + static RabbitMQTransport GetRabbitMQTransport(ITransportCustomization transportCustomization) + { + if (transportCustomization is IRabbitMQTransportExtensions rabbitMQTransportCustomization) { - InitialiseErrors.Add("API url configured is invalid"); + return rabbitMQTransportCustomization.GetTransport(); } - var defaultCredential = new NetworkCredential(userName, password); - - if (InitialiseErrors.Count == 0) - { - // ideally we would use the HttpClientFactory, but it would be a bit more involved to set that up - // so for now we are using a virtual method that can be overriden in tests - // https://github.com/Particular/ServiceControl/issues/4493 - httpClient = CreateHttpClient(defaultCredential, apiUrl); - } + throw new InvalidOperationException($"Expected a RabbitMQTransport but received {transportCustomization.GetType().Name}."); } - string GetSettingsValue(ReadOnlyDictionary settings, string key, string defaultValue) + void CheckLegacySettings(ReadOnlyDictionary settings, string key) { - if (!settings.TryGetValue(key, out string? value) || - string.IsNullOrEmpty(value)) - { - logger.LogInformation($"Using {key} from connection string"); - value = defaultValue; - _ = Diagnostics.AppendLine( - $"{key} not set, defaulted to using {key} from the ConnectionString used by instance"); - } - else + if (settings.TryGetValue(key, out _)) { - if (key == RabbitMQSettings.Password) - { - _ = Diagnostics.AppendLine($"{key} set."); - } - - _ = Diagnostics.AppendLine($"{key} set to {value}."); + logger.LogInformation($"The legacy LicensingComponent/{key} is still defined in the app.config or environment variables"); + _ = Diagnostics.AppendLine($"LicensingComponent/{key} is still defined in the app.config or environment variables"); } - - return value; } + // TODO: Determine if this needs to be updated in the RabbitMQ Transport protected virtual HttpClient CreateHttpClient(NetworkCredential defaultCredential, string apiUrl) => new(new SocketsHttpHandler { @@ -130,7 +84,16 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro var url = $"/api/queues/{HttpUtility.UrlEncode(queue.VHost)}/{HttpUtility.UrlEncode(queue.QueueName)}"; logger.LogDebug($"Querying {url}"); - var newReading = await pipeline.ExecuteAsync(async token => new RabbitMQBrokerQueueDetails(await httpClient!.GetFromJsonAsync(url, token)), cancellationToken); + + var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, cancellationToken), cancellationToken); + + if (!response.HasValue) + { + throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); + } + + var newReading = new RabbitMQBrokerQueueDetails(response.Value); + _ = queue.CalculateThroughputFrom(newReading); // looping for 24hrs, in 4 increments of 15 minutes @@ -138,8 +101,14 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro { await Task.Delay(TimeSpan.FromMinutes(15), timeProvider, cancellationToken); logger.LogDebug($"Querying {url}"); - newReading = await pipeline.ExecuteAsync(async token => new RabbitMQBrokerQueueDetails(await httpClient!.GetFromJsonAsync(url, token)), cancellationToken); + response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, cancellationToken), cancellationToken); + if (!response.HasValue) + { + throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); + } + + newReading = new RabbitMQBrokerQueueDetails(response.Value); var newTotalThroughput = queue.CalculateThroughputFrom(newReading); yield return new QueueThroughput { @@ -151,31 +120,32 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro async Task<(string rabbitVersion, string managementVersion)> GetRabbitDetails(bool skipResiliencePipeline, CancellationToken cancellationToken) { - var overviewUrl = "/api/overview"; + Response response = skipResiliencePipeline + ? await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken) + : await pipeline.ExecuteAsync(async async => await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken), cancellationToken); - JsonObject obj; + var overview = GetResponseValue(response); - if (skipResiliencePipeline) + if (overview.DisableStats) { - obj = (await httpClient!.GetFromJsonAsync(overviewUrl, cancellationToken))!; - } - else - { - obj = (await pipeline.ExecuteAsync(async token => - await httpClient!.GetFromJsonAsync(overviewUrl, token), cancellationToken))!; + throw new Exception("The RabbitMQ broker is configured with 'management.disable_stats = true' or 'management_agent.disable_metrics_collector = true' " + + "and as a result queue statistics cannot be collected using this tool. Consider changing the configuration of the RabbitMQ broker."); } - var statsDisabled = obj["disable_stats"]?.GetValue() ?? false; + var rabbitVersion = response.Value?.BrokerVersion ?? response.Value?.ProductVersion; + var mgmtVersion = response.Value?.ManagementVersion; - if (statsDisabled) + return (rabbitVersion?.ToString() ?? "Unknown", mgmtVersion?.ToString() ?? "Unknown"); + } + + static T GetResponseValue(Response response) where T : class + { + if (!response.HasValue || response.Value is null) { - throw new Exception("The RabbitMQ broker is configured with 'management.disable_stats = true' or 'management_agent.disable_metrics_collector = true' and as a result queue statistics cannot be collected using this tool. Consider changing the configuration of the RabbitMQ broker."); + throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); } - var rabbitVersion = obj["rabbitmq_version"] ?? obj["product_version"]; - var mgmtVersion = obj["management_version"]; - - return (rabbitVersion?.GetValue() ?? "Unknown", mgmtVersion?.GetValue() ?? "Unknown"); + return response.Value; } public override async IAsyncEnumerable GetQueueNames( @@ -219,16 +189,15 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can { try { - var bindingsUrl = $"/api/queues/{HttpUtility.UrlEncode(brokerQueue.VHost)}/{HttpUtility.UrlEncode(brokerQueue.QueueName)}/bindings"; - var bindings = await pipeline.ExecuteAsync(async token => await httpClient!.GetFromJsonAsync(bindingsUrl, token), cancellationToken); - var conventionalBindingFound = bindings?.Any(binding => binding!["source"]?.GetValue() == brokerQueue.QueueName - && binding["vhost"]?.GetValue() == brokerQueue.VHost - && binding["destination"]?.GetValue() == brokerQueue.QueueName - && binding["destination_type"]?.GetValue() == "queue" - && binding["routing_key"]?.GetValue() == string.Empty - && binding["properties_key"]?.GetValue() == "~") ?? false; - - if (conventionalBindingFound) + var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueueBindings(brokerQueue.QueueName, cancellationToken), cancellationToken); + + // Check if conventional binding is found + if (response.Value.Any(binding => binding?.Source == brokerQueue.QueueName + && binding?.Vhost == brokerQueue.VHost + && binding?.Destination == brokerQueue.QueueName + && binding?.DestinationType == "queue" + && binding?.RoutingKey == string.Empty + && binding?.PropertiesKey == "~")) { brokerQueue.EndpointIndicators.Add("ConventionalTopologyBinding"); } @@ -240,20 +209,14 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can try { - var exchangeUrl = $"/api/exchanges/{HttpUtility.UrlEncode(brokerQueue.VHost)}/{HttpUtility.UrlEncode(brokerQueue.QueueName)}/bindings/destination"; - var bindings = await pipeline.ExecuteAsync(async token => await httpClient!.GetFromJsonAsync(exchangeUrl, token), cancellationToken); - var delayBindingFound = bindings?.Any(binding => - { - var source = binding!["source"]?.GetValue(); - - return source is "nsb.v2.delay-delivery" or "nsb.delay-delivery" - && binding["vhost"]?.GetValue() == brokerQueue.VHost - && binding["destination"]?.GetValue() == brokerQueue.QueueName - && binding["destination_type"]?.GetValue() == "exchange" - && binding["routing_key"]?.GetValue() == $"#.{brokerQueue.QueueName}"; - }) ?? false; - - if (delayBindingFound) + var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetExchangeBindingsDestination(brokerQueue.QueueName, cancellationToken), cancellationToken); + + // Check if delayed binding is found + if (response.Value.Any(binding => binding?.Source is "nsb.v2.delay-delivery" or "nsb.delay-delivery" + && binding?.Vhost == brokerQueue.VHost + && binding?.Destination == brokerQueue.QueueName + && binding?.DestinationType == "exchange" + && binding?.RoutingKey == $"#.{brokerQueue.QueueName}")) { brokerQueue.EndpointIndicators.Add("DelayBinding"); } @@ -264,41 +227,41 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can } } - public async Task<(RabbitMQBrokerQueueDetails[]?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) + internal async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) { - var url = $"/api/queues/{HttpUtility.UrlEncode(connectionConfiguration.VirtualHost)}?page={page}&page_size=500&name=&use_regex=false&pagination=true"; - - var container = await pipeline.ExecuteAsync(async token => await httpClient!.GetFromJsonAsync(url, token), cancellationToken); - switch (container) + var pagination = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetPage(page, cancellationToken), cancellationToken); + switch (pagination.Value) { - case JsonObject obj: + case Pagination obj: { - var pageCount = obj["page_count"]!.GetValue(); - var pageReturned = obj["page"]!.GetValue(); + var pageCount = obj.PageCount; + var pageReturned = obj.Page; - if (obj["items"] is not JsonArray items) + if (obj.Items is null) //is not JsonArray items { return (null, false); } - return (MaterializeQueueDetails(items), pageCount > pageReturned); + return (MaterializeQueueDetails(obj.Items), pageCount > pageReturned); } // Older versions of RabbitMQ API did not have paging and returned the array of items directly - case JsonArray arr: - { - return (MaterializeQueueDetails(arr), false); - } + //case JsonArray arr: + // { + // return (MaterializeQueueDetails(arr), false); + // } default: throw new Exception("Was not able to get list of queues from RabbitMQ broker."); } } - static RabbitMQBrokerQueueDetails[] MaterializeQueueDetails(JsonArray items) + static List MaterializeQueueDetails(List items) { - // It is not possible to directly operated on the JsonNode. When the JsonNode is a JObject - // and the indexer is access the internal dictionary is initialized which can cause key not found exceptions - // when the payload contains the same key multiple times (which happened in the past). - var queues = items.Select(item => new RabbitMQBrokerQueueDetails(item!.Deserialize())).ToArray(); + var queues = new List(); + foreach (var item in items) + { + queues.Add(new RabbitMQBrokerQueueDetails(item)); + } + return queues; } From 8afde0cc7e87532b9a071ad0d14c67143ab5abd3 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Sat, 22 Feb 2025 00:09:10 -0800 Subject: [PATCH 21/60] Fix text --- .../RabbitMQConventionalRoutingTransportCustomization.cs | 4 ++-- .../RabbitMQDirectRoutingTransportCustomization.cs | 2 +- .../RabbitMQQueryTests.cs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index 5f42b65108..188b4ca722 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -30,10 +30,10 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport var connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); var connectionStringDictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); - var ValidateDeliveryLimitsString = GetValue(connectionStringDictionary, "ValidateDeliveryLimit", "false"); + var ValidateDeliveryLimitsString = GetValue(connectionStringDictionary, "ValidateDeliveryLimits", "false"); if (!bool.TryParse(ValidateDeliveryLimitsString, out var validateDeliveryLimits)) { - throw new ArgumentException("The value for 'ValidateDeliveryLimit' must be either 'true' or 'false'"); + throw new ArgumentException("The value for 'ValidateDeliveryLimits' must be either 'true' or 'false'"); } var transport = new RabbitMQTransport(RoutingTopology.Conventional(queueType), transportSettings.ConnectionString, enableDelayedDelivery: false); diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 5b8c9cd8b2..066816a5f5 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -31,7 +31,7 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport var ValidateDeliveryLimitsString = GetValue(connectionStringDictionary, "ValidateDeliveryLimits", "true"); if (!bool.TryParse(ValidateDeliveryLimitsString, out var validateDeliveryLimits)) { - throw new ArgumentException("The value for 'ValidateDeliveryLimit' must be either 'true' or 'false'"); + throw new ArgumentException("The value for 'ValidateDeliveryLimits' must be either 'true' or 'false'"); } var transport = new RabbitMQTransport(RoutingTopology.Direct(queueType, routingKeyConvention: type => type.FullName.Replace(".", "-")), transportSettings.ConnectionString, enableDelayedDelivery: false); diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs index 8cd6a4efea..4a552ae196 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs +++ b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs @@ -123,7 +123,7 @@ public async Task RunScenario() reset.Set(); await runScenarioAndAdvanceTime.WaitAsync(token); - // Asserting that we have one message per hour during 24 hours, the first snapshot is not counted hence the 23 assertion. + // Asserting that we have one message per hour during 24 hours, the first snapshot is not counted hence the 23 assertion. Assert.That(total, Is.GreaterThan(numMessagesToIngest)); } } \ No newline at end of file From b8fa270dc92b7ae0beb434c5753afd3634e30a74 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Tue, 25 Feb 2025 21:24:03 -0800 Subject: [PATCH 22/60] Update API calls --- .../RabbitMQQuery.cs | 68 +++++++------------ 1 file changed, 24 insertions(+), 44 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index c5de9c1914..4a686241d4 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -41,8 +41,8 @@ public RabbitMQQuery(ILogger logger, protected override void InitializeCore(ReadOnlyDictionary settings) { - //// TODO: Update documentation - //// https://docs.particular.net/servicecontrol/servicecontrol-instances/configuration#usage-reporting-when-using-the-rabbitmq-transport + // TODO: Update documentation + // https://docs.particular.net/servicecontrol/servicecontrol-instances/configuration#usage-reporting-when-using-the-rabbitmq-transport CheckLegacySettings(settings, RabbitMQSettings.UserName); CheckLegacySettings(settings, RabbitMQSettings.Password); CheckLegacySettings(settings, RabbitMQSettings.API); @@ -87,7 +87,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, cancellationToken), cancellationToken); - if (!response.HasValue) + if (response.Value is null) { throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); } @@ -103,7 +103,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro logger.LogDebug($"Querying {url}"); response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, cancellationToken), cancellationToken); - if (!response.HasValue) + if (response.Value is not null) { throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); } @@ -118,46 +118,44 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro } } - async Task<(string rabbitVersion, string managementVersion)> GetRabbitDetails(bool skipResiliencePipeline, CancellationToken cancellationToken) + async Task GetRabbitDetails(bool skipResiliencePipeline, CancellationToken cancellationToken) { - Response response = skipResiliencePipeline + + var response = skipResiliencePipeline ? await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken) : await pipeline.ExecuteAsync(async async => await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken), cancellationToken); - var overview = GetResponseValue(response); + ValidateResponse(response); - if (overview.DisableStats) + if (response.Value!.DisableStats) { throw new Exception("The RabbitMQ broker is configured with 'management.disable_stats = true' or 'management_agent.disable_metrics_collector = true' " + "and as a result queue statistics cannot be collected using this tool. Consider changing the configuration of the RabbitMQ broker."); } - var rabbitVersion = response.Value?.BrokerVersion ?? response.Value?.ProductVersion; - var mgmtVersion = response.Value?.ManagementVersion; - - return (rabbitVersion?.ToString() ?? "Unknown", mgmtVersion?.ToString() ?? "Unknown"); + Data["RabbitMQVersion"] = response.Value?.BrokerVersion ?? "Unknown"; } - static T GetResponseValue(Response response) where T : class + void ValidateResponse((HttpStatusCode StatusCode, string Reason, T? Value) response) { - if (!response.HasValue || response.Value is null) + if (response.StatusCode != HttpStatusCode.OK) { - throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); + throw new HttpRequestException($"Request failed with status code {response.StatusCode}: {response.Reason}"); } - return response.Value; + if (response.Value is null) + { + throw new InvalidOperationException("Request was successful, but the response body was null when a value was expected"); + } } - public override async IAsyncEnumerable GetQueueNames( - [EnumeratorCancellation] CancellationToken cancellationToken) + public override async IAsyncEnumerable GetQueueNames([EnumeratorCancellation] CancellationToken cancellationToken) { var page = 1; bool morePages; var vHosts = new HashSet(StringComparer.CurrentCultureIgnoreCase); - (string rabbitVersion, string managementVersion) = await GetRabbitDetails(false, cancellationToken); - Data["RabbitMQVersion"] = rabbitVersion; - Data["RabbitMQManagementVersionVersion"] = managementVersion; + await GetRabbitDetails(false, cancellationToken); do { @@ -189,7 +187,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can { try { - var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueueBindings(brokerQueue.QueueName, cancellationToken), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetBindingsForQueue(brokerQueue.QueueName, cancellationToken), cancellationToken); // Check if conventional binding is found if (response.Value.Any(binding => binding?.Source == brokerQueue.QueueName @@ -209,7 +207,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can try { - var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetExchangeBindingsDestination(brokerQueue.QueueName, cancellationToken), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetBindingsForExchange(brokerQueue.QueueName, cancellationToken), cancellationToken); // Check if delayed binding is found if (response.Value.Any(binding => binding?.Source is "nsb.v2.delay-delivery" or "nsb.delay-delivery" @@ -229,29 +227,11 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can internal async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) { - var pagination = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetPage(page, cancellationToken), cancellationToken); - switch (pagination.Value) - { - case Pagination obj: - { - var pageCount = obj.PageCount; - var pageReturned = obj.Page; + var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueues(page, 500, cancellationToken), cancellationToken); - if (obj.Items is null) //is not JsonArray items - { - return (null, false); - } + ValidateResponse((StatusCode, Reason, Value)); - return (MaterializeQueueDetails(obj.Items), pageCount > pageReturned); - } - // Older versions of RabbitMQ API did not have paging and returned the array of items directly - //case JsonArray arr: - // { - // return (MaterializeQueueDetails(arr), false); - // } - default: - throw new Exception("Was not able to get list of queues from RabbitMQ broker."); - } + return (MaterializeQueueDetails(Value), MorePages); } static List MaterializeQueueDetails(List items) From 7b5bb3b1233adc80448b5ca49d2dd4b3a620e26d Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Tue, 25 Feb 2025 21:33:09 -0800 Subject: [PATCH 23/60] Remove HttpClient and testing for it --- .../RabbitMQQuery.cs | 9 -- .../RabbitMQQuery_ResponseParsing_Tests.cs | 149 ------------------ 2 files changed, 158 deletions(-) delete mode 100644 src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQuery_ResponseParsing_Tests.cs diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 4a686241d4..7a134d2a57 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -67,15 +67,6 @@ void CheckLegacySettings(ReadOnlyDictionary settings, string key } } - // TODO: Determine if this needs to be updated in the RabbitMQ Transport - protected virtual HttpClient CreateHttpClient(NetworkCredential defaultCredential, string apiUrl) => - new(new SocketsHttpHandler - { - Credentials = defaultCredential, - PooledConnectionLifetime = TimeSpan.FromMinutes(2) - }) - { BaseAddress = new Uri(apiUrl) }; - public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQuery_ResponseParsing_Tests.cs b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQuery_ResponseParsing_Tests.cs deleted file mode 100644 index fd7aace0da..0000000000 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQuery_ResponseParsing_Tests.cs +++ /dev/null @@ -1,149 +0,0 @@ -namespace ServiceControl.Transport.Tests; - -using System; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging.Abstractions; -using Microsoft.Extensions.Time.Testing; -using NUnit.Framework; -using Transports; -using Transports.RabbitMQ; -using System.Net.Http; -using Particular.Approvals; -using System.Collections.ObjectModel; -using System.Net; - -[TestFixture] -class RabbitMQQuery_ResponseParsing_Tests : TransportTestFixture -{ - FakeTimeProvider provider; - TransportSettings transportSettings; - FakeHttpHandler httpHandler; - RabbitMQQuery rabbitMQQuery; - - [SetUp] - public void SetUp() - { - provider = new(); - provider.SetUtcNow(DateTimeOffset.UtcNow); - transportSettings = new TransportSettings - { - ConnectionString = "host=localhost;username=rabbitmq;password=rabbitmq", - MaxConcurrency = 1, - EndpointName = Guid.NewGuid().ToString("N") - }; - httpHandler = new FakeHttpHandler(); - var httpClient = new HttpClient(httpHandler) { BaseAddress = new Uri("http://localhost:15672") }; - - rabbitMQQuery = new TestableRabbitMQQuery(provider, transportSettings, httpClient, configuration.TransportCustomization); - rabbitMQQuery.Initialize(ReadOnlyDictionary.Empty); - } - - [TearDown] - public void TearDown() => httpHandler.Dispose(); - - public Func SendCallback - { - get => httpHandler.SendCallback; - set => httpHandler.SendCallback = value; - } - - [Test] - public async Task Should_handle_duplicated_json_data() - { - SendCallback = _ => - { - var response = new HttpResponseMessage - { - Content = new StringContent(""" - { - "items": [ - { - "name": "queue1", - "vhost": "vhost1", - "memory": 1024, - "memory": 1024, - "message_stats": { - "ack": 1 - } - }, - { - "name": "queue2", - "vhost": "vhost2", - "vhost": "vhost2", - "message_stats": { - "ack": 2 - } - } - ], - "page": 1, - "page_count": 1, - "page_size": 500, - "total_count": 2 - } - """) - }; - return response; - }; - - var queues = (await rabbitMQQuery.GetPage(1, default)).Item1; - Approver.Verify(queues); - } - - [Test] - public async Task Should_fetch_queue_details_in_old_format() - { - SendCallback = _ => - { - var response = new HttpResponseMessage - { - Content = new StringContent(""" - [ - { - "name": "queue1", - "vhost": "vhost1", - "memory": 1024, - "message_stats": { - "ack": 1 - } - }, - { - "name": "queue2", - "vhost": "vhost2", - "message_stats": { - "ack": 2 - } - }, - { - "name": "queue3", - "vhost": "vhost1" - } - ] - """) - }; - return response; - }; - - var queues = (await rabbitMQQuery.GetPage(1, default)).Item1; - Approver.Verify(queues); - } - - sealed class TestableRabbitMQQuery( - TimeProvider timeProvider, - TransportSettings transportSettings, - HttpClient customHttpClient, - ITransportCustomization transportCustomization) - : RabbitMQQuery(NullLogger.Instance, timeProvider, transportSettings, transportCustomization) - { - protected override HttpClient CreateHttpClient(NetworkCredential defaultCredential, string apiUrl) => customHttpClient; - } - - sealed class FakeHttpHandler : HttpClientHandler - { - public Func SendCallback { get; set; } - - protected override HttpResponseMessage Send(HttpRequestMessage request, CancellationToken cancellationToken) => SendCallback(request); - - protected override Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) => Task.FromResult(SendCallback(request)); - } -} \ No newline at end of file From cea5d620822b7e80e2cfa56eed5ad264a2e2b1e0 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Tue, 25 Feb 2025 21:35:28 -0800 Subject: [PATCH 24/60] Comment out Vhost property for now --- .../RabbitMQBrokerQueueDetails.cs | 4 ++-- .../RabbitMQQuery.cs | 17 ++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs index 686778637d..5f3f921e3f 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs @@ -9,8 +9,8 @@ class RabbitMQBrokerQueueDetails(Queue queue) : IBrokerQueue { public string QueueName { get; } = queue.Name; public string SanitizedName => QueueName; - public string Scope => VHost; - public string VHost { get; } = queue.Vhost; + public string? Scope => null; + //public string VHost { get; } = queue.Vhost; public List EndpointIndicators { get; } = []; long? AckedMessages { get; set; } = queue.MessageStats?.Ack; long Baseline { get; set; } = queue.MessageStats?.Ack ?? 0; diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 7a134d2a57..7e781b5645 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -10,7 +10,6 @@ namespace ServiceControl.Transports.RabbitMQ; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; -using System.Web; using Microsoft.Extensions.Logging; using NServiceBus; using Polly; @@ -72,9 +71,9 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro [EnumeratorCancellation] CancellationToken cancellationToken = default) { var queue = (RabbitMQBrokerQueueDetails)brokerQueue; - var url = $"/api/queues/{HttpUtility.UrlEncode(queue.VHost)}/{HttpUtility.UrlEncode(queue.QueueName)}"; + //var url = $"/api/queues/{HttpUtility.UrlEncode(queue.VHost)}/{HttpUtility.UrlEncode(queue.QueueName)}"; - logger.LogDebug($"Querying {url}"); + //logger.LogDebug($"Querying {url}"); var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, cancellationToken), cancellationToken); @@ -91,7 +90,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro for (var i = 0; i < 24 * 4; i++) { await Task.Delay(TimeSpan.FromMinutes(15), timeProvider, cancellationToken); - logger.LogDebug($"Querying {url}"); + //logger.LogDebug($"Querying {url}"); response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, cancellationToken), cancellationToken); if (response.Value is not null) @@ -144,7 +143,7 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa { var page = 1; bool morePages; - var vHosts = new HashSet(StringComparer.CurrentCultureIgnoreCase); + //var vHosts = new HashSet(StringComparer.CurrentCultureIgnoreCase); await GetRabbitDetails(false, cancellationToken); @@ -162,7 +161,7 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa { continue; } - vHosts.Add(rabbitMQQueueDetails.VHost); + //vHosts.Add(rabbitMQQueueDetails.VHost); await AddAdditionalQueueDetails(rabbitMQQueueDetails, cancellationToken); yield return rabbitMQQueueDetails; } @@ -171,7 +170,7 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa page++; } while (morePages); - ScopeType = vHosts.Count > 1 ? "VirtualHost" : null; + //ScopeType = vHosts.Count > 1 ? "VirtualHost" : null; } async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, CancellationToken cancellationToken) @@ -182,7 +181,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can // Check if conventional binding is found if (response.Value.Any(binding => binding?.Source == brokerQueue.QueueName - && binding?.Vhost == brokerQueue.VHost + //&& binding?.Vhost == brokerQueue.VHost && binding?.Destination == brokerQueue.QueueName && binding?.DestinationType == "queue" && binding?.RoutingKey == string.Empty @@ -202,7 +201,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can // Check if delayed binding is found if (response.Value.Any(binding => binding?.Source is "nsb.v2.delay-delivery" or "nsb.delay-delivery" - && binding?.Vhost == brokerQueue.VHost + //&& binding?.Vhost == brokerQueue.VHost && binding?.Destination == brokerQueue.QueueName && binding?.DestinationType == "exchange" && binding?.RoutingKey == $"#.{brokerQueue.QueueName}")) From 1e0a1c74dd381518d1de509d14d97e3f7e60f749 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 11:39:15 -0500 Subject: [PATCH 25/60] Update transport package --- src/Directory.Packages.props | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 6698b7a8f3..b81bb38833 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -38,8 +38,7 @@ - - + From 6289b46ffd8ed4ef4eb89822d4a430cec92000af Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 11:39:41 -0500 Subject: [PATCH 26/60] Get code compiling --- src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 7e781b5645..4dc1d4dbee 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -12,14 +12,13 @@ namespace ServiceControl.Transports.RabbitMQ; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using NServiceBus; +using NServiceBus.Transport.RabbitMQ.ManagementApi; using Polly; using Polly.Retry; using ServiceControl.Transports.BrokerThroughput; -using NServiceBus.Transport.RabbitMQ.ManagementApi; public class RabbitMQQuery : BrokerThroughputQuery { - HttpClient? httpClient; readonly ResiliencePipeline pipeline = new ResiliencePipelineBuilder() .AddRetry(new RetryStrategyOptions()) // Add retry using the default options .AddTimeout(TimeSpan.FromMinutes(2)) // Add timeout if it keeps failing @@ -93,7 +92,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro //logger.LogDebug($"Querying {url}"); response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, cancellationToken), cancellationToken); - if (response.Value is not null) + if (response.Value is null) { throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); } @@ -251,7 +250,7 @@ static List MaterializeQueueDetails(List item } catch (HttpRequestException e) { - throw new Exception($"Failed to connect to '{httpClient!.BaseAddress}'", e); + throw new Exception($"Failed to connect to management API", e); } return (true, []); From bce209de520962ea518437ef3ead521dcc0af849 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 11:44:03 -0500 Subject: [PATCH 27/60] Remove InternalsVisibleTo --- .../ServiceControl.Transports.RabbitMQ.csproj | 4 ---- ...ontrol.Transports.RabbitMQQuorumDirectRouting.Tests.csproj | 2 -- 2 files changed, 6 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj b/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj index f8c635dca9..3450935934 100644 --- a/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj +++ b/src/ServiceControl.Transports.RabbitMQ/ServiceControl.Transports.RabbitMQ.csproj @@ -25,8 +25,4 @@ - - - - \ No newline at end of file diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj index 99947dda0a..6cb4aacd7d 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj +++ b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj @@ -2,8 +2,6 @@ net8.0 - true - ..\NServiceBus.snk From 95777ecb8b4b62c1bdb2bbcf1954d9cfcf27d8a3 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 11:49:46 -0500 Subject: [PATCH 28/60] Fix formatting error --- .../RabbitMQConventionalRoutingTransportCustomization.cs | 3 ++- .../RabbitMQDirectRoutingTransportCustomization.cs | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index 188b4ca722..c3010be912 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -71,7 +71,8 @@ RabbitMQTransport IRabbitMQTransportExtensions.GetTransport() if (rabbitMQTransport == null) { throw new InvalidOperationException("Transport instance has not been created yet. Make sure CreateTransport() is called before accessing the transport."); - }; + } + return rabbitMQTransport; } } diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 066816a5f5..82eb3b4f6b 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -69,7 +69,8 @@ RabbitMQTransport IRabbitMQTransportExtensions.GetTransport() if (rabbitMQTransport == null) { throw new InvalidOperationException("Transport instance has not been created yet. Make sure CreateTransport() is called before accessing the transport."); - }; + } + return rabbitMQTransport; } } From 07c9c37306697de5794654635853842bcac7c3a5 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 12:56:17 -0500 Subject: [PATCH 29/60] First pass cleanup --- .../RabbitMQQuery.cs | 72 ++++++++----------- 1 file changed, 31 insertions(+), 41 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 4dc1d4dbee..baf9961a72 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -19,18 +19,15 @@ namespace ServiceControl.Transports.RabbitMQ; public class RabbitMQQuery : BrokerThroughputQuery { - readonly ResiliencePipeline pipeline = new ResiliencePipelineBuilder() - .AddRetry(new RetryStrategyOptions()) // Add retry using the default options - .AddTimeout(TimeSpan.FromMinutes(2)) // Add timeout if it keeps failing - .Build(); readonly ILogger logger; readonly TimeProvider timeProvider; readonly RabbitMQTransport rabbitMQTransport; + readonly ResiliencePipeline pipeline = new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions()) // Add retry using the default options + .AddTimeout(TimeSpan.FromMinutes(2)) // Add timeout if it keeps failing + .Build(); - public RabbitMQQuery(ILogger logger, - TimeProvider timeProvider, - TransportSettings transportSettings, - ITransportCustomization transportCustomization) : base(logger, "RabbitMQ") + public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, TransportSettings transportSettings, ITransportCustomization transportCustomization) : base(logger, "RabbitMQ") { this.logger = logger; this.timeProvider = timeProvider; @@ -65,16 +62,10 @@ void CheckLegacySettings(ReadOnlyDictionary settings, string key } } - public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, - DateOnly startDate, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var queue = (RabbitMQBrokerQueueDetails)brokerQueue; - //var url = $"/api/queues/{HttpUtility.UrlEncode(queue.VHost)}/{HttpUtility.UrlEncode(queue.QueueName)}"; - - //logger.LogDebug($"Querying {url}"); - - var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, cancellationToken), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, token), cancellationToken); if (response.Value is null) { @@ -89,8 +80,8 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro for (var i = 0; i < 24 * 4; i++) { await Task.Delay(TimeSpan.FromMinutes(15), timeProvider, cancellationToken); - //logger.LogDebug($"Querying {url}"); - response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, cancellationToken), cancellationToken); + + response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, token), cancellationToken); if (response.Value is null) { @@ -99,6 +90,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro newReading = new RabbitMQBrokerQueueDetails(response.Value); var newTotalThroughput = queue.CalculateThroughputFrom(newReading); + yield return new QueueThroughput { DateUTC = DateOnly.FromDateTime(timeProvider.GetUtcNow().DateTime), @@ -107,12 +99,9 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro } } - async Task GetRabbitDetails(bool skipResiliencePipeline, CancellationToken cancellationToken) + async Task GetRabbitDetails(CancellationToken cancellationToken) { - - var response = skipResiliencePipeline - ? await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken) - : await pipeline.ExecuteAsync(async async => await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken), cancellationToken); + var response = await pipeline.ExecuteAsync(async async => await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken), cancellationToken); ValidateResponse(response); @@ -142,15 +131,14 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa { var page = 1; bool morePages; - //var vHosts = new HashSet(StringComparer.CurrentCultureIgnoreCase); - await GetRabbitDetails(false, cancellationToken); + await GetRabbitDetails(cancellationToken); do { (var queues, morePages) = await GetPage(page, cancellationToken); - if (queues != null) + if (queues is not null) { foreach (var rabbitMQQueueDetails in queues) { @@ -160,7 +148,7 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa { continue; } - //vHosts.Add(rabbitMQQueueDetails.VHost); + await AddAdditionalQueueDetails(rabbitMQQueueDetails, cancellationToken); yield return rabbitMQQueueDetails; } @@ -168,19 +156,16 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa page++; } while (morePages); - - //ScopeType = vHosts.Count > 1 ? "VirtualHost" : null; } async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, CancellationToken cancellationToken) { try { - var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetBindingsForQueue(brokerQueue.QueueName, cancellationToken), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetBindingsForQueue(brokerQueue.QueueName, token), cancellationToken); // Check if conventional binding is found if (response.Value.Any(binding => binding?.Source == brokerQueue.QueueName - //&& binding?.Vhost == brokerQueue.VHost && binding?.Destination == brokerQueue.QueueName && binding?.DestinationType == "queue" && binding?.RoutingKey == string.Empty @@ -196,11 +181,10 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can try { - var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetBindingsForExchange(brokerQueue.QueueName, cancellationToken), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetBindingsForExchange(brokerQueue.QueueName, token), cancellationToken); // Check if delayed binding is found if (response.Value.Any(binding => binding?.Source is "nsb.v2.delay-delivery" or "nsb.delay-delivery" - //&& binding?.Vhost == brokerQueue.VHost && binding?.Destination == brokerQueue.QueueName && binding?.DestinationType == "exchange" && binding?.RoutingKey == $"#.{brokerQueue.QueueName}")) @@ -216,7 +200,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can internal async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) { - var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueues(page, 500, cancellationToken), cancellationToken); + var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueues(page, 500, token), cancellationToken); ValidateResponse((StatusCode, Reason, Value)); @@ -241,19 +225,25 @@ static List MaterializeQueueDetails(List item new KeyDescriptionPair(RabbitMQSettings.Password, RabbitMQSettings.PasswordDescription) ]; - protected override async Task<(bool Success, List Errors)> TestConnectionCore( - CancellationToken cancellationToken) + protected override async Task<(bool Success, List Errors)> TestConnectionCore(CancellationToken cancellationToken) { try { - await GetRabbitDetails(true, cancellationToken); + var (statusCode, reason, value) = await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken); + + if (value is not null) + { + return (true, []); + } + else + { + return (false, [$"{statusCode}: {reason}"]); + } } - catch (HttpRequestException e) + catch (HttpRequestException ex) { - throw new Exception($"Failed to connect to management API", e); + throw new Exception($"Failed to connect to RabbitMQ management API", ex); } - - return (true, []); } public static class RabbitMQSettings From 15e69dedef92efdb004ea3e6e29d8bfbed71c7bc Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 15:45:01 -0500 Subject: [PATCH 30/60] Update to 10.0.0-beta.1 --- src/Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index b81bb38833..e3789f9fad 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -38,7 +38,7 @@ - + From 93d0e449c836394d7b5a4c5128ad593b04031ed7 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 16:28:47 -0500 Subject: [PATCH 31/60] Improve how custom connection string settings are set --- ...nventionalRoutingTransportCustomization.cs | 32 ++------------- ...itMQDirectRoutingTransportCustomization.cs | 29 ++------------ .../RabbitMQTransportExtensions.cs | 40 +++++++++++++++++++ 3 files changed, 46 insertions(+), 55 deletions(-) create mode 100644 src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index c3010be912..332c0ccd9c 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -1,17 +1,13 @@ namespace ServiceControl.Transports.RabbitMQ { using System; - using System.Collections.Generic; using System.Linq; - using System.Text; using BrokerThroughput; using Microsoft.Extensions.DependencyInjection; using NServiceBus; - public abstract class RabbitMQConventionalRoutingTransportCustomization(QueueType queueType) - : TransportCustomization, IRabbitMQTransportExtensions + public abstract class RabbitMQConventionalRoutingTransportCustomization(QueueType queueType) : TransportCustomization, IRabbitMQTransportExtensions { - RabbitMQTransport rabbitMQTransport; protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } @@ -22,33 +18,14 @@ protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfigur protected override RabbitMQTransport CreateTransport(TransportSettings transportSettings, TransportTransactionMode preferredTransactionMode = TransportTransactionMode.ReceiveOnly) { - if (transportSettings.ConnectionString == null) + if (transportSettings.ConnectionString is null) { throw new InvalidOperationException("Connection string not configured"); } - var connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); - var connectionStringDictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); - - var ValidateDeliveryLimitsString = GetValue(connectionStringDictionary, "ValidateDeliveryLimits", "false"); - if (!bool.TryParse(ValidateDeliveryLimitsString, out var validateDeliveryLimits)) - { - throw new ArgumentException("The value for 'ValidateDeliveryLimits' must be either 'true' or 'false'"); - } - var transport = new RabbitMQTransport(RoutingTopology.Conventional(queueType), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; - transport.ValidateDeliveryLimits = validateDeliveryLimits; - - var url = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); - - if (!string.IsNullOrEmpty(url)) - { - var username = GetValue(connectionStringDictionary, "ManagementApiUserName", connectionConfiguration.UserName); - var password = GetValue(connectionStringDictionary, "ManagementApiPassword", connectionConfiguration.Password); - transport.ManagementApiConfiguration = !string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password) - ? new ManagementApiConfiguration(url, username, password) : new ManagementApiConfiguration(url); - } + transport.SetCustomSettingsFromConnectionString(transportSettings.ConnectionString); rabbitMQTransport = transport; return transport; @@ -63,9 +40,6 @@ protected sealed override void AddTransportForMonitoringCore(IServiceCollection services.AddHostedService(provider => provider.GetRequiredService()); } - static string GetValue(Dictionary dictionary, string key, string defaultValue) - => dictionary.TryGetValue(key, out var value) ? value : defaultValue; - RabbitMQTransport IRabbitMQTransportExtensions.GetTransport() { if (rabbitMQTransport == null) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 82eb3b4f6b..57c608cd6d 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -1,9 +1,7 @@ namespace ServiceControl.Transports.RabbitMQ { using System; - using System.Collections.Generic; using System.Linq; - using System.Text; using BrokerThroughput; using Microsoft.Extensions.DependencyInjection; using NServiceBus; @@ -18,35 +16,17 @@ protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfigurati protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } + protected override RabbitMQTransport CreateTransport(TransportSettings transportSettings, TransportTransactionMode preferredTransactionMode = TransportTransactionMode.ReceiveOnly) { - if (transportSettings.ConnectionString == null) + if (transportSettings.ConnectionString is null) { throw new InvalidOperationException("Connection string not configured"); } - var connectionConfiguration = ConnectionConfiguration.Create(transportSettings.ConnectionString, string.Empty); - var connectionStringDictionary = ConnectionConfiguration.ParseNServiceBusConnectionString(transportSettings.ConnectionString, new StringBuilder()); - - var ValidateDeliveryLimitsString = GetValue(connectionStringDictionary, "ValidateDeliveryLimits", "true"); - if (!bool.TryParse(ValidateDeliveryLimitsString, out var validateDeliveryLimits)) - { - throw new ArgumentException("The value for 'ValidateDeliveryLimits' must be either 'true' or 'false'"); - } - var transport = new RabbitMQTransport(RoutingTopology.Direct(queueType, routingKeyConvention: type => type.FullName.Replace(".", "-")), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; - transport.ValidateDeliveryLimits = validateDeliveryLimits; - - var url = GetValue(connectionStringDictionary, "ManagementApiUrl", string.Empty); - - if (!string.IsNullOrEmpty(url)) - { - var username = GetValue(connectionStringDictionary, "ManagementApiUserName", connectionConfiguration.UserName); - var password = GetValue(connectionStringDictionary, "ManagementApiPassword", connectionConfiguration.Password); - transport.ManagementApiConfiguration = !string.IsNullOrEmpty(username) && !string.IsNullOrEmpty(password) - ? new ManagementApiConfiguration(url, username, password) : new ManagementApiConfiguration(url); - } + transport.SetCustomSettingsFromConnectionString(transportSettings.ConnectionString); rabbitMQTransport = transport; return transport; @@ -61,9 +41,6 @@ protected sealed override void AddTransportForMonitoringCore(IServiceCollection services.AddHostedService(provider => provider.GetRequiredService()); } - static string GetValue(Dictionary dictionary, string key, string defaultValue) - => dictionary.TryGetValue(key, out var value) ? value : defaultValue; - RabbitMQTransport IRabbitMQTransportExtensions.GetTransport() { if (rabbitMQTransport == null) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs new file mode 100644 index 0000000000..77189ddcc0 --- /dev/null +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs @@ -0,0 +1,40 @@ +namespace ServiceControl.Transports.RabbitMQ; + +using System; +using System.Collections.Generic; +using System.Data.Common; +using System.Linq; +using NServiceBus; + +static class RabbitMQTransportExtensions +{ + public static void SetCustomSettingsFromConnectionString(this RabbitMQTransport transport, string connectionString) + { + if (connectionString.StartsWith("amqp", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var dictionary = new DbConnectionStringBuilder { ConnectionString = connectionString } + .OfType>() + .ToDictionary(pair => pair.Key, pair => pair.Value.ToString(), StringComparer.OrdinalIgnoreCase); + + if (dictionary.TryGetValue("ValidateDeliveryLimits", out var validateString)) + { + _ = bool.TryParse(validateString, out var validate); + transport.ValidateDeliveryLimits = validate; + } + + if (dictionary.TryGetValue("ManagementApiUrl", out var url)) + { + if (dictionary.TryGetValue("ManagementApiUserName", out var userName) && dictionary.TryGetValue("ManagementApiPassword", out var password)) + { + transport.ManagementApiConfiguration = new(url, userName, password); + } + else + { + transport.ManagementApiConfiguration = new(url); + } + } + } +} From 39b2807c05226070640b5e8050adbbd84a14519b Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 17:42:33 -0500 Subject: [PATCH 32/60] Change how ManagementClient gets passed in --- .../IManagementClientProvider.cs | 8 ++++ .../IRabbitMQTransportExtentions.cs | 9 ----- ...nventionalRoutingTransportCustomization.cs | 18 +++------ ...itMQDirectRoutingTransportCustomization.cs | 19 +++------- .../RabbitMQQuery.cs | 38 +++++++++---------- 5 files changed, 36 insertions(+), 56 deletions(-) create mode 100644 src/ServiceControl.Transports.RabbitMQ/IManagementClientProvider.cs delete mode 100644 src/ServiceControl.Transports.RabbitMQ/IRabbitMQTransportExtentions.cs diff --git a/src/ServiceControl.Transports.RabbitMQ/IManagementClientProvider.cs b/src/ServiceControl.Transports.RabbitMQ/IManagementClientProvider.cs new file mode 100644 index 0000000000..70f069b955 --- /dev/null +++ b/src/ServiceControl.Transports.RabbitMQ/IManagementClientProvider.cs @@ -0,0 +1,8 @@ +namespace ServiceControl.Transports.RabbitMQ; + +using NServiceBus.Transport.RabbitMQ.ManagementApi; + +interface IManagementClientProvider +{ + ManagementClient ManagementClient { get; } +} diff --git a/src/ServiceControl.Transports.RabbitMQ/IRabbitMQTransportExtentions.cs b/src/ServiceControl.Transports.RabbitMQ/IRabbitMQTransportExtentions.cs deleted file mode 100644 index 8eb09e50bc..0000000000 --- a/src/ServiceControl.Transports.RabbitMQ/IRabbitMQTransportExtentions.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace ServiceControl.Transports.RabbitMQ -{ - using NServiceBus; - - public interface IRabbitMQTransportExtensions - { - RabbitMQTransport GetTransport(); - } -} diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index 332c0ccd9c..a7d8df4898 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -5,12 +5,15 @@ using BrokerThroughput; using Microsoft.Extensions.DependencyInjection; using NServiceBus; + using NServiceBus.Transport.RabbitMQ.ManagementApi; - public abstract class RabbitMQConventionalRoutingTransportCustomization(QueueType queueType) : TransportCustomization, IRabbitMQTransportExtensions + public abstract class RabbitMQConventionalRoutingTransportCustomization(NServiceBus.QueueType queueType) : TransportCustomization, IManagementClientProvider { RabbitMQTransport rabbitMQTransport; - protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } + ManagementClient IManagementClientProvider.ManagementClient => rabbitMQTransport?.ManagementClient ?? throw new InvalidOperationException("Transport instance has not been created yet."); + + protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } @@ -27,7 +30,6 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; transport.SetCustomSettingsFromConnectionString(transportSettings.ConnectionString); - rabbitMQTransport = transport; return transport; } @@ -39,15 +41,5 @@ protected sealed override void AddTransportForMonitoringCore(IServiceCollection services.AddSingleton(); services.AddHostedService(provider => provider.GetRequiredService()); } - - RabbitMQTransport IRabbitMQTransportExtensions.GetTransport() - { - if (rabbitMQTransport == null) - { - throw new InvalidOperationException("Transport instance has not been created yet. Make sure CreateTransport() is called before accessing the transport."); - } - - return rabbitMQTransport; - } } } \ No newline at end of file diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 57c608cd6d..049e34b226 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -5,13 +5,15 @@ using BrokerThroughput; using Microsoft.Extensions.DependencyInjection; using NServiceBus; + using NServiceBus.Transport.RabbitMQ.ManagementApi; - public abstract class RabbitMQDirectRoutingTransportCustomization(QueueType queueType) - : TransportCustomization, IRabbitMQTransportExtensions + public abstract class RabbitMQDirectRoutingTransportCustomization(NServiceBus.QueueType queueType) : TransportCustomization, IManagementClientProvider { RabbitMQTransport rabbitMQTransport; - protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } + ManagementClient IManagementClientProvider.ManagementClient => rabbitMQTransport?.ManagementClient ?? throw new InvalidOperationException("Transport instance has not been created yet."); + + protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } @@ -28,7 +30,6 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; transport.SetCustomSettingsFromConnectionString(transportSettings.ConnectionString); - rabbitMQTransport = transport; return transport; } @@ -40,15 +41,5 @@ protected sealed override void AddTransportForMonitoringCore(IServiceCollection services.AddSingleton(); services.AddHostedService(provider => provider.GetRequiredService()); } - - RabbitMQTransport IRabbitMQTransportExtensions.GetTransport() - { - if (rabbitMQTransport == null) - { - throw new InvalidOperationException("Transport instance has not been created yet. Make sure CreateTransport() is called before accessing the transport."); - } - - return rabbitMQTransport; - } } } \ No newline at end of file diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index baf9961a72..737750b110 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -11,7 +11,6 @@ namespace ServiceControl.Transports.RabbitMQ; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; -using NServiceBus; using NServiceBus.Transport.RabbitMQ.ManagementApi; using Polly; using Polly.Retry; @@ -21,7 +20,8 @@ public class RabbitMQQuery : BrokerThroughputQuery { readonly ILogger logger; readonly TimeProvider timeProvider; - readonly RabbitMQTransport rabbitMQTransport; + readonly ManagementClient managementClient; + readonly ResiliencePipeline pipeline = new ResiliencePipelineBuilder() .AddRetry(new RetryStrategyOptions()) // Add retry using the default options .AddTimeout(TimeSpan.FromMinutes(2)) // Add timeout if it keeps failing @@ -31,7 +31,15 @@ public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, T { this.logger = logger; this.timeProvider = timeProvider; - rabbitMQTransport = GetRabbitMQTransport(transportCustomization); + + if (transportCustomization is IManagementClientProvider provider) + { + managementClient = provider.ManagementClient; + } + else + { + throw new ArgumentException($"Transport customization does not implement {nameof(IManagementClientProvider)}. Type: {transportCustomization.GetType().Name}", nameof(transportCustomization)); + } } protected override void InitializeCore(ReadOnlyDictionary settings) @@ -43,16 +51,6 @@ protected override void InitializeCore(ReadOnlyDictionary settin CheckLegacySettings(settings, RabbitMQSettings.API); } - static RabbitMQTransport GetRabbitMQTransport(ITransportCustomization transportCustomization) - { - if (transportCustomization is IRabbitMQTransportExtensions rabbitMQTransportCustomization) - { - return rabbitMQTransportCustomization.GetTransport(); - } - - throw new InvalidOperationException($"Expected a RabbitMQTransport but received {transportCustomization.GetType().Name}."); - } - void CheckLegacySettings(ReadOnlyDictionary settings, string key) { if (settings.TryGetValue(key, out _)) @@ -65,7 +63,7 @@ void CheckLegacySettings(ReadOnlyDictionary settings, string key public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var queue = (RabbitMQBrokerQueueDetails)brokerQueue; - var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, token), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await managementClient.GetQueue(queue.QueueName, token), cancellationToken); if (response.Value is null) { @@ -81,7 +79,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro { await Task.Delay(TimeSpan.FromMinutes(15), timeProvider, cancellationToken); - response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueue(queue.QueueName, token), cancellationToken); + response = await pipeline.ExecuteAsync(async token => await managementClient.GetQueue(queue.QueueName, token), cancellationToken); if (response.Value is null) { @@ -101,7 +99,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro async Task GetRabbitDetails(CancellationToken cancellationToken) { - var response = await pipeline.ExecuteAsync(async async => await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken), cancellationToken); + var response = await pipeline.ExecuteAsync(async async => await managementClient.GetOverview(cancellationToken), cancellationToken); ValidateResponse(response); @@ -162,7 +160,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can { try { - var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetBindingsForQueue(brokerQueue.QueueName, token), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await managementClient.GetBindingsForQueue(brokerQueue.QueueName, token), cancellationToken); // Check if conventional binding is found if (response.Value.Any(binding => binding?.Source == brokerQueue.QueueName @@ -181,7 +179,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can try { - var response = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetBindingsForExchange(brokerQueue.QueueName, token), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await managementClient.GetBindingsForExchange(brokerQueue.QueueName, token), cancellationToken); // Check if delayed binding is found if (response.Value.Any(binding => binding?.Source is "nsb.v2.delay-delivery" or "nsb.delay-delivery" @@ -200,7 +198,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can internal async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) { - var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await rabbitMQTransport.ManagementClient.GetQueues(page, 500, token), cancellationToken); + var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await managementClient.GetQueues(page, 500, token), cancellationToken); ValidateResponse((StatusCode, Reason, Value)); @@ -229,7 +227,7 @@ static List MaterializeQueueDetails(List item { try { - var (statusCode, reason, value) = await rabbitMQTransport.ManagementClient.GetOverview(cancellationToken); + var (statusCode, reason, value) = await managementClient.GetOverview(cancellationToken); if (value is not null) { From 82903247a0af7294cbe1ca7f9ab39e126f361195 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 18:18:11 -0500 Subject: [PATCH 33/60] Wire up settings that apparently have been lost --- .../QueueLengthProvider.cs | 8 ++-- .../RabbitMQTransportExtensions.cs | 18 ++++++-- .../TransportConfigurationExtensions.cs | 44 ------------------- 3 files changed, 19 insertions(+), 51 deletions(-) delete mode 100644 src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs diff --git a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs index abdf41c37e..0651073e23 100644 --- a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs +++ b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs @@ -2,7 +2,6 @@ { using System; using System.Collections.Concurrent; - using System.Data.Common; using System.Threading; using System.Threading.Tasks; using global::RabbitMQ.Client; @@ -109,13 +108,14 @@ public void Initialize() var connectionConfiguration = ConnectionConfiguration.Create(connectionString, "ServiceControl.Monitoring"); - var dbConnectionStringBuilder = new DbConnectionStringBuilder { ConnectionString = connectionString }; + // TODO Fix this up + //var dbConnectionStringBuilder = new DbConnectionStringBuilder { ConnectionString = connectionString }; connectionFactory = new ConnectionFactory("ServiceControl.Monitoring", connectionConfiguration, null, //providing certificates is not supported yet - dbConnectionStringBuilder.GetBooleanValue("DisableRemoteCertificateValidation"), - dbConnectionStringBuilder.GetBooleanValue("UseExternalAuthMechanism"), + false,//dbConnectionStringBuilder.GetBooleanValue("DisableRemoteCertificateValidation"), + false, //dbConnectionStringBuilder.GetBooleanValue("UseExternalAuthMechanism"), null, // value would come from config API in actual transport null); // value would come from config API in actual transport } diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs index 77189ddcc0..32ae263731 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs @@ -19,10 +19,10 @@ public static void SetCustomSettingsFromConnectionString(this RabbitMQTransport .OfType>() .ToDictionary(pair => pair.Key, pair => pair.Value.ToString(), StringComparer.OrdinalIgnoreCase); - if (dictionary.TryGetValue("ValidateDeliveryLimits", out var validateString)) + if (dictionary.TryGetValue("ValidateDeliveryLimits", out var validateDeliveryLimitsString)) { - _ = bool.TryParse(validateString, out var validate); - transport.ValidateDeliveryLimits = validate; + _ = bool.TryParse(validateDeliveryLimitsString, out var validateDeliveryLimits); + transport.ValidateDeliveryLimits = validateDeliveryLimits; } if (dictionary.TryGetValue("ManagementApiUrl", out var url)) @@ -36,5 +36,17 @@ public static void SetCustomSettingsFromConnectionString(this RabbitMQTransport transport.ManagementApiConfiguration = new(url); } } + + if (dictionary.TryGetValue("DisableRemoteCertificateValidation", out var disableRemoteCertificateValidationString)) + { + _ = bool.TryParse(disableRemoteCertificateValidationString, out var disableRemoteCertificateValidation); + transport.ValidateRemoteCertificate = !disableRemoteCertificateValidation; + } + + if (dictionary.TryGetValue("UseExternalAuthMechanism", out var useExternalAuthMechanismString)) + { + _ = bool.TryParse(useExternalAuthMechanismString, out var useExternalAuthMechanism); + transport.UseExternalAuthMechanism = useExternalAuthMechanism; + } } } diff --git a/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs b/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs deleted file mode 100644 index ee10f15afb..0000000000 --- a/src/ServiceControl.Transports.RabbitMQ/TransportConfigurationExtensions.cs +++ /dev/null @@ -1,44 +0,0 @@ -namespace ServiceControl.Transports.RabbitMQ -{ - using NServiceBus; - using System; - using System.Data.Common; - - static class TransportConfigurationExtensions - { - public static void ApplyConnectionString(this TransportExtensions transport, string connectionString) - { - if (!connectionString.StartsWith("amqp", StringComparison.InvariantCultureIgnoreCase)) - { - var builder = new DbConnectionStringBuilder { ConnectionString = connectionString }; - - if (builder.GetBooleanValue("DisableRemoteCertificateValidation")) - { - transport.DisableRemoteCertificateValidation(); - } - - if (builder.GetBooleanValue("UseExternalAuthMechanism")) - { - transport.UseExternalAuthMechanism(); - } - } - - transport.ConnectionString(connectionString); - } - - public static bool GetBooleanValue(this DbConnectionStringBuilder dbConnectionStringBuilder, string key) - { - if (!dbConnectionStringBuilder.TryGetValue(key, out var rawValue)) - { - return false; - } - - if (!bool.TryParse(rawValue.ToString(), out var value)) - { - throw new Exception($"Can't parse key '{key}'. '{rawValue}' is not a valid boolean value."); - } - - return value; - } - } -} \ No newline at end of file From a6c4fe27cd482aea0d6a66c075bbac034fc41fdf Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 18:59:27 -0500 Subject: [PATCH 34/60] Use classes from transport --- .../ConnectionConfiguration.cs | 286 ------------------ .../ConnectionFactory.cs | 103 ------- .../QueueLengthProvider.cs | 14 +- 3 files changed, 8 insertions(+), 395 deletions(-) delete mode 100644 src/ServiceControl.Transports.RabbitMQ/ConnectionConfiguration.cs delete mode 100644 src/ServiceControl.Transports.RabbitMQ/ConnectionFactory.cs diff --git a/src/ServiceControl.Transports.RabbitMQ/ConnectionConfiguration.cs b/src/ServiceControl.Transports.RabbitMQ/ConnectionConfiguration.cs deleted file mode 100644 index e6ea32a5c7..0000000000 --- a/src/ServiceControl.Transports.RabbitMQ/ConnectionConfiguration.cs +++ /dev/null @@ -1,286 +0,0 @@ -namespace ServiceControl.Transports.RabbitMQ -{ - using System; - using System.Collections.Generic; - using System.Data.Common; - using System.Diagnostics; - using System.IO; - using System.Linq; - using System.Text; - using NServiceBus; - using NServiceBus.Support; - - class ConnectionConfiguration - { - const bool defaultUseTls = false; - const int defaultPort = 5672; - const int defaultTlsPort = 5671; - const string defaultVirtualHost = "/"; - const string defaultUserName = "guest"; - const string defaultPassword = "guest"; - const ushort defaultRequestedHeartbeat = 60; - static readonly TimeSpan defaultRetryDelay = TimeSpan.FromSeconds(10); - const string defaultCertPath = ""; - const string defaultCertPassphrase = null; - - public string Host { get; } - - public int Port { get; } - - public string VirtualHost { get; } - - public string UserName { get; } - - public string Password { get; } - - public TimeSpan RequestedHeartbeat { get; } - - public TimeSpan RetryDelay { get; } - - public bool UseTls { get; } - - public string CertPath { get; } - - public string CertPassphrase { get; } - - public Dictionary ClientProperties { get; } - - ConnectionConfiguration( - string host, - int port, - string virtualHost, - string userName, - string password, - TimeSpan requestedHeartbeat, - TimeSpan retryDelay, - bool useTls, - string certPath, - string certPassphrase, - Dictionary clientProperties) - { - Host = host; - Port = port; - VirtualHost = virtualHost; - UserName = userName; - Password = password; - RequestedHeartbeat = requestedHeartbeat; - RetryDelay = retryDelay; - UseTls = useTls; - CertPath = certPath; - CertPassphrase = certPassphrase; - ClientProperties = clientProperties; - } - - public static ConnectionConfiguration Create(string connectionString, string endpointName) - { - Dictionary dictionary; - var invalidOptionsMessage = new StringBuilder(); - - if (connectionString.StartsWith("amqp", StringComparison.OrdinalIgnoreCase)) - { - dictionary = ParseAmqpConnectionString(connectionString, invalidOptionsMessage); - } - else - { - dictionary = ParseNServiceBusConnectionString(connectionString, invalidOptionsMessage); - } - - var host = GetValue(dictionary, "host", default); - var useTls = GetValue(dictionary, "useTls", bool.TryParse, defaultUseTls, invalidOptionsMessage); - var port = GetValue(dictionary, "port", int.TryParse, useTls ? defaultTlsPort : defaultPort, invalidOptionsMessage); - var virtualHost = GetValue(dictionary, "virtualHost", defaultVirtualHost); - var userName = GetValue(dictionary, "userName", defaultUserName); - var password = GetValue(dictionary, "password", defaultPassword); - - var requestedHeartbeatSeconds = GetValue(dictionary, "requestedHeartbeat", ushort.TryParse, defaultRequestedHeartbeat, invalidOptionsMessage); - var requestedHeartbeat = TimeSpan.FromSeconds(requestedHeartbeatSeconds); - - var retryDelay = GetValue(dictionary, "retryDelay", TimeSpan.TryParse, defaultRetryDelay, invalidOptionsMessage); - var certPath = GetValue(dictionary, "certPath", defaultCertPath); - var certPassPhrase = GetValue(dictionary, "certPassphrase", defaultCertPassphrase); - - if (invalidOptionsMessage.Length > 0) - { - throw new NotSupportedException(invalidOptionsMessage.ToString().TrimEnd('\r', '\n')); - } - - var nsbVersion = FileVersionInfo.GetVersionInfo(typeof(Endpoint).Assembly.Location); - var nsbFileVersion = $"{nsbVersion.FileMajorPart}.{nsbVersion.FileMinorPart}.{nsbVersion.FileBuildPart}"; - - var rabbitMQVersion = - FileVersionInfo.GetVersionInfo(typeof(ConnectionConfiguration).Assembly.Location); - var rabbitMQFileVersion = $"{rabbitMQVersion.FileMajorPart}.{rabbitMQVersion.FileMinorPart}.{rabbitMQVersion.FileBuildPart}"; - - var applicationNameAndPath = Environment.GetCommandLineArgs()[0]; - var applicationName = Path.GetFileName(applicationNameAndPath); - var applicationPath = Path.GetDirectoryName(applicationNameAndPath); - - var hostname = RuntimeEnvironment.MachineName; - - var clientProperties = new Dictionary - { - { "client_api", "NServiceBus" }, - { "nservicebus_version", nsbFileVersion }, - { "nservicebus.rabbitmq_version", rabbitMQFileVersion }, - { "application", applicationName }, - { "application_location", applicationPath }, - { "machine_name", hostname }, - { "user", userName }, - { "endpoint_name", endpointName }, - }; - - return new ConnectionConfiguration( - host, port, virtualHost, userName, password, requestedHeartbeat, retryDelay, useTls, certPath, certPassPhrase, clientProperties); - } - - static Dictionary ParseAmqpConnectionString(string connectionString, StringBuilder invalidOptionsMessage) - { - var dictionary = new Dictionary(); - var uri = new Uri(connectionString); - - var usingTls = string.Equals("amqps", uri.Scheme, StringComparison.OrdinalIgnoreCase) ? bool.TrueString : bool.FalseString; - dictionary.Add("useTls", usingTls); - - dictionary.Add("host", uri.Host); - - if (!uri.IsDefaultPort) - { - dictionary.Add("port", uri.Port.ToString()); - } - - if (!string.IsNullOrEmpty(uri.UserInfo)) - { - var userPass = uri.UserInfo.Split(':'); - - if (userPass.Length > 2) - { - invalidOptionsMessage.AppendLine($"Bad user info in AMQP URI: {uri.UserInfo}"); - } - else - { - dictionary.Add("userName", UriDecode(userPass[0])); - - if (userPass.Length == 2) - { - dictionary.Add("password", UriDecode(userPass[1])); - } - } - } - - if (uri.Segments.Length > 2) - { - invalidOptionsMessage.AppendLine($"Multiple segments in path of AMQP URI: {string.Join(", ", uri.Segments)}"); - } - else if (uri.Segments.Length == 2) - { - dictionary.Add("virtualHost", UriDecode(uri.Segments[1])); - } - - return dictionary; - } - - internal static Dictionary ParseNServiceBusConnectionString(string connectionString, StringBuilder invalidOptionsMessage) - { - var dictionary = new DbConnectionStringBuilder { ConnectionString = connectionString } - .OfType>() - .ToDictionary(pair => pair.Key, pair => pair.Value.ToString(), StringComparer.OrdinalIgnoreCase); - - RegisterDeprecatedSettingsAsInvalidOptions(dictionary, invalidOptionsMessage); - - if (dictionary.TryGetValue("port", out var portValue) && !int.TryParse(portValue, out var port)) - { - invalidOptionsMessage.AppendLine($"'{portValue}' is not a valid Int32 value for the 'port' connection string option."); - } - - if (dictionary.TryGetValue("host", out var value)) - { - var firstHostAndPort = value.Split(',')[0]; - var parts = firstHostAndPort.Split(':'); - var host = parts.ElementAt(0); - - if (host.Length == 0) - { - invalidOptionsMessage.AppendLine("Empty host name in 'host' connection string option."); - } - - dictionary["host"] = host; - - if (parts.Length > 1) - { - if (!int.TryParse(parts[1], out port)) - { - invalidOptionsMessage.AppendLine($"'{parts[1]}' is not a valid Int32 value for the port in the 'host' connection string option."); - } - else - { - dictionary["port"] = port.ToString(); - } - } - } - else - { - invalidOptionsMessage.AppendLine("Invalid connection string. 'host' value must be supplied. e.g: \"host=myServer\""); - } - - return dictionary; - } - - static void RegisterDeprecatedSettingsAsInvalidOptions(Dictionary dictionary, StringBuilder invalidOptionsMessage) - { - if (dictionary.TryGetValue("host", out var value)) - { - var hostsAndPorts = value.Split(','); - - if (hostsAndPorts.Length > 1) - { - invalidOptionsMessage.AppendLine("Multiple hosts are no longer supported. If using RabbitMQ in a cluster, consider using a load balancer to represent the nodes as a single host."); - } - } - - if (dictionary.ContainsKey("dequeuetimeout")) - { - invalidOptionsMessage.AppendLine("The 'DequeueTimeout' connection string option has been removed. Consult the documentation for further information."); - } - - if (dictionary.ContainsKey("maxwaittimeforconfirms")) - { - invalidOptionsMessage.AppendLine("The 'MaxWaitTimeForConfirms' connection string option has been removed. Consult the documentation for further information."); - } - - if (dictionary.ContainsKey("prefetchcount")) - { - invalidOptionsMessage.AppendLine("The 'PrefetchCount' connection string option has been removed. Use 'EndpointConfiguration.UseTransport().PrefetchCount' instead."); - } - - if (dictionary.ContainsKey("usepublisherconfirms")) - { - invalidOptionsMessage.AppendLine("The 'UsePublisherConfirms' connection string option has been removed. Consult the documentation for further information."); - } - } - - static string GetValue(Dictionary dictionary, string key, string defaultValue) - { - return dictionary.TryGetValue(key, out var value) ? value : defaultValue; - } - - static string UriDecode(string value) - { - return Uri.UnescapeDataString(value); - } - - static T GetValue(Dictionary dictionary, string key, Convert convert, T defaultValue, StringBuilder invalidOptionsMessage) - { - if (dictionary.TryGetValue(key, out var value)) - { - if (!convert(value, out defaultValue)) - { - invalidOptionsMessage.AppendLine($"'{value}' is not a valid {typeof(T).Name} value for the '{key}' connection string option."); - } - } - - return defaultValue; - } - - delegate bool Convert(string input, out T output); - } -} diff --git a/src/ServiceControl.Transports.RabbitMQ/ConnectionFactory.cs b/src/ServiceControl.Transports.RabbitMQ/ConnectionFactory.cs deleted file mode 100644 index fce580c92c..0000000000 --- a/src/ServiceControl.Transports.RabbitMQ/ConnectionFactory.cs +++ /dev/null @@ -1,103 +0,0 @@ -namespace ServiceControl.Transports.RabbitMQ -{ - using System; - using System.Net.Security; - using System.Security.Authentication; - using System.Security.Cryptography.X509Certificates; - using System.Threading; - using System.Threading.Tasks; - using global::RabbitMQ.Client; - - class ConnectionFactory - { - readonly string endpointName; - readonly global::RabbitMQ.Client.ConnectionFactory connectionFactory; - readonly SemaphoreSlim semaphoreSlim = new(1, 1); - - public ConnectionFactory(string endpointName, ConnectionConfiguration connectionConfiguration, - X509Certificate2Collection clientCertificateCollection, bool disableRemoteCertificateValidation, - bool useExternalAuthMechanism, TimeSpan? heartbeatInterval, TimeSpan? networkRecoveryInterval) - { - if (endpointName is null) - { - throw new ArgumentNullException(nameof(endpointName)); - } - - if (endpointName == string.Empty) - { - throw new ArgumentException("The endpoint name cannot be empty.", nameof(endpointName)); - } - - this.endpointName = endpointName; - - if (connectionConfiguration == null) - { - throw new ArgumentNullException(nameof(connectionConfiguration)); - } - - if (connectionConfiguration.Host == null) - { - throw new ArgumentException("The connectionConfiguration has a null Host.", nameof(connectionConfiguration)); - } - - connectionFactory = new global::RabbitMQ.Client.ConnectionFactory - { - HostName = connectionConfiguration.Host, - Port = connectionConfiguration.Port, - VirtualHost = connectionConfiguration.VirtualHost, - UserName = connectionConfiguration.UserName, - Password = connectionConfiguration.Password, - RequestedHeartbeat = heartbeatInterval ?? connectionConfiguration.RequestedHeartbeat, - NetworkRecoveryInterval = networkRecoveryInterval ?? connectionConfiguration.RetryDelay, - }; - - connectionFactory.Ssl.ServerName = connectionConfiguration.Host; - connectionFactory.Ssl.Certs = clientCertificateCollection; - connectionFactory.Ssl.CertPath = connectionConfiguration.CertPath; - connectionFactory.Ssl.CertPassphrase = connectionConfiguration.CertPassphrase; - connectionFactory.Ssl.Version = SslProtocols.Tls12; - connectionFactory.Ssl.Enabled = connectionConfiguration.UseTls; - - if (disableRemoteCertificateValidation) - { - connectionFactory.Ssl.AcceptablePolicyErrors = SslPolicyErrors.RemoteCertificateChainErrors | - SslPolicyErrors.RemoteCertificateNameMismatch | - SslPolicyErrors.RemoteCertificateNotAvailable; - } - - if (useExternalAuthMechanism) - { - connectionFactory.AuthMechanisms = new[] { new ExternalMechanismFactory() }; - } - - connectionFactory.ClientProperties.Clear(); - - foreach (var item in connectionConfiguration.ClientProperties) - { - connectionFactory.ClientProperties.Add(item.Key, item.Value); - } - } - - public async Task CreatePublishConnection(CancellationToken cancellationToken) => await CreateConnection($"{endpointName} Publish", false, cancellationToken); - - public Task CreateAdministrationConnection(CancellationToken cancellationToken) => CreateConnection($"{endpointName} Administration", false, cancellationToken); - - public async Task CreateConnection(string connectionName, bool automaticRecoveryEnabled = true, CancellationToken cancellationToken = default) - { - await semaphoreSlim.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - connectionFactory.AutomaticRecoveryEnabled = automaticRecoveryEnabled; - connectionFactory.ClientProperties["connected"] = DateTime.UtcNow.ToString("G"); - - var connection = await connectionFactory.CreateConnectionAsync(connectionName, cancellationToken); - - return connection; - } - finally - { - _ = semaphoreSlim.Release(); - } - } - } -} diff --git a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs index 0651073e23..5c19fc8e26 100644 --- a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs +++ b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs @@ -6,6 +6,8 @@ using System.Threading.Tasks; using global::RabbitMQ.Client; using NServiceBus.Logging; + using NServiceBus.Transport.RabbitMQ; + using ConnectionFactory = NServiceBus.Transport.RabbitMQ.ConnectionFactory; class QueueLengthProvider : AbstractQueueLengthProvider { @@ -105,18 +107,18 @@ class QueryExecutor(string connectionString) : IDisposable public void Initialize() { - var connectionConfiguration = - ConnectionConfiguration.Create(connectionString, "ServiceControl.Monitoring"); + var connectionConfiguration = ConnectionConfiguration.Create(connectionString); // TODO Fix this up //var dbConnectionStringBuilder = new DbConnectionStringBuilder { ConnectionString = connectionString }; - connectionFactory = new ConnectionFactory("ServiceControl.Monitoring", + connectionFactory = new("ServiceControl.Monitoring", connectionConfiguration, null, //providing certificates is not supported yet - false,//dbConnectionStringBuilder.GetBooleanValue("DisableRemoteCertificateValidation"), - false, //dbConnectionStringBuilder.GetBooleanValue("UseExternalAuthMechanism"), - null, // value would come from config API in actual transport + false, // TODO Fix dbConnectionStringBuilder.GetBooleanValue("DisableRemoteCertificateValidation"), + false, // TODO fix dbConnectionStringBuilder.GetBooleanValue("UseExternalAuthMechanism"), + TimeSpan.FromSeconds(60), // value would come from config API in actual transport + TimeSpan.FromSeconds(10), // value would come from config API in actual transport null); // value would come from config API in actual transport } From 5b45224b77893c3e8bf001374545647993e6cf3d Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 26 Feb 2025 19:10:02 -0500 Subject: [PATCH 35/60] Formatting --- .../RabbitMQBrokerQueueDetails.cs | 10 ++++++++-- .../BrokerThroughput/IBrokerQueue.cs | 11 +++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs index 5f3f921e3f..4a4f8d0f68 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs @@ -2,22 +2,27 @@ namespace ServiceControl.Transports.RabbitMQ; using System.Collections.Generic; -using ServiceControl.Transports.BrokerThroughput; using NServiceBus.Transport.RabbitMQ.ManagementApi; +using ServiceControl.Transports.BrokerThroughput; class RabbitMQBrokerQueueDetails(Queue queue) : IBrokerQueue { public string QueueName { get; } = queue.Name; + public string SanitizedName => QueueName; + public string? Scope => null; - //public string VHost { get; } = queue.Vhost; + public List EndpointIndicators { get; } = []; + long? AckedMessages { get; set; } = queue.MessageStats?.Ack; + long Baseline { get; set; } = queue.MessageStats?.Ack ?? 0; public long CalculateThroughputFrom(RabbitMQBrokerQueueDetails newReading) { var newlyAckedMessages = 0L; + if (newReading.AckedMessages is null) { return newlyAckedMessages; @@ -28,6 +33,7 @@ public long CalculateThroughputFrom(RabbitMQBrokerQueueDetails newReading) newlyAckedMessages = newReading.AckedMessages.Value - Baseline; AckedMessages += newlyAckedMessages; } + Baseline = newReading.AckedMessages.Value; return newlyAckedMessages; diff --git a/src/ServiceControl.Transports/BrokerThroughput/IBrokerQueue.cs b/src/ServiceControl.Transports/BrokerThroughput/IBrokerQueue.cs index 01fb7cb5b5..6cd791267e 100644 --- a/src/ServiceControl.Transports/BrokerThroughput/IBrokerQueue.cs +++ b/src/ServiceControl.Transports/BrokerThroughput/IBrokerQueue.cs @@ -7,8 +7,11 @@ namespace ServiceControl.Transports.BrokerThroughput; public interface IBrokerQueue #pragma warning restore CA1711 { - public string QueueName { get; } - public string SanitizedName { get; } - public string? Scope { get; } - public List EndpointIndicators { get; } + string QueueName { get; } + + string SanitizedName { get; } + + string? Scope { get; } + + List EndpointIndicators { get; } } \ No newline at end of file From 84b0332069bb1d4214268e28597c315a1f4a0974 Mon Sep 17 00:00:00 2001 From: Travis Nickels Date: Wed, 26 Feb 2025 19:00:28 -0800 Subject: [PATCH 36/60] Fix text for transport manifest --- src/ServiceControl.Transports.RabbitMQ/transport.manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/transport.manifest b/src/ServiceControl.Transports.RabbitMQ/transport.manifest index a7aa1ef04b..66bc54e505 100644 --- a/src/ServiceControl.Transports.RabbitMQ/transport.manifest +++ b/src/ServiceControl.Transports.RabbitMQ/transport.manifest @@ -27,7 +27,7 @@ "DisplayName": "RabbitMQ - Conventional routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumConventionalRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=;ManagementApiUrl=;ManagementApiPassword=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=;ManagementApiUrl=;ManagementApiUserName=;ManagementApiPassword=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumConventialRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" @@ -38,7 +38,7 @@ "DisplayName": "RabbitMQ - Direct routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumDirectRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=ManagementApiUrl=;ManagementApiPassword=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=ManagementApiUrl=;ManagementApiUserName=;ManagementApiPassword=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumDirectRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" From 1f712c1c6358e53e2692d238f7b1760bef8481f2 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Thu, 27 Feb 2025 11:10:46 -0500 Subject: [PATCH 37/60] Fix connection string examples --- src/ServiceControl.Transports.RabbitMQ/transport.manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/transport.manifest b/src/ServiceControl.Transports.RabbitMQ/transport.manifest index 66bc54e505..7203eb3f81 100644 --- a/src/ServiceControl.Transports.RabbitMQ/transport.manifest +++ b/src/ServiceControl.Transports.RabbitMQ/transport.manifest @@ -27,7 +27,7 @@ "DisplayName": "RabbitMQ - Conventional routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumConventionalRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=;ManagementApiUrl=;ManagementApiUserName=;ManagementApiPassword=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=;ManagementApiUrl=;ManagementApiUserName=;ManagementApiPassword=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumConventialRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" @@ -38,7 +38,7 @@ "DisplayName": "RabbitMQ - Direct routing topology (quorum queues)", "AssemblyName": "ServiceControl.Transports.RabbitMQ", "TypeName": "ServiceControl.Transports.RabbitMQ.RabbitMQQuorumDirectRoutingTransportCustomization, ServiceControl.Transports.RabbitMQ", - "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=ManagementApiUrl=;ManagementApiUserName=;ManagementApiPassword=", + "SampleConnectionString": "host=;username=;password=;DisableRemoteCertificateValidation=;UseExternalAuthMechanism=;ValidateDeliveryLimits=;ManagementApiUrl=;ManagementApiUserName=;ManagementApiPassword=", "AvailableInSCMU": true, "Aliases": [ "ServiceControl.Transports.RabbitMQ.QuorumDirectRoutingTopologyRabbitMQTransport, ServiceControl.Transports.RabbitMQ" From e3b7c8530e843d3bc73c6ade08841a4075c4c895 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Thu, 27 Feb 2025 11:17:08 -0500 Subject: [PATCH 38/60] Remove signing from TestHelper --- src/TestHelper/TestHelper.csproj | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/TestHelper/TestHelper.csproj b/src/TestHelper/TestHelper.csproj index 870af44c6a..077bb6b110 100644 --- a/src/TestHelper/TestHelper.csproj +++ b/src/TestHelper/TestHelper.csproj @@ -2,8 +2,6 @@ net8.0 - true - ..\NServiceBus.snk @@ -16,5 +14,5 @@ - + From ece2a418a73d1e415037282fe8e4467db6481d63 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Thu, 27 Feb 2025 11:21:01 -0500 Subject: [PATCH 39/60] Remove approval files for deleted tests --- ...h_queue_details_in_old_format.approved.txt | 23 ------------------- ...d_handle_duplicated_json_data.approved.txt | 16 ------------- 2 files changed, 39 deletions(-) delete mode 100644 src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQuery_ResponseParsing_Tests.Should_fetch_queue_details_in_old_format.approved.txt delete mode 100644 src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQuery_ResponseParsing_Tests.Should_handle_duplicated_json_data.approved.txt diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQuery_ResponseParsing_Tests.Should_fetch_queue_details_in_old_format.approved.txt b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQuery_ResponseParsing_Tests.Should_fetch_queue_details_in_old_format.approved.txt deleted file mode 100644 index 063ba3dd64..0000000000 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQuery_ResponseParsing_Tests.Should_fetch_queue_details_in_old_format.approved.txt +++ /dev/null @@ -1,23 +0,0 @@ -[ - { - "QueueName": "queue1", - "SanitizedName": "queue1", - "Scope": "vhost1", - "VHost": "vhost1", - "EndpointIndicators": [] - }, - { - "QueueName": "queue2", - "SanitizedName": "queue2", - "Scope": "vhost2", - "VHost": "vhost2", - "EndpointIndicators": [] - }, - { - "QueueName": "queue3", - "SanitizedName": "queue3", - "Scope": "vhost1", - "VHost": "vhost1", - "EndpointIndicators": [] - } -] \ No newline at end of file diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQuery_ResponseParsing_Tests.Should_handle_duplicated_json_data.approved.txt b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQuery_ResponseParsing_Tests.Should_handle_duplicated_json_data.approved.txt deleted file mode 100644 index 016fd18d12..0000000000 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQuery_ResponseParsing_Tests.Should_handle_duplicated_json_data.approved.txt +++ /dev/null @@ -1,16 +0,0 @@ -[ - { - "QueueName": "queue1", - "SanitizedName": "queue1", - "Scope": "vhost1", - "VHost": "vhost1", - "EndpointIndicators": [] - }, - { - "QueueName": "queue2", - "SanitizedName": "queue2", - "Scope": "vhost2", - "VHost": "vhost2", - "EndpointIndicators": [] - } -] \ No newline at end of file From 8965b450efa25674e3a3e285d4a17018bbaa32bd Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Thu, 27 Feb 2025 14:11:02 -0500 Subject: [PATCH 40/60] Update QueueLengthProvider to use management client --- src/Directory.Packages.props | 2 +- .../QueueLengthProvider.cs | 99 +++++-------------- ...nventionalRoutingTransportCustomization.cs | 2 +- ...itMQDirectRoutingTransportCustomization.cs | 2 +- 4 files changed, 28 insertions(+), 77 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index e3789f9fad..03f1125dc7 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -38,7 +38,7 @@ - + diff --git a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs index 5c19fc8e26..de2bcdc162 100644 --- a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs +++ b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs @@ -4,17 +4,21 @@ using System.Collections.Concurrent; using System.Threading; using System.Threading.Tasks; - using global::RabbitMQ.Client; using NServiceBus.Logging; - using NServiceBus.Transport.RabbitMQ; - using ConnectionFactory = NServiceBus.Transport.RabbitMQ.ConnectionFactory; + using NServiceBus.Transport.RabbitMQ.ManagementApi; class QueueLengthProvider : AbstractQueueLengthProvider { - public QueueLengthProvider(TransportSettings settings, Action store) : base(settings, store) + public QueueLengthProvider(TransportSettings settings, Action store, ITransportCustomization transportCustomization) : base(settings, store) { - queryExecutor = new QueryExecutor(ConnectionString); - queryExecutor.Initialize(); + if (transportCustomization is IManagementClientProvider provider) + { + managementClient = provider.ManagementClient; + } + else + { + throw new ArgumentException($"Transport customization does not implement {nameof(IManagementClientProvider)}. Type: {transportCustomization.GetType().Name}", nameof(transportCustomization)); + } } public override void TrackEndpointInputQueue(EndpointToQueueMapping queueToTrack) => @@ -76,90 +80,37 @@ async Task FetchQueueLengths(CancellationToken cancellationToken) { foreach (var endpointQueuePair in endpointQueues) { - await queryExecutor.Execute(async m => - { - var queueName = endpointQueuePair.Value; - - try - { - var size = (int)await m.MessageCountAsync(queueName, cancellationToken).ConfigureAwait(false); - - sizes.AddOrUpdate(queueName, _ => size, (_, __) => size); - } - catch (Exception e) - { - Logger.Warn($"Error querying queue length for {queueName}", e); - } - }, cancellationToken); - } - } - - readonly QueryExecutor queryExecutor; - static readonly TimeSpan QueryDelayInterval = TimeSpan.FromMilliseconds(200); - - readonly ConcurrentDictionary endpointQueues = new ConcurrentDictionary(); - readonly ConcurrentDictionary sizes = new ConcurrentDictionary(); - - static readonly ILog Logger = LogManager.GetLogger(); - - class QueryExecutor(string connectionString) : IDisposable - { - - public void Initialize() - { - var connectionConfiguration = ConnectionConfiguration.Create(connectionString); - - // TODO Fix this up - //var dbConnectionStringBuilder = new DbConnectionStringBuilder { ConnectionString = connectionString }; - - connectionFactory = new("ServiceControl.Monitoring", - connectionConfiguration, - null, //providing certificates is not supported yet - false, // TODO Fix dbConnectionStringBuilder.GetBooleanValue("DisableRemoteCertificateValidation"), - false, // TODO fix dbConnectionStringBuilder.GetBooleanValue("UseExternalAuthMechanism"), - TimeSpan.FromSeconds(60), // value would come from config API in actual transport - TimeSpan.FromSeconds(10), // value would come from config API in actual transport - null); // value would come from config API in actual transport - } + var queueName = endpointQueuePair.Value; - public async Task Execute(Action action, CancellationToken cancellationToken = default) - { try { - connection ??= await connectionFactory.CreateConnection("queue length monitor", cancellationToken: cancellationToken); + var (statusCode, reason, queue) = await managementClient.GetQueue(queueName, cancellationToken); - //Connection implements reconnection logic - while (!connection.IsOpen) + if (queue is not null) { - await Task.Delay(ReconnectionDelay, cancellationToken); + var size = queue.MessageCount; + sizes.AddOrUpdate(queueName, _ => size, (_, _) => size); } - - if (channel == null || channel.IsClosed) + else { - channel?.Dispose(); - - channel = await connection.CreateChannelAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + Logger.Warn($"Error querying queue length for {queueName}. {statusCode}: {reason}"); } - action(channel); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - // no-op } catch (Exception e) { - Logger.Warn("Error querying queue length.", e); + Logger.Warn($"Error querying queue length for {queueName}", e); } } + } - public void Dispose() => connection?.Dispose(); + static readonly TimeSpan QueryDelayInterval = TimeSpan.FromMilliseconds(200); - IConnection connection; - IChannel channel; - ConnectionFactory connectionFactory; + readonly ConcurrentDictionary endpointQueues = new(); + readonly ConcurrentDictionary sizes = new(); - static readonly TimeSpan ReconnectionDelay = TimeSpan.FromSeconds(5); - } + static readonly ILog Logger = LogManager.GetLogger(); + + readonly ManagementClient managementClient; } } diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index a7d8df4898..70768d7284 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -17,7 +17,7 @@ public abstract class RabbitMQConventionalRoutingTransportCustomization(NService protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } - protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } + protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; protected override RabbitMQTransport CreateTransport(TransportSettings transportSettings, TransportTransactionMode preferredTransactionMode = TransportTransactionMode.ReceiveOnly) { diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 049e34b226..bf1a70791b 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -17,7 +17,7 @@ public abstract class RabbitMQDirectRoutingTransportCustomization(NServiceBus.Qu protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } - protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } + protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; protected override RabbitMQTransport CreateTransport(TransportSettings transportSettings, TransportTransactionMode preferredTransactionMode = TransportTransactionMode.ReceiveOnly) { From 94728c81b6fefd515461d39d8aacf5463b9258e3 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Thu, 27 Feb 2025 18:31:47 -0500 Subject: [PATCH 41/60] Get RabbitMQQuery tests working --- ...nventionalRoutingTransportCustomization.cs | 2 +- ...itMQDirectRoutingTransportCustomization.cs | 2 +- .../RabbitMQQueryTests.cs | 18 +++-- ...ConnectionWithInvalidSettings.approved.txt | 7 -- ...stConnectionWithValidSettings.approved.txt | 6 -- .../RabbitMQQueryTests.cs | 77 ++++++++++--------- ...s.RabbitMQQuorumDirectRouting.Tests.csproj | 1 - 7 files changed, 53 insertions(+), 60 deletions(-) delete mode 100644 src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQueryTests.TestConnectionWithInvalidSettings.approved.txt delete mode 100644 src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQueryTests.TestConnectionWithValidSettings.approved.txt diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index 70768d7284..b14759646d 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -11,7 +11,7 @@ public abstract class RabbitMQConventionalRoutingTransportCustomization(NService { RabbitMQTransport rabbitMQTransport; - ManagementClient IManagementClientProvider.ManagementClient => rabbitMQTransport?.ManagementClient ?? throw new InvalidOperationException("Transport instance has not been created yet."); + ManagementClient IManagementClientProvider.ManagementClient => rabbitMQTransport?.ManagementClient ?? new ManagementClient(rabbitMQTransport.ConnectionConfiguration, rabbitMQTransport.ManagementApiConfiguration); protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index bf1a70791b..97415d66ce 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -11,7 +11,7 @@ public abstract class RabbitMQDirectRoutingTransportCustomization(NServiceBus.Qu { RabbitMQTransport rabbitMQTransport; - ManagementClient IManagementClientProvider.ManagementClient => rabbitMQTransport?.ManagementClient ?? throw new InvalidOperationException("Transport instance has not been created yet."); + ManagementClient IManagementClientProvider.ManagementClient => rabbitMQTransport?.ManagementClient ?? new ManagementClient(rabbitMQTransport.ConnectionConfiguration, rabbitMQTransport.ManagementApiConfiguration); protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; diff --git a/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs b/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs index ac38788349..ccfc2938cc 100644 --- a/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs +++ b/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs @@ -8,10 +8,11 @@ namespace ServiceControl.Transport.Tests; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; +using NServiceBus; using NUnit.Framework; +using ServiceControl.Transports.BrokerThroughput; using Transports; using Transports.RabbitMQ; -using ServiceControl.Transports.BrokerThroughput; [TestFixture] class RabbitMQQueryTests : TransportTestFixture @@ -20,25 +21,30 @@ class RabbitMQQueryTests : TransportTestFixture public async Task GetQueueNames_FindsQueues() { using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); - CancellationToken token = cancellationTokenSource.Token; + var token = cancellationTokenSource.Token; + var provider = new FakeTimeProvider(); + var transportSettings = new TransportSettings { ConnectionString = configuration.ConnectionString, - MaxConcurrency = 1, EndpointName = Guid.NewGuid().ToString("N") }; - var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); - string[] additionalQueues = Enumerable.Range(1, 10).Select(i => $"myqueue{i}").ToArray(); + + configuration.TransportCustomization.CustomizePrimaryEndpoint(new EndpointConfiguration(transportSettings.EndpointName), transportSettings); + + var additionalQueues = Enumerable.Range(1, 10).Select(i => $"myqueue{i}").ToArray(); await configuration.TransportCustomization.ProvisionQueues(transportSettings, additionalQueues); + var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); query.Initialize(ReadOnlyDictionary.Empty); var queueNames = new List(); + await foreach (IBrokerQueue queueName in query.GetQueueNames(token)) { queueNames.Add(queueName); - Assert.That(queueName.Scope, Is.EqualTo("/")); + if (queueName.QueueName == transportSettings.EndpointName) { Assert.That(queueName.EndpointIndicators, Has.Member("ConventionalTopologyBinding")); diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQueryTests.TestConnectionWithInvalidSettings.approved.txt b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQueryTests.TestConnectionWithInvalidSettings.approved.txt deleted file mode 100644 index 5790b9e0da..0000000000 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQueryTests.TestConnectionWithInvalidSettings.approved.txt +++ /dev/null @@ -1,7 +0,0 @@ -Connection test to RabbitMQ failed: -Failed to connect to 'http://localhost:12345/' - -Connection attempted with the following settings: -Username not set, defaulted to using "xxxxx" username from the ConnectionString used by instance -Password not set, defaulted to using password from the ConnectionString used by instance -RabbitMQ API Url set to "http://localhost:12345" diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQueryTests.TestConnectionWithValidSettings.approved.txt b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQueryTests.TestConnectionWithValidSettings.approved.txt deleted file mode 100644 index f64695447c..0000000000 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ApprovalFiles/RabbitMQQueryTests.TestConnectionWithValidSettings.approved.txt +++ /dev/null @@ -1,6 +0,0 @@ -Connection test to RabbitMQ was successful - -Connection settings used: -Username not set, defaulted to using "xxxxx" username from the ConnectionString used by instance -Password not set, defaulted to using password from the ConnectionString used by instance -RabbitMQ API Url not set, defaulted to using "xxxx" from the ConnectionString used by instance diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs index 4a552ae196..bf261a52c2 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs +++ b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs @@ -3,75 +3,65 @@ namespace ServiceControl.Transport.Tests; using System; using System.Collections.Generic; using System.Collections.ObjectModel; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging.Abstractions; using Microsoft.Extensions.Time.Testing; +using NServiceBus; using NUnit.Framework; -using Particular.Approvals; +using ServiceControl.Transports.BrokerThroughput; using Transports; using Transports.RabbitMQ; -using ServiceControl.Transports.BrokerThroughput; [TestFixture] class RabbitMQQueryTests : TransportTestFixture { - FakeTimeProvider provider; - TransportSettings transportSettings; - RabbitMQQuery query; - - [SetUp] - public void Initialise() - { - provider = new(); - provider.SetUtcNow(DateTimeOffset.UtcNow); - transportSettings = new TransportSettings - { - ConnectionString = configuration.ConnectionString, - MaxConcurrency = 1, - EndpointName = Guid.NewGuid().ToString("N") - }; - query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); - } - [Test] public async Task TestConnectionWithInvalidSettings() { using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var token = cancellationTokenSource.Token; - var dictionary = new Dictionary + var provider = new FakeTimeProvider(DateTimeOffset.UtcNow); + + var transportSettings = new TransportSettings { - { RabbitMQQuery.RabbitMQSettings.API, "http://localhost:12345" } + ConnectionString = configuration.ConnectionString + ";ManagementApiUrl=http://localhost:12345", + EndpointName = Guid.NewGuid().ToString("N") }; - query.Initialize(new ReadOnlyDictionary(dictionary)); + + configuration.TransportCustomization.CustomizePrimaryEndpoint(new EndpointConfiguration(transportSettings.EndpointName), transportSettings); + + var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); + query.Initialize(ReadOnlyDictionary.Empty); + (bool success, _, string diagnostics) = await query.TestConnection(cancellationTokenSource.Token); Assert.That(success, Is.False); - Approver.Verify(diagnostics, - s => Regex.Replace(s, "defaulted to using \"\\w*\" username", "defaulted to using \"xxxxx\" username", - RegexOptions.Multiline)); } [Test] public async Task TestConnectionWithValidSettings() { using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var token = cancellationTokenSource.Token; + var provider = new FakeTimeProvider(DateTimeOffset.UtcNow); + + var transportSettings = new TransportSettings + { + ConnectionString = configuration.ConnectionString, + EndpointName = Guid.NewGuid().ToString("N") + }; + + configuration.TransportCustomization.CustomizePrimaryEndpoint(new EndpointConfiguration(transportSettings.EndpointName), transportSettings); + + var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); query.Initialize(ReadOnlyDictionary.Empty); + (bool success, _, string diagnostics) = await query.TestConnection(cancellationTokenSource.Token); Assert.That(success, Is.True); - Approver.Verify(diagnostics, - s => - { - s = Regex.Replace(s, - "RabbitMQ API Url not set, defaulted to using \"http://[\\w.]*:15672\" from the ConnectionString used by instance", - "RabbitMQ API Url not set, defaulted to using \"xxxx\" from the ConnectionString used by instance", - RegexOptions.Multiline); - return Regex.Replace(s, "defaulted to using \"\\w*\" username", "defaulted to using \"xxxxx\" username", - RegexOptions.Multiline); - }); } [Test] @@ -79,10 +69,21 @@ public async Task RunScenario() { // We need to wait a bit of time, because the scenario running takes on average 1 sec per run. using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(3)); - CancellationToken token = cancellationTokenSource.Token; + var token = cancellationTokenSource.Token; + + var provider = new FakeTimeProvider(DateTimeOffset.UtcNow); + + var transportSettings = new TransportSettings + { + ConnectionString = configuration.ConnectionString, + EndpointName = Guid.NewGuid().ToString("N") + }; + + configuration.TransportCustomization.CustomizePrimaryEndpoint(new EndpointConfiguration(transportSettings.EndpointName), transportSettings); await CreateTestQueue(transportSettings.EndpointName); + var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); query.Initialize(ReadOnlyDictionary.Empty); var queueNames = new List(); diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj index 6cb4aacd7d..32f68e736f 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj +++ b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests.csproj @@ -20,7 +20,6 @@ - From c69587ff4cf8eec3c64ce94ec84864cedbd488bd Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Thu, 27 Feb 2025 18:34:16 -0500 Subject: [PATCH 42/60] Remove unused parameter from RabbitMQQuery ctor --- src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs | 2 +- .../RabbitMQQueryTests.cs | 2 +- .../RabbitMQQueryTests.cs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 737750b110..3c4bdd71ab 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -27,7 +27,7 @@ public class RabbitMQQuery : BrokerThroughputQuery .AddTimeout(TimeSpan.FromMinutes(2)) // Add timeout if it keeps failing .Build(); - public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, TransportSettings transportSettings, ITransportCustomization transportCustomization) : base(logger, "RabbitMQ") + public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, ITransportCustomization transportCustomization) : base(logger, "RabbitMQ") { this.logger = logger; this.timeProvider = timeProvider; diff --git a/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs b/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs index ccfc2938cc..25116c8a47 100644 --- a/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs +++ b/src/ServiceControl.Transports.RabbitMQClassicConventionalRouting.Tests/RabbitMQQueryTests.cs @@ -36,7 +36,7 @@ public async Task GetQueueNames_FindsQueues() var additionalQueues = Enumerable.Range(1, 10).Select(i => $"myqueue{i}").ToArray(); await configuration.TransportCustomization.ProvisionQueues(transportSettings, additionalQueues); - var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); + var query = new RabbitMQQuery(NullLogger.Instance, provider, configuration.TransportCustomization); query.Initialize(ReadOnlyDictionary.Empty); var queueNames = new List(); diff --git a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs index bf261a52c2..38a8b6f6ca 100644 --- a/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs +++ b/src/ServiceControl.Transports.RabbitMQQuorumDirectRouting.Tests/RabbitMQQueryTests.cs @@ -32,7 +32,7 @@ public async Task TestConnectionWithInvalidSettings() configuration.TransportCustomization.CustomizePrimaryEndpoint(new EndpointConfiguration(transportSettings.EndpointName), transportSettings); - var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); + var query = new RabbitMQQuery(NullLogger.Instance, provider, configuration.TransportCustomization); query.Initialize(ReadOnlyDictionary.Empty); (bool success, _, string diagnostics) = await query.TestConnection(cancellationTokenSource.Token); @@ -56,7 +56,7 @@ public async Task TestConnectionWithValidSettings() configuration.TransportCustomization.CustomizePrimaryEndpoint(new EndpointConfiguration(transportSettings.EndpointName), transportSettings); - var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); + var query = new RabbitMQQuery(NullLogger.Instance, provider, configuration.TransportCustomization); query.Initialize(ReadOnlyDictionary.Empty); (bool success, _, string diagnostics) = await query.TestConnection(cancellationTokenSource.Token); @@ -83,7 +83,7 @@ public async Task RunScenario() await CreateTestQueue(transportSettings.EndpointName); - var query = new RabbitMQQuery(NullLogger.Instance, provider, transportSettings, configuration.TransportCustomization); + var query = new RabbitMQQuery(NullLogger.Instance, provider, configuration.TransportCustomization); query.Initialize(ReadOnlyDictionary.Empty); var queueNames = new List(); From 0a5c9abd30889ed4ee7e6df7f695f18f026f71e6 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Fri, 28 Feb 2025 11:56:46 -0500 Subject: [PATCH 43/60] Get QueueLengthProvider tests working --- .../QueueLengthMonitoringTests.cs | 2 +- .../TransportTestFixture.cs | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Transports.Tests/QueueLengthMonitoringTests.cs b/src/ServiceControl.Transports.Tests/QueueLengthMonitoringTests.cs index 5504962847..c121bcae6d 100644 --- a/src/ServiceControl.Transports.Tests/QueueLengthMonitoringTests.cs +++ b/src/ServiceControl.Transports.Tests/QueueLengthMonitoringTests.cs @@ -10,7 +10,7 @@ class QueueLengthMonitoringTests : TransportTestFixture [Test] public async Task Should_report_queue_length() { - var queueName = GetTestQueueName("queuelenght"); + var queueName = GetTestQueueName("queuelength"); await CreateTestQueue(queueName); diff --git a/src/ServiceControl.Transports.Tests/TransportTestFixture.cs b/src/ServiceControl.Transports.Tests/TransportTestFixture.cs index c6eefe193b..0f6d9027e1 100644 --- a/src/ServiceControl.Transports.Tests/TransportTestFixture.cs +++ b/src/ServiceControl.Transports.Tests/TransportTestFixture.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; + using NServiceBus; using NServiceBus.Logging; using NServiceBus.Transport; using NUnit.Framework; @@ -87,21 +88,24 @@ protected async Task StartQueueLengthProvider(string queueName // currently working around by creating a service collection per start call and then disposing the provider // as part of the method scope. This could lead to potential problems later once we add disposable resources // but this code probably requires a major overhaul anyway. - var serviceCollection = new ServiceCollection(); + var transportSettings = new TransportSettings { ConnectionString = configuration.ConnectionString, EndpointName = queueName, MaxConcurrency = 1 }; + + var serviceCollection = new ServiceCollection(); + configuration.TransportCustomization.AddTransportForMonitoring(serviceCollection, transportSettings); - serviceCollection.AddSingleton>((qlt, _) => - onQueueLengthReported(qlt.First())); + configuration.TransportCustomization.CustomizeMonitoringEndpoint(new EndpointConfiguration("queueName"), transportSettings); + + serviceCollection.AddSingleton>((qlt, _) => onQueueLengthReported(qlt.First())); var serviceProvider = serviceCollection.BuildServiceProvider(); - queueLengthProvider = serviceProvider.GetRequiredService(); + queueLengthProvider = serviceProvider.GetRequiredService(); await queueLengthProvider.StartAsync(CancellationToken.None); - queueLengthProvider.TrackEndpointInputQueue(new EndpointToQueueMapping(queueName, queueName)); return new QueueLengthProviderScope(serviceProvider); From d02e60e42468fba0a0aa69a2a2679df6d54a5223 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Fri, 28 Feb 2025 13:03:11 -0500 Subject: [PATCH 44/60] Make IManagementClientProvider lazy to ensure endpoints have started --- .../IManagementClientProvider.cs | 3 ++- .../QueueLengthProvider.cs | 6 ++--- ...nventionalRoutingTransportCustomization.cs | 27 ++++++++++++++++--- ...itMQDirectRoutingTransportCustomization.cs | 27 ++++++++++++++++--- .../RabbitMQQuery.cs | 18 ++++++------- 5 files changed, 60 insertions(+), 21 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/IManagementClientProvider.cs b/src/ServiceControl.Transports.RabbitMQ/IManagementClientProvider.cs index 70f069b955..ab0bd30b3b 100644 --- a/src/ServiceControl.Transports.RabbitMQ/IManagementClientProvider.cs +++ b/src/ServiceControl.Transports.RabbitMQ/IManagementClientProvider.cs @@ -1,8 +1,9 @@ namespace ServiceControl.Transports.RabbitMQ; +using System; using NServiceBus.Transport.RabbitMQ.ManagementApi; interface IManagementClientProvider { - ManagementClient ManagementClient { get; } + Lazy GetManagementClient(); } diff --git a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs index de2bcdc162..d37f1885de 100644 --- a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs +++ b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs @@ -13,7 +13,7 @@ public QueueLengthProvider(TransportSettings settings, Action(); - readonly ManagementClient managementClient; + readonly Lazy managementClient; } } diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index b14759646d..2112151da3 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -9,15 +9,34 @@ public abstract class RabbitMQConventionalRoutingTransportCustomization(NServiceBus.QueueType queueType) : TransportCustomization, IManagementClientProvider { - RabbitMQTransport rabbitMQTransport; + RabbitMQTransport transport; - ManagementClient IManagementClientProvider.ManagementClient => rabbitMQTransport?.ManagementClient ?? new ManagementClient(rabbitMQTransport.ConnectionConfiguration, rabbitMQTransport.ManagementApiConfiguration); + Lazy IManagementClientProvider.GetManagementClient() + { + return new(() => Get()); + + ManagementClient Get() + { + if (transport is null) + { + throw new InvalidOperationException("Management client not available because a CustomizeTransport method has not been called first."); + } + + // Since some tests don't actually start an endpoint, this is needed to ensure a management client is available + if (transport.ManagementClient is null) + { + return new ManagementClient(transport.ConnectionConfiguration, transport.ManagementApiConfiguration); + } + + return transport.ManagementClient; + } + } - protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; + protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } - protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; + protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; protected override RabbitMQTransport CreateTransport(TransportSettings transportSettings, TransportTransactionMode preferredTransactionMode = TransportTransactionMode.ReceiveOnly) { diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 97415d66ce..6a6a88d4d4 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -9,15 +9,34 @@ public abstract class RabbitMQDirectRoutingTransportCustomization(NServiceBus.QueueType queueType) : TransportCustomization, IManagementClientProvider { - RabbitMQTransport rabbitMQTransport; + RabbitMQTransport transport; - ManagementClient IManagementClientProvider.ManagementClient => rabbitMQTransport?.ManagementClient ?? new ManagementClient(rabbitMQTransport.ConnectionConfiguration, rabbitMQTransport.ManagementApiConfiguration); + Lazy IManagementClientProvider.GetManagementClient() + { + return new(() => Get()); + + ManagementClient Get() + { + if (transport is null) + { + throw new InvalidOperationException("Management client not available because a CustomizeTransport method has not been called first."); + } + + // Since some tests don't actually start an endpoint, this is needed to ensure a management client is available + if (transport.ManagementClient is null) + { + return new ManagementClient(transport.ConnectionConfiguration, transport.ManagementApiConfiguration); + } + + return transport.ManagementClient; + } + } - protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; + protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } - protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => rabbitMQTransport = transportDefinition; + protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; protected override RabbitMQTransport CreateTransport(TransportSettings transportSettings, TransportTransactionMode preferredTransactionMode = TransportTransactionMode.ReceiveOnly) { diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 3c4bdd71ab..826114ba7c 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -20,7 +20,7 @@ public class RabbitMQQuery : BrokerThroughputQuery { readonly ILogger logger; readonly TimeProvider timeProvider; - readonly ManagementClient managementClient; + readonly Lazy managementClient; readonly ResiliencePipeline pipeline = new ResiliencePipelineBuilder() .AddRetry(new RetryStrategyOptions()) // Add retry using the default options @@ -34,7 +34,7 @@ public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, I if (transportCustomization is IManagementClientProvider provider) { - managementClient = provider.ManagementClient; + managementClient = provider.GetManagementClient(); } else { @@ -63,7 +63,7 @@ void CheckLegacySettings(ReadOnlyDictionary settings, string key public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var queue = (RabbitMQBrokerQueueDetails)brokerQueue; - var response = await pipeline.ExecuteAsync(async token => await managementClient.GetQueue(queue.QueueName, token), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueue(queue.QueueName, token), cancellationToken); if (response.Value is null) { @@ -79,7 +79,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro { await Task.Delay(TimeSpan.FromMinutes(15), timeProvider, cancellationToken); - response = await pipeline.ExecuteAsync(async token => await managementClient.GetQueue(queue.QueueName, token), cancellationToken); + response = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueue(queue.QueueName, token), cancellationToken); if (response.Value is null) { @@ -99,7 +99,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro async Task GetRabbitDetails(CancellationToken cancellationToken) { - var response = await pipeline.ExecuteAsync(async async => await managementClient.GetOverview(cancellationToken), cancellationToken); + var response = await pipeline.ExecuteAsync(async async => await managementClient.Value.GetOverview(cancellationToken), cancellationToken); ValidateResponse(response); @@ -160,7 +160,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can { try { - var response = await pipeline.ExecuteAsync(async token => await managementClient.GetBindingsForQueue(brokerQueue.QueueName, token), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetBindingsForQueue(brokerQueue.QueueName, token), cancellationToken); // Check if conventional binding is found if (response.Value.Any(binding => binding?.Source == brokerQueue.QueueName @@ -179,7 +179,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can try { - var response = await pipeline.ExecuteAsync(async token => await managementClient.GetBindingsForExchange(brokerQueue.QueueName, token), cancellationToken); + var response = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetBindingsForExchange(brokerQueue.QueueName, token), cancellationToken); // Check if delayed binding is found if (response.Value.Any(binding => binding?.Source is "nsb.v2.delay-delivery" or "nsb.delay-delivery" @@ -198,7 +198,7 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can internal async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) { - var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await managementClient.GetQueues(page, 500, token), cancellationToken); + var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); ValidateResponse((StatusCode, Reason, Value)); @@ -227,7 +227,7 @@ static List MaterializeQueueDetails(List item { try { - var (statusCode, reason, value) = await managementClient.GetOverview(cancellationToken); + var (statusCode, reason, value) = await managementClient.Value.GetOverview(cancellationToken); if (value is not null) { From be035ca3cf00433d90ff3fe913dc49a532ca862b Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Mon, 3 Mar 2025 17:26:05 -0500 Subject: [PATCH 45/60] Remove old settings --- .../RabbitMQQuery.cs | 34 ++----------------- 1 file changed, 2 insertions(+), 32 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 826114ba7c..ed67b91ddf 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -18,7 +18,6 @@ namespace ServiceControl.Transports.RabbitMQ; public class RabbitMQQuery : BrokerThroughputQuery { - readonly ILogger logger; readonly TimeProvider timeProvider; readonly Lazy managementClient; @@ -29,7 +28,6 @@ public class RabbitMQQuery : BrokerThroughputQuery public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, ITransportCustomization transportCustomization) : base(logger, "RabbitMQ") { - this.logger = logger; this.timeProvider = timeProvider; if (transportCustomization is IManagementClientProvider provider) @@ -44,20 +42,7 @@ public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, I protected override void InitializeCore(ReadOnlyDictionary settings) { - // TODO: Update documentation - // https://docs.particular.net/servicecontrol/servicecontrol-instances/configuration#usage-reporting-when-using-the-rabbitmq-transport - CheckLegacySettings(settings, RabbitMQSettings.UserName); - CheckLegacySettings(settings, RabbitMQSettings.Password); - CheckLegacySettings(settings, RabbitMQSettings.API); - } - - void CheckLegacySettings(ReadOnlyDictionary settings, string key) - { - if (settings.TryGetValue(key, out _)) - { - logger.LogInformation($"The legacy LicensingComponent/{key} is still defined in the app.config or environment variables"); - _ = Diagnostics.AppendLine($"LicensingComponent/{key} is still defined in the app.config or environment variables"); - } + Diagnostics.AppendLine("Using settings from connection string"); } public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -216,12 +201,7 @@ static List MaterializeQueueDetails(List item return queues; } - public override KeyDescriptionPair[] Settings => - [ - new KeyDescriptionPair(RabbitMQSettings.API, RabbitMQSettings.APIDescription), - new KeyDescriptionPair(RabbitMQSettings.UserName, RabbitMQSettings.UserNameDescription), - new KeyDescriptionPair(RabbitMQSettings.Password, RabbitMQSettings.PasswordDescription) - ]; + public override KeyDescriptionPair[] Settings => []; protected override async Task<(bool Success, List Errors)> TestConnectionCore(CancellationToken cancellationToken) { @@ -243,15 +223,5 @@ static List MaterializeQueueDetails(List item throw new Exception($"Failed to connect to RabbitMQ management API", ex); } } - - public static class RabbitMQSettings - { - public static readonly string API = "RabbitMQ/ApiUrl"; - public static readonly string APIDescription = "RabbitMQ management URL"; - public static readonly string UserName = "RabbitMQ/UserName"; - public static readonly string UserNameDescription = "Username to access the RabbitMQ management interface"; - public static readonly string Password = "RabbitMQ/Password"; - public static readonly string PasswordDescription = "Password to access the RabbitMQ management interface"; - } } From 2f68449648bb699eae0321f145eb5eedd567617a Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Mon, 3 Mar 2025 17:39:27 -0500 Subject: [PATCH 46/60] Reorder methods --- .../RabbitMQQuery.cs | 81 +++++++++---------- 1 file changed, 39 insertions(+), 42 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index ed67b91ddf..cb4527743f 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -40,11 +40,6 @@ public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, I } } - protected override void InitializeCore(ReadOnlyDictionary settings) - { - Diagnostics.AppendLine("Using settings from connection string"); - } - public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) { var queue = (RabbitMQBrokerQueueDetails)brokerQueue; @@ -82,34 +77,6 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro } } - async Task GetRabbitDetails(CancellationToken cancellationToken) - { - var response = await pipeline.ExecuteAsync(async async => await managementClient.Value.GetOverview(cancellationToken), cancellationToken); - - ValidateResponse(response); - - if (response.Value!.DisableStats) - { - throw new Exception("The RabbitMQ broker is configured with 'management.disable_stats = true' or 'management_agent.disable_metrics_collector = true' " + - "and as a result queue statistics cannot be collected using this tool. Consider changing the configuration of the RabbitMQ broker."); - } - - Data["RabbitMQVersion"] = response.Value?.BrokerVersion ?? "Unknown"; - } - - void ValidateResponse((HttpStatusCode StatusCode, string Reason, T? Value) response) - { - if (response.StatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException($"Request failed with status code {response.StatusCode}: {response.Reason}"); - } - - if (response.Value is null) - { - throw new InvalidOperationException("Request was successful, but the response body was null when a value was expected"); - } - } - public override async IAsyncEnumerable GetQueueNames([EnumeratorCancellation] CancellationToken cancellationToken) { var page = 1; @@ -141,6 +108,30 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa } while (morePages); } + async Task GetRabbitDetails(CancellationToken cancellationToken) + { + var response = await pipeline.ExecuteAsync(async async => await managementClient.Value.GetOverview(cancellationToken), cancellationToken); + + ValidateResponse(response); + + if (response.Value!.DisableStats) + { + throw new Exception("The RabbitMQ broker is configured with 'management.disable_stats = true' or 'management_agent.disable_metrics_collector = true' " + + "and as a result queue statistics cannot be collected using this tool. Consider changing the configuration of the RabbitMQ broker."); + } + + Data["RabbitMQVersion"] = response.Value?.BrokerVersion ?? "Unknown"; + } + + internal async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) + { + var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); + + ValidateResponse((StatusCode, Reason, Value)); + + return (MaterializeQueueDetails(Value), MorePages); + } + async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, CancellationToken cancellationToken) { try @@ -181,15 +172,6 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can } } - internal async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) - { - var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); - - ValidateResponse((StatusCode, Reason, Value)); - - return (MaterializeQueueDetails(Value), MorePages); - } - static List MaterializeQueueDetails(List items) { var queues = new List(); @@ -201,6 +183,19 @@ static List MaterializeQueueDetails(List item return queues; } + void ValidateResponse((HttpStatusCode StatusCode, string Reason, T? Value) response) + { + if (response.StatusCode != HttpStatusCode.OK) + { + throw new HttpRequestException($"Request failed with status code {response.StatusCode}: {response.Reason}"); + } + + if (response.Value is null) + { + throw new InvalidOperationException("Request was successful, but the response body was null when a value was expected"); + } + } + public override KeyDescriptionPair[] Settings => []; protected override async Task<(bool Success, List Errors)> TestConnectionCore(CancellationToken cancellationToken) @@ -223,5 +218,7 @@ static List MaterializeQueueDetails(List item throw new Exception($"Failed to connect to RabbitMQ management API", ex); } } + + protected override void InitializeCore(ReadOnlyDictionary settings) => Diagnostics.AppendLine("Using settings from connection string"); } From 5d41a6c39084df1e6c9ce73d91f68739e66b9ffd Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Mon, 3 Mar 2025 17:39:51 -0500 Subject: [PATCH 47/60] Make GetPage private --- src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index cb4527743f..237f4fe3cc 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -123,7 +123,7 @@ async Task GetRabbitDetails(CancellationToken cancellationToken) Data["RabbitMQVersion"] = response.Value?.BrokerVersion ?? "Unknown"; } - internal async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) + async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) { var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); From 5ce6b8f2264b9171aad7cd1ac0edd4fcd579ed69 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Mon, 3 Mar 2025 17:58:06 -0500 Subject: [PATCH 48/60] Remove unneeded conversion to RabbitMQBrokerQueueDetails --- .../RabbitMQQuery.cs | 32 +++++++------------ 1 file changed, 11 insertions(+), 21 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 237f4fe3cc..3e78fe9f07 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -90,17 +90,18 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa if (queues is not null) { - foreach (var rabbitMQQueueDetails in queues) + foreach (var queue in queues) { - if (rabbitMQQueueDetails.QueueName.StartsWith("nsb.delay-level-") || - rabbitMQQueueDetails.QueueName.StartsWith("nsb.v2.delay-level-") || - rabbitMQQueueDetails.QueueName.StartsWith("nsb.v2.verify-")) + if (queue.Name.StartsWith("nsb.delay-level-") || + queue.Name.StartsWith("nsb.v2.delay-level-") || + queue.Name.StartsWith("nsb.v2.verify-")) { continue; } - await AddAdditionalQueueDetails(rabbitMQQueueDetails, cancellationToken); - yield return rabbitMQQueueDetails; + var queueDetails = new RabbitMQBrokerQueueDetails(queue); + await AddAdditionalQueueDetails(queueDetails, cancellationToken); + yield return queueDetails; } } @@ -123,13 +124,13 @@ async Task GetRabbitDetails(CancellationToken cancellationToken) Data["RabbitMQVersion"] = response.Value?.BrokerVersion ?? "Unknown"; } - async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) + async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) { - var (StatusCode, Reason, Value, MorePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); + var (statusCode, reason, value, morePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); - ValidateResponse((StatusCode, Reason, Value)); + ValidateResponse((statusCode, reason, value)); - return (MaterializeQueueDetails(Value), MorePages); + return (value, morePages); } async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, CancellationToken cancellationToken) @@ -172,17 +173,6 @@ async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, Can } } - static List MaterializeQueueDetails(List items) - { - var queues = new List(); - foreach (var item in items) - { - queues.Add(new RabbitMQBrokerQueueDetails(item)); - } - - return queues; - } - void ValidateResponse((HttpStatusCode StatusCode, string Reason, T? Value) response) { if (response.StatusCode != HttpStatusCode.OK) From 8f8939851fbefc8a24b184ea97fa6f388e602a57 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Mon, 3 Mar 2025 18:31:18 -0500 Subject: [PATCH 49/60] Remove GetPage --- .../RabbitMQQuery.cs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 3e78fe9f07..e5215d008b 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -86,7 +86,9 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa do { - (var queues, morePages) = await GetPage(page, cancellationToken); + (var statusCode, var reason, var queues, morePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); + + ValidateResponse((statusCode, reason, queues)); if (queues is not null) { @@ -124,15 +126,6 @@ async Task GetRabbitDetails(CancellationToken cancellationToken) Data["RabbitMQVersion"] = response.Value?.BrokerVersion ?? "Unknown"; } - async Task<(List?, bool morePages)> GetPage(int page, CancellationToken cancellationToken) - { - var (statusCode, reason, value, morePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); - - ValidateResponse((statusCode, reason, value)); - - return (value, morePages); - } - async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, CancellationToken cancellationToken) { try From b996030c85e9310ed960c2a638aadf4bce077c95 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Mon, 3 Mar 2025 18:31:47 -0500 Subject: [PATCH 50/60] Check for DisableStats in TestConnectionCore --- .../RabbitMQQuery.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index e5215d008b..19235aaca0 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -117,10 +117,9 @@ async Task GetRabbitDetails(CancellationToken cancellationToken) ValidateResponse(response); - if (response.Value!.DisableStats) + if (response.Value?.DisableStats ?? false) { - throw new Exception("The RabbitMQ broker is configured with 'management.disable_stats = true' or 'management_agent.disable_metrics_collector = true' " + - "and as a result queue statistics cannot be collected using this tool. Consider changing the configuration of the RabbitMQ broker."); + throw new Exception(disableStatsErrorMessage); } Data["RabbitMQVersion"] = response.Value?.BrokerVersion ?? "Unknown"; @@ -189,6 +188,11 @@ void ValidateResponse((HttpStatusCode StatusCode, string Reason, T? Value) re if (value is not null) { + if (value.DisableStats) + { + return (false, [disableStatsErrorMessage]); + } + return (true, []); } else @@ -203,5 +207,7 @@ void ValidateResponse((HttpStatusCode StatusCode, string Reason, T? Value) re } protected override void InitializeCore(ReadOnlyDictionary settings) => Diagnostics.AppendLine("Using settings from connection string"); + + const string disableStatsErrorMessage = "The RabbitMQ broker is configured with 'management.disable_stats = true' or 'management_agent.disable_metrics_collector = true' and as a result queue statistics cannot be collected using this tool. Consider changing the configuration of the RabbitMQ broker."; } From 00a6668a1818fed66d6df0c861779f7583d7e3b1 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Mon, 3 Mar 2025 18:44:14 -0500 Subject: [PATCH 51/60] Update to 10.0.0-beta.3 --- src/Directory.Packages.props | 2 +- .../RabbitMQTransportExtensions.cs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index 03f1125dc7..e76ce7bdf0 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -38,7 +38,7 @@ - + diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs index 32ae263731..4484614d98 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs @@ -36,6 +36,10 @@ public static void SetCustomSettingsFromConnectionString(this RabbitMQTransport transport.ManagementApiConfiguration = new(url); } } + else if (dictionary.TryGetValue("ManagementApiUserName", out var userName) && dictionary.TryGetValue("ManagementApiPassword", out var password)) + { + transport.ManagementApiConfiguration = new(userName, password); + } if (dictionary.TryGetValue("DisableRemoteCertificateValidation", out var disableRemoteCertificateValidationString)) { From ea59075f722e738b73918ed1ef6cb8f8e9188f47 Mon Sep 17 00:00:00 2001 From: Andreas Bednarz <110360248+abparticular@users.noreply.github.com> Date: Wed, 5 Mar 2025 04:34:44 +1100 Subject: [PATCH 52/60] Migrate RabbitMQ management API settings into connection string * Migrate RabbitMQ management API settings from separate app.config entries into the connection string * Renamed settings related to Licensing component's use of RabbitMQ management API Removed unnecessary LINQ usage when working with DbConnectionStringBuilder in ServiceControlAppConfig --- .../ServiceControl/ServiceControlAppConfig.cs | 46 ++++++++++++++++++- .../ServiceControl/SettingsList.cs | 20 +++++++- 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs index c00b1d8ee5..3f93c088e4 100644 --- a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs +++ b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs @@ -1,5 +1,7 @@ namespace ServiceControlInstaller.Engine.Configuration.ServiceControl { + using System; + using System.Data.Common; using System.IO; using Instances; @@ -12,7 +14,7 @@ public ServiceControlAppConfig(IServiceControlInstance instance) : base(Path.Com protected override void UpdateSettings() { - Config.ConnectionStrings.ConnectionStrings.Set("NServiceBus/Transport", details.ConnectionString); + Config.ConnectionStrings.ConnectionStrings.Set("NServiceBus/Transport", UpdateConnectionString()); var settings = Config.AppSettings.Settings; var version = details.Version; settings.Set(ServiceControlSettings.InstanceName, details.InstanceName, version); @@ -43,6 +45,9 @@ protected override void UpdateSettings() settings.RemoveIfRetired(ServiceControlSettings.AuditLogQueue, version); settings.RemoveIfRetired(ServiceControlSettings.ForwardAuditMessages, version); settings.RemoveIfRetired(ServiceControlSettings.InternalQueueName, version); + settings.RemoveIfRetired(ServiceControlSettings.LicensingComponentRabbitMqManagementApiUrl, version); + settings.RemoveIfRetired(ServiceControlSettings.LicensingComponentRabbitMqManagementApiUsername, version); + settings.RemoveIfRetired(ServiceControlSettings.LicensingComponentRabbitMqManagementApiPassword, version); RemoveRavenDB35Settings(settings, version); } @@ -68,6 +73,43 @@ public override void SetTransportType(string transportTypeName) settings.Set(ServiceControlSettings.TransportType, transportTypeName, version); } - IServiceControlInstance details; + string UpdateConnectionString() + { + var connectionStringBuilder = new DbConnectionStringBuilder { ConnectionString = details.ConnectionString }; + + MigrateLicensingComponentRabbitMqManagementApiSettings(connectionStringBuilder); + + return connectionStringBuilder.ConnectionString; + } + + void MigrateLicensingComponentRabbitMqManagementApiSettings(DbConnectionStringBuilder connectionStringBuilder) + { + if (!details.TransportPackage.Name.Contains("rabbitmq", StringComparison.OrdinalIgnoreCase)) + { + return; + } + + var settings = Config.AppSettings.Settings; + + var legacySetting = settings[ServiceControlSettings.LicensingComponentRabbitMqManagementApiUrl.Name]; + if (legacySetting is not null && !connectionStringBuilder.ContainsKey("ManagementApiUrl")) + { + connectionStringBuilder.Add("ManagementApiUrl", legacySetting.Value); + } + + legacySetting = settings[ServiceControlSettings.LicensingComponentRabbitMqManagementApiUsername.Name]; + if (legacySetting is not null && !connectionStringBuilder.ContainsKey("ManagementApiUserName")) + { + connectionStringBuilder.Add("ManagementApiUserName", legacySetting.Value); + } + + legacySetting = settings[ServiceControlSettings.LicensingComponentRabbitMqManagementApiPassword.Name]; + if (legacySetting is not null && !connectionStringBuilder.ContainsKey("ManagementApiPassword")) + { + connectionStringBuilder.Add("ManagementApiPassword", legacySetting.Value); + } + } + + readonly IServiceControlInstance details; } } \ No newline at end of file diff --git a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/SettingsList.cs b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/SettingsList.cs index f982b8bc34..9dc2166000 100644 --- a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/SettingsList.cs +++ b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/SettingsList.cs @@ -95,5 +95,23 @@ public static class ServiceControlSettings Name = "ServiceControl/ShutdownTimeout", SupportedFrom = new SemanticVersion(6, 4, 1) }; + + public static readonly SettingInfo LicensingComponentRabbitMqManagementApiUrl = new() + { + Name = "LicensingComponent/RabbitMQ/ApiUrl", + RemovedFrom = new SemanticVersion(6, 5, 0) + }; + + public static readonly SettingInfo LicensingComponentRabbitMqManagementApiUsername = new() + { + Name = "LicensingComponent/RabbitMQ/UserName", + RemovedFrom = new SemanticVersion(6, 5, 0) + }; + + public static readonly SettingInfo LicensingComponentRabbitMqManagementApiPassword = new() + { + Name = "LicensingComponent/RabbitMQ/Password", + RemovedFrom = new SemanticVersion(6, 5, 0) + }; } -} \ No newline at end of file +} From cff47bc261e8d0c3be8067d269ed26d2e6feaef6 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Tue, 4 Mar 2025 13:03:14 -0500 Subject: [PATCH 53/60] Rename extension method --- .../RabbitMQConventionalRoutingTransportCustomization.cs | 2 +- .../RabbitMQDirectRoutingTransportCustomization.cs | 2 +- .../RabbitMQTransportExtensions.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index 2112151da3..c234d91f36 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -47,7 +47,7 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport var transport = new RabbitMQTransport(RoutingTopology.Conventional(queueType), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; - transport.SetCustomSettingsFromConnectionString(transportSettings.ConnectionString); + transport.ApplySettingsFromConnectionString(transportSettings.ConnectionString); return transport; } diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index 6a6a88d4d4..d2a7a775e4 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -47,7 +47,7 @@ protected override RabbitMQTransport CreateTransport(TransportSettings transport var transport = new RabbitMQTransport(RoutingTopology.Direct(queueType, routingKeyConvention: type => type.FullName.Replace(".", "-")), transportSettings.ConnectionString, enableDelayedDelivery: false); transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; - transport.SetCustomSettingsFromConnectionString(transportSettings.ConnectionString); + transport.ApplySettingsFromConnectionString(transportSettings.ConnectionString); return transport; } diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs index 4484614d98..36e62780c4 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs @@ -8,7 +8,7 @@ static class RabbitMQTransportExtensions { - public static void SetCustomSettingsFromConnectionString(this RabbitMQTransport transport, string connectionString) + public static void ApplySettingsFromConnectionString(this RabbitMQTransport transport, string connectionString) { if (connectionString.StartsWith("amqp", StringComparison.OrdinalIgnoreCase)) { From fcb217c04051faee7569edddc287202876c08e45 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Tue, 4 Mar 2025 16:38:46 -0500 Subject: [PATCH 54/60] Improve names --- ...rQueueDetails.cs => RabbitMQBrokerQueue.cs} | 4 ++-- .../RabbitMQQuery.cs | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) rename src/ServiceControl.Transports.RabbitMQ/{RabbitMQBrokerQueueDetails.cs => RabbitMQBrokerQueue.cs} (87%) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueue.cs similarity index 87% rename from src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs rename to src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueue.cs index 4a4f8d0f68..542e94ef7e 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueueDetails.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQBrokerQueue.cs @@ -5,7 +5,7 @@ namespace ServiceControl.Transports.RabbitMQ; using NServiceBus.Transport.RabbitMQ.ManagementApi; using ServiceControl.Transports.BrokerThroughput; -class RabbitMQBrokerQueueDetails(Queue queue) : IBrokerQueue +class RabbitMQBrokerQueue(Queue queue) : IBrokerQueue { public string QueueName { get; } = queue.Name; @@ -19,7 +19,7 @@ class RabbitMQBrokerQueueDetails(Queue queue) : IBrokerQueue long Baseline { get; set; } = queue.MessageStats?.Ack ?? 0; - public long CalculateThroughputFrom(RabbitMQBrokerQueueDetails newReading) + public long CalculateThroughputFrom(RabbitMQBrokerQueue newReading) { var newlyAckedMessages = 0L; diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index 19235aaca0..a968ab610b 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -42,7 +42,7 @@ public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, I public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var queue = (RabbitMQBrokerQueueDetails)brokerQueue; + var queue = (RabbitMQBrokerQueue)brokerQueue; var response = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueue(queue.QueueName, token), cancellationToken); if (response.Value is null) @@ -50,7 +50,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); } - var newReading = new RabbitMQBrokerQueueDetails(response.Value); + var newReading = new RabbitMQBrokerQueue(response.Value); _ = queue.CalculateThroughputFrom(newReading); @@ -66,7 +66,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); } - newReading = new RabbitMQBrokerQueueDetails(response.Value); + newReading = new RabbitMQBrokerQueue(response.Value); var newTotalThroughput = queue.CalculateThroughputFrom(newReading); yield return new QueueThroughput @@ -82,7 +82,7 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa var page = 1; bool morePages; - await GetRabbitDetails(cancellationToken); + await GetBrokerDetails(cancellationToken); do { @@ -101,9 +101,9 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa continue; } - var queueDetails = new RabbitMQBrokerQueueDetails(queue); - await AddAdditionalQueueDetails(queueDetails, cancellationToken); - yield return queueDetails; + var brokerQueue = new RabbitMQBrokerQueue(queue); + await AddEndpointIndicators(brokerQueue, cancellationToken); + yield return brokerQueue; } } @@ -111,7 +111,7 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa } while (morePages); } - async Task GetRabbitDetails(CancellationToken cancellationToken) + async Task GetBrokerDetails(CancellationToken cancellationToken) { var response = await pipeline.ExecuteAsync(async async => await managementClient.Value.GetOverview(cancellationToken), cancellationToken); @@ -125,7 +125,7 @@ async Task GetRabbitDetails(CancellationToken cancellationToken) Data["RabbitMQVersion"] = response.Value?.BrokerVersion ?? "Unknown"; } - async Task AddAdditionalQueueDetails(RabbitMQBrokerQueueDetails brokerQueue, CancellationToken cancellationToken) + async Task AddEndpointIndicators(RabbitMQBrokerQueue brokerQueue, CancellationToken cancellationToken) { try { From a33446b9e20aa84c33c5d4a7944da01bdce56960 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Fri, 7 Mar 2025 12:06:57 -0500 Subject: [PATCH 55/60] Update to 10.0.0-beta.4 --- src/Directory.Packages.props | 2 +- .../QueueLengthProvider.cs | 14 +-- .../RabbitMQQuery.cs | 110 ++++++------------ .../RabbitMQTransportExtensions.cs | 20 +--- 4 files changed, 45 insertions(+), 101 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index e76ce7bdf0..a46f67be7c 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -38,7 +38,7 @@ - + diff --git a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs index d37f1885de..3c60019abe 100644 --- a/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs +++ b/src/ServiceControl.Transports.RabbitMQ/QueueLengthProvider.cs @@ -84,18 +84,10 @@ async Task FetchQueueLengths(CancellationToken cancellationToken) try { - var (statusCode, reason, queue) = await managementClient.Value.GetQueue(queueName, cancellationToken); - - if (queue is not null) - { - var size = queue.MessageCount; - sizes.AddOrUpdate(queueName, _ => size, (_, _) => size); - } - else - { - Logger.Warn($"Error querying queue length for {queueName}. {statusCode}: {reason}"); - } + var queue = await managementClient.Value.GetQueue(queueName, cancellationToken); + var size = queue.MessageCount; + sizes.AddOrUpdate(queueName, _ => size, (_, _) => size); } catch (Exception e) { diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs index a968ab610b..8367d46bcb 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQQuery.cs @@ -42,32 +42,21 @@ public RabbitMQQuery(ILogger logger, TimeProvider timeProvider, I public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var queue = (RabbitMQBrokerQueue)brokerQueue; - var response = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueue(queue.QueueName, token), cancellationToken); + var rabbitBrokerQueue = (RabbitMQBrokerQueue)brokerQueue; + var queue = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueue(rabbitBrokerQueue.QueueName, token), cancellationToken); + var newReading = new RabbitMQBrokerQueue(queue); - if (response.Value is null) - { - throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); - } - - var newReading = new RabbitMQBrokerQueue(response.Value); - - _ = queue.CalculateThroughputFrom(newReading); + _ = rabbitBrokerQueue.CalculateThroughputFrom(newReading); // looping for 24hrs, in 4 increments of 15 minutes for (var i = 0; i < 24 * 4; i++) { await Task.Delay(TimeSpan.FromMinutes(15), timeProvider, cancellationToken); - response = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueue(queue.QueueName, token), cancellationToken); - - if (response.Value is null) - { - throw new InvalidOperationException($"Could not access RabbitMQ Management API. ({response.StatusCode}: {response.Reason})"); - } + queue = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueue(rabbitBrokerQueue.QueueName, token), cancellationToken); + newReading = new RabbitMQBrokerQueue(queue); - newReading = new RabbitMQBrokerQueue(response.Value); - var newTotalThroughput = queue.CalculateThroughputFrom(newReading); + var newTotalThroughput = rabbitBrokerQueue.CalculateThroughputFrom(newReading); yield return new QueueThroughput { @@ -86,25 +75,20 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa do { - (var statusCode, var reason, var queues, morePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); - - ValidateResponse((statusCode, reason, queues)); + (var queues, morePages) = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetQueues(page, 500, token), cancellationToken); - if (queues is not null) + foreach (var queue in queues) { - foreach (var queue in queues) + if (queue.Name.StartsWith("nsb.delay-level-") || + queue.Name.StartsWith("nsb.v2.delay-level-") || + queue.Name.StartsWith("nsb.v2.verify-")) { - if (queue.Name.StartsWith("nsb.delay-level-") || - queue.Name.StartsWith("nsb.v2.delay-level-") || - queue.Name.StartsWith("nsb.v2.verify-")) - { - continue; - } - - var brokerQueue = new RabbitMQBrokerQueue(queue); - await AddEndpointIndicators(brokerQueue, cancellationToken); - yield return brokerQueue; + continue; } + + var brokerQueue = new RabbitMQBrokerQueue(queue); + await AddEndpointIndicators(brokerQueue, cancellationToken); + yield return brokerQueue; } page++; @@ -113,30 +97,28 @@ public override async IAsyncEnumerable GetQueueNames([EnumeratorCa async Task GetBrokerDetails(CancellationToken cancellationToken) { - var response = await pipeline.ExecuteAsync(async async => await managementClient.Value.GetOverview(cancellationToken), cancellationToken); + var overview = await pipeline.ExecuteAsync(async async => await managementClient.Value.GetOverview(cancellationToken), cancellationToken); - ValidateResponse(response); - - if (response.Value?.DisableStats ?? false) + if (overview.DisableStats) { throw new Exception(disableStatsErrorMessage); } - Data["RabbitMQVersion"] = response.Value?.BrokerVersion ?? "Unknown"; + Data["RabbitMQVersion"] = overview.BrokerVersion ?? "Unknown"; } async Task AddEndpointIndicators(RabbitMQBrokerQueue brokerQueue, CancellationToken cancellationToken) { try { - var response = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetBindingsForQueue(brokerQueue.QueueName, token), cancellationToken); + var bindings = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetBindingsForQueue(brokerQueue.QueueName, token), cancellationToken); // Check if conventional binding is found - if (response.Value.Any(binding => binding?.Source == brokerQueue.QueueName - && binding?.Destination == brokerQueue.QueueName - && binding?.DestinationType == "queue" - && binding?.RoutingKey == string.Empty - && binding?.PropertiesKey == "~")) + if (bindings.Any(binding => binding.Source == brokerQueue.QueueName + && binding.Destination == brokerQueue.QueueName + && binding.DestinationType == "queue" + && binding.RoutingKey == string.Empty + && binding.PropertiesKey == "~")) { brokerQueue.EndpointIndicators.Add("ConventionalTopologyBinding"); } @@ -148,13 +130,13 @@ async Task AddEndpointIndicators(RabbitMQBrokerQueue brokerQueue, CancellationTo try { - var response = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetBindingsForExchange(brokerQueue.QueueName, token), cancellationToken); + var bindings = await pipeline.ExecuteAsync(async token => await managementClient.Value.GetBindingsForExchange(brokerQueue.QueueName, token), cancellationToken); // Check if delayed binding is found - if (response.Value.Any(binding => binding?.Source is "nsb.v2.delay-delivery" or "nsb.delay-delivery" - && binding?.Destination == brokerQueue.QueueName - && binding?.DestinationType == "exchange" - && binding?.RoutingKey == $"#.{brokerQueue.QueueName}")) + if (bindings.Any(binding => binding.Source is "nsb.v2.delay-delivery" or "nsb.delay-delivery" + && binding.Destination == brokerQueue.QueueName + && binding.DestinationType == "exchange" + && binding.RoutingKey == $"#.{brokerQueue.QueueName}")) { brokerQueue.EndpointIndicators.Add("DelayBinding"); } @@ -165,40 +147,20 @@ async Task AddEndpointIndicators(RabbitMQBrokerQueue brokerQueue, CancellationTo } } - void ValidateResponse((HttpStatusCode StatusCode, string Reason, T? Value) response) - { - if (response.StatusCode != HttpStatusCode.OK) - { - throw new HttpRequestException($"Request failed with status code {response.StatusCode}: {response.Reason}"); - } - - if (response.Value is null) - { - throw new InvalidOperationException("Request was successful, but the response body was null when a value was expected"); - } - } - public override KeyDescriptionPair[] Settings => []; protected override async Task<(bool Success, List Errors)> TestConnectionCore(CancellationToken cancellationToken) { try { - var (statusCode, reason, value) = await managementClient.Value.GetOverview(cancellationToken); + var overview = await managementClient.Value.GetOverview(cancellationToken); - if (value is not null) + if (overview.DisableStats) { - if (value.DisableStats) - { - return (false, [disableStatsErrorMessage]); - } - - return (true, []); - } - else - { - return (false, [$"{statusCode}: {reason}"]); + return (false, [disableStatsErrorMessage]); } + + return (true, []); } catch (HttpRequestException ex) { diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs index 36e62780c4..ebf285839c 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQTransportExtensions.cs @@ -25,21 +25,11 @@ public static void ApplySettingsFromConnectionString(this RabbitMQTransport tran transport.ValidateDeliveryLimits = validateDeliveryLimits; } - if (dictionary.TryGetValue("ManagementApiUrl", out var url)) - { - if (dictionary.TryGetValue("ManagementApiUserName", out var userName) && dictionary.TryGetValue("ManagementApiPassword", out var password)) - { - transport.ManagementApiConfiguration = new(url, userName, password); - } - else - { - transport.ManagementApiConfiguration = new(url); - } - } - else if (dictionary.TryGetValue("ManagementApiUserName", out var userName) && dictionary.TryGetValue("ManagementApiPassword", out var password)) - { - transport.ManagementApiConfiguration = new(userName, password); - } + dictionary.TryGetValue("ManagementApiUrl", out var url); + dictionary.TryGetValue("ManagementApiUserName", out var userName); + dictionary.TryGetValue("ManagementApiPassword", out var password); + + transport.ManagementApiConfiguration = ManagementApiConfiguration.Create(url, userName, password); if (dictionary.TryGetValue("DisableRemoteCertificateValidation", out var disableRemoteCertificateValidationString)) { From 126c117350f5e24af73780ab99727a1ac1571dd7 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Wed, 12 Mar 2025 17:43:55 -0400 Subject: [PATCH 56/60] Update to RTM --- src/Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index a46f67be7c..bc50b29325 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -38,7 +38,7 @@ - + From 9ba6316e410bddd7c07642bbeef33b0700ccd153 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Thu, 13 Mar 2025 17:16:30 -0400 Subject: [PATCH 57/60] Ensure connection string settings are migrated during upgrade --- .../ServiceControl/ServiceControlAppConfig.cs | 48 +++++++++++-------- .../Interfaces.cs | 2 +- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs index 3f93c088e4..c3c79743bf 100644 --- a/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs +++ b/src/ServiceControlInstaller.Engine/Configuration/ServiceControl/ServiceControlAppConfig.cs @@ -1,9 +1,11 @@ namespace ServiceControlInstaller.Engine.Configuration.ServiceControl { using System; + using System.Configuration; using System.Data.Common; using System.IO; using Instances; + using NuGet.Versioning; public class ServiceControlAppConfig : AppConfig { @@ -14,9 +16,12 @@ public ServiceControlAppConfig(IServiceControlInstance instance) : base(Path.Com protected override void UpdateSettings() { - Config.ConnectionStrings.ConnectionStrings.Set("NServiceBus/Transport", UpdateConnectionString()); + UpdateConnectionString(); + Config.ConnectionStrings.ConnectionStrings.Set("NServiceBus/Transport", details.ConnectionString); + var settings = Config.AppSettings.Settings; var version = details.Version; + settings.Set(ServiceControlSettings.InstanceName, details.InstanceName, version); settings.Set(ServiceControlSettings.VirtualDirectory, details.VirtualDirectory); settings.Set(ServiceControlSettings.Port, details.Port.ToString()); @@ -73,40 +78,41 @@ public override void SetTransportType(string transportTypeName) settings.Set(ServiceControlSettings.TransportType, transportTypeName, version); } - string UpdateConnectionString() + void UpdateConnectionString() { - var connectionStringBuilder = new DbConnectionStringBuilder { ConnectionString = details.ConnectionString }; - - MigrateLicensingComponentRabbitMqManagementApiSettings(connectionStringBuilder); - - return connectionStringBuilder.ConnectionString; + if (details.TransportPackage.Name.Contains("rabbitmq", StringComparison.OrdinalIgnoreCase)) + { + MigrateLicensingComponentRabbitMqManagementApiSettings(); + } } - void MigrateLicensingComponentRabbitMqManagementApiSettings(DbConnectionStringBuilder connectionStringBuilder) + void MigrateLicensingComponentRabbitMqManagementApiSettings() { - if (!details.TransportPackage.Name.Contains("rabbitmq", StringComparison.OrdinalIgnoreCase)) + if (details.ConnectionString.StartsWith("amqp", StringComparison.OrdinalIgnoreCase)) { return; } - var settings = Config.AppSettings.Settings; + var shouldMigrate = VersionComparer.Version.Compare(details.Version, new SemanticVersion(6, 5, 0)) >= 0; - var legacySetting = settings[ServiceControlSettings.LicensingComponentRabbitMqManagementApiUrl.Name]; - if (legacySetting is not null && !connectionStringBuilder.ContainsKey("ManagementApiUrl")) + if (shouldMigrate) { - connectionStringBuilder.Add("ManagementApiUrl", legacySetting.Value); - } + var connectionStringBuilder = new DbConnectionStringBuilder { ConnectionString = details.ConnectionString }; + var settings = Config.AppSettings.Settings; - legacySetting = settings[ServiceControlSettings.LicensingComponentRabbitMqManagementApiUsername.Name]; - if (legacySetting is not null && !connectionStringBuilder.ContainsKey("ManagementApiUserName")) - { - connectionStringBuilder.Add("ManagementApiUserName", legacySetting.Value); + MigrateSetting(connectionStringBuilder, settings[ServiceControlSettings.LicensingComponentRabbitMqManagementApiUrl.Name], "ManagementApiUrl"); + MigrateSetting(connectionStringBuilder, settings[ServiceControlSettings.LicensingComponentRabbitMqManagementApiUsername.Name], "ManagementApiUserName"); + MigrateSetting(connectionStringBuilder, settings[ServiceControlSettings.LicensingComponentRabbitMqManagementApiPassword.Name], "ManagementApiPassword"); + + details.ConnectionString = connectionStringBuilder.ConnectionString; } - legacySetting = settings[ServiceControlSettings.LicensingComponentRabbitMqManagementApiPassword.Name]; - if (legacySetting is not null && !connectionStringBuilder.ContainsKey("ManagementApiPassword")) + static void MigrateSetting(DbConnectionStringBuilder connectionStringBuilder, KeyValueConfigurationElement setting, string connectionStringSettingName) { - connectionStringBuilder.Add("ManagementApiPassword", legacySetting.Value); + if (setting is not null && !connectionStringBuilder.ContainsKey(connectionStringSettingName)) + { + connectionStringBuilder.Add(connectionStringSettingName, setting.Value); + } } } diff --git a/src/ServiceControlInstaller.Engine/Interfaces.cs b/src/ServiceControlInstaller.Engine/Interfaces.cs index 54a1c8728f..914be2cc43 100644 --- a/src/ServiceControlInstaller.Engine/Interfaces.cs +++ b/src/ServiceControlInstaller.Engine/Interfaces.cs @@ -16,7 +16,7 @@ public interface ILogging public interface ITransportConfig { TransportInfo TransportPackage { get; } - string ConnectionString { get; } + string ConnectionString { get; set; } } public interface IPersistenceConfig From d4b114d648822efbc3d40a42c76c8993f2d2e884 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Thu, 13 Mar 2025 17:31:11 -0400 Subject: [PATCH 58/60] Make customize methods consistent --- .../RabbitMQConventionalRoutingTransportCustomization.cs | 2 +- .../RabbitMQDirectRoutingTransportCustomization.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs index c234d91f36..0a096c5222 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQConventionalRoutingTransportCustomization.cs @@ -34,7 +34,7 @@ ManagementClient Get() protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; - protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } + protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; diff --git a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs index d2a7a775e4..2d497877d0 100644 --- a/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs +++ b/src/ServiceControl.Transports.RabbitMQ/RabbitMQDirectRoutingTransportCustomization.cs @@ -34,7 +34,7 @@ ManagementClient Get() protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; - protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) { } + protected override void CustomizeTransportForAuditEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfiguration endpointConfiguration, RabbitMQTransport transportDefinition, TransportSettings transportSettings) => transport = transportDefinition; From ab97415996a2f0740094d7d6e3913be88e6d48a0 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Thu, 13 Mar 2025 19:31:57 -0400 Subject: [PATCH 59/60] Update the RabbitMQ prereq prompts --- .../Commands/ScmuCommandChecks.cs | 2 +- .../Validation/AcknowledgementValues.cs | 1 + .../Validation/PowerShellCommandChecks.cs | 18 +++++++++++++----- .../Validation/AbstractCommandChecks.cs | 18 ++++++++---------- 4 files changed, 23 insertions(+), 16 deletions(-) diff --git a/src/ServiceControl.Config/Commands/ScmuCommandChecks.cs b/src/ServiceControl.Config/Commands/ScmuCommandChecks.cs index b89291e01c..94a528ca22 100644 --- a/src/ServiceControl.Config/Commands/ScmuCommandChecks.cs +++ b/src/ServiceControl.Config/Commands/ScmuCommandChecks.cs @@ -21,7 +21,7 @@ protected override async Task PromptForRabbitMqCheck(bool isUpgrade) { var title = isUpgrade ? "UPGRADE WARNING" : "INSTALL WARNING"; var beforeWhat = isUpgrade ? "upgrading" : "installing"; - var message = $"ServiceControl version {Constants.CurrentVersion} requires RabbitMQ broker version 3.10.0 or higher. Also, the stream_queue and quorum_queue feature flags must be enabled on the broker. Please confirm your broker meets the minimum requirements before {beforeWhat}."; + var message = $"ServiceControl version {Constants.CurrentVersion} requires:\n• RabbitMQ broker version 3.10.0 or higher\n• The stream_queue and quorum_queue feature flags must be enabled\n• The management plugin must be enabled\n\nPlease confirm your broker meets the minimum requirements before {beforeWhat}."; var question = "Do you want to proceed?"; var yes = "Yes, my RabbitMQ broker meets the minimum requirements"; var no = "No, cancel the install"; diff --git a/src/ServiceControl.Management.PowerShell/Validation/AcknowledgementValues.cs b/src/ServiceControl.Management.PowerShell/Validation/AcknowledgementValues.cs index 4b0b88a9c0..60de4a497d 100644 --- a/src/ServiceControl.Management.PowerShell/Validation/AcknowledgementValues.cs +++ b/src/ServiceControl.Management.PowerShell/Validation/AcknowledgementValues.cs @@ -3,5 +3,6 @@ static class AcknowledgementValues { public const string RabbitMQBrokerVersion310 = "RabbitMQBrokerVersion310"; + public const string RabbitMQManagementApi = "RabbitMQManagementApi"; } } diff --git a/src/ServiceControl.Management.PowerShell/Validation/PowerShellCommandChecks.cs b/src/ServiceControl.Management.PowerShell/Validation/PowerShellCommandChecks.cs index a53cb711a9..18dded24a7 100644 --- a/src/ServiceControl.Management.PowerShell/Validation/PowerShellCommandChecks.cs +++ b/src/ServiceControl.Management.PowerShell/Validation/PowerShellCommandChecks.cs @@ -71,15 +71,23 @@ protected override Task NotifyForMissingSystemPrerequisites(string missingPrereq protected override Task PromptForRabbitMqCheck(bool isUpgrade) { - if (acknowledgements.Any(ack => ack.Equals(AcknowledgementValues.RabbitMQBrokerVersion310, StringComparison.OrdinalIgnoreCase))) + if (!acknowledgements.Any(ack => ack.Equals(AcknowledgementValues.RabbitMQBrokerVersion310, StringComparison.OrdinalIgnoreCase))) { - return Task.FromResult(true); + var terminateMsg = $"ServiceControl version {Constants.CurrentVersion} requires RabbitMQ broker version 3.10.0 or higher. Also, the stream_queue and quorum_queue feature flags must be enabled on the broker. Use -Acknowledgements {AcknowledgementValues.RabbitMQBrokerVersion310} if you are sure your broker meets these requirements."; + + Terminate(terminateMsg, "Install Error", ErrorCategory.InvalidArgument); + return Task.FromResult(false); } - var terminateMsg = $"ServiceControl version {Constants.CurrentVersion} requires RabbitMQ broker version 3.10.0 or higher. Also, the stream_queue and quorum_queue feature flags must be enabled on the broker. Use -Acknowledgements {AcknowledgementValues.RabbitMQBrokerVersion310} if you are sure your broker meets these requirements."; + if (!acknowledgements.Any(ack => ack.Equals(AcknowledgementValues.RabbitMQManagementApi, StringComparison.OrdinalIgnoreCase))) + { + var terminateMsg = $"ServiceControl version {Constants.CurrentVersion} requires that the management plugin be enabled and the management API be accessible. Use -Acknowledgements {AcknowledgementValues.RabbitMQManagementApi} if you are sure your broker meets these requirements."; - Terminate(terminateMsg, "Install Error", ErrorCategory.InvalidArgument); - return Task.FromResult(false); + Terminate(terminateMsg, "Install Error", ErrorCategory.InvalidArgument); + return Task.FromResult(false); + } + + return Task.FromResult(true); } protected override Task PromptToStopRunningInstance(BaseService instance) diff --git a/src/ServiceControlInstaller.Engine/Validation/AbstractCommandChecks.cs b/src/ServiceControlInstaller.Engine/Validation/AbstractCommandChecks.cs index 79bb2edfa7..8400dec2b2 100644 --- a/src/ServiceControlInstaller.Engine/Validation/AbstractCommandChecks.cs +++ b/src/ServiceControlInstaller.Engine/Validation/AbstractCommandChecks.cs @@ -4,6 +4,7 @@ using System.Linq; using System.ServiceProcess; using System.Threading.Tasks; + using NuGet.Versioning; using ServiceControl.LicenseManagement; using ServiceControlInstaller.Engine.Configuration.ServiceControl; using ServiceControlInstaller.Engine.Instances; @@ -46,7 +47,7 @@ public async Task ValidateNewInstance(params IServiceInstance[] instances) .Select(i => i.TransportPackage) .First(t => t is not null); - var continueInstall = await RabbitMqCheckIsOK(transport, false).ConfigureAwait(false); + var continueInstall = await RabbitMqCheckIsOK(transport, Constants.CurrentVersion, false).ConfigureAwait(false); return continueInstall; } @@ -81,12 +82,9 @@ async Task CanEditOrDelete(BaseService instance, bool isDelete) return true; } - async Task RabbitMqCheckIsOK(TransportInfo transport, bool isUpgrade) + async Task RabbitMqCheckIsOK(TransportInfo transport, SemanticVersion instanceVersion, bool isUpgrade) { - if (transport is null) - { - throw new ArgumentNullException(nameof(transport)); - } + ArgumentNullException.ThrowIfNull(transport); if (transport.ZipName != "RabbitMQ") { @@ -94,9 +92,9 @@ async Task RabbitMqCheckIsOK(TransportInfo transport, bool isUpgrade) return true; } - // Only way we DON'T need to warn is if we're updating an instance that's already on a "new" (AvailableInSCMU) Rabbit transport - var needToWarn = !(isUpgrade && transport.AvailableInSCMU); - if (!needToWarn) + var newerThan650 = VersionComparer.Version.Compare(instanceVersion, new SemanticVersion(6, 5, 0)) > 0; + + if (isUpgrade && newerThan650) { return true; } @@ -166,7 +164,7 @@ public async Task CanUpgradeInstance(BaseService instance, bool forceUpgra } } - if (!await RabbitMqCheckIsOK(instance.TransportPackage, isUpgrade: true).ConfigureAwait(false)) + if (!await RabbitMqCheckIsOK(instance.TransportPackage, instance.Version, isUpgrade: true).ConfigureAwait(false)) { return false; } From c1e16221f2d608da81b64caffb8464344f6ed608 Mon Sep 17 00:00:00 2001 From: Brandon Ording Date: Fri, 14 Mar 2025 17:33:15 -0400 Subject: [PATCH 60/60] Update wording of prerequisites --- .../Commands/ScmuCommandChecks.cs | 12 ++++++++++-- .../Validation/PowerShellCommandChecks.cs | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Config/Commands/ScmuCommandChecks.cs b/src/ServiceControl.Config/Commands/ScmuCommandChecks.cs index 94a528ca22..b21e0b37fd 100644 --- a/src/ServiceControl.Config/Commands/ScmuCommandChecks.cs +++ b/src/ServiceControl.Config/Commands/ScmuCommandChecks.cs @@ -2,6 +2,7 @@ { using System; using System.Diagnostics; + using System.Text; using System.Threading.Tasks; using ServiceControl.Config.Framework; using ServiceControlInstaller.Engine; @@ -21,12 +22,19 @@ protected override async Task PromptForRabbitMqCheck(bool isUpgrade) { var title = isUpgrade ? "UPGRADE WARNING" : "INSTALL WARNING"; var beforeWhat = isUpgrade ? "upgrading" : "installing"; - var message = $"ServiceControl version {Constants.CurrentVersion} requires:\n• RabbitMQ broker version 3.10.0 or higher\n• The stream_queue and quorum_queue feature flags must be enabled\n• The management plugin must be enabled\n\nPlease confirm your broker meets the minimum requirements before {beforeWhat}."; var question = "Do you want to proceed?"; var yes = "Yes, my RabbitMQ broker meets the minimum requirements"; var no = "No, cancel the install"; - var continueInstall = await windowManager.ShowYesNoDialog(title, message, question, yes, no); + var message = new StringBuilder(); + message.AppendLine($"ServiceControl version {Constants.CurrentVersion} requires:"); + message.AppendLine("• RabbitMQ broker version 3.10.0 or higher"); + message.AppendLine("• The stream_queue and quorum_queue feature flags must be enabled"); + message.AppendLine($"• The management plugin API must be enabled and accessible. This might require custom settings to be added to the connection string before {beforeWhat}. See the ServiceControl documentation for details."); + message.AppendLine(); + message.AppendLine($"Please confirm your broker meets the minimum requirements before {beforeWhat}."); + + var continueInstall = await windowManager.ShowYesNoDialog(title, message.ToString(), question, yes, no); return continueInstall; } diff --git a/src/ServiceControl.Management.PowerShell/Validation/PowerShellCommandChecks.cs b/src/ServiceControl.Management.PowerShell/Validation/PowerShellCommandChecks.cs index 18dded24a7..9d9ffd27bc 100644 --- a/src/ServiceControl.Management.PowerShell/Validation/PowerShellCommandChecks.cs +++ b/src/ServiceControl.Management.PowerShell/Validation/PowerShellCommandChecks.cs @@ -81,7 +81,7 @@ protected override Task PromptForRabbitMqCheck(bool isUpgrade) if (!acknowledgements.Any(ack => ack.Equals(AcknowledgementValues.RabbitMQManagementApi, StringComparison.OrdinalIgnoreCase))) { - var terminateMsg = $"ServiceControl version {Constants.CurrentVersion} requires that the management plugin be enabled and the management API be accessible. Use -Acknowledgements {AcknowledgementValues.RabbitMQManagementApi} if you are sure your broker meets these requirements."; + var terminateMsg = $"ServiceControl version {Constants.CurrentVersion} requires that the management plugin API must be enabled and accessible. This might require custom settings to be added to the connection string. See the ServiceControl documentation for details. Use -Acknowledgements {AcknowledgementValues.RabbitMQManagementApi} if you are sure your broker meets these requirements."; Terminate(terminateMsg, "Install Error", ErrorCategory.InvalidArgument); return Task.FromResult(false);