From e320ca4b16b038bae53d8629e2bb4b81c2a958b5 Mon Sep 17 00:00:00 2001 From: SzymonPobiega Date: Tue, 18 Feb 2025 12:26:25 +0100 Subject: [PATCH 1/6] Upgrades SC to the new Azure Topology: - By default uses migration topology with budnle-1 topic - Can override the single topic via connection string - Can provide custom topology options via ServiceControl.Transport.ASBS/Topic setting --- src/Directory.Packages.props | 4 +- .../ASBSTransportCustomization.cs | 38 ++++++++++++++++--- .../AuthenticationMethod.cs | 2 +- src/ServiceControl.Transports.ASBS/Helper.cs | 32 ---------------- .../ServiceControl.Transports.ASBS.csproj | 3 +- .../SharedAccessSignatureAuthentication.cs | 4 +- .../TokenCredentialAuthentication.cs | 5 +-- .../ServiceControl.Transports.csproj | 5 +++ 8 files changed, 46 insertions(+), 47 deletions(-) delete mode 100644 src/ServiceControl.Transports.ASBS/Helper.cs diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index bb6efbd1e1..e2e5b24a78 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -28,7 +28,7 @@ - + @@ -41,7 +41,7 @@ - + diff --git a/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs b/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs index 7b84f142f5..0bb96a4a88 100644 --- a/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs +++ b/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs @@ -1,12 +1,17 @@ namespace ServiceControl.Transports.ASBS { using System.Linq; + using System.Text.Json; using BrokerThroughput; + using Configuration; using Microsoft.Extensions.DependencyInjection; using NServiceBus; + using NServiceBus.Transport.AzureServiceBus; public class ASBSTransportCustomization : TransportCustomization { + const string DefaultSingleTopic = "bundle-1"; + protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, AzureServiceBusTransport transportDefinition, TransportSettings transportSettings) => transportDefinition.TransportTransactionMode = TransportTransactionMode.SendsAtomicWithReceive; @@ -19,18 +24,39 @@ protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfigur protected override AzureServiceBusTransport CreateTransport(TransportSettings transportSettings, TransportTransactionMode preferredTransactionMode = TransportTransactionMode.ReceiveOnly) { var connectionSettings = ConnectionStringParser.Parse(transportSettings.ConnectionString); - var transport = connectionSettings.AuthenticationMethod.CreateTransportDefinition(connectionSettings); - transport.UseWebSockets = connectionSettings.UseWebSockets; + TopologyOptions topologyOptions; - if (connectionSettings.TopicName != null) + var serviceBusRootNamespace = new SettingsRootNamespace("ServiceControl.Transport.ASBS"); + if (SettingsReader.TryRead(serviceBusRootNamespace, "Topology", out var topologyJson)) + { + topologyOptions = JsonSerializer.Deserialize(topologyJson); + } + else { - transport.Topology = TopicTopology.Single(connectionSettings.TopicName); + var options = new MigrationTopologyOptions + { + TopicToPublishTo = connectionSettings.TopicName ?? DefaultSingleTopic, + TopicToSubscribeOn = connectionSettings.TopicName ?? DefaultSingleTopic, + PublishedEventToTopicsMap = + { + ["ServiceControl.Contracts.CustomCheckFailed"] = "ServiceControl.Contracts.CustomCheckFailed", + ["ServiceControl.Contracts.CustomCheckSucceeded"] = "ServiceControl.Contracts.CustomCheckSucceeded", + ["ServiceControl.Contracts.HeartbeatRestored"] = "ServiceControl.Contracts.HeartbeatRestored", + ["ServiceControl.Contracts.HeartbeatStopped"] = "ServiceControl.Contracts.HeartbeatStopped", + ["ServiceControl.Contracts.FailedMessagesArchived"] = "ServiceControl.Contracts.FailedMessagesArchived", + ["ServiceControl.Contracts.FailedMessagesUnArchived"] = "ServiceControl.Contracts.FailedMessagesUnArchived", + ["ServiceControl.Contracts.MessageFailed"] = "ServiceControl.Contracts.MessageFailed", + ["ServiceControl.Contracts.MessageFailureResolvedByRetry"] = "ServiceControl.Contracts.MessageFailureResolvedByRetry", + ["ServiceControl.Contracts.MessageFailureResolvedManually"] = "ServiceControl.Contracts.MessageFailureResolvedManually" + } + }; + topologyOptions = options; } + var transport = connectionSettings.AuthenticationMethod.CreateTransportDefinition(connectionSettings, TopicTopology.FromOptions(topologyOptions)); + transport.UseWebSockets = connectionSettings.UseWebSockets; transport.EnablePartitioning = connectionSettings.EnablePartitioning; - transport.ConfigureNameShorteners(); - transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; return transport; diff --git a/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs b/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs index a9f61fc1e3..966b2f3c3d 100644 --- a/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs +++ b/src/ServiceControl.Transports.ASBS/AuthenticationMethod.cs @@ -6,6 +6,6 @@ public abstract class AuthenticationMethod { public abstract ServiceBusAdministrationClient BuildManagementClient(); - public abstract AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings); + public abstract AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings, TopicTopology topology); } } \ No newline at end of file diff --git a/src/ServiceControl.Transports.ASBS/Helper.cs b/src/ServiceControl.Transports.ASBS/Helper.cs deleted file mode 100644 index 8d73edc2bf..0000000000 --- a/src/ServiceControl.Transports.ASBS/Helper.cs +++ /dev/null @@ -1,32 +0,0 @@ -namespace ServiceControl.Transports.ASBS -{ - using System; - using System.Security.Cryptography; - using System.Text; - using NServiceBus; - - public static class Helper - { - public static void ConfigureNameShorteners(this AzureServiceBusTransport transport) - { - transport.SubscriptionNamingConvention = n => n.Length > MaxEntityName ? MD5DeterministicNameBuilder.Build(n) : n; - transport.SubscriptionRuleNamingConvention = n => n.FullName.Length > MaxEntityName ? MD5DeterministicNameBuilder.Build(n.FullName) : n.FullName; - } - - const int MaxEntityName = 50; - - static class MD5DeterministicNameBuilder - { - public static string Build(string input) - { - var inputBytes = Encoding.Default.GetBytes(input); - - // use MD5 hash to get a 16-byte hash of the string - var hashBytes = MD5.HashData(inputBytes); - - // generate a guid from the hash: - return new Guid(hashBytes).ToString(); - } - } - } -} \ No newline at end of file diff --git a/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj b/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj index 540231446d..6f560c0344 100644 --- a/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj +++ b/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj @@ -3,6 +3,7 @@ net8.0 true + CS0618 @@ -15,7 +16,7 @@ - + diff --git a/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs b/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs index 4539f883d0..5368353371 100644 --- a/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs +++ b/src/ServiceControl.Transports.ASBS/SharedAccessSignatureAuthentication.cs @@ -12,7 +12,7 @@ public class SharedAccessSignatureAuthentication : AuthenticationMethod public override ServiceBusAdministrationClient BuildManagementClient() => new ServiceBusAdministrationClient(ConnectionString); - public override AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings) - => new AzureServiceBusTransport(ConnectionString); + public override AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings, TopicTopology topology) + => new AzureServiceBusTransport(ConnectionString, topology); } } \ No newline at end of file diff --git a/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs b/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs index 4bbf898096..a11f74463f 100644 --- a/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs +++ b/src/ServiceControl.Transports.ASBS/TokenCredentialAuthentication.cs @@ -4,7 +4,6 @@ using Azure.Identity; using Azure.Messaging.ServiceBus.Administration; using NServiceBus; - using NServiceBus.Transport; public class TokenCredentialAuthentication : AuthenticationMethod { @@ -30,9 +29,9 @@ public TokenCredentialAuthentication(string fullyQualifiedNamespace, string clie public override ServiceBusAdministrationClient BuildManagementClient() => new ServiceBusAdministrationClient(FullyQualifiedNamespace, Credential); - public override AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings) + public override AzureServiceBusTransport CreateTransportDefinition(ConnectionSettings connectionSettings, TopicTopology topology) { - var transport = new AzureServiceBusTransport(FullyQualifiedNamespace, Credential); + var transport = new AzureServiceBusTransport(FullyQualifiedNamespace, Credential, topology); return transport; } } diff --git a/src/ServiceControl.Transports/ServiceControl.Transports.csproj b/src/ServiceControl.Transports/ServiceControl.Transports.csproj index 9d2526b3e9..6bd8eeea61 100644 --- a/src/ServiceControl.Transports/ServiceControl.Transports.csproj +++ b/src/ServiceControl.Transports/ServiceControl.Transports.csproj @@ -4,6 +4,11 @@ net8.0 + + + + + From 6d744506babf9c70e0c2f71efdadd65ac4bd1618 Mon Sep 17 00:00:00 2001 From: SzymonPobiega Date: Tue, 18 Feb 2025 20:59:23 +0100 Subject: [PATCH 2/6] Use topic-per-event by default and revert to single-topic only when topic explicitly specified --- .../ASBSTransportCustomization.cs | 43 +++++++++++-------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs b/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs index 0bb96a4a88..35a852983f 100644 --- a/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs +++ b/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs @@ -24,36 +24,41 @@ protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfigur protected override AzureServiceBusTransport CreateTransport(TransportSettings transportSettings, TransportTransactionMode preferredTransactionMode = TransportTransactionMode.ReceiveOnly) { var connectionSettings = ConnectionStringParser.Parse(transportSettings.ConnectionString); - TopologyOptions topologyOptions; + TopicTopology selectedTopology; var serviceBusRootNamespace = new SettingsRootNamespace("ServiceControl.Transport.ASBS"); if (SettingsReader.TryRead(serviceBusRootNamespace, "Topology", out var topologyJson)) { - topologyOptions = JsonSerializer.Deserialize(topologyJson); + //Load topology from json + selectedTopology = TopicTopology.FromOptions(JsonSerializer.Deserialize(topologyJson)); } - else + else if (connectionSettings.TopicName != null) { - var options = new MigrationTopologyOptions + //Bundle name provided -> use migration topology + selectedTopology = TopicTopology.FromOptions(new MigrationTopologyOptions { TopicToPublishTo = connectionSettings.TopicName ?? DefaultSingleTopic, TopicToSubscribeOn = connectionSettings.TopicName ?? DefaultSingleTopic, - PublishedEventToTopicsMap = - { - ["ServiceControl.Contracts.CustomCheckFailed"] = "ServiceControl.Contracts.CustomCheckFailed", - ["ServiceControl.Contracts.CustomCheckSucceeded"] = "ServiceControl.Contracts.CustomCheckSucceeded", - ["ServiceControl.Contracts.HeartbeatRestored"] = "ServiceControl.Contracts.HeartbeatRestored", - ["ServiceControl.Contracts.HeartbeatStopped"] = "ServiceControl.Contracts.HeartbeatStopped", - ["ServiceControl.Contracts.FailedMessagesArchived"] = "ServiceControl.Contracts.FailedMessagesArchived", - ["ServiceControl.Contracts.FailedMessagesUnArchived"] = "ServiceControl.Contracts.FailedMessagesUnArchived", - ["ServiceControl.Contracts.MessageFailed"] = "ServiceControl.Contracts.MessageFailed", - ["ServiceControl.Contracts.MessageFailureResolvedByRetry"] = "ServiceControl.Contracts.MessageFailureResolvedByRetry", - ["ServiceControl.Contracts.MessageFailureResolvedManually"] = "ServiceControl.Contracts.MessageFailureResolvedManually" - } - }; - topologyOptions = options; + EventsToMigrateMap = [ + "ServiceControl.Contracts.CustomCheckFailed", + "ServiceControl.Contracts.CustomCheckSucceeded", + "ServiceControl.Contracts.HeartbeatRestored", + "ServiceControl.Contracts.HeartbeatStopped", + "ServiceControl.Contracts.FailedMessagesArchived", + "ServiceControl.Contracts.FailedMessagesUnArchived", + "ServiceControl.Contracts.MessageFailed", + "ServiceControl.Contracts.MessageFailureResolvedByRetry", + "ServiceControl.Contracts.MessageFailureResolvedManually" + ] + }); + } + else + { + //Default to topic-per-event topology + selectedTopology = TopicTopology.Default; } - var transport = connectionSettings.AuthenticationMethod.CreateTransportDefinition(connectionSettings, TopicTopology.FromOptions(topologyOptions)); + var transport = connectionSettings.AuthenticationMethod.CreateTransportDefinition(connectionSettings, selectedTopology); transport.UseWebSockets = connectionSettings.UseWebSockets; transport.EnablePartitioning = connectionSettings.EnablePartitioning; From 85673825286c62e38ba1df0d5261a3b70103e85f Mon Sep 17 00:00:00 2001 From: SzymonPobiega Date: Tue, 18 Feb 2025 20:59:33 +0100 Subject: [PATCH 3/6] Add missing reference --- .../ServiceControl.Transports.ASBS.Tests.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ServiceControl.Transports.ASBS.Tests/ServiceControl.Transports.ASBS.Tests.csproj b/src/ServiceControl.Transports.ASBS.Tests/ServiceControl.Transports.ASBS.Tests.csproj index 58b234a7e1..eb5e30a4ce 100644 --- a/src/ServiceControl.Transports.ASBS.Tests/ServiceControl.Transports.ASBS.Tests.csproj +++ b/src/ServiceControl.Transports.ASBS.Tests/ServiceControl.Transports.ASBS.Tests.csproj @@ -5,6 +5,7 @@ + From 2766d6f18dde38dc7a9d19ba941e1b2eb6e7ba48 Mon Sep 17 00:00:00 2001 From: SzymonPobiega Date: Wed, 19 Feb 2025 10:24:32 +0100 Subject: [PATCH 4/6] - Only add topic mappings if running a primary instance - Connection string has precedence over custom config --- .../ASBSTransportCustomization.cs | 63 ++++++++++++------- 1 file changed, 40 insertions(+), 23 deletions(-) diff --git a/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs b/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs index 35a852983f..79073c3393 100644 --- a/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs +++ b/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs @@ -10,8 +10,6 @@ public class ASBSTransportCustomization : TransportCustomization { - const string DefaultSingleTopic = "bundle-1"; - protected override void CustomizeTransportForPrimaryEndpoint(EndpointConfiguration endpointConfiguration, AzureServiceBusTransport transportDefinition, TransportSettings transportSettings) => transportDefinition.TransportTransactionMode = TransportTransactionMode.SendsAtomicWithReceive; @@ -24,21 +22,47 @@ protected override void CustomizeTransportForMonitoringEndpoint(EndpointConfigur protected override AzureServiceBusTransport CreateTransport(TransportSettings transportSettings, TransportTransactionMode preferredTransactionMode = TransportTransactionMode.ReceiveOnly) { var connectionSettings = ConnectionStringParser.Parse(transportSettings.ConnectionString); - TopicTopology selectedTopology; - var serviceBusRootNamespace = new SettingsRootNamespace("ServiceControl.Transport.ASBS"); - if (SettingsReader.TryRead(serviceBusRootNamespace, "Topology", out var topologyJson)) + if (!transportSettings.TryGet(out TopicTopology selectedTopology)) { - //Load topology from json - selectedTopology = TopicTopology.FromOptions(JsonSerializer.Deserialize(topologyJson)); + //Topology is pre-selected and customized only when creating transport for the primary instance + //For all other cases use the connection string to determine which topology to use + if (connectionSettings.TopicName != null) + { + selectedTopology = TopicTopology.MigrateFromNamedSingleTopic(connectionSettings.TopicName); + } + else + { + selectedTopology = TopicTopology.Default; + } } - else if (connectionSettings.TopicName != null) + + var transport = connectionSettings.AuthenticationMethod.CreateTransportDefinition(connectionSettings, selectedTopology); + transport.UseWebSockets = connectionSettings.UseWebSockets; + transport.EnablePartitioning = connectionSettings.EnablePartitioning; + + transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; + + return transport; + } + + protected override void AddTransportForPrimaryCore(IServiceCollection services, + TransportSettings transportSettings) + { + services.AddSingleton(); + + var connectionSettings = ConnectionStringParser.Parse(transportSettings.ConnectionString); + TopicTopology selectedTopology; + + var serviceBusRootNamespace = new SettingsRootNamespace("ServiceControl.Transport.ASBS"); + if (connectionSettings.TopicName != null) { //Bundle name provided -> use migration topology + //Need to explicitly specific events to be published on the single topic selectedTopology = TopicTopology.FromOptions(new MigrationTopologyOptions { - TopicToPublishTo = connectionSettings.TopicName ?? DefaultSingleTopic, - TopicToSubscribeOn = connectionSettings.TopicName ?? DefaultSingleTopic, + TopicToPublishTo = connectionSettings.TopicName, + TopicToSubscribeOn = connectionSettings.TopicName, EventsToMigrateMap = [ "ServiceControl.Contracts.CustomCheckFailed", "ServiceControl.Contracts.CustomCheckSucceeded", @@ -52,25 +76,18 @@ protected override AzureServiceBusTransport CreateTransport(TransportSettings tr ] }); } + else if (SettingsReader.TryRead(serviceBusRootNamespace, "Topology", out var topologyJson)) + { + //Load topology from json + selectedTopology = TopicTopology.FromOptions(JsonSerializer.Deserialize(topologyJson, TopologyOptionsSerializationContext.Default.TopologyOptions)); + } else { //Default to topic-per-event topology selectedTopology = TopicTopology.Default; } - var transport = connectionSettings.AuthenticationMethod.CreateTransportDefinition(connectionSettings, selectedTopology); - transport.UseWebSockets = connectionSettings.UseWebSockets; - transport.EnablePartitioning = connectionSettings.EnablePartitioning; - - transport.TransportTransactionMode = transport.GetSupportedTransactionModes().Contains(preferredTransactionMode) ? preferredTransactionMode : TransportTransactionMode.ReceiveOnly; - - return transport; - } - - protected override void AddTransportForPrimaryCore(IServiceCollection services, - TransportSettings transportSettings) - { - services.AddSingleton(); + transportSettings.Set(selectedTopology); } protected override void AddTransportForMonitoringCore(IServiceCollection services, TransportSettings transportSettings) From 6ea92a31a4c09de5140fa6163c056a88803f48f8 Mon Sep 17 00:00:00 2001 From: SzymonPobiega Date: Wed, 19 Feb 2025 10:57:33 +0100 Subject: [PATCH 5/6] Use warning suppressions in code --- .../ASBSTransportCustomization.cs | 4 ++++ .../ServiceControl.Transports.ASBS.csproj | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs b/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs index 79073c3393..5a600a385f 100644 --- a/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs +++ b/src/ServiceControl.Transports.ASBS/ASBSTransportCustomization.cs @@ -29,7 +29,9 @@ protected override AzureServiceBusTransport CreateTransport(TransportSettings tr //For all other cases use the connection string to determine which topology to use if (connectionSettings.TopicName != null) { +#pragma warning disable CS0618 // Type or member is obsolete selectedTopology = TopicTopology.MigrateFromNamedSingleTopic(connectionSettings.TopicName); +#pragma warning restore CS0618 // Type or member is obsolete } else { @@ -59,7 +61,9 @@ protected override void AddTransportForPrimaryCore(IServiceCollection services, { //Bundle name provided -> use migration topology //Need to explicitly specific events to be published on the single topic +#pragma warning disable CS0618 // Type or member is obsolete selectedTopology = TopicTopology.FromOptions(new MigrationTopologyOptions +#pragma warning restore CS0618 // Type or member is obsolete { TopicToPublishTo = connectionSettings.TopicName, TopicToSubscribeOn = connectionSettings.TopicName, diff --git a/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj b/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj index 6f560c0344..540231446d 100644 --- a/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj +++ b/src/ServiceControl.Transports.ASBS/ServiceControl.Transports.ASBS.csproj @@ -3,7 +3,6 @@ net8.0 true - CS0618 @@ -16,7 +15,7 @@ - + From 5995f4e8e5c69d757783937b6b6bc018916bfbd7 Mon Sep 17 00:00:00 2001 From: SzymonPobiega Date: Wed, 26 Feb 2025 11:11:53 +0100 Subject: [PATCH 6/6] Update to release 5 --- src/Directory.Packages.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index e2e5b24a78..9494512cec 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -41,7 +41,7 @@ - +