diff --git a/src/ServiceControl.AcceptanceTesting/HttpExtensions.cs b/src/ServiceControl.AcceptanceTesting/HttpExtensions.cs index 64c6203a30..7e5ebde17f 100644 --- a/src/ServiceControl.AcceptanceTesting/HttpExtensions.cs +++ b/src/ServiceControl.AcceptanceTesting/HttpExtensions.cs @@ -47,7 +47,7 @@ public static async Task> TryGetMany(this IAcceptanceTestInfras return ManyResult.Empty; } - return ManyResult.New(true, response); + return ManyResult.New(true, response.Where(m => condition(m)).ToList()); } public static async Task Patch(this IAcceptanceTestInfrastructureProvider provider, string url, T payload = null) where T : class diff --git a/src/ServiceControl.MultiInstance.AcceptanceTests/Recoverability/WhenRetrying.cs b/src/ServiceControl.MultiInstance.AcceptanceTests/Recoverability/WhenRetrying.cs new file mode 100644 index 0000000000..cdb7f575db --- /dev/null +++ b/src/ServiceControl.MultiInstance.AcceptanceTests/Recoverability/WhenRetrying.cs @@ -0,0 +1,22 @@ +namespace ServiceControl.MultiInstance.AcceptanceTests.Recoverability; + +using System.Threading.Tasks; +using AcceptanceTesting; +using MessageFailures; +using MessageFailures.Api; +using TestSupport; + +abstract class WhenRetrying : AcceptanceTest +{ + protected Task> GetFailedMessage(string uniqueMessageId, string instance, FailedMessageStatus expectedStatus) + { + if (uniqueMessageId == null) + { + return Task.FromResult(SingleResult.Empty); + } + + return this.TryGet($"/api/errors/{uniqueMessageId}", f => f.Status == expectedStatus, instance); + } + + protected Task> GetAllFailedMessage(string instance, FailedMessageStatus expectedStatus) => this.TryGetMany("/api/errors", f => f.Status == expectedStatus, instance); +} \ No newline at end of file diff --git a/src/ServiceControl.MultiInstance.AcceptanceTests/Recoverability/WhenRetryingSameMessageMultipleTimes.cs b/src/ServiceControl.MultiInstance.AcceptanceTests/Recoverability/WhenRetryingSameMessageMultipleTimes.cs new file mode 100644 index 0000000000..ed2bd76490 --- /dev/null +++ b/src/ServiceControl.MultiInstance.AcceptanceTests/Recoverability/WhenRetryingSameMessageMultipleTimes.cs @@ -0,0 +1,127 @@ +namespace ServiceControl.MultiInstance.AcceptanceTests.Recoverability +{ + using System; + using System.Linq; + using System.Threading.Tasks; + using AcceptanceTesting; + using AcceptanceTesting.EndpointTemplates; + using MessageFailures; + using NServiceBus; + using NServiceBus.AcceptanceTesting; + using NServiceBus.Settings; + using NUnit.Framework; + using ServiceControl.Infrastructure; + using TestSupport; + + class WhenRetryingSameMessageMultipleTimes : WhenRetrying + { + public enum RetryType + { + NoEdit, + Edit + } + + [TestCase(new[] { RetryType.NoEdit, RetryType.NoEdit, RetryType.Edit })] + [TestCase(new[] { RetryType.Edit, RetryType.NoEdit, RetryType.Edit })] + [TestCase(new[] { RetryType.NoEdit, RetryType.Edit, RetryType.NoEdit })] + [TestCase(new[] { RetryType.Edit, RetryType.Edit, RetryType.NoEdit })] + public async Task WithMixOfRetryTypes(RetryType[] retryTypes) + { + CustomServiceControlPrimarySettings = s => { s.AllowMessageEditing = true; }; + + await Define() + .WithEndpoint(b => + b.When(bus => bus.SendLocal(new MyMessage())).DoNotFailOnErrorMessages()) + .Done(async c => + { + if (c.RetryCount >= retryTypes.Length) // Are all retries done? + { + return !(await GetAllFailedMessage(ServiceControlInstanceName, FailedMessageStatus.Unresolved)) + .HasResult; // Should return true if all failed messages are no longer unresolved + } + + if (retryTypes[c.RetryCount] == RetryType.Edit) + { + var results = await GetAllFailedMessage(ServiceControlInstanceName, + FailedMessageStatus.Unresolved); + if (!results.HasResult) + { + return false; // No failed messages yet + } + + var result = results.Items.Single(); + + c.MessageId = result.MessageId; + } + + var failedMessage = await GetFailedMessage(c.UniqueMessageId, ServiceControlInstanceName, FailedMessageStatus.Unresolved); + if (!failedMessage.HasResult) + { + return false; // No failed message yet + } + + if (retryTypes[c.RetryCount] == RetryType.Edit) + { + await this.Post($"/api/edit/{failedMessage.Item.UniqueMessageId}", + new + { + MessageBody = $"{{ \"Name\": \"Hello {c.RetryCount}\" }}", + MessageHeaders = failedMessage.Item.ProcessingAttempts[^1].Headers + }, null, + ServiceControlInstanceName); + } + else + { + await this.Post($"/api/errors/{failedMessage.Item.UniqueMessageId}/retry", null, null, + ServiceControlInstanceName); + } + + c.RetryCount++; + + return false; + + }) + .Run(TimeSpan.FromMinutes(2)); + } + + class FailureEndpoint : EndpointConfigurationBuilder + { + public FailureEndpoint() => EndpointSetup(c => { c.NoRetries(); }); + + public class MyMessageHandler(MyContext testContext, IReadOnlySettings settings) + : IHandleMessages + { + public Task Handle(MyMessage message, IMessageHandlerContext context) + { + testContext.MessageId = context.MessageId.Replace(@"\", "-"); + testContext.EndpointNameOfReceivingEndpoint = settings.EndpointName(); + + if (testContext.RetryCount < 3) + { + Console.Out.WriteLine("Throwing exception"); + throw new Exception("Simulated exception"); + } + + Console.Out.WriteLine("Handling message"); + + return Task.CompletedTask; + } + } + } + + class MyMessage : ICommand + { + public string Name { get; set; } + } + + class MyContext : ScenarioContext + { + public string MessageId { get; set; } + + public string EndpointNameOfReceivingEndpoint { get; set; } + + public string UniqueMessageId => DeterministicGuid.MakeId(MessageId, EndpointNameOfReceivingEndpoint).ToString(); + public int RetryCount { get; set; } + } + } +} \ No newline at end of file diff --git a/src/ServiceControl.MultiInstance.AcceptanceTests/Recoverability/WhenRetryingWithEdit.cs b/src/ServiceControl.MultiInstance.AcceptanceTests/Recoverability/WhenRetryingWithEdit.cs new file mode 100644 index 0000000000..f824abff29 --- /dev/null +++ b/src/ServiceControl.MultiInstance.AcceptanceTests/Recoverability/WhenRetryingWithEdit.cs @@ -0,0 +1,92 @@ +namespace ServiceControl.MultiInstance.AcceptanceTests.Recoverability; + +using System; +using System.Threading.Tasks; +using AcceptanceTesting; +using AcceptanceTesting.EndpointTemplates; +using MessageFailures; +using NServiceBus; +using NServiceBus.AcceptanceTesting; +using NServiceBus.Settings; +using NUnit.Framework; +using TestSupport; +using ServiceControl.Infrastructure; + +class WhenRetryingWithEdit : WhenRetrying +{ + [Test] + public async Task ShouldCreateNewMessageAndResolveEditedMessage() + { + CustomServiceControlPrimarySettings = s => { s.AllowMessageEditing = true; }; + + await Define() + .WithEndpoint(b => + b.When(bus => bus.SendLocal(new MyMessage { Password = "Bad password!" })).DoNotFailOnErrorMessages()) + .Done(async c => + { + if (!c.ErrorRetried) + { + var failedMessage = await GetFailedMessage(c.UniqueMessageId, ServiceControlInstanceName, + FailedMessageStatus.Unresolved); + if (!failedMessage.HasResult) + { + return false; // No failed message yet + } + + await this.Post($"/api/edit/{failedMessage.Item.UniqueMessageId}", + new + { + MessageBody = "{ \"Password\": \"VerySecretPassword\" }", + MessageHeaders = failedMessage.Item.ProcessingAttempts[^1].Headers + }, null, + ServiceControlInstanceName); + c.ErrorRetried = true; + + return false; + } + + var failedResolvedMessage = await GetFailedMessage(c.UniqueMessageId, ServiceControlInstanceName, FailedMessageStatus.Resolved); + + return failedResolvedMessage.HasResult; // If there is a result it means the message was resolved + }) + .Run(TimeSpan.FromMinutes(2)); + } + + class FailureEndpoint : EndpointConfigurationBuilder + { + public FailureEndpoint() => EndpointSetup(c => { c.NoRetries(); }); + + public class MyMessageHandler(MyContext testContext, IReadOnlySettings settings) : IHandleMessages + { + public Task Handle(MyMessage message, IMessageHandlerContext context) + { + if (message.Password == "VerySecretPassword") + { + Console.Out.WriteLine("Handling message"); + return Task.CompletedTask; + } + + testContext.MessageId = context.MessageId.Replace(@"\", "-"); + testContext.EndpointNameOfReceivingEndpoint = settings.EndpointName(); + + Console.Out.WriteLine("Throwing exception"); + throw new Exception("Simulated exception"); + } + } + } + + class MyMessage : ICommand + { + public string Password { get; set; } + } + + class MyContext : ScenarioContext + { + public string MessageId { get; set; } + + public string EndpointNameOfReceivingEndpoint { get; set; } + + public string UniqueMessageId => DeterministicGuid.MakeId(MessageId, EndpointNameOfReceivingEndpoint).ToString(); + public bool ErrorRetried { get; set; } + } +} \ No newline at end of file diff --git a/src/ServiceControl/Recoverability/Editing/EditHandler.cs b/src/ServiceControl/Recoverability/Editing/EditHandler.cs index 2a49ada79b..eb263bd3be 100644 --- a/src/ServiceControl/Recoverability/Editing/EditHandler.cs +++ b/src/ServiceControl/Recoverability/Editing/EditHandler.cs @@ -67,7 +67,7 @@ public async Task Handle(EditAndSend message, IMessageHandlerContext context) var outgoingMessage = BuildMessage(message); // mark the new message with a link to the original message id outgoingMessage.Headers.Add("ServiceControl.EditOf", message.FailedMessageId); - outgoingMessage.Headers["ServiceControl.Retry.AcknowledgementQueue"] = ""; + outgoingMessage.Headers.Remove("ServiceControl.Retry.AcknowledgementQueue"); var address = ApplyRedirect(attempt.FailureDetails.AddressOfFailingEndpoint, redirects); if (outgoingMessage.Headers.TryGetValue("ServiceControl.RetryTo", out var retryTo))