From ab91a772688193dad616983b6e192c4a1f49b10b Mon Sep 17 00:00:00 2001 From: John Simons Date: Mon, 31 Mar 2025 16:00:31 +1000 Subject: [PATCH 1/4] Adding time sent range api Also fixed a bug where the results could return more than the page size --- .../InMemoryAuditDataStore.cs | 8 +-- .../Extensions/RavenQueryExtensions.cs | 40 +++++++++++ .../RavenAuditDataStore.cs | 12 ++-- .../RetentionTests.cs | 4 +- .../AuditTests.cs | 12 ++-- .../IAuditDataStore.cs | 8 +-- .../MessagesView/GetMessages2Controller.cs | 56 ++++++++++++++++ .../MessagesView/GetMessagesController.cs | 12 ++-- .../ErrorMessagesDataStore.cs | 16 +++-- .../RavenQueryExtensions.cs | 34 ++++++++++ .../ReturnToSenderDequeuerTests.cs | 12 ++-- .../IErrorMessageDatastore.cs | 9 ++- .../Messages/GetAllMessagesApi.cs | 2 +- .../Messages/GetAllMessagesForEndpointApi.cs | 7 +- .../Messages/GetMessages2Controller.cs | 66 +++++++++++++++++++ .../Messages/GetMessagesController.cs | 2 +- .../Messages/ScatterGatherApiMessageView.cs | 7 +- .../CompositeViews/Messages/SearchApi.cs | 7 +- .../Messages/SearchEndpointApi.cs | 7 +- 19 files changed, 267 insertions(+), 54 deletions(-) create mode 100644 src/ServiceControl.Audit/Auditing/MessagesView/GetMessages2Controller.cs create mode 100644 src/ServiceControl/CompositeViews/Messages/GetMessages2Controller.cs diff --git a/src/ServiceControl.Audit.Persistence.InMemory/InMemoryAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.InMemory/InMemoryAuditDataStore.cs index 622cbb5437..9436d3caf2 100644 --- a/src/ServiceControl.Audit.Persistence.InMemory/InMemoryAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.InMemory/InMemoryAuditDataStore.cs @@ -42,7 +42,7 @@ public async Task> QuerySagaHistoryById(Guid input, Can return await Task.FromResult(new QueryResult(sagaHistory, new QueryStatsInfo(string.Empty, 1))); } - public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) { var matched = messageViews .Where(w => !w.IsSystemMessage || includeSystemMessages) @@ -51,7 +51,7 @@ public async Task>> GetMessages(bool includeSyst return await Task.FromResult(new QueryResult>(matched, new QueryStatsInfo(string.Empty, matched.Count))); } - public async Task>> QueryMessages(string keyword, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + public async Task>> QueryMessages(string keyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) { var messages = GetMessageIdsMatchingQuery(keyword); @@ -61,7 +61,7 @@ public async Task>> QueryMessages(string keyword return await Task.FromResult(new QueryResult>(matched, new QueryStatsInfo(string.Empty, matched.Count()))); } - public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) { var messages = GetMessageIdsMatchingQuery(keyword); @@ -69,7 +69,7 @@ public async Task>> QueryMessagesByReceivingEndp return await Task.FromResult(new QueryResult>(matched, new QueryStatsInfo(string.Empty, matched.Count))); } - public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) { var matched = messageViews.Where(w => w.ReceivingEndpoint.Name == endpointName).ToList(); return await Task.FromResult(new QueryResult>(matched, new QueryStatsInfo(string.Empty, matched.Count))); diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/Extensions/RavenQueryExtensions.cs b/src/ServiceControl.Audit.Persistence.RavenDB/Extensions/RavenQueryExtensions.cs index 40e5da425a..071d6c1fee 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/Extensions/RavenQueryExtensions.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/Extensions/RavenQueryExtensions.cs @@ -1,6 +1,7 @@ namespace ServiceControl.Audit.Persistence.RavenDB.Extensions { using System; + using System.Globalization; using System.Linq; using System.Linq.Expressions; using Indexes; @@ -19,6 +20,45 @@ static class RavenQueryExtensions return source; } + public static IQueryable FilterBySentTimeRange(this IQueryable source, string range) + { + if (string.IsNullOrWhiteSpace(range)) + { + return source; + } + + var filters = range.Split(SplitChars, StringSplitOptions.None); + DateTime from, to; + try + { + + if (filters.Length == 2) + { + from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + to = DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + source.Where(m => m.TimeSent >= from && m.TimeSent <= to); + + } + else + { + from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + source.Where(m => m.TimeSent >= from); + } + } + catch (Exception) + { + throw new Exception( + "Invalid sent time date range, dates need to be in ISO8601 format and it needs to be a range eg. 2016-03-11T00:27:15.474Z...2016-03-16T03:27:15.474Z"); + } + + return source; + } + + static string[] SplitChars = + { + "..." + }; + public static IQueryable Paging(this IQueryable source, PagingInfo pagingInfo) => source.Skip(pagingInfo.Offset).Take(pagingInfo.PageSize); diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs index 9f6a120613..e454c321bd 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs @@ -31,11 +31,12 @@ public async Task> QuerySagaHistoryById(Guid input, Can return sagaHistory == null ? QueryResult.Empty() : new QueryResult(sagaHistory, new QueryStatsInfo($"{stats.ResultEtag}", stats.TotalResults)); } - public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) .Statistics(out var stats) + .FilterBySentTimeRange(timeSentRange) .IncludeSystemMessagesWhere(includeSystemMessages) .Sort(sortInfo) .Paging(pagingInfo) @@ -45,12 +46,13 @@ public async Task>> GetMessages(bool includeSyst return new QueryResult>(results, stats.ToQueryStatsInfo()); } - public async Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + public async Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) .Statistics(out var stats) .Search(x => x.Query, searchParam) + .FilterBySentTimeRange(timeSentRange) .Sort(sortInfo) .Paging(pagingInfo) .ToMessagesView() @@ -59,13 +61,14 @@ public async Task>> QueryMessages(string searchP return new QueryResult>(results, stats.ToQueryStatsInfo()); } - public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) .Statistics(out var stats) .Search(x => x.Query, keyword) .Where(m => m.ReceivingEndpointName == endpoint) + .FilterBySentTimeRange(timeSentRange) .Sort(sortInfo) .Paging(pagingInfo) .ToMessagesView() @@ -74,13 +77,14 @@ public async Task>> QueryMessagesByReceivingEndp return new QueryResult>(results, stats.ToQueryStatsInfo()); } - public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken) + public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) .Statistics(out var stats) .IncludeSystemMessagesWhere(includeSystemMessages) .Where(m => m.ReceivingEndpointName == endpointName) + .FilterBySentTimeRange(timeSentRange) .Sort(sortInfo) .Paging(pagingInfo) .ToMessagesView() diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RetentionTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RetentionTests.cs index 29c6938bd9..2377993e02 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RetentionTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RetentionTests.cs @@ -32,11 +32,11 @@ await IngestProcessedMessagesAudits( message ); - var queryResultBeforeExpiration = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), TestContext.CurrentContext.CancellationToken); + var queryResultBeforeExpiration = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), null, TestContext.CurrentContext.CancellationToken); await Task.Delay(4000); - var queryResultAfterExpiration = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), TestContext.CurrentContext.CancellationToken); + var queryResultAfterExpiration = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), null, TestContext.CurrentContext.CancellationToken); Assert.That(queryResultBeforeExpiration.Results, Has.Count.EqualTo(1)); Assert.Multiple(() => diff --git a/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs index 212a3b8621..dc95a6cf99 100644 --- a/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs @@ -30,7 +30,7 @@ await IngestProcessedMessagesAudits( message ); - var queryResult = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), null, TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.Results, Has.Count.EqualTo(1)); Assert.That(queryResult.Results[0].MessageId, Is.EqualTo("MyMessageId")); @@ -40,7 +40,7 @@ await IngestProcessedMessagesAudits( public async Task Handles_no_results_gracefully() { var nonExistingMessage = Guid.NewGuid().ToString(); - var queryResult = await DataStore.QueryMessages(nonExistingMessage, new PagingInfo(), new SortInfo("Id", "asc"), TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.QueryMessages(nonExistingMessage, new PagingInfo(), new SortInfo("Id", "asc"), null, TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.Results, Is.Empty); } @@ -73,7 +73,7 @@ await IngestProcessedMessagesAudits( ); var queryResult = await DataStore.QueryMessages("MyMessageType", new PagingInfo(), - new SortInfo("message_id", "asc"), TestContext.CurrentContext.CancellationToken); + new SortInfo("message_id", "asc"), null, TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.Results, Has.Count.EqualTo(2)); } @@ -158,7 +158,7 @@ public async Task Deduplicates_messages_in_same_batch() await configuration.CompleteDBOperation(); - var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), null, TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(1)); } @@ -182,7 +182,7 @@ public async Task Deduplicates_messages_in_different_batches() await configuration.CompleteDBOperation(); - var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), null, TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(1)); } @@ -205,7 +205,7 @@ public async Task Does_not_deduplicate_with_different_processing_started_header( await configuration.CompleteDBOperation(); - var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), null, TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(2)); } diff --git a/src/ServiceControl.Audit.Persistence/IAuditDataStore.cs b/src/ServiceControl.Audit.Persistence/IAuditDataStore.cs index 9408ceeac7..29e465f793 100644 --- a/src/ServiceControl.Audit.Persistence/IAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence/IAuditDataStore.cs @@ -14,10 +14,10 @@ public interface IAuditDataStore { Task>> QueryKnownEndpoints(CancellationToken cancellationToken); Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken); - Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken); - Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken); - Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken); - Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken); + Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken); + Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken); + Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken); + Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken); Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken); Task GetMessageBody(string messageId, CancellationToken cancellationToken); Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken); diff --git a/src/ServiceControl.Audit/Auditing/MessagesView/GetMessages2Controller.cs b/src/ServiceControl.Audit/Auditing/MessagesView/GetMessages2Controller.cs new file mode 100644 index 0000000000..a023d5a697 --- /dev/null +++ b/src/ServiceControl.Audit/Auditing/MessagesView/GetMessages2Controller.cs @@ -0,0 +1,56 @@ +namespace ServiceControl.Audit.Auditing.MessagesView; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Infrastructure; +using Infrastructure.WebApi; +using Microsoft.AspNetCore.Mvc; +using Persistence; + +[ApiController] +[Route("api")] +public class GetMessages2Controller(IAuditDataStore dataStore) : ControllerBase +{ + [Route("messages2")] + [HttpGet] + public async Task> GetAllMessages( + [FromQuery] SortInfo sortInfo, + [FromQuery(Name = "page_size")] int pageSize, + [FromQuery(Name = "endpoint_name")] string endpointName, + [FromQuery(Name = "range")] string range, + string q, + CancellationToken cancellationToken) + { + QueryResult> result; + var pagingInfo = new PagingInfo(pageSize: pageSize); + if (string.IsNullOrWhiteSpace(endpointName)) + { + if (string.IsNullOrWhiteSpace(q)) + { + result = await dataStore.GetMessages(false, pagingInfo, sortInfo, range, cancellationToken); + } + else + { + result = await dataStore.QueryMessages(q, pagingInfo, sortInfo, range, cancellationToken); + } + } + else + { + if (string.IsNullOrWhiteSpace(q)) + { + result = await dataStore.QueryMessagesByReceivingEndpoint(false, endpointName, pagingInfo, sortInfo, + range, cancellationToken); + } + else + { + result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpointName, q, pagingInfo, + sortInfo, range, cancellationToken); + } + } + + Response.WithTotalCount(result.QueryStats.TotalCount); + + return result.Results; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Audit/Auditing/MessagesView/GetMessagesController.cs b/src/ServiceControl.Audit/Auditing/MessagesView/GetMessagesController.cs index 7c33da8d3e..041cd36c0a 100644 --- a/src/ServiceControl.Audit/Auditing/MessagesView/GetMessagesController.cs +++ b/src/ServiceControl.Audit/Auditing/MessagesView/GetMessagesController.cs @@ -17,7 +17,7 @@ public class GetMessagesController(IAuditDataStore dataStore) : ControllerBase [HttpGet] public async Task> GetAllMessages([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, [FromQuery(Name = "include_system_messages")] bool includeSystemMessages, CancellationToken cancellationToken) { - var result = await dataStore.GetMessages(includeSystemMessages, pagingInfo, sortInfo, cancellationToken); + var result = await dataStore.GetMessages(includeSystemMessages, pagingInfo, sortInfo, null, cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -26,7 +26,7 @@ public async Task> GetAllMessages([FromQuery] PagingInfo pag [HttpGet] public async Task> GetEndpointMessages([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, [FromQuery(Name = "include_system_messages")] bool includeSystemMessages, string endpoint, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessagesByReceivingEndpoint(includeSystemMessages, endpoint, pagingInfo, sortInfo, cancellationToken); + var result = await dataStore.QueryMessagesByReceivingEndpoint(includeSystemMessages, endpoint, pagingInfo, sortInfo, null, cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -70,7 +70,7 @@ public async Task Get(string id, CancellationToken cancellationTo [HttpGet] public async Task> Search([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, string q, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessages(q, pagingInfo, sortInfo, cancellationToken); + var result = await dataStore.QueryMessages(q, pagingInfo, sortInfo, null, cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -79,7 +79,7 @@ public async Task> Search([FromQuery] PagingInfo pagingInfo, [HttpGet] public async Task> SearchByKeyWord([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, string keyword, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessages(keyword, pagingInfo, sortInfo, cancellationToken); + var result = await dataStore.QueryMessages(keyword, pagingInfo, sortInfo, null, cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -88,7 +88,7 @@ public async Task> SearchByKeyWord([FromQuery] PagingInfo pa [HttpGet] public async Task> Search([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, string endpoint, string q, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpoint, q, pagingInfo, sortInfo, cancellationToken); + var result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpoint, q, pagingInfo, sortInfo, null, cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -97,7 +97,7 @@ public async Task> Search([FromQuery] PagingInfo pagingInfo, [HttpGet] public async Task> SearchByKeyword([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, string endpoint, string keyword, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpoint, keyword, pagingInfo, sortInfo, cancellationToken); + var result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpoint, keyword, pagingInfo, sortInfo, null, cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } diff --git a/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs index 4b9cdfb9d5..f6b4479e42 100644 --- a/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs @@ -34,12 +34,14 @@ class ErrorMessagesDataStore( public async Task>> GetAllMessages( PagingInfo pagingInfo, SortInfo sortInfo, - bool includeSystemMessages + bool includeSystemMessages, + string timeSentRange ) { using var session = await sessionProvider.OpenSession(); var query = session.Query() .IncludeSystemMessagesWhere(includeSystemMessages) + .FilterBySentTimeRange(timeSentRange) .Statistics(out var stats) .Sort(sortInfo) .Paging(pagingInfo) @@ -55,12 +57,14 @@ public async Task>> GetAllMessagesForEndpoint( string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, - bool includeSystemMessages + bool includeSystemMessages, + string timeSentRange ) { using var session = await sessionProvider.OpenSession(); var query = session.Query() .IncludeSystemMessagesWhere(includeSystemMessages) + .FilterBySentTimeRange(timeSentRange) .Where(m => m.ReceivingEndpointName == endpointName) .Statistics(out var stats) .Sort(sortInfo) @@ -78,7 +82,8 @@ public async Task>> SearchEndpointMessages( string endpointName, string searchKeyword, PagingInfo pagingInfo, - SortInfo sortInfo + SortInfo sortInfo, + string timeSentRange ) { using var session = await sessionProvider.OpenSession(); @@ -86,6 +91,7 @@ SortInfo sortInfo .Statistics(out var stats) .Search(x => x.Query, searchKeyword) .Where(m => m.ReceivingEndpointName == endpointName) + .FilterBySentTimeRange(timeSentRange) .Sort(sortInfo) .Paging(pagingInfo) .OfType() @@ -120,13 +126,15 @@ bool includeSystemMessages public async Task>> GetAllMessagesForSearch( string searchTerms, PagingInfo pagingInfo, - SortInfo sortInfo + SortInfo sortInfo, + string timeSentRange ) { using var session = await sessionProvider.OpenSession(); var query = session.Query() .Statistics(out var stats) .Search(x => x.Query, searchTerms) + .FilterBySentTimeRange(timeSentRange) .Sort(sortInfo) .Paging(pagingInfo) .OfType() diff --git a/src/ServiceControl.Persistence.RavenDB/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDB/RavenQueryExtensions.cs index 24ed511940..794e8542f3 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenQueryExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenQueryExtensions.cs @@ -174,6 +174,40 @@ public static IAsyncDocumentQuery FilterByLastModifiedRange(this IAsyncDoc return source; } + public static IRavenQueryable FilterBySentTimeRange(this IRavenQueryable source, string range) + { + if (string.IsNullOrWhiteSpace(range)) + { + return source; + } + + var filters = range.Split(SplitChars, StringSplitOptions.None); + DateTime from, to; + try + { + + if (filters.Length == 2) + { + from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + to = DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + source.Where(m => m.TimeSent >= from && m.TimeSent <= to); + + } + else + { + from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + source.Where(m => m.TimeSent >= from); + } + } + catch (Exception) + { + throw new Exception( + "Invalid sent time date range, dates need to be in ISO8601 format and it needs to be a range eg. 2016-03-11T00:27:15.474Z...2016-03-16T03:27:15.474Z"); + } + + return source; + } + public static IAsyncDocumentQuery FilterByQueueAddress(this IAsyncDocumentQuery source, string queueAddress) { if (string.IsNullOrWhiteSpace(queueAddress)) diff --git a/src/ServiceControl.Persistence.Tests/Recoverability/ReturnToSenderDequeuerTests.cs b/src/ServiceControl.Persistence.Tests/Recoverability/ReturnToSenderDequeuerTests.cs index 1bdf6c9fb6..7d6db4c2a5 100644 --- a/src/ServiceControl.Persistence.Tests/Recoverability/ReturnToSenderDequeuerTests.cs +++ b/src/ServiceControl.Persistence.Tests/Recoverability/ReturnToSenderDequeuerTests.cs @@ -170,17 +170,21 @@ class FakeErrorMessageDataStore : IErrorMessageDataStore { public Task FetchFromFailedMessage(string bodyId) => Task.FromResult(Encoding.UTF8.GetBytes(bodyId)); - public Task>> GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages) => throw new NotImplementedException(); + public Task>> GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, string timeSentRange) => throw new NotImplementedException(); public Task>> GetAllMessagesForEndpoint(string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, - bool includeSystemMessages) => + bool includeSystemMessages, string timeSentRange) => throw new NotImplementedException(); public Task>> GetAllMessagesByConversation(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages) => throw new NotImplementedException(); - public Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo) => throw new NotImplementedException(); + public Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange) => throw new NotImplementedException(); + + public Task>> SearchEndpointMessages(string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo, + string timeSentRange) => + throw new NotImplementedException(); public Task FailedMessageMarkAsArchived(string failedMessageId) => throw new NotImplementedException(); @@ -236,8 +240,6 @@ public Task>> ErrorsByEndpointName(string s public Task StoreEventLogItem(EventLogItem logItem) => throw new NotImplementedException(); - public Task>> SearchEndpointMessages(string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo) => throw new NotImplementedException(); - public Task StoreFailedMessagesForTestsOnly(params FailedMessage[] failedMessages) => throw new NotImplementedException(); } } diff --git a/src/ServiceControl.Persistence/IErrorMessageDatastore.cs b/src/ServiceControl.Persistence/IErrorMessageDatastore.cs index 2282319835..220ea3cac2 100644 --- a/src/ServiceControl.Persistence/IErrorMessageDatastore.cs +++ b/src/ServiceControl.Persistence/IErrorMessageDatastore.cs @@ -13,10 +13,11 @@ public interface IErrorMessageDataStore { - Task>> GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages); - Task>> GetAllMessagesForEndpoint(string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages); + Task>> GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, string timeSentRange = null); + Task>> GetAllMessagesForEndpoint(string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, string timeSentRange = null); Task>> GetAllMessagesByConversation(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages); - Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo); + Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange = null); + Task>> SearchEndpointMessages(string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange = null); Task FailedMessageMarkAsArchived(string failedMessageId); Task FailedMessagesFetch(Guid[] ids); Task StoreFailedErrorImport(FailedErrorImport failure); @@ -71,8 +72,6 @@ public interface IErrorMessageDataStore // AuditEventLogWriter Task StoreEventLogItem(EventLogItem logItem); - Task>> SearchEndpointMessages(string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo); - Task StoreFailedMessagesForTestsOnly(params FailedMessage[] failedMessages); } } \ No newline at end of file diff --git a/src/ServiceControl/CompositeViews/Messages/GetAllMessagesApi.cs b/src/ServiceControl/CompositeViews/Messages/GetAllMessagesApi.cs index 1055c7c8e5..37d83ac49d 100644 --- a/src/ServiceControl/CompositeViews/Messages/GetAllMessagesApi.cs +++ b/src/ServiceControl/CompositeViews/Messages/GetAllMessagesApi.cs @@ -16,7 +16,7 @@ public GetAllMessagesApi(IErrorMessageDataStore dataStore, Settings settings, protected override Task>> LocalQuery(ScatterGatherApiMessageViewWithSystemMessagesContext input) { - return DataStore.GetAllMessages(input.PagingInfo, input.SortInfo, input.IncludeSystemMessages); + return DataStore.GetAllMessages(input.PagingInfo, input.SortInfo, input.IncludeSystemMessages, input.TimeSentRange); } } } \ No newline at end of file diff --git a/src/ServiceControl/CompositeViews/Messages/GetAllMessagesForEndpointApi.cs b/src/ServiceControl/CompositeViews/Messages/GetAllMessagesForEndpointApi.cs index c49919ff49..dedd83b31c 100644 --- a/src/ServiceControl/CompositeViews/Messages/GetAllMessagesForEndpointApi.cs +++ b/src/ServiceControl/CompositeViews/Messages/GetAllMessagesForEndpointApi.cs @@ -11,8 +11,9 @@ public record AllMessagesForEndpointContext( PagingInfo PagingInfo, SortInfo SortInfo, bool IncludeSystemMessages, - string EndpointName) - : ScatterGatherApiMessageViewWithSystemMessagesContext(PagingInfo, SortInfo, IncludeSystemMessages); + string EndpointName, + string TimeSentRange = null) + : ScatterGatherApiMessageViewWithSystemMessagesContext(PagingInfo, SortInfo, IncludeSystemMessages, TimeSentRange); public class GetAllMessagesForEndpointApi : ScatterGatherApiMessageView { @@ -21,6 +22,6 @@ public GetAllMessagesForEndpointApi(IErrorMessageDataStore dataStore, Settings s { } - protected override Task>> LocalQuery(AllMessagesForEndpointContext input) => DataStore.GetAllMessagesForEndpoint(input.EndpointName, input.PagingInfo, input.SortInfo, input.IncludeSystemMessages); + protected override Task>> LocalQuery(AllMessagesForEndpointContext input) => DataStore.GetAllMessagesForEndpoint(input.EndpointName, input.PagingInfo, input.SortInfo, input.IncludeSystemMessages, input.TimeSentRange); } } \ No newline at end of file diff --git a/src/ServiceControl/CompositeViews/Messages/GetMessages2Controller.cs b/src/ServiceControl/CompositeViews/Messages/GetMessages2Controller.cs new file mode 100644 index 0000000000..4ad0b961ff --- /dev/null +++ b/src/ServiceControl/CompositeViews/Messages/GetMessages2Controller.cs @@ -0,0 +1,66 @@ +namespace ServiceControl.CompositeViews.Messages; + +using System.Collections.Generic; +using System.Threading.Tasks; +using Infrastructure.WebApi; +using Microsoft.AspNetCore.Http.Extensions; +using Microsoft.AspNetCore.Mvc; +using Persistence.Infrastructure; + +[ApiController] +[Route("api")] +public class GetMessages2Controller( + GetAllMessagesApi allMessagesApi, + GetAllMessagesForEndpointApi allMessagesEndpointApi, + SearchApi searchApi, + SearchEndpointApi searchEndpointApi) + : ControllerBase +{ + [Route("messages2")] + [HttpGet] + public async Task> Messages( + [FromQuery] SortInfo sortInfo, + [FromQuery(Name = "page_size")] int pageSize, + [FromQuery(Name = "endpoint_name")] string endpointName, + [FromQuery(Name = "range")] string range, + string q) + { + QueryResult> result; + var pagingInfo = new PagingInfo(pageSize: pageSize); + if (string.IsNullOrWhiteSpace(endpointName)) + { + if (string.IsNullOrWhiteSpace(q)) + { + result = await allMessagesApi.Execute( + new ScatterGatherApiMessageViewWithSystemMessagesContext(pagingInfo, + sortInfo, false, range), + Request.GetEncodedPathAndQuery()); + } + else + { + result = await searchApi.Execute( + new SearchApiContext(pagingInfo, sortInfo, q, range), + Request.GetEncodedPathAndQuery()); + } + } + else + { + if (string.IsNullOrWhiteSpace(q)) + { + result = await allMessagesEndpointApi.Execute( + new AllMessagesForEndpointContext(pagingInfo, sortInfo, false, + endpointName, range), + Request.GetEncodedPathAndQuery()); + } + else + { + result = await searchEndpointApi.Execute(new SearchEndpointContext(pagingInfo, sortInfo, q, endpointName, range), + Request.GetEncodedPathAndQuery()); + } + } + + Response.WithTotalCount(result.QueryStats.TotalCount); + + return result.Results; + } +} \ No newline at end of file diff --git a/src/ServiceControl/CompositeViews/Messages/GetMessagesController.cs b/src/ServiceControl/CompositeViews/Messages/GetMessagesController.cs index 502b48bddf..95c82bf5ac 100644 --- a/src/ServiceControl/CompositeViews/Messages/GetMessagesController.cs +++ b/src/ServiceControl/CompositeViews/Messages/GetMessagesController.cs @@ -106,7 +106,7 @@ public async Task Get(string id, [FromQuery(Name = "instance_id") var forwarderError = await forwarder.SendAsync(HttpContext, remote.BaseAddress, httpMessageInvoker); if (forwarderError != ForwarderError.None && HttpContext.GetForwarderErrorFeature()?.Exception is { } exception) { - logger.Warn($"Failed to forward the request ot remote instance at {remote.BaseAddress + HttpContext.Request.GetEncodedPathAndQuery()}.", exception); + logger.Warn($"Failed to forward the request to remote instance at {remote.BaseAddress}{HttpContext.Request.GetEncodedPathAndQuery()}.", exception); } return Empty; diff --git a/src/ServiceControl/CompositeViews/Messages/ScatterGatherApiMessageView.cs b/src/ServiceControl/CompositeViews/Messages/ScatterGatherApiMessageView.cs index 1388ad45a4..840a190415 100644 --- a/src/ServiceControl/CompositeViews/Messages/ScatterGatherApiMessageView.cs +++ b/src/ServiceControl/CompositeViews/Messages/ScatterGatherApiMessageView.cs @@ -9,9 +9,10 @@ namespace ServiceControl.CompositeViews.Messages public record ScatterGatherApiMessageViewWithSystemMessagesContext( PagingInfo PagingInfo, SortInfo SortInfo, - bool IncludeSystemMessages) : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo); + bool IncludeSystemMessages, + string TimeSentRange = null) : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo, TimeSentRange); - public record ScatterGatherApiMessageViewContext(PagingInfo PagingInfo, SortInfo SortInfo) : ScatterGatherContext(PagingInfo); + public record ScatterGatherApiMessageViewContext(PagingInfo PagingInfo, SortInfo SortInfo, string TimeSentRange = null) : ScatterGatherContext(PagingInfo); public abstract class ScatterGatherApiMessageView : ScatterGatherApi> where TInput : ScatterGatherApiMessageViewContext @@ -51,7 +52,7 @@ protected override IList ProcessResults(TInput input, QueryResult< combined.Sort(comparer); } - return combined; + return combined.Take(input.PagingInfo.PageSize).ToList(); } IComparer FinalOrder(SortInfo sortInfo) => MessageViewComparer.FromSortInfo(sortInfo); diff --git a/src/ServiceControl/CompositeViews/Messages/SearchApi.cs b/src/ServiceControl/CompositeViews/Messages/SearchApi.cs index 97b63b7e81..b59382998b 100644 --- a/src/ServiceControl/CompositeViews/Messages/SearchApi.cs +++ b/src/ServiceControl/CompositeViews/Messages/SearchApi.cs @@ -10,8 +10,9 @@ namespace ServiceControl.CompositeViews.Messages public record SearchApiContext( PagingInfo PagingInfo, SortInfo SortInfo, - string SearchQuery) - : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo); + string SearchQuery, + string TimeSentRange = null) + : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo, TimeSentRange); public class SearchApi : ScatterGatherApiMessageView { @@ -21,6 +22,6 @@ public SearchApi(IErrorMessageDataStore dataStore, Settings settings, IHttpClien } protected override Task>> LocalQuery(SearchApiContext input) => - DataStore.GetAllMessagesForSearch(input.SearchQuery, input.PagingInfo, input.SortInfo); + DataStore.GetAllMessagesForSearch(input.SearchQuery, input.PagingInfo, input.SortInfo, input.TimeSentRange); } } \ No newline at end of file diff --git a/src/ServiceControl/CompositeViews/Messages/SearchEndpointApi.cs b/src/ServiceControl/CompositeViews/Messages/SearchEndpointApi.cs index b94a902812..f8dd51ecfb 100644 --- a/src/ServiceControl/CompositeViews/Messages/SearchEndpointApi.cs +++ b/src/ServiceControl/CompositeViews/Messages/SearchEndpointApi.cs @@ -11,8 +11,9 @@ public record SearchEndpointContext( PagingInfo PagingInfo, SortInfo SortInfo, string Keyword, - string Endpoint) - : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo); + string Endpoint, + string TimeSentRange = null) + : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo, TimeSentRange); public class SearchEndpointApi : ScatterGatherApiMessageView { @@ -22,6 +23,6 @@ public SearchEndpointApi(IErrorMessageDataStore dataStore, Settings settings, } protected override Task>> LocalQuery(SearchEndpointContext input) => - DataStore.SearchEndpointMessages(input.Endpoint, input.Keyword, input.PagingInfo, input.SortInfo); + DataStore.SearchEndpointMessages(input.Endpoint, input.Keyword, input.PagingInfo, input.SortInfo, input.TimeSentRange); } } \ No newline at end of file From 19f662f626a3ae91ce87655727f8ce2db5a81e11 Mon Sep 17 00:00:00 2001 From: John Simons Date: Wed, 2 Apr 2025 14:16:06 +1000 Subject: [PATCH 2/4] Updates time range to DateTimeRange object Changes time range from string to DateTimeRange object for message queries. This provides type safety and removes parsing logic and error handling from the data access layer. Also allows for optional To property on the DateTimeRange. --- .../InMemoryAuditDataStore.cs | 8 ++--- .../Extensions/RavenQueryExtensions.cs | 34 ++++--------------- .../RavenAuditDataStore.cs | 8 ++--- .../RetentionTests.cs | 8 ++--- .../AuditTests.cs | 12 +++---- .../IAuditDataStore.cs | 8 ++--- .../Infrastructure/DateTimeRange.cs | 28 +++++++++++++++ .../MessagesView/GetMessages2Controller.cs | 11 +++--- .../MessagesView/GetMessagesController.cs | 12 +++---- .../ErrorMessagesDataStore.cs | 8 ++--- .../RavenQueryExtensions.cs | 28 ++++----------- .../ReturnToSenderDequeuerTests.cs | 15 ++++---- .../IErrorMessageDatastore.cs | 8 ++--- .../Infrastructure/DateTimeRange.cs | 28 +++++++++++++++ .../Messages/GetAllMessagesForEndpointApi.cs | 2 +- .../Messages/GetMessages2Controller.cs | 11 +++--- .../Messages/ScatterGatherApiMessageView.cs | 4 +-- .../CompositeViews/Messages/SearchApi.cs | 2 +- .../Messages/SearchEndpointApi.cs | 2 +- 19 files changed, 131 insertions(+), 106 deletions(-) create mode 100644 src/ServiceControl.Audit.Persistence/Infrastructure/DateTimeRange.cs create mode 100644 src/ServiceControl.Persistence/Infrastructure/DateTimeRange.cs diff --git a/src/ServiceControl.Audit.Persistence.InMemory/InMemoryAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.InMemory/InMemoryAuditDataStore.cs index 9436d3caf2..feaab3d080 100644 --- a/src/ServiceControl.Audit.Persistence.InMemory/InMemoryAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.InMemory/InMemoryAuditDataStore.cs @@ -42,7 +42,7 @@ public async Task> QuerySagaHistoryById(Guid input, Can return await Task.FromResult(new QueryResult(sagaHistory, new QueryStatsInfo(string.Empty, 1))); } - public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) + public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { var matched = messageViews .Where(w => !w.IsSystemMessage || includeSystemMessages) @@ -51,7 +51,7 @@ public async Task>> GetMessages(bool includeSyst return await Task.FromResult(new QueryResult>(matched, new QueryStatsInfo(string.Empty, matched.Count))); } - public async Task>> QueryMessages(string keyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) + public async Task>> QueryMessages(string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { var messages = GetMessageIdsMatchingQuery(keyword); @@ -61,7 +61,7 @@ public async Task>> QueryMessages(string keyword return await Task.FromResult(new QueryResult>(matched, new QueryStatsInfo(string.Empty, matched.Count()))); } - public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) + public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { var messages = GetMessageIdsMatchingQuery(keyword); @@ -69,7 +69,7 @@ public async Task>> QueryMessagesByReceivingEndp return await Task.FromResult(new QueryResult>(matched, new QueryStatsInfo(string.Empty, matched.Count))); } - public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) + public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { var matched = messageViews.Where(w => w.ReceivingEndpoint.Name == endpointName).ToList(); return await Task.FromResult(new QueryResult>(matched, new QueryStatsInfo(string.Empty, matched.Count))); diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/Extensions/RavenQueryExtensions.cs b/src/ServiceControl.Audit.Persistence.RavenDB/Extensions/RavenQueryExtensions.cs index 071d6c1fee..c9e36f377d 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/Extensions/RavenQueryExtensions.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/Extensions/RavenQueryExtensions.cs @@ -1,7 +1,6 @@ namespace ServiceControl.Audit.Persistence.RavenDB.Extensions { using System; - using System.Globalization; using System.Linq; using System.Linq.Expressions; using Indexes; @@ -20,45 +19,26 @@ static class RavenQueryExtensions return source; } - public static IQueryable FilterBySentTimeRange(this IQueryable source, string range) + public static IQueryable FilterBySentTimeRange(this IQueryable source, DateTimeRange range) { - if (string.IsNullOrWhiteSpace(range)) + if (range == null) { return source; } - var filters = range.Split(SplitChars, StringSplitOptions.None); - DateTime from, to; - try + if (range.From.HasValue) { - - if (filters.Length == 2) - { - from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - to = DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - source.Where(m => m.TimeSent >= from && m.TimeSent <= to); - - } - else - { - from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - source.Where(m => m.TimeSent >= from); - } + source = source.Where(m => m.TimeSent >= range.From); } - catch (Exception) + + if (range.To.HasValue) { - throw new Exception( - "Invalid sent time date range, dates need to be in ISO8601 format and it needs to be a range eg. 2016-03-11T00:27:15.474Z...2016-03-16T03:27:15.474Z"); + source = source.Where(m => m.TimeSent <= range.To); } return source; } - static string[] SplitChars = - { - "..." - }; - public static IQueryable Paging(this IQueryable source, PagingInfo pagingInfo) => source.Skip(pagingInfo.Offset).Take(pagingInfo.PageSize); diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs index e454c321bd..8a465fb029 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenAuditDataStore.cs @@ -31,7 +31,7 @@ public async Task> QuerySagaHistoryById(Guid input, Can return sagaHistory == null ? QueryResult.Empty() : new QueryResult(sagaHistory, new QueryStatsInfo($"{stats.ResultEtag}", stats.TotalResults)); } - public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) + public async Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) @@ -46,7 +46,7 @@ public async Task>> GetMessages(bool includeSyst return new QueryResult>(results, stats.ToQueryStatsInfo()); } - public async Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) + public async Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) @@ -61,7 +61,7 @@ public async Task>> QueryMessages(string searchP return new QueryResult>(results, stats.ToQueryStatsInfo()); } - public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) + public async Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) @@ -77,7 +77,7 @@ public async Task>> QueryMessagesByReceivingEndp return new QueryResult>(results, stats.ToQueryStatsInfo()); } - public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken) + public async Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange, CancellationToken cancellationToken) { using var session = await sessionProvider.OpenSession(cancellationToken: cancellationToken); var results = await session.Query(GetIndexName(isFullTextSearchEnabled)) diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RetentionTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RetentionTests.cs index 2377993e02..9fef1ed7db 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RetentionTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/RetentionTests.cs @@ -28,15 +28,13 @@ public async Task AuditMessageRetention() { var message = MakeMessage("MyMessageId"); - await IngestProcessedMessagesAudits( - message - ); + await IngestProcessedMessagesAudits(message); - var queryResultBeforeExpiration = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), null, TestContext.CurrentContext.CancellationToken); + var queryResultBeforeExpiration = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); await Task.Delay(4000); - var queryResultAfterExpiration = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), null, TestContext.CurrentContext.CancellationToken); + var queryResultAfterExpiration = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); Assert.That(queryResultBeforeExpiration.Results, Has.Count.EqualTo(1)); Assert.Multiple(() => diff --git a/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs index dc95a6cf99..4d89044420 100644 --- a/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests/AuditTests.cs @@ -30,7 +30,7 @@ await IngestProcessedMessagesAudits( message ); - var queryResult = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), null, TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.QueryMessages("MyMessageId", new PagingInfo(), new SortInfo("Id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.Results, Has.Count.EqualTo(1)); Assert.That(queryResult.Results[0].MessageId, Is.EqualTo("MyMessageId")); @@ -40,7 +40,7 @@ await IngestProcessedMessagesAudits( public async Task Handles_no_results_gracefully() { var nonExistingMessage = Guid.NewGuid().ToString(); - var queryResult = await DataStore.QueryMessages(nonExistingMessage, new PagingInfo(), new SortInfo("Id", "asc"), null, TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.QueryMessages(nonExistingMessage, new PagingInfo(), new SortInfo("Id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.Results, Is.Empty); } @@ -73,7 +73,7 @@ await IngestProcessedMessagesAudits( ); var queryResult = await DataStore.QueryMessages("MyMessageType", new PagingInfo(), - new SortInfo("message_id", "asc"), null, TestContext.CurrentContext.CancellationToken); + new SortInfo("message_id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.Results, Has.Count.EqualTo(2)); } @@ -158,7 +158,7 @@ public async Task Deduplicates_messages_in_same_batch() await configuration.CompleteDBOperation(); - var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), null, TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(1)); } @@ -182,7 +182,7 @@ public async Task Deduplicates_messages_in_different_batches() await configuration.CompleteDBOperation(); - var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), null, TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(1)); } @@ -205,7 +205,7 @@ public async Task Does_not_deduplicate_with_different_processing_started_header( await configuration.CompleteDBOperation(); - var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), null, TestContext.CurrentContext.CancellationToken); + var queryResult = await DataStore.GetMessages(false, new PagingInfo(), new SortInfo("message_id", "asc"), cancellationToken: TestContext.CurrentContext.CancellationToken); Assert.That(queryResult.QueryStats.TotalCount, Is.EqualTo(2)); } diff --git a/src/ServiceControl.Audit.Persistence/IAuditDataStore.cs b/src/ServiceControl.Audit.Persistence/IAuditDataStore.cs index 29e465f793..bc7258ffc3 100644 --- a/src/ServiceControl.Audit.Persistence/IAuditDataStore.cs +++ b/src/ServiceControl.Audit.Persistence/IAuditDataStore.cs @@ -14,10 +14,10 @@ public interface IAuditDataStore { Task>> QueryKnownEndpoints(CancellationToken cancellationToken); Task> QuerySagaHistoryById(Guid input, CancellationToken cancellationToken); - Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken); - Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken); - Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken); - Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange, CancellationToken cancellationToken); + Task>> GetMessages(bool includeSystemMessages, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default); + Task>> QueryMessages(string searchParam, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default); + Task>> QueryMessagesByReceivingEndpointAndKeyword(string endpoint, string keyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default); + Task>> QueryMessagesByReceivingEndpoint(bool includeSystemMessages, string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null, CancellationToken cancellationToken = default); Task>> QueryMessagesByConversationId(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, CancellationToken cancellationToken); Task GetMessageBody(string messageId, CancellationToken cancellationToken); Task>> QueryAuditCounts(string endpointName, CancellationToken cancellationToken); diff --git a/src/ServiceControl.Audit.Persistence/Infrastructure/DateTimeRange.cs b/src/ServiceControl.Audit.Persistence/Infrastructure/DateTimeRange.cs new file mode 100644 index 0000000000..eb641d437b --- /dev/null +++ b/src/ServiceControl.Audit.Persistence/Infrastructure/DateTimeRange.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Audit.Infrastructure; + +using System; +using System.Globalization; + +public class DateTimeRange +{ + public DateTime? From { get; } + public DateTime? To { get; } + + public DateTimeRange(string from = null, string to = null) + { + if (from != null) + { + From = DateTime.Parse(from, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + } + if (to != null) + { + To = DateTime.Parse(to, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + } + } + + public DateTimeRange(DateTime? from = null, DateTime? to = null) + { + From = from; + To = to; + } +} \ No newline at end of file diff --git a/src/ServiceControl.Audit/Auditing/MessagesView/GetMessages2Controller.cs b/src/ServiceControl.Audit/Auditing/MessagesView/GetMessages2Controller.cs index a023d5a697..db187bcd77 100644 --- a/src/ServiceControl.Audit/Auditing/MessagesView/GetMessages2Controller.cs +++ b/src/ServiceControl.Audit/Auditing/MessagesView/GetMessages2Controller.cs @@ -18,7 +18,8 @@ public async Task> GetAllMessages( [FromQuery] SortInfo sortInfo, [FromQuery(Name = "page_size")] int pageSize, [FromQuery(Name = "endpoint_name")] string endpointName, - [FromQuery(Name = "range")] string range, + [FromQuery(Name = "from")] string from, + [FromQuery(Name = "to")] string to, string q, CancellationToken cancellationToken) { @@ -28,11 +29,11 @@ public async Task> GetAllMessages( { if (string.IsNullOrWhiteSpace(q)) { - result = await dataStore.GetMessages(false, pagingInfo, sortInfo, range, cancellationToken); + result = await dataStore.GetMessages(false, pagingInfo, sortInfo, new DateTimeRange(from, to), cancellationToken); } else { - result = await dataStore.QueryMessages(q, pagingInfo, sortInfo, range, cancellationToken); + result = await dataStore.QueryMessages(q, pagingInfo, sortInfo, new DateTimeRange(from, to), cancellationToken); } } else @@ -40,12 +41,12 @@ public async Task> GetAllMessages( if (string.IsNullOrWhiteSpace(q)) { result = await dataStore.QueryMessagesByReceivingEndpoint(false, endpointName, pagingInfo, sortInfo, - range, cancellationToken); + new DateTimeRange(from, to), cancellationToken); } else { result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpointName, q, pagingInfo, - sortInfo, range, cancellationToken); + sortInfo, new DateTimeRange(from, to), cancellationToken); } } diff --git a/src/ServiceControl.Audit/Auditing/MessagesView/GetMessagesController.cs b/src/ServiceControl.Audit/Auditing/MessagesView/GetMessagesController.cs index 041cd36c0a..ca3183c1b9 100644 --- a/src/ServiceControl.Audit/Auditing/MessagesView/GetMessagesController.cs +++ b/src/ServiceControl.Audit/Auditing/MessagesView/GetMessagesController.cs @@ -17,7 +17,7 @@ public class GetMessagesController(IAuditDataStore dataStore) : ControllerBase [HttpGet] public async Task> GetAllMessages([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, [FromQuery(Name = "include_system_messages")] bool includeSystemMessages, CancellationToken cancellationToken) { - var result = await dataStore.GetMessages(includeSystemMessages, pagingInfo, sortInfo, null, cancellationToken); + var result = await dataStore.GetMessages(includeSystemMessages, pagingInfo, sortInfo, cancellationToken: cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -26,7 +26,7 @@ public async Task> GetAllMessages([FromQuery] PagingInfo pag [HttpGet] public async Task> GetEndpointMessages([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, [FromQuery(Name = "include_system_messages")] bool includeSystemMessages, string endpoint, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessagesByReceivingEndpoint(includeSystemMessages, endpoint, pagingInfo, sortInfo, null, cancellationToken); + var result = await dataStore.QueryMessagesByReceivingEndpoint(includeSystemMessages, endpoint, pagingInfo, sortInfo, cancellationToken: cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -70,7 +70,7 @@ public async Task Get(string id, CancellationToken cancellationTo [HttpGet] public async Task> Search([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, string q, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessages(q, pagingInfo, sortInfo, null, cancellationToken); + var result = await dataStore.QueryMessages(q, pagingInfo, sortInfo, cancellationToken: cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -79,7 +79,7 @@ public async Task> Search([FromQuery] PagingInfo pagingInfo, [HttpGet] public async Task> SearchByKeyWord([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, string keyword, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessages(keyword, pagingInfo, sortInfo, null, cancellationToken); + var result = await dataStore.QueryMessages(keyword, pagingInfo, sortInfo, cancellationToken: cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -88,7 +88,7 @@ public async Task> SearchByKeyWord([FromQuery] PagingInfo pa [HttpGet] public async Task> Search([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, string endpoint, string q, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpoint, q, pagingInfo, sortInfo, null, cancellationToken); + var result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpoint, q, pagingInfo, sortInfo, cancellationToken: cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } @@ -97,7 +97,7 @@ public async Task> Search([FromQuery] PagingInfo pagingInfo, [HttpGet] public async Task> SearchByKeyword([FromQuery] PagingInfo pagingInfo, [FromQuery] SortInfo sortInfo, string endpoint, string keyword, CancellationToken cancellationToken) { - var result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpoint, keyword, pagingInfo, sortInfo, null, cancellationToken); + var result = await dataStore.QueryMessagesByReceivingEndpointAndKeyword(endpoint, keyword, pagingInfo, sortInfo, cancellationToken: cancellationToken); Response.WithQueryStatsAndPagingInfo(result.QueryStats, pagingInfo); return result.Results; } diff --git a/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs b/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs index f6b4479e42..4c3f77bf6d 100644 --- a/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs +++ b/src/ServiceControl.Persistence.RavenDB/ErrorMessagesDataStore.cs @@ -35,7 +35,7 @@ public async Task>> GetAllMessages( PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, - string timeSentRange + DateTimeRange timeSentRange ) { using var session = await sessionProvider.OpenSession(); @@ -58,7 +58,7 @@ public async Task>> GetAllMessagesForEndpoint( PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, - string timeSentRange + DateTimeRange timeSentRange ) { using var session = await sessionProvider.OpenSession(); @@ -83,7 +83,7 @@ public async Task>> SearchEndpointMessages( string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo, - string timeSentRange + DateTimeRange timeSentRange ) { using var session = await sessionProvider.OpenSession(); @@ -127,7 +127,7 @@ public async Task>> GetAllMessagesForSearch( string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo, - string timeSentRange + DateTimeRange timeSentRange ) { using var session = await sessionProvider.OpenSession(); diff --git a/src/ServiceControl.Persistence.RavenDB/RavenQueryExtensions.cs b/src/ServiceControl.Persistence.RavenDB/RavenQueryExtensions.cs index 794e8542f3..47afe4dcd1 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenQueryExtensions.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenQueryExtensions.cs @@ -174,35 +174,21 @@ public static IAsyncDocumentQuery FilterByLastModifiedRange(this IAsyncDoc return source; } - public static IRavenQueryable FilterBySentTimeRange(this IRavenQueryable source, string range) + public static IRavenQueryable FilterBySentTimeRange(this IRavenQueryable source, DateTimeRange range) { - if (string.IsNullOrWhiteSpace(range)) + if (range == null) { return source; } - var filters = range.Split(SplitChars, StringSplitOptions.None); - DateTime from, to; - try + if (range.From.HasValue) { - - if (filters.Length == 2) - { - from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - to = DateTime.Parse(filters[1], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - source.Where(m => m.TimeSent >= from && m.TimeSent <= to); - - } - else - { - from = DateTime.Parse(filters[0], CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); - source.Where(m => m.TimeSent >= from); - } + source = source.Where(m => m.TimeSent >= range.From); } - catch (Exception) + + if (range.To.HasValue) { - throw new Exception( - "Invalid sent time date range, dates need to be in ISO8601 format and it needs to be a range eg. 2016-03-11T00:27:15.474Z...2016-03-16T03:27:15.474Z"); + source = source.Where(m => m.TimeSent <= range.To); } return source; diff --git a/src/ServiceControl.Persistence.Tests/Recoverability/ReturnToSenderDequeuerTests.cs b/src/ServiceControl.Persistence.Tests/Recoverability/ReturnToSenderDequeuerTests.cs index 7d6db4c2a5..6df26ed702 100644 --- a/src/ServiceControl.Persistence.Tests/Recoverability/ReturnToSenderDequeuerTests.cs +++ b/src/ServiceControl.Persistence.Tests/Recoverability/ReturnToSenderDequeuerTests.cs @@ -168,22 +168,24 @@ public Task Dispatch(TransportOperations outgoingMessages, TransportTransaction class FakeErrorMessageDataStore : IErrorMessageDataStore { - public Task FetchFromFailedMessage(string bodyId) => Task.FromResult(Encoding.UTF8.GetBytes(bodyId)); - - public Task>> GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, string timeSentRange) => throw new NotImplementedException(); + public Task>> GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, + DateTimeRange timeSentRange = null) => + throw new NotImplementedException(); public Task>> GetAllMessagesForEndpoint(string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, - bool includeSystemMessages, string timeSentRange) => + bool includeSystemMessages, DateTimeRange timeSentRange = null) => throw new NotImplementedException(); public Task>> GetAllMessagesByConversation(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages) => throw new NotImplementedException(); - public Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange) => throw new NotImplementedException(); + public Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo, + DateTimeRange timeSentRange = null) => + throw new NotImplementedException(); public Task>> SearchEndpointMessages(string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo, - string timeSentRange) => + DateTimeRange timeSentRange = null) => throw new NotImplementedException(); public Task FailedMessageMarkAsArchived(string failedMessageId) => throw new NotImplementedException(); @@ -238,6 +240,7 @@ public Task>> ErrorsByEndpointName(string s public Task GetRetryPendingMessages(DateTime from, DateTime to, string queueAddress) => throw new NotImplementedException(); + public Task FetchFromFailedMessage(string bodyId) => Task.FromResult(Encoding.UTF8.GetBytes(bodyId)); public Task StoreEventLogItem(EventLogItem logItem) => throw new NotImplementedException(); public Task StoreFailedMessagesForTestsOnly(params FailedMessage[] failedMessages) => throw new NotImplementedException(); diff --git a/src/ServiceControl.Persistence/IErrorMessageDatastore.cs b/src/ServiceControl.Persistence/IErrorMessageDatastore.cs index 220ea3cac2..0dd2c6aad0 100644 --- a/src/ServiceControl.Persistence/IErrorMessageDatastore.cs +++ b/src/ServiceControl.Persistence/IErrorMessageDatastore.cs @@ -13,11 +13,11 @@ public interface IErrorMessageDataStore { - Task>> GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, string timeSentRange = null); - Task>> GetAllMessagesForEndpoint(string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, string timeSentRange = null); + Task>> GetAllMessages(PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, DateTimeRange timeSentRange = null); + Task>> GetAllMessagesForEndpoint(string endpointName, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages, DateTimeRange timeSentRange = null); Task>> GetAllMessagesByConversation(string conversationId, PagingInfo pagingInfo, SortInfo sortInfo, bool includeSystemMessages); - Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange = null); - Task>> SearchEndpointMessages(string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo, string timeSentRange = null); + Task>> GetAllMessagesForSearch(string searchTerms, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null); + Task>> SearchEndpointMessages(string endpointName, string searchKeyword, PagingInfo pagingInfo, SortInfo sortInfo, DateTimeRange timeSentRange = null); Task FailedMessageMarkAsArchived(string failedMessageId); Task FailedMessagesFetch(Guid[] ids); Task StoreFailedErrorImport(FailedErrorImport failure); diff --git a/src/ServiceControl.Persistence/Infrastructure/DateTimeRange.cs b/src/ServiceControl.Persistence/Infrastructure/DateTimeRange.cs new file mode 100644 index 0000000000..562df20c90 --- /dev/null +++ b/src/ServiceControl.Persistence/Infrastructure/DateTimeRange.cs @@ -0,0 +1,28 @@ +namespace ServiceControl.Persistence.Infrastructure; + +using System; +using System.Globalization; + +public class DateTimeRange +{ + public DateTime? From { get; } + public DateTime? To { get; } + + public DateTimeRange(string from = null, string to = null) + { + if (from != null) + { + From = DateTime.Parse(from, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + } + if (to != null) + { + To = DateTime.Parse(to, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + } + } + + public DateTimeRange(DateTime? from = null, DateTime? to = null) + { + From = from; + To = to; + } +} \ No newline at end of file diff --git a/src/ServiceControl/CompositeViews/Messages/GetAllMessagesForEndpointApi.cs b/src/ServiceControl/CompositeViews/Messages/GetAllMessagesForEndpointApi.cs index dedd83b31c..1fdd327913 100644 --- a/src/ServiceControl/CompositeViews/Messages/GetAllMessagesForEndpointApi.cs +++ b/src/ServiceControl/CompositeViews/Messages/GetAllMessagesForEndpointApi.cs @@ -12,7 +12,7 @@ public record AllMessagesForEndpointContext( SortInfo SortInfo, bool IncludeSystemMessages, string EndpointName, - string TimeSentRange = null) + DateTimeRange TimeSentRange = null) : ScatterGatherApiMessageViewWithSystemMessagesContext(PagingInfo, SortInfo, IncludeSystemMessages, TimeSentRange); public class GetAllMessagesForEndpointApi : ScatterGatherApiMessageView diff --git a/src/ServiceControl/CompositeViews/Messages/GetMessages2Controller.cs b/src/ServiceControl/CompositeViews/Messages/GetMessages2Controller.cs index 4ad0b961ff..1ee79cfbbc 100644 --- a/src/ServiceControl/CompositeViews/Messages/GetMessages2Controller.cs +++ b/src/ServiceControl/CompositeViews/Messages/GetMessages2Controller.cs @@ -22,7 +22,8 @@ public async Task> Messages( [FromQuery] SortInfo sortInfo, [FromQuery(Name = "page_size")] int pageSize, [FromQuery(Name = "endpoint_name")] string endpointName, - [FromQuery(Name = "range")] string range, + [FromQuery(Name = "from")] string from, + [FromQuery(Name = "to")] string to, string q) { QueryResult> result; @@ -33,13 +34,13 @@ public async Task> Messages( { result = await allMessagesApi.Execute( new ScatterGatherApiMessageViewWithSystemMessagesContext(pagingInfo, - sortInfo, false, range), + sortInfo, false, new DateTimeRange(from, to)), Request.GetEncodedPathAndQuery()); } else { result = await searchApi.Execute( - new SearchApiContext(pagingInfo, sortInfo, q, range), + new SearchApiContext(pagingInfo, sortInfo, q, new DateTimeRange(from, to)), Request.GetEncodedPathAndQuery()); } } @@ -49,12 +50,12 @@ public async Task> Messages( { result = await allMessagesEndpointApi.Execute( new AllMessagesForEndpointContext(pagingInfo, sortInfo, false, - endpointName, range), + endpointName, new DateTimeRange(from, to)), Request.GetEncodedPathAndQuery()); } else { - result = await searchEndpointApi.Execute(new SearchEndpointContext(pagingInfo, sortInfo, q, endpointName, range), + result = await searchEndpointApi.Execute(new SearchEndpointContext(pagingInfo, sortInfo, q, endpointName, new DateTimeRange(from, to)), Request.GetEncodedPathAndQuery()); } } diff --git a/src/ServiceControl/CompositeViews/Messages/ScatterGatherApiMessageView.cs b/src/ServiceControl/CompositeViews/Messages/ScatterGatherApiMessageView.cs index 840a190415..542ba6daba 100644 --- a/src/ServiceControl/CompositeViews/Messages/ScatterGatherApiMessageView.cs +++ b/src/ServiceControl/CompositeViews/Messages/ScatterGatherApiMessageView.cs @@ -10,9 +10,9 @@ public record ScatterGatherApiMessageViewWithSystemMessagesContext( PagingInfo PagingInfo, SortInfo SortInfo, bool IncludeSystemMessages, - string TimeSentRange = null) : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo, TimeSentRange); + DateTimeRange TimeSentRange = null) : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo, TimeSentRange); - public record ScatterGatherApiMessageViewContext(PagingInfo PagingInfo, SortInfo SortInfo, string TimeSentRange = null) : ScatterGatherContext(PagingInfo); + public record ScatterGatherApiMessageViewContext(PagingInfo PagingInfo, SortInfo SortInfo, DateTimeRange TimeSentRange = null) : ScatterGatherContext(PagingInfo); public abstract class ScatterGatherApiMessageView : ScatterGatherApi> where TInput : ScatterGatherApiMessageViewContext diff --git a/src/ServiceControl/CompositeViews/Messages/SearchApi.cs b/src/ServiceControl/CompositeViews/Messages/SearchApi.cs index b59382998b..dd6ba8afb3 100644 --- a/src/ServiceControl/CompositeViews/Messages/SearchApi.cs +++ b/src/ServiceControl/CompositeViews/Messages/SearchApi.cs @@ -11,7 +11,7 @@ public record SearchApiContext( PagingInfo PagingInfo, SortInfo SortInfo, string SearchQuery, - string TimeSentRange = null) + DateTimeRange TimeSentRange = null) : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo, TimeSentRange); public class SearchApi : ScatterGatherApiMessageView diff --git a/src/ServiceControl/CompositeViews/Messages/SearchEndpointApi.cs b/src/ServiceControl/CompositeViews/Messages/SearchEndpointApi.cs index f8dd51ecfb..990beae427 100644 --- a/src/ServiceControl/CompositeViews/Messages/SearchEndpointApi.cs +++ b/src/ServiceControl/CompositeViews/Messages/SearchEndpointApi.cs @@ -12,7 +12,7 @@ public record SearchEndpointContext( SortInfo SortInfo, string Keyword, string Endpoint, - string TimeSentRange = null) + DateTimeRange TimeSentRange = null) : ScatterGatherApiMessageViewContext(PagingInfo, SortInfo, TimeSentRange); public class SearchEndpointApi : ScatterGatherApiMessageView From cc8183b1d4dbff1ddcf7e5feb24b9e29a261dc47 Mon Sep 17 00:00:00 2001 From: John Simons Date: Tue, 8 Apr 2025 12:09:03 +1000 Subject: [PATCH 3/4] Updated approval files --- .../ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt | 1 + .../ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt index 67a59c4d88..c9303ead9c 100644 --- a/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt +++ b/src/ServiceControl.Audit.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt @@ -12,4 +12,5 @@ GET /messages => ServiceControl.Audit.Auditing.MessagesView.GetMessagesControlle GET /messages/{id}/body => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:Get(String id, CancellationToken cancellationToken) GET /messages/search => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:Search(PagingInfo pagingInfo, SortInfo sortInfo, String q, CancellationToken cancellationToken) GET /messages/search/{keyword} => ServiceControl.Audit.Auditing.MessagesView.GetMessagesController:SearchByKeyWord(PagingInfo pagingInfo, SortInfo sortInfo, String keyword, CancellationToken cancellationToken) +GET /messages2 => ServiceControl.Audit.Auditing.MessagesView.GetMessages2Controller:GetAllMessages(SortInfo sortInfo, Int32 pageSize, String endpointName, String from, String to, String q, CancellationToken cancellationToken) GET /sagas/{id} => ServiceControl.Audit.SagaAudit.SagasController:Sagas(PagingInfo pagingInfo, Guid id, CancellationToken cancellationToken) diff --git a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt index dc3a14e4ad..c33a62237d 100644 --- a/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt +++ b/src/ServiceControl.UnitTests/ApprovalFiles/APIApprovals.HttpApiRoutes.approved.txt @@ -45,6 +45,7 @@ GET /messages => ServiceControl.CompositeViews.Messages.GetMessagesController:Me GET /messages/{id}/body => ServiceControl.CompositeViews.Messages.GetMessagesController:Get(String id, String instanceId) GET /messages/search => ServiceControl.CompositeViews.Messages.GetMessagesController:Search(PagingInfo pagingInfo, SortInfo sortInfo, String q) GET /messages/search/{keyword} => ServiceControl.CompositeViews.Messages.GetMessagesController:SearchByKeyWord(PagingInfo pagingInfo, SortInfo sortInfo, String keyword) +GET /messages2 => ServiceControl.CompositeViews.Messages.GetMessages2Controller:Messages(SortInfo sortInfo, Int32 pageSize, String endpointName, String from, String to, String q) GET /notifications/email => ServiceControl.Notifications.Api.NotificationsController:GetEmailNotificationsSettings() POST /notifications/email => ServiceControl.Notifications.Api.NotificationsController:UpdateSettings(UpdateEmailNotificationsSettingsRequest request) POST /notifications/email/test => ServiceControl.Notifications.Api.NotificationsController:SendTestEmail() From ed60f9feb1cea76a4f5709f2e249436642772b73 Mon Sep 17 00:00:00 2001 From: John Simons Date: Tue, 8 Apr 2025 12:29:27 +1000 Subject: [PATCH 4/4] Add missing paging info --- .../ScatterGather/MessageView_ScatterGatherTest.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.UnitTests/ScatterGather/MessageView_ScatterGatherTest.cs b/src/ServiceControl.UnitTests/ScatterGather/MessageView_ScatterGatherTest.cs index 7ddb2cbdbf..d74e2cac8c 100644 --- a/src/ServiceControl.UnitTests/ScatterGather/MessageView_ScatterGatherTest.cs +++ b/src/ServiceControl.UnitTests/ScatterGather/MessageView_ScatterGatherTest.cs @@ -17,7 +17,7 @@ public void SetUp() { var api = new TestApi(null, null, null); - Results = api.AggregateResults(new ScatterGatherApiMessageViewContext(null, new SortInfo()), GetData()); + Results = api.AggregateResults(new ScatterGatherApiMessageViewContext(new PagingInfo(), new SortInfo()), GetData()); } protected abstract QueryResult>[] GetData();