From 6caf83f9f62e3357ed41d7d8e2aba5451c091ea6 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 16 Dec 2025 18:46:20 +0100 Subject: [PATCH 01/11] Update SQS query logic to improve time handling and CloudWatch metric collection - Adjusted `Initialize` to use the most eastern timezone UTC offset for accurate time handling to validate internally using TUTC - Fixed Start/End timestamps to correctly be Utc - Upgraded `AWSSDK.CloudWatch` package to version 4.0.6.1 for latest fixes and improvements. - Replaced manual cancellation handling by utilizing `TestContext.CurrentContext.CancellationToken` to improve test cancellation flow. --- src/Directory.Packages.props | 2 +- .../AmazonSQSQueryTests.cs | 22 +++++++++---------- .../AmazonSQSQuery.cs | 21 +++++++++++------- 3 files changed, 25 insertions(+), 20 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index dcdb33271a..cc16aa7e84 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -5,7 +5,7 @@ - + diff --git a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs index 0f537814bd..e03b665eab 100644 --- a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs +++ b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs @@ -25,12 +25,13 @@ class AmazonSQSQueryTests : TransportTestFixture public void Initialise() { provider = new(); - provider.SetUtcNow(DateTimeOffset.UtcNow); + + var kiribati = TimeZoneInfo.FindSystemTimeZoneById("Pacific/Kiritimati"); + var furthestAhead = TimeZoneInfo.ConvertTime(DateTimeOffset.UtcNow, kiribati); + provider.SetUtcNow(furthestAhead); transportSettings = new TransportSettings { - ConnectionString = configuration.ConnectionString, - MaxConcurrency = 1, - EndpointName = Guid.NewGuid().ToString("N") + ConnectionString = configuration.ConnectionString, MaxConcurrency = 1, EndpointName = Guid.NewGuid().ToString("N") }; query = new AmazonSQSQuery(NullLogger.Instance, provider, transportSettings); } @@ -94,11 +95,9 @@ public async Task TestConnectionWithValidSettings() } [Test] + [CancelAfter(6 * 60 * 1000)] public async Task RunScenario() { - // We need to wait a bit of time, to ensure AWS metrics are retrievable - using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromMinutes(6)); - CancellationToken token = cancellationTokenSource.Token; const int numMessagesToIngest = 15; await CreateTestQueue(transportSettings.EndpointName); @@ -120,12 +119,13 @@ public async Task RunScenario() dictionary.Add(AmazonSQSQuery.AmazonSQSSettings.Region, connectionString.Region); } - query.Initialize(new ReadOnlyDictionary(dictionary)); + query.Initialize(dictionary.AsReadOnly()); - await Task.Delay(TimeSpan.FromMinutes(2), token); + // Wait for metrics to become visible, usually takes 20-30 seconds + await Task.Delay(TimeSpan.FromMinutes(1), TestContext.CurrentContext.CancellationToken); var queueNames = new List(); - await foreach (IBrokerQueue queueName in query.GetQueueNames(token)) + await foreach (IBrokerQueue queueName in query.GetQueueNames(TestContext.CurrentContext.CancellationToken)) { queueNames.Add(queueName); } @@ -137,7 +137,7 @@ public async Task RunScenario() DateTime startDate = provider.GetUtcNow().DateTime; provider.Advance(TimeSpan.FromDays(1)); - await foreach (QueueThroughput queueThroughput in query.GetThroughputPerDay(queue, DateOnly.FromDateTime(startDate), token)) + await foreach (QueueThroughput queueThroughput in query.GetThroughputPerDay(queue, DateOnly.FromDateTime(startDate), TestContext.CurrentContext.CancellationToken)) { total += queueThroughput.TotalThroughput; } diff --git a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs index 0c54ca7f0b..d4960cb257 100644 --- a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs +++ b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs @@ -197,24 +197,30 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro DateOnly startDate, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var endDate = DateOnly.FromDateTime(timeProvider.GetUtcNow().DateTime).AddDays(-1); + var utcNow = timeProvider.GetUtcNow(); + var endDate = DateOnly.FromDateTime(utcNow.DateTime).AddDays(-1); // Query date up to but not including today if (endDate < startDate) { yield break; } + var startUtc = startDate.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); + var endUtc = endDate.AddDays(1).ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); + + const int SecondsInDay = 24 * 60 * 60;; + var req = new GetMetricStatisticsRequest { Namespace = "AWS/SQS", MetricName = "NumberOfMessagesDeleted", - StartTime = startDate.ToDateTime(TimeOnly.MinValue), - EndTime = endDate.ToDateTime(TimeOnly.MaxValue), - Period = 24 * 60 * 60, // 1 day + StartTime = startUtc, + EndTime = endUtc, // exclusive + Period = SecondsInDay, Statistics = ["Sum"], Dimensions = [ new Dimension { Name = "QueueName", Value = brokerQueue.QueueName } - ] + ], }; var resp = await cloudWatch!.GetMetricStatisticsAsync(req, cancellationToken); @@ -228,11 +234,10 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro currentDate = currentDate.AddDays(1); } + // Cloudwatch returns data points per 5 minutes in UTC foreach (var datapoint in resp.Datapoints ?? []) { - // There is a bug in the AWS SDK. The timestamp is actually UTC time, eventhough the DateTime returned type says Local - // See https://github.com/aws/aws-sdk-net/issues/167 - // So do not convert the timestamp to UTC time! + logger.LogInformation("\tDatapoint {Timestamp:O} {Sum} {Unit}", datapoint.Timestamp, datapoint.Sum, datapoint.Unit); if (datapoint.Timestamp.HasValue) { data[DateOnly.FromDateTime(datapoint.Timestamp.Value)].TotalThroughput = (long)datapoint.Sum.GetValueOrDefault(0); From e233c8eafc3c834aefae23f00015f901389b2300 Mon Sep 17 00:00:00 2001 From: Mauro Servienti Date: Tue, 16 Dec 2025 20:51:53 +0100 Subject: [PATCH 02/11] Apply suggestion from @andreasohlund MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Andreas Öhlund --- src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs index d4960cb257..0dee140a4e 100644 --- a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs +++ b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs @@ -208,7 +208,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro var startUtc = startDate.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); var endUtc = endDate.AddDays(1).ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); - const int SecondsInDay = 24 * 60 * 60;; + const int SecondsInDay = 24 * 60 * 60; var req = new GetMetricStatisticsRequest { From db30467607775be4d079ba148c5d31aad3474efc Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 17 Dec 2025 10:01:56 +0100 Subject: [PATCH 03/11] Clean up formatting --- src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs index 0dee140a4e..8eeabc3885 100644 --- a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs +++ b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs @@ -215,12 +215,16 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro Namespace = "AWS/SQS", MetricName = "NumberOfMessagesDeleted", StartTime = startUtc, - EndTime = endUtc, // exclusive + EndTime = endUtc, // exclusive Period = SecondsInDay, Statistics = ["Sum"], - Dimensions = [ - new Dimension { Name = "QueueName", Value = brokerQueue.QueueName } - ], + Dimensions = + [ + new Dimension + { + Name = "QueueName", Value = brokerQueue.QueueName + } + ] }; var resp = await cloudWatch!.GetMetricStatisticsAsync(req, cancellationToken); From 590f0874e08b1933bc96f9a1c0cdaa57c89b8fca Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 17 Dec 2025 10:03:13 +0100 Subject: [PATCH 04/11] Defencive handling for when we get more data from the api outside the range requested --- src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs index 8eeabc3885..c1e26ba336 100644 --- a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs +++ b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs @@ -244,7 +244,14 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro logger.LogInformation("\tDatapoint {Timestamp:O} {Sum} {Unit}", datapoint.Timestamp, datapoint.Sum, datapoint.Unit); if (datapoint.Timestamp.HasValue) { - data[DateOnly.FromDateTime(datapoint.Timestamp.Value)].TotalThroughput = (long)datapoint.Sum.GetValueOrDefault(0); + if (data.TryGetValue(DateOnly.FromDateTime(datapoint.Timestamp.Value), out var queueThroughput)) + { + queueThroughput.TotalThroughput = (long)datapoint.Sum.GetValueOrDefault(0); + } + else + { + logger.LogWarning("Datapoint for unknown date {Timestamp:O}", datapoint.Timestamp); + } } } From 1ebc43c53fe825f7465269e987cc682bb49d3f0f Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 17 Dec 2025 10:31:28 +0100 Subject: [PATCH 05/11] Reduce test timeout and improve reliable throughput validation in `AmazonSQSQueryTests`. --- .../AmazonSQSQueryTests.cs | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs index e03b665eab..914fdd4ce3 100644 --- a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs +++ b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs @@ -95,7 +95,7 @@ public async Task TestConnectionWithValidSettings() } [Test] - [CancelAfter(6 * 60 * 1000)] + [CancelAfter(2 * 60 * 1000)] public async Task RunScenario() { const int numMessagesToIngest = 15; @@ -110,10 +110,12 @@ public async Task RunScenario() { dictionary.Add(AmazonSQSQuery.AmazonSQSSettings.AccessKey, connectionString.AccessKey); } + if (!string.IsNullOrEmpty(connectionString.SecretKey)) { dictionary.Add(AmazonSQSQuery.AmazonSQSSettings.SecretKey, connectionString.SecretKey); } + if (!string.IsNullOrEmpty(connectionString.Region)) { dictionary.Add(AmazonSQSQuery.AmazonSQSSettings.Region, connectionString.Region); @@ -121,27 +123,39 @@ public async Task RunScenario() query.Initialize(dictionary.AsReadOnly()); - // Wait for metrics to become visible, usually takes 20-30 seconds - await Task.Delay(TimeSpan.FromMinutes(1), TestContext.CurrentContext.CancellationToken); + DateTime startDate = provider.GetUtcNow().DateTime; + provider.Advance(TimeSpan.FromDays(1)); - var queueNames = new List(); - await foreach (IBrokerQueue queueName in query.GetQueueNames(TestContext.CurrentContext.CancellationToken)) + while (!TestContext.CurrentContext.CancellationToken.IsCancellationRequested) { - queueNames.Add(queueName); - } + await Task.Delay(TimeSpan.FromSeconds(5), TestContext.CurrentContext.CancellationToken); - IBrokerQueue queue = queueNames.Find(name => name.QueueName == $"{connectionString.QueueNamePrefix}{transportSettings.EndpointName}"); - Assert.That(queue, Is.Not.Null); + var queueNames = new List(); + await foreach (IBrokerQueue queueName in query.GetQueueNames(TestContext.CurrentContext.CancellationToken)) + { + queueNames.Add(queueName); + } - long total = 0L; + IBrokerQueue queue = queueNames.Find(name => name.QueueName == $"{connectionString.QueueNamePrefix}{transportSettings.EndpointName}"); - DateTime startDate = provider.GetUtcNow().DateTime; - provider.Advance(TimeSpan.FromDays(1)); - await foreach (QueueThroughput queueThroughput in query.GetThroughputPerDay(queue, DateOnly.FromDateTime(startDate), TestContext.CurrentContext.CancellationToken)) - { - total += queueThroughput.TotalThroughput; + if (queue == null) + { + continue; + } + + long total = 0L; + + await foreach (QueueThroughput queueThroughput in query.GetThroughputPerDay(queue, DateOnly.FromDateTime(startDate), TestContext.CurrentContext.CancellationToken)) + { + total += queueThroughput.TotalThroughput; + } + + if (total == numMessagesToIngest) + { + return; + } } - Assert.That(total, Is.EqualTo(numMessagesToIngest)); + Assert.Fail("Timeout waiting for expected throughput to be report"); } } \ No newline at end of file From 33d14d3210e36f6a5fd3e235735e7a312bc6c09a Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 17 Dec 2025 10:52:41 +0100 Subject: [PATCH 06/11] revert formatting --- .../AmazonSQSQueryTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs index 914fdd4ce3..52adde748e 100644 --- a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs +++ b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs @@ -31,7 +31,9 @@ public void Initialise() provider.SetUtcNow(furthestAhead); transportSettings = new TransportSettings { - ConnectionString = configuration.ConnectionString, MaxConcurrency = 1, EndpointName = Guid.NewGuid().ToString("N") + ConnectionString = configuration.ConnectionString, + MaxConcurrency = 1, + EndpointName = Guid.NewGuid().ToString("N") }; query = new AmazonSQSQuery(NullLogger.Instance, provider, transportSettings); } From b17d7f5ea9e0f4f2b989cba59092377b8d8f21eb Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 18 Dec 2025 10:53:05 +0100 Subject: [PATCH 07/11] Improve logging and parameter handling in `GetThroughputPerDay` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added trace-level logging for skipped dates and query execution details. - Renamed variables for clarity (`startUtc` → `queryStartUtc`, `endUtc` → `queryendUtc`). - Adjusted log level for CloudWatch datapoints (`Information` → `Trace`). --- .../AmazonSQSQuery.cs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs index c1e26ba336..e5e255ec34 100644 --- a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs +++ b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs @@ -195,18 +195,23 @@ class AwsHttpClientFactory : HttpClientFactory public override async IAsyncEnumerable GetThroughputPerDay(IBrokerQueue brokerQueue, DateOnly startDate, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + [EnumeratorCancellation] CancellationToken cancellationToken) { var utcNow = timeProvider.GetUtcNow(); var endDate = DateOnly.FromDateTime(utcNow.DateTime).AddDays(-1); // Query date up to but not including today - if (endDate < startDate) + var isBeforeStartDate = endDate < startDate; + + if (isBeforeStartDate) { + logger.LogTrace("Skipping {Start} {End} {UtcNow}, ", startDate, endDate, utcNow); yield break; } - var startUtc = startDate.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); - var endUtc = endDate.AddDays(1).ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); + var queryStartUtc = startDate.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); + var queryendUtc = endDate.AddDays(1).ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); + + logger.LogDebug("GetThroughputPerDay {QueueName} {UtcNow} {StartDate} {EndDate} {QueryStart} {QueryEnd}", brokerQueue.QueueName, utcNow, startDate, endDate, queryStartUtc, queryendUtc); const int SecondsInDay = 24 * 60 * 60; @@ -214,8 +219,8 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro { Namespace = "AWS/SQS", MetricName = "NumberOfMessagesDeleted", - StartTime = startUtc, - EndTime = endUtc, // exclusive + StartTime = queryStartUtc, + EndTime = queryendUtc, // exclusive Period = SecondsInDay, Statistics = ["Sum"], Dimensions = @@ -241,7 +246,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro // Cloudwatch returns data points per 5 minutes in UTC foreach (var datapoint in resp.Datapoints ?? []) { - logger.LogInformation("\tDatapoint {Timestamp:O} {Sum} {Unit}", datapoint.Timestamp, datapoint.Sum, datapoint.Unit); + logger.LogTrace("Datapoint {Timestamp:O} {Sum:N0}", datapoint.Timestamp, datapoint.Sum); if (datapoint.Timestamp.HasValue) { if (data.TryGetValue(DateOnly.FromDateTime(datapoint.Timestamp.Value), out var queueThroughput)) From 6971fb14b0f2915cb1dbd34dc30bbb1b91491aa6 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 18 Dec 2025 11:23:14 +0100 Subject: [PATCH 08/11] Added comments to document WHY the logic is like it is --- src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs index e5e255ec34..f502909c78 100644 --- a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs +++ b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs @@ -208,10 +208,15 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro yield break; } + // Convert DATES that state up to but INCLUDING the TO value to a timestamp from X to Y EXCLUDING + // Example: 2025-01-01 to 2025-01-10 => 2025-01-01T00:00:00 to 2025-01-11T00:00:00 + var queryStartUtc = startDate.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); - var queryendUtc = endDate.AddDays(1).ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); + var queryEndUtc = endDate + .AddDays(1) // Convert from INCLUDING to EXCLUDING, thus need to bump one day, using ToDateTime(TimeOnly.MaxValue) would be wrong + .ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc); - logger.LogDebug("GetThroughputPerDay {QueueName} {UtcNow} {StartDate} {EndDate} {QueryStart} {QueryEnd}", brokerQueue.QueueName, utcNow, startDate, endDate, queryStartUtc, queryendUtc); + logger.LogDebug("GetThroughputPerDay {QueueName} {UtcNow} {StartDate} {EndDate} {QueryStart} {QueryEnd}", brokerQueue.QueueName, utcNow, startDate, endDate, queryStartUtc, queryEndUtc); const int SecondsInDay = 24 * 60 * 60; @@ -220,7 +225,7 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro Namespace = "AWS/SQS", MetricName = "NumberOfMessagesDeleted", StartTime = queryStartUtc, - EndTime = queryendUtc, // exclusive + EndTime = queryEndUtc, // The value specified is exclusive; results include data points up to the specified time stamp. Period = SecondsInDay, Statistics = ["Sum"], Dimensions = From a07635221c5e00baf78b62a39c37aa4bd11510d6 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 18 Dec 2025 12:33:44 +0100 Subject: [PATCH 09/11] Replace usage of `NullLogger` with `LoggerFactory` with console output in `AmazonSQSQueryTests` --- src/Directory.Packages.props | 1 + .../AmazonSQSQueryTests.cs | 10 ++++++---- .../ServiceControl.Transports.SQS.Tests.csproj | 1 + 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Directory.Packages.props b/src/Directory.Packages.props index cc16aa7e84..e9cefcd8da 100644 --- a/src/Directory.Packages.props +++ b/src/Directory.Packages.props @@ -27,6 +27,7 @@ + diff --git a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs index 52adde748e..badf2c71c6 100644 --- a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs +++ b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs @@ -6,7 +6,7 @@ namespace ServiceControl.Transport.Tests; using System.Linq; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Time.Testing; using NUnit.Framework; using Particular.Approvals; @@ -35,7 +35,9 @@ public void Initialise() MaxConcurrency = 1, EndpointName = Guid.NewGuid().ToString("N") }; - query = new AmazonSQSQuery(NullLogger.Instance, provider, transportSettings); + var loggerFactory = LoggerFactory.Create(builder => builder.AddSimpleConsole().SetMinimumLevel(LogLevel.Trace)); + var logger = loggerFactory.CreateLogger(); + query = new AmazonSQSQuery(logger, provider, transportSettings); } [Test] @@ -125,7 +127,7 @@ public async Task RunScenario() query.Initialize(dictionary.AsReadOnly()); - DateTime startDate = provider.GetUtcNow().DateTime; + var startDate = DateOnly.FromDateTime(provider.GetUtcNow().DateTime); provider.Advance(TimeSpan.FromDays(1)); while (!TestContext.CurrentContext.CancellationToken.IsCancellationRequested) @@ -147,7 +149,7 @@ public async Task RunScenario() long total = 0L; - await foreach (QueueThroughput queueThroughput in query.GetThroughputPerDay(queue, DateOnly.FromDateTime(startDate), TestContext.CurrentContext.CancellationToken)) + await foreach (QueueThroughput queueThroughput in query.GetThroughputPerDay(queue, startDate, TestContext.CurrentContext.CancellationToken)) { total += queueThroughput.TotalThroughput; } diff --git a/src/ServiceControl.Transports.SQS.Tests/ServiceControl.Transports.SQS.Tests.csproj b/src/ServiceControl.Transports.SQS.Tests/ServiceControl.Transports.SQS.Tests.csproj index 655018e20c..2f2d4bb0e5 100644 --- a/src/ServiceControl.Transports.SQS.Tests/ServiceControl.Transports.SQS.Tests.csproj +++ b/src/ServiceControl.Transports.SQS.Tests/ServiceControl.Transports.SQS.Tests.csproj @@ -13,6 +13,7 @@ + From 92c5aa986fefe3a35bc8835c56243e3e9c398941 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 18 Dec 2025 12:36:28 +0100 Subject: [PATCH 10/11] `FakeTimeProder` does not work as expected when passing a DateTimeOffset with an actuall offset, `.GetUtcNow();` return a DateTimeOffset with that same offset (!!) while this MUST be a value without an offset. --- .../AmazonSQSQueryTests.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs index badf2c71c6..c993b61be8 100644 --- a/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs +++ b/src/ServiceControl.Transports.SQS.Tests/AmazonSQSQueryTests.cs @@ -25,10 +25,7 @@ class AmazonSQSQueryTests : TransportTestFixture public void Initialise() { provider = new(); - - var kiribati = TimeZoneInfo.FindSystemTimeZoneById("Pacific/Kiritimati"); - var furthestAhead = TimeZoneInfo.ConvertTime(DateTimeOffset.UtcNow, kiribati); - provider.SetUtcNow(furthestAhead); + provider.SetUtcNow(DateTimeOffset.UtcNow); transportSettings = new TransportSettings { ConnectionString = configuration.ConnectionString, From e36f244212ac4d82c9f7c063a7d732a788293cec Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 18 Dec 2025 12:44:02 +0100 Subject: [PATCH 11/11] Remove while loop --- .../AmazonSQSQuery.cs | 33 +++++-------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs index f502909c78..c1d44ad09d 100644 --- a/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs +++ b/src/ServiceControl.Transports.SQS/AmazonSQSQuery.cs @@ -238,36 +238,19 @@ public override async IAsyncEnumerable GetThroughputPerDay(IBro }; var resp = await cloudWatch!.GetMetricStatisticsAsync(req, cancellationToken); + var dataPoints = resp.Datapoints.ToDictionary(x => DateOnly.FromDateTime(x.Timestamp!.Value.Date), x => (long)(x.Sum ?? 0)); - DateOnly currentDate = startDate; - var data = new Dictionary(); - while (currentDate <= endDate) + for (DateOnly currentDate = startDate; currentDate <= endDate; currentDate = currentDate.AddDays(1)) { - data.Add(currentDate, new QueueThroughput { TotalThroughput = 0, DateUTC = currentDate }); + dataPoints.TryGetValue(currentDate, out var sum); - currentDate = currentDate.AddDays(1); - } + logger.LogTrace("Queue throughput {QueueName} {Date} {Total}", brokerQueue.QueueName, currentDate, sum); - // Cloudwatch returns data points per 5 minutes in UTC - foreach (var datapoint in resp.Datapoints ?? []) - { - logger.LogTrace("Datapoint {Timestamp:O} {Sum:N0}", datapoint.Timestamp, datapoint.Sum); - if (datapoint.Timestamp.HasValue) + yield return new QueueThroughput { - if (data.TryGetValue(DateOnly.FromDateTime(datapoint.Timestamp.Value), out var queueThroughput)) - { - queueThroughput.TotalThroughput = (long)datapoint.Sum.GetValueOrDefault(0); - } - else - { - logger.LogWarning("Datapoint for unknown date {Timestamp:O}", datapoint.Timestamp); - } - } - } - - foreach (QueueThroughput queueThroughput in data.Values) - { - yield return queueThroughput; + TotalThroughput = sum, + DateUTC = currentDate + }; } }