Skip to content

Commit 3374b11

Browse files
andreasohlundandreasohlund
authored
Better exception management for Azure ServiceBus (#1131)
* Check result from azure monitor better * Better output for azure service bus results * Formatting * Better output * Include dates * Output dates for not found
1 parent 66802e7 commit 3374b11

File tree

2 files changed

+74
-35
lines changed

2 files changed

+74
-35
lines changed

‎src/AppCommon/Commands/AzureServiceBusCommand.cs‎

Lines changed: 37 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,7 @@ public static Command CreateCommand()
7474
string[] queueNames;
7575

7676
public AzureServiceBusCommand(SharedOptions shared, string resourceId, string serviceBusDomain, string region, string metricsDomain)
77-
: base(shared)
77+
: base(shared)
7878
{
7979
azure = new AzureClient(resourceId, serviceBusDomain, region, metricsDomain, Out.WriteLine);
8080
RunInfo.Add("AzureServiceBusNamespace", azure.FullyQualifiedNamespace);
@@ -84,8 +84,8 @@ protected override async Task<QueueDetails> GetData(CancellationToken cancellati
8484
{
8585
try
8686
{
87-
var endTime = DateOnly.FromDateTime(DateTime.UtcNow).AddDays(-1);
88-
var startTime = endTime.AddDays(-90);
87+
var endTime = DateOnly.FromDateTime(DateTime.UtcNow);
88+
var startTime = endTime.AddDays(-90); // Azure Monitor only gives a data for a month back, but we ask for more just in case
8989
var results = new List<QueueThroughput>();
9090

9191
azure.ResetConnectionQueue();
@@ -95,40 +95,51 @@ protected override async Task<QueueDetails> GetData(CancellationToken cancellati
9595
{
9696
var queueName = queueNames[i];
9797

98-
Out.WriteLine($"Gathering metrics for queue {i + 1}/{queueNames.Length}: {queueName}");
98+
Out.Write($"Gathering metrics for queue {i + 1}/{queueNames.Length}: {queueName}");
9999

100100
var metricValues = (await azure.GetMetrics(queueName, startTime, endTime, cancellationToken)).OrderBy(m => m.TimeStamp).ToArray();
101101

102-
if (metricValues is not null)
103-
{
104-
var maxThroughput = metricValues.Select(timeEntry => timeEntry.Total).Max();
102+
var maxThroughput = metricValues.Select(timeEntry => timeEntry.Total).Max();
103+
var start = DateOnly.FromDateTime(metricValues.First().TimeStamp.UtcDateTime);
104+
var end = DateOnly.FromDateTime(metricValues.Last().TimeStamp.UtcDateTime);
105105

106-
// Since we get 90 days of data, if there's no throughput in that amount of time, hard to legitimately call it an endpoint
107-
if (maxThroughput is not null and not 0)
106+
// If there's no throughput in that amount of time, hard to legitimately call it an endpoint
107+
if (maxThroughput is not null and not 0)
108+
{
109+
var currentDate = start;
110+
var data = new Dictionary<DateOnly, DailyThroughput>();
111+
while (currentDate <= end)
108112
{
109-
var start = DateOnly.FromDateTime(metricValues.First().TimeStamp.UtcDateTime);
110-
var end = DateOnly.FromDateTime(metricValues.Last().TimeStamp.UtcDateTime);
111-
var currentDate = start;
112-
var data = new Dictionary<DateOnly, DailyThroughput>();
113-
while (currentDate <= end)
113+
data.Add(currentDate, new DailyThroughput
114114
{
115-
data.Add(currentDate, new DailyThroughput { MessageCount = 0, DateUTC = currentDate });
115+
MessageCount = 0,
116+
DateUTC = currentDate
117+
});
116118

117-
currentDate = currentDate.AddDays(1);
118-
}
119+
currentDate = currentDate.AddDays(1);
120+
}
119121

120-
foreach (var metricValue in metricValues)
122+
foreach (var metricValue in metricValues)
123+
{
124+
currentDate = DateOnly.FromDateTime(metricValue.TimeStamp.UtcDateTime);
125+
data[currentDate] = new DailyThroughput
121126
{
122-
currentDate = DateOnly.FromDateTime(metricValue.TimeStamp.UtcDateTime);
123-
data[currentDate] = new DailyThroughput { MessageCount = (long)(metricValue.Total ?? 0), DateUTC = currentDate };
124-
}
125-
126-
results.Add(new QueueThroughput { QueueName = queueName, Throughput = (long?)maxThroughput, DailyThroughputFromBroker = [.. data.Values] });
127+
MessageCount = (long)(metricValue.Total ?? 0),
128+
DateUTC = currentDate
129+
};
127130
}
128-
else
131+
132+
results.Add(new QueueThroughput
129133
{
130-
Out.WriteLine(" - No throughput detected in 90 days, ignoring");
131-
}
134+
QueueName = queueName,
135+
Throughput = (long?)maxThroughput,
136+
DailyThroughputFromBroker = [.. data.Values]
137+
});
138+
Out.WriteLine($" - Max daily throughput: {maxThroughput} ({start.ToShortDateString()} - {end.ToShortDateString()})");
139+
}
140+
else
141+
{
142+
Out.WriteLine($" - No throughput detected for the period {start.ToShortDateString()} - {end.ToShortDateString()}, ignoring");
132143
}
133144
}
134145

‎src/Query/AzureServiceBus/AzureClient.cs‎

Lines changed: 37 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,16 @@ public class AzureClient
2020

2121
Queue<AuthenticatedClientSet> connectionQueue;
2222
AuthenticatedClientSet currentClients;
23+
const string CompleteMessageMetricName = "CompleteMessage";
24+
const string MicrosoftServicebusNamespacesMetricsNamespace = "Microsoft.ServiceBus/Namespaces";
2325

2426
public string FullyQualifiedNamespace { get; }
27+
2528
public AzureClient(string resourceId, string serviceBusDomain, string region, string metricsDomain, Action<string> log = null)
2629
{
2730
this.resourceId = ResourceIdentifier.Parse(resourceId);
2831

29-
this.log = log ?? new(msg => { });
32+
this.log = log ?? (_ => { });
3033

3134
FullyQualifiedNamespace = $"{this.resourceId.Name}.{serviceBusDomain}";
3235

@@ -43,9 +46,14 @@ IEnumerable<TokenCredential> CreateCredentials()
4346
yield return new VisualStudioCredential();
4447

4548
// Don't really need this one to take 100s * 4 tries to finally time out
46-
var opts = new TokenCredentialOptions();
47-
opts.Retry.MaxRetries = 1;
48-
opts.Retry.NetworkTimeout = TimeSpan.FromSeconds(10);
49+
var opts = new TokenCredentialOptions
50+
{
51+
Retry =
52+
{
53+
MaxRetries = 1,
54+
NetworkTimeout = TimeSpan.FromSeconds(10)
55+
}
56+
};
4957
yield return new ManagedIdentityCredential(FullyQualifiedNamespace, opts);
5058
}
5159

@@ -108,8 +116,8 @@ public Task<IList<MetricValue>> GetMetrics(string queueName, DateOnly startTime,
108116
{
109117
var response = await currentClients.Metrics.QueryResourcesAsync(
110118
[resourceId],
111-
["CompleteMessage"],
112-
"Microsoft.ServiceBus/Namespaces",
119+
[CompleteMessageMetricName],
120+
MicrosoftServicebusNamespacesMetricsNamespace,
113121
new MetricsQueryResourcesOptions
114122
{
115123
Filter = $"EntityName eq '{queueName}'",
@@ -119,8 +127,28 @@ public Task<IList<MetricValue>> GetMetrics(string queueName, DateOnly startTime,
119127
},
120128
token).ConfigureAwait(false);
121129

122-
// Yeah, it's buried deep
123-
return response.Value.Values.FirstOrDefault()?.Metrics.FirstOrDefault()?.TimeSeries.FirstOrDefault()?.Values ?? [];
130+
var metricQueryResult = response.Value.Values.SingleOrDefault(mr => mr.Namespace == MicrosoftServicebusNamespacesMetricsNamespace);
131+
132+
if (metricQueryResult is null)
133+
{
134+
throw new Exception("No metrics query results returned for Microsoft.ServiceBus/Namespace");
135+
}
136+
137+
var metricResult = metricQueryResult.GetMetricByName(CompleteMessageMetricName);
138+
139+
if (metricResult.Error.Message is not null)
140+
{
141+
throw new Exception($"Metrics query result for '{metricResult.Name}' failed: {metricResult.Error.Message}");
142+
}
143+
144+
var timeSeries = metricResult.TimeSeries.SingleOrDefault();
145+
146+
if (timeSeries is null)
147+
{
148+
throw new Exception($"Metrics query result for '{metricResult.Name}' contained no time series");
149+
}
150+
151+
return timeSeries.Values;
124152
}
125153
catch (Azure.RequestFailedException reqFailed) when (reqFailed.Message.Contains("ResourceGroupNotFound"))
126154
{
@@ -136,7 +164,7 @@ public Task<string[]> GetQueueNames(CancellationToken cancellationToken = defaul
136164
return GetDataWithCurrentCredentials(async token =>
137165
{
138166
var queueList = new List<string>();
139-
await foreach (var queue in currentClients.ServiceBus.GetQueuesAsync(cancellationToken).WithCancellation(cancellationToken))
167+
await foreach (var queue in currentClients.ServiceBus.GetQueuesAsync(token))
140168
{
141169
queueList.Add(queue.Name);
142170
}

0 commit comments

Comments
 (0)