diff --git a/src/ServiceControl.Infrastructure/AsyncTimer.cs b/src/ServiceControl.Infrastructure/AsyncTimer.cs index 34aa3b62b9..045e8d229e 100644 --- a/src/ServiceControl.Infrastructure/AsyncTimer.cs +++ b/src/ServiceControl.Infrastructure/AsyncTimer.cs @@ -1,96 +1,108 @@ -namespace ServiceControl.Infrastructure.BackgroundTasks +namespace ServiceControl.Infrastructure.BackgroundTasks; + +using System; +using System.Threading; +using System.Threading.Tasks; + +public enum TimerJobExecutionResult { - using System; - using System.Threading; - using System.Threading.Tasks; + ScheduleNextExecution, + ExecuteImmediately, + DoNotContinueExecuting +} - public enum TimerJobExecutionResult +public class TimerJob +{ + public TimerJob(Func> callback, TimeSpan due, TimeSpan interval, Action errorCallback) { - ScheduleNextExecution, - ExecuteImmediately, - DoNotContinueExecuting - } + tokenSource = new CancellationTokenSource(); + var token = tokenSource.Token; - public class TimerJob - { - public TimerJob(Func> callback, TimeSpan due, TimeSpan interval, Action errorCallback) + task = Task.Run(async () => { - tokenSource = new CancellationTokenSource(); - var token = tokenSource.Token; - - task = Task.Run(async () => + try { - try - { - await Task.Delay(due, token).ConfigureAwait(false); + await Task.Delay(due, token).ConfigureAwait(false); + + var consecutiveFailures = 0; - while (!token.IsCancellationRequested) + while (!token.IsCancellationRequested) + { + try { - try - { - var result = await callback(token).ConfigureAwait(false); - if (result == TimerJobExecutionResult.DoNotContinueExecuting) - { - tokenSource.Cancel(); - } - else if (result == TimerJobExecutionResult.ScheduleNextExecution) - { - await Task.Delay(interval, token).ConfigureAwait(false); - } - - //Otherwise execute immediately - } - catch (OperationCanceledException) when (token.IsCancellationRequested) + var result = await callback(token).ConfigureAwait(false); + + consecutiveFailures = 0; + if (result == TimerJobExecutionResult.DoNotContinueExecuting) { - break; + tokenSource.Cancel(); } - catch (Exception ex) + else if (result == TimerJobExecutionResult.ScheduleNextExecution) { - errorCallback(ex); + await Task.Delay(interval, token).ConfigureAwait(false); } + + //Otherwise execute immediately } - } - catch (OperationCanceledException) when (token.IsCancellationRequested) - { - // no-op - } - }, CancellationToken.None); - } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + break; + } + catch (Exception ex) + { + consecutiveFailures++; + const int MaxDelayDurationInSeconds = 60; + var delayInSeconds = consecutiveFailures * 10; + var backoffDelay = TimeSpan.FromSeconds(int.Min(MaxDelayDurationInSeconds, delayInSeconds)); - public async Task Stop() - { - if (tokenSource == null) + await Task.Delay(backoffDelay, token).ConfigureAwait(false); + + errorCallback(ex); + } + } + } + catch (OperationCanceledException) when (token.IsCancellationRequested) { - return; + // no-op } + }, CancellationToken.None); + } - await tokenSource.CancelAsync().ConfigureAwait(false); - tokenSource.Dispose(); + public async Task Stop(CancellationToken cancellationToken) + { + if (tokenSource == null) + { + return; + } - if (task != null) - { - try - { - await task.ConfigureAwait(false); - } - catch (OperationCanceledException) when (tokenSource.IsCancellationRequested) - { - //NOOP - } - } + await tokenSource.CancelAsync().ConfigureAwait(false); + tokenSource.Dispose(); + + if (task == null) + { + return; } - Task task; - CancellationTokenSource tokenSource; + try + { + await task.WaitAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) when (tokenSource.IsCancellationRequested) + { + //NOOP + } } - public interface IAsyncTimer - { - TimerJob Schedule(Func> callback, TimeSpan due, TimeSpan interval, Action errorCallback); - } + readonly Task task; + readonly CancellationTokenSource tokenSource; +} - public class AsyncTimer : IAsyncTimer - { - public TimerJob Schedule(Func> callback, TimeSpan due, TimeSpan interval, Action errorCallback) => new TimerJob(callback, due, interval, errorCallback); - } +public interface IAsyncTimer +{ + TimerJob Schedule(Func> callback, TimeSpan due, TimeSpan interval, Action errorCallback); +} + +public class AsyncTimer : IAsyncTimer +{ + public TimerJob Schedule(Func> callback, TimeSpan due, TimeSpan interval, Action errorCallback) => new(callback, due, interval, errorCallback); } \ No newline at end of file diff --git a/src/ServiceControl.Monitoring/Licensing/LicenseCheckHostedService.cs b/src/ServiceControl.Monitoring/Licensing/LicenseCheckHostedService.cs index 1842f8603d..a5b10f73b5 100644 --- a/src/ServiceControl.Monitoring/Licensing/LicenseCheckHostedService.cs +++ b/src/ServiceControl.Monitoring/Licensing/LicenseCheckHostedService.cs @@ -20,7 +20,7 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(); + public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken); TimerJob timer; diff --git a/src/ServiceControl/CustomChecks/InternalCustomChecks/InternalCustomCheckManager.cs b/src/ServiceControl/CustomChecks/InternalCustomChecks/InternalCustomCheckManager.cs index 38853a667c..e3443cd2e0 100644 --- a/src/ServiceControl/CustomChecks/InternalCustomChecks/InternalCustomCheckManager.cs +++ b/src/ServiceControl/CustomChecks/InternalCustomChecks/InternalCustomCheckManager.cs @@ -67,7 +67,7 @@ async Task Run(CancellationToken cancellationToken) : TimerJobExecutionResult.DoNotContinueExecuting; } - public Task Stop() => timer?.Stop() ?? Task.CompletedTask; + public Task Stop() => timer?.Stop(CancellationToken.None) ?? Task.CompletedTask; TimerJob timer; readonly ICustomCheck check; diff --git a/src/ServiceControl/Licensing/LicenseCheckHostedService.cs b/src/ServiceControl/Licensing/LicenseCheckHostedService.cs index 0bcb97608d..3afcc3c2ba 100644 --- a/src/ServiceControl/Licensing/LicenseCheckHostedService.cs +++ b/src/ServiceControl/Licensing/LicenseCheckHostedService.cs @@ -21,7 +21,7 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(); + public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken); TimerJob timer; diff --git a/src/ServiceControl/Monitoring/HeartbeatMonitoringHostedService.cs b/src/ServiceControl/Monitoring/HeartbeatMonitoringHostedService.cs index a3d86a01ba..1f9d6edaa5 100644 --- a/src/ServiceControl/Monitoring/HeartbeatMonitoringHostedService.cs +++ b/src/ServiceControl/Monitoring/HeartbeatMonitoringHostedService.cs @@ -24,17 +24,7 @@ public async Task StartAsync(CancellationToken cancellationToken) timer = scheduler.Schedule(_ => CheckEndpoints(), TimeSpan.Zero, TimeSpan.FromSeconds(5), e => { log.Error("Exception occurred when monitoring endpoint instances", e); }); } - public async Task StopAsync(CancellationToken cancellationToken) - { - try - { - await timer.Stop(); - } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) - { - //NOOP, invoked Stop does not - } - } + public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken); async Task CheckEndpoints() { diff --git a/src/ServiceControl/Recoverability/RecoverabilityComponent.cs b/src/ServiceControl/Recoverability/RecoverabilityComponent.cs index ce2abbfa7c..1736572799 100644 --- a/src/ServiceControl/Recoverability/RecoverabilityComponent.cs +++ b/src/ServiceControl/Recoverability/RecoverabilityComponent.cs @@ -174,10 +174,7 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - public Task StopAsync(CancellationToken cancellationToken) - { - return timer?.Stop() ?? Task.CompletedTask; - } + public Task StopAsync(CancellationToken cancellationToken) => timer?.Stop(cancellationToken) ?? Task.CompletedTask; async Task ProcessRequestedBulkRetryOperations() { @@ -237,10 +234,7 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - public Task StopAsync(CancellationToken cancellationToken) - { - return timer.Stop(); - } + public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken); TimerJob timer; readonly IAsyncTimer scheduler; @@ -266,10 +260,7 @@ public Task StartAsync(CancellationToken cancellationToken) return Task.CompletedTask; } - public async Task StopAsync(CancellationToken cancellationToken) - { - await timer.Stop(); - } + public Task StopAsync(CancellationToken cancellationToken) => timer.Stop(cancellationToken); async Task Process(CancellationToken cancellationToken) {