From 4810ad4395aea5f7c6168200f81a1755907e3b31 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 18 Feb 2025 17:10:20 +0100 Subject: [PATCH 01/15] Teardown `Raven.Embedded.EmbeddedServer` as part of stop sequence so that we kill the instance if needed as adhering to cancellation is not possible in host builder root container dispose. --- .../IRavenPersistenceLifecycle.cs | 1 + .../RavenEmbeddedPersistenceLifecycle.cs | 2 + .../RavenExternalPersistenceLifecycle.cs | 2 + .../RavenPersistenceLifecycleHostedService.cs | 2 +- .../EmbeddedDatabase.cs | 60 +++++++++++++++++-- 5 files changed, 60 insertions(+), 7 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/IRavenPersistenceLifecycle.cs b/src/ServiceControl.Audit.Persistence.RavenDB/IRavenPersistenceLifecycle.cs index 9c8c896d02..44bfe38473 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/IRavenPersistenceLifecycle.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/IRavenPersistenceLifecycle.cs @@ -6,5 +6,6 @@ interface IRavenPersistenceLifecycle { Task Initialize(CancellationToken cancellationToken = default); + Task Stop(CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs index 14494e673f..6949455510 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs @@ -72,6 +72,8 @@ public async Task Initialize(CancellationToken cancellationToken = default) } } + public Task Stop(CancellationToken cancellationToken) => database!.Stop(cancellationToken); + public void Dispose() { documentStore?.Dispose(); diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs index 81cfbeda5e..b2b8603590 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs @@ -64,6 +64,8 @@ public async Task Initialize(CancellationToken cancellationToken = default) } } + public Task Stop(CancellationToken cancellationToken) => Task.CompletedTask; // We are not stopping an external instance + public void Dispose() => documentStore?.Dispose(); IDocumentStore? documentStore; diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceLifecycleHostedService.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceLifecycleHostedService.cs index f61b5c2fc3..f266850953 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceLifecycleHostedService.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceLifecycleHostedService.cs @@ -8,6 +8,6 @@ sealed class RavenPersistenceLifecycleHostedService(IRavenPersistenceLifecycle l { public Task StartAsync(CancellationToken cancellationToken) => lifecycle.Initialize(cancellationToken); - public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + public Task StopAsync(CancellationToken cancellationToken) => lifecycle.Stop(cancellationToken); } } \ No newline at end of file diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index d54f1b8467..28c614d99d 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -53,7 +53,7 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi var nugetPackagesPath = Path.Combine(databaseConfiguration.DbPath, "Packages", "NuGet"); Logger.InfoFormat("Loading RavenDB license from {0}", licenseFileNameAndServerDirectory.LicenseFileName); - var serverOptions = new ServerOptions + serverOptions = new ServerOptions { CommandLineArgs = [ @@ -162,10 +162,48 @@ public async Task Connect(CancellationToken cancellationToken) public async Task DeleteDatabase(string dbName) { - using var store = await EmbeddedServer.Instance.GetDocumentStoreAsync(new DatabaseOptions(dbName) { SkipCreatingDatabase = true }); + using var store = await EmbeddedServer.Instance.GetDocumentStoreAsync(new DatabaseOptions(dbName) + { + SkipCreatingDatabase = true + }); await store.Maintenance.Server.SendAsync(new DeleteDatabasesOperation(dbName, true)); } + public async Task Stop(CancellationToken cancellationToken) + { + Logger.Debug("Stopping RavenDB server"); + EmbeddedServer.Instance.ServerProcessExited -= OnServerProcessExited; + + await shutdownTokenSource.CancelAsync(); + + // TODO: await EmbeddedServer.Instance.StopAsync(cancellationToken); + + var processId = await EmbeddedServer.Instance.GetServerProcessIdAsync(cancellationToken); + + var waitForCancellationTask = Task.Delay(Timeout.Infinite, cancellationToken); + var firstTask = await Task.WhenAny( + Task.Run(() => EmbeddedServer.Instance.Dispose(), cancellationToken), + waitForCancellationTask + ); + + if (firstTask == waitForCancellationTask) + { + try + { + Logger.Warn("Killing RavenDB child process because host cancelled"); + var ravenChildProcess = Process.GetProcessById(processId); + ravenChildProcess.Kill(entireProcessTree: true); + } + catch (Exception e) + { + Logger.Error("Killing RavenDB child process failed", e); + } + } + + EmbeddedServer.Instance = null!; + Logger.Debug("Stopped RavenDB server"); + } + public void Dispose() { if (disposed) @@ -173,12 +211,21 @@ public void Dispose() return; } - EmbeddedServer.Instance.ServerProcessExited -= OnServerProcessExited; + if (EmbeddedServer.Instance != null) + { + EmbeddedServer.Instance.ServerProcessExited -= OnServerProcessExited; + } shutdownTokenSource.Cancel(); - Logger.Debug("Disposing RavenDB server"); - EmbeddedServer.Instance.Dispose(); - Logger.Debug("Dispose RavenDB server"); + + if (EmbeddedServer.Instance != null) + { + serverOptions.GracefulShutdownTimeout = TimeSpan.Zero; + Logger.Debug("Disposing RavenDB server"); + EmbeddedServer.Instance.Dispose(); + Logger.Debug("Disposed RavenDB server"); + } + shutdownTokenSource.Dispose(); applicationStoppingRegistration.Dispose(); @@ -262,6 +309,7 @@ static long DirSize(DirectoryInfo d) readonly EmbeddedDatabaseConfiguration configuration; readonly CancellationToken shutdownCancellationToken; readonly CancellationTokenRegistration applicationStoppingRegistration; + static ServerOptions serverOptions; static TimeSpan delayBetweenRestarts = TimeSpan.FromSeconds(60); static readonly ILog Logger = LogManager.GetLogger(); From f2bcbe0f53cc112c885ddb17de5acd0866e759d3 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 19 Feb 2025 12:03:25 +0100 Subject: [PATCH 02/15] Removed static --- src/ServiceControl.RavenDB/EmbeddedDatabase.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 28c614d99d..632e023a0b 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -53,7 +53,7 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi var nugetPackagesPath = Path.Combine(databaseConfiguration.DbPath, "Packages", "NuGet"); Logger.InfoFormat("Loading RavenDB license from {0}", licenseFileNameAndServerDirectory.LicenseFileName); - serverOptions = new ServerOptions + var serverOptions = new ServerOptions { CommandLineArgs = [ @@ -85,6 +85,7 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi void Start(ServerOptions serverOptions) { + this.serverOptions = serverOptions; EmbeddedServer.Instance.ServerProcessExited += OnServerProcessExited; EmbeddedServer.Instance.StartServer(serverOptions); @@ -220,7 +221,7 @@ public void Dispose() if (EmbeddedServer.Instance != null) { - serverOptions.GracefulShutdownTimeout = TimeSpan.Zero; + serverOptions!.GracefulShutdownTimeout = TimeSpan.Zero; Logger.Debug("Disposing RavenDB server"); EmbeddedServer.Instance.Dispose(); Logger.Debug("Disposed RavenDB server"); @@ -309,7 +310,7 @@ static long DirSize(DirectoryInfo d) readonly EmbeddedDatabaseConfiguration configuration; readonly CancellationToken shutdownCancellationToken; readonly CancellationTokenRegistration applicationStoppingRegistration; - static ServerOptions serverOptions; + ServerOptions? serverOptions; static TimeSpan delayBetweenRestarts = TimeSpan.FromSeconds(60); static readonly ILog Logger = LogManager.GetLogger(); From 7a4ef15b5b6c659fddca1a31399d5c44ffe02d4f Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 19 Feb 2025 12:08:07 +0100 Subject: [PATCH 03/15] Add why we set `GracefulShutdownTimeout` to Zero during dispose --- src/ServiceControl.RavenDB/EmbeddedDatabase.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 632e023a0b..1256d5de47 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -221,6 +221,8 @@ public void Dispose() if (EmbeddedServer.Instance != null) { + // Set GracefulShutdownTimeout to Zero and exist ASAP, under normal operation instance would already + // have been allowed to gracefully stop during "Stop" method. serverOptions!.GracefulShutdownTimeout = TimeSpan.Zero; Logger.Debug("Disposing RavenDB server"); EmbeddedServer.Instance.Dispose(); From 86893a5973d77fe213ccb914ee77ea19f0d0fb25 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 19 Feb 2025 13:22:51 +0100 Subject: [PATCH 04/15] Enable Raven.Embedded console and file logging --- src/ServiceControl.RavenDB/EmbeddedDatabase.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 1256d5de47..d0b6956145 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -15,6 +15,7 @@ namespace ServiceControl.RavenDB using Raven.Client.Documents.Conventions; using Raven.Client.ServerWide.Operations; using Raven.Embedded; + using Sparrow.Logging; public sealed class EmbeddedDatabase : IDisposable { @@ -52,6 +53,16 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi var nugetPackagesPath = Path.Combine(databaseConfiguration.DbPath, "Packages", "NuGet"); + LoggingSource.Instance.SetupLogMode( + (LogMode)255, + Path.Combine(databaseConfiguration.LogPath, "Raven.Embedded"), + retentionTime: TimeSpan.FromDays(14), + retentionSize: 1024 * 1024 * 10, + compress: false + ); + + LoggingSource.Instance.EnableConsoleLogging(); + Logger.InfoFormat("Loading RavenDB license from {0}", licenseFileNameAndServerDirectory.LicenseFileName); var serverOptions = new ServerOptions { @@ -191,13 +202,13 @@ public async Task Stop(CancellationToken cancellationToken) { try { - Logger.Warn("Killing RavenDB child process because host cancelled"); + Logger.WarnFormat("Killing RavenDB server PID {0} child process because host cancelled", processId); var ravenChildProcess = Process.GetProcessById(processId); ravenChildProcess.Kill(entireProcessTree: true); } catch (Exception e) { - Logger.Error("Killing RavenDB child process failed", e); + Logger.ErrorFormat("Failed to kill RavenDB server PID {0}\n{1}", processId, e); } } @@ -263,6 +274,7 @@ static long DataSize(EmbeddedDatabaseConfiguration configuration) { return -1; } + return info.Length; } catch From 735446ace5ef31b7a905fc98a4fd977bf07bd4f8 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 19 Feb 2025 13:23:29 +0100 Subject: [PATCH 05/15] Set `GracefulShutdownTimeout` to 1 hour, as default is 30 seconds. --- src/ServiceControl.RavenDB/EmbeddedDatabase.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index d0b6956145..4900f149a6 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -66,6 +66,7 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi Logger.InfoFormat("Loading RavenDB license from {0}", licenseFileNameAndServerDirectory.LicenseFileName); var serverOptions = new ServerOptions { + GracefulShutdownTimeout = TimeSpan.FromHours(1), // During Stop/Dispose we manually control this CommandLineArgs = [ $"--Logs.Mode={databaseConfiguration.LogsMode}", From c7760a50ecfcb3fa168c4b2ff3983bff657a3141 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 19 Feb 2025 17:09:52 +0100 Subject: [PATCH 06/15] Workaround for failing tests that seems to re-use the instance. Is that even supported? --- src/ServiceControl.RavenDB/EmbeddedDatabase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 4900f149a6..f770b2c003 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -213,7 +213,7 @@ public async Task Stop(CancellationToken cancellationToken) } } - EmbeddedServer.Instance = null!; + serverOptions = null!; Logger.Debug("Stopped RavenDB server"); } @@ -224,14 +224,14 @@ public void Dispose() return; } - if (EmbeddedServer.Instance != null) + if (serverOptions != null) { EmbeddedServer.Instance.ServerProcessExited -= OnServerProcessExited; } shutdownTokenSource.Cancel(); - if (EmbeddedServer.Instance != null) + if (serverOptions != null) { // Set GracefulShutdownTimeout to Zero and exist ASAP, under normal operation instance would already // have been allowed to gracefully stop during "Stop" method. From 70f75d8812c264ceb362b4347a5c8168709a5f71 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 20 Feb 2025 18:42:18 +0100 Subject: [PATCH 07/15] WaitForExitAsync is needed to wait for process to exit after kill signal --- src/ServiceControl.RavenDB/EmbeddedDatabase.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index f770b2c003..8fa4ed5ae5 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -206,6 +206,9 @@ public async Task Stop(CancellationToken cancellationToken) Logger.WarnFormat("Killing RavenDB server PID {0} child process because host cancelled", processId); var ravenChildProcess = Process.GetProcessById(processId); ravenChildProcess.Kill(entireProcessTree: true); + // Kill only signals + Logger.WarnFormat("Waiting for RavenDB server PID {0} to exit... ", processId); + await ravenChildProcess.WaitForExitAsync(CancellationToken.None); } catch (Exception e) { From 9c0bd7ac8557d8b33c32b7f632caba6552592010 Mon Sep 17 00:00:00 2001 From: Mauro Servienti Date: Tue, 25 Feb 2025 10:13:19 +0100 Subject: [PATCH 08/15] Apply suggestions from Daniel's code review Co-authored-by: Daniel Marbach --- .../IRavenPersistenceLifecycle.cs | 2 +- .../RavenEmbeddedPersistenceLifecycle.cs | 2 +- .../RavenExternalPersistenceLifecycle.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/IRavenPersistenceLifecycle.cs b/src/ServiceControl.Audit.Persistence.RavenDB/IRavenPersistenceLifecycle.cs index 44bfe38473..3b2fb9cb1a 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/IRavenPersistenceLifecycle.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/IRavenPersistenceLifecycle.cs @@ -6,6 +6,6 @@ interface IRavenPersistenceLifecycle { Task Initialize(CancellationToken cancellationToken = default); - Task Stop(CancellationToken cancellationToken); + Task Stop(CancellationToken cancellationToken = default); } } \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs index 6949455510..aa8bd0d447 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs @@ -72,7 +72,7 @@ public async Task Initialize(CancellationToken cancellationToken = default) } } - public Task Stop(CancellationToken cancellationToken) => database!.Stop(cancellationToken); + public Task Stop(CancellationToken cancellationToken = default) => database!.Stop(cancellationToken); public void Dispose() { diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs b/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs index b2b8603590..3b6d382b9c 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs @@ -64,7 +64,7 @@ public async Task Initialize(CancellationToken cancellationToken = default) } } - public Task Stop(CancellationToken cancellationToken) => Task.CompletedTask; // We are not stopping an external instance + public Task Stop(CancellationToken cancellationToken = default) => Task.CompletedTask; // We are not stopping an external instance public void Dispose() => documentStore?.Dispose(); From 974a96af6bc7e2fa993920f15d5256f13882021d Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 12 Mar 2025 12:09:58 +0100 Subject: [PATCH 09/15] Ported disposing of RavenDB during stop to primary instance --- .../IRavenPersistenceLifecycle.cs | 1 + .../RavenEmbeddedPersistenceLifecycle.cs | 8 ++++++++ .../RavenExternalPersistenceLifecycle.cs | 2 ++ .../RavenPersistenceLifecycleHostedService.cs | 2 +- 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDB/IRavenPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/IRavenPersistenceLifecycle.cs index f84c01c2e5..a4ca673de6 100644 --- a/src/ServiceControl.Persistence.RavenDB/IRavenPersistenceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDB/IRavenPersistenceLifecycle.cs @@ -8,5 +8,6 @@ namespace ServiceControl.Persistence.RavenDB interface IRavenPersistenceLifecycle { Task Initialize(CancellationToken cancellationToken = default); + Task Stop(CancellationToken cancellationToken); } } \ No newline at end of file diff --git a/src/ServiceControl.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs index 1f20dad905..f1971af65c 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenEmbeddedPersistenceLifecycle.cs @@ -69,6 +69,14 @@ public async Task Initialize(CancellationToken cancellationToken) } } + public async Task Stop(CancellationToken cancellationToken) + { + if (database != null) + { + await database.Stop(cancellationToken); + } + } + public void Dispose() { documentStore?.Dispose(); diff --git a/src/ServiceControl.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs index 0d108d218c..a6c8d265f9 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenExternalPersistenceLifecycle.cs @@ -59,6 +59,8 @@ public async Task Initialize(CancellationToken cancellationToken) } } + public Task Stop(CancellationToken cancellationToken) => Task.CompletedTask; + public void Dispose() => documentStore?.Dispose(); IDocumentStore? documentStore; diff --git a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceLifecycleHostedService.cs b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceLifecycleHostedService.cs index b93391909a..57abf7d7ca 100644 --- a/src/ServiceControl.Persistence.RavenDB/RavenPersistenceLifecycleHostedService.cs +++ b/src/ServiceControl.Persistence.RavenDB/RavenPersistenceLifecycleHostedService.cs @@ -10,6 +10,6 @@ class RavenPersistenceLifecycleHostedService(IRavenPersistenceLifecycle persiste { public Task StartAsync(CancellationToken cancellationToken) => persistenceLifecycle.Initialize(cancellationToken); - public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask; + public Task StopAsync(CancellationToken cancellationToken) => persistenceLifecycle.Stop(cancellationToken); } } \ No newline at end of file From 7482a6da4454f344070bd29a42b113928be2d680 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 13 Mar 2025 15:41:58 +0100 Subject: [PATCH 10/15] Dispose Process instance `using var`, ensure EmbeddedServer.Instance.Dispose always gets invoked, added comment and improved code readability --- .../EmbeddedDatabase.cs | 31 +++++++++++++------ 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 8fa4ed5ae5..41db70d1f1 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -66,7 +66,6 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi Logger.InfoFormat("Loading RavenDB license from {0}", licenseFileNameAndServerDirectory.LicenseFileName); var serverOptions = new ServerOptions { - GracefulShutdownTimeout = TimeSpan.FromHours(1), // During Stop/Dispose we manually control this CommandLineArgs = [ $"--Logs.Mode={databaseConfiguration.LogsMode}", @@ -189,22 +188,34 @@ public async Task Stop(CancellationToken cancellationToken) await shutdownTokenSource.CancelAsync(); - // TODO: await EmbeddedServer.Instance.StopAsync(cancellationToken); + // EmbeddedServer does not have have an async Stop method, the Dispose operation blocks and waits + // until its GracefulShutdownTimeout is reached and then does a Process.Kill + // + // This logic gets the child process ID uses a Task.Delay with infinite. + // + // a. The Task.Delay gets cancelled first + // b. The EmbeddedServer.Dispose completes first + // + // If the Task.Delay gets cancelled first this means Dispose is still running and + // then we kill the process + + serverOptions!.GracefulShutdownTimeout = TimeSpan.FromHours(1); // During Stop/Dispose we manually control this var processId = await EmbeddedServer.Instance.GetServerProcessIdAsync(cancellationToken); + // Dispose always need to be invoked, even when already cancelled + var disposeTask = Task.Run(() => EmbeddedServer.Instance.Dispose(), CancellationToken.None); + var waitForCancellationTask = Task.Delay(Timeout.Infinite, cancellationToken); - var firstTask = await Task.WhenAny( - Task.Run(() => EmbeddedServer.Instance.Dispose(), cancellationToken), - waitForCancellationTask - ); - if (firstTask == waitForCancellationTask) + var delayIsCancelled = waitForCancellationTask == await Task.WhenAny(disposeTask, waitForCancellationTask); + + if (delayIsCancelled) { try { - Logger.WarnFormat("Killing RavenDB server PID {0} child process because host cancelled", processId); - var ravenChildProcess = Process.GetProcessById(processId); + Logger.WarnFormat("Killing RavenDB server PID {0} because host cancelled", processId); + using var ravenChildProcess = Process.GetProcessById(processId); ravenChildProcess.Kill(entireProcessTree: true); // Kill only signals Logger.WarnFormat("Waiting for RavenDB server PID {0} to exit... ", processId); @@ -212,7 +223,7 @@ public async Task Stop(CancellationToken cancellationToken) } catch (Exception e) { - Logger.ErrorFormat("Failed to kill RavenDB server PID {0}\n{1}", processId, e); + Logger.ErrorFormat("Failed to kill RavenDB server PID {0} shutdown\n{1}", processId, e); } } From 6c5340442d6123c8d77db4cb345abc385a8b8c9b Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 13 Mar 2025 15:52:24 +0100 Subject: [PATCH 11/15] Keep logging behavior default as it, only create client logs with LogMode.Information --- .../EmbeddedDatabase.cs | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 41db70d1f1..91340071e7 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -53,15 +53,19 @@ public static EmbeddedDatabase Start(EmbeddedDatabaseConfiguration databaseConfi var nugetPackagesPath = Path.Combine(databaseConfiguration.DbPath, "Packages", "NuGet"); - LoggingSource.Instance.SetupLogMode( - (LogMode)255, - Path.Combine(databaseConfiguration.LogPath, "Raven.Embedded"), - retentionTime: TimeSpan.FromDays(14), - retentionSize: 1024 * 1024 * 10, - compress: false - ); - - LoggingSource.Instance.EnableConsoleLogging(); + var logMode = Enum.Parse(databaseConfiguration.LogsMode); + + if (logMode == LogMode.Information) // Most verbose + { + LoggingSource.Instance.EnableConsoleLogging(); + LoggingSource.Instance.SetupLogMode( + logMode, + Path.Combine(databaseConfiguration.LogPath, "Raven.Embedded"), + retentionTime: TimeSpan.FromDays(14), + retentionSize: 1024 * 1024 * 10, + compress: false + ); + } Logger.InfoFormat("Loading RavenDB license from {0}", licenseFileNameAndServerDirectory.LicenseFileName); var serverOptions = new ServerOptions From 8edecb7cef03dd1b67f3b7b516d72b6bd83d052d Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 13 Mar 2025 15:56:02 +0100 Subject: [PATCH 12/15] Update src/ServiceControl.RavenDB/EmbeddedDatabase.cs --- src/ServiceControl.RavenDB/EmbeddedDatabase.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 91340071e7..ad8de23345 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -251,7 +251,7 @@ public void Dispose() if (serverOptions != null) { - // Set GracefulShutdownTimeout to Zero and exist ASAP, under normal operation instance would already + // Set GracefulShutdownTimeout to Zero and exit ASAP, under normal operation instance would already // have been allowed to gracefully stop during "Stop" method. serverOptions!.GracefulShutdownTimeout = TimeSpan.Zero; Logger.Debug("Disposing RavenDB server"); From e07f9ed68b8fbe1dee1a0d088d69d9efc1e18928 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 14 Mar 2025 10:03:18 +0100 Subject: [PATCH 13/15] Update src/ServiceControl.Persistence.RavenDB/IRavenPersistenceLifecycle.cs Co-authored-by: Mauro Servienti --- .../IRavenPersistenceLifecycle.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/ServiceControl.Persistence.RavenDB/IRavenPersistenceLifecycle.cs b/src/ServiceControl.Persistence.RavenDB/IRavenPersistenceLifecycle.cs index a4ca673de6..dc51ff8104 100644 --- a/src/ServiceControl.Persistence.RavenDB/IRavenPersistenceLifecycle.cs +++ b/src/ServiceControl.Persistence.RavenDB/IRavenPersistenceLifecycle.cs @@ -8,6 +8,6 @@ namespace ServiceControl.Persistence.RavenDB interface IRavenPersistenceLifecycle { Task Initialize(CancellationToken cancellationToken = default); - Task Stop(CancellationToken cancellationToken); + Task Stop(CancellationToken cancellationToken = default); } } \ No newline at end of file From dee2cfe82760aa962fc29e11b0fb322534630959 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 14 Mar 2025 10:15:10 +0100 Subject: [PATCH 14/15] Improved comments and cancellation behavior --- src/ServiceControl.RavenDB/EmbeddedDatabase.cs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index ad8de23345..7fec870071 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -192,37 +192,45 @@ public async Task Stop(CancellationToken cancellationToken) await shutdownTokenSource.CancelAsync(); - // EmbeddedServer does not have have an async Stop method, the Dispose operation blocks and waits - // until its GracefulShutdownTimeout is reached and then does a Process.Kill + // This is a workaround until the EmbeddedServer properly supports cancellation! // - // This logic gets the child process ID uses a Task.Delay with infinite. + // EmbeddedServer does not have an async Stop method, the Dispose operation blocks and waits + // until its GracefulShutdownTimeout is reached and then does a Process.Kill. Due to this behavior this can + // be shorter or longer than the allowed stop duration. + // + // With the Task.WhenAny 2 things can happen: // // a. The Task.Delay gets cancelled first // b. The EmbeddedServer.Dispose completes first // // If the Task.Delay gets cancelled first this means Dispose is still running and - // then we kill the process + // then we try and kill the process serverOptions!.GracefulShutdownTimeout = TimeSpan.FromHours(1); // During Stop/Dispose we manually control this - var processId = await EmbeddedServer.Instance.GetServerProcessIdAsync(cancellationToken); // Dispose always need to be invoked, even when already cancelled var disposeTask = Task.Run(() => EmbeddedServer.Instance.Dispose(), CancellationToken.None); + // Runs "infinite" but will eventually get cancelled var waitForCancellationTask = Task.Delay(Timeout.Infinite, cancellationToken); var delayIsCancelled = waitForCancellationTask == await Task.WhenAny(disposeTask, waitForCancellationTask); if (delayIsCancelled) { + int processId = 0; try { + // We always want to try and kill the process, even when already cancelled + processId = await EmbeddedServer.Instance.GetServerProcessIdAsync(CancellationToken.None); Logger.WarnFormat("Killing RavenDB server PID {0} because host cancelled", processId); using var ravenChildProcess = Process.GetProcessById(processId); ravenChildProcess.Kill(entireProcessTree: true); // Kill only signals Logger.WarnFormat("Waiting for RavenDB server PID {0} to exit... ", processId); + // When WaitForExitAsync returns, the process could still exist but in a frozen state to flush + // memory mapped pages to storage. await ravenChildProcess.WaitForExitAsync(CancellationToken.None); } catch (Exception e) From a5119d74221c8892da85365292dd3e1ae8cae294 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 14 Mar 2025 10:32:40 +0100 Subject: [PATCH 15/15] Fix comment --- src/ServiceControl.RavenDB/EmbeddedDatabase.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs index 7fec870071..590792fe20 100644 --- a/src/ServiceControl.RavenDB/EmbeddedDatabase.cs +++ b/src/ServiceControl.RavenDB/EmbeddedDatabase.cs @@ -198,13 +198,13 @@ public async Task Stop(CancellationToken cancellationToken) // until its GracefulShutdownTimeout is reached and then does a Process.Kill. Due to this behavior this can // be shorter or longer than the allowed stop duration. // - // With the Task.WhenAny 2 things can happen: + // When Task.WhenAny is called, 2 things can happen: // // a. The Task.Delay gets cancelled first // b. The EmbeddedServer.Dispose completes first // // If the Task.Delay gets cancelled first this means Dispose is still running and - // then we try and kill the process + // then we try and kill the process, if not disposed completed and we're done. serverOptions!.GracefulShutdownTimeout = TimeSpan.FromHours(1); // During Stop/Dispose we manually control this @@ -356,4 +356,4 @@ static long DirSize(DirectoryInfo d) static TimeSpan delayBetweenRestarts = TimeSpan.FromSeconds(60); static readonly ILog Logger = LogManager.GetLogger(); } -} \ No newline at end of file +}