Allow RavenDB search engine type to be configured

src/ServiceControl.Audit.Persistence.RavenDB/DatabaseConfiguration.cs

@@ -1,6 +1,7 @@
 namespace ServiceControl.Audit.Persistence.RavenDB
 {
     using System;
+    using Raven.Client.Documents.Indexes;
     using Sparrow.Json;
 
     public class DatabaseConfiguration(
@@ -12,7 +13,8 @@ public class DatabaseConfiguration(
         int dataSpaceRemainingThreshold,
         int minimumStorageLeftRequiredForIngestion,
         ServerConfiguration serverConfiguration,
-        TimeSpan bulkInsertCommitTimeout)
+        TimeSpan bulkInsertCommitTimeout,
+        SearchEngineType searchEngineType)
     {
         public string Name { get; } = name;
 
@@ -33,5 +35,7 @@ public class DatabaseConfiguration(
         public int MinimumStorageLeftRequiredForIngestion { get; internal set; } = minimumStorageLeftRequiredForIngestion; //Setting for ATT only
 
         public TimeSpan BulkInsertCommitTimeout { get; } = bulkInsertCommitTimeout;
+
+        public SearchEngineType SearchEngineType { get; } = searchEngineType;
     }
-}
+}

src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs

@@ -12,8 +12,8 @@
     using Raven.Client.ServerWide;
     using Raven.Client.ServerWide.Operations;
     using Raven.Client.ServerWide.Operations.Configuration;
-    using ServiceControl.Audit.Persistence.RavenDB.Indexes;
-    using ServiceControl.SagaAudit;
+    using Indexes;
+    using SagaAudit;
 
     class DatabaseSetup(DatabaseConfiguration configuration)
     {
@@ -36,8 +36,8 @@ async Task CreateDatabase(IDocumentStore documentStore, string databaseName, Can
                 try
                 {
                     var databaseRecord = new DatabaseRecord(databaseName);
-                    databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Corax");
-                    databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Corax");
+                    databaseRecord.Settings.Add(AutoSearchEngineTypeKey, configuration.SearchEngineType.ToString());
+                    databaseRecord.Settings.Add(StaticSearchEngineTypeKey, configuration.SearchEngineType.ToString());
 
                     await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken);
                 }
@@ -59,8 +59,33 @@ async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseN
 
             var updated = false;
 
-            updated |= dbRecord.Settings.TryAdd("Indexing.Auto.SearchEngineType", "Corax");
-            updated |= dbRecord.Settings.TryAdd("Indexing.Static.SearchEngineType", "Corax");
+            if (dbRecord.Settings.TryGetValue(AutoSearchEngineTypeKey, out var searchEngineTypeAuto))
+            {
+                if (searchEngineTypeAuto != configuration.SearchEngineType.ToString())
+                {
+                    updated = true;
+                }
+            }
+            else
+            {
+                updated = true;
+            }
+
+            dbRecord.Settings[AutoSearchEngineTypeKey] = configuration.SearchEngineType.ToString();
+
+            if (dbRecord.Settings.TryGetValue(StaticSearchEngineTypeKey, out var searchEngineTypeStatic))
+            {
+                if (searchEngineTypeStatic != configuration.SearchEngineType.ToString())
+                {
+                    updated = true;
+                }
+            }
+            else
+            {
+                updated = true;
+            }
+
+            dbRecord.Settings[StaticSearchEngineTypeKey] = configuration.SearchEngineType.ToString();
 
             if (updated)
             {
@@ -70,19 +95,19 @@ async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseN
             }
         }
 
-        public static async Task DeleteLegacySagaDetailsIndex(IDocumentStore documentStore, CancellationToken cancellationToken)
+        internal static async Task DeleteLegacySagaDetailsIndex(IDocumentStore documentStore, CancellationToken cancellationToken)
         {
             // If the SagaDetailsIndex exists but does not have a .Take(50000), then we remove the current SagaDetailsIndex and
             // create a new one. If we do not remove the current one, then RavenDB will attempt to do a side-by-side migration.
             // Doing a side-by-side migration results in the index never swapping if there is constant ingestion as RavenDB will wait.
             // for the index to not be stale before swapping to the new index. Constant ingestion means the index will never be not-stale.
             // This needs to stay in place until the next major version as the user could upgrade from an older version of the current
             // Major (v5.x.x) which might still have the incorrect index.
-            var sagaDetailsIndexOperation = new GetIndexOperation("SagaDetailsIndex");
+            var sagaDetailsIndexOperation = new GetIndexOperation(SagaDetailsIndexName);
             var sagaDetailsIndexDefinition = await documentStore.Maintenance.SendAsync(sagaDetailsIndexOperation, cancellationToken);
             if (sagaDetailsIndexDefinition != null && !sagaDetailsIndexDefinition.Reduce.Contains("Take(50000)"))
             {
-                await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("SagaDetailsIndex"), cancellationToken);
+                await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(SagaDetailsIndexName), cancellationToken);
             }
         }
 
@@ -95,12 +120,12 @@ async Task CreateIndexes(IDocumentStore documentStore, CancellationToken cancell
             if (configuration.EnableFullTextSearch)
             {
                 indexList.Add(new MessagesViewIndexWithFullTextSearch());
-                await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndex"), cancellationToken);
+                await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(MessagesViewIndexName), cancellationToken);
             }
             else
             {
                 indexList.Add(new MessagesViewIndex());
-                await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndexWithFullTextSearch"), cancellationToken);
+                await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(MessagesViewIndexWithFullTextSearchIndexName), cancellationToken);
             }
 
             await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken);
@@ -116,5 +141,11 @@ async Task ConfigureExpiration(IDocumentStore documentStore, CancellationToken c
 
             await documentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(expirationConfig), cancellationToken);
         }
+
+        const string AutoSearchEngineTypeKey = "Indexing.Auto.SearchEngineType";
+        const string StaticSearchEngineTypeKey = "Indexing.Static.SearchEngineType";
+        internal const string MessagesViewIndexName = "MessagesViewIndex";
+        internal const string MessagesViewIndexWithFullTextSearchIndexName = "MessagesViewIndexWithFullTextSearch";
+        internal const string SagaDetailsIndexName = "SagaDetailsIndex";
     }
-}
+}

src/ServiceControl.Audit.Persistence.RavenDB/RavenPersistenceConfiguration.cs

@@ -6,6 +6,7 @@
     using System.Reflection;
     using CustomChecks;
     using NServiceBus.Logging;
+    using Raven.Client.Documents.Indexes;
 
     public class RavenPersistenceConfiguration : IPersistenceConfiguration
     {
@@ -22,22 +23,9 @@ public class RavenPersistenceConfiguration : IPersistenceConfiguration
         public const string MinimumStorageLeftRequiredForIngestionKey = "MinimumStorageLeftRequiredForIngestion";
         public const string BulkInsertCommitTimeoutInSecondsKey = "BulkInsertCommitTimeoutInSeconds";
         public const string DataSpaceRemainingThresholdKey = "DataSpaceRemainingThreshold";
+        public const string SearchEngineTypeKey = "RavenDB/SearchEngineType";
 
-        public IEnumerable<string> ConfigurationKeys => new[]{
-            DatabaseNameKey,
-            DatabasePathKey,
-            ConnectionStringKey,
-            ClientCertificatePathKey,
-            ClientCertificateBase64Key,
-            ClientCertificatePasswordKey,
-            DatabaseMaintenancePortKey,
-            ExpirationProcessTimerInSecondsKey,
-            LogPathKey,
-            RavenDbLogLevelKey,
-            DataSpaceRemainingThresholdKey,
-            MinimumStorageLeftRequiredForIngestionKey,
-            BulkInsertCommitTimeoutInSecondsKey
-        };
+        public IEnumerable<string> ConfigurationKeys => new[] { DatabaseNameKey, DatabasePathKey, ConnectionStringKey, ClientCertificatePathKey, ClientCertificateBase64Key, ClientCertificatePasswordKey, DatabaseMaintenancePortKey, ExpirationProcessTimerInSecondsKey, LogPathKey, RavenDbLogLevelKey, DataSpaceRemainingThresholdKey, MinimumStorageLeftRequiredForIngestionKey, BulkInsertCommitTimeoutInSecondsKey, SearchEngineTypeKey };
 
         public string Name => "RavenDB";
 
@@ -70,10 +58,12 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin
                 {
                     serverConfiguration.ClientCertificatePath = clientCertificatePath;
                 }
+
                 if (settings.PersisterSpecificSettings.TryGetValue(ClientCertificateBase64Key, out var clientCertificateBase64))
                 {
                     serverConfiguration.ClientCertificateBase64 = clientCertificateBase64;
                 }
+
                 if (settings.PersisterSpecificSettings.TryGetValue(ClientCertificatePasswordKey, out var clientCertificatePassword))
                 {
                     serverConfiguration.ClientCertificatePassword = clientCertificatePassword;
@@ -120,6 +110,18 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin
 
             var bulkInsertTimeout = TimeSpan.FromSeconds(GetBulkInsertCommitTimeout(settings));
 
+            var searchEngineType = SearchEngineTypeDefault;
+
+            if (settings.PersisterSpecificSettings.TryGetValue(SearchEngineTypeKey, out var searchEngineTypeValue))
+            {
+                if (!Enum.TryParse<SearchEngineType>(searchEngineTypeValue, out var explicitSearchEngineType))
+                {
+                    throw new InvalidOperationException($"{searchEngineTypeValue} is not supported.");
+                }
+
+                searchEngineType = explicitSearchEngineType;
+            }
+
             return new DatabaseConfiguration(
                 databaseName,
                 expirationProcessTimerInSeconds,
@@ -129,7 +131,8 @@ internal static DatabaseConfiguration GetDatabaseConfiguration(PersistenceSettin
                 dataSpaceRemainingThreshold,
                 minimumStorageLeftRequiredForIngestion,
                 serverConfiguration,
-                bulkInsertTimeout);
+                bulkInsertTimeout,
+                searchEngineType);
         }
 
         static int GetExpirationProcessTimerInSeconds(PersistenceSettings settings)
@@ -197,5 +200,6 @@ static string GetLogPath(PersistenceSettings settings)
 
         const int ExpirationProcessTimerInSecondsDefault = 600;
         const int BulkInsertCommitTimeoutInSecondsDefault = 60;
+        internal static readonly SearchEngineType SearchEngineTypeDefault = SearchEngineType.Corax;
     }
-}
+}

src/ServiceControl.Audit.Persistence.Tests.RavenDB/PersistenceTestsConfiguration.cs

@@ -84,7 +84,10 @@ public async Task Configure(Action<PersistenceSettings> setSettings)
             var documentStoreProvider = host.Services.GetRequiredService<IRavenDocumentStoreProvider>();
             DocumentStore = await documentStoreProvider.GetDocumentStore();
             var bulkInsert = DocumentStore.BulkInsert(
-                options: new BulkInsertOptions { SkipOverwriteIfUnchanged = true, });
+                options: new BulkInsertOptions
+                {
+                    SkipOverwriteIfUnchanged = true,
+                });
 
             var sessionProvider = host.Services.GetRequiredService<IRavenSessionProvider>();
 
@@ -103,11 +106,18 @@ public async Task Cleanup()
             if (DocumentStore != null)
             {
                 await DocumentStore.Maintenance.Server.SendAsync(new DeleteDatabasesOperation(
-                    new DeleteDatabasesOperation.Parameters { DatabaseNames = [databaseName], HardDelete = true }));
+                    new DeleteDatabasesOperation.Parameters
+                    {
+                        DatabaseNames = [databaseName],
+                        HardDelete = true
+                    }));
             }
 
-            await host.StopAsync();
-            host.Dispose();
+            if (host != null)
+            {
+                await host.StopAsync();
+                host.Dispose();
+            }
         }
 
 

src/ServiceControl.Audit.Persistence.Tests.RavenDB/SagaDetailsIndexTests.cs

@@ -4,6 +4,7 @@
     using System.Threading;
     using System.Threading.Tasks;
     using NUnit.Framework;
+    using Persistence.RavenDB;
     using Raven.Client.Documents.Indexes;
     using Raven.Client.Documents.Operations.Indexes;
     using ServiceControl.SagaAudit;
@@ -14,14 +15,14 @@ class SagaDetailsIndexTests : PersistenceTestFixture
         [Test]
         public async Task Deletes_index_that_does_not_have_cap_of_50000()
         {
-            await configuration.DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation("SagaDetailsIndex"));
+            await configuration.DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation(DatabaseSetup.SagaDetailsIndexName));
 
             var indexWithout50000capDefinition = new IndexDefinition
             {
-                Name = "SagaDetailsIndex",
+                Name = DatabaseSetup.SagaDetailsIndexName,
                 Maps =
-                            [
-                                @"from doc in docs
+                [
+                    @"from doc in docs
                                                      select new
                                                      {
                                                          doc.SagaId,
@@ -41,7 +42,7 @@ public async Task Deletes_index_that_does_not_have_cap_of_50000()
                                     }
                                 }
             }"
-                            ],
+                ],
                 Reduce = @"from result in results
                                             group result by result.SagaId
                             into g
@@ -61,12 +62,12 @@ into g
 
             await configuration.DocumentStore.Maintenance.SendAsync(putIndexesOp);
 
-            var sagaDetailsIndexOperation = new GetIndexOperation("SagaDetailsIndex");
+            var sagaDetailsIndexOperation = new GetIndexOperation(DatabaseSetup.SagaDetailsIndexName);
             var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation);
 
             Assert.That(sagaDetailsIndexDefinition, Is.Not.Null);
 
-            await Persistence.RavenDB.DatabaseSetup.DeleteLegacySagaDetailsIndex(configuration.DocumentStore, CancellationToken.None);
+            await DatabaseSetup.DeleteLegacySagaDetailsIndex(configuration.DocumentStore, CancellationToken.None);
 
             sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation);
 
@@ -76,9 +77,9 @@ into g
         [Test]
         public async Task Does_not_delete_index_that_does_have_cap_of_50000()
         {
-            await Persistence.RavenDB.DatabaseSetup.DeleteLegacySagaDetailsIndex(configuration.DocumentStore, CancellationToken.None);
+            await DatabaseSetup.DeleteLegacySagaDetailsIndex(configuration.DocumentStore, CancellationToken.None);
 
-            var sagaDetailsIndexOperation = new GetIndexOperation("SagaDetailsIndex");
+            var sagaDetailsIndexOperation = new GetIndexOperation(DatabaseSetup.SagaDetailsIndexName);
             var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation);
 
             Assert.That(sagaDetailsIndexDefinition, Is.Not.Null);
@@ -100,13 +101,10 @@ await IngestSagaAudits(new SagaSnapshot
 
             await configuration.CompleteDBOperation();
 
-            using (var session = configuration.DocumentStore.OpenAsyncSession())
-            {
-                var sagaDetailsIndexOperation = new GetIndexOperation("SagaDetailsIndex");
-                var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation);
+            var sagaDetailsIndexOperation = new GetIndexOperation(DatabaseSetup.SagaDetailsIndexName);
+            var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation);
 
-                Assert.That(sagaDetailsIndexDefinition.Reduce, Does.Contain("Take(50000)"), "The SagaDetails index definition does not contain a .Take(50000) to limit the number of saga state changes that are reduced by the map/reduce");
-            }
+            Assert.That(sagaDetailsIndexDefinition.Reduce, Does.Contain("Take(50000)"), "The SagaDetails index definition does not contain a .Take(50000) to limit the number of saga state changes that are reduced by the map/reduce");
         }
 
         async Task IngestSagaAudits(params SagaSnapshot[] snapshots)
@@ -116,6 +114,7 @@ async Task IngestSagaAudits(params SagaSnapshot[] snapshots)
             {
                 await unitOfWork.RecordSagaSnapshot(snapshot);
             }
+
             await unitOfWork.DisposeAsync();
             await configuration.CompleteDBOperation();
         }

src/ServiceControl.Audit.Persistence.Tests.RavenDB/SearchEngineTypeTests.cs

@@ -0,0 +1,22 @@
+namespace ServiceControl.Audit.Persistence.Tests;
+
+using System.Threading.Tasks;
+using NUnit.Framework;
+using Persistence.RavenDB;
+using Raven.Client.Documents.Indexes;
+using Raven.Client.Documents.Operations.Indexes;
+
+[TestFixture]
+class SearchEngineTypeTests : PersistenceTestFixture
+{
+    [Test]
+    public async Task Free_text_search_should_be_on_using_corax_by_default()
+    {
+        var freeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(DatabaseSetup.MessagesViewIndexWithFullTextSearchIndexName));
+        var nonFreeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexName));
+
+        Assert.That(nonFreeTextIndex, Is.Null);
+        Assert.That(freeTextIndex, Is.Not.Null);
+        Assert.That(freeTextIndex.SearchEngineType, Is.EqualTo(SearchEngineType.Corax));
+    }
+}

src/ServiceControl.Audit.Persistence.Tests.RavenDB/SharedEmbeddedServer.cs

@@ -36,7 +36,7 @@ public static async Task<EmbeddedDatabase> GetInstance(CancellationToken cancell
                 var logsMode = "Operations";
                 var serverUrl = $"http://localhost:{PortUtility.FindAvailablePort(33334)}";
 
-                var databaseConfiguration = new DatabaseConfiguration("audit", 60, true, TimeSpan.FromMinutes(5), 120000, 5, 5, new ServerConfiguration(dbPath, serverUrl, logPath, logsMode), TimeSpan.FromSeconds(60));
+                var databaseConfiguration = new DatabaseConfiguration("audit", 60, true, TimeSpan.FromMinutes(5), 120000, 5, 5, new ServerConfiguration(dbPath, serverUrl, logPath, logsMode), TimeSpan.FromSeconds(60), RavenPersistenceConfiguration.SearchEngineTypeDefault);
                 var serverConfig = databaseConfiguration.ServerConfiguration;
 
                 // TODO: See if more refactoring can be done in configuration classes