From 5fd9e63b0bd332bf4d009e72db328154d18031fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 28 Feb 2025 08:26:52 +0100 Subject: [PATCH 1/8] Initial cleanup --- .../DatabaseSetup.cs | 192 +++++++++--------- 1 file changed, 99 insertions(+), 93 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs b/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs index cb31bf977a..015a3d8504 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs @@ -1,120 +1,126 @@ -namespace ServiceControl.Audit.Persistence.RavenDB +namespace ServiceControl.Audit.Persistence.RavenDB; + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Raven.Client.Documents; +using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations.Expiration; +using Raven.Client.Documents.Operations.Indexes; +using Raven.Client.Exceptions; +using Raven.Client.ServerWide; +using Raven.Client.ServerWide.Operations; +using Raven.Client.ServerWide.Operations.Configuration; +using Indexes; +using SagaAudit; + +class DatabaseSetup(DatabaseConfiguration configuration) { - using System; - using System.Collections.Generic; - using System.Threading; - using System.Threading.Tasks; - using Raven.Client.Documents; - using Raven.Client.Documents.Indexes; - using Raven.Client.Documents.Operations.Expiration; - using Raven.Client.Documents.Operations.Indexes; - using Raven.Client.Exceptions; - using Raven.Client.ServerWide; - using Raven.Client.ServerWide.Operations; - using Raven.Client.ServerWide.Operations.Configuration; - using ServiceControl.Audit.Persistence.RavenDB.Indexes; - using ServiceControl.SagaAudit; - - class DatabaseSetup(DatabaseConfiguration configuration) + public async Task Execute(IDocumentStore documentStore, CancellationToken cancellationToken) { - public async Task Execute(IDocumentStore documentStore, CancellationToken cancellationToken) - { - await CreateDatabase(documentStore, configuration.Name, cancellationToken); - await UpdateDatabaseSettings(documentStore, configuration.Name, cancellationToken); + await CreateDatabase(documentStore, configuration.Name, cancellationToken); - await CreateIndexes(documentStore, cancellationToken); + await UpdateDatabaseSettings(documentStore, configuration.Name, cancellationToken); - await ConfigureExpiration(documentStore, cancellationToken); - } + await CreateIndexes(documentStore, cancellationToken); - async Task CreateDatabase(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken) - { - var dbRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken); + await ConfigureExpiration(documentStore, cancellationToken); + } - if (dbRecord is null) - { - try - { - var databaseRecord = new DatabaseRecord(databaseName); - databaseRecord.Settings.Add("Indexing.Auto.SearchEngineType", "Corax"); - databaseRecord.Settings.Add("Indexing.Static.SearchEngineType", "Corax"); - - await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken); - } - catch (ConcurrencyException) - { - // The database was already created before calling CreateDatabaseOperation - } - } - } + async Task CreateDatabase(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken) + { + var dbRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken); - async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken) + if (dbRecord is null) { - var dbRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken); - - if (dbRecord is null) + try { - throw new InvalidOperationException($"Database '{databaseName}' does not exist."); - } - - var updated = false; + var databaseRecord = new DatabaseRecord(databaseName); - updated |= dbRecord.Settings.TryAdd("Indexing.Auto.SearchEngineType", "Corax"); - updated |= dbRecord.Settings.TryAdd("Indexing.Static.SearchEngineType", "Corax"); + SetSearchEngineType(databaseRecord, SearchEngineType.Corax); - if (updated) + await documentStore.Maintenance.Server.SendAsync(new CreateDatabaseOperation(databaseRecord), cancellationToken); + } + catch (ConcurrencyException) { - await documentStore.Maintenance.ForDatabase(databaseName).SendAsync(new PutDatabaseSettingsOperation(databaseName, dbRecord.Settings), cancellationToken); - await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, true), cancellationToken); - await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, false), cancellationToken); + // The database was already created before calling CreateDatabaseOperation } } + } - public static async Task DeleteLegacySagaDetailsIndex(IDocumentStore documentStore, CancellationToken cancellationToken) + async Task UpdateDatabaseSettings(IDocumentStore documentStore, string databaseName, CancellationToken cancellationToken) + { + var databaseRecord = await documentStore.Maintenance.Server.SendAsync(new GetDatabaseRecordOperation(databaseName), cancellationToken) ?? throw new InvalidOperationException($"Database '{databaseName}' does not exist."); + + if (!SetSearchEngineType(databaseRecord, SearchEngineType.Corax)) { - // If the SagaDetailsIndex exists but does not have a .Take(50000), then we remove the current SagaDetailsIndex and - // create a new one. If we do not remove the current one, then RavenDB will attempt to do a side-by-side migration. - // Doing a side-by-side migration results in the index never swapping if there is constant ingestion as RavenDB will wait. - // for the index to not be stale before swapping to the new index. Constant ingestion means the index will never be not-stale. - // This needs to stay in place until the next major version as the user could upgrade from an older version of the current - // Major (v5.x.x) which might still have the incorrect index. - var sagaDetailsIndexOperation = new GetIndexOperation("SagaDetailsIndex"); - var sagaDetailsIndexDefinition = await documentStore.Maintenance.SendAsync(sagaDetailsIndexOperation, cancellationToken); - if (sagaDetailsIndexDefinition != null && !sagaDetailsIndexDefinition.Reduce.Contains("Take(50000)")) - { - await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("SagaDetailsIndex"), cancellationToken); - } + return; } - async Task CreateIndexes(IDocumentStore documentStore, CancellationToken cancellationToken) + await documentStore.Maintenance.ForDatabase(databaseName).SendAsync(new PutDatabaseSettingsOperation(databaseName, databaseRecord.Settings), cancellationToken); + await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, true), cancellationToken); + await documentStore.Maintenance.Server.SendAsync(new ToggleDatabasesStateOperation(databaseName, false), cancellationToken); + } + + public static async Task DeleteLegacySagaDetailsIndex(IDocumentStore documentStore, CancellationToken cancellationToken) + { + // If the SagaDetailsIndex exists but does not have a .Take(50000), then we remove the current SagaDetailsIndex and + // create a new one. If we do not remove the current one, then RavenDB will attempt to do a side-by-side migration. + // Doing a side-by-side migration results in the index never swapping if there is constant ingestion as RavenDB will wait. + // for the index to not be stale before swapping to the new index. Constant ingestion means the index will never be not-stale. + // This needs to stay in place until the next major version as the user could upgrade from an older version of the current + // Major (v5.x.x) which might still have the incorrect index. + var sagaDetailsIndexOperation = new GetIndexOperation(SagaDetailsIndexName); + var sagaDetailsIndexDefinition = await documentStore.Maintenance.SendAsync(sagaDetailsIndexOperation, cancellationToken); + if (sagaDetailsIndexDefinition != null && !sagaDetailsIndexDefinition.Reduce.Contains("Take(50000)")) { - await DeleteLegacySagaDetailsIndex(documentStore, cancellationToken); + await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(SagaDetailsIndexName), cancellationToken); + } + } - List indexList = [new FailedAuditImportIndex(), new SagaDetailsIndex()]; + async Task CreateIndexes(IDocumentStore documentStore, CancellationToken cancellationToken) + { + await DeleteLegacySagaDetailsIndex(documentStore, cancellationToken); - if (configuration.EnableFullTextSearch) - { - indexList.Add(new MessagesViewIndexWithFullTextSearch()); - await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndex"), cancellationToken); - } - else - { - indexList.Add(new MessagesViewIndex()); - await documentStore.Maintenance.SendAsync(new DeleteIndexOperation("MessagesViewIndexWithFullTextSearch"), cancellationToken); - } + List indexList = [new FailedAuditImportIndex(), new SagaDetailsIndex()]; - await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken); + if (configuration.EnableFullTextSearch) + { + indexList.Add(new MessagesViewIndexWithFullTextSearch()); + await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(MessagesViewIndexName), cancellationToken); } + else + { + indexList.Add(new MessagesViewIndex()); + await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(MessagesViewIndexWithFulltextSearchName), cancellationToken); + } + + await IndexCreation.CreateIndexesAsync(indexList, documentStore, null, null, cancellationToken); + } - async Task ConfigureExpiration(IDocumentStore documentStore, CancellationToken cancellationToken) + async Task ConfigureExpiration(IDocumentStore documentStore, CancellationToken cancellationToken) + { + var expirationConfig = new ExpirationConfiguration { - var expirationConfig = new ExpirationConfiguration - { - Disabled = false, - DeleteFrequencyInSec = configuration.ExpirationProcessTimerInSeconds - }; + Disabled = false, + DeleteFrequencyInSec = configuration.ExpirationProcessTimerInSeconds + }; - await documentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(expirationConfig), cancellationToken); - } + await documentStore.Maintenance.SendAsync(new ConfigureExpirationOperation(expirationConfig), cancellationToken); + } + + bool SetSearchEngineType(DatabaseRecord database, SearchEngineType searchEngineType) + { + var updated = false; + + updated |= database.Settings.TryAdd("Indexing.Auto.SearchEngineType", searchEngineType.ToString()); + updated |= database.Settings.TryAdd("Indexing.Static.SearchEngineType", searchEngineType.ToString()); + + return updated; } -} + + internal const string MessagesViewIndexWithFulltextSearchName = "MessagesViewIndexWithFullTextSearch"; + internal const string SagaDetailsIndexName = "SagaDetailsIndex"; + internal const string MessagesViewIndexName = "MessagesViewIndex"; +} \ No newline at end of file From 1fe5c7c06bebf0f99b91b8376e4c21f0b7a3f5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 28 Feb 2025 08:46:52 +0100 Subject: [PATCH 2/8] Add tests to verify default behavior --- .../IndexSetupTests.cs | 34 +++++ .../SagaDetailsIndexTests.cs | 142 +++++++++--------- 2 files changed, 104 insertions(+), 72 deletions(-) create mode 100644 src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs new file mode 100644 index 0000000000..64ef3a550e --- /dev/null +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs @@ -0,0 +1,34 @@ +namespace ServiceControl.Audit.Persistence.Tests; + +using System; +using System.Threading.Tasks; +using global::ServiceControl.Audit.Persistence.RavenDB; +using NUnit.Framework; +using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations.Indexes; + +[TestFixture] +class IndexSetupTests : PersistenceTestFixture +{ + [Test] + public async Task Corax_should_the_defaul_search_engine_type() + { + var indexes = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexesOperation(0, int.MaxValue)); + + foreach (var index in indexes) + { + var indexStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName)); + Assert.That(indexStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax), $"{index.Name} is not using Corax"); + } + } + + [Test] + public async Task Free_text_search_index_should_be_used_by_default() + { + var freeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName)); + var nonFreeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexName)); + + Assert.That(nonFreeTextIndex, Is.Null); + Assert.That(freeTextIndex, Is.Not.Null); + } +} \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SagaDetailsIndexTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SagaDetailsIndexTests.cs index 4dc9a2d5d8..71ec7400d2 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SagaDetailsIndexTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/SagaDetailsIndexTests.cs @@ -1,27 +1,28 @@ -namespace ServiceControl.Audit.Persistence.Tests +namespace ServiceControl.Audit.Persistence.Tests; + +using System; +using System.Threading; +using System.Threading.Tasks; +using NUnit.Framework; +using Persistence.RavenDB; +using Raven.Client.Documents.Indexes; +using Raven.Client.Documents.Operations.Indexes; +using SagaAudit; + +[TestFixture] +class SagaDetailsIndexTests : PersistenceTestFixture { - using System; - using System.Threading; - using System.Threading.Tasks; - using NUnit.Framework; - using Raven.Client.Documents.Indexes; - using Raven.Client.Documents.Operations.Indexes; - using ServiceControl.SagaAudit; - - [TestFixture] - class SagaDetailsIndexTests : PersistenceTestFixture + [Test] + public async Task Deletes_index_that_does_not_have_cap_of_50000() { - [Test] - public async Task Deletes_index_that_does_not_have_cap_of_50000() + await configuration.DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation(DatabaseSetup.SagaDetailsIndexName)); + + var indexWithout50000capDefinition = new IndexDefinition { - await configuration.DocumentStore.Maintenance.SendAsync(new DeleteIndexOperation("SagaDetailsIndex")); - - var indexWithout50000capDefinition = new IndexDefinition - { - Name = "SagaDetailsIndex", - Maps = - [ - @"from doc in docs + Name = DatabaseSetup.SagaDetailsIndexName, + Maps = + [ + @"from doc in docs select new { doc.SagaId, @@ -41,8 +42,8 @@ public async Task Deletes_index_that_does_not_have_cap_of_50000() } } }" - ], - Reduce = @"from result in results + ], + Reduce = @"from result in results group result by result.SagaId into g let first = g.First() @@ -55,69 +56,66 @@ into g .OrderByDescending(x => x.FinishTime) .ToList() }" - }; + }; - var putIndexesOp = new PutIndexesOperation(indexWithout50000capDefinition); + var putIndexesOp = new PutIndexesOperation(indexWithout50000capDefinition); - await configuration.DocumentStore.Maintenance.SendAsync(putIndexesOp); + await configuration.DocumentStore.Maintenance.SendAsync(putIndexesOp); - var sagaDetailsIndexOperation = new GetIndexOperation("SagaDetailsIndex"); - var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation); + var sagaDetailsIndexOperation = new GetIndexOperation(DatabaseSetup.SagaDetailsIndexName); + var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation); - Assert.That(sagaDetailsIndexDefinition, Is.Not.Null); + Assert.That(sagaDetailsIndexDefinition, Is.Not.Null); - await Persistence.RavenDB.DatabaseSetup.DeleteLegacySagaDetailsIndex(configuration.DocumentStore, CancellationToken.None); + await DatabaseSetup.DeleteLegacySagaDetailsIndex(configuration.DocumentStore, CancellationToken.None); - sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation); + sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation); - Assert.That(sagaDetailsIndexDefinition, Is.Null); - } + Assert.That(sagaDetailsIndexDefinition, Is.Null); + } - [Test] - public async Task Does_not_delete_index_that_does_have_cap_of_50000() - { - await Persistence.RavenDB.DatabaseSetup.DeleteLegacySagaDetailsIndex(configuration.DocumentStore, CancellationToken.None); + [Test] + public async Task Does_not_delete_index_that_does_have_cap_of_50000() + { + await DatabaseSetup.DeleteLegacySagaDetailsIndex(configuration.DocumentStore, CancellationToken.None); - var sagaDetailsIndexOperation = new GetIndexOperation("SagaDetailsIndex"); - var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation); + var sagaDetailsIndexOperation = new GetIndexOperation(DatabaseSetup.SagaDetailsIndexName); + var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation); - Assert.That(sagaDetailsIndexDefinition, Is.Not.Null); - } + Assert.That(sagaDetailsIndexDefinition, Is.Not.Null); + } + + [Test] + public async Task Should_only_reduce_the_last_50000_saga_state_changes() + { + var sagaType = "MySagaType"; + var sagaState = "some-saga-state"; - [Test] - public async Task Should_only_reduce_the_last_50000_saga_state_changes() + await IngestSagaAudits(new SagaSnapshot { - var sagaType = "MySagaType"; - var sagaState = "some-saga-state"; - - await IngestSagaAudits(new SagaSnapshot - { - SagaId = Guid.NewGuid(), - SagaType = sagaType, - Status = SagaStateChangeStatus.New, - StateAfterChange = sagaState - }); - - await configuration.CompleteDBOperation(); - - using (var session = configuration.DocumentStore.OpenAsyncSession()) - { - var sagaDetailsIndexOperation = new GetIndexOperation("SagaDetailsIndex"); - var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation); - - Assert.That(sagaDetailsIndexDefinition.Reduce, Does.Contain("Take(50000)"), "The SagaDetails index definition does not contain a .Take(50000) to limit the number of saga state changes that are reduced by the map/reduce"); - } - } + SagaId = Guid.NewGuid(), + SagaType = sagaType, + Status = SagaStateChangeStatus.New, + StateAfterChange = sagaState + }); + + await configuration.CompleteDBOperation(); + + var sagaDetailsIndexOperation = new GetIndexOperation(DatabaseSetup.SagaDetailsIndexName); + var sagaDetailsIndexDefinition = await configuration.DocumentStore.Maintenance.SendAsync(sagaDetailsIndexOperation); - async Task IngestSagaAudits(params SagaSnapshot[] snapshots) + Assert.That(sagaDetailsIndexDefinition.Reduce, Does.Contain("Take(50000)"), "The SagaDetails index definition does not contain a .Take(50000) to limit the number of saga state changes that are reduced by the map/reduce"); + } + + async Task IngestSagaAudits(params SagaSnapshot[] snapshots) + { + var unitOfWork = await StartAuditUnitOfWork(snapshots.Length); + foreach (var snapshot in snapshots) { - var unitOfWork = await StartAuditUnitOfWork(snapshots.Length); - foreach (var snapshot in snapshots) - { - await unitOfWork.RecordSagaSnapshot(snapshot); - } - await unitOfWork.DisposeAsync(); - await configuration.CompleteDBOperation(); + await unitOfWork.RecordSagaSnapshot(snapshot); } + + await unitOfWork.DisposeAsync(); + await configuration.CompleteDBOperation(); } } \ No newline at end of file From c8bf6bea9a9a671f50d897cdd70e301da7f5e469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 28 Feb 2025 14:00:58 +0100 Subject: [PATCH 3/8] Add test to check that indexes are reset on setup --- .../DatabaseSetup.cs | 6 ++-- .../IndexSetupTests.cs | 30 +++++++++++++++++-- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs b/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs index 015a3d8504..a10410408c 100644 --- a/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs +++ b/src/ServiceControl.Audit.Persistence.RavenDB/DatabaseSetup.cs @@ -23,7 +23,7 @@ public async Task Execute(IDocumentStore documentStore, CancellationToken cancel await UpdateDatabaseSettings(documentStore, configuration.Name, cancellationToken); - await CreateIndexes(documentStore, cancellationToken); + await CreateIndexes(documentStore, configuration.EnableFullTextSearch, cancellationToken); await ConfigureExpiration(documentStore, cancellationToken); } @@ -79,13 +79,13 @@ public static async Task DeleteLegacySagaDetailsIndex(IDocumentStore documentSto } } - async Task CreateIndexes(IDocumentStore documentStore, CancellationToken cancellationToken) + internal static async Task CreateIndexes(IDocumentStore documentStore, bool enableFreeTextSearch, CancellationToken cancellationToken) { await DeleteLegacySagaDetailsIndex(documentStore, cancellationToken); List indexList = [new FailedAuditImportIndex(), new SagaDetailsIndex()]; - if (configuration.EnableFullTextSearch) + if (enableFreeTextSearch) { indexList.Add(new MessagesViewIndexWithFullTextSearch()); await documentStore.Maintenance.SendAsync(new DeleteIndexOperation(MessagesViewIndexName), cancellationToken); diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs index 64ef3a550e..131a668518 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs @@ -1,9 +1,10 @@ namespace ServiceControl.Audit.Persistence.Tests; -using System; +using System.Threading; using System.Threading.Tasks; -using global::ServiceControl.Audit.Persistence.RavenDB; using NUnit.Framework; +using Persistence.RavenDB; +using Persistence.RavenDB.Indexes; using Raven.Client.Documents.Indexes; using Raven.Client.Documents.Operations.Indexes; @@ -11,7 +12,7 @@ namespace ServiceControl.Audit.Persistence.Tests; class IndexSetupTests : PersistenceTestFixture { [Test] - public async Task Corax_should_the_defaul_search_engine_type() + public async Task Corax_should_be_the_default_search_engine_type() { var indexes = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexesOperation(0, int.MaxValue)); @@ -31,4 +32,27 @@ public async Task Free_text_search_index_should_be_used_by_default() Assert.That(nonFreeTextIndex, Is.Null); Assert.That(freeTextIndex, Is.Not.Null); } + + [Test] + public async Task Indexes_should_be_reset_on_setup() + { + var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } }; + + await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore); + + //TODO: find a better way + await Task.Delay(1000); + + var indexStatsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + + Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); + + await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None); + + //TODO: find a better way + await Task.Delay(1000); + + var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Corax)); + } } \ No newline at end of file From 0b9794518f82fa432fe8425a5a8580552688cee6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 28 Feb 2025 14:59:07 +0100 Subject: [PATCH 4/8] Add lock mode tests --- .../IndexSetupTests.cs | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs index 131a668518..151a084366 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs @@ -1,5 +1,6 @@ namespace ServiceControl.Audit.Persistence.Tests; +using System; using System.Threading; using System.Threading.Tasks; using NUnit.Framework; @@ -7,6 +8,7 @@ namespace ServiceControl.Audit.Persistence.Tests; using Persistence.RavenDB.Indexes; using Raven.Client.Documents.Indexes; using Raven.Client.Documents.Operations.Indexes; +using Raven.Client.Exceptions.Documents.Indexes; [TestFixture] class IndexSetupTests : PersistenceTestFixture @@ -55,4 +57,50 @@ public async Task Indexes_should_be_reset_on_setup() var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Corax)); } + + [Test] + public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_ignore() + { + var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } }; + + await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore); + + await configuration.DocumentStore.Maintenance.SendAsync(new SetIndexesLockOperation(new SetIndexesLockOperation.Parameters + { + IndexNames = [index.IndexName], + Mode = IndexLockMode.LockedIgnore + })); + + //TODO: find a better way + await Task.Delay(1000); + + var indexStatsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + + Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); + + + await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None); + + //TODO: find a better way + await Task.Delay(1000); + + var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); + } + + [Test] + public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_error() + { + var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } }; + + await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore); + + await configuration.DocumentStore.Maintenance.SendAsync(new SetIndexesLockOperation(new SetIndexesLockOperation.Parameters + { + IndexNames = [index.IndexName], + Mode = IndexLockMode.LockedError + })); + + Assert.ThrowsAsync(async () => await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None)); + } } \ No newline at end of file From 7764a16d9b561d549a7f56768256ebf841f5f704 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Fri, 28 Feb 2025 15:51:57 +0100 Subject: [PATCH 5/8] Add test for non free text search --- .../IndexSetupTests.cs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs index 151a084366..3d5a1447d6 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs @@ -35,6 +35,21 @@ public async Task Free_text_search_index_should_be_used_by_default() Assert.That(freeTextIndex, Is.Not.Null); } + [Test] + public async Task Free_text_search_index_can_be_opted_out_from() + { + await DatabaseSetup.CreateIndexes(configuration.DocumentStore, false, CancellationToken.None); + + //TODO: find a better way + await Task.Delay(1000); + + var freeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName)); + var nonFreeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexName)); + + Assert.That(freeTextIndex, Is.Null); + Assert.That(nonFreeTextIndex, Is.Not.Null); + } + [Test] public async Task Indexes_should_be_reset_on_setup() { From 0aa1060c2326421282287f7e21af437ccd73fc1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Sat, 1 Mar 2025 12:13:24 +0100 Subject: [PATCH 6/8] Handle race conditions --- .../IndexSetupTests.cs | 79 +++++++++++-------- 1 file changed, 44 insertions(+), 35 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs index 3d5a1447d6..10449a4f8f 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs @@ -40,9 +40,6 @@ public async Task Free_text_search_index_can_be_opted_out_from() { await DatabaseSetup.CreateIndexes(configuration.DocumentStore, false, CancellationToken.None); - //TODO: find a better way - await Task.Delay(1000); - var freeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName)); var nonFreeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexName)); @@ -55,48 +52,35 @@ public async Task Indexes_should_be_reset_on_setup() { var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } }; - await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore); - - //TODO: find a better way - await Task.Delay(1000); - - var indexStatsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + var indexWithCustomConfigStats = await UpdateIndex(index); - Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); + Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None); - //TODO: find a better way - await Task.Delay(1000); + WaitForIndexDefinitionUpdate(indexWithCustomConfigStats); - var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); - Assert.That(indexStatsAfter.SearchEngineType, Is.EqualTo(SearchEngineType.Corax)); + var indexAfterResetStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + + Assert.That(indexAfterResetStats.SearchEngineType, Is.EqualTo(SearchEngineType.Corax)); } [Test] public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_ignore() { - var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } }; - - await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore); - - await configuration.DocumentStore.Maintenance.SendAsync(new SetIndexesLockOperation(new SetIndexesLockOperation.Parameters + var index = new MessagesViewIndexWithFullTextSearch { - IndexNames = [index.IndexName], - Mode = IndexLockMode.LockedIgnore - })); - - //TODO: find a better way - await Task.Delay(1000); + Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() }, + LockMode = IndexLockMode.LockedIgnore + }; - var indexStatsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + var indexStatsBefore = await UpdateIndex(index); Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); - await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None); - //TODO: find a better way + // raven will ignore the update since index was locked, so best we can do is wait a bit and check that settings hasn't changed await Task.Delay(1000); var indexStatsAfter = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); @@ -106,16 +90,41 @@ await configuration.DocumentStore.Maintenance.SendAsync(new SetIndexesLockOperat [Test] public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_error() { - var index = new MessagesViewIndexWithFullTextSearch { Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() } }; + var index = new MessagesViewIndexWithFullTextSearch + { + Configuration = { ["Indexing.Static.SearchEngineType"] = SearchEngineType.Lucene.ToString() }, + LockMode = IndexLockMode.LockedError + }; + + await UpdateIndex(index); + + Assert.ThrowsAsync(async () => await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None)); + } + + async Task UpdateIndex(IAbstractIndexCreationTask index) + { + var statsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore); - await configuration.DocumentStore.Maintenance.SendAsync(new SetIndexesLockOperation(new SetIndexesLockOperation.Parameters - { - IndexNames = [index.IndexName], - Mode = IndexLockMode.LockedError - })); + WaitForIndexDefinitionUpdate(statsBefore); - Assert.ThrowsAsync(async () => await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None)); + return await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + } + + void WaitForIndexDefinitionUpdate(IndexStats indexStats) + { + Assert.That(SpinWait.SpinUntil(() => + { + try + { + return configuration.DocumentStore.Maintenance.Send(new GetIndexStatisticsOperation(indexStats.Name)).CreatedTimestamp > indexStats.CreatedTimestamp; + } + catch (OperationCanceledException) + { + // keep going since we can get this if we query right when the update happens + return false; + } + }, TimeSpan.FromSeconds(10)), Is.True); } } \ No newline at end of file From 8818f4a868c1facfec06db32a6619118518309d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Sat, 1 Mar 2025 12:48:42 +0100 Subject: [PATCH 7/8] Use async wait --- .../IndexSetupTests.cs | 28 +++++++++++-------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs index 10449a4f8f..3fc9d35d45 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs @@ -58,7 +58,7 @@ public async Task Indexes_should_be_reset_on_setup() await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None); - WaitForIndexDefinitionUpdate(indexWithCustomConfigStats); + await WaitForIndexDefinitionUpdate(indexWithCustomConfigStats); var indexAfterResetStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); @@ -101,30 +101,34 @@ public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_error() Assert.ThrowsAsync(async () => await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None)); } - async Task UpdateIndex(IAbstractIndexCreationTask index) + async Task UpdateIndex(IAbstractIndexCreationTask index, CancellationToken cancellationToken = default) { - var statsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + var statsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName), cancellationToken); - await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore); + await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore, null, null, cancellationToken); - WaitForIndexDefinitionUpdate(statsBefore); - - return await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName)); + return await WaitForIndexDefinitionUpdate(statsBefore, cancellationToken); } - void WaitForIndexDefinitionUpdate(IndexStats indexStats) + async Task WaitForIndexDefinitionUpdate(IndexStats oldStats, CancellationToken cancellationToken = default) { - Assert.That(SpinWait.SpinUntil(() => + while (true) { try { - return configuration.DocumentStore.Maintenance.Send(new GetIndexStatisticsOperation(indexStats.Name)).CreatedTimestamp > indexStats.CreatedTimestamp; + var newStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(oldStats.Name), cancellationToken); + + if (newStats.CreatedTimestamp > oldStats.CreatedTimestamp) + { + return newStats; + } } catch (OperationCanceledException) { // keep going since we can get this if we query right when the update happens - return false; } - }, TimeSpan.FromSeconds(10)), Is.True); + + await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken); + } } } \ No newline at end of file From 943e0c5d617629e0ce8f0ee5b72b5072cfed531a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andreas=20=C3=96hlund?= Date: Sat, 1 Mar 2025 15:59:56 +0100 Subject: [PATCH 8/8] Add test cancellation --- .../IndexSetupTests.cs | 22 +++++++++---------- .../PersistenceTestFixture.cs | 11 ++++++++++ 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs index 3fc9d35d45..678915f391 100644 --- a/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs +++ b/src/ServiceControl.Audit.Persistence.Tests.RavenDB/IndexSetupTests.cs @@ -38,7 +38,7 @@ public async Task Free_text_search_index_should_be_used_by_default() [Test] public async Task Free_text_search_index_can_be_opted_out_from() { - await DatabaseSetup.CreateIndexes(configuration.DocumentStore, false, CancellationToken.None); + await DatabaseSetup.CreateIndexes(configuration.DocumentStore, false, TestTimeoutCancellationToken); var freeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexWithFulltextSearchName)); var nonFreeTextIndex = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexOperation(DatabaseSetup.MessagesViewIndexName)); @@ -56,7 +56,7 @@ public async Task Indexes_should_be_reset_on_setup() Assert.That(indexWithCustomConfigStats.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); - await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None); + await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken); await WaitForIndexDefinitionUpdate(indexWithCustomConfigStats); @@ -78,7 +78,7 @@ public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_ignore() Assert.That(indexStatsBefore.SearchEngineType, Is.EqualTo(SearchEngineType.Lucene)); - await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None); + await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken); // raven will ignore the update since index was locked, so best we can do is wait a bit and check that settings hasn't changed await Task.Delay(1000); @@ -98,25 +98,25 @@ public async Task Indexes_should_not_be_reset_on_setup_when_locked_as_error() await UpdateIndex(index); - Assert.ThrowsAsync(async () => await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, CancellationToken.None)); + Assert.ThrowsAsync(async () => await DatabaseSetup.CreateIndexes(configuration.DocumentStore, true, TestTimeoutCancellationToken)); } - async Task UpdateIndex(IAbstractIndexCreationTask index, CancellationToken cancellationToken = default) + async Task UpdateIndex(IAbstractIndexCreationTask index) { - var statsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName), cancellationToken); + var statsBefore = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(index.IndexName), TestTimeoutCancellationToken); - await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore, null, null, cancellationToken); + await IndexCreation.CreateIndexesAsync([index], configuration.DocumentStore, null, null, TestTimeoutCancellationToken); - return await WaitForIndexDefinitionUpdate(statsBefore, cancellationToken); + return await WaitForIndexDefinitionUpdate(statsBefore); } - async Task WaitForIndexDefinitionUpdate(IndexStats oldStats, CancellationToken cancellationToken = default) + async Task WaitForIndexDefinitionUpdate(IndexStats oldStats) { while (true) { try { - var newStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(oldStats.Name), cancellationToken); + var newStats = await configuration.DocumentStore.Maintenance.SendAsync(new GetIndexStatisticsOperation(oldStats.Name), TestTimeoutCancellationToken); if (newStats.CreatedTimestamp > oldStats.CreatedTimestamp) { @@ -128,7 +128,7 @@ async Task WaitForIndexDefinitionUpdate(IndexStats oldStats, Cancell // keep going since we can get this if we query right when the update happens } - await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken); + await Task.Delay(TimeSpan.FromMilliseconds(100), TestTimeoutCancellationToken); } } } \ No newline at end of file diff --git a/src/ServiceControl.Audit.Persistence.Tests/PersistenceTestFixture.cs b/src/ServiceControl.Audit.Persistence.Tests/PersistenceTestFixture.cs index e33d6a9c7a..9aa9205675 100644 --- a/src/ServiceControl.Audit.Persistence.Tests/PersistenceTestFixture.cs +++ b/src/ServiceControl.Audit.Persistence.Tests/PersistenceTestFixture.cs @@ -1,8 +1,10 @@ namespace ServiceControl.Audit.Persistence.Tests { using System; + using System.Diagnostics; using System.IO; using System.Linq; + using System.Threading; using System.Threading.Tasks; using Auditing.BodyStorage; using NUnit.Framework; @@ -18,12 +20,15 @@ public virtual Task Setup() { configuration = new PersistenceTestsConfiguration(); + testCancellationTokenSource = Debugger.IsAttached ? new CancellationTokenSource() : new CancellationTokenSource(TestTimeout); + return configuration.Configure(SetSettings); } [TearDown] public virtual Task Cleanup() { + testCancellationTokenSource?.Dispose(); return configuration?.Cleanup(); } @@ -64,5 +69,11 @@ protected ValueTask StartAuditUnitOfWork(int batchSiz protected IServiceProvider ServiceProvider => configuration.ServiceProvider; protected PersistenceTestsConfiguration configuration; + + protected CancellationToken TestTimeoutCancellationToken => testCancellationTokenSource.Token; + + CancellationTokenSource testCancellationTokenSource; + + static readonly TimeSpan TestTimeout = TimeSpan.FromSeconds(30); } } \ No newline at end of file