From 6562f207799c21cadebe141dcde04a326ca97128 Mon Sep 17 00:00:00 2001 From: Ian Nelson Date: Fri, 13 Jun 2025 10:43:53 +0100 Subject: [PATCH] feat: DTOSS-9346 introduce MeshFileEvents audit trail --- .../Functions/FileDiscoveryFunction.cs | 44 ++- .../Functions/FileExtractFunction.cs | 25 +- .../Functions/FileRetryFunction.cs | 16 +- .../Functions/FileTransformFunction.cs | 25 +- .../Functions/MeshFileFunctionBase.cs | 31 ++ ...0250612132848_AddMeshFileEvent.Designer.cs | 275 ++++++++++++++++++ .../20250612132848_AddMeshFileEvent.cs | 49 ++++ .../ServiceLayerDbContextModelSnapshot.cs | 45 +++ .../Data/Models/FileEventSource.cs | 9 + .../Data/Models/MeshFileEvent.cs | 21 ++ .../Data/ServiceLayerDbContext.cs | 23 +- .../Functions/FileDiscoveryFunctionTests.cs | 4 +- .../Functions/FileExtractFunctionTests.cs | 6 +- .../Functions/FileRetryFunctionTests.cs | 10 +- .../Functions/FileTransformFunctionTests.cs | 8 +- .../Functions/FunctionTestBase.cs | 32 +- 16 files changed, 547 insertions(+), 76 deletions(-) create mode 100644 src/ServiceLayer.Mesh/Functions/MeshFileFunctionBase.cs create mode 100644 src/ServiceLayer.Shared/Data/Migrations/20250612132848_AddMeshFileEvent.Designer.cs create mode 100644 src/ServiceLayer.Shared/Data/Migrations/20250612132848_AddMeshFileEvent.cs create mode 100644 src/ServiceLayer.Shared/Data/Models/FileEventSource.cs create mode 100644 src/ServiceLayer.Shared/Data/Models/MeshFileEvent.cs diff --git a/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs b/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs index 573b06bf..e63bdbbd 100644 --- a/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileDiscoveryFunction.cs @@ -15,6 +15,7 @@ public class FileDiscoveryFunction( IMeshInboxService meshInboxService, ServiceLayerDbContext serviceLayerDbContext, IFileExtractQueueClient fileExtractQueueClient) + : MeshFileFunctionBase(serviceLayerDbContext) { [Function("FileDiscoveryFunction")] public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo myTimer) @@ -26,28 +27,16 @@ public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo // TODO - check if response.IsSuccessful before proceeding to dereference the Response.Messages foreach (var messageId in response.Response.Messages) { - await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); + await using var transaction = await ServiceLayerDbContext.Database.BeginTransactionAsync(); - var existing = await serviceLayerDbContext.MeshFiles + var existing = await ServiceLayerDbContext.MeshFiles .AnyAsync(f => f.FileId == messageId); if (!existing) { - var file = new MeshFile - { - FileId = messageId, - FileType = MeshFileType.NbssAppointmentEvents, - MailboxId = configuration.NbssMeshMailboxId, - Status = MeshFileStatus.Discovered, - FirstSeenUtc = DateTime.UtcNow, - LastUpdatedUtc = DateTime.UtcNow - }; - - serviceLayerDbContext.MeshFiles.Add(file); - - await serviceLayerDbContext.SaveChangesAsync(); - await transaction.CommitAsync(); + var file = await CreateMeshFile(messageId); + await transaction.CommitAsync(); await fileExtractQueueClient.EnqueueFileExtractAsync(file); } else @@ -56,4 +45,27 @@ public async Task Run([TimerTrigger("%FileDiscoveryTimerExpression%")] TimerInfo } } } + + private async Task CreateMeshFile(string messageId) + { + var now = DateTime.UtcNow; + + var file = new MeshFile + { + FileId = messageId, + FileType = MeshFileType.NbssAppointmentEvents, + MailboxId = configuration.NbssMeshMailboxId, + Status = MeshFileStatus.Discovered, + FirstSeenUtc = now, + LastUpdatedUtc = now + }; + + ServiceLayerDbContext.MeshFiles.Add(file); + + await UpdateMeshFile(file, MeshFileStatus.Discovered); + + return file; + } + + protected override FileEventSource Source => FileEventSource.DiscoveryFunction; } diff --git a/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs b/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs index a8c62de0..2e5e34cd 100644 --- a/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileExtractFunction.cs @@ -17,14 +17,14 @@ public class FileExtractFunction( ServiceLayerDbContext serviceLayerDbContext, IFileTransformQueueClient fileTransformQueueClient, IFileExtractQueueClient fileExtractQueueClient, - IMeshFilesBlobStore meshFileBlobStore) + IMeshFilesBlobStore meshFileBlobStore) : MeshFileFunctionBase(serviceLayerDbContext) { [Function("FileExtractFunction")] public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueMessage message) { logger.LogInformation("{FunctionName} started. Processing fileId: {FileId}", nameof(FileExtractFunction), message.FileId); - await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); + await using var transaction = await ServiceLayerDbContext.Database.BeginTransactionAsync(); var file = await GetFileAsync(message.FileId); if (file == null || !IsFileSuitableForExtraction(file)) @@ -32,7 +32,7 @@ public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueM return; } - await UpdateFileStatusForExtraction(file); + await UpdateMeshFile(file, MeshFileStatus.Extracting); await transaction.CommitAsync(); try @@ -47,7 +47,7 @@ public async Task Run([QueueTrigger("%FileExtractQueueName%")] FileExtractQueueM private async Task GetFileAsync(string fileId) { - var file = await serviceLayerDbContext.MeshFiles + var file = await ServiceLayerDbContext.MeshFiles .FirstOrDefaultAsync(f => f.FileId == fileId); if (file == null) @@ -76,13 +76,6 @@ private bool IsFileSuitableForExtraction(MeshFile file) return true; } - private async Task UpdateFileStatusForExtraction(MeshFile file) - { - file.Status = MeshFileStatus.Extracting; - file.LastUpdatedUtc = DateTime.UtcNow; - await serviceLayerDbContext.SaveChangesAsync(); - } - private async Task ProcessFileExtraction(MeshFile file) { var meshResponse = await meshInboxService.GetMessageByIdAsync(file.MailboxId, file.FileId); @@ -100,9 +93,7 @@ private async Task ProcessFileExtraction(MeshFile file) } file.BlobPath = blobPath; - file.Status = MeshFileStatus.Extracted; - file.LastUpdatedUtc = DateTime.UtcNow; - await serviceLayerDbContext.SaveChangesAsync(); + await UpdateMeshFile(file, MeshFileStatus.Extracted); await fileTransformQueueClient.EnqueueFileTransformAsync(file); } @@ -110,9 +101,9 @@ private async Task ProcessFileExtraction(MeshFile file) private async Task HandleExtractionError(MeshFile file, FileExtractQueueMessage message, Exception ex) { logger.LogError(ex, "An exception occurred during file extraction for fileId: {FileId}", message.FileId); - file.Status = MeshFileStatus.FailedExtract; - file.LastUpdatedUtc = DateTime.UtcNow; - await serviceLayerDbContext.SaveChangesAsync(); + await UpdateMeshFile(file, MeshFileStatus.FailedExtract); await fileExtractQueueClient.SendToPoisonQueueAsync(message); } + + protected override FileEventSource Source => FileEventSource.ExtractFunction; } diff --git a/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs b/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs index afd893cb..62f8b338 100644 --- a/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileRetryFunction.cs @@ -13,7 +13,7 @@ public class FileRetryFunction( ServiceLayerDbContext serviceLayerDbContext, IFileExtractQueueClient fileExtractQueueClient, IFileTransformQueueClient fileTransformQueueClient, - IFileRetryFunctionConfiguration configuration) + IFileRetryFunctionConfiguration configuration) : MeshFileFunctionBase(serviceLayerDbContext) { [Function("FileRetryFunction")] public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myTimer) @@ -28,7 +28,7 @@ public async Task Run([TimerTrigger("%FileRetryTimerExpression%")] TimerInfo myT private async Task RetryStaleExtractions(DateTime staleDateTimeUtc) { - var staleFiles = await serviceLayerDbContext.MeshFiles + var staleFiles = await ServiceLayerDbContext.MeshFiles .Where(f => (f.Status == MeshFileStatus.Discovered || f.Status == MeshFileStatus.Extracting) && f.LastUpdatedUtc <= staleDateTimeUtc) @@ -39,15 +39,15 @@ private async Task RetryStaleExtractions(DateTime staleDateTimeUtc) foreach (var file in staleFiles) { await fileExtractQueueClient.EnqueueFileExtractAsync(file); - file.LastUpdatedUtc = DateTime.UtcNow; - await serviceLayerDbContext.SaveChangesAsync(); + await UpdateMeshFile(file, file.Status); + await ServiceLayerDbContext.SaveChangesAsync(); logger.LogInformation("FileRetryFunction: File {FileFileId} enqueued to Extract queue", file.FileId); } } private async Task RetryStaleTransformations(DateTime staleDateTimeUtc) { - var staleFiles = await serviceLayerDbContext.MeshFiles + var staleFiles = await ServiceLayerDbContext.MeshFiles .Where(f => (f.Status == MeshFileStatus.Extracted || f.Status == MeshFileStatus.Transforming) && f.LastUpdatedUtc <= staleDateTimeUtc) @@ -58,9 +58,11 @@ private async Task RetryStaleTransformations(DateTime staleDateTimeUtc) foreach (var file in staleFiles) { await fileTransformQueueClient.EnqueueFileTransformAsync(file); - file.LastUpdatedUtc = DateTime.UtcNow; - await serviceLayerDbContext.SaveChangesAsync(); + await UpdateMeshFile(file, file.Status); + await ServiceLayerDbContext.SaveChangesAsync(); logger.LogInformation("FileRetryFunction: File {FileFileId} enqueued to Transform queue", file.FileId); } } + + protected override FileEventSource Source => FileEventSource.RetryFunction; } diff --git a/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs b/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs index cd61816f..8ddabdaa 100644 --- a/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs +++ b/src/ServiceLayer.Mesh/Functions/FileTransformFunction.cs @@ -18,7 +18,7 @@ public class FileTransformFunction( ServiceLayerDbContext serviceLayerDbContext, IFileTransformQueueClient fileTransformQueueClient, IMeshFilesBlobStore meshFileBlobStore, - IEnumerable fileTransformers) + IEnumerable fileTransformers) : MeshFileFunctionBase(serviceLayerDbContext) { private static readonly JsonSerializerOptions ValidationErrorJsonOptions = new() { @@ -34,7 +34,7 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu logger.LogInformation("{FunctionName} started. Processing fileId: {FileId}", nameof(FileTransformFunction), message.FileId); - await using var transaction = await serviceLayerDbContext.Database.BeginTransactionAsync(); + await using var transaction = await ServiceLayerDbContext.Database.BeginTransactionAsync(); var file = await GetFileAsync(message.FileId); if (file == null || !IsFileSuitableForTransformation(file)) @@ -42,7 +42,7 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu return; } - await UpdateFileStatusForTransformation(file); + await UpdateMeshFile(file, MeshFileStatus.Transforming); await transaction.CommitAsync(); try @@ -57,7 +57,7 @@ public async Task Run([QueueTrigger("%FileTransformQueueName%")] FileTransformQu private async Task GetFileAsync(string fileId) { - var file = await serviceLayerDbContext.MeshFiles + var file = await ServiceLayerDbContext.MeshFiles .FirstOrDefaultAsync(f => f.FileId == fileId); if (file == null) @@ -87,13 +87,6 @@ private bool IsFileSuitableForTransformation(MeshFile file) return true; } - private async Task UpdateFileStatusForTransformation(MeshFile file) - { - file.Status = MeshFileStatus.Transforming; - file.LastUpdatedUtc = DateTime.UtcNow; - await serviceLayerDbContext.SaveChangesAsync(); - } - private async Task ProcessFileTransformation(MeshFile file) { var transformer = GetTransformerFor(file.FileType); @@ -107,9 +100,7 @@ private async Task ProcessFileTransformation(MeshFile file) throw new InvalidOperationException("Validation errors encountered"); } - file.Status = MeshFileStatus.Transformed; - file.LastUpdatedUtc = DateTime.UtcNow; - await serviceLayerDbContext.SaveChangesAsync(); + await UpdateMeshFile(file, MeshFileStatus.Transformed); } private IFileTransformer GetTransformerFor(MeshFileType type) @@ -129,9 +120,9 @@ private IFileTransformer GetTransformerFor(MeshFileType type) private async Task HandleTransformationError(MeshFile file, FileTransformQueueMessage message, Exception ex) { logger.LogError(ex, "An exception occurred during file transformation for fileId: {FileId}", message.FileId); - file.Status = MeshFileStatus.FailedTransform; - file.LastUpdatedUtc = DateTime.UtcNow; - await serviceLayerDbContext.SaveChangesAsync(); + await UpdateMeshFile(file, MeshFileStatus.FailedTransform); await fileTransformQueueClient.SendToPoisonQueueAsync(message); } + + protected override FileEventSource Source => FileEventSource.TransformFunction; } diff --git a/src/ServiceLayer.Mesh/Functions/MeshFileFunctionBase.cs b/src/ServiceLayer.Mesh/Functions/MeshFileFunctionBase.cs new file mode 100644 index 00000000..ec1af79a --- /dev/null +++ b/src/ServiceLayer.Mesh/Functions/MeshFileFunctionBase.cs @@ -0,0 +1,31 @@ +using ServiceLayer.Data; +using ServiceLayer.Data.Models; + +namespace ServiceLayer.Mesh.Functions; + +public abstract class MeshFileFunctionBase(ServiceLayerDbContext serviceLayerDbContext) +{ + protected ServiceLayerDbContext ServiceLayerDbContext { get; } = serviceLayerDbContext; + + protected abstract FileEventSource Source { get; } + + protected async Task UpdateMeshFile(MeshFile file, MeshFileStatus status) + { + var now = DateTime.UtcNow; + + file.Status = status; + file.LastUpdatedUtc = now; + + var fileEvent = new MeshFileEvent + { + FileId = file.FileId, + Status = status, + TimestampUtc = now, + Source = Source + }; + + ServiceLayerDbContext.MeshFileEvents.Add(fileEvent); + + await ServiceLayerDbContext.SaveChangesAsync(); + } +} diff --git a/src/ServiceLayer.Shared/Data/Migrations/20250612132848_AddMeshFileEvent.Designer.cs b/src/ServiceLayer.Shared/Data/Migrations/20250612132848_AddMeshFileEvent.Designer.cs new file mode 100644 index 00000000..2c42b6ac --- /dev/null +++ b/src/ServiceLayer.Shared/Data/Migrations/20250612132848_AddMeshFileEvent.Designer.cs @@ -0,0 +1,275 @@ +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Metadata; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using ServiceLayer.Data; + +#nullable disable + +namespace ServiceLayer.Mesh.Migrations +{ + [DbContext(typeof(ServiceLayerDbContext))] + [Migration("20250612132848_AddMeshFileEvent")] + partial class AddMeshFileEvent + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "9.0.5") + .HasAnnotation("Relational:MaxIdentifierLength", 128); + + SqlServerModelBuilderExtensions.UseIdentityColumns(modelBuilder); + + modelBuilder.Entity("ServiceLayer.Data.Models.MeshFile", b => + { + b.Property("FileId") + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("BlobPath") + .HasMaxLength(1024) + .HasColumnType("nvarchar(1024)"); + + b.Property("FileType") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("FirstSeenUtc") + .HasColumnType("datetime2"); + + b.Property("LastUpdatedUtc") + .HasColumnType("datetime2"); + + b.Property("MailboxId") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("nvarchar(50)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("ValidationErrors") + .HasColumnType("nvarchar(max)"); + + b.HasKey("FileId"); + + b.ToTable("MeshFiles"); + }); + + modelBuilder.Entity("ServiceLayer.Data.Models.MeshFileEvent", b => + { + b.Property("EventId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("NewStatus") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("OldStatus") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("Source") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("TimestampUtc") + .HasColumnType("datetime2"); + + b.HasKey("EventId"); + + b.HasIndex("FileId"); + + b.ToTable("MeshFileEvents"); + }); + + modelBuilder.Entity("ServiceLayer.Data.Models.NbssAppointmentEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("Action") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("ActionTimestamp") + .HasColumnType("datetime2(0)"); + + b.Property("AppointmentDateTime") + .HasColumnType("datetime2(0)"); + + b.Property("AppointmentId") + .IsRequired() + .HasMaxLength(27) + .HasColumnType("varchar(27)"); + + b.Property("AppointmentType") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("AttendedNotScreened") + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("BSO") + .IsRequired() + .HasMaxLength(3) + .HasColumnType("char(3)"); + + b.Property("BatchId") + .IsRequired() + .HasMaxLength(9) + .HasColumnType("varchar(9)"); + + b.Property("BookedBy") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("CancelledBy") + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("ClinicAddressLine1") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicAddressLine2") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicAddressLine3") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicAddressLine4") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicAddressLine5") + .IsRequired() + .HasMaxLength(30) + .HasColumnType("varchar(30)"); + + b.Property("ClinicCode") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("varchar(5)"); + + b.Property("ClinicName") + .IsRequired() + .HasMaxLength(40) + .HasColumnType("varchar(40)"); + + b.Property("ClinicNameOnLetters") + .IsRequired() + .HasMaxLength(50) + .HasColumnType("varchar(50)"); + + b.Property("ClinicPostcode") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("varchar(8)"); + + b.Property("EpisodeStart") + .HasColumnType("date"); + + b.Property("EpisodeType") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("ExtractId") + .IsRequired() + .HasMaxLength(8) + .HasColumnType("char(8)"); + + b.Property("HoldingClinic") + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.Property("Location") + .IsRequired() + .HasMaxLength(5) + .HasColumnType("varchar(5)"); + + b.Property("MeshFileId") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("NhsNumber") + .IsRequired() + .HasMaxLength(10) + .HasColumnType("char(10)"); + + b.Property("ScreeningAppointmentNumber") + .HasColumnType("tinyint"); + + b.Property("Sequence") + .IsRequired() + .HasMaxLength(6) + .HasColumnType("char(6)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(1) + .HasColumnType("char(1)"); + + b.HasKey("Id"); + + b.HasIndex("MeshFileId"); + + b.ToTable("NbssAppointmentEvents"); + }); + + modelBuilder.Entity("ServiceLayer.Data.Models.MeshFileEvent", b => + { + b.HasOne("ServiceLayer.Data.Models.MeshFile", null) + .WithMany("Events") + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ServiceLayer.Data.Models.NbssAppointmentEvent", b => + { + b.HasOne("ServiceLayer.Data.Models.MeshFile", null) + .WithMany() + .HasForeignKey("MeshFileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + + modelBuilder.Entity("ServiceLayer.Data.Models.MeshFile", b => + { + b.Navigation("Events"); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/src/ServiceLayer.Shared/Data/Migrations/20250612132848_AddMeshFileEvent.cs b/src/ServiceLayer.Shared/Data/Migrations/20250612132848_AddMeshFileEvent.cs new file mode 100644 index 00000000..17a2b2cc --- /dev/null +++ b/src/ServiceLayer.Shared/Data/Migrations/20250612132848_AddMeshFileEvent.cs @@ -0,0 +1,49 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace ServiceLayer.Mesh.Migrations +{ + /// + public partial class AddMeshFileEvent : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "MeshFileEvents", + columns: table => new + { + EventId = table.Column(type: "uniqueidentifier", nullable: false), + FileId = table.Column(type: "nvarchar(255)", maxLength: 255, nullable: false), + TimestampUtc = table.Column(type: "datetime2", nullable: false), + OldStatus = table.Column(type: "nvarchar(20)", maxLength: 20, nullable: false), + NewStatus = table.Column(type: "nvarchar(20)", maxLength: 20, nullable: false), + Source = table.Column(type: "nvarchar(20)", maxLength: 20, nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_MeshFileEvents", x => x.EventId); + table.ForeignKey( + name: "FK_MeshFileEvents_MeshFiles_FileId", + column: x => x.FileId, + principalTable: "MeshFiles", + principalColumn: "FileId", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "IX_MeshFileEvents_FileId", + table: "MeshFileEvents", + column: "FileId"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "MeshFileEvents"); + } + } +} diff --git a/src/ServiceLayer.Shared/Data/Migrations/ServiceLayerDbContextModelSnapshot.cs b/src/ServiceLayer.Shared/Data/Migrations/ServiceLayerDbContextModelSnapshot.cs index ec54481e..1441aba8 100644 --- a/src/ServiceLayer.Shared/Data/Migrations/ServiceLayerDbContextModelSnapshot.cs +++ b/src/ServiceLayer.Shared/Data/Migrations/ServiceLayerDbContextModelSnapshot.cs @@ -61,6 +61,37 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("MeshFiles"); }); + modelBuilder.Entity("ServiceLayer.Data.Models.MeshFileEvent", b => + { + b.Property("EventId") + .ValueGeneratedOnAdd() + .HasColumnType("uniqueidentifier"); + + b.Property("FileId") + .IsRequired() + .HasMaxLength(255) + .HasColumnType("nvarchar(255)"); + + b.Property("Status") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("Source") + .IsRequired() + .HasMaxLength(20) + .HasColumnType("nvarchar(20)"); + + b.Property("TimestampUtc") + .HasColumnType("datetime2"); + + b.HasKey("EventId"); + + b.HasIndex("FileId"); + + b.ToTable("MeshFileEvents"); + }); + modelBuilder.Entity("ServiceLayer.Data.Models.NbssAppointmentEvent", b => { b.Property("Id") @@ -208,6 +239,15 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("NbssAppointmentEvents"); }); + modelBuilder.Entity("ServiceLayer.Data.Models.MeshFileEvent", b => + { + b.HasOne("ServiceLayer.Data.Models.MeshFile", null) + .WithMany("Events") + .HasForeignKey("FileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + }); + modelBuilder.Entity("ServiceLayer.Data.Models.NbssAppointmentEvent", b => { b.HasOne("ServiceLayer.Data.Models.MeshFile", null) @@ -216,6 +256,11 @@ protected override void BuildModel(ModelBuilder modelBuilder) .OnDelete(DeleteBehavior.Cascade) .IsRequired(); }); + + modelBuilder.Entity("ServiceLayer.Data.Models.MeshFile", b => + { + b.Navigation("Events"); + }); #pragma warning restore 612, 618 } } diff --git a/src/ServiceLayer.Shared/Data/Models/FileEventSource.cs b/src/ServiceLayer.Shared/Data/Models/FileEventSource.cs new file mode 100644 index 00000000..9505c294 --- /dev/null +++ b/src/ServiceLayer.Shared/Data/Models/FileEventSource.cs @@ -0,0 +1,9 @@ +namespace ServiceLayer.Data.Models; + +public enum FileEventSource +{ + DiscoveryFunction, + ExtractFunction, + TransformFunction, + RetryFunction +} diff --git a/src/ServiceLayer.Shared/Data/Models/MeshFileEvent.cs b/src/ServiceLayer.Shared/Data/Models/MeshFileEvent.cs new file mode 100644 index 00000000..4c078748 --- /dev/null +++ b/src/ServiceLayer.Shared/Data/Models/MeshFileEvent.cs @@ -0,0 +1,21 @@ +using System.ComponentModel.DataAnnotations; +using System.ComponentModel.DataAnnotations.Schema; + +namespace ServiceLayer.Data.Models; + +[Table("MeshFileEvents")] +public class MeshFileEvent +{ + public Guid EventId { get; set; } = Guid.NewGuid(); + + [MaxLength(255)] + public required string FileId { get; set; } + + public required DateTime TimestampUtc { get; set; } + + [MaxLength(20)] + public required MeshFileStatus Status { get; set; } + + [MaxLength(20)] + public required FileEventSource Source { get; set; } +} diff --git a/src/ServiceLayer.Shared/Data/ServiceLayerDbContext.cs b/src/ServiceLayer.Shared/Data/ServiceLayerDbContext.cs index 6d39aaaa..0518ad59 100644 --- a/src/ServiceLayer.Shared/Data/ServiceLayerDbContext.cs +++ b/src/ServiceLayer.Shared/Data/ServiceLayerDbContext.cs @@ -7,14 +7,35 @@ public class ServiceLayerDbContext(DbContextOptions optio { public DbSet MeshFiles { get; set; } public DbSet NbssAppointmentEvents { get; set; } + public DbSet MeshFileEvents { get; set; } protected override void OnModelCreating(ModelBuilder modelBuilder) { - // Configure relationships, keys, etc. + ConfigureMeshFiles(modelBuilder); + ConfigureMeshFileEvents(modelBuilder); + ConfigureNbssAppointmentEvents(modelBuilder); + } + + private static void ConfigureMeshFiles(ModelBuilder modelBuilder) + { modelBuilder.Entity().HasKey(p => p.FileId); modelBuilder.Entity().Property(e => e.Status).HasConversion(); modelBuilder.Entity().Property(e => e.FileType).HasConversion(); + } + + private static void ConfigureMeshFileEvents(ModelBuilder modelBuilder) + { + modelBuilder.Entity().HasKey(e => e.EventId); + modelBuilder.Entity().Property(e => e.Status).HasConversion(); + modelBuilder.Entity().Property(e => e.Source).HasConversion(); + modelBuilder.Entity() + .HasOne() + .WithMany() + .HasForeignKey(e => e.FileId); + } + private static void ConfigureNbssAppointmentEvents(ModelBuilder modelBuilder) + { modelBuilder.Entity().HasKey(e => e.Id); modelBuilder.Entity() .HasOne() diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs index 61aa477b..11a88323 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileDiscoveryFunctionTests.cs @@ -48,7 +48,7 @@ public async Task Run_AddsNewMessageToDbAndQueue() await _function.Run(new TimerInfo()); // Assert - var meshFile = AssertFileUpdated(testMessageId, MeshFileStatus.Discovered); + var meshFile = AssertFileUpdated(testMessageId, MeshFileStatus.Discovered, FileEventSource.DiscoveryFunction); Assert.Equal("test-mailbox", meshFile.MailboxId); // TODO - replace the It.IsAny with a more specific matcher, or use a callback @@ -113,7 +113,7 @@ public async Task Run_MultipleMessagesInInbox_AllAreProcessed() // Assert foreach (var id in messageIds) { - var savedFile = AssertFileUpdated(id, MeshFileStatus.Discovered); + var savedFile = AssertFileUpdated(id, MeshFileStatus.Discovered, FileEventSource.DiscoveryFunction); Assert.Equal("test-mailbox", savedFile.MailboxId); } diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs index c19697ec..1d56004a 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileExtractFunctionTests.cs @@ -119,7 +119,7 @@ public async Task Run_FileValid_FileUploadedToBlobAndAcknowledgedAndEnqueued(Mes _meshInboxServiceMock.Verify(m => m.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId), Times.Once); _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(file), Times.Once); - var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.Extracted); + var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.Extracted, FileEventSource.ExtractFunction); Assert.Equal(blobPath, updatedFile.BlobPath); } @@ -156,7 +156,7 @@ public async Task Run_GetMessageFails_ErrorLoggedAndFileSentToPoisonQueue() _meshInboxServiceMock.Verify(m => m.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId), Times.Never); _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.IsAny()), Times.Never); _fileExtractQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Once); - var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.FailedExtract); + var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.FailedExtract, FileEventSource.ExtractFunction); Assert.Null(updatedFile.BlobPath); } @@ -204,7 +204,7 @@ public async Task Run_AcknowledgeMessageFails_WarningLoggedAndProcessingContinue _meshInboxServiceMock.Verify(m => m.AcknowledgeMessageByIdAsync(file.MailboxId, file.FileId), Times.Once); _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(file), Times.Once); _fileExtractQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Never); - var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.Extracted); + var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.Extracted, FileEventSource.ExtractFunction); Assert.Equal(blobPath, updatedFile.BlobPath); } } diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileRetryFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileRetryFunctionTests.cs index d989a375..a88993d7 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/FileRetryFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileRetryFunctionTests.cs @@ -45,7 +45,7 @@ public async Task Run_EnqueuesDiscoveredOrExtractingFilesOlderThan12Hours(MeshFi _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is(f => f.FileId == file.FileId)), Times.Once); _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == file.FileId)), Times.Never); - AssertFileUpdated(file.FileId, testStatus); + AssertFileUpdated(file.FileId, testStatus, FileEventSource.RetryFunction); } [Theory] @@ -63,7 +63,7 @@ public async Task Run_EnqueuesExtractedOrTransformingFilesOlderThan12Hours(MeshF _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == file.FileId)), Times.Once); _fileExtractQueueClientMock.Verify(q => q.EnqueueFileExtractAsync(It.Is(f => f.FileId == file.FileId)), Times.Never); - AssertFileUpdated(file.FileId, testStatus); + AssertFileUpdated(file.FileId, testStatus, FileEventSource.RetryFunction); } [Theory] @@ -130,9 +130,9 @@ public async Task Run_ProcessesMultipleEligibleFiles() _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == file2.FileId)), Times.Once); _fileTransformQueueClientMock.Verify(q => q.EnqueueFileTransformAsync(It.Is(f => f.FileId == file3.FileId)), Times.Once); - AssertFileUpdated(file1.FileId, MeshFileStatus.Discovered); - AssertFileUpdated(file2.FileId, MeshFileStatus.Extracted); - AssertFileUpdated(file3.FileId, MeshFileStatus.Transforming); + AssertFileUpdated(file1.FileId, MeshFileStatus.Discovered, FileEventSource.RetryFunction); + AssertFileUpdated(file2.FileId, MeshFileStatus.Extracted, FileEventSource.RetryFunction); + AssertFileUpdated(file3.FileId, MeshFileStatus.Transforming, FileEventSource.RetryFunction); } } diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs index 9a66f513..6f0718c4 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FileTransformFunctionTests.cs @@ -99,7 +99,7 @@ public async Task Run_FileValidNoTransformersExist_ErrorLoggedAndStatusUpdated() _blobStoreMock.Verify(x => x.DownloadAsync(file), Times.Never); _fileTransformQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Once); - AssertFileUpdated(file.FileId, MeshFileStatus.FailedTransform); + AssertFileUpdated(file.FileId, MeshFileStatus.FailedTransform, FileEventSource.TransformFunction); } [Fact] @@ -125,7 +125,7 @@ public async Task Run_FileValidMultipleTransformersExist_ErrorLoggedAndStatusUpd _blobStoreMock.Verify(x => x.DownloadAsync(file), Times.Never); _fileTransformQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Once); - AssertFileUpdated(file.FileId, MeshFileStatus.FailedTransform); + AssertFileUpdated(file.FileId, MeshFileStatus.FailedTransform, FileEventSource.TransformFunction); } [Fact] @@ -159,7 +159,7 @@ public async Task Run_FileHasValidationErrors_ErrorLoggedAndStatusAndValidationE _blobStoreMock.Verify(x => x.DownloadAsync(file), Times.Once); _fileTransformQueueClientMock.Verify(q => q.SendToPoisonQueueAsync(message), Times.Once); - var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.FailedTransform); + var updatedFile = AssertFileUpdated(file.FileId, MeshFileStatus.FailedTransform, FileEventSource.TransformFunction); var savedValidationErrors = DeserializeValidationErrorsFromMeshFile(updatedFile); Assert.Equal(validationErrors, savedValidationErrors, new ValidationErrorComparer()); } @@ -187,7 +187,7 @@ public async Task Run_FileValid_FileTransformedAndStatusUpdated(MeshFileStatus v LoggerMock.VerifyNoLogs(LogLevel.Warning); _blobStoreMock.Verify(x => x.DownloadAsync(file), Times.Once); _fileTransformerMock.Verify(x => x.TransformFileAsync(expectedStream, file), Times.Once); - AssertFileUpdated(file.FileId, MeshFileStatus.Transformed); + AssertFileUpdated(file.FileId, MeshFileStatus.Transformed, FileEventSource.TransformFunction); } private static readonly JsonSerializerOptions ValidationErrorJsonOptions = new() diff --git a/tests/ServiceLayer.Mesh.Tests/Functions/FunctionTestBase.cs b/tests/ServiceLayer.Mesh.Tests/Functions/FunctionTestBase.cs index dda7bc0f..ebc3a44a 100644 --- a/tests/ServiceLayer.Mesh.Tests/Functions/FunctionTestBase.cs +++ b/tests/ServiceLayer.Mesh.Tests/Functions/FunctionTestBase.cs @@ -24,15 +24,28 @@ protected FunctionTestBase() protected MeshFile SaveMeshFile(MeshFileStatus status = MeshFileStatus.Extracted, int hoursOld = 1) { + var lastUpdated = DateTime.UtcNow.AddHours(-hoursOld); + var fileId = Guid.NewGuid().ToString(); + var file = new MeshFile { FileType = MeshFileType.NbssAppointmentEvents, MailboxId = Guid.NewGuid().ToString(), - FileId = Guid.NewGuid().ToString(), + FileId = fileId, + Status = status, + LastUpdatedUtc = lastUpdated + }; + + var fileEvent = new MeshFileEvent + { + FileId = fileId, Status = status, - LastUpdatedUtc = DateTime.UtcNow.AddHours(-hoursOld), + TimestampUtc = lastUpdated, + Source = FileEventSource.DiscoveryFunction }; + DbContext.MeshFiles.Add(file); + DbContext.MeshFileEvents.Add(fileEvent); DbContext.SaveChanges(); return file; } @@ -46,11 +59,22 @@ protected MeshFile AssertFileUnchanged(string fileId, MeshFileStatus expectedSta return unchanged; } - protected MeshFile AssertFileUpdated(string fileId, MeshFileStatus expectedStatus) + protected MeshFile AssertFileUpdated(string fileId, MeshFileStatus expectedStatus, FileEventSource expectedSource) { - var updated = DbContext.MeshFiles.Single(x => x.FileId == fileId); + var updated = DbContext.MeshFiles + .Single(x => x.FileId == fileId); Assert.Equal(expectedStatus, updated.Status); Assert.True(updated.LastUpdatedUtc > DateTime.UtcNow.AddSeconds(-10)); + + var lastEvent = DbContext.MeshFileEvents + .Where(x => x.FileId == fileId) + .OrderByDescending(x => x.TimestampUtc) + .First(); + + Assert.Equal(expectedStatus, lastEvent.Status); + Assert.True(lastEvent.TimestampUtc > DateTime.UtcNow.AddSeconds(-10)); + Assert.Equal(expectedSource, lastEvent.Source); + return updated; } }