Feature/ccm 12950 transform publish updates

infrastructure/terraform/components/api/README.md

@@ -18,7 +18,7 @@ No requirements.
 | <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
 | <a name="input_disable_gateway_execute_endpoint"></a> [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no |
 | <a name="input_enable_api_data_trace"></a> [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no |
-| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no |
+| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no |
 | <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
 | <a name="input_eventpub_control_plane_bus_arn"></a> [eventpub\_control\_plane\_bus\_arn](#input\_eventpub\_control\_plane\_bus\_arn) | ARN of the EventBridge control plane bus for eventpub | `string` | `""` | no |
@@ -42,6 +42,8 @@ No requirements.
 
 | Name | Source | Version |
 |------|--------|---------|
+| <a name="module_amendment_event_transformer"></a> [amendment\_event\_transformer](#module\_amendment\_event\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
+| <a name="module_amendments_queue"></a> [amendments\_queue](#module\_amendments\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
 | <a name="module_authorizer_lambda"></a> [authorizer\_lambda](#module\_authorizer\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_domain_truststore"></a> [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a |
 | <a name="module_eventpub"></a> [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-eventpub.zip | n/a |
@@ -51,8 +53,6 @@ No requirements.
 | <a name="module_get_letters"></a> [get\_letters](#module\_get\_letters) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_get_status"></a> [get\_status](#module\_get\_status) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_kms"></a> [kms](#module\_kms) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-kms.zip | n/a |
-| <a name="module_letter_status_update"></a> [letter\_status\_update](#module\_letter\_status\_update) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
-| <a name="module_letter_status_updates_queue"></a> [letter\_status\_updates\_queue](#module\_letter\_status\_updates\_queue) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip | n/a |
 | <a name="module_letter_updates_transformer"></a> [letter\_updates\_transformer](#module\_letter\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_logging_bucket"></a> [logging\_bucket](#module\_logging\_bucket) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a |
 | <a name="module_mi_updates_transformer"></a> [mi\_updates\_transformer](#module\_mi\_updates\_transformer) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-lambda.zip | n/a |

infrastructure/terraform/components/api/event_source_mapping_status_updates_to_handler.tf

@@ -1,12 +1,12 @@
 resource "aws_lambda_event_source_mapping" "status_updates_sqs_to_status_update_handler" {
-  event_source_arn                   = module.letter_status_updates_queue.sqs_queue_arn
-  function_name                      = module.letter_status_update.function_arn
+  event_source_arn                   = module.amendments_queue.sqs_queue_arn
+  function_name                      = module.amendment_event_transformer.function_arn
   batch_size                         = 10
   maximum_batching_window_in_seconds = 1
   scaling_config { maximum_concurrency = 10 }
 
   depends_on = [
-    module.letter_status_updates_queue, # ensures queue exists
-    module.letter_status_update         # ensures update handler exists
+    module.amendments_queue,           # ensures queue exists
+    module.amendment_event_transformer # ensures update handler exists
   ]
 }

infrastructure/terraform/components/api/lambda_event_source_mapping_letter_status_update.tf

@@ -0,0 +1,9 @@
+resource "aws_lambda_event_source_mapping" "amendment_event_transformer" {
+  event_source_arn                   = module.amendments_queue.sqs_queue_arn
+  function_name                      = module.amendment_event_transformer.function_name
+  batch_size                         = 10
+  maximum_batching_window_in_seconds = 5
+  function_response_types = [
+    "ReportBatchItemFailures"
+  ]
+}

infrastructure/terraform/components/api/locals.tf

@@ -27,6 +27,8 @@ locals {
     SUPPLIER_ID_HEADER       = "nhsd-supplier-id",
     APIM_CORRELATION_HEADER  = "nhsd-correlation-id",
     DOWNLOAD_URL_TTL_SECONDS = 60
+    SNS_TOPIC_ARN            = "${module.eventsub.sns_topic.arn}",
+    EVENT_SOURCE             = "/data-plane/supplier-api/${var.group}/${var.environment}/letters"
   }
 
   core_pdf_bucket_arn        = "arn:aws:s3:::comms-${var.core_account_id}-eu-west-2-${var.core_environment}-api-stg-pdf-pipeline"

infrastructure/terraform/components/api/module_lambda_letter_status_update.tf → infrastructure/terraform/components/api/module_lambda_amendment_event_transformer.tf

@@ -1,7 +1,7 @@
-module "letter_status_update" {
+module "amendment_event_transformer" {
   source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip"
 
-  function_name = "letter_status_update"
+  function_name = "amendment_event_transformer"
   description   = "Processes letter status updates"
 
   aws_account_id = var.aws_account_id
@@ -15,14 +15,14 @@ module "letter_status_update" {
   kms_key_arn           = module.kms.key_arn
 
   iam_policy_document = {
-    body = data.aws_iam_policy_document.letter_status_update.json
+    body = data.aws_iam_policy_document.amendment_event_transformer.json
   }
 
   function_s3_bucket      = local.acct.s3_buckets["lambda_function_artefacts"]["id"]
   function_code_base_path = local.aws_lambda_functions_dir_path
   function_code_dir       = "api-handler/dist"
   function_include_common = true
-  handler_function_name   = "letterStatusUpdate"
+  handler_function_name   = "transformAmendmentEvent"
   runtime                 = "nodejs22.x"
   memory                  = 512
   timeout                 = 29
@@ -37,7 +37,7 @@ module "letter_status_update" {
   lambda_env_vars = merge(local.common_lambda_env_vars, {})
 }
 
-data "aws_iam_policy_document" "letter_status_update" {
+data "aws_iam_policy_document" "amendment_event_transformer" {
   statement {
     sid    = "KMSPermissions"
     effect = "Allow"
@@ -59,7 +59,6 @@ data "aws_iam_policy_document" "letter_status_update" {
     actions = [
       "dynamodb:GetItem",
       "dynamodb:Query",
-      "dynamodb:UpdateItem",
     ]
 
     resources = [
@@ -79,7 +78,20 @@ data "aws_iam_policy_document" "letter_status_update" {
     ]
 
     resources = [
-      module.letter_status_updates_queue.sqs_queue_arn
+      module.amendments_queue.sqs_queue_arn
+    ]
+  }
+
+  statement {
+    sid    = "AllowSNSPublish"
+    effect = "Allow"
+
+    actions = [
+      "sns:Publish"
+    ]
+
+    resources = [
+      module.eventsub.sns_topic.arn
     ]
   }
 }

infrastructure/terraform/components/api/module_lambda_patch_letter.tf

@@ -35,7 +35,7 @@ module "patch_letter" {
   log_subscription_role_arn = local.acct.log_subscription_role_arn
 
   lambda_env_vars = merge(local.common_lambda_env_vars, {
-    QUEUE_URL = module.letter_status_updates_queue.sqs_queue_url
+    QUEUE_URL = module.amendments_queue.sqs_queue_url
   })
 }
 
@@ -64,7 +64,7 @@ data "aws_iam_policy_document" "patch_letter_lambda" {
     ]
 
     resources = [
-      module.letter_status_updates_queue.sqs_queue_arn
+      module.amendments_queue.sqs_queue_arn
     ]
   }
 }

infrastructure/terraform/components/api/module_lambda_post_letters.tf

@@ -35,7 +35,7 @@ module "post_letters" {
   log_subscription_role_arn = local.acct.log_subscription_role_arn
 
   lambda_env_vars = merge(local.common_lambda_env_vars, {
-    QUEUE_URL = module.letter_status_updates_queue.sqs_queue_url,
+    QUEUE_URL = module.amendments_queue.sqs_queue_url,
     MAX_LIMIT = var.max_get_limit
   })
 }
@@ -65,7 +65,7 @@ data "aws_iam_policy_document" "post_letters" {
     ]
 
     resources = [
-      module.letter_status_updates_queue.sqs_queue_arn
+      module.amendments_queue.sqs_queue_arn
     ]
   }
 }

infrastructure/terraform/components/api/module_sqs_letter_status_updates.tf → infrastructure/terraform/components/api/module_sqs_amendments.tf

@@ -1,8 +1,8 @@
-# Queue to transport update letter status messages
-module "letter_status_updates_queue" {
+# Queue to transport letter status amendment messages
+module "amendments_queue" {
   source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.24/terraform-sqs.zip"
 
-  name = "letter_status_updates_queue"
+  name = "amendments_queue"
 
   aws_account_id = var.aws_account_id
   component      = var.component

infrastructure/terraform/components/api/module_sqs_letter_updates.tf

@@ -16,6 +16,7 @@ module "sqs_letter_updates" {
   sqs_policy_overload = data.aws_iam_policy_document.letter_updates_queue_policy.json
 }
 
+
 data "aws_iam_policy_document" "letter_updates_queue_policy" {
   version = "2012-10-17"
   statement {
@@ -41,31 +42,4 @@ data "aws_iam_policy_document" "letter_updates_queue_policy" {
       values   = [module.eventsub.sns_topic.arn]
     }
   }
-
-  statement {
-    sid    = "AllowSNSPermissions"
-    effect = "Allow"
-
-    principals {
-      type        = "Service"
-      identifiers = ["sns.amazonaws.com"]
-    }
-
-    actions = [
-      "sqs:SendMessage",
-      "sqs:ListQueueTags",
-      "sqs:GetQueueUrl",
-      "sqs:GetQueueAttributes",
-    ]
-
-    resources = [
-      "arn:aws:sqs:${var.region}:${var.aws_account_id}:${var.project}-${var.environment}-${var.component}-letter-updates-queue"
-    ]
-
-    condition {
-      test     = "ArnEquals"
-      variable = "aws:SourceArn"
-      values   = [module.eventsub.sns_topic.arn]
-    }
-  }
 }

infrastructure/terraform/components/api/moved.tf

@@ -0,0 +1,11 @@
+# Moved blocks to handle resource renames without destroy/recreate
+
+moved {
+  from = module.letter_status_updates_queue
+  to   = module.amendments_queue
+}
+
+moved {
+  from = module.letter_status_update
+  to   = module.amendment_event_transformer
+}

infrastructure/terraform/components/api/sns_topic_subscription_eventsub_sqs_letter_updates.tf

@@ -2,4 +2,9 @@ resource "aws_sns_topic_subscription" "eventsub_sqs_letter_updates" {
   topic_arn = module.eventsub.sns_topic.arn
   protocol  = "sqs"
   endpoint  = module.sqs_letter_updates.sqs_queue_arn
+
+  filter_policy_scope = "MessageBody"
+  filter_policy = jsonencode({
+    type = [{ prefix = "uk.nhs.notify.supplier-api.letter" }]
+  })
 }

infrastructure/terraform/components/api/sns_topic_subscription_eventsub_sqs_supplier_allocator.tf

@@ -0,0 +1,11 @@
+resource "aws_sns_topic_subscription" "eventsub_sqs_supplier_allocator" {
+  # The supplier allocator queue will be introduced by another ticket. For now, route events directly to the letter updates queue.
+  topic_arn = module.eventsub.sns_topic.arn
+  protocol  = "sqs"
+  endpoint  = module.sqs_letter_updates.sqs_queue_arn
+
+  filter_policy_scope = "MessageBody"
+  filter_policy = jsonencode({
+    type = [{ prefix = "uk.nhs.notify.letter-rendering.letter-request.prepared" }]
+  })
+}

infrastructure/terraform/components/api/variables.tf

@@ -167,7 +167,7 @@ variable "core_environment" {
 variable "enable_event_cache" {
   type        = bool
   description = "Enable caching of events to an S3 bucket"
-  default     = false
+  default     = true
 }
 
 variable "enable_sns_delivery_logging" {

internal/datastore/src/__test__/letter-repository.test.ts

@@ -14,10 +14,12 @@ function createLetter(
   supplierId: string,
   letterId: string,
   status: Letter["status"] = "PENDING",
+  eventId?: string,
 ): InsertLetter {
   const now = new Date().toISOString();
   return {
     id: letterId,
+    eventId,
     supplierId,
     specificationId: "specification1",
     groupId: "group1",
@@ -168,6 +170,7 @@ describe("LetterRepository", () => {
 
     const updateLetter: UpdateLetter = {
       id: "letter1",
+      eventId: "event1",
       supplierId: "supplier1",
       status: "REJECTED",
       reasonCode: "R01",
@@ -180,6 +183,7 @@ describe("LetterRepository", () => {
       "letter1",
     );
     expect(updatedLetter.status).toBe("REJECTED");
+    expect(updatedLetter.previousStatus).toBe("PENDING");
     expect(updatedLetter.reasonCode).toBe("R01");
     expect(updatedLetter.reasonText).toBe("Reason text");
   });
@@ -199,6 +203,7 @@ describe("LetterRepository", () => {
     jest.setSystemTime(new Date(2020, 1, 2));
     const letterDto: UpdateLetter = {
       id: "letter1",
+      eventId: "event1",
       supplierId: "supplier1",
       status: "DELIVERED",
     };
@@ -215,6 +220,7 @@ describe("LetterRepository", () => {
   test("can't update a letter that does not exist", async () => {
     const updateLetter: UpdateLetter = {
       id: "letter1",
+      eventId: "event1",
       supplierId: "supplier1",
       status: "DELIVERED",
     };
@@ -233,6 +239,7 @@ describe("LetterRepository", () => {
 
     const updateLetter: UpdateLetter = {
       id: "letter1",
+      eventId: "event1",
       supplierId: "supplier1",
       status: "DELIVERED",
     };
@@ -241,6 +248,52 @@ describe("LetterRepository", () => {
     ).rejects.toThrow("Cannot do operations on a non-existent table");
   });
 
+  test("does not update a letter if the same eventId is used", async () => {
+    const letter = createLetter("supplier1", "letter1", "DELIVERED", "event1");
+    await letterRepository.putLetter(letter);
+
+    const duplicateUpdate: UpdateLetter = {
+      id: "letter1",
+      eventId: "event1",
+      supplierId: "supplier1",
+      status: "REJECTED",
+      reasonCode: "R01",
+    };
+    const result = await letterRepository.updateLetterStatus(duplicateUpdate);
+
+    expect(result).toBeUndefined();
+    const unchangedLetter = await letterRepository.getLetterById(
+      "supplier1",
+      "letter1",
+    );
+    expect(unchangedLetter.status).toBe("DELIVERED");
+    expect(unchangedLetter.eventId).toBe("event1");
+    expect(unchangedLetter.reasonCode).toBeUndefined();
+  });
+
+  test("updates a letter if a different eventId is used", async () => {
+    const letter = createLetter("supplier1", "letter1", "DELIVERED", "event1");
+    await letterRepository.putLetter(letter);
+
+    const duplicateUpdate: UpdateLetter = {
+      id: "letter1",
+      eventId: "event2",
+      supplierId: "supplier1",
+      status: "REJECTED",
+      reasonCode: "R01",
+    };
+    const result = await letterRepository.updateLetterStatus(duplicateUpdate);
+
+    expect(result).toBeDefined();
+    const changedLetter = await letterRepository.getLetterById(
+      "supplier1",
+      "letter1",
+    );
+    expect(changedLetter.status).toBe("REJECTED");
+    expect(changedLetter.eventId).toBe("event2");
+    expect(changedLetter.reasonCode).toBe("R01");
+  });
+
   test("should return a list of letters matching status", async () => {
     await letterRepository.putLetter(createLetter("supplier1", "letter1"));
     await letterRepository.putLetter(createLetter("supplier1", "letter2"));
@@ -278,6 +331,7 @@ describe("LetterRepository", () => {
 
     const updateLetter: UpdateLetter = {
       id: "letter1",
+      eventId: "event1",
       supplierId: "supplier1",
       status: "DELIVERED",
     };