From bb594dfed2666eaa54bfcf1db8e2591060ff2c6c Mon Sep 17 00:00:00 2001 From: David Wass Date: Mon, 19 Jan 2026 11:30:20 +0000 Subject: [PATCH 1/9] workflow --- .github/workflows/manual-proxy-environment-deploy.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/manual-proxy-environment-deploy.yaml b/.github/workflows/manual-proxy-environment-deploy.yaml index d5e502309..123126132 100644 --- a/.github/workflows/manual-proxy-environment-deploy.yaml +++ b/.github/workflows/manual-proxy-environment-deploy.yaml @@ -36,11 +36,10 @@ jobs: node-version: 22 - name: Npm install - working-directory: . - env: - NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: npm ci - shell: bash + uses: ./.github/actions/node-install + with: + node-version: 22 + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: "Check if pull request exists for this branch and set ENVIRONMENT/APIM_ENV" id: pr_exists From e226cf620e524227ee195d806a1614d6a0519680 Mon Sep 17 00:00:00 2001 From: David Wass Date: Fri, 16 Jan 2026 10:52:54 +0000 Subject: [PATCH 2/9] Enable firehose --- infrastructure/terraform/modules/eventsub/README.md | 6 +++--- infrastructure/terraform/modules/eventsub/variables.tf | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/infrastructure/terraform/modules/eventsub/README.md b/infrastructure/terraform/modules/eventsub/README.md index 859c5fd5b..b75c55450 100644 --- a/infrastructure/terraform/modules/eventsub/README.md +++ b/infrastructure/terraform/modules/eventsub/README.md @@ -14,9 +14,9 @@ | [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes | | [component](#input\_component) | The name of the terraformscaffold component calling this module | `string` | n/a | yes | | [default\_tags](#input\_default\_tags) | Default tag map for application to all taggable resources in the module | `map(string)` | `{}` | no | -| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no | -| [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `false` | no | -| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no | +| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no | +| [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `true` | no | +| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no | | [environment](#input\_environment) | The name of the terraformscaffold environment the module is called for | `string` | n/a | yes | | [event\_cache\_buffer\_interval](#input\_event\_cache\_buffer\_interval) | The buffer interval for data firehose | `number` | `500` | no | | [event\_cache\_expiry\_days](#input\_event\_cache\_expiry\_days) | s3 archiving expiry in days | `number` | `30` | no | diff --git a/infrastructure/terraform/modules/eventsub/variables.tf b/infrastructure/terraform/modules/eventsub/variables.tf index 4b73d4523..79a1114c6 100644 --- a/infrastructure/terraform/modules/eventsub/variables.tf +++ b/infrastructure/terraform/modules/eventsub/variables.tf @@ -70,7 +70,7 @@ variable "event_cache_buffer_interval" { variable "enable_sns_delivery_logging" { type = bool description = "Enable SNS Delivery Failure Notifications" - default = false + default = true } variable "sns_success_logging_sample_percent" { @@ -94,13 +94,13 @@ variable "event_cache_expiry_days" { variable "enable_event_cache" { type = bool description = "Enable caching of events to an S3 bucket" - default = false + default = true } variable "enable_firehose_raw_message_delivery" { type = bool description = "Enables raw message delivery on firehose subscription" - default = false + default = true } variable "force_destroy" { From d0673b2e5758f87ab733d282f181157d9e3f4951 Mon Sep 17 00:00:00 2001 From: David Wass Date: Wed, 21 Jan 2026 11:10:34 +0000 Subject: [PATCH 3/9] Status updates for sub --- .../terraform/components/api/README.md | 5 +- .../api/glue_catalog_database_supplier.tf | 4 + .../api/glue_crawler_event_crawler.tf | 24 ++++ .../terraform/components/api/iam_role_glue.tf | 105 ++++++++++++++++++ .../terraform/components/api/locals.tf | 3 + .../api/module_lambda_letter_status_update.tf | 13 +++ .../components/api/modules_eventsub.tf | 2 + .../api/s3_bucket_policy_eventcache.tf | 51 +++++++++ .../components/api/s3_event_reporting.tf | 19 ++++ .../terraform/components/api/variables.tf | 10 +- .../terraform/modules/eventsub/README.md | 3 +- .../modules/eventsub/iam_role_sns.tf | 2 +- .../kinesis_firehose_delivery_stream.tf | 2 +- .../terraform/modules/eventsub/locals.tf | 12 ++ .../eventsub/s3_bucket_policy_eventcache.tf | 48 ++++++++ .../sns_topic_subscription_firehose.tf | 2 +- .../terraform/modules/eventsub/variables.tf | 7 +- 17 files changed, 303 insertions(+), 9 deletions(-) create mode 100644 infrastructure/terraform/components/api/glue_catalog_database_supplier.tf create mode 100644 infrastructure/terraform/components/api/glue_crawler_event_crawler.tf create mode 100644 infrastructure/terraform/components/api/iam_role_glue.tf create mode 100644 infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf create mode 100644 infrastructure/terraform/components/api/s3_event_reporting.tf create mode 100644 infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index 01fe4c4be..6bf2c39d6 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -18,8 +18,9 @@ No requirements. | [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no | | [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no | | [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no | -| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no | -| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no | +| [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no | +| [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no | +| [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no | | [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes | | [eventpub\_control\_plane\_bus\_arn](#input\_eventpub\_control\_plane\_bus\_arn) | ARN of the EventBridge control plane bus for eventpub | `string` | `""` | no | | [eventpub\_data\_plane\_bus\_arn](#input\_eventpub\_data\_plane\_bus\_arn) | ARN of the EventBridge data plane bus for eventpub | `string` | `""` | no | diff --git a/infrastructure/terraform/components/api/glue_catalog_database_supplier.tf b/infrastructure/terraform/components/api/glue_catalog_database_supplier.tf new file mode 100644 index 000000000..ae64cab23 --- /dev/null +++ b/infrastructure/terraform/components/api/glue_catalog_database_supplier.tf @@ -0,0 +1,4 @@ +resource "aws_glue_catalog_database" "supplier" { + name = "${local.csi}-supplier" + description = "Glue catalog database for Suppliers API" +} diff --git a/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf new file mode 100644 index 000000000..8d5569127 --- /dev/null +++ b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf @@ -0,0 +1,24 @@ +resource "aws_glue_crawler" "event_crawler" { + count = local.event_cache_bucket_name != null ? 1 : 0 + name = "event-crawler-${aws_glue_catalog_table.events.name}" + database_name = aws_glue_catalog_database.supplier.name + role = aws_iam_role.glue_role.arn + + table_prefix = "" + s3_target { + path = "s3://${local.event_cache_bucket_name}/" + } + + s3_target { + path = "s3://${local.eventsub_event_cache_bucket_name}/" + } + recrawl_policy { + recrawl_behavior = "CRAWL_EVERYTHING" + } + + schema_change_policy { + delete_behavior = "LOG" + update_behavior = "UPDATE_IN_DATABASE" + } + +} diff --git a/infrastructure/terraform/components/api/iam_role_glue.tf b/infrastructure/terraform/components/api/iam_role_glue.tf new file mode 100644 index 000000000..3084110e6 --- /dev/null +++ b/infrastructure/terraform/components/api/iam_role_glue.tf @@ -0,0 +1,105 @@ +resource "aws_iam_role" "glue_role" { + name = "${local.csi}-glue-role" + assume_role_policy = data.aws_iam_policy_document.glue_assume_role.json +} + +data "aws_iam_policy_document" "glue_assume_role" { + statement { + sid = "AllowGlueServiceAssumeRole" + effect = "Allow" + + principals { + type = "Service" + identifiers = ["glue.amazonaws.com"] + } + + actions = [ + "sts:AssumeRole", + ] + } +} + +resource "aws_iam_policy" "glue_service_policy" { + name = "${local.csi}-glue-service-policy" + description = "Policy for ${local.csi} Glue Service Role" + policy = data.aws_iam_policy_document.glue_service_policy.json +} + +data "aws_iam_policy_document" "glue_service_policy" { + statement { + sid = "AllowGlueLogging" + effect = "Allow" + + actions = [ + "logs:CreateLogGroup", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + resources = ["arn:aws:logs:*:*:*"] + } + + statement { + sid = "AllowListBucketAndGetLocation" + effect = "Allow" + + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation" + ] + + resources = [ + "arn:aws:s3:::${local.event_cache_bucket_name}", + "arn:aws:s3:::${local.eventsub_event_cache_bucket_name}" + ] + } + statement { + sid = "AllowS3Access" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:DeleteObject" + ] + resources = [ + "arn:aws:s3:::${local.event_cache_bucket_name}/*", + "arn:aws:s3:::${local.eventsub_event_cache_bucket_name}/*" + ] + } + statement { + sid = "GlueCatalogAccess" + effect = "Allow" + actions = [ + "glue:GetDatabase", + "glue:GetDatabases", + "glue:GetTable", + "glue:GetTables", + "glue:CreateTable", + "glue:UpdateTable", + "glue:CreatePartition", + "glue:BatchCreatePartition", + "glue:GetPartition", + "glue:BatchGetPartition", + "glue:UpdatePartition" + ] + resources = ["*"] + } + statement { + sid = "S3TempAndGlueETL" + effect = "Allow" + actions = [ + "s3:PutObject", + "s3:GetObject" + ] + resources = [ + "arn:aws:s3:::aws-glue-*", + "arn:aws:s3:::aws-glue-*/*" + ] + } +} + +resource "aws_iam_role_policy_attachment" "gllue_attach_policy" { + role = aws_iam_role.glue_role.name + policy_arn = aws_iam_policy.glue_service_policy.arn +} diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index 683156a05..d928fe372 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -31,4 +31,7 @@ locals { core_pdf_bucket_arn = "arn:aws:s3:::comms-${var.core_account_id}-eu-west-2-${var.core_environment}-api-stg-pdf-pipeline" core_s3_kms_key_alias_name = "alias/comms-${var.core_environment}-api-s3" + + event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null) + eventsub_event_cache_bucket_name = lookup(module.eventsub.s3bucket_event_cache, "bucket", null) } diff --git a/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf b/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf index b9d97fec5..f66c3c9d5 100644 --- a/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf +++ b/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf @@ -82,4 +82,17 @@ data "aws_iam_policy_document" "letter_status_update" { module.letter_status_updates_queue.sqs_queue_arn ] } + + statement { + sid = "AllowSNSPublish" + effect = "Allow" + + actions = [ + "sns:Publish" + ] + + resources = [ + module.eventsub.sns_topic.arn + ] + } } diff --git a/infrastructure/terraform/components/api/modules_eventsub.tf b/infrastructure/terraform/components/api/modules_eventsub.tf index 661ff9999..c97b5a908 100644 --- a/infrastructure/terraform/components/api/modules_eventsub.tf +++ b/infrastructure/terraform/components/api/modules_eventsub.tf @@ -12,6 +12,8 @@ module "eventsub" { default_tags = local.default_tags + glue_role_arn = aws_iam_role.glue_role.arn + kms_key_arn = module.kms.key_arn log_retention_in_days = var.log_retention_in_days log_level = "INFO" diff --git a/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf b/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf new file mode 100644 index 000000000..788a2581c --- /dev/null +++ b/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf @@ -0,0 +1,51 @@ +resource "aws_s3_bucket_policy" "eventcache" { + count = local.event_cache_bucket_name != null ? 1 : 0 + bucket = local.event_cache_bucket_name + policy = data.aws_iam_policy_document.eventcache[0].json + + depends_on = [module.eventpub] +} + +data "aws_iam_policy_document" "eventcache" { + count = local.event_cache_bucket_name != null ? 1 : 0 + statement { + sid = "AllowGlueListBucketAndGetLocation" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [aws_iam_role.glue_role.arn] + } + + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation" + ] + + resources = [ + "arn:aws:s3:::${local.csi_global}-eventcache" + ] + } + + # Object-level permissions: Get/Put/Delete objects + statement { + sid = "AllowGlueObjectAccess" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [aws_iam_role.glue_role.arn] + } + + actions = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:DeleteObject" + ] + + resources = [ + "arn:aws:s3:::${local.csi_global}-eventcache/*" + ] + } +} diff --git a/infrastructure/terraform/components/api/s3_event_reporting.tf b/infrastructure/terraform/components/api/s3_event_reporting.tf new file mode 100644 index 000000000..e61602d0c --- /dev/null +++ b/infrastructure/terraform/components/api/s3_event_reporting.tf @@ -0,0 +1,19 @@ +resource "aws_s3_bucket" "event_reporting" { + bucket = "${local.csi_global}-event-reporting" + + tags = merge(local.default_tags, { "Enable-Backup" = var.enable_backups }, { "Enable-S3-Continuous-Backup" = var.enable_backups }) +} +resource "aws_s3_bucket_ownership_controls" "event_reporting" { + bucket = aws_s3_bucket.event_reporting.id + + rule { + object_ownership = "BucketOwnerPreferred" + } +} +resource "aws_s3_bucket_versioning" "event_reporting" { + bucket = aws_s3_bucket.event_reporting.id + + versioning_configuration { + status = "Enabled" + } +} diff --git a/infrastructure/terraform/components/api/variables.tf b/infrastructure/terraform/components/api/variables.tf index 47928a960..a9032d75c 100644 --- a/infrastructure/terraform/components/api/variables.tf +++ b/infrastructure/terraform/components/api/variables.tf @@ -163,17 +163,23 @@ variable "core_environment" { } +variable "enable_backups" { + type = bool + description = "Enable backups" + default = false +} + # Event Pub/Sub cache settings variable "enable_event_cache" { type = bool description = "Enable caching of events to an S3 bucket" - default = false + default = true } variable "enable_sns_delivery_logging" { type = bool description = "Enable SNS Delivery Failure Notifications" - default = false + default = true } variable "sns_success_logging_sample_percent" { diff --git a/infrastructure/terraform/modules/eventsub/README.md b/infrastructure/terraform/modules/eventsub/README.md index b75c55450..9aadbb3f2 100644 --- a/infrastructure/terraform/modules/eventsub/README.md +++ b/infrastructure/terraform/modules/eventsub/README.md @@ -15,12 +15,13 @@ | [component](#input\_component) | The name of the terraformscaffold component calling this module | `string` | n/a | yes | | [default\_tags](#input\_default\_tags) | Default tag map for application to all taggable resources in the module | `map(string)` | `{}` | no | | [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no | -| [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `true` | no | +| [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `false` | no | | [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no | | [environment](#input\_environment) | The name of the terraformscaffold environment the module is called for | `string` | n/a | yes | | [event\_cache\_buffer\_interval](#input\_event\_cache\_buffer\_interval) | The buffer interval for data firehose | `number` | `500` | no | | [event\_cache\_expiry\_days](#input\_event\_cache\_expiry\_days) | s3 archiving expiry in days | `number` | `30` | no | | [force\_destroy](#input\_force\_destroy) | When enabled will force destroy event-cache S3 bucket | `bool` | `false` | no | +| [glue\_role\_arn](#input\_glue\_role\_arn) | ARN of the Glue execution role from the parent | `string` | n/a | yes | | [group](#input\_group) | The name of the tfscaffold group | `string` | `null` | no | | [kms\_key\_arn](#input\_kms\_key\_arn) | KMS key arn to use for this function | `string` | n/a | yes | | [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"WARN"` | no | diff --git a/infrastructure/terraform/modules/eventsub/iam_role_sns.tf b/infrastructure/terraform/modules/eventsub/iam_role_sns.tf index 97bdc99af..d8514e418 100644 --- a/infrastructure/terraform/modules/eventsub/iam_role_sns.tf +++ b/infrastructure/terraform/modules/eventsub/iam_role_sns.tf @@ -45,7 +45,7 @@ data "aws_iam_policy_document" "firehose_delivery" { ] resources = [ - "${aws_kinesis_firehose_delivery_stream.main[0].arn}", + "${aws_kinesis_firehose_delivery_stream.eventsub[0].arn}", ] } } diff --git a/infrastructure/terraform/modules/eventsub/kinesis_firehose_delivery_stream.tf b/infrastructure/terraform/modules/eventsub/kinesis_firehose_delivery_stream.tf index 186372d8e..acccd884b 100644 --- a/infrastructure/terraform/modules/eventsub/kinesis_firehose_delivery_stream.tf +++ b/infrastructure/terraform/modules/eventsub/kinesis_firehose_delivery_stream.tf @@ -1,4 +1,4 @@ -resource "aws_kinesis_firehose_delivery_stream" "main" { +resource "aws_kinesis_firehose_delivery_stream" "eventsub" { count = var.enable_event_cache ? 1 : 0 name = local.csi diff --git a/infrastructure/terraform/modules/eventsub/locals.tf b/infrastructure/terraform/modules/eventsub/locals.tf index e421035ac..1141f727b 100644 --- a/infrastructure/terraform/modules/eventsub/locals.tf +++ b/infrastructure/terraform/modules/eventsub/locals.tf @@ -12,6 +12,18 @@ locals { "_", "", ) + csi_global = replace( + format( + "%s-%s-%s-%s-%s", + var.project, + var.aws_account_id, + var.region, + var.environment, + var.component, + ), + "_", + "", + ) default_tags = merge( var.default_tags, { diff --git a/infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf b/infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf new file mode 100644 index 000000000..830b243ae --- /dev/null +++ b/infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf @@ -0,0 +1,48 @@ +resource "aws_s3_bucket_policy" "eventcache" { + bucket = module.s3bucket_event_cache[0].bucket + policy = data.aws_iam_policy_document.eventcache.json + count = var.enable_event_cache ? 1 : 0 +} + +data "aws_iam_policy_document" "eventcache" { + statement { + sid = "AllowGlueListBucketAndGetLocation" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [var.glue_role_arn] + } + + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation" + ] + + resources = [ + "arn:aws:s3:::${module.s3bucket_event_cache[0].bucket}" + ] + } + + # Object-level permissions: Get/Put/Delete objects + statement { + sid = "AllowGlueObjectAccess" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [var.glue_role_arn] + } + + actions = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:DeleteObject" + ] + + resources = [ + "arn:aws:s3:::${module.s3bucket_event_cache[0].bucket}/*" + ] + } +} diff --git a/infrastructure/terraform/modules/eventsub/sns_topic_subscription_firehose.tf b/infrastructure/terraform/modules/eventsub/sns_topic_subscription_firehose.tf index 42457f6de..9b11f2793 100644 --- a/infrastructure/terraform/modules/eventsub/sns_topic_subscription_firehose.tf +++ b/infrastructure/terraform/modules/eventsub/sns_topic_subscription_firehose.tf @@ -4,6 +4,6 @@ resource "aws_sns_topic_subscription" "firehose" { topic_arn = aws_sns_topic.main.arn protocol = "firehose" subscription_role_arn = aws_iam_role.sns_role.arn - endpoint = aws_kinesis_firehose_delivery_stream.main[0].arn + endpoint = aws_kinesis_firehose_delivery_stream.eventsub[0].arn raw_message_delivery = var.enable_firehose_raw_message_delivery } diff --git a/infrastructure/terraform/modules/eventsub/variables.tf b/infrastructure/terraform/modules/eventsub/variables.tf index 79a1114c6..f808bcb4a 100644 --- a/infrastructure/terraform/modules/eventsub/variables.tf +++ b/infrastructure/terraform/modules/eventsub/variables.tf @@ -100,7 +100,7 @@ variable "enable_event_cache" { variable "enable_firehose_raw_message_delivery" { type = bool description = "Enables raw message delivery on firehose subscription" - default = true + default = false } variable "force_destroy" { @@ -114,3 +114,8 @@ variable "shared_infra_account_id" { description = "The AWS Account ID of the shared infrastructure account" default = "000000000000" } + +variable "glue_role_arn" { + type = string + description = "ARN of the Glue execution role from the parent" +} From 56471c73e88f109506a9e3b9a4d587f6ca52ef2b Mon Sep 17 00:00:00 2001 From: David Wass Date: Mon, 2 Feb 2026 13:47:39 +0000 Subject: [PATCH 4/9] override fast-xml-parser --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 4b2a1acd9..ca391b99d 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,9 @@ "pretty-format": { "react-is": "19.0.0" }, - "react": "^19.0.0" + "react": "^19.0.0", + "fast-xml-parser": "^5.3.4" + }, "scripts": { "build": "npm run bundle-oas && npm run generate --buildver=$npm_config_buildver", From bfd4e2a4c322347dde009365e71f634ac65a2f03 Mon Sep 17 00:00:00 2001 From: David Wass Date: Tue, 3 Feb 2026 13:39:43 +0000 Subject: [PATCH 5/9] add bucket to eventsub outputs --- .../terraform/components/api/glue_crawler_event_crawler.tf | 4 ++-- infrastructure/terraform/components/api/locals.tf | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf index 8d5569127..3145f2bab 100644 --- a/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf +++ b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf @@ -1,6 +1,6 @@ resource "aws_glue_crawler" "event_crawler" { - count = local.event_cache_bucket_name != null ? 1 : 0 - name = "event-crawler-${aws_glue_catalog_table.events.name}" + count = local.event_cache_bucket_name != null ? 1 : 0 + name = "${local.csi}-audit-event-crawler" database_name = aws_glue_catalog_database.supplier.name role = aws_iam_role.glue_role.arn diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index d928fe372..7b599681e 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -33,5 +33,5 @@ locals { core_s3_kms_key_alias_name = "alias/comms-${var.core_environment}-api-s3" event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null) - eventsub_event_cache_bucket_name = lookup(module.eventsub.s3bucket_event_cache, "bucket", null) + eventsub_event_cache_bucket_name = lookup(module.eventsub.s3_bucket_event_cache, "bucket", null) } From 6b31d9dd11849c13a2da987f6683e3ee2d4efa45 Mon Sep 17 00:00:00 2001 From: David Wass Date: Wed, 4 Feb 2026 11:34:05 +0000 Subject: [PATCH 6/9] add schedule --- .../terraform/components/api/glue_crawler_event_crawler.tf | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf index 3145f2bab..6bb6b2b99 100644 --- a/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf +++ b/infrastructure/terraform/components/api/glue_crawler_event_crawler.tf @@ -12,13 +12,15 @@ resource "aws_glue_crawler" "event_crawler" { s3_target { path = "s3://${local.eventsub_event_cache_bucket_name}/" } + + schedule = "cron(0 * * * ? *)" recrawl_policy { - recrawl_behavior = "CRAWL_EVERYTHING" + recrawl_behavior = "CRAWL_NEW_FOLDERS_ONLY" } schema_change_policy { delete_behavior = "LOG" - update_behavior = "UPDATE_IN_DATABASE" + update_behavior = "LOG" } } From 4808c5e6d68fb66f4c9362352b1afeb9bfd969fc Mon Sep 17 00:00:00 2001 From: David Wass Date: Thu, 5 Feb 2026 08:32:05 +0000 Subject: [PATCH 7/9] review changes --- .../api/module_lambda_letter_status_update.tf | 13 ------------- .../terraform/modules/eventsub/iam_role_sns.tf | 2 +- .../eventsub/kinesis_firehose_delivery_stream.tf | 2 +- .../eventsub/sns_topic_subscription_firehose.tf | 2 +- 4 files changed, 3 insertions(+), 16 deletions(-) diff --git a/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf b/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf index f66c3c9d5..b9d97fec5 100644 --- a/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf +++ b/infrastructure/terraform/components/api/module_lambda_letter_status_update.tf @@ -82,17 +82,4 @@ data "aws_iam_policy_document" "letter_status_update" { module.letter_status_updates_queue.sqs_queue_arn ] } - - statement { - sid = "AllowSNSPublish" - effect = "Allow" - - actions = [ - "sns:Publish" - ] - - resources = [ - module.eventsub.sns_topic.arn - ] - } } diff --git a/infrastructure/terraform/modules/eventsub/iam_role_sns.tf b/infrastructure/terraform/modules/eventsub/iam_role_sns.tf index d8514e418..97bdc99af 100644 --- a/infrastructure/terraform/modules/eventsub/iam_role_sns.tf +++ b/infrastructure/terraform/modules/eventsub/iam_role_sns.tf @@ -45,7 +45,7 @@ data "aws_iam_policy_document" "firehose_delivery" { ] resources = [ - "${aws_kinesis_firehose_delivery_stream.eventsub[0].arn}", + "${aws_kinesis_firehose_delivery_stream.main[0].arn}", ] } } diff --git a/infrastructure/terraform/modules/eventsub/kinesis_firehose_delivery_stream.tf b/infrastructure/terraform/modules/eventsub/kinesis_firehose_delivery_stream.tf index acccd884b..186372d8e 100644 --- a/infrastructure/terraform/modules/eventsub/kinesis_firehose_delivery_stream.tf +++ b/infrastructure/terraform/modules/eventsub/kinesis_firehose_delivery_stream.tf @@ -1,4 +1,4 @@ -resource "aws_kinesis_firehose_delivery_stream" "eventsub" { +resource "aws_kinesis_firehose_delivery_stream" "main" { count = var.enable_event_cache ? 1 : 0 name = local.csi diff --git a/infrastructure/terraform/modules/eventsub/sns_topic_subscription_firehose.tf b/infrastructure/terraform/modules/eventsub/sns_topic_subscription_firehose.tf index 9b11f2793..42457f6de 100644 --- a/infrastructure/terraform/modules/eventsub/sns_topic_subscription_firehose.tf +++ b/infrastructure/terraform/modules/eventsub/sns_topic_subscription_firehose.tf @@ -4,6 +4,6 @@ resource "aws_sns_topic_subscription" "firehose" { topic_arn = aws_sns_topic.main.arn protocol = "firehose" subscription_role_arn = aws_iam_role.sns_role.arn - endpoint = aws_kinesis_firehose_delivery_stream.eventsub[0].arn + endpoint = aws_kinesis_firehose_delivery_stream.main[0].arn raw_message_delivery = var.enable_firehose_raw_message_delivery } From 66c2ab4dfd2cfce7f5c9543e6c037403298a7035 Mon Sep 17 00:00:00 2001 From: David Wass Date: Thu, 5 Feb 2026 15:16:56 +0000 Subject: [PATCH 8/9] moved s3 policy documents to module --- .../terraform/components/api/README.md | 2 +- .../components/api/modules_eventpub.tf | 49 +++++++++++++++++- .../api/s3_bucket_policy_eventcache.tf | 51 ------------------- .../eventsub/module_s3bucket_event_cache.tf | 40 +++++++++++++++ .../eventsub/s3_bucket_policy_eventcache.tf | 48 ----------------- 5 files changed, 89 insertions(+), 101 deletions(-) delete mode 100644 infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf delete mode 100644 infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index 6bf2c39d6..75faa887f 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -45,7 +45,7 @@ No requirements. |------|--------|---------| | [authorizer\_lambda](#module\_authorizer\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a | -| [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-eventpub.zip | n/a | +| [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.31/terraform-eventpub.zip | n/a | | [eventsub](#module\_eventsub) | ../../modules/eventsub | n/a | | [get\_letter](#module\_get\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | | [get\_letter\_data](#module\_get\_letter\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a | diff --git a/infrastructure/terraform/components/api/modules_eventpub.tf b/infrastructure/terraform/components/api/modules_eventpub.tf index f3090b111..83547ea62 100644 --- a/infrastructure/terraform/components/api/modules_eventpub.tf +++ b/infrastructure/terraform/components/api/modules_eventpub.tf @@ -1,5 +1,5 @@ module "eventpub" { - source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-eventpub.zip" + source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.31/terraform-eventpub.zip" name = "eventpub" @@ -27,4 +27,51 @@ module "eventpub" { data_plane_bus_arn = var.eventpub_data_plane_bus_arn control_plane_bus_arn = var.eventpub_control_plane_bus_arn + + additional_policies_for_event_cache_bucket = [ + data.aws_iam_policy_document.eventcache[0].json + ] +} +data "aws_iam_policy_document" "eventcache" { + count = local.event_cache_bucket_name != null ? 1 : 0 + statement { + sid = "AllowGlueListBucketAndGetLocation" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [aws_iam_role.glue_role.arn] + } + + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation" + ] + + resources = [ + "arn:aws:s3:::${local.csi_global}-eventcache" + ] + } + + # Object-level permissions: Get/Put/Delete objects + statement { + sid = "AllowGlueObjectAccess" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [aws_iam_role.glue_role.arn] + } + + actions = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:DeleteObject" + ] + + resources = [ + "arn:aws:s3:::${local.csi_global}-eventcache/*" + ] + } } diff --git a/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf b/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf deleted file mode 100644 index 788a2581c..000000000 --- a/infrastructure/terraform/components/api/s3_bucket_policy_eventcache.tf +++ /dev/null @@ -1,51 +0,0 @@ -resource "aws_s3_bucket_policy" "eventcache" { - count = local.event_cache_bucket_name != null ? 1 : 0 - bucket = local.event_cache_bucket_name - policy = data.aws_iam_policy_document.eventcache[0].json - - depends_on = [module.eventpub] -} - -data "aws_iam_policy_document" "eventcache" { - count = local.event_cache_bucket_name != null ? 1 : 0 - statement { - sid = "AllowGlueListBucketAndGetLocation" - effect = "Allow" - - principals { - type = "AWS" - identifiers = [aws_iam_role.glue_role.arn] - } - - actions = [ - "s3:ListBucket", - "s3:GetBucketLocation" - ] - - resources = [ - "arn:aws:s3:::${local.csi_global}-eventcache" - ] - } - - # Object-level permissions: Get/Put/Delete objects - statement { - sid = "AllowGlueObjectAccess" - effect = "Allow" - - principals { - type = "AWS" - identifiers = [aws_iam_role.glue_role.arn] - } - - actions = [ - "s3:GetObject", - "s3:GetObjectVersion", - "s3:PutObject", - "s3:DeleteObject" - ] - - resources = [ - "arn:aws:s3:::${local.csi_global}-eventcache/*" - ] - } -} diff --git a/infrastructure/terraform/modules/eventsub/module_s3bucket_event_cache.tf b/infrastructure/terraform/modules/eventsub/module_s3bucket_event_cache.tf index d51f34267..0d072a76c 100644 --- a/infrastructure/terraform/modules/eventsub/module_s3bucket_event_cache.tf +++ b/infrastructure/terraform/modules/eventsub/module_s3bucket_event_cache.tf @@ -126,4 +126,44 @@ data "aws_iam_policy_document" "s3bucket_event_cache" { ] } } + statement { + sid = "AllowGlueListBucketAndGetLocation" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [var.glue_role_arn] + } + + actions = [ + "s3:ListBucket", + "s3:GetBucketLocation" + ] + + resources = [ + "arn:aws:s3:::${module.s3bucket_event_cache[0].bucket}" + ] + } + + # Object-level permissions: Get/Put/Delete objects + statement { + sid = "AllowGlueObjectAccess" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [var.glue_role_arn] + } + + actions = [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject", + "s3:DeleteObject" + ] + + resources = [ + "arn:aws:s3:::${module.s3bucket_event_cache[0].bucket}/*" + ] + } } diff --git a/infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf b/infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf deleted file mode 100644 index 830b243ae..000000000 --- a/infrastructure/terraform/modules/eventsub/s3_bucket_policy_eventcache.tf +++ /dev/null @@ -1,48 +0,0 @@ -resource "aws_s3_bucket_policy" "eventcache" { - bucket = module.s3bucket_event_cache[0].bucket - policy = data.aws_iam_policy_document.eventcache.json - count = var.enable_event_cache ? 1 : 0 -} - -data "aws_iam_policy_document" "eventcache" { - statement { - sid = "AllowGlueListBucketAndGetLocation" - effect = "Allow" - - principals { - type = "AWS" - identifiers = [var.glue_role_arn] - } - - actions = [ - "s3:ListBucket", - "s3:GetBucketLocation" - ] - - resources = [ - "arn:aws:s3:::${module.s3bucket_event_cache[0].bucket}" - ] - } - - # Object-level permissions: Get/Put/Delete objects - statement { - sid = "AllowGlueObjectAccess" - effect = "Allow" - - principals { - type = "AWS" - identifiers = [var.glue_role_arn] - } - - actions = [ - "s3:GetObject", - "s3:GetObjectVersion", - "s3:PutObject", - "s3:DeleteObject" - ] - - resources = [ - "arn:aws:s3:::${module.s3bucket_event_cache[0].bucket}/*" - ] - } -} From 9cf0b59c8eeab1be15ba9b15ccba99183a1d60cd Mon Sep 17 00:00:00 2001 From: David Wass Date: Thu, 12 Feb 2026 13:06:13 +0000 Subject: [PATCH 9/9] terraform format --- infrastructure/terraform/components/api/locals.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/api/locals.tf b/infrastructure/terraform/components/api/locals.tf index 7b599681e..72ae70ba6 100644 --- a/infrastructure/terraform/components/api/locals.tf +++ b/infrastructure/terraform/components/api/locals.tf @@ -32,6 +32,6 @@ locals { core_pdf_bucket_arn = "arn:aws:s3:::comms-${var.core_account_id}-eu-west-2-${var.core_environment}-api-stg-pdf-pipeline" core_s3_kms_key_alias_name = "alias/comms-${var.core_environment}-api-s3" - event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null) + event_cache_bucket_name = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null) eventsub_event_cache_bucket_name = lookup(module.eventsub.s3_bucket_event_cache, "bucket", null) }