CCM-12939 event audit pipeline

.github/workflows/manual-proxy-environment-deploy.yaml

@@ -36,11 +36,10 @@ jobs:
           node-version: 22
 
       - name: Npm install
-        working-directory: .
-        env:
-          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: npm ci
-        shell: bash
+        uses: ./.github/actions/node-install
+        with:
+          node-version: 22
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
       - name: "Check if pull request exists for this branch and set ENVIRONMENT/APIM_ENV"
         id: pr_exists

infrastructure/terraform/components/api/README.md

@@ -18,8 +18,9 @@ No requirements.
 | <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | A map of default tags to apply to all taggable resources within the component | `map(string)` | `{}` | no |
 | <a name="input_disable_gateway_execute_endpoint"></a> [disable\_gateway\_execute\_endpoint](#input\_disable\_gateway\_execute\_endpoint) | Disable the execution endpoint for the API Gateway | `bool` | `true` | no |
 | <a name="input_enable_api_data_trace"></a> [enable\_api\_data\_trace](#input\_enable\_api\_data\_trace) | Enable API Gateway data trace logging | `bool` | `false` | no |
-| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no |
-| <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no |
+| <a name="input_enable_backups"></a> [enable\_backups](#input\_enable\_backups) | Enable backups | `bool` | `false` | no |
+| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no |
+| <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the tfscaffold environment | `string` | n/a | yes |
 | <a name="input_eventpub_control_plane_bus_arn"></a> [eventpub\_control\_plane\_bus\_arn](#input\_eventpub\_control\_plane\_bus\_arn) | ARN of the EventBridge control plane bus for eventpub | `string` | `""` | no |
 | <a name="input_eventpub_data_plane_bus_arn"></a> [eventpub\_data\_plane\_bus\_arn](#input\_eventpub\_data\_plane\_bus\_arn) | ARN of the EventBridge data plane bus for eventpub | `string` | `""` | no |
@@ -44,7 +45,7 @@ No requirements.
 |------|--------|---------|
 | <a name="module_authorizer_lambda"></a> [authorizer\_lambda](#module\_authorizer\_lambda) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_domain_truststore"></a> [domain\_truststore](#module\_domain\_truststore) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-s3bucket.zip | n/a |
-| <a name="module_eventpub"></a> [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-eventpub.zip | n/a |
+| <a name="module_eventpub"></a> [eventpub](#module\_eventpub) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.31/terraform-eventpub.zip | n/a |
 | <a name="module_eventsub"></a> [eventsub](#module\_eventsub) | ../../modules/eventsub | n/a |
 | <a name="module_get_letter"></a> [get\_letter](#module\_get\_letter) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |
 | <a name="module_get_letter_data"></a> [get\_letter\_data](#module\_get\_letter\_data) | https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.29/terraform-lambda.zip | n/a |

infrastructure/terraform/components/api/glue_catalog_database_supplier.tf

@@ -0,0 +1,4 @@
+resource "aws_glue_catalog_database" "supplier" {
+  name        = "${local.csi}-supplier"
+  description = "Glue catalog database for Suppliers API"
+}

infrastructure/terraform/components/api/glue_crawler_event_crawler.tf

@@ -0,0 +1,26 @@
+resource "aws_glue_crawler" "event_crawler" {
+  count         = local.event_cache_bucket_name != null ? 1 : 0
+  name          = "${local.csi}-audit-event-crawler"
+  database_name = aws_glue_catalog_database.supplier.name
+  role          = aws_iam_role.glue_role.arn
+
+  table_prefix = ""
+  s3_target {
+    path = "s3://${local.event_cache_bucket_name}/"
+  }
+
+  s3_target {
+    path = "s3://${local.eventsub_event_cache_bucket_name}/"
+  }
+
+  schedule = "cron(0 * * * ? *)"
+  recrawl_policy {
+    recrawl_behavior = "CRAWL_NEW_FOLDERS_ONLY"
+  }
+
+  schema_change_policy {
+    delete_behavior = "LOG"
+    update_behavior = "LOG"
+  }
+
+}

infrastructure/terraform/components/api/iam_role_glue.tf

@@ -0,0 +1,105 @@
+resource "aws_iam_role" "glue_role" {
+  name               = "${local.csi}-glue-role"
+  assume_role_policy = data.aws_iam_policy_document.glue_assume_role.json
+}
+
+data "aws_iam_policy_document" "glue_assume_role" {
+  statement {
+    sid    = "AllowGlueServiceAssumeRole"
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["glue.amazonaws.com"]
+    }
+
+    actions = [
+      "sts:AssumeRole",
+    ]
+  }
+}
+
+resource "aws_iam_policy" "glue_service_policy" {
+  name        = "${local.csi}-glue-service-policy"
+  description = "Policy for ${local.csi} Glue Service Role"
+  policy      = data.aws_iam_policy_document.glue_service_policy.json
+}
+
+data "aws_iam_policy_document" "glue_service_policy" {
+  statement {
+    sid    = "AllowGlueLogging"
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+    resources = ["arn:aws:logs:*:*:*"]
+  }
+
+  statement {
+    sid    = "AllowListBucketAndGetLocation"
+    effect = "Allow"
+
+    actions = [
+      "s3:ListBucket",
+      "s3:GetBucketLocation"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${local.event_cache_bucket_name}",
+      "arn:aws:s3:::${local.eventsub_event_cache_bucket_name}"
+    ]
+  }
+  statement {
+    sid    = "AllowS3Access"
+    effect = "Allow"
+
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectVersion",
+      "s3:PutObject",
+      "s3:DeleteObject"
+    ]
+    resources = [
+      "arn:aws:s3:::${local.event_cache_bucket_name}/*",
+      "arn:aws:s3:::${local.eventsub_event_cache_bucket_name}/*"
+    ]
+  }
+  statement {
+    sid    = "GlueCatalogAccess"
+    effect = "Allow"
+    actions = [
+      "glue:GetDatabase",
+      "glue:GetDatabases",
+      "glue:GetTable",
+      "glue:GetTables",
+      "glue:CreateTable",
+      "glue:UpdateTable",
+      "glue:CreatePartition",
+      "glue:BatchCreatePartition",
+      "glue:GetPartition",
+      "glue:BatchGetPartition",
+      "glue:UpdatePartition"
+    ]
+    resources = ["*"]
+  }
+  statement {
+    sid    = "S3TempAndGlueETL"
+    effect = "Allow"
+    actions = [
+      "s3:PutObject",
+      "s3:GetObject"
+    ]
+    resources = [
+      "arn:aws:s3:::aws-glue-*",
+      "arn:aws:s3:::aws-glue-*/*"
+    ]
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "gllue_attach_policy" {
+  role       = aws_iam_role.glue_role.name
+  policy_arn = aws_iam_policy.glue_service_policy.arn
+}

infrastructure/terraform/components/api/locals.tf

@@ -31,4 +31,7 @@ locals {
 
   core_pdf_bucket_arn        = "arn:aws:s3:::comms-${var.core_account_id}-eu-west-2-${var.core_environment}-api-stg-pdf-pipeline"
   core_s3_kms_key_alias_name = "alias/comms-${var.core_environment}-api-s3"
+
+  event_cache_bucket_name          = lookup(module.eventpub.s3_bucket_event_cache, "bucket", null)
+  eventsub_event_cache_bucket_name = lookup(module.eventsub.s3_bucket_event_cache, "bucket", null)
 }

infrastructure/terraform/components/api/modules_eventpub.tf

@@ -1,5 +1,5 @@
 module "eventpub" {
-  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.26/terraform-eventpub.zip"
+  source = "https://github.com/NHSDigital/nhs-notify-shared-modules/releases/download/v2.0.31/terraform-eventpub.zip"
 
   name = "eventpub"
 
@@ -27,4 +27,51 @@ module "eventpub" {
 
   data_plane_bus_arn    = var.eventpub_data_plane_bus_arn
   control_plane_bus_arn = var.eventpub_control_plane_bus_arn
+
+  additional_policies_for_event_cache_bucket = [
+    data.aws_iam_policy_document.eventcache[0].json
+  ]
+}
+data "aws_iam_policy_document" "eventcache" {
+  count = local.event_cache_bucket_name != null ? 1 : 0
+  statement {
+    sid    = "AllowGlueListBucketAndGetLocation"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = [aws_iam_role.glue_role.arn]
+    }
+
+    actions = [
+      "s3:ListBucket",
+      "s3:GetBucketLocation"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${local.csi_global}-eventcache"
+    ]
+  }
+
+  # Object-level permissions: Get/Put/Delete objects
+  statement {
+    sid    = "AllowGlueObjectAccess"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = [aws_iam_role.glue_role.arn]
+    }
+
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectVersion",
+      "s3:PutObject",
+      "s3:DeleteObject"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${local.csi_global}-eventcache/*"
+    ]
+  }
 }

infrastructure/terraform/components/api/modules_eventsub.tf

@@ -12,6 +12,8 @@ module "eventsub" {
 
   default_tags = local.default_tags
 
+  glue_role_arn = aws_iam_role.glue_role.arn
+
   kms_key_arn           = module.kms.key_arn
   log_retention_in_days = var.log_retention_in_days
   log_level             = "INFO"

infrastructure/terraform/components/api/s3_event_reporting.tf

@@ -0,0 +1,19 @@
+resource "aws_s3_bucket" "event_reporting" {
+  bucket = "${local.csi_global}-event-reporting"
+
+  tags = merge(local.default_tags, { "Enable-Backup" = var.enable_backups }, { "Enable-S3-Continuous-Backup" = var.enable_backups })
+}
+resource "aws_s3_bucket_ownership_controls" "event_reporting" {
+  bucket = aws_s3_bucket.event_reporting.id
+
+  rule {
+    object_ownership = "BucketOwnerPreferred"
+  }
+}
+resource "aws_s3_bucket_versioning" "event_reporting" {
+  bucket = aws_s3_bucket.event_reporting.id
+
+  versioning_configuration {
+    status = "Enabled"
+  }
+}

infrastructure/terraform/components/api/variables.tf

@@ -163,17 +163,23 @@ variable "core_environment" {
 
 }
 
+variable "enable_backups" {
+  type        = bool
+  description = "Enable backups"
+  default     = false
+}
+
 # Event Pub/Sub cache settings
 variable "enable_event_cache" {
   type        = bool
   description = "Enable caching of events to an S3 bucket"
-  default     = false
+  default     = true
 }
 
 variable "enable_sns_delivery_logging" {
   type        = bool
   description = "Enable SNS Delivery Failure Notifications"
-  default     = false
+  default     = true
 }
 
 variable "sns_success_logging_sample_percent" {

infrastructure/terraform/modules/eventsub/README.md

@@ -14,13 +14,14 @@
 | <a name="input_aws_account_id"></a> [aws\_account\_id](#input\_aws\_account\_id) | The AWS Account ID (numeric) | `string` | n/a | yes |
 | <a name="input_component"></a> [component](#input\_component) | The name of the terraformscaffold component calling this module | `string` | n/a | yes |
 | <a name="input_default_tags"></a> [default\_tags](#input\_default\_tags) | Default tag map for application to all taggable resources in the module | `map(string)` | `{}` | no |
-| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `false` | no |
+| <a name="input_enable_event_cache"></a> [enable\_event\_cache](#input\_enable\_event\_cache) | Enable caching of events to an S3 bucket | `bool` | `true` | no |
 | <a name="input_enable_firehose_raw_message_delivery"></a> [enable\_firehose\_raw\_message\_delivery](#input\_enable\_firehose\_raw\_message\_delivery) | Enables raw message delivery on firehose subscription | `bool` | `false` | no |
-| <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `false` | no |
+| <a name="input_enable_sns_delivery_logging"></a> [enable\_sns\_delivery\_logging](#input\_enable\_sns\_delivery\_logging) | Enable SNS Delivery Failure Notifications | `bool` | `true` | no |
 | <a name="input_environment"></a> [environment](#input\_environment) | The name of the terraformscaffold environment the module is called for | `string` | n/a | yes |
 | <a name="input_event_cache_buffer_interval"></a> [event\_cache\_buffer\_interval](#input\_event\_cache\_buffer\_interval) | The buffer interval for data firehose | `number` | `500` | no |
 | <a name="input_event_cache_expiry_days"></a> [event\_cache\_expiry\_days](#input\_event\_cache\_expiry\_days) | s3 archiving expiry in days | `number` | `30` | no |
 | <a name="input_force_destroy"></a> [force\_destroy](#input\_force\_destroy) | When enabled will force destroy event-cache S3 bucket | `bool` | `false` | no |
+| <a name="input_glue_role_arn"></a> [glue\_role\_arn](#input\_glue\_role\_arn) | ARN of the Glue execution role from the parent | `string` | n/a | yes |
 | <a name="input_group"></a> [group](#input\_group) | The name of the tfscaffold group | `string` | `null` | no |
 | <a name="input_kms_key_arn"></a> [kms\_key\_arn](#input\_kms\_key\_arn) | KMS key arn to use for this function | `string` | n/a | yes |
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"WARN"` | no |

infrastructure/terraform/modules/eventsub/locals.tf

@@ -12,6 +12,18 @@ locals {
     "_",
     "",
   )
+  csi_global = replace(
+    format(
+      "%s-%s-%s-%s-%s",
+      var.project,
+      var.aws_account_id,
+      var.region,
+      var.environment,
+      var.component,
+    ),
+    "_",
+    "",
+  )
   default_tags = merge(
     var.default_tags,
     {

infrastructure/terraform/modules/eventsub/module_s3bucket_event_cache.tf

@@ -126,4 +126,44 @@ data "aws_iam_policy_document" "s3bucket_event_cache" {
       ]
     }
   }
+  statement {
+    sid    = "AllowGlueListBucketAndGetLocation"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = [var.glue_role_arn]
+    }
+
+    actions = [
+      "s3:ListBucket",
+      "s3:GetBucketLocation"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${module.s3bucket_event_cache[0].bucket}"
+    ]
+  }
+
+  # Object-level permissions: Get/Put/Delete objects
+  statement {
+    sid    = "AllowGlueObjectAccess"
+    effect = "Allow"
+
+    principals {
+      type        = "AWS"
+      identifiers = [var.glue_role_arn]
+    }
+
+    actions = [
+      "s3:GetObject",
+      "s3:GetObjectVersion",
+      "s3:PutObject",
+      "s3:DeleteObject"
+    ]
+
+    resources = [
+      "arn:aws:s3:::${module.s3bucket_event_cache[0].bucket}/*"
+    ]
+  }
 }