CCM-13295: Ingest reporting metadata into Glue table

infrastructure/terraform/components/dl/README.md

@@ -29,6 +29,7 @@ No requirements.
 | <a name="input_log_level"></a> [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"INFO"` | no |
 | <a name="input_log_retention_in_days"></a> [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no |
 | <a name="input_mesh_poll_schedule"></a> [mesh\_poll\_schedule](#input\_mesh\_poll\_schedule) | Schedule to poll MESH for messages | `string` | `"rate(5 minutes)"` | no |
+| <a name="input_metadata_refresh_schedule"></a> [metadata\_refresh\_schedule](#input\_metadata\_refresh\_schedule) | Schedule for refreshing reporting metadata. | `string` | `"cron(10 6-22 * * ? *)"` | no |
 | <a name="input_parent_acct_environment"></a> [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no |
 | <a name="input_pdm_mock_access_token"></a> [pdm\_mock\_access\_token](#input\_pdm\_mock\_access\_token) | Mock access token for PDM API authentication (used in local/dev environments) | `string` | `"mock-pdm-token"` | no |
 | <a name="input_pdm_use_non_mock_token"></a> [pdm\_use\_non\_mock\_token](#input\_pdm\_use\_non\_mock\_token) | Whether to use the shared APIM access token from SSM (/component/environment/apim/access\_token) instead of the mock token | `bool` | `false` | no |

infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_aborted.tf

@@ -0,0 +1,16 @@
+resource "aws_cloudwatch_metric_alarm" "metadata_refresh_executions_aborted" {
+  alarm_name          = "${local.csi}-metadata-refresh-execution-aborted"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = "ExecutionsAborted"
+  namespace           = "AWS/States"
+  period              = 60
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "This metric monitors aborted step function executions"
+  treat_missing_data  = "notBreaching"
+
+  dimensions = {
+    StateMachineArn = aws_sfn_state_machine.metadata_refresh.arn
+  }
+}

infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_failed.tf

@@ -0,0 +1,16 @@
+resource "aws_cloudwatch_metric_alarm" "metadata_refresh_executions_failed" {
+  alarm_name          = "${local.csi}-metadata-refresh-executions-failed"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = "ExecutionsFailed"
+  namespace           = "AWS/States"
+  period              = 60
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "This metric monitors failed step function executions"
+  treat_missing_data  = "notBreaching"
+
+  dimensions = {
+    StateMachineArn = aws_sfn_state_machine.metadata_refresh.arn
+  }
+}

infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_timedout.tf

@@ -0,0 +1,16 @@
+resource "aws_cloudwatch_metric_alarm" "metadata_refresh_executions_timedout" {
+  alarm_name          = "${local.csi}-metadata-refresh-executions-timedout"
+  comparison_operator = "GreaterThanOrEqualToThreshold"
+  evaluation_periods  = 1
+  metric_name         = "ExecutionsTimedOut"
+  namespace           = "AWS/States"
+  period              = 60
+  statistic           = "Sum"
+  threshold           = 1
+  alarm_description   = "This metric monitors step function execution timeouts"
+  treat_missing_data  = "notBreaching"
+
+  dimensions = {
+    StateMachineArn = aws_sfn_state_machine.metadata_refresh.arn
+  }
+}

infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf

@@ -24,6 +24,14 @@ resource "aws_glue_catalog_table" "event_record" {
       name = "pagecount"
       type = "int"
     }
+    columns {
+      name = "reasoncode"
+      type = "string"
+    }
+    columns {
+      name = "reasontext"
+      type = "string"
+    }
     columns {
       name = "supplierid"
       type = "string"

infrastructure/terraform/components/dl/module_sqs_mesh_acknowledge.tf

@@ -34,5 +34,11 @@ data "aws_iam_policy_document" "sqs_mesh_acknowledge" {
     resources = [
       "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-mesh-acknowledge-queue"
     ]
+
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+      values   = [aws_cloudwatch_event_rule.mesh_inbox_message_downloaded.arn]
+    }
   }
 }

infrastructure/terraform/components/dl/module_sqs_mesh_download.tf

@@ -34,5 +34,11 @@ data "aws_iam_policy_document" "sqs_mesh_download" {
     resources = [
       "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-mesh-download-queue"
     ]
+
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+      values   = [aws_cloudwatch_event_rule.mesh_inbox_message_received.arn]
+    }
   }
 }

infrastructure/terraform/components/dl/module_sqs_pdm_poll.tf

@@ -31,5 +31,11 @@ data "aws_iam_policy_document" "sqs_pdm_poll" {
     resources = [
       "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-pdm-poll-queue"
     ]
+
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+      values   = [aws_cloudwatch_event_rule.pdm_resource_submitted.arn, aws_cloudwatch_event_rule.pdm_resource_unavailable.arn]
+    }
   }
 }

infrastructure/terraform/components/dl/module_sqs_pdm_uploader.tf

@@ -34,5 +34,11 @@ data "aws_iam_policy_document" "sqs_pdm_uploader" {
     resources = [
       "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-pdm-uploader-queue"
     ]
+
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+      values   = [aws_cloudwatch_event_rule.mesh_inbox_message_downloaded.arn]
+    }
   }
 }

infrastructure/terraform/components/dl/module_sqs_print_status_handler.tf

@@ -32,5 +32,11 @@ data "aws_iam_policy_document" "sqs_print_status_handler" {
     resources = [
       "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-print-status-handler-queue"
     ]
+
+    condition {
+      test     = "ArnEquals"
+      variable = "aws:SourceArn"
+      values   = [aws_cloudwatch_event_rule.print_status_changed.arn]
+    }
   }
 }

infrastructure/terraform/components/dl/module_sqs_ttl.tf

@@ -34,5 +34,11 @@ data "aws_iam_policy_document" "sqs_ttl" {
     resources = [
       "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-ttl-queue"
     ]
+
+    condition {
+      test     = "ArnLike"
+      variable = "aws:SourceArn"
+      values   = [aws_cloudwatch_event_rule.mesh_inbox_message_downloaded.arn]
+    }
   }
 }

infrastructure/terraform/components/dl/scheduler_schedule_sf_metadata_refresh_scheduler.tf

@@ -0,0 +1,69 @@
+resource "aws_scheduler_schedule" "sf_metadata_refresh_scheduler" {
+  name        = "${local.csi}-metadata-refresh-scheduler"
+  description = "Scheduler to trigger Step Function to run metadata refresh queries"
+  group_name  = "default"
+
+  flexible_time_window {
+    mode = "OFF"
+  }
+
+  schedule_expression          = var.metadata_refresh_schedule
+  schedule_expression_timezone = "Europe/London"
+
+  target {
+    arn      = aws_sfn_state_machine.metadata_refresh.arn
+    role_arn = aws_iam_role.sf_metadata_refresh_scheduler.arn
+  }
+}
+
+resource "aws_iam_role" "sf_metadata_refresh_scheduler" {
+  name               = "${local.csi}-sf-metadata-refresh-scheduler-role"
+  description        = "Role used by the State Machine Metadata Refresh Scheduler"
+  assume_role_policy = data.aws_iam_policy_document.metadata_refresh_scheduler_assumerole.json
+}
+
+data "aws_iam_policy_document" "metadata_refresh_scheduler_assumerole" {
+  statement {
+    sid    = "EcsAssumeRole"
+    effect = "Allow"
+
+    actions = [
+      "sts:AssumeRole",
+    ]
+
+    principals {
+      type = "Service"
+
+      identifiers = [
+        "scheduler.amazonaws.com"
+      ]
+    }
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "sf_metadata_refresh_scheduler" {
+  role       = aws_iam_role.sf_metadata_refresh_scheduler.name
+  policy_arn = aws_iam_policy.sf_metadata_refresh_scheduler.arn
+}
+
+resource "aws_iam_policy" "sf_metadata_refresh_scheduler" {
+  name        = "${local.csi}-sfn-metadata-refresh-scheduler-policy"
+  description = "Allow Scheduler to execute State Machine"
+  path        = "/"
+  policy      = data.aws_iam_policy_document.sf_metadata_refresh_scheduler.json
+}
+
+data "aws_iam_policy_document" "sf_metadata_refresh_scheduler" {
+  statement {
+    sid    = "AllowStepFunctionExecution"
+    effect = "Allow"
+
+    actions = [
+      "states:StartExecution"
+    ]
+
+    resources = [
+      aws_sfn_state_machine.metadata_refresh.arn
+    ]
+  }
+}

infrastructure/terraform/components/dl/sfn_state_machine_metadata_refresh.tf

@@ -0,0 +1,159 @@
+resource "aws_sfn_state_machine" "metadata_refresh" {
+  name     = "${local.csi}-state-machine-metadata-refresh"
+  role_arn = aws_iam_role.sfn_metadata_refresh.arn
+
+  definition = jsonencode({
+    "Comment" : "Workflow to update the metadata in the reporting tables.",
+    "StartAt" : "Update Metadata",
+    "States" : {
+      "Update Metadata" : {
+        "Type" : "Task",
+        "Resource" : "arn:aws:states:::athena:startQueryExecution",
+        "Parameters" : {
+          "QueryString" : "MSCK REPAIR TABLE ${aws_glue_catalog_table.event_record.name}",
+          "WorkGroup" : "${aws_athena_workgroup.reporting.name}",
+          "QueryExecutionContext" : {
+            "Database" : "${aws_glue_catalog_database.reporting.name}"
+          }
+        },
+        "End" : true
+      }
+    }
+  })
+
+  logging_configuration {
+    log_destination        = "${aws_cloudwatch_log_group.reporting.arn}:*"
+    include_execution_data = true
+    level                  = "ERROR"
+  }
+}
+
+resource "aws_cloudwatch_log_group" "reporting" {
+  name              = "/aws/sfn-state-machine-metadata-refresh/${local.csi}"
+  retention_in_days = var.log_retention_in_days
+}
+
+resource "aws_iam_role" "sfn_metadata_refresh" {
+  name               = "${local.csi}-sf-metadata-refresh-role"
+  description        = "Role used by the State Machine for Athena metadata refresh queries"
+  assume_role_policy = data.aws_iam_policy_document.sfn_assumerole_metadata_refresh.json
+}
+
+data "aws_iam_policy_document" "sfn_assumerole_metadata_refresh" {
+  statement {
+    sid    = "StateMachineAssumeRole"
+    effect = "Allow"
+
+    actions = [
+      "sts:AssumeRole"
+    ]
+
+    principals {
+      type = "Service"
+
+      identifiers = [
+        "states.amazonaws.com",
+        "glue.amazonaws.com"
+      ]
+    }
+  }
+}
+
+resource "aws_iam_role_policy_attachment" "sfn_metadata_refresh" {
+  role       = aws_iam_role.sfn_metadata_refresh.name
+  policy_arn = aws_iam_policy.sfn_metadata_refresh.arn
+}
+
+resource "aws_iam_policy" "sfn_metadata_refresh" {
+  name        = "${local.csi}-sfn-metadata-refresh-policy"
+  description = "Allow Step Function State Machine to run Athena metadata refresh queries"
+  path        = "/"
+  policy      = data.aws_iam_policy_document.sfn_metadata_refresh.json
+}
+
+data "aws_iam_policy_document" "sfn_metadata_refresh" {
+  statement {
+    sid    = "AllowAthena"
+    effect = "Allow"
+
+    actions = [
+      "athena:startQueryExecution",
+    ]
+
+    resources = [
+      aws_athena_workgroup.reporting.arn,
+      "arn:aws:athena:${var.region}:${var.aws_account_id}:datacatalog/*"
+    ]
+  }
+
+  statement {
+    sid    = "AllowGlueCurrent"
+    effect = "Allow"
+
+    actions = [
+      "glue:Get*",
+      "glue:BatchCreatePartition"
+    ]
+
+    resources = [
+      "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog",
+      aws_glue_catalog_database.reporting.arn,
+      "arn:aws:glue:${var.region}:${var.aws_account_id}:table/${aws_glue_catalog_database.reporting.name}/*"
+    ]
+  }
+
+  statement {
+    sid    = "AllowS3Current"
+    effect = "Allow"
+
+    actions = [
+      "s3:PutObject",
+      "s3:ListBucket",
+      "s3:GetObject",
+      "s3:GetBucketLocation",
+      "s3:DescribeJob",
+    ]
+
+    resources = [
+      module.s3bucket_reporting.arn,
+      "${module.s3bucket_reporting.arn}/*"
+    ]
+  }
+
+  statement {
+    sid    = "AllowKMSCurrent"
+    effect = "Allow"
+
+    actions = [
+      "kms:GenerateDataKey*",
+      "kms:Encrypt",
+      "kms:DescribeKey",
+    ]
+
+    resources = [
+      module.kms.key_arn
+    ]
+  }
+
+  statement {
+    sid    = "AllowCloudwatchLogging"
+    effect = "Allow"
+
+    actions = [
+      "logs:CreateLogDelivery",
+      "logs:GetLogDelivery",
+      "logs:UpdateLogDelivery",
+      "logs:DeleteLogDelivery",
+      "logs:ListLogDeliveries",
+      "logs:PutResourcePolicy",
+      "logs:DescribeResourcePolicies",
+      "logs:DescribeLogGroups",
+      "logs:CreateLogStream",
+      "logs:PutLogEvents"
+    ]
+
+    resources = [
+      "*", # See https://docs.aws.amazon.com/step-functions/latest/dg/cw-logs.html & https://github.com/aws/aws-cdk/issues/7158
+    ]
+  }
+}

infrastructure/terraform/components/dl/variables.tf

@@ -198,3 +198,9 @@ variable "default_cloudwatch_event_bus_name" {
   description = "The name of the default cloudwatch event bus. This is needed as GuardDuty Scan Result events are sent to the default bus"
   default     = "default"
 }
+
+variable "metadata_refresh_schedule" {
+  type        = string
+  description = "Schedule for refreshing reporting metadata."
+  default     = "cron(10 6-22 * * ? *)" # 10 minutes past the hour, between 06:00 - 22:00
+}

lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts

@@ -46,8 +46,10 @@ describe('Firehose Handler', () => {
       );
       expect(decodedData).toEqual({
         messageReference: digitalLettersEvent.data.messageReference,
-        senderId: digitalLettersEvent.data.senderId,
         pageCount: digitalLettersEvent.data.pageCount,
+        reasonCode: digitalLettersEvent.data.reasonCode,
+        reasonText: digitalLettersEvent.data.reasonText,
+        senderId: digitalLettersEvent.data.senderId,
         supplierId: digitalLettersEvent.data.supplierId,
         time: digitalLettersEvent.time,
         type: digitalLettersEvent.type,