diff --git a/infrastructure/terraform/components/dl/README.md b/infrastructure/terraform/components/dl/README.md index 93dff3f7..ac8f25b4 100644 --- a/infrastructure/terraform/components/dl/README.md +++ b/infrastructure/terraform/components/dl/README.md @@ -32,6 +32,7 @@ No requirements. | [log\_level](#input\_log\_level) | The log level to be used in lambda functions within the component. Any log with a lower severity than the configured value will not be logged: https://docs.python.org/3/library/logging.html#levels | `string` | `"INFO"` | no | | [log\_retention\_in\_days](#input\_log\_retention\_in\_days) | The retention period in days for the Cloudwatch Logs events to be retained, default of 0 is indefinite | `number` | `0` | no | | [mesh\_poll\_schedule](#input\_mesh\_poll\_schedule) | Schedule to poll MESH for messages | `string` | `"rate(5 minutes)"` | no | +| [metadata\_refresh\_schedule](#input\_metadata\_refresh\_schedule) | Schedule for refreshing reporting metadata. | `string` | `"cron(10 6-22 * * ? *)"` | no | | [parent\_acct\_environment](#input\_parent\_acct\_environment) | Name of the environment responsible for the acct resources used, affects things like DNS zone. Useful for named dev environments | `string` | `"main"` | no | | [pdm\_mock\_access\_token](#input\_pdm\_mock\_access\_token) | Mock access token for PDM API authentication (used in local/dev environments) | `string` | `"mock-pdm-token"` | no | | [pdm\_use\_non\_mock\_token](#input\_pdm\_use\_non\_mock\_token) | Whether to use the shared APIM access token from SSM (/component/environment/apim/access\_token) instead of the mock token | `bool` | `false` | no | diff --git a/infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_aborted.tf b/infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_aborted.tf new file mode 100644 index 00000000..b791a60e --- /dev/null +++ b/infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_aborted.tf @@ -0,0 +1,16 @@ +resource "aws_cloudwatch_metric_alarm" "metadata_refresh_executions_aborted" { + alarm_name = "${local.csi}-metadata-refresh-execution-aborted" + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = 1 + metric_name = "ExecutionsAborted" + namespace = "AWS/States" + period = 60 + statistic = "Sum" + threshold = 1 + alarm_description = "This metric monitors aborted step function executions" + treat_missing_data = "notBreaching" + + dimensions = { + StateMachineArn = aws_sfn_state_machine.metadata_refresh.arn + } +} diff --git a/infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_failed.tf b/infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_failed.tf new file mode 100644 index 00000000..7a526531 --- /dev/null +++ b/infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_failed.tf @@ -0,0 +1,16 @@ +resource "aws_cloudwatch_metric_alarm" "metadata_refresh_executions_failed" { + alarm_name = "${local.csi}-metadata-refresh-executions-failed" + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = 1 + metric_name = "ExecutionsFailed" + namespace = "AWS/States" + period = 60 + statistic = "Sum" + threshold = 1 + alarm_description = "This metric monitors failed step function executions" + treat_missing_data = "notBreaching" + + dimensions = { + StateMachineArn = aws_sfn_state_machine.metadata_refresh.arn + } +} diff --git a/infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_timedout.tf b/infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_timedout.tf new file mode 100644 index 00000000..6204e712 --- /dev/null +++ b/infrastructure/terraform/components/dl/cloudwatch_metric_alarm_metadata_refresh_executions_timedout.tf @@ -0,0 +1,16 @@ +resource "aws_cloudwatch_metric_alarm" "metadata_refresh_executions_timedout" { + alarm_name = "${local.csi}-metadata-refresh-executions-timedout" + comparison_operator = "GreaterThanOrEqualToThreshold" + evaluation_periods = 1 + metric_name = "ExecutionsTimedOut" + namespace = "AWS/States" + period = 60 + statistic = "Sum" + threshold = 1 + alarm_description = "This metric monitors step function execution timeouts" + treat_missing_data = "notBreaching" + + dimensions = { + StateMachineArn = aws_sfn_state_machine.metadata_refresh.arn + } +} diff --git a/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf b/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf index f03c9586..81f851df 100644 --- a/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf +++ b/infrastructure/terraform/components/dl/glue_catalog_table_event_record.tf @@ -24,6 +24,14 @@ resource "aws_glue_catalog_table" "event_record" { name = "pagecount" type = "int" } + columns { + name = "reasoncode" + type = "string" + } + columns { + name = "reasontext" + type = "string" + } columns { name = "supplierid" type = "string" diff --git a/infrastructure/terraform/components/dl/module_sqs_mesh_acknowledge.tf b/infrastructure/terraform/components/dl/module_sqs_mesh_acknowledge.tf index 67716352..ee5b66ce 100644 --- a/infrastructure/terraform/components/dl/module_sqs_mesh_acknowledge.tf +++ b/infrastructure/terraform/components/dl/module_sqs_mesh_acknowledge.tf @@ -34,5 +34,11 @@ data "aws_iam_policy_document" "sqs_mesh_acknowledge" { resources = [ "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-mesh-acknowledge-queue" ] + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.mesh_inbox_message_downloaded.arn] + } } } diff --git a/infrastructure/terraform/components/dl/module_sqs_mesh_download.tf b/infrastructure/terraform/components/dl/module_sqs_mesh_download.tf index bde3d796..16cc16c0 100644 --- a/infrastructure/terraform/components/dl/module_sqs_mesh_download.tf +++ b/infrastructure/terraform/components/dl/module_sqs_mesh_download.tf @@ -34,5 +34,11 @@ data "aws_iam_policy_document" "sqs_mesh_download" { resources = [ "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-mesh-download-queue" ] + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.mesh_inbox_message_received.arn] + } } } diff --git a/infrastructure/terraform/components/dl/module_sqs_pdm_poll.tf b/infrastructure/terraform/components/dl/module_sqs_pdm_poll.tf index 2f3b22d9..79f0464b 100644 --- a/infrastructure/terraform/components/dl/module_sqs_pdm_poll.tf +++ b/infrastructure/terraform/components/dl/module_sqs_pdm_poll.tf @@ -31,5 +31,11 @@ data "aws_iam_policy_document" "sqs_pdm_poll" { resources = [ "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-pdm-poll-queue" ] + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.pdm_resource_submitted.arn, aws_cloudwatch_event_rule.pdm_resource_unavailable.arn] + } } } diff --git a/infrastructure/terraform/components/dl/module_sqs_pdm_uploader.tf b/infrastructure/terraform/components/dl/module_sqs_pdm_uploader.tf index a9936526..d1b6669a 100644 --- a/infrastructure/terraform/components/dl/module_sqs_pdm_uploader.tf +++ b/infrastructure/terraform/components/dl/module_sqs_pdm_uploader.tf @@ -34,5 +34,11 @@ data "aws_iam_policy_document" "sqs_pdm_uploader" { resources = [ "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-pdm-uploader-queue" ] + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.mesh_inbox_message_downloaded.arn] + } } } diff --git a/infrastructure/terraform/components/dl/module_sqs_print_status_handler.tf b/infrastructure/terraform/components/dl/module_sqs_print_status_handler.tf index 075b5950..be1e5a3b 100644 --- a/infrastructure/terraform/components/dl/module_sqs_print_status_handler.tf +++ b/infrastructure/terraform/components/dl/module_sqs_print_status_handler.tf @@ -32,5 +32,11 @@ data "aws_iam_policy_document" "sqs_print_status_handler" { resources = [ "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-print-status-handler-queue" ] + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.print_status_changed.arn] + } } } diff --git a/infrastructure/terraform/components/dl/module_sqs_ttl.tf b/infrastructure/terraform/components/dl/module_sqs_ttl.tf index 20b0e7ee..a78a2fd1 100644 --- a/infrastructure/terraform/components/dl/module_sqs_ttl.tf +++ b/infrastructure/terraform/components/dl/module_sqs_ttl.tf @@ -34,5 +34,11 @@ data "aws_iam_policy_document" "sqs_ttl" { resources = [ "arn:aws:sqs:${var.region}:${var.aws_account_id}:${local.csi}-ttl-queue" ] + + condition { + test = "ArnLike" + variable = "aws:SourceArn" + values = [aws_cloudwatch_event_rule.mesh_inbox_message_downloaded.arn] + } } } diff --git a/infrastructure/terraform/components/dl/scheduler_schedule_sf_metadata_refresh_scheduler.tf b/infrastructure/terraform/components/dl/scheduler_schedule_sf_metadata_refresh_scheduler.tf new file mode 100644 index 00000000..f3bb3edd --- /dev/null +++ b/infrastructure/terraform/components/dl/scheduler_schedule_sf_metadata_refresh_scheduler.tf @@ -0,0 +1,69 @@ +resource "aws_scheduler_schedule" "sf_metadata_refresh_scheduler" { + name = "${local.csi}-metadata-refresh-scheduler" + description = "Scheduler to trigger Step Function to run metadata refresh queries" + group_name = "default" + + flexible_time_window { + mode = "OFF" + } + + schedule_expression = var.metadata_refresh_schedule + schedule_expression_timezone = "Europe/London" + + target { + arn = aws_sfn_state_machine.metadata_refresh.arn + role_arn = aws_iam_role.sf_metadata_refresh_scheduler.arn + } +} + +resource "aws_iam_role" "sf_metadata_refresh_scheduler" { + name = "${local.csi}-sf-metadata-refresh-scheduler-role" + description = "Role used by the State Machine Metadata Refresh Scheduler" + assume_role_policy = data.aws_iam_policy_document.metadata_refresh_scheduler_assumerole.json +} + +data "aws_iam_policy_document" "metadata_refresh_scheduler_assumerole" { + statement { + sid = "EcsAssumeRole" + effect = "Allow" + + actions = [ + "sts:AssumeRole", + ] + + principals { + type = "Service" + + identifiers = [ + "scheduler.amazonaws.com" + ] + } + } +} + +resource "aws_iam_role_policy_attachment" "sf_metadata_refresh_scheduler" { + role = aws_iam_role.sf_metadata_refresh_scheduler.name + policy_arn = aws_iam_policy.sf_metadata_refresh_scheduler.arn +} + +resource "aws_iam_policy" "sf_metadata_refresh_scheduler" { + name = "${local.csi}-sfn-metadata-refresh-scheduler-policy" + description = "Allow Scheduler to execute State Machine" + path = "/" + policy = data.aws_iam_policy_document.sf_metadata_refresh_scheduler.json +} + +data "aws_iam_policy_document" "sf_metadata_refresh_scheduler" { + statement { + sid = "AllowStepFunctionExecution" + effect = "Allow" + + actions = [ + "states:StartExecution" + ] + + resources = [ + aws_sfn_state_machine.metadata_refresh.arn + ] + } +} diff --git a/infrastructure/terraform/components/dl/sfn_state_machine_metadata_refresh.tf b/infrastructure/terraform/components/dl/sfn_state_machine_metadata_refresh.tf new file mode 100644 index 00000000..18de576c --- /dev/null +++ b/infrastructure/terraform/components/dl/sfn_state_machine_metadata_refresh.tf @@ -0,0 +1,159 @@ +resource "aws_sfn_state_machine" "metadata_refresh" { + name = "${local.csi}-state-machine-metadata-refresh" + role_arn = aws_iam_role.sfn_metadata_refresh.arn + + definition = jsonencode({ + "Comment" : "Workflow to update the metadata in the reporting tables.", + "StartAt" : "Update Metadata", + "States" : { + "Update Metadata" : { + "Type" : "Task", + "Resource" : "arn:aws:states:::athena:startQueryExecution", + "Parameters" : { + "QueryString" : "MSCK REPAIR TABLE ${aws_glue_catalog_table.event_record.name}", + "WorkGroup" : "${aws_athena_workgroup.reporting.name}", + "QueryExecutionContext" : { + "Database" : "${aws_glue_catalog_database.reporting.name}" + } + }, + "End" : true + } + } + }) + + logging_configuration { + log_destination = "${aws_cloudwatch_log_group.reporting.arn}:*" + include_execution_data = true + level = "ERROR" + } +} + +resource "aws_cloudwatch_log_group" "reporting" { + name = "/aws/sfn-state-machine-metadata-refresh/${local.csi}" + retention_in_days = var.log_retention_in_days +} + +resource "aws_iam_role" "sfn_metadata_refresh" { + name = "${local.csi}-sf-metadata-refresh-role" + description = "Role used by the State Machine for Athena metadata refresh queries" + assume_role_policy = data.aws_iam_policy_document.sfn_assumerole_metadata_refresh.json +} + +data "aws_iam_policy_document" "sfn_assumerole_metadata_refresh" { + statement { + sid = "StateMachineAssumeRole" + effect = "Allow" + + actions = [ + "sts:AssumeRole" + ] + + principals { + type = "Service" + + identifiers = [ + "states.amazonaws.com", + "glue.amazonaws.com" + ] + } + } +} + +resource "aws_iam_role_policy_attachment" "sfn_metadata_refresh" { + role = aws_iam_role.sfn_metadata_refresh.name + policy_arn = aws_iam_policy.sfn_metadata_refresh.arn +} + +resource "aws_iam_policy" "sfn_metadata_refresh" { + name = "${local.csi}-sfn-metadata-refresh-policy" + description = "Allow Step Function State Machine to run Athena metadata refresh queries" + path = "/" + policy = data.aws_iam_policy_document.sfn_metadata_refresh.json +} + +data "aws_iam_policy_document" "sfn_metadata_refresh" { + statement { + sid = "AllowAthena" + effect = "Allow" + + actions = [ + "athena:startQueryExecution", + ] + + resources = [ + aws_athena_workgroup.reporting.arn, + "arn:aws:athena:${var.region}:${var.aws_account_id}:datacatalog/*" + ] + } + + statement { + sid = "AllowGlueCurrent" + effect = "Allow" + + actions = [ + "glue:Get*", + "glue:BatchCreatePartition" + ] + + resources = [ + "arn:aws:glue:${var.region}:${var.aws_account_id}:catalog", + aws_glue_catalog_database.reporting.arn, + "arn:aws:glue:${var.region}:${var.aws_account_id}:table/${aws_glue_catalog_database.reporting.name}/*" + ] + } + + statement { + sid = "AllowS3Current" + effect = "Allow" + + actions = [ + "s3:PutObject", + "s3:ListBucket", + "s3:GetObject", + "s3:GetBucketLocation", + "s3:DescribeJob", + ] + + resources = [ + module.s3bucket_reporting.arn, + "${module.s3bucket_reporting.arn}/*" + ] + } + + statement { + sid = "AllowKMSCurrent" + effect = "Allow" + + actions = [ + "kms:GenerateDataKey*", + "kms:Encrypt", + "kms:DescribeKey", + ] + + resources = [ + module.kms.key_arn + ] + } + + statement { + sid = "AllowCloudwatchLogging" + effect = "Allow" + + actions = [ + "logs:CreateLogDelivery", + "logs:GetLogDelivery", + "logs:UpdateLogDelivery", + "logs:DeleteLogDelivery", + "logs:ListLogDeliveries", + "logs:PutResourcePolicy", + "logs:DescribeResourcePolicies", + "logs:DescribeLogGroups", + "logs:CreateLogStream", + "logs:PutLogEvents" + ] + + resources = [ + "*", # See https://docs.aws.amazon.com/step-functions/latest/dg/cw-logs.html & https://github.com/aws/aws-cdk/issues/7158 + ] + } +} diff --git a/infrastructure/terraform/components/dl/variables.tf b/infrastructure/terraform/components/dl/variables.tf index 01c959a8..844617b2 100644 --- a/infrastructure/terraform/components/dl/variables.tf +++ b/infrastructure/terraform/components/dl/variables.tf @@ -213,3 +213,9 @@ variable "default_cloudwatch_event_bus_name" { description = "The name of the default cloudwatch event bus. This is needed as GuardDuty Scan Result events are sent to the default bus" default = "default" } + +variable "metadata_refresh_schedule" { + type = string + description = "Schedule for refreshing reporting metadata." + default = "cron(10 6-22 * * ? *)" # 10 minutes past the hour, between 06:00 - 22:00 +} diff --git a/lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts b/lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts index 10942086..f65b566b 100644 --- a/lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts +++ b/lambdas/report-event-transformer/src/__tests__/apis/firehose-handler.test.ts @@ -46,8 +46,10 @@ describe('Firehose Handler', () => { ); expect(decodedData).toEqual({ messageReference: digitalLettersEvent.data.messageReference, - senderId: digitalLettersEvent.data.senderId, pageCount: digitalLettersEvent.data.pageCount, + reasonCode: digitalLettersEvent.data.reasonCode, + reasonText: digitalLettersEvent.data.reasonText, + senderId: digitalLettersEvent.data.senderId, supplierId: digitalLettersEvent.data.supplierId, time: digitalLettersEvent.time, type: digitalLettersEvent.type, diff --git a/lambdas/report-event-transformer/src/__tests__/test-data.ts b/lambdas/report-event-transformer/src/__tests__/test-data.ts index e8043885..31927313 100644 --- a/lambdas/report-event-transformer/src/__tests__/test-data.ts +++ b/lambdas/report-event-transformer/src/__tests__/test-data.ts @@ -1,7 +1,7 @@ import { FirehoseTransformationEvent } from 'aws-lambda'; import { DigitalLettersEvent } from 'types/events'; -const baseEvent = { +export const digitalLettersEvent = { id: '550e8400-e29b-41d4-a716-446655440001', specversion: '1.0', source: @@ -15,20 +15,17 @@ const baseEvent = { traceparent: '00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01', datacontenttype: 'application/json', dataschema: - 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10/digital-letter-base-data.schema.json', + 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-pdm-resource-submitted-data.schema.json', severitytext: 'INFO', data: { resourceId: 'a2bcbb42-ab7e-42b6-88d6-74f8d3ca4a09', messageReference: 'ref1', + pageCount: 5, + reasonCode: 'FAILURE001', + reasonText: 'Letter has too many pages', senderId: 'sender1', + supplierId: 'supplier1', }, -}; - -export const digitalLettersEvent = { - ...baseEvent, - type: 'uk.nhs.notify.digital.letters.pdm.resource.submitted.v1', - dataschema: - 'https://notify.nhs.uk/cloudevents/schemas/digital-letters/2025-10-draft/data/digital-letters-pdm-resource-submitted-data.schema.json', } as DigitalLettersEvent; export const firehoseEvent = ( diff --git a/lambdas/report-event-transformer/src/apis/firehose-handler.ts b/lambdas/report-event-transformer/src/apis/firehose-handler.ts index 171003a2..5b46340e 100644 --- a/lambdas/report-event-transformer/src/apis/firehose-handler.ts +++ b/lambdas/report-event-transformer/src/apis/firehose-handler.ts @@ -56,16 +56,24 @@ function validateRecord( } function generateReportEvent(validatedRecord: ValidatedRecord): ReportEvent { - const { messageReference, pageCount, senderId, supplierId } = - validatedRecord.event.data; + const { + messageReference, + pageCount, + reasonCode, + reasonText, + senderId, + supplierId, + } = validatedRecord.event.data; const { time, type } = validatedRecord.event; const eventTime = new Date(time); const flattenedEvent: FlatDigitalLettersEvent = { messageReference, - senderId, pageCount, + senderId, supplierId, + reasonCode, + reasonText, time, type, }; @@ -100,7 +108,12 @@ export const createHandler = ({ logger }: HandlerDependencies) => if (validated) { validEvents.push(generateReportEvent(validated)); } else { - failedEvents.push({ ...record, result: 'ProcessingFailed' }); + const failedEvent: FirehoseTransformationResultRecord = { + recordId: record.recordId, + result: 'ProcessingFailed', + data: record.data, + }; + failedEvents.push(failedEvent); } } diff --git a/lambdas/report-event-transformer/src/types/events.ts b/lambdas/report-event-transformer/src/types/events.ts index c018f0f2..d88818f6 100644 --- a/lambdas/report-event-transformer/src/types/events.ts +++ b/lambdas/report-event-transformer/src/types/events.ts @@ -3,8 +3,10 @@ import { z } from 'zod'; export const $DigitalLettersEvent = z.object({ data: z.object({ messageReference: z.string(), - senderId: z.string(), pageCount: z.number().optional(), + reasonCode: z.string().optional(), + reasonText: z.string().optional(), + senderId: z.string(), supplierId: z.string().optional(), }), time: z.string(), @@ -15,9 +17,11 @@ export type DigitalLettersEvent = z.infer; export type FlatDigitalLettersEvent = { messageReference: string; - senderId: string; pageCount?: number; + senderId: string; supplierId?: string; + reasonCode?: string; + reasonText?: string; time: string; type: string; }; diff --git a/tests/playwright/constants/backend-constants.ts b/tests/playwright/constants/backend-constants.ts index 0296d41d..9c98f58b 100644 --- a/tests/playwright/constants/backend-constants.ts +++ b/tests/playwright/constants/backend-constants.ts @@ -12,6 +12,7 @@ export const MESH_POLL_LAMBDA_NAME = `${CSI}-mesh-poll`; export const TTL_CREATE_LAMBDA_NAME = `${CSI}-ttl-create`; export const TTL_POLL_LAMBDA_NAME = `${CSI}-ttl-poll`; export const CORE_NOTIFIER_LAMBDA_NAME = `${CSI}-core-notifier`; +export const REPORT_EVENT_TRANSFORMER_LAMBDA_NAME = `${CSI}-report-event-transformer`; export const REPORT_SCHEDULER_LAMBDA_NAME = `${CSI}-report-scheduler`; // Queue Names diff --git a/tests/playwright/digital-letters-component-tests/report-event-transformer.component.spec.ts b/tests/playwright/digital-letters-component-tests/report-event-transformer.component.spec.ts new file mode 100644 index 00000000..4cab04a2 --- /dev/null +++ b/tests/playwright/digital-letters-component-tests/report-event-transformer.component.spec.ts @@ -0,0 +1,112 @@ +import { expect, test } from '@playwright/test'; +import type { + FirehoseTransformationEvent, + FirehoseTransformationEventRecord, + FirehoseTransformationResult, +} from 'aws-lambda'; +import { REPORT_EVENT_TRANSFORMER_LAMBDA_NAME } from 'constants/backend-constants'; +import { invokeLambdaSync } from 'helpers/lambda-helpers'; + +const year = 2026; +const month = 1; +const day = 1; +const timestamp = new Date(Date.UTC(year, month - 1, day)).toISOString(); +const eventType = 'test-event-type'; + +function createRecord( + recordId: string, + data: any, +): FirehoseTransformationEventRecord { + return { + recordId, + data: Buffer.from( + JSON.stringify({ + detail: { + data, + time: timestamp, + type: eventType, + }, + }), + ).toString('base64'), + approximateArrivalTimestamp: Date.now(), + }; +} + +test.describe('Digital Letters - Report Event Transformer', () => { + test('should transform events as expected', async () => { + const validData = { + messageReference: 'test-message-ref', + pageCount: 3, + reasonCode: 'test-reason-code', + reasonText: 'test-reason-text', + senderId: 'test-sender-id', + supplierId: 'test-supplier-id', + }; + const validRecord = createRecord('test-record-id', validData); + + const invalidRecord = createRecord('test-invalid-record-id', { + // Missing required messageReference field + pageCount: 3, + reasonCode: 'test-reason-code', + reasonText: 'test-reason-text', + senderId: 'test-sender-id', + supplierId: 'test-supplier-id', + }); + + const invalidJSONRecord: FirehoseTransformationEventRecord = { + recordId: 'test-invalid-json-record-id', + data: Buffer.from('invalid-json').toString('base64'), + approximateArrivalTimestamp: Date.now(), + }; + + const payload: FirehoseTransformationEvent = { + invocationId: 'test-invocation-id', + deliveryStreamArn: 'test-delivery-stream-arn', + sourceKinesisStreamArn: 'test-source-kinesis-stream-arn', + region: 'eu-west-2', + records: [validRecord, invalidRecord, invalidJSONRecord], + }; + + const result = await invokeLambdaSync( + REPORT_EVENT_TRANSFORMER_LAMBDA_NAME, + payload, + ); + + expect(result).toBeDefined(); + expect(result!.records).toHaveLength(3); + expect(result!.records).toContainEqual({ + recordId: validRecord.recordId, + result: 'Ok', + data: Buffer.from( + JSON.stringify({ + messageReference: validData.messageReference, + pageCount: validData.pageCount, + senderId: validData.senderId, + supplierId: validData.supplierId, + reasonCode: validData.reasonCode, + reasonText: validData.reasonText, + time: timestamp, + type: eventType, + }), + ).toString('base64'), + metadata: { + partitionKeys: { + year: year.toString(), + month: month.toString(), + day: day.toString(), + senderId: validData.senderId, + }, + }, + }); + expect(result!.records).toContainEqual({ + recordId: invalidRecord.recordId, + result: 'ProcessingFailed', + data: invalidRecord.data, + }); + expect(result!.records).toContainEqual({ + recordId: invalidJSONRecord.recordId, + result: 'ProcessingFailed', + data: invalidJSONRecord.data, + }); + }); +}); diff --git a/tests/playwright/helpers/lambda-helpers.ts b/tests/playwright/helpers/lambda-helpers.ts index bc23a646..f516678c 100644 --- a/tests/playwright/helpers/lambda-helpers.ts +++ b/tests/playwright/helpers/lambda-helpers.ts @@ -6,7 +6,7 @@ const lambda = new LambdaClient({ async function invokeLambda( functionName: string, - payload?: Record, + payload?: any, ): Promise { await lambda.send( new InvokeCommand({ @@ -19,7 +19,7 @@ async function invokeLambda( async function invokeLambdaSync( functionName: string, - payload?: Record, + payload?: any, ): Promise { const response = await lambda.send( new InvokeCommand({