From 4a61c682217e29fe62f931482931fdc9f1a7b232 Mon Sep 17 00:00:00 2001 From: Matt Jarvis Date: Wed, 11 Feb 2026 17:09:47 +0000 Subject: [PATCH 1/3] VED-1029: Specify KMS key when copying extended attributes files to DPS. --- .../environments/prod/blue/variables.tfvars | 2 +- .../environments/prod/green/variables.tfvars | 2 +- .../instance/file_name_processor.tf | 25 ++++++++++--------- infrastructure/instance/variables.tf | 2 +- lambdas/filenameprocessor/src/constants.py | 1 + .../src/file_name_processor.py | 2 ++ .../mock_environment_variables.py | 3 ++- lambdas/shared/src/common/aws_s3_utils.py | 20 +++++++++------ 8 files changed, 34 insertions(+), 23 deletions(-) diff --git a/infrastructure/instance/environments/prod/blue/variables.tfvars b/infrastructure/instance/environments/prod/blue/variables.tfvars index 5319e42c0..1840ce556 100644 --- a/infrastructure/instance/environments/prod/blue/variables.tfvars +++ b/infrastructure/instance/environments/prod/blue/variables.tfvars @@ -10,4 +10,4 @@ mesh_no_invocation_period_seconds = 86400 create_mesh_processor = true has_sub_environment_scope = false dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload" -dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key" +dspp_submission_kms_key_alias = "alias/nhsd-dspp-core-prod-s3-submission-upload-key" diff --git a/infrastructure/instance/environments/prod/green/variables.tfvars b/infrastructure/instance/environments/prod/green/variables.tfvars index 5319e42c0..1840ce556 100644 --- a/infrastructure/instance/environments/prod/green/variables.tfvars +++ b/infrastructure/instance/environments/prod/green/variables.tfvars @@ -10,4 +10,4 @@ mesh_no_invocation_period_seconds = 86400 create_mesh_processor = true has_sub_environment_scope = false dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload" -dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key" +dspp_submission_kms_key_alias = "alias/nhsd-dspp-core-prod-s3-submission-upload-key" diff --git a/infrastructure/instance/file_name_processor.tf b/infrastructure/instance/file_name_processor.tf index 106a504a2..f2417795b 100644 --- a/infrastructure/instance/file_name_processor.tf +++ b/infrastructure/instance/file_name_processor.tf @@ -262,7 +262,7 @@ resource "aws_iam_policy" "filenameprocessor_dps_extended_attribute_kms_policy" Resource = "arn:aws:kms:${var.aws_region}:${var.dspp_core_account_id}:key/*", "Condition" = { "ForAnyValue:StringEquals" = { - "kms:ResourceAliases" = "alias/${var.dspp_submission_kms_key_alias}" + "kms:ResourceAliases" = var.dspp_submission_kms_key_alias } } } @@ -316,17 +316,18 @@ resource "aws_lambda_function" "file_processor_lambda" { environment { variables = { - ACCOUNT_ID = var.immunisation_account_id - DPS_ACCOUNT_ID = var.dspp_core_account_id - SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket - ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket - DPS_BUCKET_NAME = var.dspp_submission_s3_bucket_name - QUEUE_URL = aws_sqs_queue.batch_file_created.url - REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address - REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port - SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name - AUDIT_TABLE_NAME = aws_dynamodb_table.audit-table.name - AUDIT_TABLE_TTL_DAYS = 60 + ACCOUNT_ID = var.immunisation_account_id + DPS_ACCOUNT_ID = var.dspp_core_account_id + SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket + ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket + DPS_BUCKET_NAME = var.dspp_submission_s3_bucket_name + DPS_BUCKET_KMS_KEY_ALIAS = var.dspp_submission_kms_key_alias + QUEUE_URL = aws_sqs_queue.batch_file_created.url + REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address + REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port + SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name + AUDIT_TABLE_NAME = aws_dynamodb_table.audit-table.name + AUDIT_TABLE_TTL_DAYS = 60 } } kms_key_arn = data.aws_kms_key.existing_lambda_encryption_key.arn diff --git a/infrastructure/instance/variables.tf b/infrastructure/instance/variables.tf index 51d271df9..07b0e049c 100644 --- a/infrastructure/instance/variables.tf +++ b/infrastructure/instance/variables.tf @@ -36,7 +36,7 @@ variable "dspp_submission_s3_bucket_name" { variable "dspp_submission_kms_key_alias" { description = "Alias of the DSPP (DPS) KMS key required to encrypt extended attributes files" type = string - default = "nhsd-dspp-core-ref-s3-submission-upload-key" + default = "alias/nhsd-dspp-core-ref-s3-submission-upload-key" } variable "create_mesh_processor" { diff --git a/lambdas/filenameprocessor/src/constants.py b/lambdas/filenameprocessor/src/constants.py index 4dda64113..7fcaba854 100644 --- a/lambdas/filenameprocessor/src/constants.py +++ b/lambdas/filenameprocessor/src/constants.py @@ -10,6 +10,7 @@ ) DPS_DESTINATION_BUCKET_NAME = os.getenv("DPS_BUCKET_NAME") +DPS_DESTINATION_BUCKET_KMS_KEY_ALIAS = os.getenv("DPS_BUCKET_KMS_KEY_ALIAS") EXPECTED_SOURCE_BUCKET_ACCOUNT = os.getenv("ACCOUNT_ID") EXPECTED_DPS_DESTINATION_ACCOUNT = os.getenv("DPS_ACCOUNT_ID") AUDIT_TABLE_TTL_DAYS = os.getenv("AUDIT_TABLE_TTL_DAYS") diff --git a/lambdas/filenameprocessor/src/file_name_processor.py b/lambdas/filenameprocessor/src/file_name_processor.py index 6b2c58b95..a249a5c3e 100644 --- a/lambdas/filenameprocessor/src/file_name_processor.py +++ b/lambdas/filenameprocessor/src/file_name_processor.py @@ -19,6 +19,7 @@ from common.models.batch_constants import SOURCE_BUCKET_NAME, FileStatus from common.models.errors import UnhandledAuditTableError from constants import ( + DPS_DESTINATION_BUCKET_KMS_KEY_ALIAS, DPS_DESTINATION_BUCKET_NAME, DPS_DESTINATION_PREFIX, ERROR_TYPE_TO_STATUS_CODE_MAP, @@ -262,6 +263,7 @@ def handle_extended_attributes_file( dest_file_key, EXPECTED_DPS_DESTINATION_ACCOUNT, EXPECTED_SOURCE_BUCKET_ACCOUNT, + DPS_DESTINATION_BUCKET_KMS_KEY_ALIAS, ) move_file(bucket_name, file_key, f"{EXTENDED_ATTRIBUTES_ARCHIVE_PREFIX}/{file_key}") diff --git a/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py b/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py index 8c16d525b..ab72c8bf6 100644 --- a/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py +++ b/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py @@ -12,7 +12,7 @@ class BucketNames: CONFIG = "immunisation-batch-internal-dev-data-configs" SOURCE = "immunisation-batch-internal-dev-data-sources" DESTINATION = "immunisation-batch-internal-dev-data-destinations" - DPS_DESTINATION = "nhsd-dspp-core-ref-extended-attributes-gdp" + DPS_DESTINATION = "nhsd-dspp-core-ref-s3-submission-upload" # Mock firehose bucket used for testing only (due to limitations of the moto testing package) MOCK_FIREHOSE = "mock-firehose-bucket" @@ -38,6 +38,7 @@ class Sqs: "SOURCE_BUCKET_NAME": BucketNames.SOURCE, "ACK_BUCKET_NAME": BucketNames.DESTINATION, "DPS_BUCKET_NAME": BucketNames.DPS_DESTINATION, + "DPS_BUCKET_KMS_KEY_ALIAS": "alias/nhsd-dspp-core-ref-s3-submission-upload-key", "ACCOUNT_ID": MOCK_ACCOUNT_ID, "DPS_ACCOUNT_ID": MOCK_ACCOUNT_ID, "QUEUE_URL": "https://sqs.eu-west-2.amazonaws.com/123456789012/imms-batch-file-created-queue.fifo", diff --git a/lambdas/shared/src/common/aws_s3_utils.py b/lambdas/shared/src/common/aws_s3_utils.py index 22291daa3..3b2b9e579 100644 --- a/lambdas/shared/src/common/aws_s3_utils.py +++ b/lambdas/shared/src/common/aws_s3_utils.py @@ -22,15 +22,21 @@ def copy_file_to_external_bucket( destination_key: str, expected_bucket_owner: str, expected_source_bucket_owner: str, + sse_kms_key_id: str | None = None, ) -> None: + copy_params = { + "CopySource": {"Bucket": source_bucket, "Key": source_key}, + "Bucket": destination_bucket, + "Key": destination_key, + "ExpectedBucketOwner": expected_bucket_owner, + "ExpectedSourceBucketOwner": expected_source_bucket_owner, + } + + if sse_kms_key_id: + copy_params["SSEKMSKeyId"] = sse_kms_key_id + s3_client = get_s3_client() - s3_client.copy_object( - CopySource={"Bucket": source_bucket, "Key": source_key}, - Bucket=destination_bucket, - Key=destination_key, - ExpectedBucketOwner=expected_bucket_owner, - ExpectedSourceBucketOwner=expected_source_bucket_owner, - ) + s3_client.copy_object(**copy_params) def delete_file( From 30b7593c33aebd1fa958787458094c28e0fe65ca Mon Sep 17 00:00:00 2001 From: Matt Jarvis Date: Wed, 11 Feb 2026 22:52:05 +0000 Subject: [PATCH 2/3] VED-1029: Specify SSE via KMS when a KMS key ID is specified. --- lambdas/shared/src/common/aws_s3_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lambdas/shared/src/common/aws_s3_utils.py b/lambdas/shared/src/common/aws_s3_utils.py index 3b2b9e579..4cd56b0b8 100644 --- a/lambdas/shared/src/common/aws_s3_utils.py +++ b/lambdas/shared/src/common/aws_s3_utils.py @@ -33,6 +33,7 @@ def copy_file_to_external_bucket( } if sse_kms_key_id: + copy_params["ServerSideEncryption"] = "aws:kms" copy_params["SSEKMSKeyId"] = sse_kms_key_id s3_client = get_s3_client() From 84acb7fb58f0f97c98f3690be58bedf3a483f5a8 Mon Sep 17 00:00:00 2001 From: Matt Jarvis Date: Wed, 11 Feb 2026 23:10:10 +0000 Subject: [PATCH 3/3] VED-1029: Specify KMS key ARN instead of alias. --- .../instance/file_name_processor.tf | 24 +++++++++---------- lambdas/filenameprocessor/src/constants.py | 2 +- .../src/file_name_processor.py | 4 ++-- .../mock_environment_variables.py | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/infrastructure/instance/file_name_processor.tf b/infrastructure/instance/file_name_processor.tf index f2417795b..b5393395e 100644 --- a/infrastructure/instance/file_name_processor.tf +++ b/infrastructure/instance/file_name_processor.tf @@ -316,18 +316,18 @@ resource "aws_lambda_function" "file_processor_lambda" { environment { variables = { - ACCOUNT_ID = var.immunisation_account_id - DPS_ACCOUNT_ID = var.dspp_core_account_id - SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket - ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket - DPS_BUCKET_NAME = var.dspp_submission_s3_bucket_name - DPS_BUCKET_KMS_KEY_ALIAS = var.dspp_submission_kms_key_alias - QUEUE_URL = aws_sqs_queue.batch_file_created.url - REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address - REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port - SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name - AUDIT_TABLE_NAME = aws_dynamodb_table.audit-table.name - AUDIT_TABLE_TTL_DAYS = 60 + ACCOUNT_ID = var.immunisation_account_id + DPS_ACCOUNT_ID = var.dspp_core_account_id + SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket + ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket + DPS_BUCKET_NAME = var.dspp_submission_s3_bucket_name + DPS_BUCKET_KMS_KEY_ARN = "arn:aws:kms:${var.aws_region}:${var.dspp_core_account_id}:${var.dspp_submission_kms_key_alias}" + QUEUE_URL = aws_sqs_queue.batch_file_created.url + REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address + REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port + SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name + AUDIT_TABLE_NAME = aws_dynamodb_table.audit-table.name + AUDIT_TABLE_TTL_DAYS = 60 } } kms_key_arn = data.aws_kms_key.existing_lambda_encryption_key.arn diff --git a/lambdas/filenameprocessor/src/constants.py b/lambdas/filenameprocessor/src/constants.py index 7fcaba854..de4ec1825 100644 --- a/lambdas/filenameprocessor/src/constants.py +++ b/lambdas/filenameprocessor/src/constants.py @@ -10,7 +10,7 @@ ) DPS_DESTINATION_BUCKET_NAME = os.getenv("DPS_BUCKET_NAME") -DPS_DESTINATION_BUCKET_KMS_KEY_ALIAS = os.getenv("DPS_BUCKET_KMS_KEY_ALIAS") +DPS_DESTINATION_BUCKET_KMS_KEY_ARN = os.getenv("DPS_BUCKET_KMS_KEY_ARN") EXPECTED_SOURCE_BUCKET_ACCOUNT = os.getenv("ACCOUNT_ID") EXPECTED_DPS_DESTINATION_ACCOUNT = os.getenv("DPS_ACCOUNT_ID") AUDIT_TABLE_TTL_DAYS = os.getenv("AUDIT_TABLE_TTL_DAYS") diff --git a/lambdas/filenameprocessor/src/file_name_processor.py b/lambdas/filenameprocessor/src/file_name_processor.py index a249a5c3e..9e01574df 100644 --- a/lambdas/filenameprocessor/src/file_name_processor.py +++ b/lambdas/filenameprocessor/src/file_name_processor.py @@ -19,7 +19,7 @@ from common.models.batch_constants import SOURCE_BUCKET_NAME, FileStatus from common.models.errors import UnhandledAuditTableError from constants import ( - DPS_DESTINATION_BUCKET_KMS_KEY_ALIAS, + DPS_DESTINATION_BUCKET_KMS_KEY_ARN, DPS_DESTINATION_BUCKET_NAME, DPS_DESTINATION_PREFIX, ERROR_TYPE_TO_STATUS_CODE_MAP, @@ -263,7 +263,7 @@ def handle_extended_attributes_file( dest_file_key, EXPECTED_DPS_DESTINATION_ACCOUNT, EXPECTED_SOURCE_BUCKET_ACCOUNT, - DPS_DESTINATION_BUCKET_KMS_KEY_ALIAS, + DPS_DESTINATION_BUCKET_KMS_KEY_ARN, ) move_file(bucket_name, file_key, f"{EXTENDED_ATTRIBUTES_ARCHIVE_PREFIX}/{file_key}") diff --git a/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py b/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py index ab72c8bf6..a3fd5a0d7 100644 --- a/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py +++ b/lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py @@ -38,7 +38,7 @@ class Sqs: "SOURCE_BUCKET_NAME": BucketNames.SOURCE, "ACK_BUCKET_NAME": BucketNames.DESTINATION, "DPS_BUCKET_NAME": BucketNames.DPS_DESTINATION, - "DPS_BUCKET_KMS_KEY_ALIAS": "alias/nhsd-dspp-core-ref-s3-submission-upload-key", + "DPS_BUCKET_KMS_KEY_ARN": "arn:aws:kms:eu-west-2:123456789012:alias/nhsd-dspp-core-ref-s3-submission-upload-key", "ACCOUNT_ID": MOCK_ACCOUNT_ID, "DPS_ACCOUNT_ID": MOCK_ACCOUNT_ID, "QUEUE_URL": "https://sqs.eu-west-2.amazonaws.com/123456789012/imms-batch-file-created-queue.fifo",