Skip to content

VED-1029: Specify KMS key when copying extended attributes files to DPS. #1197

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
mfjarvis wants to merge 3 commits into from
Closed

VED-1029: Specify KMS key when copying extended attributes files to DPS. #1197

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4a61c68
VED-1029: Specify KMS key when copying extended attributes files to DPS.
mfjarvis Feb 11, 2026
30b7593
VED-1029: Specify SSE via KMS when a KMS key ID is specified.
mfjarvis Feb 11, 2026
84acb7f
VED-1029: Specify KMS key ARN instead of alias.
mfjarvis Feb 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion infrastructure/instance/environments/prod/blue/variables.tfvars
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ mesh_no_invocation_period_seconds = 86400
create_mesh_processor = true
has_sub_environment_scope = false
dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload"
dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key"
dspp_submission_kms_key_alias = "alias/nhsd-dspp-core-prod-s3-submission-upload-key"
2 changes: 1 addition & 1 deletion infrastructure/instance/environments/prod/green/variables.tfvars
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,4 @@ mesh_no_invocation_period_seconds = 86400
create_mesh_processor = true
has_sub_environment_scope = false
dspp_submission_s3_bucket_name = "nhsd-dspp-core-prod-s3-submission-upload"
dspp_submission_kms_key_alias = "nhsd-dspp-core-prod-s3-submission-upload-key"
dspp_submission_kms_key_alias = "alias/nhsd-dspp-core-prod-s3-submission-upload-key"
25 changes: 13 additions & 12 deletions infrastructure/instance/file_name_processor.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -262,7 +262,7 @@ resource "aws_iam_policy" "filenameprocessor_dps_extended_attribute_kms_policy"
Resource = "arn:aws:kms:${var.aws_region}:${var.dspp_core_account_id}:key/*",
"Condition" = {
"ForAnyValue:StringEquals" = {
"kms:ResourceAliases" = "alias/${var.dspp_submission_kms_key_alias}"
"kms:ResourceAliases" = var.dspp_submission_kms_key_alias
}
}
}
Expand Down Expand Up @@ -316,17 +316,18 @@ resource "aws_lambda_function" "file_processor_lambda" {

environment {
variables = {
ACCOUNT_ID = var.immunisation_account_id
DPS_ACCOUNT_ID = var.dspp_core_account_id
SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket
ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket
DPS_BUCKET_NAME = var.dspp_submission_s3_bucket_name
QUEUE_URL = aws_sqs_queue.batch_file_created.url
REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address
REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port
SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name
AUDIT_TABLE_NAME = aws_dynamodb_table.audit-table.name
AUDIT_TABLE_TTL_DAYS = 60
ACCOUNT_ID = var.immunisation_account_id
DPS_ACCOUNT_ID = var.dspp_core_account_id
SOURCE_BUCKET_NAME = aws_s3_bucket.batch_data_source_bucket.bucket
ACK_BUCKET_NAME = aws_s3_bucket.batch_data_destination_bucket.bucket
DPS_BUCKET_NAME = var.dspp_submission_s3_bucket_name
DPS_BUCKET_KMS_KEY_ARN = "arn:aws:kms:${var.aws_region}:${var.dspp_core_account_id}:${var.dspp_submission_kms_key_alias}"
QUEUE_URL = aws_sqs_queue.batch_file_created.url
REDIS_HOST = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].address
REDIS_PORT = data.aws_elasticache_cluster.existing_redis.cache_nodes[0].port
SPLUNK_FIREHOSE_NAME = module.splunk.firehose_stream_name
AUDIT_TABLE_NAME = aws_dynamodb_table.audit-table.name
AUDIT_TABLE_TTL_DAYS = 60
}
}
kms_key_arn = data.aws_kms_key.existing_lambda_encryption_key.arn
Expand Down
2 changes: 1 addition & 1 deletion infrastructure/instance/variables.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ variable "dspp_submission_s3_bucket_name" {
variable "dspp_submission_kms_key_alias" {
description = "Alias of the DSPP (DPS) KMS key required to encrypt extended attributes files"
type = string
default = "nhsd-dspp-core-ref-s3-submission-upload-key"
default = "alias/nhsd-dspp-core-ref-s3-submission-upload-key"
}

variable "create_mesh_processor" {
Expand Down
1 change: 1 addition & 0 deletions lambdas/filenameprocessor/src/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
)

DPS_DESTINATION_BUCKET_NAME = os.getenv("DPS_BUCKET_NAME")
DPS_DESTINATION_BUCKET_KMS_KEY_ARN = os.getenv("DPS_BUCKET_KMS_KEY_ARN")
EXPECTED_SOURCE_BUCKET_ACCOUNT = os.getenv("ACCOUNT_ID")
EXPECTED_DPS_DESTINATION_ACCOUNT = os.getenv("DPS_ACCOUNT_ID")
AUDIT_TABLE_TTL_DAYS = os.getenv("AUDIT_TABLE_TTL_DAYS")
Expand Down
2 changes: 2 additions & 0 deletions lambdas/filenameprocessor/src/file_name_processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from common.models.batch_constants import SOURCE_BUCKET_NAME, FileStatus
from common.models.errors import UnhandledAuditTableError
from constants import (
DPS_DESTINATION_BUCKET_KMS_KEY_ARN,
DPS_DESTINATION_BUCKET_NAME,
DPS_DESTINATION_PREFIX,
ERROR_TYPE_TO_STATUS_CODE_MAP,
Expand Down Expand Up @@ -262,6 +263,7 @@ def handle_extended_attributes_file(
dest_file_key,
EXPECTED_DPS_DESTINATION_ACCOUNT,
EXPECTED_SOURCE_BUCKET_ACCOUNT,
DPS_DESTINATION_BUCKET_KMS_KEY_ARN,
)

move_file(bucket_name, file_key, f"{EXTENDED_ATTRIBUTES_ARCHIVE_PREFIX}/{file_key}")
Expand Down
3 changes: 2 additions & 1 deletion lambdas/filenameprocessor/tests/utils_for_tests/mock_environment_variables.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ class BucketNames:
CONFIG = "immunisation-batch-internal-dev-data-configs"
SOURCE = "immunisation-batch-internal-dev-data-sources"
DESTINATION = "immunisation-batch-internal-dev-data-destinations"
DPS_DESTINATION = "nhsd-dspp-core-ref-extended-attributes-gdp"
DPS_DESTINATION = "nhsd-dspp-core-ref-s3-submission-upload"
# Mock firehose bucket used for testing only (due to limitations of the moto testing package)
MOCK_FIREHOSE = "mock-firehose-bucket"

Expand All @@ -38,6 +38,7 @@ class Sqs:
"SOURCE_BUCKET_NAME": BucketNames.SOURCE,
"ACK_BUCKET_NAME": BucketNames.DESTINATION,
"DPS_BUCKET_NAME": BucketNames.DPS_DESTINATION,
"DPS_BUCKET_KMS_KEY_ARN": "arn:aws:kms:eu-west-2:123456789012:alias/nhsd-dspp-core-ref-s3-submission-upload-key",
"ACCOUNT_ID": MOCK_ACCOUNT_ID,
"DPS_ACCOUNT_ID": MOCK_ACCOUNT_ID,
"QUEUE_URL": "https://sqs.eu-west-2.amazonaws.com/123456789012/imms-batch-file-created-queue.fifo",
Expand Down
21 changes: 14 additions & 7 deletions lambdas/shared/src/common/aws_s3_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,15 +22,22 @@ def copy_file_to_external_bucket(
destination_key: str,
expected_bucket_owner: str,
expected_source_bucket_owner: str,
sse_kms_key_id: str | None = None,
) -> None:
copy_params = {
"CopySource": {"Bucket": source_bucket, "Key": source_key},
"Bucket": destination_bucket,
"Key": destination_key,
"ExpectedBucketOwner": expected_bucket_owner,
"ExpectedSourceBucketOwner": expected_source_bucket_owner,
}

if sse_kms_key_id:
copy_params["ServerSideEncryption"] = "aws:kms"
copy_params["SSEKMSKeyId"] = sse_kms_key_id

s3_client = get_s3_client()
s3_client.copy_object(
CopySource={"Bucket": source_bucket, "Key": source_key},
Bucket=destination_bucket,
Key=destination_key,
ExpectedBucketOwner=expected_bucket_owner,
ExpectedSourceBucketOwner=expected_source_bucket_owner,
)
s3_client.copy_object(**copy_params)


def delete_file(
Expand Down
Loading