From e61d3bd7a8f5ca3a7f15176a206b2c9443ffb0d4 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 17 Dec 2025 13:36:25 +0000 Subject: [PATCH 01/60] [PRMP-1054] Create report_orchestration --- .../base-lambdas-reusable-deploy-all.yml | 14 ++++++ .../handlers/report_orchestration_handler.py | 40 +++++++++++++++ .../reporting/reporting_dynamo_repository.py | 32 ++++++++++++ .../reporting/excel_report_generator.py | 46 +++++++++++++++++ .../reporting/report_orchestration_service.py | 49 +++++++++++++++++++ 5 files changed, 181 insertions(+) create mode 100644 lambdas/handlers/report_orchestration_handler.py create mode 100644 lambdas/repositories/reporting/reporting_dynamo_repository.py create mode 100644 lambdas/services/reporting/excel_report_generator.py create mode 100644 lambdas/services/reporting/report_orchestration_service.py diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index fe1faa70cb..1b9edf7299 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -766,3 +766,17 @@ jobs: lambda_layer_names: "core_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} + + deploy_report_orchestration_lambda: + name: Deploy Search Document Review + uses: ./.github/workflows/base-lambdas-reusable-deploy.yml + with: + environment: ${{ inputs.environment }} + python_version: ${{ inputs.python_version }} + build_branch: ${{ inputs.build_branch }} + sandbox: ${{ inputs.sandbox }} + lambda_handler_name: report_orchestration_handler + lambda_aws_name: reportOrchestration + lambda_layer_names: "core_lambda_layer" + secrets: + AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py new file mode 100644 index 0000000000..bf3743a218 --- /dev/null +++ b/lambdas/handlers/report_orchestration_handler.py @@ -0,0 +1,40 @@ +import os +from datetime import datetime, timedelta, timezone + +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from services.reporting.excel_report_generator import ExcelReportGenerator +from services.reporting.report_orchestration_service import ReportOrchestrationService + + +def _calculate_reporting_window(): + now = datetime.now(timezone.utc) + today_7am = now.replace(hour=7, minute=0, second=0, microsecond=0) + + if now < today_7am: + today_7am -= timedelta(days=1) + + yesterday_7am = today_7am - timedelta(days=1) + + return ( + int(yesterday_7am.timestamp()), + int(today_7am.timestamp()), + ) + + +def lambda_handler(event, context): + table_name = os.getenv("REPORTING_DYNAMODB_TABLE") + + repository = ReportingDynamoRepository(table_name) + excel_generator = ExcelReportGenerator() + + service = ReportOrchestrationService( + repository=repository, + excel_generator=excel_generator, + ) + + window_start, window_end = _calculate_reporting_window() + + service.process_reporting_window( + window_start_ts=window_start, + window_end_ts=window_end, + ) diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py new file mode 100644 index 0000000000..90424dd0bf --- /dev/null +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -0,0 +1,32 @@ +from datetime import datetime, timezone +from services.base.dynamo_service import DynamoDBService + +class ReportingDynamoRepository: + def __init__(self, table_name: str): + self.table_name = table_name + self.dynamo_service = DynamoDBService() + + def get_records_for_time_window( + self, + start_timestamp: int, + end_timestamp: int, + ) -> list[dict]: + filter_expression = ( + "#ts BETWEEN :start AND :end" + ) + + expression_attribute_names = { + "#ts": "Timestamp" + } + + expression_attribute_values = { + ":start": start_timestamp, + ":end": end_timestamp, + } + + return self.dynamo_service.scan_whole_table( + table_name=self.table_name, + filter_expression=filter_expression, + expression_attribute_names=expression_attribute_names, + expression_attribute_values=expression_attribute_values, + ) diff --git a/lambdas/services/reporting/excel_report_generator.py b/lambdas/services/reporting/excel_report_generator.py new file mode 100644 index 0000000000..7dd3346d68 --- /dev/null +++ b/lambdas/services/reporting/excel_report_generator.py @@ -0,0 +1,46 @@ +from openpyxl import Workbook +from datetime import datetime + + +class ExcelReportGenerator: + def create_report_orchestration_xlsx( + self, + ods_code: str, + records: list[dict], + output_path: str, + ) -> str: + wb = Workbook() + ws = wb.active + ws.title = "Daily Upload Report" + + # Report metadata + ws.append([f"ODS Code: {ods_code}"]) + ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) + ws.append([]) + + # Header row + ws.append([ + "ID", + "Date", + "NHS Number", + "Uploader ODS", + "PDS ODS", + "Upload Status", + "Reason", + "File Path", + ]) + + for record in records: + ws.append([ + record.get("ID"), + record.get("Date"), + record.get("NhsNumber"), + record.get("UploaderOdsCode"), + record.get("PdsOdsCode"), + record.get("UploadStatus"), + record.get("Reason"), + record.get("FilePath"), + ]) + + wb.save(output_path) + return output_path diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py new file mode 100644 index 0000000000..4a19e13247 --- /dev/null +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -0,0 +1,49 @@ +import tempfile +from collections import defaultdict + +class ReportOrchestrationService: + def __init__( + self, + repository, + excel_generator, + ): + self.repository = repository + self.excel_generator = excel_generator + + def process_reporting_window( + self, + window_start_ts: int, + window_end_ts: int, + ): + records = self.repository.get_records_for_time_window( + window_start_ts, + window_end_ts, + ) + + records_by_ods = self.group_records_by_ods(records) + + for ods_code, ods_records in records_by_ods.items(): + self.generate_ods_report(ods_code, ods_records) + + @staticmethod + def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: + grouped = defaultdict(list) + for record in records: + ods_code = record.get("UploaderOdsCode") or "UNKNOWN" + grouped[ods_code].append(record) + return grouped + + def generate_ods_report( + self, + ods_code: str, + records: list[dict], + ): + with tempfile.NamedTemporaryFile( + suffix=f"_{ods_code}.xlsx", + delete=False, + ) as tmp: + self.excel_generator.create_report_orchestration_xlsx( + ods_code=ods_code, + records=records, + output_path=tmp.name, + ) From 3d9a2915cce25894984adf5de12a61813e9ac6bd Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 17 Dec 2025 13:42:35 +0000 Subject: [PATCH 02/60] [PRMP-1054] Updated variable name --- lambdas/handlers/report_orchestration_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index bf3743a218..51431deca3 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -22,7 +22,7 @@ def _calculate_reporting_window(): def lambda_handler(event, context): - table_name = os.getenv("REPORTING_DYNAMODB_TABLE") + table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") repository = ReportingDynamoRepository(table_name) excel_generator = ExcelReportGenerator() From 528efe0422347326bc9fe8b5dfac0d1d44398509 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 17 Dec 2025 14:24:37 +0000 Subject: [PATCH 03/60] [PRMP-1054] Updated method name --- lambdas/handlers/report_orchestration_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 51431deca3..c01e82f4b5 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -6,7 +6,7 @@ from services.reporting.report_orchestration_service import ReportOrchestrationService -def _calculate_reporting_window(): +def calculate_reporting_window(): now = datetime.now(timezone.utc) today_7am = now.replace(hour=7, minute=0, second=0, microsecond=0) @@ -32,7 +32,7 @@ def lambda_handler(event, context): excel_generator=excel_generator, ) - window_start, window_end = _calculate_reporting_window() + window_start, window_end = calculate_reporting_window() service.process_reporting_window( window_start_ts=window_start, From 7c5548103a2caae7b1c1d4038d530be57e6d0e2e Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 17 Dec 2025 15:44:03 +0000 Subject: [PATCH 04/60] [PRMP-1054] Created tests and formated code --- .../handlers/report_orchestration_handler.py | 6 +- .../reporting/reporting_dynamo_repository.py | 20 ++- .../reporting/excel_report_generator.py | 46 ------ .../excel_report_generator_service.py | 58 +++++++ .../reporting/report_orchestration_service.py | 12 ++ .../test_report_orchestration_handler.py | 66 ++++++++ .../test_reporting_dynamo_repository.py | 53 ++++++ .../test_excel_report_generator_service.py | 154 ++++++++++++++++++ .../test_report_orchestration_service.py | 107 ++++++++++++ 9 files changed, 470 insertions(+), 52 deletions(-) delete mode 100644 lambdas/services/reporting/excel_report_generator.py create mode 100644 lambdas/services/reporting/excel_report_generator_service.py create mode 100644 lambdas/tests/unit/handlers/test_report_orchestration_handler.py create mode 100644 lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py create mode 100644 lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py create mode 100644 lambdas/tests/unit/services/reporting/test_report_orchestration_service.py diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index c01e82f4b5..e8e16d5e52 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -2,8 +2,11 @@ from datetime import datetime, timedelta, timezone from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository -from services.reporting.excel_report_generator import ExcelReportGenerator +from services.reporting.excel_report_generator_service import ExcelReportGenerator from services.reporting.report_orchestration_service import ReportOrchestrationService +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) def calculate_reporting_window(): @@ -22,6 +25,7 @@ def calculate_reporting_window(): def lambda_handler(event, context): + logger.info("Report orchestration lambda invoked") table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") repository = ReportingDynamoRepository(table_name) diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 90424dd0bf..1233a0a4e1 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,5 +1,10 @@ -from datetime import datetime, timezone +from typing import Dict, List + from services.base.dynamo_service import DynamoDBService +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + class ReportingDynamoRepository: def __init__(self, table_name: str): @@ -10,13 +15,18 @@ def get_records_for_time_window( self, start_timestamp: int, end_timestamp: int, - ) -> list[dict]: - filter_expression = ( - "#ts BETWEEN :start AND :end" + ) -> List[Dict]: + logger.info( + f"Querying reporting table for window, " + f"table_name: {self.table_name}," + f"start_timestamp: {start_timestamp}," + f"end_timestamp: {end_timestamp}", ) + filter_expression = "#ts BETWEEN :start AND :end" + expression_attribute_names = { - "#ts": "Timestamp" + "#ts": "Timestamp", } expression_attribute_values = { diff --git a/lambdas/services/reporting/excel_report_generator.py b/lambdas/services/reporting/excel_report_generator.py deleted file mode 100644 index 7dd3346d68..0000000000 --- a/lambdas/services/reporting/excel_report_generator.py +++ /dev/null @@ -1,46 +0,0 @@ -from openpyxl import Workbook -from datetime import datetime - - -class ExcelReportGenerator: - def create_report_orchestration_xlsx( - self, - ods_code: str, - records: list[dict], - output_path: str, - ) -> str: - wb = Workbook() - ws = wb.active - ws.title = "Daily Upload Report" - - # Report metadata - ws.append([f"ODS Code: {ods_code}"]) - ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) - ws.append([]) - - # Header row - ws.append([ - "ID", - "Date", - "NHS Number", - "Uploader ODS", - "PDS ODS", - "Upload Status", - "Reason", - "File Path", - ]) - - for record in records: - ws.append([ - record.get("ID"), - record.get("Date"), - record.get("NhsNumber"), - record.get("UploaderOdsCode"), - record.get("PdsOdsCode"), - record.get("UploadStatus"), - record.get("Reason"), - record.get("FilePath"), - ]) - - wb.save(output_path) - return output_path diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py new file mode 100644 index 0000000000..36c6c09466 --- /dev/null +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -0,0 +1,58 @@ +from datetime import datetime + +from openpyxl import Workbook +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + + +class ExcelReportGenerator: + def create_report_orchestration_xlsx( + self, + ods_code: str, + records: list[dict], + output_path: str, + ) -> str: + logger.info( + f"Creating Excel report for ods code {ods_code} and records {records}" + ) + wb = Workbook() + ws = wb.active + ws.title = "Daily Upload Report" + + # Report metadata + ws.append([f"ODS Code: {ods_code}"]) + ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) + ws.append([]) + + # Header row + ws.append( + [ + "ID", + "Date", + "NHS Number", + "Uploader ODS", + "PDS ODS", + "Upload Status", + "Reason", + "File Path", + ] + ) + + for record in records: + ws.append( + [ + record.get("ID"), + record.get("Date"), + record.get("NhsNumber"), + record.get("UploaderOdsCode"), + record.get("PdsOdsCode"), + record.get("UploadStatus"), + record.get("Reason"), + record.get("FilePath"), + ] + ) + + wb.save(output_path) + logger.info(f"Excel report written successfully for for ods code {ods_code}") + return output_path diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index 4a19e13247..e6528d1bcb 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -1,6 +1,11 @@ import tempfile from collections import defaultdict +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + + class ReportOrchestrationService: def __init__( self, @@ -19,11 +24,18 @@ def process_reporting_window( window_start_ts, window_end_ts, ) + if not records: + logger.info("No records found for reporting window") + return records_by_ods = self.group_records_by_ods(records) for ods_code, ods_records in records_by_ods.items(): + logger.info( + f"Generating report for ODS ods_code = {ods_code} record_count = len(ods_records)" + ) self.generate_ods_report(ods_code, ods_records) + logger.info("Report orchestration completed") @staticmethod def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py new file mode 100644 index 0000000000..59fe6b3c35 --- /dev/null +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -0,0 +1,66 @@ +from unittest.mock import MagicMock + +import pytest +from handlers.report_orchestration_handler import lambda_handler + + +@pytest.fixture(autouse=True) +def mock_env(monkeypatch): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + + +@pytest.fixture +def mock_logger(mocker): + return mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) + + +@pytest.fixture +def mock_repo(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.ReportingDynamoRepository", autospec=True + ) + + +@pytest.fixture +def mock_excel_generator(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.ExcelReportGenerator", autospec=True + ) + + +@pytest.fixture +def mock_service(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.ReportOrchestrationService", + autospec=True, + ) + + +@pytest.fixture +def mock_window(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.calculate_reporting_window", + return_value=(100, 200), + ) + + +def test_lambda_handler_calls_service( + mock_logger, mock_repo, mock_excel_generator, mock_service, mock_window +): + lambda_handler(event={}, context={}) + + mock_repo.assert_called_once_with("TestTable") + mock_excel_generator.assert_called_once_with() + + mock_service.assert_called_once() + instance = mock_service.return_value + instance.process_reporting_window.assert_called_once_with( + window_start_ts=100, window_end_ts=200 + ) + + mock_logger.info.assert_called_with("Report orchestration lambda invoked") + + +def test_lambda_handler_calls_window_function(mock_service, mock_window): + lambda_handler(event={}, context={}) + mock_window.assert_called_once() diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py new file mode 100644 index 0000000000..e88d98a145 --- /dev/null +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -0,0 +1,53 @@ +from unittest.mock import MagicMock + +import pytest +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository + + +@pytest.fixture +def mock_dynamo_service(mocker): + mock_service = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.DynamoDBService" + ) + instance = mock_service.return_value + instance.scan_whole_table = MagicMock() + return instance + + +@pytest.fixture +def reporting_repo(mock_dynamo_service): + return ReportingDynamoRepository(table_name="TestTable") + + +def test_get_records_for_time_window_calls_scan(mock_dynamo_service, reporting_repo): + start_ts = 100 + end_ts = 200 + expected_result = [ + {"ID": 1, "Timestamp": 150}, + {"ID": 2, "Timestamp": 180}, + ] + mock_dynamo_service.scan_whole_table.return_value = expected_result + + result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + + mock_dynamo_service.scan_whole_table.assert_called_once_with( + table_name="TestTable", + filter_expression="#ts BETWEEN :start AND :end", + expression_attribute_names={"#ts": "Timestamp"}, + expression_attribute_values={":start": start_ts, ":end": end_ts}, + ) + + assert result == expected_result + + +def test_get_records_for_time_window_returns_empty_list( + mock_dynamo_service, reporting_repo +): + start_ts = 0 + end_ts = 50 + mock_dynamo_service.scan_whole_table.return_value = [] + + result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + + assert result == [] + mock_dynamo_service.scan_whole_table.assert_called_once() diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py new file mode 100644 index 0000000000..f16257c0fa --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -0,0 +1,154 @@ + +import pytest +from freezegun import freeze_time +from openpyxl import load_workbook +from services.reporting.excel_report_generator_service import ExcelReportGenerator + + +@pytest.fixture +def excel_report_generator(): + return ExcelReportGenerator() + + +@freeze_time("2025-01-01T12:00:00") +def test_create_report_orchestration_xlsx_happy_path( + excel_report_generator, + tmp_path, +): + output_file = tmp_path / "report.xlsx" + + ods_code = "Y12345" + records = [ + { + "ID": 1, + "Date": "2025-01-01", + "NhsNumber": "1234567890", + "UploaderOdsCode": "Y12345", + "PdsOdsCode": "A99999", + "UploadStatus": "SUCCESS", + "Reason": "", + "FilePath": "/path/file1.pdf", + }, + { + "ID": 2, + "Date": "2025-01-02", + "NhsNumber": "123456789", + "UploaderOdsCode": "Y12345", + "PdsOdsCode": "B88888", + "UploadStatus": "FAILED", + "Reason": "Invalid NHS number", + "FilePath": "/path/file2.pdf", + }, + ] + + result = excel_report_generator.create_report_orchestration_xlsx( + ods_code=ods_code, + records=records, + output_path=str(output_file), + ) + + # File path returned + assert result == str(output_file) + assert output_file.exists() + + wb = load_workbook(output_file) + ws = wb.active + + # Sheet name + assert ws.title == "Daily Upload Report" + + # Metadata rows + assert ws["A1"].value == f"ODS Code: {ods_code}" + assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" + assert ws["A3"].value is None # blank row + + # Header row + assert [cell.value for cell in ws[4]] == [ + "ID", + "Date", + "NHS Number", + "Uploader ODS", + "PDS ODS", + "Upload Status", + "Reason", + "File Path", + ] + + # First data row + assert [cell.value for cell in ws[5]] == [ + 1, + "2025-01-01", + "1234567890", + "Y12345", + "A99999", + "SUCCESS", + None, + "/path/file1.pdf", + ] + + # Second data row + assert [cell.value for cell in ws[6]] == [ + 2, + "2025-01-02", + "123456789", + "Y12345", + "B88888", + "FAILED", + "Invalid NHS number", + "/path/file2.pdf", + ] + + +def test_create_report_orchestration_xlsx_with_no_records( + excel_report_generator, + tmp_path, +): + output_file = tmp_path / "empty_report.xlsx" + + excel_report_generator.create_report_orchestration_xlsx( + ods_code="Y12345", + records=[], + output_path=str(output_file), + ) + + wb = load_workbook(output_file) + ws = wb.active + + # Only metadata + header rows should exist + assert ws.max_row == 4 + + +def test_create_report_orchestration_xlsx_handles_missing_fields( + excel_report_generator, + tmp_path, +): + output_file = tmp_path / "partial.xlsx" + + records = [ + { + "ID": 1, + "NhsNumber": "1234567890", + } + ] + + excel_report_generator.create_report_orchestration_xlsx( + ods_code="Y12345", + records=records, + output_path=str(output_file), + ) + + wb = load_workbook(output_file) + ws = wb.active + + row = [cell.value for cell in ws[5]] + + assert row == [ + 1, + None, + "1234567890", + None, + None, + None, + None, + None, + ] diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py new file mode 100644 index 0000000000..c3b21a77ef --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -0,0 +1,107 @@ +import pytest +from services.reporting.report_orchestration_service import ReportOrchestrationService + + +@pytest.fixture +def mock_repository(mocker): + repo = mocker.Mock() + repo.get_records_for_time_window.return_value = [] + return repo + + +@pytest.fixture +def mock_excel_generator(mocker): + return mocker.Mock() + + +@pytest.fixture +def report_orchestration_service(mock_repository, mock_excel_generator): + return ReportOrchestrationService( + repository=mock_repository, + excel_generator=mock_excel_generator, + ) + + +def test_process_reporting_window_no_records( + report_orchestration_service, mock_repository, mock_excel_generator +): + mock_repository.get_records_for_time_window.return_value = [] + + report_orchestration_service.process_reporting_window(100, 200) + + mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() + + +def test_group_records_by_ods_groups_correctly(): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + {"ID": 4}, # missing ODS + {"UploaderOdsCode": None, "ID": 5}, # null ODS + ] + + result = ReportOrchestrationService.group_records_by_ods(records) + + assert result["Y12345"] == [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ] + assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] + assert result["UNKNOWN"] == [ + {"ID": 4}, + {"UploaderOdsCode": None, "ID": 5}, + ] + + +def test_process_reporting_window_generates_reports_per_ods( + report_orchestration_service, mock_repository, mocker +): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + ] + mock_repository.get_records_for_time_window.return_value = records + + mocked_generate = mocker.patch.object( + report_orchestration_service, "generate_ods_report" + ) + + report_orchestration_service.process_reporting_window(100, 200) + + mocked_generate.assert_any_call( + "Y12345", + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ], + ) + mocked_generate.assert_any_call( + "A99999", + [{"UploaderOdsCode": "A99999", "ID": 3}], + ) + assert mocked_generate.call_count == 2 + + +def test_generate_ods_report_creates_excel_report( + report_orchestration_service, mock_excel_generator, mocker +): + fake_tmp = mocker.MagicMock() + fake_tmp.__enter__.return_value = fake_tmp + fake_tmp.name = "/tmp/fake_Y12345.xlsx" + + mocker.patch( + "services.reporting.report_orchestration_service.tempfile.NamedTemporaryFile", + return_value=fake_tmp, + ) + + records = [{"ID": 1, "UploaderOdsCode": "Y12345"}] + + report_orchestration_service.generate_ods_report("Y12345", records) + + mock_excel_generator.create_report_orchestration_xlsx.assert_called_once_with( + ods_code="Y12345", + records=records, + output_path=fake_tmp.name, + ) From dfbff8da57eb321e20bf39f5d70ed704e871f70a Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 18 Dec 2025 11:17:00 +0000 Subject: [PATCH 05/60] [PRMP-1054] Small updates to allow library to be imported and used --- lambdas/handlers/report_orchestration_handler.py | 14 +++++++++++++- .../reporting/excel_report_generator_service.py | 4 ++-- .../reporting/report_orchestration_service.py | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index e8e16d5e52..84f6372ab8 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,10 +1,15 @@ import os +import tempfile from datetime import datetime, timedelta, timezone from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.reporting.excel_report_generator_service import ExcelReportGenerator from services.reporting.report_orchestration_service import ReportOrchestrationService from utils.audit_logging_setup import LoggingService +from utils.decorators.ensure_env_var import ensure_environment_variables +from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions +from utils.decorators.override_error_check import override_error_check +from utils.decorators.set_audit_arg import set_request_context_for_logging logger = LoggingService(__name__) @@ -23,7 +28,12 @@ def calculate_reporting_window(): int(today_7am.timestamp()), ) - +@ensure_environment_variables( + names=["BULK_UPLOAD_REPORT_TABLE_NAME"] +) +@override_error_check +@handle_lambda_exceptions +@set_request_context_for_logging def lambda_handler(event, context): logger.info("Report orchestration lambda invoked") table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") @@ -37,8 +47,10 @@ def lambda_handler(event, context): ) window_start, window_end = calculate_reporting_window() + tmp_dir = tempfile.mkdtemp() service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, + output_dir=tmp_dir, ) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index 36c6c09466..2f8286a569 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -1,6 +1,6 @@ from datetime import datetime -from openpyxl import Workbook +from openpyxl.workbook import Workbook from utils.audit_logging_setup import LoggingService logger = LoggingService(__name__) @@ -14,7 +14,7 @@ def create_report_orchestration_xlsx( output_path: str, ) -> str: logger.info( - f"Creating Excel report for ods code {ods_code} and records {records}" + f"Creating Excel report for ods code {ods_code} and records {len(records)}" ) wb = Workbook() ws = wb.active diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index e6528d1bcb..5e6e7580cc 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -19,6 +19,7 @@ def process_reporting_window( self, window_start_ts: int, window_end_ts: int, + output_dir: str, ): records = self.repository.get_records_for_time_window( window_start_ts, From f69afbf54341cc064c31576d90ea827cc6c20e81 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 18 Dec 2025 11:26:22 +0000 Subject: [PATCH 06/60] [PRMP-1054] fixed unit tests --- .../test_report_orchestration_handler.py | 22 +++++++++++++------ .../test_report_orchestration_service.py | 4 ++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 59fe6b3c35..c9ee2ff849 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,9 +1,13 @@ +from unittest import mock from unittest.mock import MagicMock - import pytest from handlers.report_orchestration_handler import lambda_handler +class FakeContext: + aws_request_id = "test-request-id" + + @pytest.fixture(autouse=True) def mock_env(monkeypatch): monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") @@ -17,14 +21,16 @@ def mock_logger(mocker): @pytest.fixture def mock_repo(mocker): return mocker.patch( - "handlers.report_orchestration_handler.ReportingDynamoRepository", autospec=True + "handlers.report_orchestration_handler.ReportingDynamoRepository", + autospec=True, ) @pytest.fixture def mock_excel_generator(mocker): return mocker.patch( - "handlers.report_orchestration_handler.ExcelReportGenerator", autospec=True + "handlers.report_orchestration_handler.ExcelReportGenerator", + autospec=True, ) @@ -47,7 +53,7 @@ def mock_window(mocker): def test_lambda_handler_calls_service( mock_logger, mock_repo, mock_excel_generator, mock_service, mock_window ): - lambda_handler(event={}, context={}) + lambda_handler(event={}, context=FakeContext()) mock_repo.assert_called_once_with("TestTable") mock_excel_generator.assert_called_once_with() @@ -55,12 +61,14 @@ def test_lambda_handler_calls_service( mock_service.assert_called_once() instance = mock_service.return_value instance.process_reporting_window.assert_called_once_with( - window_start_ts=100, window_end_ts=200 + window_start_ts=100, + window_end_ts=200, + output_dir=mock.ANY, ) - mock_logger.info.assert_called_with("Report orchestration lambda invoked") + mock_logger.info.assert_any_call("Report orchestration lambda invoked") def test_lambda_handler_calls_window_function(mock_service, mock_window): - lambda_handler(event={}, context={}) + lambda_handler(event={}, context=FakeContext()) mock_window.assert_called_once() diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c3b21a77ef..c068ba0c16 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -27,7 +27,7 @@ def test_process_reporting_window_no_records( ): mock_repository.get_records_for_time_window.return_value = [] - report_orchestration_service.process_reporting_window(100, 200) + report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() @@ -68,7 +68,7 @@ def test_process_reporting_window_generates_reports_per_ods( report_orchestration_service, "generate_ods_report" ) - report_orchestration_service.process_reporting_window(100, 200) + report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") mocked_generate.assert_any_call( "Y12345", From 326a6c888fe92a1f04f6985c8e0417ba831f6077 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 19 Dec 2025 10:41:26 +0000 Subject: [PATCH 07/60] [PRMP-1054] added lambda layer to report_orchestration_lambda --- .github/workflows/base-lambdas-reusable-deploy-all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index 1b9edf7299..16801885bd 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -777,6 +777,6 @@ jobs: sandbox: ${{ inputs.sandbox }} lambda_handler_name: report_orchestration_handler lambda_aws_name: reportOrchestration - lambda_layer_names: "core_lambda_layer" + lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} From 31fc5e4de83b521572d6a1efffb3499c8ce1ebe7 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 19 Dec 2025 13:49:39 +0000 Subject: [PATCH 08/60] [PRMP-1054] fixed how the search works --- .../reporting/reporting_dynamo_repository.py | 21 +++++++------------ .../reporting/report_orchestration_service.py | 2 +- .../test_reporting_dynamo_repository.py | 20 ++++-------------- 3 files changed, 12 insertions(+), 31 deletions(-) diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 1233a0a4e1..7a694edf69 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,5 +1,6 @@ from typing import Dict, List +from boto3.dynamodb.conditions import Attr from services.base.dynamo_service import DynamoDBService from utils.audit_logging_setup import LoggingService @@ -18,25 +19,17 @@ def get_records_for_time_window( ) -> List[Dict]: logger.info( f"Querying reporting table for window, " - f"table_name: {self.table_name}," - f"start_timestamp: {start_timestamp}," + f"table_name: {self.table_name}, " + f"start_timestamp: {start_timestamp}, " f"end_timestamp: {end_timestamp}", ) - filter_expression = "#ts BETWEEN :start AND :end" - - expression_attribute_names = { - "#ts": "Timestamp", - } - - expression_attribute_values = { - ":start": start_timestamp, - ":end": end_timestamp, - } + filter_expression = Attr("Timestamp").between( + start_timestamp, + end_timestamp, + ) return self.dynamo_service.scan_whole_table( table_name=self.table_name, filter_expression=filter_expression, - expression_attribute_names=expression_attribute_names, - expression_attribute_values=expression_attribute_values, ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index 5e6e7580cc..a9500735b9 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -33,7 +33,7 @@ def process_reporting_window( for ods_code, ods_records in records_by_ods.items(): logger.info( - f"Generating report for ODS ods_code = {ods_code} record_count = len(ods_records)" + f"Generating report for ODS ods_code = {ods_code} record_count = {len(ods_records)}" ) self.generate_ods_report(ods_code, ods_records) logger.info("Report orchestration completed") diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index e88d98a145..df3c553044 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -20,24 +20,12 @@ def reporting_repo(mock_dynamo_service): def test_get_records_for_time_window_calls_scan(mock_dynamo_service, reporting_repo): - start_ts = 100 - end_ts = 200 - expected_result = [ - {"ID": 1, "Timestamp": 150}, - {"ID": 2, "Timestamp": 180}, - ] - mock_dynamo_service.scan_whole_table.return_value = expected_result + mock_dynamo_service.scan_whole_table.return_value = [] - result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + reporting_repo.get_records_for_time_window(100, 200) - mock_dynamo_service.scan_whole_table.assert_called_once_with( - table_name="TestTable", - filter_expression="#ts BETWEEN :start AND :end", - expression_attribute_names={"#ts": "Timestamp"}, - expression_attribute_values={":start": start_ts, ":end": end_ts}, - ) - - assert result == expected_result + mock_dynamo_service.scan_whole_table.assert_called_once() + assert "filter_expression" in mock_dynamo_service.scan_whole_table.call_args.kwargs def test_get_records_for_time_window_returns_empty_list( From d24739468cea9d7410bc69372ed0c599e57351d4 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 19 Dec 2025 16:21:52 +0000 Subject: [PATCH 09/60] [PRMP-1054] fixed comment --- lambdas/services/reporting/excel_report_generator_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index 2f8286a569..b8bef9a5cf 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -14,7 +14,7 @@ def create_report_orchestration_xlsx( output_path: str, ) -> str: logger.info( - f"Creating Excel report for ods code {ods_code} and records {len(records)}" + f"Creating Excel report for ODS code {ods_code} and records {len(records)}" ) wb = Workbook() ws = wb.active From 25fda598603656d13b0e109fb90a680a9fa0760f Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 5 Jan 2026 14:27:05 +0000 Subject: [PATCH 10/60] [PRMP-1057] created distribution lambda --- .../base-lambdas-reusable-deploy-all.yml | 14 + .../handlers/report_distribution_handler.py | 50 +++ .../handlers/report_orchestration_handler.py | 68 +++- .../reporting/report_contact_repository.py | 16 + .../requirements_reports_lambda_layer.txt | 3 +- lambdas/services/email_service.py | 104 +++++++ .../reporting/report_distribution_service.py | 137 ++++++++ .../reporting/report_orchestration_service.py | 30 +- .../test_report_distribution_handler.py | 89 ++++++ .../test_report_orchestration_handler.py | 164 +++++++++- .../test_report_contact_repository.py | 59 ++++ .../services/reporting/test_email_service.py | 154 +++++++++ .../test_report_distribution_service.py | 293 ++++++++++++++++++ .../test_report_orchestration_service.py | 136 ++++++-- lambdas/utils/zip_utils.py | 20 ++ 15 files changed, 1276 insertions(+), 61 deletions(-) create mode 100644 lambdas/handlers/report_distribution_handler.py create mode 100644 lambdas/repositories/reporting/report_contact_repository.py create mode 100644 lambdas/services/email_service.py create mode 100644 lambdas/services/reporting/report_distribution_service.py create mode 100644 lambdas/tests/unit/handlers/test_report_distribution_handler.py create mode 100644 lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py create mode 100644 lambdas/tests/unit/services/reporting/test_email_service.py create mode 100644 lambdas/tests/unit/services/reporting/test_report_distribution_service.py create mode 100644 lambdas/utils/zip_utils.py diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index e5bd7798d3..5128113a49 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -810,3 +810,17 @@ jobs: lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} + + deploy_report_distribution_lambda: + name: Deploy Report Distribution + uses: ./.github/workflows/base-lambdas-reusable-deploy.yml + with: + environment: ${{ inputs.environment }} + python_version: ${{ inputs.python_version }} + build_branch: ${{ inputs.build_branch }} + sandbox: ${{ inputs.sandbox }} + lambda_handler_name: report_distribution_handler + lambda_aws_name: reportDistribution + lambda_layer_names: "core_lambda_layer,reports_lambda_layer" + secrets: + AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py new file mode 100644 index 0000000000..9df15879d1 --- /dev/null +++ b/lambdas/handlers/report_distribution_handler.py @@ -0,0 +1,50 @@ +import os + +from repositories.reporting.report_contact_repository import ReportContactRepository +from services.base.s3_service import S3Service +from services.email_service import EmailService +from services.reporting.report_distribution_service import ReportDistributionService +from utils.audit_logging_setup import LoggingService +from utils.decorators.ensure_env_var import ensure_environment_variables +from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions +from utils.decorators.override_error_check import override_error_check +from utils.decorators.set_audit_arg import set_request_context_for_logging + +logger = LoggingService(__name__) + + +@ensure_environment_variables( + names=[ + "REPORT_BUCKET_NAME", + "CONTACT_TABLE_NAME", + "PRM_MAILBOX_EMAIL", + "SES_FROM_ADDRESS", + ] +) +@override_error_check +@handle_lambda_exceptions +@set_request_context_for_logging +def lambda_handler(event, context): + report_date = event["report_date"] + + bucket = os.environ["REPORT_BUCKET_NAME"] + contact_table = os.environ["CONTACT_TABLE_NAME"] + prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] + from_address = os.environ["SES_FROM_ADDRESS"] + + s3_service = S3Service() + contact_repo = ReportContactRepository(contact_table) + email_service = EmailService() + + service = ReportDistributionService( + s3_service=s3_service, + contact_repo=contact_repo, + email_service=email_service, + bucket=bucket, + from_address=from_address, + prm_mailbox=prm_mailbox, + ) + + result = service.distribute_reports_for_date(report_date) + logger.info(f"Daily Report distribution summary: {result}") + return result diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 84f6372ab8..6dc70db306 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,8 +1,11 @@ +import json import os -import tempfile from datetime import datetime, timedelta, timezone +import boto3 + from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from services.base.s3_service import S3Service from services.reporting.excel_report_generator_service import ExcelReportGenerator from services.reporting.report_orchestration_service import ReportOrchestrationService from utils.audit_logging_setup import LoggingService @@ -13,6 +16,8 @@ logger = LoggingService(__name__) +def _get_lambda_client(): + return boto3.client("lambda") def calculate_reporting_window(): now = datetime.now(timezone.utc) @@ -23,23 +28,38 @@ def calculate_reporting_window(): yesterday_7am = today_7am - timedelta(days=1) - return ( - int(yesterday_7am.timestamp()), - int(today_7am.timestamp()), - ) + return int(yesterday_7am.timestamp()), int(today_7am.timestamp()) + + +def get_report_date_folder() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d") + + +def build_s3_key(ods_code: str, report_date: str) -> str: + return f"Report-Orchestration/{report_date}/{ods_code}.xlsx" + @ensure_environment_variables( - names=["BULK_UPLOAD_REPORT_TABLE_NAME"] + names=[ + "BULK_UPLOAD_REPORT_TABLE_NAME", + "REPORT_BUCKET_NAME", + "REPORT_DISTRIBUTION_LAMBDA", + ] ) @override_error_check @handle_lambda_exceptions @set_request_context_for_logging def lambda_handler(event, context): logger.info("Report orchestration lambda invoked") - table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") + + lambda_client = _get_lambda_client() + table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] + report_bucket = os.environ["REPORT_BUCKET_NAME"] + distribution_lambda = os.environ["REPORT_DISTRIBUTION_LAMBDA"] repository = ReportingDynamoRepository(table_name) excel_generator = ExcelReportGenerator() + s3_service = S3Service() service = ReportOrchestrationService( repository=repository, @@ -47,10 +67,38 @@ def lambda_handler(event, context): ) window_start, window_end = calculate_reporting_window() - tmp_dir = tempfile.mkdtemp() + # TODO delete line below since it is only for testing + window_end += 24 * 60 * 60 + + report_date = get_report_date_folder() - service.process_reporting_window( + generated_files = service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, - output_dir=tmp_dir, ) + + if not generated_files: + logger.info("No reports generated; exiting") + return + + # Upload each generated report to S3 + for ods_code, local_path in generated_files.items(): + key = build_s3_key(ods_code, report_date) + s3_service.upload_file_with_extra_args( + file_name=local_path, + s3_bucket_name=report_bucket, + file_key=key, + extra_args={ + "ServerSideEncryption": "aws:kms", + }, + ) + logger.info(f"Uploaded report for ODS={ods_code} to s3://{report_bucket}/{key}") + + # Invoke distribution lambda once for the day folder + lambda_client.invoke( + FunctionName=distribution_lambda, + InvocationType="Event", + Payload=json.dumps({"report_date": report_date}), + ) + + logger.info(f"Invoked distribution lambda for report_date={report_date}") diff --git a/lambdas/repositories/reporting/report_contact_repository.py b/lambdas/repositories/reporting/report_contact_repository.py new file mode 100644 index 0000000000..9399e222ee --- /dev/null +++ b/lambdas/repositories/reporting/report_contact_repository.py @@ -0,0 +1,16 @@ +from services.base.dynamo_service import DynamoDBService + + +class ReportContactRepository: + def __init__(self, table_name: str): + self.table_name = table_name + self.dynamo = DynamoDBService() + + def get_contact_email(self, ods_code: str) -> str | None: + item = self.dynamo.get_item( + table_name=self.table_name, + key={"OdsCode": ods_code}, + ) + if not item: + return None + return item.get("Email") diff --git a/lambdas/requirements/layers/requirements_reports_lambda_layer.txt b/lambdas/requirements/layers/requirements_reports_lambda_layer.txt index 44709d7496..872e8ad167 100644 --- a/lambdas/requirements/layers/requirements_reports_lambda_layer.txt +++ b/lambdas/requirements/layers/requirements_reports_lambda_layer.txt @@ -1,2 +1,3 @@ openpyxl==3.1.5 -reportlab==4.3.1 \ No newline at end of file +reportlab==4.3.1 +pyzipper==0.3.6 \ No newline at end of file diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py new file mode 100644 index 0000000000..fb12e223dd --- /dev/null +++ b/lambdas/services/email_service.py @@ -0,0 +1,104 @@ +import boto3 +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.application import MIMEApplication +from typing import Iterable, Optional + +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + + +class EmailService: + """ + General email sender via SES (AWS Simple Email Service) Raw Email (supports attachments). + Higher-level methods prepare inputs and call send_email(). + """ + + def __init__(self): + self.ses = boto3.client("ses") + + def send_email( + self, + *, + to_address: str, + subject: str, + body_text: str, + from_address: str, + attachments: Optional[Iterable[str]] = None, + ): + msg = MIMEMultipart() + msg["Subject"] = subject + msg["To"] = to_address + msg["From"] = from_address + + msg.attach(MIMEText(body_text, "plain")) + + for attachment_path in attachments or []: + with open(attachment_path, "rb") as f: + part = MIMEApplication(f.read()) + part.add_header( + "Content-Disposition", + "attachment", + filename=attachment_path.split("/")[-1], + ) + msg.attach(part) + + self._send_raw(msg, to_address) + + def _send_raw(self, msg: MIMEMultipart, to_address: str): + logger.info(f"Sending SES raw email to {to_address}") + self.ses.send_raw_email( + RawMessage={"Data": msg.as_string()}, + Destinations=[to_address], + ) + + def send_report_email( + self, + *, + to_address: str, + from_address: str, + attachment_path: str, + ): + self.send_email( + to_address=to_address, + from_address=from_address, + subject="Daily Upload Report", + body_text="Please find your encrypted daily upload report attached.", + attachments=[attachment_path], + ) + + def send_password_email( + self, + *, + to_address: str, + from_address: str, + password: str, + ): + self.send_email( + to_address=to_address, + from_address=from_address, + subject="Daily Upload Report Password", + body_text=f"Password for your report:\n\n{password}", + ) + + def send_prm_missing_contact_email( + self, + *, + prm_mailbox: str, + from_address: str, + ods_code: str, + attachment_path: str, + password: str, + ): + self.send_email( + to_address=prm_mailbox, + from_address=from_address, + subject=f"Missing contact for ODS {ods_code}", + body_text=( + f"No contact found for ODS {ods_code}.\n\n" + f"Password: {password}\n\n" + f"Please resolve the contact and forward the report." + ), + attachments=[attachment_path], + ) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py new file mode 100644 index 0000000000..acaf65d0a5 --- /dev/null +++ b/lambdas/services/reporting/report_distribution_service.py @@ -0,0 +1,137 @@ +import os +import secrets +import tempfile +from typing import List, Dict + +import boto3 + +from repositories.reporting.report_contact_repository import ReportContactRepository +from services.base.s3_service import S3Service +from services.email_service import EmailService +from utils.audit_logging_setup import LoggingService +from utils.zip_utils import zip_encrypt_file + +logger = LoggingService(__name__) + + +class ReportDistributionService: + def __init__( + self, + *, + s3_service: S3Service, + contact_repo: ReportContactRepository, + email_service: EmailService, + bucket: str, + from_address: str, + prm_mailbox: str, + ): + self.s3_service = s3_service + self.contact_repo = contact_repo + self.email_service = email_service + self.bucket = bucket + self.from_address = from_address + self.prm_mailbox = prm_mailbox + self._s3_client = boto3.client("s3") + + def distribute_reports_for_date(self, report_date: str) -> Dict[str, int]: + day_prefix = f"Report-Orchestration/{report_date}/" + keys = self.list_xlsx_keys(prefix=day_prefix) + + logger.info(f"Found {len(keys)} report(s) under s3://{self.bucket}/{day_prefix}") + + succeeded = 0 + failed = 0 + + for key in keys: + ods_code = key.split("/")[-1].replace(".xlsx", "") + + try: + self.process_one_report(ods_code=ods_code, key=key) + succeeded += 1 + except Exception as e: + failed += 1 + logger.exception( + f"Failed processing report for ODS={ods_code}, key={key}: {e}" + ) + + logger.info( + f"Distribution completed for {report_date}: " + f"succeeded={succeeded}, failed={failed}" + ) + return {"succeeded": succeeded, "failed": failed} + + def list_xlsx_keys(self, prefix: str) -> List[str]: + paginator = self._s3_client.get_paginator("list_objects_v2") + keys: List[str] = [] + + for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): + for obj in page.get("Contents", []): + key = obj["Key"] + if key.endswith(".xlsx"): + keys.append(key) + + return keys + + def process_one_report(self, *, ods_code: str, key: str) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + local_xlsx = os.path.join(tmpdir, f"{ods_code}.xlsx") + local_zip = os.path.join(tmpdir, f"{ods_code}.zip") + + self.s3_service.download_file(self.bucket, key, local_xlsx) + + password = secrets.token_urlsafe(16) + zip_encrypt_file( + input_path=local_xlsx, + output_zip=local_zip, + password=password, + ) + + self.send_report_emails( + ods_code=ods_code, + attachment_path=local_zip, + password=password, + ) + + def send_report_emails(self, *, ods_code: str, attachment_path: str, password: str) -> None: + contact_email = self.contact_repo.get_contact_email(ods_code) + + if contact_email: + logger.info(f"Contact found for ODS={ods_code}, emailing {contact_email}") + self.email_contact( + to_address=contact_email, + attachment_path=attachment_path, + password=password, + ) + return + + logger.info(f"No contact found for ODS={ods_code}, sending to PRM mailbox") + self.email_prm_missing_contact( + ods_code=ods_code, + attachment_path=attachment_path, + password=password, + ) + + def email_contact(self, *, to_address: str, attachment_path: str, password: str) -> None: + # TODO delete next line since it is only for testing + to_address = "" + self.email_service.send_report_email( + to_address=to_address, + from_address=self.from_address, + attachment_path=attachment_path, + ) + self.email_service.send_password_email( + to_address=to_address, + from_address=self.from_address, + password=password, + ) + + def email_prm_missing_contact( + self, *, ods_code: str, attachment_path: str, password: str + ) -> None: + self.email_service.send_prm_missing_contact_email( + prm_mailbox=self.prm_mailbox, + from_address=self.from_address, + ods_code=ods_code, + attachment_path=attachment_path, + password=password, + ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index a9500735b9..69a919cc6e 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -1,5 +1,6 @@ import tempfile from collections import defaultdict +from typing import Dict from utils.audit_logging_setup import LoggingService @@ -7,11 +8,7 @@ class ReportOrchestrationService: - def __init__( - self, - repository, - excel_generator, - ): + def __init__(self, repository, excel_generator): self.repository = repository self.excel_generator = excel_generator @@ -19,24 +16,28 @@ def process_reporting_window( self, window_start_ts: int, window_end_ts: int, - output_dir: str, - ): + ) -> Dict[str, str]: records = self.repository.get_records_for_time_window( window_start_ts, window_end_ts, ) + if not records: logger.info("No records found for reporting window") - return + return {} records_by_ods = self.group_records_by_ods(records) + generated_files: Dict[str, str] = {} for ods_code, ods_records in records_by_ods.items(): logger.info( - f"Generating report for ODS ods_code = {ods_code} record_count = {len(ods_records)}" + f"Generating report for ODS={ods_code}, records={len(ods_records)}" ) - self.generate_ods_report(ods_code, ods_records) - logger.info("Report orchestration completed") + file_path = self.generate_ods_report(ods_code, ods_records) + generated_files[ods_code] = file_path + + logger.info(f"Generated {len(generated_files)} report(s)") + return generated_files @staticmethod def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: @@ -46,11 +47,7 @@ def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: grouped[ods_code].append(record) return grouped - def generate_ods_report( - self, - ods_code: str, - records: list[dict], - ): + def generate_ods_report(self, ods_code: str, records: list[dict]) -> str: with tempfile.NamedTemporaryFile( suffix=f"_{ods_code}.xlsx", delete=False, @@ -60,3 +57,4 @@ def generate_ods_report( records=records, output_path=tmp.name, ) + return tmp.name diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py new file mode 100644 index 0000000000..054af3eace --- /dev/null +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -0,0 +1,89 @@ +import importlib +import os +import pytest + +MODULE_UNDER_TEST = "handlers.report_distribution_handler" + + +@pytest.fixture +def handler_module(): + return importlib.import_module(MODULE_UNDER_TEST) + + +@pytest.fixture +def required_env(mocker): + mocker.patch.dict( + os.environ, + { + "REPORT_BUCKET_NAME": "my-report-bucket", + "CONTACT_TABLE_NAME": "contact-table", + "PRM_MAILBOX_EMAIL": "prm@example.com", + "SES_FROM_ADDRESS": "from@example.com", + }, + clear=False, + ) + + +def test_lambda_handler_wires_dependencies_and_returns_result( + mocker, handler_module, required_env +): + event = {"report_date": "2026-01-01"} + context = mocker.Mock() + + s3_instance = mocker.Mock(name="S3ServiceInstance") + contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") + email_instance = mocker.Mock(name="EmailServiceInstance") + + svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") + svc_instance.distribute_reports_for_date.return_value = {"succeeded": 2, "failed": 1} + + mocked_s3_cls = mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=s3_instance) + mocked_contact_repo_cls = mocker.patch.object( + handler_module, + "ReportContactRepository", + autospec=True, + return_value=contact_repo_instance, + ) + mocked_email_cls = mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=email_instance) + mocked_dist_svc_cls = mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + + result = handler_module.lambda_handler(event, context) + + mocked_s3_cls.assert_called_once_with() + mocked_contact_repo_cls.assert_called_once_with("contact-table") + mocked_email_cls.assert_called_once_with() + + mocked_dist_svc_cls.assert_called_once_with( + s3_service=s3_instance, + contact_repo=contact_repo_instance, + email_service=email_instance, + bucket="my-report-bucket", + from_address="from@example.com", + prm_mailbox="prm@example.com", + ) + + svc_instance.distribute_reports_for_date.assert_called_once_with("2026-01-01") + assert result == {"succeeded": 2, "failed": 1} + + +def test_lambda_handler_uses_report_date_from_event(mocker, handler_module, required_env): + event = {"report_date": "2099-12-31"} + context = mocker.Mock() + + svc_instance = mocker.Mock() + svc_instance.distribute_reports_for_date.return_value = {"succeeded": 0, "failed": 0} + mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + + mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + + result = handler_module.lambda_handler(event, context) + + svc_instance.distribute_reports_for_date.assert_called_once_with("2099-12-31") + assert result == {"succeeded": 0, "failed": 0} diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index c9ee2ff849..2654f313c0 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,6 +1,8 @@ -from unittest import mock +import json +import os from unittest.mock import MagicMock import pytest + from handlers.report_orchestration_handler import lambda_handler @@ -9,8 +11,16 @@ class FakeContext: @pytest.fixture(autouse=True) -def mock_env(monkeypatch): - monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") +def mock_env(mocker): + mocker.patch.dict( + os.environ, + { + "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", + "REPORT_BUCKET_NAME": "test-report-bucket", + "REPORT_DISTRIBUTION_LAMBDA": "test-distribution-lambda", + }, + clear=False, + ) @pytest.fixture @@ -34,6 +44,14 @@ def mock_excel_generator(mocker): ) +@pytest.fixture +def mock_s3_service(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.S3Service", + autospec=True, + ) + + @pytest.fixture def mock_service(mocker): return mocker.patch( @@ -50,25 +68,153 @@ def mock_window(mocker): ) -def test_lambda_handler_calls_service( - mock_logger, mock_repo, mock_excel_generator, mock_service, mock_window +@pytest.fixture +def mock_report_date(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.get_report_date_folder", + return_value="2026-01-02", + ) + + +@pytest.fixture +def mock_lambda_client(mocker): + fake_client = MagicMock() + return mocker.patch( + "handlers.report_orchestration_handler._get_lambda_client", + return_value=fake_client, + ) + + +def test_lambda_handler_calls_service_and_dependencies( + mock_logger, + mock_repo, + mock_excel_generator, + mock_s3_service, + mock_service, + mock_window, + mock_report_date, + mock_lambda_client, ): + service_instance = mock_service.return_value + service_instance.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + "B67890": "/tmp/B67890.xlsx", + } + lambda_handler(event={}, context=FakeContext()) mock_repo.assert_called_once_with("TestTable") mock_excel_generator.assert_called_once_with() + mock_s3_service.assert_called_once_with() mock_service.assert_called_once() - instance = mock_service.return_value - instance.process_reporting_window.assert_called_once_with( + service_instance.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, - output_dir=mock.ANY, ) mock_logger.info.assert_any_call("Report orchestration lambda invoked") -def test_lambda_handler_calls_window_function(mock_service, mock_window): +def test_lambda_handler_calls_window_function( + mock_service, + mock_window, + mock_report_date, + mock_lambda_client, + mock_s3_service, +): + mock_service.return_value.process_reporting_window.return_value = {} + lambda_handler(event={}, context=FakeContext()) + mock_window.assert_called_once() + + +def test_lambda_handler_returns_early_when_no_reports_generated( + mock_service, + mock_logger, + mock_s3_service, + mock_lambda_client, + mock_window, + mock_report_date, +): + mock_service.return_value.process_reporting_window.return_value = {} + + result = lambda_handler(event={}, context=FakeContext()) + + assert result is None + mock_s3_service.return_value.upload_file_with_extra_args.assert_not_called() + mock_lambda_client.return_value.invoke.assert_not_called() + mock_logger.info.assert_any_call("No reports generated; exiting") + + +def test_lambda_handler_uploads_each_report_to_s3( + mock_service, + mock_s3_service, + mock_report_date, + mock_lambda_client, + mock_window, +): + mock_service.return_value.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + "UNKNOWN": "/tmp/UNKNOWN.xlsx", + } + + lambda_handler(event={}, context=FakeContext()) + + s3_instance = mock_s3_service.return_value + assert s3_instance.upload_file_with_extra_args.call_count == 2 + + s3_instance.upload_file_with_extra_args.assert_any_call( + file_name="/tmp/A12345.xlsx", + s3_bucket_name="test-report-bucket", + file_key="Report-Orchestration/2026-01-02/A12345.xlsx", + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + + s3_instance.upload_file_with_extra_args.assert_any_call( + file_name="/tmp/UNKNOWN.xlsx", + s3_bucket_name="test-report-bucket", + file_key="Report-Orchestration/2026-01-02/UNKNOWN.xlsx", + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + + +def test_lambda_handler_invokes_distribution_lambda_once( + mock_service, + mock_report_date, + mock_lambda_client, + mock_window, + mock_s3_service, +): + mock_service.return_value.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + } + + lambda_handler(event={}, context=FakeContext()) + + client = mock_lambda_client.return_value + client.invoke.assert_called_once() + + kwargs = client.invoke.call_args.kwargs + assert kwargs["FunctionName"] == "test-distribution-lambda" + assert kwargs["InvocationType"] == "Event" + + payload = kwargs["Payload"] + assert '"report_date": "2026-01-02"' in payload + + +def test_lambda_handler_returns_error_when_required_env_missing(): + os.environ.pop("REPORT_BUCKET_NAME", None) + + result = lambda_handler(event={}, context=FakeContext()) + + assert isinstance(result, dict) + assert result["statusCode"] == 500 + + body = json.loads(result["body"]) + assert body["err_code"] == "ENV_5001" + assert "REPORT_BUCKET_NAME" in body["message"] + + if "interaction_id" in body and body["interaction_id"] is not None: + assert body["interaction_id"] == "test-request-id" \ No newline at end of file diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py new file mode 100644 index 0000000000..5cae95f0f2 --- /dev/null +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -0,0 +1,59 @@ +import pytest + +from repositories.reporting.report_contact_repository import ReportContactRepository + + +@pytest.fixture +def mock_dynamo(mocker): + mock = mocker.Mock() + mocker.patch( + "repositories.reporting.report_contact_repository.DynamoDBService", + return_value=mock, + ) + return mock + + +@pytest.fixture +def repo(mock_dynamo): + return ReportContactRepository(table_name="report-contacts") + + +def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): + mock_dynamo.get_item.return_value = { + "OdsCode": "Y12345", + "Email": "contact@example.com", + } + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result == "contact@example.com" + + +def test_get_contact_email_returns_none_when_item_missing(repo, mock_dynamo): + mock_dynamo.get_item.return_value = None + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result is None + + +def test_get_contact_email_returns_none_when_email_missing(repo, mock_dynamo): + mock_dynamo.get_item.return_value = { + "OdsCode": "Y12345", + } + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result is None diff --git a/lambdas/tests/unit/services/reporting/test_email_service.py b/lambdas/tests/unit/services/reporting/test_email_service.py new file mode 100644 index 0000000000..efde7f7251 --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_email_service.py @@ -0,0 +1,154 @@ +import pytest + +from services.email_service import EmailService + + +@pytest.fixture +def email_service(mocker): + mocker.patch("services.email_service.boto3.client", autospec=True) + svc = EmailService() + svc.ses = mocker.Mock() + return svc + +def test_send_email_sends_raw_email_without_attachments(email_service, mocker): + mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) + + email_service.send_email( + to_address="to@example.com", + subject="Hello", + body_text="Body text", + from_address="from@example.com", + attachments=None, + ) + + mocked_send_raw.assert_called_once() + + call_args, call_kwargs = mocked_send_raw.call_args + assert call_kwargs == {} + + msg_arg = call_args[0] + to_arg = call_args[1] + + assert to_arg == "to@example.com" + assert msg_arg["Subject"] == "Hello" + assert msg_arg["To"] == "to@example.com" + assert msg_arg["From"] == "from@example.com" + + raw = msg_arg.as_string() + assert "Body text" in raw + + + +def test_send_email_attaches_files_and_sets_filenames(email_service, mocker): + file_bytes_1 = b"zipbytes1" + file_bytes_2 = b"zipbytes2" + + m1 = mocker.mock_open(read_data=file_bytes_1) + m2 = mocker.mock_open(read_data=file_bytes_2) + + mocked_open = mocker.patch("services.email_service.open", create=True) + mocked_open.side_effect = [m1.return_value, m2.return_value] + + mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) + + email_service.send_email( + to_address="to@example.com", + subject="With Attachments", + body_text="See attached", + from_address="from@example.com", + attachments=["/tmp/a.zip", "/var/tmp/b.zip"], + ) + + assert mocked_open.call_count == 2 + mocked_open.assert_any_call("/tmp/a.zip", "rb") + mocked_open.assert_any_call("/var/tmp/b.zip", "rb") + + mocked_send_raw.assert_called_once() + + call_args, call_kwargs = mocked_send_raw.call_args + assert call_kwargs == {} + msg = call_args[0] + raw = msg.as_string() + + assert 'filename="a.zip"' in raw + assert 'filename="b.zip"' in raw + assert "See attached" in raw + + + +def test_send_raw_calls_ses_send_raw_email(email_service, mocker): + from email.mime.multipart import MIMEMultipart + msg = MIMEMultipart() + msg["Subject"] = "S" + msg["To"] = "to@example.com" + msg["From"] = "from@example.com" + + email_service._send_raw(msg, "to@example.com") + + email_service.ses.send_raw_email.assert_called_once() + call_kwargs = email_service.ses.send_raw_email.call_args.kwargs + + assert call_kwargs["Destinations"] == ["to@example.com"] + assert "RawMessage" in call_kwargs + assert "Data" in call_kwargs["RawMessage"] + assert isinstance(call_kwargs["RawMessage"]["Data"], str) + assert "Subject: S" in call_kwargs["RawMessage"]["Data"] + + +def test_send_report_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_report_email( + to_address="to@example.com", + from_address="from@example.com", + attachment_path="/tmp/report.zip", + ) + + mocked_send_email.assert_called_once_with( + to_address="to@example.com", + from_address="from@example.com", + subject="Daily Upload Report", + body_text="Please find your encrypted daily upload report attached.", + attachments=["/tmp/report.zip"], + ) + + +def test_send_password_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_password_email( + to_address="to@example.com", + from_address="from@example.com", + password="pw123", + ) + + mocked_send_email.assert_called_once_with( + to_address="to@example.com", + from_address="from@example.com", + subject="Daily Upload Report Password", + body_text="Password for your report:\n\npw123", + ) + + +def test_send_prm_missing_contact_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_prm_missing_contact_email( + prm_mailbox="prm@example.com", + from_address="from@example.com", + ods_code="Y12345", + attachment_path="/tmp/report.zip", + password="pw123", + ) + + mocked_send_email.assert_called_once_with( + to_address="prm@example.com", + from_address="from@example.com", + subject="Missing contact for ODS Y12345", + body_text=( + "No contact found for ODS Y12345.\n\n" + "Password: pw123\n\n" + "Please resolve the contact and forward the report." + ), + attachments=["/tmp/report.zip"], + ) diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py new file mode 100644 index 0000000000..06ca8721a8 --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -0,0 +1,293 @@ +import os +import pytest + +from services.reporting.report_distribution_service import ReportDistributionService + +@pytest.fixture +def mock_s3_service(mocker): + return mocker.Mock() + + +@pytest.fixture +def mock_contact_repo(mocker): + repo = mocker.Mock() + repo.get_contact_email.return_value = None + return repo + + +@pytest.fixture +def mock_email_service(mocker): + return mocker.Mock() + + +@pytest.fixture +def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): + mocker.patch("services.reporting.report_distribution_service.boto3.client", autospec=True) + + return ReportDistributionService( + s3_service=mock_s3_service, + contact_repo=mock_contact_repo, + email_service=mock_email_service, + bucket="my-bucket", + from_address="from@example.com", + prm_mailbox="prm@example.com", + ) + +def test_list_xlsx_keys_filters_only_xlsx(service, mocker): + paginator = mocker.Mock() + paginator.paginate.return_value = [ + { + "Contents": [ + {"Key": "Report-Orchestration/2026-01-01/A123.xlsx"}, + {"Key": "Report-Orchestration/2026-01-01/readme.txt"}, + {"Key": "Report-Orchestration/2026-01-01/B456.xls"}, + {"Key": "Report-Orchestration/2026-01-01/C789.xlsx"}, + ] + }, + {"Contents": [{"Key": "Report-Orchestration/2026-01-01/D000.xlsx"}]}, + {}, + ] + + service._s3_client.get_paginator.return_value = paginator + + keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + + assert keys == [ + "Report-Orchestration/2026-01-01/A123.xlsx", + "Report-Orchestration/2026-01-01/C789.xlsx", + "Report-Orchestration/2026-01-01/D000.xlsx", + ] + + service._s3_client.get_paginator.assert_called_once_with("list_objects_v2") + paginator.paginate.assert_called_once_with( + Bucket="my-bucket", + Prefix="Report-Orchestration/2026-01-01/", + ) + + +def test_list_xlsx_keys_returns_empty_when_no_objects(service, mocker): + paginator = mocker.Mock() + paginator.paginate.return_value = [{"Contents": []}, {}] + service._s3_client.get_paginator.return_value = paginator + + keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + + assert keys == [] + +def test_distribute_reports_for_date_when_no_keys_returns_zeroes(service, mocker): + mocker.patch.object(service, "list_xlsx_keys", return_value=[]) + mocked_process = mocker.patch.object(service, "process_one_report") + + result = service.distribute_reports_for_date("2026-01-01") + + assert result == {"succeeded": 0, "failed": 0} + mocked_process.assert_not_called() + + +def test_distribute_reports_for_date_extracts_ods_from_key(service, mocker): + mocker.patch.object( + service, + "list_xlsx_keys", + return_value=["Report-Orchestration/2026-01-01/some/path/Y12345.xlsx"], + ) + mocked_process = mocker.patch.object(service, "process_one_report", return_value=None) + + result = service.distribute_reports_for_date("2026-01-01") + + assert result == {"succeeded": 1, "failed": 0} + mocked_process.assert_called_once_with( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/some/path/Y12345.xlsx", + ) + + +def test_distribute_reports_for_date_counts_success_and_failure(service, mocker): + mocker.patch.object( + service, + "list_xlsx_keys", + return_value=[ + "Report-Orchestration/2026-01-01/Y12345.xlsx", + "Report-Orchestration/2026-01-01/A99999.xlsx", + ], + ) + + mocked_process = mocker.patch.object(service, "process_one_report") + mocked_process.side_effect = [None, RuntimeError("boom")] + + result = service.distribute_reports_for_date("2026-01-01") + + assert result == {"succeeded": 1, "failed": 1} + assert mocked_process.call_count == 2 + + mocked_process.assert_any_call( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + mocked_process.assert_any_call( + ods_code="A99999", + key="Report-Orchestration/2026-01-01/A99999.xlsx", + ) + +def test_process_one_report_downloads_encrypts_and_delegates_email( + service, mocker, mock_s3_service +): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="fixed-password", + ) + + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + service.process_one_report( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + + local_xlsx = os.path.join(fake_tmp, "Y12345.xlsx") + local_zip = os.path.join(fake_tmp, "Y12345.zip") + + mock_s3_service.download_file.assert_called_once_with( + "my-bucket", + "Report-Orchestration/2026-01-01/Y12345.xlsx", + local_xlsx, + ) + + mocked_zip.assert_called_once_with( + input_path=local_xlsx, + output_zip=local_zip, + password="fixed-password", + ) + + mocked_send.assert_called_once_with( + ods_code="Y12345", + attachment_path=local_zip, + password="fixed-password", + ) + + +def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_service): + mock_s3_service.download_file.side_effect = RuntimeError("download failed") + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="pw", + ) + mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="download failed"): + service.process_one_report( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + + mocked_send.assert_not_called() + + +def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="pw", + ) + mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + side_effect=RuntimeError("zip failed"), + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="zip failed"): + service.process_one_report( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + + mocked_send.assert_not_called() + +def test_send_report_emails_with_contact_calls_email_contact(service, mock_contact_repo, mocker): + mock_contact_repo.get_contact_email.return_value = "contact@example.com" + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="Y12345", + attachment_path="/tmp/Y12345.zip", + password="pw", + ) + + mock_contact_repo.get_contact_email.assert_called_once_with("Y12345") + mocked_email_contact.assert_called_once_with( + to_address="contact@example.com", + attachment_path="/tmp/Y12345.zip", + password="pw", + ) + mocked_email_prm.assert_not_called() + + +def test_send_report_emails_without_contact_calls_email_prm(service, mock_contact_repo, mocker): + mock_contact_repo.get_contact_email.return_value = None + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + mock_contact_repo.get_contact_email.assert_called_once_with("A99999") + mocked_email_prm.assert_called_once_with( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + mocked_email_contact.assert_not_called() + + +def test_email_contact_sends_report_and_password(service, mock_email_service): + service.email_contact( + to_address="contact@example.com", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_report_email.assert_called_once_with( + to_address="contact@example.com", + from_address="from@example.com", + attachment_path="/tmp/file.zip", + ) + mock_email_service.send_password_email.assert_called_once_with( + to_address="contact@example.com", + from_address="from@example.com", + password="pw", + ) + + +def test_email_prm_missing_contact_sends_prm_missing_contact_email(service, mock_email_service): + service.email_prm_missing_contact( + ods_code="X11111", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_prm_missing_contact_email.assert_called_once_with( + prm_mailbox="prm@example.com", + from_address="from@example.com", + ods_code="X11111", + attachment_path="/tmp/file.zip", + password="pw", + ) diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c068ba0c16..baf54577fe 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -1,4 +1,5 @@ import pytest + from services.reporting.report_orchestration_service import ReportOrchestrationService @@ -22,36 +23,27 @@ def report_orchestration_service(mock_repository, mock_excel_generator): ) -def test_process_reporting_window_no_records( +def test_process_reporting_window_no_records_returns_empty_dict_and_does_not_generate( report_orchestration_service, mock_repository, mock_excel_generator ): mock_repository.get_records_for_time_window.return_value = [] - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + result = report_orchestration_service.process_reporting_window(100, 200) + assert result == {} mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() -def test_group_records_by_ods_groups_correctly(): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - {"UploaderOdsCode": "A99999", "ID": 3}, - {"ID": 4}, # missing ODS - {"UploaderOdsCode": None, "ID": 5}, # null ODS - ] +def test_process_reporting_window_calls_repository_with_window_args( + report_orchestration_service, mock_repository, mocker +): + mock_repository.get_records_for_time_window.return_value = [{"UploaderOdsCode": "X1", "ID": 1}] + mocked_generate = mocker.patch.object(report_orchestration_service, "generate_ods_report", return_value="/tmp/x.xlsx") - result = ReportOrchestrationService.group_records_by_ods(records) + report_orchestration_service.process_reporting_window(100, 200) - assert result["Y12345"] == [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - ] - assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] - assert result["UNKNOWN"] == [ - {"ID": 4}, - {"UploaderOdsCode": None, "ID": 5}, - ] + mock_repository.get_records_for_time_window.assert_called_once_with(100, 200) + mocked_generate.assert_called_once() def test_process_reporting_window_generates_reports_per_ods( @@ -65,10 +57,10 @@ def test_process_reporting_window_generates_reports_per_ods( mock_repository.get_records_for_time_window.return_value = records mocked_generate = mocker.patch.object( - report_orchestration_service, "generate_ods_report" + report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" ) - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + report_orchestration_service.process_reporting_window(100, 200) mocked_generate.assert_any_call( "Y12345", @@ -84,21 +76,115 @@ def test_process_reporting_window_generates_reports_per_ods( assert mocked_generate.call_count == 2 -def test_generate_ods_report_creates_excel_report( +def test_process_reporting_window_returns_mapping_of_ods_to_generated_file_path( + report_orchestration_service, mock_repository, mocker +): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "A99999", "ID": 2}, + ] + mock_repository.get_records_for_time_window.return_value = records + + def _side_effect(ods_code, ods_records): + return f"/tmp/{ods_code}.xlsx" + + mocker.patch.object( + report_orchestration_service, + "generate_ods_report", + side_effect=_side_effect, + ) + + result = report_orchestration_service.process_reporting_window(100, 200) + + assert result == { + "Y12345": "/tmp/Y12345.xlsx", + "A99999": "/tmp/A99999.xlsx", + } + + +def test_process_reporting_window_includes_unknown_ods_group( + report_orchestration_service, mock_repository, mocker +): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"ID": 2}, # missing ODS -> UNKNOWN + {"UploaderOdsCode": None, "ID": 3}, # null ODS -> UNKNOWN + ] + mock_repository.get_records_for_time_window.return_value = records + + mocked_generate = mocker.patch.object( + report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" + ) + + report_orchestration_service.process_reporting_window(100, 200) + + # Expect 2 groups: Y12345 and UNKNOWN + assert mocked_generate.call_count == 2 + mocked_generate.assert_any_call( + "Y12345", + [{"UploaderOdsCode": "Y12345", "ID": 1}], + ) + mocked_generate.assert_any_call( + "UNKNOWN", + [{"ID": 2}, {"UploaderOdsCode": None, "ID": 3}], + ) + + +def test_group_records_by_ods_groups_correctly(): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + {"ID": 4}, # missing ODS + {"UploaderOdsCode": None, "ID": 5}, # null ODS + ] + + result = ReportOrchestrationService.group_records_by_ods(records) + + assert result["Y12345"] == [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ] + assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] + assert result["UNKNOWN"] == [ + {"ID": 4}, + {"UploaderOdsCode": None, "ID": 5}, + ] + + +def test_group_records_by_ods_empty_input_returns_empty_mapping(): + result = ReportOrchestrationService.group_records_by_ods([]) + assert dict(result) == {} + + +def test_group_records_by_ods_treats_empty_string_as_unknown(): + records = [{"UploaderOdsCode": "", "ID": 1}] + result = ReportOrchestrationService.group_records_by_ods(records) + assert result["UNKNOWN"] == [{"UploaderOdsCode": "", "ID": 1}] + + +def test_generate_ods_report_creates_excel_report_and_returns_path( report_orchestration_service, mock_excel_generator, mocker ): fake_tmp = mocker.MagicMock() fake_tmp.__enter__.return_value = fake_tmp fake_tmp.name = "/tmp/fake_Y12345.xlsx" - mocker.patch( + mocked_ntf = mocker.patch( "services.reporting.report_orchestration_service.tempfile.NamedTemporaryFile", return_value=fake_tmp, ) records = [{"ID": 1, "UploaderOdsCode": "Y12345"}] - report_orchestration_service.generate_ods_report("Y12345", records) + result_path = report_orchestration_service.generate_ods_report("Y12345", records) + + assert result_path == fake_tmp.name + + mocked_ntf.assert_called_once_with( + suffix="_Y12345.xlsx", + delete=False, + ) mock_excel_generator.create_report_orchestration_xlsx.assert_called_once_with( ods_code="Y12345", diff --git a/lambdas/utils/zip_utils.py b/lambdas/utils/zip_utils.py new file mode 100644 index 0000000000..a09e8934f0 --- /dev/null +++ b/lambdas/utils/zip_utils.py @@ -0,0 +1,20 @@ +import os +import pyzipper + + +def zip_encrypt_file(*, input_path: str, output_zip: str, password: str) -> None: + """ + Create an AES-encrypted ZIP file containing a single file. + + :param input_path: Path to the file to zip + :param output_zip: Path of the zip file to create + :param password: Password for AES encryption + """ + with pyzipper.AESZipFile( + output_zip, + "w", + compression=pyzipper.ZIP_DEFLATED, + encryption=pyzipper.WZ_AES, + ) as zf: + zf.setpassword(password.encode("utf-8")) + zf.write(input_path, arcname=os.path.basename(input_path)) From 4d9b46647a06e9a7471f62a67df4f0713e040d24 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 5 Jan 2026 15:46:42 +0000 Subject: [PATCH 11/60] [PRMP-1057] updated for the case it fails to access contacts table --- lambdas/services/reporting/report_distribution_service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index acaf65d0a5..694085685d 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -93,7 +93,13 @@ def process_one_report(self, *, ods_code: str, key: str) -> None: ) def send_report_emails(self, *, ods_code: str, attachment_path: str, password: str) -> None: - contact_email = self.contact_repo.get_contact_email(ods_code) + try: + contact_email = self.contact_repo.get_contact_email(ods_code) + except Exception as e: + logger.exception( + f"Contact lookup failed for ODS={ods_code}; falling back to PRM. Error: {e}" + ) + contact_email = None if contact_email: logger.info(f"Contact found for ODS={ods_code}, emailing {contact_email}") From 4ebc15e35bf47c72cd84f96741ef2dbd011f6094 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 6 Jan 2026 14:05:29 +0000 Subject: [PATCH 12/60] [PRMP-1182] trying with step functions --- .../handlers/report_distribution_handler.py | 24 ++++++++--- .../handlers/report_orchestration_handler.py | 43 +++++++++---------- .../reporting/report_distribution_service.py | 32 +++----------- 3 files changed, 43 insertions(+), 56 deletions(-) diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index 9df15879d1..d65314b781 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -1,4 +1,5 @@ import os +from typing import Any, Dict from repositories.reporting.report_contact_repository import ReportContactRepository from services.base.s3_service import S3Service @@ -24,10 +25,12 @@ @override_error_check @handle_lambda_exceptions @set_request_context_for_logging -def lambda_handler(event, context): - report_date = event["report_date"] +def lambda_handler(event, context) -> Dict[str, Any]: + action = event.get("action") + if action not in {"list", "process_one"}: + raise ValueError("Invalid action. Expected 'list' or 'process_one'.") - bucket = os.environ["REPORT_BUCKET_NAME"] + bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] contact_table = os.environ["CONTACT_TABLE_NAME"] prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] from_address = os.environ["SES_FROM_ADDRESS"] @@ -45,6 +48,15 @@ def lambda_handler(event, context): prm_mailbox=prm_mailbox, ) - result = service.distribute_reports_for_date(report_date) - logger.info(f"Daily Report distribution summary: {result}") - return result + if action == "list": + prefix = event["prefix"] + keys = service.list_xlsx_keys(prefix=prefix) + logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") + return {"bucket": bucket, "prefix": prefix, "keys": keys} + + # action == "process_one" + key = event["key"] + ods_code = service.extract_ods_code_from_key(key) + service.process_one_report(ods_code=ods_code, key=key) + logger.info(f"Process-one mode: processed ods={ods_code}, key={key}") + return {"status": "ok", "bucket": bucket, "key": key, "ods_code": ods_code} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 6dc70db306..d1d746d14f 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,8 +1,6 @@ -import json import os from datetime import datetime, timedelta, timezone - -import boto3 +from typing import Any, Dict from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.base.s3_service import S3Service @@ -16,8 +14,6 @@ logger = LoggingService(__name__) -def _get_lambda_client(): - return boto3.client("lambda") def calculate_reporting_window(): now = datetime.now(timezone.utc) @@ -43,19 +39,16 @@ def build_s3_key(ods_code: str, report_date: str) -> str: names=[ "BULK_UPLOAD_REPORT_TABLE_NAME", "REPORT_BUCKET_NAME", - "REPORT_DISTRIBUTION_LAMBDA", ] ) @override_error_check @handle_lambda_exceptions @set_request_context_for_logging -def lambda_handler(event, context): +def lambda_handler(event, context) -> Dict[str, Any]: logger.info("Report orchestration lambda invoked") - lambda_client = _get_lambda_client() table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] report_bucket = os.environ["REPORT_BUCKET_NAME"] - distribution_lambda = os.environ["REPORT_DISTRIBUTION_LAMBDA"] repository = ReportingDynamoRepository(table_name) excel_generator = ExcelReportGenerator() @@ -71,6 +64,7 @@ def lambda_handler(event, context): window_end += 24 * 60 * 60 report_date = get_report_date_folder() + prefix = f"Report-Orchestration/{report_date}/" generated_files = service.process_reporting_window( window_start_ts=window_start, @@ -79,26 +73,29 @@ def lambda_handler(event, context): if not generated_files: logger.info("No reports generated; exiting") - return - - # Upload each generated report to S3 + return { + "report_date": report_date, + "bucket": report_bucket, + "prefix": prefix, + "keys": [], + } + + keys = [] for ods_code, local_path in generated_files.items(): key = build_s3_key(ods_code, report_date) s3_service.upload_file_with_extra_args( file_name=local_path, s3_bucket_name=report_bucket, file_key=key, - extra_args={ - "ServerSideEncryption": "aws:kms", - }, + extra_args={"ServerSideEncryption": "aws:kms"}, ) + keys.append(key) logger.info(f"Uploaded report for ODS={ods_code} to s3://{report_bucket}/{key}") - # Invoke distribution lambda once for the day folder - lambda_client.invoke( - FunctionName=distribution_lambda, - InvocationType="Event", - Payload=json.dumps({"report_date": report_date}), - ) - - logger.info(f"Invoked distribution lambda for report_date={report_date}") + logger.info(f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}") + return { + "report_date": report_date, + "bucket": report_bucket, + "prefix": prefix, + "keys": keys, + } diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index 694085685d..bbb380faad 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -1,7 +1,7 @@ import os import secrets import tempfile -from typing import List, Dict +from typing import List import boto3 @@ -33,32 +33,10 @@ def __init__( self.prm_mailbox = prm_mailbox self._s3_client = boto3.client("s3") - def distribute_reports_for_date(self, report_date: str) -> Dict[str, int]: - day_prefix = f"Report-Orchestration/{report_date}/" - keys = self.list_xlsx_keys(prefix=day_prefix) - - logger.info(f"Found {len(keys)} report(s) under s3://{self.bucket}/{day_prefix}") - - succeeded = 0 - failed = 0 - - for key in keys: - ods_code = key.split("/")[-1].replace(".xlsx", "") - - try: - self.process_one_report(ods_code=ods_code, key=key) - succeeded += 1 - except Exception as e: - failed += 1 - logger.exception( - f"Failed processing report for ODS={ods_code}, key={key}: {e}" - ) - - logger.info( - f"Distribution completed for {report_date}: " - f"succeeded={succeeded}, failed={failed}" - ) - return {"succeeded": succeeded, "failed": failed} + @staticmethod + def extract_ods_code_from_key(key: str) -> str: + filename = key.split("/")[-1] + return filename[:-5] if filename.lower().endswith(".xlsx") else filename def list_xlsx_keys(self, prefix: str) -> List[str]: paginator = self._s3_client.get_paginator("list_objects_v2") From 5cecec9a1995cab0d565a9bff64944d1e007694c Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 6 Jan 2026 15:56:24 +0000 Subject: [PATCH 13/60] [PRMP-1182] fixed unit tests --- .../test_report_distribution_handler.py | 65 +++++-- .../test_report_orchestration_handler.py | 133 +++++++------- .../test_report_distribution_service.py | 170 +++++++++++------- 3 files changed, 223 insertions(+), 145 deletions(-) diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index 054af3eace..d129d2ed36 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -24,27 +24,32 @@ def required_env(mocker): ) -def test_lambda_handler_wires_dependencies_and_returns_result( +def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( mocker, handler_module, required_env ): - event = {"report_date": "2026-01-01"} + event = {"action": "list", "prefix": "reports/2026-01-01/"} context = mocker.Mock() + context.aws_request_id = "req-123" # avoid JSON serialization issues in decorators s3_instance = mocker.Mock(name="S3ServiceInstance") contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") email_instance = mocker.Mock(name="EmailServiceInstance") svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") - svc_instance.distribute_reports_for_date.return_value = {"succeeded": 2, "failed": 1} + svc_instance.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] - mocked_s3_cls = mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=s3_instance) + mocked_s3_cls = mocker.patch.object( + handler_module, "S3Service", autospec=True, return_value=s3_instance + ) mocked_contact_repo_cls = mocker.patch.object( handler_module, "ReportContactRepository", autospec=True, return_value=contact_repo_instance, ) - mocked_email_cls = mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=email_instance) + mocked_email_cls = mocker.patch.object( + handler_module, "EmailService", autospec=True, return_value=email_instance + ) mocked_dist_svc_cls = mocker.patch.object( handler_module, "ReportDistributionService", @@ -67,23 +72,59 @@ def test_lambda_handler_wires_dependencies_and_returns_result( prm_mailbox="prm@example.com", ) - svc_instance.distribute_reports_for_date.assert_called_once_with("2026-01-01") - assert result == {"succeeded": 2, "failed": 1} + svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") + assert result == { + "bucket": "my-report-bucket", + "prefix": "reports/2026-01-01/", + "keys": ["a.xlsx", "b.xlsx"], + } -def test_lambda_handler_uses_report_date_from_event(mocker, handler_module, required_env): - event = {"report_date": "2099-12-31"} +def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( + mocker, handler_module, required_env +): + event = {"action": "list", "prefix": "p/", "bucket": "override-bucket"} context = mocker.Mock() + context.aws_request_id = "req-456" svc_instance = mocker.Mock() - svc_instance.distribute_reports_for_date.return_value = {"succeeded": 0, "failed": 0} + svc_instance.list_xlsx_keys.return_value = [] + mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + + result = handler_module.lambda_handler(event, context) + svc_instance.list_xlsx_keys.assert_called_once_with(prefix="p/") + assert result == {"bucket": "override-bucket", "prefix": "p/", "keys": []} + + +def test_lambda_handler_process_one_mode_happy_path( + mocker, handler_module, required_env +): + event = {"action": "process_one", "key": "reports/ABC/whatever.xlsx"} + context = mocker.Mock() + context.aws_request_id = "req-789" + + svc_instance = mocker.Mock() + svc_instance.extract_ods_code_from_key.return_value = "ABC" + svc_instance.process_one_report.return_value = None + + mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) result = handler_module.lambda_handler(event, context) - svc_instance.distribute_reports_for_date.assert_called_once_with("2099-12-31") - assert result == {"succeeded": 0, "failed": 0} + svc_instance.extract_ods_code_from_key.assert_called_once_with("reports/ABC/whatever.xlsx") + svc_instance.process_one_report.assert_called_once_with(ods_code="ABC", key="reports/ABC/whatever.xlsx") + + assert result == { + "status": "ok", + "bucket": "my-report-bucket", + "key": "reports/ABC/whatever.xlsx", + "ods_code": "ABC", + } diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 2654f313c0..10364c5e42 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,6 +1,7 @@ import json import os from unittest.mock import MagicMock + import pytest from handlers.report_orchestration_handler import lambda_handler @@ -17,7 +18,6 @@ def mock_env(mocker): { "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", "REPORT_BUCKET_NAME": "test-report-bucket", - "REPORT_DISTRIBUTION_LAMBDA": "test-distribution-lambda", }, clear=False, ) @@ -29,7 +29,7 @@ def mock_logger(mocker): @pytest.fixture -def mock_repo(mocker): +def mock_repo_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.ReportingDynamoRepository", autospec=True, @@ -37,7 +37,7 @@ def mock_repo(mocker): @pytest.fixture -def mock_excel_generator(mocker): +def mock_excel_generator_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.ExcelReportGenerator", autospec=True, @@ -45,7 +45,7 @@ def mock_excel_generator(mocker): @pytest.fixture -def mock_s3_service(mocker): +def mock_s3_service_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.S3Service", autospec=True, @@ -53,7 +53,7 @@ def mock_s3_service(mocker): @pytest.fixture -def mock_service(mocker): +def mock_service_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.ReportOrchestrationService", autospec=True, @@ -76,38 +76,32 @@ def mock_report_date(mocker): ) -@pytest.fixture -def mock_lambda_client(mocker): - fake_client = MagicMock() - return mocker.patch( - "handlers.report_orchestration_handler._get_lambda_client", - return_value=fake_client, - ) - - -def test_lambda_handler_calls_service_and_dependencies( +def test_lambda_handler_wires_dependencies_and_calls_service( mock_logger, - mock_repo, - mock_excel_generator, - mock_s3_service, - mock_service, + mock_repo_cls, + mock_excel_generator_cls, + mock_s3_service_cls, + mock_service_cls, mock_window, mock_report_date, - mock_lambda_client, ): - service_instance = mock_service.return_value + service_instance = mock_service_cls.return_value service_instance.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "B67890": "/tmp/B67890.xlsx", } - lambda_handler(event={}, context=FakeContext()) + result = lambda_handler(event={}, context=FakeContext()) - mock_repo.assert_called_once_with("TestTable") - mock_excel_generator.assert_called_once_with() - mock_s3_service.assert_called_once_with() + mock_repo_cls.assert_called_once_with("TestTable") + mock_excel_generator_cls.assert_called_once_with() + mock_s3_service_cls.assert_called_once_with() + + mock_service_cls.assert_called_once_with( + repository=mock_repo_cls.return_value, + excel_generator=mock_excel_generator_cls.return_value, + ) - mock_service.assert_called_once() service_instance.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, @@ -115,54 +109,64 @@ def test_lambda_handler_calls_service_and_dependencies( mock_logger.info.assert_any_call("Report orchestration lambda invoked") + assert result["report_date"] == "2026-01-02" + assert result["bucket"] == "test-report-bucket" + assert result["prefix"] == "Report-Orchestration/2026-01-02/" + assert set(result["keys"]) == { + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/B67890.xlsx", + } + def test_lambda_handler_calls_window_function( - mock_service, + mock_service_cls, mock_window, mock_report_date, - mock_lambda_client, - mock_s3_service, + mock_s3_service_cls, ): - mock_service.return_value.process_reporting_window.return_value = {} + mock_service_cls.return_value.process_reporting_window.return_value = {} lambda_handler(event={}, context=FakeContext()) mock_window.assert_called_once() -def test_lambda_handler_returns_early_when_no_reports_generated( - mock_service, +def test_lambda_handler_returns_empty_keys_when_no_reports_generated( + mock_service_cls, mock_logger, - mock_s3_service, - mock_lambda_client, + mock_s3_service_cls, mock_window, mock_report_date, ): - mock_service.return_value.process_reporting_window.return_value = {} + mock_service_cls.return_value.process_reporting_window.return_value = {} result = lambda_handler(event={}, context=FakeContext()) - assert result is None - mock_s3_service.return_value.upload_file_with_extra_args.assert_not_called() - mock_lambda_client.return_value.invoke.assert_not_called() + assert result == { + "report_date": "2026-01-02", + "bucket": "test-report-bucket", + "prefix": "Report-Orchestration/2026-01-02/", + "keys": [], + } + + mock_s3_service_cls.return_value.upload_file_with_extra_args.assert_not_called() mock_logger.info.assert_any_call("No reports generated; exiting") -def test_lambda_handler_uploads_each_report_to_s3( - mock_service, - mock_s3_service, +def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( + mock_service_cls, + mock_s3_service_cls, mock_report_date, - mock_lambda_client, mock_window, ): - mock_service.return_value.process_reporting_window.return_value = { + mock_service_cls.return_value.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "UNKNOWN": "/tmp/UNKNOWN.xlsx", } - lambda_handler(event={}, context=FakeContext()) + result = lambda_handler(event={}, context=FakeContext()) - s3_instance = mock_s3_service.return_value + s3_instance = mock_s3_service_cls.return_value assert s3_instance.upload_file_with_extra_args.call_count == 2 s3_instance.upload_file_with_extra_args.assert_any_call( @@ -179,35 +183,20 @@ def test_lambda_handler_uploads_each_report_to_s3( extra_args={"ServerSideEncryption": "aws:kms"}, ) + assert result["keys"] == [ + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/UNKNOWN.xlsx", + ] -def test_lambda_handler_invokes_distribution_lambda_once( - mock_service, - mock_report_date, - mock_lambda_client, - mock_window, - mock_s3_service, -): - mock_service.return_value.process_reporting_window.return_value = { - "A12345": "/tmp/A12345.xlsx", - } - - lambda_handler(event={}, context=FakeContext()) - - client = mock_lambda_client.return_value - client.invoke.assert_called_once() - - kwargs = client.invoke.call_args.kwargs - assert kwargs["FunctionName"] == "test-distribution-lambda" - assert kwargs["InvocationType"] == "Event" - - payload = kwargs["Payload"] - assert '"report_date": "2026-01-02"' in payload - -def test_lambda_handler_returns_error_when_required_env_missing(): +def test_lambda_handler_returns_error_when_required_env_missing(mocker): + mocker.patch.dict(os.environ, {"BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable"}, clear=False) os.environ.pop("REPORT_BUCKET_NAME", None) - result = lambda_handler(event={}, context=FakeContext()) + ctx = FakeContext() + ctx.aws_request_id = "test-request-id" + + result = lambda_handler(event={}, context=ctx) assert isinstance(result, dict) assert result["statusCode"] == 500 @@ -216,5 +205,5 @@ def test_lambda_handler_returns_error_when_required_env_missing(): assert body["err_code"] == "ENV_5001" assert "REPORT_BUCKET_NAME" in body["message"] - if "interaction_id" in body and body["interaction_id"] is not None: - assert body["interaction_id"] == "test-request-id" \ No newline at end of file + if body.get("interaction_id") is not None: + assert body["interaction_id"] == "test-request-id" diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index 06ca8721a8..56b5150e03 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -33,6 +33,19 @@ def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): prm_mailbox="prm@example.com", ) +def test_extract_ods_code_from_key_strips_xlsx_extension(): + assert ReportDistributionService.extract_ods_code_from_key( + "Report-Orchestration/2026-01-01/Y12345.xlsx" + ) == "Y12345" + + +def test_extract_ods_code_from_key_is_case_insensitive(): + assert ReportDistributionService.extract_ods_code_from_key("a/b/C789.XLSX") == "C789" + + +def test_extract_ods_code_from_key_keeps_non_xlsx_filename(): + assert ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") == "report.csv" + def test_list_xlsx_keys_filters_only_xlsx(service, mocker): paginator = mocker.Mock() paginator.paginate.return_value = [ @@ -74,59 +87,16 @@ def test_list_xlsx_keys_returns_empty_when_no_objects(service, mocker): assert keys == [] -def test_distribute_reports_for_date_when_no_keys_returns_zeroes(service, mocker): - mocker.patch.object(service, "list_xlsx_keys", return_value=[]) - mocked_process = mocker.patch.object(service, "process_one_report") - - result = service.distribute_reports_for_date("2026-01-01") - - assert result == {"succeeded": 0, "failed": 0} - mocked_process.assert_not_called() - - -def test_distribute_reports_for_date_extracts_ods_from_key(service, mocker): - mocker.patch.object( - service, - "list_xlsx_keys", - return_value=["Report-Orchestration/2026-01-01/some/path/Y12345.xlsx"], - ) - mocked_process = mocker.patch.object(service, "process_one_report", return_value=None) - - result = service.distribute_reports_for_date("2026-01-01") - - assert result == {"succeeded": 1, "failed": 0} - mocked_process.assert_called_once_with( - ods_code="Y12345", - key="Report-Orchestration/2026-01-01/some/path/Y12345.xlsx", - ) - -def test_distribute_reports_for_date_counts_success_and_failure(service, mocker): - mocker.patch.object( - service, - "list_xlsx_keys", - return_value=[ - "Report-Orchestration/2026-01-01/Y12345.xlsx", - "Report-Orchestration/2026-01-01/A99999.xlsx", - ], - ) - - mocked_process = mocker.patch.object(service, "process_one_report") - mocked_process.side_effect = [None, RuntimeError("boom")] +def test_list_xlsx_keys_skips_pages_without_contents(service, mocker): + paginator = mocker.Mock() + paginator.paginate.return_value = [{}, {"Contents": [{"Key": "p/X.xlsx"}]}] + service._s3_client.get_paginator.return_value = paginator - result = service.distribute_reports_for_date("2026-01-01") + keys = service.list_xlsx_keys(prefix="p/") - assert result == {"succeeded": 1, "failed": 1} - assert mocked_process.call_count == 2 + assert keys == ["p/X.xlsx"] - mocked_process.assert_any_call( - ods_code="Y12345", - key="Report-Orchestration/2026-01-01/Y12345.xlsx", - ) - mocked_process.assert_any_call( - ods_code="A99999", - key="Report-Orchestration/2026-01-01/A99999.xlsx", - ) def test_process_one_report_downloads_encrypts_and_delegates_email( service, mocker, mock_s3_service @@ -180,27 +150,39 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_service): mock_s3_service.download_file.side_effect = RuntimeError("download failed") + + td = mocker.MagicMock() + td.__enter__.return_value = "/tmp/fake_tmpdir" + td.__exit__.return_value = False mocker.patch( - "services.reporting.report_distribution_service.secrets.token_urlsafe", - return_value="pw", + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, ) - mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) + + mocked_zip = mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) with pytest.raises(RuntimeError, match="download failed"): - service.process_one_report( - ods_code="Y12345", - key="Report-Orchestration/2026-01-01/Y12345.xlsx", - ) + service.process_one_report(ods_code="Y12345", key="k.xlsx") + mocked_zip.assert_not_called() mocked_send.assert_not_called() -def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker): +def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker, mock_s3_service): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", return_value="pw", ) + + td = mocker.MagicMock() + td.__enter__.return_value = "/tmp/fake_tmpdir" + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", side_effect=RuntimeError("zip failed"), @@ -209,11 +191,44 @@ def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker): mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) with pytest.raises(RuntimeError, match="zip failed"): - service.process_one_report( - ods_code="Y12345", - key="Report-Orchestration/2026-01-01/Y12345.xlsx", - ) + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mocked_send.assert_not_called() + + +def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fails( + service, mocker, mock_s3_service +): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + side_effect=RuntimeError("secrets failed"), + ) + + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="secrets failed"): + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mock_s3_service.download_file.assert_called_once_with( + "my-bucket", + "k.xlsx", + os.path.join(fake_tmp, "Y12345.xlsx"), + ) + mocked_zip.assert_not_called() mocked_send.assert_not_called() def test_send_report_emails_with_contact_calls_email_contact(service, mock_contact_repo, mocker): @@ -258,6 +273,26 @@ def test_send_report_emails_without_contact_calls_email_prm(service, mock_contac mocked_email_contact.assert_not_called() +def test_send_report_emails_contact_lookup_exception_falls_back_to_prm(service, mock_contact_repo, mocker): + mock_contact_repo.get_contact_email.side_effect = RuntimeError("ddb down") + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + mocked_email_contact.assert_not_called() + mocked_email_prm.assert_called_once_with( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + def test_email_contact_sends_report_and_password(service, mock_email_service): service.email_contact( to_address="contact@example.com", @@ -277,6 +312,19 @@ def test_email_contact_sends_report_and_password(service, mock_email_service): ) +def test_email_contact_sends_password_even_if_report_email_fails(service, mock_email_service): + mock_email_service.send_report_email.side_effect = RuntimeError("SES down") + + with pytest.raises(RuntimeError, match="SES down"): + service.email_contact( + to_address="contact@example.com", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_password_email.assert_not_called() + + def test_email_prm_missing_contact_sends_prm_missing_contact_email(service, mock_email_service): service.email_prm_missing_contact( ods_code="X11111", From 8cd233599e712dac6ebc8d8e230ce1b1136b3e59 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 7 Jan 2026 13:02:12 +0000 Subject: [PATCH 14/60] [PRMP-1182] Improved query to dynamo, to use specific ranges --- .../reporting/reporting_dynamo_repository.py | 45 +++++-- lambdas/services/base/dynamo_service.py | 40 +++++++ .../test_reporting_dynamo_repository.py | 77 +++++++++--- .../unit/services/base/test_dynamo_service.py | 112 ++++++++++++++++++ lambdas/tests/unit/utils/test_utilities.py | 17 ++- lambdas/utils/utilities.py | 6 +- 6 files changed, 268 insertions(+), 29 deletions(-) diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 7a694edf69..484db9fd10 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,12 +1,12 @@ from typing import Dict, List -from boto3.dynamodb.conditions import Attr +from boto3.dynamodb.conditions import Key from services.base.dynamo_service import DynamoDBService from utils.audit_logging_setup import LoggingService +from utils.utilities import utc_date_string logger = LoggingService(__name__) - class ReportingDynamoRepository: def __init__(self, table_name: str): self.table_name = table_name @@ -17,19 +17,42 @@ def get_records_for_time_window( start_timestamp: int, end_timestamp: int, ) -> List[Dict]: + timestamp_index_name = "TimestampIndex" logger.info( - f"Querying reporting table for window, " - f"table_name: {self.table_name}, " - f"start_timestamp: {start_timestamp}, " - f"end_timestamp: {end_timestamp}", + "Querying reporting table via TimestampIndex for window, " + f"table_name={self.table_name}, start_timestamp={start_timestamp}, end_timestamp={end_timestamp}", ) - filter_expression = Attr("Timestamp").between( - start_timestamp, - end_timestamp, + start_date_string = utc_date_string(start_timestamp) + end_date_string = utc_date_string(end_timestamp) + + if start_date_string == end_date_string: + key_condition_expression = ( + Key("Date").eq(start_date_string) + & Key("Timestamp").between(start_timestamp, end_timestamp) + ) + return self.dynamo_service.query_by_key_condition_expression( + table_name=self.table_name, + index_name=timestamp_index_name, + key_condition_expression=key_condition_expression, + ) + + start_day_key_condition = ( + Key("Date").eq(start_date_string) & Key("Timestamp").gte(start_timestamp) + ) + end_day_key_condition = ( + Key("Date").eq(end_date_string) & Key("Timestamp").lte(end_timestamp) ) - return self.dynamo_service.scan_whole_table( + start_day_items = self.dynamo_service.query_by_key_condition_expression( table_name=self.table_name, - filter_expression=filter_expression, + index_name=timestamp_index_name, + key_condition_expression=start_day_key_condition, ) + end_day_items = self.dynamo_service.query_by_key_condition_expression( + table_name=self.table_name, + index_name=timestamp_index_name, + key_condition_expression=end_day_key_condition, + ) + + return start_day_items + end_day_items diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index ec8de0b908..c97418b5a5 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -429,3 +429,43 @@ def build_update_transaction_item( }, } } + + def query_by_key_condition_expression( + self, + table_name: str, + key_condition_expression: ConditionBase, + index_name: str | None = None, + query_filter: Attr | ConditionBase | None = None, + limit: int | None = None, + ) -> list[dict]: + table = self.get_table(table_name) + + collected_items: list[dict] = [] + exclusive_start_key: dict | None = None + + while True: + query_params: dict = {"KeyConditionExpression": key_condition_expression} + + if index_name: + query_params["IndexName"] = index_name + if query_filter: + query_params["FilterExpression"] = query_filter + if exclusive_start_key: + query_params["ExclusiveStartKey"] = exclusive_start_key + if limit: + query_params["Limit"] = limit + + try: + response = table.query(**query_params) + except ClientError as exc: + logger.error(str(exc), {"Result": f"Unable to query table: {table_name}"}) + raise + + collected_items.extend(response.get("Items", [])) + + exclusive_start_key = response.get("LastEvaluatedKey") + if not exclusive_start_key: + break + + return collected_items + diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index df3c553044..f6a9ec9568 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -6,12 +6,12 @@ @pytest.fixture def mock_dynamo_service(mocker): - mock_service = mocker.patch( + mock_service_class = mocker.patch( "repositories.reporting.reporting_dynamo_repository.DynamoDBService" ) - instance = mock_service.return_value - instance.scan_whole_table = MagicMock() - return instance + mock_instance = mock_service_class.return_value + mock_instance.query_by_key_condition_expression = MagicMock() + return mock_instance @pytest.fixture @@ -19,23 +19,68 @@ def reporting_repo(mock_dynamo_service): return ReportingDynamoRepository(table_name="TestTable") -def test_get_records_for_time_window_calls_scan(mock_dynamo_service, reporting_repo): - mock_dynamo_service.scan_whole_table.return_value = [] +def test_get_records_for_time_window_same_date_queries_once( + mocker, mock_dynamo_service, reporting_repo +): + mock_utc_date_string = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date_string" + ) + mock_utc_date_string.side_effect = ["2026-01-07", "2026-01-07"] + + mock_dynamo_service.query_by_key_condition_expression.return_value = [{"ID": "one"}] + + result = reporting_repo.get_records_for_time_window(100, 200) + + assert result == [{"ID": "one"}] + mock_dynamo_service.query_by_key_condition_expression.assert_called_once() + + call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args.kwargs + assert call_kwargs["table_name"] == "TestTable" + assert call_kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in call_kwargs + + +def test_get_records_for_time_window_different_dates_queries_twice( + mocker, mock_dynamo_service, reporting_repo +): + mock_utc_date_string = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date_string" + ) + mock_utc_date_string.side_effect = ["2026-01-06", "2026-01-07"] + + mock_dynamo_service.query_by_key_condition_expression.side_effect = [ + [{"ID": "start-day"}], + [{"ID": "end-day"}], + ] + + result = reporting_repo.get_records_for_time_window(100, 200) - reporting_repo.get_records_for_time_window(100, 200) + assert result == [{"ID": "start-day"}, {"ID": "end-day"}] + assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 - mock_dynamo_service.scan_whole_table.assert_called_once() - assert "filter_expression" in mock_dynamo_service.scan_whole_table.call_args.kwargs + first_call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args_list[0].kwargs + second_call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args_list[1].kwargs + assert first_call_kwargs["table_name"] == "TestTable" + assert first_call_kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in first_call_kwargs -def test_get_records_for_time_window_returns_empty_list( - mock_dynamo_service, reporting_repo + assert second_call_kwargs["table_name"] == "TestTable" + assert second_call_kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in second_call_kwargs + + +def test_get_records_for_time_window_returns_empty_list_when_no_items( + mocker, mock_dynamo_service, reporting_repo ): - start_ts = 0 - end_ts = 50 - mock_dynamo_service.scan_whole_table.return_value = [] + mock_utc_date_string = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date_string" + ) + mock_utc_date_string.side_effect = ["2026-01-06", "2026-01-07"] + + mock_dynamo_service.query_by_key_condition_expression.side_effect = [[], []] - result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + result = reporting_repo.get_records_for_time_window(100, 200) assert result == [] - mock_dynamo_service.scan_whole_table.assert_called_once() + assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index 80fa1ea911..b0f7f06396 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -1196,3 +1196,115 @@ def test_build_key_condition_non_matching_list_lengths( mock_service.build_key_condition( search_key=search_key, search_condition=search_condition ) + +def test_query_by_key_condition_expression_single_page_returns_items( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.return_value = { + "Items": [{"ID": "item-1"}, {"ID": "item-2"}] + } + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert result == [{"ID": "item-1"}, {"ID": "item-2"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.query.assert_called_once_with( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ) + + +def test_query_by_key_condition_expression_handles_pagination( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.side_effect = [ + { + "Items": [{"ID": "item-1"}], + "LastEvaluatedKey": {"ID": "page-2"}, + }, + { + "Items": [{"ID": "item-2"}], + "LastEvaluatedKey": {"ID": "page-3"}, + }, + { + "Items": [{"ID": "item-3"}], + }, + ] + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert result == [{"ID": "item-1"}, {"ID": "item-2"}, {"ID": "item-3"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + + expected_calls = [ + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ), + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ExclusiveStartKey={"ID": "page-2"}, + ), + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ExclusiveStartKey={"ID": "page-3"}, + ), + ] + mock_table.return_value.query.assert_has_calls(expected_calls) + + +def test_query_by_key_condition_expression_passes_filter_and_limit( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").lte(1767780025) + filter_expression = Attr("UploadStatus").eq("complete") + + mock_table.return_value.query.return_value = {"Items": [{"ID": "item-1"}]} + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + query_filter=filter_expression, + limit=25, + ) + + assert result == [{"ID": "item-1"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.query.assert_called_once_with( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + FilterExpression=filter_expression, + Limit=25, + ) + + +def test_query_by_key_condition_expression_client_error_raises_exception( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.side_effect = MOCK_CLIENT_ERROR + + with pytest.raises(ClientError) as exc_info: + mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert exc_info.value == MOCK_CLIENT_ERROR diff --git a/lambdas/tests/unit/utils/test_utilities.py b/lambdas/tests/unit/utils/test_utilities.py index 68ddb60e5b..23a3b3745b 100755 --- a/lambdas/tests/unit/utils/test_utilities.py +++ b/lambdas/tests/unit/utils/test_utilities.py @@ -12,7 +12,7 @@ get_pds_service, parse_date, redact_id_to_last_4_chars, - validate_nhs_number, + validate_nhs_number, utc_date_string, ) @@ -135,3 +135,18 @@ def test_format_cloudfront_url_valid(): def test_parse_date_returns_correct_date_for_valid_formats(input_date, expected_date): result = parse_date(input_date) assert result == expected_date + +@pytest.mark.parametrize( + "timestamp_seconds, expected_date_string", + [ + (0, "1970-01-01"), + (1704067200, "2024-01-01"), + (1767780025, "2026-01-07"), + (1704153599, "2024-01-01"), + (1704153600, "2024-01-02"), + ], +) +def test_utc_date_string_returns_correct_utc_date( + timestamp_seconds, expected_date_string +): + assert utc_date_string(timestamp_seconds) == expected_date_string diff --git a/lambdas/utils/utilities.py b/lambdas/utils/utilities.py index b83af68e4b..92d99d8f02 100755 --- a/lambdas/utils/utilities.py +++ b/lambdas/utils/utilities.py @@ -2,7 +2,7 @@ import os import re import uuid -from datetime import datetime +from datetime import datetime, timezone from urllib.parse import urlparse from inflection import camelize @@ -127,3 +127,7 @@ def parse_date(date_string: str) -> datetime | None: except ValueError: continue return None + + +def utc_date_string(timestamp_seconds: int) -> str: + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).strftime("%Y-%m-%d") \ No newline at end of file From aa1ac5cfdde5ca16c6d0fdacad0f2520e1902938 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 7 Jan 2026 14:24:55 +0000 Subject: [PATCH 15/60] [PRMP-1057] adjusted time querie to dynamo --- .../handlers/report_orchestration_handler.py | 1 + .../reporting/reporting_dynamo_repository.py | 50 +++++++++---------- lambdas/services/email_service.py | 1 + lambdas/utils/utilities.py | 14 +++++- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index d1d746d14f..c8516b730d 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -61,6 +61,7 @@ def lambda_handler(event, context) -> Dict[str, Any]: window_start, window_end = calculate_reporting_window() # TODO delete line below since it is only for testing + window_start -= 24 * 60 * 60 window_end += 24 * 60 * 60 report_date = get_report_date_folder() diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 484db9fd10..86d595d87e 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,9 +1,10 @@ +from datetime import timedelta from typing import Dict, List from boto3.dynamodb.conditions import Key from services.base.dynamo_service import DynamoDBService from utils.audit_logging_setup import LoggingService -from utils.utilities import utc_date_string +from utils.utilities import utc_date_string, utc_date, utc_day_start_timestamp, utc_day_end_timestamp logger = LoggingService(__name__) @@ -23,36 +24,31 @@ def get_records_for_time_window( f"table_name={self.table_name}, start_timestamp={start_timestamp}, end_timestamp={end_timestamp}", ) - start_date_string = utc_date_string(start_timestamp) - end_date_string = utc_date_string(end_timestamp) + start_date = utc_date(start_timestamp) + end_date = utc_date(end_timestamp) - if start_date_string == end_date_string: - key_condition_expression = ( - Key("Date").eq(start_date_string) - & Key("Timestamp").between(start_timestamp, end_timestamp) + records_for_window: List[Dict] = [] + current_date = start_date + + while current_date <= end_date: + day_start_ts = utc_day_start_timestamp(current_date) + day_end_ts = utc_day_end_timestamp(current_date) + + effective_start_ts = max(start_timestamp, day_start_ts) + effective_end_ts = min(end_timestamp, day_end_ts) + + key_condition = ( + Key("Date").eq(current_date.isoformat()) + & Key("Timestamp").between(effective_start_ts, effective_end_ts) ) - return self.dynamo_service.query_by_key_condition_expression( + + records_for_day = self.dynamo_service.query_by_key_condition_expression( table_name=self.table_name, index_name=timestamp_index_name, - key_condition_expression=key_condition_expression, + key_condition_expression=key_condition, ) - start_day_key_condition = ( - Key("Date").eq(start_date_string) & Key("Timestamp").gte(start_timestamp) - ) - end_day_key_condition = ( - Key("Date").eq(end_date_string) & Key("Timestamp").lte(end_timestamp) - ) - - start_day_items = self.dynamo_service.query_by_key_condition_expression( - table_name=self.table_name, - index_name=timestamp_index_name, - key_condition_expression=start_day_key_condition, - ) - end_day_items = self.dynamo_service.query_by_key_condition_expression( - table_name=self.table_name, - index_name=timestamp_index_name, - key_condition_expression=end_day_key_condition, - ) + records_for_window.extend(records_for_day) + current_date += timedelta(days=1) - return start_day_items + end_day_items + return records_for_window \ No newline at end of file diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index fb12e223dd..b946c0259d 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -49,6 +49,7 @@ def send_email( def _send_raw(self, msg: MIMEMultipart, to_address: str): logger.info(f"Sending SES raw email to {to_address}") self.ses.send_raw_email( + Source=msg["From"], RawMessage={"Data": msg.as_string()}, Destinations=[to_address], ) diff --git a/lambdas/utils/utilities.py b/lambdas/utils/utilities.py index 92d99d8f02..bb2de83eab 100755 --- a/lambdas/utils/utilities.py +++ b/lambdas/utils/utilities.py @@ -2,7 +2,7 @@ import os import re import uuid -from datetime import datetime, timezone +from datetime import datetime, timezone, date, time from urllib.parse import urlparse from inflection import camelize @@ -130,4 +130,14 @@ def parse_date(date_string: str) -> datetime | None: def utc_date_string(timestamp_seconds: int) -> str: - return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).strftime("%Y-%m-%d") \ No newline at end of file + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).strftime("%Y-%m-%d") + +def utc_date(timestamp_seconds: int) -> date: + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).date() + +def utc_day_start_timestamp(day: date) -> int: + return int( + datetime.combine(day, time.min, tzinfo=timezone.utc).timestamp() + ) +def utc_day_end_timestamp(day: date) -> int: + return utc_day_start_timestamp(day) + 24 * 60 * 60 - 1 \ No newline at end of file From 5d2a08900c0e463943e3d35b00f2bdcf5ca31bb4 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 8 Jan 2026 08:54:51 +0000 Subject: [PATCH 16/60] [PRMP-1057] added logging, removed 1 todo --- lambdas/services/email_service.py | 22 +++++++++++++------ .../reporting/report_distribution_service.py | 4 ++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index b946c0259d..8a11a9bd88 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -26,7 +26,7 @@ def send_email( body_text: str, from_address: str, attachments: Optional[Iterable[str]] = None, - ): + )->MIMEMultipart: msg = MIMEMultipart() msg["Subject"] = subject msg["To"] = to_address @@ -43,17 +43,25 @@ def send_email( filename=attachment_path.split("/")[-1], ) msg.attach(part) + logger.info( + f"Sending email: from={from_address!r}, to={to_address!r}, subject={subject!r}, " + f"attachments={len(list(attachments or []))}" + ) + return self._send_raw(msg, to_address) - self._send_raw(msg, to_address) - - def _send_raw(self, msg: MIMEMultipart, to_address: str): - logger.info(f"Sending SES raw email to {to_address}") - self.ses.send_raw_email( - Source=msg["From"], + def _send_raw(self, msg: MIMEMultipart, to_address: str)->MIMEMultipart: + subject = msg.get("Subject", "") + from_address = msg.get("From", "") + logger.info(f"Sending SES raw email: subject={subject!r}, from={from_address!r}, to={to_address!r}") + resp = self.ses.send_raw_email( + Source=from_address, RawMessage={"Data": msg.as_string()}, Destinations=[to_address], ) + logger.info(f"SES accepted email: subject={subject!r}, message_id={resp.get('MessageId')}") + return resp + def send_report_email( self, *, diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index bbb380faad..f64c632c67 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -96,13 +96,13 @@ def send_report_emails(self, *, ods_code: str, attachment_path: str, password: s ) def email_contact(self, *, to_address: str, attachment_path: str, password: str) -> None: - # TODO delete next line since it is only for testing - to_address = "" + logger.info(f"Sending report email to {to_address}") self.email_service.send_report_email( to_address=to_address, from_address=self.from_address, attachment_path=attachment_path, ) + logger.info(f"Sending password email to {to_address}") self.email_service.send_password_email( to_address=to_address, from_address=self.from_address, From 68f7d3b2ea74cd40d0e4762c76838ac94b409ba1 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 9 Jan 2026 10:37:32 +0000 Subject: [PATCH 17/60] [PRMP-1057] fixed how items are grabbed from contact report table --- lambdas/repositories/reporting/report_contact_repository.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lambdas/repositories/reporting/report_contact_repository.py b/lambdas/repositories/reporting/report_contact_repository.py index 9399e222ee..84bd06cab1 100644 --- a/lambdas/repositories/reporting/report_contact_repository.py +++ b/lambdas/repositories/reporting/report_contact_repository.py @@ -7,10 +7,11 @@ def __init__(self, table_name: str): self.dynamo = DynamoDBService() def get_contact_email(self, ods_code: str) -> str | None: - item = self.dynamo.get_item( + resp = self.dynamo.get_item( table_name=self.table_name, key={"OdsCode": ods_code}, ) + item = (resp or {}).get("Item") if not item: return None return item.get("Email") From cb0fd89c68d3b86247f8bd632d62c372669ae26c Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 9 Jan 2026 10:50:22 +0000 Subject: [PATCH 18/60] [PRMP-1057] fixed tests --- .../handlers/report_orchestration_handler.py | 4 -- .../test_report_contact_repository.py | 12 ++++-- .../test_reporting_dynamo_repository.py | 42 +++++-------------- 3 files changed, 18 insertions(+), 40 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index c8516b730d..be9ea77779 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -60,10 +60,6 @@ def lambda_handler(event, context) -> Dict[str, Any]: ) window_start, window_end = calculate_reporting_window() - # TODO delete line below since it is only for testing - window_start -= 24 * 60 * 60 - window_end += 24 * 60 * 60 - report_date = get_report_date_folder() prefix = f"Report-Orchestration/{report_date}/" diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py index 5cae95f0f2..d534267321 100644 --- a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -20,8 +20,10 @@ def repo(mock_dynamo): def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): mock_dynamo.get_item.return_value = { - "OdsCode": "Y12345", - "Email": "contact@example.com", + "Item": { + "OdsCode": "Y12345", + "Email": "contact@example.com", + } } result = repo.get_contact_email("Y12345") @@ -34,7 +36,7 @@ def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): def test_get_contact_email_returns_none_when_item_missing(repo, mock_dynamo): - mock_dynamo.get_item.return_value = None + mock_dynamo.get_item.return_value = {} # or None result = repo.get_contact_email("Y12345") @@ -47,7 +49,9 @@ def test_get_contact_email_returns_none_when_item_missing(repo, mock_dynamo): def test_get_contact_email_returns_none_when_email_missing(repo, mock_dynamo): mock_dynamo.get_item.return_value = { - "OdsCode": "Y12345", + "Item": { + "OdsCode": "Y12345", + } } result = repo.get_contact_email("Y12345") diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index f6a9ec9568..5697eb112d 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -1,3 +1,4 @@ +from datetime import date from unittest.mock import MagicMock import pytest @@ -19,13 +20,9 @@ def reporting_repo(mock_dynamo_service): return ReportingDynamoRepository(table_name="TestTable") -def test_get_records_for_time_window_same_date_queries_once( - mocker, mock_dynamo_service, reporting_repo -): - mock_utc_date_string = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date_string" - ) - mock_utc_date_string.side_effect = ["2026-01-07", "2026-01-07"] +def test_get_records_for_time_window_same_date_queries_once(mocker, mock_dynamo_service, reporting_repo): + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [date(2026, 1, 7), date(2026, 1, 7)] mock_dynamo_service.query_by_key_condition_expression.return_value = [{"ID": "one"}] @@ -40,13 +37,9 @@ def test_get_records_for_time_window_same_date_queries_once( assert "key_condition_expression" in call_kwargs -def test_get_records_for_time_window_different_dates_queries_twice( - mocker, mock_dynamo_service, reporting_repo -): - mock_utc_date_string = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date_string" - ) - mock_utc_date_string.side_effect = ["2026-01-06", "2026-01-07"] +def test_get_records_for_time_window_different_dates_queries_twice(mocker, mock_dynamo_service, reporting_repo): + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] mock_dynamo_service.query_by_key_condition_expression.side_effect = [ [{"ID": "start-day"}], @@ -58,25 +51,10 @@ def test_get_records_for_time_window_different_dates_queries_twice( assert result == [{"ID": "start-day"}, {"ID": "end-day"}] assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 - first_call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args_list[0].kwargs - second_call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args_list[1].kwargs - - assert first_call_kwargs["table_name"] == "TestTable" - assert first_call_kwargs["index_name"] == "TimestampIndex" - assert "key_condition_expression" in first_call_kwargs - assert second_call_kwargs["table_name"] == "TestTable" - assert second_call_kwargs["index_name"] == "TimestampIndex" - assert "key_condition_expression" in second_call_kwargs - - -def test_get_records_for_time_window_returns_empty_list_when_no_items( - mocker, mock_dynamo_service, reporting_repo -): - mock_utc_date_string = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date_string" - ) - mock_utc_date_string.side_effect = ["2026-01-06", "2026-01-07"] +def test_get_records_for_time_window_returns_empty_list_when_no_items(mocker, mock_dynamo_service, reporting_repo): + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] mock_dynamo_service.query_by_key_condition_expression.side_effect = [[], []] From 5035fed5b4f07e83d8e280d8c325a5a75e983b30 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 9 Jan 2026 14:05:37 +0000 Subject: [PATCH 19/60] [PRMP-1057] removed commented line --- lambdas/handlers/report_distribution_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index d65314b781..30c8547af8 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -54,7 +54,6 @@ def lambda_handler(event, context) -> Dict[str, Any]: logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") return {"bucket": bucket, "prefix": prefix, "keys": keys} - # action == "process_one" key = event["key"] ods_code = service.extract_ods_code_from_key(key) service.process_one_report(ods_code=ods_code, key=key) From 9698a82987f91718b64cee89b4be736eb358df5b Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 12 Jan 2026 15:21:45 +0000 Subject: [PATCH 20/60] [PRMP-1058] Report rejections handling --- .../handlers/report_distribution_handler.py | 6 +- .../handlers/ses_feedback_monitor_handler.py | 29 ++++ lambdas/services/email_service.py | 80 +++++++--- .../reporting/report_distribution_service.py | 45 +++++- .../services/ses_feedback_monitor_service.py | 149 ++++++++++++++++++ 5 files changed, 283 insertions(+), 26 deletions(-) create mode 100644 lambdas/handlers/ses_feedback_monitor_handler.py create mode 100644 lambdas/services/ses_feedback_monitor_service.py diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index 30c8547af8..664a5028ae 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -20,6 +20,7 @@ "CONTACT_TABLE_NAME", "PRM_MAILBOX_EMAIL", "SES_FROM_ADDRESS", + "SES_CONFIGURATION_SET", ] ) @override_error_check @@ -35,9 +36,12 @@ def lambda_handler(event, context) -> Dict[str, Any]: prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] from_address = os.environ["SES_FROM_ADDRESS"] + configuration_set = os.environ["SES_CONFIGURATION_SET"] + s3_service = S3Service() contact_repo = ReportContactRepository(contact_table) - email_service = EmailService() + + email_service = EmailService(default_configuration_set=configuration_set) service = ReportDistributionService( s3_service=s3_service, diff --git a/lambdas/handlers/ses_feedback_monitor_handler.py b/lambdas/handlers/ses_feedback_monitor_handler.py new file mode 100644 index 0000000000..800f58012c --- /dev/null +++ b/lambdas/handlers/ses_feedback_monitor_handler.py @@ -0,0 +1,29 @@ +import os +import boto3 +from typing import Any, Dict + +from services.email_service import EmailService +from services.ses_feedback_monitor_service import SesFeedbackMonitorConfig, SesFeedbackMonitorService + + +def parse_alert_types(configured: str) -> set[str]: + return {s.strip().upper() for s in configured.split(",") if s.strip()} + + +def lambda_handler(event, context) -> Dict[str, Any]: + config = SesFeedbackMonitorConfig( + feedback_bucket=os.environ["SES_FEEDBACK_BUCKET_NAME"], + feedback_prefix=os.environ["SES_FEEDBACK_PREFIX"], + prm_mailbox=os.environ["PRM_MAILBOX_EMAIL"], + from_address=os.environ["SES_FROM_ADDRESS"], + alert_on_event_types=parse_alert_types( + os.environ["ALERT_ON_EVENT_TYPES"] + ), + ) + + service = SesFeedbackMonitorService( + s3_client=boto3.client("s3"), + email_service=EmailService(), + config=config, + ) + return service.process_ses_feedback_event(event) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index 8a11a9bd88..6a9e9b083c 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -2,7 +2,7 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.application import MIMEApplication -from typing import Iterable, Optional +from typing import Iterable, Optional, Dict, Any from utils.audit_logging_setup import LoggingService @@ -15,8 +15,9 @@ class EmailService: Higher-level methods prepare inputs and call send_email(). """ - def __init__(self): + def __init__(self, *, default_configuration_set: Optional[str] = None): self.ses = boto3.client("ses") + self.default_configuration_set = default_configuration_set def send_email( self, @@ -26,7 +27,14 @@ def send_email( body_text: str, from_address: str, attachments: Optional[Iterable[str]] = None, - )->MIMEMultipart: + configuration_set: Optional[str] = None, + tags: Optional[Dict[str, str]] = None, + ) -> Dict[str, Any]: + """ + Sends an email using SES SendRawEmail. + + If configuration_set is not provided, self.default_configuration_set is used (if set). + """ msg = MIMEMultipart() msg["Subject"] = subject msg["To"] = to_address @@ -34,7 +42,8 @@ def send_email( msg.attach(MIMEText(body_text, "plain")) - for attachment_path in attachments or []: + attachment_list = list(attachments or []) + for attachment_path in attachment_list: with open(attachment_path, "rb") as f: part = MIMEApplication(f.read()) part.add_header( @@ -43,22 +52,51 @@ def send_email( filename=attachment_path.split("/")[-1], ) msg.attach(part) + + effective_config_set = configuration_set or self.default_configuration_set + logger.info( f"Sending email: from={from_address!r}, to={to_address!r}, subject={subject!r}, " - f"attachments={len(list(attachments or []))}" + f"attachments={len(attachment_list)}, configuration_set={effective_config_set!r}, tags={tags!r}" + ) + + return self._send_raw( + msg=msg, + to_address=to_address, + configuration_set=effective_config_set, + tags=tags, ) - return self._send_raw(msg, to_address) - def _send_raw(self, msg: MIMEMultipart, to_address: str)->MIMEMultipart: + def _send_raw( + self, + *, + msg: MIMEMultipart, + to_address: str, + configuration_set: Optional[str] = None, + tags: Optional[Dict[str, str]] = None, + ) -> Dict[str, Any]: subject = msg.get("Subject", "") from_address = msg.get("From", "") - logger.info(f"Sending SES raw email: subject={subject!r}, from={from_address!r}, to={to_address!r}") - resp = self.ses.send_raw_email( - Source=from_address, - RawMessage={"Data": msg.as_string()}, - Destinations=[to_address], + + logger.info( + f"Sending SES raw email: subject={subject!r}, from={from_address!r}, to={to_address!r}, " + f"configuration_set={configuration_set!r}, tags={tags!r}" ) + kwargs: Dict[str, Any] = { + "Source": from_address, + "RawMessage": {"Data": msg.as_string()}, + "Destinations": [to_address], + } + + if configuration_set: + kwargs["ConfigurationSetName"] = configuration_set + + if tags: + kwargs["Tags"] = [{"Name": k, "Value": v} for k, v in tags.items()] + + resp = self.ses.send_raw_email(**kwargs) + logger.info(f"SES accepted email: subject={subject!r}, message_id={resp.get('MessageId')}") return resp @@ -68,13 +106,15 @@ def send_report_email( to_address: str, from_address: str, attachment_path: str, - ): - self.send_email( + tags: Optional[Dict[str, str]] = None, + ) -> Dict[str, Any]: + return self.send_email( to_address=to_address, from_address=from_address, subject="Daily Upload Report", body_text="Please find your encrypted daily upload report attached.", attachments=[attachment_path], + tags=tags, ) def send_password_email( @@ -83,12 +123,14 @@ def send_password_email( to_address: str, from_address: str, password: str, - ): - self.send_email( + tags: Optional[Dict[str, str]] = None, + ) -> Dict[str, Any]: + return self.send_email( to_address=to_address, from_address=from_address, subject="Daily Upload Report Password", body_text=f"Password for your report:\n\n{password}", + tags=tags, ) def send_prm_missing_contact_email( @@ -99,8 +141,9 @@ def send_prm_missing_contact_email( ods_code: str, attachment_path: str, password: str, - ): - self.send_email( + tags: Optional[Dict[str, str]] = None, + ) -> Dict[str, Any]: + return self.send_email( to_address=prm_mailbox, from_address=from_address, subject=f"Missing contact for ODS {ods_code}", @@ -110,4 +153,5 @@ def send_prm_missing_contact_email( f"Please resolve the contact and forward the report." ), attachments=[attachment_path], + tags=tags, ) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index f64c632c67..e448fe1b73 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -1,7 +1,6 @@ -import os import secrets import tempfile -from typing import List +from typing import List, Dict import boto3 @@ -51,9 +50,11 @@ def list_xlsx_keys(self, prefix: str) -> List[str]: return keys def process_one_report(self, *, ods_code: str, key: str) -> None: + base_tags: Dict[str, str] = {"ods_code": ods_code, "report_key": key} + with tempfile.TemporaryDirectory() as tmpdir: - local_xlsx = os.path.join(tmpdir, f"{ods_code}.xlsx") - local_zip = os.path.join(tmpdir, f"{ods_code}.zip") + local_xlsx = f"{tmpdir}/{ods_code}.xlsx" + local_zip = f"{tmpdir}/{ods_code}.zip" self.s3_service.download_file(self.bucket, key, local_xlsx) @@ -68,9 +69,17 @@ def process_one_report(self, *, ods_code: str, key: str) -> None: ods_code=ods_code, attachment_path=local_zip, password=password, + base_tags=base_tags, ) - def send_report_emails(self, *, ods_code: str, attachment_path: str, password: str) -> None: + def send_report_emails( + self, + *, + ods_code: str, + attachment_path: str, + password: str, + base_tags: Dict[str, str], + ) -> None: try: contact_email = self.contact_repo.get_contact_email(ods_code) except Exception as e: @@ -85,6 +94,7 @@ def send_report_emails(self, *, ods_code: str, attachment_path: str, password: s to_address=contact_email, attachment_path=attachment_path, password=password, + base_tags=base_tags, ) return @@ -93,29 +103,50 @@ def send_report_emails(self, *, ods_code: str, attachment_path: str, password: s ods_code=ods_code, attachment_path=attachment_path, password=password, + base_tags=base_tags, ) - def email_contact(self, *, to_address: str, attachment_path: str, password: str) -> None: + def email_contact( + self, + *, + to_address: str, + attachment_path: str, + password: str, + base_tags: Dict[str, str], + ) -> None: + tags = {**base_tags, "email": to_address} + logger.info(f"Sending report email to {to_address}") self.email_service.send_report_email( to_address=to_address, from_address=self.from_address, attachment_path=attachment_path, + tags=tags, ) + logger.info(f"Sending password email to {to_address}") self.email_service.send_password_email( to_address=to_address, from_address=self.from_address, password=password, + tags=tags, ) def email_prm_missing_contact( - self, *, ods_code: str, attachment_path: str, password: str + self, + *, + ods_code: str, + attachment_path: str, + password: str, + base_tags: Dict[str, str], ) -> None: + tags = {**base_tags, "email": self.prm_mailbox} + self.email_service.send_prm_missing_contact_email( prm_mailbox=self.prm_mailbox, from_address=self.from_address, ods_code=ods_code, attachment_path=attachment_path, password=password, + tags=tags, ) diff --git a/lambdas/services/ses_feedback_monitor_service.py b/lambdas/services/ses_feedback_monitor_service.py new file mode 100644 index 0000000000..5df4d78557 --- /dev/null +++ b/lambdas/services/ses_feedback_monitor_service.py @@ -0,0 +1,149 @@ +import json +from dataclasses import dataclass +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple, Protocol + +from utils.audit_logging_setup import LoggingService +from services.email_service import EmailService + +logger = LoggingService(__name__) + + +class S3Client(Protocol): + def put_object(self, **kwargs): # pragma: no cover (protocol) + ... + + +@dataclass(frozen=True) +class SesFeedbackMonitorConfig: + feedback_bucket: str + feedback_prefix: str + prm_mailbox: str + from_address: str + alert_on_event_types: set[str] # {"BOUNCE","REJECT"} + + +class SesFeedbackMonitorService: + def __init__(self, *, s3_client: S3Client, email_service: EmailService, config: SesFeedbackMonitorConfig): + self.s3 = s3_client + self.email_service = email_service + self.config = config + + @staticmethod + def parse_sns_message(record: Dict[str, Any]) -> Dict[str, Any]: + msg = record["Sns"]["Message"] + return json.loads(msg) if isinstance(msg, str) else msg + + @staticmethod + def event_type(payload: Dict[str, Any]) -> str: + return (payload.get("eventType") or payload.get("notificationType") or "UNKNOWN").upper() + + @staticmethod + def message_id(payload: Dict[str, Any]) -> str: + mail = payload.get("mail") or {} + return mail.get("messageId") or payload.get("mailMessageId") or "unknown-message-id" + + @staticmethod + def extract_tags(payload: Dict[str, Any]) -> Dict[str, List[str]]: + mail = payload.get("mail") or {} + tags = mail.get("tags") or {} + return tags if isinstance(tags, dict) else {} + + @staticmethod + def extract_recipients_and_diagnostic(payload: Dict[str, Any]) -> Tuple[List[str], Optional[str]]: + recipients: List[str] = [] + diagnostic: Optional[str] = None + + if "bounce" in payload: + b = payload.get("bounce") or {} + bounced = b.get("bouncedRecipients") or [] + for r in bounced: + email_addr = r.get("emailAddress") + if email_addr: + recipients.append(email_addr) + if bounced: + diagnostic = (bounced[0] or {}).get("diagnosticCode") + diagnostic = diagnostic or b.get("smtpResponse") + return recipients, diagnostic + + if "complaint" in payload: + c = payload.get("complaint") or {} + complained = c.get("complainedRecipients") or [] + for r in complained: + email_addr = r.get("emailAddress") + if email_addr: + recipients.append(email_addr) + return recipients, diagnostic + + if "reject" in payload: + r = payload.get("reject") or {} + diagnostic = r.get("reason") or r.get("message") + return recipients, diagnostic + + return recipients, diagnostic + + @staticmethod + def build_s3_key(prefix: str, event_type: str, message_id: str) -> str: + now = datetime.now(timezone.utc) + return f"{prefix.rstrip('/')}/{event_type}/{now:%Y/%m/%d}/{message_id}.json" + + def process_ses_feedback_event(self, event: Dict[str, Any]) -> Dict[str, Any]: + stored = 0 + alerted = 0 + + for record in event.get("Records") or []: + payload = self.parse_sns_message(record) + et = self.event_type(payload) + mid = self.message_id(payload) + + s3_key = self.build_s3_key(self.config.feedback_prefix, et, mid) + + self.s3.put_object( + Bucket=self.config.feedback_bucket, + Key=s3_key, + Body=json.dumps(payload).encode("utf-8"), + ContentType="application/json", + ) + stored += 1 + logger.info(f"Stored SES feedback event: type={et}, message_id={mid}, s3=s3://{self.config.feedback_bucket}/{s3_key}") + + if et in self.config.alert_on_event_types: + subject, body = self.build_prm_email(payload, s3_key) + self.email_service.send_email( + to_address=self.config.prm_mailbox, + from_address=self.config.from_address, + subject=subject, + body_text=body, + ) + alerted += 1 + logger.info(f"Emailed PRM for SES feedback event: type={et}, message_id={mid}") + + return {"status": "ok", "stored": stored, "alerted": alerted} + + def build_prm_email(self, payload: Dict[str, Any], s3_key: str) -> Tuple[str, str]: + et = self.event_type(payload) + mid = self.message_id(payload) + tags = self.extract_tags(payload) + recipients, diagnostic = self.extract_recipients_and_diagnostic(payload) + + ods_code = (tags.get("ods_code") or [None])[0] + report_key = (tags.get("report_key") or [None])[0] + email_tag = (tags.get("email") or [None])[0] + + subject = f"SES {et}: messageId={mid}" + body_lines = [ + f"Event type: {et}", + f"Message ID: {mid}", + f"Affected recipients: {', '.join(recipients) if recipients else '(none parsed)'}", + f"Diagnostic: {diagnostic or '(none parsed)'}", + "", + f"ODS code tag: {ods_code or '(none)'}", + f"Email tag: {email_tag or '(none)'}", + f"Report key tag: {report_key or '(none)'}", + "", + f"Stored at: s3://{self.config.feedback_bucket}/{s3_key}", + "", + "Raw event JSON:", + json.dumps(payload, indent=2, sort_keys=True), + ] + return subject, "\n".join(body_lines) From 583a5b58e59c2d3131c95f80c49e360dabe6550f Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 12 Jan 2026 15:22:52 +0000 Subject: [PATCH 21/60] [PRMP-1058] added lambda layer call --- .../workflows/base-lambdas-reusable-deploy-all.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index 5128113a49..4055ee7b97 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -824,3 +824,17 @@ jobs: lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} + + deploy_ses_feedback_monitor_lambda: + name: Deploy SES Feedback Monitor + uses: ./.github/workflows/base-lambdas-reusable-deploy.yml + with: + environment: ${{ inputs.environment }} + python_version: ${{ inputs.python_version }} + build_branch: ${{ inputs.build_branch }} + sandbox: ${{ inputs.sandbox }} + lambda_handler_name: ses_feedback_monitor_handler + lambda_aws_name: sesFeedbackMonitor + lambda_layer_names: "core_lambda_layer,reports_lambda_layer" + secrets: + AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} \ No newline at end of file From aef729c475d444abd6ad80c1a99f3342b72bdf38 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 14 Jan 2026 15:47:53 +0000 Subject: [PATCH 22/60] [PRMP-1058] Sanitised tags --- .../services/reporting/report_distribution_service.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index e448fe1b73..1c86409b6d 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -1,3 +1,4 @@ +import re import secrets import tempfile from typing import List, Dict @@ -12,6 +13,9 @@ logger = LoggingService(__name__) +_SES_TAG_VALUE_ALLOWED = re.compile(r"[^A-Za-z0-9_\-\.@]") +def _sanitize_ses_tag_value(value: str) -> str: + return _SES_TAG_VALUE_ALLOWED.sub("_", str(value)) class ReportDistributionService: def __init__( @@ -50,8 +54,10 @@ def list_xlsx_keys(self, prefix: str) -> List[str]: return keys def process_one_report(self, *, ods_code: str, key: str) -> None: - base_tags: Dict[str, str] = {"ods_code": ods_code, "report_key": key} - + base_tags: Dict[str, str] = { + "ods_code": _sanitize_ses_tag_value(ods_code), + "report_key": _sanitize_ses_tag_value(key), + } with tempfile.TemporaryDirectory() as tmpdir: local_xlsx = f"{tmpdir}/{ods_code}.xlsx" local_zip = f"{tmpdir}/{ods_code}.zip" From 462209c8de0268f9d012538bae94c64663208108 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 15 Jan 2026 09:44:36 +0000 Subject: [PATCH 23/60] [PRMP-1058] Sanitised tags --- .../handlers/report_orchestration_handler.py | 4 +- .../handlers/ses_feedback_monitor_handler.py | 11 +- .../reporting/reporting_dynamo_repository.py | 16 +- lambdas/services/email_service.py | 10 +- .../reporting/report_distribution_service.py | 6 +- .../services/ses_feedback_monitor_service.py | 34 +- .../test_report_distribution_handler.py | 56 +++- .../test_ses_feedback_monitor_handler.py | 164 ++++++++++ .../services/reporting/test_email_service.py | 125 +++++++- .../test_report_distribution_service.py | 147 +++++++-- .../test_ses_feedback_monitor_service.py | 293 ++++++++++++++++++ 11 files changed, 790 insertions(+), 76 deletions(-) create mode 100644 lambdas/tests/unit/handlers/test_ses_feedback_monitor_handler.py create mode 100644 lambdas/tests/unit/services/test_ses_feedback_monitor_service.py diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index be9ea77779..696ce2e039 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -89,7 +89,9 @@ def lambda_handler(event, context) -> Dict[str, Any]: keys.append(key) logger.info(f"Uploaded report for ODS={ods_code} to s3://{report_bucket}/{key}") - logger.info(f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}") + logger.info( + f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}" + ) return { "report_date": report_date, "bucket": report_bucket, diff --git a/lambdas/handlers/ses_feedback_monitor_handler.py b/lambdas/handlers/ses_feedback_monitor_handler.py index 800f58012c..6f1b0003c5 100644 --- a/lambdas/handlers/ses_feedback_monitor_handler.py +++ b/lambdas/handlers/ses_feedback_monitor_handler.py @@ -1,9 +1,12 @@ import os -import boto3 from typing import Any, Dict +import boto3 from services.email_service import EmailService -from services.ses_feedback_monitor_service import SesFeedbackMonitorConfig, SesFeedbackMonitorService +from services.ses_feedback_monitor_service import ( + SesFeedbackMonitorConfig, + SesFeedbackMonitorService, +) def parse_alert_types(configured: str) -> set[str]: @@ -16,9 +19,7 @@ def lambda_handler(event, context) -> Dict[str, Any]: feedback_prefix=os.environ["SES_FEEDBACK_PREFIX"], prm_mailbox=os.environ["PRM_MAILBOX_EMAIL"], from_address=os.environ["SES_FROM_ADDRESS"], - alert_on_event_types=parse_alert_types( - os.environ["ALERT_ON_EVENT_TYPES"] - ), + alert_on_event_types=parse_alert_types(os.environ["ALERT_ON_EVENT_TYPES"]), ) service = SesFeedbackMonitorService( diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 86d595d87e..0afb2300dd 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -4,10 +4,15 @@ from boto3.dynamodb.conditions import Key from services.base.dynamo_service import DynamoDBService from utils.audit_logging_setup import LoggingService -from utils.utilities import utc_date_string, utc_date, utc_day_start_timestamp, utc_day_end_timestamp +from utils.utilities import ( + utc_date, + utc_day_end_timestamp, + utc_day_start_timestamp, +) logger = LoggingService(__name__) + class ReportingDynamoRepository: def __init__(self, table_name: str): self.table_name = table_name @@ -37,10 +42,9 @@ def get_records_for_time_window( effective_start_ts = max(start_timestamp, day_start_ts) effective_end_ts = min(end_timestamp, day_end_ts) - key_condition = ( - Key("Date").eq(current_date.isoformat()) - & Key("Timestamp").between(effective_start_ts, effective_end_ts) - ) + key_condition = Key("Date").eq(current_date.isoformat()) & Key( + "Timestamp" + ).between(effective_start_ts, effective_end_ts) records_for_day = self.dynamo_service.query_by_key_condition_expression( table_name=self.table_name, @@ -51,4 +55,4 @@ def get_records_for_time_window( records_for_window.extend(records_for_day) current_date += timedelta(days=1) - return records_for_window \ No newline at end of file + return records_for_window diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index 6a9e9b083c..5467d8812f 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -1,9 +1,9 @@ -import boto3 +from email.mime.application import MIMEApplication from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText -from email.mime.application import MIMEApplication -from typing import Iterable, Optional, Dict, Any +from typing import Any, Dict, Iterable, Optional +import boto3 from utils.audit_logging_setup import LoggingService logger = LoggingService(__name__) @@ -97,7 +97,9 @@ def _send_raw( resp = self.ses.send_raw_email(**kwargs) - logger.info(f"SES accepted email: subject={subject!r}, message_id={resp.get('MessageId')}") + logger.info( + f"SES accepted email: subject={subject!r}, message_id={resp.get('MessageId')}" + ) return resp def send_report_email( diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index 1c86409b6d..0afca276bc 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -1,10 +1,9 @@ import re import secrets import tempfile -from typing import List, Dict +from typing import Dict, List import boto3 - from repositories.reporting.report_contact_repository import ReportContactRepository from services.base.s3_service import S3Service from services.email_service import EmailService @@ -14,9 +13,12 @@ logger = LoggingService(__name__) _SES_TAG_VALUE_ALLOWED = re.compile(r"[^A-Za-z0-9_\-\.@]") + + def _sanitize_ses_tag_value(value: str) -> str: return _SES_TAG_VALUE_ALLOWED.sub("_", str(value)) + class ReportDistributionService: def __init__( self, diff --git a/lambdas/services/ses_feedback_monitor_service.py b/lambdas/services/ses_feedback_monitor_service.py index 5df4d78557..6a924f4733 100644 --- a/lambdas/services/ses_feedback_monitor_service.py +++ b/lambdas/services/ses_feedback_monitor_service.py @@ -1,10 +1,10 @@ import json from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Tuple, Protocol +from typing import Any, Dict, List, Optional, Protocol, Tuple -from utils.audit_logging_setup import LoggingService from services.email_service import EmailService +from utils.audit_logging_setup import LoggingService logger = LoggingService(__name__) @@ -24,7 +24,13 @@ class SesFeedbackMonitorConfig: class SesFeedbackMonitorService: - def __init__(self, *, s3_client: S3Client, email_service: EmailService, config: SesFeedbackMonitorConfig): + def __init__( + self, + *, + s3_client: S3Client, + email_service: EmailService, + config: SesFeedbackMonitorConfig, + ): self.s3 = s3_client self.email_service = email_service self.config = config @@ -36,12 +42,18 @@ def parse_sns_message(record: Dict[str, Any]) -> Dict[str, Any]: @staticmethod def event_type(payload: Dict[str, Any]) -> str: - return (payload.get("eventType") or payload.get("notificationType") or "UNKNOWN").upper() + return ( + payload.get("eventType") or payload.get("notificationType") or "UNKNOWN" + ).upper() @staticmethod def message_id(payload: Dict[str, Any]) -> str: mail = payload.get("mail") or {} - return mail.get("messageId") or payload.get("mailMessageId") or "unknown-message-id" + return ( + mail.get("messageId") + or payload.get("mailMessageId") + or "unknown-message-id" + ) @staticmethod def extract_tags(payload: Dict[str, Any]) -> Dict[str, List[str]]: @@ -50,7 +62,9 @@ def extract_tags(payload: Dict[str, Any]) -> Dict[str, List[str]]: return tags if isinstance(tags, dict) else {} @staticmethod - def extract_recipients_and_diagnostic(payload: Dict[str, Any]) -> Tuple[List[str], Optional[str]]: + def extract_recipients_and_diagnostic( + payload: Dict[str, Any] + ) -> Tuple[List[str], Optional[str]]: recipients: List[str] = [] diagnostic: Optional[str] = None @@ -105,7 +119,9 @@ def process_ses_feedback_event(self, event: Dict[str, Any]) -> Dict[str, Any]: ContentType="application/json", ) stored += 1 - logger.info(f"Stored SES feedback event: type={et}, message_id={mid}, s3=s3://{self.config.feedback_bucket}/{s3_key}") + logger.info( + f"Stored SES feedback event: type={et}, message_id={mid}, s3=s3://{self.config.feedback_bucket}/{s3_key}" + ) if et in self.config.alert_on_event_types: subject, body = self.build_prm_email(payload, s3_key) @@ -116,7 +132,9 @@ def process_ses_feedback_event(self, event: Dict[str, Any]) -> Dict[str, Any]: body_text=body, ) alerted += 1 - logger.info(f"Emailed PRM for SES feedback event: type={et}, message_id={mid}") + logger.info( + f"Emailed PRM for SES feedback event: type={et}, message_id={mid}" + ) return {"status": "ok", "stored": stored, "alerted": alerted} diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index d129d2ed36..b0d319cebe 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -1,5 +1,6 @@ import importlib import os + import pytest MODULE_UNDER_TEST = "handlers.report_distribution_handler" @@ -19,6 +20,7 @@ def required_env(mocker): "CONTACT_TABLE_NAME": "contact-table", "PRM_MAILBOX_EMAIL": "prm@example.com", "SES_FROM_ADDRESS": "from@example.com", + "SES_CONFIGURATION_SET": "my-config-set", }, clear=False, ) @@ -61,7 +63,7 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( mocked_s3_cls.assert_called_once_with() mocked_contact_repo_cls.assert_called_once_with("contact-table") - mocked_email_cls.assert_called_once_with() + mocked_email_cls.assert_called_once_with(default_configuration_set="my-config-set") mocked_dist_svc_cls.assert_called_once_with( s3_service=s3_instance, @@ -90,10 +92,24 @@ def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( svc_instance = mocker.Mock() svc_instance.list_xlsx_keys.return_value = [] - mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) - mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) - mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + mocker.patch.object( + handler_module, "S3Service", autospec=True, return_value=mocker.Mock() + ) + mocker.patch.object( + handler_module, + "ReportContactRepository", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch.object( + handler_module, "EmailService", autospec=True, return_value=mocker.Mock() + ) result = handler_module.lambda_handler(event, context) @@ -112,15 +128,33 @@ def test_lambda_handler_process_one_mode_happy_path( svc_instance.extract_ods_code_from_key.return_value = "ABC" svc_instance.process_one_report.return_value = None - mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) - mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) - mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + mocker.patch.object( + handler_module, "S3Service", autospec=True, return_value=mocker.Mock() + ) + mocker.patch.object( + handler_module, + "ReportContactRepository", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch.object( + handler_module, "EmailService", autospec=True, return_value=mocker.Mock() + ) result = handler_module.lambda_handler(event, context) - svc_instance.extract_ods_code_from_key.assert_called_once_with("reports/ABC/whatever.xlsx") - svc_instance.process_one_report.assert_called_once_with(ods_code="ABC", key="reports/ABC/whatever.xlsx") + svc_instance.extract_ods_code_from_key.assert_called_once_with( + "reports/ABC/whatever.xlsx" + ) + svc_instance.process_one_report.assert_called_once_with( + ods_code="ABC", key="reports/ABC/whatever.xlsx" + ) assert result == { "status": "ok", diff --git a/lambdas/tests/unit/handlers/test_ses_feedback_monitor_handler.py b/lambdas/tests/unit/handlers/test_ses_feedback_monitor_handler.py new file mode 100644 index 0000000000..b0d319cebe --- /dev/null +++ b/lambdas/tests/unit/handlers/test_ses_feedback_monitor_handler.py @@ -0,0 +1,164 @@ +import importlib +import os + +import pytest + +MODULE_UNDER_TEST = "handlers.report_distribution_handler" + + +@pytest.fixture +def handler_module(): + return importlib.import_module(MODULE_UNDER_TEST) + + +@pytest.fixture +def required_env(mocker): + mocker.patch.dict( + os.environ, + { + "REPORT_BUCKET_NAME": "my-report-bucket", + "CONTACT_TABLE_NAME": "contact-table", + "PRM_MAILBOX_EMAIL": "prm@example.com", + "SES_FROM_ADDRESS": "from@example.com", + "SES_CONFIGURATION_SET": "my-config-set", + }, + clear=False, + ) + + +def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( + mocker, handler_module, required_env +): + event = {"action": "list", "prefix": "reports/2026-01-01/"} + context = mocker.Mock() + context.aws_request_id = "req-123" # avoid JSON serialization issues in decorators + + s3_instance = mocker.Mock(name="S3ServiceInstance") + contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") + email_instance = mocker.Mock(name="EmailServiceInstance") + + svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") + svc_instance.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] + + mocked_s3_cls = mocker.patch.object( + handler_module, "S3Service", autospec=True, return_value=s3_instance + ) + mocked_contact_repo_cls = mocker.patch.object( + handler_module, + "ReportContactRepository", + autospec=True, + return_value=contact_repo_instance, + ) + mocked_email_cls = mocker.patch.object( + handler_module, "EmailService", autospec=True, return_value=email_instance + ) + mocked_dist_svc_cls = mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + + result = handler_module.lambda_handler(event, context) + + mocked_s3_cls.assert_called_once_with() + mocked_contact_repo_cls.assert_called_once_with("contact-table") + mocked_email_cls.assert_called_once_with(default_configuration_set="my-config-set") + + mocked_dist_svc_cls.assert_called_once_with( + s3_service=s3_instance, + contact_repo=contact_repo_instance, + email_service=email_instance, + bucket="my-report-bucket", + from_address="from@example.com", + prm_mailbox="prm@example.com", + ) + + svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") + assert result == { + "bucket": "my-report-bucket", + "prefix": "reports/2026-01-01/", + "keys": ["a.xlsx", "b.xlsx"], + } + + +def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( + mocker, handler_module, required_env +): + event = {"action": "list", "prefix": "p/", "bucket": "override-bucket"} + context = mocker.Mock() + context.aws_request_id = "req-456" + + svc_instance = mocker.Mock() + svc_instance.list_xlsx_keys.return_value = [] + + mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + mocker.patch.object( + handler_module, "S3Service", autospec=True, return_value=mocker.Mock() + ) + mocker.patch.object( + handler_module, + "ReportContactRepository", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch.object( + handler_module, "EmailService", autospec=True, return_value=mocker.Mock() + ) + + result = handler_module.lambda_handler(event, context) + + svc_instance.list_xlsx_keys.assert_called_once_with(prefix="p/") + assert result == {"bucket": "override-bucket", "prefix": "p/", "keys": []} + + +def test_lambda_handler_process_one_mode_happy_path( + mocker, handler_module, required_env +): + event = {"action": "process_one", "key": "reports/ABC/whatever.xlsx"} + context = mocker.Mock() + context.aws_request_id = "req-789" + + svc_instance = mocker.Mock() + svc_instance.extract_ods_code_from_key.return_value = "ABC" + svc_instance.process_one_report.return_value = None + + mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + mocker.patch.object( + handler_module, "S3Service", autospec=True, return_value=mocker.Mock() + ) + mocker.patch.object( + handler_module, + "ReportContactRepository", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch.object( + handler_module, "EmailService", autospec=True, return_value=mocker.Mock() + ) + + result = handler_module.lambda_handler(event, context) + + svc_instance.extract_ods_code_from_key.assert_called_once_with( + "reports/ABC/whatever.xlsx" + ) + svc_instance.process_one_report.assert_called_once_with( + ods_code="ABC", key="reports/ABC/whatever.xlsx" + ) + + assert result == { + "status": "ok", + "bucket": "my-report-bucket", + "key": "reports/ABC/whatever.xlsx", + "ods_code": "ABC", + } diff --git a/lambdas/tests/unit/services/reporting/test_email_service.py b/lambdas/tests/unit/services/reporting/test_email_service.py index efde7f7251..fe5912834f 100644 --- a/lambdas/tests/unit/services/reporting/test_email_service.py +++ b/lambdas/tests/unit/services/reporting/test_email_service.py @@ -1,5 +1,4 @@ import pytest - from services.email_service import EmailService @@ -10,6 +9,7 @@ def email_service(mocker): svc.ses = mocker.Mock() return svc + def test_send_email_sends_raw_email_without_attachments(email_service, mocker): mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) @@ -22,14 +22,14 @@ def test_send_email_sends_raw_email_without_attachments(email_service, mocker): ) mocked_send_raw.assert_called_once() + call_kwargs = mocked_send_raw.call_args.kwargs - call_args, call_kwargs = mocked_send_raw.call_args - assert call_kwargs == {} + assert set(call_kwargs.keys()) == {"msg", "to_address", "configuration_set", "tags"} + assert call_kwargs["to_address"] == "to@example.com" + assert call_kwargs["configuration_set"] is None + assert call_kwargs["tags"] is None - msg_arg = call_args[0] - to_arg = call_args[1] - - assert to_arg == "to@example.com" + msg_arg = call_kwargs["msg"] assert msg_arg["Subject"] == "Hello" assert msg_arg["To"] == "to@example.com" assert msg_arg["From"] == "from@example.com" @@ -38,6 +38,60 @@ def test_send_email_sends_raw_email_without_attachments(email_service, mocker): assert "Body text" in raw +def test_send_email_uses_default_configuration_set_when_not_provided(mocker): + mocker.patch("services.email_service.boto3.client", autospec=True) + svc = EmailService(default_configuration_set="DEFAULT_CFG") + svc.ses = mocker.Mock() + + mocked_send_raw = mocker.patch.object(svc, "_send_raw", autospec=True) + + svc.send_email( + to_address="to@example.com", + subject="Hello", + body_text="Body text", + from_address="from@example.com", + attachments=None, + configuration_set=None, + ) + + mocked_send_raw.assert_called_once() + assert mocked_send_raw.call_args.kwargs["configuration_set"] == "DEFAULT_CFG" + + +def test_send_email_configuration_set_overrides_default(mocker): + mocker.patch("services.email_service.boto3.client", autospec=True) + svc = EmailService(default_configuration_set="DEFAULT_CFG") + svc.ses = mocker.Mock() + + mocked_send_raw = mocker.patch.object(svc, "_send_raw", autospec=True) + + svc.send_email( + to_address="to@example.com", + subject="Hello", + body_text="Body text", + from_address="from@example.com", + attachments=None, + configuration_set="OVERRIDE_CFG", + ) + + mocked_send_raw.assert_called_once() + assert mocked_send_raw.call_args.kwargs["configuration_set"] == "OVERRIDE_CFG" + + +def test_send_email_passes_tags_through_to_send_raw(email_service, mocker): + mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) + + email_service.send_email( + to_address="to@example.com", + subject="Hello", + body_text="Body text", + from_address="from@example.com", + tags={"k1": "v1", "k2": "v2"}, + ) + + mocked_send_raw.assert_called_once() + assert mocked_send_raw.call_args.kwargs["tags"] == {"k1": "v1", "k2": "v2"} + def test_send_email_attaches_files_and_sets_filenames(email_service, mocker): file_bytes_1 = b"zipbytes1" @@ -64,10 +118,7 @@ def test_send_email_attaches_files_and_sets_filenames(email_service, mocker): mocked_open.assert_any_call("/var/tmp/b.zip", "rb") mocked_send_raw.assert_called_once() - - call_args, call_kwargs = mocked_send_raw.call_args - assert call_kwargs == {} - msg = call_args[0] + msg = mocked_send_raw.call_args.kwargs["msg"] raw = msg.as_string() assert 'filename="a.zip"' in raw @@ -75,25 +126,57 @@ def test_send_email_attaches_files_and_sets_filenames(email_service, mocker): assert "See attached" in raw - -def test_send_raw_calls_ses_send_raw_email(email_service, mocker): +def test_send_raw_calls_ses_send_raw_email_minimal(email_service): from email.mime.multipart import MIMEMultipart + msg = MIMEMultipart() msg["Subject"] = "S" msg["To"] = "to@example.com" msg["From"] = "from@example.com" - email_service._send_raw(msg, "to@example.com") + email_service.ses.send_raw_email.return_value = {"MessageId": "abc123"} + + resp = email_service._send_raw(msg=msg, to_address="to@example.com") + assert resp == {"MessageId": "abc123"} email_service.ses.send_raw_email.assert_called_once() call_kwargs = email_service.ses.send_raw_email.call_args.kwargs + assert call_kwargs["Source"] == "from@example.com" assert call_kwargs["Destinations"] == ["to@example.com"] assert "RawMessage" in call_kwargs assert "Data" in call_kwargs["RawMessage"] assert isinstance(call_kwargs["RawMessage"]["Data"], str) assert "Subject: S" in call_kwargs["RawMessage"]["Data"] + assert "ConfigurationSetName" not in call_kwargs + assert "Tags" not in call_kwargs + + +def test_send_raw_includes_configuration_set_and_tags(email_service): + from email.mime.multipart import MIMEMultipart + + msg = MIMEMultipart() + msg["Subject"] = "S" + msg["To"] = "to@example.com" + msg["From"] = "from@example.com" + + email_service.ses.send_raw_email.return_value = {"MessageId": "abc123"} + + email_service._send_raw( + msg=msg, + to_address="to@example.com", + configuration_set="CFG", + tags={"env": "test", "team": "data"}, + ) + + call_kwargs = email_service.ses.send_raw_email.call_args.kwargs + assert call_kwargs["ConfigurationSetName"] == "CFG" + assert call_kwargs["Tags"] == [ + {"Name": "env", "Value": "test"}, + {"Name": "team", "Value": "data"}, + ] + def test_send_report_email_calls_send_email_with_expected_inputs(email_service, mocker): mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) @@ -102,6 +185,7 @@ def test_send_report_email_calls_send_email_with_expected_inputs(email_service, to_address="to@example.com", from_address="from@example.com", attachment_path="/tmp/report.zip", + tags={"k": "v"}, ) mocked_send_email.assert_called_once_with( @@ -110,16 +194,20 @@ def test_send_report_email_calls_send_email_with_expected_inputs(email_service, subject="Daily Upload Report", body_text="Please find your encrypted daily upload report attached.", attachments=["/tmp/report.zip"], + tags={"k": "v"}, ) -def test_send_password_email_calls_send_email_with_expected_inputs(email_service, mocker): +def test_send_password_email_calls_send_email_with_expected_inputs( + email_service, mocker +): mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) email_service.send_password_email( to_address="to@example.com", from_address="from@example.com", password="pw123", + tags={"k": "v"}, ) mocked_send_email.assert_called_once_with( @@ -127,10 +215,13 @@ def test_send_password_email_calls_send_email_with_expected_inputs(email_service from_address="from@example.com", subject="Daily Upload Report Password", body_text="Password for your report:\n\npw123", + tags={"k": "v"}, ) -def test_send_prm_missing_contact_email_calls_send_email_with_expected_inputs(email_service, mocker): +def test_send_prm_missing_contact_email_calls_send_email_with_expected_inputs( + email_service, mocker +): mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) email_service.send_prm_missing_contact_email( @@ -139,6 +230,7 @@ def test_send_prm_missing_contact_email_calls_send_email_with_expected_inputs(em ods_code="Y12345", attachment_path="/tmp/report.zip", password="pw123", + tags={"k": "v"}, ) mocked_send_email.assert_called_once_with( @@ -151,4 +243,5 @@ def test_send_prm_missing_contact_email_calls_send_email_with_expected_inputs(em "Please resolve the contact and forward the report." ), attachments=["/tmp/report.zip"], + tags={"k": "v"}, ) diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index 56b5150e03..850982db4b 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -1,7 +1,11 @@ import os + import pytest +from services.reporting.report_distribution_service import ( + ReportDistributionService, + _sanitize_ses_tag_value, +) -from services.reporting.report_distribution_service import ReportDistributionService @pytest.fixture def mock_s3_service(mocker): @@ -22,7 +26,9 @@ def mock_email_service(mocker): @pytest.fixture def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): - mocker.patch("services.reporting.report_distribution_service.boto3.client", autospec=True) + mocker.patch( + "services.reporting.report_distribution_service.boto3.client", autospec=True + ) return ReportDistributionService( s3_service=mock_s3_service, @@ -33,18 +39,34 @@ def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): prm_mailbox="prm@example.com", ) + +def test_sanitize_ses_tag_value_replaces_disallowed_chars(): + assert _sanitize_ses_tag_value("A B/C") == "A_B_C" + assert _sanitize_ses_tag_value("x@y.com") == "x@y.com" # @ allowed + assert _sanitize_ses_tag_value("a.b-c_d") == "a.b-c_d" # . - _ allowed + + def test_extract_ods_code_from_key_strips_xlsx_extension(): - assert ReportDistributionService.extract_ods_code_from_key( - "Report-Orchestration/2026-01-01/Y12345.xlsx" - ) == "Y12345" + assert ( + ReportDistributionService.extract_ods_code_from_key( + "Report-Orchestration/2026-01-01/Y12345.xlsx" + ) + == "Y12345" + ) def test_extract_ods_code_from_key_is_case_insensitive(): - assert ReportDistributionService.extract_ods_code_from_key("a/b/C789.XLSX") == "C789" + assert ( + ReportDistributionService.extract_ods_code_from_key("a/b/C789.XLSX") == "C789" + ) def test_extract_ods_code_from_key_keeps_non_xlsx_filename(): - assert ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") == "report.csv" + assert ( + ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") + == "report.csv" + ) + def test_list_xlsx_keys_filters_only_xlsx(service, mocker): paginator = mocker.Mock() @@ -141,14 +163,52 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( password="fixed-password", ) - mocked_send.assert_called_once_with( - ods_code="Y12345", - attachment_path=local_zip, - password="fixed-password", + # Updated: send_report_emails now requires base_tags + mocked_send.assert_called_once() + call_kwargs = mocked_send.call_args.kwargs + assert call_kwargs["ods_code"] == "Y12345" + assert call_kwargs["attachment_path"] == local_zip + assert call_kwargs["password"] == "fixed-password" + assert call_kwargs["base_tags"] == { + "ods_code": "Y12345", + "report_key": "Report-Orchestration_2026-01-01_Y12345.xlsx", + } + + +def test_process_one_report_sanitizes_tags(service, mocker, mock_s3_service): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="pw", + ) + + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, ) + mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + service.process_one_report( + ods_code="Y 12/345", + key="prefix/2026-01-01/Y 12/345.xlsx", + ) + + assert mocked_send.call_args.kwargs["base_tags"] == { + "ods_code": "Y_12_345", + "report_key": "prefix_2026-01-01_Y_12_345.xlsx", + } -def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_service): + +def test_process_one_report_propagates_download_errors( + service, mocker, mock_s3_service +): mock_s3_service.download_file.side_effect = RuntimeError("download failed") td = mocker.MagicMock() @@ -159,7 +219,10 @@ def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_ return_value=td, ) - mocked_zip = mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + autospec=True, + ) mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) with pytest.raises(RuntimeError, match="download failed"): @@ -169,7 +232,7 @@ def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_ mocked_send.assert_not_called() -def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker, mock_s3_service): +def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", return_value="pw", @@ -231,16 +294,22 @@ def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fa mocked_zip.assert_not_called() mocked_send.assert_not_called() -def test_send_report_emails_with_contact_calls_email_contact(service, mock_contact_repo, mocker): + +def test_send_report_emails_with_contact_calls_email_contact( + service, mock_contact_repo, mocker +): mock_contact_repo.get_contact_email.return_value = "contact@example.com" mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + mocked_email_prm = mocker.patch.object( + service, "email_prm_missing_contact", autospec=True + ) service.send_report_emails( ods_code="Y12345", attachment_path="/tmp/Y12345.zip", password="pw", + base_tags={"ods_code": "Y12345", "report_key": "k.xlsx"}, ) mock_contact_repo.get_contact_email.assert_called_once_with("Y12345") @@ -248,20 +317,26 @@ def test_send_report_emails_with_contact_calls_email_contact(service, mock_conta to_address="contact@example.com", attachment_path="/tmp/Y12345.zip", password="pw", + base_tags={"ods_code": "Y12345", "report_key": "k.xlsx"}, ) mocked_email_prm.assert_not_called() -def test_send_report_emails_without_contact_calls_email_prm(service, mock_contact_repo, mocker): +def test_send_report_emails_without_contact_calls_email_prm( + service, mock_contact_repo, mocker +): mock_contact_repo.get_contact_email.return_value = None mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + mocked_email_prm = mocker.patch.object( + service, "email_prm_missing_contact", autospec=True + ) service.send_report_emails( ods_code="A99999", attachment_path="/tmp/A99999.zip", password="pw", + base_tags={"ods_code": "A99999", "report_key": "k.xlsx"}, ) mock_contact_repo.get_contact_email.assert_called_once_with("A99999") @@ -269,20 +344,26 @@ def test_send_report_emails_without_contact_calls_email_prm(service, mock_contac ods_code="A99999", attachment_path="/tmp/A99999.zip", password="pw", + base_tags={"ods_code": "A99999", "report_key": "k.xlsx"}, ) mocked_email_contact.assert_not_called() -def test_send_report_emails_contact_lookup_exception_falls_back_to_prm(service, mock_contact_repo, mocker): +def test_send_report_emails_contact_lookup_exception_falls_back_to_prm( + service, mock_contact_repo, mocker +): mock_contact_repo.get_contact_email.side_effect = RuntimeError("ddb down") mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + mocked_email_prm = mocker.patch.object( + service, "email_prm_missing_contact", autospec=True + ) service.send_report_emails( ods_code="A99999", attachment_path="/tmp/A99999.zip", password="pw", + base_tags={"ods_code": "A99999", "report_key": "k.xlsx"}, ) mocked_email_contact.assert_not_called() @@ -290,29 +371,40 @@ def test_send_report_emails_contact_lookup_exception_falls_back_to_prm(service, ods_code="A99999", attachment_path="/tmp/A99999.zip", password="pw", + base_tags={"ods_code": "A99999", "report_key": "k.xlsx"}, ) -def test_email_contact_sends_report_and_password(service, mock_email_service): +def test_email_contact_sends_report_and_password_with_tags(service, mock_email_service): + base_tags = {"ods_code": "Y12345", "report_key": "k.xlsx"} + service.email_contact( to_address="contact@example.com", attachment_path="/tmp/file.zip", password="pw", + base_tags=base_tags, ) + expected_tags = {**base_tags, "email": "contact@example.com"} + mock_email_service.send_report_email.assert_called_once_with( to_address="contact@example.com", from_address="from@example.com", attachment_path="/tmp/file.zip", + tags=expected_tags, ) mock_email_service.send_password_email.assert_called_once_with( to_address="contact@example.com", from_address="from@example.com", password="pw", + tags=expected_tags, ) -def test_email_contact_sends_password_even_if_report_email_fails(service, mock_email_service): +def test_email_contact_does_not_send_password_if_report_email_fails( + service, mock_email_service +): + base_tags = {"ods_code": "Y12345", "report_key": "k.xlsx"} mock_email_service.send_report_email.side_effect = RuntimeError("SES down") with pytest.raises(RuntimeError, match="SES down"): @@ -320,22 +412,31 @@ def test_email_contact_sends_password_even_if_report_email_fails(service, mock_e to_address="contact@example.com", attachment_path="/tmp/file.zip", password="pw", + base_tags=base_tags, ) mock_email_service.send_password_email.assert_not_called() -def test_email_prm_missing_contact_sends_prm_missing_contact_email(service, mock_email_service): +def test_email_prm_missing_contact_sends_prm_missing_contact_email_with_tags( + service, mock_email_service +): + base_tags = {"ods_code": "X11111", "report_key": "k.xlsx"} + service.email_prm_missing_contact( ods_code="X11111", attachment_path="/tmp/file.zip", password="pw", + base_tags=base_tags, ) + expected_tags = {**base_tags, "email": "prm@example.com"} + mock_email_service.send_prm_missing_contact_email.assert_called_once_with( prm_mailbox="prm@example.com", from_address="from@example.com", ods_code="X11111", attachment_path="/tmp/file.zip", password="pw", + tags=expected_tags, ) diff --git a/lambdas/tests/unit/services/test_ses_feedback_monitor_service.py b/lambdas/tests/unit/services/test_ses_feedback_monitor_service.py new file mode 100644 index 0000000000..7cc164f06c --- /dev/null +++ b/lambdas/tests/unit/services/test_ses_feedback_monitor_service.py @@ -0,0 +1,293 @@ +import json +from datetime import datetime, timezone + +import pytest +from services.ses_feedback_monitor_service import ( + SesFeedbackMonitorConfig, + SesFeedbackMonitorService, +) + + +@pytest.fixture +def config(): + return SesFeedbackMonitorConfig( + feedback_bucket="feedback-bucket", + feedback_prefix="ses/feedback", + prm_mailbox="prm@example.com", + from_address="from@example.com", + alert_on_event_types={"BOUNCE", "REJECT"}, + ) + + +@pytest.fixture +def s3_client(mocker): + return mocker.Mock() + + +@pytest.fixture +def email_service(mocker): + return mocker.Mock() + + +@pytest.fixture +def svc(s3_client, email_service, config): + return SesFeedbackMonitorService( + s3_client=s3_client, email_service=email_service, config=config + ) + + +def test_parse_sns_message_parses_json_string(): + record = {"Sns": {"Message": json.dumps({"a": 1, "eventType": "bounce"})}} + payload = SesFeedbackMonitorService.parse_sns_message(record) + assert payload == {"a": 1, "eventType": "bounce"} + + +def test_parse_sns_message_returns_non_string_message_as_is(): + record = {"Sns": {"Message": {"a": 1}}} + payload = SesFeedbackMonitorService.parse_sns_message(record) + assert payload == {"a": 1} + + +@pytest.mark.parametrize( + "payload, expected", + [ + ({"eventType": "bounce"}, "BOUNCE"), + ({"notificationType": "complaint"}, "COMPLAINT"), + ({"eventType": None, "notificationType": None}, "UNKNOWN"), + ({}, "UNKNOWN"), + ], +) +def test_event_type(payload, expected): + assert SesFeedbackMonitorService.event_type(payload) == expected + + +@pytest.mark.parametrize( + "payload, expected", + [ + ({"mail": {"messageId": "m1"}}, "m1"), + ({"mailMessageId": "legacy"}, "legacy"), + ({}, "unknown-message-id"), + ({"mail": {}}, "unknown-message-id"), + ], +) +def test_message_id(payload, expected): + assert SesFeedbackMonitorService.message_id(payload) == expected + + +def test_extract_tags_returns_dict_or_empty(): + assert SesFeedbackMonitorService.extract_tags({"mail": {"tags": {"k": ["v"]}}}) == { + "k": ["v"] + } + assert ( + SesFeedbackMonitorService.extract_tags({"mail": {"tags": ["not-a-dict"]}}) == {} + ) + assert SesFeedbackMonitorService.extract_tags({"mail": {}}) == {} + assert SesFeedbackMonitorService.extract_tags({}) == {} + + +def test_extract_recipients_and_diagnostic_for_bounce_uses_diagnostic_code_first(): + payload = { + "bounce": { + "bouncedRecipients": [ + {"emailAddress": "a@example.com", "diagnosticCode": "550 5.1.1 bad"}, + {"emailAddress": "b@example.com"}, + ], + "smtpResponse": "fallback smtp", + } + } + recipients, diagnostic = ( + SesFeedbackMonitorService.extract_recipients_and_diagnostic(payload) + ) + assert recipients == ["a@example.com", "b@example.com"] + assert diagnostic == "550 5.1.1 bad" + + +def test_extract_recipients_and_diagnostic_for_bounce_falls_back_to_smtp_response(): + payload = { + "bounce": { + "bouncedRecipients": [{"emailAddress": "a@example.com"}], + "smtpResponse": "smtp fallback", + } + } + recipients, diagnostic = ( + SesFeedbackMonitorService.extract_recipients_and_diagnostic(payload) + ) + assert recipients == ["a@example.com"] + assert diagnostic == "smtp fallback" + + +def test_extract_recipients_and_diagnostic_for_complaint(): + payload = { + "complaint": {"complainedRecipients": [{"emailAddress": "x@example.com"}]} + } + recipients, diagnostic = ( + SesFeedbackMonitorService.extract_recipients_and_diagnostic(payload) + ) + assert recipients == ["x@example.com"] + assert diagnostic is None + + +def test_extract_recipients_and_diagnostic_for_reject(): + payload = {"reject": {"reason": "Bad content"}} + recipients, diagnostic = ( + SesFeedbackMonitorService.extract_recipients_and_diagnostic(payload) + ) + assert recipients == [] + assert diagnostic == "Bad content" + + +def test_extract_recipients_and_diagnostic_unknown_payload(): + recipients, diagnostic = ( + SesFeedbackMonitorService.extract_recipients_and_diagnostic({"eventType": "x"}) + ) + assert recipients == [] + assert diagnostic is None + + +def test_build_s3_key_uses_prefix_and_date_and_message_id(mocker): + + fixed_now = datetime(2026, 1, 15, 12, 34, 56, tzinfo=timezone.utc) + + module = __import__(SesFeedbackMonitorService.__module__, fromlist=["datetime"]) + mock_dt = mocker.patch.object(module, "datetime", autospec=True) + mock_dt.now.return_value = fixed_now + + key = SesFeedbackMonitorService.build_s3_key( + prefix="ses/feedback/", + event_type="BOUNCE", + message_id="mid123", + ) + + assert key == "ses/feedback/BOUNCE/2026/01/15/mid123.json" + + +def test_build_s3_key_strips_trailing_slash(mocker): + + fixed_now = datetime(2026, 1, 15, 0, 0, 0, tzinfo=timezone.utc) + + module = __import__(SesFeedbackMonitorService.__module__, fromlist=["datetime"]) + mock_dt = mocker.patch.object(module, "datetime", autospec=True) + mock_dt.now.return_value = fixed_now + + key = SesFeedbackMonitorService.build_s3_key( + prefix="pfx////", event_type="REJECT", message_id="m" + ) + + assert key == "pfx/REJECT/2026/01/15/m.json" + + +def test_build_prm_email_includes_expected_fields(svc): + payload = { + "eventType": "bounce", + "mail": { + "messageId": "m-1", + "tags": { + "ods_code": ["Y12345"], + "report_key": ["Report-Orchestration/2026-01-01/Y12345.xlsx"], + "email": ["contact@example.com"], + }, + }, + "bounce": { + "bouncedRecipients": [ + {"emailAddress": "a@example.com", "diagnosticCode": "550 bad"} + ] + }, + } + + subject, body = svc.build_prm_email( + payload, s3_key="ses/feedback/BOUNCE/2026/01/15/m-1.json" + ) + + assert subject == "SES BOUNCE: messageId=m-1" + assert "Event type: BOUNCE" in body + assert "Message ID: m-1" in body + assert "Affected recipients: a@example.com" in body + assert "Diagnostic: 550 bad" in body + assert "ODS code tag: Y12345" in body + assert "Email tag: contact@example.com" in body + assert "Report key tag: Report-Orchestration/2026-01-01/Y12345.xlsx" in body + assert ( + "Stored at: s3://feedback-bucket/ses/feedback/BOUNCE/2026/01/15/m-1.json" + in body + ) + assert "Raw event JSON:" in body + assert '"eventType": "bounce"' in body + + +def test_process_ses_feedback_event_stores_each_record_and_alerts_only_configured_types( + svc, s3_client, email_service, mocker +): + mocker.patch.object( + SesFeedbackMonitorService, + "build_s3_key", + return_value="ses/feedback/BOUNCE/2026/01/15/m.json", + ) + + bounce_payload = { + "eventType": "bounce", + "mail": {"messageId": "m"}, + "bounce": {"bouncedRecipients": []}, + } + complaint_payload = { + "notificationType": "complaint", + "mail": {"messageId": "c"}, + "complaint": {"complainedRecipients": []}, + } + + event = { + "Records": [ + {"Sns": {"Message": json.dumps(bounce_payload)}}, + {"Sns": {"Message": json.dumps(complaint_payload)}}, + ] + } + + mocker.patch.object(svc, "build_prm_email", return_value=("subj", "body")) + + resp = svc.process_ses_feedback_event(event) + + assert resp == {"status": "ok", "stored": 2, "alerted": 1} + + assert s3_client.put_object.call_count == 2 + put1 = s3_client.put_object.call_args_list[0].kwargs + assert put1["Bucket"] == "feedback-bucket" + assert put1["Key"] == "ses/feedback/BOUNCE/2026/01/15/m.json" + assert put1["ContentType"] == "application/json" + assert json.loads(put1["Body"].decode("utf-8")) == bounce_payload + + email_service.send_email.assert_called_once_with( + to_address="prm@example.com", + from_address="from@example.com", + subject="subj", + body_text="body", + ) + + +def test_process_ses_feedback_event_handles_empty_records( + svc, s3_client, email_service +): + resp = svc.process_ses_feedback_event({"Records": []}) + assert resp == {"status": "ok", "stored": 0, "alerted": 0} + s3_client.put_object.assert_not_called() + email_service.send_email.assert_not_called() + + +def test_process_ses_feedback_event_message_can_be_dict_not_json_string( + svc, s3_client, email_service, mocker +): + mocker.patch.object( + SesFeedbackMonitorService, "build_s3_key", return_value="k.json" + ) + mocker.patch.object(svc, "build_prm_email", return_value=("s", "b")) + + payload = { + "eventType": "reject", + "mail": {"messageId": "m"}, + "reject": {"reason": "nope"}, + } + event = {"Records": [{"Sns": {"Message": payload}}]} + + resp = svc.process_ses_feedback_event(event) + + assert resp == {"status": "ok", "stored": 1, "alerted": 1} + s3_client.put_object.assert_called_once() + email_service.send_email.assert_called_once() From 0264411fc294e2b30facc4bbf12137a589443d53 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 15 Jan 2026 10:45:01 +0000 Subject: [PATCH 24/60] [PRMP-1057] removed redundancy in error message --- lambdas/services/reporting/report_distribution_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index f64c632c67..9548ddbca0 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -75,7 +75,7 @@ def send_report_emails(self, *, ods_code: str, attachment_path: str, password: s contact_email = self.contact_repo.get_contact_email(ods_code) except Exception as e: logger.exception( - f"Contact lookup failed for ODS={ods_code}; falling back to PRM. Error: {e}" + f"Contact lookup failed for ODS={ods_code}; falling back to PRM." ) contact_email = None From 0ff40e1af828c95e10c23779ab93e387cac34011 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 15 Jan 2026 15:25:55 +0000 Subject: [PATCH 25/60] [PRMP-1058] updated service logic --- .../handlers/ses_feedback_monitor_handler.py | 6 +- lambdas/services/base/s3_service.py | 18 ++ .../services/ses_feedback_monitor_service.py | 24 ++- .../test_ses_feedback_monitor_handler.py | 165 +++++------------- .../unit/services/base/test_s3_service.py | 106 ++++++++--- .../test_ses_feedback_monitor_service.py | 30 ++-- 6 files changed, 175 insertions(+), 174 deletions(-) diff --git a/lambdas/handlers/ses_feedback_monitor_handler.py b/lambdas/handlers/ses_feedback_monitor_handler.py index 6f1b0003c5..0e254688d6 100644 --- a/lambdas/handlers/ses_feedback_monitor_handler.py +++ b/lambdas/handlers/ses_feedback_monitor_handler.py @@ -1,7 +1,7 @@ import os from typing import Any, Dict -import boto3 +from services.base.s3_service import S3Service from services.email_service import EmailService from services.ses_feedback_monitor_service import ( SesFeedbackMonitorConfig, @@ -22,8 +22,10 @@ def lambda_handler(event, context) -> Dict[str, Any]: alert_on_event_types=parse_alert_types(os.environ["ALERT_ON_EVENT_TYPES"]), ) + s3_service = S3Service() + service = SesFeedbackMonitorService( - s3_client=boto3.client("s3"), + s3_service=s3_service, email_service=EmailService(), config=config, ) diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index 3ee42b4e24..e0f2e4a7e5 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -1,4 +1,5 @@ import io +import json from datetime import datetime, timedelta, timezone from io import BytesIO from typing import Any, Mapping @@ -27,6 +28,8 @@ def __new__(cls, *args, **kwargs): def __init__(self, custom_aws_role=None): if not self.initialised: self.config = BotoConfig( + connect_timeout=3, + read_timeout=5, retries={"max_attempts": 3, "mode": "standard"}, s3={"addressing_style": "virtual"}, signature_version="s3v4", @@ -43,6 +46,21 @@ def __init__(self, custom_aws_role=None): self.custom_aws_role, "s3", config=self.config ) + def put_json( + self, + bucket: str, + key: str, + payload: Mapping[str, Any], + *, + content_type: str = "application/json", + ): + return self.client.put_object( + Bucket=bucket, + Key=key, + Body=json.dumps(payload).encode("utf-8"), + ContentType=content_type, + ) + # S3 Location should be a minimum of a s3_object_key but can also be a directory location in the form of # {{directory}}/{{s3_object_key}} def create_upload_presigned_url(self, s3_bucket_name: str, s3_object_location: str): diff --git a/lambdas/services/ses_feedback_monitor_service.py b/lambdas/services/ses_feedback_monitor_service.py index 6a924f4733..31e1caedcf 100644 --- a/lambdas/services/ses_feedback_monitor_service.py +++ b/lambdas/services/ses_feedback_monitor_service.py @@ -1,19 +1,15 @@ import json from dataclasses import dataclass from datetime import datetime, timezone -from typing import Any, Dict, List, Optional, Protocol, Tuple +from typing import Any, Dict, List, Optional, Tuple +from services.base.s3_service import S3Service from services.email_service import EmailService from utils.audit_logging_setup import LoggingService logger = LoggingService(__name__) -class S3Client(Protocol): - def put_object(self, **kwargs): # pragma: no cover (protocol) - ... - - @dataclass(frozen=True) class SesFeedbackMonitorConfig: feedback_bucket: str @@ -27,11 +23,11 @@ class SesFeedbackMonitorService: def __init__( self, *, - s3_client: S3Client, + s3_service: S3Service, email_service: EmailService, config: SesFeedbackMonitorConfig, ): - self.s3 = s3_client + self.s3 = s3_service self.email_service = email_service self.config = config @@ -112,15 +108,15 @@ def process_ses_feedback_event(self, event: Dict[str, Any]) -> Dict[str, Any]: s3_key = self.build_s3_key(self.config.feedback_prefix, et, mid) - self.s3.put_object( - Bucket=self.config.feedback_bucket, - Key=s3_key, - Body=json.dumps(payload).encode("utf-8"), - ContentType="application/json", + self.s3.put_json( + self.config.feedback_bucket, + s3_key, + payload, ) stored += 1 logger.info( - f"Stored SES feedback event: type={et}, message_id={mid}, s3=s3://{self.config.feedback_bucket}/{s3_key}" + f"Stored SES feedback event: type={et}, message_id={mid}, " + f"s3=s3://{self.config.feedback_bucket}/{s3_key}" ) if et in self.config.alert_on_event_types: diff --git a/lambdas/tests/unit/handlers/test_ses_feedback_monitor_handler.py b/lambdas/tests/unit/handlers/test_ses_feedback_monitor_handler.py index b0d319cebe..6251d0258e 100644 --- a/lambdas/tests/unit/handlers/test_ses_feedback_monitor_handler.py +++ b/lambdas/tests/unit/handlers/test_ses_feedback_monitor_handler.py @@ -3,7 +3,7 @@ import pytest -MODULE_UNDER_TEST = "handlers.report_distribution_handler" +MODULE_UNDER_TEST = "handlers.ses_feedback_monitor_handler" @pytest.fixture @@ -16,45 +16,42 @@ def required_env(mocker): mocker.patch.dict( os.environ, { - "REPORT_BUCKET_NAME": "my-report-bucket", - "CONTACT_TABLE_NAME": "contact-table", + "SES_FEEDBACK_BUCKET_NAME": "my-feedback-bucket", + "SES_FEEDBACK_PREFIX": "ses-feedback/", "PRM_MAILBOX_EMAIL": "prm@example.com", "SES_FROM_ADDRESS": "from@example.com", - "SES_CONFIGURATION_SET": "my-config-set", + "ALERT_ON_EVENT_TYPES": "BOUNCE, REJECT", }, clear=False, ) -def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( +def test_lambda_handler_wires_dependencies_and_returns_service_result( mocker, handler_module, required_env ): - event = {"action": "list", "prefix": "reports/2026-01-01/"} + event = {"Records": []} context = mocker.Mock() - context.aws_request_id = "req-123" # avoid JSON serialization issues in decorators + context.aws_request_id = "req-123" s3_instance = mocker.Mock(name="S3ServiceInstance") - contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") email_instance = mocker.Mock(name="EmailServiceInstance") - svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") - svc_instance.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] + svc_instance = mocker.Mock(name="SesFeedbackMonitorServiceInstance") + svc_instance.process_ses_feedback_event.return_value = { + "status": "ok", + "stored": 1, + "alerted": 0, + } mocked_s3_cls = mocker.patch.object( handler_module, "S3Service", autospec=True, return_value=s3_instance ) - mocked_contact_repo_cls = mocker.patch.object( - handler_module, - "ReportContactRepository", - autospec=True, - return_value=contact_repo_instance, - ) mocked_email_cls = mocker.patch.object( handler_module, "EmailService", autospec=True, return_value=email_instance ) - mocked_dist_svc_cls = mocker.patch.object( + mocked_svc_cls = mocker.patch.object( handler_module, - "ReportDistributionService", + "SesFeedbackMonitorService", autospec=True, return_value=svc_instance, ) @@ -62,103 +59,35 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( result = handler_module.lambda_handler(event, context) mocked_s3_cls.assert_called_once_with() - mocked_contact_repo_cls.assert_called_once_with("contact-table") - mocked_email_cls.assert_called_once_with(default_configuration_set="my-config-set") - - mocked_dist_svc_cls.assert_called_once_with( - s3_service=s3_instance, - contact_repo=contact_repo_instance, - email_service=email_instance, - bucket="my-report-bucket", - from_address="from@example.com", - prm_mailbox="prm@example.com", - ) - - svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") - assert result == { - "bucket": "my-report-bucket", - "prefix": "reports/2026-01-01/", - "keys": ["a.xlsx", "b.xlsx"], - } - - -def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( - mocker, handler_module, required_env -): - event = {"action": "list", "prefix": "p/", "bucket": "override-bucket"} - context = mocker.Mock() - context.aws_request_id = "req-456" - - svc_instance = mocker.Mock() - svc_instance.list_xlsx_keys.return_value = [] - - mocker.patch.object( - handler_module, - "ReportDistributionService", - autospec=True, - return_value=svc_instance, - ) - mocker.patch.object( - handler_module, "S3Service", autospec=True, return_value=mocker.Mock() - ) - mocker.patch.object( - handler_module, - "ReportContactRepository", - autospec=True, - return_value=mocker.Mock(), - ) - mocker.patch.object( - handler_module, "EmailService", autospec=True, return_value=mocker.Mock() - ) - - result = handler_module.lambda_handler(event, context) - - svc_instance.list_xlsx_keys.assert_called_once_with(prefix="p/") - assert result == {"bucket": "override-bucket", "prefix": "p/", "keys": []} - - -def test_lambda_handler_process_one_mode_happy_path( - mocker, handler_module, required_env -): - event = {"action": "process_one", "key": "reports/ABC/whatever.xlsx"} - context = mocker.Mock() - context.aws_request_id = "req-789" - - svc_instance = mocker.Mock() - svc_instance.extract_ods_code_from_key.return_value = "ABC" - svc_instance.process_one_report.return_value = None - - mocker.patch.object( - handler_module, - "ReportDistributionService", - autospec=True, - return_value=svc_instance, - ) - mocker.patch.object( - handler_module, "S3Service", autospec=True, return_value=mocker.Mock() - ) - mocker.patch.object( - handler_module, - "ReportContactRepository", - autospec=True, - return_value=mocker.Mock(), - ) - mocker.patch.object( - handler_module, "EmailService", autospec=True, return_value=mocker.Mock() - ) - - result = handler_module.lambda_handler(event, context) - - svc_instance.extract_ods_code_from_key.assert_called_once_with( - "reports/ABC/whatever.xlsx" - ) - svc_instance.process_one_report.assert_called_once_with( - ods_code="ABC", key="reports/ABC/whatever.xlsx" - ) - - assert result == { - "status": "ok", - "bucket": "my-report-bucket", - "key": "reports/ABC/whatever.xlsx", - "ods_code": "ABC", - } + mocked_email_cls.assert_called_once_with() + + mocked_svc_cls.assert_called_once() + _, kwargs = mocked_svc_cls.call_args + + assert kwargs["s3_service"] is s3_instance + assert kwargs["email_service"] is email_instance + + cfg = kwargs["config"] + assert cfg.feedback_bucket == "my-feedback-bucket" + assert cfg.feedback_prefix == "ses-feedback/" + assert cfg.prm_mailbox == "prm@example.com" + assert cfg.from_address == "from@example.com" + assert cfg.alert_on_event_types == {"BOUNCE", "REJECT"} + + svc_instance.process_ses_feedback_event.assert_called_once_with(event) + assert result == {"status": "ok", "stored": 1, "alerted": 0} + + +@pytest.mark.parametrize( + "configured, expected", + [ + ("BOUNCE,REJECT", {"BOUNCE", "REJECT"}), + (" bounce , reject ", {"BOUNCE", "REJECT"}), + ("BOUNCE,, ,REJECT,", {"BOUNCE", "REJECT"}), + ("", set()), + (" ", set()), + ("complaint", {"COMPLAINT"}), + ], +) +def test_parse_alert_types(handler_module, configured, expected): + assert handler_module.parse_alert_types(configured) == expected diff --git a/lambdas/tests/unit/services/base/test_s3_service.py b/lambdas/tests/unit/services/base/test_s3_service.py index b113024b3c..e325ee4962 100755 --- a/lambdas/tests/unit/services/base/test_s3_service.py +++ b/lambdas/tests/unit/services/base/test_s3_service.py @@ -1,4 +1,5 @@ import datetime +import json from io import BytesIO import pytest @@ -21,12 +22,6 @@ from utils.exceptions import TagNotFoundException TEST_DOWNLOAD_PATH = "test_path" -MOCK_EVENT_BODY = { - "resourceType": "DocumentReference", - "subject": {"identifier": {"value": 111111000}}, - "content": [{"attachment": {"contentType": "application/pdf"}}], - "description": "test_filename.pdf", -} def flatten(list_of_lists): @@ -37,12 +32,16 @@ def flatten(list_of_lists): @freeze_time("2023-10-30T10:25:00") @pytest.fixture def mock_service(mocker, set_env): - mocker.patch("boto3.client") + S3Service._instance = None + + mock_boto_client = mocker.patch("boto3.client") mocker.patch("services.base.iam_service.IAMService") + service = S3Service(custom_aws_role="mock_arn_custom_role") service.expiration_time = datetime.datetime.now( datetime.timezone.utc ) + datetime.timedelta(hours=1) + yield service S3Service._instance = None @@ -65,6 +64,43 @@ def mock_list_objects_paginate(mock_client): return mock_paginator_method +def test_s3_service_constructs_boto_client_with_timeouts(mocker): + S3Service._instance = None + mocked_boto_client = mocker.patch("boto3.client") + + _ = S3Service() + + mocked_boto_client.assert_called_once() + _, kwargs = mocked_boto_client.call_args + assert kwargs["config"].connect_timeout == 3 + assert kwargs["config"].read_timeout == 5 + + S3Service._instance = None + + +def test_put_json_calls_put_object_with_encoded_json(mock_service, mock_client): + payload = {"a": 1, "b": {"c": 2}} + + mock_service.put_json("bucket", "key.json", payload) + + mock_client.put_object.assert_called_once() + _, kwargs = mock_client.put_object.call_args + + assert kwargs["Bucket"] == "bucket" + assert kwargs["Key"] == "key.json" + assert kwargs["ContentType"] == "application/json" + assert json.loads(kwargs["Body"].decode("utf-8")) == payload + + +def test_put_json_allows_custom_content_type(mock_service, mock_client): + payload = {"hello": "world"} + + mock_service.put_json("bucket", "key", payload, content_type="application/x-ndjson") + + _, kwargs = mock_client.put_object.call_args + assert kwargs["ContentType"] == "application/x-ndjson" + + def test_create_upload_presigned_url(mock_service, mocker, mock_custom_client): mock_custom_client.generate_presigned_post.return_value = ( MOCK_PRESIGNED_URL_RESPONSE @@ -89,6 +125,7 @@ def test_create_download_presigned_url(mock_service, mocker, mock_custom_client) mock_custom_client, mock_service.expiration_time, ) + response = mock_service.create_download_presigned_url(MOCK_BUCKET, TEST_FILE_KEY) assert response == MOCK_PRESIGNED_URL_RESPONSE @@ -160,7 +197,7 @@ def test_copy_across_bucket_if_none_match(mock_service, mock_client): def test_delete_object(mock_service, mock_client): mock_service.delete_object(s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME) - mock_client.delete_object_assert_called_once_with( + mock_client.delete_object.assert_called_once_with( Bucket=MOCK_BUCKET, Key=TEST_FILE_NAME ) @@ -200,8 +237,7 @@ def test_get_tag_value(mock_service, mock_client): actual = mock_service.get_tag_value( s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME, tag_key=test_tag_key ) - expected = test_tag_value - assert actual == expected + assert actual == test_tag_value mock_client.get_object_tagging.assert_called_once_with( Bucket=MOCK_BUCKET, @@ -245,11 +281,12 @@ def test_file_exist_on_s3_return_true_if_object_exists(mock_service, mock_client mock_client.head_object.return_value = mock_response - expected = True - actual = mock_service.file_exist_on_s3( - s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME + assert ( + mock_service.file_exist_on_s3( + s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME + ) + is True ) - assert actual == expected mock_client.head_object.assert_called_once_with( Bucket=MOCK_BUCKET, @@ -267,13 +304,13 @@ def test_file_exist_on_s3_return_false_if_object_does_not_exist( mock_client.head_object.side_effect = mock_error - expected = False - actual = mock_service.file_exist_on_s3( - s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME + assert ( + mock_service.file_exist_on_s3( + s3_bucket_name=MOCK_BUCKET, file_key=TEST_FILE_NAME + ) + is False ) - assert actual == expected - mock_client.head_object.assert_called_with( Bucket=MOCK_BUCKET, Key=TEST_FILE_NAME, @@ -302,6 +339,7 @@ def test_file_exist_on_s3_raises_client_error_if_unexpected_response( def test_s3_service_singleton_instance(mocker): + S3Service._instance = None mocker.patch("boto3.client") instance_1 = S3Service() @@ -309,8 +347,11 @@ def test_s3_service_singleton_instance(mocker): assert instance_1 is instance_2 + S3Service._instance = None + def test_not_created_presigned_url_without_custom_client(mocker): + S3Service._instance = None mocker.patch("boto3.client") mock_service = S3Service() @@ -318,8 +359,11 @@ def test_not_created_presigned_url_without_custom_client(mocker): assert response is None + S3Service._instance = None + def test_not_created_custom_client_without_client_role(mocker): + S3Service._instance = None mocker.patch("boto3.client") iam_service = mocker.patch("services.base.iam_service.IAMService") @@ -328,6 +372,8 @@ def test_not_created_custom_client_without_client_role(mocker): iam_service.assert_not_called() assert mock_service.custom_client is None + S3Service._instance = None + @freeze_time("2023-10-30T10:25:00") def test_created_custom_client_when_client_role_is_passed(mocker): @@ -351,6 +397,8 @@ def test_created_custom_client_when_client_role_is_passed(mocker): assert mock_service.custom_client == custom_client_mock iam_service_instance.assume_role.assert_called() + S3Service._instance = None + def test_list_all_objects_return_a_list_of_file_details( mock_service, mock_client, mock_list_objects_paginate @@ -428,7 +476,7 @@ def test_save_or_create_file(mock_service, mock_client): mock_service.save_or_create_file(MOCK_BUCKET, TEST_FILE_NAME, body) mock_client.put_object.assert_called() - args, kwargs = mock_client.put_object.call_args + _, kwargs = mock_client.put_object.call_args assert kwargs["Bucket"] == MOCK_BUCKET assert kwargs["Key"] == TEST_FILE_NAME @@ -442,6 +490,13 @@ def test_returns_binary_file_content_when_file_exists( "Body": mocker.Mock(read=lambda: b"file-content") } + body = mock_service.get_object_stream(bucket=MOCK_BUCKET, key=TEST_FILE_KEY) + assert body.read() == b"file-content" + + mock_client.get_object.assert_called_once_with( + Bucket=MOCK_BUCKET, Key=TEST_FILE_KEY + ) + def test_raises_exception_when_file_does_not_exist(mock_service, mock_client): mock_client.get_object.side_effect = MOCK_CLIENT_ERROR @@ -570,12 +625,15 @@ def test_copy_across_bucket_retries_on_409_conflict(mock_service, mock_client): mock_client.copy_object.side_effect = [ ClientError( { - "Error": {"Code": "PreconditionFailed", "Message": "Precondition Failed"}, - "ResponseMetadata": {"HTTPStatusCode": 409} + "Error": { + "Code": "PreconditionFailed", + "Message": "Precondition Failed", + }, + "ResponseMetadata": {"HTTPStatusCode": 409}, }, - "CopyObject" + "CopyObject", ), - {"CopyObjectResult": {"ETag": "mock-etag"}} # Success on retry + {"CopyObjectResult": {"ETag": "mock-etag"}}, # Success on retry ] mock_service.copy_across_bucket( diff --git a/lambdas/tests/unit/services/test_ses_feedback_monitor_service.py b/lambdas/tests/unit/services/test_ses_feedback_monitor_service.py index 7cc164f06c..0384afde2b 100644 --- a/lambdas/tests/unit/services/test_ses_feedback_monitor_service.py +++ b/lambdas/tests/unit/services/test_ses_feedback_monitor_service.py @@ -20,7 +20,7 @@ def config(): @pytest.fixture -def s3_client(mocker): +def s3_service(mocker): return mocker.Mock() @@ -30,9 +30,9 @@ def email_service(mocker): @pytest.fixture -def svc(s3_client, email_service, config): +def svc(s3_service, email_service, config): return SesFeedbackMonitorService( - s3_client=s3_client, email_service=email_service, config=config + s3_service=s3_service, email_service=email_service, config=config ) @@ -145,7 +145,6 @@ def test_extract_recipients_and_diagnostic_unknown_payload(): def test_build_s3_key_uses_prefix_and_date_and_message_id(mocker): - fixed_now = datetime(2026, 1, 15, 12, 34, 56, tzinfo=timezone.utc) module = __import__(SesFeedbackMonitorService.__module__, fromlist=["datetime"]) @@ -162,7 +161,6 @@ def test_build_s3_key_uses_prefix_and_date_and_message_id(mocker): def test_build_s3_key_strips_trailing_slash(mocker): - fixed_now = datetime(2026, 1, 15, 0, 0, 0, tzinfo=timezone.utc) module = __import__(SesFeedbackMonitorService.__module__, fromlist=["datetime"]) @@ -215,7 +213,7 @@ def test_build_prm_email_includes_expected_fields(svc): def test_process_ses_feedback_event_stores_each_record_and_alerts_only_configured_types( - svc, s3_client, email_service, mocker + svc, s3_service, email_service, mocker ): mocker.patch.object( SesFeedbackMonitorService, @@ -247,12 +245,12 @@ def test_process_ses_feedback_event_stores_each_record_and_alerts_only_configure assert resp == {"status": "ok", "stored": 2, "alerted": 1} - assert s3_client.put_object.call_count == 2 - put1 = s3_client.put_object.call_args_list[0].kwargs - assert put1["Bucket"] == "feedback-bucket" - assert put1["Key"] == "ses/feedback/BOUNCE/2026/01/15/m.json" - assert put1["ContentType"] == "application/json" - assert json.loads(put1["Body"].decode("utf-8")) == bounce_payload + assert s3_service.put_json.call_count == 2 + + first_call = s3_service.put_json.call_args_list[0] + assert first_call.args[0] == "feedback-bucket" + assert first_call.args[1] == "ses/feedback/BOUNCE/2026/01/15/m.json" + assert first_call.args[2] == bounce_payload email_service.send_email.assert_called_once_with( to_address="prm@example.com", @@ -263,16 +261,16 @@ def test_process_ses_feedback_event_stores_each_record_and_alerts_only_configure def test_process_ses_feedback_event_handles_empty_records( - svc, s3_client, email_service + svc, s3_service, email_service ): resp = svc.process_ses_feedback_event({"Records": []}) assert resp == {"status": "ok", "stored": 0, "alerted": 0} - s3_client.put_object.assert_not_called() + s3_service.put_json.assert_not_called() email_service.send_email.assert_not_called() def test_process_ses_feedback_event_message_can_be_dict_not_json_string( - svc, s3_client, email_service, mocker + svc, s3_service, email_service, mocker ): mocker.patch.object( SesFeedbackMonitorService, "build_s3_key", return_value="k.json" @@ -289,5 +287,5 @@ def test_process_ses_feedback_event_message_can_be_dict_not_json_string( resp = svc.process_ses_feedback_event(event) assert resp == {"status": "ok", "stored": 1, "alerted": 1} - s3_client.put_object.assert_called_once() + s3_service.put_json.assert_called_once_with("feedback-bucket", "k.json", payload) email_service.send_email.assert_called_once() From 4c2efcfa478918db8eb9a94802b0da4eec3674f5 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 16 Jan 2026 09:05:24 +0000 Subject: [PATCH 26/60] [PRMP-1058] fixed test --- lambdas/tests/unit/services/base/test_s3_service.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lambdas/tests/unit/services/base/test_s3_service.py b/lambdas/tests/unit/services/base/test_s3_service.py index e325ee4962..38fb8c3fca 100755 --- a/lambdas/tests/unit/services/base/test_s3_service.py +++ b/lambdas/tests/unit/services/base/test_s3_service.py @@ -34,7 +34,7 @@ def flatten(list_of_lists): def mock_service(mocker, set_env): S3Service._instance = None - mock_boto_client = mocker.patch("boto3.client") + mocker.patch("boto3.client") mocker.patch("services.base.iam_service.IAMService") service = S3Service(custom_aws_role="mock_arn_custom_role") @@ -483,9 +483,7 @@ def test_save_or_create_file(mock_service, mock_client): assert kwargs["Body"].read() == body -def test_returns_binary_file_content_when_file_exists( - mock_service, mock_client, mocker -): +def test_returns_binary_file_content_when_file_exists(mock_service, mock_client, mocker): mock_client.get_object.return_value = { "Body": mocker.Mock(read=lambda: b"file-content") } From 3370f69295a2f8c232207e302cbadf39f217d510 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 16 Jan 2026 11:52:25 +0000 Subject: [PATCH 27/60] [PRMP-1058] minor updates --- lambdas/handlers/ses_feedback_monitor_handler.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lambdas/handlers/ses_feedback_monitor_handler.py b/lambdas/handlers/ses_feedback_monitor_handler.py index 0e254688d6..f74b31bc4b 100644 --- a/lambdas/handlers/ses_feedback_monitor_handler.py +++ b/lambdas/handlers/ses_feedback_monitor_handler.py @@ -7,12 +7,17 @@ SesFeedbackMonitorConfig, SesFeedbackMonitorService, ) +from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions +from utils.decorators.override_error_check import override_error_check +from utils.decorators.set_audit_arg import set_request_context_for_logging def parse_alert_types(configured: str) -> set[str]: return {s.strip().upper() for s in configured.split(",") if s.strip()} - +@override_error_check +@handle_lambda_exceptions +@set_request_context_for_logging def lambda_handler(event, context) -> Dict[str, Any]: config = SesFeedbackMonitorConfig( feedback_bucket=os.environ["SES_FEEDBACK_BUCKET_NAME"], From f7d63df04ad43ee740dc5bb872f31e101b8c3f4d Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 16 Jan 2026 14:42:18 +0000 Subject: [PATCH 28/60] [PRMP-1058] minor updates --- lambdas/services/ses_feedback_monitor_service.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lambdas/services/ses_feedback_monitor_service.py b/lambdas/services/ses_feedback_monitor_service.py index 31e1caedcf..e20ba041ad 100644 --- a/lambdas/services/ses_feedback_monitor_service.py +++ b/lambdas/services/ses_feedback_monitor_service.py @@ -103,10 +103,10 @@ def process_ses_feedback_event(self, event: Dict[str, Any]) -> Dict[str, Any]: for record in event.get("Records") or []: payload = self.parse_sns_message(record) - et = self.event_type(payload) - mid = self.message_id(payload) + event_type = self.event_type(payload) + message_id = self.message_id(payload) - s3_key = self.build_s3_key(self.config.feedback_prefix, et, mid) + s3_key = self.build_s3_key(self.config.feedback_prefix, event_type, message_id) self.s3.put_json( self.config.feedback_bucket, @@ -115,11 +115,11 @@ def process_ses_feedback_event(self, event: Dict[str, Any]) -> Dict[str, Any]: ) stored += 1 logger.info( - f"Stored SES feedback event: type={et}, message_id={mid}, " + f"Stored SES feedback event: type={event_type}, message_id={message_id}, " f"s3=s3://{self.config.feedback_bucket}/{s3_key}" ) - if et in self.config.alert_on_event_types: + if event_type in self.config.alert_on_event_types: subject, body = self.build_prm_email(payload, s3_key) self.email_service.send_email( to_address=self.config.prm_mailbox, @@ -129,7 +129,7 @@ def process_ses_feedback_event(self, event: Dict[str, Any]) -> Dict[str, Any]: ) alerted += 1 logger.info( - f"Emailed PRM for SES feedback event: type={et}, message_id={mid}" + f"Emailed PRM for SES feedback event: type={event_type}, message_id={message_id}" ) return {"status": "ok", "stored": stored, "alerted": alerted} From 8f74fe338d0e073a1601dd11708f701c4b0ecdd9 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 16 Jan 2026 15:28:22 +0000 Subject: [PATCH 29/60] [PRMP-1057] fixed comments --- .../handlers/report_orchestration_handler.py | 98 +++++++++++-------- lambdas/utils/utilities.py | 1 + 2 files changed, 59 insertions(+), 40 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index be9ea77779..d5367068df 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,6 +1,6 @@ import os from datetime import datetime, timedelta, timezone -from typing import Any, Dict +from typing import Any, Dict, Tuple from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.base.s3_service import S3Service @@ -15,7 +15,7 @@ logger = LoggingService(__name__) -def calculate_reporting_window(): +def calculate_reporting_window() -> Tuple[int, int]: now = datetime.now(timezone.utc) today_7am = now.replace(hour=7, minute=0, second=0, microsecond=0) @@ -23,7 +23,6 @@ def calculate_reporting_window(): today_7am -= timedelta(days=1) yesterday_7am = today_7am - timedelta(days=1) - return int(yesterday_7am.timestamp()), int(today_7am.timestamp()) @@ -35,6 +34,50 @@ def build_s3_key(ods_code: str, report_date: str) -> str: return f"Report-Orchestration/{report_date}/{ods_code}.xlsx" +def get_config() -> Tuple[str, str]: + return os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"], os.environ["REPORT_BUCKET_NAME"] + + +def build_services(table_name: str) -> Tuple[ReportOrchestrationService, S3Service]: + repository = ReportingDynamoRepository(table_name) + excel_generator = ExcelReportGenerator() + orchestration_service = ReportOrchestrationService( + repository=repository, + excel_generator=excel_generator, + ) + return orchestration_service, S3Service() + + +def upload_generated_reports( + s3_service: S3Service, + bucket: str, + report_date: str, + generated_files: Dict[str, str], +) -> list[str]: + keys: list[str] = [] + for ods_code, local_path in generated_files.items(): + key = build_s3_key(ods_code, report_date) + s3_service.upload_file_with_extra_args( + file_name=local_path, + s3_bucket_name=bucket, + file_key=key, + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + keys.append(key) + logger.info(f"Uploaded report for ODS={ods_code} to s3://{bucket}/{key}") + return keys + + +def build_response(report_date: str, bucket: str, keys: list[str]) -> Dict[str, Any]: + prefix = f"Report-Orchestration/{report_date}/" + return { + "report_date": report_date, + "bucket": bucket, + "prefix": prefix, + "keys": keys, + } + + @ensure_environment_variables( names=[ "BULK_UPLOAD_REPORT_TABLE_NAME", @@ -47,52 +90,27 @@ def build_s3_key(ods_code: str, report_date: str) -> str: def lambda_handler(event, context) -> Dict[str, Any]: logger.info("Report orchestration lambda invoked") - table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] - report_bucket = os.environ["REPORT_BUCKET_NAME"] - - repository = ReportingDynamoRepository(table_name) - excel_generator = ExcelReportGenerator() - s3_service = S3Service() - - service = ReportOrchestrationService( - repository=repository, - excel_generator=excel_generator, - ) + table_name, report_bucket = get_config() + orchestration_service, s3_service = build_services(table_name) window_start, window_end = calculate_reporting_window() report_date = get_report_date_folder() - prefix = f"Report-Orchestration/{report_date}/" - generated_files = service.process_reporting_window( + generated_files = orchestration_service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, ) if not generated_files: logger.info("No reports generated; exiting") - return { - "report_date": report_date, - "bucket": report_bucket, - "prefix": prefix, - "keys": [], - } - - keys = [] - for ods_code, local_path in generated_files.items(): - key = build_s3_key(ods_code, report_date) - s3_service.upload_file_with_extra_args( - file_name=local_path, - s3_bucket_name=report_bucket, - file_key=key, - extra_args={"ServerSideEncryption": "aws:kms"}, - ) - keys.append(key) - logger.info(f"Uploaded report for ODS={ods_code} to s3://{report_bucket}/{key}") + return build_response(report_date=report_date, bucket=report_bucket, keys=[]) + + keys = upload_generated_reports( + s3_service=s3_service, + bucket=report_bucket, + report_date=report_date, + generated_files=generated_files, + ) logger.info(f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}") - return { - "report_date": report_date, - "bucket": report_bucket, - "prefix": prefix, - "keys": keys, - } + return build_response(report_date=report_date, bucket=report_bucket, keys=keys) diff --git a/lambdas/utils/utilities.py b/lambdas/utils/utilities.py index bb2de83eab..f23720e9b1 100755 --- a/lambdas/utils/utilities.py +++ b/lambdas/utils/utilities.py @@ -139,5 +139,6 @@ def utc_day_start_timestamp(day: date) -> int: return int( datetime.combine(day, time.min, tzinfo=timezone.utc).timestamp() ) + def utc_day_end_timestamp(day: date) -> int: return utc_day_start_timestamp(day) + 24 * 60 * 60 - 1 \ No newline at end of file From 509f3e9b96e39a353d261ad7ab1e3f3cf3fbc0d1 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 16 Jan 2026 15:34:35 +0000 Subject: [PATCH 30/60] [PRMP-1057] updated tests --- lambdas/services/email_service.py | 6 +- .../test_report_orchestration_handler.py | 99 +++++++------------ 2 files changed, 38 insertions(+), 67 deletions(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index 8a11a9bd88..e72bbcfde4 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -2,7 +2,7 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.application import MIMEApplication -from typing import Iterable, Optional +from typing import Iterable, Optional, Dict, Any from utils.audit_logging_setup import LoggingService @@ -26,7 +26,7 @@ def send_email( body_text: str, from_address: str, attachments: Optional[Iterable[str]] = None, - )->MIMEMultipart: + )->Dict[str, Any]: msg = MIMEMultipart() msg["Subject"] = subject msg["To"] = to_address @@ -49,7 +49,7 @@ def send_email( ) return self._send_raw(msg, to_address) - def _send_raw(self, msg: MIMEMultipart, to_address: str)->MIMEMultipart: + def _send_raw(self, msg: MIMEMultipart, to_address: str)->Dict[str, Any]: subject = msg.get("Subject", "") from_address = msg.get("From", "") logger.info(f"Sending SES raw email: subject={subject!r}, from={from_address!r}, to={to_address!r}") diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 10364c5e42..6458a2f8a6 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -28,38 +28,6 @@ def mock_logger(mocker): return mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) -@pytest.fixture -def mock_repo_cls(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.ReportingDynamoRepository", - autospec=True, - ) - - -@pytest.fixture -def mock_excel_generator_cls(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.ExcelReportGenerator", - autospec=True, - ) - - -@pytest.fixture -def mock_s3_service_cls(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.S3Service", - autospec=True, - ) - - -@pytest.fixture -def mock_service_cls(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.ReportOrchestrationService", - autospec=True, - ) - - @pytest.fixture def mock_window(mocker): return mocker.patch( @@ -76,38 +44,40 @@ def mock_report_date(mocker): ) -def test_lambda_handler_wires_dependencies_and_calls_service( +@pytest.fixture +def mock_build_services(mocker): + orchestration_service = MagicMock() + s3_service = MagicMock() + patcher = mocker.patch( + "handlers.report_orchestration_handler.build_services", + return_value=(orchestration_service, s3_service), + ) + return patcher, orchestration_service, s3_service + + +def test_lambda_handler_calls_service_and_returns_expected_response( mock_logger, - mock_repo_cls, - mock_excel_generator_cls, - mock_s3_service_cls, - mock_service_cls, + mock_build_services, mock_window, mock_report_date, ): - service_instance = mock_service_cls.return_value - service_instance.process_reporting_window.return_value = { + build_services_patcher, orchestration_service, s3_service = mock_build_services + + orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "B67890": "/tmp/B67890.xlsx", } result = lambda_handler(event={}, context=FakeContext()) - mock_repo_cls.assert_called_once_with("TestTable") - mock_excel_generator_cls.assert_called_once_with() - mock_s3_service_cls.assert_called_once_with() - - mock_service_cls.assert_called_once_with( - repository=mock_repo_cls.return_value, - excel_generator=mock_excel_generator_cls.return_value, - ) + build_services_patcher.assert_called_once_with("TestTable") - service_instance.process_reporting_window.assert_called_once_with( + orchestration_service.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, ) - mock_logger.info.assert_any_call("Report orchestration lambda invoked") + assert s3_service.upload_file_with_extra_args.call_count == 2 assert result["report_date"] == "2026-01-02" assert result["bucket"] == "test-report-bucket" @@ -117,14 +87,16 @@ def test_lambda_handler_wires_dependencies_and_calls_service( "Report-Orchestration/2026-01-02/B67890.xlsx", } + mock_logger.info.assert_any_call("Report orchestration lambda invoked") + def test_lambda_handler_calls_window_function( - mock_service_cls, + mock_build_services, mock_window, mock_report_date, - mock_s3_service_cls, ): - mock_service_cls.return_value.process_reporting_window.return_value = {} + _, orchestration_service, _ = mock_build_services + orchestration_service.process_reporting_window.return_value = {} lambda_handler(event={}, context=FakeContext()) @@ -132,13 +104,13 @@ def test_lambda_handler_calls_window_function( def test_lambda_handler_returns_empty_keys_when_no_reports_generated( - mock_service_cls, + mock_build_services, mock_logger, - mock_s3_service_cls, mock_window, mock_report_date, ): - mock_service_cls.return_value.process_reporting_window.return_value = {} + _, orchestration_service, s3_service = mock_build_services + orchestration_service.process_reporting_window.return_value = {} result = lambda_handler(event={}, context=FakeContext()) @@ -149,34 +121,33 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( "keys": [], } - mock_s3_service_cls.return_value.upload_file_with_extra_args.assert_not_called() + s3_service.upload_file_with_extra_args.assert_not_called() mock_logger.info.assert_any_call("No reports generated; exiting") def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( - mock_service_cls, - mock_s3_service_cls, - mock_report_date, + mock_build_services, mock_window, + mock_report_date, ): - mock_service_cls.return_value.process_reporting_window.return_value = { + _, orchestration_service, s3_service = mock_build_services + orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "UNKNOWN": "/tmp/UNKNOWN.xlsx", } result = lambda_handler(event={}, context=FakeContext()) - s3_instance = mock_s3_service_cls.return_value - assert s3_instance.upload_file_with_extra_args.call_count == 2 + assert s3_service.upload_file_with_extra_args.call_count == 2 - s3_instance.upload_file_with_extra_args.assert_any_call( + s3_service.upload_file_with_extra_args.assert_any_call( file_name="/tmp/A12345.xlsx", s3_bucket_name="test-report-bucket", file_key="Report-Orchestration/2026-01-02/A12345.xlsx", extra_args={"ServerSideEncryption": "aws:kms"}, ) - s3_instance.upload_file_with_extra_args.assert_any_call( + s3_service.upload_file_with_extra_args.assert_any_call( file_name="/tmp/UNKNOWN.xlsx", s3_bucket_name="test-report-bucket", file_key="Report-Orchestration/2026-01-02/UNKNOWN.xlsx", From 0cf7002eabaf0a3521803783b8fd5ab1455ffb8a Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 19 Jan 2026 13:51:37 +0000 Subject: [PATCH 31/60] [PRMP-1057] updated lambda names --- .github/workflows/base-lambdas-reusable-deploy-all.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index 5128113a49..69df066397 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -806,7 +806,7 @@ jobs: build_branch: ${{ inputs.build_branch }} sandbox: ${{ inputs.sandbox }} lambda_handler_name: report_orchestration_handler - lambda_aws_name: reportOrchestration + lambda_aws_name: ReportOrchestration lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} @@ -820,7 +820,7 @@ jobs: build_branch: ${{ inputs.build_branch }} sandbox: ${{ inputs.sandbox }} lambda_handler_name: report_distribution_handler - lambda_aws_name: reportDistribution + lambda_aws_name: ReportDistribution lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} From 2948ffe2578ab08d62e9f3d181087ef7c0d98b25 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 20 Jan 2026 09:39:02 +0000 Subject: [PATCH 32/60] [PRMP-1057] updated report columns --- .../reporting/excel_report_generator_service.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index b8bef9a5cf..406e52ae7a 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -16,6 +16,7 @@ def create_report_orchestration_xlsx( logger.info( f"Creating Excel report for ODS code {ods_code} and records {len(records)}" ) + wb = Workbook() ws = wb.active ws.title = "Daily Upload Report" @@ -25,12 +26,11 @@ def create_report_orchestration_xlsx( ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) ws.append([]) - # Header row + # Header row (ID removed, NHS Number first) ws.append( [ - "ID", - "Date", "NHS Number", + "Date", "Uploader ODS", "PDS ODS", "Upload Status", @@ -42,9 +42,8 @@ def create_report_orchestration_xlsx( for record in records: ws.append( [ - record.get("ID"), - record.get("Date"), record.get("NhsNumber"), + record.get("Date"), record.get("UploaderOdsCode"), record.get("PdsOdsCode"), record.get("UploadStatus"), @@ -54,5 +53,5 @@ def create_report_orchestration_xlsx( ) wb.save(output_path) - logger.info(f"Excel report written successfully for for ods code {ods_code}") + logger.info(f"Excel report written successfully for ods code {ods_code}") return output_path From c67f895336ac74d83c0456ca3b120b8f239e6288 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 20 Jan 2026 09:39:41 +0000 Subject: [PATCH 33/60] [PRMP-1057] removed comment --- lambdas/services/reporting/excel_report_generator_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index 406e52ae7a..5a4a03cfc5 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -26,7 +26,6 @@ def create_report_orchestration_xlsx( ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) ws.append([]) - # Header row (ID removed, NHS Number first) ws.append( [ "NHS Number", From 895863f5e9ceab5c23bfb1337c7eb67a05444b41 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 20 Jan 2026 09:45:04 +0000 Subject: [PATCH 34/60] [PRMP-1057] updated tests --- .../test_excel_report_generator_service.py | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index f16257c0fa..2f48552855 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -1,4 +1,3 @@ - import pytest from freezegun import freeze_time from openpyxl import load_workbook @@ -47,26 +46,20 @@ def test_create_report_orchestration_xlsx_happy_path( output_path=str(output_file), ) - # File path returned assert result == str(output_file) assert output_file.exists() wb = load_workbook(output_file) ws = wb.active - # Sheet name assert ws.title == "Daily Upload Report" - - # Metadata rows assert ws["A1"].value == f"ODS Code: {ods_code}" assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" assert ws["A3"].value is None # blank row - # Header row assert [cell.value for cell in ws[4]] == [ - "ID", - "Date", "NHS Number", + "Date", "Uploader ODS", "PDS ODS", "Upload Status", @@ -74,11 +67,10 @@ def test_create_report_orchestration_xlsx_happy_path( "File Path", ] - # First data row + # First data row (no ID column) assert [cell.value for cell in ws[5]] == [ - 1, - "2025-01-01", "1234567890", + "2025-01-01", "Y12345", "A99999", "SUCCESS", @@ -86,11 +78,9 @@ def test_create_report_orchestration_xlsx_happy_path( "/path/file1.pdf", ] - # Second data row assert [cell.value for cell in ws[6]] == [ - 2, - "2025-01-02", "123456789", + "2025-01-02", "Y12345", "B88888", "FAILED", @@ -114,7 +104,6 @@ def test_create_report_orchestration_xlsx_with_no_records( wb = load_workbook(output_file) ws = wb.active - # Only metadata + header rows should exist assert ws.max_row == 4 @@ -143,12 +132,11 @@ def test_create_report_orchestration_xlsx_handles_missing_fields( row = [cell.value for cell in ws[5]] assert row == [ - 1, - None, "1234567890", None, None, None, None, None, + None, ] From 717f32cdc791af577ae458df2e6acc811c03b0bb Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 20 Jan 2026 09:56:16 +0000 Subject: [PATCH 35/60] [PRMP-1057] removed unused local variable --- lambdas/services/reporting/report_distribution_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index 9548ddbca0..a83eb5ae19 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -73,7 +73,7 @@ def process_one_report(self, *, ods_code: str, key: str) -> None: def send_report_emails(self, *, ods_code: str, attachment_path: str, password: str) -> None: try: contact_email = self.contact_repo.get_contact_email(ods_code) - except Exception as e: + except Exception: logger.exception( f"Contact lookup failed for ODS={ods_code}; falling back to PRM." ) From 4c9ed8f88f08051e697a68abe6010332429aee51 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 22 Jan 2026 10:42:34 +0000 Subject: [PATCH 36/60] [PRMP-1054] fixed sonnar --- lambdas/services/reporting/excel_report_generator_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index b8bef9a5cf..a3579df501 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from openpyxl.workbook import Workbook from utils.audit_logging_setup import LoggingService @@ -22,7 +22,7 @@ def create_report_orchestration_xlsx( # Report metadata ws.append([f"ODS Code: {ods_code}"]) - ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) + ws.append([f"Generated at (UTC): {datetime.now(timezone.utc).isoformat()}"]) ws.append([]) # Header row From b05e171e8eb377f1a577d9f507b5a545d0d8f6e2 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 22 Jan 2026 10:47:34 +0000 Subject: [PATCH 37/60] [PRMP-1054] fixed sonar --- lambdas/handlers/report_orchestration_handler.py | 2 -- lambdas/services/reporting/report_orchestration_service.py | 1 - .../tests/unit/handlers/test_report_orchestration_handler.py | 1 - .../services/reporting/test_excel_report_generator_service.py | 2 +- .../services/reporting/test_report_orchestration_service.py | 4 ++-- 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 84f6372ab8..d9501b3a8e 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -47,10 +47,8 @@ def lambda_handler(event, context): ) window_start, window_end = calculate_reporting_window() - tmp_dir = tempfile.mkdtemp() service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, - output_dir=tmp_dir, ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index a9500735b9..7d4ea48659 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -19,7 +19,6 @@ def process_reporting_window( self, window_start_ts: int, window_end_ts: int, - output_dir: str, ): records = self.repository.get_records_for_time_window( window_start_ts, diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index c9ee2ff849..1e7af584c9 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -63,7 +63,6 @@ def test_lambda_handler_calls_service( instance.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, - output_dir=mock.ANY, ) mock_logger.info.assert_any_call("Report orchestration lambda invoked") diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index f16257c0fa..6f170bafcd 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -59,7 +59,7 @@ def test_create_report_orchestration_xlsx_happy_path( # Metadata rows assert ws["A1"].value == f"ODS Code: {ods_code}" - assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" + assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00+00:00" assert ws["A3"].value is None # blank row # Header row diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c068ba0c16..c3b21a77ef 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -27,7 +27,7 @@ def test_process_reporting_window_no_records( ): mock_repository.get_records_for_time_window.return_value = [] - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + report_orchestration_service.process_reporting_window(100, 200) mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() @@ -68,7 +68,7 @@ def test_process_reporting_window_generates_reports_per_ods( report_orchestration_service, "generate_ods_report" ) - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + report_orchestration_service.process_reporting_window(100, 200) mocked_generate.assert_any_call( "Y12345", From 792334631f93762c833d246b54f50dd9d70419f9 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 22 Jan 2026 10:57:01 +0000 Subject: [PATCH 38/60] Revert "[PRMP-1054] fixed sonar" This reverts commit b05e171e8eb377f1a577d9f507b5a545d0d8f6e2. --- lambdas/handlers/report_orchestration_handler.py | 2 ++ lambdas/services/reporting/report_orchestration_service.py | 1 + .../tests/unit/handlers/test_report_orchestration_handler.py | 1 + .../services/reporting/test_excel_report_generator_service.py | 2 +- .../services/reporting/test_report_orchestration_service.py | 4 ++-- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index d9501b3a8e..84f6372ab8 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -47,8 +47,10 @@ def lambda_handler(event, context): ) window_start, window_end = calculate_reporting_window() + tmp_dir = tempfile.mkdtemp() service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, + output_dir=tmp_dir, ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index 7d4ea48659..a9500735b9 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -19,6 +19,7 @@ def process_reporting_window( self, window_start_ts: int, window_end_ts: int, + output_dir: str, ): records = self.repository.get_records_for_time_window( window_start_ts, diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 1e7af584c9..c9ee2ff849 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -63,6 +63,7 @@ def test_lambda_handler_calls_service( instance.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, + output_dir=mock.ANY, ) mock_logger.info.assert_any_call("Report orchestration lambda invoked") diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index 6f170bafcd..f16257c0fa 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -59,7 +59,7 @@ def test_create_report_orchestration_xlsx_happy_path( # Metadata rows assert ws["A1"].value == f"ODS Code: {ods_code}" - assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00+00:00" + assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" assert ws["A3"].value is None # blank row # Header row diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c3b21a77ef..c068ba0c16 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -27,7 +27,7 @@ def test_process_reporting_window_no_records( ): mock_repository.get_records_for_time_window.return_value = [] - report_orchestration_service.process_reporting_window(100, 200) + report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() @@ -68,7 +68,7 @@ def test_process_reporting_window_generates_reports_per_ods( report_orchestration_service, "generate_ods_report" ) - report_orchestration_service.process_reporting_window(100, 200) + report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") mocked_generate.assert_any_call( "Y12345", From c78120eda4ac80441f1bdb5223ea7d80cc40d3e3 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 22 Jan 2026 10:59:24 +0000 Subject: [PATCH 39/60] [PRMP-1054] fixed issues --- .../services/reporting/test_excel_report_generator_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index f16257c0fa..8b8bd484c7 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -59,7 +59,7 @@ def test_create_report_orchestration_xlsx_happy_path( # Metadata rows assert ws["A1"].value == f"ODS Code: {ods_code}" - assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" + assert ws["A2"].value.startswith("Generated at (UTC): ") assert ws["A3"].value is None # blank row # Header row From f1c5973c95e4ea5bc99a2aaf209c184134ecbe3e Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 28 Jan 2026 15:06:36 +0000 Subject: [PATCH 40/60] [PRMP-1058] updated pypdf version --- .github/workflows/base-lambdas-reusable-deploy-all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index 77f4f3bb0e..6e7423202a 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -834,7 +834,7 @@ jobs: build_branch: ${{ inputs.build_branch }} sandbox: ${{ inputs.sandbox }} lambda_handler_name: ses_feedback_monitor_handler - lambda_aws_name: sesFeedbackMonitor + lambda_aws_name: SesFeedbackMonitor lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} \ No newline at end of file From 5e9cb3e6f5ac765ee8cee19ba0ed2802922d715f Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 30 Jan 2026 13:16:45 +0000 Subject: [PATCH 41/60] [PRMP-1057-2] fixed comments --- lambdas/enums/lambda_error.py | 4 + .../handlers/report_distribution_handler.py | 22 ++- .../handlers/report_orchestration_handler.py | 1 + lambdas/services/base/s3_service.py | 12 +- .../reporting/report_distribution_service.py | 13 +- lambdas/tests/unit/conftest.py | 14 ++ lambdas/tests/unit/handlers/conftest.py | 49 +++++++ .../test_report_distribution_handler.py | 114 ++++++++++----- .../test_report_orchestration_handler.py | 85 +++++++----- .../test_reporting_dynamo_repository.py | 130 +++++++++++++----- .../unit/services/base/test_s3_service.py | 31 +++++ .../services/reporting/test_email_service.py | 6 +- .../test_report_distribution_service.py | 109 +++++++-------- lambdas/utils/lambda_exceptions.py | 3 + lambdas/utils/zip_utils.py | 13 +- poetry.lock | 68 ++++++++- pyproject.toml | 1 + 17 files changed, 486 insertions(+), 189 deletions(-) diff --git a/lambdas/enums/lambda_error.py b/lambdas/enums/lambda_error.py index 16269d9d23..871425dade 100644 --- a/lambdas/enums/lambda_error.py +++ b/lambdas/enums/lambda_error.py @@ -729,3 +729,7 @@ def create_error_body( "message": "An internal server error occurred", "fhir_coding": FhirIssueCoding.EXCEPTION, } + InvalidAction = { + "err_code": "RD_IA", + "message": "Invalid action. Expected 'list' or 'process_one'.", + } diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index 30c8547af8..da32e9d409 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -1,6 +1,7 @@ import os from typing import Any, Dict +from enums.lambda_error import LambdaError from repositories.reporting.report_contact_repository import ReportContactRepository from services.base.s3_service import S3Service from services.email_service import EmailService @@ -10,6 +11,7 @@ from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions from utils.decorators.override_error_check import override_error_check from utils.decorators.set_audit_arg import set_request_context_for_logging +from utils.lambda_exceptions import ReportDistributionException logger = LoggingService(__name__) @@ -28,7 +30,8 @@ def lambda_handler(event, context) -> Dict[str, Any]: action = event.get("action") if action not in {"list", "process_one"}: - raise ValueError("Invalid action. Expected 'list' or 'process_one'.") + logger.error(f"Invalid action. Expected 'list' or 'process_one'.") + raise ReportDistributionException(400, LambdaError.InvalidAction) bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] contact_table = os.environ["CONTACT_TABLE_NAME"] @@ -48,14 +51,19 @@ def lambda_handler(event, context) -> Dict[str, Any]: prm_mailbox=prm_mailbox, ) + response = {"status": "ok", "bucket": bucket} + if action == "list": prefix = event["prefix"] keys = service.list_xlsx_keys(prefix=prefix) logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") - return {"bucket": bucket, "prefix": prefix, "keys": keys} + response.update({"prefix": prefix, "keys": keys}) + else: + key = event["key"] + ods_code = service.extract_ods_code_from_key(key) + service.process_one_report(ods_code=ods_code, key=key) + logger.info(f"Process-one mode: processed ods={ods_code}, key={key}") + response.update({"key": key, "ods_code": ods_code}) + + return response - key = event["key"] - ods_code = service.extract_ods_code_from_key(key) - service.process_one_report(ods_code=ods_code, key=key) - logger.info(f"Process-one mode: processed ods={ods_code}, key={key}") - return {"status": "ok", "bucket": bucket, "key": key, "ods_code": ods_code} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index d5367068df..38a450e76f 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -71,6 +71,7 @@ def upload_generated_reports( def build_response(report_date: str, bucket: str, keys: list[str]) -> Dict[str, Any]: prefix = f"Report-Orchestration/{report_date}/" return { + "status": "ok", "report_date": report_date, "bucket": bucket, "prefix": prefix, diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index 3ee42b4e24..d3bf770fcd 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -1,7 +1,7 @@ import io from datetime import datetime, timedelta, timezone from io import BytesIO -from typing import Any, Mapping +from typing import Any, Mapping, Callable import boto3 from botocore.client import Config as BotoConfig @@ -255,3 +255,13 @@ def save_or_create_file(self, source_bucket: str, file_key: str, body: bytes): return self.client.put_object( Bucket=source_bucket, Key=file_key, Body=BytesIO(body) ) + + def list_object_keys(self, bucket_name: str, prefix: str) -> list[str]: + paginator = self.client.get_paginator("list_objects_v2") + keys: list[str] = [] + + for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix): + for obj in page.get("Contents", []): + keys.append(obj["Key"]) + + return keys \ No newline at end of file diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index a83eb5ae19..d93274e5e1 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -31,7 +31,6 @@ def __init__( self.bucket = bucket self.from_address = from_address self.prm_mailbox = prm_mailbox - self._s3_client = boto3.client("s3") @staticmethod def extract_ods_code_from_key(key: str) -> str: @@ -39,16 +38,8 @@ def extract_ods_code_from_key(key: str) -> str: return filename[:-5] if filename.lower().endswith(".xlsx") else filename def list_xlsx_keys(self, prefix: str) -> List[str]: - paginator = self._s3_client.get_paginator("list_objects_v2") - keys: List[str] = [] - - for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): - for obj in page.get("Contents", []): - key = obj["Key"] - if key.endswith(".xlsx"): - keys.append(key) - - return keys + keys = self.s3_service.list_object_keys(bucket_name=self.bucket, prefix=prefix) + return [k for k in keys if k.endswith(".xlsx")] def process_one_report(self, *, ods_code: str, key: str) -> None: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index b3afba0f82..dd3a98e4b1 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -13,6 +13,7 @@ from pydantic import ValidationError from pypdf import PdfWriter from requests import Response +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from tests.unit.helpers.data.pds.pds_patient_response import PDS_PATIENT from utils.audit_logging_setup import LoggingService @@ -416,3 +417,16 @@ def valid_pdf_stream(): @pytest.fixture def corrupt_pdf_stream(): return BytesIO(b"This is not a valid PDF content") + + +@pytest.fixture +def mock_reporting_dynamo_service(mocker): + mock_cls = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.DynamoDBService" + ) + return mock_cls.return_value + + +@pytest.fixture +def reporting_repo(mock_reporting_dynamo_service): + return ReportingDynamoRepository(table_name="TestTable") diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index 8eca1979f2..a7cd749de9 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -1,5 +1,8 @@ +import os + import pytest from enums.feature_flags import FeatureFlags +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.feature_flags_service import FeatureFlagService @@ -219,3 +222,49 @@ def mock_upload_document_iteration_3_disabled(mocker): FeatureFlags.UPLOAD_DOCUMENT_ITERATION_3_ENABLED: False } yield mock_feature_flag + + +@pytest.fixture +def required_report_distribution_env(mocker): + mocker.patch.dict( + os.environ, + { + "REPORT_BUCKET_NAME": "my-report-bucket", + "CONTACT_TABLE_NAME": "contact-table", + "PRM_MAILBOX_EMAIL": "prm@example.com", + "SES_FROM_ADDRESS": "from@example.com", + }, + clear=False, + ) + + +@pytest.fixture +def lambda_context(mocker): + ctx = mocker.Mock() + ctx.aws_request_id = "req-123" + return ctx + + +@pytest.fixture +def required_report_orchestration_env(mocker): + mocker.patch.dict( + os.environ, + { + "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", + "REPORT_BUCKET_NAME": "test-report-bucket", + }, + clear=False, + ) + + +@pytest.fixture +def mock_reporting_dynamo_service(mocker): + mock_cls = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.DynamoDBService" + ) + return mock_cls.return_value + + +@pytest.fixture +def reporting_repo(mock_reporting_dynamo_service): + return ReportingDynamoRepository(table_name="TestTable") diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index d129d2ed36..ed7371a634 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -1,5 +1,7 @@ import importlib +import json import os + import pytest MODULE_UNDER_TEST = "handlers.report_distribution_handler" @@ -10,26 +12,10 @@ def handler_module(): return importlib.import_module(MODULE_UNDER_TEST) -@pytest.fixture -def required_env(mocker): - mocker.patch.dict( - os.environ, - { - "REPORT_BUCKET_NAME": "my-report-bucket", - "CONTACT_TABLE_NAME": "contact-table", - "PRM_MAILBOX_EMAIL": "prm@example.com", - "SES_FROM_ADDRESS": "from@example.com", - }, - clear=False, - ) - - def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( - mocker, handler_module, required_env + mocker, handler_module, required_report_distribution_env, lambda_context ): event = {"action": "list", "prefix": "reports/2026-01-01/"} - context = mocker.Mock() - context.aws_request_id = "req-123" # avoid JSON serialization issues in decorators s3_instance = mocker.Mock(name="S3ServiceInstance") contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") @@ -57,7 +43,7 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( return_value=svc_instance, ) - result = handler_module.lambda_handler(event, context) + result = handler_module.lambda_handler(event, lambda_context) mocked_s3_cls.assert_called_once_with() mocked_contact_repo_cls.assert_called_once_with("contact-table") @@ -73,7 +59,9 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( ) svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") + assert result == { + "status": "ok", "bucket": "my-report-bucket", "prefix": "reports/2026-01-01/", "keys": ["a.xlsx", "b.xlsx"], @@ -81,46 +69,63 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( - mocker, handler_module, required_env + mocker, handler_module, required_report_distribution_env, lambda_context ): event = {"action": "list", "prefix": "p/", "bucket": "override-bucket"} - context = mocker.Mock() - context.aws_request_id = "req-456" svc_instance = mocker.Mock() svc_instance.list_xlsx_keys.return_value = [] - mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object( + handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock() + ) mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) - result = handler_module.lambda_handler(event, context) + result = handler_module.lambda_handler(event, lambda_context) svc_instance.list_xlsx_keys.assert_called_once_with(prefix="p/") - assert result == {"bucket": "override-bucket", "prefix": "p/", "keys": []} + assert result == { + "status": "ok", + "bucket": "override-bucket", + "prefix": "p/", + "keys": [], + } def test_lambda_handler_process_one_mode_happy_path( - mocker, handler_module, required_env + mocker, handler_module, required_report_distribution_env, lambda_context ): event = {"action": "process_one", "key": "reports/ABC/whatever.xlsx"} - context = mocker.Mock() - context.aws_request_id = "req-789" svc_instance = mocker.Mock() svc_instance.extract_ods_code_from_key.return_value = "ABC" svc_instance.process_one_report.return_value = None - mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object( + handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock() + ) mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) - result = handler_module.lambda_handler(event, context) + result = handler_module.lambda_handler(event, lambda_context) svc_instance.extract_ods_code_from_key.assert_called_once_with("reports/ABC/whatever.xlsx") - svc_instance.process_one_report.assert_called_once_with(ods_code="ABC", key="reports/ABC/whatever.xlsx") + svc_instance.process_one_report.assert_called_once_with( + ods_code="ABC", key="reports/ABC/whatever.xlsx" + ) assert result == { "status": "ok", @@ -128,3 +133,48 @@ def test_lambda_handler_process_one_mode_happy_path( "key": "reports/ABC/whatever.xlsx", "ods_code": "ABC", } + + +def test_lambda_handler_returns_400_when_action_invalid( + handler_module, required_report_distribution_env, lambda_context +): + event = {"action": "nope"} + + result = handler_module.lambda_handler(event, lambda_context) + + assert isinstance(result, dict) + assert result["statusCode"] == 400 + + body = json.loads(result["body"]) + assert body["err_code"] == handler_module.LambdaError.InvalidAction.value["err_code"] + assert "Invalid action" in body["message"] + + if body.get("interaction_id") is not None: + assert body["interaction_id"] == lambda_context.aws_request_id + + +def test_lambda_handler_returns_500_when_required_env_missing(mocker, handler_module, lambda_context): + mocker.patch.dict( + os.environ, + { + "CONTACT_TABLE_NAME": "contact-table", + "PRM_MAILBOX_EMAIL": "prm@example.com", + "SES_FROM_ADDRESS": "from@example.com", + }, + clear=False, + ) + os.environ.pop("REPORT_BUCKET_NAME", None) + + event = {"action": "list", "prefix": "p/"} + + result = handler_module.lambda_handler(event, lambda_context) + + assert isinstance(result, dict) + assert result["statusCode"] == 500 + + body = json.loads(result["body"]) + assert body["err_code"] == "ENV_5001" + assert "REPORT_BUCKET_NAME" in body["message"] + + if body.get("interaction_id") is not None: + assert body["interaction_id"] == lambda_context.aws_request_id diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 6458a2f8a6..0ab0dac213 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,61 +1,57 @@ +import importlib import json import os from unittest.mock import MagicMock import pytest -from handlers.report_orchestration_handler import lambda_handler +MODULE_UNDER_TEST = "handlers.report_orchestration_handler" -class FakeContext: - aws_request_id = "test-request-id" - - -@pytest.fixture(autouse=True) -def mock_env(mocker): - mocker.patch.dict( - os.environ, - { - "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", - "REPORT_BUCKET_NAME": "test-report-bucket", - }, - clear=False, - ) +@pytest.fixture +def handler_module(): + return importlib.import_module(MODULE_UNDER_TEST) @pytest.fixture -def mock_logger(mocker): - return mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) +def mock_logger(mocker, handler_module): + return mocker.patch.object(handler_module, "logger", new=MagicMock()) @pytest.fixture -def mock_window(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.calculate_reporting_window", +def mock_window(mocker, handler_module): + return mocker.patch.object( + handler_module, + "calculate_reporting_window", return_value=(100, 200), ) @pytest.fixture -def mock_report_date(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.get_report_date_folder", +def mock_report_date(mocker, handler_module): + return mocker.patch.object( + handler_module, + "get_report_date_folder", return_value="2026-01-02", ) @pytest.fixture -def mock_build_services(mocker): +def mock_build_services(mocker, handler_module): orchestration_service = MagicMock() s3_service = MagicMock() - patcher = mocker.patch( - "handlers.report_orchestration_handler.build_services", + patcher = mocker.patch.object( + handler_module, + "build_services", return_value=(orchestration_service, s3_service), ) return patcher, orchestration_service, s3_service def test_lambda_handler_calls_service_and_returns_expected_response( + handler_module, + required_report_orchestration_env, + lambda_context, mock_logger, mock_build_services, mock_window, @@ -68,7 +64,7 @@ def test_lambda_handler_calls_service_and_returns_expected_response( "B67890": "/tmp/B67890.xlsx", } - result = lambda_handler(event={}, context=FakeContext()) + result = handler_module.lambda_handler(event={}, context=lambda_context) build_services_patcher.assert_called_once_with("TestTable") @@ -91,6 +87,9 @@ def test_lambda_handler_calls_service_and_returns_expected_response( def test_lambda_handler_calls_window_function( + handler_module, + required_report_orchestration_env, + lambda_context, mock_build_services, mock_window, mock_report_date, @@ -98,12 +97,15 @@ def test_lambda_handler_calls_window_function( _, orchestration_service, _ = mock_build_services orchestration_service.process_reporting_window.return_value = {} - lambda_handler(event={}, context=FakeContext()) + handler_module.lambda_handler(event={}, context=lambda_context) mock_window.assert_called_once() def test_lambda_handler_returns_empty_keys_when_no_reports_generated( + handler_module, + required_report_orchestration_env, + lambda_context, mock_build_services, mock_logger, mock_window, @@ -112,9 +114,10 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( _, orchestration_service, s3_service = mock_build_services orchestration_service.process_reporting_window.return_value = {} - result = lambda_handler(event={}, context=FakeContext()) + result = handler_module.lambda_handler(event={}, context=lambda_context) assert result == { + "status": "ok", "report_date": "2026-01-02", "bucket": "test-report-bucket", "prefix": "Report-Orchestration/2026-01-02/", @@ -126,6 +129,9 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( + handler_module, + required_report_orchestration_env, + lambda_context, mock_build_services, mock_window, mock_report_date, @@ -136,7 +142,7 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( "UNKNOWN": "/tmp/UNKNOWN.xlsx", } - result = lambda_handler(event={}, context=FakeContext()) + result = handler_module.lambda_handler(event={}, context=lambda_context) assert s3_service.upload_file_with_extra_args.call_count == 2 @@ -160,14 +166,19 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( ] -def test_lambda_handler_returns_error_when_required_env_missing(mocker): - mocker.patch.dict(os.environ, {"BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable"}, clear=False) +def test_lambda_handler_returns_error_when_required_env_missing( + handler_module, + lambda_context, + mocker, +): + mocker.patch.dict( + os.environ, + {"BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable"}, + clear=False, + ) os.environ.pop("REPORT_BUCKET_NAME", None) - ctx = FakeContext() - ctx.aws_request_id = "test-request-id" - - result = lambda_handler(event={}, context=ctx) + result = handler_module.lambda_handler(event={}, context=lambda_context) assert isinstance(result, dict) assert result["statusCode"] == 500 @@ -177,4 +188,4 @@ def test_lambda_handler_returns_error_when_required_env_missing(mocker): assert "REPORT_BUCKET_NAME" in body["message"] if body.get("interaction_id") is not None: - assert body["interaction_id"] == "test-request-id" + assert body["interaction_id"] == lambda_context.aws_request_id diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index 5697eb112d..7626bde154 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -1,64 +1,122 @@ from datetime import date -from unittest.mock import MagicMock -import pytest -from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository - - -@pytest.fixture -def mock_dynamo_service(mocker): - mock_service_class = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.DynamoDBService" +def test_get_records_for_time_window_same_date_queries_once( + mocker, mock_reporting_dynamo_service, reporting_repo +): + mock_utc_date = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date" ) - mock_instance = mock_service_class.return_value - mock_instance.query_by_key_condition_expression = MagicMock() - return mock_instance - - -@pytest.fixture -def reporting_repo(mock_dynamo_service): - return ReportingDynamoRepository(table_name="TestTable") - - -def test_get_records_for_time_window_same_date_queries_once(mocker, mock_dynamo_service, reporting_repo): - mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") mock_utc_date.side_effect = [date(2026, 1, 7), date(2026, 1, 7)] - mock_dynamo_service.query_by_key_condition_expression.return_value = [{"ID": "one"}] + mock_reporting_dynamo_service.query_by_key_condition_expression.return_value = [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ] result = reporting_repo.get_records_for_time_window(100, 200) - assert result == [{"ID": "one"}] - mock_dynamo_service.query_by_key_condition_expression.assert_called_once() + assert result == [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ] + mock_reporting_dynamo_service.query_by_key_condition_expression.assert_called_once() - call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args.kwargs + call_kwargs = ( + mock_reporting_dynamo_service.query_by_key_condition_expression.call_args.kwargs + ) assert call_kwargs["table_name"] == "TestTable" assert call_kwargs["index_name"] == "TimestampIndex" assert "key_condition_expression" in call_kwargs -def test_get_records_for_time_window_different_dates_queries_twice(mocker, mock_dynamo_service, reporting_repo): - mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") +def test_get_records_for_time_window_different_dates_queries_twice( + mocker, mock_reporting_dynamo_service, reporting_repo +): + mock_utc_date = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date" + ) mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] - mock_dynamo_service.query_by_key_condition_expression.side_effect = [ - [{"ID": "start-day"}], - [{"ID": "end-day"}], + mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = [ + [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + } + ], + [ + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ], ] result = reporting_repo.get_records_for_time_window(100, 200) - assert result == [{"ID": "start-day"}, {"ID": "end-day"}] - assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 + assert result == [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ] + assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == 2 -def test_get_records_for_time_window_returns_empty_list_when_no_items(mocker, mock_dynamo_service, reporting_repo): - mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") +def test_get_records_for_time_window_returns_empty_list_when_no_items( + mocker, mock_reporting_dynamo_service, reporting_repo +): + mock_utc_date = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date" + ) mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] - mock_dynamo_service.query_by_key_condition_expression.side_effect = [[], []] + mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = [ + [], + [], + ] result = reporting_repo.get_records_for_time_window(100, 200) assert result == [] - assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 + assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == 2 diff --git a/lambdas/tests/unit/services/base/test_s3_service.py b/lambdas/tests/unit/services/base/test_s3_service.py index b9e9aa2cb0..6510a45140 100755 --- a/lambdas/tests/unit/services/base/test_s3_service.py +++ b/lambdas/tests/unit/services/base/test_s3_service.py @@ -597,3 +597,34 @@ def test_copy_across_bucket_retries_on_409_conflict(mock_service, mock_client): "StorageClass": "INTELLIGENT_TIERING", } mock_client.copy_object.assert_called_with(**expected_call) + +def test_list_object_keys_returns_keys_for_prefix( + mock_service, mock_client, mock_list_objects_paginate +): + mock_list_objects_paginate.return_value = [MOCK_LIST_OBJECTS_RESPONSE] + + prefix = "some/prefix/" + expected = [obj["Key"] for obj in MOCK_LIST_OBJECTS_RESPONSE["Contents"]] + + actual = mock_service.list_object_keys(bucket_name=MOCK_BUCKET, prefix=prefix) + + assert actual == expected + mock_client.get_paginator.assert_called_with("list_objects_v2") + mock_list_objects_paginate.assert_called_with(Bucket=MOCK_BUCKET, Prefix=prefix) + + +def test_list_object_keys_handles_paginated_responses( + mock_service, mock_client, mock_list_objects_paginate +): + mock_list_objects_paginate.return_value = MOCK_LIST_OBJECTS_PAGINATED_RESPONSES + + prefix = "some/prefix/" + expected = flatten( + [[obj["Key"] for obj in page.get("Contents", [])] for page in MOCK_LIST_OBJECTS_PAGINATED_RESPONSES] + ) + + actual = mock_service.list_object_keys(bucket_name=MOCK_BUCKET, prefix=prefix) + + assert actual == expected + mock_client.get_paginator.assert_called_with("list_objects_v2") + mock_list_objects_paginate.assert_called_with(Bucket=MOCK_BUCKET, Prefix=prefix) diff --git a/lambdas/tests/unit/services/reporting/test_email_service.py b/lambdas/tests/unit/services/reporting/test_email_service.py index efde7f7251..3c84799cbc 100644 --- a/lambdas/tests/unit/services/reporting/test_email_service.py +++ b/lambdas/tests/unit/services/reporting/test_email_service.py @@ -6,9 +6,9 @@ @pytest.fixture def email_service(mocker): mocker.patch("services.email_service.boto3.client", autospec=True) - svc = EmailService() - svc.ses = mocker.Mock() - return svc + service = EmailService() + service.ses = mocker.Mock() + return service def test_send_email_sends_raw_email_without_attachments(email_service, mocker): mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index 56b5150e03..3bff899731 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -1,29 +1,29 @@ import os + import pytest from services.reporting.report_distribution_service import ReportDistributionService + @pytest.fixture def mock_s3_service(mocker): - return mocker.Mock() + return mocker.Mock(name="S3Service") @pytest.fixture def mock_contact_repo(mocker): - repo = mocker.Mock() + repo = mocker.Mock(name="ReportContactRepository") repo.get_contact_email.return_value = None return repo @pytest.fixture def mock_email_service(mocker): - return mocker.Mock() + return mocker.Mock(name="EmailService") @pytest.fixture -def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): - mocker.patch("services.reporting.report_distribution_service.boto3.client", autospec=True) - +def service(mock_s3_service, mock_contact_repo, mock_email_service): return ReportDistributionService( s3_service=mock_s3_service, contact_repo=mock_contact_repo, @@ -33,10 +33,14 @@ def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): prm_mailbox="prm@example.com", ) + def test_extract_ods_code_from_key_strips_xlsx_extension(): - assert ReportDistributionService.extract_ods_code_from_key( - "Report-Orchestration/2026-01-01/Y12345.xlsx" - ) == "Y12345" + assert ( + ReportDistributionService.extract_ods_code_from_key( + "Report-Orchestration/2026-01-01/Y12345.xlsx" + ) + == "Y12345" + ) def test_extract_ods_code_from_key_is_case_insensitive(): @@ -44,58 +48,41 @@ def test_extract_ods_code_from_key_is_case_insensitive(): def test_extract_ods_code_from_key_keeps_non_xlsx_filename(): - assert ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") == "report.csv" - -def test_list_xlsx_keys_filters_only_xlsx(service, mocker): - paginator = mocker.Mock() - paginator.paginate.return_value = [ - { - "Contents": [ - {"Key": "Report-Orchestration/2026-01-01/A123.xlsx"}, - {"Key": "Report-Orchestration/2026-01-01/readme.txt"}, - {"Key": "Report-Orchestration/2026-01-01/B456.xls"}, - {"Key": "Report-Orchestration/2026-01-01/C789.xlsx"}, - ] - }, - {"Contents": [{"Key": "Report-Orchestration/2026-01-01/D000.xlsx"}]}, - {}, - ] + assert ( + ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") + == "report.csv" + ) - service._s3_client.get_paginator.return_value = paginator + +def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): + mock_s3_service.list_object_keys.return_value = [ + "Report-Orchestration/2026-01-01/A123.xlsx", + "Report-Orchestration/2026-01-01/readme.txt", + "Report-Orchestration/2026-01-01/B456.xls", + "Report-Orchestration/2026-01-01/C789.xlsx", + "Report-Orchestration/2026-01-01/D000.xlsx", + ] keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + mock_s3_service.list_object_keys.assert_called_once_with( + bucket_name="my-bucket", + prefix="Report-Orchestration/2026-01-01/", + ) assert keys == [ "Report-Orchestration/2026-01-01/A123.xlsx", "Report-Orchestration/2026-01-01/C789.xlsx", "Report-Orchestration/2026-01-01/D000.xlsx", ] - service._s3_client.get_paginator.assert_called_once_with("list_objects_v2") - paginator.paginate.assert_called_once_with( - Bucket="my-bucket", - Prefix="Report-Orchestration/2026-01-01/", - ) - -def test_list_xlsx_keys_returns_empty_when_no_objects(service, mocker): - paginator = mocker.Mock() - paginator.paginate.return_value = [{"Contents": []}, {}] - service._s3_client.get_paginator.return_value = paginator +def test_list_xlsx_keys_returns_empty_when_no_objects(service, mock_s3_service): + mock_s3_service.list_object_keys.return_value = [] keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") assert keys == [] - - -def test_list_xlsx_keys_skips_pages_without_contents(service, mocker): - paginator = mocker.Mock() - paginator.paginate.return_value = [{}, {"Contents": [{"Key": "p/X.xlsx"}]}] - service._s3_client.get_paginator.return_value = paginator - - keys = service.list_xlsx_keys(prefix="p/") - - assert keys == ["p/X.xlsx"] + mock_s3_service.list_object_keys.assert_called_once() def test_process_one_report_downloads_encrypts_and_delegates_email( @@ -134,13 +121,11 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( "Report-Orchestration/2026-01-01/Y12345.xlsx", local_xlsx, ) - mocked_zip.assert_called_once_with( input_path=local_xlsx, output_zip=local_zip, password="fixed-password", ) - mocked_send.assert_called_once_with( ods_code="Y12345", attachment_path=local_zip, @@ -159,7 +144,9 @@ def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_ return_value=td, ) - mocked_zip = mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True + ) mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) with pytest.raises(RuntimeError, match="download failed"): @@ -227,15 +214,19 @@ def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fa "k.xlsx", os.path.join(fake_tmp, "Y12345.xlsx"), ) - mocked_zip.assert_not_called() mocked_send.assert_not_called() -def test_send_report_emails_with_contact_calls_email_contact(service, mock_contact_repo, mocker): + +def test_send_report_emails_with_contact_calls_email_contact( + service, mock_contact_repo, mocker +): mock_contact_repo.get_contact_email.return_value = "contact@example.com" mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + mocked_email_prm = mocker.patch.object( + service, "email_prm_missing_contact", autospec=True + ) service.send_report_emails( ods_code="Y12345", @@ -256,7 +247,9 @@ def test_send_report_emails_without_contact_calls_email_prm(service, mock_contac mock_contact_repo.get_contact_email.return_value = None mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + mocked_email_prm = mocker.patch.object( + service, "email_prm_missing_contact", autospec=True + ) service.send_report_emails( ods_code="A99999", @@ -273,11 +266,15 @@ def test_send_report_emails_without_contact_calls_email_prm(service, mock_contac mocked_email_contact.assert_not_called() -def test_send_report_emails_contact_lookup_exception_falls_back_to_prm(service, mock_contact_repo, mocker): +def test_send_report_emails_contact_lookup_exception_falls_back_to_prm( + service, mock_contact_repo, mocker +): mock_contact_repo.get_contact_email.side_effect = RuntimeError("ddb down") mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + mocked_email_prm = mocker.patch.object( + service, "email_prm_missing_contact", autospec=True + ) service.send_report_emails( ods_code="A99999", @@ -312,7 +309,7 @@ def test_email_contact_sends_report_and_password(service, mock_email_service): ) -def test_email_contact_sends_password_even_if_report_email_fails(service, mock_email_service): +def test_email_contact_does_not_send_password_if_report_email_fails(service, mock_email_service): mock_email_service.send_report_email.side_effect = RuntimeError("SES down") with pytest.raises(RuntimeError, match="SES down"): diff --git a/lambdas/utils/lambda_exceptions.py b/lambdas/utils/lambda_exceptions.py index 9a30812eed..cc4af788a4 100644 --- a/lambdas/utils/lambda_exceptions.py +++ b/lambdas/utils/lambda_exceptions.py @@ -120,3 +120,6 @@ class DocumentReviewLambdaException(LambdaException): class UpdateDocumentReviewException(LambdaException): pass + +class ReportDistributionException(LambdaException): + pass \ No newline at end of file diff --git a/lambdas/utils/zip_utils.py b/lambdas/utils/zip_utils.py index a09e8934f0..43775059e7 100644 --- a/lambdas/utils/zip_utils.py +++ b/lambdas/utils/zip_utils.py @@ -3,12 +3,15 @@ def zip_encrypt_file(*, input_path: str, output_zip: str, password: str) -> None: - """ - Create an AES-encrypted ZIP file containing a single file. + """Create an AES-encrypted ZIP file containing a single file. + + Args: + input_path: Path to the file to zip. + output_zip: Path of the ZIP file to create. + password: Password used for AES encryption. - :param input_path: Path to the file to zip - :param output_zip: Path of the zip file to create - :param password: Password for AES encryption + Returns: + None """ with pyzipper.AESZipFile( output_zip, diff --git a/poetry.lock b/poetry.lock index 51ea36ac57..eb1c448afe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1671,6 +1671,57 @@ files = [ {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, ] +[[package]] +name = "pycryptodomex" +version = "3.23.0" +description = "Cryptographic library for Python" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["reports-lambda"] +files = [ + {file = "pycryptodomex-3.23.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:add243d204e125f189819db65eed55e6b4713f70a7e9576c043178656529cec7"}, + {file = "pycryptodomex-3.23.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:1c6d919fc8429e5cb228ba8c0d4d03d202a560b421c14867a65f6042990adc8e"}, + {file = "pycryptodomex-3.23.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:1c3a65ad441746b250d781910d26b7ed0a396733c6f2dbc3327bd7051ec8a541"}, + {file = "pycryptodomex-3.23.0-cp27-cp27m-win32.whl", hash = "sha256:47f6d318fe864d02d5e59a20a18834819596c4ed1d3c917801b22b92b3ffa648"}, + {file = "pycryptodomex-3.23.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:d9825410197a97685d6a1fa2a86196430b01877d64458a20e95d4fd00d739a08"}, + {file = "pycryptodomex-3.23.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:267a3038f87a8565bd834317dbf053a02055915acf353bf42ededb9edaf72010"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:7b37e08e3871efe2187bc1fd9320cc81d87caf19816c648f24443483005ff886"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:91979028227543010d7b2ba2471cf1d1e398b3f183cb105ac584df0c36dac28d"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b8962204c47464d5c1c4038abeadd4514a133b28748bcd9fa5b6d62e3cec6fa"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a33986a0066860f7fcf7c7bd2bc804fa90e434183645595ae7b33d01f3c91ed8"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7947ab8d589e3178da3d7cdeabe14f841b391e17046954f2fbcd941705762b5"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c25e30a20e1b426e1f0fa00131c516f16e474204eee1139d1603e132acffc314"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:da4fa650cef02db88c2b98acc5434461e027dce0ae8c22dd5a69013eaf510006"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:58b851b9effd0d072d4ca2e4542bf2a4abcf13c82a29fd2c93ce27ee2a2e9462"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:a9d446e844f08299236780f2efa9898c818fe7e02f17263866b8550c7d5fb328"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:bc65bdd9fc8de7a35a74cab1c898cab391a4add33a8fe740bda00f5976ca4708"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:c885da45e70139464f082018ac527fdaad26f1657a99ee13eecdce0f0ca24ab4"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:06698f957fe1ab229a99ba2defeeae1c09af185baa909a31a5d1f9d42b1aaed6"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2c2537863eccef2d41061e82a881dcabb04944c5c06c5aa7110b577cc487545"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43c446e2ba8df8889e0e16f02211c25b4934898384c1ec1ec04d7889c0333587"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f489c4765093fb60e2edafdf223397bc716491b2b69fe74367b70d6999257a5c"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdc69d0d3d989a1029df0eed67cc5e8e5d968f3724f4519bd03e0ec68df7543c"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6bbcb1dd0f646484939e142462d9e532482bc74475cecf9c4903d4e1cd21f003"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:8a4fcd42ccb04c31268d1efeecfccfd1249612b4de6374205376b8f280321744"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55ccbe27f049743a4caf4f4221b166560d3438d0b1e5ab929e07ae1702a4d6fd"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-win32.whl", hash = "sha256:189afbc87f0b9f158386bf051f720e20fa6145975f1e76369303d0f31d1a8d7c"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:52e5ca58c3a0b0bd5e100a9fbc8015059b05cffc6c66ce9d98b4b45e023443b9"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:02d87b80778c171445d67e23d1caef279bf4b25c3597050ccd2e13970b57fd51"}, + {file = "pycryptodomex-3.23.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:febec69c0291efd056c65691b6d9a339f8b4bc43c6635b8699471248fe897fea"}, + {file = "pycryptodomex-3.23.0-pp27-pypy_73-win32.whl", hash = "sha256:c84b239a1f4ec62e9c789aafe0543f0594f0acd90c8d9e15bcece3efe55eca66"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ebfff755c360d674306e5891c564a274a47953562b42fb74a5c25b8fc1fb1cb5"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eca54f4bb349d45afc17e3011ed4264ef1cc9e266699874cdd1349c504e64798"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2596e643d4365e14d0879dc5aafe6355616c61c2176009270f3048f6d9a61f"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdfac7cda115bca3a5abb2f9e43bc2fb66c2b65ab074913643803ca7083a79ea"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:14c37aaece158d0ace436f76a7bb19093db3b4deade9797abfc39ec6cd6cc2fe"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7de1e40a41a5d7f1ac42b6569b10bcdded34339950945948529067d8426d2785"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bffc92138d75664b6d543984db7893a628559b9e78658563b0395e2a5fb47ed9"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df027262368334552db2c0ce39706b3fb32022d1dce34673d0f9422df004b96a"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e79f1aaff5a3a374e92eb462fa9e598585452135012e2945f96874ca6eeb1ff"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:27e13c80ac9a0a1d050ef0a7e0a18cc04c8850101ec891815b6c5a0375e8a245"}, + {file = "pycryptodomex-3.23.0.tar.gz", hash = "sha256:71909758f010c82bc99b0abf4ea12012c98962fbf0583c2164f8b84533c2e4da"}, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -2065,6 +2116,21 @@ files = [ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] +[[package]] +name = "pyzipper" +version = "0.3.6" +description = "AES encryption for zipfile." +optional = false +python-versions = ">=3.4" +groups = ["reports-lambda"] +files = [ + {file = "pyzipper-0.3.6-py2.py3-none-any.whl", hash = "sha256:6d097f465bfa47796b1494e12ea65d1478107d38e13bc56f6e58eedc4f6c1a87"}, + {file = "pyzipper-0.3.6.tar.gz", hash = "sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc"}, +] + +[package.dependencies] +pycryptodomex = "*" + [[package]] name = "regex" version = "2023.12.25" @@ -2541,4 +2607,4 @@ requests = "*" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "5bd6a9f1c4a220910e01dd3a74cb94f89d8fa04704ac1c70d97c6f4867ff7d25" +content-hash = "1a7b4dc509284d68e04fb77167a43fd560d0d05d979eb0bcd605263573ccb354" diff --git a/pyproject.toml b/pyproject.toml index 5bdb4d955e..e19c0ec384 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,3 +65,4 @@ polars = "1.31.0" [tool.poetry.group.reports_lambda.dependencies] openpyxl = "^3.1.5" reportlab = "^4.3.1" +pyzipper = "^0.3.6" From 0c98c3489635ae3aed329b6654b968bf476bacf6 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 30 Jan 2026 15:50:59 +0000 Subject: [PATCH 42/60] [PRMP-1057-2] fixed comments --- lambdas/enums/report_distribution_action.py | 6 + .../handlers/report_distribution_handler.py | 11 +- lambdas/tests/unit/handlers/conftest.py | 103 ++++++-- .../test_report_distribution_handler.py | 61 ++--- .../test_report_orchestration_handler.py | 90 ++----- .../test_report_contact_repository.py | 69 +++-- .../test_reporting_dynamo_repository.py | 228 ++++++++--------- .../test_excel_report_generator_service.py | 145 ++++++----- .../test_report_distribution_service.py | 187 ++++++-------- .../test_report_orchestration_service.py | 237 ++++++++---------- 10 files changed, 535 insertions(+), 602 deletions(-) create mode 100644 lambdas/enums/report_distribution_action.py diff --git a/lambdas/enums/report_distribution_action.py b/lambdas/enums/report_distribution_action.py new file mode 100644 index 0000000000..903314b771 --- /dev/null +++ b/lambdas/enums/report_distribution_action.py @@ -0,0 +1,6 @@ +from enum import StrEnum + + +class ReportDistributionAction(StrEnum): + LIST = "list" + PROCESS_ONE = "process_one" \ No newline at end of file diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index da32e9d409..af6c0f2c7e 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -2,6 +2,7 @@ from typing import Any, Dict from enums.lambda_error import LambdaError +from enums.report_distribution_action import ReportDistributionAction from repositories.reporting.report_contact_repository import ReportContactRepository from services.base.s3_service import S3Service from services.email_service import EmailService @@ -29,8 +30,11 @@ @set_request_context_for_logging def lambda_handler(event, context) -> Dict[str, Any]: action = event.get("action") - if action not in {"list", "process_one"}: - logger.error(f"Invalid action. Expected 'list' or 'process_one'.") + if action not in { + ReportDistributionAction.LIST, + ReportDistributionAction.PROCESS_ONE, + }: + logger.error("Invalid action. Expected 'list' or 'process_one'.") raise ReportDistributionException(400, LambdaError.InvalidAction) bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] @@ -53,7 +57,7 @@ def lambda_handler(event, context) -> Dict[str, Any]: response = {"status": "ok", "bucket": bucket} - if action == "list": + if action == ReportDistributionAction.LIST: prefix = event["prefix"] keys = service.list_xlsx_keys(prefix=prefix) logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") @@ -66,4 +70,3 @@ def lambda_handler(event, context) -> Dict[str, Any]: response.update({"key": key, "ods_code": ods_code}) return response - diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index a7cd749de9..c744cb885b 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -2,6 +2,7 @@ import pytest from enums.feature_flags import FeatureFlags +from enums.report_distribution_action import ReportDistributionAction from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.feature_flags_service import FeatureFlagService @@ -225,17 +226,11 @@ def mock_upload_document_iteration_3_disabled(mocker): @pytest.fixture -def required_report_distribution_env(mocker): - mocker.patch.dict( - os.environ, - { - "REPORT_BUCKET_NAME": "my-report-bucket", - "CONTACT_TABLE_NAME": "contact-table", - "PRM_MAILBOX_EMAIL": "prm@example.com", - "SES_FROM_ADDRESS": "from@example.com", - }, - clear=False, - ) +def required_report_distribution_env(monkeypatch): + monkeypatch.setenv("REPORT_BUCKET_NAME", "my-report-bucket") + monkeypatch.setenv("CONTACT_TABLE_NAME", "contact-table") + monkeypatch.setenv("PRM_MAILBOX_EMAIL", "prm@example.com") + monkeypatch.setenv("SES_FROM_ADDRESS", "from@example.com") @pytest.fixture @@ -246,15 +241,10 @@ def lambda_context(mocker): @pytest.fixture -def required_report_orchestration_env(mocker): - mocker.patch.dict( - os.environ, - { - "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", - "REPORT_BUCKET_NAME": "test-report-bucket", - }, - clear=False, - ) +def required_report_orchestration_env(monkeypatch): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + monkeypatch.setenv("REPORT_BUCKET_NAME", "test-report-bucket") + @pytest.fixture @@ -268,3 +258,76 @@ def mock_reporting_dynamo_service(mocker): @pytest.fixture def reporting_repo(mock_reporting_dynamo_service): return ReportingDynamoRepository(table_name="TestTable") + + +@pytest.fixture +def report_distribution_list_event(): + return {"action": ReportDistributionAction.LIST, "prefix": "p/"} + + +@pytest.fixture +def report_distribution_process_one_event(): + return {"action": ReportDistributionAction.PROCESS_ONE, "key": "reports/ABC/whatever.xlsx"} + + +@pytest.fixture +def mock_report_distribution_wiring(mocker): + svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") + + mocker.patch( + "handlers.report_distribution_handler.S3Service", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch( + "handlers.report_distribution_handler.ReportContactRepository", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch( + "handlers.report_distribution_handler.EmailService", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch( + "handlers.report_distribution_handler.ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + + return svc_instance + +from unittest.mock import MagicMock + +import pytest + + +@pytest.fixture +def mock_report_orchestration_wiring(mocker): + logger = mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) + + mock_window = mocker.patch( + "handlers.report_orchestration_handler.calculate_reporting_window", + return_value=(100, 200), + ) + mock_report_date = mocker.patch( + "handlers.report_orchestration_handler.get_report_date_folder", + return_value="2026-01-02", + ) + + orchestration_service = MagicMock(name="ReportOrchestrationService") + s3_service = MagicMock(name="S3Service") + + build_services = mocker.patch( + "handlers.report_orchestration_handler.build_services", + return_value=(orchestration_service, s3_service), + ) + + return { + "logger": logger, + "mock_window": mock_window, + "mock_report_date": mock_report_date, + "build_services": build_services, + "orchestration_service": orchestration_service, + "s3_service": s3_service, + } diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index ed7371a634..9e5e2cb5b2 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -15,7 +15,7 @@ def handler_module(): def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( mocker, handler_module, required_report_distribution_env, lambda_context ): - event = {"action": "list", "prefix": "reports/2026-01-01/"} + event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "reports/2026-01-01/"} s3_instance = mocker.Mock(name="S3ServiceInstance") contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") @@ -69,28 +69,18 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( - mocker, handler_module, required_report_distribution_env, lambda_context + handler_module, + required_report_distribution_env, + lambda_context, + report_distribution_list_event, + mock_report_distribution_wiring, ): - event = {"action": "list", "prefix": "p/", "bucket": "override-bucket"} - - svc_instance = mocker.Mock() - svc_instance.list_xlsx_keys.return_value = [] - - mocker.patch.object( - handler_module, - "ReportDistributionService", - autospec=True, - return_value=svc_instance, - ) - mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object( - handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock() - ) - mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + event = {**report_distribution_list_event, "bucket": "override-bucket"} + mock_report_distribution_wiring.list_xlsx_keys.return_value = [] result = handler_module.lambda_handler(event, lambda_context) - svc_instance.list_xlsx_keys.assert_called_once_with(prefix="p/") + mock_report_distribution_wiring.list_xlsx_keys.assert_called_once_with(prefix="p/") assert result == { "status": "ok", "bucket": "override-bucket", @@ -100,30 +90,23 @@ def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( def test_lambda_handler_process_one_mode_happy_path( - mocker, handler_module, required_report_distribution_env, lambda_context + handler_module, + required_report_distribution_env, + lambda_context, + report_distribution_process_one_event, + mock_report_distribution_wiring, ): - event = {"action": "process_one", "key": "reports/ABC/whatever.xlsx"} + event = {**report_distribution_process_one_event, "key": "reports/ABC/whatever.xlsx"} - svc_instance = mocker.Mock() - svc_instance.extract_ods_code_from_key.return_value = "ABC" - svc_instance.process_one_report.return_value = None - - mocker.patch.object( - handler_module, - "ReportDistributionService", - autospec=True, - return_value=svc_instance, - ) - mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object( - handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock() - ) - mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + mock_report_distribution_wiring.extract_ods_code_from_key.return_value = "ABC" + mock_report_distribution_wiring.process_one_report.return_value = None result = handler_module.lambda_handler(event, lambda_context) - svc_instance.extract_ods_code_from_key.assert_called_once_with("reports/ABC/whatever.xlsx") - svc_instance.process_one_report.assert_called_once_with( + mock_report_distribution_wiring.extract_ods_code_from_key.assert_called_once_with( + "reports/ABC/whatever.xlsx" + ) + mock_report_distribution_wiring.process_one_report.assert_called_once_with( ods_code="ABC", key="reports/ABC/whatever.xlsx" ) @@ -165,7 +148,7 @@ def test_lambda_handler_returns_500_when_required_env_missing(mocker, handler_mo ) os.environ.pop("REPORT_BUCKET_NAME", None) - event = {"action": "list", "prefix": "p/"} + event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "p/"} result = handler_module.lambda_handler(event, lambda_context) diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 0ab0dac213..241a12e3fb 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,7 +1,5 @@ import importlib import json -import os -from unittest.mock import MagicMock import pytest @@ -13,51 +11,14 @@ def handler_module(): return importlib.import_module(MODULE_UNDER_TEST) -@pytest.fixture -def mock_logger(mocker, handler_module): - return mocker.patch.object(handler_module, "logger", new=MagicMock()) - - -@pytest.fixture -def mock_window(mocker, handler_module): - return mocker.patch.object( - handler_module, - "calculate_reporting_window", - return_value=(100, 200), - ) - - -@pytest.fixture -def mock_report_date(mocker, handler_module): - return mocker.patch.object( - handler_module, - "get_report_date_folder", - return_value="2026-01-02", - ) - - -@pytest.fixture -def mock_build_services(mocker, handler_module): - orchestration_service = MagicMock() - s3_service = MagicMock() - patcher = mocker.patch.object( - handler_module, - "build_services", - return_value=(orchestration_service, s3_service), - ) - return patcher, orchestration_service, s3_service - - def test_lambda_handler_calls_service_and_returns_expected_response( handler_module, required_report_orchestration_env, lambda_context, - mock_logger, - mock_build_services, - mock_window, - mock_report_date, + mock_report_orchestration_wiring, ): - build_services_patcher, orchestration_service, s3_service = mock_build_services + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + s3_service = mock_report_orchestration_wiring["s3_service"] orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", @@ -66,7 +27,7 @@ def test_lambda_handler_calls_service_and_returns_expected_response( result = handler_module.lambda_handler(event={}, context=lambda_context) - build_services_patcher.assert_called_once_with("TestTable") + mock_report_orchestration_wiring["build_services"].assert_called_once_with("TestTable") orchestration_service.process_reporting_window.assert_called_once_with( window_start_ts=100, @@ -83,35 +44,34 @@ def test_lambda_handler_calls_service_and_returns_expected_response( "Report-Orchestration/2026-01-02/B67890.xlsx", } - mock_logger.info.assert_any_call("Report orchestration lambda invoked") + mock_report_orchestration_wiring["logger"].info.assert_any_call( + "Report orchestration lambda invoked" + ) def test_lambda_handler_calls_window_function( handler_module, required_report_orchestration_env, lambda_context, - mock_build_services, - mock_window, - mock_report_date, + mock_report_orchestration_wiring, ): - _, orchestration_service, _ = mock_build_services + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] orchestration_service.process_reporting_window.return_value = {} handler_module.lambda_handler(event={}, context=lambda_context) - mock_window.assert_called_once() + mock_report_orchestration_wiring["mock_window"].assert_called_once() def test_lambda_handler_returns_empty_keys_when_no_reports_generated( handler_module, required_report_orchestration_env, lambda_context, - mock_build_services, - mock_logger, - mock_window, - mock_report_date, + mock_report_orchestration_wiring, ): - _, orchestration_service, s3_service = mock_build_services + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + s3_service = mock_report_orchestration_wiring["s3_service"] + orchestration_service.process_reporting_window.return_value = {} result = handler_module.lambda_handler(event={}, context=lambda_context) @@ -125,18 +85,20 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( } s3_service.upload_file_with_extra_args.assert_not_called() - mock_logger.info.assert_any_call("No reports generated; exiting") + mock_report_orchestration_wiring["logger"].info.assert_any_call( + "No reports generated; exiting" + ) def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( handler_module, required_report_orchestration_env, lambda_context, - mock_build_services, - mock_window, - mock_report_date, + mock_report_orchestration_wiring, ): - _, orchestration_service, s3_service = mock_build_services + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + s3_service = mock_report_orchestration_wiring["s3_service"] + orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "UNKNOWN": "/tmp/UNKNOWN.xlsx", @@ -169,14 +131,10 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( def test_lambda_handler_returns_error_when_required_env_missing( handler_module, lambda_context, - mocker, + monkeypatch, ): - mocker.patch.dict( - os.environ, - {"BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable"}, - clear=False, - ) - os.environ.pop("REPORT_BUCKET_NAME", None) + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + monkeypatch.delenv("REPORT_BUCKET_NAME", raising=False) result = handler_module.lambda_handler(event={}, context=lambda_context) diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py index d534267321..4216c3407c 100644 --- a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -5,12 +5,12 @@ @pytest.fixture def mock_dynamo(mocker): - mock = mocker.Mock() + dynamo = mocker.Mock() mocker.patch( "repositories.reporting.report_contact_repository.DynamoDBService", - return_value=mock, + return_value=dynamo, ) - return mock + return dynamo @pytest.fixture @@ -18,13 +18,32 @@ def repo(mock_dynamo): return ReportContactRepository(table_name="report-contacts") -def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): - mock_dynamo.get_item.return_value = { - "Item": { - "OdsCode": "Y12345", - "Email": "contact@example.com", - } - } +@pytest.mark.parametrize( + "dynamo_response, expected_email", + [ + ( + { + "Item": { + "OdsCode": "Y12345", + "Email": "contact@example.com", + } + }, + "contact@example.com", + ), + ({}, None), + ( + { + "Item": { + "OdsCode": "Y12345", + } + }, + None, + ), + (None, None), + ], +) +def test_get_contact_email(repo, mock_dynamo, dynamo_response, expected_email): + mock_dynamo.get_item.return_value = dynamo_response result = repo.get_contact_email("Y12345") @@ -32,32 +51,4 @@ def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): table_name="report-contacts", key={"OdsCode": "Y12345"}, ) - assert result == "contact@example.com" - - -def test_get_contact_email_returns_none_when_item_missing(repo, mock_dynamo): - mock_dynamo.get_item.return_value = {} # or None - - result = repo.get_contact_email("Y12345") - - mock_dynamo.get_item.assert_called_once_with( - table_name="report-contacts", - key={"OdsCode": "Y12345"}, - ) - assert result is None - - -def test_get_contact_email_returns_none_when_email_missing(repo, mock_dynamo): - mock_dynamo.get_item.return_value = { - "Item": { - "OdsCode": "Y12345", - } - } - - result = repo.get_contact_email("Y12345") - - mock_dynamo.get_item.assert_called_once_with( - table_name="report-contacts", - key={"OdsCode": "Y12345"}, - ) - assert result is None + assert result == expected_email diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index 7626bde154..40f193c2df 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -1,122 +1,124 @@ from datetime import date -def test_get_records_for_time_window_same_date_queries_once( - mocker, mock_reporting_dynamo_service, reporting_repo -): - mock_utc_date = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date" - ) - mock_utc_date.side_effect = [date(2026, 1, 7), date(2026, 1, 7)] - - mock_reporting_dynamo_service.query_by_key_condition_expression.return_value = [ - { - "Date": "2026-01-07", - "Timestamp": 1704601000, - "UploaderOdsCode": "A12345", - "DocumentId": "doc-001", - "Status": "UPLOADED", - } - ] - - result = reporting_repo.get_records_for_time_window(100, 200) +import pytest - assert result == [ - { - "Date": "2026-01-07", - "Timestamp": 1704601000, - "UploaderOdsCode": "A12345", - "DocumentId": "doc-001", - "Status": "UPLOADED", - } - ] - mock_reporting_dynamo_service.query_by_key_condition_expression.assert_called_once() - call_kwargs = ( - mock_reporting_dynamo_service.query_by_key_condition_expression.call_args.kwargs - ) - assert call_kwargs["table_name"] == "TestTable" - assert call_kwargs["index_name"] == "TimestampIndex" - assert "key_condition_expression" in call_kwargs - - -def test_get_records_for_time_window_different_dates_queries_twice( - mocker, mock_reporting_dynamo_service, reporting_repo +@pytest.mark.parametrize( + "start_dt,end_dt,service_side_effect,expected,expected_call_count", + [ + ( + date(2026, 1, 7), + date(2026, 1, 7), + [ + [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ] + ], + [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ], + 1, + ), + ( + date(2026, 1, 6), + date(2026, 1, 7), + [ + [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + } + ], + [ + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ], + ], + [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ], + 2, + ), + ( + date(2026, 1, 6), + date(2026, 1, 7), + [ + [], + [], + ], + [], + 2, + ), + ], +) +def test_get_records_for_time_window( + mocker, + mock_reporting_dynamo_service, + reporting_repo, + start_dt, + end_dt, + service_side_effect, + expected, + expected_call_count, ): - mock_utc_date = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date" - ) - mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] - - mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = [ - [ - { - "Date": "2026-01-06", - "Timestamp": 1704517200, - "UploaderOdsCode": "A12345", - "DocumentId": "doc-101", - "Status": "UPLOADED", - } - ], - [ - { - "Date": "2026-01-07", - "Timestamp": 1704590000, - "UploaderOdsCode": "B67890", - "DocumentId": "doc-202", - "Status": "PROCESSED", - }, - { - "Date": "2026-01-07", - "Timestamp": 1704593600, - "UploaderOdsCode": "B67890", - "DocumentId": "doc-203", - "Status": "PROCESSED", - }, - ], - ] + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [start_dt, end_dt] + mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = service_side_effect result = reporting_repo.get_records_for_time_window(100, 200) - assert result == [ - { - "Date": "2026-01-06", - "Timestamp": 1704517200, - "UploaderOdsCode": "A12345", - "DocumentId": "doc-101", - "Status": "UPLOADED", - }, - { - "Date": "2026-01-07", - "Timestamp": 1704590000, - "UploaderOdsCode": "B67890", - "DocumentId": "doc-202", - "Status": "PROCESSED", - }, - { - "Date": "2026-01-07", - "Timestamp": 1704593600, - "UploaderOdsCode": "B67890", - "DocumentId": "doc-203", - "Status": "PROCESSED", - }, - ] - assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == 2 - - -def test_get_records_for_time_window_returns_empty_list_when_no_items( - mocker, mock_reporting_dynamo_service, reporting_repo -): - mock_utc_date = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date" - ) - mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] - - mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = [ - [], - [], - ] - - result = reporting_repo.get_records_for_time_window(100, 200) + assert result == expected + assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == expected_call_count - assert result == [] - assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == 2 + for call in mock_reporting_dynamo_service.query_by_key_condition_expression.call_args_list: + kwargs = call.kwargs + assert kwargs["table_name"] == "TestTable" + assert kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in kwargs diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index f81559bea3..225227983b 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -1,6 +1,7 @@ import pytest from freezegun import freeze_time from openpyxl import load_workbook + from services.reporting.excel_report_generator_service import ExcelReportGenerator @@ -9,13 +10,43 @@ def excel_report_generator(): return ExcelReportGenerator() -@freeze_time("2025-01-01T12:00:00") -def test_create_report_orchestration_xlsx_happy_path( - excel_report_generator, - tmp_path, -): - output_file = tmp_path / "report.xlsx" +@pytest.fixture +def make_report(tmp_path, excel_report_generator): + def _make(*, ods_code="Y12345", records=None, filename="report.xlsx"): + records = records or [] + output_file = tmp_path / filename + + result = excel_report_generator.create_report_orchestration_xlsx( + ods_code=ods_code, + records=records, + output_path=str(output_file), + ) + + assert result == str(output_file) + assert output_file.exists() + + wb = load_workbook(output_file) + ws = wb.active + return output_file, ws + return _make + + +@pytest.fixture +def expected_header_row(): + return [ + "NHS Number", + "Date", + "Uploader ODS", + "PDS ODS", + "Upload Status", + "Reason", + "File Path", + ] + + +@freeze_time("2025-01-01T12:00:00") +def test_create_report_orchestration_xlsx_happy_path(make_report, expected_header_row): ods_code = "Y12345" records = [ { @@ -40,34 +71,15 @@ def test_create_report_orchestration_xlsx_happy_path( }, ] - result = excel_report_generator.create_report_orchestration_xlsx( - ods_code=ods_code, - records=records, - output_path=str(output_file), - ) - - assert result == str(output_file) - assert output_file.exists() - - wb = load_workbook(output_file) - ws = wb.active + _, ws = make_report(ods_code=ods_code, records=records, filename="report.xlsx") assert ws.title == "Daily Upload Report" assert ws["A1"].value == f"ODS Code: {ods_code}" - assert ws["A2"].value.startswith("Generated at (UTC): ") - assert ws["A3"].value is None # blank row + assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00+00:00" + assert ws["A3"].value is None - assert [cell.value for cell in ws[4]] == [ - "NHS Number", - "Date", - "Uploader ODS", - "PDS ODS", - "Upload Status", - "Reason", - "File Path", - ] + assert [cell.value for cell in ws[4]] == expected_header_row - # First data row (no ID column) assert [cell.value for cell in ws[5]] == [ "1234567890", "2025-01-01", @@ -89,54 +101,37 @@ def test_create_report_orchestration_xlsx_happy_path( ] -def test_create_report_orchestration_xlsx_with_no_records( - excel_report_generator, - tmp_path, -): - output_file = tmp_path / "empty_report.xlsx" - - excel_report_generator.create_report_orchestration_xlsx( - ods_code="Y12345", - records=[], - output_path=str(output_file), - ) - - wb = load_workbook(output_file) - ws = wb.active +def test_create_report_orchestration_xlsx_with_no_records(make_report, expected_header_row): + _, ws = make_report(records=[], filename="empty_report.xlsx") assert ws.max_row == 4 - - + assert [cell.value for cell in ws[4]] == expected_header_row + + +@pytest.mark.parametrize( + "records, expected_row", + [ + ( + [{"ID": 1, "NhsNumber": "1234567890"}], + ["1234567890", None, None, None, None, None, None], + ), + ( + [{"Date": "2025-01-01"}], + [None, "2025-01-01", None, None, None, None, None], + ), + ( + [{"UploaderOdsCode": "Y12345", "UploadStatus": "SUCCESS"}], + [None, None, "Y12345", None, "SUCCESS", None, None], + ), + ], +) def test_create_report_orchestration_xlsx_handles_missing_fields( - excel_report_generator, - tmp_path, + make_report, + expected_header_row, + records, + expected_row, ): - output_file = tmp_path / "partial.xlsx" - - records = [ - { - "ID": 1, - "NhsNumber": "1234567890", - } - ] - - excel_report_generator.create_report_orchestration_xlsx( - ods_code="Y12345", - records=records, - output_path=str(output_file), - ) - - wb = load_workbook(output_file) - ws = wb.active + _, ws = make_report(records=records, filename="partial.xlsx") - row = [cell.value for cell in ws[5]] - - assert row == [ - "1234567890", - None, - None, - None, - None, - None, - None, - ] + assert [cell.value for cell in ws[4]] == expected_header_row + assert [cell.value for cell in ws[5]] == expected_row diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index 3bff899731..b8ecde73fe 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -34,24 +34,31 @@ def service(mock_s3_service, mock_contact_repo, mock_email_service): ) -def test_extract_ods_code_from_key_strips_xlsx_extension(): - assert ( - ReportDistributionService.extract_ods_code_from_key( - "Report-Orchestration/2026-01-01/Y12345.xlsx" - ) - == "Y12345" +@pytest.fixture +def fixed_tmpdir(mocker): + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, ) + return fake_tmp -def test_extract_ods_code_from_key_is_case_insensitive(): - assert ReportDistributionService.extract_ods_code_from_key("a/b/C789.XLSX") == "C789" - - -def test_extract_ods_code_from_key_keeps_non_xlsx_filename(): - assert ( - ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") - == "report.csv" - ) +@pytest.mark.parametrize( + "key, expected", + [ + ("Report-Orchestration/2026-01-01/Y12345.xlsx", "Y12345"), + ("a/b/C789.XLSX", "C789"), + ("a/b/report.csv", "report.csv"), + ("just-a-name", "just-a-name"), + ("a/b/noext", "noext"), + ], +) +def test_extract_ods_code_from_key(key, expected): + assert ReportDistributionService.extract_ods_code_from_key(key) == expected def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): @@ -81,27 +88,21 @@ def test_list_xlsx_keys_returns_empty_when_no_objects(service, mock_s3_service): keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + mock_s3_service.list_object_keys.assert_called_once_with( + bucket_name="my-bucket", + prefix="Report-Orchestration/2026-01-01/", + ) assert keys == [] - mock_s3_service.list_object_keys.assert_called_once() def test_process_one_report_downloads_encrypts_and_delegates_email( - service, mocker, mock_s3_service + service, mocker, mock_s3_service, fixed_tmpdir ): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", return_value="fixed-password", ) - fake_tmp = "/tmp/fake_tmpdir" - td = mocker.MagicMock() - td.__enter__.return_value = fake_tmp - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, - ) - mocked_zip = mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True, @@ -113,8 +114,8 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( key="Report-Orchestration/2026-01-01/Y12345.xlsx", ) - local_xlsx = os.path.join(fake_tmp, "Y12345.xlsx") - local_zip = os.path.join(fake_tmp, "Y12345.zip") + local_xlsx = os.path.join(fixed_tmpdir, "Y12345.xlsx") + local_zip = os.path.join(fixed_tmpdir, "Y12345.zip") mock_s3_service.download_file.assert_called_once_with( "my-bucket", @@ -133,17 +134,11 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( ) -def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_service): +def test_process_one_report_propagates_download_errors( + service, mocker, mock_s3_service, fixed_tmpdir +): mock_s3_service.download_file.side_effect = RuntimeError("download failed") - td = mocker.MagicMock() - td.__enter__.return_value = "/tmp/fake_tmpdir" - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, - ) - mocked_zip = mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True ) @@ -156,20 +151,13 @@ def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_ mocked_send.assert_not_called() -def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker, mock_s3_service): +def test_process_one_report_does_not_send_email_if_zip_fails( + service, mocker, mock_s3_service, fixed_tmpdir +): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", return_value="pw", ) - - td = mocker.MagicMock() - td.__enter__.return_value = "/tmp/fake_tmpdir" - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, - ) - mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", side_effect=RuntimeError("zip failed"), @@ -184,22 +172,13 @@ def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker, mo def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fails( - service, mocker, mock_s3_service + service, mocker, mock_s3_service, fixed_tmpdir ): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", side_effect=RuntimeError("secrets failed"), ) - fake_tmp = "/tmp/fake_tmpdir" - td = mocker.MagicMock() - td.__enter__.return_value = fake_tmp - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, - ) - mocked_zip = mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True, @@ -212,44 +191,35 @@ def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fa mock_s3_service.download_file.assert_called_once_with( "my-bucket", "k.xlsx", - os.path.join(fake_tmp, "Y12345.xlsx"), + os.path.join(fixed_tmpdir, "Y12345.xlsx"), ) mocked_zip.assert_not_called() mocked_send.assert_not_called() -def test_send_report_emails_with_contact_calls_email_contact( - service, mock_contact_repo, mocker +@pytest.mark.parametrize( + "contact_lookup_result, contact_lookup_side_effect, expected_method", + [ + ("contact@example.com", None, "email_contact"), + (None, None, "email_prm_missing_contact"), + (None, RuntimeError("ddb down"), "email_prm_missing_contact"), + ], +) +def test_send_report_emails_routes_correctly( + service, + mock_contact_repo, + mocker, + contact_lookup_result, + contact_lookup_side_effect, + expected_method, ): - mock_contact_repo.get_contact_email.return_value = "contact@example.com" + if contact_lookup_side_effect is not None: + mock_contact_repo.get_contact_email.side_effect = contact_lookup_side_effect + else: + mock_contact_repo.get_contact_email.return_value = contact_lookup_result mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object( - service, "email_prm_missing_contact", autospec=True - ) - - service.send_report_emails( - ods_code="Y12345", - attachment_path="/tmp/Y12345.zip", - password="pw", - ) - - mock_contact_repo.get_contact_email.assert_called_once_with("Y12345") - mocked_email_contact.assert_called_once_with( - to_address="contact@example.com", - attachment_path="/tmp/Y12345.zip", - password="pw", - ) - mocked_email_prm.assert_not_called() - - -def test_send_report_emails_without_contact_calls_email_prm(service, mock_contact_repo, mocker): - mock_contact_repo.get_contact_email.return_value = None - - mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object( - service, "email_prm_missing_contact", autospec=True - ) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) service.send_report_emails( ods_code="A99999", @@ -258,36 +228,21 @@ def test_send_report_emails_without_contact_calls_email_prm(service, mock_contac ) mock_contact_repo.get_contact_email.assert_called_once_with("A99999") - mocked_email_prm.assert_called_once_with( - ods_code="A99999", - attachment_path="/tmp/A99999.zip", - password="pw", - ) - mocked_email_contact.assert_not_called() - - -def test_send_report_emails_contact_lookup_exception_falls_back_to_prm( - service, mock_contact_repo, mocker -): - mock_contact_repo.get_contact_email.side_effect = RuntimeError("ddb down") - - mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object( - service, "email_prm_missing_contact", autospec=True - ) - - service.send_report_emails( - ods_code="A99999", - attachment_path="/tmp/A99999.zip", - password="pw", - ) - mocked_email_contact.assert_not_called() - mocked_email_prm.assert_called_once_with( - ods_code="A99999", - attachment_path="/tmp/A99999.zip", - password="pw", - ) + if expected_method == "email_contact": + mocked_email_contact.assert_called_once_with( + to_address="contact@example.com", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + mocked_email_prm.assert_not_called() + else: + mocked_email_prm.assert_called_once_with( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + mocked_email_contact.assert_not_called() def test_email_contact_sends_report_and_password(service, mock_email_service): diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index baf54577fe..85685c8449 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -23,158 +23,135 @@ def report_orchestration_service(mock_repository, mock_excel_generator): ) -def test_process_reporting_window_no_records_returns_empty_dict_and_does_not_generate( - report_orchestration_service, mock_repository, mock_excel_generator -): - mock_repository.get_records_for_time_window.return_value = [] - - result = report_orchestration_service.process_reporting_window(100, 200) - - assert result == {} - mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() - - -def test_process_reporting_window_calls_repository_with_window_args( - report_orchestration_service, mock_repository, mocker -): - mock_repository.get_records_for_time_window.return_value = [{"UploaderOdsCode": "X1", "ID": 1}] - mocked_generate = mocker.patch.object(report_orchestration_service, "generate_ods_report", return_value="/tmp/x.xlsx") - - report_orchestration_service.process_reporting_window(100, 200) - - mock_repository.get_records_for_time_window.assert_called_once_with(100, 200) - mocked_generate.assert_called_once() - - -def test_process_reporting_window_generates_reports_per_ods( - report_orchestration_service, mock_repository, mocker -): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - {"UploaderOdsCode": "A99999", "ID": 3}, - ] - mock_repository.get_records_for_time_window.return_value = records - - mocked_generate = mocker.patch.object( - report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" - ) - - report_orchestration_service.process_reporting_window(100, 200) - - mocked_generate.assert_any_call( - "Y12345", - [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - ], - ) - mocked_generate.assert_any_call( - "A99999", - [{"UploaderOdsCode": "A99999", "ID": 3}], - ) - assert mocked_generate.call_count == 2 - - -def test_process_reporting_window_returns_mapping_of_ods_to_generated_file_path( - report_orchestration_service, mock_repository, mocker -): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "A99999", "ID": 2}, - ] - mock_repository.get_records_for_time_window.return_value = records - - def _side_effect(ods_code, ods_records): - return f"/tmp/{ods_code}.xlsx" - - mocker.patch.object( +@pytest.fixture +def mocked_generate(report_orchestration_service, mocker): + return mocker.patch.object( report_orchestration_service, "generate_ods_report", - side_effect=_side_effect, + autospec=True, + side_effect=lambda ods_code, _records: f"/tmp/{ods_code}.xlsx", ) - result = report_orchestration_service.process_reporting_window(100, 200) - - assert result == { - "Y12345": "/tmp/Y12345.xlsx", - "A99999": "/tmp/A99999.xlsx", - } - -def test_process_reporting_window_includes_unknown_ods_group( - report_orchestration_service, mock_repository, mocker +@pytest.mark.parametrize( + "records, expected_generate_calls, expected_result", + [ + ( + [], + [], + {}, + ), + ( + [{"UploaderOdsCode": "X1", "ID": 1}], + [ + ("X1", [{"UploaderOdsCode": "X1", "ID": 1}]), + ], + {"X1": "/tmp/X1.xlsx"}, + ), + ( + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + ], + [ + ("Y12345", [{"UploaderOdsCode": "Y12345", "ID": 1}, {"UploaderOdsCode": "Y12345", "ID": 2}]), + ("A99999", [{"UploaderOdsCode": "A99999", "ID": 3}]), + ], + {"Y12345": "/tmp/Y12345.xlsx", "A99999": "/tmp/A99999.xlsx"}, + ), + ( + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"ID": 2}, + {"UploaderOdsCode": None, "ID": 3}, + ], + [ + ("Y12345", [{"UploaderOdsCode": "Y12345", "ID": 1}]), + ("UNKNOWN", [{"ID": 2}, {"UploaderOdsCode": None, "ID": 3}]), + ], + {"Y12345": "/tmp/Y12345.xlsx", "UNKNOWN": "/tmp/UNKNOWN.xlsx"}, + ), + ( + [{"UploaderOdsCode": "", "ID": 1}], + [ + ("UNKNOWN", [{"UploaderOdsCode": "", "ID": 1}]), + ], + {"UNKNOWN": "/tmp/UNKNOWN.xlsx"}, + ), + ], +) +def test_process_reporting_window_behaviour( + report_orchestration_service, + mock_repository, + mocked_generate, + records, + expected_generate_calls, + expected_result, ): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"ID": 2}, # missing ODS -> UNKNOWN - {"UploaderOdsCode": None, "ID": 3}, # null ODS -> UNKNOWN - ] mock_repository.get_records_for_time_window.return_value = records - mocked_generate = mocker.patch.object( - report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" - ) - - report_orchestration_service.process_reporting_window(100, 200) - - # Expect 2 groups: Y12345 and UNKNOWN - assert mocked_generate.call_count == 2 - mocked_generate.assert_any_call( - "Y12345", - [{"UploaderOdsCode": "Y12345", "ID": 1}], - ) - mocked_generate.assert_any_call( - "UNKNOWN", - [{"ID": 2}, {"UploaderOdsCode": None, "ID": 3}], - ) - - -def test_group_records_by_ods_groups_correctly(): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - {"UploaderOdsCode": "A99999", "ID": 3}, - {"ID": 4}, # missing ODS - {"UploaderOdsCode": None, "ID": 5}, # null ODS - ] - - result = ReportOrchestrationService.group_records_by_ods(records) - - assert result["Y12345"] == [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - ] - assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] - assert result["UNKNOWN"] == [ - {"ID": 4}, - {"UploaderOdsCode": None, "ID": 5}, - ] - - -def test_group_records_by_ods_empty_input_returns_empty_mapping(): - result = ReportOrchestrationService.group_records_by_ods([]) - assert dict(result) == {} + result = report_orchestration_service.process_reporting_window(100, 200) + mock_repository.get_records_for_time_window.assert_called_once_with(100, 200) -def test_group_records_by_ods_treats_empty_string_as_unknown(): - records = [{"UploaderOdsCode": "", "ID": 1}] + assert mocked_generate.call_count == len(expected_generate_calls) + for ods_code, ods_records in expected_generate_calls: + mocked_generate.assert_any_call(ods_code, ods_records) + + assert result == expected_result + + +@pytest.mark.parametrize( + "records, expected", + [ + ( + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + {"ID": 4}, # missing ODS + {"UploaderOdsCode": None, "ID": 5}, # null ODS + ], + { + "Y12345": [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ], + "A99999": [{"UploaderOdsCode": "A99999", "ID": 3}], + "UNKNOWN": [{"ID": 4}, {"UploaderOdsCode": None, "ID": 5}], + }, + ), + ([], {}), + ( + [{"UploaderOdsCode": "", "ID": 1}], + {"UNKNOWN": [{"UploaderOdsCode": "", "ID": 1}]}, + ), + ], +) +def test_group_records_by_ods(records, expected): result = ReportOrchestrationService.group_records_by_ods(records) - assert result["UNKNOWN"] == [{"UploaderOdsCode": "", "ID": 1}] + assert dict(result) == expected -def test_generate_ods_report_creates_excel_report_and_returns_path( - report_orchestration_service, mock_excel_generator, mocker -): +@pytest.fixture +def fake_named_tmpfile(mocker): fake_tmp = mocker.MagicMock() fake_tmp.__enter__.return_value = fake_tmp + fake_tmp.__exit__.return_value = False fake_tmp.name = "/tmp/fake_Y12345.xlsx" mocked_ntf = mocker.patch( "services.reporting.report_orchestration_service.tempfile.NamedTemporaryFile", return_value=fake_tmp, ) + return mocked_ntf, fake_tmp + +def test_generate_ods_report_creates_excel_report_and_returns_path( + report_orchestration_service, mock_excel_generator, fake_named_tmpfile +): + mocked_ntf, fake_tmp = fake_named_tmpfile records = [{"ID": 1, "UploaderOdsCode": "Y12345"}] result_path = report_orchestration_service.generate_ods_report("Y12345", records) From a755a54b537d6ff9450153f35c26270e31c0c6c1 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 30 Jan 2026 16:24:07 +0000 Subject: [PATCH 43/60] [PRMP-1057-2] fixed tests --- lambdas/tests/unit/handlers/conftest.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index c744cb885b..70eedc7414 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -1,4 +1,5 @@ import os +from unittest.mock import MagicMock import pytest from enums.feature_flags import FeatureFlags @@ -6,7 +7,6 @@ from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.feature_flags_service import FeatureFlagService - @pytest.fixture def valid_id_event_without_auth_header(): api_gateway_proxy_event = { @@ -297,11 +297,6 @@ def mock_report_distribution_wiring(mocker): return svc_instance -from unittest.mock import MagicMock - -import pytest - - @pytest.fixture def mock_report_orchestration_wiring(mocker): logger = mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) From 4012ab9f69d0da17221c78c01ebb7f9a6e8ba2f0 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 2 Feb 2026 09:01:26 +0000 Subject: [PATCH 44/60] [PRMP-1057-2] fixed tests --- lambdas/tests/unit/conftest.py | 15 +- lambdas/tests/unit/handlers/conftest.py | 14 +- .../test_report_distribution_handler.py | 85 +++------ .../test_report_distribution_service.py | 165 +++++++----------- 4 files changed, 115 insertions(+), 164 deletions(-) diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index dd3a98e4b1..c367205f4b 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -12,8 +12,8 @@ from models.pds_models import Patient, PatientDetails from pydantic import ValidationError from pypdf import PdfWriter -from requests import Response from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from requests import Response from tests.unit.helpers.data.pds.pds_patient_response import PDS_PATIENT from utils.audit_logging_setup import LoggingService @@ -430,3 +430,16 @@ def mock_reporting_dynamo_service(mocker): @pytest.fixture def reporting_repo(mock_reporting_dynamo_service): return ReportingDynamoRepository(table_name="TestTable") + + +@pytest.fixture +def fixed_tmpdir(mocker): + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + return fake_tmp diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index 70eedc7414..ee83467361 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -1,4 +1,3 @@ -import os from unittest.mock import MagicMock import pytest @@ -7,6 +6,7 @@ from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.feature_flags_service import FeatureFlagService + @pytest.fixture def valid_id_event_without_auth_header(): api_gateway_proxy_event = { @@ -231,6 +231,7 @@ def required_report_distribution_env(monkeypatch): monkeypatch.setenv("CONTACT_TABLE_NAME", "contact-table") monkeypatch.setenv("PRM_MAILBOX_EMAIL", "prm@example.com") monkeypatch.setenv("SES_FROM_ADDRESS", "from@example.com") + monkeypatch.setenv("SES_CONFIGURATION_SET", "my-config-set") @pytest.fixture @@ -246,7 +247,6 @@ def required_report_orchestration_env(monkeypatch): monkeypatch.setenv("REPORT_BUCKET_NAME", "test-report-bucket") - @pytest.fixture def mock_reporting_dynamo_service(mocker): mock_cls = mocker.patch( @@ -267,7 +267,10 @@ def report_distribution_list_event(): @pytest.fixture def report_distribution_process_one_event(): - return {"action": ReportDistributionAction.PROCESS_ONE, "key": "reports/ABC/whatever.xlsx"} + return { + "action": ReportDistributionAction.PROCESS_ONE, + "key": "reports/ABC/whatever.xlsx", + } @pytest.fixture @@ -297,9 +300,12 @@ def mock_report_distribution_wiring(mocker): return svc_instance + @pytest.fixture def mock_report_orchestration_wiring(mocker): - logger = mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) + logger = mocker.patch( + "handlers.report_orchestration_handler.logger", new=MagicMock() + ) mock_window = mocker.patch( "handlers.report_orchestration_handler.calculate_reporting_window", diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index 34c5959b6a..b002d53275 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -1,6 +1,7 @@ import importlib import json import os + import pytest MODULE_UNDER_TEST = "handlers.report_distribution_handler" @@ -11,69 +12,21 @@ def handler_module(): return importlib.import_module(MODULE_UNDER_TEST) -@pytest.fixture -def required_env(mocker): - mocker.patch.dict( - os.environ, - { - "REPORT_BUCKET_NAME": "my-report-bucket", - "CONTACT_TABLE_NAME": "contact-table", - "PRM_MAILBOX_EMAIL": "prm@example.com", - "SES_FROM_ADDRESS": "from@example.com", - "SES_CONFIGURATION_SET": "my-config-set", - }, - clear=False, - ) - - def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( - mocker, handler_module, required_report_distribution_env, lambda_context + handler_module, + required_report_distribution_env, + lambda_context, + mock_report_distribution_wiring, + report_distribution_list_event, ): - event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "reports/2026-01-01/"} - - s3_instance = mocker.Mock(name="S3ServiceInstance") - contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") - email_instance = mocker.Mock(name="EmailServiceInstance") - - svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") - svc_instance.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] - - mocked_s3_cls = mocker.patch.object( - handler_module, "S3Service", autospec=True, return_value=s3_instance - ) - mocked_contact_repo_cls = mocker.patch.object( - handler_module, - "ReportContactRepository", - autospec=True, - return_value=contact_repo_instance, - ) - mocked_email_cls = mocker.patch.object( - handler_module, "EmailService", autospec=True, return_value=email_instance - ) - mocked_dist_svc_cls = mocker.patch.object( - handler_module, - "ReportDistributionService", - autospec=True, - return_value=svc_instance, - ) + event = {**report_distribution_list_event, "prefix": "reports/2026-01-01/"} + mock_report_distribution_wiring.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] result = handler_module.lambda_handler(event, lambda_context) - mocked_s3_cls.assert_called_once_with() - mocked_contact_repo_cls.assert_called_once_with("contact-table") - mocked_email_cls.assert_called_once_with(default_configuration_set="my-config-set") - - mocked_dist_svc_cls.assert_called_once_with( - s3_service=s3_instance, - contact_repo=contact_repo_instance, - email_service=email_instance, - bucket="my-report-bucket", - from_address="from@example.com", - prm_mailbox="prm@example.com", + mock_report_distribution_wiring.list_xlsx_keys.assert_called_once_with( + prefix="reports/2026-01-01/" ) - - svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") - assert result == { "status": "ok", "bucket": "my-report-bucket", @@ -110,7 +63,10 @@ def test_lambda_handler_process_one_mode_happy_path( report_distribution_process_one_event, mock_report_distribution_wiring, ): - event = {**report_distribution_process_one_event, "key": "reports/ABC/whatever.xlsx"} + event = { + **report_distribution_process_one_event, + "key": "reports/ABC/whatever.xlsx", + } mock_report_distribution_wiring.extract_ods_code_from_key.return_value = "ABC" mock_report_distribution_wiring.process_one_report.return_value = None @@ -143,14 +99,18 @@ def test_lambda_handler_returns_400_when_action_invalid( assert result["statusCode"] == 400 body = json.loads(result["body"]) - assert body["err_code"] == handler_module.LambdaError.InvalidAction.value["err_code"] + assert ( + body["err_code"] == handler_module.LambdaError.InvalidAction.value["err_code"] + ) assert "Invalid action" in body["message"] if body.get("interaction_id") is not None: assert body["interaction_id"] == lambda_context.aws_request_id -def test_lambda_handler_returns_500_when_required_env_missing(mocker, handler_module, lambda_context): +def test_lambda_handler_returns_500_when_required_env_missing( + mocker, handler_module, lambda_context +): mocker.patch.dict( os.environ, { @@ -161,6 +121,7 @@ def test_lambda_handler_returns_500_when_required_env_missing(mocker, handler_mo clear=False, ) os.environ.pop("REPORT_BUCKET_NAME", None) + os.environ.pop("SES_CONFIGURATION_SET", None) event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "p/"} @@ -171,7 +132,9 @@ def test_lambda_handler_returns_500_when_required_env_missing(mocker, handler_mo body = json.loads(result["body"]) assert body["err_code"] == "ENV_5001" - assert "REPORT_BUCKET_NAME" in body["message"] + assert ("REPORT_BUCKET_NAME" in body["message"]) or ( + "SES_CONFIGURATION_SET" in body["message"] + ) if body.get("interaction_id") is not None: assert body["interaction_id"] == lambda_context.aws_request_id diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index 54374f84d9..1e6bb3406e 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -7,7 +7,6 @@ ) - @pytest.fixture def mock_s3_service(mocker): return mocker.Mock(name="S3Service") @@ -37,36 +36,31 @@ def service(mock_s3_service, mock_contact_repo, mock_email_service): ) -def test_sanitize_ses_tag_value_replaces_disallowed_chars(): - assert _sanitize_ses_tag_value("A B/C") == "A_B_C" - assert _sanitize_ses_tag_value("x@y.com") == "x@y.com" # @ allowed - assert _sanitize_ses_tag_value("a.b-c_d") == "a.b-c_d" # . - _ allowed +@pytest.fixture +def patch_password(mocker): + return mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="pw", + ) -def test_extract_ods_code_from_key_strips_xlsx_extension(): - assert ( - ReportDistributionService.extract_ods_code_from_key( - "Report-Orchestration/2026-01-01/Y12345.xlsx" - ) - == "Y12345" - ) +@pytest.fixture +def patch_send_report_emails(mocker, service): + return mocker.patch.object(service, "send_report_emails") + @pytest.fixture -def fixed_tmpdir(mocker): - fake_tmp = "/tmp/fake_tmpdir" - td = mocker.MagicMock() - td.__enter__.return_value = fake_tmp - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, +def patch_zip_encrypt(mocker): + return mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file" ) - return fake_tmp -def test_extract_ods_code_from_key_is_case_insensitive(): - assert ( - ReportDistributionService.extract_ods_code_from_key("a/b/C789.XLSX") == "C789" - ) + +def test_sanitize_ses_tag_value_replaces_disallowed_chars(): + assert _sanitize_ses_tag_value("A B/C") == "A_B_C" + assert _sanitize_ses_tag_value("x@y.com") == "x@y.com" + assert _sanitize_ses_tag_value("a.b-c_d") == "a.b-c_d" + @pytest.mark.parametrize( "key, expected", @@ -81,12 +75,6 @@ def test_extract_ods_code_from_key_is_case_insensitive(): def test_extract_ods_code_from_key(key, expected): assert ReportDistributionService.extract_ods_code_from_key(key) == expected -def test_extract_ods_code_from_key_keeps_non_xlsx_filename(): - assert ( - ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") - == "report.csv" - ) - def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): mock_s3_service.list_object_keys.return_value = [ @@ -123,19 +111,18 @@ def test_list_xlsx_keys_returns_empty_when_no_objects(service, mock_s3_service): def test_process_one_report_downloads_encrypts_and_delegates_email( - service, mocker, mock_s3_service, fixed_tmpdir + service, + mocker, + mock_s3_service, + fixed_tmpdir, + patch_zip_encrypt, + patch_send_report_emails, ): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", return_value="fixed-password", ) - mocked_zip = mocker.patch( - "services.reporting.report_distribution_service.zip_encrypt_file", - autospec=True, - ) - mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) - service.process_one_report( ods_code="Y12345", key="Report-Orchestration/2026-01-01/Y12345.xlsx", @@ -149,14 +136,15 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( "Report-Orchestration/2026-01-01/Y12345.xlsx", local_xlsx, ) - mocked_zip.assert_called_once_with( + + patch_zip_encrypt.assert_called_once_with( input_path=local_xlsx, output_zip=local_zip, password="fixed-password", ) - mocked_send.assert_called_once() - call_kwargs = mocked_send.call_args.kwargs + patch_send_report_emails.assert_called_once() + call_kwargs = patch_send_report_emails.call_args.kwargs assert call_kwargs["ods_code"] == "Y12345" assert call_kwargs["attachment_path"] == local_zip assert call_kwargs["password"] == "fixed-password" @@ -166,54 +154,32 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( } -def test_process_one_report_sanitizes_tags(service, mocker, mock_s3_service): - mocker.patch( - "services.reporting.report_distribution_service.secrets.token_urlsafe", - return_value="pw", - ) - - fake_tmp = "/tmp/fake_tmpdir" - td = mocker.MagicMock() - td.__enter__.return_value = fake_tmp - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, - ) - -def test_process_one_report_propagates_download_errors( - service, mocker, mock_s3_service, fixed_tmpdir +def test_process_one_report_sanitizes_tags_in_base_tags( + service, + patch_password, + patch_zip_encrypt, + patch_send_report_emails, ): - mocker.patch( - "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True - ) - mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) - service.process_one_report( ods_code="Y 12/345", key="prefix/2026-01-01/Y 12/345.xlsx", ) - assert mocked_send.call_args.kwargs["base_tags"] == { + assert patch_send_report_emails.call_args.kwargs["base_tags"] == { "ods_code": "Y_12_345", "report_key": "prefix_2026-01-01_Y_12_345.xlsx", } def test_process_one_report_propagates_download_errors( - service, mocker, mock_s3_service + service, mock_s3_service, mocker ): mock_s3_service.download_file.side_effect = RuntimeError("download failed") mocked_zip = mocker.patch( - "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True - ) - - mocked_zip = mocker.patch( - "services.reporting.report_distribution_service.zip_encrypt_file", - autospec=True, + "services.reporting.report_distribution_service.zip_encrypt_file" ) - mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + mocked_send = mocker.patch.object(service, "send_report_emails") with pytest.raises(RuntimeError, match="download failed"): service.process_one_report(ods_code="Y12345", key="k.xlsx") @@ -223,18 +189,15 @@ def test_process_one_report_propagates_download_errors( def test_process_one_report_does_not_send_email_if_zip_fails( - service, mocker, mock_s3_service, fixed_tmpdir + service, + mocker, + patch_password, ): - mocker.patch( - "services.reporting.report_distribution_service.secrets.token_urlsafe", - return_value="pw", - ) mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", side_effect=RuntimeError("zip failed"), - autospec=True, ) - mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + mocked_send = mocker.patch.object(service, "send_report_emails") with pytest.raises(RuntimeError, match="zip failed"): service.process_one_report(ods_code="Y12345", key="k.xlsx") @@ -251,10 +214,9 @@ def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fa ) mocked_zip = mocker.patch( - "services.reporting.report_distribution_service.zip_encrypt_file", - autospec=True, + "services.reporting.report_distribution_service.zip_encrypt_file" ) - mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + mocked_send = mocker.patch.object(service, "send_report_emails") with pytest.raises(RuntimeError, match="secrets failed"): service.process_one_report(ods_code="Y12345", key="k.xlsx") @@ -289,33 +251,37 @@ def test_send_report_emails_routes_correctly( else: mock_contact_repo.get_contact_email.return_value = contact_lookup_result - mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object( - service, "email_prm_missing_contact", autospec=True - ) + mocked_email_contact = mocker.patch.object(service, "email_contact") + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact") + + base_tags = {"ods_code": "A99999", "report_key": "k.xlsx"} service.send_report_emails( ods_code="A99999", attachment_path="/tmp/A99999.zip", password="pw", - base_tags={"ods_code": "Y12345", "report_key": "k.xlsx"}, + base_tags=base_tags, ) mock_contact_repo.get_contact_email.assert_called_once_with("A99999") if expected_method == "email_contact": - mocked_email_contact.assert_called_once_with( - to_address="contact@example.com", - attachment_path="/tmp/A99999.zip", - password="pw", - ) + mocked_email_contact.assert_called_once() + assert mocked_email_contact.call_args.kwargs == { + "to_address": "contact@example.com", + "attachment_path": "/tmp/A99999.zip", + "password": "pw", + "base_tags": base_tags, + } mocked_email_prm.assert_not_called() else: - mocked_email_prm.assert_called_once_with( - ods_code="A99999", - attachment_path="/tmp/A99999.zip", - password="pw", - ) + mocked_email_prm.assert_called_once() + assert mocked_email_prm.call_args.kwargs == { + "ods_code": "A99999", + "attachment_path": "/tmp/A99999.zip", + "password": "pw", + "base_tags": base_tags, + } mocked_email_contact.assert_not_called() @@ -345,7 +311,10 @@ def test_email_contact_sends_report_and_password_with_tags(service, mock_email_s ) -def test_email_contact_does_not_send_password_if_report_email_fails(service, mock_email_service): +def test_email_contact_does_not_send_password_if_report_email_fails( + service, mock_email_service +): + base_tags = {"ods_code": "Y12345", "report_key": "k.xlsx"} mock_email_service.send_report_email.side_effect = RuntimeError("SES down") with pytest.raises(RuntimeError, match="SES down"): From d2961b90aa21f040bc5074f4241dd441dc02d8f6 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 2 Feb 2026 09:02:50 +0000 Subject: [PATCH 45/60] [PRMP-1058] formating --- lambdas/handlers/report_orchestration_handler.py | 4 +++- .../services/reporting/report_distribution_service.py | 1 - .../unit/handlers/test_report_orchestration_handler.py | 4 +++- .../reporting/test_report_orchestration_service.py | 9 +++++++-- 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 38a450e76f..a653bcc34c 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -113,5 +113,7 @@ def lambda_handler(event, context) -> Dict[str, Any]: generated_files=generated_files, ) - logger.info(f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}") + logger.info( + f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}" + ) return build_response(report_date=report_date, bucket=report_bucket, keys=keys) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index 7cb0cbb6ef..c532edcb7d 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -3,7 +3,6 @@ import tempfile from typing import Dict, List -import boto3 from repositories.reporting.report_contact_repository import ReportContactRepository from services.base.s3_service import S3Service from services.email_service import EmailService diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 241a12e3fb..d89fafe178 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -27,7 +27,9 @@ def test_lambda_handler_calls_service_and_returns_expected_response( result = handler_module.lambda_handler(event={}, context=lambda_context) - mock_report_orchestration_wiring["build_services"].assert_called_once_with("TestTable") + mock_report_orchestration_wiring["build_services"].assert_called_once_with( + "TestTable" + ) orchestration_service.process_reporting_window.assert_called_once_with( window_start_ts=100, diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index 85685c8449..3159baa78d 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -1,5 +1,4 @@ import pytest - from services.reporting.report_orchestration_service import ReportOrchestrationService @@ -55,7 +54,13 @@ def mocked_generate(report_orchestration_service, mocker): {"UploaderOdsCode": "A99999", "ID": 3}, ], [ - ("Y12345", [{"UploaderOdsCode": "Y12345", "ID": 1}, {"UploaderOdsCode": "Y12345", "ID": 2}]), + ( + "Y12345", + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ], + ), ("A99999", [{"UploaderOdsCode": "A99999", "ID": 3}]), ], {"Y12345": "/tmp/Y12345.xlsx", "A99999": "/tmp/A99999.xlsx"}, From 2361c309227be6c3daa4be5f29fb13f37ed831d7 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 5 Feb 2026 15:43:09 +0000 Subject: [PATCH 46/60] [PRMP-1057-2] update handlers ans services --- .../handlers/report_distribution_handler.py | 14 +--- .../handlers/report_orchestration_handler.py | 19 +---- .../reporting/report_contact_repository.py | 6 +- .../reporting/reporting_dynamo_repository.py | 5 +- .../reporting/report_distribution_service.py | 15 ++-- .../reporting/report_orchestration_service.py | 8 +- lambdas/tests/unit/handlers/conftest.py | 63 +++++++--------- .../test_report_distribution_handler.py | 54 +++---------- .../test_report_orchestration_handler.py | 49 +++++------- .../test_report_contact_repository.py | 25 ++++--- .../test_reporting_dynamo_repository.py | 8 ++ .../test_report_distribution_service.py | 75 ++++++++++++++++--- .../test_report_orchestration_service.py | 54 ++++++++++--- 13 files changed, 209 insertions(+), 186 deletions(-) diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index af6c0f2c7e..9936a7559e 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -38,21 +38,9 @@ def lambda_handler(event, context) -> Dict[str, Any]: raise ReportDistributionException(400, LambdaError.InvalidAction) bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] - contact_table = os.environ["CONTACT_TABLE_NAME"] - prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] - from_address = os.environ["SES_FROM_ADDRESS"] - - s3_service = S3Service() - contact_repo = ReportContactRepository(contact_table) - email_service = EmailService() service = ReportDistributionService( - s3_service=s3_service, - contact_repo=contact_repo, - email_service=email_service, - bucket=bucket, - from_address=from_address, - prm_mailbox=prm_mailbox, + bucket=bucket ) response = {"status": "ok", "bucket": bucket} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 38a450e76f..73ce6c6850 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -34,19 +34,6 @@ def build_s3_key(ods_code: str, report_date: str) -> str: return f"Report-Orchestration/{report_date}/{ods_code}.xlsx" -def get_config() -> Tuple[str, str]: - return os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"], os.environ["REPORT_BUCKET_NAME"] - - -def build_services(table_name: str) -> Tuple[ReportOrchestrationService, S3Service]: - repository = ReportingDynamoRepository(table_name) - excel_generator = ExcelReportGenerator() - orchestration_service = ReportOrchestrationService( - repository=repository, - excel_generator=excel_generator, - ) - return orchestration_service, S3Service() - def upload_generated_reports( s3_service: S3Service, @@ -91,8 +78,10 @@ def build_response(report_date: str, bucket: str, keys: list[str]) -> Dict[str, def lambda_handler(event, context) -> Dict[str, Any]: logger.info("Report orchestration lambda invoked") - table_name, report_bucket = get_config() - orchestration_service, s3_service = build_services(table_name) + report_bucket = os.environ["REPORT_BUCKET_NAME"] + orchestration_service = ReportOrchestrationService( + ) + s3_service = S3Service() window_start, window_end = calculate_reporting_window() report_date = get_report_date_folder() diff --git a/lambdas/repositories/reporting/report_contact_repository.py b/lambdas/repositories/reporting/report_contact_repository.py index 84bd06cab1..d9c21c0c2e 100644 --- a/lambdas/repositories/reporting/report_contact_repository.py +++ b/lambdas/repositories/reporting/report_contact_repository.py @@ -1,9 +1,11 @@ +import os + from services.base.dynamo_service import DynamoDBService class ReportContactRepository: - def __init__(self, table_name: str): - self.table_name = table_name + def __init__(self): + self.table_name = os.environ["CONTACT_TABLE_NAME"] self.dynamo = DynamoDBService() def get_contact_email(self, ods_code: str) -> str | None: diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 86d595d87e..9bb017a7cc 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,3 +1,4 @@ +import os from datetime import timedelta from typing import Dict, List @@ -9,8 +10,8 @@ logger = LoggingService(__name__) class ReportingDynamoRepository: - def __init__(self, table_name: str): - self.table_name = table_name + def __init__(self): + self.table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] self.dynamo_service = DynamoDBService() def get_records_for_time_window( diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index d93274e5e1..5dd75e8f46 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -18,19 +18,14 @@ class ReportDistributionService: def __init__( self, *, - s3_service: S3Service, - contact_repo: ReportContactRepository, - email_service: EmailService, bucket: str, - from_address: str, - prm_mailbox: str, ): - self.s3_service = s3_service - self.contact_repo = contact_repo - self.email_service = email_service + self.s3_service = S3Service() + self.contact_repo = ReportContactRepository() + self.email_service = EmailService() self.bucket = bucket - self.from_address = from_address - self.prm_mailbox = prm_mailbox + self.from_address = os.environ["SES_FROM_ADDRESS"] + self.prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] @staticmethod def extract_ods_code_from_key(key: str) -> str: diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index 69a919cc6e..3842419284 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -2,15 +2,17 @@ from collections import defaultdict from typing import Dict +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from services.reporting.excel_report_generator_service import ExcelReportGenerator from utils.audit_logging_setup import LoggingService logger = LoggingService(__name__) class ReportOrchestrationService: - def __init__(self, repository, excel_generator): - self.repository = repository - self.excel_generator = excel_generator + def __init__(self): + self.repository = ReportingDynamoRepository() + self.excel_generator = ExcelReportGenerator() def process_reporting_window( self, diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index 70eedc7414..f0f72bab4a 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -273,56 +273,51 @@ def report_distribution_process_one_event(): @pytest.fixture def mock_report_distribution_wiring(mocker): svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") - - mocker.patch( - "handlers.report_distribution_handler.S3Service", - autospec=True, - return_value=mocker.Mock(), - ) - mocker.patch( - "handlers.report_distribution_handler.ReportContactRepository", - autospec=True, - return_value=mocker.Mock(), - ) - mocker.patch( - "handlers.report_distribution_handler.EmailService", - autospec=True, - return_value=mocker.Mock(), - ) mocker.patch( "handlers.report_distribution_handler.ReportDistributionService", autospec=True, return_value=svc_instance, ) - return svc_instance +import pytest + + @pytest.fixture def mock_report_orchestration_wiring(mocker): - logger = mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) + from handlers import report_orchestration_handler as handler_module - mock_window = mocker.patch( - "handlers.report_orchestration_handler.calculate_reporting_window", - return_value=(100, 200), + orchestration_service = mocker.Mock(name="ReportOrchestrationServiceInstance") + s3_service = mocker.Mock(name="S3ServiceInstance") + + mocker.patch.object( + handler_module, + "ReportOrchestrationService", + autospec=True, + return_value=orchestration_service, ) - mock_report_date = mocker.patch( - "handlers.report_orchestration_handler.get_report_date_folder", - return_value="2026-01-02", + mocker.patch.object( + handler_module, + "S3Service", + autospec=True, + return_value=s3_service, ) - orchestration_service = MagicMock(name="ReportOrchestrationService") - s3_service = MagicMock(name="S3Service") - - build_services = mocker.patch( - "handlers.report_orchestration_handler.build_services", - return_value=(orchestration_service, s3_service), + mock_window = mocker.patch.object( + handler_module, + "calculate_reporting_window", + return_value=(100, 200), + ) + mock_report_date = mocker.patch.object( + handler_module, + "get_report_date_folder", + return_value="2026-01-02", ) return { - "logger": logger, - "mock_window": mock_window, - "mock_report_date": mock_report_date, - "build_services": build_services, + "handler_module": handler_module, "orchestration_service": orchestration_service, "s3_service": s3_service, + "mock_window": mock_window, + "mock_report_date": mock_report_date, } diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index 9e5e2cb5b2..d70178948e 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -1,41 +1,21 @@ -import importlib import json import os -import pytest -MODULE_UNDER_TEST = "handlers.report_distribution_handler" +from handlers import report_distribution_handler as handler_module -@pytest.fixture -def handler_module(): - return importlib.import_module(MODULE_UNDER_TEST) - - -def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( - mocker, handler_module, required_report_distribution_env, lambda_context +def test_lambda_handler_wires_service_and_returns_result_list_mode( + mocker, required_report_distribution_env, lambda_context ): - event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "reports/2026-01-01/"} - - s3_instance = mocker.Mock(name="S3ServiceInstance") - contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") - email_instance = mocker.Mock(name="EmailServiceInstance") + event = { + "action": handler_module.ReportDistributionAction.LIST, + "prefix": "reports/2026-01-01/", + } svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") svc_instance.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] - mocked_s3_cls = mocker.patch.object( - handler_module, "S3Service", autospec=True, return_value=s3_instance - ) - mocked_contact_repo_cls = mocker.patch.object( - handler_module, - "ReportContactRepository", - autospec=True, - return_value=contact_repo_instance, - ) - mocked_email_cls = mocker.patch.object( - handler_module, "EmailService", autospec=True, return_value=email_instance - ) mocked_dist_svc_cls = mocker.patch.object( handler_module, "ReportDistributionService", @@ -45,19 +25,7 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( result = handler_module.lambda_handler(event, lambda_context) - mocked_s3_cls.assert_called_once_with() - mocked_contact_repo_cls.assert_called_once_with("contact-table") - mocked_email_cls.assert_called_once_with() - - mocked_dist_svc_cls.assert_called_once_with( - s3_service=s3_instance, - contact_repo=contact_repo_instance, - email_service=email_instance, - bucket="my-report-bucket", - from_address="from@example.com", - prm_mailbox="prm@example.com", - ) - + mocked_dist_svc_cls.assert_called_once_with(bucket="my-report-bucket") svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") assert result == { @@ -69,7 +37,6 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( - handler_module, required_report_distribution_env, lambda_context, report_distribution_list_event, @@ -90,7 +57,6 @@ def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( def test_lambda_handler_process_one_mode_happy_path( - handler_module, required_report_distribution_env, lambda_context, report_distribution_process_one_event, @@ -119,7 +85,7 @@ def test_lambda_handler_process_one_mode_happy_path( def test_lambda_handler_returns_400_when_action_invalid( - handler_module, required_report_distribution_env, lambda_context + required_report_distribution_env, lambda_context ): event = {"action": "nope"} @@ -136,7 +102,7 @@ def test_lambda_handler_returns_400_when_action_invalid( assert body["interaction_id"] == lambda_context.aws_request_id -def test_lambda_handler_returns_500_when_required_env_missing(mocker, handler_module, lambda_context): +def test_lambda_handler_returns_500_when_required_env_missing(mocker, lambda_context): mocker.patch.dict( os.environ, { diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 241a12e3fb..3f4eb05632 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,24 +1,17 @@ -import importlib import json -import pytest - -MODULE_UNDER_TEST = "handlers.report_orchestration_handler" - - -@pytest.fixture -def handler_module(): - return importlib.import_module(MODULE_UNDER_TEST) +from handlers import report_orchestration_handler as handler_module def test_lambda_handler_calls_service_and_returns_expected_response( - handler_module, required_report_orchestration_env, lambda_context, mock_report_orchestration_wiring, ): orchestration_service = mock_report_orchestration_wiring["orchestration_service"] s3_service = mock_report_orchestration_wiring["s3_service"] + mock_window = mock_report_orchestration_wiring["mock_window"] + mock_report_date = mock_report_orchestration_wiring["mock_report_date"] orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", @@ -27,44 +20,43 @@ def test_lambda_handler_calls_service_and_returns_expected_response( result = handler_module.lambda_handler(event={}, context=lambda_context) - mock_report_orchestration_wiring["build_services"].assert_called_once_with("TestTable") + mock_window.assert_called_once() + mock_report_date.assert_called_once() orchestration_service.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, ) - assert s3_service.upload_file_with_extra_args.call_count == 2 - assert result["report_date"] == "2026-01-02" - assert result["bucket"] == "test-report-bucket" - assert result["prefix"] == "Report-Orchestration/2026-01-02/" - assert set(result["keys"]) == { - "Report-Orchestration/2026-01-02/A12345.xlsx", - "Report-Orchestration/2026-01-02/B67890.xlsx", + assert result == { + "status": "ok", + "report_date": "2026-01-02", + "bucket": "test-report-bucket", + "prefix": "Report-Orchestration/2026-01-02/", + "keys": [ + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/B67890.xlsx", + ], } - mock_report_orchestration_wiring["logger"].info.assert_any_call( - "Report orchestration lambda invoked" - ) - def test_lambda_handler_calls_window_function( - handler_module, required_report_orchestration_env, lambda_context, mock_report_orchestration_wiring, ): orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + mock_window = mock_report_orchestration_wiring["mock_window"] + orchestration_service.process_reporting_window.return_value = {} handler_module.lambda_handler(event={}, context=lambda_context) - mock_report_orchestration_wiring["mock_window"].assert_called_once() + mock_window.assert_called_once() def test_lambda_handler_returns_empty_keys_when_no_reports_generated( - handler_module, required_report_orchestration_env, lambda_context, mock_report_orchestration_wiring, @@ -83,15 +75,10 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( "prefix": "Report-Orchestration/2026-01-02/", "keys": [], } - s3_service.upload_file_with_extra_args.assert_not_called() - mock_report_orchestration_wiring["logger"].info.assert_any_call( - "No reports generated; exiting" - ) def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( - handler_module, required_report_orchestration_env, lambda_context, mock_report_orchestration_wiring, @@ -114,7 +101,6 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( file_key="Report-Orchestration/2026-01-02/A12345.xlsx", extra_args={"ServerSideEncryption": "aws:kms"}, ) - s3_service.upload_file_with_extra_args.assert_any_call( file_name="/tmp/UNKNOWN.xlsx", s3_bucket_name="test-report-bucket", @@ -129,7 +115,6 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( def test_lambda_handler_returns_error_when_required_env_missing( - handler_module, lambda_context, monkeypatch, ): diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py index 4216c3407c..13f91bac99 100644 --- a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -3,19 +3,25 @@ from repositories.reporting.report_contact_repository import ReportContactRepository +@pytest.fixture +def required_contact_repo_env(monkeypatch): + monkeypatch.setenv("CONTACT_TABLE_NAME", "report-contacts") + + @pytest.fixture def mock_dynamo(mocker): - dynamo = mocker.Mock() + dynamo = mocker.Mock(name="DynamoDBServiceInstance") mocker.patch( "repositories.reporting.report_contact_repository.DynamoDBService", + autospec=True, return_value=dynamo, ) return dynamo @pytest.fixture -def repo(mock_dynamo): - return ReportContactRepository(table_name="report-contacts") +def repo(required_contact_repo_env, mock_dynamo): + return ReportContactRepository() @pytest.mark.parametrize( @@ -31,14 +37,7 @@ def repo(mock_dynamo): "contact@example.com", ), ({}, None), - ( - { - "Item": { - "OdsCode": "Y12345", - } - }, - None, - ), + ({"Item": {"OdsCode": "Y12345"}}, None), (None, None), ], ) @@ -52,3 +51,7 @@ def test_get_contact_email(repo, mock_dynamo, dynamo_response, expected_email): key={"OdsCode": "Y12345"}, ) assert result == expected_email + + +def test_init_reads_table_name_from_env(repo): + assert repo.table_name == "report-contacts" diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index 40f193c2df..21bd57f3a8 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -2,6 +2,14 @@ import pytest +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository + + +@pytest.fixture +def reporting_repo(monkeypatch, mock_reporting_dynamo_service): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + return ReportingDynamoRepository() + @pytest.mark.parametrize( "start_dt,end_dt,service_side_effect,expected,expected_call_count", diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index b8ecde73fe..db8b1e3bff 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -5,34 +5,55 @@ from services.reporting.report_distribution_service import ReportDistributionService +@pytest.fixture +def required_report_distribution_env(monkeypatch): + monkeypatch.setenv("SES_FROM_ADDRESS", "from@example.com") + monkeypatch.setenv("PRM_MAILBOX_EMAIL", "prm@example.com") + + @pytest.fixture def mock_s3_service(mocker): - return mocker.Mock(name="S3Service") + return mocker.Mock(name="S3ServiceInstance") @pytest.fixture def mock_contact_repo(mocker): - repo = mocker.Mock(name="ReportContactRepository") + repo = mocker.Mock(name="ReportContactRepositoryInstance") repo.get_contact_email.return_value = None return repo @pytest.fixture def mock_email_service(mocker): - return mocker.Mock(name="EmailService") + return mocker.Mock(name="EmailServiceInstance") @pytest.fixture -def service(mock_s3_service, mock_contact_repo, mock_email_service): - return ReportDistributionService( - s3_service=mock_s3_service, - contact_repo=mock_contact_repo, - email_service=mock_email_service, - bucket="my-bucket", - from_address="from@example.com", - prm_mailbox="prm@example.com", +def service( + required_report_distribution_env, + mocker, + mock_s3_service, + mock_contact_repo, + mock_email_service, +): + mocker.patch( + "services.reporting.report_distribution_service.S3Service", + autospec=True, + return_value=mock_s3_service, + ) + mocker.patch( + "services.reporting.report_distribution_service.ReportContactRepository", + autospec=True, + return_value=mock_contact_repo, + ) + mocker.patch( + "services.reporting.report_distribution_service.EmailService", + autospec=True, + return_value=mock_email_service, ) + return ReportDistributionService(bucket="my-bucket") + @pytest.fixture def fixed_tmpdir(mocker): @@ -61,6 +82,37 @@ def test_extract_ods_code_from_key(key, expected): assert ReportDistributionService.extract_ods_code_from_key(key) == expected +def test_init_reads_env_and_wires_dependencies(required_report_distribution_env, mocker): + mock_s3 = mocker.Mock(name="S3ServiceInstance") + mock_repo = mocker.Mock(name="ReportContactRepositoryInstance") + mock_email = mocker.Mock(name="EmailServiceInstance") + + mocker.patch( + "services.reporting.report_distribution_service.S3Service", + autospec=True, + return_value=mock_s3, + ) + mocker.patch( + "services.reporting.report_distribution_service.ReportContactRepository", + autospec=True, + return_value=mock_repo, + ) + mocker.patch( + "services.reporting.report_distribution_service.EmailService", + autospec=True, + return_value=mock_email, + ) + + svc = ReportDistributionService(bucket="bucket-1") + + assert svc.bucket == "bucket-1" + assert svc.from_address == "from@example.com" + assert svc.prm_mailbox == "prm@example.com" + assert svc.s3_service is mock_s3 + assert svc.contact_repo is mock_repo + assert svc.email_service is mock_email + + def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): mock_s3_service.list_object_keys.return_value = [ "Report-Orchestration/2026-01-01/A123.xlsx", @@ -68,6 +120,7 @@ def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): "Report-Orchestration/2026-01-01/B456.xls", "Report-Orchestration/2026-01-01/C789.xlsx", "Report-Orchestration/2026-01-01/D000.xlsx", + "Report-Orchestration/2026-01-01/E111.xlsx.tmp", ] keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index 85685c8449..2487fb235c 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -5,22 +5,29 @@ @pytest.fixture def mock_repository(mocker): - repo = mocker.Mock() + repo = mocker.Mock(name="ReportingDynamoRepositoryInstance") repo.get_records_for_time_window.return_value = [] return repo @pytest.fixture def mock_excel_generator(mocker): - return mocker.Mock() + return mocker.Mock(name="ExcelReportGeneratorInstance") @pytest.fixture -def report_orchestration_service(mock_repository, mock_excel_generator): - return ReportOrchestrationService( - repository=mock_repository, - excel_generator=mock_excel_generator, +def report_orchestration_service(mocker, mock_repository, mock_excel_generator): + mocker.patch( + "services.reporting.report_orchestration_service.ReportingDynamoRepository", + autospec=True, + return_value=mock_repository, + ) + mocker.patch( + "services.reporting.report_orchestration_service.ExcelReportGenerator", + autospec=True, + return_value=mock_excel_generator, ) + return ReportOrchestrationService() @pytest.fixture @@ -55,7 +62,13 @@ def mocked_generate(report_orchestration_service, mocker): {"UploaderOdsCode": "A99999", "ID": 3}, ], [ - ("Y12345", [{"UploaderOdsCode": "Y12345", "ID": 1}, {"UploaderOdsCode": "Y12345", "ID": 2}]), + ( + "Y12345", + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ], + ), ("A99999", [{"UploaderOdsCode": "A99999", "ID": 3}]), ], {"Y12345": "/tmp/Y12345.xlsx", "A99999": "/tmp/A99999.xlsx"}, @@ -110,8 +123,8 @@ def test_process_reporting_window_behaviour( {"UploaderOdsCode": "Y12345", "ID": 1}, {"UploaderOdsCode": "Y12345", "ID": 2}, {"UploaderOdsCode": "A99999", "ID": 3}, - {"ID": 4}, # missing ODS - {"UploaderOdsCode": None, "ID": 5}, # null ODS + {"ID": 4}, + {"UploaderOdsCode": None, "ID": 5}, ], { "Y12345": [ @@ -168,3 +181,26 @@ def test_generate_ods_report_creates_excel_report_and_returns_path( records=records, output_path=fake_tmp.name, ) + + +def test_init_constructs_repository_and_excel_generator(mocker): + mock_repo = mocker.Mock(name="ReportingDynamoRepositoryInstance") + mock_excel = mocker.Mock(name="ExcelReportGeneratorInstance") + + mocked_repo_cls = mocker.patch( + "services.reporting.report_orchestration_service.ReportingDynamoRepository", + autospec=True, + return_value=mock_repo, + ) + mocked_excel_cls = mocker.patch( + "services.reporting.report_orchestration_service.ExcelReportGenerator", + autospec=True, + return_value=mock_excel, + ) + + svc = ReportOrchestrationService() + + mocked_repo_cls.assert_called_once_with() + mocked_excel_cls.assert_called_once_with() + assert svc.repository is mock_repo + assert svc.excel_generator is mock_excel From 59f05313df0833a763b625f299c773d3bacd1ee7 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 5 Feb 2026 16:25:00 +0000 Subject: [PATCH 47/60] [PRMP-1057-2] small fix --- lambdas/tests/unit/handlers/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index f0f72bab4a..caaade5496 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -280,8 +280,6 @@ def mock_report_distribution_wiring(mocker): ) return svc_instance -import pytest - @pytest.fixture def mock_report_orchestration_wiring(mocker): From b4bbe40b3328dd052e176fe749c5a2c72a0aa4e6 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 6 Feb 2026 09:33:21 +0000 Subject: [PATCH 48/60] [PRMP-1058] merged and updated with 1057 --- .../handlers/report_distribution_handler.py | 19 +------------ .../handlers/report_orchestration_handler.py | 6 +---- .../reporting/report_distribution_service.py | 5 +++- lambdas/tests/unit/conftest.py | 5 ++-- lambdas/tests/unit/handlers/conftest.py | 8 +++--- .../test_report_distribution_handler.py | 27 ++++++------------- .../test_report_orchestration_handler.py | 5 ++-- .../test_report_distribution_service.py | 21 ++++++++------- 8 files changed, 34 insertions(+), 62 deletions(-) diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index 1aea4e5f43..487c46c16e 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -3,9 +3,6 @@ from enums.lambda_error import LambdaError from enums.report_distribution_action import ReportDistributionAction -from repositories.reporting.report_contact_repository import ReportContactRepository -from services.base.s3_service import S3Service -from services.email_service import EmailService from services.reporting.report_distribution_service import ReportDistributionService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables @@ -39,22 +36,8 @@ def lambda_handler(event, context) -> Dict[str, Any]: raise ReportDistributionException(400, LambdaError.InvalidAction) bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] - # contact_table = os.environ["CONTACT_TABLE_NAME"] - # prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] - # from_address = os.environ["SES_FROM_ADDRESS"] - configuration_set = os.environ["SES_CONFIGURATION_SET"] - s3_service = S3Service() - # contact_repo = ReportContactRepository(contact_table) - email_service = EmailService(default_configuration_set=configuration_set) - service = ReportDistributionService( - s3_service=s3_service, - contact_repo=contact_repo, - email_service=email_service, - bucket=bucket, - from_address=from_address, - prm_mailbox=prm_mailbox, - ) + service = ReportDistributionService(bucket=bucket) response = {"status": "ok", "bucket": bucket} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 0f33f2b1cb..7befb3a2b2 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -2,9 +2,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Dict, Tuple -from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.base.s3_service import S3Service -from services.reporting.excel_report_generator_service import ExcelReportGenerator from services.reporting.report_orchestration_service import ReportOrchestrationService from utils.audit_logging_setup import LoggingService from utils.decorators.ensure_env_var import ensure_environment_variables @@ -34,7 +32,6 @@ def build_s3_key(ods_code: str, report_date: str) -> str: return f"Report-Orchestration/{report_date}/{ods_code}.xlsx" - def upload_generated_reports( s3_service: S3Service, bucket: str, @@ -79,8 +76,7 @@ def lambda_handler(event, context) -> Dict[str, Any]: logger.info("Report orchestration lambda invoked") report_bucket = os.environ["REPORT_BUCKET_NAME"] - orchestration_service = ReportOrchestrationService( - ) + orchestration_service = ReportOrchestrationService() s3_service = S3Service() window_start, window_end = calculate_reporting_window() diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index f68b337e56..6b9ea7e705 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -1,3 +1,4 @@ +import os import re import secrets import tempfile @@ -26,7 +27,9 @@ def __init__( ): self.s3_service = S3Service() self.contact_repo = ReportContactRepository() - self.email_service = EmailService() + self.email_service = EmailService( + default_configuration_set=os.environ["SES_CONFIGURATION_SET"] + ) self.bucket = bucket self.from_address = os.environ["SES_FROM_ADDRESS"] self.prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index c367205f4b..bd2867031d 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -428,8 +428,9 @@ def mock_reporting_dynamo_service(mocker): @pytest.fixture -def reporting_repo(mock_reporting_dynamo_service): - return ReportingDynamoRepository(table_name="TestTable") +def reporting_repo(monkeypatch, mock_reporting_dynamo_service): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + return ReportingDynamoRepository() @pytest.fixture diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index 0300c16c5c..f6ed2de784 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -1,5 +1,3 @@ -import os -from unittest.mock import MagicMock import pytest from enums.feature_flags import FeatureFlags @@ -248,7 +246,6 @@ def required_report_orchestration_env(monkeypatch): monkeypatch.setenv("REPORT_BUCKET_NAME", "test-report-bucket") - @pytest.fixture def mock_reporting_dynamo_service(mocker): mock_cls = mocker.patch( @@ -258,8 +255,9 @@ def mock_reporting_dynamo_service(mocker): @pytest.fixture -def reporting_repo(mock_reporting_dynamo_service): - return ReportingDynamoRepository(table_name="TestTable") +def reporting_repo(monkeypatch, mock_reporting_dynamo_service): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + return ReportingDynamoRepository() @pytest.fixture diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index c16d02275c..14f552bf2f 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -1,13 +1,10 @@ import json import os - from handlers import report_distribution_handler as handler_module - def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( - handler_module, required_report_distribution_env, lambda_context, mock_report_distribution_wiring, @@ -100,20 +97,14 @@ def test_lambda_handler_returns_400_when_action_invalid( assert body["interaction_id"] == lambda_context.aws_request_id -def test_lambda_handler_returns_500_when_required_env_missing( - mocker, handler_module, lambda_context -): - mocker.patch.dict( - os.environ, - { - "CONTACT_TABLE_NAME": "contact-table", - "PRM_MAILBOX_EMAIL": "prm@example.com", - "SES_FROM_ADDRESS": "from@example.com", - }, - clear=False, - ) +def test_lambda_handler_returns_500_when_required_env_missing(mocker, lambda_context): + mocker.patch.dict(os.environ, {}, clear=False) + + os.environ["CONTACT_TABLE_NAME"] = "contact-table" + os.environ["PRM_MAILBOX_EMAIL"] = "prm@example.com" + os.environ["SES_FROM_ADDRESS"] = "from@example.com" + os.environ["SES_CONFIGURATION_SET"] = "my-config-set" os.environ.pop("REPORT_BUCKET_NAME", None) - os.environ.pop("SES_CONFIGURATION_SET", None) event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "p/"} @@ -124,9 +115,7 @@ def test_lambda_handler_returns_500_when_required_env_missing( body = json.loads(result["body"]) assert body["err_code"] == "ENV_5001" - assert ("REPORT_BUCKET_NAME" in body["message"]) or ( - "SES_CONFIGURATION_SET" in body["message"] - ) + assert "REPORT_BUCKET_NAME" in body["message"] if body.get("interaction_id") is not None: assert body["interaction_id"] == lambda_context.aws_request_id diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 3a8d078a83..3f4eb05632 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -20,9 +20,8 @@ def test_lambda_handler_calls_service_and_returns_expected_response( result = handler_module.lambda_handler(event={}, context=lambda_context) - mock_report_orchestration_wiring["build_services"].assert_called_once_with( - "TestTable" - ) + mock_window.assert_called_once() + mock_report_date.assert_called_once() orchestration_service.process_reporting_window.assert_called_once_with( window_start_ts=100, diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index 232a194b20..f62256bbc1 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -11,6 +11,7 @@ def required_report_distribution_env(monkeypatch): monkeypatch.setenv("SES_FROM_ADDRESS", "from@example.com") monkeypatch.setenv("PRM_MAILBOX_EMAIL", "prm@example.com") + monkeypatch.setenv("SES_CONFIGURATION_SET", "my-config-set") @pytest.fixture @@ -97,22 +98,24 @@ def test_extract_ods_code_from_key(key, expected): assert ReportDistributionService.extract_ods_code_from_key(key) == expected -def test_init_reads_env_and_wires_dependencies(required_report_distribution_env, mocker): +def test_init_reads_env_and_wires_dependencies( + required_report_distribution_env, mocker +): mock_s3 = mocker.Mock(name="S3ServiceInstance") mock_repo = mocker.Mock(name="ReportContactRepositoryInstance") mock_email = mocker.Mock(name="EmailServiceInstance") - mocker.patch( + mocked_s3_cls = mocker.patch( "services.reporting.report_distribution_service.S3Service", autospec=True, return_value=mock_s3, ) - mocker.patch( + mocked_repo_cls = mocker.patch( "services.reporting.report_distribution_service.ReportContactRepository", autospec=True, return_value=mock_repo, ) - mocker.patch( + mocked_email_cls = mocker.patch( "services.reporting.report_distribution_service.EmailService", autospec=True, return_value=mock_email, @@ -120,6 +123,10 @@ def test_init_reads_env_and_wires_dependencies(required_report_distribution_env, svc = ReportDistributionService(bucket="bucket-1") + mocked_s3_cls.assert_called_once_with() + mocked_repo_cls.assert_called_once_with() + mocked_email_cls.assert_called_once_with(default_configuration_set="my-config-set") + assert svc.bucket == "bucket-1" assert svc.from_address == "from@example.com" assert svc.prm_mailbox == "prm@example.com" @@ -242,9 +249,7 @@ def test_process_one_report_propagates_download_errors( def test_process_one_report_does_not_send_email_if_zip_fails( - service, - mocker, - patch_password, + service, mocker, patch_password ): mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", @@ -319,7 +324,6 @@ def test_send_report_emails_routes_correctly( mock_contact_repo.get_contact_email.assert_called_once_with("A99999") if expected_method == "email_contact": - mocked_email_contact.assert_called_once() assert mocked_email_contact.call_args.kwargs == { "to_address": "contact@example.com", "attachment_path": "/tmp/A99999.zip", @@ -328,7 +332,6 @@ def test_send_report_emails_routes_correctly( } mocked_email_prm.assert_not_called() else: - mocked_email_prm.assert_called_once() assert mocked_email_prm.call_args.kwargs == { "ods_code": "A99999", "attachment_path": "/tmp/A99999.zip", From 2d986197578d4026ecbefd961d087dd7d06579a5 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 6 Feb 2026 10:40:59 +0000 Subject: [PATCH 49/60] [PRMP-1058] fixed comment --- lambdas/repositories/reporting/reporting_dynamo_repository.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 8b4569dc73..fa1933cdfe 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -30,11 +30,10 @@ def get_records_for_time_window( f"table_name={self.table_name}, start_timestamp={start_timestamp}, end_timestamp={end_timestamp}", ) - start_date = utc_date(start_timestamp) + current_date = utc_date(start_timestamp) end_date = utc_date(end_timestamp) records_for_window: List[Dict] = [] - current_date = start_date while current_date <= end_date: day_start_ts = utc_day_start_timestamp(current_date) From 933ca592bc674316b80e5dc6f56c15c4eccbf90a Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 09:27:02 +0000 Subject: [PATCH 50/60] [PRMP-1058] updated email message --- lambdas/services/email_service.py | 97 ++++++++++++------- .../services/reporting/test_email_service.py | 4 +- 2 files changed, 64 insertions(+), 37 deletions(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index 5467d8812f..8671ba9b99 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -20,15 +20,15 @@ def __init__(self, *, default_configuration_set: Optional[str] = None): self.default_configuration_set = default_configuration_set def send_email( - self, - *, - to_address: str, - subject: str, - body_text: str, - from_address: str, - attachments: Optional[Iterable[str]] = None, - configuration_set: Optional[str] = None, - tags: Optional[Dict[str, str]] = None, + self, + *, + to_address: str, + subject: str, + body_text: str, + from_address: str, + attachments: Optional[Iterable[str]] = None, + configuration_set: Optional[str] = None, + tags: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: """ Sends an email using SES SendRawEmail. @@ -68,12 +68,12 @@ def send_email( ) def _send_raw( - self, - *, - msg: MIMEMultipart, - to_address: str, - configuration_set: Optional[str] = None, - tags: Optional[Dict[str, str]] = None, + self, + *, + msg: MIMEMultipart, + to_address: str, + configuration_set: Optional[str] = None, + tags: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: subject = msg.get("Subject", "") from_address = msg.get("From", "") @@ -103,29 +103,54 @@ def _send_raw( return resp def send_report_email( - self, - *, - to_address: str, - from_address: str, - attachment_path: str, - tags: Optional[Dict[str, str]] = None, + self, + *, + to_address: str, + from_address: str, + attachment_path: str, + tags: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: + practice_name = tags.get("ods_code", "") + body_text = (f"Dear {practice_name}, \n" + f"We are pleased to share that the transfer of your patient records " + f"to the National Document Repository has been successful.\n" + f"Please find attached an encrypted summary of the transfer \n" + f" and a report for successes and rejections.\n\n" + f"Important update – new functionality coming in March " + f"From March, you will have the ability to review rejected records directly on the NDR" + f" and choose whether to accept them into the access and store service.\n" + f"Our default position is that we will hold the rejected records " + f"until this functionality is available, so you can review and act on them online." + f"If you would prefer not to wait and for us to return the rejected records to you now, " + f"please let us know and we can arrange to do this via egress.\n\n" + f"In the meantime, the attached report is for information only if you wish to use " + f"the new review portal from march.\n\n" + f"Guidance can be sent should you wish to review the rejected records manually before " + f"the new functionality is released. In summary, this involves:\n" + f"Checking NHS numbers and patient demographic details are correct." + f"Confirming the correct number of files were sent.Ensuring file names follow " + f"the required naming standard (see guidance attached).\n\n " + f"Once the files have been successfully reviewed and renamed, they can be resent for upload.\n\n" + f"If you have any questions, please don’t hesitate to contact us at england.prm@nhs.net.\n\n" + f"Kind regards,\n" + f"PRM team") + return self.send_email( to_address=to_address, from_address=from_address, subject="Daily Upload Report", - body_text="Please find your encrypted daily upload report attached.", + body_text=body_text, attachments=[attachment_path], tags=tags, ) def send_password_email( - self, - *, - to_address: str, - from_address: str, - password: str, - tags: Optional[Dict[str, str]] = None, + self, + *, + to_address: str, + from_address: str, + password: str, + tags: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: return self.send_email( to_address=to_address, @@ -136,14 +161,14 @@ def send_password_email( ) def send_prm_missing_contact_email( - self, - *, - prm_mailbox: str, - from_address: str, - ods_code: str, - attachment_path: str, - password: str, - tags: Optional[Dict[str, str]] = None, + self, + *, + prm_mailbox: str, + from_address: str, + ods_code: str, + attachment_path: str, + password: str, + tags: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: return self.send_email( to_address=prm_mailbox, diff --git a/lambdas/tests/unit/services/reporting/test_email_service.py b/lambdas/tests/unit/services/reporting/test_email_service.py index 934265c23d..b7574c22a3 100644 --- a/lambdas/tests/unit/services/reporting/test_email_service.py +++ b/lambdas/tests/unit/services/reporting/test_email_service.py @@ -1,3 +1,5 @@ +from unittest.mock import ANY + import pytest from services.email_service import EmailService @@ -192,7 +194,7 @@ def test_send_report_email_calls_send_email_with_expected_inputs(email_service, to_address="to@example.com", from_address="from@example.com", subject="Daily Upload Report", - body_text="Please find your encrypted daily upload report attached.", + body_text=ANY, attachments=["/tmp/report.zip"], tags={"k": "v"}, ) From b77f61ea110a2b56dce7496131a87708ec399882 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 10:55:49 +0000 Subject: [PATCH 51/60] [PRMP-1058] updated email message --- lambdas/services/email_service.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index 8671ba9b99..ff77c3e5f1 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -41,6 +41,7 @@ def send_email( msg["From"] = from_address msg.attach(MIMEText(body_text, "plain")) + msg.attach(MIMEText(body_text, "html")) attachment_list = list(attachments or []) for attachment_path in attachment_list: @@ -116,7 +117,7 @@ def send_report_email( f"to the National Document Repository has been successful.\n" f"Please find attached an encrypted summary of the transfer \n" f" and a report for successes and rejections.\n\n" - f"Important update – new functionality coming in March " + f"Important update – new functionality coming in March\n" f"From March, you will have the ability to review rejected records directly on the NDR" f" and choose whether to accept them into the access and store service.\n" f"Our default position is that we will hold the rejected records " @@ -124,12 +125,12 @@ def send_report_email( f"If you would prefer not to wait and for us to return the rejected records to you now, " f"please let us know and we can arrange to do this via egress.\n\n" f"In the meantime, the attached report is for information only if you wish to use " - f"the new review portal from march.\n\n" + f"the new review portal from March.\n\n" f"Guidance can be sent should you wish to review the rejected records manually before " - f"the new functionality is released. In summary, this involves:\n" - f"Checking NHS numbers and patient demographic details are correct." + f"the new functionality is released. \nIn summary, this involves:\n" + f"Checking NHS numbers and patient demographic details are correct.\n" f"Confirming the correct number of files were sent.Ensuring file names follow " - f"the required naming standard (see guidance attached).\n\n " + f"the required naming standard (see guidance attached).\n\n" f"Once the files have been successfully reviewed and renamed, they can be resent for upload.\n\n" f"If you have any questions, please don’t hesitate to contact us at england.prm@nhs.net.\n\n" f"Kind regards,\n" @@ -152,6 +153,16 @@ def send_password_email( password: str, tags: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: + practice_name = tags.get("ods_code", "") + body_text = (f"Dear {practice_name},\n" + f"You have been issued a temporary password to access your reports.\n\n" + f"Temporary Password:\n" + f"{password}\n\n" + f"For security reasons, " + f"please log in as soon as possible and change this password immediately.\n" + f"If you did not request access, or if you experience any issues logging in, " + f"please contact\n" + f"england.prm@nhs.net.") return self.send_email( to_address=to_address, from_address=from_address, From 365c758e4b12fefcdb9a186b0f605b0fb9a2f12a Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 11:22:45 +0000 Subject: [PATCH 52/60] Update lambdas/services/email_service.py Co-authored-by: Mohammad Iqbal <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> --- lambdas/services/email_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index ff77c3e5f1..bd75760ecb 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -115,7 +115,7 @@ def send_report_email( body_text = (f"Dear {practice_name}, \n" f"We are pleased to share that the transfer of your patient records " f"to the National Document Repository has been successful.\n" - f"Please find attached an encrypted summary of the transfer \n" + f"Please find attached an encrypted summary of the transfer" f" and a report for successes and rejections.\n\n" f"Important update – new functionality coming in March\n" f"From March, you will have the ability to review rejected records directly on the NDR" From 37ed6d05a17685071fb158a542e31e700a2df62d Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 11:23:22 +0000 Subject: [PATCH 53/60] Update lambdas/services/email_service.py Co-authored-by: Mohammad Iqbal <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> --- lambdas/services/email_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index bd75760ecb..f668e2abf5 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -119,7 +119,7 @@ def send_report_email( f" and a report for successes and rejections.\n\n" f"Important update – new functionality coming in March\n" f"From March, you will have the ability to review rejected records directly on the NDR" - f" and choose whether to accept them into the access and store service.\n" + f" and choose whether to accept them into the Access and Store service.\n" f"Our default position is that we will hold the rejected records " f"until this functionality is available, so you can review and act on them online." f"If you would prefer not to wait and for us to return the rejected records to you now, " From 896a7c9d8b24957758f5d7e10fa65e7a33afcbee Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 11:24:06 +0000 Subject: [PATCH 54/60] Update lambdas/services/email_service.py Co-authored-by: Mohammad Iqbal <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> --- lambdas/services/email_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index f668e2abf5..03f17949f4 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -129,7 +129,7 @@ def send_report_email( f"Guidance can be sent should you wish to review the rejected records manually before " f"the new functionality is released. \nIn summary, this involves:\n" f"Checking NHS numbers and patient demographic details are correct.\n" - f"Confirming the correct number of files were sent.Ensuring file names follow " + f"Confirming the correct number of files were sent.\nEnsuring file names follow " f"the required naming standard (see guidance attached).\n\n" f"Once the files have been successfully reviewed and renamed, they can be resent for upload.\n\n" f"If you have any questions, please don’t hesitate to contact us at england.prm@nhs.net.\n\n" From 246c238cdcc6c362ecd837ff3dfbb3484bad45af Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 11:24:22 +0000 Subject: [PATCH 55/60] Update lambdas/services/email_service.py Co-authored-by: Mohammad Iqbal <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> --- lambdas/services/email_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index 03f17949f4..bb87b15db0 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -131,7 +131,7 @@ def send_report_email( f"Checking NHS numbers and patient demographic details are correct.\n" f"Confirming the correct number of files were sent.\nEnsuring file names follow " f"the required naming standard (see guidance attached).\n\n" - f"Once the files have been successfully reviewed and renamed, they can be resent for upload.\n\n" + f"Once the files have been successfully reviewed and renamed, they can be re-sent for upload.\n\n" f"If you have any questions, please don’t hesitate to contact us at england.prm@nhs.net.\n\n" f"Kind regards,\n" f"PRM team") From fc573cb0188f4ac81566da0ced40efc42931418c Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 11:24:50 +0000 Subject: [PATCH 56/60] Update lambdas/services/email_service.py Co-authored-by: Mohammad Iqbal <127403145+MohammadIqbalAD-NHS@users.noreply.github.com> --- lambdas/services/email_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index bb87b15db0..d6b64ad2f3 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -121,7 +121,7 @@ def send_report_email( f"From March, you will have the ability to review rejected records directly on the NDR" f" and choose whether to accept them into the Access and Store service.\n" f"Our default position is that we will hold the rejected records " - f"until this functionality is available, so you can review and act on them online." + f"until this functionality is available, so you can review and act on them online.\n" f"If you would prefer not to wait and for us to return the rejected records to you now, " f"please let us know and we can arrange to do this via egress.\n\n" f"In the meantime, the attached report is for information only if you wish to use " From 1a5394cda489621a785790aa52d77e08b3528846 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 11:26:36 +0000 Subject: [PATCH 57/60] [PRMP-1058] updated email message --- lambdas/services/email_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index ff77c3e5f1..8dd3a04287 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -129,7 +129,7 @@ def send_report_email( f"Guidance can be sent should you wish to review the rejected records manually before " f"the new functionality is released. \nIn summary, this involves:\n" f"Checking NHS numbers and patient demographic details are correct.\n" - f"Confirming the correct number of files were sent.Ensuring file names follow " + f"Confirming the correct number of files were sent.\nEnsuring file names follow " f"the required naming standard (see guidance attached).\n\n" f"Once the files have been successfully reviewed and renamed, they can be resent for upload.\n\n" f"If you have any questions, please don’t hesitate to contact us at england.prm@nhs.net.\n\n" @@ -167,7 +167,7 @@ def send_password_email( to_address=to_address, from_address=from_address, subject="Daily Upload Report Password", - body_text=f"Password for your report:\n\n{password}", + body_text=body_text, tags=tags, ) From dbbc9f48cae2835589fc4e283d80c2ff45f95a80 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 11:34:44 +0000 Subject: [PATCH 58/60] [PRMP-1058] updated email subject and body --- lambdas/services/email_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index 4e969397a6..cab1329b7d 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -139,7 +139,7 @@ def send_report_email( return self.send_email( to_address=to_address, from_address=from_address, - subject="Daily Upload Report", + subject=f"{practice_name} - National document repository bulk upload reports", body_text=body_text, attachments=[attachment_path], tags=tags, @@ -166,7 +166,7 @@ def send_password_email( return self.send_email( to_address=to_address, from_address=from_address, - subject="Daily Upload Report Password", + subject=f"{practice_name} - National document repository bulk upload reports password", body_text=body_text, tags=tags, ) From 38d30aa0ee87993888a15a584682c8689390853b Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 10 Feb 2026 15:25:25 +0000 Subject: [PATCH 59/60] [PRMP-939] updated NDR to National document repository --- lambdas/services/email_service.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index cab1329b7d..b9c28b3475 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -111,14 +111,15 @@ def send_report_email( attachment_path: str, tags: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: - practice_name = tags.get("ods_code", "") - body_text = (f"Dear {practice_name}, \n" + practice_ods = tags.get("ods_code", "") + body_text = (f"Dear {practice_ods}, \n" f"We are pleased to share that the transfer of your patient records " f"to the National Document Repository has been successful.\n" f"Please find attached an encrypted summary of the transfer" f" and a report for successes and rejections.\n\n" f"Important update – new functionality coming in March\n" - f"From March, you will have the ability to review rejected records directly on the NDR" + f"From March, you will have the ability to review rejected records directly on " + f"the National Document Repository" f" and choose whether to accept them into the Access and Store service.\n" f"Our default position is that we will hold the rejected records " f"until this functionality is available, so you can review and act on them online.\n" @@ -134,12 +135,12 @@ def send_report_email( f"Once the files have been successfully reviewed and renamed, they can be re-sent for upload.\n\n" f"If you have any questions, please don’t hesitate to contact us at england.prm@nhs.net.\n\n" f"Kind regards,\n" - f"PRM team") + f"Patient Record Management Team") return self.send_email( to_address=to_address, from_address=from_address, - subject=f"{practice_name} - National document repository bulk upload reports", + subject=f"{practice_ods} - National Document Repository bulk upload reports", body_text=body_text, attachments=[attachment_path], tags=tags, @@ -153,8 +154,8 @@ def send_password_email( password: str, tags: Optional[Dict[str, str]] = None, ) -> Dict[str, Any]: - practice_name = tags.get("ods_code", "") - body_text = (f"Dear {practice_name},\n" + practice_ods = tags.get("ods_code", "") + body_text = (f"Dear {practice_ods},\n" f"You have been issued a temporary password to access your reports.\n\n" f"Temporary Password:\n" f"{password}\n\n" @@ -166,7 +167,7 @@ def send_password_email( return self.send_email( to_address=to_address, from_address=from_address, - subject=f"{practice_name} - National document repository bulk upload reports password", + subject=f"{practice_ods} - National Document Repository bulk upload reports password", body_text=body_text, tags=tags, ) From 01f4e02ce931f389d1333dd52e7468b4d5dc1bf9 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 11 Feb 2026 11:36:05 +0000 Subject: [PATCH 60/60] [PRMP-1058] remove unused line --- lambdas/services/email_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index b9c28b3475..032b885349 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -41,7 +41,6 @@ def send_email( msg["From"] = from_address msg.attach(MIMEText(body_text, "plain")) - msg.attach(MIMEText(body_text, "html")) attachment_list = list(attachments or []) for attachment_path in attachment_list: