From e61d3bd7a8f5ca3a7f15176a206b2c9443ffb0d4 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 17 Dec 2025 13:36:25 +0000 Subject: [PATCH 01/37] [PRMP-1054] Create report_orchestration --- .../base-lambdas-reusable-deploy-all.yml | 14 ++++++ .../handlers/report_orchestration_handler.py | 40 +++++++++++++++ .../reporting/reporting_dynamo_repository.py | 32 ++++++++++++ .../reporting/excel_report_generator.py | 46 +++++++++++++++++ .../reporting/report_orchestration_service.py | 49 +++++++++++++++++++ 5 files changed, 181 insertions(+) create mode 100644 lambdas/handlers/report_orchestration_handler.py create mode 100644 lambdas/repositories/reporting/reporting_dynamo_repository.py create mode 100644 lambdas/services/reporting/excel_report_generator.py create mode 100644 lambdas/services/reporting/report_orchestration_service.py diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index fe1faa70cb..1b9edf7299 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -766,3 +766,17 @@ jobs: lambda_layer_names: "core_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} + + deploy_report_orchestration_lambda: + name: Deploy Search Document Review + uses: ./.github/workflows/base-lambdas-reusable-deploy.yml + with: + environment: ${{ inputs.environment }} + python_version: ${{ inputs.python_version }} + build_branch: ${{ inputs.build_branch }} + sandbox: ${{ inputs.sandbox }} + lambda_handler_name: report_orchestration_handler + lambda_aws_name: reportOrchestration + lambda_layer_names: "core_lambda_layer" + secrets: + AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py new file mode 100644 index 0000000000..bf3743a218 --- /dev/null +++ b/lambdas/handlers/report_orchestration_handler.py @@ -0,0 +1,40 @@ +import os +from datetime import datetime, timedelta, timezone + +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from services.reporting.excel_report_generator import ExcelReportGenerator +from services.reporting.report_orchestration_service import ReportOrchestrationService + + +def _calculate_reporting_window(): + now = datetime.now(timezone.utc) + today_7am = now.replace(hour=7, minute=0, second=0, microsecond=0) + + if now < today_7am: + today_7am -= timedelta(days=1) + + yesterday_7am = today_7am - timedelta(days=1) + + return ( + int(yesterday_7am.timestamp()), + int(today_7am.timestamp()), + ) + + +def lambda_handler(event, context): + table_name = os.getenv("REPORTING_DYNAMODB_TABLE") + + repository = ReportingDynamoRepository(table_name) + excel_generator = ExcelReportGenerator() + + service = ReportOrchestrationService( + repository=repository, + excel_generator=excel_generator, + ) + + window_start, window_end = _calculate_reporting_window() + + service.process_reporting_window( + window_start_ts=window_start, + window_end_ts=window_end, + ) diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py new file mode 100644 index 0000000000..90424dd0bf --- /dev/null +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -0,0 +1,32 @@ +from datetime import datetime, timezone +from services.base.dynamo_service import DynamoDBService + +class ReportingDynamoRepository: + def __init__(self, table_name: str): + self.table_name = table_name + self.dynamo_service = DynamoDBService() + + def get_records_for_time_window( + self, + start_timestamp: int, + end_timestamp: int, + ) -> list[dict]: + filter_expression = ( + "#ts BETWEEN :start AND :end" + ) + + expression_attribute_names = { + "#ts": "Timestamp" + } + + expression_attribute_values = { + ":start": start_timestamp, + ":end": end_timestamp, + } + + return self.dynamo_service.scan_whole_table( + table_name=self.table_name, + filter_expression=filter_expression, + expression_attribute_names=expression_attribute_names, + expression_attribute_values=expression_attribute_values, + ) diff --git a/lambdas/services/reporting/excel_report_generator.py b/lambdas/services/reporting/excel_report_generator.py new file mode 100644 index 0000000000..7dd3346d68 --- /dev/null +++ b/lambdas/services/reporting/excel_report_generator.py @@ -0,0 +1,46 @@ +from openpyxl import Workbook +from datetime import datetime + + +class ExcelReportGenerator: + def create_report_orchestration_xlsx( + self, + ods_code: str, + records: list[dict], + output_path: str, + ) -> str: + wb = Workbook() + ws = wb.active + ws.title = "Daily Upload Report" + + # Report metadata + ws.append([f"ODS Code: {ods_code}"]) + ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) + ws.append([]) + + # Header row + ws.append([ + "ID", + "Date", + "NHS Number", + "Uploader ODS", + "PDS ODS", + "Upload Status", + "Reason", + "File Path", + ]) + + for record in records: + ws.append([ + record.get("ID"), + record.get("Date"), + record.get("NhsNumber"), + record.get("UploaderOdsCode"), + record.get("PdsOdsCode"), + record.get("UploadStatus"), + record.get("Reason"), + record.get("FilePath"), + ]) + + wb.save(output_path) + return output_path diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py new file mode 100644 index 0000000000..4a19e13247 --- /dev/null +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -0,0 +1,49 @@ +import tempfile +from collections import defaultdict + +class ReportOrchestrationService: + def __init__( + self, + repository, + excel_generator, + ): + self.repository = repository + self.excel_generator = excel_generator + + def process_reporting_window( + self, + window_start_ts: int, + window_end_ts: int, + ): + records = self.repository.get_records_for_time_window( + window_start_ts, + window_end_ts, + ) + + records_by_ods = self.group_records_by_ods(records) + + for ods_code, ods_records in records_by_ods.items(): + self.generate_ods_report(ods_code, ods_records) + + @staticmethod + def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: + grouped = defaultdict(list) + for record in records: + ods_code = record.get("UploaderOdsCode") or "UNKNOWN" + grouped[ods_code].append(record) + return grouped + + def generate_ods_report( + self, + ods_code: str, + records: list[dict], + ): + with tempfile.NamedTemporaryFile( + suffix=f"_{ods_code}.xlsx", + delete=False, + ) as tmp: + self.excel_generator.create_report_orchestration_xlsx( + ods_code=ods_code, + records=records, + output_path=tmp.name, + ) From 3d9a2915cce25894984adf5de12a61813e9ac6bd Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 17 Dec 2025 13:42:35 +0000 Subject: [PATCH 02/37] [PRMP-1054] Updated variable name --- lambdas/handlers/report_orchestration_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index bf3743a218..51431deca3 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -22,7 +22,7 @@ def _calculate_reporting_window(): def lambda_handler(event, context): - table_name = os.getenv("REPORTING_DYNAMODB_TABLE") + table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") repository = ReportingDynamoRepository(table_name) excel_generator = ExcelReportGenerator() From 528efe0422347326bc9fe8b5dfac0d1d44398509 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 17 Dec 2025 14:24:37 +0000 Subject: [PATCH 03/37] [PRMP-1054] Updated method name --- lambdas/handlers/report_orchestration_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 51431deca3..c01e82f4b5 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -6,7 +6,7 @@ from services.reporting.report_orchestration_service import ReportOrchestrationService -def _calculate_reporting_window(): +def calculate_reporting_window(): now = datetime.now(timezone.utc) today_7am = now.replace(hour=7, minute=0, second=0, microsecond=0) @@ -32,7 +32,7 @@ def lambda_handler(event, context): excel_generator=excel_generator, ) - window_start, window_end = _calculate_reporting_window() + window_start, window_end = calculate_reporting_window() service.process_reporting_window( window_start_ts=window_start, From 7c5548103a2caae7b1c1d4038d530be57e6d0e2e Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 17 Dec 2025 15:44:03 +0000 Subject: [PATCH 04/37] [PRMP-1054] Created tests and formated code --- .../handlers/report_orchestration_handler.py | 6 +- .../reporting/reporting_dynamo_repository.py | 20 ++- .../reporting/excel_report_generator.py | 46 ------ .../excel_report_generator_service.py | 58 +++++++ .../reporting/report_orchestration_service.py | 12 ++ .../test_report_orchestration_handler.py | 66 ++++++++ .../test_reporting_dynamo_repository.py | 53 ++++++ .../test_excel_report_generator_service.py | 154 ++++++++++++++++++ .../test_report_orchestration_service.py | 107 ++++++++++++ 9 files changed, 470 insertions(+), 52 deletions(-) delete mode 100644 lambdas/services/reporting/excel_report_generator.py create mode 100644 lambdas/services/reporting/excel_report_generator_service.py create mode 100644 lambdas/tests/unit/handlers/test_report_orchestration_handler.py create mode 100644 lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py create mode 100644 lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py create mode 100644 lambdas/tests/unit/services/reporting/test_report_orchestration_service.py diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index c01e82f4b5..e8e16d5e52 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -2,8 +2,11 @@ from datetime import datetime, timedelta, timezone from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository -from services.reporting.excel_report_generator import ExcelReportGenerator +from services.reporting.excel_report_generator_service import ExcelReportGenerator from services.reporting.report_orchestration_service import ReportOrchestrationService +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) def calculate_reporting_window(): @@ -22,6 +25,7 @@ def calculate_reporting_window(): def lambda_handler(event, context): + logger.info("Report orchestration lambda invoked") table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") repository = ReportingDynamoRepository(table_name) diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 90424dd0bf..1233a0a4e1 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,5 +1,10 @@ -from datetime import datetime, timezone +from typing import Dict, List + from services.base.dynamo_service import DynamoDBService +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + class ReportingDynamoRepository: def __init__(self, table_name: str): @@ -10,13 +15,18 @@ def get_records_for_time_window( self, start_timestamp: int, end_timestamp: int, - ) -> list[dict]: - filter_expression = ( - "#ts BETWEEN :start AND :end" + ) -> List[Dict]: + logger.info( + f"Querying reporting table for window, " + f"table_name: {self.table_name}," + f"start_timestamp: {start_timestamp}," + f"end_timestamp: {end_timestamp}", ) + filter_expression = "#ts BETWEEN :start AND :end" + expression_attribute_names = { - "#ts": "Timestamp" + "#ts": "Timestamp", } expression_attribute_values = { diff --git a/lambdas/services/reporting/excel_report_generator.py b/lambdas/services/reporting/excel_report_generator.py deleted file mode 100644 index 7dd3346d68..0000000000 --- a/lambdas/services/reporting/excel_report_generator.py +++ /dev/null @@ -1,46 +0,0 @@ -from openpyxl import Workbook -from datetime import datetime - - -class ExcelReportGenerator: - def create_report_orchestration_xlsx( - self, - ods_code: str, - records: list[dict], - output_path: str, - ) -> str: - wb = Workbook() - ws = wb.active - ws.title = "Daily Upload Report" - - # Report metadata - ws.append([f"ODS Code: {ods_code}"]) - ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) - ws.append([]) - - # Header row - ws.append([ - "ID", - "Date", - "NHS Number", - "Uploader ODS", - "PDS ODS", - "Upload Status", - "Reason", - "File Path", - ]) - - for record in records: - ws.append([ - record.get("ID"), - record.get("Date"), - record.get("NhsNumber"), - record.get("UploaderOdsCode"), - record.get("PdsOdsCode"), - record.get("UploadStatus"), - record.get("Reason"), - record.get("FilePath"), - ]) - - wb.save(output_path) - return output_path diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py new file mode 100644 index 0000000000..36c6c09466 --- /dev/null +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -0,0 +1,58 @@ +from datetime import datetime + +from openpyxl import Workbook +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + + +class ExcelReportGenerator: + def create_report_orchestration_xlsx( + self, + ods_code: str, + records: list[dict], + output_path: str, + ) -> str: + logger.info( + f"Creating Excel report for ods code {ods_code} and records {records}" + ) + wb = Workbook() + ws = wb.active + ws.title = "Daily Upload Report" + + # Report metadata + ws.append([f"ODS Code: {ods_code}"]) + ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) + ws.append([]) + + # Header row + ws.append( + [ + "ID", + "Date", + "NHS Number", + "Uploader ODS", + "PDS ODS", + "Upload Status", + "Reason", + "File Path", + ] + ) + + for record in records: + ws.append( + [ + record.get("ID"), + record.get("Date"), + record.get("NhsNumber"), + record.get("UploaderOdsCode"), + record.get("PdsOdsCode"), + record.get("UploadStatus"), + record.get("Reason"), + record.get("FilePath"), + ] + ) + + wb.save(output_path) + logger.info(f"Excel report written successfully for for ods code {ods_code}") + return output_path diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index 4a19e13247..e6528d1bcb 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -1,6 +1,11 @@ import tempfile from collections import defaultdict +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + + class ReportOrchestrationService: def __init__( self, @@ -19,11 +24,18 @@ def process_reporting_window( window_start_ts, window_end_ts, ) + if not records: + logger.info("No records found for reporting window") + return records_by_ods = self.group_records_by_ods(records) for ods_code, ods_records in records_by_ods.items(): + logger.info( + f"Generating report for ODS ods_code = {ods_code} record_count = len(ods_records)" + ) self.generate_ods_report(ods_code, ods_records) + logger.info("Report orchestration completed") @staticmethod def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py new file mode 100644 index 0000000000..59fe6b3c35 --- /dev/null +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -0,0 +1,66 @@ +from unittest.mock import MagicMock + +import pytest +from handlers.report_orchestration_handler import lambda_handler + + +@pytest.fixture(autouse=True) +def mock_env(monkeypatch): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + + +@pytest.fixture +def mock_logger(mocker): + return mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) + + +@pytest.fixture +def mock_repo(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.ReportingDynamoRepository", autospec=True + ) + + +@pytest.fixture +def mock_excel_generator(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.ExcelReportGenerator", autospec=True + ) + + +@pytest.fixture +def mock_service(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.ReportOrchestrationService", + autospec=True, + ) + + +@pytest.fixture +def mock_window(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.calculate_reporting_window", + return_value=(100, 200), + ) + + +def test_lambda_handler_calls_service( + mock_logger, mock_repo, mock_excel_generator, mock_service, mock_window +): + lambda_handler(event={}, context={}) + + mock_repo.assert_called_once_with("TestTable") + mock_excel_generator.assert_called_once_with() + + mock_service.assert_called_once() + instance = mock_service.return_value + instance.process_reporting_window.assert_called_once_with( + window_start_ts=100, window_end_ts=200 + ) + + mock_logger.info.assert_called_with("Report orchestration lambda invoked") + + +def test_lambda_handler_calls_window_function(mock_service, mock_window): + lambda_handler(event={}, context={}) + mock_window.assert_called_once() diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py new file mode 100644 index 0000000000..e88d98a145 --- /dev/null +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -0,0 +1,53 @@ +from unittest.mock import MagicMock + +import pytest +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository + + +@pytest.fixture +def mock_dynamo_service(mocker): + mock_service = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.DynamoDBService" + ) + instance = mock_service.return_value + instance.scan_whole_table = MagicMock() + return instance + + +@pytest.fixture +def reporting_repo(mock_dynamo_service): + return ReportingDynamoRepository(table_name="TestTable") + + +def test_get_records_for_time_window_calls_scan(mock_dynamo_service, reporting_repo): + start_ts = 100 + end_ts = 200 + expected_result = [ + {"ID": 1, "Timestamp": 150}, + {"ID": 2, "Timestamp": 180}, + ] + mock_dynamo_service.scan_whole_table.return_value = expected_result + + result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + + mock_dynamo_service.scan_whole_table.assert_called_once_with( + table_name="TestTable", + filter_expression="#ts BETWEEN :start AND :end", + expression_attribute_names={"#ts": "Timestamp"}, + expression_attribute_values={":start": start_ts, ":end": end_ts}, + ) + + assert result == expected_result + + +def test_get_records_for_time_window_returns_empty_list( + mock_dynamo_service, reporting_repo +): + start_ts = 0 + end_ts = 50 + mock_dynamo_service.scan_whole_table.return_value = [] + + result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + + assert result == [] + mock_dynamo_service.scan_whole_table.assert_called_once() diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py new file mode 100644 index 0000000000..f16257c0fa --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -0,0 +1,154 @@ + +import pytest +from freezegun import freeze_time +from openpyxl import load_workbook +from services.reporting.excel_report_generator_service import ExcelReportGenerator + + +@pytest.fixture +def excel_report_generator(): + return ExcelReportGenerator() + + +@freeze_time("2025-01-01T12:00:00") +def test_create_report_orchestration_xlsx_happy_path( + excel_report_generator, + tmp_path, +): + output_file = tmp_path / "report.xlsx" + + ods_code = "Y12345" + records = [ + { + "ID": 1, + "Date": "2025-01-01", + "NhsNumber": "1234567890", + "UploaderOdsCode": "Y12345", + "PdsOdsCode": "A99999", + "UploadStatus": "SUCCESS", + "Reason": "", + "FilePath": "/path/file1.pdf", + }, + { + "ID": 2, + "Date": "2025-01-02", + "NhsNumber": "123456789", + "UploaderOdsCode": "Y12345", + "PdsOdsCode": "B88888", + "UploadStatus": "FAILED", + "Reason": "Invalid NHS number", + "FilePath": "/path/file2.pdf", + }, + ] + + result = excel_report_generator.create_report_orchestration_xlsx( + ods_code=ods_code, + records=records, + output_path=str(output_file), + ) + + # File path returned + assert result == str(output_file) + assert output_file.exists() + + wb = load_workbook(output_file) + ws = wb.active + + # Sheet name + assert ws.title == "Daily Upload Report" + + # Metadata rows + assert ws["A1"].value == f"ODS Code: {ods_code}" + assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" + assert ws["A3"].value is None # blank row + + # Header row + assert [cell.value for cell in ws[4]] == [ + "ID", + "Date", + "NHS Number", + "Uploader ODS", + "PDS ODS", + "Upload Status", + "Reason", + "File Path", + ] + + # First data row + assert [cell.value for cell in ws[5]] == [ + 1, + "2025-01-01", + "1234567890", + "Y12345", + "A99999", + "SUCCESS", + None, + "/path/file1.pdf", + ] + + # Second data row + assert [cell.value for cell in ws[6]] == [ + 2, + "2025-01-02", + "123456789", + "Y12345", + "B88888", + "FAILED", + "Invalid NHS number", + "/path/file2.pdf", + ] + + +def test_create_report_orchestration_xlsx_with_no_records( + excel_report_generator, + tmp_path, +): + output_file = tmp_path / "empty_report.xlsx" + + excel_report_generator.create_report_orchestration_xlsx( + ods_code="Y12345", + records=[], + output_path=str(output_file), + ) + + wb = load_workbook(output_file) + ws = wb.active + + # Only metadata + header rows should exist + assert ws.max_row == 4 + + +def test_create_report_orchestration_xlsx_handles_missing_fields( + excel_report_generator, + tmp_path, +): + output_file = tmp_path / "partial.xlsx" + + records = [ + { + "ID": 1, + "NhsNumber": "1234567890", + } + ] + + excel_report_generator.create_report_orchestration_xlsx( + ods_code="Y12345", + records=records, + output_path=str(output_file), + ) + + wb = load_workbook(output_file) + ws = wb.active + + row = [cell.value for cell in ws[5]] + + assert row == [ + 1, + None, + "1234567890", + None, + None, + None, + None, + None, + ] diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py new file mode 100644 index 0000000000..c3b21a77ef --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -0,0 +1,107 @@ +import pytest +from services.reporting.report_orchestration_service import ReportOrchestrationService + + +@pytest.fixture +def mock_repository(mocker): + repo = mocker.Mock() + repo.get_records_for_time_window.return_value = [] + return repo + + +@pytest.fixture +def mock_excel_generator(mocker): + return mocker.Mock() + + +@pytest.fixture +def report_orchestration_service(mock_repository, mock_excel_generator): + return ReportOrchestrationService( + repository=mock_repository, + excel_generator=mock_excel_generator, + ) + + +def test_process_reporting_window_no_records( + report_orchestration_service, mock_repository, mock_excel_generator +): + mock_repository.get_records_for_time_window.return_value = [] + + report_orchestration_service.process_reporting_window(100, 200) + + mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() + + +def test_group_records_by_ods_groups_correctly(): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + {"ID": 4}, # missing ODS + {"UploaderOdsCode": None, "ID": 5}, # null ODS + ] + + result = ReportOrchestrationService.group_records_by_ods(records) + + assert result["Y12345"] == [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ] + assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] + assert result["UNKNOWN"] == [ + {"ID": 4}, + {"UploaderOdsCode": None, "ID": 5}, + ] + + +def test_process_reporting_window_generates_reports_per_ods( + report_orchestration_service, mock_repository, mocker +): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + ] + mock_repository.get_records_for_time_window.return_value = records + + mocked_generate = mocker.patch.object( + report_orchestration_service, "generate_ods_report" + ) + + report_orchestration_service.process_reporting_window(100, 200) + + mocked_generate.assert_any_call( + "Y12345", + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ], + ) + mocked_generate.assert_any_call( + "A99999", + [{"UploaderOdsCode": "A99999", "ID": 3}], + ) + assert mocked_generate.call_count == 2 + + +def test_generate_ods_report_creates_excel_report( + report_orchestration_service, mock_excel_generator, mocker +): + fake_tmp = mocker.MagicMock() + fake_tmp.__enter__.return_value = fake_tmp + fake_tmp.name = "/tmp/fake_Y12345.xlsx" + + mocker.patch( + "services.reporting.report_orchestration_service.tempfile.NamedTemporaryFile", + return_value=fake_tmp, + ) + + records = [{"ID": 1, "UploaderOdsCode": "Y12345"}] + + report_orchestration_service.generate_ods_report("Y12345", records) + + mock_excel_generator.create_report_orchestration_xlsx.assert_called_once_with( + ods_code="Y12345", + records=records, + output_path=fake_tmp.name, + ) From dfbff8da57eb321e20bf39f5d70ed704e871f70a Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 18 Dec 2025 11:17:00 +0000 Subject: [PATCH 05/37] [PRMP-1054] Small updates to allow library to be imported and used --- lambdas/handlers/report_orchestration_handler.py | 14 +++++++++++++- .../reporting/excel_report_generator_service.py | 4 ++-- .../reporting/report_orchestration_service.py | 1 + 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index e8e16d5e52..84f6372ab8 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,10 +1,15 @@ import os +import tempfile from datetime import datetime, timedelta, timezone from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.reporting.excel_report_generator_service import ExcelReportGenerator from services.reporting.report_orchestration_service import ReportOrchestrationService from utils.audit_logging_setup import LoggingService +from utils.decorators.ensure_env_var import ensure_environment_variables +from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions +from utils.decorators.override_error_check import override_error_check +from utils.decorators.set_audit_arg import set_request_context_for_logging logger = LoggingService(__name__) @@ -23,7 +28,12 @@ def calculate_reporting_window(): int(today_7am.timestamp()), ) - +@ensure_environment_variables( + names=["BULK_UPLOAD_REPORT_TABLE_NAME"] +) +@override_error_check +@handle_lambda_exceptions +@set_request_context_for_logging def lambda_handler(event, context): logger.info("Report orchestration lambda invoked") table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") @@ -37,8 +47,10 @@ def lambda_handler(event, context): ) window_start, window_end = calculate_reporting_window() + tmp_dir = tempfile.mkdtemp() service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, + output_dir=tmp_dir, ) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index 36c6c09466..2f8286a569 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -1,6 +1,6 @@ from datetime import datetime -from openpyxl import Workbook +from openpyxl.workbook import Workbook from utils.audit_logging_setup import LoggingService logger = LoggingService(__name__) @@ -14,7 +14,7 @@ def create_report_orchestration_xlsx( output_path: str, ) -> str: logger.info( - f"Creating Excel report for ods code {ods_code} and records {records}" + f"Creating Excel report for ods code {ods_code} and records {len(records)}" ) wb = Workbook() ws = wb.active diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index e6528d1bcb..5e6e7580cc 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -19,6 +19,7 @@ def process_reporting_window( self, window_start_ts: int, window_end_ts: int, + output_dir: str, ): records = self.repository.get_records_for_time_window( window_start_ts, From f69afbf54341cc064c31576d90ea827cc6c20e81 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 18 Dec 2025 11:26:22 +0000 Subject: [PATCH 06/37] [PRMP-1054] fixed unit tests --- .../test_report_orchestration_handler.py | 22 +++++++++++++------ .../test_report_orchestration_service.py | 4 ++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 59fe6b3c35..c9ee2ff849 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,9 +1,13 @@ +from unittest import mock from unittest.mock import MagicMock - import pytest from handlers.report_orchestration_handler import lambda_handler +class FakeContext: + aws_request_id = "test-request-id" + + @pytest.fixture(autouse=True) def mock_env(monkeypatch): monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") @@ -17,14 +21,16 @@ def mock_logger(mocker): @pytest.fixture def mock_repo(mocker): return mocker.patch( - "handlers.report_orchestration_handler.ReportingDynamoRepository", autospec=True + "handlers.report_orchestration_handler.ReportingDynamoRepository", + autospec=True, ) @pytest.fixture def mock_excel_generator(mocker): return mocker.patch( - "handlers.report_orchestration_handler.ExcelReportGenerator", autospec=True + "handlers.report_orchestration_handler.ExcelReportGenerator", + autospec=True, ) @@ -47,7 +53,7 @@ def mock_window(mocker): def test_lambda_handler_calls_service( mock_logger, mock_repo, mock_excel_generator, mock_service, mock_window ): - lambda_handler(event={}, context={}) + lambda_handler(event={}, context=FakeContext()) mock_repo.assert_called_once_with("TestTable") mock_excel_generator.assert_called_once_with() @@ -55,12 +61,14 @@ def test_lambda_handler_calls_service( mock_service.assert_called_once() instance = mock_service.return_value instance.process_reporting_window.assert_called_once_with( - window_start_ts=100, window_end_ts=200 + window_start_ts=100, + window_end_ts=200, + output_dir=mock.ANY, ) - mock_logger.info.assert_called_with("Report orchestration lambda invoked") + mock_logger.info.assert_any_call("Report orchestration lambda invoked") def test_lambda_handler_calls_window_function(mock_service, mock_window): - lambda_handler(event={}, context={}) + lambda_handler(event={}, context=FakeContext()) mock_window.assert_called_once() diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c3b21a77ef..c068ba0c16 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -27,7 +27,7 @@ def test_process_reporting_window_no_records( ): mock_repository.get_records_for_time_window.return_value = [] - report_orchestration_service.process_reporting_window(100, 200) + report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() @@ -68,7 +68,7 @@ def test_process_reporting_window_generates_reports_per_ods( report_orchestration_service, "generate_ods_report" ) - report_orchestration_service.process_reporting_window(100, 200) + report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") mocked_generate.assert_any_call( "Y12345", From 326a6c888fe92a1f04f6985c8e0417ba831f6077 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 19 Dec 2025 10:41:26 +0000 Subject: [PATCH 07/37] [PRMP-1054] added lambda layer to report_orchestration_lambda --- .github/workflows/base-lambdas-reusable-deploy-all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index 1b9edf7299..16801885bd 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -777,6 +777,6 @@ jobs: sandbox: ${{ inputs.sandbox }} lambda_handler_name: report_orchestration_handler lambda_aws_name: reportOrchestration - lambda_layer_names: "core_lambda_layer" + lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} From 31fc5e4de83b521572d6a1efffb3499c8ce1ebe7 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 19 Dec 2025 13:49:39 +0000 Subject: [PATCH 08/37] [PRMP-1054] fixed how the search works --- .../reporting/reporting_dynamo_repository.py | 21 +++++++------------ .../reporting/report_orchestration_service.py | 2 +- .../test_reporting_dynamo_repository.py | 20 ++++-------------- 3 files changed, 12 insertions(+), 31 deletions(-) diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 1233a0a4e1..7a694edf69 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,5 +1,6 @@ from typing import Dict, List +from boto3.dynamodb.conditions import Attr from services.base.dynamo_service import DynamoDBService from utils.audit_logging_setup import LoggingService @@ -18,25 +19,17 @@ def get_records_for_time_window( ) -> List[Dict]: logger.info( f"Querying reporting table for window, " - f"table_name: {self.table_name}," - f"start_timestamp: {start_timestamp}," + f"table_name: {self.table_name}, " + f"start_timestamp: {start_timestamp}, " f"end_timestamp: {end_timestamp}", ) - filter_expression = "#ts BETWEEN :start AND :end" - - expression_attribute_names = { - "#ts": "Timestamp", - } - - expression_attribute_values = { - ":start": start_timestamp, - ":end": end_timestamp, - } + filter_expression = Attr("Timestamp").between( + start_timestamp, + end_timestamp, + ) return self.dynamo_service.scan_whole_table( table_name=self.table_name, filter_expression=filter_expression, - expression_attribute_names=expression_attribute_names, - expression_attribute_values=expression_attribute_values, ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index 5e6e7580cc..a9500735b9 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -33,7 +33,7 @@ def process_reporting_window( for ods_code, ods_records in records_by_ods.items(): logger.info( - f"Generating report for ODS ods_code = {ods_code} record_count = len(ods_records)" + f"Generating report for ODS ods_code = {ods_code} record_count = {len(ods_records)}" ) self.generate_ods_report(ods_code, ods_records) logger.info("Report orchestration completed") diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index e88d98a145..df3c553044 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -20,24 +20,12 @@ def reporting_repo(mock_dynamo_service): def test_get_records_for_time_window_calls_scan(mock_dynamo_service, reporting_repo): - start_ts = 100 - end_ts = 200 - expected_result = [ - {"ID": 1, "Timestamp": 150}, - {"ID": 2, "Timestamp": 180}, - ] - mock_dynamo_service.scan_whole_table.return_value = expected_result + mock_dynamo_service.scan_whole_table.return_value = [] - result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + reporting_repo.get_records_for_time_window(100, 200) - mock_dynamo_service.scan_whole_table.assert_called_once_with( - table_name="TestTable", - filter_expression="#ts BETWEEN :start AND :end", - expression_attribute_names={"#ts": "Timestamp"}, - expression_attribute_values={":start": start_ts, ":end": end_ts}, - ) - - assert result == expected_result + mock_dynamo_service.scan_whole_table.assert_called_once() + assert "filter_expression" in mock_dynamo_service.scan_whole_table.call_args.kwargs def test_get_records_for_time_window_returns_empty_list( From d24739468cea9d7410bc69372ed0c599e57351d4 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 19 Dec 2025 16:21:52 +0000 Subject: [PATCH 09/37] [PRMP-1054] fixed comment --- lambdas/services/reporting/excel_report_generator_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index 2f8286a569..b8bef9a5cf 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -14,7 +14,7 @@ def create_report_orchestration_xlsx( output_path: str, ) -> str: logger.info( - f"Creating Excel report for ods code {ods_code} and records {len(records)}" + f"Creating Excel report for ODS code {ods_code} and records {len(records)}" ) wb = Workbook() ws = wb.active From 25fda598603656d13b0e109fb90a680a9fa0760f Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 5 Jan 2026 14:27:05 +0000 Subject: [PATCH 10/37] [PRMP-1057] created distribution lambda --- .../base-lambdas-reusable-deploy-all.yml | 14 + .../handlers/report_distribution_handler.py | 50 +++ .../handlers/report_orchestration_handler.py | 68 +++- .../reporting/report_contact_repository.py | 16 + .../requirements_reports_lambda_layer.txt | 3 +- lambdas/services/email_service.py | 104 +++++++ .../reporting/report_distribution_service.py | 137 ++++++++ .../reporting/report_orchestration_service.py | 30 +- .../test_report_distribution_handler.py | 89 ++++++ .../test_report_orchestration_handler.py | 164 +++++++++- .../test_report_contact_repository.py | 59 ++++ .../services/reporting/test_email_service.py | 154 +++++++++ .../test_report_distribution_service.py | 293 ++++++++++++++++++ .../test_report_orchestration_service.py | 136 ++++++-- lambdas/utils/zip_utils.py | 20 ++ 15 files changed, 1276 insertions(+), 61 deletions(-) create mode 100644 lambdas/handlers/report_distribution_handler.py create mode 100644 lambdas/repositories/reporting/report_contact_repository.py create mode 100644 lambdas/services/email_service.py create mode 100644 lambdas/services/reporting/report_distribution_service.py create mode 100644 lambdas/tests/unit/handlers/test_report_distribution_handler.py create mode 100644 lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py create mode 100644 lambdas/tests/unit/services/reporting/test_email_service.py create mode 100644 lambdas/tests/unit/services/reporting/test_report_distribution_service.py create mode 100644 lambdas/utils/zip_utils.py diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index e5bd7798d3..5128113a49 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -810,3 +810,17 @@ jobs: lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} + + deploy_report_distribution_lambda: + name: Deploy Report Distribution + uses: ./.github/workflows/base-lambdas-reusable-deploy.yml + with: + environment: ${{ inputs.environment }} + python_version: ${{ inputs.python_version }} + build_branch: ${{ inputs.build_branch }} + sandbox: ${{ inputs.sandbox }} + lambda_handler_name: report_distribution_handler + lambda_aws_name: reportDistribution + lambda_layer_names: "core_lambda_layer,reports_lambda_layer" + secrets: + AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py new file mode 100644 index 0000000000..9df15879d1 --- /dev/null +++ b/lambdas/handlers/report_distribution_handler.py @@ -0,0 +1,50 @@ +import os + +from repositories.reporting.report_contact_repository import ReportContactRepository +from services.base.s3_service import S3Service +from services.email_service import EmailService +from services.reporting.report_distribution_service import ReportDistributionService +from utils.audit_logging_setup import LoggingService +from utils.decorators.ensure_env_var import ensure_environment_variables +from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions +from utils.decorators.override_error_check import override_error_check +from utils.decorators.set_audit_arg import set_request_context_for_logging + +logger = LoggingService(__name__) + + +@ensure_environment_variables( + names=[ + "REPORT_BUCKET_NAME", + "CONTACT_TABLE_NAME", + "PRM_MAILBOX_EMAIL", + "SES_FROM_ADDRESS", + ] +) +@override_error_check +@handle_lambda_exceptions +@set_request_context_for_logging +def lambda_handler(event, context): + report_date = event["report_date"] + + bucket = os.environ["REPORT_BUCKET_NAME"] + contact_table = os.environ["CONTACT_TABLE_NAME"] + prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] + from_address = os.environ["SES_FROM_ADDRESS"] + + s3_service = S3Service() + contact_repo = ReportContactRepository(contact_table) + email_service = EmailService() + + service = ReportDistributionService( + s3_service=s3_service, + contact_repo=contact_repo, + email_service=email_service, + bucket=bucket, + from_address=from_address, + prm_mailbox=prm_mailbox, + ) + + result = service.distribute_reports_for_date(report_date) + logger.info(f"Daily Report distribution summary: {result}") + return result diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 84f6372ab8..6dc70db306 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,8 +1,11 @@ +import json import os -import tempfile from datetime import datetime, timedelta, timezone +import boto3 + from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from services.base.s3_service import S3Service from services.reporting.excel_report_generator_service import ExcelReportGenerator from services.reporting.report_orchestration_service import ReportOrchestrationService from utils.audit_logging_setup import LoggingService @@ -13,6 +16,8 @@ logger = LoggingService(__name__) +def _get_lambda_client(): + return boto3.client("lambda") def calculate_reporting_window(): now = datetime.now(timezone.utc) @@ -23,23 +28,38 @@ def calculate_reporting_window(): yesterday_7am = today_7am - timedelta(days=1) - return ( - int(yesterday_7am.timestamp()), - int(today_7am.timestamp()), - ) + return int(yesterday_7am.timestamp()), int(today_7am.timestamp()) + + +def get_report_date_folder() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d") + + +def build_s3_key(ods_code: str, report_date: str) -> str: + return f"Report-Orchestration/{report_date}/{ods_code}.xlsx" + @ensure_environment_variables( - names=["BULK_UPLOAD_REPORT_TABLE_NAME"] + names=[ + "BULK_UPLOAD_REPORT_TABLE_NAME", + "REPORT_BUCKET_NAME", + "REPORT_DISTRIBUTION_LAMBDA", + ] ) @override_error_check @handle_lambda_exceptions @set_request_context_for_logging def lambda_handler(event, context): logger.info("Report orchestration lambda invoked") - table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") + + lambda_client = _get_lambda_client() + table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] + report_bucket = os.environ["REPORT_BUCKET_NAME"] + distribution_lambda = os.environ["REPORT_DISTRIBUTION_LAMBDA"] repository = ReportingDynamoRepository(table_name) excel_generator = ExcelReportGenerator() + s3_service = S3Service() service = ReportOrchestrationService( repository=repository, @@ -47,10 +67,38 @@ def lambda_handler(event, context): ) window_start, window_end = calculate_reporting_window() - tmp_dir = tempfile.mkdtemp() + # TODO delete line below since it is only for testing + window_end += 24 * 60 * 60 + + report_date = get_report_date_folder() - service.process_reporting_window( + generated_files = service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, - output_dir=tmp_dir, ) + + if not generated_files: + logger.info("No reports generated; exiting") + return + + # Upload each generated report to S3 + for ods_code, local_path in generated_files.items(): + key = build_s3_key(ods_code, report_date) + s3_service.upload_file_with_extra_args( + file_name=local_path, + s3_bucket_name=report_bucket, + file_key=key, + extra_args={ + "ServerSideEncryption": "aws:kms", + }, + ) + logger.info(f"Uploaded report for ODS={ods_code} to s3://{report_bucket}/{key}") + + # Invoke distribution lambda once for the day folder + lambda_client.invoke( + FunctionName=distribution_lambda, + InvocationType="Event", + Payload=json.dumps({"report_date": report_date}), + ) + + logger.info(f"Invoked distribution lambda for report_date={report_date}") diff --git a/lambdas/repositories/reporting/report_contact_repository.py b/lambdas/repositories/reporting/report_contact_repository.py new file mode 100644 index 0000000000..9399e222ee --- /dev/null +++ b/lambdas/repositories/reporting/report_contact_repository.py @@ -0,0 +1,16 @@ +from services.base.dynamo_service import DynamoDBService + + +class ReportContactRepository: + def __init__(self, table_name: str): + self.table_name = table_name + self.dynamo = DynamoDBService() + + def get_contact_email(self, ods_code: str) -> str | None: + item = self.dynamo.get_item( + table_name=self.table_name, + key={"OdsCode": ods_code}, + ) + if not item: + return None + return item.get("Email") diff --git a/lambdas/requirements/layers/requirements_reports_lambda_layer.txt b/lambdas/requirements/layers/requirements_reports_lambda_layer.txt index 44709d7496..872e8ad167 100644 --- a/lambdas/requirements/layers/requirements_reports_lambda_layer.txt +++ b/lambdas/requirements/layers/requirements_reports_lambda_layer.txt @@ -1,2 +1,3 @@ openpyxl==3.1.5 -reportlab==4.3.1 \ No newline at end of file +reportlab==4.3.1 +pyzipper==0.3.6 \ No newline at end of file diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py new file mode 100644 index 0000000000..fb12e223dd --- /dev/null +++ b/lambdas/services/email_service.py @@ -0,0 +1,104 @@ +import boto3 +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.application import MIMEApplication +from typing import Iterable, Optional + +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + + +class EmailService: + """ + General email sender via SES (AWS Simple Email Service) Raw Email (supports attachments). + Higher-level methods prepare inputs and call send_email(). + """ + + def __init__(self): + self.ses = boto3.client("ses") + + def send_email( + self, + *, + to_address: str, + subject: str, + body_text: str, + from_address: str, + attachments: Optional[Iterable[str]] = None, + ): + msg = MIMEMultipart() + msg["Subject"] = subject + msg["To"] = to_address + msg["From"] = from_address + + msg.attach(MIMEText(body_text, "plain")) + + for attachment_path in attachments or []: + with open(attachment_path, "rb") as f: + part = MIMEApplication(f.read()) + part.add_header( + "Content-Disposition", + "attachment", + filename=attachment_path.split("/")[-1], + ) + msg.attach(part) + + self._send_raw(msg, to_address) + + def _send_raw(self, msg: MIMEMultipart, to_address: str): + logger.info(f"Sending SES raw email to {to_address}") + self.ses.send_raw_email( + RawMessage={"Data": msg.as_string()}, + Destinations=[to_address], + ) + + def send_report_email( + self, + *, + to_address: str, + from_address: str, + attachment_path: str, + ): + self.send_email( + to_address=to_address, + from_address=from_address, + subject="Daily Upload Report", + body_text="Please find your encrypted daily upload report attached.", + attachments=[attachment_path], + ) + + def send_password_email( + self, + *, + to_address: str, + from_address: str, + password: str, + ): + self.send_email( + to_address=to_address, + from_address=from_address, + subject="Daily Upload Report Password", + body_text=f"Password for your report:\n\n{password}", + ) + + def send_prm_missing_contact_email( + self, + *, + prm_mailbox: str, + from_address: str, + ods_code: str, + attachment_path: str, + password: str, + ): + self.send_email( + to_address=prm_mailbox, + from_address=from_address, + subject=f"Missing contact for ODS {ods_code}", + body_text=( + f"No contact found for ODS {ods_code}.\n\n" + f"Password: {password}\n\n" + f"Please resolve the contact and forward the report." + ), + attachments=[attachment_path], + ) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py new file mode 100644 index 0000000000..acaf65d0a5 --- /dev/null +++ b/lambdas/services/reporting/report_distribution_service.py @@ -0,0 +1,137 @@ +import os +import secrets +import tempfile +from typing import List, Dict + +import boto3 + +from repositories.reporting.report_contact_repository import ReportContactRepository +from services.base.s3_service import S3Service +from services.email_service import EmailService +from utils.audit_logging_setup import LoggingService +from utils.zip_utils import zip_encrypt_file + +logger = LoggingService(__name__) + + +class ReportDistributionService: + def __init__( + self, + *, + s3_service: S3Service, + contact_repo: ReportContactRepository, + email_service: EmailService, + bucket: str, + from_address: str, + prm_mailbox: str, + ): + self.s3_service = s3_service + self.contact_repo = contact_repo + self.email_service = email_service + self.bucket = bucket + self.from_address = from_address + self.prm_mailbox = prm_mailbox + self._s3_client = boto3.client("s3") + + def distribute_reports_for_date(self, report_date: str) -> Dict[str, int]: + day_prefix = f"Report-Orchestration/{report_date}/" + keys = self.list_xlsx_keys(prefix=day_prefix) + + logger.info(f"Found {len(keys)} report(s) under s3://{self.bucket}/{day_prefix}") + + succeeded = 0 + failed = 0 + + for key in keys: + ods_code = key.split("/")[-1].replace(".xlsx", "") + + try: + self.process_one_report(ods_code=ods_code, key=key) + succeeded += 1 + except Exception as e: + failed += 1 + logger.exception( + f"Failed processing report for ODS={ods_code}, key={key}: {e}" + ) + + logger.info( + f"Distribution completed for {report_date}: " + f"succeeded={succeeded}, failed={failed}" + ) + return {"succeeded": succeeded, "failed": failed} + + def list_xlsx_keys(self, prefix: str) -> List[str]: + paginator = self._s3_client.get_paginator("list_objects_v2") + keys: List[str] = [] + + for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): + for obj in page.get("Contents", []): + key = obj["Key"] + if key.endswith(".xlsx"): + keys.append(key) + + return keys + + def process_one_report(self, *, ods_code: str, key: str) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + local_xlsx = os.path.join(tmpdir, f"{ods_code}.xlsx") + local_zip = os.path.join(tmpdir, f"{ods_code}.zip") + + self.s3_service.download_file(self.bucket, key, local_xlsx) + + password = secrets.token_urlsafe(16) + zip_encrypt_file( + input_path=local_xlsx, + output_zip=local_zip, + password=password, + ) + + self.send_report_emails( + ods_code=ods_code, + attachment_path=local_zip, + password=password, + ) + + def send_report_emails(self, *, ods_code: str, attachment_path: str, password: str) -> None: + contact_email = self.contact_repo.get_contact_email(ods_code) + + if contact_email: + logger.info(f"Contact found for ODS={ods_code}, emailing {contact_email}") + self.email_contact( + to_address=contact_email, + attachment_path=attachment_path, + password=password, + ) + return + + logger.info(f"No contact found for ODS={ods_code}, sending to PRM mailbox") + self.email_prm_missing_contact( + ods_code=ods_code, + attachment_path=attachment_path, + password=password, + ) + + def email_contact(self, *, to_address: str, attachment_path: str, password: str) -> None: + # TODO delete next line since it is only for testing + to_address = "" + self.email_service.send_report_email( + to_address=to_address, + from_address=self.from_address, + attachment_path=attachment_path, + ) + self.email_service.send_password_email( + to_address=to_address, + from_address=self.from_address, + password=password, + ) + + def email_prm_missing_contact( + self, *, ods_code: str, attachment_path: str, password: str + ) -> None: + self.email_service.send_prm_missing_contact_email( + prm_mailbox=self.prm_mailbox, + from_address=self.from_address, + ods_code=ods_code, + attachment_path=attachment_path, + password=password, + ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index a9500735b9..69a919cc6e 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -1,5 +1,6 @@ import tempfile from collections import defaultdict +from typing import Dict from utils.audit_logging_setup import LoggingService @@ -7,11 +8,7 @@ class ReportOrchestrationService: - def __init__( - self, - repository, - excel_generator, - ): + def __init__(self, repository, excel_generator): self.repository = repository self.excel_generator = excel_generator @@ -19,24 +16,28 @@ def process_reporting_window( self, window_start_ts: int, window_end_ts: int, - output_dir: str, - ): + ) -> Dict[str, str]: records = self.repository.get_records_for_time_window( window_start_ts, window_end_ts, ) + if not records: logger.info("No records found for reporting window") - return + return {} records_by_ods = self.group_records_by_ods(records) + generated_files: Dict[str, str] = {} for ods_code, ods_records in records_by_ods.items(): logger.info( - f"Generating report for ODS ods_code = {ods_code} record_count = {len(ods_records)}" + f"Generating report for ODS={ods_code}, records={len(ods_records)}" ) - self.generate_ods_report(ods_code, ods_records) - logger.info("Report orchestration completed") + file_path = self.generate_ods_report(ods_code, ods_records) + generated_files[ods_code] = file_path + + logger.info(f"Generated {len(generated_files)} report(s)") + return generated_files @staticmethod def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: @@ -46,11 +47,7 @@ def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: grouped[ods_code].append(record) return grouped - def generate_ods_report( - self, - ods_code: str, - records: list[dict], - ): + def generate_ods_report(self, ods_code: str, records: list[dict]) -> str: with tempfile.NamedTemporaryFile( suffix=f"_{ods_code}.xlsx", delete=False, @@ -60,3 +57,4 @@ def generate_ods_report( records=records, output_path=tmp.name, ) + return tmp.name diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py new file mode 100644 index 0000000000..054af3eace --- /dev/null +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -0,0 +1,89 @@ +import importlib +import os +import pytest + +MODULE_UNDER_TEST = "handlers.report_distribution_handler" + + +@pytest.fixture +def handler_module(): + return importlib.import_module(MODULE_UNDER_TEST) + + +@pytest.fixture +def required_env(mocker): + mocker.patch.dict( + os.environ, + { + "REPORT_BUCKET_NAME": "my-report-bucket", + "CONTACT_TABLE_NAME": "contact-table", + "PRM_MAILBOX_EMAIL": "prm@example.com", + "SES_FROM_ADDRESS": "from@example.com", + }, + clear=False, + ) + + +def test_lambda_handler_wires_dependencies_and_returns_result( + mocker, handler_module, required_env +): + event = {"report_date": "2026-01-01"} + context = mocker.Mock() + + s3_instance = mocker.Mock(name="S3ServiceInstance") + contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") + email_instance = mocker.Mock(name="EmailServiceInstance") + + svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") + svc_instance.distribute_reports_for_date.return_value = {"succeeded": 2, "failed": 1} + + mocked_s3_cls = mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=s3_instance) + mocked_contact_repo_cls = mocker.patch.object( + handler_module, + "ReportContactRepository", + autospec=True, + return_value=contact_repo_instance, + ) + mocked_email_cls = mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=email_instance) + mocked_dist_svc_cls = mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + + result = handler_module.lambda_handler(event, context) + + mocked_s3_cls.assert_called_once_with() + mocked_contact_repo_cls.assert_called_once_with("contact-table") + mocked_email_cls.assert_called_once_with() + + mocked_dist_svc_cls.assert_called_once_with( + s3_service=s3_instance, + contact_repo=contact_repo_instance, + email_service=email_instance, + bucket="my-report-bucket", + from_address="from@example.com", + prm_mailbox="prm@example.com", + ) + + svc_instance.distribute_reports_for_date.assert_called_once_with("2026-01-01") + assert result == {"succeeded": 2, "failed": 1} + + +def test_lambda_handler_uses_report_date_from_event(mocker, handler_module, required_env): + event = {"report_date": "2099-12-31"} + context = mocker.Mock() + + svc_instance = mocker.Mock() + svc_instance.distribute_reports_for_date.return_value = {"succeeded": 0, "failed": 0} + mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + + mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + + result = handler_module.lambda_handler(event, context) + + svc_instance.distribute_reports_for_date.assert_called_once_with("2099-12-31") + assert result == {"succeeded": 0, "failed": 0} diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index c9ee2ff849..2654f313c0 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,6 +1,8 @@ -from unittest import mock +import json +import os from unittest.mock import MagicMock import pytest + from handlers.report_orchestration_handler import lambda_handler @@ -9,8 +11,16 @@ class FakeContext: @pytest.fixture(autouse=True) -def mock_env(monkeypatch): - monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") +def mock_env(mocker): + mocker.patch.dict( + os.environ, + { + "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", + "REPORT_BUCKET_NAME": "test-report-bucket", + "REPORT_DISTRIBUTION_LAMBDA": "test-distribution-lambda", + }, + clear=False, + ) @pytest.fixture @@ -34,6 +44,14 @@ def mock_excel_generator(mocker): ) +@pytest.fixture +def mock_s3_service(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.S3Service", + autospec=True, + ) + + @pytest.fixture def mock_service(mocker): return mocker.patch( @@ -50,25 +68,153 @@ def mock_window(mocker): ) -def test_lambda_handler_calls_service( - mock_logger, mock_repo, mock_excel_generator, mock_service, mock_window +@pytest.fixture +def mock_report_date(mocker): + return mocker.patch( + "handlers.report_orchestration_handler.get_report_date_folder", + return_value="2026-01-02", + ) + + +@pytest.fixture +def mock_lambda_client(mocker): + fake_client = MagicMock() + return mocker.patch( + "handlers.report_orchestration_handler._get_lambda_client", + return_value=fake_client, + ) + + +def test_lambda_handler_calls_service_and_dependencies( + mock_logger, + mock_repo, + mock_excel_generator, + mock_s3_service, + mock_service, + mock_window, + mock_report_date, + mock_lambda_client, ): + service_instance = mock_service.return_value + service_instance.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + "B67890": "/tmp/B67890.xlsx", + } + lambda_handler(event={}, context=FakeContext()) mock_repo.assert_called_once_with("TestTable") mock_excel_generator.assert_called_once_with() + mock_s3_service.assert_called_once_with() mock_service.assert_called_once() - instance = mock_service.return_value - instance.process_reporting_window.assert_called_once_with( + service_instance.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, - output_dir=mock.ANY, ) mock_logger.info.assert_any_call("Report orchestration lambda invoked") -def test_lambda_handler_calls_window_function(mock_service, mock_window): +def test_lambda_handler_calls_window_function( + mock_service, + mock_window, + mock_report_date, + mock_lambda_client, + mock_s3_service, +): + mock_service.return_value.process_reporting_window.return_value = {} + lambda_handler(event={}, context=FakeContext()) + mock_window.assert_called_once() + + +def test_lambda_handler_returns_early_when_no_reports_generated( + mock_service, + mock_logger, + mock_s3_service, + mock_lambda_client, + mock_window, + mock_report_date, +): + mock_service.return_value.process_reporting_window.return_value = {} + + result = lambda_handler(event={}, context=FakeContext()) + + assert result is None + mock_s3_service.return_value.upload_file_with_extra_args.assert_not_called() + mock_lambda_client.return_value.invoke.assert_not_called() + mock_logger.info.assert_any_call("No reports generated; exiting") + + +def test_lambda_handler_uploads_each_report_to_s3( + mock_service, + mock_s3_service, + mock_report_date, + mock_lambda_client, + mock_window, +): + mock_service.return_value.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + "UNKNOWN": "/tmp/UNKNOWN.xlsx", + } + + lambda_handler(event={}, context=FakeContext()) + + s3_instance = mock_s3_service.return_value + assert s3_instance.upload_file_with_extra_args.call_count == 2 + + s3_instance.upload_file_with_extra_args.assert_any_call( + file_name="/tmp/A12345.xlsx", + s3_bucket_name="test-report-bucket", + file_key="Report-Orchestration/2026-01-02/A12345.xlsx", + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + + s3_instance.upload_file_with_extra_args.assert_any_call( + file_name="/tmp/UNKNOWN.xlsx", + s3_bucket_name="test-report-bucket", + file_key="Report-Orchestration/2026-01-02/UNKNOWN.xlsx", + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + + +def test_lambda_handler_invokes_distribution_lambda_once( + mock_service, + mock_report_date, + mock_lambda_client, + mock_window, + mock_s3_service, +): + mock_service.return_value.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + } + + lambda_handler(event={}, context=FakeContext()) + + client = mock_lambda_client.return_value + client.invoke.assert_called_once() + + kwargs = client.invoke.call_args.kwargs + assert kwargs["FunctionName"] == "test-distribution-lambda" + assert kwargs["InvocationType"] == "Event" + + payload = kwargs["Payload"] + assert '"report_date": "2026-01-02"' in payload + + +def test_lambda_handler_returns_error_when_required_env_missing(): + os.environ.pop("REPORT_BUCKET_NAME", None) + + result = lambda_handler(event={}, context=FakeContext()) + + assert isinstance(result, dict) + assert result["statusCode"] == 500 + + body = json.loads(result["body"]) + assert body["err_code"] == "ENV_5001" + assert "REPORT_BUCKET_NAME" in body["message"] + + if "interaction_id" in body and body["interaction_id"] is not None: + assert body["interaction_id"] == "test-request-id" \ No newline at end of file diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py new file mode 100644 index 0000000000..5cae95f0f2 --- /dev/null +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -0,0 +1,59 @@ +import pytest + +from repositories.reporting.report_contact_repository import ReportContactRepository + + +@pytest.fixture +def mock_dynamo(mocker): + mock = mocker.Mock() + mocker.patch( + "repositories.reporting.report_contact_repository.DynamoDBService", + return_value=mock, + ) + return mock + + +@pytest.fixture +def repo(mock_dynamo): + return ReportContactRepository(table_name="report-contacts") + + +def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): + mock_dynamo.get_item.return_value = { + "OdsCode": "Y12345", + "Email": "contact@example.com", + } + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result == "contact@example.com" + + +def test_get_contact_email_returns_none_when_item_missing(repo, mock_dynamo): + mock_dynamo.get_item.return_value = None + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result is None + + +def test_get_contact_email_returns_none_when_email_missing(repo, mock_dynamo): + mock_dynamo.get_item.return_value = { + "OdsCode": "Y12345", + } + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result is None diff --git a/lambdas/tests/unit/services/reporting/test_email_service.py b/lambdas/tests/unit/services/reporting/test_email_service.py new file mode 100644 index 0000000000..efde7f7251 --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_email_service.py @@ -0,0 +1,154 @@ +import pytest + +from services.email_service import EmailService + + +@pytest.fixture +def email_service(mocker): + mocker.patch("services.email_service.boto3.client", autospec=True) + svc = EmailService() + svc.ses = mocker.Mock() + return svc + +def test_send_email_sends_raw_email_without_attachments(email_service, mocker): + mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) + + email_service.send_email( + to_address="to@example.com", + subject="Hello", + body_text="Body text", + from_address="from@example.com", + attachments=None, + ) + + mocked_send_raw.assert_called_once() + + call_args, call_kwargs = mocked_send_raw.call_args + assert call_kwargs == {} + + msg_arg = call_args[0] + to_arg = call_args[1] + + assert to_arg == "to@example.com" + assert msg_arg["Subject"] == "Hello" + assert msg_arg["To"] == "to@example.com" + assert msg_arg["From"] == "from@example.com" + + raw = msg_arg.as_string() + assert "Body text" in raw + + + +def test_send_email_attaches_files_and_sets_filenames(email_service, mocker): + file_bytes_1 = b"zipbytes1" + file_bytes_2 = b"zipbytes2" + + m1 = mocker.mock_open(read_data=file_bytes_1) + m2 = mocker.mock_open(read_data=file_bytes_2) + + mocked_open = mocker.patch("services.email_service.open", create=True) + mocked_open.side_effect = [m1.return_value, m2.return_value] + + mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) + + email_service.send_email( + to_address="to@example.com", + subject="With Attachments", + body_text="See attached", + from_address="from@example.com", + attachments=["/tmp/a.zip", "/var/tmp/b.zip"], + ) + + assert mocked_open.call_count == 2 + mocked_open.assert_any_call("/tmp/a.zip", "rb") + mocked_open.assert_any_call("/var/tmp/b.zip", "rb") + + mocked_send_raw.assert_called_once() + + call_args, call_kwargs = mocked_send_raw.call_args + assert call_kwargs == {} + msg = call_args[0] + raw = msg.as_string() + + assert 'filename="a.zip"' in raw + assert 'filename="b.zip"' in raw + assert "See attached" in raw + + + +def test_send_raw_calls_ses_send_raw_email(email_service, mocker): + from email.mime.multipart import MIMEMultipart + msg = MIMEMultipart() + msg["Subject"] = "S" + msg["To"] = "to@example.com" + msg["From"] = "from@example.com" + + email_service._send_raw(msg, "to@example.com") + + email_service.ses.send_raw_email.assert_called_once() + call_kwargs = email_service.ses.send_raw_email.call_args.kwargs + + assert call_kwargs["Destinations"] == ["to@example.com"] + assert "RawMessage" in call_kwargs + assert "Data" in call_kwargs["RawMessage"] + assert isinstance(call_kwargs["RawMessage"]["Data"], str) + assert "Subject: S" in call_kwargs["RawMessage"]["Data"] + + +def test_send_report_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_report_email( + to_address="to@example.com", + from_address="from@example.com", + attachment_path="/tmp/report.zip", + ) + + mocked_send_email.assert_called_once_with( + to_address="to@example.com", + from_address="from@example.com", + subject="Daily Upload Report", + body_text="Please find your encrypted daily upload report attached.", + attachments=["/tmp/report.zip"], + ) + + +def test_send_password_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_password_email( + to_address="to@example.com", + from_address="from@example.com", + password="pw123", + ) + + mocked_send_email.assert_called_once_with( + to_address="to@example.com", + from_address="from@example.com", + subject="Daily Upload Report Password", + body_text="Password for your report:\n\npw123", + ) + + +def test_send_prm_missing_contact_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_prm_missing_contact_email( + prm_mailbox="prm@example.com", + from_address="from@example.com", + ods_code="Y12345", + attachment_path="/tmp/report.zip", + password="pw123", + ) + + mocked_send_email.assert_called_once_with( + to_address="prm@example.com", + from_address="from@example.com", + subject="Missing contact for ODS Y12345", + body_text=( + "No contact found for ODS Y12345.\n\n" + "Password: pw123\n\n" + "Please resolve the contact and forward the report." + ), + attachments=["/tmp/report.zip"], + ) diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py new file mode 100644 index 0000000000..06ca8721a8 --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -0,0 +1,293 @@ +import os +import pytest + +from services.reporting.report_distribution_service import ReportDistributionService + +@pytest.fixture +def mock_s3_service(mocker): + return mocker.Mock() + + +@pytest.fixture +def mock_contact_repo(mocker): + repo = mocker.Mock() + repo.get_contact_email.return_value = None + return repo + + +@pytest.fixture +def mock_email_service(mocker): + return mocker.Mock() + + +@pytest.fixture +def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): + mocker.patch("services.reporting.report_distribution_service.boto3.client", autospec=True) + + return ReportDistributionService( + s3_service=mock_s3_service, + contact_repo=mock_contact_repo, + email_service=mock_email_service, + bucket="my-bucket", + from_address="from@example.com", + prm_mailbox="prm@example.com", + ) + +def test_list_xlsx_keys_filters_only_xlsx(service, mocker): + paginator = mocker.Mock() + paginator.paginate.return_value = [ + { + "Contents": [ + {"Key": "Report-Orchestration/2026-01-01/A123.xlsx"}, + {"Key": "Report-Orchestration/2026-01-01/readme.txt"}, + {"Key": "Report-Orchestration/2026-01-01/B456.xls"}, + {"Key": "Report-Orchestration/2026-01-01/C789.xlsx"}, + ] + }, + {"Contents": [{"Key": "Report-Orchestration/2026-01-01/D000.xlsx"}]}, + {}, + ] + + service._s3_client.get_paginator.return_value = paginator + + keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + + assert keys == [ + "Report-Orchestration/2026-01-01/A123.xlsx", + "Report-Orchestration/2026-01-01/C789.xlsx", + "Report-Orchestration/2026-01-01/D000.xlsx", + ] + + service._s3_client.get_paginator.assert_called_once_with("list_objects_v2") + paginator.paginate.assert_called_once_with( + Bucket="my-bucket", + Prefix="Report-Orchestration/2026-01-01/", + ) + + +def test_list_xlsx_keys_returns_empty_when_no_objects(service, mocker): + paginator = mocker.Mock() + paginator.paginate.return_value = [{"Contents": []}, {}] + service._s3_client.get_paginator.return_value = paginator + + keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + + assert keys == [] + +def test_distribute_reports_for_date_when_no_keys_returns_zeroes(service, mocker): + mocker.patch.object(service, "list_xlsx_keys", return_value=[]) + mocked_process = mocker.patch.object(service, "process_one_report") + + result = service.distribute_reports_for_date("2026-01-01") + + assert result == {"succeeded": 0, "failed": 0} + mocked_process.assert_not_called() + + +def test_distribute_reports_for_date_extracts_ods_from_key(service, mocker): + mocker.patch.object( + service, + "list_xlsx_keys", + return_value=["Report-Orchestration/2026-01-01/some/path/Y12345.xlsx"], + ) + mocked_process = mocker.patch.object(service, "process_one_report", return_value=None) + + result = service.distribute_reports_for_date("2026-01-01") + + assert result == {"succeeded": 1, "failed": 0} + mocked_process.assert_called_once_with( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/some/path/Y12345.xlsx", + ) + + +def test_distribute_reports_for_date_counts_success_and_failure(service, mocker): + mocker.patch.object( + service, + "list_xlsx_keys", + return_value=[ + "Report-Orchestration/2026-01-01/Y12345.xlsx", + "Report-Orchestration/2026-01-01/A99999.xlsx", + ], + ) + + mocked_process = mocker.patch.object(service, "process_one_report") + mocked_process.side_effect = [None, RuntimeError("boom")] + + result = service.distribute_reports_for_date("2026-01-01") + + assert result == {"succeeded": 1, "failed": 1} + assert mocked_process.call_count == 2 + + mocked_process.assert_any_call( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + mocked_process.assert_any_call( + ods_code="A99999", + key="Report-Orchestration/2026-01-01/A99999.xlsx", + ) + +def test_process_one_report_downloads_encrypts_and_delegates_email( + service, mocker, mock_s3_service +): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="fixed-password", + ) + + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + service.process_one_report( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + + local_xlsx = os.path.join(fake_tmp, "Y12345.xlsx") + local_zip = os.path.join(fake_tmp, "Y12345.zip") + + mock_s3_service.download_file.assert_called_once_with( + "my-bucket", + "Report-Orchestration/2026-01-01/Y12345.xlsx", + local_xlsx, + ) + + mocked_zip.assert_called_once_with( + input_path=local_xlsx, + output_zip=local_zip, + password="fixed-password", + ) + + mocked_send.assert_called_once_with( + ods_code="Y12345", + attachment_path=local_zip, + password="fixed-password", + ) + + +def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_service): + mock_s3_service.download_file.side_effect = RuntimeError("download failed") + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="pw", + ) + mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="download failed"): + service.process_one_report( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + + mocked_send.assert_not_called() + + +def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="pw", + ) + mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + side_effect=RuntimeError("zip failed"), + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="zip failed"): + service.process_one_report( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + + mocked_send.assert_not_called() + +def test_send_report_emails_with_contact_calls_email_contact(service, mock_contact_repo, mocker): + mock_contact_repo.get_contact_email.return_value = "contact@example.com" + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="Y12345", + attachment_path="/tmp/Y12345.zip", + password="pw", + ) + + mock_contact_repo.get_contact_email.assert_called_once_with("Y12345") + mocked_email_contact.assert_called_once_with( + to_address="contact@example.com", + attachment_path="/tmp/Y12345.zip", + password="pw", + ) + mocked_email_prm.assert_not_called() + + +def test_send_report_emails_without_contact_calls_email_prm(service, mock_contact_repo, mocker): + mock_contact_repo.get_contact_email.return_value = None + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + mock_contact_repo.get_contact_email.assert_called_once_with("A99999") + mocked_email_prm.assert_called_once_with( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + mocked_email_contact.assert_not_called() + + +def test_email_contact_sends_report_and_password(service, mock_email_service): + service.email_contact( + to_address="contact@example.com", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_report_email.assert_called_once_with( + to_address="contact@example.com", + from_address="from@example.com", + attachment_path="/tmp/file.zip", + ) + mock_email_service.send_password_email.assert_called_once_with( + to_address="contact@example.com", + from_address="from@example.com", + password="pw", + ) + + +def test_email_prm_missing_contact_sends_prm_missing_contact_email(service, mock_email_service): + service.email_prm_missing_contact( + ods_code="X11111", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_prm_missing_contact_email.assert_called_once_with( + prm_mailbox="prm@example.com", + from_address="from@example.com", + ods_code="X11111", + attachment_path="/tmp/file.zip", + password="pw", + ) diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c068ba0c16..baf54577fe 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -1,4 +1,5 @@ import pytest + from services.reporting.report_orchestration_service import ReportOrchestrationService @@ -22,36 +23,27 @@ def report_orchestration_service(mock_repository, mock_excel_generator): ) -def test_process_reporting_window_no_records( +def test_process_reporting_window_no_records_returns_empty_dict_and_does_not_generate( report_orchestration_service, mock_repository, mock_excel_generator ): mock_repository.get_records_for_time_window.return_value = [] - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + result = report_orchestration_service.process_reporting_window(100, 200) + assert result == {} mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() -def test_group_records_by_ods_groups_correctly(): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - {"UploaderOdsCode": "A99999", "ID": 3}, - {"ID": 4}, # missing ODS - {"UploaderOdsCode": None, "ID": 5}, # null ODS - ] +def test_process_reporting_window_calls_repository_with_window_args( + report_orchestration_service, mock_repository, mocker +): + mock_repository.get_records_for_time_window.return_value = [{"UploaderOdsCode": "X1", "ID": 1}] + mocked_generate = mocker.patch.object(report_orchestration_service, "generate_ods_report", return_value="/tmp/x.xlsx") - result = ReportOrchestrationService.group_records_by_ods(records) + report_orchestration_service.process_reporting_window(100, 200) - assert result["Y12345"] == [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - ] - assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] - assert result["UNKNOWN"] == [ - {"ID": 4}, - {"UploaderOdsCode": None, "ID": 5}, - ] + mock_repository.get_records_for_time_window.assert_called_once_with(100, 200) + mocked_generate.assert_called_once() def test_process_reporting_window_generates_reports_per_ods( @@ -65,10 +57,10 @@ def test_process_reporting_window_generates_reports_per_ods( mock_repository.get_records_for_time_window.return_value = records mocked_generate = mocker.patch.object( - report_orchestration_service, "generate_ods_report" + report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" ) - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + report_orchestration_service.process_reporting_window(100, 200) mocked_generate.assert_any_call( "Y12345", @@ -84,21 +76,115 @@ def test_process_reporting_window_generates_reports_per_ods( assert mocked_generate.call_count == 2 -def test_generate_ods_report_creates_excel_report( +def test_process_reporting_window_returns_mapping_of_ods_to_generated_file_path( + report_orchestration_service, mock_repository, mocker +): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "A99999", "ID": 2}, + ] + mock_repository.get_records_for_time_window.return_value = records + + def _side_effect(ods_code, ods_records): + return f"/tmp/{ods_code}.xlsx" + + mocker.patch.object( + report_orchestration_service, + "generate_ods_report", + side_effect=_side_effect, + ) + + result = report_orchestration_service.process_reporting_window(100, 200) + + assert result == { + "Y12345": "/tmp/Y12345.xlsx", + "A99999": "/tmp/A99999.xlsx", + } + + +def test_process_reporting_window_includes_unknown_ods_group( + report_orchestration_service, mock_repository, mocker +): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"ID": 2}, # missing ODS -> UNKNOWN + {"UploaderOdsCode": None, "ID": 3}, # null ODS -> UNKNOWN + ] + mock_repository.get_records_for_time_window.return_value = records + + mocked_generate = mocker.patch.object( + report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" + ) + + report_orchestration_service.process_reporting_window(100, 200) + + # Expect 2 groups: Y12345 and UNKNOWN + assert mocked_generate.call_count == 2 + mocked_generate.assert_any_call( + "Y12345", + [{"UploaderOdsCode": "Y12345", "ID": 1}], + ) + mocked_generate.assert_any_call( + "UNKNOWN", + [{"ID": 2}, {"UploaderOdsCode": None, "ID": 3}], + ) + + +def test_group_records_by_ods_groups_correctly(): + records = [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + {"ID": 4}, # missing ODS + {"UploaderOdsCode": None, "ID": 5}, # null ODS + ] + + result = ReportOrchestrationService.group_records_by_ods(records) + + assert result["Y12345"] == [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ] + assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] + assert result["UNKNOWN"] == [ + {"ID": 4}, + {"UploaderOdsCode": None, "ID": 5}, + ] + + +def test_group_records_by_ods_empty_input_returns_empty_mapping(): + result = ReportOrchestrationService.group_records_by_ods([]) + assert dict(result) == {} + + +def test_group_records_by_ods_treats_empty_string_as_unknown(): + records = [{"UploaderOdsCode": "", "ID": 1}] + result = ReportOrchestrationService.group_records_by_ods(records) + assert result["UNKNOWN"] == [{"UploaderOdsCode": "", "ID": 1}] + + +def test_generate_ods_report_creates_excel_report_and_returns_path( report_orchestration_service, mock_excel_generator, mocker ): fake_tmp = mocker.MagicMock() fake_tmp.__enter__.return_value = fake_tmp fake_tmp.name = "/tmp/fake_Y12345.xlsx" - mocker.patch( + mocked_ntf = mocker.patch( "services.reporting.report_orchestration_service.tempfile.NamedTemporaryFile", return_value=fake_tmp, ) records = [{"ID": 1, "UploaderOdsCode": "Y12345"}] - report_orchestration_service.generate_ods_report("Y12345", records) + result_path = report_orchestration_service.generate_ods_report("Y12345", records) + + assert result_path == fake_tmp.name + + mocked_ntf.assert_called_once_with( + suffix="_Y12345.xlsx", + delete=False, + ) mock_excel_generator.create_report_orchestration_xlsx.assert_called_once_with( ods_code="Y12345", diff --git a/lambdas/utils/zip_utils.py b/lambdas/utils/zip_utils.py new file mode 100644 index 0000000000..a09e8934f0 --- /dev/null +++ b/lambdas/utils/zip_utils.py @@ -0,0 +1,20 @@ +import os +import pyzipper + + +def zip_encrypt_file(*, input_path: str, output_zip: str, password: str) -> None: + """ + Create an AES-encrypted ZIP file containing a single file. + + :param input_path: Path to the file to zip + :param output_zip: Path of the zip file to create + :param password: Password for AES encryption + """ + with pyzipper.AESZipFile( + output_zip, + "w", + compression=pyzipper.ZIP_DEFLATED, + encryption=pyzipper.WZ_AES, + ) as zf: + zf.setpassword(password.encode("utf-8")) + zf.write(input_path, arcname=os.path.basename(input_path)) From 4d9b46647a06e9a7471f62a67df4f0713e040d24 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 5 Jan 2026 15:46:42 +0000 Subject: [PATCH 11/37] [PRMP-1057] updated for the case it fails to access contacts table --- lambdas/services/reporting/report_distribution_service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index acaf65d0a5..694085685d 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -93,7 +93,13 @@ def process_one_report(self, *, ods_code: str, key: str) -> None: ) def send_report_emails(self, *, ods_code: str, attachment_path: str, password: str) -> None: - contact_email = self.contact_repo.get_contact_email(ods_code) + try: + contact_email = self.contact_repo.get_contact_email(ods_code) + except Exception as e: + logger.exception( + f"Contact lookup failed for ODS={ods_code}; falling back to PRM. Error: {e}" + ) + contact_email = None if contact_email: logger.info(f"Contact found for ODS={ods_code}, emailing {contact_email}") From 4ebc15e35bf47c72cd84f96741ef2dbd011f6094 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 6 Jan 2026 14:05:29 +0000 Subject: [PATCH 12/37] [PRMP-1182] trying with step functions --- .../handlers/report_distribution_handler.py | 24 ++++++++--- .../handlers/report_orchestration_handler.py | 43 +++++++++---------- .../reporting/report_distribution_service.py | 32 +++----------- 3 files changed, 43 insertions(+), 56 deletions(-) diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index 9df15879d1..d65314b781 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -1,4 +1,5 @@ import os +from typing import Any, Dict from repositories.reporting.report_contact_repository import ReportContactRepository from services.base.s3_service import S3Service @@ -24,10 +25,12 @@ @override_error_check @handle_lambda_exceptions @set_request_context_for_logging -def lambda_handler(event, context): - report_date = event["report_date"] +def lambda_handler(event, context) -> Dict[str, Any]: + action = event.get("action") + if action not in {"list", "process_one"}: + raise ValueError("Invalid action. Expected 'list' or 'process_one'.") - bucket = os.environ["REPORT_BUCKET_NAME"] + bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] contact_table = os.environ["CONTACT_TABLE_NAME"] prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] from_address = os.environ["SES_FROM_ADDRESS"] @@ -45,6 +48,15 @@ def lambda_handler(event, context): prm_mailbox=prm_mailbox, ) - result = service.distribute_reports_for_date(report_date) - logger.info(f"Daily Report distribution summary: {result}") - return result + if action == "list": + prefix = event["prefix"] + keys = service.list_xlsx_keys(prefix=prefix) + logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") + return {"bucket": bucket, "prefix": prefix, "keys": keys} + + # action == "process_one" + key = event["key"] + ods_code = service.extract_ods_code_from_key(key) + service.process_one_report(ods_code=ods_code, key=key) + logger.info(f"Process-one mode: processed ods={ods_code}, key={key}") + return {"status": "ok", "bucket": bucket, "key": key, "ods_code": ods_code} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 6dc70db306..d1d746d14f 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,8 +1,6 @@ -import json import os from datetime import datetime, timedelta, timezone - -import boto3 +from typing import Any, Dict from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.base.s3_service import S3Service @@ -16,8 +14,6 @@ logger = LoggingService(__name__) -def _get_lambda_client(): - return boto3.client("lambda") def calculate_reporting_window(): now = datetime.now(timezone.utc) @@ -43,19 +39,16 @@ def build_s3_key(ods_code: str, report_date: str) -> str: names=[ "BULK_UPLOAD_REPORT_TABLE_NAME", "REPORT_BUCKET_NAME", - "REPORT_DISTRIBUTION_LAMBDA", ] ) @override_error_check @handle_lambda_exceptions @set_request_context_for_logging -def lambda_handler(event, context): +def lambda_handler(event, context) -> Dict[str, Any]: logger.info("Report orchestration lambda invoked") - lambda_client = _get_lambda_client() table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] report_bucket = os.environ["REPORT_BUCKET_NAME"] - distribution_lambda = os.environ["REPORT_DISTRIBUTION_LAMBDA"] repository = ReportingDynamoRepository(table_name) excel_generator = ExcelReportGenerator() @@ -71,6 +64,7 @@ def lambda_handler(event, context): window_end += 24 * 60 * 60 report_date = get_report_date_folder() + prefix = f"Report-Orchestration/{report_date}/" generated_files = service.process_reporting_window( window_start_ts=window_start, @@ -79,26 +73,29 @@ def lambda_handler(event, context): if not generated_files: logger.info("No reports generated; exiting") - return - - # Upload each generated report to S3 + return { + "report_date": report_date, + "bucket": report_bucket, + "prefix": prefix, + "keys": [], + } + + keys = [] for ods_code, local_path in generated_files.items(): key = build_s3_key(ods_code, report_date) s3_service.upload_file_with_extra_args( file_name=local_path, s3_bucket_name=report_bucket, file_key=key, - extra_args={ - "ServerSideEncryption": "aws:kms", - }, + extra_args={"ServerSideEncryption": "aws:kms"}, ) + keys.append(key) logger.info(f"Uploaded report for ODS={ods_code} to s3://{report_bucket}/{key}") - # Invoke distribution lambda once for the day folder - lambda_client.invoke( - FunctionName=distribution_lambda, - InvocationType="Event", - Payload=json.dumps({"report_date": report_date}), - ) - - logger.info(f"Invoked distribution lambda for report_date={report_date}") + logger.info(f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}") + return { + "report_date": report_date, + "bucket": report_bucket, + "prefix": prefix, + "keys": keys, + } diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index 694085685d..bbb380faad 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -1,7 +1,7 @@ import os import secrets import tempfile -from typing import List, Dict +from typing import List import boto3 @@ -33,32 +33,10 @@ def __init__( self.prm_mailbox = prm_mailbox self._s3_client = boto3.client("s3") - def distribute_reports_for_date(self, report_date: str) -> Dict[str, int]: - day_prefix = f"Report-Orchestration/{report_date}/" - keys = self.list_xlsx_keys(prefix=day_prefix) - - logger.info(f"Found {len(keys)} report(s) under s3://{self.bucket}/{day_prefix}") - - succeeded = 0 - failed = 0 - - for key in keys: - ods_code = key.split("/")[-1].replace(".xlsx", "") - - try: - self.process_one_report(ods_code=ods_code, key=key) - succeeded += 1 - except Exception as e: - failed += 1 - logger.exception( - f"Failed processing report for ODS={ods_code}, key={key}: {e}" - ) - - logger.info( - f"Distribution completed for {report_date}: " - f"succeeded={succeeded}, failed={failed}" - ) - return {"succeeded": succeeded, "failed": failed} + @staticmethod + def extract_ods_code_from_key(key: str) -> str: + filename = key.split("/")[-1] + return filename[:-5] if filename.lower().endswith(".xlsx") else filename def list_xlsx_keys(self, prefix: str) -> List[str]: paginator = self._s3_client.get_paginator("list_objects_v2") From 5cecec9a1995cab0d565a9bff64944d1e007694c Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 6 Jan 2026 15:56:24 +0000 Subject: [PATCH 13/37] [PRMP-1182] fixed unit tests --- .../test_report_distribution_handler.py | 65 +++++-- .../test_report_orchestration_handler.py | 133 +++++++------- .../test_report_distribution_service.py | 170 +++++++++++------- 3 files changed, 223 insertions(+), 145 deletions(-) diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index 054af3eace..d129d2ed36 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -24,27 +24,32 @@ def required_env(mocker): ) -def test_lambda_handler_wires_dependencies_and_returns_result( +def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( mocker, handler_module, required_env ): - event = {"report_date": "2026-01-01"} + event = {"action": "list", "prefix": "reports/2026-01-01/"} context = mocker.Mock() + context.aws_request_id = "req-123" # avoid JSON serialization issues in decorators s3_instance = mocker.Mock(name="S3ServiceInstance") contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") email_instance = mocker.Mock(name="EmailServiceInstance") svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") - svc_instance.distribute_reports_for_date.return_value = {"succeeded": 2, "failed": 1} + svc_instance.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] - mocked_s3_cls = mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=s3_instance) + mocked_s3_cls = mocker.patch.object( + handler_module, "S3Service", autospec=True, return_value=s3_instance + ) mocked_contact_repo_cls = mocker.patch.object( handler_module, "ReportContactRepository", autospec=True, return_value=contact_repo_instance, ) - mocked_email_cls = mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=email_instance) + mocked_email_cls = mocker.patch.object( + handler_module, "EmailService", autospec=True, return_value=email_instance + ) mocked_dist_svc_cls = mocker.patch.object( handler_module, "ReportDistributionService", @@ -67,23 +72,59 @@ def test_lambda_handler_wires_dependencies_and_returns_result( prm_mailbox="prm@example.com", ) - svc_instance.distribute_reports_for_date.assert_called_once_with("2026-01-01") - assert result == {"succeeded": 2, "failed": 1} + svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") + assert result == { + "bucket": "my-report-bucket", + "prefix": "reports/2026-01-01/", + "keys": ["a.xlsx", "b.xlsx"], + } -def test_lambda_handler_uses_report_date_from_event(mocker, handler_module, required_env): - event = {"report_date": "2099-12-31"} +def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( + mocker, handler_module, required_env +): + event = {"action": "list", "prefix": "p/", "bucket": "override-bucket"} context = mocker.Mock() + context.aws_request_id = "req-456" svc_instance = mocker.Mock() - svc_instance.distribute_reports_for_date.return_value = {"succeeded": 0, "failed": 0} + svc_instance.list_xlsx_keys.return_value = [] + mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + + result = handler_module.lambda_handler(event, context) + svc_instance.list_xlsx_keys.assert_called_once_with(prefix="p/") + assert result == {"bucket": "override-bucket", "prefix": "p/", "keys": []} + + +def test_lambda_handler_process_one_mode_happy_path( + mocker, handler_module, required_env +): + event = {"action": "process_one", "key": "reports/ABC/whatever.xlsx"} + context = mocker.Mock() + context.aws_request_id = "req-789" + + svc_instance = mocker.Mock() + svc_instance.extract_ods_code_from_key.return_value = "ABC" + svc_instance.process_one_report.return_value = None + + mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) result = handler_module.lambda_handler(event, context) - svc_instance.distribute_reports_for_date.assert_called_once_with("2099-12-31") - assert result == {"succeeded": 0, "failed": 0} + svc_instance.extract_ods_code_from_key.assert_called_once_with("reports/ABC/whatever.xlsx") + svc_instance.process_one_report.assert_called_once_with(ods_code="ABC", key="reports/ABC/whatever.xlsx") + + assert result == { + "status": "ok", + "bucket": "my-report-bucket", + "key": "reports/ABC/whatever.xlsx", + "ods_code": "ABC", + } diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 2654f313c0..10364c5e42 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,6 +1,7 @@ import json import os from unittest.mock import MagicMock + import pytest from handlers.report_orchestration_handler import lambda_handler @@ -17,7 +18,6 @@ def mock_env(mocker): { "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", "REPORT_BUCKET_NAME": "test-report-bucket", - "REPORT_DISTRIBUTION_LAMBDA": "test-distribution-lambda", }, clear=False, ) @@ -29,7 +29,7 @@ def mock_logger(mocker): @pytest.fixture -def mock_repo(mocker): +def mock_repo_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.ReportingDynamoRepository", autospec=True, @@ -37,7 +37,7 @@ def mock_repo(mocker): @pytest.fixture -def mock_excel_generator(mocker): +def mock_excel_generator_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.ExcelReportGenerator", autospec=True, @@ -45,7 +45,7 @@ def mock_excel_generator(mocker): @pytest.fixture -def mock_s3_service(mocker): +def mock_s3_service_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.S3Service", autospec=True, @@ -53,7 +53,7 @@ def mock_s3_service(mocker): @pytest.fixture -def mock_service(mocker): +def mock_service_cls(mocker): return mocker.patch( "handlers.report_orchestration_handler.ReportOrchestrationService", autospec=True, @@ -76,38 +76,32 @@ def mock_report_date(mocker): ) -@pytest.fixture -def mock_lambda_client(mocker): - fake_client = MagicMock() - return mocker.patch( - "handlers.report_orchestration_handler._get_lambda_client", - return_value=fake_client, - ) - - -def test_lambda_handler_calls_service_and_dependencies( +def test_lambda_handler_wires_dependencies_and_calls_service( mock_logger, - mock_repo, - mock_excel_generator, - mock_s3_service, - mock_service, + mock_repo_cls, + mock_excel_generator_cls, + mock_s3_service_cls, + mock_service_cls, mock_window, mock_report_date, - mock_lambda_client, ): - service_instance = mock_service.return_value + service_instance = mock_service_cls.return_value service_instance.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "B67890": "/tmp/B67890.xlsx", } - lambda_handler(event={}, context=FakeContext()) + result = lambda_handler(event={}, context=FakeContext()) - mock_repo.assert_called_once_with("TestTable") - mock_excel_generator.assert_called_once_with() - mock_s3_service.assert_called_once_with() + mock_repo_cls.assert_called_once_with("TestTable") + mock_excel_generator_cls.assert_called_once_with() + mock_s3_service_cls.assert_called_once_with() + + mock_service_cls.assert_called_once_with( + repository=mock_repo_cls.return_value, + excel_generator=mock_excel_generator_cls.return_value, + ) - mock_service.assert_called_once() service_instance.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, @@ -115,54 +109,64 @@ def test_lambda_handler_calls_service_and_dependencies( mock_logger.info.assert_any_call("Report orchestration lambda invoked") + assert result["report_date"] == "2026-01-02" + assert result["bucket"] == "test-report-bucket" + assert result["prefix"] == "Report-Orchestration/2026-01-02/" + assert set(result["keys"]) == { + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/B67890.xlsx", + } + def test_lambda_handler_calls_window_function( - mock_service, + mock_service_cls, mock_window, mock_report_date, - mock_lambda_client, - mock_s3_service, + mock_s3_service_cls, ): - mock_service.return_value.process_reporting_window.return_value = {} + mock_service_cls.return_value.process_reporting_window.return_value = {} lambda_handler(event={}, context=FakeContext()) mock_window.assert_called_once() -def test_lambda_handler_returns_early_when_no_reports_generated( - mock_service, +def test_lambda_handler_returns_empty_keys_when_no_reports_generated( + mock_service_cls, mock_logger, - mock_s3_service, - mock_lambda_client, + mock_s3_service_cls, mock_window, mock_report_date, ): - mock_service.return_value.process_reporting_window.return_value = {} + mock_service_cls.return_value.process_reporting_window.return_value = {} result = lambda_handler(event={}, context=FakeContext()) - assert result is None - mock_s3_service.return_value.upload_file_with_extra_args.assert_not_called() - mock_lambda_client.return_value.invoke.assert_not_called() + assert result == { + "report_date": "2026-01-02", + "bucket": "test-report-bucket", + "prefix": "Report-Orchestration/2026-01-02/", + "keys": [], + } + + mock_s3_service_cls.return_value.upload_file_with_extra_args.assert_not_called() mock_logger.info.assert_any_call("No reports generated; exiting") -def test_lambda_handler_uploads_each_report_to_s3( - mock_service, - mock_s3_service, +def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( + mock_service_cls, + mock_s3_service_cls, mock_report_date, - mock_lambda_client, mock_window, ): - mock_service.return_value.process_reporting_window.return_value = { + mock_service_cls.return_value.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "UNKNOWN": "/tmp/UNKNOWN.xlsx", } - lambda_handler(event={}, context=FakeContext()) + result = lambda_handler(event={}, context=FakeContext()) - s3_instance = mock_s3_service.return_value + s3_instance = mock_s3_service_cls.return_value assert s3_instance.upload_file_with_extra_args.call_count == 2 s3_instance.upload_file_with_extra_args.assert_any_call( @@ -179,35 +183,20 @@ def test_lambda_handler_uploads_each_report_to_s3( extra_args={"ServerSideEncryption": "aws:kms"}, ) + assert result["keys"] == [ + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/UNKNOWN.xlsx", + ] -def test_lambda_handler_invokes_distribution_lambda_once( - mock_service, - mock_report_date, - mock_lambda_client, - mock_window, - mock_s3_service, -): - mock_service.return_value.process_reporting_window.return_value = { - "A12345": "/tmp/A12345.xlsx", - } - - lambda_handler(event={}, context=FakeContext()) - - client = mock_lambda_client.return_value - client.invoke.assert_called_once() - - kwargs = client.invoke.call_args.kwargs - assert kwargs["FunctionName"] == "test-distribution-lambda" - assert kwargs["InvocationType"] == "Event" - - payload = kwargs["Payload"] - assert '"report_date": "2026-01-02"' in payload - -def test_lambda_handler_returns_error_when_required_env_missing(): +def test_lambda_handler_returns_error_when_required_env_missing(mocker): + mocker.patch.dict(os.environ, {"BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable"}, clear=False) os.environ.pop("REPORT_BUCKET_NAME", None) - result = lambda_handler(event={}, context=FakeContext()) + ctx = FakeContext() + ctx.aws_request_id = "test-request-id" + + result = lambda_handler(event={}, context=ctx) assert isinstance(result, dict) assert result["statusCode"] == 500 @@ -216,5 +205,5 @@ def test_lambda_handler_returns_error_when_required_env_missing(): assert body["err_code"] == "ENV_5001" assert "REPORT_BUCKET_NAME" in body["message"] - if "interaction_id" in body and body["interaction_id"] is not None: - assert body["interaction_id"] == "test-request-id" \ No newline at end of file + if body.get("interaction_id") is not None: + assert body["interaction_id"] == "test-request-id" diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index 06ca8721a8..56b5150e03 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -33,6 +33,19 @@ def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): prm_mailbox="prm@example.com", ) +def test_extract_ods_code_from_key_strips_xlsx_extension(): + assert ReportDistributionService.extract_ods_code_from_key( + "Report-Orchestration/2026-01-01/Y12345.xlsx" + ) == "Y12345" + + +def test_extract_ods_code_from_key_is_case_insensitive(): + assert ReportDistributionService.extract_ods_code_from_key("a/b/C789.XLSX") == "C789" + + +def test_extract_ods_code_from_key_keeps_non_xlsx_filename(): + assert ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") == "report.csv" + def test_list_xlsx_keys_filters_only_xlsx(service, mocker): paginator = mocker.Mock() paginator.paginate.return_value = [ @@ -74,59 +87,16 @@ def test_list_xlsx_keys_returns_empty_when_no_objects(service, mocker): assert keys == [] -def test_distribute_reports_for_date_when_no_keys_returns_zeroes(service, mocker): - mocker.patch.object(service, "list_xlsx_keys", return_value=[]) - mocked_process = mocker.patch.object(service, "process_one_report") - - result = service.distribute_reports_for_date("2026-01-01") - - assert result == {"succeeded": 0, "failed": 0} - mocked_process.assert_not_called() - - -def test_distribute_reports_for_date_extracts_ods_from_key(service, mocker): - mocker.patch.object( - service, - "list_xlsx_keys", - return_value=["Report-Orchestration/2026-01-01/some/path/Y12345.xlsx"], - ) - mocked_process = mocker.patch.object(service, "process_one_report", return_value=None) - - result = service.distribute_reports_for_date("2026-01-01") - - assert result == {"succeeded": 1, "failed": 0} - mocked_process.assert_called_once_with( - ods_code="Y12345", - key="Report-Orchestration/2026-01-01/some/path/Y12345.xlsx", - ) - -def test_distribute_reports_for_date_counts_success_and_failure(service, mocker): - mocker.patch.object( - service, - "list_xlsx_keys", - return_value=[ - "Report-Orchestration/2026-01-01/Y12345.xlsx", - "Report-Orchestration/2026-01-01/A99999.xlsx", - ], - ) - - mocked_process = mocker.patch.object(service, "process_one_report") - mocked_process.side_effect = [None, RuntimeError("boom")] +def test_list_xlsx_keys_skips_pages_without_contents(service, mocker): + paginator = mocker.Mock() + paginator.paginate.return_value = [{}, {"Contents": [{"Key": "p/X.xlsx"}]}] + service._s3_client.get_paginator.return_value = paginator - result = service.distribute_reports_for_date("2026-01-01") + keys = service.list_xlsx_keys(prefix="p/") - assert result == {"succeeded": 1, "failed": 1} - assert mocked_process.call_count == 2 + assert keys == ["p/X.xlsx"] - mocked_process.assert_any_call( - ods_code="Y12345", - key="Report-Orchestration/2026-01-01/Y12345.xlsx", - ) - mocked_process.assert_any_call( - ods_code="A99999", - key="Report-Orchestration/2026-01-01/A99999.xlsx", - ) def test_process_one_report_downloads_encrypts_and_delegates_email( service, mocker, mock_s3_service @@ -180,27 +150,39 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_service): mock_s3_service.download_file.side_effect = RuntimeError("download failed") + + td = mocker.MagicMock() + td.__enter__.return_value = "/tmp/fake_tmpdir" + td.__exit__.return_value = False mocker.patch( - "services.reporting.report_distribution_service.secrets.token_urlsafe", - return_value="pw", + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, ) - mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) + + mocked_zip = mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) with pytest.raises(RuntimeError, match="download failed"): - service.process_one_report( - ods_code="Y12345", - key="Report-Orchestration/2026-01-01/Y12345.xlsx", - ) + service.process_one_report(ods_code="Y12345", key="k.xlsx") + mocked_zip.assert_not_called() mocked_send.assert_not_called() -def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker): +def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker, mock_s3_service): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", return_value="pw", ) + + td = mocker.MagicMock() + td.__enter__.return_value = "/tmp/fake_tmpdir" + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", side_effect=RuntimeError("zip failed"), @@ -209,11 +191,44 @@ def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker): mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) with pytest.raises(RuntimeError, match="zip failed"): - service.process_one_report( - ods_code="Y12345", - key="Report-Orchestration/2026-01-01/Y12345.xlsx", - ) + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mocked_send.assert_not_called() + + +def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fails( + service, mocker, mock_s3_service +): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + side_effect=RuntimeError("secrets failed"), + ) + + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="secrets failed"): + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mock_s3_service.download_file.assert_called_once_with( + "my-bucket", + "k.xlsx", + os.path.join(fake_tmp, "Y12345.xlsx"), + ) + mocked_zip.assert_not_called() mocked_send.assert_not_called() def test_send_report_emails_with_contact_calls_email_contact(service, mock_contact_repo, mocker): @@ -258,6 +273,26 @@ def test_send_report_emails_without_contact_calls_email_prm(service, mock_contac mocked_email_contact.assert_not_called() +def test_send_report_emails_contact_lookup_exception_falls_back_to_prm(service, mock_contact_repo, mocker): + mock_contact_repo.get_contact_email.side_effect = RuntimeError("ddb down") + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + mocked_email_contact.assert_not_called() + mocked_email_prm.assert_called_once_with( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + def test_email_contact_sends_report_and_password(service, mock_email_service): service.email_contact( to_address="contact@example.com", @@ -277,6 +312,19 @@ def test_email_contact_sends_report_and_password(service, mock_email_service): ) +def test_email_contact_sends_password_even_if_report_email_fails(service, mock_email_service): + mock_email_service.send_report_email.side_effect = RuntimeError("SES down") + + with pytest.raises(RuntimeError, match="SES down"): + service.email_contact( + to_address="contact@example.com", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_password_email.assert_not_called() + + def test_email_prm_missing_contact_sends_prm_missing_contact_email(service, mock_email_service): service.email_prm_missing_contact( ods_code="X11111", From 8cd233599e712dac6ebc8d8e230ce1b1136b3e59 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 7 Jan 2026 13:02:12 +0000 Subject: [PATCH 14/37] [PRMP-1182] Improved query to dynamo, to use specific ranges --- .../reporting/reporting_dynamo_repository.py | 45 +++++-- lambdas/services/base/dynamo_service.py | 40 +++++++ .../test_reporting_dynamo_repository.py | 77 +++++++++--- .../unit/services/base/test_dynamo_service.py | 112 ++++++++++++++++++ lambdas/tests/unit/utils/test_utilities.py | 17 ++- lambdas/utils/utilities.py | 6 +- 6 files changed, 268 insertions(+), 29 deletions(-) diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 7a694edf69..484db9fd10 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,12 +1,12 @@ from typing import Dict, List -from boto3.dynamodb.conditions import Attr +from boto3.dynamodb.conditions import Key from services.base.dynamo_service import DynamoDBService from utils.audit_logging_setup import LoggingService +from utils.utilities import utc_date_string logger = LoggingService(__name__) - class ReportingDynamoRepository: def __init__(self, table_name: str): self.table_name = table_name @@ -17,19 +17,42 @@ def get_records_for_time_window( start_timestamp: int, end_timestamp: int, ) -> List[Dict]: + timestamp_index_name = "TimestampIndex" logger.info( - f"Querying reporting table for window, " - f"table_name: {self.table_name}, " - f"start_timestamp: {start_timestamp}, " - f"end_timestamp: {end_timestamp}", + "Querying reporting table via TimestampIndex for window, " + f"table_name={self.table_name}, start_timestamp={start_timestamp}, end_timestamp={end_timestamp}", ) - filter_expression = Attr("Timestamp").between( - start_timestamp, - end_timestamp, + start_date_string = utc_date_string(start_timestamp) + end_date_string = utc_date_string(end_timestamp) + + if start_date_string == end_date_string: + key_condition_expression = ( + Key("Date").eq(start_date_string) + & Key("Timestamp").between(start_timestamp, end_timestamp) + ) + return self.dynamo_service.query_by_key_condition_expression( + table_name=self.table_name, + index_name=timestamp_index_name, + key_condition_expression=key_condition_expression, + ) + + start_day_key_condition = ( + Key("Date").eq(start_date_string) & Key("Timestamp").gte(start_timestamp) + ) + end_day_key_condition = ( + Key("Date").eq(end_date_string) & Key("Timestamp").lte(end_timestamp) ) - return self.dynamo_service.scan_whole_table( + start_day_items = self.dynamo_service.query_by_key_condition_expression( table_name=self.table_name, - filter_expression=filter_expression, + index_name=timestamp_index_name, + key_condition_expression=start_day_key_condition, ) + end_day_items = self.dynamo_service.query_by_key_condition_expression( + table_name=self.table_name, + index_name=timestamp_index_name, + key_condition_expression=end_day_key_condition, + ) + + return start_day_items + end_day_items diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index ec8de0b908..c97418b5a5 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -429,3 +429,43 @@ def build_update_transaction_item( }, } } + + def query_by_key_condition_expression( + self, + table_name: str, + key_condition_expression: ConditionBase, + index_name: str | None = None, + query_filter: Attr | ConditionBase | None = None, + limit: int | None = None, + ) -> list[dict]: + table = self.get_table(table_name) + + collected_items: list[dict] = [] + exclusive_start_key: dict | None = None + + while True: + query_params: dict = {"KeyConditionExpression": key_condition_expression} + + if index_name: + query_params["IndexName"] = index_name + if query_filter: + query_params["FilterExpression"] = query_filter + if exclusive_start_key: + query_params["ExclusiveStartKey"] = exclusive_start_key + if limit: + query_params["Limit"] = limit + + try: + response = table.query(**query_params) + except ClientError as exc: + logger.error(str(exc), {"Result": f"Unable to query table: {table_name}"}) + raise + + collected_items.extend(response.get("Items", [])) + + exclusive_start_key = response.get("LastEvaluatedKey") + if not exclusive_start_key: + break + + return collected_items + diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index df3c553044..f6a9ec9568 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -6,12 +6,12 @@ @pytest.fixture def mock_dynamo_service(mocker): - mock_service = mocker.patch( + mock_service_class = mocker.patch( "repositories.reporting.reporting_dynamo_repository.DynamoDBService" ) - instance = mock_service.return_value - instance.scan_whole_table = MagicMock() - return instance + mock_instance = mock_service_class.return_value + mock_instance.query_by_key_condition_expression = MagicMock() + return mock_instance @pytest.fixture @@ -19,23 +19,68 @@ def reporting_repo(mock_dynamo_service): return ReportingDynamoRepository(table_name="TestTable") -def test_get_records_for_time_window_calls_scan(mock_dynamo_service, reporting_repo): - mock_dynamo_service.scan_whole_table.return_value = [] +def test_get_records_for_time_window_same_date_queries_once( + mocker, mock_dynamo_service, reporting_repo +): + mock_utc_date_string = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date_string" + ) + mock_utc_date_string.side_effect = ["2026-01-07", "2026-01-07"] + + mock_dynamo_service.query_by_key_condition_expression.return_value = [{"ID": "one"}] + + result = reporting_repo.get_records_for_time_window(100, 200) + + assert result == [{"ID": "one"}] + mock_dynamo_service.query_by_key_condition_expression.assert_called_once() + + call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args.kwargs + assert call_kwargs["table_name"] == "TestTable" + assert call_kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in call_kwargs + + +def test_get_records_for_time_window_different_dates_queries_twice( + mocker, mock_dynamo_service, reporting_repo +): + mock_utc_date_string = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date_string" + ) + mock_utc_date_string.side_effect = ["2026-01-06", "2026-01-07"] + + mock_dynamo_service.query_by_key_condition_expression.side_effect = [ + [{"ID": "start-day"}], + [{"ID": "end-day"}], + ] + + result = reporting_repo.get_records_for_time_window(100, 200) - reporting_repo.get_records_for_time_window(100, 200) + assert result == [{"ID": "start-day"}, {"ID": "end-day"}] + assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 - mock_dynamo_service.scan_whole_table.assert_called_once() - assert "filter_expression" in mock_dynamo_service.scan_whole_table.call_args.kwargs + first_call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args_list[0].kwargs + second_call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args_list[1].kwargs + assert first_call_kwargs["table_name"] == "TestTable" + assert first_call_kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in first_call_kwargs -def test_get_records_for_time_window_returns_empty_list( - mock_dynamo_service, reporting_repo + assert second_call_kwargs["table_name"] == "TestTable" + assert second_call_kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in second_call_kwargs + + +def test_get_records_for_time_window_returns_empty_list_when_no_items( + mocker, mock_dynamo_service, reporting_repo ): - start_ts = 0 - end_ts = 50 - mock_dynamo_service.scan_whole_table.return_value = [] + mock_utc_date_string = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date_string" + ) + mock_utc_date_string.side_effect = ["2026-01-06", "2026-01-07"] + + mock_dynamo_service.query_by_key_condition_expression.side_effect = [[], []] - result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + result = reporting_repo.get_records_for_time_window(100, 200) assert result == [] - mock_dynamo_service.scan_whole_table.assert_called_once() + assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index 80fa1ea911..b0f7f06396 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -1196,3 +1196,115 @@ def test_build_key_condition_non_matching_list_lengths( mock_service.build_key_condition( search_key=search_key, search_condition=search_condition ) + +def test_query_by_key_condition_expression_single_page_returns_items( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.return_value = { + "Items": [{"ID": "item-1"}, {"ID": "item-2"}] + } + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert result == [{"ID": "item-1"}, {"ID": "item-2"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.query.assert_called_once_with( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ) + + +def test_query_by_key_condition_expression_handles_pagination( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.side_effect = [ + { + "Items": [{"ID": "item-1"}], + "LastEvaluatedKey": {"ID": "page-2"}, + }, + { + "Items": [{"ID": "item-2"}], + "LastEvaluatedKey": {"ID": "page-3"}, + }, + { + "Items": [{"ID": "item-3"}], + }, + ] + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert result == [{"ID": "item-1"}, {"ID": "item-2"}, {"ID": "item-3"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + + expected_calls = [ + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ), + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ExclusiveStartKey={"ID": "page-2"}, + ), + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ExclusiveStartKey={"ID": "page-3"}, + ), + ] + mock_table.return_value.query.assert_has_calls(expected_calls) + + +def test_query_by_key_condition_expression_passes_filter_and_limit( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").lte(1767780025) + filter_expression = Attr("UploadStatus").eq("complete") + + mock_table.return_value.query.return_value = {"Items": [{"ID": "item-1"}]} + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + query_filter=filter_expression, + limit=25, + ) + + assert result == [{"ID": "item-1"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.query.assert_called_once_with( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + FilterExpression=filter_expression, + Limit=25, + ) + + +def test_query_by_key_condition_expression_client_error_raises_exception( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.side_effect = MOCK_CLIENT_ERROR + + with pytest.raises(ClientError) as exc_info: + mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert exc_info.value == MOCK_CLIENT_ERROR diff --git a/lambdas/tests/unit/utils/test_utilities.py b/lambdas/tests/unit/utils/test_utilities.py index 68ddb60e5b..23a3b3745b 100755 --- a/lambdas/tests/unit/utils/test_utilities.py +++ b/lambdas/tests/unit/utils/test_utilities.py @@ -12,7 +12,7 @@ get_pds_service, parse_date, redact_id_to_last_4_chars, - validate_nhs_number, + validate_nhs_number, utc_date_string, ) @@ -135,3 +135,18 @@ def test_format_cloudfront_url_valid(): def test_parse_date_returns_correct_date_for_valid_formats(input_date, expected_date): result = parse_date(input_date) assert result == expected_date + +@pytest.mark.parametrize( + "timestamp_seconds, expected_date_string", + [ + (0, "1970-01-01"), + (1704067200, "2024-01-01"), + (1767780025, "2026-01-07"), + (1704153599, "2024-01-01"), + (1704153600, "2024-01-02"), + ], +) +def test_utc_date_string_returns_correct_utc_date( + timestamp_seconds, expected_date_string +): + assert utc_date_string(timestamp_seconds) == expected_date_string diff --git a/lambdas/utils/utilities.py b/lambdas/utils/utilities.py index b83af68e4b..92d99d8f02 100755 --- a/lambdas/utils/utilities.py +++ b/lambdas/utils/utilities.py @@ -2,7 +2,7 @@ import os import re import uuid -from datetime import datetime +from datetime import datetime, timezone from urllib.parse import urlparse from inflection import camelize @@ -127,3 +127,7 @@ def parse_date(date_string: str) -> datetime | None: except ValueError: continue return None + + +def utc_date_string(timestamp_seconds: int) -> str: + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).strftime("%Y-%m-%d") \ No newline at end of file From aa1ac5cfdde5ca16c6d0fdacad0f2520e1902938 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Wed, 7 Jan 2026 14:24:55 +0000 Subject: [PATCH 15/37] [PRMP-1057] adjusted time querie to dynamo --- .../handlers/report_orchestration_handler.py | 1 + .../reporting/reporting_dynamo_repository.py | 50 +++++++++---------- lambdas/services/email_service.py | 1 + lambdas/utils/utilities.py | 14 +++++- 4 files changed, 37 insertions(+), 29 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index d1d746d14f..c8516b730d 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -61,6 +61,7 @@ def lambda_handler(event, context) -> Dict[str, Any]: window_start, window_end = calculate_reporting_window() # TODO delete line below since it is only for testing + window_start -= 24 * 60 * 60 window_end += 24 * 60 * 60 report_date = get_report_date_folder() diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 484db9fd10..86d595d87e 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,9 +1,10 @@ +from datetime import timedelta from typing import Dict, List from boto3.dynamodb.conditions import Key from services.base.dynamo_service import DynamoDBService from utils.audit_logging_setup import LoggingService -from utils.utilities import utc_date_string +from utils.utilities import utc_date_string, utc_date, utc_day_start_timestamp, utc_day_end_timestamp logger = LoggingService(__name__) @@ -23,36 +24,31 @@ def get_records_for_time_window( f"table_name={self.table_name}, start_timestamp={start_timestamp}, end_timestamp={end_timestamp}", ) - start_date_string = utc_date_string(start_timestamp) - end_date_string = utc_date_string(end_timestamp) + start_date = utc_date(start_timestamp) + end_date = utc_date(end_timestamp) - if start_date_string == end_date_string: - key_condition_expression = ( - Key("Date").eq(start_date_string) - & Key("Timestamp").between(start_timestamp, end_timestamp) + records_for_window: List[Dict] = [] + current_date = start_date + + while current_date <= end_date: + day_start_ts = utc_day_start_timestamp(current_date) + day_end_ts = utc_day_end_timestamp(current_date) + + effective_start_ts = max(start_timestamp, day_start_ts) + effective_end_ts = min(end_timestamp, day_end_ts) + + key_condition = ( + Key("Date").eq(current_date.isoformat()) + & Key("Timestamp").between(effective_start_ts, effective_end_ts) ) - return self.dynamo_service.query_by_key_condition_expression( + + records_for_day = self.dynamo_service.query_by_key_condition_expression( table_name=self.table_name, index_name=timestamp_index_name, - key_condition_expression=key_condition_expression, + key_condition_expression=key_condition, ) - start_day_key_condition = ( - Key("Date").eq(start_date_string) & Key("Timestamp").gte(start_timestamp) - ) - end_day_key_condition = ( - Key("Date").eq(end_date_string) & Key("Timestamp").lte(end_timestamp) - ) - - start_day_items = self.dynamo_service.query_by_key_condition_expression( - table_name=self.table_name, - index_name=timestamp_index_name, - key_condition_expression=start_day_key_condition, - ) - end_day_items = self.dynamo_service.query_by_key_condition_expression( - table_name=self.table_name, - index_name=timestamp_index_name, - key_condition_expression=end_day_key_condition, - ) + records_for_window.extend(records_for_day) + current_date += timedelta(days=1) - return start_day_items + end_day_items + return records_for_window \ No newline at end of file diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index fb12e223dd..b946c0259d 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -49,6 +49,7 @@ def send_email( def _send_raw(self, msg: MIMEMultipart, to_address: str): logger.info(f"Sending SES raw email to {to_address}") self.ses.send_raw_email( + Source=msg["From"], RawMessage={"Data": msg.as_string()}, Destinations=[to_address], ) diff --git a/lambdas/utils/utilities.py b/lambdas/utils/utilities.py index 92d99d8f02..bb2de83eab 100755 --- a/lambdas/utils/utilities.py +++ b/lambdas/utils/utilities.py @@ -2,7 +2,7 @@ import os import re import uuid -from datetime import datetime, timezone +from datetime import datetime, timezone, date, time from urllib.parse import urlparse from inflection import camelize @@ -130,4 +130,14 @@ def parse_date(date_string: str) -> datetime | None: def utc_date_string(timestamp_seconds: int) -> str: - return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).strftime("%Y-%m-%d") \ No newline at end of file + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).strftime("%Y-%m-%d") + +def utc_date(timestamp_seconds: int) -> date: + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).date() + +def utc_day_start_timestamp(day: date) -> int: + return int( + datetime.combine(day, time.min, tzinfo=timezone.utc).timestamp() + ) +def utc_day_end_timestamp(day: date) -> int: + return utc_day_start_timestamp(day) + 24 * 60 * 60 - 1 \ No newline at end of file From 5d2a08900c0e463943e3d35b00f2bdcf5ca31bb4 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 8 Jan 2026 08:54:51 +0000 Subject: [PATCH 16/37] [PRMP-1057] added logging, removed 1 todo --- lambdas/services/email_service.py | 22 +++++++++++++------ .../reporting/report_distribution_service.py | 4 ++-- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index b946c0259d..8a11a9bd88 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -26,7 +26,7 @@ def send_email( body_text: str, from_address: str, attachments: Optional[Iterable[str]] = None, - ): + )->MIMEMultipart: msg = MIMEMultipart() msg["Subject"] = subject msg["To"] = to_address @@ -43,17 +43,25 @@ def send_email( filename=attachment_path.split("/")[-1], ) msg.attach(part) + logger.info( + f"Sending email: from={from_address!r}, to={to_address!r}, subject={subject!r}, " + f"attachments={len(list(attachments or []))}" + ) + return self._send_raw(msg, to_address) - self._send_raw(msg, to_address) - - def _send_raw(self, msg: MIMEMultipart, to_address: str): - logger.info(f"Sending SES raw email to {to_address}") - self.ses.send_raw_email( - Source=msg["From"], + def _send_raw(self, msg: MIMEMultipart, to_address: str)->MIMEMultipart: + subject = msg.get("Subject", "") + from_address = msg.get("From", "") + logger.info(f"Sending SES raw email: subject={subject!r}, from={from_address!r}, to={to_address!r}") + resp = self.ses.send_raw_email( + Source=from_address, RawMessage={"Data": msg.as_string()}, Destinations=[to_address], ) + logger.info(f"SES accepted email: subject={subject!r}, message_id={resp.get('MessageId')}") + return resp + def send_report_email( self, *, diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index bbb380faad..f64c632c67 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -96,13 +96,13 @@ def send_report_emails(self, *, ods_code: str, attachment_path: str, password: s ) def email_contact(self, *, to_address: str, attachment_path: str, password: str) -> None: - # TODO delete next line since it is only for testing - to_address = "" + logger.info(f"Sending report email to {to_address}") self.email_service.send_report_email( to_address=to_address, from_address=self.from_address, attachment_path=attachment_path, ) + logger.info(f"Sending password email to {to_address}") self.email_service.send_password_email( to_address=to_address, from_address=self.from_address, From 68f7d3b2ea74cd40d0e4762c76838ac94b409ba1 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 9 Jan 2026 10:37:32 +0000 Subject: [PATCH 17/37] [PRMP-1057] fixed how items are grabbed from contact report table --- lambdas/repositories/reporting/report_contact_repository.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lambdas/repositories/reporting/report_contact_repository.py b/lambdas/repositories/reporting/report_contact_repository.py index 9399e222ee..84bd06cab1 100644 --- a/lambdas/repositories/reporting/report_contact_repository.py +++ b/lambdas/repositories/reporting/report_contact_repository.py @@ -7,10 +7,11 @@ def __init__(self, table_name: str): self.dynamo = DynamoDBService() def get_contact_email(self, ods_code: str) -> str | None: - item = self.dynamo.get_item( + resp = self.dynamo.get_item( table_name=self.table_name, key={"OdsCode": ods_code}, ) + item = (resp or {}).get("Item") if not item: return None return item.get("Email") From cb0fd89c68d3b86247f8bd632d62c372669ae26c Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 9 Jan 2026 10:50:22 +0000 Subject: [PATCH 18/37] [PRMP-1057] fixed tests --- .../handlers/report_orchestration_handler.py | 4 -- .../test_report_contact_repository.py | 12 ++++-- .../test_reporting_dynamo_repository.py | 42 +++++-------------- 3 files changed, 18 insertions(+), 40 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index c8516b730d..be9ea77779 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -60,10 +60,6 @@ def lambda_handler(event, context) -> Dict[str, Any]: ) window_start, window_end = calculate_reporting_window() - # TODO delete line below since it is only for testing - window_start -= 24 * 60 * 60 - window_end += 24 * 60 * 60 - report_date = get_report_date_folder() prefix = f"Report-Orchestration/{report_date}/" diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py index 5cae95f0f2..d534267321 100644 --- a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -20,8 +20,10 @@ def repo(mock_dynamo): def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): mock_dynamo.get_item.return_value = { - "OdsCode": "Y12345", - "Email": "contact@example.com", + "Item": { + "OdsCode": "Y12345", + "Email": "contact@example.com", + } } result = repo.get_contact_email("Y12345") @@ -34,7 +36,7 @@ def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): def test_get_contact_email_returns_none_when_item_missing(repo, mock_dynamo): - mock_dynamo.get_item.return_value = None + mock_dynamo.get_item.return_value = {} # or None result = repo.get_contact_email("Y12345") @@ -47,7 +49,9 @@ def test_get_contact_email_returns_none_when_item_missing(repo, mock_dynamo): def test_get_contact_email_returns_none_when_email_missing(repo, mock_dynamo): mock_dynamo.get_item.return_value = { - "OdsCode": "Y12345", + "Item": { + "OdsCode": "Y12345", + } } result = repo.get_contact_email("Y12345") diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index f6a9ec9568..5697eb112d 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -1,3 +1,4 @@ +from datetime import date from unittest.mock import MagicMock import pytest @@ -19,13 +20,9 @@ def reporting_repo(mock_dynamo_service): return ReportingDynamoRepository(table_name="TestTable") -def test_get_records_for_time_window_same_date_queries_once( - mocker, mock_dynamo_service, reporting_repo -): - mock_utc_date_string = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date_string" - ) - mock_utc_date_string.side_effect = ["2026-01-07", "2026-01-07"] +def test_get_records_for_time_window_same_date_queries_once(mocker, mock_dynamo_service, reporting_repo): + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [date(2026, 1, 7), date(2026, 1, 7)] mock_dynamo_service.query_by_key_condition_expression.return_value = [{"ID": "one"}] @@ -40,13 +37,9 @@ def test_get_records_for_time_window_same_date_queries_once( assert "key_condition_expression" in call_kwargs -def test_get_records_for_time_window_different_dates_queries_twice( - mocker, mock_dynamo_service, reporting_repo -): - mock_utc_date_string = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date_string" - ) - mock_utc_date_string.side_effect = ["2026-01-06", "2026-01-07"] +def test_get_records_for_time_window_different_dates_queries_twice(mocker, mock_dynamo_service, reporting_repo): + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] mock_dynamo_service.query_by_key_condition_expression.side_effect = [ [{"ID": "start-day"}], @@ -58,25 +51,10 @@ def test_get_records_for_time_window_different_dates_queries_twice( assert result == [{"ID": "start-day"}, {"ID": "end-day"}] assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 - first_call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args_list[0].kwargs - second_call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args_list[1].kwargs - - assert first_call_kwargs["table_name"] == "TestTable" - assert first_call_kwargs["index_name"] == "TimestampIndex" - assert "key_condition_expression" in first_call_kwargs - assert second_call_kwargs["table_name"] == "TestTable" - assert second_call_kwargs["index_name"] == "TimestampIndex" - assert "key_condition_expression" in second_call_kwargs - - -def test_get_records_for_time_window_returns_empty_list_when_no_items( - mocker, mock_dynamo_service, reporting_repo -): - mock_utc_date_string = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date_string" - ) - mock_utc_date_string.side_effect = ["2026-01-06", "2026-01-07"] +def test_get_records_for_time_window_returns_empty_list_when_no_items(mocker, mock_dynamo_service, reporting_repo): + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] mock_dynamo_service.query_by_key_condition_expression.side_effect = [[], []] From 5035fed5b4f07e83d8e280d8c325a5a75e983b30 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 9 Jan 2026 14:05:37 +0000 Subject: [PATCH 19/37] [PRMP-1057] removed commented line --- lambdas/handlers/report_distribution_handler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index d65314b781..30c8547af8 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -54,7 +54,6 @@ def lambda_handler(event, context) -> Dict[str, Any]: logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") return {"bucket": bucket, "prefix": prefix, "keys": keys} - # action == "process_one" key = event["key"] ods_code = service.extract_ods_code_from_key(key) service.process_one_report(ods_code=ods_code, key=key) From 0264411fc294e2b30facc4bbf12137a589443d53 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 15 Jan 2026 10:45:01 +0000 Subject: [PATCH 20/37] [PRMP-1057] removed redundancy in error message --- lambdas/services/reporting/report_distribution_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index f64c632c67..9548ddbca0 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -75,7 +75,7 @@ def send_report_emails(self, *, ods_code: str, attachment_path: str, password: s contact_email = self.contact_repo.get_contact_email(ods_code) except Exception as e: logger.exception( - f"Contact lookup failed for ODS={ods_code}; falling back to PRM. Error: {e}" + f"Contact lookup failed for ODS={ods_code}; falling back to PRM." ) contact_email = None From 8f74fe338d0e073a1601dd11708f701c4b0ecdd9 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 16 Jan 2026 15:28:22 +0000 Subject: [PATCH 21/37] [PRMP-1057] fixed comments --- .../handlers/report_orchestration_handler.py | 98 +++++++++++-------- lambdas/utils/utilities.py | 1 + 2 files changed, 59 insertions(+), 40 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index be9ea77779..d5367068df 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,6 +1,6 @@ import os from datetime import datetime, timedelta, timezone -from typing import Any, Dict +from typing import Any, Dict, Tuple from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.base.s3_service import S3Service @@ -15,7 +15,7 @@ logger = LoggingService(__name__) -def calculate_reporting_window(): +def calculate_reporting_window() -> Tuple[int, int]: now = datetime.now(timezone.utc) today_7am = now.replace(hour=7, minute=0, second=0, microsecond=0) @@ -23,7 +23,6 @@ def calculate_reporting_window(): today_7am -= timedelta(days=1) yesterday_7am = today_7am - timedelta(days=1) - return int(yesterday_7am.timestamp()), int(today_7am.timestamp()) @@ -35,6 +34,50 @@ def build_s3_key(ods_code: str, report_date: str) -> str: return f"Report-Orchestration/{report_date}/{ods_code}.xlsx" +def get_config() -> Tuple[str, str]: + return os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"], os.environ["REPORT_BUCKET_NAME"] + + +def build_services(table_name: str) -> Tuple[ReportOrchestrationService, S3Service]: + repository = ReportingDynamoRepository(table_name) + excel_generator = ExcelReportGenerator() + orchestration_service = ReportOrchestrationService( + repository=repository, + excel_generator=excel_generator, + ) + return orchestration_service, S3Service() + + +def upload_generated_reports( + s3_service: S3Service, + bucket: str, + report_date: str, + generated_files: Dict[str, str], +) -> list[str]: + keys: list[str] = [] + for ods_code, local_path in generated_files.items(): + key = build_s3_key(ods_code, report_date) + s3_service.upload_file_with_extra_args( + file_name=local_path, + s3_bucket_name=bucket, + file_key=key, + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + keys.append(key) + logger.info(f"Uploaded report for ODS={ods_code} to s3://{bucket}/{key}") + return keys + + +def build_response(report_date: str, bucket: str, keys: list[str]) -> Dict[str, Any]: + prefix = f"Report-Orchestration/{report_date}/" + return { + "report_date": report_date, + "bucket": bucket, + "prefix": prefix, + "keys": keys, + } + + @ensure_environment_variables( names=[ "BULK_UPLOAD_REPORT_TABLE_NAME", @@ -47,52 +90,27 @@ def build_s3_key(ods_code: str, report_date: str) -> str: def lambda_handler(event, context) -> Dict[str, Any]: logger.info("Report orchestration lambda invoked") - table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] - report_bucket = os.environ["REPORT_BUCKET_NAME"] - - repository = ReportingDynamoRepository(table_name) - excel_generator = ExcelReportGenerator() - s3_service = S3Service() - - service = ReportOrchestrationService( - repository=repository, - excel_generator=excel_generator, - ) + table_name, report_bucket = get_config() + orchestration_service, s3_service = build_services(table_name) window_start, window_end = calculate_reporting_window() report_date = get_report_date_folder() - prefix = f"Report-Orchestration/{report_date}/" - generated_files = service.process_reporting_window( + generated_files = orchestration_service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, ) if not generated_files: logger.info("No reports generated; exiting") - return { - "report_date": report_date, - "bucket": report_bucket, - "prefix": prefix, - "keys": [], - } - - keys = [] - for ods_code, local_path in generated_files.items(): - key = build_s3_key(ods_code, report_date) - s3_service.upload_file_with_extra_args( - file_name=local_path, - s3_bucket_name=report_bucket, - file_key=key, - extra_args={"ServerSideEncryption": "aws:kms"}, - ) - keys.append(key) - logger.info(f"Uploaded report for ODS={ods_code} to s3://{report_bucket}/{key}") + return build_response(report_date=report_date, bucket=report_bucket, keys=[]) + + keys = upload_generated_reports( + s3_service=s3_service, + bucket=report_bucket, + report_date=report_date, + generated_files=generated_files, + ) logger.info(f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}") - return { - "report_date": report_date, - "bucket": report_bucket, - "prefix": prefix, - "keys": keys, - } + return build_response(report_date=report_date, bucket=report_bucket, keys=keys) diff --git a/lambdas/utils/utilities.py b/lambdas/utils/utilities.py index bb2de83eab..f23720e9b1 100755 --- a/lambdas/utils/utilities.py +++ b/lambdas/utils/utilities.py @@ -139,5 +139,6 @@ def utc_day_start_timestamp(day: date) -> int: return int( datetime.combine(day, time.min, tzinfo=timezone.utc).timestamp() ) + def utc_day_end_timestamp(day: date) -> int: return utc_day_start_timestamp(day) + 24 * 60 * 60 - 1 \ No newline at end of file From 509f3e9b96e39a353d261ad7ab1e3f3cf3fbc0d1 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 16 Jan 2026 15:34:35 +0000 Subject: [PATCH 22/37] [PRMP-1057] updated tests --- lambdas/services/email_service.py | 6 +- .../test_report_orchestration_handler.py | 99 +++++++------------ 2 files changed, 38 insertions(+), 67 deletions(-) diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py index 8a11a9bd88..e72bbcfde4 100644 --- a/lambdas/services/email_service.py +++ b/lambdas/services/email_service.py @@ -2,7 +2,7 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.mime.application import MIMEApplication -from typing import Iterable, Optional +from typing import Iterable, Optional, Dict, Any from utils.audit_logging_setup import LoggingService @@ -26,7 +26,7 @@ def send_email( body_text: str, from_address: str, attachments: Optional[Iterable[str]] = None, - )->MIMEMultipart: + )->Dict[str, Any]: msg = MIMEMultipart() msg["Subject"] = subject msg["To"] = to_address @@ -49,7 +49,7 @@ def send_email( ) return self._send_raw(msg, to_address) - def _send_raw(self, msg: MIMEMultipart, to_address: str)->MIMEMultipart: + def _send_raw(self, msg: MIMEMultipart, to_address: str)->Dict[str, Any]: subject = msg.get("Subject", "") from_address = msg.get("From", "") logger.info(f"Sending SES raw email: subject={subject!r}, from={from_address!r}, to={to_address!r}") diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 10364c5e42..6458a2f8a6 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -28,38 +28,6 @@ def mock_logger(mocker): return mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) -@pytest.fixture -def mock_repo_cls(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.ReportingDynamoRepository", - autospec=True, - ) - - -@pytest.fixture -def mock_excel_generator_cls(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.ExcelReportGenerator", - autospec=True, - ) - - -@pytest.fixture -def mock_s3_service_cls(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.S3Service", - autospec=True, - ) - - -@pytest.fixture -def mock_service_cls(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.ReportOrchestrationService", - autospec=True, - ) - - @pytest.fixture def mock_window(mocker): return mocker.patch( @@ -76,38 +44,40 @@ def mock_report_date(mocker): ) -def test_lambda_handler_wires_dependencies_and_calls_service( +@pytest.fixture +def mock_build_services(mocker): + orchestration_service = MagicMock() + s3_service = MagicMock() + patcher = mocker.patch( + "handlers.report_orchestration_handler.build_services", + return_value=(orchestration_service, s3_service), + ) + return patcher, orchestration_service, s3_service + + +def test_lambda_handler_calls_service_and_returns_expected_response( mock_logger, - mock_repo_cls, - mock_excel_generator_cls, - mock_s3_service_cls, - mock_service_cls, + mock_build_services, mock_window, mock_report_date, ): - service_instance = mock_service_cls.return_value - service_instance.process_reporting_window.return_value = { + build_services_patcher, orchestration_service, s3_service = mock_build_services + + orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "B67890": "/tmp/B67890.xlsx", } result = lambda_handler(event={}, context=FakeContext()) - mock_repo_cls.assert_called_once_with("TestTable") - mock_excel_generator_cls.assert_called_once_with() - mock_s3_service_cls.assert_called_once_with() - - mock_service_cls.assert_called_once_with( - repository=mock_repo_cls.return_value, - excel_generator=mock_excel_generator_cls.return_value, - ) + build_services_patcher.assert_called_once_with("TestTable") - service_instance.process_reporting_window.assert_called_once_with( + orchestration_service.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, ) - mock_logger.info.assert_any_call("Report orchestration lambda invoked") + assert s3_service.upload_file_with_extra_args.call_count == 2 assert result["report_date"] == "2026-01-02" assert result["bucket"] == "test-report-bucket" @@ -117,14 +87,16 @@ def test_lambda_handler_wires_dependencies_and_calls_service( "Report-Orchestration/2026-01-02/B67890.xlsx", } + mock_logger.info.assert_any_call("Report orchestration lambda invoked") + def test_lambda_handler_calls_window_function( - mock_service_cls, + mock_build_services, mock_window, mock_report_date, - mock_s3_service_cls, ): - mock_service_cls.return_value.process_reporting_window.return_value = {} + _, orchestration_service, _ = mock_build_services + orchestration_service.process_reporting_window.return_value = {} lambda_handler(event={}, context=FakeContext()) @@ -132,13 +104,13 @@ def test_lambda_handler_calls_window_function( def test_lambda_handler_returns_empty_keys_when_no_reports_generated( - mock_service_cls, + mock_build_services, mock_logger, - mock_s3_service_cls, mock_window, mock_report_date, ): - mock_service_cls.return_value.process_reporting_window.return_value = {} + _, orchestration_service, s3_service = mock_build_services + orchestration_service.process_reporting_window.return_value = {} result = lambda_handler(event={}, context=FakeContext()) @@ -149,34 +121,33 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( "keys": [], } - mock_s3_service_cls.return_value.upload_file_with_extra_args.assert_not_called() + s3_service.upload_file_with_extra_args.assert_not_called() mock_logger.info.assert_any_call("No reports generated; exiting") def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( - mock_service_cls, - mock_s3_service_cls, - mock_report_date, + mock_build_services, mock_window, + mock_report_date, ): - mock_service_cls.return_value.process_reporting_window.return_value = { + _, orchestration_service, s3_service = mock_build_services + orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "UNKNOWN": "/tmp/UNKNOWN.xlsx", } result = lambda_handler(event={}, context=FakeContext()) - s3_instance = mock_s3_service_cls.return_value - assert s3_instance.upload_file_with_extra_args.call_count == 2 + assert s3_service.upload_file_with_extra_args.call_count == 2 - s3_instance.upload_file_with_extra_args.assert_any_call( + s3_service.upload_file_with_extra_args.assert_any_call( file_name="/tmp/A12345.xlsx", s3_bucket_name="test-report-bucket", file_key="Report-Orchestration/2026-01-02/A12345.xlsx", extra_args={"ServerSideEncryption": "aws:kms"}, ) - s3_instance.upload_file_with_extra_args.assert_any_call( + s3_service.upload_file_with_extra_args.assert_any_call( file_name="/tmp/UNKNOWN.xlsx", s3_bucket_name="test-report-bucket", file_key="Report-Orchestration/2026-01-02/UNKNOWN.xlsx", From 0cf7002eabaf0a3521803783b8fd5ab1455ffb8a Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Mon, 19 Jan 2026 13:51:37 +0000 Subject: [PATCH 23/37] [PRMP-1057] updated lambda names --- .github/workflows/base-lambdas-reusable-deploy-all.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index 5128113a49..69df066397 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -806,7 +806,7 @@ jobs: build_branch: ${{ inputs.build_branch }} sandbox: ${{ inputs.sandbox }} lambda_handler_name: report_orchestration_handler - lambda_aws_name: reportOrchestration + lambda_aws_name: ReportOrchestration lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} @@ -820,7 +820,7 @@ jobs: build_branch: ${{ inputs.build_branch }} sandbox: ${{ inputs.sandbox }} lambda_handler_name: report_distribution_handler - lambda_aws_name: reportDistribution + lambda_aws_name: ReportDistribution lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} From 2948ffe2578ab08d62e9f3d181087ef7c0d98b25 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 20 Jan 2026 09:39:02 +0000 Subject: [PATCH 24/37] [PRMP-1057] updated report columns --- .../reporting/excel_report_generator_service.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index b8bef9a5cf..406e52ae7a 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -16,6 +16,7 @@ def create_report_orchestration_xlsx( logger.info( f"Creating Excel report for ODS code {ods_code} and records {len(records)}" ) + wb = Workbook() ws = wb.active ws.title = "Daily Upload Report" @@ -25,12 +26,11 @@ def create_report_orchestration_xlsx( ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) ws.append([]) - # Header row + # Header row (ID removed, NHS Number first) ws.append( [ - "ID", - "Date", "NHS Number", + "Date", "Uploader ODS", "PDS ODS", "Upload Status", @@ -42,9 +42,8 @@ def create_report_orchestration_xlsx( for record in records: ws.append( [ - record.get("ID"), - record.get("Date"), record.get("NhsNumber"), + record.get("Date"), record.get("UploaderOdsCode"), record.get("PdsOdsCode"), record.get("UploadStatus"), @@ -54,5 +53,5 @@ def create_report_orchestration_xlsx( ) wb.save(output_path) - logger.info(f"Excel report written successfully for for ods code {ods_code}") + logger.info(f"Excel report written successfully for ods code {ods_code}") return output_path From c67f895336ac74d83c0456ca3b120b8f239e6288 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 20 Jan 2026 09:39:41 +0000 Subject: [PATCH 25/37] [PRMP-1057] removed comment --- lambdas/services/reporting/excel_report_generator_service.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index 406e52ae7a..5a4a03cfc5 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -26,7 +26,6 @@ def create_report_orchestration_xlsx( ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) ws.append([]) - # Header row (ID removed, NHS Number first) ws.append( [ "NHS Number", From 895863f5e9ceab5c23bfb1337c7eb67a05444b41 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 20 Jan 2026 09:45:04 +0000 Subject: [PATCH 26/37] [PRMP-1057] updated tests --- .../test_excel_report_generator_service.py | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index f16257c0fa..2f48552855 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -1,4 +1,3 @@ - import pytest from freezegun import freeze_time from openpyxl import load_workbook @@ -47,26 +46,20 @@ def test_create_report_orchestration_xlsx_happy_path( output_path=str(output_file), ) - # File path returned assert result == str(output_file) assert output_file.exists() wb = load_workbook(output_file) ws = wb.active - # Sheet name assert ws.title == "Daily Upload Report" - - # Metadata rows assert ws["A1"].value == f"ODS Code: {ods_code}" assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" assert ws["A3"].value is None # blank row - # Header row assert [cell.value for cell in ws[4]] == [ - "ID", - "Date", "NHS Number", + "Date", "Uploader ODS", "PDS ODS", "Upload Status", @@ -74,11 +67,10 @@ def test_create_report_orchestration_xlsx_happy_path( "File Path", ] - # First data row + # First data row (no ID column) assert [cell.value for cell in ws[5]] == [ - 1, - "2025-01-01", "1234567890", + "2025-01-01", "Y12345", "A99999", "SUCCESS", @@ -86,11 +78,9 @@ def test_create_report_orchestration_xlsx_happy_path( "/path/file1.pdf", ] - # Second data row assert [cell.value for cell in ws[6]] == [ - 2, - "2025-01-02", "123456789", + "2025-01-02", "Y12345", "B88888", "FAILED", @@ -114,7 +104,6 @@ def test_create_report_orchestration_xlsx_with_no_records( wb = load_workbook(output_file) ws = wb.active - # Only metadata + header rows should exist assert ws.max_row == 4 @@ -143,12 +132,11 @@ def test_create_report_orchestration_xlsx_handles_missing_fields( row = [cell.value for cell in ws[5]] assert row == [ - 1, - None, "1234567890", None, None, None, None, None, + None, ] From 717f32cdc791af577ae458df2e6acc811c03b0bb Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Tue, 20 Jan 2026 09:56:16 +0000 Subject: [PATCH 27/37] [PRMP-1057] removed unused local variable --- lambdas/services/reporting/report_distribution_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index 9548ddbca0..a83eb5ae19 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -73,7 +73,7 @@ def process_one_report(self, *, ods_code: str, key: str) -> None: def send_report_emails(self, *, ods_code: str, attachment_path: str, password: str) -> None: try: contact_email = self.contact_repo.get_contact_email(ods_code) - except Exception as e: + except Exception: logger.exception( f"Contact lookup failed for ODS={ods_code}; falling back to PRM." ) From 4c9ed8f88f08051e697a68abe6010332429aee51 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 22 Jan 2026 10:42:34 +0000 Subject: [PATCH 28/37] [PRMP-1054] fixed sonnar --- lambdas/services/reporting/excel_report_generator_service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index b8bef9a5cf..a3579df501 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -1,4 +1,4 @@ -from datetime import datetime +from datetime import datetime, timezone from openpyxl.workbook import Workbook from utils.audit_logging_setup import LoggingService @@ -22,7 +22,7 @@ def create_report_orchestration_xlsx( # Report metadata ws.append([f"ODS Code: {ods_code}"]) - ws.append([f"Generated at (UTC): {datetime.utcnow().isoformat()}"]) + ws.append([f"Generated at (UTC): {datetime.now(timezone.utc).isoformat()}"]) ws.append([]) # Header row From b05e171e8eb377f1a577d9f507b5a545d0d8f6e2 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 22 Jan 2026 10:47:34 +0000 Subject: [PATCH 29/37] [PRMP-1054] fixed sonar --- lambdas/handlers/report_orchestration_handler.py | 2 -- lambdas/services/reporting/report_orchestration_service.py | 1 - .../tests/unit/handlers/test_report_orchestration_handler.py | 1 - .../services/reporting/test_excel_report_generator_service.py | 2 +- .../services/reporting/test_report_orchestration_service.py | 4 ++-- 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 84f6372ab8..d9501b3a8e 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -47,10 +47,8 @@ def lambda_handler(event, context): ) window_start, window_end = calculate_reporting_window() - tmp_dir = tempfile.mkdtemp() service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, - output_dir=tmp_dir, ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index a9500735b9..7d4ea48659 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -19,7 +19,6 @@ def process_reporting_window( self, window_start_ts: int, window_end_ts: int, - output_dir: str, ): records = self.repository.get_records_for_time_window( window_start_ts, diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index c9ee2ff849..1e7af584c9 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -63,7 +63,6 @@ def test_lambda_handler_calls_service( instance.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, - output_dir=mock.ANY, ) mock_logger.info.assert_any_call("Report orchestration lambda invoked") diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index f16257c0fa..6f170bafcd 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -59,7 +59,7 @@ def test_create_report_orchestration_xlsx_happy_path( # Metadata rows assert ws["A1"].value == f"ODS Code: {ods_code}" - assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" + assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00+00:00" assert ws["A3"].value is None # blank row # Header row diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c068ba0c16..c3b21a77ef 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -27,7 +27,7 @@ def test_process_reporting_window_no_records( ): mock_repository.get_records_for_time_window.return_value = [] - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + report_orchestration_service.process_reporting_window(100, 200) mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() @@ -68,7 +68,7 @@ def test_process_reporting_window_generates_reports_per_ods( report_orchestration_service, "generate_ods_report" ) - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") + report_orchestration_service.process_reporting_window(100, 200) mocked_generate.assert_any_call( "Y12345", From 792334631f93762c833d246b54f50dd9d70419f9 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 22 Jan 2026 10:57:01 +0000 Subject: [PATCH 30/37] Revert "[PRMP-1054] fixed sonar" This reverts commit b05e171e8eb377f1a577d9f507b5a545d0d8f6e2. --- lambdas/handlers/report_orchestration_handler.py | 2 ++ lambdas/services/reporting/report_orchestration_service.py | 1 + .../tests/unit/handlers/test_report_orchestration_handler.py | 1 + .../services/reporting/test_excel_report_generator_service.py | 2 +- .../services/reporting/test_report_orchestration_service.py | 4 ++-- 5 files changed, 7 insertions(+), 3 deletions(-) diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index d9501b3a8e..84f6372ab8 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -47,8 +47,10 @@ def lambda_handler(event, context): ) window_start, window_end = calculate_reporting_window() + tmp_dir = tempfile.mkdtemp() service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, + output_dir=tmp_dir, ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index 7d4ea48659..a9500735b9 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -19,6 +19,7 @@ def process_reporting_window( self, window_start_ts: int, window_end_ts: int, + output_dir: str, ): records = self.repository.get_records_for_time_window( window_start_ts, diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 1e7af584c9..c9ee2ff849 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -63,6 +63,7 @@ def test_lambda_handler_calls_service( instance.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, + output_dir=mock.ANY, ) mock_logger.info.assert_any_call("Report orchestration lambda invoked") diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index 6f170bafcd..f16257c0fa 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -59,7 +59,7 @@ def test_create_report_orchestration_xlsx_happy_path( # Metadata rows assert ws["A1"].value == f"ODS Code: {ods_code}" - assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00+00:00" + assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" assert ws["A3"].value is None # blank row # Header row diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c3b21a77ef..c068ba0c16 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -27,7 +27,7 @@ def test_process_reporting_window_no_records( ): mock_repository.get_records_for_time_window.return_value = [] - report_orchestration_service.process_reporting_window(100, 200) + report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() @@ -68,7 +68,7 @@ def test_process_reporting_window_generates_reports_per_ods( report_orchestration_service, "generate_ods_report" ) - report_orchestration_service.process_reporting_window(100, 200) + report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") mocked_generate.assert_any_call( "Y12345", From c78120eda4ac80441f1bdb5223ea7d80cc40d3e3 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 22 Jan 2026 10:59:24 +0000 Subject: [PATCH 31/37] [PRMP-1054] fixed issues --- .../services/reporting/test_excel_report_generator_service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index f16257c0fa..8b8bd484c7 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -59,7 +59,7 @@ def test_create_report_orchestration_xlsx_happy_path( # Metadata rows assert ws["A1"].value == f"ODS Code: {ods_code}" - assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00" + assert ws["A2"].value.startswith("Generated at (UTC): ") assert ws["A3"].value is None # blank row # Header row From 5e9cb3e6f5ac765ee8cee19ba0ed2802922d715f Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 30 Jan 2026 13:16:45 +0000 Subject: [PATCH 32/37] [PRMP-1057-2] fixed comments --- lambdas/enums/lambda_error.py | 4 + .../handlers/report_distribution_handler.py | 22 ++- .../handlers/report_orchestration_handler.py | 1 + lambdas/services/base/s3_service.py | 12 +- .../reporting/report_distribution_service.py | 13 +- lambdas/tests/unit/conftest.py | 14 ++ lambdas/tests/unit/handlers/conftest.py | 49 +++++++ .../test_report_distribution_handler.py | 114 ++++++++++----- .../test_report_orchestration_handler.py | 85 +++++++----- .../test_reporting_dynamo_repository.py | 130 +++++++++++++----- .../unit/services/base/test_s3_service.py | 31 +++++ .../services/reporting/test_email_service.py | 6 +- .../test_report_distribution_service.py | 109 +++++++-------- lambdas/utils/lambda_exceptions.py | 3 + lambdas/utils/zip_utils.py | 13 +- poetry.lock | 68 ++++++++- pyproject.toml | 1 + 17 files changed, 486 insertions(+), 189 deletions(-) diff --git a/lambdas/enums/lambda_error.py b/lambdas/enums/lambda_error.py index 16269d9d23..871425dade 100644 --- a/lambdas/enums/lambda_error.py +++ b/lambdas/enums/lambda_error.py @@ -729,3 +729,7 @@ def create_error_body( "message": "An internal server error occurred", "fhir_coding": FhirIssueCoding.EXCEPTION, } + InvalidAction = { + "err_code": "RD_IA", + "message": "Invalid action. Expected 'list' or 'process_one'.", + } diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index 30c8547af8..da32e9d409 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -1,6 +1,7 @@ import os from typing import Any, Dict +from enums.lambda_error import LambdaError from repositories.reporting.report_contact_repository import ReportContactRepository from services.base.s3_service import S3Service from services.email_service import EmailService @@ -10,6 +11,7 @@ from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions from utils.decorators.override_error_check import override_error_check from utils.decorators.set_audit_arg import set_request_context_for_logging +from utils.lambda_exceptions import ReportDistributionException logger = LoggingService(__name__) @@ -28,7 +30,8 @@ def lambda_handler(event, context) -> Dict[str, Any]: action = event.get("action") if action not in {"list", "process_one"}: - raise ValueError("Invalid action. Expected 'list' or 'process_one'.") + logger.error(f"Invalid action. Expected 'list' or 'process_one'.") + raise ReportDistributionException(400, LambdaError.InvalidAction) bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] contact_table = os.environ["CONTACT_TABLE_NAME"] @@ -48,14 +51,19 @@ def lambda_handler(event, context) -> Dict[str, Any]: prm_mailbox=prm_mailbox, ) + response = {"status": "ok", "bucket": bucket} + if action == "list": prefix = event["prefix"] keys = service.list_xlsx_keys(prefix=prefix) logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") - return {"bucket": bucket, "prefix": prefix, "keys": keys} + response.update({"prefix": prefix, "keys": keys}) + else: + key = event["key"] + ods_code = service.extract_ods_code_from_key(key) + service.process_one_report(ods_code=ods_code, key=key) + logger.info(f"Process-one mode: processed ods={ods_code}, key={key}") + response.update({"key": key, "ods_code": ods_code}) + + return response - key = event["key"] - ods_code = service.extract_ods_code_from_key(key) - service.process_one_report(ods_code=ods_code, key=key) - logger.info(f"Process-one mode: processed ods={ods_code}, key={key}") - return {"status": "ok", "bucket": bucket, "key": key, "ods_code": ods_code} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index d5367068df..38a450e76f 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -71,6 +71,7 @@ def upload_generated_reports( def build_response(report_date: str, bucket: str, keys: list[str]) -> Dict[str, Any]: prefix = f"Report-Orchestration/{report_date}/" return { + "status": "ok", "report_date": report_date, "bucket": bucket, "prefix": prefix, diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index 3ee42b4e24..d3bf770fcd 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -1,7 +1,7 @@ import io from datetime import datetime, timedelta, timezone from io import BytesIO -from typing import Any, Mapping +from typing import Any, Mapping, Callable import boto3 from botocore.client import Config as BotoConfig @@ -255,3 +255,13 @@ def save_or_create_file(self, source_bucket: str, file_key: str, body: bytes): return self.client.put_object( Bucket=source_bucket, Key=file_key, Body=BytesIO(body) ) + + def list_object_keys(self, bucket_name: str, prefix: str) -> list[str]: + paginator = self.client.get_paginator("list_objects_v2") + keys: list[str] = [] + + for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix): + for obj in page.get("Contents", []): + keys.append(obj["Key"]) + + return keys \ No newline at end of file diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index a83eb5ae19..d93274e5e1 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -31,7 +31,6 @@ def __init__( self.bucket = bucket self.from_address = from_address self.prm_mailbox = prm_mailbox - self._s3_client = boto3.client("s3") @staticmethod def extract_ods_code_from_key(key: str) -> str: @@ -39,16 +38,8 @@ def extract_ods_code_from_key(key: str) -> str: return filename[:-5] if filename.lower().endswith(".xlsx") else filename def list_xlsx_keys(self, prefix: str) -> List[str]: - paginator = self._s3_client.get_paginator("list_objects_v2") - keys: List[str] = [] - - for page in paginator.paginate(Bucket=self.bucket, Prefix=prefix): - for obj in page.get("Contents", []): - key = obj["Key"] - if key.endswith(".xlsx"): - keys.append(key) - - return keys + keys = self.s3_service.list_object_keys(bucket_name=self.bucket, prefix=prefix) + return [k for k in keys if k.endswith(".xlsx")] def process_one_report(self, *, ods_code: str, key: str) -> None: with tempfile.TemporaryDirectory() as tmpdir: diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index b3afba0f82..dd3a98e4b1 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -13,6 +13,7 @@ from pydantic import ValidationError from pypdf import PdfWriter from requests import Response +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from tests.unit.helpers.data.pds.pds_patient_response import PDS_PATIENT from utils.audit_logging_setup import LoggingService @@ -416,3 +417,16 @@ def valid_pdf_stream(): @pytest.fixture def corrupt_pdf_stream(): return BytesIO(b"This is not a valid PDF content") + + +@pytest.fixture +def mock_reporting_dynamo_service(mocker): + mock_cls = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.DynamoDBService" + ) + return mock_cls.return_value + + +@pytest.fixture +def reporting_repo(mock_reporting_dynamo_service): + return ReportingDynamoRepository(table_name="TestTable") diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index 8eca1979f2..a7cd749de9 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -1,5 +1,8 @@ +import os + import pytest from enums.feature_flags import FeatureFlags +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.feature_flags_service import FeatureFlagService @@ -219,3 +222,49 @@ def mock_upload_document_iteration_3_disabled(mocker): FeatureFlags.UPLOAD_DOCUMENT_ITERATION_3_ENABLED: False } yield mock_feature_flag + + +@pytest.fixture +def required_report_distribution_env(mocker): + mocker.patch.dict( + os.environ, + { + "REPORT_BUCKET_NAME": "my-report-bucket", + "CONTACT_TABLE_NAME": "contact-table", + "PRM_MAILBOX_EMAIL": "prm@example.com", + "SES_FROM_ADDRESS": "from@example.com", + }, + clear=False, + ) + + +@pytest.fixture +def lambda_context(mocker): + ctx = mocker.Mock() + ctx.aws_request_id = "req-123" + return ctx + + +@pytest.fixture +def required_report_orchestration_env(mocker): + mocker.patch.dict( + os.environ, + { + "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", + "REPORT_BUCKET_NAME": "test-report-bucket", + }, + clear=False, + ) + + +@pytest.fixture +def mock_reporting_dynamo_service(mocker): + mock_cls = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.DynamoDBService" + ) + return mock_cls.return_value + + +@pytest.fixture +def reporting_repo(mock_reporting_dynamo_service): + return ReportingDynamoRepository(table_name="TestTable") diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index d129d2ed36..ed7371a634 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -1,5 +1,7 @@ import importlib +import json import os + import pytest MODULE_UNDER_TEST = "handlers.report_distribution_handler" @@ -10,26 +12,10 @@ def handler_module(): return importlib.import_module(MODULE_UNDER_TEST) -@pytest.fixture -def required_env(mocker): - mocker.patch.dict( - os.environ, - { - "REPORT_BUCKET_NAME": "my-report-bucket", - "CONTACT_TABLE_NAME": "contact-table", - "PRM_MAILBOX_EMAIL": "prm@example.com", - "SES_FROM_ADDRESS": "from@example.com", - }, - clear=False, - ) - - def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( - mocker, handler_module, required_env + mocker, handler_module, required_report_distribution_env, lambda_context ): event = {"action": "list", "prefix": "reports/2026-01-01/"} - context = mocker.Mock() - context.aws_request_id = "req-123" # avoid JSON serialization issues in decorators s3_instance = mocker.Mock(name="S3ServiceInstance") contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") @@ -57,7 +43,7 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( return_value=svc_instance, ) - result = handler_module.lambda_handler(event, context) + result = handler_module.lambda_handler(event, lambda_context) mocked_s3_cls.assert_called_once_with() mocked_contact_repo_cls.assert_called_once_with("contact-table") @@ -73,7 +59,9 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( ) svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") + assert result == { + "status": "ok", "bucket": "my-report-bucket", "prefix": "reports/2026-01-01/", "keys": ["a.xlsx", "b.xlsx"], @@ -81,46 +69,63 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( - mocker, handler_module, required_env + mocker, handler_module, required_report_distribution_env, lambda_context ): event = {"action": "list", "prefix": "p/", "bucket": "override-bucket"} - context = mocker.Mock() - context.aws_request_id = "req-456" svc_instance = mocker.Mock() svc_instance.list_xlsx_keys.return_value = [] - mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object( + handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock() + ) mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) - result = handler_module.lambda_handler(event, context) + result = handler_module.lambda_handler(event, lambda_context) svc_instance.list_xlsx_keys.assert_called_once_with(prefix="p/") - assert result == {"bucket": "override-bucket", "prefix": "p/", "keys": []} + assert result == { + "status": "ok", + "bucket": "override-bucket", + "prefix": "p/", + "keys": [], + } def test_lambda_handler_process_one_mode_happy_path( - mocker, handler_module, required_env + mocker, handler_module, required_report_distribution_env, lambda_context ): event = {"action": "process_one", "key": "reports/ABC/whatever.xlsx"} - context = mocker.Mock() - context.aws_request_id = "req-789" svc_instance = mocker.Mock() svc_instance.extract_ods_code_from_key.return_value = "ABC" svc_instance.process_one_report.return_value = None - mocker.patch.object(handler_module, "ReportDistributionService", autospec=True, return_value=svc_instance) + mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object(handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock()) + mocker.patch.object( + handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock() + ) mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) - result = handler_module.lambda_handler(event, context) + result = handler_module.lambda_handler(event, lambda_context) svc_instance.extract_ods_code_from_key.assert_called_once_with("reports/ABC/whatever.xlsx") - svc_instance.process_one_report.assert_called_once_with(ods_code="ABC", key="reports/ABC/whatever.xlsx") + svc_instance.process_one_report.assert_called_once_with( + ods_code="ABC", key="reports/ABC/whatever.xlsx" + ) assert result == { "status": "ok", @@ -128,3 +133,48 @@ def test_lambda_handler_process_one_mode_happy_path( "key": "reports/ABC/whatever.xlsx", "ods_code": "ABC", } + + +def test_lambda_handler_returns_400_when_action_invalid( + handler_module, required_report_distribution_env, lambda_context +): + event = {"action": "nope"} + + result = handler_module.lambda_handler(event, lambda_context) + + assert isinstance(result, dict) + assert result["statusCode"] == 400 + + body = json.loads(result["body"]) + assert body["err_code"] == handler_module.LambdaError.InvalidAction.value["err_code"] + assert "Invalid action" in body["message"] + + if body.get("interaction_id") is not None: + assert body["interaction_id"] == lambda_context.aws_request_id + + +def test_lambda_handler_returns_500_when_required_env_missing(mocker, handler_module, lambda_context): + mocker.patch.dict( + os.environ, + { + "CONTACT_TABLE_NAME": "contact-table", + "PRM_MAILBOX_EMAIL": "prm@example.com", + "SES_FROM_ADDRESS": "from@example.com", + }, + clear=False, + ) + os.environ.pop("REPORT_BUCKET_NAME", None) + + event = {"action": "list", "prefix": "p/"} + + result = handler_module.lambda_handler(event, lambda_context) + + assert isinstance(result, dict) + assert result["statusCode"] == 500 + + body = json.loads(result["body"]) + assert body["err_code"] == "ENV_5001" + assert "REPORT_BUCKET_NAME" in body["message"] + + if body.get("interaction_id") is not None: + assert body["interaction_id"] == lambda_context.aws_request_id diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 6458a2f8a6..0ab0dac213 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,61 +1,57 @@ +import importlib import json import os from unittest.mock import MagicMock import pytest -from handlers.report_orchestration_handler import lambda_handler +MODULE_UNDER_TEST = "handlers.report_orchestration_handler" -class FakeContext: - aws_request_id = "test-request-id" - - -@pytest.fixture(autouse=True) -def mock_env(mocker): - mocker.patch.dict( - os.environ, - { - "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", - "REPORT_BUCKET_NAME": "test-report-bucket", - }, - clear=False, - ) +@pytest.fixture +def handler_module(): + return importlib.import_module(MODULE_UNDER_TEST) @pytest.fixture -def mock_logger(mocker): - return mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) +def mock_logger(mocker, handler_module): + return mocker.patch.object(handler_module, "logger", new=MagicMock()) @pytest.fixture -def mock_window(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.calculate_reporting_window", +def mock_window(mocker, handler_module): + return mocker.patch.object( + handler_module, + "calculate_reporting_window", return_value=(100, 200), ) @pytest.fixture -def mock_report_date(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.get_report_date_folder", +def mock_report_date(mocker, handler_module): + return mocker.patch.object( + handler_module, + "get_report_date_folder", return_value="2026-01-02", ) @pytest.fixture -def mock_build_services(mocker): +def mock_build_services(mocker, handler_module): orchestration_service = MagicMock() s3_service = MagicMock() - patcher = mocker.patch( - "handlers.report_orchestration_handler.build_services", + patcher = mocker.patch.object( + handler_module, + "build_services", return_value=(orchestration_service, s3_service), ) return patcher, orchestration_service, s3_service def test_lambda_handler_calls_service_and_returns_expected_response( + handler_module, + required_report_orchestration_env, + lambda_context, mock_logger, mock_build_services, mock_window, @@ -68,7 +64,7 @@ def test_lambda_handler_calls_service_and_returns_expected_response( "B67890": "/tmp/B67890.xlsx", } - result = lambda_handler(event={}, context=FakeContext()) + result = handler_module.lambda_handler(event={}, context=lambda_context) build_services_patcher.assert_called_once_with("TestTable") @@ -91,6 +87,9 @@ def test_lambda_handler_calls_service_and_returns_expected_response( def test_lambda_handler_calls_window_function( + handler_module, + required_report_orchestration_env, + lambda_context, mock_build_services, mock_window, mock_report_date, @@ -98,12 +97,15 @@ def test_lambda_handler_calls_window_function( _, orchestration_service, _ = mock_build_services orchestration_service.process_reporting_window.return_value = {} - lambda_handler(event={}, context=FakeContext()) + handler_module.lambda_handler(event={}, context=lambda_context) mock_window.assert_called_once() def test_lambda_handler_returns_empty_keys_when_no_reports_generated( + handler_module, + required_report_orchestration_env, + lambda_context, mock_build_services, mock_logger, mock_window, @@ -112,9 +114,10 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( _, orchestration_service, s3_service = mock_build_services orchestration_service.process_reporting_window.return_value = {} - result = lambda_handler(event={}, context=FakeContext()) + result = handler_module.lambda_handler(event={}, context=lambda_context) assert result == { + "status": "ok", "report_date": "2026-01-02", "bucket": "test-report-bucket", "prefix": "Report-Orchestration/2026-01-02/", @@ -126,6 +129,9 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( + handler_module, + required_report_orchestration_env, + lambda_context, mock_build_services, mock_window, mock_report_date, @@ -136,7 +142,7 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( "UNKNOWN": "/tmp/UNKNOWN.xlsx", } - result = lambda_handler(event={}, context=FakeContext()) + result = handler_module.lambda_handler(event={}, context=lambda_context) assert s3_service.upload_file_with_extra_args.call_count == 2 @@ -160,14 +166,19 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( ] -def test_lambda_handler_returns_error_when_required_env_missing(mocker): - mocker.patch.dict(os.environ, {"BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable"}, clear=False) +def test_lambda_handler_returns_error_when_required_env_missing( + handler_module, + lambda_context, + mocker, +): + mocker.patch.dict( + os.environ, + {"BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable"}, + clear=False, + ) os.environ.pop("REPORT_BUCKET_NAME", None) - ctx = FakeContext() - ctx.aws_request_id = "test-request-id" - - result = lambda_handler(event={}, context=ctx) + result = handler_module.lambda_handler(event={}, context=lambda_context) assert isinstance(result, dict) assert result["statusCode"] == 500 @@ -177,4 +188,4 @@ def test_lambda_handler_returns_error_when_required_env_missing(mocker): assert "REPORT_BUCKET_NAME" in body["message"] if body.get("interaction_id") is not None: - assert body["interaction_id"] == "test-request-id" + assert body["interaction_id"] == lambda_context.aws_request_id diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index 5697eb112d..7626bde154 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -1,64 +1,122 @@ from datetime import date -from unittest.mock import MagicMock -import pytest -from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository - - -@pytest.fixture -def mock_dynamo_service(mocker): - mock_service_class = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.DynamoDBService" +def test_get_records_for_time_window_same_date_queries_once( + mocker, mock_reporting_dynamo_service, reporting_repo +): + mock_utc_date = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date" ) - mock_instance = mock_service_class.return_value - mock_instance.query_by_key_condition_expression = MagicMock() - return mock_instance - - -@pytest.fixture -def reporting_repo(mock_dynamo_service): - return ReportingDynamoRepository(table_name="TestTable") - - -def test_get_records_for_time_window_same_date_queries_once(mocker, mock_dynamo_service, reporting_repo): - mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") mock_utc_date.side_effect = [date(2026, 1, 7), date(2026, 1, 7)] - mock_dynamo_service.query_by_key_condition_expression.return_value = [{"ID": "one"}] + mock_reporting_dynamo_service.query_by_key_condition_expression.return_value = [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ] result = reporting_repo.get_records_for_time_window(100, 200) - assert result == [{"ID": "one"}] - mock_dynamo_service.query_by_key_condition_expression.assert_called_once() + assert result == [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ] + mock_reporting_dynamo_service.query_by_key_condition_expression.assert_called_once() - call_kwargs = mock_dynamo_service.query_by_key_condition_expression.call_args.kwargs + call_kwargs = ( + mock_reporting_dynamo_service.query_by_key_condition_expression.call_args.kwargs + ) assert call_kwargs["table_name"] == "TestTable" assert call_kwargs["index_name"] == "TimestampIndex" assert "key_condition_expression" in call_kwargs -def test_get_records_for_time_window_different_dates_queries_twice(mocker, mock_dynamo_service, reporting_repo): - mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") +def test_get_records_for_time_window_different_dates_queries_twice( + mocker, mock_reporting_dynamo_service, reporting_repo +): + mock_utc_date = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date" + ) mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] - mock_dynamo_service.query_by_key_condition_expression.side_effect = [ - [{"ID": "start-day"}], - [{"ID": "end-day"}], + mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = [ + [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + } + ], + [ + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ], ] result = reporting_repo.get_records_for_time_window(100, 200) - assert result == [{"ID": "start-day"}, {"ID": "end-day"}] - assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 + assert result == [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ] + assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == 2 -def test_get_records_for_time_window_returns_empty_list_when_no_items(mocker, mock_dynamo_service, reporting_repo): - mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") +def test_get_records_for_time_window_returns_empty_list_when_no_items( + mocker, mock_reporting_dynamo_service, reporting_repo +): + mock_utc_date = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.utc_date" + ) mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] - mock_dynamo_service.query_by_key_condition_expression.side_effect = [[], []] + mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = [ + [], + [], + ] result = reporting_repo.get_records_for_time_window(100, 200) assert result == [] - assert mock_dynamo_service.query_by_key_condition_expression.call_count == 2 + assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == 2 diff --git a/lambdas/tests/unit/services/base/test_s3_service.py b/lambdas/tests/unit/services/base/test_s3_service.py index b9e9aa2cb0..6510a45140 100755 --- a/lambdas/tests/unit/services/base/test_s3_service.py +++ b/lambdas/tests/unit/services/base/test_s3_service.py @@ -597,3 +597,34 @@ def test_copy_across_bucket_retries_on_409_conflict(mock_service, mock_client): "StorageClass": "INTELLIGENT_TIERING", } mock_client.copy_object.assert_called_with(**expected_call) + +def test_list_object_keys_returns_keys_for_prefix( + mock_service, mock_client, mock_list_objects_paginate +): + mock_list_objects_paginate.return_value = [MOCK_LIST_OBJECTS_RESPONSE] + + prefix = "some/prefix/" + expected = [obj["Key"] for obj in MOCK_LIST_OBJECTS_RESPONSE["Contents"]] + + actual = mock_service.list_object_keys(bucket_name=MOCK_BUCKET, prefix=prefix) + + assert actual == expected + mock_client.get_paginator.assert_called_with("list_objects_v2") + mock_list_objects_paginate.assert_called_with(Bucket=MOCK_BUCKET, Prefix=prefix) + + +def test_list_object_keys_handles_paginated_responses( + mock_service, mock_client, mock_list_objects_paginate +): + mock_list_objects_paginate.return_value = MOCK_LIST_OBJECTS_PAGINATED_RESPONSES + + prefix = "some/prefix/" + expected = flatten( + [[obj["Key"] for obj in page.get("Contents", [])] for page in MOCK_LIST_OBJECTS_PAGINATED_RESPONSES] + ) + + actual = mock_service.list_object_keys(bucket_name=MOCK_BUCKET, prefix=prefix) + + assert actual == expected + mock_client.get_paginator.assert_called_with("list_objects_v2") + mock_list_objects_paginate.assert_called_with(Bucket=MOCK_BUCKET, Prefix=prefix) diff --git a/lambdas/tests/unit/services/reporting/test_email_service.py b/lambdas/tests/unit/services/reporting/test_email_service.py index efde7f7251..3c84799cbc 100644 --- a/lambdas/tests/unit/services/reporting/test_email_service.py +++ b/lambdas/tests/unit/services/reporting/test_email_service.py @@ -6,9 +6,9 @@ @pytest.fixture def email_service(mocker): mocker.patch("services.email_service.boto3.client", autospec=True) - svc = EmailService() - svc.ses = mocker.Mock() - return svc + service = EmailService() + service.ses = mocker.Mock() + return service def test_send_email_sends_raw_email_without_attachments(email_service, mocker): mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index 56b5150e03..3bff899731 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -1,29 +1,29 @@ import os + import pytest from services.reporting.report_distribution_service import ReportDistributionService + @pytest.fixture def mock_s3_service(mocker): - return mocker.Mock() + return mocker.Mock(name="S3Service") @pytest.fixture def mock_contact_repo(mocker): - repo = mocker.Mock() + repo = mocker.Mock(name="ReportContactRepository") repo.get_contact_email.return_value = None return repo @pytest.fixture def mock_email_service(mocker): - return mocker.Mock() + return mocker.Mock(name="EmailService") @pytest.fixture -def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): - mocker.patch("services.reporting.report_distribution_service.boto3.client", autospec=True) - +def service(mock_s3_service, mock_contact_repo, mock_email_service): return ReportDistributionService( s3_service=mock_s3_service, contact_repo=mock_contact_repo, @@ -33,10 +33,14 @@ def service(mocker, mock_s3_service, mock_contact_repo, mock_email_service): prm_mailbox="prm@example.com", ) + def test_extract_ods_code_from_key_strips_xlsx_extension(): - assert ReportDistributionService.extract_ods_code_from_key( - "Report-Orchestration/2026-01-01/Y12345.xlsx" - ) == "Y12345" + assert ( + ReportDistributionService.extract_ods_code_from_key( + "Report-Orchestration/2026-01-01/Y12345.xlsx" + ) + == "Y12345" + ) def test_extract_ods_code_from_key_is_case_insensitive(): @@ -44,58 +48,41 @@ def test_extract_ods_code_from_key_is_case_insensitive(): def test_extract_ods_code_from_key_keeps_non_xlsx_filename(): - assert ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") == "report.csv" - -def test_list_xlsx_keys_filters_only_xlsx(service, mocker): - paginator = mocker.Mock() - paginator.paginate.return_value = [ - { - "Contents": [ - {"Key": "Report-Orchestration/2026-01-01/A123.xlsx"}, - {"Key": "Report-Orchestration/2026-01-01/readme.txt"}, - {"Key": "Report-Orchestration/2026-01-01/B456.xls"}, - {"Key": "Report-Orchestration/2026-01-01/C789.xlsx"}, - ] - }, - {"Contents": [{"Key": "Report-Orchestration/2026-01-01/D000.xlsx"}]}, - {}, - ] + assert ( + ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") + == "report.csv" + ) - service._s3_client.get_paginator.return_value = paginator + +def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): + mock_s3_service.list_object_keys.return_value = [ + "Report-Orchestration/2026-01-01/A123.xlsx", + "Report-Orchestration/2026-01-01/readme.txt", + "Report-Orchestration/2026-01-01/B456.xls", + "Report-Orchestration/2026-01-01/C789.xlsx", + "Report-Orchestration/2026-01-01/D000.xlsx", + ] keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + mock_s3_service.list_object_keys.assert_called_once_with( + bucket_name="my-bucket", + prefix="Report-Orchestration/2026-01-01/", + ) assert keys == [ "Report-Orchestration/2026-01-01/A123.xlsx", "Report-Orchestration/2026-01-01/C789.xlsx", "Report-Orchestration/2026-01-01/D000.xlsx", ] - service._s3_client.get_paginator.assert_called_once_with("list_objects_v2") - paginator.paginate.assert_called_once_with( - Bucket="my-bucket", - Prefix="Report-Orchestration/2026-01-01/", - ) - -def test_list_xlsx_keys_returns_empty_when_no_objects(service, mocker): - paginator = mocker.Mock() - paginator.paginate.return_value = [{"Contents": []}, {}] - service._s3_client.get_paginator.return_value = paginator +def test_list_xlsx_keys_returns_empty_when_no_objects(service, mock_s3_service): + mock_s3_service.list_object_keys.return_value = [] keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") assert keys == [] - - -def test_list_xlsx_keys_skips_pages_without_contents(service, mocker): - paginator = mocker.Mock() - paginator.paginate.return_value = [{}, {"Contents": [{"Key": "p/X.xlsx"}]}] - service._s3_client.get_paginator.return_value = paginator - - keys = service.list_xlsx_keys(prefix="p/") - - assert keys == ["p/X.xlsx"] + mock_s3_service.list_object_keys.assert_called_once() def test_process_one_report_downloads_encrypts_and_delegates_email( @@ -134,13 +121,11 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( "Report-Orchestration/2026-01-01/Y12345.xlsx", local_xlsx, ) - mocked_zip.assert_called_once_with( input_path=local_xlsx, output_zip=local_zip, password="fixed-password", ) - mocked_send.assert_called_once_with( ods_code="Y12345", attachment_path=local_zip, @@ -159,7 +144,9 @@ def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_ return_value=td, ) - mocked_zip = mocker.patch("services.reporting.report_distribution_service.zip_encrypt_file", autospec=True) + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True + ) mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) with pytest.raises(RuntimeError, match="download failed"): @@ -227,15 +214,19 @@ def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fa "k.xlsx", os.path.join(fake_tmp, "Y12345.xlsx"), ) - mocked_zip.assert_not_called() mocked_send.assert_not_called() -def test_send_report_emails_with_contact_calls_email_contact(service, mock_contact_repo, mocker): + +def test_send_report_emails_with_contact_calls_email_contact( + service, mock_contact_repo, mocker +): mock_contact_repo.get_contact_email.return_value = "contact@example.com" mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + mocked_email_prm = mocker.patch.object( + service, "email_prm_missing_contact", autospec=True + ) service.send_report_emails( ods_code="Y12345", @@ -256,7 +247,9 @@ def test_send_report_emails_without_contact_calls_email_prm(service, mock_contac mock_contact_repo.get_contact_email.return_value = None mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + mocked_email_prm = mocker.patch.object( + service, "email_prm_missing_contact", autospec=True + ) service.send_report_emails( ods_code="A99999", @@ -273,11 +266,15 @@ def test_send_report_emails_without_contact_calls_email_prm(service, mock_contac mocked_email_contact.assert_not_called() -def test_send_report_emails_contact_lookup_exception_falls_back_to_prm(service, mock_contact_repo, mocker): +def test_send_report_emails_contact_lookup_exception_falls_back_to_prm( + service, mock_contact_repo, mocker +): mock_contact_repo.get_contact_email.side_effect = RuntimeError("ddb down") mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + mocked_email_prm = mocker.patch.object( + service, "email_prm_missing_contact", autospec=True + ) service.send_report_emails( ods_code="A99999", @@ -312,7 +309,7 @@ def test_email_contact_sends_report_and_password(service, mock_email_service): ) -def test_email_contact_sends_password_even_if_report_email_fails(service, mock_email_service): +def test_email_contact_does_not_send_password_if_report_email_fails(service, mock_email_service): mock_email_service.send_report_email.side_effect = RuntimeError("SES down") with pytest.raises(RuntimeError, match="SES down"): diff --git a/lambdas/utils/lambda_exceptions.py b/lambdas/utils/lambda_exceptions.py index 9a30812eed..cc4af788a4 100644 --- a/lambdas/utils/lambda_exceptions.py +++ b/lambdas/utils/lambda_exceptions.py @@ -120,3 +120,6 @@ class DocumentReviewLambdaException(LambdaException): class UpdateDocumentReviewException(LambdaException): pass + +class ReportDistributionException(LambdaException): + pass \ No newline at end of file diff --git a/lambdas/utils/zip_utils.py b/lambdas/utils/zip_utils.py index a09e8934f0..43775059e7 100644 --- a/lambdas/utils/zip_utils.py +++ b/lambdas/utils/zip_utils.py @@ -3,12 +3,15 @@ def zip_encrypt_file(*, input_path: str, output_zip: str, password: str) -> None: - """ - Create an AES-encrypted ZIP file containing a single file. + """Create an AES-encrypted ZIP file containing a single file. + + Args: + input_path: Path to the file to zip. + output_zip: Path of the ZIP file to create. + password: Password used for AES encryption. - :param input_path: Path to the file to zip - :param output_zip: Path of the zip file to create - :param password: Password for AES encryption + Returns: + None """ with pyzipper.AESZipFile( output_zip, diff --git a/poetry.lock b/poetry.lock index 51ea36ac57..eb1c448afe 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1671,6 +1671,57 @@ files = [ {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, ] +[[package]] +name = "pycryptodomex" +version = "3.23.0" +description = "Cryptographic library for Python" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["reports-lambda"] +files = [ + {file = "pycryptodomex-3.23.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:add243d204e125f189819db65eed55e6b4713f70a7e9576c043178656529cec7"}, + {file = "pycryptodomex-3.23.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:1c6d919fc8429e5cb228ba8c0d4d03d202a560b421c14867a65f6042990adc8e"}, + {file = "pycryptodomex-3.23.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:1c3a65ad441746b250d781910d26b7ed0a396733c6f2dbc3327bd7051ec8a541"}, + {file = "pycryptodomex-3.23.0-cp27-cp27m-win32.whl", hash = "sha256:47f6d318fe864d02d5e59a20a18834819596c4ed1d3c917801b22b92b3ffa648"}, + {file = "pycryptodomex-3.23.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:d9825410197a97685d6a1fa2a86196430b01877d64458a20e95d4fd00d739a08"}, + {file = "pycryptodomex-3.23.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:267a3038f87a8565bd834317dbf053a02055915acf353bf42ededb9edaf72010"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:7b37e08e3871efe2187bc1fd9320cc81d87caf19816c648f24443483005ff886"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:91979028227543010d7b2ba2471cf1d1e398b3f183cb105ac584df0c36dac28d"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b8962204c47464d5c1c4038abeadd4514a133b28748bcd9fa5b6d62e3cec6fa"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a33986a0066860f7fcf7c7bd2bc804fa90e434183645595ae7b33d01f3c91ed8"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7947ab8d589e3178da3d7cdeabe14f841b391e17046954f2fbcd941705762b5"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c25e30a20e1b426e1f0fa00131c516f16e474204eee1139d1603e132acffc314"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:da4fa650cef02db88c2b98acc5434461e027dce0ae8c22dd5a69013eaf510006"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:58b851b9effd0d072d4ca2e4542bf2a4abcf13c82a29fd2c93ce27ee2a2e9462"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:a9d446e844f08299236780f2efa9898c818fe7e02f17263866b8550c7d5fb328"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:bc65bdd9fc8de7a35a74cab1c898cab391a4add33a8fe740bda00f5976ca4708"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:c885da45e70139464f082018ac527fdaad26f1657a99ee13eecdce0f0ca24ab4"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:06698f957fe1ab229a99ba2defeeae1c09af185baa909a31a5d1f9d42b1aaed6"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2c2537863eccef2d41061e82a881dcabb04944c5c06c5aa7110b577cc487545"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43c446e2ba8df8889e0e16f02211c25b4934898384c1ec1ec04d7889c0333587"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f489c4765093fb60e2edafdf223397bc716491b2b69fe74367b70d6999257a5c"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdc69d0d3d989a1029df0eed67cc5e8e5d968f3724f4519bd03e0ec68df7543c"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6bbcb1dd0f646484939e142462d9e532482bc74475cecf9c4903d4e1cd21f003"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:8a4fcd42ccb04c31268d1efeecfccfd1249612b4de6374205376b8f280321744"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55ccbe27f049743a4caf4f4221b166560d3438d0b1e5ab929e07ae1702a4d6fd"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-win32.whl", hash = "sha256:189afbc87f0b9f158386bf051f720e20fa6145975f1e76369303d0f31d1a8d7c"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:52e5ca58c3a0b0bd5e100a9fbc8015059b05cffc6c66ce9d98b4b45e023443b9"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:02d87b80778c171445d67e23d1caef279bf4b25c3597050ccd2e13970b57fd51"}, + {file = "pycryptodomex-3.23.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:febec69c0291efd056c65691b6d9a339f8b4bc43c6635b8699471248fe897fea"}, + {file = "pycryptodomex-3.23.0-pp27-pypy_73-win32.whl", hash = "sha256:c84b239a1f4ec62e9c789aafe0543f0594f0acd90c8d9e15bcece3efe55eca66"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ebfff755c360d674306e5891c564a274a47953562b42fb74a5c25b8fc1fb1cb5"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eca54f4bb349d45afc17e3011ed4264ef1cc9e266699874cdd1349c504e64798"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2596e643d4365e14d0879dc5aafe6355616c61c2176009270f3048f6d9a61f"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdfac7cda115bca3a5abb2f9e43bc2fb66c2b65ab074913643803ca7083a79ea"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:14c37aaece158d0ace436f76a7bb19093db3b4deade9797abfc39ec6cd6cc2fe"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7de1e40a41a5d7f1ac42b6569b10bcdded34339950945948529067d8426d2785"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bffc92138d75664b6d543984db7893a628559b9e78658563b0395e2a5fb47ed9"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df027262368334552db2c0ce39706b3fb32022d1dce34673d0f9422df004b96a"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e79f1aaff5a3a374e92eb462fa9e598585452135012e2945f96874ca6eeb1ff"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:27e13c80ac9a0a1d050ef0a7e0a18cc04c8850101ec891815b6c5a0375e8a245"}, + {file = "pycryptodomex-3.23.0.tar.gz", hash = "sha256:71909758f010c82bc99b0abf4ea12012c98962fbf0583c2164f8b84533c2e4da"}, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -2065,6 +2116,21 @@ files = [ {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, ] +[[package]] +name = "pyzipper" +version = "0.3.6" +description = "AES encryption for zipfile." +optional = false +python-versions = ">=3.4" +groups = ["reports-lambda"] +files = [ + {file = "pyzipper-0.3.6-py2.py3-none-any.whl", hash = "sha256:6d097f465bfa47796b1494e12ea65d1478107d38e13bc56f6e58eedc4f6c1a87"}, + {file = "pyzipper-0.3.6.tar.gz", hash = "sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc"}, +] + +[package.dependencies] +pycryptodomex = "*" + [[package]] name = "regex" version = "2023.12.25" @@ -2541,4 +2607,4 @@ requests = "*" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "5bd6a9f1c4a220910e01dd3a74cb94f89d8fa04704ac1c70d97c6f4867ff7d25" +content-hash = "1a7b4dc509284d68e04fb77167a43fd560d0d05d979eb0bcd605263573ccb354" diff --git a/pyproject.toml b/pyproject.toml index 5bdb4d955e..e19c0ec384 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,3 +65,4 @@ polars = "1.31.0" [tool.poetry.group.reports_lambda.dependencies] openpyxl = "^3.1.5" reportlab = "^4.3.1" +pyzipper = "^0.3.6" From 0c98c3489635ae3aed329b6654b968bf476bacf6 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 30 Jan 2026 15:50:59 +0000 Subject: [PATCH 33/37] [PRMP-1057-2] fixed comments --- lambdas/enums/report_distribution_action.py | 6 + .../handlers/report_distribution_handler.py | 11 +- lambdas/tests/unit/handlers/conftest.py | 103 ++++++-- .../test_report_distribution_handler.py | 61 ++--- .../test_report_orchestration_handler.py | 90 ++----- .../test_report_contact_repository.py | 69 +++-- .../test_reporting_dynamo_repository.py | 228 ++++++++--------- .../test_excel_report_generator_service.py | 145 ++++++----- .../test_report_distribution_service.py | 187 ++++++-------- .../test_report_orchestration_service.py | 237 ++++++++---------- 10 files changed, 535 insertions(+), 602 deletions(-) create mode 100644 lambdas/enums/report_distribution_action.py diff --git a/lambdas/enums/report_distribution_action.py b/lambdas/enums/report_distribution_action.py new file mode 100644 index 0000000000..903314b771 --- /dev/null +++ b/lambdas/enums/report_distribution_action.py @@ -0,0 +1,6 @@ +from enum import StrEnum + + +class ReportDistributionAction(StrEnum): + LIST = "list" + PROCESS_ONE = "process_one" \ No newline at end of file diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index da32e9d409..af6c0f2c7e 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -2,6 +2,7 @@ from typing import Any, Dict from enums.lambda_error import LambdaError +from enums.report_distribution_action import ReportDistributionAction from repositories.reporting.report_contact_repository import ReportContactRepository from services.base.s3_service import S3Service from services.email_service import EmailService @@ -29,8 +30,11 @@ @set_request_context_for_logging def lambda_handler(event, context) -> Dict[str, Any]: action = event.get("action") - if action not in {"list", "process_one"}: - logger.error(f"Invalid action. Expected 'list' or 'process_one'.") + if action not in { + ReportDistributionAction.LIST, + ReportDistributionAction.PROCESS_ONE, + }: + logger.error("Invalid action. Expected 'list' or 'process_one'.") raise ReportDistributionException(400, LambdaError.InvalidAction) bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] @@ -53,7 +57,7 @@ def lambda_handler(event, context) -> Dict[str, Any]: response = {"status": "ok", "bucket": bucket} - if action == "list": + if action == ReportDistributionAction.LIST: prefix = event["prefix"] keys = service.list_xlsx_keys(prefix=prefix) logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") @@ -66,4 +70,3 @@ def lambda_handler(event, context) -> Dict[str, Any]: response.update({"key": key, "ods_code": ods_code}) return response - diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index a7cd749de9..c744cb885b 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -2,6 +2,7 @@ import pytest from enums.feature_flags import FeatureFlags +from enums.report_distribution_action import ReportDistributionAction from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.feature_flags_service import FeatureFlagService @@ -225,17 +226,11 @@ def mock_upload_document_iteration_3_disabled(mocker): @pytest.fixture -def required_report_distribution_env(mocker): - mocker.patch.dict( - os.environ, - { - "REPORT_BUCKET_NAME": "my-report-bucket", - "CONTACT_TABLE_NAME": "contact-table", - "PRM_MAILBOX_EMAIL": "prm@example.com", - "SES_FROM_ADDRESS": "from@example.com", - }, - clear=False, - ) +def required_report_distribution_env(monkeypatch): + monkeypatch.setenv("REPORT_BUCKET_NAME", "my-report-bucket") + monkeypatch.setenv("CONTACT_TABLE_NAME", "contact-table") + monkeypatch.setenv("PRM_MAILBOX_EMAIL", "prm@example.com") + monkeypatch.setenv("SES_FROM_ADDRESS", "from@example.com") @pytest.fixture @@ -246,15 +241,10 @@ def lambda_context(mocker): @pytest.fixture -def required_report_orchestration_env(mocker): - mocker.patch.dict( - os.environ, - { - "BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable", - "REPORT_BUCKET_NAME": "test-report-bucket", - }, - clear=False, - ) +def required_report_orchestration_env(monkeypatch): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + monkeypatch.setenv("REPORT_BUCKET_NAME", "test-report-bucket") + @pytest.fixture @@ -268,3 +258,76 @@ def mock_reporting_dynamo_service(mocker): @pytest.fixture def reporting_repo(mock_reporting_dynamo_service): return ReportingDynamoRepository(table_name="TestTable") + + +@pytest.fixture +def report_distribution_list_event(): + return {"action": ReportDistributionAction.LIST, "prefix": "p/"} + + +@pytest.fixture +def report_distribution_process_one_event(): + return {"action": ReportDistributionAction.PROCESS_ONE, "key": "reports/ABC/whatever.xlsx"} + + +@pytest.fixture +def mock_report_distribution_wiring(mocker): + svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") + + mocker.patch( + "handlers.report_distribution_handler.S3Service", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch( + "handlers.report_distribution_handler.ReportContactRepository", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch( + "handlers.report_distribution_handler.EmailService", + autospec=True, + return_value=mocker.Mock(), + ) + mocker.patch( + "handlers.report_distribution_handler.ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + + return svc_instance + +from unittest.mock import MagicMock + +import pytest + + +@pytest.fixture +def mock_report_orchestration_wiring(mocker): + logger = mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) + + mock_window = mocker.patch( + "handlers.report_orchestration_handler.calculate_reporting_window", + return_value=(100, 200), + ) + mock_report_date = mocker.patch( + "handlers.report_orchestration_handler.get_report_date_folder", + return_value="2026-01-02", + ) + + orchestration_service = MagicMock(name="ReportOrchestrationService") + s3_service = MagicMock(name="S3Service") + + build_services = mocker.patch( + "handlers.report_orchestration_handler.build_services", + return_value=(orchestration_service, s3_service), + ) + + return { + "logger": logger, + "mock_window": mock_window, + "mock_report_date": mock_report_date, + "build_services": build_services, + "orchestration_service": orchestration_service, + "s3_service": s3_service, + } diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index ed7371a634..9e5e2cb5b2 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -15,7 +15,7 @@ def handler_module(): def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( mocker, handler_module, required_report_distribution_env, lambda_context ): - event = {"action": "list", "prefix": "reports/2026-01-01/"} + event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "reports/2026-01-01/"} s3_instance = mocker.Mock(name="S3ServiceInstance") contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") @@ -69,28 +69,18 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( - mocker, handler_module, required_report_distribution_env, lambda_context + handler_module, + required_report_distribution_env, + lambda_context, + report_distribution_list_event, + mock_report_distribution_wiring, ): - event = {"action": "list", "prefix": "p/", "bucket": "override-bucket"} - - svc_instance = mocker.Mock() - svc_instance.list_xlsx_keys.return_value = [] - - mocker.patch.object( - handler_module, - "ReportDistributionService", - autospec=True, - return_value=svc_instance, - ) - mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object( - handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock() - ) - mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + event = {**report_distribution_list_event, "bucket": "override-bucket"} + mock_report_distribution_wiring.list_xlsx_keys.return_value = [] result = handler_module.lambda_handler(event, lambda_context) - svc_instance.list_xlsx_keys.assert_called_once_with(prefix="p/") + mock_report_distribution_wiring.list_xlsx_keys.assert_called_once_with(prefix="p/") assert result == { "status": "ok", "bucket": "override-bucket", @@ -100,30 +90,23 @@ def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( def test_lambda_handler_process_one_mode_happy_path( - mocker, handler_module, required_report_distribution_env, lambda_context + handler_module, + required_report_distribution_env, + lambda_context, + report_distribution_process_one_event, + mock_report_distribution_wiring, ): - event = {"action": "process_one", "key": "reports/ABC/whatever.xlsx"} + event = {**report_distribution_process_one_event, "key": "reports/ABC/whatever.xlsx"} - svc_instance = mocker.Mock() - svc_instance.extract_ods_code_from_key.return_value = "ABC" - svc_instance.process_one_report.return_value = None - - mocker.patch.object( - handler_module, - "ReportDistributionService", - autospec=True, - return_value=svc_instance, - ) - mocker.patch.object(handler_module, "S3Service", autospec=True, return_value=mocker.Mock()) - mocker.patch.object( - handler_module, "ReportContactRepository", autospec=True, return_value=mocker.Mock() - ) - mocker.patch.object(handler_module, "EmailService", autospec=True, return_value=mocker.Mock()) + mock_report_distribution_wiring.extract_ods_code_from_key.return_value = "ABC" + mock_report_distribution_wiring.process_one_report.return_value = None result = handler_module.lambda_handler(event, lambda_context) - svc_instance.extract_ods_code_from_key.assert_called_once_with("reports/ABC/whatever.xlsx") - svc_instance.process_one_report.assert_called_once_with( + mock_report_distribution_wiring.extract_ods_code_from_key.assert_called_once_with( + "reports/ABC/whatever.xlsx" + ) + mock_report_distribution_wiring.process_one_report.assert_called_once_with( ods_code="ABC", key="reports/ABC/whatever.xlsx" ) @@ -165,7 +148,7 @@ def test_lambda_handler_returns_500_when_required_env_missing(mocker, handler_mo ) os.environ.pop("REPORT_BUCKET_NAME", None) - event = {"action": "list", "prefix": "p/"} + event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "p/"} result = handler_module.lambda_handler(event, lambda_context) diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 0ab0dac213..241a12e3fb 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,7 +1,5 @@ import importlib import json -import os -from unittest.mock import MagicMock import pytest @@ -13,51 +11,14 @@ def handler_module(): return importlib.import_module(MODULE_UNDER_TEST) -@pytest.fixture -def mock_logger(mocker, handler_module): - return mocker.patch.object(handler_module, "logger", new=MagicMock()) - - -@pytest.fixture -def mock_window(mocker, handler_module): - return mocker.patch.object( - handler_module, - "calculate_reporting_window", - return_value=(100, 200), - ) - - -@pytest.fixture -def mock_report_date(mocker, handler_module): - return mocker.patch.object( - handler_module, - "get_report_date_folder", - return_value="2026-01-02", - ) - - -@pytest.fixture -def mock_build_services(mocker, handler_module): - orchestration_service = MagicMock() - s3_service = MagicMock() - patcher = mocker.patch.object( - handler_module, - "build_services", - return_value=(orchestration_service, s3_service), - ) - return patcher, orchestration_service, s3_service - - def test_lambda_handler_calls_service_and_returns_expected_response( handler_module, required_report_orchestration_env, lambda_context, - mock_logger, - mock_build_services, - mock_window, - mock_report_date, + mock_report_orchestration_wiring, ): - build_services_patcher, orchestration_service, s3_service = mock_build_services + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + s3_service = mock_report_orchestration_wiring["s3_service"] orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", @@ -66,7 +27,7 @@ def test_lambda_handler_calls_service_and_returns_expected_response( result = handler_module.lambda_handler(event={}, context=lambda_context) - build_services_patcher.assert_called_once_with("TestTable") + mock_report_orchestration_wiring["build_services"].assert_called_once_with("TestTable") orchestration_service.process_reporting_window.assert_called_once_with( window_start_ts=100, @@ -83,35 +44,34 @@ def test_lambda_handler_calls_service_and_returns_expected_response( "Report-Orchestration/2026-01-02/B67890.xlsx", } - mock_logger.info.assert_any_call("Report orchestration lambda invoked") + mock_report_orchestration_wiring["logger"].info.assert_any_call( + "Report orchestration lambda invoked" + ) def test_lambda_handler_calls_window_function( handler_module, required_report_orchestration_env, lambda_context, - mock_build_services, - mock_window, - mock_report_date, + mock_report_orchestration_wiring, ): - _, orchestration_service, _ = mock_build_services + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] orchestration_service.process_reporting_window.return_value = {} handler_module.lambda_handler(event={}, context=lambda_context) - mock_window.assert_called_once() + mock_report_orchestration_wiring["mock_window"].assert_called_once() def test_lambda_handler_returns_empty_keys_when_no_reports_generated( handler_module, required_report_orchestration_env, lambda_context, - mock_build_services, - mock_logger, - mock_window, - mock_report_date, + mock_report_orchestration_wiring, ): - _, orchestration_service, s3_service = mock_build_services + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + s3_service = mock_report_orchestration_wiring["s3_service"] + orchestration_service.process_reporting_window.return_value = {} result = handler_module.lambda_handler(event={}, context=lambda_context) @@ -125,18 +85,20 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( } s3_service.upload_file_with_extra_args.assert_not_called() - mock_logger.info.assert_any_call("No reports generated; exiting") + mock_report_orchestration_wiring["logger"].info.assert_any_call( + "No reports generated; exiting" + ) def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( handler_module, required_report_orchestration_env, lambda_context, - mock_build_services, - mock_window, - mock_report_date, + mock_report_orchestration_wiring, ): - _, orchestration_service, s3_service = mock_build_services + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + s3_service = mock_report_orchestration_wiring["s3_service"] + orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", "UNKNOWN": "/tmp/UNKNOWN.xlsx", @@ -169,14 +131,10 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( def test_lambda_handler_returns_error_when_required_env_missing( handler_module, lambda_context, - mocker, + monkeypatch, ): - mocker.patch.dict( - os.environ, - {"BULK_UPLOAD_REPORT_TABLE_NAME": "TestTable"}, - clear=False, - ) - os.environ.pop("REPORT_BUCKET_NAME", None) + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + monkeypatch.delenv("REPORT_BUCKET_NAME", raising=False) result = handler_module.lambda_handler(event={}, context=lambda_context) diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py index d534267321..4216c3407c 100644 --- a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -5,12 +5,12 @@ @pytest.fixture def mock_dynamo(mocker): - mock = mocker.Mock() + dynamo = mocker.Mock() mocker.patch( "repositories.reporting.report_contact_repository.DynamoDBService", - return_value=mock, + return_value=dynamo, ) - return mock + return dynamo @pytest.fixture @@ -18,13 +18,32 @@ def repo(mock_dynamo): return ReportContactRepository(table_name="report-contacts") -def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): - mock_dynamo.get_item.return_value = { - "Item": { - "OdsCode": "Y12345", - "Email": "contact@example.com", - } - } +@pytest.mark.parametrize( + "dynamo_response, expected_email", + [ + ( + { + "Item": { + "OdsCode": "Y12345", + "Email": "contact@example.com", + } + }, + "contact@example.com", + ), + ({}, None), + ( + { + "Item": { + "OdsCode": "Y12345", + } + }, + None, + ), + (None, None), + ], +) +def test_get_contact_email(repo, mock_dynamo, dynamo_response, expected_email): + mock_dynamo.get_item.return_value = dynamo_response result = repo.get_contact_email("Y12345") @@ -32,32 +51,4 @@ def test_get_contact_email_returns_email_when_item_exists(repo, mock_dynamo): table_name="report-contacts", key={"OdsCode": "Y12345"}, ) - assert result == "contact@example.com" - - -def test_get_contact_email_returns_none_when_item_missing(repo, mock_dynamo): - mock_dynamo.get_item.return_value = {} # or None - - result = repo.get_contact_email("Y12345") - - mock_dynamo.get_item.assert_called_once_with( - table_name="report-contacts", - key={"OdsCode": "Y12345"}, - ) - assert result is None - - -def test_get_contact_email_returns_none_when_email_missing(repo, mock_dynamo): - mock_dynamo.get_item.return_value = { - "Item": { - "OdsCode": "Y12345", - } - } - - result = repo.get_contact_email("Y12345") - - mock_dynamo.get_item.assert_called_once_with( - table_name="report-contacts", - key={"OdsCode": "Y12345"}, - ) - assert result is None + assert result == expected_email diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index 7626bde154..40f193c2df 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -1,122 +1,124 @@ from datetime import date -def test_get_records_for_time_window_same_date_queries_once( - mocker, mock_reporting_dynamo_service, reporting_repo -): - mock_utc_date = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date" - ) - mock_utc_date.side_effect = [date(2026, 1, 7), date(2026, 1, 7)] - - mock_reporting_dynamo_service.query_by_key_condition_expression.return_value = [ - { - "Date": "2026-01-07", - "Timestamp": 1704601000, - "UploaderOdsCode": "A12345", - "DocumentId": "doc-001", - "Status": "UPLOADED", - } - ] - - result = reporting_repo.get_records_for_time_window(100, 200) +import pytest - assert result == [ - { - "Date": "2026-01-07", - "Timestamp": 1704601000, - "UploaderOdsCode": "A12345", - "DocumentId": "doc-001", - "Status": "UPLOADED", - } - ] - mock_reporting_dynamo_service.query_by_key_condition_expression.assert_called_once() - call_kwargs = ( - mock_reporting_dynamo_service.query_by_key_condition_expression.call_args.kwargs - ) - assert call_kwargs["table_name"] == "TestTable" - assert call_kwargs["index_name"] == "TimestampIndex" - assert "key_condition_expression" in call_kwargs - - -def test_get_records_for_time_window_different_dates_queries_twice( - mocker, mock_reporting_dynamo_service, reporting_repo +@pytest.mark.parametrize( + "start_dt,end_dt,service_side_effect,expected,expected_call_count", + [ + ( + date(2026, 1, 7), + date(2026, 1, 7), + [ + [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ] + ], + [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ], + 1, + ), + ( + date(2026, 1, 6), + date(2026, 1, 7), + [ + [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + } + ], + [ + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ], + ], + [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ], + 2, + ), + ( + date(2026, 1, 6), + date(2026, 1, 7), + [ + [], + [], + ], + [], + 2, + ), + ], +) +def test_get_records_for_time_window( + mocker, + mock_reporting_dynamo_service, + reporting_repo, + start_dt, + end_dt, + service_side_effect, + expected, + expected_call_count, ): - mock_utc_date = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date" - ) - mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] - - mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = [ - [ - { - "Date": "2026-01-06", - "Timestamp": 1704517200, - "UploaderOdsCode": "A12345", - "DocumentId": "doc-101", - "Status": "UPLOADED", - } - ], - [ - { - "Date": "2026-01-07", - "Timestamp": 1704590000, - "UploaderOdsCode": "B67890", - "DocumentId": "doc-202", - "Status": "PROCESSED", - }, - { - "Date": "2026-01-07", - "Timestamp": 1704593600, - "UploaderOdsCode": "B67890", - "DocumentId": "doc-203", - "Status": "PROCESSED", - }, - ], - ] + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [start_dt, end_dt] + mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = service_side_effect result = reporting_repo.get_records_for_time_window(100, 200) - assert result == [ - { - "Date": "2026-01-06", - "Timestamp": 1704517200, - "UploaderOdsCode": "A12345", - "DocumentId": "doc-101", - "Status": "UPLOADED", - }, - { - "Date": "2026-01-07", - "Timestamp": 1704590000, - "UploaderOdsCode": "B67890", - "DocumentId": "doc-202", - "Status": "PROCESSED", - }, - { - "Date": "2026-01-07", - "Timestamp": 1704593600, - "UploaderOdsCode": "B67890", - "DocumentId": "doc-203", - "Status": "PROCESSED", - }, - ] - assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == 2 - - -def test_get_records_for_time_window_returns_empty_list_when_no_items( - mocker, mock_reporting_dynamo_service, reporting_repo -): - mock_utc_date = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.utc_date" - ) - mock_utc_date.side_effect = [date(2026, 1, 6), date(2026, 1, 7)] - - mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = [ - [], - [], - ] - - result = reporting_repo.get_records_for_time_window(100, 200) + assert result == expected + assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == expected_call_count - assert result == [] - assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == 2 + for call in mock_reporting_dynamo_service.query_by_key_condition_expression.call_args_list: + kwargs = call.kwargs + assert kwargs["table_name"] == "TestTable" + assert kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in kwargs diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index f81559bea3..225227983b 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -1,6 +1,7 @@ import pytest from freezegun import freeze_time from openpyxl import load_workbook + from services.reporting.excel_report_generator_service import ExcelReportGenerator @@ -9,13 +10,43 @@ def excel_report_generator(): return ExcelReportGenerator() -@freeze_time("2025-01-01T12:00:00") -def test_create_report_orchestration_xlsx_happy_path( - excel_report_generator, - tmp_path, -): - output_file = tmp_path / "report.xlsx" +@pytest.fixture +def make_report(tmp_path, excel_report_generator): + def _make(*, ods_code="Y12345", records=None, filename="report.xlsx"): + records = records or [] + output_file = tmp_path / filename + + result = excel_report_generator.create_report_orchestration_xlsx( + ods_code=ods_code, + records=records, + output_path=str(output_file), + ) + + assert result == str(output_file) + assert output_file.exists() + + wb = load_workbook(output_file) + ws = wb.active + return output_file, ws + return _make + + +@pytest.fixture +def expected_header_row(): + return [ + "NHS Number", + "Date", + "Uploader ODS", + "PDS ODS", + "Upload Status", + "Reason", + "File Path", + ] + + +@freeze_time("2025-01-01T12:00:00") +def test_create_report_orchestration_xlsx_happy_path(make_report, expected_header_row): ods_code = "Y12345" records = [ { @@ -40,34 +71,15 @@ def test_create_report_orchestration_xlsx_happy_path( }, ] - result = excel_report_generator.create_report_orchestration_xlsx( - ods_code=ods_code, - records=records, - output_path=str(output_file), - ) - - assert result == str(output_file) - assert output_file.exists() - - wb = load_workbook(output_file) - ws = wb.active + _, ws = make_report(ods_code=ods_code, records=records, filename="report.xlsx") assert ws.title == "Daily Upload Report" assert ws["A1"].value == f"ODS Code: {ods_code}" - assert ws["A2"].value.startswith("Generated at (UTC): ") - assert ws["A3"].value is None # blank row + assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00+00:00" + assert ws["A3"].value is None - assert [cell.value for cell in ws[4]] == [ - "NHS Number", - "Date", - "Uploader ODS", - "PDS ODS", - "Upload Status", - "Reason", - "File Path", - ] + assert [cell.value for cell in ws[4]] == expected_header_row - # First data row (no ID column) assert [cell.value for cell in ws[5]] == [ "1234567890", "2025-01-01", @@ -89,54 +101,37 @@ def test_create_report_orchestration_xlsx_happy_path( ] -def test_create_report_orchestration_xlsx_with_no_records( - excel_report_generator, - tmp_path, -): - output_file = tmp_path / "empty_report.xlsx" - - excel_report_generator.create_report_orchestration_xlsx( - ods_code="Y12345", - records=[], - output_path=str(output_file), - ) - - wb = load_workbook(output_file) - ws = wb.active +def test_create_report_orchestration_xlsx_with_no_records(make_report, expected_header_row): + _, ws = make_report(records=[], filename="empty_report.xlsx") assert ws.max_row == 4 - - + assert [cell.value for cell in ws[4]] == expected_header_row + + +@pytest.mark.parametrize( + "records, expected_row", + [ + ( + [{"ID": 1, "NhsNumber": "1234567890"}], + ["1234567890", None, None, None, None, None, None], + ), + ( + [{"Date": "2025-01-01"}], + [None, "2025-01-01", None, None, None, None, None], + ), + ( + [{"UploaderOdsCode": "Y12345", "UploadStatus": "SUCCESS"}], + [None, None, "Y12345", None, "SUCCESS", None, None], + ), + ], +) def test_create_report_orchestration_xlsx_handles_missing_fields( - excel_report_generator, - tmp_path, + make_report, + expected_header_row, + records, + expected_row, ): - output_file = tmp_path / "partial.xlsx" - - records = [ - { - "ID": 1, - "NhsNumber": "1234567890", - } - ] - - excel_report_generator.create_report_orchestration_xlsx( - ods_code="Y12345", - records=records, - output_path=str(output_file), - ) - - wb = load_workbook(output_file) - ws = wb.active + _, ws = make_report(records=records, filename="partial.xlsx") - row = [cell.value for cell in ws[5]] - - assert row == [ - "1234567890", - None, - None, - None, - None, - None, - None, - ] + assert [cell.value for cell in ws[4]] == expected_header_row + assert [cell.value for cell in ws[5]] == expected_row diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index 3bff899731..b8ecde73fe 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -34,24 +34,31 @@ def service(mock_s3_service, mock_contact_repo, mock_email_service): ) -def test_extract_ods_code_from_key_strips_xlsx_extension(): - assert ( - ReportDistributionService.extract_ods_code_from_key( - "Report-Orchestration/2026-01-01/Y12345.xlsx" - ) - == "Y12345" +@pytest.fixture +def fixed_tmpdir(mocker): + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, ) + return fake_tmp -def test_extract_ods_code_from_key_is_case_insensitive(): - assert ReportDistributionService.extract_ods_code_from_key("a/b/C789.XLSX") == "C789" - - -def test_extract_ods_code_from_key_keeps_non_xlsx_filename(): - assert ( - ReportDistributionService.extract_ods_code_from_key("a/b/report.csv") - == "report.csv" - ) +@pytest.mark.parametrize( + "key, expected", + [ + ("Report-Orchestration/2026-01-01/Y12345.xlsx", "Y12345"), + ("a/b/C789.XLSX", "C789"), + ("a/b/report.csv", "report.csv"), + ("just-a-name", "just-a-name"), + ("a/b/noext", "noext"), + ], +) +def test_extract_ods_code_from_key(key, expected): + assert ReportDistributionService.extract_ods_code_from_key(key) == expected def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): @@ -81,27 +88,21 @@ def test_list_xlsx_keys_returns_empty_when_no_objects(service, mock_s3_service): keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + mock_s3_service.list_object_keys.assert_called_once_with( + bucket_name="my-bucket", + prefix="Report-Orchestration/2026-01-01/", + ) assert keys == [] - mock_s3_service.list_object_keys.assert_called_once() def test_process_one_report_downloads_encrypts_and_delegates_email( - service, mocker, mock_s3_service + service, mocker, mock_s3_service, fixed_tmpdir ): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", return_value="fixed-password", ) - fake_tmp = "/tmp/fake_tmpdir" - td = mocker.MagicMock() - td.__enter__.return_value = fake_tmp - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, - ) - mocked_zip = mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True, @@ -113,8 +114,8 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( key="Report-Orchestration/2026-01-01/Y12345.xlsx", ) - local_xlsx = os.path.join(fake_tmp, "Y12345.xlsx") - local_zip = os.path.join(fake_tmp, "Y12345.zip") + local_xlsx = os.path.join(fixed_tmpdir, "Y12345.xlsx") + local_zip = os.path.join(fixed_tmpdir, "Y12345.zip") mock_s3_service.download_file.assert_called_once_with( "my-bucket", @@ -133,17 +134,11 @@ def test_process_one_report_downloads_encrypts_and_delegates_email( ) -def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_service): +def test_process_one_report_propagates_download_errors( + service, mocker, mock_s3_service, fixed_tmpdir +): mock_s3_service.download_file.side_effect = RuntimeError("download failed") - td = mocker.MagicMock() - td.__enter__.return_value = "/tmp/fake_tmpdir" - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, - ) - mocked_zip = mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True ) @@ -156,20 +151,13 @@ def test_process_one_report_propagates_download_errors(service, mocker, mock_s3_ mocked_send.assert_not_called() -def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker, mock_s3_service): +def test_process_one_report_does_not_send_email_if_zip_fails( + service, mocker, mock_s3_service, fixed_tmpdir +): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", return_value="pw", ) - - td = mocker.MagicMock() - td.__enter__.return_value = "/tmp/fake_tmpdir" - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, - ) - mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", side_effect=RuntimeError("zip failed"), @@ -184,22 +172,13 @@ def test_process_one_report_does_not_send_email_if_zip_fails(service, mocker, mo def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fails( - service, mocker, mock_s3_service + service, mocker, mock_s3_service, fixed_tmpdir ): mocker.patch( "services.reporting.report_distribution_service.secrets.token_urlsafe", side_effect=RuntimeError("secrets failed"), ) - fake_tmp = "/tmp/fake_tmpdir" - td = mocker.MagicMock() - td.__enter__.return_value = fake_tmp - td.__exit__.return_value = False - mocker.patch( - "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", - return_value=td, - ) - mocked_zip = mocker.patch( "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True, @@ -212,44 +191,35 @@ def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fa mock_s3_service.download_file.assert_called_once_with( "my-bucket", "k.xlsx", - os.path.join(fake_tmp, "Y12345.xlsx"), + os.path.join(fixed_tmpdir, "Y12345.xlsx"), ) mocked_zip.assert_not_called() mocked_send.assert_not_called() -def test_send_report_emails_with_contact_calls_email_contact( - service, mock_contact_repo, mocker +@pytest.mark.parametrize( + "contact_lookup_result, contact_lookup_side_effect, expected_method", + [ + ("contact@example.com", None, "email_contact"), + (None, None, "email_prm_missing_contact"), + (None, RuntimeError("ddb down"), "email_prm_missing_contact"), + ], +) +def test_send_report_emails_routes_correctly( + service, + mock_contact_repo, + mocker, + contact_lookup_result, + contact_lookup_side_effect, + expected_method, ): - mock_contact_repo.get_contact_email.return_value = "contact@example.com" + if contact_lookup_side_effect is not None: + mock_contact_repo.get_contact_email.side_effect = contact_lookup_side_effect + else: + mock_contact_repo.get_contact_email.return_value = contact_lookup_result mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object( - service, "email_prm_missing_contact", autospec=True - ) - - service.send_report_emails( - ods_code="Y12345", - attachment_path="/tmp/Y12345.zip", - password="pw", - ) - - mock_contact_repo.get_contact_email.assert_called_once_with("Y12345") - mocked_email_contact.assert_called_once_with( - to_address="contact@example.com", - attachment_path="/tmp/Y12345.zip", - password="pw", - ) - mocked_email_prm.assert_not_called() - - -def test_send_report_emails_without_contact_calls_email_prm(service, mock_contact_repo, mocker): - mock_contact_repo.get_contact_email.return_value = None - - mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object( - service, "email_prm_missing_contact", autospec=True - ) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) service.send_report_emails( ods_code="A99999", @@ -258,36 +228,21 @@ def test_send_report_emails_without_contact_calls_email_prm(service, mock_contac ) mock_contact_repo.get_contact_email.assert_called_once_with("A99999") - mocked_email_prm.assert_called_once_with( - ods_code="A99999", - attachment_path="/tmp/A99999.zip", - password="pw", - ) - mocked_email_contact.assert_not_called() - - -def test_send_report_emails_contact_lookup_exception_falls_back_to_prm( - service, mock_contact_repo, mocker -): - mock_contact_repo.get_contact_email.side_effect = RuntimeError("ddb down") - - mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) - mocked_email_prm = mocker.patch.object( - service, "email_prm_missing_contact", autospec=True - ) - - service.send_report_emails( - ods_code="A99999", - attachment_path="/tmp/A99999.zip", - password="pw", - ) - mocked_email_contact.assert_not_called() - mocked_email_prm.assert_called_once_with( - ods_code="A99999", - attachment_path="/tmp/A99999.zip", - password="pw", - ) + if expected_method == "email_contact": + mocked_email_contact.assert_called_once_with( + to_address="contact@example.com", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + mocked_email_prm.assert_not_called() + else: + mocked_email_prm.assert_called_once_with( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + mocked_email_contact.assert_not_called() def test_email_contact_sends_report_and_password(service, mock_email_service): diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index baf54577fe..85685c8449 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -23,158 +23,135 @@ def report_orchestration_service(mock_repository, mock_excel_generator): ) -def test_process_reporting_window_no_records_returns_empty_dict_and_does_not_generate( - report_orchestration_service, mock_repository, mock_excel_generator -): - mock_repository.get_records_for_time_window.return_value = [] - - result = report_orchestration_service.process_reporting_window(100, 200) - - assert result == {} - mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() - - -def test_process_reporting_window_calls_repository_with_window_args( - report_orchestration_service, mock_repository, mocker -): - mock_repository.get_records_for_time_window.return_value = [{"UploaderOdsCode": "X1", "ID": 1}] - mocked_generate = mocker.patch.object(report_orchestration_service, "generate_ods_report", return_value="/tmp/x.xlsx") - - report_orchestration_service.process_reporting_window(100, 200) - - mock_repository.get_records_for_time_window.assert_called_once_with(100, 200) - mocked_generate.assert_called_once() - - -def test_process_reporting_window_generates_reports_per_ods( - report_orchestration_service, mock_repository, mocker -): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - {"UploaderOdsCode": "A99999", "ID": 3}, - ] - mock_repository.get_records_for_time_window.return_value = records - - mocked_generate = mocker.patch.object( - report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" - ) - - report_orchestration_service.process_reporting_window(100, 200) - - mocked_generate.assert_any_call( - "Y12345", - [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - ], - ) - mocked_generate.assert_any_call( - "A99999", - [{"UploaderOdsCode": "A99999", "ID": 3}], - ) - assert mocked_generate.call_count == 2 - - -def test_process_reporting_window_returns_mapping_of_ods_to_generated_file_path( - report_orchestration_service, mock_repository, mocker -): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "A99999", "ID": 2}, - ] - mock_repository.get_records_for_time_window.return_value = records - - def _side_effect(ods_code, ods_records): - return f"/tmp/{ods_code}.xlsx" - - mocker.patch.object( +@pytest.fixture +def mocked_generate(report_orchestration_service, mocker): + return mocker.patch.object( report_orchestration_service, "generate_ods_report", - side_effect=_side_effect, + autospec=True, + side_effect=lambda ods_code, _records: f"/tmp/{ods_code}.xlsx", ) - result = report_orchestration_service.process_reporting_window(100, 200) - - assert result == { - "Y12345": "/tmp/Y12345.xlsx", - "A99999": "/tmp/A99999.xlsx", - } - -def test_process_reporting_window_includes_unknown_ods_group( - report_orchestration_service, mock_repository, mocker +@pytest.mark.parametrize( + "records, expected_generate_calls, expected_result", + [ + ( + [], + [], + {}, + ), + ( + [{"UploaderOdsCode": "X1", "ID": 1}], + [ + ("X1", [{"UploaderOdsCode": "X1", "ID": 1}]), + ], + {"X1": "/tmp/X1.xlsx"}, + ), + ( + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + ], + [ + ("Y12345", [{"UploaderOdsCode": "Y12345", "ID": 1}, {"UploaderOdsCode": "Y12345", "ID": 2}]), + ("A99999", [{"UploaderOdsCode": "A99999", "ID": 3}]), + ], + {"Y12345": "/tmp/Y12345.xlsx", "A99999": "/tmp/A99999.xlsx"}, + ), + ( + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"ID": 2}, + {"UploaderOdsCode": None, "ID": 3}, + ], + [ + ("Y12345", [{"UploaderOdsCode": "Y12345", "ID": 1}]), + ("UNKNOWN", [{"ID": 2}, {"UploaderOdsCode": None, "ID": 3}]), + ], + {"Y12345": "/tmp/Y12345.xlsx", "UNKNOWN": "/tmp/UNKNOWN.xlsx"}, + ), + ( + [{"UploaderOdsCode": "", "ID": 1}], + [ + ("UNKNOWN", [{"UploaderOdsCode": "", "ID": 1}]), + ], + {"UNKNOWN": "/tmp/UNKNOWN.xlsx"}, + ), + ], +) +def test_process_reporting_window_behaviour( + report_orchestration_service, + mock_repository, + mocked_generate, + records, + expected_generate_calls, + expected_result, ): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"ID": 2}, # missing ODS -> UNKNOWN - {"UploaderOdsCode": None, "ID": 3}, # null ODS -> UNKNOWN - ] mock_repository.get_records_for_time_window.return_value = records - mocked_generate = mocker.patch.object( - report_orchestration_service, "generate_ods_report", return_value="/tmp/ignored.xlsx" - ) - - report_orchestration_service.process_reporting_window(100, 200) - - # Expect 2 groups: Y12345 and UNKNOWN - assert mocked_generate.call_count == 2 - mocked_generate.assert_any_call( - "Y12345", - [{"UploaderOdsCode": "Y12345", "ID": 1}], - ) - mocked_generate.assert_any_call( - "UNKNOWN", - [{"ID": 2}, {"UploaderOdsCode": None, "ID": 3}], - ) - - -def test_group_records_by_ods_groups_correctly(): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - {"UploaderOdsCode": "A99999", "ID": 3}, - {"ID": 4}, # missing ODS - {"UploaderOdsCode": None, "ID": 5}, # null ODS - ] - - result = ReportOrchestrationService.group_records_by_ods(records) - - assert result["Y12345"] == [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - ] - assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] - assert result["UNKNOWN"] == [ - {"ID": 4}, - {"UploaderOdsCode": None, "ID": 5}, - ] - - -def test_group_records_by_ods_empty_input_returns_empty_mapping(): - result = ReportOrchestrationService.group_records_by_ods([]) - assert dict(result) == {} + result = report_orchestration_service.process_reporting_window(100, 200) + mock_repository.get_records_for_time_window.assert_called_once_with(100, 200) -def test_group_records_by_ods_treats_empty_string_as_unknown(): - records = [{"UploaderOdsCode": "", "ID": 1}] + assert mocked_generate.call_count == len(expected_generate_calls) + for ods_code, ods_records in expected_generate_calls: + mocked_generate.assert_any_call(ods_code, ods_records) + + assert result == expected_result + + +@pytest.mark.parametrize( + "records, expected", + [ + ( + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + {"ID": 4}, # missing ODS + {"UploaderOdsCode": None, "ID": 5}, # null ODS + ], + { + "Y12345": [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ], + "A99999": [{"UploaderOdsCode": "A99999", "ID": 3}], + "UNKNOWN": [{"ID": 4}, {"UploaderOdsCode": None, "ID": 5}], + }, + ), + ([], {}), + ( + [{"UploaderOdsCode": "", "ID": 1}], + {"UNKNOWN": [{"UploaderOdsCode": "", "ID": 1}]}, + ), + ], +) +def test_group_records_by_ods(records, expected): result = ReportOrchestrationService.group_records_by_ods(records) - assert result["UNKNOWN"] == [{"UploaderOdsCode": "", "ID": 1}] + assert dict(result) == expected -def test_generate_ods_report_creates_excel_report_and_returns_path( - report_orchestration_service, mock_excel_generator, mocker -): +@pytest.fixture +def fake_named_tmpfile(mocker): fake_tmp = mocker.MagicMock() fake_tmp.__enter__.return_value = fake_tmp + fake_tmp.__exit__.return_value = False fake_tmp.name = "/tmp/fake_Y12345.xlsx" mocked_ntf = mocker.patch( "services.reporting.report_orchestration_service.tempfile.NamedTemporaryFile", return_value=fake_tmp, ) + return mocked_ntf, fake_tmp + +def test_generate_ods_report_creates_excel_report_and_returns_path( + report_orchestration_service, mock_excel_generator, fake_named_tmpfile +): + mocked_ntf, fake_tmp = fake_named_tmpfile records = [{"ID": 1, "UploaderOdsCode": "Y12345"}] result_path = report_orchestration_service.generate_ods_report("Y12345", records) From a755a54b537d6ff9450153f35c26270e31c0c6c1 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 30 Jan 2026 16:24:07 +0000 Subject: [PATCH 34/37] [PRMP-1057-2] fixed tests --- lambdas/tests/unit/handlers/conftest.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index c744cb885b..70eedc7414 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -1,4 +1,5 @@ import os +from unittest.mock import MagicMock import pytest from enums.feature_flags import FeatureFlags @@ -6,7 +7,6 @@ from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.feature_flags_service import FeatureFlagService - @pytest.fixture def valid_id_event_without_auth_header(): api_gateway_proxy_event = { @@ -297,11 +297,6 @@ def mock_report_distribution_wiring(mocker): return svc_instance -from unittest.mock import MagicMock - -import pytest - - @pytest.fixture def mock_report_orchestration_wiring(mocker): logger = mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) From 2361c309227be6c3daa4be5f29fb13f37ed831d7 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 5 Feb 2026 15:43:09 +0000 Subject: [PATCH 35/37] [PRMP-1057-2] update handlers ans services --- .../handlers/report_distribution_handler.py | 14 +--- .../handlers/report_orchestration_handler.py | 19 +---- .../reporting/report_contact_repository.py | 6 +- .../reporting/reporting_dynamo_repository.py | 5 +- .../reporting/report_distribution_service.py | 15 ++-- .../reporting/report_orchestration_service.py | 8 +- lambdas/tests/unit/handlers/conftest.py | 63 +++++++--------- .../test_report_distribution_handler.py | 54 +++---------- .../test_report_orchestration_handler.py | 49 +++++------- .../test_report_contact_repository.py | 25 ++++--- .../test_reporting_dynamo_repository.py | 8 ++ .../test_report_distribution_service.py | 75 ++++++++++++++++--- .../test_report_orchestration_service.py | 54 ++++++++++--- 13 files changed, 209 insertions(+), 186 deletions(-) diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py index af6c0f2c7e..9936a7559e 100644 --- a/lambdas/handlers/report_distribution_handler.py +++ b/lambdas/handlers/report_distribution_handler.py @@ -38,21 +38,9 @@ def lambda_handler(event, context) -> Dict[str, Any]: raise ReportDistributionException(400, LambdaError.InvalidAction) bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] - contact_table = os.environ["CONTACT_TABLE_NAME"] - prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] - from_address = os.environ["SES_FROM_ADDRESS"] - - s3_service = S3Service() - contact_repo = ReportContactRepository(contact_table) - email_service = EmailService() service = ReportDistributionService( - s3_service=s3_service, - contact_repo=contact_repo, - email_service=email_service, - bucket=bucket, - from_address=from_address, - prm_mailbox=prm_mailbox, + bucket=bucket ) response = {"status": "ok", "bucket": bucket} diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index 38a450e76f..73ce6c6850 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -34,19 +34,6 @@ def build_s3_key(ods_code: str, report_date: str) -> str: return f"Report-Orchestration/{report_date}/{ods_code}.xlsx" -def get_config() -> Tuple[str, str]: - return os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"], os.environ["REPORT_BUCKET_NAME"] - - -def build_services(table_name: str) -> Tuple[ReportOrchestrationService, S3Service]: - repository = ReportingDynamoRepository(table_name) - excel_generator = ExcelReportGenerator() - orchestration_service = ReportOrchestrationService( - repository=repository, - excel_generator=excel_generator, - ) - return orchestration_service, S3Service() - def upload_generated_reports( s3_service: S3Service, @@ -91,8 +78,10 @@ def build_response(report_date: str, bucket: str, keys: list[str]) -> Dict[str, def lambda_handler(event, context) -> Dict[str, Any]: logger.info("Report orchestration lambda invoked") - table_name, report_bucket = get_config() - orchestration_service, s3_service = build_services(table_name) + report_bucket = os.environ["REPORT_BUCKET_NAME"] + orchestration_service = ReportOrchestrationService( + ) + s3_service = S3Service() window_start, window_end = calculate_reporting_window() report_date = get_report_date_folder() diff --git a/lambdas/repositories/reporting/report_contact_repository.py b/lambdas/repositories/reporting/report_contact_repository.py index 84bd06cab1..d9c21c0c2e 100644 --- a/lambdas/repositories/reporting/report_contact_repository.py +++ b/lambdas/repositories/reporting/report_contact_repository.py @@ -1,9 +1,11 @@ +import os + from services.base.dynamo_service import DynamoDBService class ReportContactRepository: - def __init__(self, table_name: str): - self.table_name = table_name + def __init__(self): + self.table_name = os.environ["CONTACT_TABLE_NAME"] self.dynamo = DynamoDBService() def get_contact_email(self, ods_code: str) -> str | None: diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 86d595d87e..9bb017a7cc 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,3 +1,4 @@ +import os from datetime import timedelta from typing import Dict, List @@ -9,8 +10,8 @@ logger = LoggingService(__name__) class ReportingDynamoRepository: - def __init__(self, table_name: str): - self.table_name = table_name + def __init__(self): + self.table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] self.dynamo_service = DynamoDBService() def get_records_for_time_window( diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py index d93274e5e1..5dd75e8f46 100644 --- a/lambdas/services/reporting/report_distribution_service.py +++ b/lambdas/services/reporting/report_distribution_service.py @@ -18,19 +18,14 @@ class ReportDistributionService: def __init__( self, *, - s3_service: S3Service, - contact_repo: ReportContactRepository, - email_service: EmailService, bucket: str, - from_address: str, - prm_mailbox: str, ): - self.s3_service = s3_service - self.contact_repo = contact_repo - self.email_service = email_service + self.s3_service = S3Service() + self.contact_repo = ReportContactRepository() + self.email_service = EmailService() self.bucket = bucket - self.from_address = from_address - self.prm_mailbox = prm_mailbox + self.from_address = os.environ["SES_FROM_ADDRESS"] + self.prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] @staticmethod def extract_ods_code_from_key(key: str) -> str: diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index 69a919cc6e..3842419284 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -2,15 +2,17 @@ from collections import defaultdict from typing import Dict +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from services.reporting.excel_report_generator_service import ExcelReportGenerator from utils.audit_logging_setup import LoggingService logger = LoggingService(__name__) class ReportOrchestrationService: - def __init__(self, repository, excel_generator): - self.repository = repository - self.excel_generator = excel_generator + def __init__(self): + self.repository = ReportingDynamoRepository() + self.excel_generator = ExcelReportGenerator() def process_reporting_window( self, diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index 70eedc7414..f0f72bab4a 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -273,56 +273,51 @@ def report_distribution_process_one_event(): @pytest.fixture def mock_report_distribution_wiring(mocker): svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") - - mocker.patch( - "handlers.report_distribution_handler.S3Service", - autospec=True, - return_value=mocker.Mock(), - ) - mocker.patch( - "handlers.report_distribution_handler.ReportContactRepository", - autospec=True, - return_value=mocker.Mock(), - ) - mocker.patch( - "handlers.report_distribution_handler.EmailService", - autospec=True, - return_value=mocker.Mock(), - ) mocker.patch( "handlers.report_distribution_handler.ReportDistributionService", autospec=True, return_value=svc_instance, ) - return svc_instance +import pytest + + @pytest.fixture def mock_report_orchestration_wiring(mocker): - logger = mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) + from handlers import report_orchestration_handler as handler_module - mock_window = mocker.patch( - "handlers.report_orchestration_handler.calculate_reporting_window", - return_value=(100, 200), + orchestration_service = mocker.Mock(name="ReportOrchestrationServiceInstance") + s3_service = mocker.Mock(name="S3ServiceInstance") + + mocker.patch.object( + handler_module, + "ReportOrchestrationService", + autospec=True, + return_value=orchestration_service, ) - mock_report_date = mocker.patch( - "handlers.report_orchestration_handler.get_report_date_folder", - return_value="2026-01-02", + mocker.patch.object( + handler_module, + "S3Service", + autospec=True, + return_value=s3_service, ) - orchestration_service = MagicMock(name="ReportOrchestrationService") - s3_service = MagicMock(name="S3Service") - - build_services = mocker.patch( - "handlers.report_orchestration_handler.build_services", - return_value=(orchestration_service, s3_service), + mock_window = mocker.patch.object( + handler_module, + "calculate_reporting_window", + return_value=(100, 200), + ) + mock_report_date = mocker.patch.object( + handler_module, + "get_report_date_folder", + return_value="2026-01-02", ) return { - "logger": logger, - "mock_window": mock_window, - "mock_report_date": mock_report_date, - "build_services": build_services, + "handler_module": handler_module, "orchestration_service": orchestration_service, "s3_service": s3_service, + "mock_window": mock_window, + "mock_report_date": mock_report_date, } diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py index 9e5e2cb5b2..d70178948e 100644 --- a/lambdas/tests/unit/handlers/test_report_distribution_handler.py +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -1,41 +1,21 @@ -import importlib import json import os -import pytest -MODULE_UNDER_TEST = "handlers.report_distribution_handler" +from handlers import report_distribution_handler as handler_module -@pytest.fixture -def handler_module(): - return importlib.import_module(MODULE_UNDER_TEST) - - -def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( - mocker, handler_module, required_report_distribution_env, lambda_context +def test_lambda_handler_wires_service_and_returns_result_list_mode( + mocker, required_report_distribution_env, lambda_context ): - event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "reports/2026-01-01/"} - - s3_instance = mocker.Mock(name="S3ServiceInstance") - contact_repo_instance = mocker.Mock(name="ReportContactRepositoryInstance") - email_instance = mocker.Mock(name="EmailServiceInstance") + event = { + "action": handler_module.ReportDistributionAction.LIST, + "prefix": "reports/2026-01-01/", + } svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") svc_instance.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] - mocked_s3_cls = mocker.patch.object( - handler_module, "S3Service", autospec=True, return_value=s3_instance - ) - mocked_contact_repo_cls = mocker.patch.object( - handler_module, - "ReportContactRepository", - autospec=True, - return_value=contact_repo_instance, - ) - mocked_email_cls = mocker.patch.object( - handler_module, "EmailService", autospec=True, return_value=email_instance - ) mocked_dist_svc_cls = mocker.patch.object( handler_module, "ReportDistributionService", @@ -45,19 +25,7 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( result = handler_module.lambda_handler(event, lambda_context) - mocked_s3_cls.assert_called_once_with() - mocked_contact_repo_cls.assert_called_once_with("contact-table") - mocked_email_cls.assert_called_once_with() - - mocked_dist_svc_cls.assert_called_once_with( - s3_service=s3_instance, - contact_repo=contact_repo_instance, - email_service=email_instance, - bucket="my-report-bucket", - from_address="from@example.com", - prm_mailbox="prm@example.com", - ) - + mocked_dist_svc_cls.assert_called_once_with(bucket="my-report-bucket") svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") assert result == { @@ -69,7 +37,6 @@ def test_lambda_handler_wires_dependencies_and_returns_result_list_mode( def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( - handler_module, required_report_distribution_env, lambda_context, report_distribution_list_event, @@ -90,7 +57,6 @@ def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( def test_lambda_handler_process_one_mode_happy_path( - handler_module, required_report_distribution_env, lambda_context, report_distribution_process_one_event, @@ -119,7 +85,7 @@ def test_lambda_handler_process_one_mode_happy_path( def test_lambda_handler_returns_400_when_action_invalid( - handler_module, required_report_distribution_env, lambda_context + required_report_distribution_env, lambda_context ): event = {"action": "nope"} @@ -136,7 +102,7 @@ def test_lambda_handler_returns_400_when_action_invalid( assert body["interaction_id"] == lambda_context.aws_request_id -def test_lambda_handler_returns_500_when_required_env_missing(mocker, handler_module, lambda_context): +def test_lambda_handler_returns_500_when_required_env_missing(mocker, lambda_context): mocker.patch.dict( os.environ, { diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 241a12e3fb..3f4eb05632 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,24 +1,17 @@ -import importlib import json -import pytest - -MODULE_UNDER_TEST = "handlers.report_orchestration_handler" - - -@pytest.fixture -def handler_module(): - return importlib.import_module(MODULE_UNDER_TEST) +from handlers import report_orchestration_handler as handler_module def test_lambda_handler_calls_service_and_returns_expected_response( - handler_module, required_report_orchestration_env, lambda_context, mock_report_orchestration_wiring, ): orchestration_service = mock_report_orchestration_wiring["orchestration_service"] s3_service = mock_report_orchestration_wiring["s3_service"] + mock_window = mock_report_orchestration_wiring["mock_window"] + mock_report_date = mock_report_orchestration_wiring["mock_report_date"] orchestration_service.process_reporting_window.return_value = { "A12345": "/tmp/A12345.xlsx", @@ -27,44 +20,43 @@ def test_lambda_handler_calls_service_and_returns_expected_response( result = handler_module.lambda_handler(event={}, context=lambda_context) - mock_report_orchestration_wiring["build_services"].assert_called_once_with("TestTable") + mock_window.assert_called_once() + mock_report_date.assert_called_once() orchestration_service.process_reporting_window.assert_called_once_with( window_start_ts=100, window_end_ts=200, ) - assert s3_service.upload_file_with_extra_args.call_count == 2 - assert result["report_date"] == "2026-01-02" - assert result["bucket"] == "test-report-bucket" - assert result["prefix"] == "Report-Orchestration/2026-01-02/" - assert set(result["keys"]) == { - "Report-Orchestration/2026-01-02/A12345.xlsx", - "Report-Orchestration/2026-01-02/B67890.xlsx", + assert result == { + "status": "ok", + "report_date": "2026-01-02", + "bucket": "test-report-bucket", + "prefix": "Report-Orchestration/2026-01-02/", + "keys": [ + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/B67890.xlsx", + ], } - mock_report_orchestration_wiring["logger"].info.assert_any_call( - "Report orchestration lambda invoked" - ) - def test_lambda_handler_calls_window_function( - handler_module, required_report_orchestration_env, lambda_context, mock_report_orchestration_wiring, ): orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + mock_window = mock_report_orchestration_wiring["mock_window"] + orchestration_service.process_reporting_window.return_value = {} handler_module.lambda_handler(event={}, context=lambda_context) - mock_report_orchestration_wiring["mock_window"].assert_called_once() + mock_window.assert_called_once() def test_lambda_handler_returns_empty_keys_when_no_reports_generated( - handler_module, required_report_orchestration_env, lambda_context, mock_report_orchestration_wiring, @@ -83,15 +75,10 @@ def test_lambda_handler_returns_empty_keys_when_no_reports_generated( "prefix": "Report-Orchestration/2026-01-02/", "keys": [], } - s3_service.upload_file_with_extra_args.assert_not_called() - mock_report_orchestration_wiring["logger"].info.assert_any_call( - "No reports generated; exiting" - ) def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( - handler_module, required_report_orchestration_env, lambda_context, mock_report_orchestration_wiring, @@ -114,7 +101,6 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( file_key="Report-Orchestration/2026-01-02/A12345.xlsx", extra_args={"ServerSideEncryption": "aws:kms"}, ) - s3_service.upload_file_with_extra_args.assert_any_call( file_name="/tmp/UNKNOWN.xlsx", s3_bucket_name="test-report-bucket", @@ -129,7 +115,6 @@ def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( def test_lambda_handler_returns_error_when_required_env_missing( - handler_module, lambda_context, monkeypatch, ): diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py index 4216c3407c..13f91bac99 100644 --- a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -3,19 +3,25 @@ from repositories.reporting.report_contact_repository import ReportContactRepository +@pytest.fixture +def required_contact_repo_env(monkeypatch): + monkeypatch.setenv("CONTACT_TABLE_NAME", "report-contacts") + + @pytest.fixture def mock_dynamo(mocker): - dynamo = mocker.Mock() + dynamo = mocker.Mock(name="DynamoDBServiceInstance") mocker.patch( "repositories.reporting.report_contact_repository.DynamoDBService", + autospec=True, return_value=dynamo, ) return dynamo @pytest.fixture -def repo(mock_dynamo): - return ReportContactRepository(table_name="report-contacts") +def repo(required_contact_repo_env, mock_dynamo): + return ReportContactRepository() @pytest.mark.parametrize( @@ -31,14 +37,7 @@ def repo(mock_dynamo): "contact@example.com", ), ({}, None), - ( - { - "Item": { - "OdsCode": "Y12345", - } - }, - None, - ), + ({"Item": {"OdsCode": "Y12345"}}, None), (None, None), ], ) @@ -52,3 +51,7 @@ def test_get_contact_email(repo, mock_dynamo, dynamo_response, expected_email): key={"OdsCode": "Y12345"}, ) assert result == expected_email + + +def test_init_reads_table_name_from_env(repo): + assert repo.table_name == "report-contacts" diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index 40f193c2df..21bd57f3a8 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -2,6 +2,14 @@ import pytest +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository + + +@pytest.fixture +def reporting_repo(monkeypatch, mock_reporting_dynamo_service): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + return ReportingDynamoRepository() + @pytest.mark.parametrize( "start_dt,end_dt,service_side_effect,expected,expected_call_count", diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py index b8ecde73fe..db8b1e3bff 100644 --- a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -5,34 +5,55 @@ from services.reporting.report_distribution_service import ReportDistributionService +@pytest.fixture +def required_report_distribution_env(monkeypatch): + monkeypatch.setenv("SES_FROM_ADDRESS", "from@example.com") + monkeypatch.setenv("PRM_MAILBOX_EMAIL", "prm@example.com") + + @pytest.fixture def mock_s3_service(mocker): - return mocker.Mock(name="S3Service") + return mocker.Mock(name="S3ServiceInstance") @pytest.fixture def mock_contact_repo(mocker): - repo = mocker.Mock(name="ReportContactRepository") + repo = mocker.Mock(name="ReportContactRepositoryInstance") repo.get_contact_email.return_value = None return repo @pytest.fixture def mock_email_service(mocker): - return mocker.Mock(name="EmailService") + return mocker.Mock(name="EmailServiceInstance") @pytest.fixture -def service(mock_s3_service, mock_contact_repo, mock_email_service): - return ReportDistributionService( - s3_service=mock_s3_service, - contact_repo=mock_contact_repo, - email_service=mock_email_service, - bucket="my-bucket", - from_address="from@example.com", - prm_mailbox="prm@example.com", +def service( + required_report_distribution_env, + mocker, + mock_s3_service, + mock_contact_repo, + mock_email_service, +): + mocker.patch( + "services.reporting.report_distribution_service.S3Service", + autospec=True, + return_value=mock_s3_service, + ) + mocker.patch( + "services.reporting.report_distribution_service.ReportContactRepository", + autospec=True, + return_value=mock_contact_repo, + ) + mocker.patch( + "services.reporting.report_distribution_service.EmailService", + autospec=True, + return_value=mock_email_service, ) + return ReportDistributionService(bucket="my-bucket") + @pytest.fixture def fixed_tmpdir(mocker): @@ -61,6 +82,37 @@ def test_extract_ods_code_from_key(key, expected): assert ReportDistributionService.extract_ods_code_from_key(key) == expected +def test_init_reads_env_and_wires_dependencies(required_report_distribution_env, mocker): + mock_s3 = mocker.Mock(name="S3ServiceInstance") + mock_repo = mocker.Mock(name="ReportContactRepositoryInstance") + mock_email = mocker.Mock(name="EmailServiceInstance") + + mocker.patch( + "services.reporting.report_distribution_service.S3Service", + autospec=True, + return_value=mock_s3, + ) + mocker.patch( + "services.reporting.report_distribution_service.ReportContactRepository", + autospec=True, + return_value=mock_repo, + ) + mocker.patch( + "services.reporting.report_distribution_service.EmailService", + autospec=True, + return_value=mock_email, + ) + + svc = ReportDistributionService(bucket="bucket-1") + + assert svc.bucket == "bucket-1" + assert svc.from_address == "from@example.com" + assert svc.prm_mailbox == "prm@example.com" + assert svc.s3_service is mock_s3 + assert svc.contact_repo is mock_repo + assert svc.email_service is mock_email + + def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): mock_s3_service.list_object_keys.return_value = [ "Report-Orchestration/2026-01-01/A123.xlsx", @@ -68,6 +120,7 @@ def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): "Report-Orchestration/2026-01-01/B456.xls", "Report-Orchestration/2026-01-01/C789.xlsx", "Report-Orchestration/2026-01-01/D000.xlsx", + "Report-Orchestration/2026-01-01/E111.xlsx.tmp", ] keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index 85685c8449..2487fb235c 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -5,22 +5,29 @@ @pytest.fixture def mock_repository(mocker): - repo = mocker.Mock() + repo = mocker.Mock(name="ReportingDynamoRepositoryInstance") repo.get_records_for_time_window.return_value = [] return repo @pytest.fixture def mock_excel_generator(mocker): - return mocker.Mock() + return mocker.Mock(name="ExcelReportGeneratorInstance") @pytest.fixture -def report_orchestration_service(mock_repository, mock_excel_generator): - return ReportOrchestrationService( - repository=mock_repository, - excel_generator=mock_excel_generator, +def report_orchestration_service(mocker, mock_repository, mock_excel_generator): + mocker.patch( + "services.reporting.report_orchestration_service.ReportingDynamoRepository", + autospec=True, + return_value=mock_repository, + ) + mocker.patch( + "services.reporting.report_orchestration_service.ExcelReportGenerator", + autospec=True, + return_value=mock_excel_generator, ) + return ReportOrchestrationService() @pytest.fixture @@ -55,7 +62,13 @@ def mocked_generate(report_orchestration_service, mocker): {"UploaderOdsCode": "A99999", "ID": 3}, ], [ - ("Y12345", [{"UploaderOdsCode": "Y12345", "ID": 1}, {"UploaderOdsCode": "Y12345", "ID": 2}]), + ( + "Y12345", + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ], + ), ("A99999", [{"UploaderOdsCode": "A99999", "ID": 3}]), ], {"Y12345": "/tmp/Y12345.xlsx", "A99999": "/tmp/A99999.xlsx"}, @@ -110,8 +123,8 @@ def test_process_reporting_window_behaviour( {"UploaderOdsCode": "Y12345", "ID": 1}, {"UploaderOdsCode": "Y12345", "ID": 2}, {"UploaderOdsCode": "A99999", "ID": 3}, - {"ID": 4}, # missing ODS - {"UploaderOdsCode": None, "ID": 5}, # null ODS + {"ID": 4}, + {"UploaderOdsCode": None, "ID": 5}, ], { "Y12345": [ @@ -168,3 +181,26 @@ def test_generate_ods_report_creates_excel_report_and_returns_path( records=records, output_path=fake_tmp.name, ) + + +def test_init_constructs_repository_and_excel_generator(mocker): + mock_repo = mocker.Mock(name="ReportingDynamoRepositoryInstance") + mock_excel = mocker.Mock(name="ExcelReportGeneratorInstance") + + mocked_repo_cls = mocker.patch( + "services.reporting.report_orchestration_service.ReportingDynamoRepository", + autospec=True, + return_value=mock_repo, + ) + mocked_excel_cls = mocker.patch( + "services.reporting.report_orchestration_service.ExcelReportGenerator", + autospec=True, + return_value=mock_excel, + ) + + svc = ReportOrchestrationService() + + mocked_repo_cls.assert_called_once_with() + mocked_excel_cls.assert_called_once_with() + assert svc.repository is mock_repo + assert svc.excel_generator is mock_excel From 59f05313df0833a763b625f299c773d3bacd1ee7 Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Thu, 5 Feb 2026 16:25:00 +0000 Subject: [PATCH 36/37] [PRMP-1057-2] small fix --- lambdas/tests/unit/handlers/conftest.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index f0f72bab4a..caaade5496 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -280,8 +280,6 @@ def mock_report_distribution_wiring(mocker): ) return svc_instance -import pytest - @pytest.fixture def mock_report_orchestration_wiring(mocker): From d8c30057808121222e9b5db9a20689bb5ac7cc0d Mon Sep 17 00:00:00 2001 From: PedroSoaresNHS Date: Fri, 6 Feb 2026 13:08:00 +0000 Subject: [PATCH 37/37] [PRMP-1057] updated poetry.lock --- poetry.lock | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index 8e0c2d78cd..7fa78253f3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2058,6 +2058,21 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pyzipper" +version = "0.3.6" +description = "AES encryption for zipfile." +optional = false +python-versions = ">=3.4" +groups = ["reports-lambda"] +files = [ + {file = "pyzipper-0.3.6-py2.py3-none-any.whl", hash = "sha256:6d097f465bfa47796b1494e12ea65d1478107d38e13bc56f6e58eedc4f6c1a87"}, + {file = "pyzipper-0.3.6.tar.gz", hash = "sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc"}, +] + +[package.dependencies] +pycryptodomex = "*" + [[package]] name = "regex" version = "2023.12.25" @@ -2501,4 +2516,4 @@ requests = "*" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "77d0249d2dd6c9fbb02b0e434ade72ac221f4df82e8d98ec49250fe0b7ff74df" +content-hash = "853336b823ceae919933474298444b50eb7935088cf43b92d713c8326f90e5be"