diff --git a/.github/workflows/base-lambdas-reusable-deploy-all.yml b/.github/workflows/base-lambdas-reusable-deploy-all.yml index 89722bf25d..c12ad8bda3 100644 --- a/.github/workflows/base-lambdas-reusable-deploy-all.yml +++ b/.github/workflows/base-lambdas-reusable-deploy-all.yml @@ -806,7 +806,21 @@ jobs: build_branch: ${{ inputs.build_branch }} sandbox: ${{ inputs.sandbox }} lambda_handler_name: report_orchestration_handler - lambda_aws_name: reportOrchestration + lambda_aws_name: ReportOrchestration + lambda_layer_names: "core_lambda_layer,reports_lambda_layer" + secrets: + AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} + + deploy_report_distribution_lambda: + name: Deploy Report Distribution + uses: ./.github/workflows/base-lambdas-reusable-deploy.yml + with: + environment: ${{ inputs.environment }} + python_version: ${{ inputs.python_version }} + build_branch: ${{ inputs.build_branch }} + sandbox: ${{ inputs.sandbox }} + lambda_handler_name: report_distribution_handler + lambda_aws_name: ReportDistribution lambda_layer_names: "core_lambda_layer,reports_lambda_layer" secrets: AWS_ASSUME_ROLE: ${{ secrets.AWS_ASSUME_ROLE }} diff --git a/lambdas/enums/lambda_error.py b/lambdas/enums/lambda_error.py index 16269d9d23..871425dade 100644 --- a/lambdas/enums/lambda_error.py +++ b/lambdas/enums/lambda_error.py @@ -729,3 +729,7 @@ def create_error_body( "message": "An internal server error occurred", "fhir_coding": FhirIssueCoding.EXCEPTION, } + InvalidAction = { + "err_code": "RD_IA", + "message": "Invalid action. Expected 'list' or 'process_one'.", + } diff --git a/lambdas/enums/report_distribution_action.py b/lambdas/enums/report_distribution_action.py new file mode 100644 index 0000000000..903314b771 --- /dev/null +++ b/lambdas/enums/report_distribution_action.py @@ -0,0 +1,6 @@ +from enum import StrEnum + + +class ReportDistributionAction(StrEnum): + LIST = "list" + PROCESS_ONE = "process_one" \ No newline at end of file diff --git a/lambdas/handlers/report_distribution_handler.py b/lambdas/handlers/report_distribution_handler.py new file mode 100644 index 0000000000..9936a7559e --- /dev/null +++ b/lambdas/handlers/report_distribution_handler.py @@ -0,0 +1,60 @@ +import os +from typing import Any, Dict + +from enums.lambda_error import LambdaError +from enums.report_distribution_action import ReportDistributionAction +from repositories.reporting.report_contact_repository import ReportContactRepository +from services.base.s3_service import S3Service +from services.email_service import EmailService +from services.reporting.report_distribution_service import ReportDistributionService +from utils.audit_logging_setup import LoggingService +from utils.decorators.ensure_env_var import ensure_environment_variables +from utils.decorators.handle_lambda_exceptions import handle_lambda_exceptions +from utils.decorators.override_error_check import override_error_check +from utils.decorators.set_audit_arg import set_request_context_for_logging +from utils.lambda_exceptions import ReportDistributionException + +logger = LoggingService(__name__) + + +@ensure_environment_variables( + names=[ + "REPORT_BUCKET_NAME", + "CONTACT_TABLE_NAME", + "PRM_MAILBOX_EMAIL", + "SES_FROM_ADDRESS", + ] +) +@override_error_check +@handle_lambda_exceptions +@set_request_context_for_logging +def lambda_handler(event, context) -> Dict[str, Any]: + action = event.get("action") + if action not in { + ReportDistributionAction.LIST, + ReportDistributionAction.PROCESS_ONE, + }: + logger.error("Invalid action. Expected 'list' or 'process_one'.") + raise ReportDistributionException(400, LambdaError.InvalidAction) + + bucket = event.get("bucket") or os.environ["REPORT_BUCKET_NAME"] + + service = ReportDistributionService( + bucket=bucket + ) + + response = {"status": "ok", "bucket": bucket} + + if action == ReportDistributionAction.LIST: + prefix = event["prefix"] + keys = service.list_xlsx_keys(prefix=prefix) + logger.info(f"List mode: returning {len(keys)} key(s) for prefix={prefix}") + response.update({"prefix": prefix, "keys": keys}) + else: + key = event["key"] + ods_code = service.extract_ods_code_from_key(key) + service.process_one_report(ods_code=ods_code, key=key) + logger.info(f"Process-one mode: processed ods={ods_code}, key={key}") + response.update({"key": key, "ods_code": ods_code}) + + return response diff --git a/lambdas/handlers/report_orchestration_handler.py b/lambdas/handlers/report_orchestration_handler.py index f2d4d59729..73ce6c6850 100644 --- a/lambdas/handlers/report_orchestration_handler.py +++ b/lambdas/handlers/report_orchestration_handler.py @@ -1,8 +1,9 @@ import os -import tempfile from datetime import datetime, timedelta, timezone +from typing import Any, Dict, Tuple from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from services.base.s3_service import S3Service from services.reporting.excel_report_generator_service import ExcelReportGenerator from services.reporting.report_orchestration_service import ReportOrchestrationService from utils.audit_logging_setup import LoggingService @@ -14,7 +15,7 @@ logger = LoggingService(__name__) -def calculate_reporting_window(): +def calculate_reporting_window() -> Tuple[int, int]: now = datetime.now(timezone.utc) today_7am = now.replace(hour=7, minute=0, second=0, microsecond=0) @@ -22,34 +23,84 @@ def calculate_reporting_window(): today_7am -= timedelta(days=1) yesterday_7am = today_7am - timedelta(days=1) + return int(yesterday_7am.timestamp()), int(today_7am.timestamp()) - return ( - int(yesterday_7am.timestamp()), - int(today_7am.timestamp()), - ) + +def get_report_date_folder() -> str: + return datetime.now(timezone.utc).strftime("%Y-%m-%d") + + +def build_s3_key(ods_code: str, report_date: str) -> str: + return f"Report-Orchestration/{report_date}/{ods_code}.xlsx" + + + +def upload_generated_reports( + s3_service: S3Service, + bucket: str, + report_date: str, + generated_files: Dict[str, str], +) -> list[str]: + keys: list[str] = [] + for ods_code, local_path in generated_files.items(): + key = build_s3_key(ods_code, report_date) + s3_service.upload_file_with_extra_args( + file_name=local_path, + s3_bucket_name=bucket, + file_key=key, + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + keys.append(key) + logger.info(f"Uploaded report for ODS={ods_code} to s3://{bucket}/{key}") + return keys -@ensure_environment_variables(names=["BULK_UPLOAD_REPORT_TABLE_NAME"]) +def build_response(report_date: str, bucket: str, keys: list[str]) -> Dict[str, Any]: + prefix = f"Report-Orchestration/{report_date}/" + return { + "status": "ok", + "report_date": report_date, + "bucket": bucket, + "prefix": prefix, + "keys": keys, + } + + +@ensure_environment_variables( + names=[ + "BULK_UPLOAD_REPORT_TABLE_NAME", + "REPORT_BUCKET_NAME", + ] +) @override_error_check @handle_lambda_exceptions @set_request_context_for_logging -def lambda_handler(event, context): +def lambda_handler(event, context) -> Dict[str, Any]: logger.info("Report orchestration lambda invoked") - table_name = os.getenv("BULK_UPLOAD_REPORT_TABLE_NAME") - repository = ReportingDynamoRepository(table_name) - excel_generator = ExcelReportGenerator() - - service = ReportOrchestrationService( - repository=repository, - excel_generator=excel_generator, + report_bucket = os.environ["REPORT_BUCKET_NAME"] + orchestration_service = ReportOrchestrationService( ) + s3_service = S3Service() window_start, window_end = calculate_reporting_window() - tmp_dir = tempfile.mkdtemp() + report_date = get_report_date_folder() - service.process_reporting_window( + generated_files = orchestration_service.process_reporting_window( window_start_ts=window_start, window_end_ts=window_end, - output_dir=tmp_dir, ) + + if not generated_files: + logger.info("No reports generated; exiting") + return build_response(report_date=report_date, bucket=report_bucket, keys=[]) + + keys = upload_generated_reports( + s3_service=s3_service, + bucket=report_bucket, + report_date=report_date, + generated_files=generated_files, + ) + + logger.info(f"Generated and uploaded {len(keys)} report(s) for report_date={report_date}") + return build_response(report_date=report_date, bucket=report_bucket, keys=keys) diff --git a/lambdas/repositories/reporting/report_contact_repository.py b/lambdas/repositories/reporting/report_contact_repository.py new file mode 100644 index 0000000000..d9c21c0c2e --- /dev/null +++ b/lambdas/repositories/reporting/report_contact_repository.py @@ -0,0 +1,19 @@ +import os + +from services.base.dynamo_service import DynamoDBService + + +class ReportContactRepository: + def __init__(self): + self.table_name = os.environ["CONTACT_TABLE_NAME"] + self.dynamo = DynamoDBService() + + def get_contact_email(self, ods_code: str) -> str | None: + resp = self.dynamo.get_item( + table_name=self.table_name, + key={"OdsCode": ods_code}, + ) + item = (resp or {}).get("Item") + if not item: + return None + return item.get("Email") diff --git a/lambdas/repositories/reporting/reporting_dynamo_repository.py b/lambdas/repositories/reporting/reporting_dynamo_repository.py index 7a694edf69..9bb017a7cc 100644 --- a/lambdas/repositories/reporting/reporting_dynamo_repository.py +++ b/lambdas/repositories/reporting/reporting_dynamo_repository.py @@ -1,15 +1,17 @@ +import os +from datetime import timedelta from typing import Dict, List -from boto3.dynamodb.conditions import Attr +from boto3.dynamodb.conditions import Key from services.base.dynamo_service import DynamoDBService from utils.audit_logging_setup import LoggingService +from utils.utilities import utc_date_string, utc_date, utc_day_start_timestamp, utc_day_end_timestamp logger = LoggingService(__name__) - class ReportingDynamoRepository: - def __init__(self, table_name: str): - self.table_name = table_name + def __init__(self): + self.table_name = os.environ["BULK_UPLOAD_REPORT_TABLE_NAME"] self.dynamo_service = DynamoDBService() def get_records_for_time_window( @@ -17,19 +19,37 @@ def get_records_for_time_window( start_timestamp: int, end_timestamp: int, ) -> List[Dict]: + timestamp_index_name = "TimestampIndex" logger.info( - f"Querying reporting table for window, " - f"table_name: {self.table_name}, " - f"start_timestamp: {start_timestamp}, " - f"end_timestamp: {end_timestamp}", + "Querying reporting table via TimestampIndex for window, " + f"table_name={self.table_name}, start_timestamp={start_timestamp}, end_timestamp={end_timestamp}", ) - filter_expression = Attr("Timestamp").between( - start_timestamp, - end_timestamp, - ) + start_date = utc_date(start_timestamp) + end_date = utc_date(end_timestamp) - return self.dynamo_service.scan_whole_table( - table_name=self.table_name, - filter_expression=filter_expression, - ) + records_for_window: List[Dict] = [] + current_date = start_date + + while current_date <= end_date: + day_start_ts = utc_day_start_timestamp(current_date) + day_end_ts = utc_day_end_timestamp(current_date) + + effective_start_ts = max(start_timestamp, day_start_ts) + effective_end_ts = min(end_timestamp, day_end_ts) + + key_condition = ( + Key("Date").eq(current_date.isoformat()) + & Key("Timestamp").between(effective_start_ts, effective_end_ts) + ) + + records_for_day = self.dynamo_service.query_by_key_condition_expression( + table_name=self.table_name, + index_name=timestamp_index_name, + key_condition_expression=key_condition, + ) + + records_for_window.extend(records_for_day) + current_date += timedelta(days=1) + + return records_for_window \ No newline at end of file diff --git a/lambdas/requirements/layers/requirements_reports_lambda_layer.txt b/lambdas/requirements/layers/requirements_reports_lambda_layer.txt index 44709d7496..872e8ad167 100644 --- a/lambdas/requirements/layers/requirements_reports_lambda_layer.txt +++ b/lambdas/requirements/layers/requirements_reports_lambda_layer.txt @@ -1,2 +1,3 @@ openpyxl==3.1.5 -reportlab==4.3.1 \ No newline at end of file +reportlab==4.3.1 +pyzipper==0.3.6 \ No newline at end of file diff --git a/lambdas/services/base/dynamo_service.py b/lambdas/services/base/dynamo_service.py index a09e3ca596..7ae9a7ea93 100644 --- a/lambdas/services/base/dynamo_service.py +++ b/lambdas/services/base/dynamo_service.py @@ -441,6 +441,46 @@ def build_update_transaction_item( } } + def query_by_key_condition_expression( + self, + table_name: str, + key_condition_expression: ConditionBase, + index_name: str | None = None, + query_filter: Attr | ConditionBase | None = None, + limit: int | None = None, + ) -> list[dict]: + table = self.get_table(table_name) + + collected_items: list[dict] = [] + exclusive_start_key: dict | None = None + + while True: + query_params: dict = {"KeyConditionExpression": key_condition_expression} + + if index_name: + query_params["IndexName"] = index_name + if query_filter: + query_params["FilterExpression"] = query_filter + if exclusive_start_key: + query_params["ExclusiveStartKey"] = exclusive_start_key + if limit: + query_params["Limit"] = limit + + try: + response = table.query(**query_params) + except ClientError as exc: + logger.error(str(exc), {"Result": f"Unable to query table: {table_name}"}) + raise + + collected_items.extend(response.get("Items", [])) + + exclusive_start_key = response.get("LastEvaluatedKey") + if not exclusive_start_key: + break + + return collected_items + + def query_table_with_paginator( self, table_name: str, diff --git a/lambdas/services/base/s3_service.py b/lambdas/services/base/s3_service.py index 3ee42b4e24..d3bf770fcd 100644 --- a/lambdas/services/base/s3_service.py +++ b/lambdas/services/base/s3_service.py @@ -1,7 +1,7 @@ import io from datetime import datetime, timedelta, timezone from io import BytesIO -from typing import Any, Mapping +from typing import Any, Mapping, Callable import boto3 from botocore.client import Config as BotoConfig @@ -255,3 +255,13 @@ def save_or_create_file(self, source_bucket: str, file_key: str, body: bytes): return self.client.put_object( Bucket=source_bucket, Key=file_key, Body=BytesIO(body) ) + + def list_object_keys(self, bucket_name: str, prefix: str) -> list[str]: + paginator = self.client.get_paginator("list_objects_v2") + keys: list[str] = [] + + for page in paginator.paginate(Bucket=bucket_name, Prefix=prefix): + for obj in page.get("Contents", []): + keys.append(obj["Key"]) + + return keys \ No newline at end of file diff --git a/lambdas/services/email_service.py b/lambdas/services/email_service.py new file mode 100644 index 0000000000..e72bbcfde4 --- /dev/null +++ b/lambdas/services/email_service.py @@ -0,0 +1,113 @@ +import boto3 +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.mime.application import MIMEApplication +from typing import Iterable, Optional, Dict, Any + +from utils.audit_logging_setup import LoggingService + +logger = LoggingService(__name__) + + +class EmailService: + """ + General email sender via SES (AWS Simple Email Service) Raw Email (supports attachments). + Higher-level methods prepare inputs and call send_email(). + """ + + def __init__(self): + self.ses = boto3.client("ses") + + def send_email( + self, + *, + to_address: str, + subject: str, + body_text: str, + from_address: str, + attachments: Optional[Iterable[str]] = None, + )->Dict[str, Any]: + msg = MIMEMultipart() + msg["Subject"] = subject + msg["To"] = to_address + msg["From"] = from_address + + msg.attach(MIMEText(body_text, "plain")) + + for attachment_path in attachments or []: + with open(attachment_path, "rb") as f: + part = MIMEApplication(f.read()) + part.add_header( + "Content-Disposition", + "attachment", + filename=attachment_path.split("/")[-1], + ) + msg.attach(part) + logger.info( + f"Sending email: from={from_address!r}, to={to_address!r}, subject={subject!r}, " + f"attachments={len(list(attachments or []))}" + ) + return self._send_raw(msg, to_address) + + def _send_raw(self, msg: MIMEMultipart, to_address: str)->Dict[str, Any]: + subject = msg.get("Subject", "") + from_address = msg.get("From", "") + logger.info(f"Sending SES raw email: subject={subject!r}, from={from_address!r}, to={to_address!r}") + resp = self.ses.send_raw_email( + Source=from_address, + RawMessage={"Data": msg.as_string()}, + Destinations=[to_address], + ) + + logger.info(f"SES accepted email: subject={subject!r}, message_id={resp.get('MessageId')}") + return resp + + def send_report_email( + self, + *, + to_address: str, + from_address: str, + attachment_path: str, + ): + self.send_email( + to_address=to_address, + from_address=from_address, + subject="Daily Upload Report", + body_text="Please find your encrypted daily upload report attached.", + attachments=[attachment_path], + ) + + def send_password_email( + self, + *, + to_address: str, + from_address: str, + password: str, + ): + self.send_email( + to_address=to_address, + from_address=from_address, + subject="Daily Upload Report Password", + body_text=f"Password for your report:\n\n{password}", + ) + + def send_prm_missing_contact_email( + self, + *, + prm_mailbox: str, + from_address: str, + ods_code: str, + attachment_path: str, + password: str, + ): + self.send_email( + to_address=prm_mailbox, + from_address=from_address, + subject=f"Missing contact for ODS {ods_code}", + body_text=( + f"No contact found for ODS {ods_code}.\n\n" + f"Password: {password}\n\n" + f"Please resolve the contact and forward the report." + ), + attachments=[attachment_path], + ) diff --git a/lambdas/services/reporting/excel_report_generator_service.py b/lambdas/services/reporting/excel_report_generator_service.py index a3579df501..8445de2881 100644 --- a/lambdas/services/reporting/excel_report_generator_service.py +++ b/lambdas/services/reporting/excel_report_generator_service.py @@ -16,6 +16,7 @@ def create_report_orchestration_xlsx( logger.info( f"Creating Excel report for ODS code {ods_code} and records {len(records)}" ) + wb = Workbook() ws = wb.active ws.title = "Daily Upload Report" @@ -25,12 +26,10 @@ def create_report_orchestration_xlsx( ws.append([f"Generated at (UTC): {datetime.now(timezone.utc).isoformat()}"]) ws.append([]) - # Header row ws.append( [ - "ID", - "Date", "NHS Number", + "Date", "Uploader ODS", "PDS ODS", "Upload Status", @@ -42,9 +41,8 @@ def create_report_orchestration_xlsx( for record in records: ws.append( [ - record.get("ID"), - record.get("Date"), record.get("NhsNumber"), + record.get("Date"), record.get("UploaderOdsCode"), record.get("PdsOdsCode"), record.get("UploadStatus"), @@ -54,5 +52,5 @@ def create_report_orchestration_xlsx( ) wb.save(output_path) - logger.info(f"Excel report written successfully for for ods code {ods_code}") + logger.info(f"Excel report written successfully for ods code {ods_code}") return output_path diff --git a/lambdas/services/reporting/report_distribution_service.py b/lambdas/services/reporting/report_distribution_service.py new file mode 100644 index 0000000000..5dd75e8f46 --- /dev/null +++ b/lambdas/services/reporting/report_distribution_service.py @@ -0,0 +1,107 @@ +import os +import secrets +import tempfile +from typing import List + +import boto3 + +from repositories.reporting.report_contact_repository import ReportContactRepository +from services.base.s3_service import S3Service +from services.email_service import EmailService +from utils.audit_logging_setup import LoggingService +from utils.zip_utils import zip_encrypt_file + +logger = LoggingService(__name__) + + +class ReportDistributionService: + def __init__( + self, + *, + bucket: str, + ): + self.s3_service = S3Service() + self.contact_repo = ReportContactRepository() + self.email_service = EmailService() + self.bucket = bucket + self.from_address = os.environ["SES_FROM_ADDRESS"] + self.prm_mailbox = os.environ["PRM_MAILBOX_EMAIL"] + + @staticmethod + def extract_ods_code_from_key(key: str) -> str: + filename = key.split("/")[-1] + return filename[:-5] if filename.lower().endswith(".xlsx") else filename + + def list_xlsx_keys(self, prefix: str) -> List[str]: + keys = self.s3_service.list_object_keys(bucket_name=self.bucket, prefix=prefix) + return [k for k in keys if k.endswith(".xlsx")] + + def process_one_report(self, *, ods_code: str, key: str) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + local_xlsx = os.path.join(tmpdir, f"{ods_code}.xlsx") + local_zip = os.path.join(tmpdir, f"{ods_code}.zip") + + self.s3_service.download_file(self.bucket, key, local_xlsx) + + password = secrets.token_urlsafe(16) + zip_encrypt_file( + input_path=local_xlsx, + output_zip=local_zip, + password=password, + ) + + self.send_report_emails( + ods_code=ods_code, + attachment_path=local_zip, + password=password, + ) + + def send_report_emails(self, *, ods_code: str, attachment_path: str, password: str) -> None: + try: + contact_email = self.contact_repo.get_contact_email(ods_code) + except Exception: + logger.exception( + f"Contact lookup failed for ODS={ods_code}; falling back to PRM." + ) + contact_email = None + + if contact_email: + logger.info(f"Contact found for ODS={ods_code}, emailing {contact_email}") + self.email_contact( + to_address=contact_email, + attachment_path=attachment_path, + password=password, + ) + return + + logger.info(f"No contact found for ODS={ods_code}, sending to PRM mailbox") + self.email_prm_missing_contact( + ods_code=ods_code, + attachment_path=attachment_path, + password=password, + ) + + def email_contact(self, *, to_address: str, attachment_path: str, password: str) -> None: + logger.info(f"Sending report email to {to_address}") + self.email_service.send_report_email( + to_address=to_address, + from_address=self.from_address, + attachment_path=attachment_path, + ) + logger.info(f"Sending password email to {to_address}") + self.email_service.send_password_email( + to_address=to_address, + from_address=self.from_address, + password=password, + ) + + def email_prm_missing_contact( + self, *, ods_code: str, attachment_path: str, password: str + ) -> None: + self.email_service.send_prm_missing_contact_email( + prm_mailbox=self.prm_mailbox, + from_address=self.from_address, + ods_code=ods_code, + attachment_path=attachment_path, + password=password, + ) diff --git a/lambdas/services/reporting/report_orchestration_service.py b/lambdas/services/reporting/report_orchestration_service.py index a9500735b9..3842419284 100644 --- a/lambdas/services/reporting/report_orchestration_service.py +++ b/lambdas/services/reporting/report_orchestration_service.py @@ -1,42 +1,45 @@ import tempfile from collections import defaultdict +from typing import Dict +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository +from services.reporting.excel_report_generator_service import ExcelReportGenerator from utils.audit_logging_setup import LoggingService logger = LoggingService(__name__) class ReportOrchestrationService: - def __init__( - self, - repository, - excel_generator, - ): - self.repository = repository - self.excel_generator = excel_generator + def __init__(self): + self.repository = ReportingDynamoRepository() + self.excel_generator = ExcelReportGenerator() def process_reporting_window( self, window_start_ts: int, window_end_ts: int, - output_dir: str, - ): + ) -> Dict[str, str]: records = self.repository.get_records_for_time_window( window_start_ts, window_end_ts, ) + if not records: logger.info("No records found for reporting window") - return + return {} records_by_ods = self.group_records_by_ods(records) + generated_files: Dict[str, str] = {} for ods_code, ods_records in records_by_ods.items(): logger.info( - f"Generating report for ODS ods_code = {ods_code} record_count = {len(ods_records)}" + f"Generating report for ODS={ods_code}, records={len(ods_records)}" ) - self.generate_ods_report(ods_code, ods_records) - logger.info("Report orchestration completed") + file_path = self.generate_ods_report(ods_code, ods_records) + generated_files[ods_code] = file_path + + logger.info(f"Generated {len(generated_files)} report(s)") + return generated_files @staticmethod def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: @@ -46,11 +49,7 @@ def group_records_by_ods(records: list[dict]) -> dict[str, list[dict]]: grouped[ods_code].append(record) return grouped - def generate_ods_report( - self, - ods_code: str, - records: list[dict], - ): + def generate_ods_report(self, ods_code: str, records: list[dict]) -> str: with tempfile.NamedTemporaryFile( suffix=f"_{ods_code}.xlsx", delete=False, @@ -60,3 +59,4 @@ def generate_ods_report( records=records, output_path=tmp.name, ) + return tmp.name diff --git a/lambdas/tests/unit/conftest.py b/lambdas/tests/unit/conftest.py index b3afba0f82..dd3a98e4b1 100644 --- a/lambdas/tests/unit/conftest.py +++ b/lambdas/tests/unit/conftest.py @@ -13,6 +13,7 @@ from pydantic import ValidationError from pypdf import PdfWriter from requests import Response +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from tests.unit.helpers.data.pds.pds_patient_response import PDS_PATIENT from utils.audit_logging_setup import LoggingService @@ -416,3 +417,16 @@ def valid_pdf_stream(): @pytest.fixture def corrupt_pdf_stream(): return BytesIO(b"This is not a valid PDF content") + + +@pytest.fixture +def mock_reporting_dynamo_service(mocker): + mock_cls = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.DynamoDBService" + ) + return mock_cls.return_value + + +@pytest.fixture +def reporting_repo(mock_reporting_dynamo_service): + return ReportingDynamoRepository(table_name="TestTable") diff --git a/lambdas/tests/unit/handlers/conftest.py b/lambdas/tests/unit/handlers/conftest.py index 8eca1979f2..caaade5496 100755 --- a/lambdas/tests/unit/handlers/conftest.py +++ b/lambdas/tests/unit/handlers/conftest.py @@ -1,8 +1,12 @@ +import os +from unittest.mock import MagicMock + import pytest from enums.feature_flags import FeatureFlags +from enums.report_distribution_action import ReportDistributionAction +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository from services.feature_flags_service import FeatureFlagService - @pytest.fixture def valid_id_event_without_auth_header(): api_gateway_proxy_event = { @@ -219,3 +223,99 @@ def mock_upload_document_iteration_3_disabled(mocker): FeatureFlags.UPLOAD_DOCUMENT_ITERATION_3_ENABLED: False } yield mock_feature_flag + + +@pytest.fixture +def required_report_distribution_env(monkeypatch): + monkeypatch.setenv("REPORT_BUCKET_NAME", "my-report-bucket") + monkeypatch.setenv("CONTACT_TABLE_NAME", "contact-table") + monkeypatch.setenv("PRM_MAILBOX_EMAIL", "prm@example.com") + monkeypatch.setenv("SES_FROM_ADDRESS", "from@example.com") + + +@pytest.fixture +def lambda_context(mocker): + ctx = mocker.Mock() + ctx.aws_request_id = "req-123" + return ctx + + +@pytest.fixture +def required_report_orchestration_env(monkeypatch): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + monkeypatch.setenv("REPORT_BUCKET_NAME", "test-report-bucket") + + + +@pytest.fixture +def mock_reporting_dynamo_service(mocker): + mock_cls = mocker.patch( + "repositories.reporting.reporting_dynamo_repository.DynamoDBService" + ) + return mock_cls.return_value + + +@pytest.fixture +def reporting_repo(mock_reporting_dynamo_service): + return ReportingDynamoRepository(table_name="TestTable") + + +@pytest.fixture +def report_distribution_list_event(): + return {"action": ReportDistributionAction.LIST, "prefix": "p/"} + + +@pytest.fixture +def report_distribution_process_one_event(): + return {"action": ReportDistributionAction.PROCESS_ONE, "key": "reports/ABC/whatever.xlsx"} + + +@pytest.fixture +def mock_report_distribution_wiring(mocker): + svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") + mocker.patch( + "handlers.report_distribution_handler.ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + return svc_instance + + +@pytest.fixture +def mock_report_orchestration_wiring(mocker): + from handlers import report_orchestration_handler as handler_module + + orchestration_service = mocker.Mock(name="ReportOrchestrationServiceInstance") + s3_service = mocker.Mock(name="S3ServiceInstance") + + mocker.patch.object( + handler_module, + "ReportOrchestrationService", + autospec=True, + return_value=orchestration_service, + ) + mocker.patch.object( + handler_module, + "S3Service", + autospec=True, + return_value=s3_service, + ) + + mock_window = mocker.patch.object( + handler_module, + "calculate_reporting_window", + return_value=(100, 200), + ) + mock_report_date = mocker.patch.object( + handler_module, + "get_report_date_folder", + return_value="2026-01-02", + ) + + return { + "handler_module": handler_module, + "orchestration_service": orchestration_service, + "s3_service": s3_service, + "mock_window": mock_window, + "mock_report_date": mock_report_date, + } diff --git a/lambdas/tests/unit/handlers/test_report_distribution_handler.py b/lambdas/tests/unit/handlers/test_report_distribution_handler.py new file mode 100644 index 0000000000..d70178948e --- /dev/null +++ b/lambdas/tests/unit/handlers/test_report_distribution_handler.py @@ -0,0 +1,129 @@ +import json +import os + + +from handlers import report_distribution_handler as handler_module + + +def test_lambda_handler_wires_service_and_returns_result_list_mode( + mocker, required_report_distribution_env, lambda_context +): + event = { + "action": handler_module.ReportDistributionAction.LIST, + "prefix": "reports/2026-01-01/", + } + + svc_instance = mocker.Mock(name="ReportDistributionServiceInstance") + svc_instance.list_xlsx_keys.return_value = ["a.xlsx", "b.xlsx"] + + mocked_dist_svc_cls = mocker.patch.object( + handler_module, + "ReportDistributionService", + autospec=True, + return_value=svc_instance, + ) + + result = handler_module.lambda_handler(event, lambda_context) + + mocked_dist_svc_cls.assert_called_once_with(bucket="my-report-bucket") + svc_instance.list_xlsx_keys.assert_called_once_with(prefix="reports/2026-01-01/") + + assert result == { + "status": "ok", + "bucket": "my-report-bucket", + "prefix": "reports/2026-01-01/", + "keys": ["a.xlsx", "b.xlsx"], + } + + +def test_lambda_handler_uses_bucket_from_event_when_provided_list_mode( + required_report_distribution_env, + lambda_context, + report_distribution_list_event, + mock_report_distribution_wiring, +): + event = {**report_distribution_list_event, "bucket": "override-bucket"} + mock_report_distribution_wiring.list_xlsx_keys.return_value = [] + + result = handler_module.lambda_handler(event, lambda_context) + + mock_report_distribution_wiring.list_xlsx_keys.assert_called_once_with(prefix="p/") + assert result == { + "status": "ok", + "bucket": "override-bucket", + "prefix": "p/", + "keys": [], + } + + +def test_lambda_handler_process_one_mode_happy_path( + required_report_distribution_env, + lambda_context, + report_distribution_process_one_event, + mock_report_distribution_wiring, +): + event = {**report_distribution_process_one_event, "key": "reports/ABC/whatever.xlsx"} + + mock_report_distribution_wiring.extract_ods_code_from_key.return_value = "ABC" + mock_report_distribution_wiring.process_one_report.return_value = None + + result = handler_module.lambda_handler(event, lambda_context) + + mock_report_distribution_wiring.extract_ods_code_from_key.assert_called_once_with( + "reports/ABC/whatever.xlsx" + ) + mock_report_distribution_wiring.process_one_report.assert_called_once_with( + ods_code="ABC", key="reports/ABC/whatever.xlsx" + ) + + assert result == { + "status": "ok", + "bucket": "my-report-bucket", + "key": "reports/ABC/whatever.xlsx", + "ods_code": "ABC", + } + + +def test_lambda_handler_returns_400_when_action_invalid( + required_report_distribution_env, lambda_context +): + event = {"action": "nope"} + + result = handler_module.lambda_handler(event, lambda_context) + + assert isinstance(result, dict) + assert result["statusCode"] == 400 + + body = json.loads(result["body"]) + assert body["err_code"] == handler_module.LambdaError.InvalidAction.value["err_code"] + assert "Invalid action" in body["message"] + + if body.get("interaction_id") is not None: + assert body["interaction_id"] == lambda_context.aws_request_id + + +def test_lambda_handler_returns_500_when_required_env_missing(mocker, lambda_context): + mocker.patch.dict( + os.environ, + { + "CONTACT_TABLE_NAME": "contact-table", + "PRM_MAILBOX_EMAIL": "prm@example.com", + "SES_FROM_ADDRESS": "from@example.com", + }, + clear=False, + ) + os.environ.pop("REPORT_BUCKET_NAME", None) + + event = {"action": handler_module.ReportDistributionAction.LIST, "prefix": "p/"} + + result = handler_module.lambda_handler(event, lambda_context) + + assert isinstance(result, dict) + assert result["statusCode"] == 500 + + body = json.loads(result["body"]) + assert body["err_code"] == "ENV_5001" + assert "REPORT_BUCKET_NAME" in body["message"] + + if body.get("interaction_id") is not None: + assert body["interaction_id"] == lambda_context.aws_request_id diff --git a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py index 750e0d2780..3f4eb05632 100644 --- a/lambdas/tests/unit/handlers/test_report_orchestration_handler.py +++ b/lambdas/tests/unit/handlers/test_report_orchestration_handler.py @@ -1,75 +1,134 @@ -from unittest import mock -from unittest.mock import MagicMock +import json -import pytest -from handlers.report_orchestration_handler import lambda_handler +from handlers import report_orchestration_handler as handler_module -class FakeContext: - aws_request_id = "test-request-id" +def test_lambda_handler_calls_service_and_returns_expected_response( + required_report_orchestration_env, + lambda_context, + mock_report_orchestration_wiring, +): + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + s3_service = mock_report_orchestration_wiring["s3_service"] + mock_window = mock_report_orchestration_wiring["mock_window"] + mock_report_date = mock_report_orchestration_wiring["mock_report_date"] + orchestration_service.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + "B67890": "/tmp/B67890.xlsx", + } -@pytest.fixture(autouse=True) -def mock_env(monkeypatch): - monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + result = handler_module.lambda_handler(event={}, context=lambda_context) + mock_window.assert_called_once() + mock_report_date.assert_called_once() -@pytest.fixture -def mock_logger(mocker): - return mocker.patch("handlers.report_orchestration_handler.logger", new=MagicMock()) + orchestration_service.process_reporting_window.assert_called_once_with( + window_start_ts=100, + window_end_ts=200, + ) + assert s3_service.upload_file_with_extra_args.call_count == 2 + + assert result == { + "status": "ok", + "report_date": "2026-01-02", + "bucket": "test-report-bucket", + "prefix": "Report-Orchestration/2026-01-02/", + "keys": [ + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/B67890.xlsx", + ], + } + + +def test_lambda_handler_calls_window_function( + required_report_orchestration_env, + lambda_context, + mock_report_orchestration_wiring, +): + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + mock_window = mock_report_orchestration_wiring["mock_window"] + orchestration_service.process_reporting_window.return_value = {} -@pytest.fixture -def mock_repo(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.ReportingDynamoRepository", - autospec=True, - ) + handler_module.lambda_handler(event={}, context=lambda_context) + mock_window.assert_called_once() -@pytest.fixture -def mock_excel_generator(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.ExcelReportGenerator", - autospec=True, - ) +def test_lambda_handler_returns_empty_keys_when_no_reports_generated( + required_report_orchestration_env, + lambda_context, + mock_report_orchestration_wiring, +): + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + s3_service = mock_report_orchestration_wiring["s3_service"] -@pytest.fixture -def mock_service(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.ReportOrchestrationService", - autospec=True, - ) + orchestration_service.process_reporting_window.return_value = {} + result = handler_module.lambda_handler(event={}, context=lambda_context) -@pytest.fixture -def mock_window(mocker): - return mocker.patch( - "handlers.report_orchestration_handler.calculate_reporting_window", - return_value=(100, 200), - ) + assert result == { + "status": "ok", + "report_date": "2026-01-02", + "bucket": "test-report-bucket", + "prefix": "Report-Orchestration/2026-01-02/", + "keys": [], + } + s3_service.upload_file_with_extra_args.assert_not_called() -def test_lambda_handler_calls_service( - mock_logger, mock_repo, mock_excel_generator, mock_service, mock_window +def test_lambda_handler_uploads_each_report_to_s3_with_kms_encryption( + required_report_orchestration_env, + lambda_context, + mock_report_orchestration_wiring, ): - lambda_handler(event={}, context=FakeContext()) + orchestration_service = mock_report_orchestration_wiring["orchestration_service"] + s3_service = mock_report_orchestration_wiring["s3_service"] - mock_repo.assert_called_once_with("TestTable") - mock_excel_generator.assert_called_once_with() + orchestration_service.process_reporting_window.return_value = { + "A12345": "/tmp/A12345.xlsx", + "UNKNOWN": "/tmp/UNKNOWN.xlsx", + } - mock_service.assert_called_once() - instance = mock_service.return_value - instance.process_reporting_window.assert_called_once_with( - window_start_ts=100, - window_end_ts=200, - output_dir=mock.ANY, + result = handler_module.lambda_handler(event={}, context=lambda_context) + + assert s3_service.upload_file_with_extra_args.call_count == 2 + + s3_service.upload_file_with_extra_args.assert_any_call( + file_name="/tmp/A12345.xlsx", + s3_bucket_name="test-report-bucket", + file_key="Report-Orchestration/2026-01-02/A12345.xlsx", + extra_args={"ServerSideEncryption": "aws:kms"}, + ) + s3_service.upload_file_with_extra_args.assert_any_call( + file_name="/tmp/UNKNOWN.xlsx", + s3_bucket_name="test-report-bucket", + file_key="Report-Orchestration/2026-01-02/UNKNOWN.xlsx", + extra_args={"ServerSideEncryption": "aws:kms"}, ) - mock_logger.info.assert_any_call("Report orchestration lambda invoked") + assert result["keys"] == [ + "Report-Orchestration/2026-01-02/A12345.xlsx", + "Report-Orchestration/2026-01-02/UNKNOWN.xlsx", + ] -def test_lambda_handler_calls_window_function(mock_service, mock_window): - lambda_handler(event={}, context=FakeContext()) - mock_window.assert_called_once() +def test_lambda_handler_returns_error_when_required_env_missing( + lambda_context, + monkeypatch, +): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + monkeypatch.delenv("REPORT_BUCKET_NAME", raising=False) + + result = handler_module.lambda_handler(event={}, context=lambda_context) + + assert isinstance(result, dict) + assert result["statusCode"] == 500 + + body = json.loads(result["body"]) + assert body["err_code"] == "ENV_5001" + assert "REPORT_BUCKET_NAME" in body["message"] + + if body.get("interaction_id") is not None: + assert body["interaction_id"] == lambda_context.aws_request_id diff --git a/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py new file mode 100644 index 0000000000..13f91bac99 --- /dev/null +++ b/lambdas/tests/unit/repositories/reporting/test_report_contact_repository.py @@ -0,0 +1,57 @@ +import pytest + +from repositories.reporting.report_contact_repository import ReportContactRepository + + +@pytest.fixture +def required_contact_repo_env(monkeypatch): + monkeypatch.setenv("CONTACT_TABLE_NAME", "report-contacts") + + +@pytest.fixture +def mock_dynamo(mocker): + dynamo = mocker.Mock(name="DynamoDBServiceInstance") + mocker.patch( + "repositories.reporting.report_contact_repository.DynamoDBService", + autospec=True, + return_value=dynamo, + ) + return dynamo + + +@pytest.fixture +def repo(required_contact_repo_env, mock_dynamo): + return ReportContactRepository() + + +@pytest.mark.parametrize( + "dynamo_response, expected_email", + [ + ( + { + "Item": { + "OdsCode": "Y12345", + "Email": "contact@example.com", + } + }, + "contact@example.com", + ), + ({}, None), + ({"Item": {"OdsCode": "Y12345"}}, None), + (None, None), + ], +) +def test_get_contact_email(repo, mock_dynamo, dynamo_response, expected_email): + mock_dynamo.get_item.return_value = dynamo_response + + result = repo.get_contact_email("Y12345") + + mock_dynamo.get_item.assert_called_once_with( + table_name="report-contacts", + key={"OdsCode": "Y12345"}, + ) + assert result == expected_email + + +def test_init_reads_table_name_from_env(repo): + assert repo.table_name == "report-contacts" diff --git a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py index df3c553044..21bd57f3a8 100644 --- a/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py +++ b/lambdas/tests/unit/repositories/reporting/test_reporting_dynamo_repository.py @@ -1,41 +1,132 @@ -from unittest.mock import MagicMock +from datetime import date import pytest -from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository - -@pytest.fixture -def mock_dynamo_service(mocker): - mock_service = mocker.patch( - "repositories.reporting.reporting_dynamo_repository.DynamoDBService" - ) - instance = mock_service.return_value - instance.scan_whole_table = MagicMock() - return instance +from repositories.reporting.reporting_dynamo_repository import ReportingDynamoRepository @pytest.fixture -def reporting_repo(mock_dynamo_service): - return ReportingDynamoRepository(table_name="TestTable") - +def reporting_repo(monkeypatch, mock_reporting_dynamo_service): + monkeypatch.setenv("BULK_UPLOAD_REPORT_TABLE_NAME", "TestTable") + return ReportingDynamoRepository() -def test_get_records_for_time_window_calls_scan(mock_dynamo_service, reporting_repo): - mock_dynamo_service.scan_whole_table.return_value = [] - reporting_repo.get_records_for_time_window(100, 200) - - mock_dynamo_service.scan_whole_table.assert_called_once() - assert "filter_expression" in mock_dynamo_service.scan_whole_table.call_args.kwargs - - -def test_get_records_for_time_window_returns_empty_list( - mock_dynamo_service, reporting_repo +@pytest.mark.parametrize( + "start_dt,end_dt,service_side_effect,expected,expected_call_count", + [ + ( + date(2026, 1, 7), + date(2026, 1, 7), + [ + [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ] + ], + [ + { + "Date": "2026-01-07", + "Timestamp": 1704601000, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-001", + "Status": "UPLOADED", + } + ], + 1, + ), + ( + date(2026, 1, 6), + date(2026, 1, 7), + [ + [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + } + ], + [ + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ], + ], + [ + { + "Date": "2026-01-06", + "Timestamp": 1704517200, + "UploaderOdsCode": "A12345", + "DocumentId": "doc-101", + "Status": "UPLOADED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704590000, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-202", + "Status": "PROCESSED", + }, + { + "Date": "2026-01-07", + "Timestamp": 1704593600, + "UploaderOdsCode": "B67890", + "DocumentId": "doc-203", + "Status": "PROCESSED", + }, + ], + 2, + ), + ( + date(2026, 1, 6), + date(2026, 1, 7), + [ + [], + [], + ], + [], + 2, + ), + ], +) +def test_get_records_for_time_window( + mocker, + mock_reporting_dynamo_service, + reporting_repo, + start_dt, + end_dt, + service_side_effect, + expected, + expected_call_count, ): - start_ts = 0 - end_ts = 50 - mock_dynamo_service.scan_whole_table.return_value = [] + mock_utc_date = mocker.patch("repositories.reporting.reporting_dynamo_repository.utc_date") + mock_utc_date.side_effect = [start_dt, end_dt] + mock_reporting_dynamo_service.query_by_key_condition_expression.side_effect = service_side_effect + + result = reporting_repo.get_records_for_time_window(100, 200) - result = reporting_repo.get_records_for_time_window(start_ts, end_ts) + assert result == expected + assert mock_reporting_dynamo_service.query_by_key_condition_expression.call_count == expected_call_count - assert result == [] - mock_dynamo_service.scan_whole_table.assert_called_once() + for call in mock_reporting_dynamo_service.query_by_key_condition_expression.call_args_list: + kwargs = call.kwargs + assert kwargs["table_name"] == "TestTable" + assert kwargs["index_name"] == "TimestampIndex" + assert "key_condition_expression" in kwargs diff --git a/lambdas/tests/unit/services/base/test_dynamo_service.py b/lambdas/tests/unit/services/base/test_dynamo_service.py index 2487e75f76..221385f57a 100755 --- a/lambdas/tests/unit/services/base/test_dynamo_service.py +++ b/lambdas/tests/unit/services/base/test_dynamo_service.py @@ -1324,6 +1324,118 @@ def test_build_key_condition_non_matching_list_lengths( search_key=search_key, search_condition=search_condition ) +def test_query_by_key_condition_expression_single_page_returns_items( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.return_value = { + "Items": [{"ID": "item-1"}, {"ID": "item-2"}] + } + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert result == [{"ID": "item-1"}, {"ID": "item-2"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.query.assert_called_once_with( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ) + + +def test_query_by_key_condition_expression_handles_pagination( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.side_effect = [ + { + "Items": [{"ID": "item-1"}], + "LastEvaluatedKey": {"ID": "page-2"}, + }, + { + "Items": [{"ID": "item-2"}], + "LastEvaluatedKey": {"ID": "page-3"}, + }, + { + "Items": [{"ID": "item-3"}], + }, + ] + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert result == [{"ID": "item-1"}, {"ID": "item-2"}, {"ID": "item-3"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + + expected_calls = [ + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ), + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ExclusiveStartKey={"ID": "page-2"}, + ), + call( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + ExclusiveStartKey={"ID": "page-3"}, + ), + ] + mock_table.return_value.query.assert_has_calls(expected_calls) + + +def test_query_by_key_condition_expression_passes_filter_and_limit( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").lte(1767780025) + filter_expression = Attr("UploadStatus").eq("complete") + + mock_table.return_value.query.return_value = {"Items": [{"ID": "item-1"}]} + + result = mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + query_filter=filter_expression, + limit=25, + ) + + assert result == [{"ID": "item-1"}] + mock_table.assert_called_with(MOCK_TABLE_NAME) + mock_table.return_value.query.assert_called_once_with( + IndexName="TimestampIndex", + KeyConditionExpression=key_condition_expression, + FilterExpression=filter_expression, + Limit=25, + ) + + +def test_query_by_key_condition_expression_client_error_raises_exception( + mock_service, mock_table +): + key_condition_expression = Key("Date").eq("2026-01-07") & Key("Timestamp").gte(1767779952) + + mock_table.return_value.query.side_effect = MOCK_CLIENT_ERROR + + with pytest.raises(ClientError) as exc_info: + mock_service.query_by_key_condition_expression( + table_name=MOCK_TABLE_NAME, + index_name="TimestampIndex", + key_condition_expression=key_condition_expression, + ) + + assert exc_info.value == MOCK_CLIENT_ERROR + def test_query_table_using_paginator(mock_service): mock_paginator = mock_service.client.get_paginator.return_value = MagicMock() diff --git a/lambdas/tests/unit/services/base/test_s3_service.py b/lambdas/tests/unit/services/base/test_s3_service.py index b9e9aa2cb0..6510a45140 100755 --- a/lambdas/tests/unit/services/base/test_s3_service.py +++ b/lambdas/tests/unit/services/base/test_s3_service.py @@ -597,3 +597,34 @@ def test_copy_across_bucket_retries_on_409_conflict(mock_service, mock_client): "StorageClass": "INTELLIGENT_TIERING", } mock_client.copy_object.assert_called_with(**expected_call) + +def test_list_object_keys_returns_keys_for_prefix( + mock_service, mock_client, mock_list_objects_paginate +): + mock_list_objects_paginate.return_value = [MOCK_LIST_OBJECTS_RESPONSE] + + prefix = "some/prefix/" + expected = [obj["Key"] for obj in MOCK_LIST_OBJECTS_RESPONSE["Contents"]] + + actual = mock_service.list_object_keys(bucket_name=MOCK_BUCKET, prefix=prefix) + + assert actual == expected + mock_client.get_paginator.assert_called_with("list_objects_v2") + mock_list_objects_paginate.assert_called_with(Bucket=MOCK_BUCKET, Prefix=prefix) + + +def test_list_object_keys_handles_paginated_responses( + mock_service, mock_client, mock_list_objects_paginate +): + mock_list_objects_paginate.return_value = MOCK_LIST_OBJECTS_PAGINATED_RESPONSES + + prefix = "some/prefix/" + expected = flatten( + [[obj["Key"] for obj in page.get("Contents", [])] for page in MOCK_LIST_OBJECTS_PAGINATED_RESPONSES] + ) + + actual = mock_service.list_object_keys(bucket_name=MOCK_BUCKET, prefix=prefix) + + assert actual == expected + mock_client.get_paginator.assert_called_with("list_objects_v2") + mock_list_objects_paginate.assert_called_with(Bucket=MOCK_BUCKET, Prefix=prefix) diff --git a/lambdas/tests/unit/services/reporting/test_email_service.py b/lambdas/tests/unit/services/reporting/test_email_service.py new file mode 100644 index 0000000000..3c84799cbc --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_email_service.py @@ -0,0 +1,154 @@ +import pytest + +from services.email_service import EmailService + + +@pytest.fixture +def email_service(mocker): + mocker.patch("services.email_service.boto3.client", autospec=True) + service = EmailService() + service.ses = mocker.Mock() + return service + +def test_send_email_sends_raw_email_without_attachments(email_service, mocker): + mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) + + email_service.send_email( + to_address="to@example.com", + subject="Hello", + body_text="Body text", + from_address="from@example.com", + attachments=None, + ) + + mocked_send_raw.assert_called_once() + + call_args, call_kwargs = mocked_send_raw.call_args + assert call_kwargs == {} + + msg_arg = call_args[0] + to_arg = call_args[1] + + assert to_arg == "to@example.com" + assert msg_arg["Subject"] == "Hello" + assert msg_arg["To"] == "to@example.com" + assert msg_arg["From"] == "from@example.com" + + raw = msg_arg.as_string() + assert "Body text" in raw + + + +def test_send_email_attaches_files_and_sets_filenames(email_service, mocker): + file_bytes_1 = b"zipbytes1" + file_bytes_2 = b"zipbytes2" + + m1 = mocker.mock_open(read_data=file_bytes_1) + m2 = mocker.mock_open(read_data=file_bytes_2) + + mocked_open = mocker.patch("services.email_service.open", create=True) + mocked_open.side_effect = [m1.return_value, m2.return_value] + + mocked_send_raw = mocker.patch.object(email_service, "_send_raw", autospec=True) + + email_service.send_email( + to_address="to@example.com", + subject="With Attachments", + body_text="See attached", + from_address="from@example.com", + attachments=["/tmp/a.zip", "/var/tmp/b.zip"], + ) + + assert mocked_open.call_count == 2 + mocked_open.assert_any_call("/tmp/a.zip", "rb") + mocked_open.assert_any_call("/var/tmp/b.zip", "rb") + + mocked_send_raw.assert_called_once() + + call_args, call_kwargs = mocked_send_raw.call_args + assert call_kwargs == {} + msg = call_args[0] + raw = msg.as_string() + + assert 'filename="a.zip"' in raw + assert 'filename="b.zip"' in raw + assert "See attached" in raw + + + +def test_send_raw_calls_ses_send_raw_email(email_service, mocker): + from email.mime.multipart import MIMEMultipart + msg = MIMEMultipart() + msg["Subject"] = "S" + msg["To"] = "to@example.com" + msg["From"] = "from@example.com" + + email_service._send_raw(msg, "to@example.com") + + email_service.ses.send_raw_email.assert_called_once() + call_kwargs = email_service.ses.send_raw_email.call_args.kwargs + + assert call_kwargs["Destinations"] == ["to@example.com"] + assert "RawMessage" in call_kwargs + assert "Data" in call_kwargs["RawMessage"] + assert isinstance(call_kwargs["RawMessage"]["Data"], str) + assert "Subject: S" in call_kwargs["RawMessage"]["Data"] + + +def test_send_report_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_report_email( + to_address="to@example.com", + from_address="from@example.com", + attachment_path="/tmp/report.zip", + ) + + mocked_send_email.assert_called_once_with( + to_address="to@example.com", + from_address="from@example.com", + subject="Daily Upload Report", + body_text="Please find your encrypted daily upload report attached.", + attachments=["/tmp/report.zip"], + ) + + +def test_send_password_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_password_email( + to_address="to@example.com", + from_address="from@example.com", + password="pw123", + ) + + mocked_send_email.assert_called_once_with( + to_address="to@example.com", + from_address="from@example.com", + subject="Daily Upload Report Password", + body_text="Password for your report:\n\npw123", + ) + + +def test_send_prm_missing_contact_email_calls_send_email_with_expected_inputs(email_service, mocker): + mocked_send_email = mocker.patch.object(email_service, "send_email", autospec=True) + + email_service.send_prm_missing_contact_email( + prm_mailbox="prm@example.com", + from_address="from@example.com", + ods_code="Y12345", + attachment_path="/tmp/report.zip", + password="pw123", + ) + + mocked_send_email.assert_called_once_with( + to_address="prm@example.com", + from_address="from@example.com", + subject="Missing contact for ODS Y12345", + body_text=( + "No contact found for ODS Y12345.\n\n" + "Password: pw123\n\n" + "Please resolve the contact and forward the report." + ), + attachments=["/tmp/report.zip"], + ) diff --git a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py index 4698edf72a..225227983b 100644 --- a/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py +++ b/lambdas/tests/unit/services/reporting/test_excel_report_generator_service.py @@ -1,6 +1,7 @@ import pytest from freezegun import freeze_time from openpyxl import load_workbook + from services.reporting.excel_report_generator_service import ExcelReportGenerator @@ -9,13 +10,43 @@ def excel_report_generator(): return ExcelReportGenerator() -@freeze_time("2025-01-01T12:00:00") -def test_create_report_orchestration_xlsx_happy_path( - excel_report_generator, - tmp_path, -): - output_file = tmp_path / "report.xlsx" +@pytest.fixture +def make_report(tmp_path, excel_report_generator): + def _make(*, ods_code="Y12345", records=None, filename="report.xlsx"): + records = records or [] + output_file = tmp_path / filename + + result = excel_report_generator.create_report_orchestration_xlsx( + ods_code=ods_code, + records=records, + output_path=str(output_file), + ) + + assert result == str(output_file) + assert output_file.exists() + + wb = load_workbook(output_file) + ws = wb.active + return output_file, ws + + return _make + +@pytest.fixture +def expected_header_row(): + return [ + "NHS Number", + "Date", + "Uploader ODS", + "PDS ODS", + "Upload Status", + "Reason", + "File Path", + ] + + +@freeze_time("2025-01-01T12:00:00") +def test_create_report_orchestration_xlsx_happy_path(make_report, expected_header_row): ods_code = "Y12345" records = [ { @@ -40,44 +71,18 @@ def test_create_report_orchestration_xlsx_happy_path( }, ] - result = excel_report_generator.create_report_orchestration_xlsx( - ods_code=ods_code, - records=records, - output_path=str(output_file), - ) - - # File path returned - assert result == str(output_file) - assert output_file.exists() + _, ws = make_report(ods_code=ods_code, records=records, filename="report.xlsx") - wb = load_workbook(output_file) - ws = wb.active - - # Sheet name assert ws.title == "Daily Upload Report" - - # Metadata rows assert ws["A1"].value == f"ODS Code: {ods_code}" - assert ws["A2"].value.startswith("Generated at (UTC): ") - assert ws["A3"].value is None # blank row + assert ws["A2"].value == "Generated at (UTC): 2025-01-01T12:00:00+00:00" + assert ws["A3"].value is None - # Header row - assert [cell.value for cell in ws[4]] == [ - "ID", - "Date", - "NHS Number", - "Uploader ODS", - "PDS ODS", - "Upload Status", - "Reason", - "File Path", - ] + assert [cell.value for cell in ws[4]] == expected_header_row - # First data row assert [cell.value for cell in ws[5]] == [ - 1, - "2025-01-01", "1234567890", + "2025-01-01", "Y12345", "A99999", "SUCCESS", @@ -85,11 +90,9 @@ def test_create_report_orchestration_xlsx_happy_path( "/path/file1.pdf", ] - # Second data row assert [cell.value for cell in ws[6]] == [ - 2, - "2025-01-02", "123456789", + "2025-01-02", "Y12345", "B88888", "FAILED", @@ -98,56 +101,37 @@ def test_create_report_orchestration_xlsx_happy_path( ] -def test_create_report_orchestration_xlsx_with_no_records( - excel_report_generator, - tmp_path, -): - output_file = tmp_path / "empty_report.xlsx" +def test_create_report_orchestration_xlsx_with_no_records(make_report, expected_header_row): + _, ws = make_report(records=[], filename="empty_report.xlsx") - excel_report_generator.create_report_orchestration_xlsx( - ods_code="Y12345", - records=[], - output_path=str(output_file), - ) - - wb = load_workbook(output_file) - ws = wb.active - - # Only metadata + header rows should exist assert ws.max_row == 4 - - + assert [cell.value for cell in ws[4]] == expected_header_row + + +@pytest.mark.parametrize( + "records, expected_row", + [ + ( + [{"ID": 1, "NhsNumber": "1234567890"}], + ["1234567890", None, None, None, None, None, None], + ), + ( + [{"Date": "2025-01-01"}], + [None, "2025-01-01", None, None, None, None, None], + ), + ( + [{"UploaderOdsCode": "Y12345", "UploadStatus": "SUCCESS"}], + [None, None, "Y12345", None, "SUCCESS", None, None], + ), + ], +) def test_create_report_orchestration_xlsx_handles_missing_fields( - excel_report_generator, - tmp_path, + make_report, + expected_header_row, + records, + expected_row, ): - output_file = tmp_path / "partial.xlsx" + _, ws = make_report(records=records, filename="partial.xlsx") - records = [ - { - "ID": 1, - "NhsNumber": "1234567890", - } - ] - - excel_report_generator.create_report_orchestration_xlsx( - ods_code="Y12345", - records=records, - output_path=str(output_file), - ) - - wb = load_workbook(output_file) - ws = wb.active - - row = [cell.value for cell in ws[5]] - - assert row == [ - 1, - None, - "1234567890", - None, - None, - None, - None, - None, - ] + assert [cell.value for cell in ws[4]] == expected_header_row + assert [cell.value for cell in ws[5]] == expected_row diff --git a/lambdas/tests/unit/services/reporting/test_report_distribution_service.py b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py new file mode 100644 index 0000000000..db8b1e3bff --- /dev/null +++ b/lambdas/tests/unit/services/reporting/test_report_distribution_service.py @@ -0,0 +1,346 @@ +import os + +import pytest + +from services.reporting.report_distribution_service import ReportDistributionService + + +@pytest.fixture +def required_report_distribution_env(monkeypatch): + monkeypatch.setenv("SES_FROM_ADDRESS", "from@example.com") + monkeypatch.setenv("PRM_MAILBOX_EMAIL", "prm@example.com") + + +@pytest.fixture +def mock_s3_service(mocker): + return mocker.Mock(name="S3ServiceInstance") + + +@pytest.fixture +def mock_contact_repo(mocker): + repo = mocker.Mock(name="ReportContactRepositoryInstance") + repo.get_contact_email.return_value = None + return repo + + +@pytest.fixture +def mock_email_service(mocker): + return mocker.Mock(name="EmailServiceInstance") + + +@pytest.fixture +def service( + required_report_distribution_env, + mocker, + mock_s3_service, + mock_contact_repo, + mock_email_service, +): + mocker.patch( + "services.reporting.report_distribution_service.S3Service", + autospec=True, + return_value=mock_s3_service, + ) + mocker.patch( + "services.reporting.report_distribution_service.ReportContactRepository", + autospec=True, + return_value=mock_contact_repo, + ) + mocker.patch( + "services.reporting.report_distribution_service.EmailService", + autospec=True, + return_value=mock_email_service, + ) + + return ReportDistributionService(bucket="my-bucket") + + +@pytest.fixture +def fixed_tmpdir(mocker): + fake_tmp = "/tmp/fake_tmpdir" + td = mocker.MagicMock() + td.__enter__.return_value = fake_tmp + td.__exit__.return_value = False + mocker.patch( + "services.reporting.report_distribution_service.tempfile.TemporaryDirectory", + return_value=td, + ) + return fake_tmp + + +@pytest.mark.parametrize( + "key, expected", + [ + ("Report-Orchestration/2026-01-01/Y12345.xlsx", "Y12345"), + ("a/b/C789.XLSX", "C789"), + ("a/b/report.csv", "report.csv"), + ("just-a-name", "just-a-name"), + ("a/b/noext", "noext"), + ], +) +def test_extract_ods_code_from_key(key, expected): + assert ReportDistributionService.extract_ods_code_from_key(key) == expected + + +def test_init_reads_env_and_wires_dependencies(required_report_distribution_env, mocker): + mock_s3 = mocker.Mock(name="S3ServiceInstance") + mock_repo = mocker.Mock(name="ReportContactRepositoryInstance") + mock_email = mocker.Mock(name="EmailServiceInstance") + + mocker.patch( + "services.reporting.report_distribution_service.S3Service", + autospec=True, + return_value=mock_s3, + ) + mocker.patch( + "services.reporting.report_distribution_service.ReportContactRepository", + autospec=True, + return_value=mock_repo, + ) + mocker.patch( + "services.reporting.report_distribution_service.EmailService", + autospec=True, + return_value=mock_email, + ) + + svc = ReportDistributionService(bucket="bucket-1") + + assert svc.bucket == "bucket-1" + assert svc.from_address == "from@example.com" + assert svc.prm_mailbox == "prm@example.com" + assert svc.s3_service is mock_s3 + assert svc.contact_repo is mock_repo + assert svc.email_service is mock_email + + +def test_list_xlsx_keys_filters_only_xlsx(service, mock_s3_service): + mock_s3_service.list_object_keys.return_value = [ + "Report-Orchestration/2026-01-01/A123.xlsx", + "Report-Orchestration/2026-01-01/readme.txt", + "Report-Orchestration/2026-01-01/B456.xls", + "Report-Orchestration/2026-01-01/C789.xlsx", + "Report-Orchestration/2026-01-01/D000.xlsx", + "Report-Orchestration/2026-01-01/E111.xlsx.tmp", + ] + + keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + + mock_s3_service.list_object_keys.assert_called_once_with( + bucket_name="my-bucket", + prefix="Report-Orchestration/2026-01-01/", + ) + assert keys == [ + "Report-Orchestration/2026-01-01/A123.xlsx", + "Report-Orchestration/2026-01-01/C789.xlsx", + "Report-Orchestration/2026-01-01/D000.xlsx", + ] + + +def test_list_xlsx_keys_returns_empty_when_no_objects(service, mock_s3_service): + mock_s3_service.list_object_keys.return_value = [] + + keys = service.list_xlsx_keys(prefix="Report-Orchestration/2026-01-01/") + + mock_s3_service.list_object_keys.assert_called_once_with( + bucket_name="my-bucket", + prefix="Report-Orchestration/2026-01-01/", + ) + assert keys == [] + + +def test_process_one_report_downloads_encrypts_and_delegates_email( + service, mocker, mock_s3_service, fixed_tmpdir +): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="fixed-password", + ) + + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + service.process_one_report( + ods_code="Y12345", + key="Report-Orchestration/2026-01-01/Y12345.xlsx", + ) + + local_xlsx = os.path.join(fixed_tmpdir, "Y12345.xlsx") + local_zip = os.path.join(fixed_tmpdir, "Y12345.zip") + + mock_s3_service.download_file.assert_called_once_with( + "my-bucket", + "Report-Orchestration/2026-01-01/Y12345.xlsx", + local_xlsx, + ) + mocked_zip.assert_called_once_with( + input_path=local_xlsx, + output_zip=local_zip, + password="fixed-password", + ) + mocked_send.assert_called_once_with( + ods_code="Y12345", + attachment_path=local_zip, + password="fixed-password", + ) + + +def test_process_one_report_propagates_download_errors( + service, mocker, mock_s3_service, fixed_tmpdir +): + mock_s3_service.download_file.side_effect = RuntimeError("download failed") + + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", autospec=True + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="download failed"): + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mocked_zip.assert_not_called() + mocked_send.assert_not_called() + + +def test_process_one_report_does_not_send_email_if_zip_fails( + service, mocker, mock_s3_service, fixed_tmpdir +): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + return_value="pw", + ) + mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + side_effect=RuntimeError("zip failed"), + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="zip failed"): + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mocked_send.assert_not_called() + + +def test_process_one_report_does_not_zip_or_send_email_if_password_generation_fails( + service, mocker, mock_s3_service, fixed_tmpdir +): + mocker.patch( + "services.reporting.report_distribution_service.secrets.token_urlsafe", + side_effect=RuntimeError("secrets failed"), + ) + + mocked_zip = mocker.patch( + "services.reporting.report_distribution_service.zip_encrypt_file", + autospec=True, + ) + mocked_send = mocker.patch.object(service, "send_report_emails", autospec=True) + + with pytest.raises(RuntimeError, match="secrets failed"): + service.process_one_report(ods_code="Y12345", key="k.xlsx") + + mock_s3_service.download_file.assert_called_once_with( + "my-bucket", + "k.xlsx", + os.path.join(fixed_tmpdir, "Y12345.xlsx"), + ) + mocked_zip.assert_not_called() + mocked_send.assert_not_called() + + +@pytest.mark.parametrize( + "contact_lookup_result, contact_lookup_side_effect, expected_method", + [ + ("contact@example.com", None, "email_contact"), + (None, None, "email_prm_missing_contact"), + (None, RuntimeError("ddb down"), "email_prm_missing_contact"), + ], +) +def test_send_report_emails_routes_correctly( + service, + mock_contact_repo, + mocker, + contact_lookup_result, + contact_lookup_side_effect, + expected_method, +): + if contact_lookup_side_effect is not None: + mock_contact_repo.get_contact_email.side_effect = contact_lookup_side_effect + else: + mock_contact_repo.get_contact_email.return_value = contact_lookup_result + + mocked_email_contact = mocker.patch.object(service, "email_contact", autospec=True) + mocked_email_prm = mocker.patch.object(service, "email_prm_missing_contact", autospec=True) + + service.send_report_emails( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + + mock_contact_repo.get_contact_email.assert_called_once_with("A99999") + + if expected_method == "email_contact": + mocked_email_contact.assert_called_once_with( + to_address="contact@example.com", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + mocked_email_prm.assert_not_called() + else: + mocked_email_prm.assert_called_once_with( + ods_code="A99999", + attachment_path="/tmp/A99999.zip", + password="pw", + ) + mocked_email_contact.assert_not_called() + + +def test_email_contact_sends_report_and_password(service, mock_email_service): + service.email_contact( + to_address="contact@example.com", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_report_email.assert_called_once_with( + to_address="contact@example.com", + from_address="from@example.com", + attachment_path="/tmp/file.zip", + ) + mock_email_service.send_password_email.assert_called_once_with( + to_address="contact@example.com", + from_address="from@example.com", + password="pw", + ) + + +def test_email_contact_does_not_send_password_if_report_email_fails(service, mock_email_service): + mock_email_service.send_report_email.side_effect = RuntimeError("SES down") + + with pytest.raises(RuntimeError, match="SES down"): + service.email_contact( + to_address="contact@example.com", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_password_email.assert_not_called() + + +def test_email_prm_missing_contact_sends_prm_missing_contact_email(service, mock_email_service): + service.email_prm_missing_contact( + ods_code="X11111", + attachment_path="/tmp/file.zip", + password="pw", + ) + + mock_email_service.send_prm_missing_contact_email.assert_called_once_with( + prm_mailbox="prm@example.com", + from_address="from@example.com", + ods_code="X11111", + attachment_path="/tmp/file.zip", + password="pw", + ) diff --git a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py index c068ba0c16..2487fb235c 100644 --- a/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py +++ b/lambdas/tests/unit/services/reporting/test_report_orchestration_service.py @@ -1,107 +1,206 @@ import pytest + from services.reporting.report_orchestration_service import ReportOrchestrationService @pytest.fixture def mock_repository(mocker): - repo = mocker.Mock() + repo = mocker.Mock(name="ReportingDynamoRepositoryInstance") repo.get_records_for_time_window.return_value = [] return repo @pytest.fixture def mock_excel_generator(mocker): - return mocker.Mock() + return mocker.Mock(name="ExcelReportGeneratorInstance") @pytest.fixture -def report_orchestration_service(mock_repository, mock_excel_generator): - return ReportOrchestrationService( - repository=mock_repository, - excel_generator=mock_excel_generator, +def report_orchestration_service(mocker, mock_repository, mock_excel_generator): + mocker.patch( + "services.reporting.report_orchestration_service.ReportingDynamoRepository", + autospec=True, + return_value=mock_repository, ) + mocker.patch( + "services.reporting.report_orchestration_service.ExcelReportGenerator", + autospec=True, + return_value=mock_excel_generator, + ) + return ReportOrchestrationService() -def test_process_reporting_window_no_records( - report_orchestration_service, mock_repository, mock_excel_generator -): - mock_repository.get_records_for_time_window.return_value = [] - - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") - - mock_excel_generator.create_report_orchestration_xlsx.assert_not_called() - - -def test_group_records_by_ods_groups_correctly(): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - {"UploaderOdsCode": "A99999", "ID": 3}, - {"ID": 4}, # missing ODS - {"UploaderOdsCode": None, "ID": 5}, # null ODS - ] - - result = ReportOrchestrationService.group_records_by_ods(records) - - assert result["Y12345"] == [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - ] - assert result["A99999"] == [{"UploaderOdsCode": "A99999", "ID": 3}] - assert result["UNKNOWN"] == [ - {"ID": 4}, - {"UploaderOdsCode": None, "ID": 5}, - ] +@pytest.fixture +def mocked_generate(report_orchestration_service, mocker): + return mocker.patch.object( + report_orchestration_service, + "generate_ods_report", + autospec=True, + side_effect=lambda ods_code, _records: f"/tmp/{ods_code}.xlsx", + ) -def test_process_reporting_window_generates_reports_per_ods( - report_orchestration_service, mock_repository, mocker +@pytest.mark.parametrize( + "records, expected_generate_calls, expected_result", + [ + ( + [], + [], + {}, + ), + ( + [{"UploaderOdsCode": "X1", "ID": 1}], + [ + ("X1", [{"UploaderOdsCode": "X1", "ID": 1}]), + ], + {"X1": "/tmp/X1.xlsx"}, + ), + ( + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + ], + [ + ( + "Y12345", + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ], + ), + ("A99999", [{"UploaderOdsCode": "A99999", "ID": 3}]), + ], + {"Y12345": "/tmp/Y12345.xlsx", "A99999": "/tmp/A99999.xlsx"}, + ), + ( + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"ID": 2}, + {"UploaderOdsCode": None, "ID": 3}, + ], + [ + ("Y12345", [{"UploaderOdsCode": "Y12345", "ID": 1}]), + ("UNKNOWN", [{"ID": 2}, {"UploaderOdsCode": None, "ID": 3}]), + ], + {"Y12345": "/tmp/Y12345.xlsx", "UNKNOWN": "/tmp/UNKNOWN.xlsx"}, + ), + ( + [{"UploaderOdsCode": "", "ID": 1}], + [ + ("UNKNOWN", [{"UploaderOdsCode": "", "ID": 1}]), + ], + {"UNKNOWN": "/tmp/UNKNOWN.xlsx"}, + ), + ], +) +def test_process_reporting_window_behaviour( + report_orchestration_service, + mock_repository, + mocked_generate, + records, + expected_generate_calls, + expected_result, ): - records = [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - {"UploaderOdsCode": "A99999", "ID": 3}, - ] mock_repository.get_records_for_time_window.return_value = records - mocked_generate = mocker.patch.object( - report_orchestration_service, "generate_ods_report" - ) - - report_orchestration_service.process_reporting_window(100, 200, output_dir="/tmp") - - mocked_generate.assert_any_call( - "Y12345", - [ - {"UploaderOdsCode": "Y12345", "ID": 1}, - {"UploaderOdsCode": "Y12345", "ID": 2}, - ], - ) - mocked_generate.assert_any_call( - "A99999", - [{"UploaderOdsCode": "A99999", "ID": 3}], - ) - assert mocked_generate.call_count == 2 + result = report_orchestration_service.process_reporting_window(100, 200) + + mock_repository.get_records_for_time_window.assert_called_once_with(100, 200) + + assert mocked_generate.call_count == len(expected_generate_calls) + for ods_code, ods_records in expected_generate_calls: + mocked_generate.assert_any_call(ods_code, ods_records) + + assert result == expected_result + + +@pytest.mark.parametrize( + "records, expected", + [ + ( + [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + {"UploaderOdsCode": "A99999", "ID": 3}, + {"ID": 4}, + {"UploaderOdsCode": None, "ID": 5}, + ], + { + "Y12345": [ + {"UploaderOdsCode": "Y12345", "ID": 1}, + {"UploaderOdsCode": "Y12345", "ID": 2}, + ], + "A99999": [{"UploaderOdsCode": "A99999", "ID": 3}], + "UNKNOWN": [{"ID": 4}, {"UploaderOdsCode": None, "ID": 5}], + }, + ), + ([], {}), + ( + [{"UploaderOdsCode": "", "ID": 1}], + {"UNKNOWN": [{"UploaderOdsCode": "", "ID": 1}]}, + ), + ], +) +def test_group_records_by_ods(records, expected): + result = ReportOrchestrationService.group_records_by_ods(records) + assert dict(result) == expected -def test_generate_ods_report_creates_excel_report( - report_orchestration_service, mock_excel_generator, mocker -): +@pytest.fixture +def fake_named_tmpfile(mocker): fake_tmp = mocker.MagicMock() fake_tmp.__enter__.return_value = fake_tmp + fake_tmp.__exit__.return_value = False fake_tmp.name = "/tmp/fake_Y12345.xlsx" - mocker.patch( + mocked_ntf = mocker.patch( "services.reporting.report_orchestration_service.tempfile.NamedTemporaryFile", return_value=fake_tmp, ) + return mocked_ntf, fake_tmp + +def test_generate_ods_report_creates_excel_report_and_returns_path( + report_orchestration_service, mock_excel_generator, fake_named_tmpfile +): + mocked_ntf, fake_tmp = fake_named_tmpfile records = [{"ID": 1, "UploaderOdsCode": "Y12345"}] - report_orchestration_service.generate_ods_report("Y12345", records) + result_path = report_orchestration_service.generate_ods_report("Y12345", records) + + assert result_path == fake_tmp.name + + mocked_ntf.assert_called_once_with( + suffix="_Y12345.xlsx", + delete=False, + ) mock_excel_generator.create_report_orchestration_xlsx.assert_called_once_with( ods_code="Y12345", records=records, output_path=fake_tmp.name, ) + + +def test_init_constructs_repository_and_excel_generator(mocker): + mock_repo = mocker.Mock(name="ReportingDynamoRepositoryInstance") + mock_excel = mocker.Mock(name="ExcelReportGeneratorInstance") + + mocked_repo_cls = mocker.patch( + "services.reporting.report_orchestration_service.ReportingDynamoRepository", + autospec=True, + return_value=mock_repo, + ) + mocked_excel_cls = mocker.patch( + "services.reporting.report_orchestration_service.ExcelReportGenerator", + autospec=True, + return_value=mock_excel, + ) + + svc = ReportOrchestrationService() + + mocked_repo_cls.assert_called_once_with() + mocked_excel_cls.assert_called_once_with() + assert svc.repository is mock_repo + assert svc.excel_generator is mock_excel diff --git a/lambdas/tests/unit/utils/test_utilities.py b/lambdas/tests/unit/utils/test_utilities.py index 68ddb60e5b..23a3b3745b 100755 --- a/lambdas/tests/unit/utils/test_utilities.py +++ b/lambdas/tests/unit/utils/test_utilities.py @@ -12,7 +12,7 @@ get_pds_service, parse_date, redact_id_to_last_4_chars, - validate_nhs_number, + validate_nhs_number, utc_date_string, ) @@ -135,3 +135,18 @@ def test_format_cloudfront_url_valid(): def test_parse_date_returns_correct_date_for_valid_formats(input_date, expected_date): result = parse_date(input_date) assert result == expected_date + +@pytest.mark.parametrize( + "timestamp_seconds, expected_date_string", + [ + (0, "1970-01-01"), + (1704067200, "2024-01-01"), + (1767780025, "2026-01-07"), + (1704153599, "2024-01-01"), + (1704153600, "2024-01-02"), + ], +) +def test_utc_date_string_returns_correct_utc_date( + timestamp_seconds, expected_date_string +): + assert utc_date_string(timestamp_seconds) == expected_date_string diff --git a/lambdas/utils/lambda_exceptions.py b/lambdas/utils/lambda_exceptions.py index 9a30812eed..cc4af788a4 100644 --- a/lambdas/utils/lambda_exceptions.py +++ b/lambdas/utils/lambda_exceptions.py @@ -120,3 +120,6 @@ class DocumentReviewLambdaException(LambdaException): class UpdateDocumentReviewException(LambdaException): pass + +class ReportDistributionException(LambdaException): + pass \ No newline at end of file diff --git a/lambdas/utils/utilities.py b/lambdas/utils/utilities.py index b83af68e4b..f23720e9b1 100755 --- a/lambdas/utils/utilities.py +++ b/lambdas/utils/utilities.py @@ -2,7 +2,7 @@ import os import re import uuid -from datetime import datetime +from datetime import datetime, timezone, date, time from urllib.parse import urlparse from inflection import camelize @@ -127,3 +127,18 @@ def parse_date(date_string: str) -> datetime | None: except ValueError: continue return None + + +def utc_date_string(timestamp_seconds: int) -> str: + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).strftime("%Y-%m-%d") + +def utc_date(timestamp_seconds: int) -> date: + return datetime.fromtimestamp(timestamp_seconds, tz=timezone.utc).date() + +def utc_day_start_timestamp(day: date) -> int: + return int( + datetime.combine(day, time.min, tzinfo=timezone.utc).timestamp() + ) + +def utc_day_end_timestamp(day: date) -> int: + return utc_day_start_timestamp(day) + 24 * 60 * 60 - 1 \ No newline at end of file diff --git a/lambdas/utils/zip_utils.py b/lambdas/utils/zip_utils.py new file mode 100644 index 0000000000..43775059e7 --- /dev/null +++ b/lambdas/utils/zip_utils.py @@ -0,0 +1,23 @@ +import os +import pyzipper + + +def zip_encrypt_file(*, input_path: str, output_zip: str, password: str) -> None: + """Create an AES-encrypted ZIP file containing a single file. + + Args: + input_path: Path to the file to zip. + output_zip: Path of the ZIP file to create. + password: Password used for AES encryption. + + Returns: + None + """ + with pyzipper.AESZipFile( + output_zip, + "w", + compression=pyzipper.ZIP_DEFLATED, + encryption=pyzipper.WZ_AES, + ) as zf: + zf.setpassword(password.encode("utf-8")) + zf.write(input_path, arcname=os.path.basename(input_path)) diff --git a/poetry.lock b/poetry.lock index c329268964..7fa78253f3 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1671,6 +1671,57 @@ files = [ {file = "pycparser-2.23.tar.gz", hash = "sha256:78816d4f24add8f10a06d6f05b4d424ad9e96cfebf68a4ddc99c65c0720d00c2"}, ] +[[package]] +name = "pycryptodomex" +version = "3.23.0" +description = "Cryptographic library for Python" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["reports-lambda"] +files = [ + {file = "pycryptodomex-3.23.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:add243d204e125f189819db65eed55e6b4713f70a7e9576c043178656529cec7"}, + {file = "pycryptodomex-3.23.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:1c6d919fc8429e5cb228ba8c0d4d03d202a560b421c14867a65f6042990adc8e"}, + {file = "pycryptodomex-3.23.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:1c3a65ad441746b250d781910d26b7ed0a396733c6f2dbc3327bd7051ec8a541"}, + {file = "pycryptodomex-3.23.0-cp27-cp27m-win32.whl", hash = "sha256:47f6d318fe864d02d5e59a20a18834819596c4ed1d3c917801b22b92b3ffa648"}, + {file = "pycryptodomex-3.23.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:d9825410197a97685d6a1fa2a86196430b01877d64458a20e95d4fd00d739a08"}, + {file = "pycryptodomex-3.23.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:267a3038f87a8565bd834317dbf053a02055915acf353bf42ededb9edaf72010"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:7b37e08e3871efe2187bc1fd9320cc81d87caf19816c648f24443483005ff886"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:91979028227543010d7b2ba2471cf1d1e398b3f183cb105ac584df0c36dac28d"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b8962204c47464d5c1c4038abeadd4514a133b28748bcd9fa5b6d62e3cec6fa"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a33986a0066860f7fcf7c7bd2bc804fa90e434183645595ae7b33d01f3c91ed8"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7947ab8d589e3178da3d7cdeabe14f841b391e17046954f2fbcd941705762b5"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:c25e30a20e1b426e1f0fa00131c516f16e474204eee1139d1603e132acffc314"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:da4fa650cef02db88c2b98acc5434461e027dce0ae8c22dd5a69013eaf510006"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:58b851b9effd0d072d4ca2e4542bf2a4abcf13c82a29fd2c93ce27ee2a2e9462"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:a9d446e844f08299236780f2efa9898c818fe7e02f17263866b8550c7d5fb328"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:bc65bdd9fc8de7a35a74cab1c898cab391a4add33a8fe740bda00f5976ca4708"}, + {file = "pycryptodomex-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:c885da45e70139464f082018ac527fdaad26f1657a99ee13eecdce0f0ca24ab4"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:06698f957fe1ab229a99ba2defeeae1c09af185baa909a31a5d1f9d42b1aaed6"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:b2c2537863eccef2d41061e82a881dcabb04944c5c06c5aa7110b577cc487545"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:43c446e2ba8df8889e0e16f02211c25b4934898384c1ec1ec04d7889c0333587"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f489c4765093fb60e2edafdf223397bc716491b2b69fe74367b70d6999257a5c"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:bdc69d0d3d989a1029df0eed67cc5e8e5d968f3724f4519bd03e0ec68df7543c"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:6bbcb1dd0f646484939e142462d9e532482bc74475cecf9c4903d4e1cd21f003"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:8a4fcd42ccb04c31268d1efeecfccfd1249612b4de6374205376b8f280321744"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:55ccbe27f049743a4caf4f4221b166560d3438d0b1e5ab929e07ae1702a4d6fd"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-win32.whl", hash = "sha256:189afbc87f0b9f158386bf051f720e20fa6145975f1e76369303d0f31d1a8d7c"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:52e5ca58c3a0b0bd5e100a9fbc8015059b05cffc6c66ce9d98b4b45e023443b9"}, + {file = "pycryptodomex-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:02d87b80778c171445d67e23d1caef279bf4b25c3597050ccd2e13970b57fd51"}, + {file = "pycryptodomex-3.23.0-pp27-pypy_73-manylinux2010_x86_64.whl", hash = "sha256:febec69c0291efd056c65691b6d9a339f8b4bc43c6635b8699471248fe897fea"}, + {file = "pycryptodomex-3.23.0-pp27-pypy_73-win32.whl", hash = "sha256:c84b239a1f4ec62e9c789aafe0543f0594f0acd90c8d9e15bcece3efe55eca66"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ebfff755c360d674306e5891c564a274a47953562b42fb74a5c25b8fc1fb1cb5"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eca54f4bb349d45afc17e3011ed4264ef1cc9e266699874cdd1349c504e64798"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2596e643d4365e14d0879dc5aafe6355616c61c2176009270f3048f6d9a61f"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fdfac7cda115bca3a5abb2f9e43bc2fb66c2b65ab074913643803ca7083a79ea"}, + {file = "pycryptodomex-3.23.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:14c37aaece158d0ace436f76a7bb19093db3b4deade9797abfc39ec6cd6cc2fe"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-macosx_10_15_x86_64.whl", hash = "sha256:7de1e40a41a5d7f1ac42b6569b10bcdded34339950945948529067d8426d2785"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bffc92138d75664b6d543984db7893a628559b9e78658563b0395e2a5fb47ed9"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:df027262368334552db2c0ce39706b3fb32022d1dce34673d0f9422df004b96a"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4e79f1aaff5a3a374e92eb462fa9e598585452135012e2945f96874ca6eeb1ff"}, + {file = "pycryptodomex-3.23.0-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:27e13c80ac9a0a1d050ef0a7e0a18cc04c8850101ec891815b6c5a0375e8a245"}, + {file = "pycryptodomex-3.23.0.tar.gz", hash = "sha256:71909758f010c82bc99b0abf4ea12012c98962fbf0583c2164f8b84533c2e4da"}, +] + [[package]] name = "pydantic" version = "2.12.5" @@ -2007,6 +2058,21 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "pyzipper" +version = "0.3.6" +description = "AES encryption for zipfile." +optional = false +python-versions = ">=3.4" +groups = ["reports-lambda"] +files = [ + {file = "pyzipper-0.3.6-py2.py3-none-any.whl", hash = "sha256:6d097f465bfa47796b1494e12ea65d1478107d38e13bc56f6e58eedc4f6c1a87"}, + {file = "pyzipper-0.3.6.tar.gz", hash = "sha256:0adca90a00c36a93fbe49bfa8c5add452bfe4ef85a1b8e3638739dd1c7b26bfc"}, +] + +[package.dependencies] +pycryptodomex = "*" + [[package]] name = "regex" version = "2023.12.25" @@ -2450,4 +2516,4 @@ requests = "*" [metadata] lock-version = "2.1" python-versions = "^3.11" -content-hash = "77d0249d2dd6c9fbb02b0e434ade72ac221f4df82e8d98ec49250fe0b7ff74df" +content-hash = "853336b823ceae919933474298444b50eb7935088cf43b92d713c8326f90e5be" diff --git a/pyproject.toml b/pyproject.toml index 4abaa1617d..1e174b6ece 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,3 +54,4 @@ polars = "1.31.0" [tool.poetry.group.reports_lambda.dependencies] openpyxl = "^3.1.5" reportlab = "^4.3.1" +pyzipper = "^0.3.6"