From 7fbf536380d907c643b74d39325ffa71cbcfe5be Mon Sep 17 00:00:00 2001 From: Harriet H-W Date: Wed, 10 Dec 2025 16:13:21 +0000 Subject: [PATCH] wip --- .../commands/store_mesh_messages.py | 2 + .../tests/integration/test_load_test.py | 48 ++++++++++ scripts/python/load_test/load_test_data.dat | 4 + .../load_test/notifications_load_test.py | 95 +++++++++++++++++++ scripts/terraform/terraform.mk | 6 ++ 5 files changed, 155 insertions(+) create mode 100644 manage_breast_screening/notifications/tests/integration/test_load_test.py create mode 100644 scripts/python/load_test/load_test_data.dat create mode 100644 scripts/python/load_test/notifications_load_test.py diff --git a/manage_breast_screening/notifications/management/commands/store_mesh_messages.py b/manage_breast_screening/notifications/management/commands/store_mesh_messages.py index 64c3e85c3..a378ce8df 100644 --- a/manage_breast_screening/notifications/management/commands/store_mesh_messages.py +++ b/manage_breast_screening/notifications/management/commands/store_mesh_messages.py @@ -28,6 +28,8 @@ def handle(self, *args, **options): logger.debug("Processing message %s", message_id) message = inbox.fetch_message(message_id) + logger.debug("Processing message with subject %s", message.subject) + BlobStorage().add( f"{today_dirname}/{message.filename}", message.read().decode("ASCII"), diff --git a/manage_breast_screening/notifications/tests/integration/test_load_test.py b/manage_breast_screening/notifications/tests/integration/test_load_test.py new file mode 100644 index 000000000..95fb6eee4 --- /dev/null +++ b/manage_breast_screening/notifications/tests/integration/test_load_test.py @@ -0,0 +1,48 @@ +import os +from datetime import datetime + +import pytest + +from manage_breast_screening.notifications.management.commands.create_appointments import ( + Command, +) +from manage_breast_screening.notifications.services.blob_storage import BlobStorage +from manage_breast_screening.notifications.tests.integration.helpers import Helpers +from scripts.python.load_test.notifications_load_test import generate_load_test_data + + +# TODO: DELETE - setup as easy way to validate data locally +@pytest.mark.integration +class TestLoadTest: + @pytest.fixture(autouse=True) + def setup(self, monkeypatch): + monkeypatch.setenv( + "BLOB_STORAGE_CONNECTION_STRING", Helpers().azurite_connection_string() + ) + monkeypatch.setenv("BLOB_CONTAINER_NAME", "nbss-appoinments-data") + + @pytest.fixture + def helpers(self): + return Helpers() + + @pytest.mark.django_db + def test_generate_test_data(self, helpers): + for number in range(10): + with open( + f"{os.path.dirname(os.path.realpath(__file__))}/../fixtures/load_data_{number}.dat", + "w", + ) as file: + data_string = generate_load_test_data(number) + file.write(data_string) + + for number in range(10): + today_dirname = datetime.today().strftime("%Y-%m-%d") + test_file_name = f"load_data_{number}.dat" + blob_name = f"{today_dirname}/{test_file_name}" + + with open( + f"{os.path.dirname(os.path.realpath(__file__))}/../fixtures/load_data_{number}.dat" + ) as test_file: + BlobStorage().add(blob_name, test_file.read()) + + Command().handle(**{"date_str": today_dirname}) diff --git a/scripts/python/load_test/load_test_data.dat b/scripts/python/load_test/load_test_data.dat new file mode 100644 index 000000000..07a528780 --- /dev/null +++ b/scripts/python/load_test/load_test_data.dat @@ -0,0 +1,4 @@ +"NBSSAPPT_HDR"|"88888888"|"20250101"|"170922"|"000003" +"NBSSAPPT_FLDS"|"Sequence"|"BSO"|"Action"|"Clinic Code"|"Holding Clinic"|"Status"|"Attended Not Scr"|"Appointment ID"|"NHS Num"|"Epsiode Type"|"Episode Start"|"BatchID"|"Screen or Asses"|"Screen Appt num"|"Booked By"|"Cancelled By"|"Appt Date"|"Appt Time"|"Location"|"Clinic Name"|"Clinic Name (Let)"|"Clinic Address 1"|"Clinic Address 2"|"Clinic Address 3"|"Clinic Address 4"|"Clinic Address 5"|"Postcode"|"Action Timestamp" +"NBSSAPPT_DATA"|"000001"|"L04D"|"B"|"L04D"|"N"|"B"|"N"|"L04D-0000000000"|"9449306621"|"F"|"20250101"|"SM0K0000"|"S"|"1"|"H"|""|"20250101"|"1445"|"SMOK"|"SMOKE TEST CLINIC"|"SMOKE TEST CLINIC"|"SMOKE TEST UNIT"|"NOT A REAL HOSPITAL"|"SMOKE TEST LANE"|"LONDON"|"E5 0AB"|"E5 0AB"|"20250101-154004" +"NBSSAPPT_END"|"00000013"|"20250101"|"17:09:22"|"000003" diff --git a/scripts/python/load_test/notifications_load_test.py b/scripts/python/load_test/notifications_load_test.py new file mode 100644 index 000000000..f51cc2db8 --- /dev/null +++ b/scripts/python/load_test/notifications_load_test.py @@ -0,0 +1,95 @@ +import logging +import os +import subprocess +import time +from datetime import datetime + +from manage_breast_screening.notifications.models import ( + Appointment, + Clinic, +) +from scripts.python.smoke_test.notifications_smoke_test import ( + mesh_client, + populate_mesh_env_vars, +) + +WORK_DIR = os.path.dirname(os.path.realpath(__file__)) +STARTUP_SCRIPT_PATH = f"{WORK_DIR}/../../bash/run_container_app_job.sh" +NUMBER_OF_ROWS = os.getenv("NUMBER_OF_ROWS", "5") +NUMBER_OF_FILES = os.getenv("NUMBER_OF_ROWS", "5") + + +def test_load(): + logging.info("Running notifications load test") + + environment = os.getenv("ENVIRONMENT") + resource_group_name = f"rg-manbrs-{environment}-container-app-uks" + + if environment == "prod": + return + + add_load_test_data_to_mesh(environment, resource_group_name) + + for job in ["smm", "cap"]: + logging.info( + "Starting notifications container app job manbrs-%s-%s", job, environment + ) + + job_result = subprocess.run( + [STARTUP_SCRIPT_PATH, environment, job], + check=True, + capture_output=True, + text=True, + ) + assert job_result.returncode == 0 + + logging.info("Finished notifications load test") + + +def add_load_test_data_to_mesh(environment: str, resource_group_name: str): + populate_mesh_env_vars(environment, resource_group_name) + for number in range(int(NUMBER_OF_FILES)): + logging.info(f"sending mesh message {number}") + mesh_client().send_message( + os.getenv("NBSS_MESH_INBOX_NAME"), + generate_load_test_data(number).encode("ASCII"), + subject="Load test data", + ) + + +def generate_load_test_data(file_sequence_number: int) -> str: + data = open(f"{WORK_DIR}/load_test_data.dat").read() + data = data.replace("20250101", datetime.now().strftime("%Y%m%d")) + data = data.replace("88888888", f"{file_sequence_number:06d}") + first_appointment_id = f"L04D-1-{time.time_ns()}" + data = data.replace("L04D-0000000000", first_appointment_id) + + rows = data.split("\n") + _header = rows[0] + _footer = rows[3] + first_data_row = rows[2] + + for number in range(2, int(NUMBER_OF_ROWS)): + # logging.info(f"creating row {number}") + new_data = first_data_row.replace("000001", f"{number:06d}") + new_data = new_data.replace( + first_appointment_id, f"L04D-{number}-{time.time_ns()}" + ) + rows.insert((number + 1), new_data) + + return "\n".join(rows) + + +def delete_all_load_test_data(): + # You can run this in a python shell in the AVD, put the following in after exec into web app: + # python manage.py shell + # from manage_breast_screening.notifications.models import ( + # Appointment, + # Clinic, + # ) + appt_count = Appointment.objects.filter(nbss_id__contains="L04D").count() + Appointment.objects.filter(nbss_id__contains="L04D").delete() + logging.info("deleting %s appointments", appt_count) + clinic_count = Clinic.objects.filter(bso_code__contains="L04D").count() + logging.info("deleting %s clinics", clinic_count) + Clinic.objects.filter(bso_code__contains="L04D").delete() diff --git a/scripts/terraform/terraform.mk b/scripts/terraform/terraform.mk index 54d07a88a..bf24a8b4e 100644 --- a/scripts/terraform/terraform.mk +++ b/scripts/terraform/terraform.mk @@ -83,3 +83,9 @@ notifications-smoke-test: else echo "Skipping notifications smoke test" fi + +notifications-load-test: + $(eval export ENVIRONMENT=${ENVIRONMENT}) + pip install pytest mesh-client + pytest -vv scripts/python/load_test/notifications_load_test.py --log-cli-level=DEBUG +