[PRMP-873] Bulk Upload Script to use new metadata processor

lambdas/services/mock_pds_service.py

@@ -5,15 +5,21 @@
 
 from requests import Response
 from services.patient_search_service import PatientSearch
+from utils.audit_logging_setup import LoggingService
 from utils.exceptions import PdsErrorException
 
+logger = LoggingService(__name__)
+
 
 class MockPdsApiService(PatientSearch):
     def __init__(self, always_pass_mock: bool = False, *args, **kwargs):
         self.always_pass_mock = always_pass_mock
         pass
 
     def pds_request(self, nhs_number: str, *args, **kwargs) -> Response:
+        logger.info(
+            f"MockPdsApiService: pds_request called with nhs_number={nhs_number}"
+        )
         mock_pds_results: list[dict] = []
 
         if os.getenv("MOCK_PDS_TOO_MANY_REQUESTS_ERROR") == "true":

lambdas/tests/e2e/api/__snapshots__/test_search_patient_ui_api/test_search_patient_found.json

@@ -0,0 +1,14 @@
+{
+  "active": true,
+  "birthDate": "2023-03-06",
+  "deceased": false,
+  "familyName": "PENDYALA",
+  "givenName": [
+    "VANITA",
+    "SAMIKSHA"
+  ],
+  "nhsNumber": "9730154813",
+  "postalCode": "KT10 0NJ",
+  "restricted": false,
+  "superseded": false
+}

lambdas/tests/e2e/api/__snapshots__/test_search_patient_ui_api/test_search_patient_not_found.json

@@ -0,0 +1,4 @@
+{
+  "err_code": "PN_4001",
+  "message": "Invalid patient number 9030154813"
+}

lambdas/tests/e2e/api/conftest.py

@@ -0,0 +1,36 @@
+import requests
+from tests.e2e.conftest import API_ENDPOINT
+
+
+def search_patient(token, patient):
+    url = f"https://{API_ENDPOINT}/SearchPatient"
+    headers = {
+        "Authorization": token,
+    }
+    params = {"patientId": patient}
+
+    response = requests.get(url, headers=headers, params=params)
+    if response.status_code != 200:
+        raise Exception(f"SearchPatient failed: {response.status_code} {response.text}")
+    return response
+
+
+def initiate_stitch(token, patient):
+    url = f"https://{API_ENDPOINT}/LloydGeorgeStitch?patientId={patient}"
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": token,
+    }
+    payload = ""
+
+    return requests.post(url, headers=headers, data=payload)
+
+
+def check_stitch(token, patient):
+    url = f"https://{API_ENDPOINT}/LloydGeorgeStitch?patientId={patient}"
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": token,
+    }
+
+    return requests.get(url, headers=headers)

lambdas/tests/e2e/api/test_search_patient_ui_api.py

@@ -0,0 +1,28 @@
+from syrupy.filters import paths
+from tests.e2e.api.conftest import search_patient
+from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
+from tests.e2e.helpers.lloyd_george_mockcis2_helper import LloydGeorgeMockcis2Helper
+
+lloyd_george_datahelper = LloydGeorgeDataHelper()
+
+
+def test_search_patient_found(snapshot_json):
+    login_helper = LloydGeorgeMockcis2Helper(ods="H81109", repository_role="gp_admin")
+    login_helper.generate_mockcis2_token()
+    patient_search_result = search_patient(login_helper.user_token, 9730154813)
+    assert (
+        patient_search_result.status_code == 200
+    ), f"Expected status code 200, but got {patient_search_result.status_code}"
+    assert patient_search_result.json() == snapshot_json()
+
+
+def test_search_patient_not_found(snapshot_json):
+    login_helper = LloydGeorgeMockcis2Helper(ods="H81109", repository_role="gp_admin")
+    login_helper.generate_mockcis2_token()
+    patient_search_result = search_patient(login_helper.user_token, 9030154813)
+    assert (
+        patient_search_result.status_code == 400
+    ), f"Expected status code 200, but got {patient_search_result.status_code}"
+    assert patient_search_result.json() == snapshot_json(
+        exclude=paths("interaction_id")
+    )

lambdas/tests/e2e/api/test_stitch_api.py

@@ -0,0 +1,51 @@
+import io
+import uuid
+
+import requests
+from tests.e2e.api.conftest import initiate_stitch, search_patient
+from tests.e2e.conftest import API_ENDPOINT, fetch_with_retry
+from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
+from tests.e2e.helpers.lloyd_george_mockcis2_helper import LloydGeorgeMockcis2Helper
+
+lloyd_george_datahelper = LloydGeorgeDataHelper()
+
+
+def test_retrieve_document(test_data):
+    lloyd_george_record = {}
+    test_data.append(lloyd_george_record)
+
+    lloyd_george_record["id"] = str(uuid.uuid4())
+    lloyd_george_record["nhs_number"] = "9449305943"
+    lloyd_george_record["data"] = io.BytesIO(b"Sample PDF Content")
+
+    lloyd_george_datahelper.create_metadata(lloyd_george_record)
+    lloyd_george_datahelper.create_resource(lloyd_george_record)
+
+    login_helper = LloydGeorgeMockcis2Helper(ods="H81109", repository_role="gp_admin")
+    login_helper.generate_mockcis2_token()
+
+    search_patient(login_helper.user_token, lloyd_george_record["nhs_number"])
+    initiate_response = initiate_stitch(
+        login_helper.user_token, lloyd_george_record["nhs_number"]
+    )
+    assert initiate_response.status_code == 200
+
+    def condition(response_json):
+        return response_json.get("jobStatus", "Pending") == "Completed"
+
+    url = f"https://{API_ENDPOINT}/LloydGeorgeStitch?patientId={lloyd_george_record['nhs_number']}"
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": login_helper.user_token,
+    }
+
+    response = fetch_with_retry(
+        url,
+        condition,
+        headers=headers,
+    )
+
+    presignedUrl = response.json().get("presignedUrl")
+    presign_response = requests.get(url=presignedUrl)
+    assert presign_response.status_code == 200
+    assert presign_response.content == io.BytesIO(b"Sample PDF Content").getvalue()

lambdas/tests/e2e/api/test_upload_document_api.py

@@ -16,6 +16,7 @@
 from tests.e2e.helpers.data_helper import LloydGeorgeDataHelper
 
 data_helper = LloydGeorgeDataHelper()
+headers = {"Authorization": "Bearer 123", "X-Api-Key": API_KEY}
 
 
 def create_upload_payload(lloyd_george_record):
@@ -92,7 +93,7 @@ def condition(response_json):
         logging.info(response_json)
         return response_json["content"][0]["attachment"].get("data", False)
 
-    raw_retrieve_response = fetch_with_retry(retrieve_url, condition)
+    raw_retrieve_response = fetch_with_retry(retrieve_url, condition, headers=headers)
     retrieve_response = raw_retrieve_response.json()
 
     attachment_url = upload_response["content"][0]["attachment"]["url"]
@@ -142,7 +143,7 @@ def condition(response_json):
         logging.info(response_json)
         return response_json["content"][0]["attachment"].get("url", False)
 
-    raw_retrieve_response = fetch_with_retry(retrieve_url, condition)
+    raw_retrieve_response = fetch_with_retry(retrieve_url, condition, headers)
     retrieve_response = raw_retrieve_response.json()
 
     expected_presign_uri = f"https://{LLOYD_GEORGE_S3_BUCKET}.s3.eu-west-2.amazonaws.com/{lloyd_george_record['nhs_number']}/{lloyd_george_record['id']}"
@@ -189,7 +190,7 @@ def condition(response_json):
         logging.info(response_json)
         return response_json.get("docStatus", False) == "cancelled"
 
-    raw_retrieve_response = fetch_with_retry(retrieve_url, condition)
+    raw_retrieve_response = fetch_with_retry(retrieve_url, condition, headers)
     retrieve_response = raw_retrieve_response.json()
 
     assert upload_response == snapshot_json(exclude=paths("id", "date"))

lambdas/tests/e2e/conftest.py

@@ -30,10 +30,9 @@ def test_data():
         data_helper.tidyup(record)
 
 
-def fetch_with_retry(url, condition_func, max_retries=5, delay=10):
+def fetch_with_retry(url, condition_func, headers, max_retries=5, delay=10):
     retries = 0
     while retries < max_retries:
-        headers = {"Authorization": "Bearer 123", "X-Api-Key": API_KEY}
         response = requests.get(url, headers=headers)
         if condition_func(response.json()):
             return response

lambdas/tests/e2e/helpers/data_helper.py

@@ -40,21 +40,21 @@ def build_record(
             record["size"] = size
         return record
 
-    def create_metadata(self, document_details):
+    def create_metadata(self, record):
         dynamo_item = {
-            "ID": document_details["id"],
+            "ID": record["id"],
             "ContentType": "application/pdf",
             "Created": datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
             "CurrentGpOds": "H81109",
             "Custodian": "H81109",
-            "DocStatus": document_details.get("doc_status", "final"),
+            "DocStatus": record.get("doc_status", "final"),
             "DocumentScanCreation": "2023-01-01",
             "DocumentSnomedCodeType": self.snomed_code,
-            "FileLocation": f"s3://{self.s3_bucket}/{document_details['nhs_number']}/{document_details['id']}",
-            "FileName": f"1of1_{self.record_type}_[Holly Lorna MAGAN]_[{document_details['nhs_number']}]_[29-05-2006].pdf",
-            "FileSize": document_details.get("size", "12345"),
+            "FileLocation": f"s3://{self.s3_bucket}/{record['nhs_number']}/{record['id']}",
+            "FileName": f"1of1_{self.record_type}_[Holly Lorna MAGAN]_[{record['nhs_number']}]_[29-05-2006].pdf",
+            "FileSize": record.get("size", "12345"),
             "LastUpdated": 1743177202,
-            "NhsNumber": document_details["nhs_number"],
+            "NhsNumber": record["nhs_number"],
             "Status": "current",
             "Uploaded": True,
             "Uploading": False,

lambdas/tests/e2e/performance/locustfile.py

@@ -0,0 +1,41 @@
+import time
+
+from locust import HttpUser, between, task
+from tests.e2e.api.conftest import search_patient
+from tests.e2e.helpers.lloyd_george_mockcis2_helper import LloydGeorgeMockcis2Helper
+
+
+class NationalDocRepositoryUser(HttpUser):
+    wait_time = between(1, 5)
+
+    @task
+    def two_tone_test(self):
+        nhs_number = "9100000582"
+        start = time.time()
+        helper = LloydGeorgeMockcis2Helper(ods="M85143", repository_role="gp_admin")
+        helper.generate_mockcis2_token()
+
+        search_patient(helper.user_token, nhs_number)
+        metric = self.environment.stats.get("extended_task", "Time to Record")
+        self.client.post(
+            f"/LloydGeorgeStitch?patientId={nhs_number}", "Initiate Stitching"
+        )
+        complete = False
+        while not complete:
+            response = self.client.get(
+                f"/LloydGeorgeStitch?patientId={nhs_number}", name="Stitch Status"
+            )
+            try:
+                data = response.json()
+                if data.get("jobStatus") == "Completed":
+                    presignedUrl = response.json().get(
+                        "presignedUrl", name="Downloading the file"
+                    )
+                    self.client.get(url=presignedUrl)
+
+                    complete = True
+            except Exception:
+                pass
+            time.sleep(3)
+        total_time_to_record = (time.time() - start) * 1000
+        metric.log(total_time_to_record)

tests/bulk-upload/scripts/run_bulk_upload.py

@@ -35,4 +35,7 @@ def invoke_lambda(lambda_name, payload={}):
     if args.start_bulk_upload or input(
         "Would you like to start the Bulk Upload Process:"
     ):
-        invoke_lambda(f"{args.environment}_BulkUploadMetadataLambda")
+        invoke_lambda(
+            f"{args.environment}_BulkUploadMetadataProcessor",
+            payload={"practiceDirectory": "test"},
+        )

tests/bulk-upload/scripts/setup_bulk_upload.py

@@ -327,9 +327,14 @@ def build_metadata_csv(
         row = build_metadata_csv_rows(
             patient=patient, total_number=total_number_for_each
         )
-        all_rows.append(row)
+        # Add "test/" prefix to FILEPATH values in each row
+        prefixed_row = [
+            "test" + r.split(",", 1)[0] + "," + r.split(",", 1)[1] if r else r
+            for r in row
+        ]
+        all_rows.append(prefixed_row)
         if patient.nhs_number in NHS_NUMBER_DUPLICATE_IN_METADATA:
-            all_rows.append(row)
+            all_rows.append(prefixed_row)
 
     flatten_rows = [row for sublist in all_rows for row in sublist]
     return "\n".join([header_row, *flatten_rows])
@@ -387,13 +392,14 @@ def upload_lg_files_to_staging():
     # this one is a bit flaky
     os.chdir("../output")
 
-    files = ["metadata.csv"] + glob("*/*Lloyd_George_Record*.pdf")
+    files = ["metadata.csv"] + glob("**/*Lloyd_George_Record*.pdf", recursive=True)
+    print(files)
     client = boto3.client("s3")
     for file in files:
         client.upload_file(
             Filename=file,
             Bucket=STAGING_BUCKET,
-            Key=file,
+            Key=f"test/{file}",
             ExtraArgs={"StorageClass": "INTELLIGENT_TIERING"},
         )
 
@@ -402,7 +408,7 @@ def upload_lg_files_to_staging():
             scan_result = "Infected"
         client.put_object_tagging(
             Bucket=STAGING_BUCKET,
-            Key=file,
+            Key=f"test/{file}",
             Tagging={
                 "TagSet": [
                     {"Key": "scan-result", "Value": scan_result},