From f96614f06fef5284ef20d18b9e121cc98ea53ce1 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Tue, 17 Dec 2024 13:41:38 +0000 Subject: [PATCH 1/4] [NRL-1231] Add script that can cleanup invalid pointers --- scripts/delete_all_invalid_pointers.py | 120 +++++++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 scripts/delete_all_invalid_pointers.py diff --git a/scripts/delete_all_invalid_pointers.py b/scripts/delete_all_invalid_pointers.py new file mode 100644 index 000000000..d99d1da25 --- /dev/null +++ b/scripts/delete_all_invalid_pointers.py @@ -0,0 +1,120 @@ +from datetime import datetime, timedelta, timezone +from typing import Any + +import boto3 +import fire + +from nrlf.consumer.fhir.r4.model import DocumentReference +from nrlf.core.logger import logger +from nrlf.core.validators import DocumentReferenceValidator + +dynamodb = boto3.client("dynamodb") +paginator = dynamodb.get_paginator("scan") + +logger.setLevel("ERROR") + + +def _validate_document(document: str): + docref = DocumentReference.model_validate_json(document) + + validator = DocumentReferenceValidator() + result = validator.validate(data=docref) + + if not result.is_valid: + raise RuntimeError("Failed to validate document: " + str(result.issues)) + + +def _find_invalid_pointers(table_name: str) -> dict[str, float | int]: + """ + Find and delete pointers in the given table that are invalid based on the FHIR model and NRLF validators. + Parameters: + - table_name: The name of the pointers table to find and delete pointer from. + """ + + print(f"Finding invalid pointers to delete in table {table_name}....") + + params: dict[str, Any] = { + "TableName": table_name, + "PaginationConfig": {"PageSize": 50}, + } + + invalid_pointers = [] + total_scanned_count = 0 + + start_time = datetime.now(tz=timezone.utc) + + for page in paginator.paginate(**params): + for item in page["Items"]: + pointer_id = item.get("id", {}).get("S") + document = item.get("document", {}).get("S", "") + try: + _validate_document(document) + except Exception as exc: + invalid_pointers.append((pointer_id, exc)) + + total_scanned_count += page["ScannedCount"] + + if total_scanned_count % 1000 == 0: + print(".", end="", flush=True) + + if total_scanned_count % 100000 == 0: + print(f"scanned={total_scanned_count} invalid={len(invalid_pointers)}") + + end_time = datetime.now(tz=timezone.utc) + + print(f" Done. Found {len(invalid_pointers)} invalid pointers") + + if len(invalid_pointers) == 0: + return { + "invalid_pointers": 0, + "scanned_count": total_scanned_count, + "took-secs": timedelta.total_seconds(end_time - start_time), + } + + print("Writing invalid pointers IDs to file ./invalid_pointers.txt ...") + with open("invalid_pointers.txt", "w") as f: + for _id, err in invalid_pointers: + f.write(f"{_id}: {err}\n") + + confirmation_input = input( + "Would you like to delete all the invalid pointers? (yes/no): " + ) + if confirmation_input != "yes": + print("Invalid pointers NOT deleted.") + return { + "invalid_pointers": len(invalid_pointers), + "scanned_count": total_scanned_count, + "took-secs": timedelta.total_seconds(end_time - start_time), + } + + print("Deleting invalid pointers...") + pointers_deleted = 0 + for _id, _ in invalid_pointers: + try: + item_key = {"S": f"D#{_id}"} + dynamodb.delete_item( + TableName=table_name, + Key={"pk": item_key, "sk": item_key}, + ReturnValues="NONE", + ) + + pointers_deleted += 1 + + if pointers_deleted % 1000 == 0: + print(".", end="", flush=True) + except Exception as exc: + print(f"Failed to delete pointer {_id}: {exc}") + + end_time = datetime.now(tz=timezone.utc) + + print(" Done") + return { + "invalid_pointers_total": len(invalid_pointers), + "invalid_pointers_deleted": pointers_deleted, + "scanned_count": total_scanned_count, + "took-secs": timedelta.total_seconds(end_time - start_time), + } + + +if __name__ == "__main__": + fire.Fire(_find_invalid_pointers) From 51b8007f16b7c7d56fe1d3d27959e722a80078ad Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Tue, 17 Dec 2024 13:57:28 +0000 Subject: [PATCH 2/4] [NRL-1231] Fixup delete_all_invalid_pointers.py script to satisfy sonarcloud --- scripts/delete_all_invalid_pointers.py | 76 +++++++++++++++++--------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/scripts/delete_all_invalid_pointers.py b/scripts/delete_all_invalid_pointers.py index d99d1da25..78792c389 100644 --- a/scripts/delete_all_invalid_pointers.py +++ b/scripts/delete_all_invalid_pointers.py @@ -24,7 +24,7 @@ def _validate_document(document: str): raise RuntimeError("Failed to validate document: " + str(result.issues)) -def _find_invalid_pointers(table_name: str) -> dict[str, float | int]: +def _find_invalid_pointers(table_name: str) -> dict[str, Any]: """ Find and delete pointers in the given table that are invalid based on the FHIR model and NRLF validators. Parameters: @@ -64,32 +64,28 @@ def _find_invalid_pointers(table_name: str) -> dict[str, float | int]: print(f" Done. Found {len(invalid_pointers)} invalid pointers") - if len(invalid_pointers) == 0: - return { - "invalid_pointers": 0, - "scanned_count": total_scanned_count, - "took-secs": timedelta.total_seconds(end_time - start_time), - } + if len(invalid_pointers) > 0: + print("Writing invalid pointers IDs to file ./invalid_pointers.txt ...") + with open("invalid_pointers.txt", "w") as f: + for _id, err in invalid_pointers: + f.write(f"{_id}: {err}\n") - print("Writing invalid pointers IDs to file ./invalid_pointers.txt ...") - with open("invalid_pointers.txt", "w") as f: - for _id, err in invalid_pointers: - f.write(f"{_id}: {err}\n") + return { + "invalid_pointers": invalid_pointers, + "scanned_count": total_scanned_count, + "find-took-secs": timedelta.total_seconds(end_time - start_time), + } - confirmation_input = input( - "Would you like to delete all the invalid pointers? (yes/no): " - ) - if confirmation_input != "yes": - print("Invalid pointers NOT deleted.") - return { - "invalid_pointers": len(invalid_pointers), - "scanned_count": total_scanned_count, - "took-secs": timedelta.total_seconds(end_time - start_time), - } + +def _delete_pointers(table_name: str, pointers_to_delete: list[str]) -> dict[str, Any]: + """ + Delete the provided pointers from the given table. + """ + start_time = datetime.now(tz=timezone.utc) print("Deleting invalid pointers...") pointers_deleted = 0 - for _id, _ in invalid_pointers: + for _id, _ in pointers_to_delete: try: item_key = {"S": f"D#{_id}"} dynamodb.delete_item( @@ -109,12 +105,38 @@ def _find_invalid_pointers(table_name: str) -> dict[str, float | int]: print(" Done") return { - "invalid_pointers_total": len(invalid_pointers), - "invalid_pointers_deleted": pointers_deleted, - "scanned_count": total_scanned_count, - "took-secs": timedelta.total_seconds(end_time - start_time), + "pointers_to_delete": len(pointers_to_delete), + "deleted_pointers": pointers_deleted, + "deletes-took-secs": timedelta.total_seconds(end_time - start_time), } +def _find_and_delete_invalid_pointers(table_name: str) -> dict[str, float | int]: + find_result = _find_invalid_pointers(table_name) + + if len(find_result["invalid_pointers"]) == 0: + return { + "invalid_pointers": 0, + "scanned_count": find_result["scanned_count"], + "find-took-secs": find_result["find-took-secs"], + } + + confirmation_input = input( + "Would you like to delete all the invalid pointers? (yes/no): " + ) + if confirmation_input != "yes": + print("Invalid pointers NOT deleted.") + find_result.pop("invalid_pointers") + return find_result + + pointers_to_delete = [_id for _id, _ in find_result["invalid_pointers"]] + + delete_result = _delete_pointers(table_name, pointers_to_delete) + + find_result.pop("invalid_pointers") + + return {**find_result, **delete_result} + + if __name__ == "__main__": - fire.Fire(_find_invalid_pointers) + fire.Fire(_find_and_delete_invalid_pointers) From a90db24858ce2c5e5f20fd58854232c85d47235b Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Tue, 17 Dec 2024 15:08:39 +0000 Subject: [PATCH 3/4] [NRL-1231] Update description comment in new script --- scripts/delete_all_invalid_pointers.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/scripts/delete_all_invalid_pointers.py b/scripts/delete_all_invalid_pointers.py index 78792c389..60609c35f 100644 --- a/scripts/delete_all_invalid_pointers.py +++ b/scripts/delete_all_invalid_pointers.py @@ -25,12 +25,6 @@ def _validate_document(document: str): def _find_invalid_pointers(table_name: str) -> dict[str, Any]: - """ - Find and delete pointers in the given table that are invalid based on the FHIR model and NRLF validators. - Parameters: - - table_name: The name of the pointers table to find and delete pointer from. - """ - print(f"Finding invalid pointers to delete in table {table_name}....") params: dict[str, Any] = { @@ -112,6 +106,11 @@ def _delete_pointers(table_name: str, pointers_to_delete: list[str]) -> dict[str def _find_and_delete_invalid_pointers(table_name: str) -> dict[str, float | int]: + """ + Find and delete any pointers in the given table that are invalid based on the FHIR model and NRLF validators. + Parameters: + - table_name: The name of the pointers table to find and delete pointer from. + """ find_result = _find_invalid_pointers(table_name) if len(find_result["invalid_pointers"]) == 0: From 9d61e7a54c7dcf9e25707978bf382619b5b019d6 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 18 Dec 2024 11:59:00 +0000 Subject: [PATCH 4/4] [NRL-1231] Switch delete script to use batch_write_item --- scripts/delete_all_invalid_pointers.py | 39 ++++++++++++++++---------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/scripts/delete_all_invalid_pointers.py b/scripts/delete_all_invalid_pointers.py index 60609c35f..738b2a59b 100644 --- a/scripts/delete_all_invalid_pointers.py +++ b/scripts/delete_all_invalid_pointers.py @@ -10,6 +10,7 @@ dynamodb = boto3.client("dynamodb") paginator = dynamodb.get_paginator("scan") +resource = boto3.resource("dynamodb") logger.setLevel("ERROR") @@ -79,21 +80,28 @@ def _delete_pointers(table_name: str, pointers_to_delete: list[str]) -> dict[str print("Deleting invalid pointers...") pointers_deleted = 0 - for _id, _ in pointers_to_delete: - try: - item_key = {"S": f"D#{_id}"} - dynamodb.delete_item( - TableName=table_name, - Key={"pk": item_key, "sk": item_key}, - ReturnValues="NONE", - ) - - pointers_deleted += 1 - - if pointers_deleted % 1000 == 0: - print(".", end="", flush=True) - except Exception as exc: - print(f"Failed to delete pointer {_id}: {exc}") + failed_to_delete = 0 + + for _batch_id in range(0, len(pointers_to_delete), 25): + batch = [ + { + "DeleteRequest": { + "Key": { + "pk": {"S": f"D#{pointer_id}"}, + "sk": {"S": f"D#{pointer_id}"}, + } + } + } + for pointer_id in pointers_to_delete[_batch_id : _batch_id + 25] + ] + + result = dynamodb.batch_write_item(RequestItems={table_name: batch}) + + unprocessed_items = len(result.get("UnprocessedItems", [])) + pointers_deleted += 25 - unprocessed_items + failed_to_delete += unprocessed_items + if pointers_deleted % 1000 == 0: + print(".", end="", flush=True) end_time = datetime.now(tz=timezone.utc) @@ -101,6 +109,7 @@ def _delete_pointers(table_name: str, pointers_to_delete: list[str]) -> dict[str return { "pointers_to_delete": len(pointers_to_delete), "deleted_pointers": pointers_deleted, + "failed_deletes": failed_to_delete, "deletes-took-secs": timedelta.total_seconds(end_time - start_time), }