diff --git a/README.md b/README.md index d7120009d..6eb9758d1 100644 --- a/README.md +++ b/README.md @@ -511,3 +511,26 @@ Once you have a new release version ready, you can deploy it through our environ 2. If any issues arise in the deployment, fix the issues, create a new release version and start this process again. 3. Once the deployments are complete, use the "Persistent Environment Deploy" Github Action workflow to deploy the release version to `ref`. 4. Once that is complete, use the "Persistent Environment Deploy" workflow to deploy the release version to `prod`. + +## Reports + +Reports are provided as scripts in the `reports/` directory. To run a report: + +1. Login to your AWS account on the command line, choosing the account that contains the resources you want to report on. +2. Run your chosen report script, giving the script the resource names and parameters it requires. See each report script for details. + +For example, to count the number of pointers from X26 in the pointers table in the dev environment: + +``` +$ poetry run python ./scripts/count_pointers_for_custodian.py \ + nhsd-nrlf--dev-pointers-table \ + X26 +``` + +### Running reports in the prod environment + +The reports scripts may require resources that could affect the performance of the live production system. Because of this, it is recommended that you take steps to minimise this impact before running reports. + +If you are running a report against the DynamoDB pointers table in prod, you should create a copy (or restore a PITR backup) of the table and run your report against the copy. + +Please ensure any duplicated resource/data is deleted from the prod environment once you have finished using it. diff --git a/api/producer/createDocumentReference/tests/test_create_document_reference.py b/api/producer/createDocumentReference/tests/test_create_document_reference.py index 57c199893..964bf1afc 100644 --- a/api/producer/createDocumentReference/tests/test_create_document_reference.py +++ b/api/producer/createDocumentReference/tests/test_create_document_reference.py @@ -238,6 +238,21 @@ def test_create_document_reference_invalid_body(): "diagnostics": "Request body could not be parsed (status: Field required)", "expression": ["status"], }, + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + } + ], + }, + "diagnostics": "Request body could not be parsed (author: Field required)", + "expression": ["author"], + }, { "severity": "error", "code": "invalid", diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index 5cadfa08b..bea387d33 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1383,6 +1383,7 @@ components: - resourceType - status - content + - author Bundle: type: object properties: diff --git a/api/producer/updateDocumentReference/tests/test_update_document_reference.py b/api/producer/updateDocumentReference/tests/test_update_document_reference.py index d5071b354..ff91d147e 100644 --- a/api/producer/updateDocumentReference/tests/test_update_document_reference.py +++ b/api/producer/updateDocumentReference/tests/test_update_document_reference.py @@ -246,6 +246,21 @@ def test_create_document_reference_invalid_body(): "diagnostics": "Request body could not be parsed (status: Field required)", "expression": ["status"], }, + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + } + ], + }, + "diagnostics": "Request body could not be parsed (author: Field required)", + "expression": ["author"], + }, { "severity": "error", "code": "invalid", diff --git a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py index b6cd09166..142cbb5fb 100644 --- a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py +++ b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py @@ -323,6 +323,21 @@ def test_upsert_document_reference_invalid_body(): "diagnostics": "Request body could not be parsed (status: Field required)", "expression": ["status"], }, + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + } + ], + }, + "diagnostics": "Request body could not be parsed (author: Field required)", + "expression": ["author"], + }, { "severity": "error", "code": "invalid", diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index da76fe858..4af54348d 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-10-28T10:30:48+00:00 +# timestamp: 2024-11-04T11:43:16+00:00 from __future__ import annotations diff --git a/layer/nrlf/core/constants.py b/layer/nrlf/core/constants.py index a572132de..650547fb9 100644 --- a/layer/nrlf/core/constants.py +++ b/layer/nrlf/core/constants.py @@ -8,7 +8,15 @@ class Source(Enum): VALID_SOURCES = frozenset(item.value for item in Source.__members__.values()) EMPTY_VALUES = ("", None, [], {}) -REQUIRED_CREATE_FIELDS = ["custodian", "id", "type", "status", "subject", "category"] +REQUIRED_CREATE_FIELDS = [ + "custodian", + "id", + "type", + "status", + "subject", + "category", + "author", +] JSON_TYPES = {dict, list} NHS_NUMBER_INDEX = "idx_nhs_number_by_id" ID_SEPARATOR = "-" diff --git a/layer/nrlf/core/errors.py b/layer/nrlf/core/errors.py index 42ebcfc9c..320d2d470 100644 --- a/layer/nrlf/core/errors.py +++ b/layer/nrlf/core/errors.py @@ -1,6 +1,7 @@ from typing import List, Optional from pydantic import ValidationError +from pydantic_core import ErrorDetails from nrlf.core.response import Response from nrlf.core.types import CodeableConcept @@ -8,6 +9,17 @@ from nrlf.producer.fhir.r4.model import OperationOutcome, OperationOutcomeIssue +def diag_for_error(error: ErrorDetails) -> str: + if error["loc"]: + return f"{error['loc'][0]}: {error['msg']}" + else: + return f"root: {error['msg']}" + + +def expression_for_error(error: ErrorDetails) -> Optional[str]: + return str(error["loc"][0] if error["loc"] else "root") + + class OperationOutcomeError(Exception): """ Will instantly trigger an OperationOutcome error response when raised @@ -59,8 +71,8 @@ def from_validation_error( severity="error", code="invalid", details=details, # type: ignore - diagnostics=f"{msg} ({error['loc'][0]}: {error['msg']})", - expression=[str(error["loc"][0])], # type: ignore + diagnostics=f"{msg} ({diag_for_error(error)})", + expression=[expression_for_error(error)], # type: ignore ) for error in exc.errors() ] diff --git a/layer/nrlf/core/tests/test_request.py b/layer/nrlf/core/tests/test_request.py index 5055b3a4c..8b5f537a6 100644 --- a/layer/nrlf/core/tests/test_request.py +++ b/layer/nrlf/core/tests/test_request.py @@ -2,8 +2,10 @@ import pytest -from nrlf.core.errors import OperationOutcomeError -from nrlf.core.request import parse_headers +from nrlf.core.errors import OperationOutcomeError, ParseError +from nrlf.core.request import parse_body, parse_headers +from nrlf.producer.fhir.r4.model import DocumentReference +from nrlf.tests.data import load_document_reference_data def test_parse_headers_empty_headers(): @@ -129,3 +131,212 @@ def test_parse_headers_case_insensitive(): assert metadata.client_rp_details.developer_app_name == "TestApp" assert metadata.client_rp_details.developer_app_id == "12345" assert metadata.ods_code_parts == ("X26", "001") + + +def test_parse_body_no_model_no_body(): + body = None + model = None + + result = parse_body(model, body) + + assert result is None + + +def test_parse_body_valid_docref(): + model = DocumentReference + docref_body = load_document_reference_data("Y05868-736253002-Valid") + + result = parse_body(model, docref_body) + + assert isinstance(result, DocumentReference) + + +def test_parse_body_no_body(): + model = DocumentReference + body = None + + with pytest.raises(OperationOutcomeError) as error: + parse_body(model, body) + + exc = error.value + + assert exc.status_code == "400" + assert exc.operation_outcome.model_dump(exclude_none=True) == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "BAD_REQUEST", + "display": "Bad request", + } + ], + }, + "diagnostics": "Request body is required", + } + ], + } + + +def test_parse_body_invalid_docref_json(): + model = DocumentReference + docref_body = load_document_reference_data("Y05868-736253002-Valid") + + docref_body = docref_body.replace('unstructured"', "unstructured") + + with pytest.raises(ParseError) as error: + parse_body(model, docref_body) + + response = error.value.response.model_dump() + + assert response["statusCode"] == "400" + assert json.loads(response["body"]) == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + } + ], + }, + "diagnostics": "Request body could not be parsed (root: Invalid JSON: control character (\\u0000-\\u001F) found while parsing a string at line 72 column 0)", + "expression": ["root"], + } + ], + } + + +def test_parse_body_invalid_json(): + model = DocumentReference + body = '{ "type": "is-not-a-docref" }' + + with pytest.raises(ParseError) as error: + parse_body(model, body) + + response = error.value.response.model_dump() + + assert response["statusCode"] == "400" + assert json.loads(response["body"]) == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + } + ] + }, + "diagnostics": "Request body could not be parsed (resourceType: Field required)", + "expression": ["resourceType"], + }, + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + } + ] + }, + "diagnostics": "Request body could not be parsed (status: Field required)", + "expression": ["status"], + }, + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + } + ] + }, + "diagnostics": "Request body could not be parsed (type: Input should be an object)", + "expression": ["type"], + }, + { + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + }, + ], + }, + "diagnostics": "Request body could not be parsed (author: Field required)", + "expression": [ + "author", + ], + "severity": "error", + }, + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + } + ] + }, + "diagnostics": "Request body could not be parsed (content: Field required)", + "expression": ["content"], + }, + ], + } + + +def test_parse_body_not_json(): + model = DocumentReference + body = "is not json" + + with pytest.raises(ParseError) as error: + parse_body(model, body) + + response = error.value.response + + assert response.statusCode == "400" + assert json.loads(response.body) == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + } + ] + }, + "diagnostics": "Request body could not be parsed (root: Invalid JSON: expected value at line 1 column 1)", + "expression": ["root"], + } + ], + } diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index 7c452e15c..52765b8f1 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -2,7 +2,7 @@ import pytest -from nrlf.core.constants import PointerTypes +from nrlf.core.constants import ODS_SYSTEM, PointerTypes from nrlf.core.errors import ParseError from nrlf.core.validators import ( DocumentReferenceValidator, @@ -640,6 +640,140 @@ def test_validate_content_extension_too_many_extensions(): } +def test_validate_author_too_many_authors(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["author"].append( + { + "identifier": { + "system": ODS_SYSTEM, + "value": "someODSCode", + } + } + ) + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid author length: 2 Author must only contain a single value", + "expression": ["author"], + } + + +def test_validate_author_system_invalid(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["author"][0] = { + "identifier": { + "system": "some system", + "value": "someODSCode", + } + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_IDENTIFIER_SYSTEM", + "display": "Invalid identifier system", + } + ] + }, + "diagnostics": f"Invalid author system: 'some system' Author system must be 'https://fhir.nhs.uk/Id/ods-organization-code'", + "expression": ["author[0].identifier.system"], + } + + +def test_validate_author_value_invalid(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["author"][0] = { + "identifier": { + "system": ODS_SYSTEM, + "value": "!!!!!!12sd", + } + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": f"Invalid author value: '!!!!!!12sd' Author value must be alphanumeric", + "expression": ["author[0].identifier.value"], + } + + +def test_validate_author_value_too_long(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["author"][0] = { + "identifier": { + "system": ODS_SYSTEM, + "value": "d1111111111111111111111111111111111111111111111", + } + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": f"Invalid author value: 'd1111111111111111111111111111111111111111111111' Author value must be less than 13 characters", + "expression": ["author[0].identifier.value"], + } + + def test_validate_content_extension_invalid_code(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 061b11134..0974e403e 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -5,7 +5,7 @@ from pydantic import ValidationError from nrlf.core.codes import SpineErrorConcept -from nrlf.core.constants import CATEGORY_ATTRIBUTES, REQUIRED_CREATE_FIELDS +from nrlf.core.constants import CATEGORY_ATTRIBUTES, ODS_SYSTEM, REQUIRED_CREATE_FIELDS from nrlf.core.errors import ParseError from nrlf.core.logger import LogReference, logger from nrlf.core.types import DocumentReference, OperationOutcomeIssue, RequestQueryType @@ -118,6 +118,7 @@ def validate(self, data: Dict[str, Any] | DocumentReference): self._validate_relates_to(resource) self._validate_ssp_asid(resource) self._validate_category(resource) + self._validate_author(resource) if resource.content[0].extension: self._validate_content_extension(resource) @@ -464,3 +465,48 @@ def _validate_content_extension(self, model: DocumentReference): field=f"content[{i}].extension[0].url", ) return + + def _validate_author(self, model: DocumentReference): + """ + Validate the author field contains an appropriate coding system and code. + """ + logger.log(LogReference.VALIDATOR001, step="author") + + if len(model.author) > 1: + self.result.add_error( + issue_code="invalid", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid author length: {len(model.author)} Author must only contain a single value", + field=f"author", + ) + return + + logger.debug("Validating author") + identifier = model.author[0].identifier + + if identifier.system != ODS_SYSTEM: + self.result.add_error( + issue_code="invalid", + error_code="INVALID_IDENTIFIER_SYSTEM", + diagnostics=f"Invalid author system: '{identifier.system}' Author system must be '{ODS_SYSTEM}'", + field=f"author[0].identifier.system", + ) + return + + if not identifier.value.isalnum(): + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid author value: '{identifier.value}' Author value must be alphanumeric", + field=f"author[0].identifier.value", + ) + return + + if len(identifier.value) > 12: + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid author value: '{identifier.value}' Author value must be less than 13 characters", + field=f"author[0].identifier.value", + ) + return diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index 697b8afdb..ecb14c05c 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-10-28T10:30:43+00:00 +# timestamp: 2024-11-04T11:43:12+00:00 from __future__ import annotations @@ -582,7 +582,7 @@ class DocumentReference(BaseModel): pattern="([0-9]([0-9]([0-9][1-9]|[1-9]0)|[1-9]00)|[1-9]000)-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])T([01][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)(\\.[0-9]+)?(Z|(\\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00))", ), ] = None - author: Optional[List[Reference]] = None + author: List[Reference] authenticator: Annotated[ Optional[Reference], Field( diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index 78545b2bf..91bdcfe36 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-10-28T10:30:45+00:00 +# timestamp: 2024-11-04T11:43:14+00:00 from __future__ import annotations @@ -505,7 +505,7 @@ class DocumentReference(BaseModel): Optional[StrictStr], Field(description="When the document reference was created."), ] = None - author: Optional[List[Reference]] = None + author: List[Reference] authenticator: Annotated[ Optional[Reference], Field( diff --git a/poetry.lock b/poetry.lock index 9672f16c4..b41e5d956 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2479,19 +2479,23 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] [[package]] name = "setuptools" -version = "69.2.0" +version = "75.4.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, - {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, + {file = "setuptools-75.4.0-py3-none-any.whl", hash = "sha256:b3c5d862f98500b06ffdf7cc4499b48c46c317d8d56cb30b5c8bce4d88f5c216"}, + {file = "setuptools-75.4.0.tar.gz", hash = "sha256:1dc484f5cf56fd3fe7216d7b8df820802e7246cfb534a1db2aa64f14fcb9cdcb"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.7.0)"] +core = ["importlib-metadata (>=6)", "jaraco.collections", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more-itertools", "more-itertools (>=8.8)", "packaging", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] +enabler = ["pytest-enabler (>=2.2)"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib-metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (>=1.12,<1.14)", "pytest-mypy"] [[package]] name = "sh" @@ -2601,13 +2605,13 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [[package]] name = "werkzeug" -version = "3.0.1" +version = "3.1.3" description = "The comprehensive WSGI web application library." optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" files = [ - {file = "werkzeug-3.0.1-py3-none-any.whl", hash = "sha256:90a285dc0e42ad56b34e696398b8122ee4c681833fb35b8334a095d82c56da10"}, - {file = "werkzeug-3.0.1.tar.gz", hash = "sha256:507e811ecea72b18a404947aded4b3390e1db8f826b494d76550ef45bb3b1dcc"}, + {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, + {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, ] [package.dependencies] diff --git a/reports/count_pointers_for_custodian.py b/reports/count_pointers_for_custodian.py new file mode 100644 index 000000000..835719526 --- /dev/null +++ b/reports/count_pointers_for_custodian.py @@ -0,0 +1,59 @@ +from datetime import datetime, timedelta, timezone +from typing import Any, Optional + +import boto3 +import fire + +dynamodb = boto3.client("dynamodb") +paginator = dynamodb.get_paginator("scan") + + +def _count_pointers( + table_name: str, ods_code: str, created_date: Optional[str] = None +) -> dict[str, float]: + """ + Count the number of pointers for a given custodian (ODS code) in the pointers table. + Parameters: + - table_name: The name of the pointers table to use. + - ods_code: The custodian ODS code to find pointers for. + - created_date: The created date to filter pointers on. (optional) + """ + + print(f"Counting pointers for {ods_code} in table {table_name}....") # noqa + + params: dict[str, Any] = { + "TableName": table_name, + "FilterExpression": "custodian = :ods_code", + "ExpressionAttributeValues": {":ods_code": {"S": ods_code}}, + "Select": "COUNT", + "PaginationConfig": {"PageSize": 100}, + } + + if created_date: + params["FilterExpression"] += " AND starts_with(created_on, :created_date)" + params["ExpressionAttributeValues"][":created_date"] = {"S": created_date} + + custodian_pointers_count = 0 + total_scanned_count = 0 + + start_time = datetime.now(tz=timezone.utc) + + for page in paginator.paginate(**params): + custodian_pointers_count += page["Count"] + total_scanned_count += page["ScannedCount"] + + if total_scanned_count % 1000 == 0: + print(".", end="", flush=True) # noqa + + end_time = datetime.now(tz=timezone.utc) + + print(" Done") # noqa + return { + "items_found": custodian_pointers_count, + "scanned_count": total_scanned_count, + "took-secs": timedelta.total_seconds(end_time - start_time), + } + + +if __name__ == "__main__": + fire.Fire(_count_pointers) diff --git a/sonar-project.properties b/sonar-project.properties index 0d308a912..20292ee79 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -3,5 +3,11 @@ sonar.organization=nhsdigital sonar.projectName=NRLF sonar.python.version=3.9.5 sonar.terraform.provider.aws.version=4.63.0 +# TODO: Some paths here are outdated and perhaps we don't want to exclude everything sonar.cpd.exclusions=api/tests/**, tests/**, api/**/tests/**, feature_tests/**, cron/seed_sandbox/tests/**, data_contracts/**/tests/**, firehose/**/tests/**, firehose/**/scripts/**, helpers/tests/**, mi/**/tests/** sonar.exclusions=scripts/**, **/scripts/**, api/tests/**, tests/**, api/**/tests/**, feature_tests/**, cron/seed_sandbox/tests/**, data_contracts/**/tests/**, firehose/**/tests/**, firehose/**/scripts/**, helpers/tests/**, mi/**/tests/** + +# Exclude snomed urls as being unsafe +sonar.issue.ignore.multicriteria=exclude_snomed_urls +sonar.issue.ignore.multicriteria.exclude_snomed_urls.ruleKey=python:S5332 +sonar.issue.ignore.multicriteria.exclude_snomed_urls.pattern=http://snomed\.info(/sct)?$ diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index fbeb83b1f..38768e81a 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -120,6 +120,80 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ + Scenario: Invalid Author + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'author' is: + """ + "author":[{ + "identifier": { + "system": "https://fhir.nhs.uk/Id/ods-organization-code", + "value": "!!!!!" + } + }] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid author value: '!!!!!' Author value must be alphanumeric", + "expression": [ + "author[0].identifier.value" + ] + } + """ + + Scenario: Invalid Author System + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'author' is: + """ + "author":[{ + "identifier": { + "system": "ddddd", + "value": "123" + } + }] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_IDENTIFIER_SYSTEM", + "display": "Invalid identifier system" + } + ] + }, + "diagnostics": "Invalid author system: 'ddddd' Author system must be 'https://fhir.nhs.uk/Id/ods-organization-code'", + "expression": [ + "author[0].identifier.system" + ] + } + """ + # Invalid document reference - invalid custodian ID # Invalid document reference - invalid relatesTo target # Invalid document reference - invalid producer ID in relatesTo target