From 97be4870cf7abc15861ca14cdd44b79dab04067b Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:10:30 +0000 Subject: [PATCH 01/11] NRL-1137 implement daily builds workflow --- .github/workflows/daily-build.yml | 85 +++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/daily-build.yml diff --git a/.github/workflows/daily-build.yml b/.github/workflows/daily-build.yml new file mode 100644 index 000000000..0fc15523c --- /dev/null +++ b/.github/workflows/daily-build.yml @@ -0,0 +1,85 @@ +name: Build NRL Project on Environment +run-name: Build NRL Project on ${{ inputs.environment || 'dev' }} +permissions: + id-token: write + contents: read + actions: write + +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + inputs: + environment: + type: environment + description: "The environment to deploy changes to" + default: "dev" + required: true + +jobs: + build: + name: Build - develop + runs-on: [self-hosted, ci] + + steps: + - name: Git clone - develop + uses: actions/checkout@v4 + with: + ref: develop + + - name: Setup asdf cache + uses: actions/cache@v4 + with: + path: ~/.asdf + key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} + restore-keys: | + ${{ runner.os }}-asdf- + + - name: Install asdf + uses: asdf-vm/actions/install@v3.0.2 + with: + asdf_branch: v0.13.1 + + - name: Install zip + run: sudo apt-get install zip + + - name: Setup Python environment + run: | + poetry install --no-root + source $(poetry env info --path)/bin/activate + + - name: Run Linting + run: make lint + + - name: Run Unit Tests + run: make test + + - name: Build Project + run: make build + + - name: Configure Management Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-2 + role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} + role-session-name: github-actions-ci-${{ inputs.environment || 'dev' }}-${{ github.run_id }} + + - name: Add S3 Permissions to Lambda + run: | + account=$(echo '${{ inputs.environment || 'dev' }}' | cut -d '-' -f1) + inactive_stack=$(poetry run python ./scripts/get_env_config.py inactive-stack ${{ inputs.environment || 'dev' }}) + make get-s3-perms ENV=${account} TF_WORKSPACE_NAME=${inactive_stack} + + - name: Save Build Artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts + path: | + dist/*.zip + !dist/nrlf_permissions.zip + + - name: Save NRLF Permissions cache + uses: actions/cache/save@v4 + with: + key: ${{ github.run_id }}-nrlf-permissions + path: dist/nrlf_permissions.zip From 4d1220a277d0c18bbe966f39754e11e693e5a64e Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:11:18 +0000 Subject: [PATCH 02/11] NRL-477 validate docStatus --- api/consumer/swagger.yaml | 2 +- .../tests/test_create_document_reference.py | 41 +++++++++++++++++++ api/producer/swagger.yaml | 2 +- .../tests/test_upsert_document_reference.py | 41 +++++++++++++++++++ layer/nrlf/consumer/fhir/r4/model.py | 4 +- layer/nrlf/producer/fhir/r4/model.py | 4 +- layer/nrlf/producer/fhir/r4/strict_model.py | 2 +- 7 files changed, 89 insertions(+), 7 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 47c1a072c..3f8141469 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -788,7 +788,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "[^\\s]+(\\s[^\\s]+)*" + pattern: "^(entered-in-error|amended|preliminary|final)$" description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" diff --git a/api/producer/createDocumentReference/tests/test_create_document_reference.py b/api/producer/createDocumentReference/tests/test_create_document_reference.py index 19e29569b..b58d3c234 100644 --- a/api/producer/createDocumentReference/tests/test_create_document_reference.py +++ b/api/producer/createDocumentReference/tests/test_create_document_reference.py @@ -411,6 +411,47 @@ def test_create_document_reference_with_no_practiceSetting(): } +def test_create_document_reference_with_invalid_docStatus(): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_ref.docStatus = "invalid" + + event = create_test_api_gateway_event( + headers=create_headers(), + body=doc_ref.model_dump_json(exclude_none=True), + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "400", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + } + ], + }, + "diagnostics": "Request body could not be parsed (docStatus: String should match pattern '^(entered-in-error|amended|preliminary|final)$')", + "expression": ["docStatus"], + }, + ], + } + + def test_create_document_reference_invalid_custodian_id(): doc_ref = load_document_reference("Y05868-736253002-Valid") diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index bdd0127b8..fa10b2790 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1351,7 +1351,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "[^\\s]+(\\s[^\\s]+)*" + pattern: "^(entered-in-error|amended|preliminary|final)$" description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" diff --git a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py index 6f3878cfd..009c12075 100644 --- a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py +++ b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py @@ -455,6 +455,47 @@ def test_upsert_document_reference_with_no_practiceSetting(): } +def test_upsert_document_reference_with_invalid_docStatus(): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_ref.docStatus = "invalid" + + event = create_test_api_gateway_event( + headers=create_headers(), + body=doc_ref.model_dump_json(exclude_none=True), + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "400", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + } + ], + }, + "diagnostics": "Request body could not be parsed (docStatus: String should match pattern '^(entered-in-error|amended|preliminary|final)$')", + "expression": ["docStatus"], + }, + ], + } + + def test_upsert_document_reference_invalid_producer_id(): doc_ref = load_document_reference("Y05868-736253002-Valid") doc_ref.id = "X26-99999-99999-999999" diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index 4665533b5..70167bf4a 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T09:43:58+00:00 +# timestamp: 2024-12-05T01:41:57+00:00 from __future__ import annotations @@ -580,7 +580,7 @@ class DocumentReference(BaseModel): Optional[str], Field( description="The status of the underlying document.", - pattern="[^\\s]+(\\s[^\\s]+)*", + pattern="^(entered-in-error|amended|preliminary|final)$", ), ] = None type: Annotated[ diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index d96b7ce73..6a7e49abb 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T10:10:52+00:00 +# timestamp: 2024-12-05T01:41:53+00:00 from __future__ import annotations @@ -564,7 +564,7 @@ class DocumentReference(BaseModel): Optional[str], Field( description="The status of the underlying document.", - pattern="[^\\s]+(\\s[^\\s]+)*", + pattern="^(entered-in-error|amended|preliminary|final)$", ), ] = None type: Annotated[ diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index e4edefc58..a7d73861a 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T10:10:54+00:00 +# timestamp: 2024-12-05T01:41:55+00:00 from __future__ import annotations From da4802ccc3ad49c0a53c262627eb8ebbecc55fc1 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:12:08 +0000 Subject: [PATCH 03/11] NRL-502 validate format code for attachment types --- layer/nrlf/core/tests/test_validators.py | 60 +++++++++++++++++++ layer/nrlf/core/validators.py | 30 ++++++++++ .../createDocumentReference-failure.feature | 46 ++++++++++++++ 3 files changed, 136 insertions(+) diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index fd02deb20..f243c1843 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1522,3 +1522,63 @@ def test_validate_ssp_content_with_multiple_asids(): "diagnostics": "Multiple ASID identifiers provided. Only a single valid ASID identifier can be provided in the context.related.", "expression": ["context.related"], } + + +def test_validate_content_format_invalid_code_for_unstructured_document(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["format"] = { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:contact", + "display": "Contact details", + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + "expression": ["content[0].format.code"], + } + + +def test_validate_content_format_invalid_code_for_contact_details(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"]["contentType"] = "text/html" + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", + "expression": ["content[0].format.code"], + } diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index bdb01b78b..907e1f1b3 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -141,6 +141,7 @@ def validate(self, data: Dict[str, Any] | DocumentReference): self._validate_category(resource) self._validate_author(resource) self._validate_type_category_mapping(resource) + self._validate_content_format(resource) if resource.content[0].extension: self._validate_content_extension(resource) @@ -481,6 +482,35 @@ def _validate_type_category_mapping(self, model: DocumentReference): field="category.coding[0].code", ) + def _validate_content_format(self, model: DocumentReference): + """ + Validate the content.format field contains an appropriate coding. + """ + logger.log(LogReference.VALIDATOR001, step="content_format") + + logger.debug("Validating format") + for i, content in enumerate(model.content): + if ( + content.attachment.contentType == "text/html" + and content.format.code != "urn:nhs-ic:record-contact" + ): + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid content format code: {content.format.code} format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", + field=f"content[{i}].format.code", + ) + elif ( + content.attachment.contentType == "application/pdf" + and content.format.code != "urn:nhs-ic:unstructured" + ): + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid content format code: {content.format.code} format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + field=f"content[{i}].format.code", + ) + def _validate_content_extension(self, model: DocumentReference): """ Validate the content.extension field contains an appropriate coding. diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index c1d09f483..1ce52efac 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -407,6 +407,52 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ + Scenario: Invalid format code for attachment type + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'X26' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "application/pdf", + "language": "en-US", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", + "size": 3654, + "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", + "title": "Mental health crisis plan report", + "creation": "2022-12-21T10:45:41+11:00" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details" + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", + "expression": ["content[0].format.code"] + } + """ + # Invalid document reference - invalid Type # NRL-769 Known issue: Type display is not validated # Scenario: Invalid type (valid code but wrong display value) From 772a93b3deaa56af2859091656f39ca7a34e8fbe Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:17:45 +0000 Subject: [PATCH 04/11] NRL-502 add integration test --- .../createDocumentReference-failure.feature | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 1ce52efac..427ce3488 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -407,7 +407,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - Scenario: Invalid format code for attachment type + Scenario: Invalid format code for attachment type contact details Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'X26' is authorised to access pointer types: | system | value | @@ -418,7 +418,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", + "contentType": "text/html", "language": "en-US", "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", "size": 3654, @@ -428,8 +428,8 @@ Feature: Producer - createDocumentReference - Failure Scenarios }, "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:record-contact", - "display": "Contact details" + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Documents" } """ Then the response status code is 400 @@ -453,6 +453,51 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ + Scenario: Invalid format code for attachment type pdf + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'X26' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "application/pdf", + "language": "en-US", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", + "size": 3654, + "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", + "title": "Mental health crisis plan report", + "creation": "2022-12-21T10:45:41+11:00" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details" + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + "expression": ["content[0].format.code"], + } + """ + # Invalid document reference - invalid Type # NRL-769 Known issue: Type display is not validated # Scenario: Invalid type (valid code but wrong display value) From a8c035268ef2ee79b82d54696d5bbe4105e64d40 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:35:04 +0000 Subject: [PATCH 05/11] NRL-502 add format to setup --- .../producer/createDocumentReference-failure.feature | 4 ++-- tests/features/utils/data.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 427ce3488..20a0f83e9 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -409,7 +409,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios Scenario: Invalid format code for attachment type contact details Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - And the organisation 'X26' is authorised to access pointer types: + And the organisation 'TSTCUS' is authorised to access pointer types: | system | value | | http://snomed.info/sct | 1363501000000100 | | http://snomed.info/sct | 736253002 | @@ -455,7 +455,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios Scenario: Invalid format code for attachment type pdf Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - And the organisation 'X26' is authorised to access pointer types: + And the organisation 'TSTCUS' is authorised to access pointer types: | system | value | | http://snomed.info/sct | 1363501000000100 | | http://snomed.info/sct | 736253002 | diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 11288945a..e47c2e0a8 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -33,7 +33,12 @@ def create_test_document_reference(items: dict) -> DocumentReference: attachment=Attachment( contentType=items.get("contentType", "application/json"), url=items["url"], - ) + ), + format=Coding( + system="https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + code="urn:nhs-ic:unstructured", + display="Unstructured document", + ), ) ], context=DocumentReferenceContext( From 37aed66ebdfe6451f20a63194674aef0c8ba7d53 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:48:06 +0000 Subject: [PATCH 06/11] NRL-502 fix tests --- .../readDocumentReference-success.feature | 10 +++++ .../createDocumentReference-failure.feature | 41 +++++++++++-------- .../readDocumentReference-success.feature | 5 +++ 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/tests/features/consumer/readDocumentReference-success.feature b/tests/features/consumer/readDocumentReference-success.feature index b1e1add80..5c0d83979 100644 --- a/tests/features/consumer/readDocumentReference-success.feature +++ b/tests/features/consumer/readDocumentReference-success.feature @@ -69,6 +69,11 @@ Feature: Consumer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" } } ], @@ -155,6 +160,11 @@ Feature: Consumer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" } } ], diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 20a0f83e9..bb1601f47 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -440,16 +440,18 @@ Feature: Producer - createDocumentReference - Failure Scenarios "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", - "expression": ["content[0].format.code"] + "diagnostics": "The Category code of the provided document 'http://snomed.info/sct|1102421000000108' must match the allowed category for pointer type 'http://snomed.info/sct|736253002' with a category value of 'http://snomed.info/sct|734163000'", + "expression": [ + "category.coding[0].code" + ] } """ @@ -482,19 +484,22 @@ Feature: Producer - createDocumentReference - Failure Scenarios And the response is an OperationOutcome with 1 issue And the OperationOutcome contains the issue: """ + { "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", - "expression": ["content[0].format.code"], + "diagnostics": "The Category code of the provided document 'http://snomed.info/sct|1102421000000108' must match the allowed category for pointer type 'http://snomed.info/sct|736253002' with a category value of 'http://snomed.info/sct|734163000'", + "expression": [ + "category.coding[0].code" + ] } """ diff --git a/tests/features/producer/readDocumentReference-success.feature b/tests/features/producer/readDocumentReference-success.feature index bf0e340b1..04c4228ec 100644 --- a/tests/features/producer/readDocumentReference-success.feature +++ b/tests/features/producer/readDocumentReference-success.feature @@ -71,6 +71,11 @@ Feature: Producer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" } } ], From 31d2dc93a666e77a13d5a62f8e2d3f0776504636 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 03:01:06 +0000 Subject: [PATCH 07/11] NRL-502 fix new tests --- .../createDocumentReference-failure.feature | 74 +++++++++---------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index bb1601f47..b30ff97bc 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -418,19 +418,16 @@ Feature: Producer - createDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "text/html", - "language": "en-US", - "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", - "size": 3654, - "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", - "title": "Mental health crisis plan report", - "creation": "2022-12-21T10:45:41+11:00" + "contentType": "text/html", + "url": "someContact.co.uk" }, "format": { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:unstructured", - "display": "Unstructured Documents" + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" + } } + ] """ Then the response status code is 400 And the response is an OperationOutcome with 1 issue @@ -440,17 +437,17 @@ Feature: Producer - createDocumentReference - Failure Scenarios "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource" - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "The Category code of the provided document 'http://snomed.info/sct|1102421000000108' must match the allowed category for pointer type 'http://snomed.info/sct|736253002' with a category value of 'http://snomed.info/sct|734163000'", + "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", "expression": [ - "category.coding[0].code" + "content[0].format.code" ] } """ @@ -466,19 +463,20 @@ Feature: Producer - createDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", - "language": "en-US", - "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", - "size": 3654, - "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", - "title": "Mental health crisis plan report", - "creation": "2022-12-21T10:45:41+11:00" + "contentType": "application/pdf", + "language": "en-UK", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", + "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", + "title": "Mental health crisis plan report", + "creation": "2022-12-21T10:45:41+11:00" }, "format": { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:record-contact", - "display": "Contact details" + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details" + } } + ] """ Then the response status code is 400 And the response is an OperationOutcome with 1 issue @@ -488,17 +486,17 @@ Feature: Producer - createDocumentReference - Failure Scenarios "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource" - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "The Category code of the provided document 'http://snomed.info/sct|1102421000000108' must match the allowed category for pointer type 'http://snomed.info/sct|736253002' with a category value of 'http://snomed.info/sct|734163000'", + "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", "expression": [ - "category.coding[0].code" + "content[0].format.code" ] } """ From cef46169005774abd8b47c32a4c14c540f103613 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Wed, 11 Dec 2024 14:19:55 +0000 Subject: [PATCH 08/11] NRL-477 update model and error messages --- api/consumer/swagger.yaml | 2 +- .../tests/test_create_document_reference.py | 2 +- api/producer/swagger.yaml | 2 +- .../tests/test_upsert_document_reference.py | 2 +- layer/nrlf/consumer/fhir/r4/model.py | 9 ++-- layer/nrlf/producer/fhir/r4/model.py | 9 ++-- layer/nrlf/producer/fhir/r4/strict_model.py | 5 ++- .../createDocumentReference-failure.feature | 42 ------------------- 8 files changed, 13 insertions(+), 60 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 3f8141469..f62fc6a90 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -788,7 +788,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "^(entered-in-error|amended|preliminary|final)$" + enum: ["entered-in-error", "amended", "preliminary", "final"] description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" diff --git a/api/producer/createDocumentReference/tests/test_create_document_reference.py b/api/producer/createDocumentReference/tests/test_create_document_reference.py index b58d3c234..56daacfc6 100644 --- a/api/producer/createDocumentReference/tests/test_create_document_reference.py +++ b/api/producer/createDocumentReference/tests/test_create_document_reference.py @@ -445,7 +445,7 @@ def test_create_document_reference_with_invalid_docStatus(): } ], }, - "diagnostics": "Request body could not be parsed (docStatus: String should match pattern '^(entered-in-error|amended|preliminary|final)$')", + "diagnostics": "Request body could not be parsed (docStatus: Input should be 'entered-in-error', 'amended', 'preliminary' or 'final')", "expression": ["docStatus"], }, ], diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index fa10b2790..25fe84fc4 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1351,7 +1351,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "^(entered-in-error|amended|preliminary|final)$" + enum: ["entered-in-error", "amended", "preliminary", "final"] description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" diff --git a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py index 009c12075..090542dae 100644 --- a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py +++ b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py @@ -489,7 +489,7 @@ def test_upsert_document_reference_with_invalid_docStatus(): } ], }, - "diagnostics": "Request body could not be parsed (docStatus: String should match pattern '^(entered-in-error|amended|preliminary|final)$')", + "diagnostics": "Request body could not be parsed (docStatus: Input should be 'entered-in-error', 'amended', 'preliminary' or 'final')", "expression": ["docStatus"], }, ], diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index 70167bf4a..a679c5084 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-05T01:41:57+00:00 +# timestamp: 2024-12-11T14:01:43+00:00 from __future__ import annotations @@ -577,11 +577,8 @@ class DocumentReference(BaseModel): ), ] docStatus: Annotated[ - Optional[str], - Field( - description="The status of the underlying document.", - pattern="^(entered-in-error|amended|preliminary|final)$", - ), + Optional[Literal["entered-in-error", "amended", "preliminary", "final"]], + Field(description="The status of the underlying document."), ] = None type: Annotated[ Optional[CodeableConcept], diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index 6a7e49abb..99f0d7d6d 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-05T01:41:53+00:00 +# timestamp: 2024-12-11T14:01:38+00:00 from __future__ import annotations @@ -561,11 +561,8 @@ class DocumentReference(BaseModel): ), ] docStatus: Annotated[ - Optional[str], - Field( - description="The status of the underlying document.", - pattern="^(entered-in-error|amended|preliminary|final)$", - ), + Optional[Literal["entered-in-error", "amended", "preliminary", "final"]], + Field(description="The status of the underlying document."), ] = None type: Annotated[ Optional[CodeableConcept], diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index a7d73861a..bd78aab06 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-05T01:41:55+00:00 +# timestamp: 2024-12-11T14:01:41+00:00 from __future__ import annotations @@ -491,7 +491,8 @@ class DocumentReference(BaseModel): StrictStr, Field(description="The status of this document reference.") ] docStatus: Annotated[ - Optional[StrictStr], Field(description="The status of the underlying document.") + Optional[Literal["entered-in-error", "amended", "preliminary", "final"]], + Field(description="The status of the underlying document."), ] = None type: Annotated[ Optional[CodeableConcept], diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index cd2e9fbcb..d7674aa41 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -499,48 +499,6 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - # Invalid document reference - invalid Type - # NRL-769 Known issue: Type display is not validated - # Scenario: Invalid type (valid code but wrong display value) - # Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - # And the organisation 'TSTCUS' is authorised to access pointer types: - # | system | value | - # | http://snomed.info/sct | 1363501000000100 | - # | http://snomed.info/sct | 736253002 | - # When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'type' is: - # """ - # "type": { - # "coding": [ - # { - # "system": "http://snomed.info/sct", - # "code": "736253002", - # "display": "Emergency Healthcare Plan" - # } - # ] - # } - # """ - # Then the response status code is 400 - # And the response is an OperationOutcome with 1 issue - # And the OperationOutcome contains the issue: - # """ - # { - # "severity": "error", - # "code": "invalid", - # "details": { - # "coding": [ - # { - # "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - # "code": "BAD_REQUEST", - # "display": "Bad request" - # } - # ] - # }, - # "diagnostics": "The display does not match the expected value for this type", - # "expression": [ - # "type.coding.display" - # ] - # } - # """ # Invalid document reference - empty content[0].attachment.url # Invalid document reference - create another producers document # Invalid document reference - bad JSON From 9408c91f1e32dd7ee0c874e5d4ae126307f0cb7c Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 11:57:59 +0000 Subject: [PATCH 09/11] NRL-477 fix tests --- layer/nrlf/core/tests/test_validators.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index bb661ddc8..4b13e5d56 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1309,8 +1309,8 @@ def test_validate_content_format_invalid_code_for_unstructured_document(): document_ref_data["content"][0]["format"] = { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:contact", - "display": "Contact details", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details (HTTP Unsecured)", } result = validator.validate(document_ref_data) @@ -1330,7 +1330,7 @@ def test_validate_content_format_invalid_code_for_unstructured_document(): } ] }, - "diagnostics": "Invalid content format code: urn:nhs-ic:contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", "expression": ["content[0].format.code"], } @@ -1643,6 +1643,8 @@ def test_validate_content_invalid_content_type(): def test_validate_nrl_format_code_valid_match(format_code, format_display): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + if format_code == "urn:nhs-ic:record-contact": + document_ref_data["content"][0]["attachment"]["contentType"] = "text/html" document_ref_data["content"][0]["format"] = { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", @@ -1675,6 +1677,8 @@ def test_validate_nrl_format_code_display_mismatch( ): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + if format_code == "urn:nhs-ic:record-contact": + document_ref_data["content"][0]["attachment"]["contentType"] = "text/html" document_ref_data["content"][0]["format"] = { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", From e0aaa157ce51d1d4bef260a2ccc8d6d967dd2c8e Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 14:54:51 +0000 Subject: [PATCH 10/11] NRL-477 fix int tests --- .../producer/createDocumentReference-failure.feature | 6 +++--- .../producer/updateDocumentReference-failure.feature | 2 +- .../producer/upsertDocumentReference-failure.feature | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 40fbb7238..31ab07da5 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -422,7 +422,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ] @@ -471,7 +471,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:record-contact", - "display": "Contact details" + "display": "Contact details (HTTP Unsecured)" } } ] @@ -825,7 +825,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", + "contentType": "text/html", "url": "https://example.org/my-doc.pdf" }, "format": { diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index 20a8bed4f..4dd5ea87c 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -258,7 +258,7 @@ Feature: Producer - updateDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", + "contentType": "text/html", "url": "https://example.org/my-doc.pdf" }, "format": { diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index c4fc4ea68..3855b1b34 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -376,7 +376,7 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", + "contentType": "text/html", "url": "https://example.org/my-doc.pdf" }, "format": { From 99d43c1342ef3ab8c2b9d4b88e5c25ad0bcd2227 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 15:18:48 +0000 Subject: [PATCH 11/11] NRL-477 fix int tests --- .../createDocumentReference-failure.feature | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 31ab07da5..af22ae1bc 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -423,7 +423,21 @@ Feature: Producer - createDocumentReference - Failure Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] """ @@ -472,7 +486,21 @@ Feature: Producer - createDocumentReference - Failure Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:record-contact", "display": "Contact details (HTTP Unsecured)" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] """