From 3f6f09294ed1e7887ff97873b33f5c25e0f45e0e Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 12 Nov 2024 16:16:46 +0000 Subject: [PATCH 001/106] NRL-1176 Fix warnings in integration tests due to pointer already deleted --- tests/features/steps/1_setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/features/steps/1_setup.py b/tests/features/steps/1_setup.py index 49269fbff..2a0a51e3b 100644 --- a/tests/features/steps/1_setup.py +++ b/tests/features/steps/1_setup.py @@ -1,5 +1,4 @@ import json -from contextlib import suppress from behave import * # noqa from behave.runner import Context @@ -55,5 +54,5 @@ def create_document_reference_step(context: Context): def clean_up_test_pointer(context: Context, doc_pointer: DocumentPointer): """Remove a pointer during cleanup without failing if it has already been deleted""" - with suppress(Exception): + if context.repository.get_by_id(doc_pointer.id): context.repository.delete(doc_pointer) From e716136dbbdf78db2aa05cd8904b32659c0ea22c Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Fri, 15 Nov 2024 11:43:32 +0000 Subject: [PATCH 002/106] NRL-853 WIP cloud backup setup --- .../dev/aws-backups.tf | 136 ++++++++++++ .../dev/dynamodb__pointers-table.tf | 5 +- .../account-wide-infrastructure/dev/s3.tf | 6 +- .../modules/backup-source/README.md | 37 ++++ .../modules/backup-source/backup_framework.tf | 149 +++++++++++++ .../backup-source/backup_notification.tf | 12 ++ .../modules/backup-source/backup_plan.tf | 85 ++++++++ .../backup-source/backup_report_plan.tf | 72 +++++++ .../backup_restore_testing.tf_disabled | 26 +++ .../modules/backup-source/backup_vault.tf | 4 + .../backup-source/backup_vault_policy.tf | 47 +++++ .../modules/backup-source/data.tf | 8 + .../modules/backup-source/iam.tf | 37 ++++ .../modules/backup-source/kms.tf | 34 +++ .../modules/backup-source/locals.tf | 3 + .../modules/backup-source/sns.tf | 35 +++ .../modules/backup-source/variables.tf | 199 ++++++++++++++++++ .../modules/permissions-store-bucket/vars.tf | 6 + .../modules/pointers-table/dynamodb.tf | 1 + .../modules/pointers-table/vars.tf | 6 + .../modules/truststore-bucket/vars.tf | 6 + .../modules/aws-backup-destination/README.md | 31 +++ .../modules/aws-backup-destination/backup.tf | 8 + .../backup_vault_lock.tf | 7 + .../backup_vault_policy.tf | 68 ++++++ .../aws-backup-destination/variables.tf | 67 ++++++ .../backup-infrastructure/test/aws-backup.tf | 42 ++++ terraform/backup-infrastructure/test/data.tf | 5 + .../backup-infrastructure/test/locals.tf | 8 + terraform/backup-infrastructure/test/main.tf | 14 ++ terraform/backup-infrastructure/test/vars.tf | 4 + 31 files changed, 1164 insertions(+), 4 deletions(-) create mode 100644 terraform/account-wide-infrastructure/dev/aws-backups.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/README.md create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/backup_framework.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/backup_notification.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/backup_report_plan.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/backup_restore_testing.tf_disabled create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/backup_vault.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/backup_vault_policy.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/data.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/iam.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/kms.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/locals.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/sns.tf create mode 100644 terraform/account-wide-infrastructure/modules/backup-source/variables.tf create mode 100644 terraform/backup-infrastructure/modules/aws-backup-destination/README.md create mode 100644 terraform/backup-infrastructure/modules/aws-backup-destination/backup.tf create mode 100644 terraform/backup-infrastructure/modules/aws-backup-destination/backup_vault_lock.tf create mode 100644 terraform/backup-infrastructure/modules/aws-backup-destination/backup_vault_policy.tf create mode 100644 terraform/backup-infrastructure/modules/aws-backup-destination/variables.tf create mode 100644 terraform/backup-infrastructure/test/aws-backup.tf create mode 100644 terraform/backup-infrastructure/test/data.tf create mode 100644 terraform/backup-infrastructure/test/locals.tf create mode 100644 terraform/backup-infrastructure/test/main.tf create mode 100644 terraform/backup-infrastructure/test/vars.tf diff --git a/terraform/account-wide-infrastructure/dev/aws-backups.tf b/terraform/account-wide-infrastructure/dev/aws-backups.tf new file mode 100644 index 000000000..09fd9da1f --- /dev/null +++ b/terraform/account-wide-infrastructure/dev/aws-backups.tf @@ -0,0 +1,136 @@ +provider "aws" { + alias = "source" + region = "eu-west-2" +} + +variable "destination_vault_arn" { + description = "ARN of the backup vault in the destination account" + type = string + default = "" +} + +#data "aws_arn" "destination_vault_arn" { +# arn = var.destination_vault_arn +#} + +data "aws_secretsmanager_secret" "backup-account-secret" { + name = "nhsd-nrlf--dev--test-backup-account-id" +} +data "aws_secretsmanager_secret_version" "destination_account_id" { + secret_id = data.aws_secretsmanager_secret.backup-account-secret.id +} + +locals { + # Adjust these as required + project_name = "dev-backups-poc" + environment_name = "dev" + + source_account_id = data.aws_caller_identity.current.account_id + # destination_account_id = data.aws_arn.destination_vault_arn.account + destination_account_id = data.aws_secretsmanager_secret_version.destination_account_id.secret_string +} + +# First, we create an S3 bucket for compliance reports. You may already have a module for creating +# S3 buckets with more refined access rules, which you may prefer to use. + +resource "aws_s3_bucket" "backup_reports" { + bucket_prefix = "${local.project_name}-backup-reports" +} + +# Now we have to configure access to the report bucket. + +resource "aws_s3_bucket_ownership_controls" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.id + rule { + object_ownership = "BucketOwnerPreferred" + } +} + +resource "aws_s3_bucket_acl" "backup_reports" { + depends_on = [aws_s3_bucket_ownership_controls.backup_reports] + + bucket = aws_s3_bucket.backup_reports.id + acl = "private" +} + +# We need a key for the SNS topic that will be used for notifications from AWS Backup. This key +# will be used to encrypt the messages sent to the topic before they are sent to the subscribers, +# but isn't needed by the recipients of the messages. + +# First we need some contextual data +data "aws_caller_identity" "current" {} +data "aws_region" "current" {} + +# Now we can define the key itself +resource "aws_kms_key" "backup_notifications" { + description = "KMS key for AWS Backup notifications" + deletion_window_in_days = 7 + enable_key_rotation = true + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Sid = "Enable IAM User Permissions" + Principal = { + AWS = "arn:aws:iam::${local.source_account_id}:root" + } + Action = "kms:*" + Resource = "*" + }, + { + Effect = "Allow" + Principal = { + Service = "sns.amazonaws.com" + } + Action = ["kms:GenerateDataKey*", "kms:Decrypt"] + Resource = "*" + }, + ] + }) +} + +# Now we can deploy the source and destination modules, referencing the resources we've created above. + +module "source" { + source = "../modules/backup-source" + + backup_copy_vault_account_id = local.destination_account_id + # backup_copy_vault_arn = data.aws_arn.destination_vault_arn.arn + environment_name = local.environment_name + bootstrap_kms_key_arn = aws_kms_key.backup_notifications.arn + project_name = local.project_name + reports_bucket = aws_s3_bucket.backup_reports.bucket + #terraform_role_arn = data.aws_caller_identity.current.arn + terraform_role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}" + + backup_plan_config = { + "compliance_resource_types" : [ + "S3" + ], + "rules" : [ + { + "copy_action" : { + "delete_after" : 4 + }, + "lifecycle" : { + "delete_after" : 2 + }, + "name" : "daily_kept_for_2_days", + "schedule" : "cron(0 0 * * ? *)" + } + ], + "selection_tag" : "NHSE-Enable-Backup" + } + # Note here that we need to explicitly disable DynamoDB backups in the source account. + # The default config in the module enables backups for all resource types. + backup_plan_config_dynamodb = { + "compliance_resource_types" : [ + "DynamoDB" + ], + "rules" : [ + ], + "enable" : false, + "selection_tag" : "NHSE-Enable-Backup" + } +} diff --git a/terraform/account-wide-infrastructure/dev/dynamodb__pointers-table.tf b/terraform/account-wide-infrastructure/dev/dynamodb__pointers-table.tf index fccaa1b00..4a6403208 100644 --- a/terraform/account-wide-infrastructure/dev/dynamodb__pointers-table.tf +++ b/terraform/account-wide-infrastructure/dev/dynamodb__pointers-table.tf @@ -1,6 +1,7 @@ module "dev-pointers-table" { - source = "../modules/pointers-table" - name_prefix = "nhsd-nrlf--dev" + source = "../modules/pointers-table" + name_prefix = "nhsd-nrlf--dev" + enable_backups = true } module "dev-sandbox-pointers-table" { diff --git a/terraform/account-wide-infrastructure/dev/s3.tf b/terraform/account-wide-infrastructure/dev/s3.tf index 472189d41..b90bf677f 100644 --- a/terraform/account-wide-infrastructure/dev/s3.tf +++ b/terraform/account-wide-infrastructure/dev/s3.tf @@ -1,6 +1,7 @@ module "dev-permissions-store-bucket" { - source = "../modules/permissions-store-bucket" - name_prefix = "nhsd-nrlf--dev" + source = "../modules/permissions-store-bucket" + name_prefix = "nhsd-nrlf--dev" + enable_backups = true } module "dev-sandbox-permissions-store-bucket" { @@ -12,6 +13,7 @@ module "dev-truststore-bucket" { source = "../modules/truststore-bucket" name_prefix = "nhsd-nrlf--dev" server_certificate_file = "../../../truststore/server/dev.pem" + enable_backups = true } module "dev-sandbox-truststore-bucket" { diff --git a/terraform/account-wide-infrastructure/modules/backup-source/README.md b/terraform/account-wide-infrastructure/modules/backup-source/README.md new file mode 100644 index 000000000..3f1b8bdb6 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/README.md @@ -0,0 +1,37 @@ +# AWS Backup Module + +The AWS Backup Module helps automates the setup of AWS Backup resources in a source account. It streamlines the process of creating, managing, and standardising backup configurations. + +## Inputs + +| Name | Description | Type | Default | Required | +| --------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | :------: | +| [backup_copy_vault_account_id](#input_backup_copy_vault_account_id) | The account id of the destination backup vault for allowing restores back into the source account. | `string` | `""` | no | +| [backup_copy_vault_arn](#input_backup_copy_vault_arn) | The ARN of the destination backup vault for cross-account backup copies. | `string` | `""` | no | +| [backup_plan_config](#input_backup_plan_config) | Configuration for backup plans | 
object({
 selection_tag = string
 compliance_resource_types = list(string)
 rules = list(object({
 name = string
 schedule = string
 enable_continuous_backup = optional(bool)
 lifecycle = object({
 delete_after = optional(number)
 cold_storage_after = optional(number)
 })
 copy_action = optional(object({
 delete_after = optional(number)
 }))
 }))
 })
| 
{
 "compliance_resource_types": [
 "S3"
 ],
 "rules": [
 {
 "copy_action": {
 "delete_after": 365
 },
 "lifecycle": {
 "delete_after": 35
 },
 "name": "daily_kept_5_weeks",
 "schedule": "cron(0 0 * * ? *)"
 },
 {
 "copy_action": {
 "delete_after": 365
 },
 "lifecycle": {
 "delete_after": 90
 },
 "name": "weekly_kept_3_months",
 "schedule": "cron(0 1 ? * SUN *)"
 },
 {
 "copy_action": {
 "delete_after": 365
 },
 "lifecycle": {
 "cold_storage_after": 30,
 "delete_after": 2555
 },
 "name": "monthly_kept_7_years",
 "schedule": "cron(0 2 1 * ? *)"
 },
 {
 "copy_action": {
 "delete_after": 365
 },
 "enable_continuous_backup": true,
 "lifecycle": {
 "delete_after": 35
 },
 "name": "point_in_time_recovery",
 "schedule": "cron(0 5 * * ? *)"
 }
 ],
 "selection_tag": "BackupLocal"
}
| no | +| [backup_plan_config_dynamodb](#input_backup_plan_config_dynamodb) | Configuration for backup plans with dynamodb | 
object({
 enable = bool
 selection_tag = string
 compliance_resource_types = list(string)
 rules = optional(list(object({
 name = string
 schedule = string
 enable_continuous_backup = optional(bool)
 lifecycle = object({
 delete_after = number
 cold_storage_after = optional(number)
 })
 copy_action = optional(object({
 delete_after = optional(number)
 }))
 })))
 })
| 
{
 "compliance_resource_types": [
 "DynamoDB"
 ],
 "enable": true,
 "rules": [
 {
 "copy_action": {
 "delete_after": 365
 },
 "lifecycle": {
 "delete_after": 35
 },
 "name": "dynamodb_daily_kept_5_weeks",
 "schedule": "cron(0 0 * * ? *)"
 },
 {
 "copy_action": {
 "delete_after": 365
 },
 "lifecycle": {
 "delete_after": 90
 },
 "name": "dynamodb_weekly_kept_3_months",
 "schedule": "cron(0 1 ? * SUN *)"
 },
 {
 "copy_action": {
 "delete_after": 365
 },
 "lifecycle": {
 "cold_storage_after": 30,
 "delete_after": 2555
 },
 "name": "dynamodb_monthly_kept_7_years",
 "schedule": "cron(0 2 1 * ? *)"
 }
 ],
 "selection_tag": "BackupDynamoDB"
}
| no | +| [bootstrap_kms_key_arn](#input_bootstrap_kms_key_arn) | The ARN of the bootstrap KMS key used for encryption at rest of the SNS topic. | `string` | n/a | yes | +| [environment_name](#input_environment_name) | The name of the environment where AWS Backup is configured. | `string` | n/a | yes | +| [notifications_target_email_address](#input_notifications_target_email_address) | The email address to which backup notifications will be sent via SNS. | `string` | `""` | no | +| [project_name](#input_project_name) | The name of the project this relates to. | `string` | n/a | yes | +| [reports_bucket](#input_reports_bucket) | Bucket to drop backup reports into | `string` | n/a | yes | +| [restore_testing_plan_algorithm](#input_restore_testing_plan_algorithm) | Algorithm of the Recovery Selection Point | `string` | `"LATEST_WITHIN_WINDOW"` | no | +| [restore_testing_plan_recovery_point_types](#input_restore_testing_plan_recovery_point_types) | Recovery Point Types | `list(string)` | 
[
 "SNAPSHOT"
]
| no | +| [restore_testing_plan_scheduled_expression](#input_restore_testing_plan_scheduled_expression) | Scheduled Expression of Recovery Selection Point | `string` | `"cron(0 1 ? * SUN *)"` | no | +| [restore_testing_plan_selection_window_days](#input_restore_testing_plan_selection_window_days) | Selection window days | `number` | `7` | no | +| [restore_testing_plan_start_window](#input_restore_testing_plan_start_window) | Start window from the scheduled time during which the test should start | `number` | `1` | no | +| [terraform_role_arn](#input_terraform_role_arn) | ARN of Terraform role used to deploy to account | `string` | n/a | yes | + +## Example + +```terraform +module "test_aws_backup" { + source = "./modules/aws-backup" + + environment_name = "environment_name" + bootstrap_kms_key_arn = kms_key[0].arn + project_name = "testproject" + reports_bucket = "compliance-reports" + terraform_role_arn = data.aws_iam_role.terraform_role.arn +} +``` diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_framework.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_framework.tf new file mode 100644 index 000000000..d10b43137 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_framework.tf @@ -0,0 +1,149 @@ +resource "aws_backup_framework" "main" { + # must be underscores instead of dashes + name = replace("${local.resource_name_prefix}-framework", "-", "_") + description = "${var.project_name} Backup Framework" + + # Evaluates if recovery points are encrypted. + control { + name = "BACKUP_RECOVERY_POINT_ENCRYPTED" + + scope { + tags = { + "environment_name" = var.environment_name + } + } + } + + # Evaluates if backup vaults do not allow manual deletion of recovery points with the exception of certain IAM roles. + control { + name = "BACKUP_RECOVERY_POINT_MANUAL_DELETION_DISABLED" + + scope { + tags = { + "environment_name" = var.environment_name + } + } + + input_parameter { + name = "principalArnList" + value = var.terraform_role_arn + } + } + + # Evaluates if recovery point retention period is at least 1 month. + control { + name = "BACKUP_RECOVERY_POINT_MINIMUM_RETENTION_CHECK" + + scope { + tags = { + "environment_name" = var.environment_name + } + } + + input_parameter { + name = "requiredRetentionDays" + value = "35" + } + } + + # Evaluates if backup plan creates backups at least every 1 day and retains them for at least 1 month before deleting them. + control { + name = "BACKUP_PLAN_MIN_FREQUENCY_AND_MIN_RETENTION_CHECK" + + scope { + tags = { + "environment_name" = var.environment_name + } + } + + input_parameter { + name = "requiredFrequencyUnit" + value = "days" + } + + input_parameter { + name = "requiredRetentionDays" + value = "35" + } + + input_parameter { + name = "requiredFrequencyValue" + value = "1" + } + } + + # Evaluates if resources are protected by a backup plan. + control { + name = "BACKUP_RESOURCES_PROTECTED_BY_BACKUP_PLAN" + + scope { + compliance_resource_types = var.backup_plan_config.compliance_resource_types + tags = { + (var.backup_plan_config.selection_tag) = "True" + } + } + } + + # Evaluates if resources have at least one recovery point created within the past 1 day. + control { + name = "BACKUP_LAST_RECOVERY_POINT_CREATED" + + input_parameter { + name = "recoveryPointAgeUnit" + value = "days" + } + + input_parameter { + name = "recoveryPointAgeValue" + value = "1" + } + + scope { + compliance_resource_types = var.backup_plan_config.compliance_resource_types + tags = { + (var.backup_plan_config.selection_tag) = "True" + } + } + } +} + +resource "aws_backup_framework" "dynamodb" { + count = var.backup_plan_config_dynamodb.enable ? 1 : 0 + # must be underscores instead of dashes + name = replace("${local.resource_name_prefix}-dynamodb-framework", "-", "_") + description = "${var.project_name} DynamoDB Backup Framework" + + # Evaluates if resources are protected by a backup plan. + control { + name = "BACKUP_RESOURCES_PROTECTED_BY_BACKUP_PLAN" + + scope { + compliance_resource_types = var.backup_plan_config_dynamodb.compliance_resource_types + tags = { + (var.backup_plan_config_dynamodb.selection_tag) = "True" + } + } + } + + # Evaluates if resources have at least one recovery point created within the past 1 day. + control { + name = "BACKUP_LAST_RECOVERY_POINT_CREATED" + + input_parameter { + name = "recoveryPointAgeUnit" + value = "days" + } + + input_parameter { + name = "recoveryPointAgeValue" + value = "1" + } + + scope { + compliance_resource_types = var.backup_plan_config_dynamodb.compliance_resource_types + tags = { + (var.backup_plan_config_dynamodb.selection_tag) = "True" + } + } + } +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_notification.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_notification.tf new file mode 100644 index 000000000..cb712321f --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_notification.tf @@ -0,0 +1,12 @@ +resource "aws_backup_vault_notifications" "backup_notification" { + count = var.notifications_target_email_address != "" ? 1 : 0 + backup_vault_name = aws_backup_vault.main.name + sns_topic_arn = aws_sns_topic.backup[0].arn + backup_vault_events = [ + "BACKUP_JOB_COMPLETED", + "RESTORE_JOB_COMPLETED", + "S3_BACKUP_OBJECT_FAILED", + "S3_RESTORE_OBJECT_FAILED", + "COPY_JOB_FAILED" + ] +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf new file mode 100644 index 000000000..4d2cf5066 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf @@ -0,0 +1,85 @@ +resource "aws_backup_plan" "default" { + name = "${local.resource_name_prefix}-plan" + + dynamic "rule" { + for_each = var.backup_plan_config.rules + content { + recovery_point_tags = { + backup_rule_name = rule.value.name + } + rule_name = rule.value.name + target_vault_name = aws_backup_vault.main.name + schedule = rule.value.schedule + enable_continuous_backup = rule.value.enable_continuous_backup != null ? rule.value.enable_continuous_backup : null + lifecycle { + delete_after = rule.value.lifecycle.delete_after != null ? rule.value.lifecycle.delete_after : null + cold_storage_after = rule.value.lifecycle.cold_storage_after != null ? rule.value.lifecycle.cold_storage_after : null + } + dynamic "copy_action" { + for_each = var.backup_copy_vault_arn != "" && var.backup_copy_vault_account_id != "" && rule.value.copy_action != null ? rule.value.copy_action : {} + content { + lifecycle { + delete_after = copy_action.value + } + destination_vault_arn = var.backup_copy_vault_arn + } + } + } + } +} + +# this backup plan shouldn't include a continous backup rule as it isn't supported for DynamoDB +resource "aws_backup_plan" "dynamodb" { + count = var.backup_plan_config_dynamodb.enable ? 1 : 0 + name = "${local.resource_name_prefix}-dynamodb-plan" + + dynamic "rule" { + for_each = var.backup_plan_config_dynamodb.rules + content { + recovery_point_tags = { + backup_rule_name = rule.value.name + } + rule_name = rule.value.name + target_vault_name = aws_backup_vault.main.name + schedule = rule.value.schedule + lifecycle { + delete_after = rule.value.lifecycle.delete_after != null ? rule.value.lifecycle.delete_after : null + cold_storage_after = rule.value.lifecycle.cold_storage_after != null ? rule.value.lifecycle.cold_storage_after : null + } + dynamic "copy_action" { + for_each = var.backup_copy_vault_arn != "" && var.backup_copy_vault_account_id != "" && rule.value.copy_action != null ? rule.value.copy_action : {} + content { + lifecycle { + delete_after = copy_action.value + } + destination_vault_arn = var.backup_copy_vault_arn + } + } + } + } +} + +resource "aws_backup_selection" "default" { + iam_role_arn = aws_iam_role.backup.arn + name = "${local.resource_name_prefix}-selection" + plan_id = aws_backup_plan.default.id + + selection_tag { + key = var.backup_plan_config.selection_tag + type = "STRINGEQUALS" + value = "True" + } +} + +resource "aws_backup_selection" "dynamodb" { + count = var.backup_plan_config_dynamodb.enable ? 1 : 0 + iam_role_arn = aws_iam_role.backup.arn + name = "${local.resource_name_prefix}-dynamodb-selection" + plan_id = aws_backup_plan.dynamodb[0].id + + selection_tag { + key = var.backup_plan_config_dynamodb.selection_tag + type = "STRINGEQUALS" + value = "True" + } +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_report_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_report_plan.tf new file mode 100644 index 000000000..e0f20d41e --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_report_plan.tf @@ -0,0 +1,72 @@ +# Create the reports +resource "aws_backup_report_plan" "backup_jobs" { + name = "backup_jobs" + description = "Report for showing whether backups ran successfully in the last 24 hours" + + report_delivery_channel { + formats = [ + "JSON" + ] + s3_bucket_name = var.reports_bucket + s3_key_prefix = "backup_jobs" + } + + report_setting { + report_template = "BACKUP_JOB_REPORT" + } +} + +# Create the restore testing completion reports +resource "aws_backup_report_plan" "backup_restore_testing_jobs" { + name = "backup_restore_testing_jobs" + description = "Report for showing whether backup restore test ran successfully in the last 24 hours" + + report_delivery_channel { + formats = [ + "JSON" + ] + s3_bucket_name = var.reports_bucket + s3_key_prefix = "backup_restore_testing_jobs" + } + + report_setting { + report_template = "RESTORE_JOB_REPORT" + } +} + +resource "aws_backup_report_plan" "resource_compliance" { + name = "resource_compliance" + description = "Report for showing whether resources are compliant with the framework" + + report_delivery_channel { + formats = [ + "JSON" + ] + s3_bucket_name = var.reports_bucket + s3_key_prefix = "resource_compliance" + } + + report_setting { + framework_arns = var.backup_plan_config_dynamodb.enable ? [aws_backup_framework.main.arn, aws_backup_framework.dynamodb[0].arn] : [aws_backup_framework.main.arn] + number_of_frameworks = 2 + report_template = "RESOURCE_COMPLIANCE_REPORT" + } +} + +resource "aws_backup_report_plan" "copy_jobs" { + count = var.backup_copy_vault_arn != "" && var.backup_copy_vault_account_id != "" ? 1 : 0 + name = "copy_jobs" + description = "Report for showing whether copies ran successfully in the last 24 hours" + + report_delivery_channel { + formats = [ + "JSON" + ] + s3_bucket_name = var.reports_bucket + s3_key_prefix = "copy_jobs" + } + + report_setting { + report_template = "COPY_JOB_REPORT" + } +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_restore_testing.tf_disabled b/terraform/account-wide-infrastructure/modules/backup-source/backup_restore_testing.tf_disabled new file mode 100644 index 000000000..6c4b6f3a9 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_restore_testing.tf_disabled @@ -0,0 +1,26 @@ +resource "awscc_backup_restore_testing_plan" "backup_restore_testing_plan" { + restore_testing_plan_name = "backup_restore_testing_plan" + schedule_expression = var.restore_testing_plan_scheduled_expression + start_window_hours = var.restore_testing_plan_start_window + recovery_point_selection = { + algorithm = var.restore_testing_plan_algorithm + include_vaults = [aws_backup_vault.main.arn] + recovery_point_types = var.restore_testing_plan_recovery_point_types + selection_window_days = var.restore_testing_plan_selection_window_days + } +} + +resource "awscc_backup_restore_testing_selection" "backup_restore_testing_selection_dynamodb" { + count = var.backup_plan_config_dynamodb.enable ? 1 : 0 + iam_role_arn = aws_iam_role.backup.arn + protected_resource_type = "DynamoDB" + restore_testing_plan_name = awscc_backup_restore_testing_plan.backup_restore_testing_plan.restore_testing_plan_name + restore_testing_selection_name = "backup_restore_testing_selection_dynamodb" + protected_resource_arns = ["*"] + protected_resource_conditions = { + string_equals = [{ + key = "aws:ResourceTag/${var.backup_plan_config_dynamodb.selection_tag}" + value = "True" + }] + } +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_vault.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_vault.tf new file mode 100644 index 000000000..49f79ca49 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_vault.tf @@ -0,0 +1,4 @@ +resource "aws_backup_vault" "main" { + name = "${local.resource_name_prefix}-vault" + kms_key_arn = aws_kms_key.aws_backup_key.arn +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_vault_policy.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_vault_policy.tf new file mode 100644 index 000000000..09d4ec117 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_vault_policy.tf @@ -0,0 +1,47 @@ +resource "aws_backup_vault_policy" "vault_policy" { + backup_vault_name = aws_backup_vault.main.name + policy = data.aws_iam_policy_document.vault_policy.json +} + +data "aws_iam_policy_document" "vault_policy" { + + + statement { + sid = "DenyApartFromTerraform" + effect = "Deny" + + principals { + type = "AWS" + identifiers = ["*"] + } + + condition { + test = "ArnNotEquals" + values = [var.terraform_role_arn] + variable = "aws:PrincipalArn" + } + + actions = [ + "backup:DeleteRecoveryPoint", + "backup:PutBackupVaultAccessPolicy", + "backup:UpdateRecoveryPointLifecycle" + ] + + resources = ["*"] + } + dynamic "statement" { + for_each = var.backup_copy_vault_arn != "" && var.backup_copy_vault_account_id != "" ? [1] : [] + content { + sid = "Allow account to copy into backup vault" + effect = "Allow" + + actions = ["backup:CopyIntoBackupVault"] + resources = ["*"] + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${var.backup_copy_vault_account_id}:root"] + } + } + } +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/data.tf b/terraform/account-wide-infrastructure/modules/backup-source/data.tf new file mode 100644 index 000000000..9275ede2e --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/data.tf @@ -0,0 +1,8 @@ +data "aws_caller_identity" "current" {} + +data "aws_region" "current" {} + +data "aws_iam_roles" "roles" { + name_regex = "AWSReservedSSO_Admin_.*" + path_prefix = "/aws-reserved/sso.amazonaws.com/" +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/iam.tf b/terraform/account-wide-infrastructure/modules/backup-source/iam.tf new file mode 100644 index 000000000..e4d58dcc4 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/iam.tf @@ -0,0 +1,37 @@ +data "aws_iam_policy_document" "assume_role" { + statement { + effect = "Allow" + + principals { + type = "Service" + identifiers = ["backup.amazonaws.com"] + } + + actions = ["sts:AssumeRole"] + } +} + +resource "aws_iam_role" "backup" { + name = "${var.project_name}BackupRole" + assume_role_policy = data.aws_iam_policy_document.assume_role.json +} + +resource "aws_iam_role_policy_attachment" "backup" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForBackup" + role = aws_iam_role.backup.name +} + +resource "aws_iam_role_policy_attachment" "restore" { + policy_arn = "arn:aws:iam::aws:policy/service-role/AWSBackupServiceRolePolicyForRestores" + role = aws_iam_role.backup.name +} + +resource "aws_iam_role_policy_attachment" "s3_restore" { + policy_arn = "arn:aws:iam::aws:policy/AWSBackupServiceRolePolicyForS3Restore" + role = aws_iam_role.backup.name +} + +resource "aws_iam_role_policy_attachment" "s3_backup" { + policy_arn = "arn:aws:iam::aws:policy/AWSBackupServiceRolePolicyForS3Backup" + role = aws_iam_role.backup.name +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/kms.tf b/terraform/account-wide-infrastructure/modules/backup-source/kms.tf new file mode 100644 index 000000000..ee8183220 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/kms.tf @@ -0,0 +1,34 @@ +resource "aws_kms_key" "aws_backup_key" { + description = "AWS Backup KMS Key" + deletion_window_in_days = 30 + enable_key_rotation = true + policy = data.aws_iam_policy_document.backup_key_policy.json +} + +resource "aws_kms_alias" "backup_key" { + name = "alias/${var.environment_name}/backup-key" + target_key_id = aws_kms_key.aws_backup_key.key_id +} + +data "aws_iam_policy_document" "backup_key_policy" { + #checkov:skip=CKV_AWS_109:See (CERSS-25168) for more info + #checkov:skip=CKV_AWS_111:See (CERSS-25169) for more info + statement { + sid = "AllowBackupUseOfKey" + principals { + type = "Service" + identifiers = ["backup.amazonaws.com"] + } + actions = ["kms:GenerateDataKey", "kms:Decrypt", "kms:Encrypt"] + resources = ["*"] + } + statement { + sid = "EnableIAMUserPermissions" + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root", data.aws_caller_identity.current.arn] + } + actions = ["kms:*"] + resources = ["*"] + } +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/locals.tf b/terraform/account-wide-infrastructure/modules/backup-source/locals.tf new file mode 100644 index 000000000..e6929817b --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/locals.tf @@ -0,0 +1,3 @@ +locals { + resource_name_prefix = "${data.aws_region.current.name}-${data.aws_caller_identity.current.account_id}-backup" +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/sns.tf b/terraform/account-wide-infrastructure/modules/backup-source/sns.tf new file mode 100644 index 000000000..cdfec7ff2 --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/sns.tf @@ -0,0 +1,35 @@ +resource "aws_sns_topic" "backup" { + count = var.notifications_target_email_address != "" ? 1 : 0 + name = "${local.resource_name_prefix}-notifications" + kms_master_key_id = var.bootstrap_kms_key_arn + policy = data.aws_iam_policy_document.allow_backup_to_sns.json +} + +data "aws_iam_policy_document" "allow_backup_to_sns" { + policy_id = "backup" + + statement { + actions = [ + "SNS:Publish", + ] + + effect = "Allow" + + principals { + type = "Service" + identifiers = ["backup.amazonaws.com"] + } + + resources = ["*"] + + sid = "allow_backup" + } +} + +resource "aws_sns_topic_subscription" "aws_backup_notifications_email_target" { + count = var.notifications_target_email_address != "" ? 1 : 0 + topic_arn = aws_sns_topic.backup[0].arn + protocol = "email" + endpoint = var.notifications_target_email_address + filter_policy = jsonencode({ "State" : [{ "anything-but" : "COMPLETED" }] }) +} diff --git a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf new file mode 100644 index 000000000..0873063fe --- /dev/null +++ b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf @@ -0,0 +1,199 @@ +variable "project_name" { + description = "The name of the project this relates to." + type = string +} + +variable "environment_name" { + description = "The name of the environment where AWS Backup is configured." + type = string +} + +variable "notifications_target_email_address" { + description = "The email address to which backup notifications will be sent via SNS." + type = string + default = "" +} + +variable "bootstrap_kms_key_arn" { + description = "The ARN of the bootstrap KMS key used for encryption at rest of the SNS topic." + type = string +} + +variable "reports_bucket" { + description = "Bucket to drop backup reports into" + type = string +} + +variable "terraform_role_arn" { + description = "ARN of Terraform role used to deploy to account" + type = string +} + +variable "restore_testing_plan_algorithm" { + description = "Algorithm of the Recovery Selection Point" + type = string + default = "LATEST_WITHIN_WINDOW" +} + +variable "restore_testing_plan_start_window" { + description = "Start window from the scheduled time during which the test should start" + type = number + default = 1 +} + +variable "restore_testing_plan_scheduled_expression" { + description = "Scheduled Expression of Recovery Selection Point" + type = string + default = "cron(0 1 ? * SUN *)" +} + +variable "restore_testing_plan_recovery_point_types" { + description = "Recovery Point Types" + type = list(string) + default = ["SNAPSHOT"] +} + +variable "restore_testing_plan_selection_window_days" { + description = "Selection window days" + type = number + default = 7 +} + +variable "backup_copy_vault_arn" { + description = "The ARN of the destination backup vault for cross-account backup copies." + type = string + default = "" +} + +variable "backup_copy_vault_account_id" { + description = "The account id of the destination backup vault for allowing restores back into the source account." + type = string + default = "" +} + +variable "backup_plan_config" { + description = "Configuration for backup plans" + type = object({ + selection_tag = string + compliance_resource_types = list(string) + rules = list(object({ + name = string + schedule = string + enable_continuous_backup = optional(bool) + lifecycle = object({ + delete_after = optional(number) + cold_storage_after = optional(number) + }) + copy_action = optional(object({ + delete_after = optional(number) + })) + })) + }) + default = { + selection_tag = "BackupLocal" + compliance_resource_types = ["S3"] + rules = [ + { + name = "daily_kept_5_weeks" + schedule = "cron(0 0 * * ? *)" + lifecycle = { + delete_after = 35 + } + copy_action = { + delete_after = 365 + } + }, + { + name = "weekly_kept_3_months" + schedule = "cron(0 1 ? * SUN *)" + lifecycle = { + delete_after = 90 + } + copy_action = { + delete_after = 365 + } + }, + { + name = "monthly_kept_7_years" + schedule = "cron(0 2 1 * ? *)" + lifecycle = { + cold_storage_after = 30 + delete_after = 2555 + } + copy_action = { + delete_after = 365 + } + }, + { + name = "point_in_time_recovery" + schedule = "cron(0 5 * * ? *)" + enable_continuous_backup = true + lifecycle = { + delete_after = 35 + } + copy_action = { + delete_after = 365 + } + } + ] + } +} + +variable "backup_plan_config_dynamodb" { + description = "Configuration for backup plans with dynamodb" + type = object({ + enable = bool + selection_tag = string + compliance_resource_types = list(string) + rules = optional(list(object({ + name = string + schedule = string + enable_continuous_backup = optional(bool) + lifecycle = object({ + delete_after = number + cold_storage_after = optional(number) + }) + copy_action = optional(object({ + delete_after = optional(number) + })) + }))) + }) + default = { + enable = true + selection_tag = "BackupDynamoDB" + compliance_resource_types = ["DynamoDB"] + rules = [ + { + name = "dynamodb_daily_kept_5_weeks" + schedule = "cron(0 0 * * ? *)" + lifecycle = { + delete_after = 35 + } + copy_action = { + delete_after = 365 + } + }, + { + name = "dynamodb_weekly_kept_3_months" + schedule = "cron(0 1 ? * SUN *)" + lifecycle = { + delete_after = 90 + } + copy_action = { + delete_after = 365 + } + }, + { + name = "dynamodb_monthly_kept_7_years" + schedule = "cron(0 2 1 * ? *)" + lifecycle = { + cold_storage_after = 30 + delete_after = 2555 + } + copy_action = { + delete_after = 365 + } + } + ] + } +} diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/vars.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/vars.tf index f593893ae..b5584a65c 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/vars.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/vars.tf @@ -8,3 +8,9 @@ variable "enable_bucket_force_destroy" { description = "A boolean flag to enable force destroy of the S3 bucket, so that all objects in the bucket are deleted when the bucket is destroyed." default = false } + +variable "enable_backups" { + type = bool + descirption = "enable AWS cloud backups" + default = false +} diff --git a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf index 1da659046..39a2d7d4f 100644 --- a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf +++ b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf @@ -51,4 +51,5 @@ resource "aws_dynamodb_table" "pointers" { point_in_time_recovery { enabled = var.enable_pitr } + #tags conditional } diff --git a/terraform/account-wide-infrastructure/modules/pointers-table/vars.tf b/terraform/account-wide-infrastructure/modules/pointers-table/vars.tf index 738e3b99e..29d04b60e 100644 --- a/terraform/account-wide-infrastructure/modules/pointers-table/vars.tf +++ b/terraform/account-wide-infrastructure/modules/pointers-table/vars.tf @@ -20,3 +20,9 @@ variable "kms_deletion_window_in_days" { description = "The duration in days after which the key is deleted after destruction of the resource." default = 7 } + +variable "enable_backups" { + type = bool + description = "Enable AwS cloud backup" + default = false +} diff --git a/terraform/account-wide-infrastructure/modules/truststore-bucket/vars.tf b/terraform/account-wide-infrastructure/modules/truststore-bucket/vars.tf index 3c6fa8790..e3b2f6f43 100644 --- a/terraform/account-wide-infrastructure/modules/truststore-bucket/vars.tf +++ b/terraform/account-wide-infrastructure/modules/truststore-bucket/vars.tf @@ -13,3 +13,9 @@ variable "enable_bucket_force_destroy" { description = "A boolean flag to enable force destroy of the S3 bucket, so that all objects in the bucket are deleted when the bucket is destroyed." default = false } + +variable "enable_backups" { + type = bool + description = "enable AWS cloud backups" + default = false +} diff --git a/terraform/backup-infrastructure/modules/aws-backup-destination/README.md b/terraform/backup-infrastructure/modules/aws-backup-destination/README.md new file mode 100644 index 000000000..10e01514b --- /dev/null +++ b/terraform/backup-infrastructure/modules/aws-backup-destination/README.md @@ -0,0 +1,31 @@ +# AWS Backup Module + +The AWS Backup Module helps automates the setup of AWS Backup resources in a destination account. It streamlines the process of creating, managing, and standardising backup configurations. + +## Inputs + +| Name | Description | Type | Default | Required | +| ------------------------------------------------------------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | -------------- | :------: | +| [account_id](#input_account_id) | The id of the account that the vault will be in | `string` | n/a | yes | +| [changeable_for_days](#input_changeable_for_days) | How long you want the vault lock to be changeable for, only applies to compliance mode. This value is expressed in days no less than 3 and no greater than 36,500; otherwise, an error will return. | `number` | `14` | no | +| [enable_vault_protection](#input_enable_vault_protection) | Flag which controls if the vault lock is enabled | `bool` | `false` | no | +| [kms_key](#input_kms_key) | The KMS key used to secure the vault | `string` | n/a | yes | +| [region](#input_region) | The region we should be operating in | `string` | `"eu-west-2"` | no | +| [source_account_id](#input_source_account_id) | The id of the account that backups will come from | `string` | n/a | yes | +| [source_account_name](#input_source_account_name) | The name of the account that backups will come from | `string` | n/a | yes | +| [vault_lock_max_retention_days](#input_vault_lock_max_retention_days) | The maximum retention period that the vault retains its recovery points | `number` | `365` | no | +| [vault_lock_min_retention_days](#input_vault_lock_min_retention_days) | The minimum retention period that the vault retains its recovery points | `number` | `365` | no | +| [vault_lock_type](#input_vault_lock_type) | The type of lock that the vault should be, will default to governance | `string` | `"governance"` | no | + +## Example + +```terraform +module "test_backup_vault" { + source = "./modules/aws_backup" + source_account_name = "test" + account_id = local.aws_accounts_ids["backup"] + source_account_id = local.aws_accounts_ids["test"] + kms_key = aws_kms_key.backup_key.arn + enable_vault_protection = true +} +``` diff --git a/terraform/backup-infrastructure/modules/aws-backup-destination/backup.tf b/terraform/backup-infrastructure/modules/aws-backup-destination/backup.tf new file mode 100644 index 000000000..1df7f2acf --- /dev/null +++ b/terraform/backup-infrastructure/modules/aws-backup-destination/backup.tf @@ -0,0 +1,8 @@ +resource "aws_backup_vault" "vault" { + name = "${var.source_account_name}-backup-vault" + kms_key_arn = var.kms_key +} + +output "vault_arn" { + value = aws_backup_vault.vault.arn +} diff --git a/terraform/backup-infrastructure/modules/aws-backup-destination/backup_vault_lock.tf b/terraform/backup-infrastructure/modules/aws-backup-destination/backup_vault_lock.tf new file mode 100644 index 000000000..e1a31781e --- /dev/null +++ b/terraform/backup-infrastructure/modules/aws-backup-destination/backup_vault_lock.tf @@ -0,0 +1,7 @@ +resource "aws_backup_vault_lock_configuration" "vault_lock" { + count = var.enable_vault_protection ? 1 : 0 + backup_vault_name = aws_backup_vault.vault.name + changeable_for_days = var.vault_lock_type == "compliance" ? var.changeable_for_days : null + max_retention_days = var.vault_lock_max_retention_days + min_retention_days = var.vault_lock_min_retention_days +} diff --git a/terraform/backup-infrastructure/modules/aws-backup-destination/backup_vault_policy.tf b/terraform/backup-infrastructure/modules/aws-backup-destination/backup_vault_policy.tf new file mode 100644 index 000000000..224904193 --- /dev/null +++ b/terraform/backup-infrastructure/modules/aws-backup-destination/backup_vault_policy.tf @@ -0,0 +1,68 @@ +resource "aws_backup_vault_policy" "vault_policy" { + backup_vault_name = aws_backup_vault.vault.name + policy = data.aws_iam_policy_document.vault_policy.json +} + +data "aws_iam_policy_document" "vault_policy" { + + statement { + sid = "AllowCopyToVault" + effect = "Allow" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${var.source_account_id}:root"] + } + + actions = [ + "backup:CopyIntoBackupVault" + ] + resources = ["*"] + } + + dynamic "statement" { + for_each = var.enable_vault_protection ? [1] : [] + content { + sid = "DenyBackupVaultAccess" + effect = "Deny" + + principals { + type = "AWS" + identifiers = ["*"] + } + actions = [ + "backup:DeleteRecoveryPoint", + "backup:PutBackupVaultAccessPolicy", + "backup:UpdateRecoveryPointLifecycle", + "backup:DeleteBackupVault", + "backup:StartRestoreJob", + "backup:DeleteBackupVaultLockConfiguration", + ] + resources = ["*"] + } + } + + dynamic "statement" { + for_each = var.enable_vault_protection ? [1] : [] + content { + sid = "DenyBackupCopyExceptToSourceAccount" + effect = "Deny" + + principals { + type = "AWS" + identifiers = ["arn:aws:iam::${var.account_id}:root"] + } + actions = [ + "backup:CopyFromBackupVault" + ] + resources = ["*"] + condition { + test = "StringNotEquals" + variable = "backup:CopyTargets" + values = [ + "arn:aws:backup:${var.region}:${var.source_account_id}:backup-vault:${var.region}-${var.source_account_id}-backup-vault" + ] + } + } + } +} diff --git a/terraform/backup-infrastructure/modules/aws-backup-destination/variables.tf b/terraform/backup-infrastructure/modules/aws-backup-destination/variables.tf new file mode 100644 index 000000000..75e620cfa --- /dev/null +++ b/terraform/backup-infrastructure/modules/aws-backup-destination/variables.tf @@ -0,0 +1,67 @@ +variable "source_account_name" { + # This is used as a prefix for the vault name, and referenced by the policy and the lock. + # It doesn't have to match anything in the source AWS account. + description = "The name of the account that backups will come from" + type = string +} + +variable "source_account_id" { + # The source account ID is used in the policy to allow permit root in the source account + # to copy backups into the vault. + description = "The id of the account that backups will come from" + type = string +} + +variable "account_id" { + # This is used to deny root from being able to copy backups from the vault + # to anywhere other than the source account. The constraint will need to + # be removed if the original source account is lost. + description = "The id of the account that the vault will be in" + type = string +} + +variable "region" { + description = "The region we should be operating in" + type = string + default = "eu-west-2" +} + +variable "kms_key" { + description = "The KMS key used to secure the vault" + type = string +} + +variable "enable_vault_protection" { + # With this set to true, privileges are locked down so that the vault can't be deleted or + # have its policy changed. The minimum and maximum retention periods are also set only if this is true. + description = "Flag which controls if the vault lock is enabled" + type = bool + default = false +} + +variable "vault_lock_type" { + description = "The type of lock that the vault should be, will default to governance" + type = string + # See toplevel README.md: + # DO NOT SET THIS TO compliance UNTIL YOU ARE SURE THAT YOU WANT TO LOCK THE VAULT PERMANENTLY + # When you do, you will also need to set "enable_vault_protection" to true for it to take effect. + default = "governance" +} + +variable "vault_lock_min_retention_days" { + description = "The minimum retention period that the vault retains its recovery points" + type = number + default = 365 +} + +variable "vault_lock_max_retention_days" { + description = "The maximum retention period that the vault retains its recovery points" + type = number + default = 365 +} + +variable "changeable_for_days" { + description = "How long you want the vault lock to be changeable for, only applies to compliance mode. This value is expressed in days no less than 3 and no greater than 36,500; otherwise, an error will return." + type = number + default = 14 +} diff --git a/terraform/backup-infrastructure/test/aws-backup.tf b/terraform/backup-infrastructure/test/aws-backup.tf new file mode 100644 index 000000000..19ee2e43a --- /dev/null +++ b/terraform/backup-infrastructure/test/aws-backup.tf @@ -0,0 +1,42 @@ + +# We need a key for the backup vaults. This key will be used to encrypt the backups themselves. +# We need one per vault (on the assumption that each vault will be in a different account). +resource "aws_kms_key" "destination_backup_key" { + description = "KMS key for AWS Backup vaults" + deletion_window_in_days = 7 + enable_key_rotation = true + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Sid = "Enable IAM User Permissions" + Principal = { + AWS = "arn:aws:iam::${local.destination_account_id}:root" + } + Action = "kms:*" + Resource = "*" + } + ] + }) +} + +module "destination" { + source = "../modules/aws-backup-destination" + + source_account_name = "test" # please note that the assigned value would be the prefix in aws_backup_vault.vault.name + account_id = local.destination_account_id + source_account_id = local.source_account_id + kms_key = aws_kms_key.destination_backup_key.arn + enable_vault_protection = false +} + +### +# Destination vault ARN output +### + +output "destination_vault_arn" { + # The ARN of the backup vault in the destination account is needed by + # the source account to copy backups into it. + value = module.destination.vault_arn +} diff --git a/terraform/backup-infrastructure/test/data.tf b/terraform/backup-infrastructure/test/data.tf new file mode 100644 index 000000000..f8b5aa3c4 --- /dev/null +++ b/terraform/backup-infrastructure/test/data.tf @@ -0,0 +1,5 @@ +data "aws_arn" "source_terraform_role" { + arn = var.source_terraform_role_arn +} + +data "aws_caller_identity" "current" {} diff --git a/terraform/backup-infrastructure/test/locals.tf b/terraform/backup-infrastructure/test/locals.tf new file mode 100644 index 000000000..0303cd337 --- /dev/null +++ b/terraform/backup-infrastructure/test/locals.tf @@ -0,0 +1,8 @@ +locals { + # Adjust these as required + project_name = "nrlf-test-backup" + environment_name = "dev" + + source_account_id = data.aws_arn.source_terraform_role.account + destination_account_id = data.aws_caller_identity.current.account_id +} diff --git a/terraform/backup-infrastructure/test/main.tf b/terraform/backup-infrastructure/test/main.tf new file mode 100644 index 000000000..f32f1468f --- /dev/null +++ b/terraform/backup-infrastructure/test/main.tf @@ -0,0 +1,14 @@ +#terraform { +# backend "s3" { +# bucket = "project-env-backup-tf-bucket" # change this to the destination account terraform state s3 bucket name +# key = "project-env-backup.tfstate" # change this to the destination account terraform state s3 key name +# dynamodb_table = "project-env-backup-lock-table" # change this to the destination account terraform state dynamodb table name +# region = "eu-west-2" +# } +#} + + +provider "aws" { + alias = "source" + region = "eu-west-2" +} diff --git a/terraform/backup-infrastructure/test/vars.tf b/terraform/backup-infrastructure/test/vars.tf new file mode 100644 index 000000000..e6e55ff45 --- /dev/null +++ b/terraform/backup-infrastructure/test/vars.tf @@ -0,0 +1,4 @@ +variable "source_terraform_role_arn" { + description = "ARN of the terraform role in the source account" + type = string +} From 732c833c52d90fa0a8e89bfb6b7682db1e41a1ea Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Mon, 18 Nov 2024 09:13:02 +0000 Subject: [PATCH 003/106] [NRL-853] Fix perms errors with AWSCC resources for backup restore testing --- terraform/account-wide-infrastructure/dev/main.tf | 7 +++++++ ...store_testing.tf_disabled => backup_restore_testing.tf} | 0 .../modules/backup-source/kms.tf | 7 +++++-- .../modules/permissions-store-bucket/vars.tf | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) rename terraform/account-wide-infrastructure/modules/backup-source/{backup_restore_testing.tf_disabled => backup_restore_testing.tf} (100%) diff --git a/terraform/account-wide-infrastructure/dev/main.tf b/terraform/account-wide-infrastructure/dev/main.tf index cfed956f2..6a15ca71b 100644 --- a/terraform/account-wide-infrastructure/dev/main.tf +++ b/terraform/account-wide-infrastructure/dev/main.tf @@ -10,7 +10,14 @@ provider "aws" { workspace = terraform.workspace } } +} + +provider "awscc" { + region = local.region + assume_role = { + role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}" + } } terraform { diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_restore_testing.tf_disabled b/terraform/account-wide-infrastructure/modules/backup-source/backup_restore_testing.tf similarity index 100% rename from terraform/account-wide-infrastructure/modules/backup-source/backup_restore_testing.tf_disabled rename to terraform/account-wide-infrastructure/modules/backup-source/backup_restore_testing.tf diff --git a/terraform/account-wide-infrastructure/modules/backup-source/kms.tf b/terraform/account-wide-infrastructure/modules/backup-source/kms.tf index ee8183220..55efeeec1 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/kms.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/kms.tf @@ -25,8 +25,11 @@ data "aws_iam_policy_document" "backup_key_policy" { statement { sid = "EnableIAMUserPermissions" principals { - type = "AWS" - identifiers = ["arn:aws:iam::${data.aws_caller_identity.current.account_id}:root", data.aws_caller_identity.current.arn] + type = "AWS" + identifiers = [ + "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root", + var.terraform_role_arn + ] } actions = ["kms:*"] resources = ["*"] diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/vars.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/vars.tf index b5584a65c..4a4db27b6 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/vars.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/vars.tf @@ -11,6 +11,6 @@ variable "enable_bucket_force_destroy" { variable "enable_backups" { type = bool - descirption = "enable AWS cloud backups" + description = "enable AWS cloud backups" default = false } From b00d59882fb26d06a602e49f96a8c2a6226e4017 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Mon, 18 Nov 2024 14:40:45 +0000 Subject: [PATCH 004/106] [NRL-853] Move notification email config out of lambda-error module and re-use for backup notifications --- terraform/account-wide-infrastructure/dev/aws-backups.tf | 2 ++ terraform/account-wide-infrastructure/dev/cloudwatch.tf | 2 ++ terraform/account-wide-infrastructure/dev/data.tf | 8 ++++++++ terraform/account-wide-infrastructure/dev/locals.tf | 2 ++ terraform/account-wide-infrastructure/dev/secrets.tf | 4 ++++ .../modules/backup-source/backup_notification.tf | 3 +-- .../modules/backup-source/sns.tf | 7 +++---- .../modules/backup-source/variables.tf | 8 ++++---- .../modules/lambda-errors-metric-alarm/secretsmanager.tf | 8 -------- .../modules/lambda-errors-metric-alarm/sns.tf | 2 +- .../modules/lambda-errors-metric-alarm/vars.tf | 6 ++++++ 11 files changed, 33 insertions(+), 19 deletions(-) delete mode 100644 terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/secretsmanager.tf diff --git a/terraform/account-wide-infrastructure/dev/aws-backups.tf b/terraform/account-wide-infrastructure/dev/aws-backups.tf index 09fd9da1f..041ef3b37 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backups.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backups.tf @@ -104,6 +104,8 @@ module "source" { #terraform_role_arn = data.aws_caller_identity.current.arn terraform_role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}" + notification_target_email_addresses = local.notification_emails + backup_plan_config = { "compliance_resource_types" : [ "S3" diff --git a/terraform/account-wide-infrastructure/dev/cloudwatch.tf b/terraform/account-wide-infrastructure/dev/cloudwatch.tf index 8a54e5854..031a6e36d 100644 --- a/terraform/account-wide-infrastructure/dev/cloudwatch.tf +++ b/terraform/account-wide-infrastructure/dev/cloudwatch.tf @@ -2,6 +2,8 @@ module "lambda_errors_cloudwatch_metric_alarm_dev" { source = "../modules/lambda-errors-metric-alarm" name_prefix = "nhsd-nrlf--dev" + notification_emails = local.notification_emails + evaluation_periods = 1 period = 60 threshold = 1 diff --git a/terraform/account-wide-infrastructure/dev/data.tf b/terraform/account-wide-infrastructure/dev/data.tf index fe0eefc7c..7b3c623de 100644 --- a/terraform/account-wide-infrastructure/dev/data.tf +++ b/terraform/account-wide-infrastructure/dev/data.tf @@ -1,3 +1,11 @@ data "aws_secretsmanager_secret_version" "identities_account_id" { secret_id = aws_secretsmanager_secret.identities_account_id.name } + +data "aws_secretsmanager_secret" "emails" { + name = "${local.prefix}-emails" +} + +data "aws_secretsmanager_secret_version" "emails" { + secret_id = data.aws_secretsmanager_secret.emails.id +} diff --git a/terraform/account-wide-infrastructure/dev/locals.tf b/terraform/account-wide-infrastructure/dev/locals.tf index 0929b0d38..9b06efdfe 100644 --- a/terraform/account-wide-infrastructure/dev/locals.tf +++ b/terraform/account-wide-infrastructure/dev/locals.tf @@ -3,4 +3,6 @@ locals { project = "nhsd-nrlf" environment = terraform.workspace prefix = "${local.project}--${local.environment}" + + notification_emails = nonsensitive(toset(tolist(jsondecode(data.aws_secretsmanager_secret_version.emails.secret_string)))) } diff --git a/terraform/account-wide-infrastructure/dev/secrets.tf b/terraform/account-wide-infrastructure/dev/secrets.tf index 46c339fc9..2559c81cd 100644 --- a/terraform/account-wide-infrastructure/dev/secrets.tf +++ b/terraform/account-wide-infrastructure/dev/secrets.tf @@ -2,6 +2,10 @@ resource "aws_secretsmanager_secret" "identities_account_id" { name = "${local.prefix}--nhs-identities-account-id" } +resource "aws_secretsmanager_secret" "notification_email_addresses" { + name = "${local.prefix}-dev-notification-email-addresses" +} + resource "aws_secretsmanager_secret" "dev_smoke_test_apigee_app" { name = "${local.prefix}--dev--apigee-app--smoke-test" description = "APIGEE App used to run Smoke Tests against the DEV environment" diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_notification.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_notification.tf index cb712321f..554f2ad49 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_notification.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_notification.tf @@ -1,7 +1,6 @@ resource "aws_backup_vault_notifications" "backup_notification" { - count = var.notifications_target_email_address != "" ? 1 : 0 backup_vault_name = aws_backup_vault.main.name - sns_topic_arn = aws_sns_topic.backup[0].arn + sns_topic_arn = aws_sns_topic.backup.arn backup_vault_events = [ "BACKUP_JOB_COMPLETED", "RESTORE_JOB_COMPLETED", diff --git a/terraform/account-wide-infrastructure/modules/backup-source/sns.tf b/terraform/account-wide-infrastructure/modules/backup-source/sns.tf index cdfec7ff2..f91b26b96 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/sns.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/sns.tf @@ -1,5 +1,4 @@ resource "aws_sns_topic" "backup" { - count = var.notifications_target_email_address != "" ? 1 : 0 name = "${local.resource_name_prefix}-notifications" kms_master_key_id = var.bootstrap_kms_key_arn policy = data.aws_iam_policy_document.allow_backup_to_sns.json @@ -27,9 +26,9 @@ data "aws_iam_policy_document" "allow_backup_to_sns" { } resource "aws_sns_topic_subscription" "aws_backup_notifications_email_target" { - count = var.notifications_target_email_address != "" ? 1 : 0 - topic_arn = aws_sns_topic.backup[0].arn + for_each = var.notification_target_email_addresses + topic_arn = aws_sns_topic.backup.arn protocol = "email" - endpoint = var.notifications_target_email_address + endpoint = each.value filter_policy = jsonencode({ "State" : [{ "anything-but" : "COMPLETED" }] }) } diff --git a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf index 0873063fe..a76f7a4d2 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf @@ -8,10 +8,10 @@ variable "environment_name" { type = string } -variable "notifications_target_email_address" { - description = "The email address to which backup notifications will be sent via SNS." - type = string - default = "" +variable "notification_target_email_addresses" { + description = "The email addresses to which backup notifications will be sent via SNS." + type = set(string) + default = [] } variable "bootstrap_kms_key_arn" { diff --git a/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/secretsmanager.tf b/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/secretsmanager.tf deleted file mode 100644 index 984bd41e3..000000000 --- a/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/secretsmanager.tf +++ /dev/null @@ -1,8 +0,0 @@ -data "aws_secretsmanager_secret" "emails" { - name = "${var.name_prefix}-emails" -} - -data "aws_secretsmanager_secret_version" "emails" { - secret_id = data.aws_secretsmanager_secret.emails.id - -} diff --git a/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/sns.tf b/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/sns.tf index 011568f53..5abaa0a6c 100644 --- a/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/sns.tf +++ b/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/sns.tf @@ -4,7 +4,7 @@ resource "aws_sns_topic" "sns_topic" { } resource "aws_sns_topic_subscription" "sns_subscription" { - for_each = nonsensitive(toset(tolist(jsondecode(data.aws_secretsmanager_secret_version.emails.secret_string)))) + for_each = var.notification_emails topic_arn = aws_sns_topic.sns_topic.arn protocol = "email" endpoint = sensitive(each.value) diff --git a/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/vars.tf b/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/vars.tf index a244243e1..605569262 100644 --- a/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/vars.tf +++ b/terraform/account-wide-infrastructure/modules/lambda-errors-metric-alarm/vars.tf @@ -25,3 +25,9 @@ variable "kms_deletion_window_in_days" { description = "The duration in days after which the key is deleted after destruction of the resource." default = 7 } + +variable "notification_emails" { + type = set(string) + description = "The email addresses to which notifications will be sent." + default = [] +} From b465e2bd7fb7a05e13fea98f93710a8f0a5a0018 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Mon, 18 Nov 2024 15:49:25 +0000 Subject: [PATCH 005/106] NRL-853 combine dynamodb and s3 policies, protect access to report bucket --- .../dev/aws-backups.tf | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backups.tf b/terraform/account-wide-infrastructure/dev/aws-backups.tf index 041ef3b37..5996ff3a3 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backups.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backups.tf @@ -37,6 +37,24 @@ resource "aws_s3_bucket" "backup_reports" { bucket_prefix = "${local.project_name}-backup-reports" } +resource "aws_s3_bucket_public_access_block" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" { + bucket = aws_s3_bucket.backup_reports.bucket + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} # Now we have to configure access to the report bucket. resource "aws_s3_bucket_ownership_controls" "backup_reports" { @@ -108,7 +126,7 @@ module "source" { backup_plan_config = { "compliance_resource_types" : [ - "S3" + "S3", "DynamoDB" ], "rules" : [ { @@ -124,15 +142,4 @@ module "source" { ], "selection_tag" : "NHSE-Enable-Backup" } - # Note here that we need to explicitly disable DynamoDB backups in the source account. - # The default config in the module enables backups for all resource types. - backup_plan_config_dynamodb = { - "compliance_resource_types" : [ - "DynamoDB" - ], - "rules" : [ - ], - "enable" : false, - "selection_tag" : "NHSE-Enable-Backup" - } } From 87b73169a3b1ec2a13c958ebb6149b4e79052861 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Mon, 18 Nov 2024 15:51:43 +0000 Subject: [PATCH 006/106] NRL-853 add backup tag to pointer table --- .../modules/pointers-table/dynamodb.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf index 39a2d7d4f..4bb3745e1 100644 --- a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf +++ b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf @@ -51,5 +51,6 @@ resource "aws_dynamodb_table" "pointers" { point_in_time_recovery { enabled = var.enable_pitr } - #tags conditional + + tags = var.enable_backups ? { NHSE-Enable-Backup : daily } : {} } From e18d11776ad5068287b773fd3f2d4bc6156533c7 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Mon, 18 Nov 2024 16:19:02 +0000 Subject: [PATCH 007/106] NRL-853 disallow http requests in s3 buckets --- .../dev/aws-backups.tf | 32 ++++++++++++++++--- .../modules/permissions-store-bucket/s3.tf | 26 +++++++++++++++ 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backups.tf b/terraform/account-wide-infrastructure/dev/aws-backups.tf index 5996ff3a3..8dd30b4e0 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backups.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backups.tf @@ -30,9 +30,7 @@ locals { destination_account_id = data.aws_secretsmanager_secret_version.destination_account_id.secret_string } -# First, we create an S3 bucket for compliance reports. You may already have a module for creating -# S3 buckets with more refined access rules, which you may prefer to use. - +# First, we create an S3 bucket for compliance reports. resource "aws_s3_bucket" "backup_reports" { bucket_prefix = "${local.project_name}-backup-reports" } @@ -55,7 +53,33 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "backup_reports" { } } } -# Now we have to configure access to the report bucket. + +resource "aws_s3_bucket_policy" "backup_reports_bucket_policy" { + bucket = aws_s3_bucket.backup_reports.id + + policy = jsonencode({ + Version = "2012-10-17" + Id = "backup_reports_bucket_policy" + Statement = [ + { + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = "*" + Action = "s3:*" + Resource = [ + aws_s3_bucket.backup_reports.arn, + "${aws_s3_bucket.backup_reports.arn}/*", + ] + Condition = { + Bool = { + "aws:SecureTransport" = "false" + } + } + }, + ] + }) +} + resource "aws_s3_bucket_ownership_controls" "backup_reports" { bucket = aws_s3_bucket.backup_reports.id diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf index ad37cff7d..5e6b80440 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf @@ -27,6 +27,32 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "authorization-sto } } +resource "aws_s3_bucket_policy" "authorization_store_bucket_policy" { + bucket = aws_s3_bucket.authorization-store.id + + policy = jsonencode({ + Version = "2012-10-17" + Id = "authorization_store_bucket_policy" + Statement = [ + { + Sid = "HTTPSOnly" + Effect = "Deny" + Principal = "*" + Action = "s3:*" + Resource = [ + aws_s3_bucket.authorization-store.arn, + "${aws_s3_bucket.authorization-store.arn}/*", + ] + Condition = { + Bool = { + "aws:SecureTransport" = "false" + } + } + }, + ] + }) +} + resource "aws_s3_bucket_versioning" "authorization-store" { bucket = aws_s3_bucket.authorization-store.id versioning_configuration { From 2c6ef7e24af24e25c6f83d16661dc29cf2ea4feb Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Mon, 18 Nov 2024 17:27:17 +0000 Subject: [PATCH 008/106] NRL-519 add practice setting codes from nrl2.8 to FHIR resources --- .../fhir/NRLF-PracticeSetting-ValueSet.json | 1889 +++++++++++++++++ 1 file changed, 1889 insertions(+) create mode 100644 resources/fhir/NRLF-PracticeSetting-ValueSet.json diff --git a/resources/fhir/NRLF-PracticeSetting-ValueSet.json b/resources/fhir/NRLF-PracticeSetting-ValueSet.json new file mode 100644 index 000000000..e81312fe7 --- /dev/null +++ b/resources/fhir/NRLF-PracticeSetting-ValueSet.json @@ -0,0 +1,1889 @@ +{ + "resourceType": "ValueSet", + "id": "NRLF-PracticeSetting", + "url": "https://fhir.nhs.uk/England/CodeSystem/England-NRLPracticeSetting", + "version": "1.1.2", + "name": "NRLF Record Practice Setting", + "status": "draft", + "date": "2024-11-18T00:00:00+00:00", + "publisher": "NHS Digital", + "contact": { + "name": "NRL Team at NHS Digital", + "telecom": { + "system": "email", + "value": "nrls@nhs.net", + "use": "work" + } + }, + "description": "A code from the SNOMED Clinical Terminology UK coding system to represent the NRL clinical practice setting.", + "copyright": "Copyright 2024 NHS Digital. This value set includes content from SNOMED CT, which is copyright 2002+ International Health Terminology Standards Development Organisation (IHTSDO), and distributed by agreement between IHTSDO and HL7. Implementer use of SNOMED CT is not covered by this agreement.", + "compose": { + "include": [ + { + "system": "http://snomed.info/sct", + "concept": [ + { + "code": "2471000175109", + "display": "Employee health service" + }, + { + "code": "828331000000102", + "display": "Homeopathy service" + }, + { + "code": "893041000000108", + "display": "Transient ischaemic attack service" + }, + { + "code": "893391000000101", + "display": "Adult cystic fibrosis service" + }, + { + "code": "828811000000104", + "display": "Child psychiatry service" + }, + { + "code": "741073001", + "display": "Neonatal intensive care service" + }, + { + "code": "3801000175108", + "display": "Pediatric pulmonology service" + }, + { + "code": "893521000000108", + "display": "Respiratory physiology service" + }, + { + "code": "892801000000107", + "display": "Audiological medicine service" + }, + { + "code": "892571000000101", + "display": "Medical oncology service" + }, + { + "code": "893421000000107", + "display": "Tropical medicine service" + }, + { + "code": "1060971000000108", + "display": "General practice service" + }, + { + "code": "907271000000106", + "display": "Genetics laboratory service" + }, + { + "code": "224891009", + "display": "Healthcare services" + }, + { + "code": "893771000000104", + "display": "Paediatric respiratory medicine service" + }, + { + "code": "893591000000106", + "display": "Paediatric metabolic disease service" + }, + { + "code": "892771000000109", + "display": "Cardiothoracic transplantation service" + }, + { + "code": "92151000000102", + "display": "Mental health crisis resolution team" + }, + { + "code": "893141000000109", + "display": "Nephrology service" + }, + { + "code": "829981000000108", + "display": "Community child health service" + }, + { + "code": "733459009", + "display": "Cardiac rehabilitation service" + }, + { + "code": "893971000000101", + "display": "Paediatric burns care service" + }, + { + "code": "931851000000100", + "display": "Oral pathology service" + }, + { + "code": "3771000175106", + "display": "Pediatric gastroenterology service" + }, + { + "code": "893621000000109", + "display": "Paediatric ear nose and throat service" + }, + { + "code": "828281000000107", + "display": "Eating disorders service" + }, + { + "code": "893121000000102", + "display": "Spinal surgery service" + }, + { + "code": "892751000000100", + "display": "Clinical immunology and allergy service" + }, + { + "code": "893601000000100", + "display": "Paediatric medical oncology service" + }, + { + "code": "3751000175100", + "display": "Pediatric emergency medical service" + }, + { + "code": "893951000000105", + "display": "Paediatric cardiology service" + }, + { + "code": "931831000000107", + "display": "Oral medicine service" + }, + { + "code": "829961000000104", + "display": "Out of hours service" + }, + { + "code": "911381000000108", + "display": "Telehealthcare service" + }, + { + "code": "828181000000101", + "display": "Community sexual and reproductive health" + }, + { + "code": "2451000175103", + "display": "Perinatology service" + }, + { + "code": "893851000000100", + "display": "Paediatric neurosurgery service" + }, + { + "code": "109201000000109", + "display": "Substance misuse team" + }, + { + "code": "893671000000108", + "display": "Paediatric urology service" + }, + { + "code": "932271000000104", + "display": "Oral and maxillofacial surgery service" + }, + { + "code": "444933003", + "display": "Home hospice service" + }, + { + "code": "444913002", + "display": "Diabetes mellitus service" + }, + { + "code": "892601000000108", + "display": "Intermediate care service" + }, + { + "code": "893801000000101", + "display": "Paediatric ophthalmology service" + }, + { + "code": "911231000000103", + "display": "Remote health monitoring service" + }, + { + "code": "893091000000103", + "display": "Infectious diseases service" + }, + { + "code": "893221000000106", + "display": "Specialist rehabilitation service" + }, + { + "code": "828381000000103", + "display": "Well woman service" + }, + { + "code": "907301000000109", + "display": "National Health Service 111 service" + }, + { + "code": "828511000000102", + "display": "National Health Service 24" + }, + { + "code": "893701000000107", + "display": "Paediatric thoracic surgery service" + }, + { + "code": "828861000000102", + "display": "Programmed pulmonary rehabilitation service" + }, + { + "code": "931781000000102", + "display": "Acute medicine service" + }, + { + "code": "827641000000101", + "display": "Anticoagulant service" + }, + { + "code": "893201000000102", + "display": "Sport and exercise medicine service" + }, + { + "code": "278032008", + "display": "Preventive service" + }, + { + "code": "892731000000107", + "display": "Dental medicine service" + }, + { + "code": "893451000000102", + "display": "Respite care service" + }, + { + "code": "2351000175106", + "display": "Sports medicine service" + }, + { + "code": "893271000000105", + "display": "Medical virology service" + }, + { + "code": "708168004", + "display": "Mental health service" + }, + { + "code": "708169007", + "display": "Respiratory therapy service" + }, + { + "code": "708171007", + "display": "Vascular ultrasound service" + }, + { + "code": "708170008", + "display": "Nursing service" + }, + { + "code": "708173005", + "display": "Obstetric ultrasound service" + }, + { + "code": "708172000", + "display": "Cardiac ultrasound service" + }, + { + "code": "708175003", + "display": "Diagnostic imaging service" + }, + { + "code": "708178001", + "display": "Cytogenetics service" + }, + { + "code": "708174004", + "display": "Interventional radiology service" + }, + { + "code": "708179009", + "display": "Molecular pathology service" + }, + { + "code": "708183009", + "display": "Anatomic pathology service" + }, + { + "code": "708182004", + "display": "Histology service" + }, + { + "code": "708180007", + "display": "Dermatopathology service" + }, + { + "code": "708187005", + "display": "Surgical pathology service" + }, + { + "code": "708185002", + "display": "Virology service" + }, + { + "code": "708184003", + "display": "Clinical pathology service" + }, + { + "code": "708188000", + "display": "Serology service" + }, + { + "code": "708194008", + "display": "Blood bank service" + }, + { + "code": "708196005", + "display": "Hematology service" + }, + { + "code": "708191000", + "display": "Toxicology service" + }, + { + "code": "708190004", + "display": "Immunology service" + }, + { + "code": "708193002", + "display": "Coagulation service" + }, + { + "code": "89301000000108", + "display": "Community mental health team" + }, + { + "code": "893301000000108", + "display": "Local specialist rehabilitation service" + }, + { + "code": "893651000000104", + "display": "Paediatric clinical immunology and allergy service" + }, + { + "code": "893171000000103", + "display": "Clinical neurophysiology service" + }, + { + "code": "711332004", + "display": "Allergy service" + }, + { + "code": "893151000000107", + "display": "Nuclear medicine service" + }, + { + "code": "894001000000107", + "display": "Paediatric audiological medicine service" + }, + { + "code": "892781000000106", + "display": "Burns care service" + }, + { + "code": "3781000175109", + "display": "Pediatric infectious disease service" + }, + { + "code": "893631000000106", + "display": "Paediatric diabetic medicine service" + }, + { + "code": "827621000000108", + "display": "Addiction service" + }, + { + "code": "893531000000105", + "display": "Psychiatric intensive care service" + }, + { + "code": "893881000000106", + "display": "Paediatric maxillofacial surgery service" + }, + { + "code": "893051000000106", + "display": "Clinical allergy service" + }, + { + "code": "893351000000109", + "display": "Complex specialised rehabilitation service" + }, + { + "code": "893001000000105", + "display": "Clinical genetics service" + }, + { + "code": "908981000000101", + "display": "Remote triage and advice service" + }, + { + "code": "931811000000104", + "display": "Histopathology service" + }, + { + "code": "1079481000000104", + "display": "Perinatal psychiatry service" + }, + { + "code": "893251000000101", + "display": "Mental health recovery and rehabilitation service" + }, + { + "code": "3531000175102", + "display": "Geriatric service" + }, + { + "code": "736622005", + "display": "Aboriginal health service" + }, + { + "code": "983641000000106", + "display": "Fracture liaison service" + }, + { + "code": "893711000000109", + "display": "Neonatal critical care service" + }, + { + "code": "828521000000108", + "display": "National Health Service Direct" + }, + { + "code": "892761000000102", + "display": "Clinical haematology service" + }, + { + "code": "901221000000102", + "display": "Perinatal mental health service" + }, + { + "code": "706902008", + "display": "Mycology service" + }, + { + "code": "706901001", + "display": "Bacteriology service" + }, + { + "code": "706903003", + "display": "Mycobacteriology service" + }, + { + "code": "706900000", + "display": "Parasitology service" + }, + { + "code": "893131000000100", + "display": "Genitourinary medicine service" + }, + { + "code": "413294000", + "display": "Community health services" + }, + { + "code": "413299005", + "display": "Early years services" + }, + { + "code": "828291000000109", + "display": "Dispensing optometry service" + }, + { + "code": "413331009", + "display": "Voluntary services" + }, + { + "code": "61831000000105", + "display": "Periodontics service" + }, + { + "code": "893231000000108", + "display": "Podiatric surgery service" + }, + { + "code": "893581000000109", + "display": "Well baby service" + }, + { + "code": "893911000000106", + "display": "Paediatric gastrointestinal surgery service" + }, + { + "code": "395104009", + "display": "Cancer primary healthcare multidisciplinary team" + }, + { + "code": "892711000000104", + "display": "Gynaecological oncology service" + }, + { + "code": "893331000000102", + "display": "Dementia assessment service" + }, + { + "code": "892611000000105", + "display": "Hepatology service" + }, + { + "code": "893681000000105", + "display": "Paediatric trauma and orthopaedics service" + }, + { + "code": "395086005", + "display": "Community specialist palliative care" + }, + { + "code": "395092004", + "display": "Specialist palliative care" + }, + { + "code": "710028007", + "display": "Maxillofacial surgery service" + }, + { + "code": "892811000000109", + "display": "Adult mental health service" + }, + { + "code": "828821000000105", + "display": "Adolescent psychiatry service" + }, + { + "code": "983341000000102", + "display": "Pharmacy First service" + }, + { + "code": "893431000000109", + "display": "Trauma and orthopaedics service" + }, + { + "code": "893781000000102", + "display": "Paediatric plastic surgery service" + }, + { + "code": "892581000000104", + "display": "Learning disability service" + }, + { + "code": "893661000000101", + "display": "Paediatric clinical haematology service" + }, + { + "code": "893311000000105", + "display": "Haemophilia service" + }, + { + "code": "373654008", + "display": "Medical referral service" + }, + { + "code": "893081000000100", + "display": "Respiratory medicine service" + }, + { + "code": "911221000000100", + "display": "Remote care environment monitoring service" + }, + { + "code": "409971007", + "display": "Emergency medical services" + }, + { + "code": "828371000000100", + "display": "Well man service" + }, + { + "code": "963151000000104", + "display": "Diabetic medicine service" + }, + { + "code": "893761000000106", + "display": "Paediatric rheumatology service" + }, + { + "code": "932841000000106", + "display": "Public health dentistry service" + }, + { + "code": "893861000000102", + "display": "Paediatric neurodisability service" + }, + { + "code": "92191000000105", + "display": "Early intervention in psychosis team" + }, + { + "code": "722424008", + "display": "Physical medicine and rehabilitation service" + }, + { + "code": "89311000000105", + "display": "Crisis prevention assessment and treatment team" + }, + { + "code": "722393008", + "display": "Legal medicine service" + }, + { + "code": "722352000", + "display": "Vascular medicine service" + }, + { + "code": "722170006", + "display": "Chiropractic service" + }, + { + "code": "722174002", + "display": "Pulmonary medicine service" + }, + { + "code": "722175001", + "display": "Psychosomatic medicine service" + }, + { + "code": "722176000", + "display": "Dentistry service" + }, + { + "code": "722140001", + "display": "Physiotherapy service" + }, + { + "code": "892561000000108", + "display": "Medical ophthalmology service" + }, + { + "code": "714088003", + "display": "Midwifery service" + }, + { + "code": "714089006", + "display": "Community midwifery service" + }, + { + "code": "3761000175103", + "display": "Pediatric endocrinology service" + }, + { + "code": "893611000000103", + "display": "Paediatric epilepsy service" + }, + { + "code": "893961000000108", + "display": "Paediatric cardiac surgery service" + }, + { + "code": "931841000000103", + "display": "Oral microbiology service" + }, + { + "code": "91901000000109", + "display": "Assertive outreach team" + }, + { + "code": "828191000000104", + "display": "Dental hygiene service" + }, + { + "code": "893031000000104", + "display": "Clinical immunology service" + }, + { + "code": "893381000000103", + "display": "Clinical psychology service" + }, + { + "code": "2461000175101", + "display": "Pulmonary rehabilitation service" + }, + { + "code": "893161000000105", + "display": "Neurology service" + }, + { + "code": "828201000000102", + "display": "Dental surgery assistance service" + }, + { + "code": "1079491000000102", + "display": "Paediatric diabetes service" + }, + { + "code": "893261000000103", + "display": "Mental health dual diagnosis service" + }, + { + "code": "310031001", + "display": "Family planning service" + }, + { + "code": "310032008", + "display": "Intensive care service" + }, + { + "code": "310030000", + "display": "Endoscopy service" + }, + { + "code": "310034009", + "display": "Pediatric intensive care service" + }, + { + "code": "310033003", + "display": "Adult intensive care service" + }, + { + "code": "310025004", + "display": "Complementary therapy service" + }, + { + "code": "310024000", + "display": "Colposcopy service" + }, + { + "code": "310027007", + "display": "Mental health counseling service" + }, + { + "code": "310026003", + "display": "Counseling service" + }, + { + "code": "310029005", + "display": "Domiciliary visit service" + }, + { + "code": "310028002", + "display": "Diagnostic investigation service" + }, + { + "code": "310020009", + "display": "Hearing therapy service" + }, + { + "code": "310022001", + "display": "Clinical oncology service" + }, + { + "code": "310021008", + "display": "Assistive listening device service" + }, + { + "code": "310023006", + "display": "Radiotherapy service" + }, + { + "code": "310017001", + "display": "Pediatric hearing aid service" + }, + { + "code": "310016005", + "display": "Adult hearing aid service" + }, + { + "code": "310015009", + "display": "Hearing aid service" + }, + { + "code": "310014008", + "display": "Pediatric cochlear implant service" + }, + { + "code": "310013002", + "display": "Adult cochlear implant service" + }, + { + "code": "310012007", + "display": "Cochlear implant service" + }, + { + "code": "310011000", + "display": "Aural rehabilitation service" + }, + { + "code": "310010004", + "display": "Distraction test audiological screening service" + }, + { + "code": "310019003", + "display": "Tinnitus management service" + }, + { + "code": "310018006", + "display": "Speech-reading training service" + }, + { + "code": "310001007", + "display": "Anesthetic service" + }, + { + "code": "310000008", + "display": "Accident and Emergency service" + }, + { + "code": "310003005", + "display": "Child assessment service" + }, + { + "code": "310002000", + "display": "Assessment service" + }, + { + "code": "310008001", + "display": "Audiological screening service" + }, + { + "code": "310009009", + "display": "Neonatal audiological screening service" + }, + { + "code": "310005003", + "display": "Diagnostic audiology service" + }, + { + "code": "310004004", + "display": "Audiological service" + }, + { + "code": "310007006", + "display": "Pediatric diagnostic audiology service" + }, + { + "code": "310006002", + "display": "Adult diagnostic audiology service" + }, + { + "code": "310085001", + "display": "Drama therapy service" + }, + { + "code": "310086000", + "display": "Music therapy service" + }, + { + "code": "310083008", + "display": "Art therapy service" + }, + { + "code": "310082003", + "display": "Arts therapy services" + }, + { + "code": "310084002", + "display": "Dance therapy service" + }, + { + "code": "310089007", + "display": "Hospital-based podiatry service" + }, + { + "code": "310087009", + "display": "Podiatry service" + }, + { + "code": "310088004", + "display": "Community-based podiatry service" + }, + { + "code": "310080006", + "display": "Pharmacy service" + }, + { + "code": "310081005", + "display": "Professional allied to medicine service" + }, + { + "code": "310079008", + "display": "Neuropathology service" + }, + { + "code": "310078000", + "display": "Medical microbiology service" + }, + { + "code": "310071006", + "display": "Pain management service" + }, + { + "code": "310070007", + "display": "Special care baby service" + }, + { + "code": "310072004", + "display": "Acute pain service" + }, + { + "code": "310073009", + "display": "Palliative care service" + }, + { + "code": "310076001", + "display": "Clinical biochemistry service" + }, + { + "code": "310074003", + "display": "Pathology service" + }, + { + "code": "310064001", + "display": "Occupational health service" + }, + { + "code": "310063007", + "display": "Obstetrics service" + }, + { + "code": "310066004", + "display": "Pediatric service" + }, + { + "code": "310065000", + "display": "Open access service" + }, + { + "code": "310068003", + "display": "Pediatric neurology service" + }, + { + "code": "310067008", + "display": "Community pediatric service" + }, + { + "code": "310069006", + "display": "Pediatric oncology service" + }, + { + "code": "310061009", + "display": "Gynecology service" + }, + { + "code": "310062002", + "display": "Pregnancy termination service" + }, + { + "code": "310060005", + "display": "Obstetrics and gynecology service" + }, + { + "code": "310099002", + "display": "Child physiotherapy service" + }, + { + "code": "310098005", + "display": "Hospital-based physiotherapy service" + }, + { + "code": "310091004", + "display": "Community-based dietetics service" + }, + { + "code": "310090003", + "display": "Dietetics service" + }, + { + "code": "310096009", + "display": "Hospital-based occupational therapy service" + }, + { + "code": "310094007", + "display": "Community-based occupational therapy service" + }, + { + "code": "310095008", + "display": "Social services occupational therapy service" + }, + { + "code": "310093001", + "display": "Occupational therapy service" + }, + { + "code": "310092006", + "display": "Hospital-based dietetics service" + }, + { + "code": "310120006", + "display": "Mental handicap psychiatry service" + }, + { + "code": "310121005", + "display": "Psychogeriatric service" + }, + { + "code": "310126000", + "display": "Breast screening service" + }, + { + "code": "310128004", + "display": "Computerized tomography service" + }, + { + "code": "310127009", + "display": "Magnetic resonance imaging service" + }, + { + "code": "310129007", + "display": "Rehabilitation service" + }, + { + "code": "310122003", + "display": "Rehabilitation psychiatry service" + }, + { + "code": "310123008", + "display": "Psychology service" + }, + { + "code": "310124002", + "display": "Psychotherapy service" + }, + { + "code": "310125001", + "display": "Radiology service" + }, + { + "code": "310117003", + "display": "Child and adolescent psychiatry service" + }, + { + "code": "310116007", + "display": "Psychiatry service" + }, + { + "code": "310119000", + "display": "Liaison psychiatry service" + }, + { + "code": "310118008", + "display": "Forensic psychiatry service" + }, + { + "code": "310114005", + "display": "Community surgical fitting service" + }, + { + "code": "310112009", + "display": "Surgical fitting service" + }, + { + "code": "310115006", + "display": "Public health service" + }, + { + "code": "310113004", + "display": "Hospital surgical fitting service" + }, + { + "code": "310110001", + "display": "Hospital orthotics service" + }, + { + "code": "310111002", + "display": "Community orthotics service" + }, + { + "code": "310109006", + "display": "Orthotics service" + }, + { + "code": "310108003", + "display": "Community orthoptics service" + }, + { + "code": "310105000", + "display": "Optometry service" + }, + { + "code": "310107008", + "display": "Hospital orthoptics service" + }, + { + "code": "310106004", + "display": "Orthoptics service" + }, + { + "code": "310104001", + "display": "Child speech and language therapy service" + }, + { + "code": "310103007", + "display": "Hospital-based speech and language therapy service" + }, + { + "code": "310102002", + "display": "Community-based speech and language therapy service" + }, + { + "code": "310141000", + "display": "Thoracic surgery service" + }, + { + "code": "310143002", + "display": "Dental surgery service" + }, + { + "code": "310142007", + "display": "Cardiac surgery service" + }, + { + "code": "310144008", + "display": "General dental surgery service" + }, + { + "code": "310145009", + "display": "Oral surgery service" + }, + { + "code": "310146005", + "display": "Orthodontics service" + }, + { + "code": "310147001", + "display": "Pediatric dentistry service" + }, + { + "code": "310148006", + "display": "Restorative dentistry service" + }, + { + "code": "310149003", + "display": "Ear, nose and throat service" + }, + { + "code": "310140004", + "display": "Cardiothoracic surgery service" + }, + { + "code": "310131003", + "display": "Community rehabilitation service" + }, + { + "code": "310130002", + "display": "Head injury rehabilitation service" + }, + { + "code": "310134006", + "display": "Social services" + }, + { + "code": "310135007", + "display": "Social services department customer services" + }, + { + "code": "310132005", + "display": "Young disabled service" + }, + { + "code": "310133000", + "display": "Swallow clinic" + }, + { + "code": "310139001", + "display": "Breast surgery service" + }, + { + "code": "310138009", + "display": "Surgical service" + }, + { + "code": "310136008", + "display": "Social services department duty team" + }, + { + "code": "310137004", + "display": "Stroke service" + }, + { + "code": "310101009", + "display": "Speech and language therapy service" + }, + { + "code": "310100005", + "display": "Play therapy service" + }, + { + "code": "734862008", + "display": "Endodontic service" + }, + { + "code": "734863003", + "display": "Prosthodontic service" + }, + { + "code": "310168000", + "display": "Vascular surgery service" + }, + { + "code": "310169008", + "display": "Ultrasonography service" + }, + { + "code": "310165002", + "display": "Transplant surgery service" + }, + { + "code": "310167005", + "display": "Urology service" + }, + { + "code": "310166001", + "display": "Trauma surgery service" + }, + { + "code": "310163009", + "display": "Pediatric surgical service" + }, + { + "code": "310164003", + "display": "Plastic surgery service" + }, + { + "code": "310161006", + "display": "Orthopedic service" + }, + { + "code": "310162004", + "display": "Pancreatic surgery service" + }, + { + "code": "310160007", + "display": "Ophthalmology service" + }, + { + "code": "310151004", + "display": "Gastrointestinal surgery service" + }, + { + "code": "310152006", + "display": "General gastrointestinal surgery service" + }, + { + "code": "310153001", + "display": "Upper gastrointestinal surgery service" + }, + { + "code": "310155008", + "display": "Colorectal surgery service" + }, + { + "code": "310150003", + "display": "Endocrine surgery service" + }, + { + "code": "310157000", + "display": "Hand surgery service" + }, + { + "code": "310156009", + "display": "General surgical service" + }, + { + "code": "310159002", + "display": "Neurosurgical service" + }, + { + "code": "310158005", + "display": "Hepatobiliary surgical service" + }, + { + "code": "828301000000108", + "display": "Electrocardiography service" + }, + { + "code": "310200001", + "display": "Cytology service" + }, + { + "code": "734920002", + "display": "Diabetes mellitus education service" + }, + { + "code": "829951000000102", + "display": "Industrial therapy service" + }, + { + "code": "931821000000105", + "display": "School nursing service" + }, + { + "code": "893941000000107", + "display": "Paediatric dermatology service" + }, + { + "code": "892741000000103", + "display": "Clinical microbiology service" + }, + { + "code": "893211000000100", + "display": "Spinal injuries service" + }, + { + "code": "92221000000103", + "display": "Mental health home treatment team" + }, + { + "code": "3621000175101", + "display": "Rheumatology service" + }, + { + "code": "408451000", + "display": "Community learning disabilities team" + }, + { + "code": "408452007", + "display": "Behavioral intervention team" + }, + { + "code": "408458006", + "display": "Specialist multidisciplinary team" + }, + { + "code": "893341000000106", + "display": "Congenital heart disease service" + }, + { + "code": "445449000", + "display": "Acute care hospice service" + }, + { + "code": "1078501000000104", + "display": "Health visiting service" + }, + { + "code": "893691000000107", + "display": "Paediatric transplantation surgery service" + }, + { + "code": "892621000000104", + "display": "Hepatobiliary and pancreatic surgery service" + }, + { + "code": "2421000175108", + "display": "Acute care inpatient service" + }, + { + "code": "931801000000101", + "display": "Community nursing service" + }, + { + "code": "699478002", + "display": "Surgical oncology service" + }, + { + "code": "893791000000100", + "display": "Paediatric pain management service" + }, + { + "code": "893541000000101", + "display": "Prosthetics service" + }, + { + "code": "699650006", + "display": "Community based physiotherapy service" + }, + { + "code": "893891000000108", + "display": "Paediatric interventional radiology service" + }, + { + "code": "827631000000105", + "display": "Emergency ambulance service" + }, + { + "code": "827981000000103", + "display": "Paediatric cystic fibrosis service" + }, + { + "code": "892821000000103", + "display": "Critical care medicine service" + }, + { + "code": "700435004", + "display": "Clinical physiology service" + }, + { + "code": "700436003", + "display": "Clinical pharmacology service" + }, + { + "code": "700434000", + "display": "Endocrinology service" + }, + { + "code": "700221004", + "display": "Care of elderly service" + }, + { + "code": "700433006", + "display": "Gastroenterology service" + }, + { + "code": "700231006", + "display": "Critical care physician service" + }, + { + "code": "700232004", + "display": "General medical service" + }, + { + "code": "700241009", + "display": "Dermatology service" + }, + { + "code": "893061000000109", + "display": "Cardiology service" + }, + { + "code": "705150003", + "display": "Domiciliary physiotherapy service" + }, + { + "code": "932241000000105", + "display": "Blood banking and transfusion service" + }, + { + "code": "3791000175107", + "display": "Pediatric nephrology service" + }, + { + "code": "892791000000108", + "display": "Blood and marrow transplantation service" + }, + { + "code": "893641000000102", + "display": "Palliative medicine service" + }, + { + "code": "431051000124102", + "display": "Dialysis service" + }, + { + "code": "23951000087100", + "display": "Opioid dependence service" + }, + { + "code": "1323651000000109", + "display": "Cardiac physiology service" + }, + { + "code": "788126001", + "display": "Prosthetic service" + }, + { + "code": "788124003", + "display": "Histopathology service" + }, + { + "code": "788125002", + "display": "Addiction service" + }, + { + "code": "788123009", + "display": "Radiation oncology service" + }, + { + "code": "788122004", + "display": "Sexual health service" + }, + { + "code": "788128000", + "display": "Critical care medicine service" + }, + { + "code": "788127005", + "display": "Child health service" + }, + { + "code": "788121006", + "display": "Clinical immunology and allergy service" + }, + { + "code": "733921009", + "display": "Transplant medicine service" + }, + { + "code": "788001008", + "display": "Infectious disease service" + }, + { + "code": "788002001", + "display": "Adult mental health service" + }, + { + "code": "788003006", + "display": "Nephrology service" + }, + { + "code": "788009005", + "display": "Nuclear medicine service" + }, + { + "code": "788006003", + "display": "Genetic laboratory service" + }, + { + "code": "788004000", + "display": "Clinical genetics service" + }, + { + "code": "788005004", + "display": "Neurology service" + }, + { + "code": "788008002", + "display": "Oral and maxillofacial surgery service" + }, + { + "code": "788007007", + "display": "General practice service" + }, + { + "code": "1326391000000100", + "display": "FNP (Family Nurse Partnership) service" + }, + { + "code": "1186717003", + "display": "Intellectual disability psychiatry service" + }, + { + "code": "830149003", + "display": "Clinical neurophysiology service" + }, + { + "code": "224930009", + "display": "Services" + }, + { + "code": "830039004", + "display": "Genitourinary medicine service" + }, + { + "code": "830038007", + "display": "Clinical allergy service" + }, + { + "code": "830037002", + "display": "Clinical immunology service" + }, + { + "code": "1326421000000106", + "display": "Safeguarding children team" + }, + { + "code": "1323551000000105", + "display": "Inherited metabolic medicine service" + }, + { + "code": "28541000087101", + "display": "Musculoskeletal service" + }, + { + "code": "24271000087103", + "display": "Adult chronic pain management service" + }, + { + "code": "1240241000000109", + "display": "Community sexual and reproductive health service" + }, + { + "code": "897188002", + "display": "Pediatric hematology service" + }, + { + "code": "1323631000000102", + "display": "Aviation and space medicine service" + }, + { + "code": "1325831000000100", + "display": "Post-COVID-19 syndrome service" + }, + { + "code": "1323431000000104", + "display": "Fetal medicine service" + }, + { + "code": "24001000087103", + "display": "Paediatric plastic surgery service" + }, + { + "code": "24351000087104", + "display": "Paediatric chronic pain management service" + }, + { + "code": "1323881000000102", + "display": "Stroke medicine service" + }, + { + "code": "1323531000000103", + "display": "Urological physiology service" + }, + { + "code": "148621000000100", + "display": "School aged immunisation service" + }, + { + "code": "24101000087102", + "display": "HIV (human immunodeficiency virus) social work service" + }, + { + "code": "23911000087104", + "display": "Medication review service" + }, + { + "code": "1148679005", + "display": "Specialist palliative care service" + }, + { + "code": "773558007", + "display": "Physical medicine service" + }, + { + "code": "24051000087102", + "display": "Breast surgical oncology service" + }, + { + "code": "896974005", + "display": "Transgender health service" + }, + { + "code": "1323611000000105", + "display": "Paediatric inherited metabolic medicine service" + }, + { + "code": "1323661000000107", + "display": "Paediatric audiovestibular medicine service" + }, + { + "code": "24331000087108", + "display": "Narcotic addiction service with chronic pain management" + }, + { + "code": "1323561000000108", + "display": "Gastrointestinal physiology service" + }, + { + "code": "34911000087100", + "display": "Amputation care service" + }, + { + "code": "1231786003", + "display": "Refugee healthcare service" + }, + { + "code": "1163002007", + "display": "Electrocardiography service" + }, + { + "code": "1163004008", + "display": "Hyperbaric medicine service" + }, + { + "code": "1163003002", + "display": "Colorectal cancer screening service" + }, + { + "code": "1323841000000105", + "display": "Paediatric palliative medicine service" + }, + { + "code": "1163054002", + "display": "Gastroscopy service" + }, + { + "code": "1231392007", + "display": "Paediatric orthopaedic service" + }, + { + "code": "1231391000", + "display": "Colonoscopy service" + }, + { + "code": "1231393002", + "display": "Spirometry service" + }, + { + "code": "1231390004", + "display": "Hand therapy service" + }, + { + "code": "1231394008", + "display": "Paediatric urology service" + }, + { + "code": "24081000087105", + "display": "HIV (human immunodeficiency virus) nurse practitioner service" + }, + { + "code": "23941000087103", + "display": "Narcotic addiction service" + }, + { + "code": "1323641000000106", + "display": "Audiovestibular medicine service" + }, + { + "code": "840587001", + "display": "Aerospace medical service" + }, + { + "code": "840586005", + "display": "Neonatal service" + }, + { + "code": "840585009", + "display": "Postnatal service" + }, + { + "code": "816075004", + "display": "Prosthetic and orthotic service" + }, + { + "code": "1323921000000108", + "display": "Neuropsychiatry service" + }, + { + "code": "24011000087101", + "display": "Vascular imaging service" + }, + { + "code": "1323691000000101", + "display": "General internal medical service" + }, + { + "code": "23891000087102", + "display": "Adult hematology service" + }, + { + "code": "1323821000000103", + "display": "Paediatric clinical pharmacology service" + }, + { + "code": "789718008", + "display": "Cardiology service" + }, + { + "code": "789714005", + "display": "Pediatric rheumatology service" + }, + { + "code": "789715006", + "display": "Paediatric respiratory therapy service" + }, + { + "code": "789716007", + "display": "Pediatric otolaryngology service" + }, + { + "code": "789717003", + "display": "Paediatric cardiology service" + }, + { + "code": "148581000000100", + "display": "Personal health record provider service" + }, + { + "code": "792849008", + "display": "Pediatric clinical genetics service" + }, + { + "code": "792847005", + "display": "Emergency ambulance service" + }, + { + "code": "792848000", + "display": "Internal medicine service" + }, + { + "code": "1323621000000104", + "display": "Medical psychotherapy service" + }, + { + "code": "1323871000000104", + "display": "Rehabilitation medicine service" + }, + { + "code": "1230046007", + "display": "Cervical cancer screening service" + }, + { + "code": "1230045006", + "display": "Cardiac diagnostic service" + }, + { + "code": "1230044005", + "display": "Cardiac specialist nursing service" + }, + { + "code": "1362761000000103", + "display": "Adult safeguarding team" + }, + { + "code": "1323901000000104", + "display": "Rare disease service" + }, + { + "code": "1136421000168109", + "display": "Sleep medicine service" + }, + { + "code": "1323701000000101", + "display": "Vascular physiology service" + }, + { + "code": "24141000087104", + "display": "Spine orthopedic surgery service" + }, + { + "code": "1323571000000101", + "display": "Orthogeriatric medicine service" + }, + { + "code": "1323801000000107", + "display": "Paediatric oral and maxillofacial surgery service" + }, + { + "code": "23871000087101", + "display": "Adult dermatology service" + }, + { + "code": "1323601000000108", + "display": "Ophthalmic and vision science service" + }, + { + "code": "1234796008", + "display": "Community nursing service" + }, + { + "code": "23901000087101", + "display": "Hepatology service" + }, + { + "code": "1324191000000107", + "display": "Intensive care medicine service" + }, + { + "code": "2391000175104", + "display": "Bariatric surgery service" + }, + { + "code": "1323851000000108", + "display": "Paediatric hepatology service" + }, + { + "code": "24291000087104", + "display": "Geriatric chronic pain management service" + }, + { + "code": "1323501000000109", + "display": "Special care dentistry service" + }, + { + "code": "1423561000000102", + "display": "Acute oncology service" + } + ] + } + ] + } +} From a4a9fd85bc1a2989ffabff581ef749104b7c51e1 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Tue, 19 Nov 2024 09:03:34 +0000 Subject: [PATCH 009/106] [NRL-853] Move backup infrastructure TF state into S3 and add README. Fixup account bootstrap script to work on non-mgmt accounts --- scripts/bootstrap.sh | 28 +++--- terraform/backup-infrastructure/README.md | 87 +++++++++++++++++++ terraform/backup-infrastructure/test/data.tf | 4 - .../backup-infrastructure/test/locals.tf | 6 +- terraform/backup-infrastructure/test/main.tf | 40 ++++++--- terraform/backup-infrastructure/test/vars.tf | 15 +++- 6 files changed, 149 insertions(+), 31 deletions(-) create mode 100644 terraform/backup-infrastructure/README.md diff --git a/scripts/bootstrap.sh b/scripts/bootstrap.sh index 89eb4baa3..b3e9069b2 100755 --- a/scripts/bootstrap.sh +++ b/scripts/bootstrap.sh @@ -1,4 +1,6 @@ #!/bin/bash +# Setup mgmt and non-mgmt AWS accounts for NRLF +set -o errexit -o nounset -o pipefail AWS_REGION_NAME="eu-west-2" PROFILE_PREFIX="nhsd-nrlf" @@ -32,18 +34,12 @@ function _check_mgmt() { } function _check_non_mgmt() { - if [[ "$(aws iam list-account-aliases --query 'AccountAliases[0]' --output text)" != 'nhsd-ddc-spine-nrlf-mgmt' ]]; then + if [[ "$(aws iam list-account-aliases --query 'AccountAliases[0]' --output text)" == 'nhsd-ddc-spine-nrlf-mgmt' ]]; then echo "Please log in as a non-mgmt account" >&2 return 1 fi } -function _get_mgmt_account(){ - if ! _check_mgmt; then return 1; fi - return $(aws sts get-caller-identity --query Account --output text) -} - - function _bootstrap() { local command=$1 local admin_policy_arn="arn:aws:iam::aws:policy/AdministratorAccess" @@ -55,7 +51,7 @@ function _bootstrap() { "create-mgmt") _check_mgmt || return 1 - cd $root/terraform/bootstrap/mgmt + cd terraform/bootstrap/mgmt aws s3api create-bucket --bucket "${truststore_bucket_name}" --region us-east-1 --create-bucket-configuration LocationConstraint="${AWS_REGION_NAME}" aws s3api create-bucket --bucket "${state_bucket_name}" --region us-east-1 --create-bucket-configuration LocationConstraint="${AWS_REGION_NAME}" aws s3api put-public-access-block --bucket "${state_bucket_name}" --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" @@ -69,7 +65,7 @@ function _bootstrap() { "delete-mgmt") _check_mgmt || return 1 - cd $root/terraform/bootstrap/mgmt + cd terraform/bootstrap/mgmt aws dynamodb delete-table --table-name "${state_lock_table_name}" || return 1 local versioned_objects versioned_objects=$(aws s3api list-object-versions \ @@ -90,10 +86,20 @@ function _bootstrap() { "create-non-mgmt") _check_non_mgmt || return 1 - cd $root/terraform/bootstrap/non-mgmt + cd terraform/bootstrap/non-mgmt local tf_assume_role_policy local mgmt_account_id - mgmt_account_id=$(_get_mgmt_account) + + set +e + mgmt_account_id=$(aws secretsmanager get-secret-value --secret-id "${MGMT_ACCOUNT_ID_LOCATION}" --query SecretString --output text) + + if [ "${mgmt_account_id}" == "" ]; then + aws secretsmanager create-secret --name "${MGMT_ACCOUNT_ID_LOCATION}" + echo "Please set ${MGMT_ACCOUNT_ID_LOCATION} in the Secrets Manager and rerun the script" + exit 1 + fi + set -e + tf_assume_role_policy=$(awk "{sub(/REPLACEME/,\"${mgmt_account_id}\")}1" terraform-trust-policy.json) aws iam create-role --role-name "${TERRAFORM_ROLE_NAME}" --assume-role-policy-document "${tf_assume_role_policy}" || return 1 aws iam attach-role-policy --policy-arn "${admin_policy_arn}" --role-name "${TERRAFORM_ROLE_NAME}" || return 1 diff --git a/terraform/backup-infrastructure/README.md b/terraform/backup-infrastructure/README.md new file mode 100644 index 000000000..8af8ae7db --- /dev/null +++ b/terraform/backup-infrastructure/README.md @@ -0,0 +1,87 @@ +# NRLF Backup Infrastructure + +This directory contains AWS backup terraform resources which are global to a given account. + +Each subdirectory corresponds to each AWS account (`prod` and `test`). + +**Backup infrastructure should be deployed manually and not be run as part of CI.** + +## Table of Contents + +1. [Prerequisites](#prerequisites) +2. [Initialise shell environment](#initialise-shell-environment) +3. [Deploy backup resources](#deploy-backup-resources) +4. [Tear down backup resources](#tear-down-backup-resources) + +## Prerequisites + +Before deploying the NRLF backup infrastructure, you will need: + +- An AWS backup account that have already been bootstrapped, as described in [bootstrap/README.md](../bootstrap/README.md). This is a one-time account setup step. + +## Deploy backup resources + +To deploy the backup resources, first login to the AWS mgmt account on the CLI. + +Then, initialise the terraform backup workspace. For the test account: + +```shell +$ cd test +$ terraform init && ( \ + terraform workspace new backup-infra-test || \ + terraform workspace select backup-infra-test ) +``` + +If you want to apply changes to prod, use the `prod` directory and the `backup-infra-prod` terraform workspace. + +Once you have your workspace set, you can plan your changes with: + +```shell +$ terraform plan \ + -var 'source_account_id=SOURCE_ACCOUNT_ID" \ + -var 'assume_account=AWS_ACCOUNT_ID' \ + -var 'assume_role=terraform' +``` + +Replacing SOURCE_ACCOUNT with the account id that will be sending backups to the backup account and AWS_ACCOUNT_ID with the AWS account id of your backup account. + +Once you're happy with your planned changes, you can apply them with: + +```shell +$ terraform apply \ + -var 'source_account_id=SOURCE_ACCOUNT_ID" \ + -var 'assume_account=AWS_ACCOUNT_ID' \ + -var 'assume_role=terraform' +``` + +Replacing SOURCE_ACCOUNT with the account id that will be sending backups to the backup account and AWS_ACCOUNT_ID with the AWS account id of your backup account. + +## Tear down backup resources + +WARNING - This action will destroy all backup resources from the AWS account. This should +only be done if you are sure that this is safe and are sure that you are signed into the correct +AWS account. + +To tear down backup resources, first login to the AWS mgmt account on the CLI. + +Then, initialise your terraform workspace. For the test account: + +```shell +$ cd test +$ terraform init && ( \ + terraform workspace new backup-infra-test || \ + terraform workspace select backup-infra-test ) +``` + +If you want to destroy resources in prod, use the `prod` directory and the `backup-infra-prod` terraform workspace. + +And then, to tear down: + +```shell +$ terraform destroy \ + -var 'source_account_id=SOURCE_ACCOUNT_ID" \ + -var 'assume_account=AWS_ACCOUNT_ID' \ + -var 'assume_role=terraform' +``` + +Replacing SOURCE_ACCOUNT with the account id that will be sending backups to the backup account and AWS_ACCOUNT_ID with the AWS account id of your backup account. diff --git a/terraform/backup-infrastructure/test/data.tf b/terraform/backup-infrastructure/test/data.tf index f8b5aa3c4..8fc4b38cc 100644 --- a/terraform/backup-infrastructure/test/data.tf +++ b/terraform/backup-infrastructure/test/data.tf @@ -1,5 +1 @@ -data "aws_arn" "source_terraform_role" { - arn = var.source_terraform_role_arn -} - data "aws_caller_identity" "current" {} diff --git a/terraform/backup-infrastructure/test/locals.tf b/terraform/backup-infrastructure/test/locals.tf index 0303cd337..6bc51d571 100644 --- a/terraform/backup-infrastructure/test/locals.tf +++ b/terraform/backup-infrastructure/test/locals.tf @@ -1,8 +1,8 @@ locals { # Adjust these as required project_name = "nrlf-test-backup" - environment_name = "dev" + environment_name = "test" - source_account_id = data.aws_arn.source_terraform_role.account - destination_account_id = data.aws_caller_identity.current.account_id + source_account_id = var.source_account_id + destination_account_id = var.assume_account } diff --git a/terraform/backup-infrastructure/test/main.tf b/terraform/backup-infrastructure/test/main.tf index f32f1468f..260e66173 100644 --- a/terraform/backup-infrastructure/test/main.tf +++ b/terraform/backup-infrastructure/test/main.tf @@ -1,14 +1,32 @@ -#terraform { -# backend "s3" { -# bucket = "project-env-backup-tf-bucket" # change this to the destination account terraform state s3 bucket name -# key = "project-env-backup.tfstate" # change this to the destination account terraform state s3 key name -# dynamodb_table = "project-env-backup-lock-table" # change this to the destination account terraform state dynamodb table name -# region = "eu-west-2" -# } -#} - - provider "aws" { - alias = "source" region = "eu-west-2" + + assume_role { + role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}" + } + + default_tags { + tags = { + project_name = local.project_name + workspace = terraform.workspace + } + } +} + +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.76.0" + } + } + + backend "s3" { + region = "eu-west-2" + bucket = "nhsd-nrlf--terraform-state" + dynamodb_table = "nhsd-nrlf--terraform-state-lock" + key = "terraform-state-dev-backup-infrastructure" + workspace_key_prefix = "nhsd-nrlf" + encrypt = false + } } diff --git a/terraform/backup-infrastructure/test/vars.tf b/terraform/backup-infrastructure/test/vars.tf index e6e55ff45..e091ee9c5 100644 --- a/terraform/backup-infrastructure/test/vars.tf +++ b/terraform/backup-infrastructure/test/vars.tf @@ -1,4 +1,15 @@ -variable "source_terraform_role_arn" { - description = "ARN of the terraform role in the source account" +variable "assume_account" { + description = "The account id to deploy the infrastructure to" + sensitive = true +} + +variable "assume_role" { + description = "Name of the role to assume to deploy the infrastructure" + type = string +} + +variable "source_account_id" { + description = "The account id of the backup source account" type = string + sensitive = true } From d6376e754b0b6192067c54703388f5a3780de711 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 22 Nov 2024 02:37:02 +0000 Subject: [PATCH 010/106] NRL-1075 split category by list for multi category search --- .../searchDocumentReference/search_document_reference.py | 2 +- .../search_post_document_reference.py | 2 +- .../searchDocumentReference/search_document_reference.py | 4 ++-- .../search_post_document_reference.py | 4 ++-- layer/nrlf/core/validators.py | 4 +++- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/api/consumer/searchDocumentReference/search_document_reference.py b/api/consumer/searchDocumentReference/search_document_reference.py index b420c7132..3638274f2 100644 --- a/api/consumer/searchDocumentReference/search_document_reference.py +++ b/api/consumer/searchDocumentReference/search_document_reference.py @@ -102,7 +102,7 @@ def handler( nhs_number=params.nhs_number, custodian=custodian_id, pointer_types=pointer_types, - categories=[params.category.root] if params.category else [], + categories=params.category.root.split(",") if params.category else [], ): try: document_reference = DocumentReference.model_validate_json(result.document) diff --git a/api/consumer/searchPostDocumentReference/search_post_document_reference.py b/api/consumer/searchPostDocumentReference/search_post_document_reference.py index 18ac8e606..c6c89033c 100644 --- a/api/consumer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/consumer/searchPostDocumentReference/search_post_document_reference.py @@ -105,7 +105,7 @@ def handler( nhs_number=body.nhs_number, custodian=custodian_id, pointer_types=pointer_types, - categories=[body.category.root] if body.category else [], + categories=body.category.root.split(",") if body.category else [], ): try: document_reference = DocumentReference.model_validate_json(result.document) diff --git a/api/producer/searchDocumentReference/search_document_reference.py b/api/producer/searchDocumentReference/search_document_reference.py index d2dad8fd5..c931f1fb5 100644 --- a/api/producer/searchDocumentReference/search_document_reference.py +++ b/api/producer/searchDocumentReference/search_document_reference.py @@ -78,7 +78,7 @@ def handler( custodian_suffix=metadata.ods_code_extension, nhs_number=params.nhs_number, pointer_types=pointer_types, - categories=[params.category.root] if params.category else [], + categories=params.category.root.split(",") if params.category else [], ) for result in repository.search( @@ -86,7 +86,7 @@ def handler( custodian_suffix=metadata.ods_code_extension, nhs_number=params.nhs_number, pointer_types=pointer_types, - categories=[params.category.root] if params.category else [], + categories=params.category.root.split(",") if params.category else [], ): try: document_reference = DocumentReference.model_validate_json(result.document) diff --git a/api/producer/searchPostDocumentReference/search_post_document_reference.py b/api/producer/searchPostDocumentReference/search_post_document_reference.py index 12c1144e1..b6d1cd264 100644 --- a/api/producer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/producer/searchPostDocumentReference/search_post_document_reference.py @@ -72,7 +72,7 @@ def handler( custodian_suffix=metadata.ods_code_extension, nhs_number=body.nhs_number, pointer_types=pointer_types, - categories=[body.category.root] if body.category else [], + categories=body.category.root.split(",") if body.category else [], ) for result in repository.search( @@ -80,7 +80,7 @@ def handler( custodian_suffix=metadata.ods_code_extension, nhs_number=body.nhs_number, pointer_types=pointer_types, - categories=[body.category.root] if body.category else [], + categories=body.category.root.split(",") if body.category else [], ): try: document_reference = DocumentReference.model_validate_json(result.document) diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 45627ccc7..b061a16fd 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -43,7 +43,9 @@ def validate_category(category_: Optional[RequestQueryCategory]) -> bool: if not category_: return True - return category_.root in Categories.list() + categories = category_.root.split(",") + + return all(category in Categories.list() for category in categories) @dataclass From 22e931b21dbaa8cd2e5252d82cdd68ae111246ca Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 22 Nov 2024 02:46:28 +0000 Subject: [PATCH 011/106] NRL-1075 less duplication --- .../searchDocumentReference/search_document_reference.py | 5 +++-- .../search_post_document_reference.py | 7 ++++--- .../searchDocumentReference/search_document_reference.py | 5 +++-- .../search_post_document_reference.py | 9 +++++---- layer/nrlf/core/validators.py | 6 ++---- 5 files changed, 17 insertions(+), 15 deletions(-) diff --git a/api/consumer/searchDocumentReference/search_document_reference.py b/api/consumer/searchDocumentReference/search_document_reference.py index 3638274f2..aee43c7ec 100644 --- a/api/consumer/searchDocumentReference/search_document_reference.py +++ b/api/consumer/searchDocumentReference/search_document_reference.py @@ -58,7 +58,8 @@ def handler( expression="type", ) - if not validate_category(params.category): + categories = params.category.root.split(",") if params.category else [] + if not validate_category(categories): logger.log( LogReference.CONSEARCH002b, category=params.category, @@ -102,7 +103,7 @@ def handler( nhs_number=params.nhs_number, custodian=custodian_id, pointer_types=pointer_types, - categories=params.category.root.split(",") if params.category else [], + categories=categories, ): try: document_reference = DocumentReference.model_validate_json(result.document) diff --git a/api/consumer/searchPostDocumentReference/search_post_document_reference.py b/api/consumer/searchPostDocumentReference/search_post_document_reference.py index c6c89033c..39964f608 100644 --- a/api/consumer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/consumer/searchPostDocumentReference/search_post_document_reference.py @@ -61,10 +61,11 @@ def handler( expression="type", ) - if not validate_category(body.category): + categories = body.category.root.split(",") if body.category else [] + if not validate_category(categories): logger.log( LogReference.CONPOSTSEARCH002b, - type=body.category, + category=body.category, ) # TODO - Should update error message once permissioning by category is implemented return SpineErrorResponse.INVALID_CODE_SYSTEM( diagnostics="The provided category is not valid", @@ -105,7 +106,7 @@ def handler( nhs_number=body.nhs_number, custodian=custodian_id, pointer_types=pointer_types, - categories=body.category.root.split(",") if body.category else [], + categories=categories, ): try: document_reference = DocumentReference.model_validate_json(result.document) diff --git a/api/producer/searchDocumentReference/search_document_reference.py b/api/producer/searchDocumentReference/search_document_reference.py index c931f1fb5..403d22b8e 100644 --- a/api/producer/searchDocumentReference/search_document_reference.py +++ b/api/producer/searchDocumentReference/search_document_reference.py @@ -59,10 +59,11 @@ def handler( expression="type", ) - if not validate_category(params.category): + categories = params.category.root.split(",") if params.category else [] + if not validate_category(categories): logger.log( LogReference.PROSEARCH002b, - type=params.category, + category=params.category, ) # TODO - Should update error message once permissioning by category is implemented return SpineErrorResponse.INVALID_CODE_SYSTEM( diagnostics="Invalid query parameter (The provided category is not valid)", diff --git a/api/producer/searchPostDocumentReference/search_post_document_reference.py b/api/producer/searchPostDocumentReference/search_post_document_reference.py index b6d1cd264..2c1159653 100644 --- a/api/producer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/producer/searchPostDocumentReference/search_post_document_reference.py @@ -53,10 +53,11 @@ def handler( expression="type", ) - if not validate_category(body.category): + categories = body.category.root.split(",") if body.category else [] + if not validate_category(categories): logger.log( LogReference.PROPOSTSEARCH002b, - type=body.category, + category=body.category, ) # TODO - Should update error message once permissioning by category is implemented return SpineErrorResponse.INVALID_CODE_SYSTEM( diagnostics="The provided category is not valid", @@ -72,7 +73,7 @@ def handler( custodian_suffix=metadata.ods_code_extension, nhs_number=body.nhs_number, pointer_types=pointer_types, - categories=body.category.root.split(",") if body.category else [], + categories=categories, ) for result in repository.search( @@ -80,7 +81,7 @@ def handler( custodian_suffix=metadata.ods_code_extension, nhs_number=body.nhs_number, pointer_types=pointer_types, - categories=body.category.root.split(",") if body.category else [], + categories=categories, ): try: document_reference = DocumentReference.model_validate_json(result.document) diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index b061a16fd..09f2c6370 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -36,15 +36,13 @@ def validate_type_system( # TODO - Validate category is in set permissions once permissioning by category is done. -def validate_category(category_: Optional[RequestQueryCategory]) -> bool: +def validate_category(categories: Optional[RequestQueryCategory]) -> bool: """ Validates if the given category is valid. """ - if not category_: + if not categories: return True - categories = category_.root.split(",") - return all(category in Categories.list() for category in categories) From bd0ce6b0a06140d5d6295a36fb7bb9d287b9f928 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Fri, 22 Nov 2024 14:44:20 +0000 Subject: [PATCH 012/106] NRL-853 fix tags for table and buckets --- .../modules/permissions-store-bucket/s3.tf | 6 +++++- .../modules/pointers-table/dynamodb.tf | 2 +- .../modules/truststore-bucket/s3.tf | 1 + 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf index 5e6b80440..0a41e073a 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf @@ -2,7 +2,11 @@ resource "aws_s3_bucket" "authorization-store" { bucket = "${var.name_prefix}-authorization-store" force_destroy = var.enable_bucket_force_destroy - tags = { + tags = var.enable_backups ? { + Name = "authorization store" + Environment = "${var.name_prefix}" + NHSE-Enable-Backup = "daily" + } : { Name = "authorization store" Environment = "${var.name_prefix}" } diff --git a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf index 4bb3745e1..0cdb15e8c 100644 --- a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf +++ b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf @@ -52,5 +52,5 @@ resource "aws_dynamodb_table" "pointers" { enabled = var.enable_pitr } - tags = var.enable_backups ? { NHSE-Enable-Backup : daily } : {} + tags = var.enable_backups ? { NHSE-Enable-Backup = "daily" } : {} } diff --git a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf index 6767ecaa5..1eac1f4fa 100644 --- a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf @@ -1,6 +1,7 @@ resource "aws_s3_bucket" "api_truststore" { bucket = "${var.name_prefix}-api-truststore" force_destroy = var.enable_bucket_force_destroy + tags = var.enable_backups ? { NHSE-Enable-Backup = "daily" } : {} } resource "aws_s3_bucket_policy" "api_truststore_bucket_policy" { From ef6a542705d65b680757d1ba6ecb3d95ea70384e Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Fri, 22 Nov 2024 15:17:39 +0000 Subject: [PATCH 013/106] NRL-853 fix tags to match policy --- .../modules/permissions-store-bucket/s3.tf | 2 +- .../modules/pointers-table/dynamodb.tf | 2 +- .../account-wide-infrastructure/modules/truststore-bucket/s3.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf index 0a41e073a..7454d13ac 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf @@ -5,7 +5,7 @@ resource "aws_s3_bucket" "authorization-store" { tags = var.enable_backups ? { Name = "authorization store" Environment = "${var.name_prefix}" - NHSE-Enable-Backup = "daily" + NHSE-Enable-Backup = "true" } : { Name = "authorization store" Environment = "${var.name_prefix}" diff --git a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf index 0cdb15e8c..19c9184cb 100644 --- a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf +++ b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf @@ -52,5 +52,5 @@ resource "aws_dynamodb_table" "pointers" { enabled = var.enable_pitr } - tags = var.enable_backups ? { NHSE-Enable-Backup = "daily" } : {} + tags = var.enable_backups ? { NHSE-Enable-Backup = "true" } : {} } diff --git a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf index 1eac1f4fa..a3d4f970b 100644 --- a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf @@ -1,7 +1,7 @@ resource "aws_s3_bucket" "api_truststore" { bucket = "${var.name_prefix}-api-truststore" force_destroy = var.enable_bucket_force_destroy - tags = var.enable_backups ? { NHSE-Enable-Backup = "daily" } : {} + tags = var.enable_backups ? { NHSE-Enable-Backup = "true" } : {} } resource "aws_s3_bucket_policy" "api_truststore_bucket_policy" { From dd524465af5e51d2f699273e895c910f1bd112d5 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 22 Nov 2024 15:39:09 +0000 Subject: [PATCH 014/106] NRL-1075 add tests --- ...test_search_document_reference_consumer.py | 77 ++++++++++++++- ...search_post_document_reference_consumer.py | 78 +++++++++++++++ ...test_search_document_reference_producer.py | 71 +++++++++++++- ...search_post_document_reference_producer.py | 73 +++++++++++++- .../searchDocumentReference-failure.feature | 28 ++++++ .../searchDocumentReference-success.feature | 95 +++++++++++++++++++ ...earchPostDocumentReference-failure.feature | 33 ++++++- ...earchPostDocumentReference-success.feature | 94 ++++++++++++++++++ 8 files changed, 543 insertions(+), 6 deletions(-) diff --git a/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py b/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py index 1b8cd23b1..1f30aafc5 100644 --- a/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py +++ b/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py @@ -3,6 +3,12 @@ from moto import mock_aws from api.consumer.searchDocumentReference.search_document_reference import handler +from nrlf.core.constants import ( + CATEGORY_ATTRIBUTES, + TYPE_ATTRIBUTES, + Categories, + PointerTypes, +) from nrlf.core.dynamodb.repository import DocumentPointer, DocumentPointerRepository from nrlf.tests.data import load_document_reference from nrlf.tests.dynamodb import mock_repository @@ -144,6 +150,19 @@ def test_search_document_reference_happy_path_with_category( doc_pointer = DocumentPointer.from_document_reference(doc_ref) repository.create(doc_pointer) + # Second pointer different category + doc_ref2 = load_document_reference("Y05868-736253002-Valid") + doc_ref2.id = "Y05868-736253002-Valid2" + doc_ref2.type.coding[0].code = PointerTypes.NEWS2_CHART.coding_value() + doc_ref2.type.coding[0].display = TYPE_ATTRIBUTES.get( + PointerTypes.NEWS2_CHART.value + ).get("display") + doc_ref2.category[0].coding[0].code = Categories.OBSERVATIONS.coding_value() + doc_ref2.category[0].coding[0].display = CATEGORY_ATTRIBUTES.get( + Categories.OBSERVATIONS.value + ).get("display") + repository.create(DocumentPointer.from_document_reference(doc_ref2)) + event = create_test_api_gateway_event( headers=create_headers(), query_string_parameters={ @@ -160,7 +179,6 @@ def test_search_document_reference_happy_path_with_category( "headers": default_response_headers(), "isBase64Encoded": False, } - parsed_body = json.loads(body) assert parsed_body == { "resourceType": "Bundle", @@ -176,6 +194,63 @@ def test_search_document_reference_happy_path_with_category( } +@mock_aws +@mock_repository +def test_search_document_reference_happy_path_with_multiple_categories( + repository: DocumentPointerRepository, +): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_pointer = DocumentPointer.from_document_reference(doc_ref) + repository.create(doc_pointer) + + # Second pointer different category + doc_ref2 = load_document_reference("Y05868-736253002-Valid") + doc_ref2.id = "Y05868-736253002-Valid2" + doc_ref2.type.coding[0].code = PointerTypes.NEWS2_CHART.coding_value() + doc_ref2.type.coding[0].display = TYPE_ATTRIBUTES.get( + PointerTypes.NEWS2_CHART.value + ).get("display") + doc_ref2.category[0].coding[0].code = Categories.OBSERVATIONS.coding_value() + doc_ref2.category[0].coding[0].display = CATEGORY_ATTRIBUTES.get( + Categories.OBSERVATIONS.value + ).get("display") + repository.create(DocumentPointer.from_document_reference(doc_ref2)) + + event = create_test_api_gateway_event( + headers=create_headers(), + query_string_parameters={ + "subject:identifier": "https://fhir.nhs.uk/Id/nhs-number|6700028191", + "category": "http://snomed.info/sct|734163000,http://snomed.info/sct|1102421000000108", + }, + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "200", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "Bundle", + "type": "searchset", + "link": [ + { + "relation": "self", + "url": "https://pytest.api.service.nhs.uk/record-locator/consumer/FHIR/R4/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|6700028191&category=http://snomed.info/sct|734163000,http://snomed.info/sct|1102421000000108", + } + ], + "total": 2, + "entry": [ + {"resource": doc_ref2.model_dump(exclude_none=True)}, + {"resource": doc_ref.model_dump(exclude_none=True)}, + ], + } + + @mock_aws @mock_repository def test_search_document_reference_happy_path_with_nicip_type( diff --git a/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py b/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py index 3dc41ba45..4e3ed4e4d 100644 --- a/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py +++ b/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py @@ -5,6 +5,12 @@ from api.consumer.searchPostDocumentReference.search_post_document_reference import ( handler, ) +from nrlf.core.constants import ( + CATEGORY_ATTRIBUTES, + TYPE_ATTRIBUTES, + Categories, + PointerTypes, +) from nrlf.core.dynamodb.repository import DocumentPointer, DocumentPointerRepository from nrlf.tests.data import load_document_reference from nrlf.tests.dynamodb import mock_repository @@ -153,6 +159,19 @@ def test_search_post_document_reference_happy_path_with_category( doc_pointer = DocumentPointer.from_document_reference(doc_ref) repository.create(doc_pointer) + # Second pointer different category + doc_ref2 = load_document_reference("Y05868-736253002-Valid") + doc_ref2.id = "Y05868-736253002-Valid2" + doc_ref2.type.coding[0].code = PointerTypes.NEWS2_CHART.coding_value() + doc_ref2.type.coding[0].display = TYPE_ATTRIBUTES.get( + PointerTypes.NEWS2_CHART.value + ).get("display") + doc_ref2.category[0].coding[0].code = Categories.OBSERVATIONS.coding_value() + doc_ref2.category[0].coding[0].display = CATEGORY_ATTRIBUTES.get( + Categories.OBSERVATIONS.value + ).get("display") + repository.create(DocumentPointer.from_document_reference(doc_ref2)) + event = create_test_api_gateway_event( headers=create_headers(), body=json.dumps( @@ -187,6 +206,65 @@ def test_search_post_document_reference_happy_path_with_category( } +@mock_aws +@mock_repository +def test_search_post_document_reference_happy_path_with_multiple_categories( + repository: DocumentPointerRepository, +): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_pointer = DocumentPointer.from_document_reference(doc_ref) + repository.create(doc_pointer) + + # Second pointer different category + doc_ref2 = load_document_reference("Y05868-736253002-Valid") + doc_ref2.id = "Y05868-736253002-Valid2" + doc_ref2.type.coding[0].code = PointerTypes.NEWS2_CHART.coding_value() + doc_ref2.type.coding[0].display = TYPE_ATTRIBUTES.get( + PointerTypes.NEWS2_CHART.value + ).get("display") + doc_ref2.category[0].coding[0].code = Categories.OBSERVATIONS.coding_value() + doc_ref2.category[0].coding[0].display = CATEGORY_ATTRIBUTES.get( + Categories.OBSERVATIONS.value + ).get("display") + repository.create(DocumentPointer.from_document_reference(doc_ref2)) + + event = create_test_api_gateway_event( + headers=create_headers(), + body=json.dumps( + { + "subject:identifier": "https://fhir.nhs.uk/Id/nhs-number|6700028191", + "category": "http://snomed.info/sct|734163000,http://snomed.info/sct|1102421000000108", + } + ), + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "200", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "Bundle", + "type": "searchset", + "link": [ + { + "relation": "self", + "url": "https://pytest.api.service.nhs.uk/record-locator/consumer/FHIR/R4/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|6700028191&category=http://snomed.info/sct|734163000,http://snomed.info/sct|1102421000000108", + } + ], + "total": 2, + "entry": [ + {"resource": doc_ref2.model_dump(exclude_none=True)}, + {"resource": doc_ref.model_dump(exclude_none=True)}, + ], + } + + @mock_aws @mock_repository def test_search_document_reference_no_results(repository: DocumentPointerRepository): diff --git a/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py b/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py index 7f3b89165..d7a46ece7 100644 --- a/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py +++ b/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py @@ -3,7 +3,12 @@ from moto import mock_aws from api.producer.searchDocumentReference.search_document_reference import handler -from nrlf.core.constants import Categories, PointerTypes +from nrlf.core.constants import ( + CATEGORY_ATTRIBUTES, + TYPE_ATTRIBUTES, + Categories, + PointerTypes, +) from nrlf.core.dynamodb.repository import DocumentPointer, DocumentPointerRepository from nrlf.tests.data import load_document_reference from nrlf.tests.dynamodb import mock_repository @@ -326,6 +331,19 @@ def test_search_document_reference_filters_by_category( doc_pointer = DocumentPointer.from_document_reference(doc_ref) repository.create(doc_pointer) + # Second pointer different category + doc_ref2 = load_document_reference("Y05868-736253002-Valid") + doc_ref2.id = "Y05868-736253002-Valid2" + doc_ref2.type.coding[0].code = PointerTypes.NEWS2_CHART.coding_value() + doc_ref2.type.coding[0].display = TYPE_ATTRIBUTES.get( + PointerTypes.NEWS2_CHART.value + ).get("display") + doc_ref2.category[0].coding[0].code = Categories.OBSERVATIONS.coding_value() + doc_ref2.category[0].coding[0].display = CATEGORY_ATTRIBUTES.get( + Categories.OBSERVATIONS.value + ).get("display") + repository.create(DocumentPointer.from_document_reference(doc_ref2)) + event = create_test_api_gateway_event( headers=create_headers(), query_string_parameters={ @@ -352,6 +370,57 @@ def test_search_document_reference_filters_by_category( } +@mock_aws +@mock_repository +def test_search_document_reference_filters_with_multiple_categories( + repository: DocumentPointerRepository, +): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_pointer = DocumentPointer.from_document_reference(doc_ref) + repository.create(doc_pointer) + + # Second pointer different category + doc_ref2 = load_document_reference("Y05868-736253002-Valid") + doc_ref2.id = "Y05868-736253002-Valid2" + doc_ref2.type.coding[0].code = PointerTypes.NEWS2_CHART.coding_value() + doc_ref2.type.coding[0].display = TYPE_ATTRIBUTES.get( + PointerTypes.NEWS2_CHART.value + ).get("display") + doc_ref2.category[0].coding[0].code = Categories.OBSERVATIONS.coding_value() + doc_ref2.category[0].coding[0].display = CATEGORY_ATTRIBUTES.get( + Categories.OBSERVATIONS.value + ).get("display") + repository.create(DocumentPointer.from_document_reference(doc_ref2)) + + event = create_test_api_gateway_event( + headers=create_headers(), + query_string_parameters={ + "subject:identifier": "https://fhir.nhs.uk/Id/nhs-number|6700028191", + "category": "http://snomed.info/sct|734163000,http://snomed.info/sct|1102421000000108", + }, + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "200", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "Bundle", + "type": "searchset", + "total": 2, + "entry": [ + {"resource": doc_ref2.model_dump(exclude_none=True)}, + {"resource": doc_ref.model_dump(exclude_none=True)}, + ], + } + + @mock_aws @mock_repository def test_search_document_reference_filters_by_pointer_types( diff --git a/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py b/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py index 66bd579a8..f17de3fb1 100644 --- a/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py +++ b/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py @@ -5,7 +5,12 @@ from api.producer.searchPostDocumentReference.search_post_document_reference import ( handler, ) -from nrlf.core.constants import Categories, PointerTypes +from nrlf.core.constants import ( + CATEGORY_ATTRIBUTES, + TYPE_ATTRIBUTES, + Categories, + PointerTypes, +) from nrlf.core.dynamodb.repository import DocumentPointer, DocumentPointerRepository from nrlf.tests.data import load_document_reference from nrlf.tests.dynamodb import mock_repository @@ -337,6 +342,19 @@ def test_search_document_reference_filters_by_category( doc_pointer = DocumentPointer.from_document_reference(doc_ref) repository.create(doc_pointer) + # Second pointer different category + doc_ref2 = load_document_reference("Y05868-736253002-Valid") + doc_ref2.id = "Y05868-736253002-Valid2" + doc_ref2.type.coding[0].code = PointerTypes.NEWS2_CHART.coding_value() + doc_ref2.type.coding[0].display = TYPE_ATTRIBUTES.get( + PointerTypes.NEWS2_CHART.value + ).get("display") + doc_ref2.category[0].coding[0].code = Categories.OBSERVATIONS.coding_value() + doc_ref2.category[0].coding[0].display = CATEGORY_ATTRIBUTES.get( + Categories.OBSERVATIONS.value + ).get("display") + repository.create(DocumentPointer.from_document_reference(doc_ref2)) + event = create_test_api_gateway_event( headers=create_headers(), body=json.dumps( @@ -365,6 +383,59 @@ def test_search_document_reference_filters_by_category( } +@mock_aws +@mock_repository +def test_search_post_document_reference_filters_with_multiple_categories( + repository: DocumentPointerRepository, +): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_pointer = DocumentPointer.from_document_reference(doc_ref) + repository.create(doc_pointer) + + # Second pointer different category + doc_ref2 = load_document_reference("Y05868-736253002-Valid") + doc_ref2.id = "Y05868-736253002-Valid2" + doc_ref2.type.coding[0].code = PointerTypes.NEWS2_CHART.coding_value() + doc_ref2.type.coding[0].display = TYPE_ATTRIBUTES.get( + PointerTypes.NEWS2_CHART.value + ).get("display") + doc_ref2.category[0].coding[0].code = Categories.OBSERVATIONS.coding_value() + doc_ref2.category[0].coding[0].display = CATEGORY_ATTRIBUTES.get( + Categories.OBSERVATIONS.value + ).get("display") + repository.create(DocumentPointer.from_document_reference(doc_ref2)) + + event = create_test_api_gateway_event( + headers=create_headers(), + body=json.dumps( + { + "subject:identifier": "https://fhir.nhs.uk/Id/nhs-number|6700028191", + "category": "http://snomed.info/sct|734163000,http://snomed.info/sct|1102421000000108", + } + ), + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "200", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "Bundle", + "type": "searchset", + "total": 2, + "entry": [ + {"resource": doc_ref2.model_dump(exclude_none=True)}, + {"resource": doc_ref.model_dump(exclude_none=True)}, + ], + } + + @mock_aws @mock_repository def test_search_document_reference_filters_by_pointer_types( diff --git a/tests/features/consumer/searchDocumentReference-failure.feature b/tests/features/consumer/searchDocumentReference-failure.feature index 8ed4300e6..1de20623a 100644 --- a/tests/features/consumer/searchDocumentReference-failure.feature +++ b/tests/features/consumer/searchDocumentReference-failure.feature @@ -219,3 +219,31 @@ Feature: Consumer - searchDocumentReference - Failure Scenarios "expression": ["category"] } """ + + Scenario: Search rejects request with multiple categories and one invalid category + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When consumer 'RX898' searches for DocumentReferences with parameters: + | parameter | value | + | subject | 9278693472 | + | category | http://snomed.info/sct\|734163000,http://snomed.info/sct\|invalid | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "code-invalid", + "details": { + "coding": [{ + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_CODE_SYSTEM", + "display": "Invalid code system" + }] + }, + "diagnostics": "Invalid query parameter (The provided category is not valid)", + "expression": ["category"] + } + """ diff --git a/tests/features/consumer/searchDocumentReference-success.feature b/tests/features/consumer/searchDocumentReference-success.feature index 960dfb823..d94cbb9d9 100644 --- a/tests/features/consumer/searchDocumentReference-success.feature +++ b/tests/features/consumer/searchDocumentReference-success.feature @@ -343,6 +343,101 @@ Feature: Consumer - searchDocumentReference - Success Scenarios | author | 02V | And the Bundle does not contain a DocumentReference with ID '02V-1111111111-SearchMultipleRefTest3' + Scenario: Search for multiple DocumentReferences by NHS number and Multiple Categories + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 824321000000109 | + And a DocumentReference resource exists with values: + | property | value | + | id | 02V-1111111111-SearchMultipleRefTest1 | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc-1.pdf | + | custodian | 02V | + | author | 02V | + And a DocumentReference resource exists with values: + | property | value | + | id | 02V-1111111111-SearchMultipleRefTest2 | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc-2.pdf | + | custodian | 02V | + | author | 02V | + And a DocumentReference resource exists with values: + | property | value | + | id | 02V-1111111111-SearchMultipleRefTest3 | + | subject | 9278693472 | + | status | current | + | type | 1363501000000100 | + | category | 1102421000000108 | + | contentType | application/pdf | + | url | https://example.org/my-doc-3.pdf | + | custodian | 02V | + | author | 02V | + And a DocumentReference resource exists with values: + | property | value | + | id | 02V-1111111111-SearchMultipleRefTest4 | + | subject | 9278693472 | + | status | current | + | type | 824321000000109 | + | category | 823651000000106 | + | contentType | application/pdf | + | url | https://example.org/my-doc-4.pdf | + | custodian | 02V | + | author | 02V | + When consumer 'RX898' searches for DocumentReferences with parameters: + | parameter | value | + | subject | 9278693472 | + | category | http://snomed.info/sct\|734163000,http://snomed.info/sct\|823651000000106 | + Then the response status code is 200 + And the response is a searchset Bundle + And the Bundle has a self link matching 'DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|9278693472&category=http://snomed.info/sct|734163000,http://snomed.info/sct\|823651000000106' + And the Bundle has a total of 3 + And the Bundle has 3 entries + And the Bundle contains an DocumentReference with values + | property | value | + | id | 02V-1111111111-SearchMultipleRefTest1 | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc-1.pdf | + | custodian | 02V | + | author | 02V | + And the Bundle contains an DocumentReference with values + | property | value | + | id | 02V-1111111111-SearchMultipleRefTest2 | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc-2.pdf | + | custodian | 02V | + | author | 02V | + And the Bundle contains an DocumentReference with values + | property | value | + | id | 02V-1111111111-SearchMultipleRefTest4 | + | subject | 9278693472 | + | status | current | + | type | 824321000000109 | + | category | 823651000000106 | + | contentType | application/pdf | + | url | https://example.org/my-doc-4.pdf | + | custodian | 02V | + | author | 02V | + And the Bundle does not contain a DocumentReference with ID '02V-1111111111-SearchMultipleRefTest3' + # No pointers found - done # Pointers exist but no permissions - covered in failure scenarios # Search by custodian - done diff --git a/tests/features/consumer/searchPostDocumentReference-failure.feature b/tests/features/consumer/searchPostDocumentReference-failure.feature index 8f5b507bd..91380c764 100644 --- a/tests/features/consumer/searchPostDocumentReference-failure.feature +++ b/tests/features/consumer/searchPostDocumentReference-failure.feature @@ -195,9 +195,8 @@ Feature: Consumer - searchDocumentReference - Failure Scenarios Scenario: Search rejects request with category system they are not allowed to use Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'RX898' is authorised to access pointer types: - | system | value | - | http://snomed.info/sct | 736253002 | - | http://snomed.info/sct | 1363501000000100 | + | system | value | + | http://snomed.info/sct | 736253002 | When consumer 'RX898' searches for DocumentReferences using POST with request body: | key | value | | subject | 9278693472 | @@ -220,3 +219,31 @@ Feature: Consumer - searchDocumentReference - Failure Scenarios "expression": ["category"] } """ + + Scenario: Search rejects request with multiple categories and one that is invalid + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When consumer 'RX898' searches for DocumentReferences using POST with request body: + | key | value | + | subject | 9278693472 | + | category | http://snomed.info/sct\|734163000,http://snomed.info/sct\|invalid | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "code-invalid", + "details": { + "coding": [{ + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_CODE_SYSTEM", + "display": "Invalid code system" + }] + }, + "diagnostics": "The provided category is not valid", + "expression": ["category"] + } + """ diff --git a/tests/features/consumer/searchPostDocumentReference-success.feature b/tests/features/consumer/searchPostDocumentReference-success.feature index 27d3fff18..d080e8f93 100644 --- a/tests/features/consumer/searchPostDocumentReference-success.feature +++ b/tests/features/consumer/searchPostDocumentReference-success.feature @@ -313,3 +313,97 @@ Feature: Consumer - searchDocumentReference - Success Scenarios | custodian | X26 | | author | X26 | And the Bundle does not contain a DocumentReference with ID 'x26-1111111111-SearchMultipleRefTest3' + + Scenario: Search for multiple DocumentReferences by NHS number and Multiple Categories + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 824321000000109 | + And a DocumentReference resource exists with values: + | property | value | + | id | X26-1111111111-SearchMultipleRefTest1 | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc-1.pdf | + | custodian | X26 | + | author | X26 | + And a DocumentReference resource exists with values: + | property | value | + | id | X26-1111111111-SearchMultipleRefTest2 | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc-2.pdf | + | custodian | X26 | + | author | X26 | + And a DocumentReference resource exists with values: + | property | value | + | id | x26-1111111111-SearchMultipleRefTest3 | + | subject | 9278693472 | + | status | current | + | type | 1363501000000100 | + | category | 1102421000000108 | + | contentType | application/pdf | + | url | https://example.org/my-doc-3.pdf | + | custodian | x26 | + | author | x26 | + And a DocumentReference resource exists with values: + | property | value | + | id | X26-1111111111-SearchMultipleRefTest4 | + | subject | 9278693472 | + | status | current | + | type | 824321000000109 | + | category | 823651000000106 | + | contentType | application/pdf | + | url | https://example.org/my-doc-4.pdf | + | custodian | X26 | + | author | X26 | + When consumer 'RX898' searches for DocumentReferences using POST with request body: + | key | value | + | subject | 9278693472 | + | category | http://snomed.info/sct\|734163000,http://snomed.info/sct\|823651000000106 | + Then the response status code is 200 + And the response is a searchset Bundle + And the Bundle has a total of 3 + And the Bundle has 3 entries + And the Bundle contains an DocumentReference with values + | property | value | + | id | X26-1111111111-SearchMultipleRefTest1 | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc-1.pdf | + | custodian | X26 | + | author | X26 | + And the Bundle contains an DocumentReference with values + | property | value | + | id | X26-1111111111-SearchMultipleRefTest2 | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc-2.pdf | + | custodian | X26 | + | author | X26 | + And the Bundle contains an DocumentReference with values + | property | value | + | id | X26-1111111111-SearchMultipleRefTest4 | + | subject | 9278693472 | + | status | current | + | type | 824321000000109 | + | category | 823651000000106 | + | contentType | application/pdf | + | url | https://example.org/my-doc-4.pdf | + | custodian | X26 | + | author | X26 | + And the Bundle does not contain a DocumentReference with ID 'x26-1111111111-SearchMultipleRefTest3' From 1d715a2b367f6683f4a8e6156770a37122522a77 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Fri, 22 Nov 2024 15:45:47 +0000 Subject: [PATCH 015/106] NRL-853 split back into different plans for s3 and ddb --- .../dev/aws-backups.tf | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backups.tf b/terraform/account-wide-infrastructure/dev/aws-backups.tf index 8dd30b4e0..6bc45623d 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backups.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backups.tf @@ -150,7 +150,7 @@ module "source" { backup_plan_config = { "compliance_resource_types" : [ - "S3", "DynamoDB" + "S3" ], "rules" : [ { @@ -166,4 +166,24 @@ module "source" { ], "selection_tag" : "NHSE-Enable-Backup" } + + backup_plan_config_dynamodb = { + "compliance_resource_types" : [ + "DynamoDB" + ], + "enable" : true, + "rules" : [ + { + "copy_action" : { + "delete_after" : 4 + }, + "lifecycle" : { + "delete_after" : 2 + }, + "name" : "daily_kept_for_2_days", + "schedule" : "cron(0 0 * * ? *)" + } + ], + "selection_tag" : "NHSE-Enable-Backup" + } } From 179eeb98454102c6b88fba3d34ee46db22a7598e Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Fri, 22 Nov 2024 15:47:03 +0000 Subject: [PATCH 016/106] NRL-853 split back into different plans for s3 and ddb --- .../modules/backup-source/backup_plan.tf | 2 +- .../modules/backup-source/variables.tf | 88 +------------------ 2 files changed, 2 insertions(+), 88 deletions(-) diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf index 4d2cf5066..b5ac3c1df 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf @@ -80,6 +80,6 @@ resource "aws_backup_selection" "dynamodb" { selection_tag { key = var.backup_plan_config_dynamodb.selection_tag type = "STRINGEQUALS" - value = "True" + value = "true" } } diff --git a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf index a76f7a4d2..88acbfd19 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf @@ -89,56 +89,7 @@ variable "backup_plan_config" { })) })) }) - default = { - selection_tag = "BackupLocal" - compliance_resource_types = ["S3"] - rules = [ - { - name = "daily_kept_5_weeks" - schedule = "cron(0 0 * * ? *)" - lifecycle = { - delete_after = 35 - } - copy_action = { - delete_after = 365 - } - }, - { - name = "weekly_kept_3_months" - schedule = "cron(0 1 ? * SUN *)" - lifecycle = { - delete_after = 90 - } - copy_action = { - delete_after = 365 - } - }, - { - name = "monthly_kept_7_years" - schedule = "cron(0 2 1 * ? *)" - lifecycle = { - cold_storage_after = 30 - delete_after = 2555 - } - copy_action = { - delete_after = 365 - } - }, - { - name = "point_in_time_recovery" - schedule = "cron(0 5 * * ? *)" - enable_continuous_backup = true - lifecycle = { - delete_after = 35 - } - copy_action = { - delete_after = 365 - } - } - ] - } } - variable "backup_plan_config_dynamodb" { description = "Configuration for backup plans with dynamodb" type = object({ @@ -158,42 +109,5 @@ variable "backup_plan_config_dynamodb" { })) }))) }) - default = { - enable = true - selection_tag = "BackupDynamoDB" - compliance_resource_types = ["DynamoDB"] - rules = [ - { - name = "dynamodb_daily_kept_5_weeks" - schedule = "cron(0 0 * * ? *)" - lifecycle = { - delete_after = 35 - } - copy_action = { - delete_after = 365 - } - }, - { - name = "dynamodb_weekly_kept_3_months" - schedule = "cron(0 1 ? * SUN *)" - lifecycle = { - delete_after = 90 - } - copy_action = { - delete_after = 365 - } - }, - { - name = "dynamodb_monthly_kept_7_years" - schedule = "cron(0 2 1 * ? *)" - lifecycle = { - cold_storage_after = 30 - delete_after = 2555 - } - copy_action = { - delete_after = 365 - } - } - ] - } + } From 29e4dae17ef87fd5ac93b664d8d43fdb5e1778ca Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 22 Nov 2024 15:56:16 +0000 Subject: [PATCH 017/106] NRL-1075 fix test --- tests/features/consumer/searchDocumentReference-success.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/features/consumer/searchDocumentReference-success.feature b/tests/features/consumer/searchDocumentReference-success.feature index d94cbb9d9..ead47a40e 100644 --- a/tests/features/consumer/searchDocumentReference-success.feature +++ b/tests/features/consumer/searchDocumentReference-success.feature @@ -400,7 +400,7 @@ Feature: Consumer - searchDocumentReference - Success Scenarios | category | http://snomed.info/sct\|734163000,http://snomed.info/sct\|823651000000106 | Then the response status code is 200 And the response is a searchset Bundle - And the Bundle has a self link matching 'DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|9278693472&category=http://snomed.info/sct|734163000,http://snomed.info/sct\|823651000000106' + And the Bundle has a self link matching 'DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|9278693472&category=http://snomed.info/sct|734163000,http://snomed.info/sct|823651000000106' And the Bundle has a total of 3 And the Bundle has 3 entries And the Bundle contains an DocumentReference with values From aae1596ccb92309643ed328d301ffc42d1d8a828 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Mon, 25 Nov 2024 14:38:59 +0000 Subject: [PATCH 018/106] [NRL-1192] Add LICENSE file with MIT license --- LICENSE | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..4388d2899 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Crown Copyright + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. From 8de586c0d161bc6f3c7fb2dddc0f2fffb93a1db3 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Tue, 26 Nov 2024 02:29:07 +0000 Subject: [PATCH 019/106] NRL-519 add practice setting validator --- layer/nrlf/core/constants.py | 472 ++++++++++++++++++++++++ layer/nrlf/core/validators.py | 73 +++- terraform/infrastructure/consumer.tftpl | 4 +- terraform/infrastructure/producer.tftpl | 10 +- 4 files changed, 554 insertions(+), 5 deletions(-) diff --git a/layer/nrlf/core/constants.py b/layer/nrlf/core/constants.py index 650547fb9..9ba8e1ffe 100644 --- a/layer/nrlf/core/constants.py +++ b/layer/nrlf/core/constants.py @@ -25,6 +25,7 @@ class Source(Enum): KEY_SEPARATOR = "#" ODS_SYSTEM = "https://fhir.nhs.uk/Id/ods-organization-code" NHS_NUMBER_SYSTEM_URL = "https://fhir.nhs.uk/Id/nhs-number" +SNOMED_SYSTEM_URL = "http://snomed.info/sct" RELATES_TO_REPLACES = "replaces" ALLOWED_RELATES_TO_CODES = { RELATES_TO_REPLACES, @@ -138,5 +139,476 @@ def coding_value(self): PointerTypes.MRI_AXILLA_BOTH.value: Categories.DIAGNOSTIC_PROCEDURE.value, } +PRACTICE_SETTING_VALUE_SET_URL = ( + "https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting" +) +SNOMED_PRACTICE_SETTINGS = { + "2471000175109": "Employee health service", + "828331000000102": "Homeopathy service", + "893041000000108": "Transient ischaemic attack service", + "893391000000101": "Adult cystic fibrosis service", + "828811000000104": "Child psychiatry service", + "741073001": "Neonatal intensive care service", + "3801000175108": "Pediatric pulmonology service", + "893521000000108": "Respiratory physiology service", + "892801000000107": "Audiological medicine service", + "892571000000101": "Medical oncology service", + "893421000000107": "Tropical medicine service", + "1060971000000108": "General practice service", + "907271000000106": "Genetics laboratory service", + "224891009": "Healthcare services", + "893771000000104": "Paediatric respiratory medicine service", + "893591000000106": "Paediatric metabolic disease service", + "892771000000109": "Cardiothoracic transplantation service", + "92151000000102": "Mental health crisis resolution team", + "893141000000109": "Nephrology service", + "829981000000108": "Community child health service", + "733459009": "Cardiac rehabilitation service", + "893971000000101": "Paediatric burns care service", + "931851000000100": "Oral pathology service", + "3771000175106": "Pediatric gastroenterology service", + "893621000000109": "Paediatric ear nose and throat service", + "828281000000107": "Eating disorders service", + "893121000000102": "Spinal surgery service", + "892751000000100": "Clinical immunology and allergy service", + "893601000000100": "Paediatric medical oncology service", + "3751000175100": "Pediatric emergency medical service", + "893951000000105": "Paediatric cardiology service", + "931831000000107": "Oral medicine service", + "829961000000104": "Out of hours service", + "911381000000108": "Telehealthcare service", + "828181000000101": "Community sexual and reproductive health", + "2451000175103": "Perinatology service", + "893851000000100": "Paediatric neurosurgery service", + "109201000000109": "Substance misuse team", + "893671000000108": "Paediatric urology service", + "932271000000104": "Oral and maxillofacial surgery service", + "444933003": "Home hospice service", + "444913002": "Diabetes mellitus service", + "892601000000108": "Intermediate care service", + "893801000000101": "Paediatric ophthalmology service", + "911231000000103": "Remote health monitoring service", + "893091000000103": "Infectious diseases service", + "893221000000106": "Specialist rehabilitation service", + "828381000000103": "Well woman service", + "907301000000109": "National Health Service 111 service", + "828511000000102": "National Health Service 24", + "893701000000107": "Paediatric thoracic surgery service", + "828861000000102": "Programmed pulmonary rehabilitation service", + "931781000000102": "Acute medicine service", + "827641000000101": "Anticoagulant service", + "893201000000102": "Sport and exercise medicine service", + "278032008": "Preventive service", + "892731000000107": "Dental medicine service", + "893451000000102": "Respite care service", + "2351000175106": "Sports medicine service", + "893271000000105": "Medical virology service", + "708168004": "Mental health service", + "708169007": "Respiratory therapy service", + "708171007": "Vascular ultrasound service", + "708170008": "Nursing service", + "708173005": "Obstetric ultrasound service", + "708172000": "Cardiac ultrasound service", + "708175003": "Diagnostic imaging service", + "708178001": "Cytogenetics service", + "708174004": "Interventional radiology service", + "708179009": "Molecular pathology service", + "708183009": "Anatomic pathology service", + "708182004": "Histology service", + "708180007": "Dermatopathology service", + "708187005": "Surgical pathology service", + "708185002": "Virology service", + "708184003": "Clinical pathology service", + "708188000": "Serology service", + "708194008": "Blood bank service", + "708196005": "Hematology service", + "708191000": "Toxicology service", + "708190004": "Immunology service", + "708193002": "Coagulation service", + "89301000000108": "Community mental health team", + "893301000000108": "Local specialist rehabilitation service", + "893651000000104": "Paediatric clinical immunology and allergy service", + "893171000000103": "Clinical neurophysiology service", + "711332004": "Allergy service", + "893151000000107": "Nuclear medicine service", + "894001000000107": "Paediatric audiological medicine service", + "892781000000106": "Burns care service", + "3781000175109": "Pediatric infectious disease service", + "893631000000106": "Paediatric diabetic medicine service", + "827621000000108": "Addiction service", + "893531000000105": "Psychiatric intensive care service", + "893881000000106": "Paediatric maxillofacial surgery service", + "893051000000106": "Clinical allergy service", + "893351000000109": "Complex specialised rehabilitation service", + "893001000000105": "Clinical genetics service", + "908981000000101": "Remote triage and advice service", + "931811000000104": "Histopathology service", + "1079481000000104": "Perinatal psychiatry service", + "893251000000101": "Mental health recovery and rehabilitation service", + "3531000175102": "Geriatric service", + "736622005": "Aboriginal health service", + "983641000000106": "Fracture liaison service", + "893711000000109": "Neonatal critical care service", + "828521000000108": "National Health Service Direct", + "892761000000102": "Clinical haematology service", + "901221000000102": "Perinatal mental health service", + "706902008": "Mycology service", + "706901001": "Bacteriology service", + "706903003": "Mycobacteriology service", + "706900000": "Parasitology service", + "893131000000100": "Genitourinary medicine service", + "413294000": "Community health services", + "413299005": "Early years services", + "828291000000109": "Dispensing optometry service", + "413331009": "Voluntary services", + "61831000000105": "Periodontics service", + "893231000000108": "Podiatric surgery service", + "893581000000109": "Well baby service", + "893911000000106": "Paediatric gastrointestinal surgery service", + "395104009": "Cancer primary healthcare multidisciplinary team", + "892711000000104": "Gynaecological oncology service", + "893331000000102": "Dementia assessment service", + "892611000000105": "Hepatology service", + "893681000000105": "Paediatric trauma and orthopaedics service", + "395086005": "Community specialist palliative care", + "395092004": "Specialist palliative care", + "710028007": "Maxillofacial surgery service", + "892811000000109": "Adult mental health service", + "828821000000105": "Adolescent psychiatry service", + "983341000000102": "Pharmacy First service", + "893431000000109": "Trauma and orthopaedics service", + "893781000000102": "Paediatric plastic surgery service", + "892581000000104": "Learning disability service", + "893661000000101": "Paediatric clinical haematology service", + "893311000000105": "Haemophilia service", + "373654008": "Medical referral service", + "893081000000100": "Respiratory medicine service", + "911221000000100": "Remote care environment monitoring service", + "409971007": "Emergency medical services", + "828371000000100": "Well man service", + "963151000000104": "Diabetic medicine service", + "893761000000106": "Paediatric rheumatology service", + "932841000000106": "Public health dentistry service", + "893861000000102": "Paediatric neurodisability service", + "92191000000105": "Early intervention in psychosis team", + "722424008": "Physical medicine and rehabilitation service", + "89311000000105": "Crisis prevention assessment and treatment team", + "722393008": "Legal medicine service", + "722352000": "Vascular medicine service", + "722170006": "Chiropractic service", + "722174002": "Pulmonary medicine service", + "722175001": "Psychosomatic medicine service", + "722176000": "Dentistry service", + "722140001": "Physiotherapy service", + "892561000000108": "Medical ophthalmology service", + "714088003": "Midwifery service", + "714089006": "Community midwifery service", + "3761000175103": "Pediatric endocrinology service", + "893611000000103": "Paediatric epilepsy service", + "893961000000108": "Paediatric cardiac surgery service", + "931841000000103": "Oral microbiology service", + "91901000000109": "Assertive outreach team", + "828191000000104": "Dental hygiene service", + "893031000000104": "Clinical immunology service", + "893381000000103": "Clinical psychology service", + "2461000175101": "Pulmonary rehabilitation service", + "893161000000105": "Neurology service", + "828201000000102": "Dental surgery assistance service", + "1079491000000102": "Paediatric diabetes service", + "893261000000103": "Mental health dual diagnosis service", + "310031001": "Family planning service", + "310032008": "Intensive care service", + "310030000": "Endoscopy service", + "310034009": "Pediatric intensive care service", + "310033003": "Adult intensive care service", + "310025004": "Complementary therapy service", + "310024000": "Colposcopy service", + "310027007": "Mental health counseling service", + "310026003": "Counseling service", + "310029005": "Domiciliary visit service", + "310028002": "Diagnostic investigation service", + "310020009": "Hearing therapy service", + "310022001": "Clinical oncology service", + "310021008": "Assistive listening device service", + "310023006": "Radiotherapy service", + "310017001": "Pediatric hearing aid service", + "310016005": "Adult hearing aid service", + "310015009": "Hearing aid service", + "310014008": "Pediatric cochlear implant service", + "310013002": "Adult cochlear implant service", + "310012007": "Cochlear implant service", + "310011000": "Aural rehabilitation service", + "310010004": "Distraction test audiological screening service", + "310019003": "Tinnitus management service", + "310018006": "Speech-reading training service", + "310001007": "Anesthetic service", + "310000008": "Accident and Emergency service", + "310003005": "Child assessment service", + "310002000": "Assessment service", + "310008001": "Audiological screening service", + "310009009": "Neonatal audiological screening service", + "310005003": "Diagnostic audiology service", + "310004004": "Audiological service", + "310007006": "Pediatric diagnostic audiology service", + "310006002": "Adult diagnostic audiology service", + "310085001": "Drama therapy service", + "310086000": "Music therapy service", + "310083008": "Art therapy service", + "310082003": "Arts therapy services", + "310084002": "Dance therapy service", + "310089007": "Hospital-based podiatry service", + "310087009": "Podiatry service", + "310088004": "Community-based podiatry service", + "310080006": "Pharmacy service", + "310081005": "Professional allied to medicine service", + "310079008": "Neuropathology service", + "310078000": "Medical microbiology service", + "310071006": "Pain management service", + "310070007": "Special care baby service", + "310072004": "Acute pain service", + "310073009": "Palliative care service", + "310076001": "Clinical biochemistry service", + "310074003": "Pathology service", + "310064001": "Occupational health service", + "310063007": "Obstetrics service", + "310066004": "Pediatric service", + "310065000": "Open access service", + "310068003": "Pediatric neurology service", + "310067008": "Community pediatric service", + "310069006": "Pediatric oncology service", + "310061009": "Gynecology service", + "310062002": "Pregnancy termination service", + "310060005": "Obstetrics and gynecology service", + "310099002": "Child physiotherapy service", + "310098005": "Hospital-based physiotherapy service", + "310091004": "Community-based dietetics service", + "310090003": "Dietetics service", + "310096009": "Hospital-based occupational therapy service", + "310094007": "Community-based occupational therapy service", + "310095008": "Social services occupational therapy service", + "310093001": "Occupational therapy service", + "310092006": "Hospital-based dietetics service", + "310120006": "Mental handicap psychiatry service", + "310121005": "Psychogeriatric service", + "310126000": "Breast screening service", + "310128004": "Computerized tomography service", + "310127009": "Magnetic resonance imaging service", + "310129007": "Rehabilitation service", + "310122003": "Rehabilitation psychiatry service", + "310123008": "Psychology service", + "310124002": "Psychotherapy service", + "310125001": "Radiology service", + "310117003": "Child and adolescent psychiatry service", + "310116007": "Psychiatry service", + "310119000": "Liaison psychiatry service", + "310118008": "Forensic psychiatry service", + "310114005": "Community surgical fitting service", + "310112009": "Surgical fitting service", + "310115006": "Public health service", + "310113004": "Hospital surgical fitting service", + "310110001": "Hospital orthotics service", + "310111002": "Community orthotics service", + "310109006": "Orthotics service", + "310108003": "Community orthoptics service", + "310105000": "Optometry service", + "310107008": "Hospital orthoptics service", + "310106004": "Orthoptics service", + "310104001": "Child speech and language therapy service", + "310103007": "Hospital-based speech and language therapy service", + "310102002": "Community-based speech and language therapy service", + "310141000": "Thoracic surgery service", + "310143002": "Dental surgery service", + "310142007": "Cardiac surgery service", + "310144008": "General dental surgery service", + "310145009": "Oral surgery service", + "310146005": "Orthodontics service", + "310147001": "Pediatric dentistry service", + "310148006": "Restorative dentistry service", + "310149003": "Ear, nose and throat service", + "310140004": "Cardiothoracic surgery service", + "310131003": "Community rehabilitation service", + "310130002": "Head injury rehabilitation service", + "310134006": "Social services", + "310135007": "Social services department customer services", + "310132005": "Young disabled service", + "310133000": "Swallow clinic", + "310139001": "Breast surgery service", + "310138009": "Surgical service", + "310136008": "Social services department duty team", + "310137004": "Stroke service", + "310101009": "Speech and language therapy service", + "310100005": "Play therapy service", + "734862008": "Endodontic service", + "734863003": "Prosthodontic service", + "310168000": "Vascular surgery service", + "310169008": "Ultrasonography service", + "310165002": "Transplant surgery service", + "310167005": "Urology service", + "310166001": "Trauma surgery service", + "310163009": "Pediatric surgical service", + "310164003": "Plastic surgery service", + "310161006": "Orthopedic service", + "310162004": "Pancreatic surgery service", + "310160007": "Ophthalmology service", + "310151004": "Gastrointestinal surgery service", + "310152006": "General gastrointestinal surgery service", + "310153001": "Upper gastrointestinal surgery service", + "310155008": "Colorectal surgery service", + "310150003": "Endocrine surgery service", + "310157000": "Hand surgery service", + "310156009": "General surgical service", + "310159002": "Neurosurgical service", + "310158005": "Hepatobiliary surgical service", + "828301000000108": "Electrocardiography service", + "310200001": "Cytology service", + "734920002": "Diabetes mellitus education service", + "829951000000102": "Industrial therapy service", + "931821000000105": "School nursing service", + "893941000000107": "Paediatric dermatology service", + "892741000000103": "Clinical microbiology service", + "893211000000100": "Spinal injuries service", + "92221000000103": "Mental health home treatment team", + "3621000175101": "Rheumatology service", + "408451000": "Community learning disabilities team", + "408452007": "Behavioral intervention team", + "408458006": "Specialist multidisciplinary team", + "893341000000106": "Congenital heart disease service", + "445449000": "Acute care hospice service", + "1078501000000104": "Health visiting service", + "893691000000107": "Paediatric transplantation surgery service", + "892621000000104": "Hepatobiliary and pancreatic surgery service", + "2421000175108": "Acute care inpatient service", + "931801000000101": "Community nursing service", + "699478002": "Surgical oncology service", + "893791000000100": "Paediatric pain management service", + "893541000000101": "Prosthetics service", + "699650006": "Community based physiotherapy service", + "893891000000108": "Paediatric interventional radiology service", + "827631000000105": "Emergency ambulance service", + "827981000000103": "Paediatric cystic fibrosis service", + "892821000000103": "Critical care medicine service", + "700435004": "Clinical physiology service", + "700436003": "Clinical pharmacology service", + "700434000": "Endocrinology service", + "700221004": "Care of elderly service", + "700433006": "Gastroenterology service", + "700231006": "Critical care physician service", + "700232004": "General medical service", + "700241009": "Dermatology service", + "893061000000109": "Cardiology service", + "705150003": "Domiciliary physiotherapy service", + "932241000000105": "Blood banking and transfusion service", + "3791000175107": "Pediatric nephrology service", + "892791000000108": "Blood and marrow transplantation service", + "893641000000102": "Palliative medicine service", + "431051000124102": "Dialysis service", + "23951000087100": "Opioid dependence service", + "1323651000000109": "Cardiac physiology service", + "788126001": "Prosthetic service", + "788124003": "Histopathology service", + "788125002": "Addiction service", + "788123009": "Radiation oncology service", + "788122004": "Sexual health service", + "788128000": "Critical care medicine service", + "788127005": "Child health service", + "788121006": "Clinical immunology and allergy service", + "733921009": "Transplant medicine service", + "788001008": "Infectious disease service", + "788002001": "Adult mental health service", + "788003006": "Nephrology service", + "788009005": "Nuclear medicine service", + "788006003": "Genetic laboratory service", + "788004000": "Clinical genetics service", + "788005004": "Neurology service", + "788008002": "Oral and maxillofacial surgery service", + "788007007": "General practice service", + "1326391000000100": "FNP (Family Nurse Partnership) service", + "1186717003": "Intellectual disability psychiatry service", + "830149003": "Clinical neurophysiology service", + "224930009": "Services", + "830039004": "Genitourinary medicine service", + "830038007": "Clinical allergy service", + "830037002": "Clinical immunology service", + "1326421000000106": "Safeguarding children team", + "1323551000000105": "Inherited metabolic medicine service", + "28541000087101": "Musculoskeletal service", + "24271000087103": "Adult chronic pain management service", + "1240241000000109": "Community sexual and reproductive health service", + "897188002": "Pediatric hematology service", + "1323631000000102": "Aviation and space medicine service", + "1325831000000100": "Post-COVID-19 syndrome service", + "1323431000000104": "Fetal medicine service", + "24001000087103": "Paediatric plastic surgery service", + "24351000087104": "Paediatric chronic pain management service", + "1323881000000102": "Stroke medicine service", + "1323531000000103": "Urological physiology service", + "148621000000100": "School aged immunisation service", + "24101000087102": "HIV (human immunodeficiency virus) social work service", + "23911000087104": "Medication review service", + "1148679005": "Specialist palliative care service", + "773558007": "Physical medicine service", + "24051000087102": "Breast surgical oncology service", + "896974005": "Transgender health service", + "1323611000000105": "Paediatric inherited metabolic medicine service", + "1323661000000107": "Paediatric audiovestibular medicine service", + "24331000087108": "Narcotic addiction service with chronic pain management", + "1323561000000108": "Gastrointestinal physiology service", + "34911000087100": "Amputation care service", + "1231786003": "Refugee healthcare service", + "1163002007": "Electrocardiography service", + "1163004008": "Hyperbaric medicine service", + "1163003002": "Colorectal cancer screening service", + "1323841000000105": "Paediatric palliative medicine service", + "1163054002": "Gastroscopy service", + "1231392007": "Paediatric orthopaedic service", + "1231391000": "Colonoscopy service", + "1231393002": "Spirometry service", + "1231390004": "Hand therapy service", + "1231394008": "Paediatric urology service", + "24081000087105": "HIV (human immunodeficiency virus) nurse practitioner service", + "23941000087103": "Narcotic addiction service", + "1323641000000106": "Audiovestibular medicine service", + "840587001": "Aerospace medical service", + "840586005": "Neonatal service", + "840585009": "Postnatal service", + "816075004": "Prosthetic and orthotic service", + "1323921000000108": "Neuropsychiatry service", + "24011000087101": "Vascular imaging service", + "1323691000000101": "General internal medical service", + "23891000087102": "Adult hematology service", + "1323821000000103": "Paediatric clinical pharmacology service", + "789718008": "Cardiology service", + "789714005": "Pediatric rheumatology service", + "789715006": "Paediatric respiratory therapy service", + "789716007": "Pediatric otolaryngology service", + "789717003": "Paediatric cardiology service", + "148581000000100": "Personal health record provider service", + "792849008": "Pediatric clinical genetics service", + "792847005": "Emergency ambulance service", + "792848000": "Internal medicine service", + "1323621000000104": "Medical psychotherapy service", + "1323871000000104": "Rehabilitation medicine service", + "1230046007": "Cervical cancer screening service", + "1230045006": "Cardiac diagnostic service", + "1230044005": "Cardiac specialist nursing service", + "1362761000000103": "Adult safeguarding team", + "1323901000000104": "Rare disease service", + "1136421000168109": "Sleep medicine service", + "1323701000000101": "Vascular physiology service", + "24141000087104": "Spine orthopedic surgery service", + "1323571000000101": "Orthogeriatric medicine service", + "1323801000000107": "Paediatric oral and maxillofacial surgery service", + "23871000087101": "Adult dermatology service", + "1323601000000108": "Ophthalmic and vision science service", + "1234796008": "Community nursing service", + "23901000087101": "Hepatology service", + "1324191000000107": "Intensive care medicine service", + "2391000175104": "Bariatric surgery service", + "1323851000000108": "Paediatric hepatology service", + "24291000087104": "Geriatric chronic pain management service", + "1323501000000109": "Special care dentistry service", + "1423561000000102": "Acute oncology service", +} + SYSTEM_SHORT_IDS = {"http://snomed.info/sct": "SCT", "https://nicip.nhs.uk": "NICIP"} diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 0974e403e..b31c72599 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -5,7 +5,14 @@ from pydantic import ValidationError from nrlf.core.codes import SpineErrorConcept -from nrlf.core.constants import CATEGORY_ATTRIBUTES, ODS_SYSTEM, REQUIRED_CREATE_FIELDS +from nrlf.core.constants import ( + CATEGORY_ATTRIBUTES, + ODS_SYSTEM, + PRACTICE_SETTING_VALUE_SET_URL, + REQUIRED_CREATE_FIELDS, + SNOMED_PRACTICE_SETTINGS, + SNOMED_SYSTEM_URL, +) from nrlf.core.errors import ParseError from nrlf.core.logger import LogReference, logger from nrlf.core.types import DocumentReference, OperationOutcomeIssue, RequestQueryType @@ -510,3 +517,67 @@ def _validate_author(self, model: DocumentReference): field=f"author[0].identifier.value", ) return + + def _validate_practiceSetting(self, model: DocumentReference): + """ + Validate the practice setting field contains an appropriate coding system and code. + """ + + if not ( + practice_setting_coding := getattr( + model.context.practiceSetting, "coding", [] + ) + ): + self.result.add_error( + issue_code="invalid", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid practice setting: must contain a Coding", + field=f"context.practiceSetting.coding", + ) + return + + if len(practice_setting_coding) != 1: + self.result.add_error( + issue_code="invalid", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid practice setting coding length: {len(model.context.practiceSetting.coding)} Practice Setting Coding must only contain a single value", + field=f"context.practiceSetting.coding", + ) + return + + if ( + practice_setting_system := getattr( + practice_setting_coding[0], "system", None + ) + != SNOMED_SYSTEM_URL + ): + self.result.add_error( + issue_code="invalid", + error_code="INVALID_VALIDATION", + diagnostics=f"Invalid practice setting system: {practice_setting_system} Practice Setting system must be {SNOMED_SYSTEM_URL}", + field=f"context.practiceSetting.coding[0].system", + ) + return + + if ( + practice_setting_value := getattr(practice_setting_coding[0], "value", None) + not in SNOMED_PRACTICE_SETTINGS.keys() + ): + self.result.add_error( + issue_code="invalid", + error_code="INVALID_VALIDATION", + diagnostics=f"Invalid practice setting code: {practice_setting_value} Practice Setting coding must be a member of value set {PRACTICE_SETTING_VALUE_SET_URL}", + field=f"context.practiceSetting.coding[0].value", + ) + return + + if practice_setting_display := getattr( + practice_setting_coding[0], "display", None + ) != SNOMED_PRACTICE_SETTINGS.get(practice_setting_value): + self.result.add_error( + issue_code="invalid", + error_code="INVALID_VALIDATION", + diagnostics=f"Invalid practice setting coding: display {practice_setting_display} does not match the expected display for {practice_setting_value} Practice Setting coding is bound to value set {PRACTICE_SETTING_VALUE_SET_URL}", + field=f"context.practiceSetting.coding[0]", + ) + return diff --git a/terraform/infrastructure/consumer.tftpl b/terraform/infrastructure/consumer.tftpl index 15ee86292..363ead22f 100644 --- a/terraform/infrastructure/consumer.tftpl +++ b/terraform/infrastructure/consumer.tftpl @@ -4,7 +4,7 @@ "url": "https://${domain}/record-locator/consumer/FHIR/R4/metadata", "name": "NRLConsumerAPICapabilityStatement", "status": "active", - "version": "1.0.0", + "version": "1.0.1", "experimental": false, "date": "2024-03-13", "publisher": "NHS England", @@ -99,7 +99,7 @@ "name": "type", "definition": "http://hl7.org/fhir/SearchParameter/DocumentReference-type", "type": "token", - "documentation": "Allows DocumentReference search results to be filtered by pointer type (SNOMED code)." + "documentation": "Allows DocumentReference search results to be filtered by pointer type (SNOMED or NICIP code)." }, { "name": "subject", diff --git a/terraform/infrastructure/producer.tftpl b/terraform/infrastructure/producer.tftpl index d52749ebd..fafe92700 100644 --- a/terraform/infrastructure/producer.tftpl +++ b/terraform/infrastructure/producer.tftpl @@ -4,7 +4,7 @@ "url": "https://${domain}/record-locator/producer/FHIR/R4/metadata", "name": "NRLProducerAPICapabilityStatement", "status": "active", - "version": "1.0.0", + "version": "1.0.1", "experimental": false, "date": "2024-03-13", "publisher": "NHS England", @@ -98,7 +98,13 @@ "name": "type", "definition": "http://hl7.org/fhir/SearchParameter/DocumentReference-type", "type": "token", - "documentation": "Allows DocumentReference search results to be filtered by pointer type (SNOMED code)." + "documentation": "Allows DocumentReference search results to be filtered by pointer type (SNOMED or NICIP code)." + }, + { + "name": "category", + "definition": "http://hl7.org/fhir/SearchParameter/DocumentReference-category", + "type": "token", + "documentation": "Allows DocumentReference search results to be filtered by pointer category (SNOMED code)." }, { "name": "subject", From aed5ca615954881a2846f98ad0cfcf45eb3f5c61 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Fri, 29 Nov 2024 09:23:26 +0000 Subject: [PATCH 020/106] NRL-519 add unit tests --- layer/nrlf/core/tests/test_validators.py | 202 +++++++++++++++++++++++ layer/nrlf/core/validators.py | 38 ++--- tests/features/utils/constants.py | 4 +- 3 files changed, 223 insertions(+), 21 deletions(-) diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index fd02deb20..169e4af2c 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1522,3 +1522,205 @@ def test_validate_ssp_content_with_multiple_asids(): "diagnostics": "Multiple ASID identifiers provided. Only a single valid ASID identifier can be provided in the context.related.", "expression": ["context.related"], } + + +def test_validate_practiceSetting_no_coding(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["context"]["practiceSetting"] = { + "text": "Description of the clinic" + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid practice setting: must contain a Coding", + "expression": ["context.practiceSetting.coding"], + } + + +def test_validate_practiceSetting_coding_invalid_system(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["context"]["practiceSetting"] = { + "coding": [ + { + "system": "http://snoooooomed/sctfffffg", + "code": "788002001", + "display": "Adult mental health service", + } + ] + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid practice setting system: http://snoooooomed/sctfffffg Practice Setting system must be 'http://snomed.info/sct'", + "expression": ["context.practiceSetting.coding[0].system"], + } + + +def test_validate_practiceSetting_coding_invalid_code(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["context"]["practiceSetting"] = { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "123", + "display": "Adult mental health service", + } + ] + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid practice setting code: 123 Practice Setting coding must be a member of value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", + "expression": ["context.practiceSetting.coding[0].code"], + } + + +def test_validate_practiceSetting_coding_missing_code(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["context"]["practiceSetting"] = { + "coding": [ + { + "system": "http://snomed.info/sct", + "display": "Adult mental health service", + } + ] + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid practice setting code: None Practice Setting coding must be a member of value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", + "expression": ["context.practiceSetting.coding[0].code"], + } + + +def test_validate_practiceSetting_coding_missing_display(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["context"]["practiceSetting"] = { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "788002001", + } + ] + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid practice setting coding: display None does not match the expected display for 788002001 Practice Setting coding is bound to value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", + "expression": ["context.practiceSetting.coding[0]"], + } + + +def test_validate_practiceSetting_coding_mismatch_code_and_display(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["context"]["practiceSetting"] = { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "788002001", + "display": "Nephrology service", + } + ] + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid practice setting coding: display Nephrology service does not match the expected display for 788002001 Practice Setting coding is bound to value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", + "expression": ["context.practiceSetting.coding[0]"], + } diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index ae8346ee6..ca5f5f894 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -13,11 +13,10 @@ REQUIRED_CREATE_FIELDS, SNOMED_PRACTICE_SETTINGS, SNOMED_SYSTEM_URL, - REQUIRED_CREATE_FIELDS, TYPE_ATTRIBUTES, TYPE_CATEGORIES, Categories, - PointerTypes + PointerTypes, ) from nrlf.core.errors import ParseError from nrlf.core.logger import LogReference, logger @@ -145,6 +144,7 @@ def validate(self, data: Dict[str, Any] | DocumentReference): self._validate_category(resource) self._validate_author(resource) self._validate_type_category_mapping(resource) + self._validate_practiceSetting(resource) if resource.content[0].extension: self._validate_content_extension(resource) @@ -615,7 +615,7 @@ def _validate_practiceSetting(self, model: DocumentReference): ) ): self.result.add_error( - issue_code="invalid", + issue_code="value", error_code="INVALID_RESOURCE", diagnostics=f"Invalid practice setting: must contain a Coding", field=f"context.practiceSetting.coding", @@ -624,7 +624,7 @@ def _validate_practiceSetting(self, model: DocumentReference): if len(practice_setting_coding) != 1: self.result.add_error( - issue_code="invalid", + issue_code="value", error_code="INVALID_RESOURCE", diagnostics=f"Invalid practice setting coding length: {len(model.context.practiceSetting.coding)} Practice Setting Coding must only contain a single value", field=f"context.practiceSetting.coding", @@ -635,34 +635,34 @@ def _validate_practiceSetting(self, model: DocumentReference): practice_setting_system := getattr( practice_setting_coding[0], "system", None ) - != SNOMED_SYSTEM_URL - ): + ) != SNOMED_SYSTEM_URL: self.result.add_error( - issue_code="invalid", - error_code="INVALID_VALIDATION", - diagnostics=f"Invalid practice setting system: {practice_setting_system} Practice Setting system must be {SNOMED_SYSTEM_URL}", + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid practice setting system: {practice_setting_system} Practice Setting system must be '{SNOMED_SYSTEM_URL}'", field=f"context.practiceSetting.coding[0].system", ) return if ( - practice_setting_value := getattr(practice_setting_coding[0], "value", None) - not in SNOMED_PRACTICE_SETTINGS.keys() - ): + practice_setting_value := getattr(practice_setting_coding[0], "code", None) + ) not in SNOMED_PRACTICE_SETTINGS: self.result.add_error( - issue_code="invalid", - error_code="INVALID_VALIDATION", + issue_code="value", + error_code="INVALID_RESOURCE", diagnostics=f"Invalid practice setting code: {practice_setting_value} Practice Setting coding must be a member of value set {PRACTICE_SETTING_VALUE_SET_URL}", - field=f"context.practiceSetting.coding[0].value", + field=f"context.practiceSetting.coding[0].code", ) return - if practice_setting_display := getattr( - practice_setting_coding[0], "display", None + if ( + practice_setting_display := getattr( + practice_setting_coding[0], "display", None + ) ) != SNOMED_PRACTICE_SETTINGS.get(practice_setting_value): self.result.add_error( - issue_code="invalid", - error_code="INVALID_VALIDATION", + issue_code="value", + error_code="INVALID_RESOURCE", diagnostics=f"Invalid practice setting coding: display {practice_setting_display} does not match the expected display for {practice_setting_value} Practice Setting coding is bound to value set {PRACTICE_SETTING_VALUE_SET_URL}", field=f"context.practiceSetting.coding[0]", ) diff --git a/tests/features/utils/constants.py b/tests/features/utils/constants.py index 48228a74d..7c656824e 100644 --- a/tests/features/utils/constants.py +++ b/tests/features/utils/constants.py @@ -64,8 +64,8 @@ "coding": [ { "system": "http://snomed.info/sct", - "code": "390826005", - "display": "Mental health caregiver support" + "code": "788002001", + "display": "Adult mental health service" } ] }, From 94f08dbdc1f8de50c971d8238e09789cf3c1f3e6 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 29 Nov 2024 15:46:51 +0000 Subject: [PATCH 021/106] SPII-000 add context to smoke test setup --- tests/smoke/setup.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index e93a81aed..65bd065e2 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -5,6 +5,7 @@ Coding, DocumentReference, DocumentReferenceContent, + DocumentReferenceContext, DocumentReferenceRelatesTo, Identifier, Reference, @@ -67,6 +68,17 @@ def build_document_reference( ] ) ], + context=DocumentReferenceContext( + practiceSetting=CodeableConcept( + coding=[ + Coding( + system="http://snomed.info/sct", + code="390826005", + display="Mental health caregiver support", + ) + ] + ) + ), ) if replaces_id: From 7ec8278c2bb36bc18d1b5152cdf00c68ce48afac Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 29 Nov 2024 15:55:47 +0000 Subject: [PATCH 022/106] NRL-1213 add smoke tests to pr deploy workflow --- .github/workflows/pr-env-deploy.yml | 50 +++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index 755953b7b..23d1da44c 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -332,3 +332,53 @@ jobs: - name: Cleanup Environment Test Data run: make test-performance-cleanup TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} + + smoke-test: + name: Run Smoke Tests + needs: [set-environment-id, deploy] + environment: pull-request + runs-on: [self-hosted, ci] + + steps: + - name: Git Clone - ${{ github.event.pull_request.head.ref }} + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + + - name: Setup asdf cache + uses: actions/cache@v4 + with: + path: ~/.asdf + key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} + restore-keys: | + ${{ runner.os }}-asdf- + + - name: Install asdf and tools + uses: asdf-vm/actions/install@v3.0.2 + with: + asdf_branch: v0.13.1 + + - name: Python Dependency Install + run: poetry install --no-root + + - name: Configure Management Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-2 + role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} + role-session-name: github-actions-ci-${{ needs.set-environment-id.outputs.environment_id }} + + - name: Retrieve Client Certificates + run: make truststore-pull-client ENV=dev + + - name: Configure Dev Account Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-2 + role-chaining: true + role-to-assume: ${{ secrets.DEPLOY_ROLE_ARN }} + role-session-name: github-actions-ci-${{ needs.set-environment-id.outputs.environment_id }} + + - name: Smoke Test + run: | + make TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal From 23095a946927b84a9872ad5553f82773c71887df Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 29 Nov 2024 16:03:22 +0000 Subject: [PATCH 023/106] NRL-000 add mising type display --- tests/smoke/setup.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index 65bd065e2..a44ebe9bb 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -1,4 +1,4 @@ -from nrlf.core.constants import Categories, PointerTypes +from nrlf.core.constants import TYPE_ATTRIBUTES, Categories, PointerTypes from nrlf.producer.fhir.r4.model import ( Attachment, CodeableConcept, @@ -36,7 +36,15 @@ def build_document_reference( ) ], type=CodeableConcept( - coding=[Coding(system="http://snomed.info/sct", code=type)] + coding=[ + Coding( + system="http://snomed.info/sct", + code=type, + display=TYPE_ATTRIBUTES.get(f"http://snomed.info/sct|{type}").get( + "display" + ), + ) + ] ), subject=Reference( identifier=Identifier( From 08520f225e1237ad6cc864e29a6d270aba5e12c2 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 29 Nov 2024 16:07:30 +0000 Subject: [PATCH 024/106] NRL-1213 make smoke test public --- .github/workflows/pr-env-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index 23d1da44c..1b8fbc04f 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -381,4 +381,4 @@ jobs: - name: Smoke Test run: | - make TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal + make TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-public From 0f71411a0ace9929efcaa2586af7dd827948c79a Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 29 Nov 2024 16:41:55 +0000 Subject: [PATCH 025/106] NRL-000 use pr env for smoke tests - internal --- .github/workflows/pr-env-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index 1b8fbc04f..23d1da44c 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -381,4 +381,4 @@ jobs: - name: Smoke Test run: | - make TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-public + make TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal From 1f3e5028854eb69229e20b45a09bb571a9bb05bd Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 29 Nov 2024 16:52:34 +0000 Subject: [PATCH 026/106] NRL-000 stay on mgmt acct for smoke tests --- .github/workflows/pr-env-deploy.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index 23d1da44c..92a0a65c7 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -371,14 +371,6 @@ jobs: - name: Retrieve Client Certificates run: make truststore-pull-client ENV=dev - - name: Configure Dev Account Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: eu-west-2 - role-chaining: true - role-to-assume: ${{ secrets.DEPLOY_ROLE_ARN }} - role-session-name: github-actions-ci-${{ needs.set-environment-id.outputs.environment_id }} - - name: Smoke Test run: | make TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal From 13a863eb21f65ae661a9e5f95bc152cdee6b399c Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 29 Nov 2024 16:53:28 +0000 Subject: [PATCH 027/106] NRL-000 run smoke tests after perf tests --- .github/workflows/pr-env-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index 92a0a65c7..25d8cfbe0 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -335,7 +335,7 @@ jobs: smoke-test: name: Run Smoke Tests - needs: [set-environment-id, deploy] + needs: [set-environment-id, performance-test] environment: pull-request runs-on: [self-hosted, ci] From 515c7e784e7dbbdc95f975ac886ef0d7757ef6ad Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 29 Nov 2024 17:34:31 +0000 Subject: [PATCH 028/106] NRL-1213 move smoke tests up and change call --- .github/workflows/pr-env-deploy.yml | 82 ++++++++++++++--------------- 1 file changed, 40 insertions(+), 42 deletions(-) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index 25d8cfbe0..56ba42efb 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -264,6 +264,46 @@ jobs: - name: Run Integration Tests run: make test-features-integration TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} + smoke-test: + name: Run Smoke Tests + needs: [set-environment-id, integration-test] + environment: pull-request + runs-on: [self-hosted, ci] + + steps: + - name: Git Clone - ${{ github.event.pull_request.head.ref }} + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + + - name: Setup asdf cache + uses: actions/cache@v4 + with: + path: ~/.asdf + key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} + restore-keys: | + ${{ runner.os }}-asdf- + + - name: Install asdf and tools + uses: asdf-vm/actions/install@v3.0.2 + with: + asdf_branch: v0.13.1 + + - name: Python Dependency Install + run: poetry install --no-root + + - name: Configure Management Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-2 + role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} + role-session-name: github-actions-ci-${{ needs.set-environment-id.outputs.environment_id }} + + - name: Smoke Test + run: | + make ENV=dev truststore-pull-client + make ENV=dev test-smoke-internal + performance-test: name: Run Performance Tests needs: [set-environment-id, integration-test] @@ -332,45 +372,3 @@ jobs: - name: Cleanup Environment Test Data run: make test-performance-cleanup TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} - - smoke-test: - name: Run Smoke Tests - needs: [set-environment-id, performance-test] - environment: pull-request - runs-on: [self-hosted, ci] - - steps: - - name: Git Clone - ${{ github.event.pull_request.head.ref }} - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.ref }} - - - name: Setup asdf cache - uses: actions/cache@v4 - with: - path: ~/.asdf - key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} - restore-keys: | - ${{ runner.os }}-asdf- - - - name: Install asdf and tools - uses: asdf-vm/actions/install@v3.0.2 - with: - asdf_branch: v0.13.1 - - - name: Python Dependency Install - run: poetry install --no-root - - - name: Configure Management Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: eu-west-2 - role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} - role-session-name: github-actions-ci-${{ needs.set-environment-id.outputs.environment_id }} - - - name: Retrieve Client Certificates - run: make truststore-pull-client ENV=dev - - - name: Smoke Test - run: | - make TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal From dd1e91db134b32a5bfd44af90e803ab9169e05d1 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Sat, 30 Nov 2024 20:16:40 +0000 Subject: [PATCH 029/106] NRL-000 trying something --- .github/workflows/pr-env-deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index 56ba42efb..be6aa1342 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -302,7 +302,7 @@ jobs: - name: Smoke Test run: | make ENV=dev truststore-pull-client - make ENV=dev test-smoke-internal + make ENV=dev TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal performance-test: name: Run Performance Tests From 2eea40c4accf10fbd9bec36bd433566ea635a8ba Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Sun, 1 Dec 2024 18:21:27 +0000 Subject: [PATCH 030/106] NRL-853 use different tags for S3 an dynamodb resources --- .../dev/aws-backups.tf | 20 +++++++++---------- .../modules/permissions-store-bucket/s3.tf | 11 ++++------ .../modules/pointers-table/dynamodb.tf | 2 +- .../modules/truststore-bucket/s3.tf | 2 +- 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backups.tf b/terraform/account-wide-infrastructure/dev/aws-backups.tf index 6bc45623d..c62666ece 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backups.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backups.tf @@ -9,9 +9,9 @@ variable "destination_vault_arn" { default = "" } -#data "aws_arn" "destination_vault_arn" { -# arn = var.destination_vault_arn -#} +data "aws_arn" "destination_vault_arn" { + arn = var.destination_vault_arn +} data "aws_secretsmanager_secret" "backup-account-secret" { name = "nhsd-nrlf--dev--test-backup-account-id" @@ -138,11 +138,11 @@ module "source" { source = "../modules/backup-source" backup_copy_vault_account_id = local.destination_account_id - # backup_copy_vault_arn = data.aws_arn.destination_vault_arn.arn - environment_name = local.environment_name - bootstrap_kms_key_arn = aws_kms_key.backup_notifications.arn - project_name = local.project_name - reports_bucket = aws_s3_bucket.backup_reports.bucket + backup_copy_vault_arn = data.aws_arn.destination_vault_arn.arn + environment_name = local.environment_name + bootstrap_kms_key_arn = aws_kms_key.backup_notifications.arn + project_name = local.project_name + reports_bucket = aws_s3_bucket.backup_reports.bucket #terraform_role_arn = data.aws_caller_identity.current.arn terraform_role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}" @@ -164,7 +164,7 @@ module "source" { "schedule" : "cron(0 0 * * ? *)" } ], - "selection_tag" : "NHSE-Enable-Backup" + "selection_tag" : "NHSE-Enable-S3-Backup" } backup_plan_config_dynamodb = { @@ -184,6 +184,6 @@ module "source" { "schedule" : "cron(0 0 * * ? *)" } ], - "selection_tag" : "NHSE-Enable-Backup" + "selection_tag" : "NHSE-Enable-DDB-Backup" } } diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf index 7454d13ac..06e61a58e 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf @@ -2,13 +2,10 @@ resource "aws_s3_bucket" "authorization-store" { bucket = "${var.name_prefix}-authorization-store" force_destroy = var.enable_bucket_force_destroy - tags = var.enable_backups ? { - Name = "authorization store" - Environment = "${var.name_prefix}" - NHSE-Enable-Backup = "true" - } : { - Name = "authorization store" - Environment = "${var.name_prefix}" + tags = { + Name = "authorization store" + Environment = "${var.name_prefix}" + NHSE-Enable-S3-Backup = "${var.enable_backups}" } } diff --git a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf index 19c9184cb..93e060fdb 100644 --- a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf +++ b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf @@ -52,5 +52,5 @@ resource "aws_dynamodb_table" "pointers" { enabled = var.enable_pitr } - tags = var.enable_backups ? { NHSE-Enable-Backup = "true" } : {} + tags = { NHSE-Enable-DDB-Backup = "${var.enable_backups}" } } diff --git a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf index a3d4f970b..aa32f2f16 100644 --- a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf @@ -1,7 +1,7 @@ resource "aws_s3_bucket" "api_truststore" { bucket = "${var.name_prefix}-api-truststore" force_destroy = var.enable_bucket_force_destroy - tags = var.enable_backups ? { NHSE-Enable-Backup = "true" } : {} + tags = { NHSE-Enable-S3-Backup = "${var.enable_backups}" } } resource "aws_s3_bucket_policy" "api_truststore_bucket_policy" { From cb8d9496b97e59199267bcb9117195ed2ef8b3fa Mon Sep 17 00:00:00 2001 From: eesa456 Date: Mon, 2 Dec 2024 09:26:09 +0000 Subject: [PATCH 031/106] NRL-000 setup python env --- .github/workflows/pr-env-deploy.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index be6aa1342..cca2d741c 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -289,8 +289,10 @@ jobs: with: asdf_branch: v0.13.1 - - name: Python Dependency Install - run: poetry install --no-root + - name: Setup Python environment + run: | + poetry install --no-root + source $(poetry env info --path)/bin/activate - name: Configure Management Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -306,7 +308,7 @@ jobs: performance-test: name: Run Performance Tests - needs: [set-environment-id, integration-test] + needs: [set-environment-id, smoke-test] environment: pull-request runs-on: [self-hosted, ci] From db37396ece031cdba2cb238b0826430601b774ba Mon Sep 17 00:00:00 2001 From: eesa456 Date: Mon, 2 Dec 2024 11:10:04 +0000 Subject: [PATCH 032/106] NRL-1213 comment out smoke test --- .github/workflows/pr-env-deploy.yml | 84 ++++++++++++++--------------- 1 file changed, 42 insertions(+), 42 deletions(-) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index cca2d741c..a445b1f2e 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -264,51 +264,51 @@ jobs: - name: Run Integration Tests run: make test-features-integration TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} - smoke-test: - name: Run Smoke Tests - needs: [set-environment-id, integration-test] - environment: pull-request - runs-on: [self-hosted, ci] - - steps: - - name: Git Clone - ${{ github.event.pull_request.head.ref }} - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.ref }} - - - name: Setup asdf cache - uses: actions/cache@v4 - with: - path: ~/.asdf - key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} - restore-keys: | - ${{ runner.os }}-asdf- - - - name: Install asdf and tools - uses: asdf-vm/actions/install@v3.0.2 - with: - asdf_branch: v0.13.1 - - - name: Setup Python environment - run: | - poetry install --no-root - source $(poetry env info --path)/bin/activate - - - name: Configure Management Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-region: eu-west-2 - role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} - role-session-name: github-actions-ci-${{ needs.set-environment-id.outputs.environment_id }} - - - name: Smoke Test - run: | - make ENV=dev truststore-pull-client - make ENV=dev TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal + # smoke-test: + # name: Run Smoke Tests + # needs: [set-environment-id, integration-test] + # environment: pull-request + # runs-on: [self-hosted, ci] + + # steps: + # - name: Git Clone - ${{ github.event.pull_request.head.ref }} + # uses: actions/checkout@v4 + # with: + # ref: ${{ github.event.pull_request.head.ref }} + + # - name: Setup asdf cache + # uses: actions/cache@v4 + # with: + # path: ~/.asdf + # key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} + # restore-keys: | + # ${{ runner.os }}-asdf- + + # - name: Install asdf and tools + # uses: asdf-vm/actions/install@v3.0.2 + # with: + # asdf_branch: v0.13.1 + + # - name: Setup Python environment + # run: | + # poetry install --no-root + # source $(poetry env info --path)/bin/activate + + # - name: Configure Management Credentials + # uses: aws-actions/configure-aws-credentials@v4 + # with: + # aws-region: eu-west-2 + # role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} + # role-session-name: github-actions-ci-${{ needs.set-environment-id.outputs.environment_id }} + + # - name: Smoke Test + # run: | + # make ENV=dev truststore-pull-client + # make ENV=dev TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal performance-test: name: Run Performance Tests - needs: [set-environment-id, smoke-test] + needs: [set-environment-id, integration-test] environment: pull-request runs-on: [self-hosted, ci] From d15fcaecbdd982007caea9706ad6f6664ff82cb1 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Mon, 2 Dec 2024 15:50:13 +0000 Subject: [PATCH 033/106] [NRL-000] Fix up smoke tests to use TF to get domain --- .github/workflows/pr-env-deploy.yml | 87 +++++++++++++++-------------- 1 file changed, 46 insertions(+), 41 deletions(-) diff --git a/.github/workflows/pr-env-deploy.yml b/.github/workflows/pr-env-deploy.yml index a445b1f2e..a90419e50 100644 --- a/.github/workflows/pr-env-deploy.yml +++ b/.github/workflows/pr-env-deploy.yml @@ -264,47 +264,52 @@ jobs: - name: Run Integration Tests run: make test-features-integration TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} - # smoke-test: - # name: Run Smoke Tests - # needs: [set-environment-id, integration-test] - # environment: pull-request - # runs-on: [self-hosted, ci] - - # steps: - # - name: Git Clone - ${{ github.event.pull_request.head.ref }} - # uses: actions/checkout@v4 - # with: - # ref: ${{ github.event.pull_request.head.ref }} - - # - name: Setup asdf cache - # uses: actions/cache@v4 - # with: - # path: ~/.asdf - # key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} - # restore-keys: | - # ${{ runner.os }}-asdf- - - # - name: Install asdf and tools - # uses: asdf-vm/actions/install@v3.0.2 - # with: - # asdf_branch: v0.13.1 - - # - name: Setup Python environment - # run: | - # poetry install --no-root - # source $(poetry env info --path)/bin/activate - - # - name: Configure Management Credentials - # uses: aws-actions/configure-aws-credentials@v4 - # with: - # aws-region: eu-west-2 - # role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} - # role-session-name: github-actions-ci-${{ needs.set-environment-id.outputs.environment_id }} - - # - name: Smoke Test - # run: | - # make ENV=dev truststore-pull-client - # make ENV=dev TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal + smoke-test: + name: Run Smoke Tests + needs: [set-environment-id, integration-test] + environment: pull-request + runs-on: [self-hosted, ci] + steps: + - name: Git Clone - ${{ github.event.pull_request.head.ref }} + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.ref }} + + - name: Setup asdf cache + uses: actions/cache@v4 + with: + path: ~/.asdf + key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} + restore-keys: | + ${{ runner.os }}-asdf- + + - name: Install asdf and tools + uses: asdf-vm/actions/install@v3.0.2 + with: + asdf_branch: v0.13.1 + + - name: Setup Python environment + run: | + poetry install --no-root + source $(poetry env info --path)/bin/activate + + - name: Configure Management Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-2 + role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} + role-session-name: github-actions-ci-${{ needs.set-environment-id.outputs.environment_id }} + + - name: Terraform Init + run: | + terraform -chdir=terraform/infrastructure init + terraform -chdir=terraform/infrastructure workspace new ${{ needs.set-environment-id.outputs.environment_id }} || \ + terraform -chdir=terraform/infrastructure workspace select ${{ needs.set-environment-id.outputs.environment_id }} + + - name: Smoke Test + run: | + make ENV=dev truststore-pull-client + make ENV=dev TF_WORKSPACE_NAME=${{ needs.set-environment-id.outputs.environment_id }} test-smoke-internal performance-test: name: Run Performance Tests From 61fad3441f7ab19fc4cb4c34fcf0537ae617aea2 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Mon, 2 Dec 2024 17:50:12 +0000 Subject: [PATCH 034/106] [NRL-853] Added backup destination vault arn as secret. Moved backup vars/data/locals into dev account-side infra files --- .../dev/{aws-backups.tf => aws-backup.tf} | 47 ++++--------------- .../account-wide-infrastructure/dev/data.tf | 4 ++ .../dev/secrets.tf | 5 ++ .../modules/backup-source/backup_plan.tf | 10 ++-- .../backup-source/backup_report_plan.tf | 2 +- .../backup-source/backup_vault_policy.tf | 2 +- .../modules/backup-source/variables.tf | 1 + 7 files changed, 26 insertions(+), 45 deletions(-) rename terraform/account-wide-infrastructure/dev/{aws-backups.tf => aws-backup.tf} (72%) diff --git a/terraform/account-wide-infrastructure/dev/aws-backups.tf b/terraform/account-wide-infrastructure/dev/aws-backup.tf similarity index 72% rename from terraform/account-wide-infrastructure/dev/aws-backups.tf rename to terraform/account-wide-infrastructure/dev/aws-backup.tf index c62666ece..fc41d32a8 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backups.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backup.tf @@ -1,38 +1,7 @@ -provider "aws" { - alias = "source" - region = "eu-west-2" -} - -variable "destination_vault_arn" { - description = "ARN of the backup vault in the destination account" - type = string - default = "" -} - -data "aws_arn" "destination_vault_arn" { - arn = var.destination_vault_arn -} - -data "aws_secretsmanager_secret" "backup-account-secret" { - name = "nhsd-nrlf--dev--test-backup-account-id" -} -data "aws_secretsmanager_secret_version" "destination_account_id" { - secret_id = data.aws_secretsmanager_secret.backup-account-secret.id -} - -locals { - # Adjust these as required - project_name = "dev-backups-poc" - environment_name = "dev" - - source_account_id = data.aws_caller_identity.current.account_id - # destination_account_id = data.aws_arn.destination_vault_arn.account - destination_account_id = data.aws_secretsmanager_secret_version.destination_account_id.secret_string -} # First, we create an S3 bucket for compliance reports. resource "aws_s3_bucket" "backup_reports" { - bucket_prefix = "${local.project_name}-backup-reports" + bucket_prefix = "${local.prefix}-backup-reports" } resource "aws_s3_bucket_public_access_block" "backup_reports" { @@ -115,7 +84,7 @@ resource "aws_kms_key" "backup_notifications" { Effect = "Allow" Sid = "Enable IAM User Permissions" Principal = { - AWS = "arn:aws:iam::${local.source_account_id}:root" + AWS = "arn:aws:iam::${var.assume_account}:root" } Action = "kms:*" Resource = "*" @@ -137,14 +106,13 @@ resource "aws_kms_key" "backup_notifications" { module "source" { source = "../modules/backup-source" - backup_copy_vault_account_id = local.destination_account_id - backup_copy_vault_arn = data.aws_arn.destination_vault_arn.arn - environment_name = local.environment_name + backup_copy_vault_account_id = jsondecode(data.aws_secretsmanager_secret_version.backup_destination_parameters.secret_string)["account-id"] + backup_copy_vault_arn = jsondecode(data.aws_secretsmanager_secret_version.backup_destination_parameters.secret_string)["vault-arn"] + environment_name = local.environment bootstrap_kms_key_arn = aws_kms_key.backup_notifications.arn - project_name = local.project_name + project_name = "${local.prefix}-" reports_bucket = aws_s3_bucket.backup_reports.bucket - #terraform_role_arn = data.aws_caller_identity.current.arn - terraform_role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}" + terraform_role_arn = "arn:aws:iam::${var.assume_account}:role/${var.assume_role}" notification_target_email_addresses = local.notification_emails @@ -152,6 +120,7 @@ module "source" { "compliance_resource_types" : [ "S3" ], + "enable" = true, "rules" : [ { "copy_action" : { diff --git a/terraform/account-wide-infrastructure/dev/data.tf b/terraform/account-wide-infrastructure/dev/data.tf index 7b3c623de..bb435ed6b 100644 --- a/terraform/account-wide-infrastructure/dev/data.tf +++ b/terraform/account-wide-infrastructure/dev/data.tf @@ -2,6 +2,10 @@ data "aws_secretsmanager_secret_version" "identities_account_id" { secret_id = aws_secretsmanager_secret.identities_account_id.name } +data "aws_secretsmanager_secret_version" "backup_destination_parameters" { + secret_id = aws_secretsmanager_secret.backup_destination_parameters.name +} + data "aws_secretsmanager_secret" "emails" { name = "${local.prefix}-emails" } diff --git a/terraform/account-wide-infrastructure/dev/secrets.tf b/terraform/account-wide-infrastructure/dev/secrets.tf index 2559c81cd..bc9b0a3cc 100644 --- a/terraform/account-wide-infrastructure/dev/secrets.tf +++ b/terraform/account-wide-infrastructure/dev/secrets.tf @@ -2,6 +2,11 @@ resource "aws_secretsmanager_secret" "identities_account_id" { name = "${local.prefix}--nhs-identities-account-id" } +resource "aws_secretsmanager_secret" "backup_destination_parameters" { + name = "${local.prefix}--backup-destination-parameters" + description = "Parameters used to configure the backup destination" +} + resource "aws_secretsmanager_secret" "notification_email_addresses" { name = "${local.prefix}-dev-notification-email-addresses" } diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf index b5ac3c1df..0e6cd4ce8 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf @@ -1,5 +1,6 @@ resource "aws_backup_plan" "default" { - name = "${local.resource_name_prefix}-plan" + count = var.backup_plan_config.enable ? 1 : 0 + name = "${local.resource_name_prefix}-plan" dynamic "rule" { for_each = var.backup_plan_config.rules @@ -16,7 +17,7 @@ resource "aws_backup_plan" "default" { cold_storage_after = rule.value.lifecycle.cold_storage_after != null ? rule.value.lifecycle.cold_storage_after : null } dynamic "copy_action" { - for_each = var.backup_copy_vault_arn != "" && var.backup_copy_vault_account_id != "" && rule.value.copy_action != null ? rule.value.copy_action : {} + for_each = rule.value.copy_action != null ? rule.value.copy_action : {} content { lifecycle { delete_after = copy_action.value @@ -47,7 +48,7 @@ resource "aws_backup_plan" "dynamodb" { cold_storage_after = rule.value.lifecycle.cold_storage_after != null ? rule.value.lifecycle.cold_storage_after : null } dynamic "copy_action" { - for_each = var.backup_copy_vault_arn != "" && var.backup_copy_vault_account_id != "" && rule.value.copy_action != null ? rule.value.copy_action : {} + for_each = rule.value.copy_action != null ? rule.value.copy_action : {} content { lifecycle { delete_after = copy_action.value @@ -60,9 +61,10 @@ resource "aws_backup_plan" "dynamodb" { } resource "aws_backup_selection" "default" { + count = var.backup_plan_config.enable ? 1 : 0 iam_role_arn = aws_iam_role.backup.arn name = "${local.resource_name_prefix}-selection" - plan_id = aws_backup_plan.default.id + plan_id = aws_backup_plan.default[0].id selection_tag { key = var.backup_plan_config.selection_tag diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_report_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_report_plan.tf index e0f20d41e..7120bfe70 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_report_plan.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_report_plan.tf @@ -54,7 +54,7 @@ resource "aws_backup_report_plan" "resource_compliance" { } resource "aws_backup_report_plan" "copy_jobs" { - count = var.backup_copy_vault_arn != "" && var.backup_copy_vault_account_id != "" ? 1 : 0 + count = var.backup_plan_config.enable || var.backup_plan_config_dynamodb.enable ? 1 : 0 name = "copy_jobs" description = "Report for showing whether copies ran successfully in the last 24 hours" diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_vault_policy.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_vault_policy.tf index 09d4ec117..f1e6222e9 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_vault_policy.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_vault_policy.tf @@ -30,7 +30,7 @@ data "aws_iam_policy_document" "vault_policy" { resources = ["*"] } dynamic "statement" { - for_each = var.backup_copy_vault_arn != "" && var.backup_copy_vault_account_id != "" ? [1] : [] + for_each = var.backup_plan_config.enable || var.backup_plan_config_dynamodb.enable ? [1] : [] content { sid = "Allow account to copy into backup vault" effect = "Allow" diff --git a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf index 88acbfd19..72cc612f6 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/variables.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/variables.tf @@ -74,6 +74,7 @@ variable "backup_copy_vault_account_id" { variable "backup_plan_config" { description = "Configuration for backup plans" type = object({ + enable = bool selection_tag = string compliance_resource_types = list(string) rules = list(object({ From 64cf5f4891d4833660528fc4263f55e7c42771d0 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Wed, 27 Nov 2024 09:23:43 +0000 Subject: [PATCH 035/106] NRL-518 Validate display as per the definition --- layer/nrlf/core/tests/test_validators.py | 42 ++++++++++++++++++- layer/nrlf/core/validators.py | 40 ++++++++++-------- .../RQI-736253002-Valid.json | 2 +- .../Y05868-736253002-Valid.json | 2 +- 4 files changed, 65 insertions(+), 21 deletions(-) diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index fd02deb20..00da67405 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1029,7 +1029,7 @@ def test_validate_content_extension_invalid_display(): } ] }, - "diagnostics": "Invalid content extension display: invalid Extension display must be the same as code either 'static' or 'dynamic'", + "diagnostics": "Invalid content extension display: invalid Extension display must be 'Static' or 'Dynamic'", "expression": [ "content[0].extension[0].valueCodeableConcept.coding[0].display" ], @@ -1522,3 +1522,43 @@ def test_validate_ssp_content_with_multiple_asids(): "diagnostics": "Multiple ASID identifiers provided. Only a single valid ASID identifier can be provided in the context.related.", "expression": ["context.related"], } + + def test_validate_content_extension_invalid_code_and_display_mismatch(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["extension"][0] = { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Dynamic", + } + ] + }, + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content extension display: Dynamic Extension display must be the same as code either 'Static' or 'Dynamic'", + "expression": [ + "content[0].extension[0].valueCodeableConcept.coding[0].display" + ], + } diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index bdb01b78b..c40eeae7e 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -508,50 +508,54 @@ def _validate_content_extension(self, model: DocumentReference): return if ( - content.extension[0].valueCodeableConcept.coding[0].system + content.extension[0].url + != "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ): + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid content extension url: {content.extension[0].url} Extension url must be 'https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability'", + field=f"content[{i}].extension[0].url", + ) + return + + coding = content.extension[0].valueCodeableConcept.coding[0] + if ( + coding.system != "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" ): self.result.add_error( issue_code="value", error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension system: {content.extension[0].valueCodeableConcept.coding[0].system} Extension system must be 'https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability'", + diagnostics=f"Invalid content extension system: {coding.system} Extension system must be 'https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability'", field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].system", ) return - if content.extension[0].valueCodeableConcept.coding[0].code not in [ - "static", - "dynamic", - ]: + if coding.code not in ["static", "dynamic"]: self.result.add_error( issue_code="value", error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension code: {content.extension[0].valueCodeableConcept.coding[0].code} Extension code must be 'static' or 'dynamic'", + diagnostics=f"Invalid content extension code: {coding.code} Extension code must be 'static' or 'dynamic'", field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].code", ) return - if ( - content.extension[0].valueCodeableConcept.coding[0].code - != content.extension[0].valueCodeableConcept.coding[0].display.lower() - ): + if coding.display not in ["Static", "Dynamic"]: self.result.add_error( issue_code="value", error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension display: {content.extension[0].valueCodeableConcept.coding[0].display} Extension display must be the same as code either 'static' or 'dynamic'", + diagnostics=f"Invalid content extension display: {coding.display} Extension display must be 'Static' or 'Dynamic'", field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].display", ) return - if ( - content.extension[0].url - != "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - ): + if coding.code != coding.display.lower(): self.result.add_error( issue_code="value", error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension url: {content.extension[0].url} Extension url must be 'https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability'", - field=f"content[{i}].extension[0].url", + diagnostics=f"Invalid content extension display: {coding.display} Extension display must be the same as code either 'Static' or 'Dynamic'", + field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].display", ) return diff --git a/tests/data/DocumentReference/RQI-736253002-Valid.json b/tests/data/DocumentReference/RQI-736253002-Valid.json index db7503fbf..429d088a4 100644 --- a/tests/data/DocumentReference/RQI-736253002-Valid.json +++ b/tests/data/DocumentReference/RQI-736253002-Valid.json @@ -79,7 +79,7 @@ { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", "code": "static", - "display": "static" + "display": "Static" } ] } diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid.json b/tests/data/DocumentReference/Y05868-736253002-Valid.json index b0a18b1d5..d83b5a3f6 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid.json @@ -79,7 +79,7 @@ { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", "code": "static", - "display": "static" + "display": "Static" } ] } From dcde0a9ff829426dd5294fabf198d96575c5751a Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Mon, 2 Dec 2024 16:47:49 +0000 Subject: [PATCH 036/106] NRL-518 Fix pydantic error reporting --- layer/nrlf/core/errors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/layer/nrlf/core/errors.py b/layer/nrlf/core/errors.py index f5bdc3a47..6e7b21686 100644 --- a/layer/nrlf/core/errors.py +++ b/layer/nrlf/core/errors.py @@ -11,14 +11,14 @@ def diag_for_error(error: ErrorDetails) -> str: if error["loc"]: - loc_string = ".".join(each for each in error["loc"]) + loc_string = ".".join(str(each) for each in error["loc"]) return f"{loc_string}: {error['msg']}" else: return f"root: {error['msg']}" def expression_for_error(error: ErrorDetails) -> Optional[str]: - return str(".".join(each for each in error["loc"]) if error["loc"] else "root") + return str(".".join(str(each) for each in error["loc"]) if error["loc"] else "root") class OperationOutcomeError(Exception): From a50f1a5eddda86b6fad385b9218a3e1fa0907a66 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Mon, 2 Dec 2024 16:48:21 +0000 Subject: [PATCH 037/106] NRL-518 Update models with mandatory fields --- api/consumer/swagger.yaml | 6 +++++- api/producer/swagger.yaml | 6 +++++- layer/nrlf/consumer/fhir/r4/model.py | 16 ++++++++-------- layer/nrlf/producer/fhir/r4/model.py | 16 ++++++++-------- layer/nrlf/producer/fhir/r4/strict_model.py | 15 +++++++-------- 5 files changed, 33 insertions(+), 26 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 47c1a072c..7caec31fd 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -1074,6 +1074,7 @@ components: description: Additional content defined by implementations. required: - attachment + - format DocumentReferenceRelatesTo: type: object properties: @@ -1100,7 +1101,7 @@ components: description: Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces. contentType: type: string - pattern: "[^\\s]+(\\s[^\\s]+)*" + pattern: "^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$" description: Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate. language: type: string @@ -1130,6 +1131,9 @@ components: type: string pattern: ([0-9]([0-9]([0-9][1-9]|[1-9]0)|[1-9]00)|[1-9]000)(-(0[1-9]|1[0-2])(-(0[1-9]|[1-2][0-9]|3[0-1])(T([01][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)(\.[0-9]+)?(Z|(\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00)))?)?)? description: The date that the attachment was first created. + required: + - contentType + - url CodeableConcept: type: object properties: diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index bdd0127b8..6b1fa4457 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1639,6 +1639,7 @@ components: description: Additional content defined by implementations. required: - attachment + - format DocumentReferenceRelatesTo: type: object properties: @@ -1665,7 +1666,7 @@ components: description: Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces. contentType: type: string - pattern: "[^\\s]+(\\s[^\\s]+)*" + pattern: "^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$" description: Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate. language: type: string @@ -1695,6 +1696,9 @@ components: type: string pattern: ([0-9]([0-9]([0-9][1-9]|[1-9]0)|[1-9]00)|[1-9]000)(-(0[1-9]|1[0-2])(-(0[1-9]|[1-2][0-9]|3[0-1])(T([01][0-9]|2[0-3]):[0-5][0-9]:([0-5][0-9]|60)(\.[0-9]+)?(Z|(\+|-)((0[0-9]|1[0-3]):[0-5][0-9]|14:00)))?)?)? description: The date that the attachment was first created. + required: + - contentType + - url CodeableConcept: type: object properties: diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index 4665533b5..a44aab138 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T09:43:58+00:00 +# timestamp: 2024-11-26T15:05:35+00:00 from __future__ import annotations @@ -133,12 +133,12 @@ class Attachment(BaseModel): ), ] = None contentType: Annotated[ - Optional[str], + str, Field( description="Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate.", - pattern="[^\\s]+(\\s[^\\s]+)*", + pattern="^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$", ), - ] = None + ] language: Annotated[ Optional[str], Field( @@ -154,9 +154,9 @@ class Attachment(BaseModel): ), ] = None url: Annotated[ - Optional[str], + str, Field(description="A location where the data can be accessed.", pattern="\\S*"), - ] = None + ] size: Annotated[ Optional[int], Field( @@ -833,11 +833,11 @@ class DocumentReferenceContent(BaseModel): ), ] format: Annotated[ - Optional[Coding], + Coding, Field( description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), - ] = None + ] extension: Optional[List[Extension]] = None diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index d96b7ce73..fe399cff7 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T10:10:52+00:00 +# timestamp: 2024-11-26T15:05:33+00:00 from __future__ import annotations @@ -133,12 +133,12 @@ class Attachment(BaseModel): ), ] = None contentType: Annotated[ - Optional[str], + str, Field( description="Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate.", - pattern="[^\\s]+(\\s[^\\s]+)*", + pattern="^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$", ), - ] = None + ] language: Annotated[ Optional[str], Field( @@ -154,9 +154,9 @@ class Attachment(BaseModel): ), ] = None url: Annotated[ - Optional[str], + str, Field(description="A location where the data can be accessed.", pattern="\\S*"), - ] = None + ] size: Annotated[ Optional[int], Field( @@ -817,11 +817,11 @@ class DocumentReferenceContent(BaseModel): ), ] format: Annotated[ - Optional[Coding], + Coding, Field( description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), - ] = None + ] extension: Optional[List[Extension]] = None diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index e4edefc58..ca3ec452f 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T10:10:54+00:00 +# timestamp: 2024-11-26T15:05:34+00:00 from __future__ import annotations @@ -126,11 +126,11 @@ class Attachment(BaseModel): ), ] = None contentType: Annotated[ - Optional[StrictStr], + StrictStr, Field( description="Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate." ), - ] = None + ] language: Annotated[ Optional[StrictStr], Field( @@ -144,9 +144,8 @@ class Attachment(BaseModel): ), ] = None url: Annotated[ - Optional[StrictStr], - Field(description="A location where the data can be accessed."), - ] = None + StrictStr, Field(description="A location where the data can be accessed.") + ] size: Annotated[ Optional[StrictInt], Field( @@ -721,11 +720,11 @@ class DocumentReferenceContent(BaseModel): ), ] format: Annotated[ - Optional[Coding], + Coding, Field( description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), - ] = None + ] extension: Optional[List[Extension]] = None From d29aa5f1655e163073881f9814a89458d2375c23 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Mon, 2 Dec 2024 17:28:45 +0000 Subject: [PATCH 038/106] NRL-518 Add step for upserts, refactor create and upsert into one method --- tests/features/steps/2_request.py | 25 +++++++++++++++++++------ tests/utilities/api_clients.py | 8 ++++++++ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/tests/features/steps/2_request.py b/tests/features/steps/2_request.py index a008aed17..4dfd974e4 100644 --- a/tests/features/steps/2_request.py +++ b/tests/features/steps/2_request.py @@ -94,17 +94,16 @@ def create_post_document_reference_step(context: Context, ods_code: str): context.add_cleanup(lambda: context.repository.delete_by_id(doc_ref_id)) -@when( - "producer 'TSTCUS' requests creation of a DocumentReference with default test values except '{section}' is" -) -def create_post_body_step(context: Context, section: str): +def _create_or_upsert_body_step( + context: Context, method: str, pointer_id: str = "TSTCUS-sample-id-00000" +): client = producer_client_from_context(context, "TSTCUS") if not context.text: raise ValueError("No document reference text snippet provided") - doc_ref = create_test_document_reference_with_defaults(section, context.text) - context.response = client.create_text(doc_ref) + doc_ref = create_test_document_reference_with_defaults("content", context.text) + context.response = getattr(client, method)(doc_ref) if context.response.status_code == 201: doc_ref_id = context.response.headers["Location"].split("/")[-1] @@ -114,6 +113,20 @@ def create_post_body_step(context: Context, section: str): context.add_cleanup(lambda: context.repository.delete_by_id(doc_ref_id)) +@when( + "producer 'TSTCUS' requests creation of a DocumentReference with default test values except '{section}' is" +) +def create_post_body_step(context: Context, section: str): + _create_or_upsert_body_step(context, "create_text") + + +@when( + "producer 'TSTCUS' requests upsert of a DocumentReference with pointerId '{pointer_id}' and default test values except '{section}' is" +) +def upsert_post_body_step(context: Context, section: str, pointer_id: str): + _create_or_upsert_body_step(context, "upsert_text", pointer_id) + + @when("producer '{ods_code}' upserts a DocumentReference with values") def create_put_document_reference_step(context: Context, ods_code: str): client = producer_client_from_context(context, ods_code) diff --git a/tests/utilities/api_clients.py b/tests/utilities/api_clients.py index 1d06bf2a8..3b1b78e9f 100644 --- a/tests/utilities/api_clients.py +++ b/tests/utilities/api_clients.py @@ -213,6 +213,14 @@ def upsert(self, doc_ref): cert=self.config.client_cert, ) + def upsert_text(self, doc_ref): + return requests.put( + f"{self.api_url}/DocumentReference", + data=doc_ref, + headers=self.request_headers, + cert=self.config.client_cert, + ) + def update(self, doc_ref, doc_ref_id: str): return requests.put( f"{self.api_url}/DocumentReference/{doc_ref_id}", From d58fdca7bd4f81ff8a04aa85648a6377bc873915 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 09:01:49 +0000 Subject: [PATCH 039/106] NRL-518 Add default format to default test document reference --- tests/features/utils/data.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 11288945a..68ee5fccb 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -28,14 +28,25 @@ def create_test_document_reference(items: dict) -> DocumentReference: base_doc_ref = DocumentReference.model_construct( resourceType="DocumentReference", status=items.get("status", "current"), - content=[ - DocumentReferenceContent( - attachment=Attachment( - contentType=items.get("contentType", "application/json"), - url=items["url"], + content=items.get( + "content", + [ + DocumentReferenceContent( + attachment=Attachment( + contentType=items.get("contentType", "application/json"), + url=items["url"], + ), + format=Coding( + system=items.get( + "formatSystem", + "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + ), + code=items.get("formatCode", "urn:nhs-ic:unstructured"), + display=items.get("formatDisplay", "Unstructured document"), + ), ) - ) - ], + ], + ), context=DocumentReferenceContext( practiceSetting=CodeableConcept( coding=[ From 548435bdb1a822d9e2e832b74b30d8abcb0cde1a Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 09:06:20 +0000 Subject: [PATCH 040/106] NRL-518 Fix parameter should not be fix to content --- tests/features/steps/2_request.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tests/features/steps/2_request.py b/tests/features/steps/2_request.py index 4dfd974e4..1ca36903c 100644 --- a/tests/features/steps/2_request.py +++ b/tests/features/steps/2_request.py @@ -95,14 +95,17 @@ def create_post_document_reference_step(context: Context, ods_code: str): def _create_or_upsert_body_step( - context: Context, method: str, pointer_id: str = "TSTCUS-sample-id-00000" + context: Context, + method: str, + section: str, + pointer_id: str = "TSTCUS-sample-id-00000", ): client = producer_client_from_context(context, "TSTCUS") if not context.text: raise ValueError("No document reference text snippet provided") - doc_ref = create_test_document_reference_with_defaults("content", context.text) + doc_ref = create_test_document_reference_with_defaults(section, context.text) context.response = getattr(client, method)(doc_ref) if context.response.status_code == 201: @@ -117,14 +120,14 @@ def _create_or_upsert_body_step( "producer 'TSTCUS' requests creation of a DocumentReference with default test values except '{section}' is" ) def create_post_body_step(context: Context, section: str): - _create_or_upsert_body_step(context, "create_text") + _create_or_upsert_body_step(context, "create_text", section) @when( "producer 'TSTCUS' requests upsert of a DocumentReference with pointerId '{pointer_id}' and default test values except '{section}' is" ) def upsert_post_body_step(context: Context, section: str, pointer_id: str): - _create_or_upsert_body_step(context, "upsert_text", pointer_id) + _create_or_upsert_body_step(context, "upsert_text", section, pointer_id) @when("producer '{ods_code}' upserts a DocumentReference with values") From 84a5f103090a0afe74aeb1cbd1de9cbd8ec5418a Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 09:07:01 +0000 Subject: [PATCH 041/106] NRL-518 Add method to change default parameters for an update --- tests/features/steps/2_request.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/features/steps/2_request.py b/tests/features/steps/2_request.py index 1ca36903c..9caaab2e9 100644 --- a/tests/features/steps/2_request.py +++ b/tests/features/steps/2_request.py @@ -1,3 +1,5 @@ +import json + from behave import * # noqa from behave.runner import Context @@ -130,6 +132,28 @@ def upsert_post_body_step(context: Context, section: str, pointer_id: str): _create_or_upsert_body_step(context, "upsert_text", section, pointer_id) +@when( + "producer 'TSTCUS' requests update of a DocumentReference with pointerId '{pointer_id}' and only changing" +) +def update_post_body_step(context: Context, pointer_id: str): + """ + Updates an existing DocumentReference with new values for a specific section + """ + consumer_client = consumer_client_from_context(context, "TSTCUS") + context.response = consumer_client.read(pointer_id) + + if context.response.status_code != 200: + raise ValueError(f"Failed to read existing pointer: {context.response.text}") + + doc_ref = context.response.json() + custom_data = json.loads(context.text) + for key in custom_data: + doc_ref[key] = custom_data[key] + + producer_client = producer_client_from_context(context, "TSTCUS") + context.response = producer_client.update(doc_ref, pointer_id) + + @when("producer '{ods_code}' upserts a DocumentReference with values") def create_put_document_reference_step(context: Context, ods_code: str): client = producer_client_from_context(context, ods_code) From d3f26c6500d93d838893744f0904421b5cb55e18 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 09:07:39 +0000 Subject: [PATCH 042/106] NRL-518 Add validator method for content and add tests --- layer/nrlf/core/tests/test_validators.py | 93 ++++++++++ layer/nrlf/core/validators.py | 34 ++++ .../createDocumentReference-failure.feature | 114 +++++++++++++ .../updateDocumentReference-failure.feature | 159 ++++++++++++++++++ .../upsertDocumentReference-failure.feature | 115 +++++++++++++ 5 files changed, 515 insertions(+) diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index 00da67405..20f08411f 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1562,3 +1562,96 @@ def test_validate_content_extension_invalid_code_and_display_mismatch(): "content[0].extension[0].valueCodeableConcept.coding[0].display" ], } + + def test_validate_content_missing_attachment(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0].pop("attachment") + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "required", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Missing attachment in content", + "expression": ["content[0].attachment"], + } + + def test_validate_content_missing_content_type(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"].pop("contentType") + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "required", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Missing contentType in content.attachment", + "expression": ["content[0].attachment.contentType"], + } + + def test_validate_content_invalid_content_type(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"][ + "contentType" + ] = "invalid/type" + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid contentType: invalid/type. Must be 'application/pdf' or 'text/html'", + "expression": ["content[0].attachment.contentType"], + } + + def test_validate_content_valid_content_type(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"][ + "contentType" + ] = "application/pdf" + + result = validator.validate(document_ref_data) + + assert result.is_valid is True + assert result.issues == [] diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index c40eeae7e..7e910913b 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -141,6 +141,7 @@ def validate(self, data: Dict[str, Any] | DocumentReference): self._validate_category(resource) self._validate_author(resource) self._validate_type_category_mapping(resource) + self._validate_content(resource) if resource.content[0].extension: self._validate_content_extension(resource) @@ -603,3 +604,36 @@ def _validate_author(self, model: DocumentReference): field=f"author[0].identifier.value", ) return + + def _validate_content(self, model: DocumentReference): + """ + Validate that the contentType is present and is either 'application/pdf' or 'text/html'. + """ + logger.log(LogReference.VALIDATOR001, step="content") + + for i, content in enumerate(model.content): + if not content.attachment: + self.result.add_error( + issue_code="required", + error_code="INVALID_RESOURCE", + diagnostics="Missing attachment in content", + field=f"content[{i}].attachment", + ) + continue + + if not content.attachment.contentType: + self.result.add_error( + issue_code="required", + error_code="INVALID_RESOURCE", + diagnostics="Missing contentType in content.attachment", + field=f"content[{i}].attachment.contentType", + ) + continue + + if content.attachment.contentType not in ["application/pdf", "text/html"]: + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid contentType: {content.attachment.contentType}. Must be 'application/pdf' or 'text/html'", + field=f"content[{i}].attachment.contentType", + ) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index c1d09f483..cf78d96bd 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -655,3 +655,117 @@ Feature: Producer - createDocumentReference - Failure Scenarios | type-system | type-code | category-code | type-display | correct-display | | https://nicip.nhs.uk | MAULR | 721981007 | "Nonsense display" | MRA Upper Limb Rt | | https://nicip.nhs.uk | MAXIB | 103693007 | "Nonsense display" | MRI Axilla Both | + + Scenario: Missing content + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content: List should have at least 1 item after validation, not 0)", + "expression": [ + "content" + ] + } + """ + + Scenario: Missing contentType + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" + } + } + ] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$')", + "expression": [ + "content.0.attachment.contentType" + ] + } + """ + + Scenario: Invalid contentType + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'ANGY1' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'ANGY1' creates a DocumentReference with values: + | property | value | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | contentType | application/invalid | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid contentType: application/invalid. Must be 'application/pdf' or 'text/html'", + "expression": [ + "content[0].attachment.contentType" + ] + } + """ diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index b909b46cf..bd0fbbe76 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -48,3 +48,162 @@ Feature: Producer - updateDocumentReference - Failure Scenarios ] } """ + + Scenario: Missing content + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | TSTCUS-1114567890-updateDocTest | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | TSTCUS | + | author | TSTCUS | + When producer 'TSTCUS' requests update of a DocumentReference with pointerId 'TSTCUS-1114567890-updateDocTest' and only changing: + """ + { + "content": [] + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content: List should have at least 1 item after validation, not 0)", + "expression": [ + "content" + ] + } + """ + + Scenario: Missing contentType + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | TSTCUS-1114567891-updateDocTest | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | TSTCUS | + | author | TSTCUS | + When producer 'TSTCUS' requests update of a DocumentReference with pointerId 'TSTCUS-1114567891-updateDocTest' and only changing: + """ + { + "content": [ + { + "attachment": { + "contentType": "", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" + } + } + ] + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$')", + "expression": [ + "content.0.attachment.contentType" + ] + } + """ + + Scenario: Invalid contentType + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | TSTCUS-1114567892-updateDocTest | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | TSTCUS | + | author | TSTCUS | + When producer 'TSTCUS' requests update of a DocumentReference with pointerId 'TSTCUS-1114567892-updateDocTest' and only changing: + """ + { + "content": [ + { + "attachment": { + "contentType": "application/invalid", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" + } + } + ] + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid contentType: application/invalid. Must be 'application/pdf' or 'text/html'", + "expression": [ + "content[0].attachment.contentType" + ] + } + """ diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index 3fb5d6ebd..5876e7db6 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -236,3 +236,118 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios | type-system | type-code | category-code | type-display | correct-display | | https://nicip.nhs.uk | MAULR | 721981007 | "Nonsense display" | MRA Upper Limb Rt | | https://nicip.nhs.uk | MAXIB | 103693007 | "Nonsense display" | MRI Axilla Both | + + Scenario: Missing content + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests upsert of a DocumentReference with pointerId 'TSTCUS-sample-id-00001' and default test values except 'content' is: + """ + "content": [] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content: List should have at least 1 item after validation, not 0)", + "expression": [ + "content" + ] + } + """ + + Scenario: Missing contentType + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests upsert of a DocumentReference with pointerId 'TSTCUS-sample-id-00002' and default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" + } + } + ] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed" + } + ] + }, + "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$')", + "expression": [ + "content.0.attachment.contentType" + ] + } + """ + + Scenario: Invalid contentType + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'ANGY1' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'ANGY1' upserts a DocumentReference with values: + | property | value | + | id | TSTCUS-sample-id-00003 | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | contentType | application/invalid | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid contentType: application/invalid. Must be 'application/pdf' or 'text/html'", + "expression": [ + "content[0].attachment.contentType" + ] + } + """ From 02a9a7e91241014f58685095d6123edd1bc8c24f Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 09:11:09 +0000 Subject: [PATCH 043/106] NRL-518 Fix default contentType --- tests/features/utils/data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 68ee5fccb..6cf930114 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -33,7 +33,7 @@ def create_test_document_reference(items: dict) -> DocumentReference: [ DocumentReferenceContent( attachment=Attachment( - contentType=items.get("contentType", "application/json"), + contentType=items.get("contentType", "application/pdf"), url=items["url"], ), format=Coding( From de9065918f04d1179a9bd887fbb87a38d45305bf Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 09:24:30 +0000 Subject: [PATCH 044/106] NRL-518 Fix integration tests expected without format --- .../consumer/readDocumentReference-success.feature | 10 ++++++++++ .../producer/readDocumentReference-success.feature | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/tests/features/consumer/readDocumentReference-success.feature b/tests/features/consumer/readDocumentReference-success.feature index b1e1add80..5c0d83979 100644 --- a/tests/features/consumer/readDocumentReference-success.feature +++ b/tests/features/consumer/readDocumentReference-success.feature @@ -69,6 +69,11 @@ Feature: Consumer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" } } ], @@ -155,6 +160,11 @@ Feature: Consumer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" } } ], diff --git a/tests/features/producer/readDocumentReference-success.feature b/tests/features/producer/readDocumentReference-success.feature index bf0e340b1..04c4228ec 100644 --- a/tests/features/producer/readDocumentReference-success.feature +++ b/tests/features/producer/readDocumentReference-success.feature @@ -71,6 +71,11 @@ Feature: Producer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" } } ], From 48ace9e5c3c79112270a8ea37cd50fa81af010bc Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 09:42:11 +0000 Subject: [PATCH 045/106] NRL-518 Add ContentStability value set --- .../fhir/NRLF-ContentStability-ValueSet.json | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 resources/fhir/NRLF-ContentStability-ValueSet.json diff --git a/resources/fhir/NRLF-ContentStability-ValueSet.json b/resources/fhir/NRLF-ContentStability-ValueSet.json new file mode 100644 index 000000000..26a2ca381 --- /dev/null +++ b/resources/fhir/NRLF-ContentStability-ValueSet.json @@ -0,0 +1,37 @@ +{ + "resourceType": "ValueSet", + "id": "NRLF-ContentStability", + "url": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "version": "1.0.0", + "name": "NRLF Content Stability", + "status": "draft", + "date": "2024-12-03T00:00:00+00:00", + "publisher": "NHS Digital", + "contact": { + "name": "NRL Team at NHS Digital", + "telecom": { + "system": "email", + "value": "nrls@nhs.net", + "use": "work" + } + }, + "description": "A code from the NRL Content Stability coding system to represent the stability of the content.", + "copyright": "Copyright 2024 NHS Digital.", + "compose": { + "include": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "concept": [ + { + "code": "static", + "display": "Static" + }, + { + "code": "dynamic", + "display": "Dynamic" + } + ] + } + ] + } +} From 7a6e4307e3c8b44f3748cb8ff55025b29ff88f4b Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 10:03:36 +0000 Subject: [PATCH 046/106] NRL-518 Fix smoke tests missing format --- tests/smoke/setup.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index a44ebe9bb..0c0a578c5 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -32,7 +32,16 @@ def build_document_reference( attachment=Attachment( contentType=content_type, url=content_url, - ) + ), + format=CodeableConcept( + coding=[ + Coding( + system="https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + code="urn:nhs-ic:unstructured", + display="Unstructured document", + ) + ] + ), ) ], type=CodeableConcept( From 3c6c1b86339ad48070eaaad93aad2bcbf177aa0a Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 10:37:58 +0000 Subject: [PATCH 047/106] NRL-518 Fix unused parameter --- tests/features/steps/2_request.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/features/steps/2_request.py b/tests/features/steps/2_request.py index 9caaab2e9..993fae1c7 100644 --- a/tests/features/steps/2_request.py +++ b/tests/features/steps/2_request.py @@ -107,7 +107,9 @@ def _create_or_upsert_body_step( if not context.text: raise ValueError("No document reference text snippet provided") - doc_ref = create_test_document_reference_with_defaults(section, context.text) + doc_ref = create_test_document_reference_with_defaults( + section, context.text, pointer_id + ) context.response = getattr(client, method)(doc_ref) if context.response.status_code == 201: From 4820d089d112064a854bcb44fe92e4bfaf8b6a59 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 10:58:17 +0000 Subject: [PATCH 048/106] NRL-518 Fix content_type for smoke test --- tests/smoke/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index 0c0a578c5..cb1423ec4 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -20,7 +20,7 @@ def build_document_reference( category: str = Categories.CARE_PLAN.coding_value(), type: str = PointerTypes.MENTAL_HEALTH_PLAN.coding_value(), author: str = "SMOKETEST", - content_type: str = "application/json", + content_type: str = "application/pdf", content_url: str = "https://testing.record-locator.national.nhs.uk/_smoke_test_pointer_content", replaces_id: str | None = None, ) -> DocumentReference: From 32e1363fb802b5baa790699ce96561eb9b556cfa Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Tue, 3 Dec 2024 12:20:22 +0000 Subject: [PATCH 049/106] NRL-518 Fix format for smoke tests --- tests/smoke/setup.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index cb1423ec4..49ddeb766 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -33,14 +33,10 @@ def build_document_reference( contentType=content_type, url=content_url, ), - format=CodeableConcept( - coding=[ - Coding( - system="https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - code="urn:nhs-ic:unstructured", - display="Unstructured document", - ) - ] + format=Coding( + system="https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + code="urn:nhs-ic:unstructured", + display="Unstructured document", ), ) ], From 7b7f2bd985ffc1efad3061aa7b7c64726944e248 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Wed, 4 Dec 2024 15:07:36 +0000 Subject: [PATCH 050/106] NRL-518 Add documentation for content validation --- swagger/producer-static/narrative.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/swagger/producer-static/narrative.yaml b/swagger/producer-static/narrative.yaml index 4341faf86..91041187e 100644 --- a/swagger/producer-static/narrative.yaml +++ b/swagger/producer-static/narrative.yaml @@ -276,7 +276,12 @@ paths: ] ``` * `content` MUST have at least one entry. - * `content[].format[]` SHOULD indicate whether the data is structured or not, e.g. + * `content` MUST include an `attachment` entry. + * `content` MUST include a `format` entry. + * `content` MUST include the content stability extension (https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability). + * `content[].attachment` MUST include a `url` to the document. + * `content[].attachment` MUST include a `contentType` and be a valid MIME type, specifically `application/pdf` for documents or `text/html` for contact details. + * `content[].format[]` MUST indicate whether the data is structured or not, e.g. ``` "format": [ { From 6b8e9e63e152e44555c00dedde9bff7c9f920db7 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Wed, 4 Dec 2024 15:08:12 +0000 Subject: [PATCH 051/106] NRL-518 Add NRLFormatCode valueset --- resources/fhir/NRLF-FormatCode-ValueSet.json | 37 ++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 resources/fhir/NRLF-FormatCode-ValueSet.json diff --git a/resources/fhir/NRLF-FormatCode-ValueSet.json b/resources/fhir/NRLF-FormatCode-ValueSet.json new file mode 100644 index 000000000..a0fe1cc77 --- /dev/null +++ b/resources/fhir/NRLF-FormatCode-ValueSet.json @@ -0,0 +1,37 @@ +{ + "resourceType": "ValueSet", + "id": "NRLF-FormatCode", + "url": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "version": "1.0.0", + "name": "NRLF Format Code", + "status": "draft", + "date": "2024-12-03T00:00:00+00:00", + "publisher": "NHS Digital", + "contact": { + "name": "NRL Team at NHS Digital", + "telecom": { + "system": "email", + "value": "nrls@nhs.net", + "use": "work" + } + }, + "description": "A ValueSet that identifies the format of the content of the target document or record of a National Record Locator pointer.", + "copyright": "Copyright © 2024 NHS Digital", + "compose": { + "include": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "concept": [ + { + "code": "urn:nhs-ic:record-contact", + "display": "Contact details (HTTP Unsecured)" + }, + { + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Document" + } + ] + } + ] + } +} From 202b4f71fdfc61b7bde56e1ae5bfa9731d438b8d Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Wed, 4 Dec 2024 15:09:28 +0000 Subject: [PATCH 052/106] NRL-518 Update postman link in documentation --- swagger/consumer-static/narrative.yaml | 2 +- swagger/producer-static/narrative.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/swagger/consumer-static/narrative.yaml b/swagger/consumer-static/narrative.yaml index a349758ea..6b4522a33 100644 --- a/swagger/consumer-static/narrative.yaml +++ b/swagger/consumer-static/narrative.yaml @@ -190,7 +190,7 @@ info: Right click the icon and save link as... to save the Postman collection to your device - [![Right click and save link as...](https://run.pstmn.io/button.svg)](https://github.com/NHSDigital/NRLF/raw/main/postman_collection.json) + [![Right click and save link as...](https://run.pstmn.io/button.svg)](https://github.com/NHSDigital/NRLF/raw/develop/postman_collection.json) ### Integration testing diff --git a/swagger/producer-static/narrative.yaml b/swagger/producer-static/narrative.yaml index 91041187e..0a3af911f 100644 --- a/swagger/producer-static/narrative.yaml +++ b/swagger/producer-static/narrative.yaml @@ -179,7 +179,7 @@ info: Right click the icon and save link as... to save the Postman collection to your device - [![Right click and save link as...](https://run.pstmn.io/button.svg)](https://github.com/NHSDigital/NRLF/raw/main/postman_collection.json) + [![Right click and save link as...](https://run.pstmn.io/button.svg)](https://github.com/NHSDigital/NRLF/raw/develop/postman_collection.json) ### Integration testing From 9f44e028a1b8f52994d6d620fbf604a09f7d82c6 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Wed, 4 Dec 2024 17:50:17 +0000 Subject: [PATCH 053/106] NRL-519 include practice setting in integration tests --- .../createDocumentReference-success.feature | 165 +++++++++--------- tests/features/utils/data.py | 40 +++-- 2 files changed, 112 insertions(+), 93 deletions(-) diff --git a/tests/features/producer/createDocumentReference-success.feature b/tests/features/producer/createDocumentReference-success.feature index 929e2a037..202595190 100644 --- a/tests/features/producer/createDocumentReference-success.feature +++ b/tests/features/producer/createDocumentReference-success.feature @@ -6,14 +6,15 @@ Feature: Producer - createDocumentReference - Success Scenarios | system | value | | http://snomed.info/sct | 736253002 | When producer 'ANGY1' creates a DocumentReference with values: - | property | value | - | subject | 9278693472 | - | status | current | - | type | 736253002 | - | category | 734163000 | - | custodian | ANGY1 | - | author | HAR1 | - | url | https://example.org/my-doc.pdf | + | property | value | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | practiceSetting | 788002001 | Then the response status code is 201 And the response is an OperationOutcome with 1 issue And the OperationOutcome contains the issue: @@ -36,14 +37,15 @@ Feature: Producer - createDocumentReference - Success Scenarios And the response has a Location header And the Location header starts with '/producer/FHIR/R4/DocumentReference/ANGY1-' And the resource in the Location header exists with values: - | property | value | - | subject | 9278693472 | - | status | current | - | type | 736253002 | - | category | 734163000 | - | custodian | ANGY1 | - | author | HAR1 | - | url | https://example.org/my-doc.pdf | + | property | value | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | practiceSetting | 788002001 | # # NRL-766 Resolve custodian suffix issues # Scenario: Successfully create a Document Pointer (care plan) with custodian suffix @@ -97,26 +99,28 @@ Feature: Producer - createDocumentReference - Success Scenarios | system | value | | http://snomed.info/sct | 736253002 | And a DocumentReference resource exists with values: - | property | value | - | id | ANGY1-111-SupercedeDocRefTest1 | - | subject | 9278693472 | - | status | current | - | type | 736253002 | - | category | 734163000 | - | contentType | application/pdf | - | url | https://example.org/my-doc.pdf | - | custodian | ANGY1 | - | author | HAR1 | + | property | value | + | id | ANGY1-111-SupercedeDocRefTest1 | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | ANGY1 | + | author | HAR1 | + | practiceSetting | 788002001 | When producer 'ANGY1' creates a DocumentReference with values: - | property | value | - | subject | 9278693472 | - | status | current | - | type | 736253002 | - | category | 734163000 | - | custodian | ANGY1 | - | author | HAR1 | - | url | https://example.org/newdoc.pdf | - | supercedes | ANGY1-111-SupercedeDocRefTest1 | + | property | value | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/newdoc.pdf | + | supercedes | ANGY1-111-SupercedeDocRefTest1 | + | practiceSetting | 788002001 | Then the response status code is 201 And the response is an OperationOutcome with 1 issue And the OperationOutcome contains the issue: @@ -139,14 +143,15 @@ Feature: Producer - createDocumentReference - Success Scenarios And the response has a Location header And the Location header starts with '/producer/FHIR/R4/DocumentReference/ANGY1-' And the resource in the Location header exists with values: - | property | value | - | subject | 9278693472 | - | status | current | - | type | 736253002 | - | category | 734163000 | - | custodian | ANGY1 | - | author | HAR1 | - | url | https://example.org/newdoc.pdf | + | property | value | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/newdoc.pdf | + | practiceSetting | 788002001 | And the resource with id 'ANGY1-111-SupercedeDocRefTest1' does not exist # Create document reference with relatesTo - not code='replaces' @@ -157,14 +162,15 @@ Feature: Producer - createDocumentReference - Success Scenarios | system | value | | http://snomed.info/sct | | When producer 'ANGY1' creates a DocumentReference with values: - | property | value | - | subject | 9278693472 | - | status | current | - | type | | - | category | | - | custodian | ANGY1 | - | author | HAR1 | - | url | https://example.org/my-doc.pdf | + | property | value | + | subject | 9278693472 | + | status | current | + | type | | + | category | | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | practiceSetting | 788002001 | Then the response status code is 201 And the response is an OperationOutcome with 1 issue And the OperationOutcome contains the issue: @@ -229,15 +235,16 @@ Feature: Producer - createDocumentReference - Success Scenarios | https://nicip.nhs.uk | MAULR | | https://nicip.nhs.uk | MAXIB | When producer 'ANGY1' creates a DocumentReference with values: - | property | value | - | subject | 9278693472 | - | status | current | - | type | MAULR | - | type_system | https://nicip.nhs.uk | - | category | 721981007 | - | custodian | ANGY1 | - | author | HAR1 | - | url | https://example.org/my-doc.pdf | + | property | value | + | subject | 9278693472 | + | status | current | + | type | MAULR | + | type_system | https://nicip.nhs.uk | + | category | 721981007 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | practiceSetting | 788002001 | Then the response status code is 201 And the response is an OperationOutcome with 1 issue And the OperationOutcome contains the issue: @@ -260,25 +267,27 @@ Feature: Producer - createDocumentReference - Success Scenarios And the response has a Location header And the Location header starts with '/producer/FHIR/R4/DocumentReference/ANGY1-' And the resource in the Location header exists with values: - | property | value | - | subject | 9278693472 | - | status | current | - | type | MAULR | - | type_system | https://nicip.nhs.uk | - | category | 721981007 | - | custodian | ANGY1 | - | author | HAR1 | - | url | https://example.org/my-doc.pdf | + | property | value | + | subject | 9278693472 | + | status | current | + | type | MAULR | + | type_system | https://nicip.nhs.uk | + | category | 721981007 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | practiceSetting | 788002001 | When producer 'ANGY1' creates a DocumentReference with values: - | property | value | - | subject | 9278693472 | - | status | current | - | type | MAXIB | - | type_system | https://nicip.nhs.uk | - | category | 103693007 | - | custodian | ANGY1 | - | author | HAR1 | - | url | https://example.org/my-doc.pdf | + | property | value | + | subject | 9278693472 | + | status | current | + | type | MAXIB | + | type_system | https://nicip.nhs.uk | + | category | 103693007 | + | custodian | ANGY1 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | practiceSetting | 788002001 | Then the response status code is 201 And the response is an OperationOutcome with 1 issue And the OperationOutcome contains the issue: diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 11288945a..da40a9424 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -1,4 +1,9 @@ -from layer.nrlf.core.constants import CATEGORY_ATTRIBUTES, TYPE_ATTRIBUTES +from layer.nrlf.core.constants import ( + CATEGORY_ATTRIBUTES, + SNOMED_PRACTICE_SETTINGS, + SNOMED_SYSTEM_URL, + TYPE_ATTRIBUTES, +) from nrlf.producer.fhir.r4.model import ( Attachment, CodeableConcept, @@ -36,24 +41,13 @@ def create_test_document_reference(items: dict) -> DocumentReference: ) ) ], - context=DocumentReferenceContext( - practiceSetting=CodeableConcept( - coding=[ - Coding( - system="http://snomed.info/sct", - code="390826005", - display="Mental health caregiver support", - ) - ] - ) - ), ) if items.get("id"): base_doc_ref.id = items["id"] if type_code := items.get("type"): - type_system = items.get("type_system", "http://snomed.info/sct") + type_system = items.get("type_system", SNOMED_SYSTEM_URL) type_str = f"{type_system}|{type_code}" type_display = items.get( "type_display", TYPE_ATTRIBUTES.get(type_str, {}).get("display") @@ -90,13 +84,13 @@ def create_test_document_reference(items: dict) -> DocumentReference: if items.get("category"): category_display = CATEGORY_ATTRIBUTES.get( - f"http://snomed.info/sct|{items['category']}", {} + f"SNOMED_SYSTEM_URL|{items['category']}", {} ).get("display") base_doc_ref.category = [ CodeableConcept( coding=[ Coding( - system="http://snomed.info/sct", + system=SNOMED_SYSTEM_URL, code=items["category"], display=category_display, ) @@ -118,6 +112,22 @@ def create_test_document_reference(items: dict) -> DocumentReference: ) ] + if practice_setting_code := items.get("practiceSetting"): + practice_setting_display = SNOMED_PRACTICE_SETTINGS.get( + str(practice_setting_code), "Unknown practice setting" + ) + base_doc_ref.context = DocumentReferenceContext( + practiceSetting=CodeableConcept( + coding=[ + Coding( + system=SNOMED_SYSTEM_URL, + code=str(practice_setting_code), + display=practice_setting_display, + ) + ] + ) + ) + return base_doc_ref From ef966237a58a0de17a79c6ab42aec4d212d34eca Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Wed, 4 Dec 2024 19:09:49 +0000 Subject: [PATCH 054/106] NRL-519 include practice setting in integration tests --- tests/features/utils/data.py | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index da40a9424..a4a23ea17 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -30,6 +30,23 @@ def create_test_document_reference(items: dict) -> DocumentReference: + + if practice_setting_code := items.get("practiceSetting"): + practice_setting_display = SNOMED_PRACTICE_SETTINGS.get( + str(practice_setting_code), "Unknown practice setting" + ) + context = DocumentReferenceContext( + practiceSetting=CodeableConcept( + coding=[ + Coding( + system=SNOMED_SYSTEM_URL, + code=str(practice_setting_code), + display=practice_setting_display, + ) + ] + ) + ) + base_doc_ref = DocumentReference.model_construct( resourceType="DocumentReference", status=items.get("status", "current"), @@ -41,6 +58,7 @@ def create_test_document_reference(items: dict) -> DocumentReference: ) ) ], + context=context, ) if items.get("id"): @@ -112,22 +130,6 @@ def create_test_document_reference(items: dict) -> DocumentReference: ) ] - if practice_setting_code := items.get("practiceSetting"): - practice_setting_display = SNOMED_PRACTICE_SETTINGS.get( - str(practice_setting_code), "Unknown practice setting" - ) - base_doc_ref.context = DocumentReferenceContext( - practiceSetting=CodeableConcept( - coding=[ - Coding( - system=SNOMED_SYSTEM_URL, - code=str(practice_setting_code), - display=practice_setting_display, - ) - ] - ) - ) - return base_doc_ref From dbde35db3a933ea97abaf95141e691f809592c33 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Wed, 4 Dec 2024 19:59:07 +0000 Subject: [PATCH 055/106] NRL-519 include practice setting in integration tests --- tests/features/utils/data.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index a4a23ea17..053a42e76 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -31,21 +31,10 @@ def create_test_document_reference(items: dict) -> DocumentReference: - if practice_setting_code := items.get("practiceSetting"): - practice_setting_display = SNOMED_PRACTICE_SETTINGS.get( - str(practice_setting_code), "Unknown practice setting" - ) - context = DocumentReferenceContext( - practiceSetting=CodeableConcept( - coding=[ - Coding( - system=SNOMED_SYSTEM_URL, - code=str(practice_setting_code), - display=practice_setting_display, - ) - ] - ) - ) + practice_setting_code = items.get("practiceSetting", "788007007") + practice_setting_display = SNOMED_PRACTICE_SETTINGS.get( + str(practice_setting_code), "General practice service" + ) base_doc_ref = DocumentReference.model_construct( resourceType="DocumentReference", @@ -58,7 +47,17 @@ def create_test_document_reference(items: dict) -> DocumentReference: ) ) ], - context=context, + context=DocumentReferenceContext( + practiceSetting=CodeableConcept( + coding=[ + Coding( + system=SNOMED_SYSTEM_URL, + code=str(practice_setting_code), + display=practice_setting_display, + ) + ] + ) + ), ) if items.get("id"): From 4b3ab850a254e478274d96cd6d45ca527bb76697 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Wed, 4 Dec 2024 20:33:25 +0000 Subject: [PATCH 056/106] NRL-519 include practice setting in integration tests --- tests/features/producer/readDocumentReference-success.feature | 4 ++-- tests/features/utils/data.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/features/producer/readDocumentReference-success.feature b/tests/features/producer/readDocumentReference-success.feature index bf0e340b1..2eb19448b 100644 --- a/tests/features/producer/readDocumentReference-success.feature +++ b/tests/features/producer/readDocumentReference-success.feature @@ -79,8 +79,8 @@ Feature: Producer - readDocumentReference - Success Scenarios "coding": [ { "system": "http://snomed.info/sct", - "code": "390826005", - "display": "Mental health caregiver support" + "code": "788007007", + "display": "General practice service" } ] } diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 053a42e76..5bc300585 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -101,7 +101,7 @@ def create_test_document_reference(items: dict) -> DocumentReference: if items.get("category"): category_display = CATEGORY_ATTRIBUTES.get( - f"SNOMED_SYSTEM_URL|{items['category']}", {} + f"{SNOMED_SYSTEM_URL}|{items['category']}", {} ).get("display") base_doc_ref.category = [ CodeableConcept( From 04b2c9ddc8aa19d01d2058c9f7ce8130ae07df17 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Wed, 4 Dec 2024 20:50:08 +0000 Subject: [PATCH 057/106] NRL-519 include practice setting in integration tests --- .../consumer/readDocumentReference-success.feature | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/features/consumer/readDocumentReference-success.feature b/tests/features/consumer/readDocumentReference-success.feature index b1e1add80..f0854af16 100644 --- a/tests/features/consumer/readDocumentReference-success.feature +++ b/tests/features/consumer/readDocumentReference-success.feature @@ -77,8 +77,8 @@ Feature: Consumer - readDocumentReference - Success Scenarios "coding": [ { "system": "http://snomed.info/sct", - "code": "390826005", - "display": "Mental health caregiver support" + "code": "788007007", + "display": "General practice service" } ] } @@ -163,8 +163,8 @@ Feature: Consumer - readDocumentReference - Success Scenarios "coding": [ { "system": "http://snomed.info/sct", - "code": "390826005", - "display": "Mental health caregiver support" + "code": "788007007", + "display": "General practice service" } ] } From 97be4870cf7abc15861ca14cdd44b79dab04067b Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:10:30 +0000 Subject: [PATCH 058/106] NRL-1137 implement daily builds workflow --- .github/workflows/daily-build.yml | 85 +++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 .github/workflows/daily-build.yml diff --git a/.github/workflows/daily-build.yml b/.github/workflows/daily-build.yml new file mode 100644 index 000000000..0fc15523c --- /dev/null +++ b/.github/workflows/daily-build.yml @@ -0,0 +1,85 @@ +name: Build NRL Project on Environment +run-name: Build NRL Project on ${{ inputs.environment || 'dev' }} +permissions: + id-token: write + contents: read + actions: write + +on: + schedule: + - cron: "0 1 * * *" + workflow_dispatch: + inputs: + environment: + type: environment + description: "The environment to deploy changes to" + default: "dev" + required: true + +jobs: + build: + name: Build - develop + runs-on: [self-hosted, ci] + + steps: + - name: Git clone - develop + uses: actions/checkout@v4 + with: + ref: develop + + - name: Setup asdf cache + uses: actions/cache@v4 + with: + path: ~/.asdf + key: ${{ runner.os }}-asdf-${{ hashFiles('**/.tool-versions') }} + restore-keys: | + ${{ runner.os }}-asdf- + + - name: Install asdf + uses: asdf-vm/actions/install@v3.0.2 + with: + asdf_branch: v0.13.1 + + - name: Install zip + run: sudo apt-get install zip + + - name: Setup Python environment + run: | + poetry install --no-root + source $(poetry env info --path)/bin/activate + + - name: Run Linting + run: make lint + + - name: Run Unit Tests + run: make test + + - name: Build Project + run: make build + + - name: Configure Management Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-west-2 + role-to-assume: ${{ secrets.MGMT_ROLE_ARN }} + role-session-name: github-actions-ci-${{ inputs.environment || 'dev' }}-${{ github.run_id }} + + - name: Add S3 Permissions to Lambda + run: | + account=$(echo '${{ inputs.environment || 'dev' }}' | cut -d '-' -f1) + inactive_stack=$(poetry run python ./scripts/get_env_config.py inactive-stack ${{ inputs.environment || 'dev' }}) + make get-s3-perms ENV=${account} TF_WORKSPACE_NAME=${inactive_stack} + + - name: Save Build Artifacts + uses: actions/upload-artifact@v4 + with: + name: build-artifacts + path: | + dist/*.zip + !dist/nrlf_permissions.zip + + - name: Save NRLF Permissions cache + uses: actions/cache/save@v4 + with: + key: ${{ github.run_id }}-nrlf-permissions + path: dist/nrlf_permissions.zip From 4d1220a277d0c18bbe966f39754e11e693e5a64e Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:11:18 +0000 Subject: [PATCH 059/106] NRL-477 validate docStatus --- api/consumer/swagger.yaml | 2 +- .../tests/test_create_document_reference.py | 41 +++++++++++++++++++ api/producer/swagger.yaml | 2 +- .../tests/test_upsert_document_reference.py | 41 +++++++++++++++++++ layer/nrlf/consumer/fhir/r4/model.py | 4 +- layer/nrlf/producer/fhir/r4/model.py | 4 +- layer/nrlf/producer/fhir/r4/strict_model.py | 2 +- 7 files changed, 89 insertions(+), 7 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 47c1a072c..3f8141469 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -788,7 +788,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "[^\\s]+(\\s[^\\s]+)*" + pattern: "^(entered-in-error|amended|preliminary|final)$" description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" diff --git a/api/producer/createDocumentReference/tests/test_create_document_reference.py b/api/producer/createDocumentReference/tests/test_create_document_reference.py index 19e29569b..b58d3c234 100644 --- a/api/producer/createDocumentReference/tests/test_create_document_reference.py +++ b/api/producer/createDocumentReference/tests/test_create_document_reference.py @@ -411,6 +411,47 @@ def test_create_document_reference_with_no_practiceSetting(): } +def test_create_document_reference_with_invalid_docStatus(): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_ref.docStatus = "invalid" + + event = create_test_api_gateway_event( + headers=create_headers(), + body=doc_ref.model_dump_json(exclude_none=True), + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "400", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + } + ], + }, + "diagnostics": "Request body could not be parsed (docStatus: String should match pattern '^(entered-in-error|amended|preliminary|final)$')", + "expression": ["docStatus"], + }, + ], + } + + def test_create_document_reference_invalid_custodian_id(): doc_ref = load_document_reference("Y05868-736253002-Valid") diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index bdd0127b8..fa10b2790 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1351,7 +1351,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "[^\\s]+(\\s[^\\s]+)*" + pattern: "^(entered-in-error|amended|preliminary|final)$" description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" diff --git a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py index 6f3878cfd..009c12075 100644 --- a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py +++ b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py @@ -455,6 +455,47 @@ def test_upsert_document_reference_with_no_practiceSetting(): } +def test_upsert_document_reference_with_invalid_docStatus(): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_ref.docStatus = "invalid" + + event = create_test_api_gateway_event( + headers=create_headers(), + body=doc_ref.model_dump_json(exclude_none=True), + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "400", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "OperationOutcome", + "issue": [ + { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "code": "MESSAGE_NOT_WELL_FORMED", + "display": "Message not well formed", + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + } + ], + }, + "diagnostics": "Request body could not be parsed (docStatus: String should match pattern '^(entered-in-error|amended|preliminary|final)$')", + "expression": ["docStatus"], + }, + ], + } + + def test_upsert_document_reference_invalid_producer_id(): doc_ref = load_document_reference("Y05868-736253002-Valid") doc_ref.id = "X26-99999-99999-999999" diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index 4665533b5..70167bf4a 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T09:43:58+00:00 +# timestamp: 2024-12-05T01:41:57+00:00 from __future__ import annotations @@ -580,7 +580,7 @@ class DocumentReference(BaseModel): Optional[str], Field( description="The status of the underlying document.", - pattern="[^\\s]+(\\s[^\\s]+)*", + pattern="^(entered-in-error|amended|preliminary|final)$", ), ] = None type: Annotated[ diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index d96b7ce73..6a7e49abb 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T10:10:52+00:00 +# timestamp: 2024-12-05T01:41:53+00:00 from __future__ import annotations @@ -564,7 +564,7 @@ class DocumentReference(BaseModel): Optional[str], Field( description="The status of the underlying document.", - pattern="[^\\s]+(\\s[^\\s]+)*", + pattern="^(entered-in-error|amended|preliminary|final)$", ), ] = None type: Annotated[ diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index e4edefc58..a7d73861a 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-20T10:10:54+00:00 +# timestamp: 2024-12-05T01:41:55+00:00 from __future__ import annotations From da4802ccc3ad49c0a53c262627eb8ebbecc55fc1 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:12:08 +0000 Subject: [PATCH 060/106] NRL-502 validate format code for attachment types --- layer/nrlf/core/tests/test_validators.py | 60 +++++++++++++++++++ layer/nrlf/core/validators.py | 30 ++++++++++ .../createDocumentReference-failure.feature | 46 ++++++++++++++ 3 files changed, 136 insertions(+) diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index fd02deb20..f243c1843 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1522,3 +1522,63 @@ def test_validate_ssp_content_with_multiple_asids(): "diagnostics": "Multiple ASID identifiers provided. Only a single valid ASID identifier can be provided in the context.related.", "expression": ["context.related"], } + + +def test_validate_content_format_invalid_code_for_unstructured_document(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["format"] = { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:contact", + "display": "Contact details", + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + "expression": ["content[0].format.code"], + } + + +def test_validate_content_format_invalid_code_for_contact_details(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"]["contentType"] = "text/html" + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", + "expression": ["content[0].format.code"], + } diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index bdb01b78b..907e1f1b3 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -141,6 +141,7 @@ def validate(self, data: Dict[str, Any] | DocumentReference): self._validate_category(resource) self._validate_author(resource) self._validate_type_category_mapping(resource) + self._validate_content_format(resource) if resource.content[0].extension: self._validate_content_extension(resource) @@ -481,6 +482,35 @@ def _validate_type_category_mapping(self, model: DocumentReference): field="category.coding[0].code", ) + def _validate_content_format(self, model: DocumentReference): + """ + Validate the content.format field contains an appropriate coding. + """ + logger.log(LogReference.VALIDATOR001, step="content_format") + + logger.debug("Validating format") + for i, content in enumerate(model.content): + if ( + content.attachment.contentType == "text/html" + and content.format.code != "urn:nhs-ic:record-contact" + ): + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid content format code: {content.format.code} format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", + field=f"content[{i}].format.code", + ) + elif ( + content.attachment.contentType == "application/pdf" + and content.format.code != "urn:nhs-ic:unstructured" + ): + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid content format code: {content.format.code} format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + field=f"content[{i}].format.code", + ) + def _validate_content_extension(self, model: DocumentReference): """ Validate the content.extension field contains an appropriate coding. diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index c1d09f483..1ce52efac 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -407,6 +407,52 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ + Scenario: Invalid format code for attachment type + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'X26' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "application/pdf", + "language": "en-US", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", + "size": 3654, + "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", + "title": "Mental health crisis plan report", + "creation": "2022-12-21T10:45:41+11:00" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details" + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", + "expression": ["content[0].format.code"] + } + """ + # Invalid document reference - invalid Type # NRL-769 Known issue: Type display is not validated # Scenario: Invalid type (valid code but wrong display value) From 772a93b3deaa56af2859091656f39ca7a34e8fbe Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:17:45 +0000 Subject: [PATCH 061/106] NRL-502 add integration test --- .../createDocumentReference-failure.feature | 53 +++++++++++++++++-- 1 file changed, 49 insertions(+), 4 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 1ce52efac..427ce3488 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -407,7 +407,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - Scenario: Invalid format code for attachment type + Scenario: Invalid format code for attachment type contact details Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'X26' is authorised to access pointer types: | system | value | @@ -418,7 +418,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", + "contentType": "text/html", "language": "en-US", "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", "size": 3654, @@ -428,8 +428,8 @@ Feature: Producer - createDocumentReference - Failure Scenarios }, "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:record-contact", - "display": "Contact details" + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured Documents" } """ Then the response status code is 400 @@ -453,6 +453,51 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ + Scenario: Invalid format code for attachment type pdf + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'X26' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "application/pdf", + "language": "en-US", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", + "size": 3654, + "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", + "title": "Mental health crisis plan report", + "creation": "2022-12-21T10:45:41+11:00" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details" + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + "expression": ["content[0].format.code"], + } + """ + # Invalid document reference - invalid Type # NRL-769 Known issue: Type display is not validated # Scenario: Invalid type (valid code but wrong display value) From a8c035268ef2ee79b82d54696d5bbe4105e64d40 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:35:04 +0000 Subject: [PATCH 062/106] NRL-502 add format to setup --- .../producer/createDocumentReference-failure.feature | 4 ++-- tests/features/utils/data.py | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 427ce3488..20a0f83e9 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -409,7 +409,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios Scenario: Invalid format code for attachment type contact details Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - And the organisation 'X26' is authorised to access pointer types: + And the organisation 'TSTCUS' is authorised to access pointer types: | system | value | | http://snomed.info/sct | 1363501000000100 | | http://snomed.info/sct | 736253002 | @@ -455,7 +455,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios Scenario: Invalid format code for attachment type pdf Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - And the organisation 'X26' is authorised to access pointer types: + And the organisation 'TSTCUS' is authorised to access pointer types: | system | value | | http://snomed.info/sct | 1363501000000100 | | http://snomed.info/sct | 736253002 | diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 11288945a..e47c2e0a8 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -33,7 +33,12 @@ def create_test_document_reference(items: dict) -> DocumentReference: attachment=Attachment( contentType=items.get("contentType", "application/json"), url=items["url"], - ) + ), + format=Coding( + system="https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + code="urn:nhs-ic:unstructured", + display="Unstructured document", + ), ) ], context=DocumentReferenceContext( From 37aed66ebdfe6451f20a63194674aef0c8ba7d53 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 02:48:06 +0000 Subject: [PATCH 063/106] NRL-502 fix tests --- .../readDocumentReference-success.feature | 10 +++++ .../createDocumentReference-failure.feature | 41 +++++++++++-------- .../readDocumentReference-success.feature | 5 +++ 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/tests/features/consumer/readDocumentReference-success.feature b/tests/features/consumer/readDocumentReference-success.feature index b1e1add80..5c0d83979 100644 --- a/tests/features/consumer/readDocumentReference-success.feature +++ b/tests/features/consumer/readDocumentReference-success.feature @@ -69,6 +69,11 @@ Feature: Consumer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" } } ], @@ -155,6 +160,11 @@ Feature: Consumer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" } } ], diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 20a0f83e9..bb1601f47 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -440,16 +440,18 @@ Feature: Producer - createDocumentReference - Failure Scenarios "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", - "expression": ["content[0].format.code"] + "diagnostics": "The Category code of the provided document 'http://snomed.info/sct|1102421000000108' must match the allowed category for pointer type 'http://snomed.info/sct|736253002' with a category value of 'http://snomed.info/sct|734163000'", + "expression": [ + "category.coding[0].code" + ] } """ @@ -482,19 +484,22 @@ Feature: Producer - createDocumentReference - Failure Scenarios And the response is an OperationOutcome with 1 issue And the OperationOutcome contains the issue: """ + { "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", - "expression": ["content[0].format.code"], + "diagnostics": "The Category code of the provided document 'http://snomed.info/sct|1102421000000108' must match the allowed category for pointer type 'http://snomed.info/sct|736253002' with a category value of 'http://snomed.info/sct|734163000'", + "expression": [ + "category.coding[0].code" + ] } """ diff --git a/tests/features/producer/readDocumentReference-success.feature b/tests/features/producer/readDocumentReference-success.feature index bf0e340b1..04c4228ec 100644 --- a/tests/features/producer/readDocumentReference-success.feature +++ b/tests/features/producer/readDocumentReference-success.feature @@ -71,6 +71,11 @@ Feature: Producer - readDocumentReference - Success Scenarios "attachment": { "contentType": "application/pdf", "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" } } ], From 31d2dc93a666e77a13d5a62f8e2d3f0776504636 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Thu, 5 Dec 2024 03:01:06 +0000 Subject: [PATCH 064/106] NRL-502 fix new tests --- .../createDocumentReference-failure.feature | 74 +++++++++---------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index bb1601f47..b30ff97bc 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -418,19 +418,16 @@ Feature: Producer - createDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "text/html", - "language": "en-US", - "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", - "size": 3654, - "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", - "title": "Mental health crisis plan report", - "creation": "2022-12-21T10:45:41+11:00" + "contentType": "text/html", + "url": "someContact.co.uk" }, "format": { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:unstructured", - "display": "Unstructured Documents" + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured", + "display": "Unstructured document" + } } + ] """ Then the response status code is 400 And the response is an OperationOutcome with 1 issue @@ -440,17 +437,17 @@ Feature: Producer - createDocumentReference - Failure Scenarios "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource" - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "The Category code of the provided document 'http://snomed.info/sct|1102421000000108' must match the allowed category for pointer type 'http://snomed.info/sct|736253002' with a category value of 'http://snomed.info/sct|734163000'", + "diagnostics": "Invalid content format code: urn:nhs-ic:unstructured format code must be 'urn:nhs-ic:record-contact' for Contact details attachments.", "expression": [ - "category.coding[0].code" + "content[0].format.code" ] } """ @@ -466,19 +463,20 @@ Feature: Producer - createDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", - "language": "en-US", - "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", - "size": 3654, - "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", - "title": "Mental health crisis plan report", - "creation": "2022-12-21T10:45:41+11:00" + "contentType": "application/pdf", + "language": "en-UK", + "url": "https://spine-proxy.national.ncrs.nhs.uk/https%3A%2F%2Fp1.nhs.uk%2FMentalhealthCrisisPlanReport.pdf", + "hash": "2jmj7l5rSw0yVb/vlWAYkK/YBwk=", + "title": "Mental health crisis plan report", + "creation": "2022-12-21T10:45:41+11:00" }, "format": { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:record-contact", - "display": "Contact details" + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details" + } } + ] """ Then the response status code is 400 And the response is an OperationOutcome with 1 issue @@ -488,17 +486,17 @@ Feature: Producer - createDocumentReference - Failure Scenarios "severity": "error", "code": "value", "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource" - } - ] + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] }, - "diagnostics": "The Category code of the provided document 'http://snomed.info/sct|1102421000000108' must match the allowed category for pointer type 'http://snomed.info/sct|736253002' with a category value of 'http://snomed.info/sct|734163000'", + "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", "expression": [ - "category.coding[0].code" + "content[0].format.code" ] } """ From 2aef83dd6026a5e9b343c935d0fb05c5fec4eb33 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Thu, 5 Dec 2024 08:44:26 +0000 Subject: [PATCH 065/106] NRL-519 add create failure scenarios for invalid practice settings --- .../createDocumentReference-failure.feature | 202 +++++++----------- 1 file changed, 81 insertions(+), 121 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index c1d09f483..9f6efda40 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -194,7 +194,6 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - # Invalid document reference - invalid custodian ID # Invalid document reference - invalid relatesTo target # Invalid document reference - invalid producer ID in relatesTo target Scenario: Unauthorised supersede - target belongs to a different custodian @@ -331,7 +330,6 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - # Credentials - missing pointer type for create Scenario: Producer lacks the permission for the pointer type requested Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'ANGY1' is authorised to access pointer types: @@ -407,93 +405,93 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - # Invalid document reference - invalid Type - # NRL-769 Known issue: Type display is not validated - # Scenario: Invalid type (valid code but wrong display value) - # Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - # And the organisation 'TSTCUS' is authorised to access pointer types: - # | system | value | - # | http://snomed.info/sct | 1363501000000100 | - # | http://snomed.info/sct | 736253002 | - # When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'type' is: - # """ - # "type": { - # "coding": [ - # { - # "system": "http://snomed.info/sct", - # "code": "736253002", - # "display": "Emergency Healthcare Plan" - # } - # ] - # } - # """ - # Then the response status code is 400 - # And the response is an OperationOutcome with 1 issue - # And the OperationOutcome contains the issue: - # """ - # { - # "severity": "error", - # "code": "invalid", - # "details": { - # "coding": [ - # { - # "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - # "code": "BAD_REQUEST", - # "display": "Bad request" - # } - # ] - # }, - # "diagnostics": "The display does not match the expected value for this type", - # "expression": [ - # "type.coding.display" - # ] - # } - # """ + Scenario: Invalid practice setting (not in value set) + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'X26' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 736253002 | + When producer 'X26' creates a DocumentReference with values: + | property | value | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 1102421000000108 | + | custodian | X26 | + | author | HAR1 | + | url | https://example.org/my-doc.pdf | + | practiceSetting | 12345 | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid practice setting code: 12345 Practice Setting coding must be a member of value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", + "expression": ["context.practiceSetting.coding[0].code"], + + } + """ + + Scenario: Invalid practice setting (valid code but wrong display value) + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 1363501000000100 | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'context' is: + """ + "context": { + "practiceSetting": { + "coding": [ + { + "system": "http://snomed.info/sct", + "code": "788002001", + "display": "Ophthalmology service" + } + ] + } + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid practice setting coding: display Ophthalmology service does not match the expected display for 788002001 Practice Setting coding is bound to value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", + "expression": [ + "context.practiceSetting.coding[0]" + ] + } + """ + # Invalid document reference - empty content[0].attachment.url # Invalid document reference - create another producers document # Invalid document reference - bad JSON - # Invalid document reference - invalid status (NRL-476 to ensure only 'current' is accepted) - # Scenario: Invalid document reference - invalid status - # Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - # And the organisation 'ANGY1' is authorised to access pointer types: - # | system | value | - # | http://snomed.info/sct | 736253002 | - # When producer 'ANGY1' creates a DocumentReference with values: - # | property | value | - # | subject | 9999999999 | - # | status | notarealStatus | - # | type | 736253002 | - # | category | 734163000 | - # | custodian | ANGY1 | - # | author | HAR1 | - # | url | https://example.org/my-doc.pdf | - # Then the response status code is 400 - # And the response is an OperationOutcome with 1 issue - # And the OperationOutcome contains the issue: - # """ - # { - # "severity": "error", - # "code": "forbidden", - # "details": { - # "coding": [ - # { - # "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - # "code": "AUTHOR_CREDENTIALS_ERROR", - # "display": "Author credentials error" - # } - # ] - # }, - # "diagnostics": "The type of the provided DocumentReference is not in the list of allowed types for this organisation", - # "expression": [ - # "type.coding[0].code" - # ] - # } - # """ - # Invalid document reference - invalid author (NRL-474) # Invalid document reference - invalid content (NRL-518) # Invalid document reference - invalid context.related for an SSP url # Invalid document reference - missing context.related for an SSP url - # Invalid document reference - invalid context.practiceSetting (NRL-519) # Invalid document reference - invalid docStatus (NRL-477) # Invalid document reference - duplicate keys # Invalid document reference - duplicate relatesTo targets in URL @@ -573,44 +571,6 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - Scenario: Mismatched Category Code for Document Reference Type - Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - And the organisation 'X26' is authorised to access pointer types: - | system | value | - | http://snomed.info/sct | 1363501000000100 | - | http://snomed.info/sct | 736253002 | - When producer 'X26' creates a DocumentReference with values: - | property | value | - | subject | 9999999999 | - | status | current | - | type | 736253002 | - | category | 1102421000000108 | - | custodian | X26 | - | author | HAR1 | - | url | https://example.org/my-doc.pdf | - Then the response status code is 400 - And the response is an OperationOutcome with 1 issue - And the OperationOutcome contains the issue: - """ - { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource" - } - ] - }, - "diagnostics": "The Category code of the provided document 'http://snomed.info/sct|1102421000000108' must match the allowed category for pointer type 'http://snomed.info/sct|736253002' with a category value of 'http://snomed.info/sct|734163000'", - "expression": [ - "category.coding[0].code" - ] - } - """ - Scenario Outline: Invalid display value for type or category (imaging) Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'ANGY1' is authorised to access pointer types: From 4f6469475d50429c1b7c8d191d63c3433808d364 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Thu, 5 Dec 2024 08:50:08 +0000 Subject: [PATCH 066/106] NRL-519 update smoketest to use a valid practiceSetting --- tests/smoke/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index a44ebe9bb..69e1f6358 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -81,8 +81,8 @@ def build_document_reference( coding=[ Coding( system="http://snomed.info/sct", - code="390826005", - display="Mental health caregiver support", + code="224891009", + display="Healthcare services", ) ] ) From 8e4aa3e4eda079a1a7ae23cb2aa8c5f55516a715 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Thu, 5 Dec 2024 09:04:30 +0000 Subject: [PATCH 067/106] NRL-519 fix confounding issue in create integration test --- tests/features/producer/createDocumentReference-failure.feature | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 9f6efda40..2aeac74c4 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -416,7 +416,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios | subject | 9999999999 | | status | current | | type | 736253002 | - | category | 1102421000000108 | + | category | 734163000 | | custodian | X26 | | author | HAR1 | | url | https://example.org/my-doc.pdf | From c52fc09480985987be52b295daae2e1f9316d74e Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Thu, 5 Dec 2024 09:16:00 +0000 Subject: [PATCH 068/106] NRL-519 fix formatting error --- .../features/producer/createDocumentReference-failure.feature | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 2aeac74c4..d407845b7 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -438,8 +438,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios ] }, "diagnostics": "Invalid practice setting code: 12345 Practice Setting coding must be a member of value set https://fhir.nhs.uk/England/ValueSet/England-PracticeSetting", - "expression": ["context.practiceSetting.coding[0].code"], - + "expression": ["context.practiceSetting.coding[0].code"] } """ From 5a46d45261b432456c840aaa6837f26f124cf997 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Thu, 5 Dec 2024 12:30:58 +0000 Subject: [PATCH 069/106] NRL-518 Remove tests no longer valid due to upstream pydantic validation --- layer/nrlf/core/tests/test_pydantic_errors.py | 86 +++++++++ layer/nrlf/core/tests/test_validators.py | 165 ++++-------------- 2 files changed, 122 insertions(+), 129 deletions(-) create mode 100644 layer/nrlf/core/tests/test_pydantic_errors.py diff --git a/layer/nrlf/core/tests/test_pydantic_errors.py b/layer/nrlf/core/tests/test_pydantic_errors.py new file mode 100644 index 000000000..4ea579d3b --- /dev/null +++ b/layer/nrlf/core/tests/test_pydantic_errors.py @@ -0,0 +1,86 @@ +from unittest.mock import Mock +import pytest +from nrlf.core.errors import ParseError +from nrlf.core.validators import DocumentReferenceValidator +from nrlf.tests.data import load_document_reference_json + +def test_validate_content_missing_attachment(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0].pop("attachment") + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content.0.attachment: Field required)", + "expression": ["content.0.attachment"], + } + +def test_validate_content_missing_content_type(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"].pop("contentType") + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content.0.attachment.contentType: Field required)", + "expression": ["content.0.attachment.contentType"], + } + +def test_validate_content_invalid_content_type(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"]["contentType"] = "invalid/type" + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content.0.attachment.contentType: String should match pattern '^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$')", + "expression": ["content.0.attachment.contentType"], + } diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index 20f08411f..d4a06d9b7 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1523,135 +1523,42 @@ def test_validate_ssp_content_with_multiple_asids(): "expression": ["context.related"], } - def test_validate_content_extension_invalid_code_and_display_mismatch(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": { - "coding": [ - { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - "code": "static", - "display": "Dynamic", - } - ] - }, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension display: Dynamic Extension display must be the same as code either 'Static' or 'Dynamic'", - "expression": [ - "content[0].extension[0].valueCodeableConcept.coding[0].display" - ], - } - - def test_validate_content_missing_attachment(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0].pop("attachment") - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "required", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Missing attachment in content", - "expression": ["content[0].attachment"], - } - - def test_validate_content_missing_content_type(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["attachment"].pop("contentType") - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "required", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Missing contentType in content.attachment", - "expression": ["content[0].attachment.contentType"], - } - - def test_validate_content_invalid_content_type(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["attachment"][ - "contentType" - ] = "invalid/type" - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid contentType: invalid/type. Must be 'application/pdf' or 'text/html'", - "expression": ["content[0].attachment.contentType"], - } - - def test_validate_content_valid_content_type(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") +def test_validate_content_extension_invalid_code_and_display_mismatch(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - document_ref_data["content"][0]["attachment"][ - "contentType" - ] = "application/pdf" + document_ref_data["content"][0]["extension"][0] = { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Dynamic", + } + ] + }, + } - result = validator.validate(document_ref_data) + result = validator.validate(document_ref_data) - assert result.is_valid is True - assert result.issues == [] + assert result.is_valid is False + assert result.resource.id == "Y05868-99999-99999-999999" + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid content extension display: Dynamic Extension display must be the same as code either 'Static' or 'Dynamic'", + "expression": [ + "content[0].extension[0].valueCodeableConcept.coding[0].display" + ], + } From b9f7d40122c5d4f73df3321b80edb77ab2229b23 Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Thu, 5 Dec 2024 13:16:29 +0000 Subject: [PATCH 070/106] NRL-519 implement sonarcloud suggestions --- layer/nrlf/core/validators.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 654827202..707ca5294 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -570,7 +570,7 @@ def _validate_author(self, model: DocumentReference): issue_code="invalid", error_code="INVALID_RESOURCE", diagnostics=f"Invalid author length: {len(model.author)} Author must only contain a single value", - field=f"author", + field="author", ) return @@ -582,7 +582,7 @@ def _validate_author(self, model: DocumentReference): issue_code="invalid", error_code="INVALID_IDENTIFIER_SYSTEM", diagnostics=f"Invalid author system: '{identifier.system}' Author system must be '{ODS_SYSTEM}'", - field=f"author[0].identifier.system", + field="author[0].identifier.system", ) return @@ -591,7 +591,7 @@ def _validate_author(self, model: DocumentReference): issue_code="value", error_code="INVALID_RESOURCE", diagnostics=f"Invalid author value: '{identifier.value}' Author value must be alphanumeric", - field=f"author[0].identifier.value", + field="author[0].identifier.value", ) return @@ -600,7 +600,7 @@ def _validate_author(self, model: DocumentReference): issue_code="value", error_code="INVALID_RESOURCE", diagnostics=f"Invalid author value: '{identifier.value}' Author value must be less than 13 characters", - field=f"author[0].identifier.value", + field="author[0].identifier.value", ) return @@ -617,8 +617,8 @@ def _validate_practiceSetting(self, model: DocumentReference): self.result.add_error( issue_code="value", error_code="INVALID_RESOURCE", - diagnostics=f"Invalid practice setting: must contain a Coding", - field=f"context.practiceSetting.coding", + diagnostics="Invalid practice setting: must contain a Coding", + field="context.practiceSetting.coding", ) return @@ -627,7 +627,7 @@ def _validate_practiceSetting(self, model: DocumentReference): issue_code="value", error_code="INVALID_RESOURCE", diagnostics=f"Invalid practice setting coding length: {len(model.context.practiceSetting.coding)} Practice Setting Coding must only contain a single value", - field=f"context.practiceSetting.coding", + field="context.practiceSetting.coding", ) return @@ -651,7 +651,7 @@ def _validate_practiceSetting(self, model: DocumentReference): issue_code="value", error_code="INVALID_RESOURCE", diagnostics=f"Invalid practice setting code: {practice_setting_value} Practice Setting coding must be a member of value set {PRACTICE_SETTING_VALUE_SET_URL}", - field=f"context.practiceSetting.coding[0].code", + field="context.practiceSetting.coding[0].code", ) return @@ -664,6 +664,6 @@ def _validate_practiceSetting(self, model: DocumentReference): issue_code="value", error_code="INVALID_RESOURCE", diagnostics=f"Invalid practice setting coding: display {practice_setting_display} does not match the expected display for {practice_setting_value} Practice Setting coding is bound to value set {PRACTICE_SETTING_VALUE_SET_URL}", - field=f"context.practiceSetting.coding[0]", + field="context.practiceSetting.coding[0]", ) return From bfc1850a3c60e8c3c61e09f121ee903e290d3c7e Mon Sep 17 00:00:00 2001 From: Kate Bobyn Date: Thu, 5 Dec 2024 13:17:34 +0000 Subject: [PATCH 071/106] NRL-519 implement sonarcloud suggestions --- layer/nrlf/core/validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 707ca5294..fa1743013 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -640,7 +640,7 @@ def _validate_practiceSetting(self, model: DocumentReference): issue_code="value", error_code="INVALID_RESOURCE", diagnostics=f"Invalid practice setting system: {practice_setting_system} Practice Setting system must be '{SNOMED_SYSTEM_URL}'", - field=f"context.practiceSetting.coding[0].system", + field="context.practiceSetting.coding[0].system", ) return From e6d10c6042ff53ebfab08455cc55752fedf8edcc Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Thu, 5 Dec 2024 16:27:32 +0000 Subject: [PATCH 072/106] NRL-518 Simplify test, fix lint warnings --- layer/nrlf/core/tests/test_pydantic_errors.py | 5 +++++ layer/nrlf/core/tests/test_validators.py | 15 +++------------ 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/layer/nrlf/core/tests/test_pydantic_errors.py b/layer/nrlf/core/tests/test_pydantic_errors.py index 4ea579d3b..dd8bfba54 100644 --- a/layer/nrlf/core/tests/test_pydantic_errors.py +++ b/layer/nrlf/core/tests/test_pydantic_errors.py @@ -1,9 +1,12 @@ from unittest.mock import Mock + import pytest + from nrlf.core.errors import ParseError from nrlf.core.validators import DocumentReferenceValidator from nrlf.tests.data import load_document_reference_json + def test_validate_content_missing_attachment(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") @@ -31,6 +34,7 @@ def test_validate_content_missing_attachment(): "expression": ["content.0.attachment"], } + def test_validate_content_missing_content_type(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") @@ -58,6 +62,7 @@ def test_validate_content_missing_content_type(): "expression": ["content.0.attachment.contentType"], } + def test_validate_content_invalid_content_type(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index d4a06d9b7..58af89bd6 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -960,18 +960,8 @@ def test_validate_content_extension_invalid_code(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - document_ref_data["content"][0]["extension"][0] = { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": { - "coding": [ - { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - "code": "invalid", - "display": "invalid", - } - ] - }, - } + content_extension = document_ref_data["content"][0]["extension"][0] + content_extension["valueCodeableConcept"]["coding"][0]["code"] = "invalid" result = validator.validate(document_ref_data) @@ -1523,6 +1513,7 @@ def test_validate_ssp_content_with_multiple_asids(): "expression": ["context.related"], } + def test_validate_content_extension_invalid_code_and_display_mismatch(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") From 1d58d6522564d5e5543c335cf04210bba4fe0359 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Thu, 5 Dec 2024 17:05:49 +0000 Subject: [PATCH 073/106] NRL-518 Revert contentType regex to simple version --- api/consumer/swagger.yaml | 2 +- api/producer/swagger.yaml | 2 +- layer/nrlf/consumer/fhir/r4/model.py | 4 +-- layer/nrlf/core/tests/test_pydantic_errors.py | 28 ------------------- layer/nrlf/producer/fhir/r4/model.py | 4 +-- layer/nrlf/producer/fhir/r4/strict_model.py | 2 +- .../createDocumentReference-failure.feature | 2 +- .../updateDocumentReference-failure.feature | 2 +- .../upsertDocumentReference-failure.feature | 2 +- 9 files changed, 10 insertions(+), 38 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 7caec31fd..10632c719 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -1101,7 +1101,7 @@ components: description: Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces. contentType: type: string - pattern: "^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$" + pattern: "[^\\s]+(\\s[^\\s]+)*" description: Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate. language: type: string diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index 6b1fa4457..cb99f13b7 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1666,7 +1666,7 @@ components: description: Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces. contentType: type: string - pattern: "^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$" + pattern: "[^\\s]+(\\s[^\\s]+)*" description: Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate. language: type: string diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index a44aab138..b36fe952e 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-26T15:05:35+00:00 +# timestamp: 2024-12-05T16:54:49+00:00 from __future__ import annotations @@ -136,7 +136,7 @@ class Attachment(BaseModel): str, Field( description="Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate.", - pattern="^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$", + pattern="[^\\s]+(\\s[^\\s]+)*", ), ] language: Annotated[ diff --git a/layer/nrlf/core/tests/test_pydantic_errors.py b/layer/nrlf/core/tests/test_pydantic_errors.py index dd8bfba54..9b63550f8 100644 --- a/layer/nrlf/core/tests/test_pydantic_errors.py +++ b/layer/nrlf/core/tests/test_pydantic_errors.py @@ -61,31 +61,3 @@ def test_validate_content_missing_content_type(): "diagnostics": "Failed to parse DocumentReference resource (content.0.attachment.contentType: Field required)", "expression": ["content.0.attachment.contentType"], } - - -def test_validate_content_invalid_content_type(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["attachment"]["contentType"] = "invalid/type" - - with pytest.raises(ParseError) as error: - validator.validate(document_ref_data) - - exc = error.value - assert len(exc.issues) == 1 - assert exc.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "invalid", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Failed to parse DocumentReference resource (content.0.attachment.contentType: String should match pattern '^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$')", - "expression": ["content.0.attachment.contentType"], - } diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index fe399cff7..17f8653c4 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-26T15:05:33+00:00 +# timestamp: 2024-12-05T16:54:46+00:00 from __future__ import annotations @@ -136,7 +136,7 @@ class Attachment(BaseModel): str, Field( description="Identifies the type of the data in the attachment and allows a method to be chosen to interpret or render the data. Includes mime type parameters such as charset where appropriate.", - pattern="^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$", + pattern="[^\\s]+(\\s[^\\s]+)*", ), ] language: Annotated[ diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index ca3ec452f..db9a93d5d 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-11-26T15:05:34+00:00 +# timestamp: 2024-12-05T16:54:48+00:00 from __future__ import annotations diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index cf78d96bd..14d824c40 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -725,7 +725,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios } ] }, - "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$')", + "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", "expression": [ "content.0.attachment.contentType" ] diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index bd0fbbe76..a19cd64d2 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -144,7 +144,7 @@ Feature: Producer - updateDocumentReference - Failure Scenarios } ] }, - "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$')", + "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", "expression": [ "content.0.attachment.contentType" ] diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index 5876e7db6..6ca9c54d8 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -306,7 +306,7 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios } ] }, - "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '^(application|audio|image|message|model|multipart|text|video)/[a-zA-Z0-9!#$&^_+.-]+(;[a-zA-Z0-9!#$&^_+.-]+=[a-zA-Z0-9!#$&^_+.-]+)*$')", + "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", "expression": [ "content.0.attachment.contentType" ] From 90822a6da61609ac1f94dc18a21411bc73dd5952 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Thu, 5 Dec 2024 17:09:03 +0000 Subject: [PATCH 074/106] NRL-518 Fix lint issue --- layer/nrlf/core/tests/test_pydantic_errors.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/layer/nrlf/core/tests/test_pydantic_errors.py b/layer/nrlf/core/tests/test_pydantic_errors.py index 9b63550f8..249151a3f 100644 --- a/layer/nrlf/core/tests/test_pydantic_errors.py +++ b/layer/nrlf/core/tests/test_pydantic_errors.py @@ -1,5 +1,3 @@ -from unittest.mock import Mock - import pytest from nrlf.core.errors import ParseError From 07fd1e3d88984d48ef2ca64bec8d615ef150da11 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Thu, 5 Dec 2024 22:21:22 +0000 Subject: [PATCH 075/106] NRL-518 Remove checks, pydantic validates that, add invalid contentType test --- layer/nrlf/core/tests/test_validators.py | 27 ++++++++++++++++++++++++ layer/nrlf/core/validators.py | 18 ---------------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index 58af89bd6..34c32a915 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1553,3 +1553,30 @@ def test_validate_content_extension_invalid_code_and_display_mismatch(): "content[0].extension[0].valueCodeableConcept.coding[0].display" ], } + + +def test_validate_content_invalid_content_type(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["attachment"]["contentType"] = "invalid/type" + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Invalid contentType: invalid/type. Must be 'application/pdf' or 'text/html'", + "expression": ["content[0].attachment.contentType"], + } diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 7e910913b..6d92be8aa 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -612,24 +612,6 @@ def _validate_content(self, model: DocumentReference): logger.log(LogReference.VALIDATOR001, step="content") for i, content in enumerate(model.content): - if not content.attachment: - self.result.add_error( - issue_code="required", - error_code="INVALID_RESOURCE", - diagnostics="Missing attachment in content", - field=f"content[{i}].attachment", - ) - continue - - if not content.attachment.contentType: - self.result.add_error( - issue_code="required", - error_code="INVALID_RESOURCE", - diagnostics="Missing contentType in content.attachment", - field=f"content[{i}].attachment.contentType", - ) - continue - if content.attachment.contentType not in ["application/pdf", "text/html"]: self.result.add_error( issue_code="value", From 3d8f0bdbabe4d951988f82c628ba3fad8e06dafc Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Thu, 5 Dec 2024 22:22:28 +0000 Subject: [PATCH 076/106] NRL-518 Improve pydantic diagnostic message --- layer/nrlf/core/errors.py | 19 ++++++---- layer/nrlf/core/tests/test_pydantic_errors.py | 36 ++++++++++++++++--- .../createDocumentReference-failure.feature | 4 +-- .../updateDocumentReference-failure.feature | 4 +-- .../upsertDocumentReference-failure.feature | 4 +-- 5 files changed, 51 insertions(+), 16 deletions(-) diff --git a/layer/nrlf/core/errors.py b/layer/nrlf/core/errors.py index 6e7b21686..ccd17628f 100644 --- a/layer/nrlf/core/errors.py +++ b/layer/nrlf/core/errors.py @@ -9,16 +9,23 @@ from nrlf.producer.fhir.r4.model import OperationOutcome, OperationOutcomeIssue +def format_error_location(loc: List) -> str: + formatted_loc = "" + for each in loc: + if isinstance(each, int): + formatted_loc = f"{formatted_loc}[{each}]" + else: + formatted_loc = f"{formatted_loc}.{each}" if formatted_loc else str(each) + return formatted_loc + + def diag_for_error(error: ErrorDetails) -> str: - if error["loc"]: - loc_string = ".".join(str(each) for each in error["loc"]) - return f"{loc_string}: {error['msg']}" - else: - return f"root: {error['msg']}" + loc_string = format_error_location(error["loc"]) + return f"{loc_string}: {error['msg']}" if loc_string else f"root: {error['msg']}" def expression_for_error(error: ErrorDetails) -> Optional[str]: - return str(".".join(str(each) for each in error["loc"]) if error["loc"] else "root") + return format_error_location(error["loc"]) or "root" class OperationOutcomeError(Exception): diff --git a/layer/nrlf/core/tests/test_pydantic_errors.py b/layer/nrlf/core/tests/test_pydantic_errors.py index 249151a3f..169fb272d 100644 --- a/layer/nrlf/core/tests/test_pydantic_errors.py +++ b/layer/nrlf/core/tests/test_pydantic_errors.py @@ -28,8 +28,8 @@ def test_validate_content_missing_attachment(): } ] }, - "diagnostics": "Failed to parse DocumentReference resource (content.0.attachment: Field required)", - "expression": ["content.0.attachment"], + "diagnostics": "Failed to parse DocumentReference resource (content[0].attachment: Field required)", + "expression": ["content[0].attachment"], } @@ -56,6 +56,34 @@ def test_validate_content_missing_content_type(): } ] }, - "diagnostics": "Failed to parse DocumentReference resource (content.0.attachment.contentType: Field required)", - "expression": ["content.0.attachment.contentType"], + "diagnostics": "Failed to parse DocumentReference resource (content[0].attachment.contentType: Field required)", + "expression": ["content[0].attachment.contentType"], + } + + +def test_validate_content_missing_format(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0].pop("format") + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].format: Field required)", + "expression": ["content[0].format"], } diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 14d824c40..4bbbac805 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -725,9 +725,9 @@ Feature: Producer - createDocumentReference - Failure Scenarios } ] }, - "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", + "diagnostics": "Request body could not be parsed (content[0].attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", "expression": [ - "content.0.attachment.contentType" + "content[0].attachment.contentType" ] } """ diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index a19cd64d2..fc134f6cb 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -144,9 +144,9 @@ Feature: Producer - updateDocumentReference - Failure Scenarios } ] }, - "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", + "diagnostics": "Request body could not be parsed (content[0].attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", "expression": [ - "content.0.attachment.contentType" + "content[0].attachment.contentType" ] } """ diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index 6ca9c54d8..f1d2412c8 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -306,9 +306,9 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios } ] }, - "diagnostics": "Request body could not be parsed (content.0.attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", + "diagnostics": "Request body could not be parsed (content[0].attachment.contentType: String should match pattern '[^\\s]+(\\s[^\\s]+)*')", "expression": [ - "content.0.attachment.contentType" + "content[0].attachment.contentType" ] } """ From 75534a3c318b98800a43ca060f40d268d76ed411 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Fri, 6 Dec 2024 03:12:50 +0000 Subject: [PATCH 077/106] NRL-518 ContentStability validation mostly on pydantic --- api/consumer/swagger.yaml | 47 +++- api/producer/swagger.yaml | 47 +++- layer/nrlf/consumer/fhir/r4/model.py | 70 ++++-- layer/nrlf/core/errors.py | 2 +- layer/nrlf/core/tests/test_pydantic_errors.py | 214 +++++++++++++++++ layer/nrlf/core/tests/test_validators.py | 220 ------------------ layer/nrlf/core/validators.py | 60 ----- layer/nrlf/producer/fhir/r4/model.py | 70 ++++-- layer/nrlf/producer/fhir/r4/strict_model.py | 68 ++++-- ...-Valid-with-date-and-meta-lastupdated.json | 16 +- .../Y05868-736253002-Valid-with-date.json | 16 +- ...736253002-Valid-with-meta-lastupdated.json | 16 +- ...5868-736253002-Valid-with-ssp-content.json | 16 +- 13 files changed, 499 insertions(+), 363 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 10632c719..e71257dfe 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -1070,11 +1070,14 @@ components: extension: type: array items: - $ref: "#/components/schemas/Extension" - description: Additional content defined by implementations. + $ref: "#/components/schemas/ContentStabilityExtension" + description: Additional extension for content stability. + minItems: 1 + maxItems: 1 required: - attachment - format + - extension DocumentReferenceRelatesTo: type: object properties: @@ -1191,6 +1194,46 @@ components: type: string pattern: \S* description: The reference details for the link. + ContentStabilityExtension: + type: object + properties: + url: + type: string + enum: + [ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + ] + valueCodeableConcept: + type: object + properties: + coding: + type: array + items: + type: object + properties: + system: + type: string + enum: + [ + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + ] + code: + type: string + enum: ["static", "dynamic"] + display: + type: string + enum: ["Static", "Dynamic"] + required: + - system + - code + - display + minItems: 1 + maxItems: 1 + required: + - coding + required: + - url + - valueCodeableConcept Identifier: type: object properties: diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index cb99f13b7..bf1fc5067 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1635,11 +1635,14 @@ components: extension: type: array items: - $ref: "#/components/schemas/Extension" - description: Additional content defined by implementations. + $ref: "#/components/schemas/ContentStabilityExtension" + description: Additional extension for content stability. + minItems: 1 + maxItems: 1 required: - attachment - format + - extension DocumentReferenceRelatesTo: type: object properties: @@ -1757,6 +1760,46 @@ components: type: string pattern: \S* description: The reference details for the link. + ContentStabilityExtension: + type: object + properties: + url: + type: string + enum: + [ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + ] + valueCodeableConcept: + type: object + properties: + coding: + type: array + items: + type: object + properties: + system: + type: string + enum: + [ + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + ] + code: + type: string + enum: ["static", "dynamic"] + display: + type: string + enum: ["Static", "Dynamic"] + required: + - system + - code + - display + minItems: 1 + maxItems: 1 + required: + - coding + required: + - url + - valueCodeableConcept Identifier: type: object properties: diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index b36fe952e..5a50e1f5c 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-05T16:54:49+00:00 +# timestamp: 2024-12-06T02:57:59+00:00 from __future__ import annotations @@ -230,6 +230,25 @@ class Coding(BaseModel): ] = None +class CodingItem(BaseModel): + system: Literal[ + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + ] + code: Literal["static", "dynamic"] + display: Literal["Static", "Dynamic"] + + +class ValueCodeableConcept(BaseModel): + coding: Annotated[List[CodingItem], Field(max_length=1, min_length=1)] + + +class ContentStabilityExtension(BaseModel): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ValueCodeableConcept + + class Period(BaseModel): id: Annotated[ Optional[str], @@ -427,6 +446,31 @@ class RequestHeaderCorrelationId(RootModel[str]): root: Annotated[str, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] +class DocumentReferenceContent(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + attachment: Annotated[ + Attachment, + Field( + description="The document or URL of the document along with critical metadata to prove content has integrity." + ), + ] + format: Annotated[ + Coding, + Field( + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." + ), + ] + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] + + class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -818,29 +862,6 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - Coding, - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] - extension: Optional[List[Extension]] = None - - class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[str], @@ -1032,7 +1053,6 @@ class Signature(BaseModel): Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() -DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() CodeableConcept.model_rebuild() Identifier.model_rebuild() diff --git a/layer/nrlf/core/errors.py b/layer/nrlf/core/errors.py index ccd17628f..9172c32e9 100644 --- a/layer/nrlf/core/errors.py +++ b/layer/nrlf/core/errors.py @@ -21,7 +21,7 @@ def format_error_location(loc: List) -> str: def diag_for_error(error: ErrorDetails) -> str: loc_string = format_error_location(error["loc"]) - return f"{loc_string}: {error['msg']}" if loc_string else f"root: {error['msg']}" + return f"{loc_string or 'root'}: {error['msg']}" def expression_for_error(error: ErrorDetails) -> Optional[str]: diff --git a/layer/nrlf/core/tests/test_pydantic_errors.py b/layer/nrlf/core/tests/test_pydantic_errors.py index 169fb272d..6b40f4a9d 100644 --- a/layer/nrlf/core/tests/test_pydantic_errors.py +++ b/layer/nrlf/core/tests/test_pydantic_errors.py @@ -87,3 +87,217 @@ def test_validate_content_missing_format(): "diagnostics": "Failed to parse DocumentReference resource (content[0].format: Field required)", "expression": ["content[0].format"], } + + +def test_validate_content_multiple_content_stability_extensions(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Add a second duplicate contentStability extension + document_ref_data["content"][0]["extension"].append( + document_ref_data["content"][0]["extension"][0] + ) + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension: List should have at most 1 item after validation, not 2)", + "expression": ["content[0].extension"], + } + + +def test_validate_content_invalid_content_stability_code(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an invalid code for contentStability extension + content_extension = document_ref_data["content"][0]["extension"][0] + content_extension["valueCodeableConcept"]["coding"][0]["code"] = "invalid" + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].code: Input should be 'static' or 'dynamic')", + "expression": ["content[0].extension[0].valueCodeableConcept.coding[0].code"], + } + + +def test_validate_content_invalid_content_stability_display(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an invalid display for contentStability extension + content_extension = document_ref_data["content"][0]["extension"][0] + content_extension["valueCodeableConcept"]["coding"][0]["display"] = "invalid" + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].display: Input should be 'Static' or 'Dynamic')", + "expression": [ + "content[0].extension[0].valueCodeableConcept.coding[0].display" + ], + } + + +def test_validate_content_invalid_content_stability_system(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an invalid system for contentStability extension + content_extension = document_ref_data["content"][0]["extension"][0] + content_extension["valueCodeableConcept"]["coding"][0]["system"] = "invalid" + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].system: Input should be 'https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability')", + "expression": ["content[0].extension[0].valueCodeableConcept.coding[0].system"], + } + + +def test_validate_content_invalid_content_stability_url(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an invalid URL for contentStability extension + document_ref_data["content"][0]["extension"][0]["url"] = "invalid" + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].url: Input should be 'https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability')", + "expression": ["content[0].extension[0].url"], + } + + +def test_validate_content_empty_content_stability_coding(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Set an empty coding list for contentStability extension + document_ref_data["content"][0]["extension"][0]["valueCodeableConcept"][ + "coding" + ] = [] + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding: List should have at least 1 item after validation, not 0)", + "expression": ["content[0].extension[0].valueCodeableConcept.coding"], + } + + +def test_validate_content_missing_content_stability_coding(): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + # Remove the coding key from contentStability extension + del document_ref_data["content"][0]["extension"][0]["valueCodeableConcept"][ + "coding" + ] + + with pytest.raises(ParseError) as error: + validator.validate(document_ref_data) + + exc = error.value + assert len(exc.issues) == 1 + assert exc.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "invalid", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding: Field required)", + "expression": ["content[0].extension[0].valueCodeableConcept.coding"], + } diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index 34c32a915..851b5011f 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -781,47 +781,6 @@ def test_validate_type_coding_display_mismatch(type_str: str, display: str): } -def test_validate_content_extension_too_many_extensions(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"].append( - { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": { - "coding": [ - { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - "code": "static", - "display": "static", - } - ] - }, - } - ) - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "invalid", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension length: 2 Extension must only contain a single value", - "expression": ["content[0].extension"], - } - - def test_validate_author_too_many_authors(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") @@ -956,185 +915,6 @@ def test_validate_author_value_too_long(): } -def test_validate_content_extension_invalid_code(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - content_extension = document_ref_data["content"][0]["extension"][0] - content_extension["valueCodeableConcept"]["coding"][0]["code"] = "invalid" - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension code: invalid Extension code must be 'static' or 'dynamic'", - "expression": ["content[0].extension[0].valueCodeableConcept.coding[0].code"], - } - - -def test_validate_content_extension_invalid_display(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": { - "coding": [ - { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - "code": "static", - "display": "invalid", - } - ] - }, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension display: invalid Extension display must be 'Static' or 'Dynamic'", - "expression": [ - "content[0].extension[0].valueCodeableConcept.coding[0].display" - ], - } - - -def test_validate_content_extension_invalid_system(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": { - "coding": [ - { - "system": "invalid", - "code": "static", - "display": "static", - } - ] - }, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension system: invalid Extension system must be 'https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability'", - "expression": ["content[0].extension[0].valueCodeableConcept.coding[0].system"], - } - - -def test_validate_content_extension_invalid_url(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "invalid", - "valueCodeableConcept": { - "coding": [ - { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - "code": "static", - "display": "static", - } - ] - }, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "value", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Invalid content extension url: invalid Extension url must be 'https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability'", - "expression": ["content[0].extension[0].url"], - } - - -def test_validate_content_extension_missing_coding(): - validator = DocumentReferenceValidator() - document_ref_data = load_document_reference_json("Y05868-736253002-Valid") - - document_ref_data["content"][0]["extension"][0] = { - "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - "valueCodeableConcept": {"coding": []}, - } - - result = validator.validate(document_ref_data) - - assert result.is_valid is False - assert result.resource.id == "Y05868-99999-99999-999999" - assert len(result.issues) == 1 - assert result.issues[0].model_dump(exclude_none=True) == { - "severity": "error", - "code": "required", - "details": { - "coding": [ - { - "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - "code": "INVALID_RESOURCE", - "display": "Invalid validation of resource", - } - ] - }, - "diagnostics": "Missing content[0].extension[0].valueCodeableConcept.coding, extension must have at least one coding.", - "expression": ["content[0].extension.valueCodeableConcept.coding"], - } - - def test_validate_identifiers_invalid_systems(): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 6d92be8aa..552fc1465 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -490,67 +490,7 @@ def _validate_content_extension(self, model: DocumentReference): logger.debug("Validating extension") for i, content in enumerate(model.content): - if len(content.extension) > 1: - self.result.add_error( - issue_code="invalid", - error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension length: {len(content.extension)} Extension must only contain a single value", - field=f"content[{i}].extension", - ) - return - - if len(content.extension[0].valueCodeableConcept.coding) < 1: - self.result.add_error( - issue_code="required", - error_code="INVALID_RESOURCE", - diagnostics=f"Missing content[{i}].extension[0].valueCodeableConcept.coding, extension must have at least one coding.", - field=f"content[{i}].extension.valueCodeableConcept.coding", - ) - return - - if ( - content.extension[0].url - != "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - ): - self.result.add_error( - issue_code="value", - error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension url: {content.extension[0].url} Extension url must be 'https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability'", - field=f"content[{i}].extension[0].url", - ) - return - coding = content.extension[0].valueCodeableConcept.coding[0] - if ( - coding.system - != "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" - ): - self.result.add_error( - issue_code="value", - error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension system: {coding.system} Extension system must be 'https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability'", - field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].system", - ) - return - - if coding.code not in ["static", "dynamic"]: - self.result.add_error( - issue_code="value", - error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension code: {coding.code} Extension code must be 'static' or 'dynamic'", - field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].code", - ) - return - - if coding.display not in ["Static", "Dynamic"]: - self.result.add_error( - issue_code="value", - error_code="INVALID_RESOURCE", - diagnostics=f"Invalid content extension display: {coding.display} Extension display must be 'Static' or 'Dynamic'", - field=f"content[{i}].extension[0].valueCodeableConcept.coding[0].display", - ) - return - if coding.code != coding.display.lower(): self.result.add_error( issue_code="value", diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index 17f8653c4..98041bca8 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-05T16:54:46+00:00 +# timestamp: 2024-12-06T02:57:56+00:00 from __future__ import annotations @@ -230,6 +230,25 @@ class Coding(BaseModel): ] = None +class CodingItem(BaseModel): + system: Literal[ + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + ] + code: Literal["static", "dynamic"] + display: Literal["Static", "Dynamic"] + + +class ValueCodeableConcept(BaseModel): + coding: Annotated[List[CodingItem], Field(max_length=1, min_length=1)] + + +class ContentStabilityExtension(BaseModel): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ValueCodeableConcept + + class Period(BaseModel): id: Annotated[ Optional[str], @@ -417,6 +436,31 @@ class RequestHeaderCorrelationId(RootModel[str]): root: Annotated[str, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] +class DocumentReferenceContent(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + attachment: Annotated[ + Attachment, + Field( + description="The document or URL of the document along with critical metadata to prove content has integrity." + ), + ] + format: Annotated[ + Coding, + Field( + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." + ), + ] + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] + + class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -802,29 +846,6 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - Coding, - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] - extension: Optional[List[Extension]] = None - - class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[str], @@ -1016,7 +1037,6 @@ class Signature(BaseModel): Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() -DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() CodeableConcept.model_rebuild() Identifier.model_rebuild() diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index db9a93d5d..3ae1d3511 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-05T16:54:48+00:00 +# timestamp: 2024-12-06T02:57:58+00:00 from __future__ import annotations @@ -207,6 +207,25 @@ class Coding(BaseModel): ] = None +class CodingItem(BaseModel): + system: Literal[ + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + ] + code: Literal["static", "dynamic"] + display: Literal["Static", "Dynamic"] + + +class ValueCodeableConcept(BaseModel): + coding: Annotated[List[CodingItem], Field(max_length=1, min_length=1)] + + +class ContentStabilityExtension(BaseModel): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ValueCodeableConcept + + class Period(BaseModel): id: Annotated[ Optional[StrictStr], @@ -365,6 +384,30 @@ class RequestHeaderCorrelationId(RootModel[StrictStr]): root: Annotated[StrictStr, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] +class DocumentReferenceContent(BaseModel): + id: Annotated[ + Optional[StrictStr], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." + ), + ] = None + attachment: Annotated[ + Attachment, + Field( + description="The document or URL of the document along with critical metadata to prove content has integrity." + ), + ] + format: Annotated[ + Coding, + Field( + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." + ), + ] + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] + + class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -706,28 +749,6 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[StrictStr], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - Coding, - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] - extension: Optional[List[Extension]] = None - - class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[StrictStr], @@ -895,7 +916,6 @@ class Signature(BaseModel): Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() -DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() CodeableConcept.model_rebuild() Identifier.model_rebuild() diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json index 2bf5f63df..8bfae4aaf 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json @@ -74,7 +74,21 @@ "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json index 0aee9f067..72389bf26 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json @@ -71,7 +71,21 @@ "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json index ca7dfbcd5..5b9e6b994 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json @@ -73,7 +73,21 @@ "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json index c88029c05..dbb356058 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json @@ -70,7 +70,21 @@ "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { From 3452fa3e57082ed7992bc1c3b7fdde1473275f92 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Fri, 6 Dec 2024 03:27:23 +0000 Subject: [PATCH 078/106] NRL-518 Better naming for scenario --- tests/features/producer/createDocumentReference-failure.feature | 2 +- tests/features/producer/updateDocumentReference-failure.feature | 2 +- tests/features/producer/upsertDocumentReference-failure.feature | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 4bbbac805..fea9ac2a7 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -688,7 +688,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - Scenario: Missing contentType + Scenario: contentType empty string Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'TSTCUS' is authorised to access pointer types: | system | value | diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index fc134f6cb..0c911d6c8 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -94,7 +94,7 @@ Feature: Producer - updateDocumentReference - Failure Scenarios } """ - Scenario: Missing contentType + Scenario: contentType empty string Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'TSTCUS' is authorised to access pointer types: | system | value | diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index f1d2412c8..5b658d3eb 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -269,7 +269,7 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios } """ - Scenario: Missing contentType + Scenario: contentType empty string Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'TSTCUS' is authorised to access pointer types: | system | value | From 9bc5d31bccfe74e312d7997c7fdfb61fe909d992 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Fri, 6 Dec 2024 03:55:59 +0000 Subject: [PATCH 079/106] NRL-518 Better format for single value enum in swagger --- api/consumer/swagger.yaml | 8 ++------ api/producer/swagger.yaml | 8 ++------ 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index e71257dfe..183fa77cf 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -1200,9 +1200,7 @@ components: url: type: string enum: - [ - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - ] + - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" valueCodeableConcept: type: object properties: @@ -1214,9 +1212,7 @@ components: system: type: string enum: - [ - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - ] + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" code: type: string enum: ["static", "dynamic"] diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index bf1fc5067..c185e1087 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1766,9 +1766,7 @@ components: url: type: string enum: - [ - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - ] + - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" valueCodeableConcept: type: object properties: @@ -1780,9 +1778,7 @@ components: system: type: string enum: - [ - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", - ] + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" code: type: string enum: ["static", "dynamic"] From 52f1d24836f04274c532d8db6155f1f256c4e38d Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Fri, 6 Dec 2024 03:56:26 +0000 Subject: [PATCH 080/106] NRL-518 Pydantic will ensure extension is always there --- layer/nrlf/core/validators.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 552fc1465..8083e323a 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -142,8 +142,7 @@ def validate(self, data: Dict[str, Any] | DocumentReference): self._validate_author(resource) self._validate_type_category_mapping(resource) self._validate_content(resource) - if resource.content[0].extension: - self._validate_content_extension(resource) + self._validate_content_extension(resource) except StopValidationError: logger.log(LogReference.VALIDATOR003) From f8663e782e7154f0eb98724531a1ddc63389386f Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Fri, 6 Dec 2024 04:13:58 +0000 Subject: [PATCH 081/106] NRL-518 Pydantic format validation, fix casing format display --- api/consumer/swagger.yaml | 26 +++++++++++++++- api/producer/swagger.yaml | 30 +++++++++++++++++-- layer/nrlf/consumer/fhir/r4/model.py | 19 ++++++++++-- layer/nrlf/producer/fhir/r4/model.py | 19 ++++++++++-- layer/nrlf/producer/fhir/r4/strict_model.py | 19 ++++++++++-- swagger/producer-static/components.yaml | 2 +- swagger/producer-static/narrative.yaml | 2 +- .../RQI-736253002-Valid.json | 2 +- ...-Valid-with-date-and-meta-lastupdated.json | 2 +- .../Y05868-736253002-Valid-with-date.json | 2 +- ...736253002-Valid-with-meta-lastupdated.json | 2 +- ...5868-736253002-Valid-with-ssp-content.json | 2 +- .../Y05868-736253002-Valid.json | 2 +- .../readDocumentReference-success.feature | 4 +-- .../createDocumentReference-failure.feature | 2 +- .../readDocumentReference-success.feature | 2 +- .../updateDocumentReference-failure.feature | 4 +-- .../upsertDocumentReference-failure.feature | 2 +- tests/features/utils/constants.py | 2 +- tests/features/utils/data.py | 2 +- tests/smoke/setup.py | 2 +- 21 files changed, 121 insertions(+), 28 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 183fa77cf..dbe549ea6 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -1065,7 +1065,7 @@ components: $ref: "#/components/schemas/Attachment" description: The document or URL of the document along with critical metadata to prove content has integrity. format: - $ref: "#/components/schemas/Coding" + $ref: "#/components/schemas/NRLFormatCode" description: An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType. extension: type: array @@ -1230,6 +1230,30 @@ components: required: - url - valueCodeableConcept + NRLFormatCode: + type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" + description: The system URL for the NRLF Format Code. + code: + type: string + enum: + - "urn:nhs-ic:record-contact" + - "urn:nhs-ic:unstructured" + description: The code representing the format of the document. + display: + type: string + enum: + - "Contact details (HTTP Unsecured)" + - "Unstructured Document" + description: The display text for the code. + required: + - system + - code + - display Identifier: type: object properties: diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index c185e1087..a00d5239e 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -334,7 +334,7 @@ paths: { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured" - "display": "Unstructured document" + "display": "Unstructured Document" } ] ``` @@ -1215,7 +1215,7 @@ components: format: system: https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode code: "urn:nhs-ic:unstructured" - display: Unstructured document + display: Unstructured Document context: event: - coding: @@ -1630,7 +1630,7 @@ components: $ref: "#/components/schemas/Attachment" description: The document or URL of the document along with critical metadata to prove content has integrity. format: - $ref: "#/components/schemas/Coding" + $ref: "#/components/schemas/NRLFormatCode" description: An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType. extension: type: array @@ -1796,6 +1796,30 @@ components: required: - url - valueCodeableConcept + NRLFormatCode: + type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" + description: The system URL for the NRLF Format Code. + code: + type: string + enum: + - "urn:nhs-ic:record-contact" + - "urn:nhs-ic:unstructured" + description: The code representing the format of the document. + display: + type: string + enum: + - "Contact details (HTTP Unsecured)" + - "Unstructured Document" + description: The display text for the code. + required: + - system + - code + - display Identifier: type: object properties: diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index 5a50e1f5c..161fabb1f 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-06T02:57:59+00:00 +# timestamp: 2024-12-06T04:10:37+00:00 from __future__ import annotations @@ -249,6 +249,21 @@ class ContentStabilityExtension(BaseModel): valueCodeableConcept: ValueCodeableConcept +class NRLFormatCode(BaseModel): + system: Annotated[ + Literal["https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode"], + Field(description="The system URL for the NRLF Format Code."), + ] + code: Annotated[ + Literal["urn:nhs-ic:record-contact", "urn:nhs-ic:unstructured"], + Field(description="The code representing the format of the document."), + ] + display: Annotated[ + Literal["Contact details (HTTP Unsecured)", "Unstructured Document"], + Field(description="The display text for the code."), + ] + + class Period(BaseModel): id: Annotated[ Optional[str], @@ -461,7 +476,7 @@ class DocumentReferenceContent(BaseModel): ), ] format: Annotated[ - Coding, + NRLFormatCode, Field( description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index 98041bca8..580836343 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-06T02:57:56+00:00 +# timestamp: 2024-12-06T04:10:35+00:00 from __future__ import annotations @@ -249,6 +249,21 @@ class ContentStabilityExtension(BaseModel): valueCodeableConcept: ValueCodeableConcept +class NRLFormatCode(BaseModel): + system: Annotated[ + Literal["https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode"], + Field(description="The system URL for the NRLF Format Code."), + ] + code: Annotated[ + Literal["urn:nhs-ic:record-contact", "urn:nhs-ic:unstructured"], + Field(description="The code representing the format of the document."), + ] + display: Annotated[ + Literal["Contact details (HTTP Unsecured)", "Unstructured Document"], + Field(description="The display text for the code."), + ] + + class Period(BaseModel): id: Annotated[ Optional[str], @@ -451,7 +466,7 @@ class DocumentReferenceContent(BaseModel): ), ] format: Annotated[ - Coding, + NRLFormatCode, Field( description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index 3ae1d3511..6d1e371b7 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-06T02:57:58+00:00 +# timestamp: 2024-12-06T04:10:36+00:00 from __future__ import annotations @@ -226,6 +226,21 @@ class ContentStabilityExtension(BaseModel): valueCodeableConcept: ValueCodeableConcept +class NRLFormatCode(BaseModel): + system: Annotated[ + Literal["https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode"], + Field(description="The system URL for the NRLF Format Code."), + ] + code: Annotated[ + Literal["urn:nhs-ic:record-contact", "urn:nhs-ic:unstructured"], + Field(description="The code representing the format of the document."), + ] + display: Annotated[ + Literal["Contact details (HTTP Unsecured)", "Unstructured Document"], + Field(description="The display text for the code."), + ] + + class Period(BaseModel): id: Annotated[ Optional[StrictStr], @@ -398,7 +413,7 @@ class DocumentReferenceContent(BaseModel): ), ] format: Annotated[ - Coding, + NRLFormatCode, Field( description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), diff --git a/swagger/producer-static/components.yaml b/swagger/producer-static/components.yaml index 22a096326..8c984b796 100644 --- a/swagger/producer-static/components.yaml +++ b/swagger/producer-static/components.yaml @@ -122,7 +122,7 @@ components: format: system: https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode code: "urn:nhs-ic:unstructured" - display: Unstructured document + display: Unstructured Document context: practiceSetting: coding: diff --git a/swagger/producer-static/narrative.yaml b/swagger/producer-static/narrative.yaml index 0a3af911f..fd567ac08 100644 --- a/swagger/producer-static/narrative.yaml +++ b/swagger/producer-static/narrative.yaml @@ -287,7 +287,7 @@ paths: { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured" - "display": "Unstructured document" + "display": "Unstructured Document" } ] ``` diff --git a/tests/data/DocumentReference/RQI-736253002-Valid.json b/tests/data/DocumentReference/RQI-736253002-Valid.json index 429d088a4..873434086 100644 --- a/tests/data/DocumentReference/RQI-736253002-Valid.json +++ b/tests/data/DocumentReference/RQI-736253002-Valid.json @@ -69,7 +69,7 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" }, "extension": [ { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json index 8bfae4aaf..de51a0df6 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date-and-meta-lastupdated.json @@ -73,7 +73,7 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" }, "extension": [ { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json index 72389bf26..49e5484ca 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-date.json @@ -70,7 +70,7 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" }, "extension": [ { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json index 5b9e6b994..9e6a0e73b 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-meta-lastupdated.json @@ -72,7 +72,7 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" }, "extension": [ { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json b/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json index dbb356058..a20f81d33 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid-with-ssp-content.json @@ -69,7 +69,7 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" }, "extension": [ { diff --git a/tests/data/DocumentReference/Y05868-736253002-Valid.json b/tests/data/DocumentReference/Y05868-736253002-Valid.json index d83b5a3f6..6b79042e8 100644 --- a/tests/data/DocumentReference/Y05868-736253002-Valid.json +++ b/tests/data/DocumentReference/Y05868-736253002-Valid.json @@ -69,7 +69,7 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" }, "extension": [ { diff --git a/tests/features/consumer/readDocumentReference-success.feature b/tests/features/consumer/readDocumentReference-success.feature index 5c0d83979..356594b57 100644 --- a/tests/features/consumer/readDocumentReference-success.feature +++ b/tests/features/consumer/readDocumentReference-success.feature @@ -73,7 +73,7 @@ Feature: Consumer - readDocumentReference - Success Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ], @@ -164,7 +164,7 @@ Feature: Consumer - readDocumentReference - Success Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ], diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index fea9ac2a7..663006526 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -704,7 +704,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ] diff --git a/tests/features/producer/readDocumentReference-success.feature b/tests/features/producer/readDocumentReference-success.feature index 04c4228ec..147c2bd81 100644 --- a/tests/features/producer/readDocumentReference-success.feature +++ b/tests/features/producer/readDocumentReference-success.feature @@ -75,7 +75,7 @@ Feature: Producer - readDocumentReference - Success Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ], diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index 0c911d6c8..a19633829 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -122,7 +122,7 @@ Feature: Producer - updateDocumentReference - Failure Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ] @@ -179,7 +179,7 @@ Feature: Producer - updateDocumentReference - Failure Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ] diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index 5b658d3eb..83f3d7de7 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -285,7 +285,7 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ] diff --git a/tests/features/utils/constants.py b/tests/features/utils/constants.py index 48228a74d..f5e479df3 100644 --- a/tests/features/utils/constants.py +++ b/tests/features/utils/constants.py @@ -112,7 +112,7 @@ "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ] diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 6cf930114..e3e5ec745 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -42,7 +42,7 @@ def create_test_document_reference(items: dict) -> DocumentReference: "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", ), code=items.get("formatCode", "urn:nhs-ic:unstructured"), - display=items.get("formatDisplay", "Unstructured document"), + display=items.get("formatDisplay", "Unstructured Document"), ), ) ], diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index 49ddeb766..7cdd73b5a 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -36,7 +36,7 @@ def build_document_reference( format=Coding( system="https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", code="urn:nhs-ic:unstructured", - display="Unstructured document", + display="Unstructured Document", ), ) ], From 5098eca959c19ab3c76a611a2a1afd5fc2c00aa8 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Fri, 6 Dec 2024 05:27:23 +0000 Subject: [PATCH 082/106] NRL-518 Fix integration tests --- .../readDocumentReference-success.feature | 32 +++++++++++++++++-- .../createDocumentReference-failure.feature | 16 +++++++++- .../readDocumentReference-success.feature | 16 +++++++++- .../updateDocumentReference-failure.feature | 32 +++++++++++++++++-- .../upsertDocumentReference-failure.feature | 16 +++++++++- tests/features/utils/constants.py | 16 +++++++++- tests/features/utils/data.py | 20 +++++++++++- 7 files changed, 139 insertions(+), 9 deletions(-) diff --git a/tests/features/consumer/readDocumentReference-success.feature b/tests/features/consumer/readDocumentReference-success.feature index 356594b57..974afa244 100644 --- a/tests/features/consumer/readDocumentReference-success.feature +++ b/tests/features/consumer/readDocumentReference-success.feature @@ -74,7 +74,21 @@ Feature: Consumer - readDocumentReference - Success Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { @@ -165,7 +179,21 @@ Feature: Consumer - readDocumentReference - Success Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 663006526..33f472a76 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -705,7 +705,21 @@ Feature: Producer - createDocumentReference - Failure Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] """ diff --git a/tests/features/producer/readDocumentReference-success.feature b/tests/features/producer/readDocumentReference-success.feature index 147c2bd81..4cc883e20 100644 --- a/tests/features/producer/readDocumentReference-success.feature +++ b/tests/features/producer/readDocumentReference-success.feature @@ -76,7 +76,21 @@ Feature: Producer - readDocumentReference - Success Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ], "context": { diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index a19633829..99754f509 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -123,7 +123,21 @@ Feature: Producer - updateDocumentReference - Failure Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] } @@ -180,7 +194,21 @@ Feature: Producer - updateDocumentReference - Failure Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] } diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index 83f3d7de7..a8de6ee35 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -286,7 +286,21 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] """ diff --git a/tests/features/utils/constants.py b/tests/features/utils/constants.py index f5e479df3..b51d6d886 100644 --- a/tests/features/utils/constants.py +++ b/tests/features/utils/constants.py @@ -113,7 +113,21 @@ "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] """ diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index e3e5ec745..723c358a6 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -3,12 +3,16 @@ Attachment, CodeableConcept, Coding, + CodingItem, + ContentStabilityExtension, DocumentReference, DocumentReferenceContent, DocumentReferenceContext, DocumentReferenceRelatesTo, Identifier, + NRLFormatCode, Reference, + ValueCodeableConcept, ) from tests.features.utils.constants import ( DEFAULT_TEST_AUTHOR, @@ -36,7 +40,7 @@ def create_test_document_reference(items: dict) -> DocumentReference: contentType=items.get("contentType", "application/pdf"), url=items["url"], ), - format=Coding( + format=NRLFormatCode( system=items.get( "formatSystem", "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", @@ -44,6 +48,20 @@ def create_test_document_reference(items: dict) -> DocumentReference: code=items.get("formatCode", "urn:nhs-ic:unstructured"), display=items.get("formatDisplay", "Unstructured Document"), ), + extension=[ + ContentStabilityExtension( + url="https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + valueCodeableConcept=ValueCodeableConcept( + coding=[ + CodingItem( + system="https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + code="static", + display="Static", + ) + ] + ), + ) + ], ) ], ), From aca3ca4dfcb85ec0e2b588f7ce9e0eb76ff87937 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Fri, 6 Dec 2024 06:24:52 +0000 Subject: [PATCH 083/106] NRL-518 Add value set for pydantic diagnostic messages --- layer/nrlf/core/errors.py | 18 +++++++++++++++++- layer/nrlf/core/tests/test_pydantic_errors.py | 12 ++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/layer/nrlf/core/errors.py b/layer/nrlf/core/errors.py index 9172c32e9..44567c974 100644 --- a/layer/nrlf/core/errors.py +++ b/layer/nrlf/core/errors.py @@ -19,9 +19,25 @@ def format_error_location(loc: List) -> str: return formatted_loc +def append_value_set_url(loc_string: str) -> str: + if loc_string.endswith(("url", "system")): + return "" + + if "content" in loc_string: + if "extension" in loc_string: + return ". See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + if "format" in loc_string: + return ". See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" + + return "" + + def diag_for_error(error: ErrorDetails) -> str: loc_string = format_error_location(error["loc"]) - return f"{loc_string or 'root'}: {error['msg']}" + print(f"Error location: {loc_string}") + msg = f"{loc_string or 'root'}: {error['msg']}" + msg += append_value_set_url(loc_string) + return msg def expression_for_error(error: ErrorDetails) -> Optional[str]: diff --git a/layer/nrlf/core/tests/test_pydantic_errors.py b/layer/nrlf/core/tests/test_pydantic_errors.py index 6b40f4a9d..7ff69e6be 100644 --- a/layer/nrlf/core/tests/test_pydantic_errors.py +++ b/layer/nrlf/core/tests/test_pydantic_errors.py @@ -84,7 +84,7 @@ def test_validate_content_missing_format(): } ] }, - "diagnostics": "Failed to parse DocumentReference resource (content[0].format: Field required)", + "diagnostics": "Failed to parse DocumentReference resource (content[0].format: Field required. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode)", "expression": ["content[0].format"], } @@ -115,7 +115,7 @@ def test_validate_content_multiple_content_stability_extensions(): } ] }, - "diagnostics": "Failed to parse DocumentReference resource (content[0].extension: List should have at most 1 item after validation, not 2)", + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension: List should have at most 1 item after validation, not 2. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", "expression": ["content[0].extension"], } @@ -145,7 +145,7 @@ def test_validate_content_invalid_content_stability_code(): } ] }, - "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].code: Input should be 'static' or 'dynamic')", + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].code: Input should be 'static' or 'dynamic'. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", "expression": ["content[0].extension[0].valueCodeableConcept.coding[0].code"], } @@ -175,7 +175,7 @@ def test_validate_content_invalid_content_stability_display(): } ] }, - "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].display: Input should be 'Static' or 'Dynamic')", + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding[0].display: Input should be 'Static' or 'Dynamic'. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", "expression": [ "content[0].extension[0].valueCodeableConcept.coding[0].display" ], @@ -267,7 +267,7 @@ def test_validate_content_empty_content_stability_coding(): } ] }, - "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding: List should have at least 1 item after validation, not 0)", + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding: List should have at least 1 item after validation, not 0. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", "expression": ["content[0].extension[0].valueCodeableConcept.coding"], } @@ -298,6 +298,6 @@ def test_validate_content_missing_content_stability_coding(): } ] }, - "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding: Field required)", + "diagnostics": "Failed to parse DocumentReference resource (content[0].extension[0].valueCodeableConcept.coding: Field required. See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability)", "expression": ["content[0].extension[0].valueCodeableConcept.coding"], } From 7886799147b05304f65b3aa81a9c12412e3314e6 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Fri, 6 Dec 2024 06:30:58 +0000 Subject: [PATCH 084/106] NRL-518 Fix smoke tests --- tests/smoke/setup.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index 7cdd73b5a..a8f8bdb7d 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -33,11 +33,25 @@ def build_document_reference( contentType=content_type, url=content_url, ), - format=Coding( + format=NRLFormatCode( system="https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", code="urn:nhs-ic:unstructured", display="Unstructured Document", ), + extension=[ + ContentStabilityExtension( + url="https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + valueCodeableConcept=ValueCodeableConcept( + coding=[ + CodingItem( + system="https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + code="static", + display="Static", + ) + ] + ), + ) + ], ) ], type=CodeableConcept( From 3137b908ab01e0ba2f028cb1f258837812991cfe Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Fri, 6 Dec 2024 06:46:14 +0000 Subject: [PATCH 085/106] NRL-518 Fix linting issue --- layer/nrlf/core/errors.py | 1 - 1 file changed, 1 deletion(-) diff --git a/layer/nrlf/core/errors.py b/layer/nrlf/core/errors.py index 44567c974..e4cc82e21 100644 --- a/layer/nrlf/core/errors.py +++ b/layer/nrlf/core/errors.py @@ -34,7 +34,6 @@ def append_value_set_url(loc_string: str) -> str: def diag_for_error(error: ErrorDetails) -> str: loc_string = format_error_location(error["loc"]) - print(f"Error location: {loc_string}") msg = f"{loc_string or 'root'}: {error['msg']}" msg += append_value_set_url(loc_string) return msg From 5260be2e9dcd5ea30980d3274a2c4683f9bab2df Mon Sep 17 00:00:00 2001 From: eesa456 Date: Wed, 11 Dec 2024 03:55:30 +0000 Subject: [PATCH 086/106] NRL-1202 validate type not just system --- .../search_document_reference.py | 5 ++-- .../search_post_document_reference.py | 4 ++-- .../search_document_reference.py | 4 ++-- .../search_post_document_reference.py | 4 ++-- layer/nrlf/core/tests/test_validators.py | 23 +++++++++++++------ layer/nrlf/core/validators.py | 11 ++------- 6 files changed, 26 insertions(+), 25 deletions(-) diff --git a/api/consumer/searchDocumentReference/search_document_reference.py b/api/consumer/searchDocumentReference/search_document_reference.py index aee43c7ec..ae590e0f8 100644 --- a/api/consumer/searchDocumentReference/search_document_reference.py +++ b/api/consumer/searchDocumentReference/search_document_reference.py @@ -9,7 +9,7 @@ from nrlf.core.logger import LogReference, logger from nrlf.core.model import ConnectionMetadata, ConsumerRequestParams from nrlf.core.response import Response, SpineErrorResponse -from nrlf.core.validators import validate_category, validate_type_system +from nrlf.core.validators import validate_category, validate_type @request_handler(params=ConsumerRequestParams) @@ -46,8 +46,7 @@ def handler( base_url = f"https://{config.ENVIRONMENT}.api.service.nhs.uk/" self_link = f"{base_url}record-locator/consumer/FHIR/R4/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|{params.nhs_number}" - # TODO - Add checks for the type code as well as system - if not validate_type_system(params.type, metadata.pointer_types): + if not validate_type(params.type, metadata.pointer_types): logger.log( LogReference.CONSEARCH002, type=params.type, diff --git a/api/consumer/searchPostDocumentReference/search_post_document_reference.py b/api/consumer/searchPostDocumentReference/search_post_document_reference.py index 39964f608..35f91366c 100644 --- a/api/consumer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/consumer/searchPostDocumentReference/search_post_document_reference.py @@ -9,7 +9,7 @@ from nrlf.core.logger import LogReference, logger from nrlf.core.model import ConnectionMetadata, ConsumerRequestParams from nrlf.core.response import Response, SpineErrorResponse -from nrlf.core.validators import validate_category, validate_type_system +from nrlf.core.validators import validate_category, validate_type @request_handler(body=ConsumerRequestParams) @@ -50,7 +50,7 @@ def handler( base_url = f"https://{config.ENVIRONMENT}.api.service.nhs.uk/" self_link = f"{base_url}record-locator/consumer/FHIR/R4/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|{body.nhs_number}" - if not validate_type_system(body.type, metadata.pointer_types): + if not validate_type(body.type, metadata.pointer_types): logger.log( LogReference.CONPOSTSEARCH002, type=body.type, diff --git a/api/producer/searchDocumentReference/search_document_reference.py b/api/producer/searchDocumentReference/search_document_reference.py index 403d22b8e..eb980f80c 100644 --- a/api/producer/searchDocumentReference/search_document_reference.py +++ b/api/producer/searchDocumentReference/search_document_reference.py @@ -6,7 +6,7 @@ from nrlf.core.logger import LogReference, logger from nrlf.core.model import ConnectionMetadata, ProducerRequestParams from nrlf.core.response import Response, SpineErrorResponse -from nrlf.core.validators import validate_category, validate_type_system +from nrlf.core.validators import validate_category, validate_type from nrlf.producer.fhir.r4.model import Bundle, DocumentReference @@ -48,7 +48,7 @@ def handler( expression="subject:identifier", ) - if not validate_type_system(params.type, metadata.pointer_types): + if not validate_type(params.type, metadata.pointer_types): logger.log( LogReference.PROSEARCH002, type=params.type, diff --git a/api/producer/searchPostDocumentReference/search_post_document_reference.py b/api/producer/searchPostDocumentReference/search_post_document_reference.py index 2c1159653..fc6d8b535 100644 --- a/api/producer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/producer/searchPostDocumentReference/search_post_document_reference.py @@ -6,7 +6,7 @@ from nrlf.core.logger import LogReference, logger from nrlf.core.model import ConnectionMetadata, ProducerRequestParams from nrlf.core.response import Response, SpineErrorResponse -from nrlf.core.validators import validate_category, validate_type_system +from nrlf.core.validators import validate_category, validate_type from nrlf.producer.fhir.r4.model import Bundle, DocumentReference @@ -42,7 +42,7 @@ def handler( expression="subject:identifier", ) - if not validate_type_system(body.type, metadata.pointer_types): + if not validate_type(body.type, metadata.pointer_types): logger.log( LogReference.PROPOSTSEARCH002, type=body.type, diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index 169e4af2c..f41e45c2c 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -13,7 +13,7 @@ from nrlf.core.validators import ( DocumentReferenceValidator, ValidationResult, - validate_type_system, + validate_type, ) from nrlf.producer.fhir.r4.model import ( DocumentReference, @@ -23,28 +23,37 @@ from nrlf.tests.data import load_document_reference_json -def test_validate_type_system_valid(): +def test_validate_type_valid(): type_ = RequestQueryType(root=PointerTypes.MENTAL_HEALTH_PLAN.value) pointer_types = [ PointerTypes.MENTAL_HEALTH_PLAN.value, PointerTypes.EOL_CARE_PLAN.value, ] - assert validate_type_system(type_, pointer_types) is True + assert validate_type(type_, pointer_types) is True -def test_validate_type_system_invalid(): +def test_validate_type_invalid_system(): type_ = RequestQueryType(root="http://snomed.info/invalid|736373009") pointer_types = [ PointerTypes.EOL_CARE_PLAN.value, PointerTypes.EOL_CARE_PLAN.value, ] - assert validate_type_system(type_, pointer_types) is False + assert validate_type(type_, pointer_types) is False -def test_validate_type_system_empty(): +def test_validate_type_invalid_code(): + type_ = RequestQueryType(root=PointerTypes.MRA_UPPER_LIMB_ARTERY.value) + pointer_types = [ + PointerTypes.MENTAL_HEALTH_PLAN.value, + PointerTypes.EOL_CARE_PLAN.value, + ] + assert validate_type(type_, pointer_types) is False + + +def test_validate_type_empty(): type_ = None pointer_types: list[str] = [] - assert validate_type_system(type_, pointer_types) is True + assert validate_type(type_, pointer_types) is True def test_validation_result_reset(): diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index fa1743013..bef650299 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -24,21 +24,14 @@ from nrlf.producer.fhir.r4 import model as producer_model -def validate_type_system( - type_: Optional[RequestQueryType], pointer_types: List[str] -) -> bool: +def validate_type(type_: Optional[RequestQueryType], pointer_types: List[str]) -> bool: """ Validates if the given type system is present in the list of pointer types. """ if not type_: return True - type_system = type_.root.split("|", 1)[0] - pointer_type_systems = [ - pointer_type.split("|", 1)[0] for pointer_type in pointer_types - ] - - return type_system in pointer_type_systems + return type_.root in pointer_types # TODO - Validate category is in set permissions once permissioning by category is done. From b67cb13eeef294723feb0d4f98901b96bbaa222c Mon Sep 17 00:00:00 2001 From: eesa456 Date: Wed, 11 Dec 2024 04:03:18 +0000 Subject: [PATCH 087/106] NRL-1202 fix int tests --- tests/features/steps/3_assert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/features/steps/3_assert.py b/tests/features/steps/3_assert.py index 1ce9c82f8..112a2f492 100644 --- a/tests/features/steps/3_assert.py +++ b/tests/features/steps/3_assert.py @@ -229,7 +229,7 @@ def assert_bundle_contains_documentreference_values_step(context: Context): raise ValueError("No id provided in the table") for entry in context.bundle.entry: - if entry.resource.id != items["id"]: + if entry.resource.get("id") != items["id"]: continue return assert_document_reference_matches_value(context, entry.resource, items) From e664a3fce99033cf80de00870cd934269fb5cb43 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Wed, 11 Dec 2024 04:17:44 +0000 Subject: [PATCH 088/106] Revert "NRL-1202 fix int tests" This reverts commit b67cb13eeef294723feb0d4f98901b96bbaa222c. --- tests/features/steps/3_assert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/features/steps/3_assert.py b/tests/features/steps/3_assert.py index 112a2f492..1ce9c82f8 100644 --- a/tests/features/steps/3_assert.py +++ b/tests/features/steps/3_assert.py @@ -229,7 +229,7 @@ def assert_bundle_contains_documentreference_values_step(context: Context): raise ValueError("No id provided in the table") for entry in context.bundle.entry: - if entry.resource.get("id") != items["id"]: + if entry.resource.id != items["id"]: continue return assert_document_reference_matches_value(context, entry.resource, items) From c113df836bcee55c9ffee85cfab67eb392d05e01 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Wed, 11 Dec 2024 13:01:34 +0000 Subject: [PATCH 089/106] NRL-518 Add validation for format code display mismatch --- layer/nrlf/core/tests/test_validators.py | 70 ++++++++++++++++++++++++ layer/nrlf/core/validators.py | 17 ++++++ 2 files changed, 87 insertions(+) diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index 851b5011f..0af045a13 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1360,3 +1360,73 @@ def test_validate_content_invalid_content_type(): "diagnostics": "Invalid contentType: invalid/type. Must be 'application/pdf' or 'text/html'", "expression": ["content[0].attachment.contentType"], } + + +@pytest.mark.parametrize( + "format_code, format_display", + [ + ("urn:nhs-ic:record-contact", "Contact details (HTTP Unsecured)"), + ("urn:nhs-ic:unstructured", "Unstructured Document"), + ], +) +def test_validate_nrl_format_code_valid_match(format_code, format_display): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["format"] = { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": format_code, + "display": format_display, + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is True + + +@pytest.mark.parametrize( + "format_code, format_display, expected_display", + [ + ( + "urn:nhs-ic:unstructured", + "Contact details (HTTP Unsecured)", + "Unstructured Document", + ), + ( + "urn:nhs-ic:record-contact", + "Unstructured Document", + "Contact details (HTTP Unsecured)", + ), + ], +) +def test_validate_nrl_format_code_display_mismatch( + format_code, format_display, expected_display +): + validator = DocumentReferenceValidator() + document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + + document_ref_data["content"][0]["format"] = { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": format_code, + "display": format_display, + } + + result = validator.validate(document_ref_data) + + assert result.is_valid is False + assert len(result.issues) == 1 + assert result.issues[0].model_dump(exclude_none=True) == { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource", + } + ] + }, + "diagnostics": f"Invalid display for format code '{format_code}'. Expected '{expected_display}'", + "expression": ["content[0].format.display"], + } diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index 8083e323a..98770748c 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -550,6 +550,11 @@ def _validate_content(self, model: DocumentReference): """ logger.log(LogReference.VALIDATOR001, step="content") + format_code_display_map = { + "urn:nhs-ic:record-contact": "Contact details (HTTP Unsecured)", + "urn:nhs-ic:unstructured": "Unstructured Document", + } + for i, content in enumerate(model.content): if content.attachment.contentType not in ["application/pdf", "text/html"]: self.result.add_error( @@ -558,3 +563,15 @@ def _validate_content(self, model: DocumentReference): diagnostics=f"Invalid contentType: {content.attachment.contentType}. Must be 'application/pdf' or 'text/html'", field=f"content[{i}].attachment.contentType", ) + + # Validate NRLFormatCode + format_code = content.format.code + format_display = content.format.display + expected_display = format_code_display_map.get(format_code) + if expected_display and format_display != expected_display: + self.result.add_error( + issue_code="value", + error_code="INVALID_RESOURCE", + diagnostics=f"Invalid display for format code '{format_code}'. Expected '{expected_display}'", + field=f"content[{i}].format.display", + ) From e505b70218f44b25e3e3b146145863870aa248db Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Wed, 11 Dec 2024 13:26:09 +0000 Subject: [PATCH 090/106] NRL-518 Add feature tests for format code display mismatch --- .../createDocumentReference-failure.feature | 58 +++++++++++++++ .../updateDocumentReference-failure.feature | 71 +++++++++++++++++++ .../upsertDocumentReference-failure.feature | 58 +++++++++++++++ 3 files changed, 187 insertions(+) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 33f472a76..9bf18f6d6 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -783,3 +783,61 @@ Feature: Producer - createDocumentReference - Failure Scenarios ] } """ + + Scenario: Mismatched format code and display + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "application/pdf", + "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid display for format code 'urn:nhs-ic:record-contact'. Expected 'Contact details (HTTP Unsecured)'", + "expression": [ + "content[0].format.display" + ] + } + """ diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index 99754f509..20a8bed4f 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -235,3 +235,74 @@ Feature: Producer - updateDocumentReference - Failure Scenarios ] } """ + + Scenario: Mismatched format code and display + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | TSTCUS-1114567893-updateDocTest | + | subject | 9999999999 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | TSTCUS | + | author | TSTCUS | + When producer 'TSTCUS' requests update of a DocumentReference with pointerId 'TSTCUS-1114567893-updateDocTest' and only changing: + """ + { + "content": [ + { + "attachment": { + "contentType": "application/pdf", + "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + } + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid display for format code 'urn:nhs-ic:record-contact'. Expected 'Contact details (HTTP Unsecured)'", + "expression": [ + "content[0].format.display" + ] + } + """ diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index a8de6ee35..c4fc4ea68 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -365,3 +365,61 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios ] } """ + + Scenario: Mismatched format code and display + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'TSTCUS' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When producer 'TSTCUS' requests upsert of a DocumentReference with pointerId 'TSTCUS-testid-upsert-0001-0001' and default test values except 'content' is: + """ + "content": [ + { + "attachment": { + "contentType": "application/pdf", + "url": "https://example.org/my-doc.pdf" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:record-contact", + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] + } + ] + """ + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "value", + "details": { + "coding": [ + { + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_RESOURCE", + "display": "Invalid validation of resource" + } + ] + }, + "diagnostics": "Invalid display for format code 'urn:nhs-ic:record-contact'. Expected 'Contact details (HTTP Unsecured)'", + "expression": [ + "content[0].format.display" + ] + } + """ From c62a99388f69b40862cc93ec66a02711888f755a Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Wed, 11 Dec 2024 13:27:20 +0000 Subject: [PATCH 091/106] [NRL-853] Fixup S3 bucket tag values --- .../account-wide-infrastructure/dev/aws-backup.tf | 11 ----------- .../modules/backup-source/backup_plan.tf | 2 +- .../modules/permissions-store-bucket/s3.tf | 2 +- .../modules/pointers-table/dynamodb.tf | 4 +++- .../modules/truststore-bucket/s3.tf | 4 +++- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/terraform/account-wide-infrastructure/dev/aws-backup.tf b/terraform/account-wide-infrastructure/dev/aws-backup.tf index fc41d32a8..d357e6b15 100644 --- a/terraform/account-wide-infrastructure/dev/aws-backup.tf +++ b/terraform/account-wide-infrastructure/dev/aws-backup.tf @@ -64,15 +64,6 @@ resource "aws_s3_bucket_acl" "backup_reports" { acl = "private" } -# We need a key for the SNS topic that will be used for notifications from AWS Backup. This key -# will be used to encrypt the messages sent to the topic before they are sent to the subscribers, -# but isn't needed by the recipients of the messages. - -# First we need some contextual data -data "aws_caller_identity" "current" {} -data "aws_region" "current" {} - -# Now we can define the key itself resource "aws_kms_key" "backup_notifications" { description = "KMS key for AWS Backup notifications" deletion_window_in_days = 7 @@ -101,8 +92,6 @@ resource "aws_kms_key" "backup_notifications" { }) } -# Now we can deploy the source and destination modules, referencing the resources we've created above. - module "source" { source = "../modules/backup-source" diff --git a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf index 0e6cd4ce8..298d654c1 100644 --- a/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf +++ b/terraform/account-wide-infrastructure/modules/backup-source/backup_plan.tf @@ -82,6 +82,6 @@ resource "aws_backup_selection" "dynamodb" { selection_tag { key = var.backup_plan_config_dynamodb.selection_tag type = "STRINGEQUALS" - value = "true" + value = "True" } } diff --git a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf index 06e61a58e..ab7fe77aa 100644 --- a/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/permissions-store-bucket/s3.tf @@ -5,7 +5,7 @@ resource "aws_s3_bucket" "authorization-store" { tags = { Name = "authorization store" Environment = "${var.name_prefix}" - NHSE-Enable-S3-Backup = "${var.enable_backups}" + NHSE-Enable-S3-Backup = var.enable_backups ? "True" : "False" } } diff --git a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf index 93e060fdb..06a7428b7 100644 --- a/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf +++ b/terraform/account-wide-infrastructure/modules/pointers-table/dynamodb.tf @@ -52,5 +52,7 @@ resource "aws_dynamodb_table" "pointers" { enabled = var.enable_pitr } - tags = { NHSE-Enable-DDB-Backup = "${var.enable_backups}" } + tags = { + NHSE-Enable-DDB-Backup = var.enable_backups ? "True" : "False" + } } diff --git a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf index aa32f2f16..1f7bd3e81 100644 --- a/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf +++ b/terraform/account-wide-infrastructure/modules/truststore-bucket/s3.tf @@ -1,7 +1,9 @@ resource "aws_s3_bucket" "api_truststore" { bucket = "${var.name_prefix}-api-truststore" force_destroy = var.enable_bucket_force_destroy - tags = { NHSE-Enable-S3-Backup = "${var.enable_backups}" } + tags = { + NHSE-Enable-S3-Backup = var.enable_backups ? "True" : "False" + } } resource "aws_s3_bucket_policy" "api_truststore_bucket_policy" { From cef46169005774abd8b47c32a4c14c540f103613 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Wed, 11 Dec 2024 14:19:55 +0000 Subject: [PATCH 092/106] NRL-477 update model and error messages --- api/consumer/swagger.yaml | 2 +- .../tests/test_create_document_reference.py | 2 +- api/producer/swagger.yaml | 2 +- .../tests/test_upsert_document_reference.py | 2 +- layer/nrlf/consumer/fhir/r4/model.py | 9 ++-- layer/nrlf/producer/fhir/r4/model.py | 9 ++-- layer/nrlf/producer/fhir/r4/strict_model.py | 5 ++- .../createDocumentReference-failure.feature | 42 ------------------- 8 files changed, 13 insertions(+), 60 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 3f8141469..f62fc6a90 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -788,7 +788,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "^(entered-in-error|amended|preliminary|final)$" + enum: ["entered-in-error", "amended", "preliminary", "final"] description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" diff --git a/api/producer/createDocumentReference/tests/test_create_document_reference.py b/api/producer/createDocumentReference/tests/test_create_document_reference.py index b58d3c234..56daacfc6 100644 --- a/api/producer/createDocumentReference/tests/test_create_document_reference.py +++ b/api/producer/createDocumentReference/tests/test_create_document_reference.py @@ -445,7 +445,7 @@ def test_create_document_reference_with_invalid_docStatus(): } ], }, - "diagnostics": "Request body could not be parsed (docStatus: String should match pattern '^(entered-in-error|amended|preliminary|final)$')", + "diagnostics": "Request body could not be parsed (docStatus: Input should be 'entered-in-error', 'amended', 'preliminary' or 'final')", "expression": ["docStatus"], }, ], diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index fa10b2790..25fe84fc4 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1351,7 +1351,7 @@ components: description: The status of this document reference. docStatus: type: string - pattern: "^(entered-in-error|amended|preliminary|final)$" + enum: ["entered-in-error", "amended", "preliminary", "final"] description: The status of the underlying document. type: $ref: "#/components/schemas/CodeableConcept" diff --git a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py index 009c12075..090542dae 100644 --- a/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py +++ b/api/producer/upsertDocumentReference/tests/test_upsert_document_reference.py @@ -489,7 +489,7 @@ def test_upsert_document_reference_with_invalid_docStatus(): } ], }, - "diagnostics": "Request body could not be parsed (docStatus: String should match pattern '^(entered-in-error|amended|preliminary|final)$')", + "diagnostics": "Request body could not be parsed (docStatus: Input should be 'entered-in-error', 'amended', 'preliminary' or 'final')", "expression": ["docStatus"], }, ], diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index 70167bf4a..a679c5084 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-05T01:41:57+00:00 +# timestamp: 2024-12-11T14:01:43+00:00 from __future__ import annotations @@ -577,11 +577,8 @@ class DocumentReference(BaseModel): ), ] docStatus: Annotated[ - Optional[str], - Field( - description="The status of the underlying document.", - pattern="^(entered-in-error|amended|preliminary|final)$", - ), + Optional[Literal["entered-in-error", "amended", "preliminary", "final"]], + Field(description="The status of the underlying document."), ] = None type: Annotated[ Optional[CodeableConcept], diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index 6a7e49abb..99f0d7d6d 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-05T01:41:53+00:00 +# timestamp: 2024-12-11T14:01:38+00:00 from __future__ import annotations @@ -561,11 +561,8 @@ class DocumentReference(BaseModel): ), ] docStatus: Annotated[ - Optional[str], - Field( - description="The status of the underlying document.", - pattern="^(entered-in-error|amended|preliminary|final)$", - ), + Optional[Literal["entered-in-error", "amended", "preliminary", "final"]], + Field(description="The status of the underlying document."), ] = None type: Annotated[ Optional[CodeableConcept], diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index a7d73861a..bd78aab06 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-05T01:41:55+00:00 +# timestamp: 2024-12-11T14:01:41+00:00 from __future__ import annotations @@ -491,7 +491,8 @@ class DocumentReference(BaseModel): StrictStr, Field(description="The status of this document reference.") ] docStatus: Annotated[ - Optional[StrictStr], Field(description="The status of the underlying document.") + Optional[Literal["entered-in-error", "amended", "preliminary", "final"]], + Field(description="The status of the underlying document."), ] = None type: Annotated[ Optional[CodeableConcept], diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index cd2e9fbcb..d7674aa41 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -499,48 +499,6 @@ Feature: Producer - createDocumentReference - Failure Scenarios } """ - # Invalid document reference - invalid Type - # NRL-769 Known issue: Type display is not validated - # Scenario: Invalid type (valid code but wrong display value) - # Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API - # And the organisation 'TSTCUS' is authorised to access pointer types: - # | system | value | - # | http://snomed.info/sct | 1363501000000100 | - # | http://snomed.info/sct | 736253002 | - # When producer 'TSTCUS' requests creation of a DocumentReference with default test values except 'type' is: - # """ - # "type": { - # "coding": [ - # { - # "system": "http://snomed.info/sct", - # "code": "736253002", - # "display": "Emergency Healthcare Plan" - # } - # ] - # } - # """ - # Then the response status code is 400 - # And the response is an OperationOutcome with 1 issue - # And the OperationOutcome contains the issue: - # """ - # { - # "severity": "error", - # "code": "invalid", - # "details": { - # "coding": [ - # { - # "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", - # "code": "BAD_REQUEST", - # "display": "Bad request" - # } - # ] - # }, - # "diagnostics": "The display does not match the expected value for this type", - # "expression": [ - # "type.coding.display" - # ] - # } - # """ # Invalid document reference - empty content[0].attachment.url # Invalid document reference - create another producers document # Invalid document reference - bad JSON From 85efa9e06e6a41a67be7adb95c4cf75cd09b3a8d Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Wed, 11 Dec 2024 17:05:36 +0000 Subject: [PATCH 093/106] NRL-518 Better model inheritance --- api/consumer/swagger.yaml | 112 ++++++++++--------- api/producer/swagger.yaml | 113 +++++++++++--------- layer/nrlf/consumer/fhir/r4/model.py | 82 +++++++------- layer/nrlf/producer/fhir/r4/model.py | 82 +++++++------- layer/nrlf/producer/fhir/r4/strict_model.py | 80 +++++++------- tests/features/utils/data.py | 8 +- tests/smoke/setup.py | 7 +- 7 files changed, 261 insertions(+), 223 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index dbe549ea6..38173f417 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -1195,65 +1195,77 @@ components: pattern: \S* description: The reference details for the link. ContentStabilityExtension: - type: object - properties: - url: - type: string - enum: - - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - valueCodeableConcept: - type: object + allOf: + - $ref: "#/components/schemas/Extension" + - type: object + properties: + url: + type: string + enum: + - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + valueCodeableConcept: + $ref: "#/components/schemas/ContentStabilityExtensionValueCodeableConcept" + required: + - url + - valueCodeableConcept + ContentStabilityExtensionValueCodeableConcept: + allOf: + - $ref: "#/components/schemas/CodeableConcept" + - type: object properties: coding: type: array items: - type: object - properties: - system: - type: string - enum: - - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" - code: - type: string - enum: ["static", "dynamic"] - display: - type: string - enum: ["Static", "Dynamic"] - required: - - system - - code - - display + $ref: "#/components/schemas/ContentStabilityExtensionCoding" minItems: 1 maxItems: 1 required: - coding - required: - - url - - valueCodeableConcept + ContentStabilityExtensionCoding: + allOf: + - $ref: "#/components/schemas/Coding" + - type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + code: + type: string + enum: ["static", "dynamic"] + display: + type: string + enum: ["Static", "Dynamic"] + required: + - system + - code + - display NRLFormatCode: - type: object - properties: - system: - type: string - enum: - - "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" - description: The system URL for the NRLF Format Code. - code: - type: string - enum: - - "urn:nhs-ic:record-contact" - - "urn:nhs-ic:unstructured" - description: The code representing the format of the document. - display: - type: string - enum: - - "Contact details (HTTP Unsecured)" - - "Unstructured Document" - description: The display text for the code. - required: - - system - - code - - display + allOf: + - $ref: "#/components/schemas/Coding" + - type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" + description: The system URL for the NRLF Format Code. + code: + type: string + enum: + - "urn:nhs-ic:record-contact" + - "urn:nhs-ic:unstructured" + description: The code representing the format of the document. + display: + type: string + enum: + - "Contact details (HTTP Unsecured)" + - "Unstructured Document" + description: The display text for the code. + required: + - system + - code + - display Identifier: type: object properties: diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index a00d5239e..8c014a3a7 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1723,7 +1723,6 @@ components: items: $ref: "#/components/schemas/Extension" description: Additional content defined by implementations. - Coding: type: object properties: @@ -1761,65 +1760,77 @@ components: pattern: \S* description: The reference details for the link. ContentStabilityExtension: - type: object - properties: - url: - type: string - enum: - - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - valueCodeableConcept: - type: object + allOf: + - $ref: "#/components/schemas/Extension" + - type: object + properties: + url: + type: string + enum: + - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + valueCodeableConcept: + $ref: "#/components/schemas/ContentStabilityExtensionValueCodeableConcept" + required: + - url + - valueCodeableConcept + ContentStabilityExtensionValueCodeableConcept: + allOf: + - $ref: "#/components/schemas/CodeableConcept" + - type: object properties: coding: type: array items: - type: object - properties: - system: - type: string - enum: - - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" - code: - type: string - enum: ["static", "dynamic"] - display: - type: string - enum: ["Static", "Dynamic"] - required: - - system - - code - - display + $ref: "#/components/schemas/ContentStabilityExtensionCoding" minItems: 1 maxItems: 1 required: - coding - required: - - url - - valueCodeableConcept + ContentStabilityExtensionCoding: + allOf: + - $ref: "#/components/schemas/Coding" + - type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + code: + type: string + enum: ["static", "dynamic"] + display: + type: string + enum: ["Static", "Dynamic"] + required: + - system + - code + - display NRLFormatCode: - type: object - properties: - system: - type: string - enum: - - "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" - description: The system URL for the NRLF Format Code. - code: - type: string - enum: - - "urn:nhs-ic:record-contact" - - "urn:nhs-ic:unstructured" - description: The code representing the format of the document. - display: - type: string - enum: - - "Contact details (HTTP Unsecured)" - - "Unstructured Document" - description: The display text for the code. - required: - - system - - code - - display + allOf: + - $ref: "#/components/schemas/Coding" + - type: object + properties: + system: + type: string + enum: + - "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" + description: The system URL for the NRLF Format Code. + code: + type: string + enum: + - "urn:nhs-ic:record-contact" + - "urn:nhs-ic:unstructured" + description: The code representing the format of the document. + display: + type: string + enum: + - "Contact details (HTTP Unsecured)" + - "Unstructured Document" + description: The display text for the code. + required: + - system + - code + - display Identifier: type: object properties: diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index 161fabb1f..3dc3d9265 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-06T04:10:37+00:00 +# timestamp: 2024-12-11T16:30:20+00:00 from __future__ import annotations @@ -230,7 +230,7 @@ class Coding(BaseModel): ] = None -class CodingItem(BaseModel): +class ContentStabilityExtensionCoding(Coding): system: Literal[ "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" ] @@ -238,18 +238,7 @@ class CodingItem(BaseModel): display: Literal["Static", "Dynamic"] -class ValueCodeableConcept(BaseModel): - coding: Annotated[List[CodingItem], Field(max_length=1, min_length=1)] - - -class ContentStabilityExtension(BaseModel): - url: Literal[ - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - ] - valueCodeableConcept: ValueCodeableConcept - - -class NRLFormatCode(BaseModel): +class NRLFormatCode(Coding): system: Annotated[ Literal["https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode"], Field(description="The system URL for the NRLF Format Code."), @@ -461,31 +450,6 @@ class RequestHeaderCorrelationId(RootModel[str]): root: Annotated[str, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - NRLFormatCode, - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] - extension: Annotated[ - List[ContentStabilityExtension], Field(max_length=1, min_length=1) - ] - - class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -877,6 +841,31 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None +class DocumentReferenceContent(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + attachment: Annotated[ + Attachment, + Field( + description="The document or URL of the document along with critical metadata to prove content has integrity." + ), + ] + format: Annotated[ + NRLFormatCode, + Field( + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." + ), + ] + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] + + class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[str], @@ -1062,12 +1051,27 @@ class Signature(BaseModel): ] = None +class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): + coding: Annotated[ + List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) + ] + + +class ContentStabilityExtension(Extension): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept + + OperationOutcome.model_rebuild() OperationOutcomeIssue.model_rebuild() DocumentReference.model_rebuild() Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() +DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() CodeableConcept.model_rebuild() Identifier.model_rebuild() +ContentStabilityExtensionValueCodeableConcept.model_rebuild() diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index 580836343..afd77dae2 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-06T04:10:35+00:00 +# timestamp: 2024-12-11T16:30:17+00:00 from __future__ import annotations @@ -230,7 +230,7 @@ class Coding(BaseModel): ] = None -class CodingItem(BaseModel): +class ContentStabilityExtensionCoding(Coding): system: Literal[ "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" ] @@ -238,18 +238,7 @@ class CodingItem(BaseModel): display: Literal["Static", "Dynamic"] -class ValueCodeableConcept(BaseModel): - coding: Annotated[List[CodingItem], Field(max_length=1, min_length=1)] - - -class ContentStabilityExtension(BaseModel): - url: Literal[ - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - ] - valueCodeableConcept: ValueCodeableConcept - - -class NRLFormatCode(BaseModel): +class NRLFormatCode(Coding): system: Annotated[ Literal["https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode"], Field(description="The system URL for the NRLF Format Code."), @@ -451,31 +440,6 @@ class RequestHeaderCorrelationId(RootModel[str]): root: Annotated[str, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - NRLFormatCode, - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] - extension: Annotated[ - List[ContentStabilityExtension], Field(max_length=1, min_length=1) - ] - - class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -861,6 +825,31 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None +class DocumentReferenceContent(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + attachment: Annotated[ + Attachment, + Field( + description="The document or URL of the document along with critical metadata to prove content has integrity." + ), + ] + format: Annotated[ + NRLFormatCode, + Field( + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." + ), + ] + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] + + class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[str], @@ -1046,12 +1035,27 @@ class Signature(BaseModel): ] = None +class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): + coding: Annotated[ + List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) + ] + + +class ContentStabilityExtension(Extension): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept + + OperationOutcome.model_rebuild() OperationOutcomeIssue.model_rebuild() DocumentReference.model_rebuild() Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() +DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() CodeableConcept.model_rebuild() Identifier.model_rebuild() +ContentStabilityExtensionValueCodeableConcept.model_rebuild() diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index 6d1e371b7..17074f3b5 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-06T04:10:36+00:00 +# timestamp: 2024-12-11T16:30:18+00:00 from __future__ import annotations @@ -207,7 +207,7 @@ class Coding(BaseModel): ] = None -class CodingItem(BaseModel): +class ContentStabilityExtensionCoding(Coding): system: Literal[ "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" ] @@ -215,18 +215,7 @@ class CodingItem(BaseModel): display: Literal["Static", "Dynamic"] -class ValueCodeableConcept(BaseModel): - coding: Annotated[List[CodingItem], Field(max_length=1, min_length=1)] - - -class ContentStabilityExtension(BaseModel): - url: Literal[ - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - ] - valueCodeableConcept: ValueCodeableConcept - - -class NRLFormatCode(BaseModel): +class NRLFormatCode(Coding): system: Annotated[ Literal["https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode"], Field(description="The system URL for the NRLF Format Code."), @@ -399,30 +388,6 @@ class RequestHeaderCorrelationId(RootModel[StrictStr]): root: Annotated[StrictStr, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[StrictStr], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - NRLFormatCode, - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] - extension: Annotated[ - List[ContentStabilityExtension], Field(max_length=1, min_length=1) - ] - - class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -764,6 +729,30 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None +class DocumentReferenceContent(BaseModel): + id: Annotated[ + Optional[StrictStr], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." + ), + ] = None + attachment: Annotated[ + Attachment, + Field( + description="The document or URL of the document along with critical metadata to prove content has integrity." + ), + ] + format: Annotated[ + NRLFormatCode, + Field( + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." + ), + ] + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] + + class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[StrictStr], @@ -925,12 +914,27 @@ class Signature(BaseModel): ] = None +class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): + coding: Annotated[ + List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) + ] + + +class ContentStabilityExtension(Extension): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept + + OperationOutcome.model_rebuild() OperationOutcomeIssue.model_rebuild() DocumentReference.model_rebuild() Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() +DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() CodeableConcept.model_rebuild() Identifier.model_rebuild() +ContentStabilityExtensionValueCodeableConcept.model_rebuild() diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 723c358a6..609a220e5 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -3,8 +3,9 @@ Attachment, CodeableConcept, Coding, - CodingItem, ContentStabilityExtension, + ContentStabilityExtensionCoding, + ContentStabilityExtensionValueCodeableConcept, DocumentReference, DocumentReferenceContent, DocumentReferenceContext, @@ -12,7 +13,6 @@ Identifier, NRLFormatCode, Reference, - ValueCodeableConcept, ) from tests.features.utils.constants import ( DEFAULT_TEST_AUTHOR, @@ -51,9 +51,9 @@ def create_test_document_reference(items: dict) -> DocumentReference: extension=[ ContentStabilityExtension( url="https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - valueCodeableConcept=ValueCodeableConcept( + valueCodeableConcept=ContentStabilityExtensionValueCodeableConcept( coding=[ - CodingItem( + ContentStabilityExtensionCoding( system="https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", code="static", display="Static", diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index a8f8bdb7d..d50f649cc 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -3,6 +3,9 @@ Attachment, CodeableConcept, Coding, + ContentStabilityExtension, + ContentStabilityExtensionCoding, + ContentStabilityExtensionValueCodeableConcept, DocumentReference, DocumentReferenceContent, DocumentReferenceContext, @@ -41,9 +44,9 @@ def build_document_reference( extension=[ ContentStabilityExtension( url="https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", - valueCodeableConcept=ValueCodeableConcept( + valueCodeableConcept=ContentStabilityExtensionValueCodeableConcept( coding=[ - CodingItem( + ContentStabilityExtensionCoding( system="https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", code="static", display="Static", From 22523d985062fe08c24417506d13b10585d81b60 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Wed, 11 Dec 2024 17:30:11 +0000 Subject: [PATCH 094/106] NRL-518 Update narrative --- swagger/producer-static/narrative.yaml | 38 ++++++++++++++++++++------ 1 file changed, 30 insertions(+), 8 deletions(-) diff --git a/swagger/producer-static/narrative.yaml b/swagger/producer-static/narrative.yaml index fd567ac08..8380e0879 100644 --- a/swagger/producer-static/narrative.yaml +++ b/swagger/producer-static/narrative.yaml @@ -276,18 +276,40 @@ paths: ] ``` * `content` MUST have at least one entry. - * `content` MUST include an `attachment` entry. - * `content` MUST include a `format` entry. - * `content` MUST include the content stability extension (https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability). + * `content[]` MUST include an `attachment` entry. + * `content[]` MUST include a `format` entry. (https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode) + * `content[]` MUST include the content stability extension (https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability). * `content[].attachment` MUST include a `url` to the document. * `content[].attachment` MUST include a `contentType` and be a valid MIME type, specifically `application/pdf` for documents or `text/html` for contact details. - * `content[].format[]` MUST indicate whether the data is structured or not, e.g. + * `content[].format` MUST indicate whether the data is structured or not + * Example of the content section: ``` - "format": [ + "content": [ { - "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:unstructured" - "display": "Unstructured Document" + "attachment": { + "contentType": "application/pdf", + "url": "https://provider-ods-code.thirdparty.nhs.uk/path/to/document.pdf", + "creation": "2022-12-22T09:45:41+11:00" + }, + "format": { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + "code": "urn:nhs-ic:unstructured" + "display": "Unstructured Document" + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] ``` From d48ab5a928b66b05283b218f942b1461bf4376ed Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Thu, 12 Dec 2024 11:27:29 +0000 Subject: [PATCH 095/106] NRL-518 Move content fhir url strings to constants --- layer/nrlf/core/constants.py | 7 +++++++ layer/nrlf/core/errors.py | 5 +++-- tests/features/utils/data.py | 14 ++++++++++---- tests/smoke/setup.py | 15 +++++++++++---- 4 files changed, 31 insertions(+), 10 deletions(-) diff --git a/layer/nrlf/core/constants.py b/layer/nrlf/core/constants.py index fa8e961ea..9fa73906d 100644 --- a/layer/nrlf/core/constants.py +++ b/layer/nrlf/core/constants.py @@ -185,3 +185,10 @@ def coding_value(self): SYSTEM_SHORT_IDS = {"http://snomed.info/sct": "SCT", "https://nicip.nhs.uk": "NICIP"} +CONTENT_STABILITY_EXTENSION_URL = ( + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" +) +CONTENT_STABILITY_SYSTEM_URL = ( + "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" +) +CONTENT_FORMAT_CODE_URL = "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" diff --git a/layer/nrlf/core/errors.py b/layer/nrlf/core/errors.py index e4cc82e21..615ef9a2a 100644 --- a/layer/nrlf/core/errors.py +++ b/layer/nrlf/core/errors.py @@ -3,6 +3,7 @@ from pydantic import ValidationError from pydantic_core import ErrorDetails +from nrlf.core.constants import CONTENT_FORMAT_CODE_URL, CONTENT_STABILITY_SYSTEM_URL from nrlf.core.response import Response from nrlf.core.types import CodeableConcept from nrlf.producer.fhir.r4 import model as producer_model @@ -25,9 +26,9 @@ def append_value_set_url(loc_string: str) -> str: if "content" in loc_string: if "extension" in loc_string: - return ". See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability" + return f". See ValueSet: {CONTENT_STABILITY_SYSTEM_URL}" if "format" in loc_string: - return ". See ValueSet: https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode" + return f". See ValueSet: {CONTENT_FORMAT_CODE_URL}" return "" diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 609a220e5..ed3a3e948 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -1,4 +1,10 @@ -from layer.nrlf.core.constants import CATEGORY_ATTRIBUTES, TYPE_ATTRIBUTES +from layer.nrlf.core.constants import ( + CATEGORY_ATTRIBUTES, + CONTENT_FORMAT_CODE_URL, + CONTENT_STABILITY_EXTENSION_URL, + CONTENT_STABILITY_SYSTEM_URL, + TYPE_ATTRIBUTES, +) from nrlf.producer.fhir.r4.model import ( Attachment, CodeableConcept, @@ -43,18 +49,18 @@ def create_test_document_reference(items: dict) -> DocumentReference: format=NRLFormatCode( system=items.get( "formatSystem", - "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + CONTENT_FORMAT_CODE_URL, ), code=items.get("formatCode", "urn:nhs-ic:unstructured"), display=items.get("formatDisplay", "Unstructured Document"), ), extension=[ ContentStabilityExtension( - url="https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + url=CONTENT_STABILITY_EXTENSION_URL, valueCodeableConcept=ContentStabilityExtensionValueCodeableConcept( coding=[ ContentStabilityExtensionCoding( - system="https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + system=CONTENT_STABILITY_SYSTEM_URL, code="static", display="Static", ) diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index d50f649cc..e40f2e823 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -1,4 +1,11 @@ -from nrlf.core.constants import TYPE_ATTRIBUTES, Categories, PointerTypes +from nrlf.core.constants import ( + CONTENT_FORMAT_CODE_URL, + CONTENT_STABILITY_EXTENSION_URL, + CONTENT_STABILITY_SYSTEM_URL, + TYPE_ATTRIBUTES, + Categories, + PointerTypes, +) from nrlf.producer.fhir.r4.model import ( Attachment, CodeableConcept, @@ -37,17 +44,17 @@ def build_document_reference( url=content_url, ), format=NRLFormatCode( - system="https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", + system=CONTENT_FORMAT_CODE_URL, code="urn:nhs-ic:unstructured", display="Unstructured Document", ), extension=[ ContentStabilityExtension( - url="https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + url=CONTENT_STABILITY_EXTENSION_URL, valueCodeableConcept=ContentStabilityExtensionValueCodeableConcept( coding=[ ContentStabilityExtensionCoding( - system="https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + system=CONTENT_STABILITY_SYSTEM_URL, code="static", display="Static", ) From 64e6312c54e03d4c5f5fe82c8624adb5cb8def45 Mon Sep 17 00:00:00 2001 From: "Axel Garcia K." Date: Thu, 12 Dec 2024 15:30:40 +0000 Subject: [PATCH 096/106] NRL-518 Remove extension from CodeableConcept --- api/consumer/swagger.yaml | 5 - api/producer/swagger.yaml | 5 - layer/nrlf/consumer/fhir/r4/model.py | 194 ++++++++++---------- layer/nrlf/producer/fhir/r4/model.py | 194 ++++++++++---------- layer/nrlf/producer/fhir/r4/strict_model.py | 176 +++++++++--------- 5 files changed, 273 insertions(+), 301 deletions(-) diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 38173f417..cb1474228 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -1153,11 +1153,6 @@ components: type: string pattern: "[ \\r\\n\\t\\S]+" description: A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user. - extension: - type: array - items: - $ref: "#/components/schemas/Extension" - description: Additional content defined by implementations. Coding: type: object properties: diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index 8c014a3a7..a0f83a3c9 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -1718,11 +1718,6 @@ components: type: string pattern: "[ \\r\\n\\t\\S]+" description: A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user. - extension: - type: array - items: - $ref: "#/components/schemas/Extension" - description: Additional content defined by implementations. Coding: type: object properties: diff --git a/layer/nrlf/consumer/fhir/r4/model.py b/layer/nrlf/consumer/fhir/r4/model.py index 3dc3d9265..d8e069ddc 100644 --- a/layer/nrlf/consumer/fhir/r4/model.py +++ b/layer/nrlf/consumer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-11T16:30:20+00:00 +# timestamp: 2024-12-12T13:19:56+00:00 from __future__ import annotations @@ -450,6 +450,43 @@ class RequestHeaderCorrelationId(RootModel[str]): root: Annotated[str, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] +class CodeableConcept(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + coding: Optional[List[Coding]] = None + text: Annotated[ + Optional[str], + Field( + description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user.", + pattern="[ \\r\\n\\t\\S]+", + ), + ] = None + + +class Extension(BaseModel): + valueCodeableConcept: Annotated[ + Optional[CodeableConcept], + Field( + description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." + ), + ] = None + url: Annotated[ + Optional[str], + Field(description="The reference details for the link.", pattern="\\S*"), + ] = None + + +class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): + coding: Annotated[ + List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) + ] + + class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -474,6 +511,52 @@ class CountRequestParams(BaseModel): ] +class OperationOutcomeIssue(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + severity: Annotated[ + str, + Field( + description="Indicates whether the issue indicates a variation from successful processing.", + pattern="[^\\s]+(\\s[^\\s]+)*", + ), + ] + code: Annotated[ + str, + Field( + description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element.", + pattern="[^\\s]+(\\s[^\\s]+)*", + ), + ] + details: Annotated[ + Optional[CodeableConcept], + Field( + description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." + ), + ] = None + diagnostics: Annotated[ + Optional[str], + Field( + description="Additional diagnostic information about the issue.", + pattern="[ \\r\\n\\t\\S]+", + ), + ] = None + location: Optional[List[LocationItem]] = None + expression: Optional[List[ExpressionItem]] = None + + +class ContentStabilityExtension(Extension): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept + + class OperationOutcome(BaseModel): resourceType: Literal["OperationOutcome"] id: Annotated[ @@ -512,7 +595,7 @@ class OperationOutcome(BaseModel): issue: Annotated[List[OperationOutcomeIssue], Field(min_length=1)] -class OperationOutcomeIssue(BaseModel): +class DocumentReferenceContent(BaseModel): id: Annotated[ Optional[str], Field( @@ -520,35 +603,21 @@ class OperationOutcomeIssue(BaseModel): pattern="[A-Za-z0-9\\-\\.]{1,64}", ), ] = None - severity: Annotated[ - str, + attachment: Annotated[ + Attachment, Field( - description="Indicates whether the issue indicates a variation from successful processing.", - pattern="[^\\s]+(\\s[^\\s]+)*", + description="The document or URL of the document along with critical metadata to prove content has integrity." ), ] - code: Annotated[ - str, + format: Annotated[ + NRLFormatCode, Field( - description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element.", - pattern="[^\\s]+(\\s[^\\s]+)*", + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), ] - details: Annotated[ - Optional[CodeableConcept], - Field( - description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." - ), - ] = None - diagnostics: Annotated[ - Optional[str], - Field( - description="Additional diagnostic information about the issue.", - pattern="[ \\r\\n\\t\\S]+", - ), - ] = None - location: Optional[List[LocationItem]] = None - expression: Optional[List[ExpressionItem]] = None + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] class DocumentReference(BaseModel): @@ -841,31 +910,6 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - NRLFormatCode, - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] - extension: Annotated[ - List[ContentStabilityExtension], Field(max_length=1, min_length=1) - ] - - class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[str], @@ -886,38 +930,6 @@ class DocumentReferenceRelatesTo(BaseModel): ] -class CodeableConcept(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - coding: Optional[List[Coding]] = None - text: Annotated[ - Optional[str], - Field( - description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user.", - pattern="[ \\r\\n\\t\\S]+", - ), - ] = None - extension: Optional[List[Extension]] = None - - -class Extension(BaseModel): - valueCodeableConcept: Annotated[ - Optional[CodeableConcept], - Field( - description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." - ), - ] = None - url: Annotated[ - Optional[str], - Field(description="The reference details for the link.", pattern="\\S*"), - ] = None - - class Identifier(BaseModel): id: Annotated[ Optional[str], @@ -1051,27 +1063,9 @@ class Signature(BaseModel): ] = None -class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): - coding: Annotated[ - List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) - ] - - -class ContentStabilityExtension(Extension): - url: Literal[ - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - ] - valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept - - -OperationOutcome.model_rebuild() -OperationOutcomeIssue.model_rebuild() DocumentReference.model_rebuild() Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() -DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() -CodeableConcept.model_rebuild() Identifier.model_rebuild() -ContentStabilityExtensionValueCodeableConcept.model_rebuild() diff --git a/layer/nrlf/producer/fhir/r4/model.py b/layer/nrlf/producer/fhir/r4/model.py index afd77dae2..538659e9d 100644 --- a/layer/nrlf/producer/fhir/r4/model.py +++ b/layer/nrlf/producer/fhir/r4/model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-11T16:30:17+00:00 +# timestamp: 2024-12-12T13:19:54+00:00 from __future__ import annotations @@ -440,6 +440,43 @@ class RequestHeaderCorrelationId(RootModel[str]): root: Annotated[str, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] +class CodeableConcept(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + coding: Optional[List[Coding]] = None + text: Annotated[ + Optional[str], + Field( + description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user.", + pattern="[ \\r\\n\\t\\S]+", + ), + ] = None + + +class Extension(BaseModel): + valueCodeableConcept: Annotated[ + Optional[CodeableConcept], + Field( + description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." + ), + ] = None + url: Annotated[ + Optional[str], + Field(description="The reference details for the link.", pattern="\\S*"), + ] = None + + +class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): + coding: Annotated[ + List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) + ] + + class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -455,6 +492,52 @@ class RequestParams(BaseModel): ] = None +class OperationOutcomeIssue(BaseModel): + id: Annotated[ + Optional[str], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", + pattern="[A-Za-z0-9\\-\\.]{1,64}", + ), + ] = None + severity: Annotated[ + str, + Field( + description="Indicates whether the issue indicates a variation from successful processing.", + pattern="[^\\s]+(\\s[^\\s]+)*", + ), + ] + code: Annotated[ + str, + Field( + description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element.", + pattern="[^\\s]+(\\s[^\\s]+)*", + ), + ] + details: Annotated[ + Optional[CodeableConcept], + Field( + description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." + ), + ] = None + diagnostics: Annotated[ + Optional[str], + Field( + description="Additional diagnostic information about the issue.", + pattern="[ \\r\\n\\t\\S]+", + ), + ] = None + location: Optional[List[LocationItem]] = None + expression: Optional[List[ExpressionItem]] = None + + +class ContentStabilityExtension(Extension): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept + + class OperationOutcome(BaseModel): resourceType: Literal["OperationOutcome"] id: Annotated[ @@ -493,7 +576,7 @@ class OperationOutcome(BaseModel): issue: Annotated[List[OperationOutcomeIssue], Field(min_length=1)] -class OperationOutcomeIssue(BaseModel): +class DocumentReferenceContent(BaseModel): id: Annotated[ Optional[str], Field( @@ -501,35 +584,21 @@ class OperationOutcomeIssue(BaseModel): pattern="[A-Za-z0-9\\-\\.]{1,64}", ), ] = None - severity: Annotated[ - str, + attachment: Annotated[ + Attachment, Field( - description="Indicates whether the issue indicates a variation from successful processing.", - pattern="[^\\s]+(\\s[^\\s]+)*", + description="The document or URL of the document along with critical metadata to prove content has integrity." ), ] - code: Annotated[ - str, + format: Annotated[ + NRLFormatCode, Field( - description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element.", - pattern="[^\\s]+(\\s[^\\s]+)*", + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), ] - details: Annotated[ - Optional[CodeableConcept], - Field( - description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." - ), - ] = None - diagnostics: Annotated[ - Optional[str], - Field( - description="Additional diagnostic information about the issue.", - pattern="[ \\r\\n\\t\\S]+", - ), - ] = None - location: Optional[List[LocationItem]] = None - expression: Optional[List[ExpressionItem]] = None + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] class DocumentReference(BaseModel): @@ -825,31 +894,6 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - NRLFormatCode, - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] - extension: Annotated[ - List[ContentStabilityExtension], Field(max_length=1, min_length=1) - ] - - class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[str], @@ -870,38 +914,6 @@ class DocumentReferenceRelatesTo(BaseModel): ] -class CodeableConcept(BaseModel): - id: Annotated[ - Optional[str], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces.", - pattern="[A-Za-z0-9\\-\\.]{1,64}", - ), - ] = None - coding: Optional[List[Coding]] = None - text: Annotated[ - Optional[str], - Field( - description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user.", - pattern="[ \\r\\n\\t\\S]+", - ), - ] = None - extension: Optional[List[Extension]] = None - - -class Extension(BaseModel): - valueCodeableConcept: Annotated[ - Optional[CodeableConcept], - Field( - description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." - ), - ] = None - url: Annotated[ - Optional[str], - Field(description="The reference details for the link.", pattern="\\S*"), - ] = None - - class Identifier(BaseModel): id: Annotated[ Optional[str], @@ -1035,27 +1047,9 @@ class Signature(BaseModel): ] = None -class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): - coding: Annotated[ - List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) - ] - - -class ContentStabilityExtension(Extension): - url: Literal[ - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - ] - valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept - - -OperationOutcome.model_rebuild() -OperationOutcomeIssue.model_rebuild() DocumentReference.model_rebuild() Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() -DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() -CodeableConcept.model_rebuild() Identifier.model_rebuild() -ContentStabilityExtensionValueCodeableConcept.model_rebuild() diff --git a/layer/nrlf/producer/fhir/r4/strict_model.py b/layer/nrlf/producer/fhir/r4/strict_model.py index 17074f3b5..cfa6f3243 100644 --- a/layer/nrlf/producer/fhir/r4/strict_model.py +++ b/layer/nrlf/producer/fhir/r4/strict_model.py @@ -1,6 +1,6 @@ # generated by datamodel-codegen: # filename: swagger.yaml -# timestamp: 2024-12-11T16:30:18+00:00 +# timestamp: 2024-12-12T13:19:55+00:00 from __future__ import annotations @@ -388,6 +388,40 @@ class RequestHeaderCorrelationId(RootModel[StrictStr]): root: Annotated[StrictStr, Field(examples=["11C46F5F-CDEF-4865-94B2-0EE0EDCC26DA"])] +class CodeableConcept(BaseModel): + id: Annotated[ + Optional[StrictStr], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." + ), + ] = None + coding: Optional[List[Coding]] = None + text: Annotated[ + Optional[StrictStr], + Field( + description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user." + ), + ] = None + + +class Extension(BaseModel): + valueCodeableConcept: Annotated[ + Optional[CodeableConcept], + Field( + description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." + ), + ] = None + url: Annotated[ + Optional[StrictStr], Field(description="The reference details for the link.") + ] = None + + +class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): + coding: Annotated[ + List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) + ] + + class RequestHeader(BaseModel): odsCode: RequestHeaderOdsCode @@ -403,6 +437,46 @@ class RequestParams(BaseModel): ] = None +class OperationOutcomeIssue(BaseModel): + id: Annotated[ + Optional[StrictStr], + Field( + description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." + ), + ] = None + severity: Annotated[ + StrictStr, + Field( + description="Indicates whether the issue indicates a variation from successful processing." + ), + ] + code: Annotated[ + StrictStr, + Field( + description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element." + ), + ] + details: Annotated[ + Optional[CodeableConcept], + Field( + description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." + ), + ] = None + diagnostics: Annotated[ + Optional[StrictStr], + Field(description="Additional diagnostic information about the issue."), + ] = None + location: Optional[List[LocationItem]] = None + expression: Optional[List[ExpressionItem]] = None + + +class ContentStabilityExtension(Extension): + url: Literal[ + "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" + ] + valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept + + class OperationOutcome(BaseModel): resourceType: Literal["OperationOutcome"] id: Annotated[ @@ -436,37 +510,28 @@ class OperationOutcome(BaseModel): issue: Annotated[List[OperationOutcomeIssue], Field(min_length=1)] -class OperationOutcomeIssue(BaseModel): +class DocumentReferenceContent(BaseModel): id: Annotated[ Optional[StrictStr], Field( description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." ), ] = None - severity: Annotated[ - StrictStr, + attachment: Annotated[ + Attachment, Field( - description="Indicates whether the issue indicates a variation from successful processing." + description="The document or URL of the document along with critical metadata to prove content has integrity." ), ] - code: Annotated[ - StrictStr, + format: Annotated[ + NRLFormatCode, Field( - description="Describes the type of the issue. The system that creates an OperationOutcome SHALL choose the most applicable code from the IssueType value set, and may additional provide its own code for the error in the details element." + description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." ), ] - details: Annotated[ - Optional[CodeableConcept], - Field( - description="Additional details about the error. This may be a text description of the error or a system code that identifies the error." - ), - ] = None - diagnostics: Annotated[ - Optional[StrictStr], - Field(description="Additional diagnostic information about the issue."), - ] = None - location: Optional[List[LocationItem]] = None - expression: Optional[List[ExpressionItem]] = None + extension: Annotated[ + List[ContentStabilityExtension], Field(max_length=1, min_length=1) + ] class DocumentReference(BaseModel): @@ -729,30 +794,6 @@ class DocumentReferenceContext(BaseModel): related: Optional[List[Reference]] = None -class DocumentReferenceContent(BaseModel): - id: Annotated[ - Optional[StrictStr], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." - ), - ] = None - attachment: Annotated[ - Attachment, - Field( - description="The document or URL of the document along with critical metadata to prove content has integrity." - ), - ] - format: Annotated[ - NRLFormatCode, - Field( - description="An identifier of the document encoding, structure, and template that the document conforms to beyond the base format indicated in the mimeType." - ), - ] - extension: Annotated[ - List[ContentStabilityExtension], Field(max_length=1, min_length=1) - ] - - class DocumentReferenceRelatesTo(BaseModel): id: Annotated[ Optional[StrictStr], @@ -771,35 +812,6 @@ class DocumentReferenceRelatesTo(BaseModel): ] -class CodeableConcept(BaseModel): - id: Annotated[ - Optional[StrictStr], - Field( - description="Unique id for the element within a resource (for internal references). This may be any string value that does not contain spaces." - ), - ] = None - coding: Optional[List[Coding]] = None - text: Annotated[ - Optional[StrictStr], - Field( - description="A human language representation of the concept as seen/selected/uttered by the user who entered the data and/or which represents the intended meaning of the user." - ), - ] = None - extension: Optional[List[Extension]] = None - - -class Extension(BaseModel): - valueCodeableConcept: Annotated[ - Optional[CodeableConcept], - Field( - description="A name which details the functional use for this link – see [http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1](http://www.iana.org/assignments/link–relations/link–relations.xhtml#link–relations–1)." - ), - ] = None - url: Annotated[ - Optional[StrictStr], Field(description="The reference details for the link.") - ] = None - - class Identifier(BaseModel): id: Annotated[ Optional[StrictStr], @@ -914,27 +926,9 @@ class Signature(BaseModel): ] = None -class ContentStabilityExtensionValueCodeableConcept(CodeableConcept): - coding: Annotated[ - List[ContentStabilityExtensionCoding], Field(max_length=1, min_length=1) - ] - - -class ContentStabilityExtension(Extension): - url: Literal[ - "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability" - ] - valueCodeableConcept: ContentStabilityExtensionValueCodeableConcept - - -OperationOutcome.model_rebuild() -OperationOutcomeIssue.model_rebuild() DocumentReference.model_rebuild() Bundle.model_rebuild() BundleEntry.model_rebuild() DocumentReferenceContext.model_rebuild() -DocumentReferenceContent.model_rebuild() DocumentReferenceRelatesTo.model_rebuild() -CodeableConcept.model_rebuild() Identifier.model_rebuild() -ContentStabilityExtensionValueCodeableConcept.model_rebuild() From a95f144698a79803ce1080588e68cdd5f641f886 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Thu, 12 Dec 2024 16:00:11 +0000 Subject: [PATCH 097/106] [NRL-1228] Fix advance care plan naming. Add report script to find invalid pointers --- api/consumer/swagger.yaml | 6 +- api/producer/swagger.yaml | 6 +- layer/nrlf/core/constants.py | 8 +- reports/find_invalid_pointers.py | 81 +++++++++++++++++++ resources/fhir/NRLF-RecordType-ValueSet.json | 2 +- .../createDocumentReference-success.feature | 2 +- tests/performance/environment.py | 2 +- 7 files changed, 94 insertions(+), 13 deletions(-) create mode 100644 reports/find_invalid_pointers.py diff --git a/api/consumer/swagger.yaml b/api/consumer/swagger.yaml index 47c1a072c..632dca1bf 100644 --- a/api/consumer/swagger.yaml +++ b/api/consumer/swagger.yaml @@ -36,7 +36,7 @@ info: * [End of Life Care Coordination Summary](http://snomed.info/sct/861421000000109) * [Emergency health care plan](http://snomed.info/sct/887701000000100) * [Lloyd George record folder](http://snomed.info/sct/16521000000101) - * [Advanced care plan](http://snomed.info/sct/736366004) + * [Advance care plan](http://snomed.info/sct/736366004) * [Treatment escalation plan](http://snomed.info/sct/735324008) * [Summary record]("http://snomed.info/sct|824321000000109") * [Personalised Care and Support Plan]("http://snomed.info/sct|2181441000000107") @@ -1519,8 +1519,8 @@ components: SNOMED_CODES_LLOYD_GEORGE_RECORD_FOLDER: summary: Lloyd George record folder value: http://snomed.info/sct|16521000000101 - SNOMED_CODES_ADVANCED_CARE_PLAN: - summary: Advanced care plan + SNOMED_CODES_ADVANCE_CARE_PLAN: + summary: Advance care plan value: http://snomed.info/sct|736366004 SNOMED_CODES_TREATMENT_ESCALATION_PLAN: summary: Treatment escalation plan diff --git a/api/producer/swagger.yaml b/api/producer/swagger.yaml index bdd0127b8..642932212 100644 --- a/api/producer/swagger.yaml +++ b/api/producer/swagger.yaml @@ -36,7 +36,7 @@ info: * [End of Life Care Coordination Summary](http://snomed.info/sct/861421000000109) * [Emergency health care plan](http://snomed.info/sct/887701000000100) * [Lloyd George record folder](http://snomed.info/sct/16521000000101) - * [Advanced care plan](http://snomed.info/sct/736366004) + * [Advance care plan](http://snomed.info/sct/736366004) * [Treatment escalation plan](http://snomed.info/sct/735324008) * [Summary record]("http://snomed.info/sct|824321000000109") * [Personalised Care and Support Plan]("http://snomed.info/sct|2181441000000107") @@ -2054,8 +2054,8 @@ components: SNOMED_CODES_LLOYD_GEORGE_RECORD_FOLDER: summary: Lloyd George record folder value: http://snomed.info/sct|16521000000101 - SNOMED_CODES_ADVANCED_CARE_PLAN: - summary: Advanced care plan + SNOMED_CODES_ADVANCE_CARE_PLAN: + summary: Advance care plan value: http://snomed.info/sct|736366004 SNOMED_CODES_TREATMENT_ESCALATION_PLAN: summary: Treatment escalation plan diff --git a/layer/nrlf/core/constants.py b/layer/nrlf/core/constants.py index 67971859f..4280f7a6d 100644 --- a/layer/nrlf/core/constants.py +++ b/layer/nrlf/core/constants.py @@ -60,7 +60,7 @@ class PointerTypes(Enum): CONTINGENCY_PLAN = "http://snomed.info/sct|325691000000100" EOL_CARE_PLAN = "http://snomed.info/sct|736373009" LLOYD_GEORGE_FOLDER = "http://snomed.info/sct|16521000000101" - ADVANCED_CARE_PLAN = "http://snomed.info/sct|736366004" + ADVANCE_CARE_PLAN = "http://snomed.info/sct|736366004" TREATMENT_ESCALATION_PLAN = "http://snomed.info/sct|735324008" SUMMARY_RECORD = "http://snomed.info/sct|824321000000109" PERSONALISED_CARE_AND_SUPPORT_PLAN = "http://snomed.info/sct|2181441000000107" @@ -139,8 +139,8 @@ def coding_value(self): PointerTypes.LLOYD_GEORGE_FOLDER.value: { "display": "Lloyd George record folder", }, - PointerTypes.ADVANCED_CARE_PLAN.value: { - "display": "Advanced care plan", + PointerTypes.ADVANCE_CARE_PLAN.value: { + "display": "Advance care plan", }, PointerTypes.TREATMENT_ESCALATION_PLAN.value: { "display": "Treatment escalation plan", @@ -169,7 +169,7 @@ def coding_value(self): PointerTypes.CONTINGENCY_PLAN.value: Categories.CARE_PLAN.value, PointerTypes.EOL_CARE_PLAN.value: Categories.CARE_PLAN.value, PointerTypes.LLOYD_GEORGE_FOLDER.value: Categories.CARE_PLAN.value, - PointerTypes.ADVANCED_CARE_PLAN.value: Categories.CARE_PLAN.value, + PointerTypes.ADVANCE_CARE_PLAN.value: Categories.CARE_PLAN.value, PointerTypes.TREATMENT_ESCALATION_PLAN.value: Categories.CARE_PLAN.value, PointerTypes.PERSONALISED_CARE_AND_SUPPORT_PLAN.value: Categories.CARE_PLAN.value, # diff --git a/reports/find_invalid_pointers.py b/reports/find_invalid_pointers.py new file mode 100644 index 000000000..23bbb8e07 --- /dev/null +++ b/reports/find_invalid_pointers.py @@ -0,0 +1,81 @@ +from datetime import datetime, timedelta, timezone +from typing import Any + +import boto3 +import fire + +from nrlf.consumer.fhir.r4.model import DocumentReference +from nrlf.core.logger import logger +from nrlf.core.validators import DocumentReferenceValidator + +dynamodb = boto3.client("dynamodb") +paginator = dynamodb.get_paginator("scan") + +logger.setLevel("ERROR") + + +def _validate_document(document: str): + docref = DocumentReference.model_validate_json(document) + + validator = DocumentReferenceValidator() + result = validator.validate(data=docref) + + if not result.is_valid: + raise Exception("Failed to validate document: " + str(result.issues)) + + +def _find_invalid_pointers(table_name: str) -> dict[str, float]: + """ + Find pointers in the given table that are invalid. + Parameters: + - table_name: The name of the pointers table to use. + """ + + print(f"Finding invalid pointers in table {table_name}....") # noqa + + params: dict[str, Any] = { + "TableName": table_name, + "PaginationConfig": {"PageSize": 50}, + } + + invalid_pointers = [] + total_scanned_count = 0 + + start_time = datetime.now(tz=timezone.utc) + + for page in paginator.paginate(**params): + for item in page["Items"]: + id = item.get("id", {}).get("S") + document = item.get("document", {}).get("S", "") + try: + _validate_document(document) + except Exception as exc: + invalid_pointers.append((id, exc)) + + total_scanned_count += page["ScannedCount"] + + if total_scanned_count % 1000 == 0: + print(".", end="", flush=True) # noqa + + if total_scanned_count % 100000 == 0: + print( # noqa + f"scanned={total_scanned_count} invalid={len(invalid_pointers)}" + ) + + end_time = datetime.now(tz=timezone.utc) + + print("Writing invalid_pointers to file ./invalid_pointers.txt ...") # noqa + with open("invalid_pointers.txt", "w") as f: + for id, err in invalid_pointers: + f.write(f"{id}: {err}\n") + + print(" Done") # noqa + return { + "invalid_pointers": len(invalid_pointers), + "scanned_count": total_scanned_count, + "took-secs": timedelta.total_seconds(end_time - start_time), + } + + +if __name__ == "__main__": + fire.Fire(_find_invalid_pointers) diff --git a/resources/fhir/NRLF-RecordType-ValueSet.json b/resources/fhir/NRLF-RecordType-ValueSet.json index 4fda682f0..08652fe5f 100644 --- a/resources/fhir/NRLF-RecordType-ValueSet.json +++ b/resources/fhir/NRLF-RecordType-ValueSet.json @@ -56,7 +56,7 @@ }, { "code": "736366004", - "display": "Advanced care plan" + "display": "Advance care plan" }, { "code": "735324008", diff --git a/tests/features/producer/createDocumentReference-success.feature b/tests/features/producer/createDocumentReference-success.feature index 202595190..8842111fb 100644 --- a/tests/features/producer/createDocumentReference-success.feature +++ b/tests/features/producer/createDocumentReference-success.feature @@ -211,7 +211,7 @@ Feature: Producer - createDocumentReference - Success Scenarios | 325691000000100 | 734163000 | CONTINGENCY_PLAN | | 736373009 | 734163000 | EOL_CARE_PLAN | | 16521000000101 | 734163000 | LLOYD_GEORGE_FOLDER | - | 736366004 | 734163000 | ADVANCED_CARE_PLAN | + | 736366004 | 734163000 | ADVANCE_CARE_PLAN | | 735324008 | 734163000 | TREATMENT_ESCALATION_PLAN | | 2181441000000107 | 734163000 | PERSONALISED_CARE_AND_SUPPORT_PLAN | diff --git a/tests/performance/environment.py b/tests/performance/environment.py index 5baa3f7f0..fc7861bd7 100644 --- a/tests/performance/environment.py +++ b/tests/performance/environment.py @@ -29,7 +29,7 @@ class LogReference: "736373009": "End of life care plan", "861421000000109": "End of life care coordination summary", "887701000000100": "Emergency Health Care Plans", - "736366004": "Advanced Care Plan", + "736366004": "Advance Care Plan", "735324008": "Treatment Escalation Plan", "824321000000109": "Summary Record", "2181441000000107": "Personalised Care and Support Plan", From d946923215ab26862080872dfb62e88f36897eb2 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 07:03:43 +0000 Subject: [PATCH 098/106] NRL-1076 add integration and unit tests for identifier accession number and error message fix --- .../search_document_reference.py | 2 +- ...test_search_document_reference_consumer.py | 44 +++++++++++++++++++ .../search_post_document_reference.py | 2 +- .../search_document_reference.py | 2 +- .../search_post_document_reference.py | 2 +- .../searchDocumentReference-failure.feature | 30 ++++++++++++- .../searchDocumentReference-success.feature | 38 ++++++++++++++++ ...earchPostDocumentReference-failure.feature | 30 ++++++++++++- tests/features/steps/3_assert.py | 8 ++++ tests/features/utils/data.py | 6 +++ 10 files changed, 158 insertions(+), 6 deletions(-) diff --git a/api/consumer/searchDocumentReference/search_document_reference.py b/api/consumer/searchDocumentReference/search_document_reference.py index ae590e0f8..d847e9ef4 100644 --- a/api/consumer/searchDocumentReference/search_document_reference.py +++ b/api/consumer/searchDocumentReference/search_document_reference.py @@ -53,7 +53,7 @@ def handler( pointer_types=metadata.pointer_types, ) return SpineErrorResponse.INVALID_CODE_SYSTEM( - diagnostics="Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + diagnostics="Invalid query parameter (The provided type does not match the allowed types for this organisation)", expression="type", ) diff --git a/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py b/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py index 1f30aafc5..0453eea2c 100644 --- a/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py +++ b/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py @@ -59,6 +59,50 @@ def test_search_document_reference_happy_path(repository: DocumentPointerReposit } +@mock_aws +@mock_repository +def test_search_document_reference_accession_number_in_pointer( + repository: DocumentPointerRepository, +): + doc_ref = load_document_reference("Y05868-736253002-Valid") + doc_ref.identifier = [ + {"type": {"text": "Accession-Number"}, "value": "Y05868.123456789"} + ] + doc_pointer = DocumentPointer.from_document_reference(doc_ref) + repository.create(doc_pointer) + + event = create_test_api_gateway_event( + headers=create_headers(), + query_string_parameters={ + "subject:identifier": "https://fhir.nhs.uk/Id/nhs-number|6700028191", + }, + ) + + result = handler(event, create_mock_context()) + body = result.pop("body") + + assert result == { + "statusCode": "200", + "headers": default_response_headers(), + "isBase64Encoded": False, + } + + parsed_body = json.loads(body) + assert parsed_body == { + "resourceType": "Bundle", + "type": "searchset", + "total": 1, + "entry": [{"resource": doc_ref.model_dump(exclude_none=True)}], + } + + created_doc_pointer = repository.get_by_id("Y05868-99999-99999-999999") + + assert created_doc_pointer is not None + assert json.loads(created_doc_pointer.document)["identifier"] == [ + {"type": {"text": "Accession-Number"}, "value": "Y05868.123456789"} + ] + + @mock_aws @mock_repository def test_search_document_reference_happy_path_with_custodian( diff --git a/api/consumer/searchPostDocumentReference/search_post_document_reference.py b/api/consumer/searchPostDocumentReference/search_post_document_reference.py index 35f91366c..5ea555cf0 100644 --- a/api/consumer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/consumer/searchPostDocumentReference/search_post_document_reference.py @@ -57,7 +57,7 @@ def handler( pointer_types=metadata.pointer_types, ) return SpineErrorResponse.INVALID_CODE_SYSTEM( - diagnostics="Invalid type (The provided type system does not match the allowed types for this organisation)", + diagnostics="Invalid type (The provided type does not match the allowed types for this organisation)", expression="type", ) diff --git a/api/producer/searchDocumentReference/search_document_reference.py b/api/producer/searchDocumentReference/search_document_reference.py index eb980f80c..e961e7b9f 100644 --- a/api/producer/searchDocumentReference/search_document_reference.py +++ b/api/producer/searchDocumentReference/search_document_reference.py @@ -55,7 +55,7 @@ def handler( pointer_types=metadata.pointer_types, ) return SpineErrorResponse.INVALID_CODE_SYSTEM( - diagnostics="Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + diagnostics="Invalid query parameter (The provided type does not match the allowed types for this organisation)", expression="type", ) diff --git a/api/producer/searchPostDocumentReference/search_post_document_reference.py b/api/producer/searchPostDocumentReference/search_post_document_reference.py index fc6d8b535..023dcfe7c 100644 --- a/api/producer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/producer/searchPostDocumentReference/search_post_document_reference.py @@ -49,7 +49,7 @@ def handler( pointer_types=metadata.pointer_types, ) return SpineErrorResponse.INVALID_CODE_SYSTEM( - diagnostics="The provided type system does not match the allowed types for this organisation", + diagnostics="The provided type does not match the allowed types for this organisation", expression="type", ) diff --git a/tests/features/consumer/searchDocumentReference-failure.feature b/tests/features/consumer/searchDocumentReference-failure.feature index 1de20623a..161e12bcd 100644 --- a/tests/features/consumer/searchDocumentReference-failure.feature +++ b/tests/features/consumer/searchDocumentReference-failure.feature @@ -77,7 +77,35 @@ Feature: Consumer - searchDocumentReference - Failure Scenarios "display": "Invalid code system" }] }, - "diagnostics": "Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "Invalid query parameter (The provided type does not match the allowed types for this organisation)", + "expression": ["type"] + } + """ + + Scenario: Search rejects request with type they are not allowed to use + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When consumer 'RX898' searches for DocumentReferences with parameters: + | parameter | value | + | subject | 9278693472 | + | type | http://snomed.info/sct\|887701000000100 | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "code-invalid", + "details": { + "coding": [{ + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_CODE_SYSTEM", + "display": "Invalid code system" + }] + }, + "diagnostics": "Invalid query parameter (The provided type does not match the allowed types for this organisation)", "expression": ["type"] } """ diff --git a/tests/features/consumer/searchDocumentReference-success.feature b/tests/features/consumer/searchDocumentReference-success.feature index ead47a40e..1fd296127 100644 --- a/tests/features/consumer/searchDocumentReference-success.feature +++ b/tests/features/consumer/searchDocumentReference-success.feature @@ -36,6 +36,44 @@ Feature: Consumer - searchDocumentReference - Success Scenarios | custodian | 02V | | author | 02V | + Scenario: Search for a DocumentReference and Accession Number is in response + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + And a DocumentReference resource exists with values: + | property | value | + | id | 02V-1111111111-SearchDocRefTest | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | 02V | + | author | 02V | + | identifier | 02V.123456789 | + When consumer 'RX898' searches for DocumentReferences with parameters: + | parameter | value | + | subject | 9278693472 | + Then the response status code is 200 + And the response is a searchset Bundle + And the Bundle has a self link matching 'DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|9278693472' + And the Bundle has a total of 1 + And the Bundle has 1 entry + And the Bundle contains an DocumentReference with values + | property | value | + | id | 02V-1111111111-SearchDocRefTest | + | subject | 9278693472 | + | status | current | + | type | 736253002 | + | category | 734163000 | + | contentType | application/pdf | + | url | https://example.org/my-doc.pdf | + | custodian | 02V | + | author | 02V | + | identifier | 02V.123456789 | + Scenario: Search for a DocumentReference by NHS Number and Custodian where both search parameters match Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API And the organisation 'RX898' is authorised to access pointer types: diff --git a/tests/features/consumer/searchPostDocumentReference-failure.feature b/tests/features/consumer/searchPostDocumentReference-failure.feature index 91380c764..c798af6ec 100644 --- a/tests/features/consumer/searchPostDocumentReference-failure.feature +++ b/tests/features/consumer/searchPostDocumentReference-failure.feature @@ -77,7 +77,35 @@ Feature: Consumer - searchDocumentReference - Failure Scenarios "display": "Invalid code system" }] }, - "diagnostics": "Invalid type (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "Invalid type (The provided type does not match the allowed types for this organisation)", + "expression": ["type"] + } + """ + + Scenario: Search rejects request with type they are not allowed to use + Given the application 'DataShare' (ID 'z00z-y11y-x22x') is registered to access the API + And the organisation 'RX898' is authorised to access pointer types: + | system | value | + | http://snomed.info/sct | 736253002 | + When consumer 'RX898' searches for DocumentReferences using POST with request body: + | key | value | + | subject | 9278693472 | + | type | http://snomed.info/sct\|887701000000100 | + Then the response status code is 400 + And the response is an OperationOutcome with 1 issue + And the OperationOutcome contains the issue: + """ + { + "severity": "error", + "code": "code-invalid", + "details": { + "coding": [{ + "system": "https://fhir.nhs.uk/ValueSet/Spine-ErrorOrWarningCode-1", + "code": "INVALID_CODE_SYSTEM", + "display": "Invalid code system" + }] + }, + "diagnostics": "Invalid type (The provided type does not match the allowed types for this organisation)", "expression": ["type"] } """ diff --git a/tests/features/steps/3_assert.py b/tests/features/steps/3_assert.py index 1ce9c82f8..445f2268d 100644 --- a/tests/features/steps/3_assert.py +++ b/tests/features/steps/3_assert.py @@ -218,6 +218,14 @@ def assert_document_reference_matches_value( context.response.json(), ) + if identifier := items.get("identifier"): + assert doc_ref.identifier[0].value == identifier, format_error( + "DocumentReference Identifier does not match", + identifier, + doc_ref.identifier[0].value, + context.response.json(), + ) + @then("the Bundle contains an DocumentReference with values") def assert_bundle_contains_documentreference_values_step(context: Context): diff --git a/tests/features/utils/data.py b/tests/features/utils/data.py index 5bc300585..f6a672eb0 100644 --- a/tests/features/utils/data.py +++ b/tests/features/utils/data.py @@ -128,6 +128,12 @@ def create_test_document_reference(items: dict) -> DocumentReference: ), ) ] + if items.get("identifier"): + base_doc_ref.identifier = [ + Identifier( + type=CodeableConcept(text="Accession-Number"), value=items["identifier"] + ) + ] return base_doc_ref From 4abb3e3724ccce686c68812c02021eb52615f301 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 07:16:42 +0000 Subject: [PATCH 099/106] NRL-1076 fix tests --- .../tests/test_search_document_reference_consumer.py | 8 +++++++- .../tests/test_search_post_document_reference_consumer.py | 2 +- .../tests/test_search_document_reference_producer.py | 2 +- .../tests/test_search_post_document_reference_producer.py | 2 +- layer/nrlf/core/validators.py | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py b/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py index 0453eea2c..88d1757c8 100644 --- a/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py +++ b/api/consumer/searchDocumentReference/tests/test_search_document_reference_consumer.py @@ -92,6 +92,12 @@ def test_search_document_reference_accession_number_in_pointer( "resourceType": "Bundle", "type": "searchset", "total": 1, + "link": [ + { + "relation": "self", + "url": "https://pytest.api.service.nhs.uk/record-locator/consumer/FHIR/R4/DocumentReference?subject:identifier=https://fhir.nhs.uk/Id/nhs-number|6700028191", + } + ], "entry": [{"resource": doc_ref.model_dump(exclude_none=True)}], } @@ -495,7 +501,7 @@ def test_search_document_reference_invalid_type(repository: DocumentPointerRepos } ] }, - "diagnostics": "Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "Invalid query parameter (The provided type does not match the allowed types for this organisation)", "expression": ["type"], } ], diff --git a/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py b/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py index 4e3ed4e4d..3e50b535b 100644 --- a/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py +++ b/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py @@ -424,7 +424,7 @@ def test_search_post_document_reference_invalid_type( } ] }, - "diagnostics": "Invalid type (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "Invalid type (The provided type does not match the allowed types for this organisation)", "expression": ["type"], } ], diff --git a/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py b/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py index d7a46ece7..b577f7754 100644 --- a/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py +++ b/api/producer/searchDocumentReference/tests/test_search_document_reference_producer.py @@ -201,7 +201,7 @@ def test_search_document_reference_invalid_type(repository: DocumentPointerRepos } ] }, - "diagnostics": "Invalid query parameter (The provided type system does not match the allowed types for this organisation)", + "diagnostics": "Invalid query parameter (The provided type does not match the allowed types for this organisation)", "expression": ["type"], } ], diff --git a/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py b/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py index f17de3fb1..8340ee0d3 100644 --- a/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py +++ b/api/producer/searchPostDocumentReference/tests/test_search_post_document_reference_producer.py @@ -206,7 +206,7 @@ def test_search_document_reference_invalid_type(repository: DocumentPointerRepos } ] }, - "diagnostics": "The provided type system does not match the allowed types for this organisation", + "diagnostics": "The provided type does not match the allowed types for this organisation", "expression": ["type"], } ], diff --git a/layer/nrlf/core/validators.py b/layer/nrlf/core/validators.py index bef650299..f75edb41f 100644 --- a/layer/nrlf/core/validators.py +++ b/layer/nrlf/core/validators.py @@ -26,7 +26,7 @@ def validate_type(type_: Optional[RequestQueryType], pointer_types: List[str]) -> bool: """ - Validates if the given type system is present in the list of pointer types. + Validates if the given type is present in the list of pointer types. """ if not type_: return True From 8b398a7466aad04e3c24084054b2b721d97aa49c Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 07:19:38 +0000 Subject: [PATCH 100/106] NRL-518 add import for smoke tests --- tests/smoke/setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/smoke/setup.py b/tests/smoke/setup.py index 4fb6d38da..f639e80f8 100644 --- a/tests/smoke/setup.py +++ b/tests/smoke/setup.py @@ -18,6 +18,7 @@ DocumentReferenceContext, DocumentReferenceRelatesTo, Identifier, + NRLFormatCode, Reference, ) from tests.utilities.api_clients import ProducerTestClient From e203cc18d1f9bbbabb6c29d8669132766dc69a73 Mon Sep 17 00:00:00 2001 From: Matt Dean Date: Fri, 13 Dec 2024 09:34:06 +0000 Subject: [PATCH 101/106] [NRL-1228] Fixup sonar cloud warnings --- reports/find_invalid_pointers.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/reports/find_invalid_pointers.py b/reports/find_invalid_pointers.py index 23bbb8e07..506f27b7a 100644 --- a/reports/find_invalid_pointers.py +++ b/reports/find_invalid_pointers.py @@ -21,10 +21,10 @@ def _validate_document(document: str): result = validator.validate(data=docref) if not result.is_valid: - raise Exception("Failed to validate document: " + str(result.issues)) + raise RuntimeError("Failed to validate document: " + str(result.issues)) -def _find_invalid_pointers(table_name: str) -> dict[str, float]: +def _find_invalid_pointers(table_name: str) -> dict[str, float | int]: """ Find pointers in the given table that are invalid. Parameters: @@ -45,12 +45,12 @@ def _find_invalid_pointers(table_name: str) -> dict[str, float]: for page in paginator.paginate(**params): for item in page["Items"]: - id = item.get("id", {}).get("S") + pointer_id = item.get("id", {}).get("S") document = item.get("document", {}).get("S", "") try: _validate_document(document) except Exception as exc: - invalid_pointers.append((id, exc)) + invalid_pointers.append((pointer_id, exc)) total_scanned_count += page["ScannedCount"] @@ -64,12 +64,13 @@ def _find_invalid_pointers(table_name: str) -> dict[str, float]: end_time = datetime.now(tz=timezone.utc) + print(" Done") # noqa + print("Writing invalid_pointers to file ./invalid_pointers.txt ...") # noqa with open("invalid_pointers.txt", "w") as f: - for id, err in invalid_pointers: - f.write(f"{id}: {err}\n") + for _id, err in invalid_pointers: + f.write(f"{_id}: {err}\n") - print(" Done") # noqa return { "invalid_pointers": len(invalid_pointers), "scanned_count": total_scanned_count, From 1754771ae617e936e67824a6bd239b4487ce4e9b Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 09:47:08 +0000 Subject: [PATCH 102/106] NRL-1202 standardise message --- .../search_post_document_reference.py | 2 +- .../tests/test_search_post_document_reference_consumer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/consumer/searchPostDocumentReference/search_post_document_reference.py b/api/consumer/searchPostDocumentReference/search_post_document_reference.py index 5ea555cf0..6cb9b7e8c 100644 --- a/api/consumer/searchPostDocumentReference/search_post_document_reference.py +++ b/api/consumer/searchPostDocumentReference/search_post_document_reference.py @@ -57,7 +57,7 @@ def handler( pointer_types=metadata.pointer_types, ) return SpineErrorResponse.INVALID_CODE_SYSTEM( - diagnostics="Invalid type (The provided type does not match the allowed types for this organisation)", + diagnostics="The provided type does not match the allowed types for this organisation", expression="type", ) diff --git a/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py b/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py index 3e50b535b..6616ec51c 100644 --- a/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py +++ b/api/consumer/searchPostDocumentReference/tests/test_search_post_document_reference_consumer.py @@ -424,7 +424,7 @@ def test_search_post_document_reference_invalid_type( } ] }, - "diagnostics": "Invalid type (The provided type does not match the allowed types for this organisation)", + "diagnostics": "The provided type does not match the allowed types for this organisation", "expression": ["type"], } ], From 123f8379b53a5c5f1384e92dd60164f470df3cff Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 10:42:16 +0000 Subject: [PATCH 103/106] NRL-1202 fix int tests --- .../consumer/searchPostDocumentReference-failure.feature | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/features/consumer/searchPostDocumentReference-failure.feature b/tests/features/consumer/searchPostDocumentReference-failure.feature index c798af6ec..3c76f708a 100644 --- a/tests/features/consumer/searchPostDocumentReference-failure.feature +++ b/tests/features/consumer/searchPostDocumentReference-failure.feature @@ -77,7 +77,7 @@ Feature: Consumer - searchDocumentReference - Failure Scenarios "display": "Invalid code system" }] }, - "diagnostics": "Invalid type (The provided type does not match the allowed types for this organisation)", + "diagnostics": "The provided type does not match the allowed types for this organisation", "expression": ["type"] } """ @@ -105,7 +105,7 @@ Feature: Consumer - searchDocumentReference - Failure Scenarios "display": "Invalid code system" }] }, - "diagnostics": "Invalid type (The provided type does not match the allowed types for this organisation)", + "diagnostics": "The provided type does not match the allowed types for this organisation", "expression": ["type"] } """ From 9408c91f1e32dd7ee0c874e5d4ae126307f0cb7c Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 11:57:59 +0000 Subject: [PATCH 104/106] NRL-477 fix tests --- layer/nrlf/core/tests/test_validators.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/layer/nrlf/core/tests/test_validators.py b/layer/nrlf/core/tests/test_validators.py index bb661ddc8..4b13e5d56 100644 --- a/layer/nrlf/core/tests/test_validators.py +++ b/layer/nrlf/core/tests/test_validators.py @@ -1309,8 +1309,8 @@ def test_validate_content_format_invalid_code_for_unstructured_document(): document_ref_data["content"][0]["format"] = { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", - "code": "urn:nhs-ic:contact", - "display": "Contact details", + "code": "urn:nhs-ic:record-contact", + "display": "Contact details (HTTP Unsecured)", } result = validator.validate(document_ref_data) @@ -1330,7 +1330,7 @@ def test_validate_content_format_invalid_code_for_unstructured_document(): } ] }, - "diagnostics": "Invalid content format code: urn:nhs-ic:contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", + "diagnostics": "Invalid content format code: urn:nhs-ic:record-contact format code must be 'urn:nhs-ic:unstructured' for Unstructured Document attachments.", "expression": ["content[0].format.code"], } @@ -1643,6 +1643,8 @@ def test_validate_content_invalid_content_type(): def test_validate_nrl_format_code_valid_match(format_code, format_display): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + if format_code == "urn:nhs-ic:record-contact": + document_ref_data["content"][0]["attachment"]["contentType"] = "text/html" document_ref_data["content"][0]["format"] = { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", @@ -1675,6 +1677,8 @@ def test_validate_nrl_format_code_display_mismatch( ): validator = DocumentReferenceValidator() document_ref_data = load_document_reference_json("Y05868-736253002-Valid") + if format_code == "urn:nhs-ic:record-contact": + document_ref_data["content"][0]["attachment"]["contentType"] = "text/html" document_ref_data["content"][0]["format"] = { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", From e0aaa157ce51d1d4bef260a2ccc8d6d967dd2c8e Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 14:54:51 +0000 Subject: [PATCH 105/106] NRL-477 fix int tests --- .../producer/createDocumentReference-failure.feature | 6 +++--- .../producer/updateDocumentReference-failure.feature | 2 +- .../producer/upsertDocumentReference-failure.feature | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 40fbb7238..31ab07da5 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -422,7 +422,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", - "display": "Unstructured document" + "display": "Unstructured Document" } } ] @@ -471,7 +471,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios "format": { "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:record-contact", - "display": "Contact details" + "display": "Contact details (HTTP Unsecured)" } } ] @@ -825,7 +825,7 @@ Feature: Producer - createDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", + "contentType": "text/html", "url": "https://example.org/my-doc.pdf" }, "format": { diff --git a/tests/features/producer/updateDocumentReference-failure.feature b/tests/features/producer/updateDocumentReference-failure.feature index 20a8bed4f..4dd5ea87c 100644 --- a/tests/features/producer/updateDocumentReference-failure.feature +++ b/tests/features/producer/updateDocumentReference-failure.feature @@ -258,7 +258,7 @@ Feature: Producer - updateDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", + "contentType": "text/html", "url": "https://example.org/my-doc.pdf" }, "format": { diff --git a/tests/features/producer/upsertDocumentReference-failure.feature b/tests/features/producer/upsertDocumentReference-failure.feature index c4fc4ea68..3855b1b34 100644 --- a/tests/features/producer/upsertDocumentReference-failure.feature +++ b/tests/features/producer/upsertDocumentReference-failure.feature @@ -376,7 +376,7 @@ Feature: Producer - upsertDocumentReference - Failure Scenarios "content": [ { "attachment": { - "contentType": "application/pdf", + "contentType": "text/html", "url": "https://example.org/my-doc.pdf" }, "format": { From 99d43c1342ef3ab8c2b9d4b88e5c25ad0bcd2227 Mon Sep 17 00:00:00 2001 From: eesa456 Date: Fri, 13 Dec 2024 15:18:48 +0000 Subject: [PATCH 106/106] NRL-477 fix int tests --- .../createDocumentReference-failure.feature | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/features/producer/createDocumentReference-failure.feature b/tests/features/producer/createDocumentReference-failure.feature index 31ab07da5..af22ae1bc 100644 --- a/tests/features/producer/createDocumentReference-failure.feature +++ b/tests/features/producer/createDocumentReference-failure.feature @@ -423,7 +423,21 @@ Feature: Producer - createDocumentReference - Failure Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:unstructured", "display": "Unstructured Document" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] """ @@ -472,7 +486,21 @@ Feature: Producer - createDocumentReference - Failure Scenarios "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLFormatCode", "code": "urn:nhs-ic:record-contact", "display": "Contact details (HTTP Unsecured)" - } + }, + "extension": [ + { + "url": "https://fhir.nhs.uk/England/StructureDefinition/Extension-England-ContentStability", + "valueCodeableConcept": { + "coding": [ + { + "system": "https://fhir.nhs.uk/England/CodeSystem/England-NRLContentStability", + "code": "static", + "display": "Static" + } + ] + } + } + ] } ] """