NRL-1666 Performance test implementation

Makefile

@@ -1,4 +1,3 @@
-
 .EXPORT_ALL_VARIABLES:
 .NOTPARALLEL:
 .PHONY: *
@@ -16,6 +15,8 @@ ACCOUNT ?= dev
 APP_ALIAS ?= default
 HOST ?= $(TF_WORKSPACE_NAME).api.record-locator.$(ENV).national.nhs.uk
 ENV_TYPE ?= $(ENV)
+PERFTEST_TABLE_NAME ?= perftest
+PERFTEST_HOST ?= perftest-1.perftest.record-locator.national.nhs.uk
 
 export PATH := $(PATH):$(PWD)/.venv/bin
 export USE_SHARED_RESOURCES := $(shell poetry run python scripts/are_resources_shared_for_stack.py $(TF_WORKSPACE_NAME))
@@ -246,3 +247,33 @@ generate-models: check-warn ## Generate Pydantic Models
 		--output ./layer/nrlf/consumer/fhir/r4/model.py \
 		--base-class nrlf.core.parent_model.Parent \
 		--output-model-type "pydantic_v2.BaseModel"
+
+
+generate-perftest-permissions: ## Generate perftest permissions and add to nrlf_permissions
+	poetry run python tests/performance/producer/generate_permissions.py --output_dir="$(DIST_PATH)/nrlf_permissions/K6PerformanceTest"
+
+perftest-producer:
+	@echo "Running producer performance tests with HOST=$(PERFTEST_HOST) and ENV_TYPE=$(ENV_TYPE) and DIST_PATH=$(DIST_PATH)"
+	k6 run tests/performance/producer/perftest.js -e HOST=$(PERFTEST_HOST) -e ENV_TYPE=$(ENV_TYPE) -e DIST_PATH=$(DIST_PATH)
+
+perftest-consumer:
+	@echo "Running consumer performance tests with HOST=$(PERFTEST_HOST) and ENV_TYPE=$(ENV_TYPE) and DIST_PATH=$(DIST_PATH)"
+	k6 run tests/performance/consumer/perftest.js -e HOST=$(PERFTEST_HOST) -e ENV_TYPE=$(ENV_TYPE) -e DIST_PATH=$(DIST_PATH)
+
+perftest-prep-generate-producer-data:
+	@echo "Generating producer reference with PERFTEST_TABLE_NAME=$(PERFTEST_TABLE_NAME) and DIST_PATH=$(DIST_PATH)"
+	mkdir -p $(DIST_PATH)
+	PYTHONPATH=. poetry run python tests/performance/perftest_environment.py generate_producer_data --output_dir="$(DIST_PATH)"
+
+perftest-prep-extract-consumer-data:
+	@echo "Generating consumer reference with PERFTEST_TABLE_NAME=$(PERFTEST_TABLE_NAME) and DIST_PATH=$(DIST_PATH)"
+	mkdir -p $(DIST_PATH)
+	PYTHONPATH=. poetry run python tests/performance/perftest_environment.py extract_consumer_data --output_dir="$(DIST_PATH)"
+
+perftest-prep-generate-pointer-table-extract:
+	@echo "Generating pointer table extract with PERFTEST_TABLE_NAME=$(PERFTEST_TABLE_NAME) and DIST_PATH=$(DIST_PATH)"
+	mkdir -p $(DIST_PATH)
+	PYTHONPATH=. poetry run python tests/performance/perftest_environment.py generate_pointer_table_extract --output_dir="$(DIST_PATH)"
+
+perftest-prepare: perftest-prep-generate-producer-data perftest-prep-extract-consumer-data perftest-prep-generate-pointer-table-extract
+	@echo "Prepared performance tests with PERFTEST_TABLE_NAME=$(PERFTEST_TABLE_NAME) and DIST_PATH=$(DIST_PATH)"

tests/performance/README.md

@@ -0,0 +1,51 @@
+# Performance Testing
+
+some high level context short
+
+## Run perf tests
+
+### Prep the environment
+
+Perf tests are generally conducted in the perftest env. There's a selection of tables in the perftest env representing different pointer volume scenarios e.g. perftest-baseline vs perftest-1million (todo: update with real names!).
+
+To reset this table to the expected state for perftests, restore the table from a backup.
+
+In the steps below, make sure the table name is the table your environment is pointing at. You might need to redeploy NRLF lambdas to point at the desired table.
+
+### Prepare to run tests
+
+#### Pull certs for env
+
+```sh
+assume management
+make truststore-pull-all ENV=perftest
+```
+
+#### Generate permissions
+
+You will need to generate pointer permissions the first time performance tests are run in an environment e.g. if the perftest environment is destroyed & recreated.
+
+```sh
+make generate permissions   # makes a bunch of json permission files
+make build  # will take all permissions & create nrlf_permissions.zip file
+
+# apply this new permissions zip file to your environment
+cd ./terraform/infrastructure
+assume test # needed?
+make init TF_WORKSPACE_NAME=perftest-1 ENV=perftest
+tf apply
+```
+
+#### Generate input files
+
+```sh
+# creates 2 csv files and a json file
+make perftest-prepare PERFTEST_TABLE_NAME=perftest-baseline
+```
+
+### Run tests
+
+```sh
+make perftest-consumer ENV_TYPE=perftest PERFTEST_HOST=perftest-1.perftest.record-locator.national.nhs.uk
+make perftest-producer ENV_TYPE=perftest PERFTEST_HOST=perftest-1.perftest.record-locator.national.nhs.uk
+```

tests/performance/constants.js

@@ -1,3 +1,5 @@
+import { CATEGORY_TYPE_GROUPS } from "./type-category-mappings.js";
+
 export const DEFAULT_TEST_RECORD = open(
   "../data/DocumentReference/Y05868-736253002-Valid.json"
 );
@@ -12,23 +14,34 @@ export const ALL_POINTER_IDS =
 export const POINTERS_TO_DELETE = ALL_POINTER_IDS.slice(0, 3500);
 export const POINTER_IDS = ALL_POINTER_IDS.slice(3500);
 export const NHS_NUMBERS = REFERENCE_DATA["nhs_numbers"];
-export const POINTER_TYPES = [
+
+// filter only 736253001, 736253002, 1363501000000100, 861421000000109, 749001000000101 for now
+export const FILTERED_POINTER_TYPES = [
+  "736253001",
   "736253002",
   "1363501000000100",
-  "1382601000000107",
-  "325691000000100",
-  "736373009",
   "861421000000109",
-  "887701000000100",
-  "736366004",
-  "735324008",
-  "824321000000109",
-  "2181441000000107",
-];
-export const CATEGORIES = [
-  "734163000",
-  "1102421000000108",
-  "823651000000106",
-  "721981007",
-  "103693007",
+  "749001000000101",
 ];
+
+export const POINTER_TYPES = FILTERED_POINTER_TYPES;
+
+export const CATEGORIES = CATEGORY_TYPE_GROUPS.map(
+  (group) => group.category.code
+);
+export const POINTER_TYPE_DISPLAY = Object.fromEntries(
+  CATEGORY_TYPE_GROUPS.flatMap((group) =>
+    group.types.map((t) => [t.code, t.display])
+  )
+);
+export const TYPE_CATEGORY_MAP = Object.fromEntries(
+  CATEGORY_TYPE_GROUPS.flatMap((group) =>
+    group.types.map((t) => [t.code, group.category.code])
+  )
+);
+export const CATEGORY_DISPLAY = Object.fromEntries(
+  CATEGORY_TYPE_GROUPS.map((group) => [
+    group.category.code,
+    group.category.display,
+  ])
+);

tests/performance/consumer/client_perftest.js

@@ -0,0 +1,228 @@
+import http from "k6/http";
+import { check } from "k6";
+import exec from "k6/execution";
+import { CATEGORY_TYPE_GROUPS } from "../type-category-mappings.js";
+
+const csvPath = __ENV.DIST_PATH
+  ? `../../../${__ENV.DIST_PATH}/producer_reference_data.csv`
+  : "../producer_reference_data.csv";
+const csv = open(csvPath);
+const lines = csv.trim().split("\n");
+// Skip header
+const dataLines = lines.slice(1);
+
+function getNextPointer() {
+  // pick the next line according to iteration in scenario
+  const iter = exec.vu.iterationInScenario;
+  const index = iter % dataLines.length;
+  const line = dataLines[index];
+  const [count, pointer_id, pointer_type, custodian, nhs_number] = line
+    .split(",")
+    .map((field) => field.trim());
+  return { pointer_id, pointer_type, nhs_number };
+}
+
+function getHeaders(odsCode) {
+  return {
+    "Content-Type": "application/fhir+json",
+    "X-Request-Id": `K6perftest-consumer-${exec.scenario.name}-${exec.vu.idInTest}-${exec.vu.iterationInScenario}`,
+    "NHSD-Correlation-Id": `K6perftest-consumer-${exec.scenario.name}-${exec.vu.idInTest}-${exec.vu.iterationInScenario}`,
+    "NHSD-Connection-Metadata": JSON.stringify({
+      "nrl.ods-code": odsCode,
+      "nrl.app-id": "K6PerformanceTest",
+    }),
+    "NHSD-Client-RP-Details": JSON.stringify({
+      "developer.app.name": "K6PerformanceTest",
+      "developer.app.id": "K6PerformanceTest",
+    }),
+  };
+}
+
+function getCustodianFromPointerId(pointer_id) {
+  // pointer_id format is "CUSTODIAN-XXXX"
+  return pointer_id.split("-")[0];
+}
+
+function checkResponse(res) {
+  const is_success = check(res, { "status is 200": (r) => r.status === 200 });
+  if (!is_success) {
+    console.warn(res.json());
+  }
+}
+
+const pointerTypeToCategoryMap = new Map();
+for (const group of CATEGORY_TYPE_GROUPS) {
+  for (const type of group.types) {
+    pointerTypeToCategoryMap.set(type.code, group.category.code);
+  }
+}
+
+export function countDocumentReference() {
+  const { pointer_id, nhs_number } = getNextPointer();
+  const custodian = getCustodianFromPointerId(pointer_id);
+  const identifier = encodeURIComponent(
+    `https://fhir.nhs.uk/Id/nhs-number|${nhs_number}`
+  );
+
+  const res = http.get(
+    `https://${__ENV.HOST}/consumer/DocumentReference?_summary=count&subject:identifier=${identifier}`,
+    {
+      headers: getHeaders(custodian),
+    }
+  );
+  checkResponse(res);
+}
+
+export function readDocumentReference() {
+  const { pointer_id } = getNextPointer();
+  const custodian = getCustodianFromPointerId(pointer_id);
+
+  const res = http.get(
+    `https://${__ENV.HOST}/consumer/DocumentReference/${pointer_id}`,
+    {
+      headers: getHeaders(custodian),
+    }
+  );
+
+  checkResponse(res);
+}
+
+export function searchDocumentReference() {
+  const { pointer_id, pointer_type, nhs_number } = getNextPointer();
+  const custodian = getCustodianFromPointerId(pointer_id);
+
+  const identifier = encodeURIComponent(
+    `https://fhir.nhs.uk/Id/nhs-number|${nhs_number}`
+  );
+  const type = encodeURIComponent(`http://snomed.info/sct|${pointer_type}`);
+
+  const res = http.get(
+    `https://${__ENV.HOST}/consumer/DocumentReference?subject:identifier=${identifier}&type=${type}`,
+    {
+      headers: getHeaders(custodian),
+    }
+  );
+  checkResponse(res);
+}
+
+export function searchDocumentReferenceByCategory() {
+  const { pointer_id, pointer_type, nhs_number } = getNextPointer();
+  const custodian = getCustodianFromPointerId(pointer_id);
+  const category_code = pointerTypeToCategoryMap.get(pointer_type);
+
+  const identifier = encodeURIComponent(
+    `https://fhir.nhs.uk/Id/nhs-number|${nhs_number}`
+  );
+  const category = encodeURIComponent(
+    `http://snomed.info/sct|${category_code}`
+  );
+
+  const res = http.get(
+    `https://${__ENV.HOST}/consumer/DocumentReference?subject:identifier=${identifier}&category=${category}`,
+    {
+      headers: getHeaders(custodian),
+    }
+  );
+  checkResponse(res);
+}
+
+export function searchPostDocumentReference() {
+  const { pointer_id, pointer_type, nhs_number } = getNextPointer();
+  const custodian = getCustodianFromPointerId(pointer_id);
+
+  const body = JSON.stringify({
+    "subject:identifier": `https://fhir.nhs.uk/Id/nhs-number|${nhs_number}`,
+    type: `http://snomed.info/sct|${pointer_type}`,
+  });
+
+  const res = http.post(
+    `https://${__ENV.HOST}/consumer/DocumentReference/_search`,
+    body,
+    {
+      headers: getHeaders(custodian),
+    }
+  );
+  checkResponse(res);
+}
+
+export function searchPostDocumentReferenceByCategory() {
+  const { pointer_id, pointer_type, nhs_number } = getNextPointer();
+  const custodian = getCustodianFromPointerId(pointer_id);
+  const category_code = pointerTypeToCategoryMap.get(pointer_type);
+
+  const body = JSON.stringify({
+    "subject:identifier": `https://fhir.nhs.uk/Id/nhs-number|${nhs_number}`,
+    category: `http://snomed.info/sct|${category_code}`,
+  });
+
+  const res = http.post(
+    `https://${__ENV.HOST}/consumer/DocumentReference/_search`,
+    body,
+    {
+      headers: getHeaders(custodian),
+    }
+  );
+  checkResponse(res);
+}
+
+export function countPostDocumentReference() {
+  const { pointer_id, nhs_number } = getNextPointer();
+  const custodian = getCustodianFromPointerId(pointer_id);
+
+  const body = JSON.stringify({
+    "subject:identifier": `https://fhir.nhs.uk/Id/nhs-number|${nhs_number}`,
+  });
+
+  const res = http.post(
+    `https://${__ENV.HOST}/consumer/DocumentReference/_search?_summary=count`,
+    body,
+    {
+      headers: getHeaders(custodian),
+    }
+  );
+  checkResponse(res);
+}
+
+export function searchPostDocumentReferenceAccessDenied() {
+  const { nhs_number, pointer_type } = getNextPointer();
+
+  const body = JSON.stringify({
+    "subject:identifier": `https://fhir.nhs.uk/Id/nhs-number|${nhs_number}`,
+    type: `http://snomed.info/sct|${pointer_type}`,
+  });
+
+  // Use a custodian that should not have access (simulate denied)
+  const deniedCustodian = "DENIED_ODS_CODE";
+  let headers = getHeaders(deniedCustodian);
+  headers["NHSD-Connection-Metadata"] = JSON.stringify({
+    "nrl.ods-code": deniedCustodian,
+    "nrl.app-id": "K6PerformanceTest",
+  });
+
+  const res = http.post(
+    `https://${__ENV.HOST}/consumer/DocumentReference/_search`,
+    body,
+    {
+      headers: headers,
+    }
+  );
+
+  const is_denied = check(res, { "status is 403": (r) => r.status === 403 });
+  if (!is_denied) {
+    console.warn(`Expected access denied but got: ${res.status}`);
+  }
+}
+
+export function readDocumentReferenceNotFound() {
+  const { custodian } = getNextPointer();
+
+  const res = http.get(
+    `https://${__ENV.HOST}/consumer/DocumentReference/NonExistentID`,
+    {
+      headers: getHeaders(custodian),
+    }
+  );
+
+  // we expect a 404 here
+  check(res, { "status is 404": (r) => r.status === 404 });
+}

tests/performance/consumer/consumer_reference_data.json

@@ -0,0 +1,9 @@
+{
+  "nhs_numbers": ["9694202043"],
+  "pointer_ids": [
+    "RQI-9347490b-6087-4be6-8c95-82ad9fb0c83f",
+    "RQI-123",
+    "RQI-7fba4cfb-acfe-4b62-ac85-916197a24868"
+  ],
+  "custodians": ["RQI"]
+}