New: [AEA-5411] - Create load test for PSU Notify

.devcontainer/Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update \
     jq apt-transport-https ca-certificates gnupg-agent \
     software-properties-common bash-completion python3-pip make libbz2-dev \
     libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev \
-    xz-utils tk-dev liblzma-dev netcat libyaml-dev
+    xz-utils tk-dev liblzma-dev netcat-traditional libyaml-dev
 
 # install aws stuff
 RUN wget -O /tmp/awscliv2.zip "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" && \

.github/workflows/run_notify_load.yml

@@ -15,14 +15,10 @@ on:
         description: 'The duration of the main test'
         required: true
         default: '900'
-      rampUpDuration:
-        description: 'The duration to ramp up to the arrival rate'
-        required: true
-        default: '900'
       maxVusers:
         description: 'Maximum number of vusers to create'
         required: true
-        default: '10000'
+        default: '1000'
 
 jobs:
   run_artillery:
@@ -38,7 +34,6 @@ jobs:
           echo "## environment : ${{ github.event.inputs.environment }}" >> "$GITHUB_STEP_SUMMARY"
           echo "## arrivalRate : ${{ github.event.inputs.arrivalRate }}" >> "$GITHUB_STEP_SUMMARY"
           echo "## duration : ${{ github.event.inputs.duration }}" >> "$GITHUB_STEP_SUMMARY"
-          echo "## rampUpDuration : ${{ github.event.inputs.rampUpDuration }}" >> "$GITHUB_STEP_SUMMARY"
           echo "## maxVusers : ${{ github.event.inputs.maxVusers }}" >> "$GITHUB_STEP_SUMMARY"
 
       - name: Checkout repo

Makefile

@@ -36,4 +36,7 @@ local-cpsu:
 	./scripts/test_cpsu_load_test.sh
 
 local-psu:
-	./scripts/test_psu_load_test.sh
+	./scripts/test_psu_load_test.sh
+
+local-notify:
+	./scripts/test_notify_load_test.sh

artillery/helper/odscodes.mjs

@@ -0,0 +1,8 @@
+// These ODS codes should match the values in AWS parameter store. They're assumed to be enabled in the whitelist
+export const allowedOdsCodes = [
+    "FA565"
+]
+
+export const blockedOdsCodes = [
+    // "B3J1Z"
+]

artillery/helper/psu.mjs

@@ -6,17 +6,19 @@ const logger = pino()
 let oauthToken
 let tokenExpiryTime
 
-export function getBody(isValid = true) {
+export function getBody(
+  isValid = true, 
+  status = "in-progress",
+  odsCode = "C9Z10", 
+  nhsNumber = "9449304130",
+  businessStatus = "With Pharmacy",
+) {
   // If this is intended to be a failed request, mangle the prescription ID.
   const prescriptionID = isValid ? shortPrescId() : invalidShortPrescId();
 
   const task_identifier = uuidv4()
   const prescriptionOrderItemNumber = uuidv4()
-  const nhsNumber = "9449304130"
   const currentTimestamp = new Date().toISOString()
-  const odsCode = "C9Z1O"
-  const status = "in-progress"
-  const businessStatus = "With Pharmacy"
   const body = {
     resourceType: "Bundle",
     type: "transaction",
@@ -130,7 +132,6 @@ export async function getPSUParams(requestParams, vuContext) {
   const isValid = vuContext.scenario.tags.isValid
   const body = getBody(isValid)
 
-  requestParams.json = body
   // This sets the body of the request and some variables so headers are unique
   requestParams.json = body
   vuContext.vars.x_request_id = uuidv4()

artillery/notify_entrypoint.mjs

@@ -0,0 +1,111 @@
+import {v4 as uuidv4}   from "uuid"
+import pino from "pino"
+import {getSharedAuthToken, getBody} from "./helper/psu.mjs"
+import {allowedOdsCodes, blockedOdsCodes} from "./helper/odscodes.mjs"
+
+export { getSharedAuthToken }
+
+const logger = pino()
+
+const LETTERS = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";
+const DIGITS = "0123456789";
+
+const randomChar = (chars) => chars[Math.floor(Math.random() * chars.length)];
+
+/** Generate one two-letter, three-digit ODS code, e.g. "AB123" */
+const generateOdsCode = () =>
+  `${randomChar(LETTERS)}${randomChar(LETTERS)}${randomChar(DIGITS)}${randomChar(DIGITS)}${randomChar(DIGITS)}`;
+
+function buildFullOdsCodes(targetCount, seedCodes) {
+  const codes = new Set(seedCodes);
+
+  while (codes.size < targetCount) {
+    codes.add(generateOdsCode());
+  }
+
+  return Array.from(codes);
+}
+
+// The complete list of ODS codes
+const fullOdsCodes = allowedOdsCodes.concat(blockedOdsCodes)
+
+function computeCheckDigit(nhsNumber) {
+    const factors = [10,9,8,7,6,5,4,3,2]
+    let total = 0
+
+    for (let i = 0; i < 9; i++) {
+        total += parseInt(nhsNumber.charAt(i),10) * factors[i]
+    }
+
+    const rem = total % 11
+    let d = 11 - rem
+    if (d === 11) d = 0
+
+    return d
+}
+
+function generateValidNhsNumber() {
+  while (true) {
+    const partial = Array.from({length:9},() => Math.floor(Math.random()*10)).join("")
+    const cd = computeCheckDigit(partial)
+    if (cd < 10) return partial + cd
+  }
+}
+
+// Apparently Math.sampleNormal isn't a function? Do a quick Box-Muller transform instead
+function sampleNormal(mean = 0, sd = 1) {
+  let u = 0, v = 0;
+  // avoid zeros because of log(0)
+  while (u === 0) u = Math.random();
+  while (v === 0) v = Math.random();
+  const z = Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
+  return z * sd + mean;
+}
+
+export function initUser(context, events, done) {
+    // Generate data for a patient
+    context.vars.odsCode = fullOdsCodes[Math.floor(Math.random()*fullOdsCodes.length)]
+    context.vars.nhsNumber = generateValidNhsNumber()
+
+    let prescriptionCount = Math.round(sampleNormal(3,1))
+    if (prescriptionCount < 1) prescriptionCount = 1 // just truncate at 1.
+    context.vars.prescriptionCount  = prescriptionCount
+    context.vars.loopcount = 0
+
+    logger.info(`Patient ${context.vars.nhsNumber}, ODS ${context.vars.odsCode} has ${context.vars.prescriptionCount} prescriptions`)
+
+    done()
+}
+
+export function generatePrescData(requestParams, context, ee, next) {
+  logger.debug(`Generating a prescription for patient ${context.vars.nhsNumber}`)
+  const body = getBody(
+    true,                   /* isValid */   
+    "completed",            /* status */    
+    context.vars.odsCode,   /* odsCode */   
+    context.vars.nhsNumber, /* nhsNumber */ 
+    "ready to collect"      /* Item status */
+  )
+  // The body is fine - it works when I put it in postman
+
+  requestParams.json = body
+  context.vars.x_request_id = uuidv4()
+  context.vars.x_correlation_id = uuidv4()
+
+  context.vars.loopcount += 1
+
+  // Wait this long between requests
+  let meanDelay = 10 // seconds
+  let stdDevDelay = 10 // seconds
+  let delay = 0
+  if (context.vars.loopcount < context.vars.prescriptionCount) {
+    delay = sampleNormal(meanDelay, stdDevDelay)
+    while (delay < 0) delay = sampleNormal(meanDelay, stdDevDelay)
+  }
+
+  context.vars.nextDelay = delay
+  logger.debug(`Patient ${context.vars.nhsNumber} (on prescription update ${context.vars.loopcount}/${context.vars.prescriptionCount}) will think for ${context.vars.nextDelay} seconds`)
+
+  next()
+}
+

artillery/notify_load_test.yml

@@ -0,0 +1,44 @@
+config:
+  processor: "./notify_entrypoint.mjs"
+  plugins:
+    expect: {}
+    apdex: {}
+  phases:
+    - name: run phase
+      duration: "{{ $env.duration }}"
+      arrivalRate: "{{ $env.arrivalRate }}"
+      maxVusers: "{{ $env.maxVusers }}"
+  environments:
+    dev:
+      target: https://internal-dev.api.service.nhs.uk/
+    ref:
+      target: https://ref.api.service.nhs.uk/
+    int:
+      target: https://int.api.service.nhs.uk/
+    pr:
+      target: https://psu-pr-{{ prNumber }}.dev.eps.national.nhs.uk/
+
+before:
+  flow:
+    - function: getSharedAuthToken
+
+scenarios:
+  - name: dynamic PSU calls for each patient
+
+    flow:
+      - function: initUser
+
+      # for each prescription (number of prescriptions is variable)
+      - loop:
+          - function: getSharedAuthToken
+          - post:
+              url: "/prescription-status-update/"
+              beforeRequest: "generatePrescData"
+              headers:
+                Authorization: "Bearer {{ authToken }}"
+                x-request-id:     "{{ x_request_id }}"
+                x-correlation-id: "{{ x_correlation_id }}"
+              expect:
+                - statusCode: 201
+          - think: "{{ nextDelay }}seconds"
+        count: "{{ prescriptionCount }}"

scripts/run_cpsu_load_test.sh

@@ -57,7 +57,7 @@ npx artillery run-fargate \
     --security-group-ids "${security_group}" \
     --subnet-ids "${vpc_subnets}" \
     --task-role-name "${artillery_worker_role_name}" \
-    --dotenv runtimeenv.env \
+    --env-file runtimeenv.env \
     --output cpsu_load_test.json \
     artillery/cpsu_load_test.yml
 

scripts/run_notify_load_test.sh

@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+
+
+if [ -z "${artillery_key}" ]; then
+    echo "artillery_key is unset or set to the empty string"
+    exit 1
+fi
+
+if [ -z "${environment}" ]; then
+    echo "environment is unset or set to the empty string"
+    exit 1
+fi
+
+if [ -z "${maxVusers}" ]; then
+    echo "maxVusers is unset or set to the empty string"
+    exit 1
+fi
+
+if [ -z "${duration}" ]; then
+    echo "duration is unset or set to the empty string"
+    exit 1
+fi
+
+if [ -z "${arrivalRate}" ]; then
+    echo "arrivalRate is unset or set to the empty string"
+    exit 1
+fi
+
+if ! [[ "${environment}" =~ ^(dev|ref)$ ]]
+then 
+    echo "environment must be dev or ref"
+    exit 1
+fi
+
+security_group=$(aws cloudformation list-exports --output json | jq -r '.Exports[] | select(.Name == "artillery-resources:ArtillerySecurityGroupId") | .Value' | grep -o '[^:]*$')
+export security_group
+vpc_subnets=$(aws cloudformation list-exports --output json | jq -r '.Exports[] | select(.Name == "vpc-resources:PrivateSubnets") | .Value' | grep -o '[^:]*$')
+export vpc_subnets
+
+artillery_worker_role_name=$(aws cloudformation list-exports --output json | jq -r '.Exports[] | select(.Name == "artillery-resources:ArtilleryWorkerRoleName") | .Value' | grep -o '[^:]*$')
+export artillery_worker_role_name
+
+cat <<EOF > runtimeenv.env
+maxVusers=$maxVusers
+duration=$duration
+arrivalRate=$arrivalRate
+EOF
+
+echo ${launch_config}
+
+# shellcheck disable=SC2090,SC2086
+npx artillery run-fargate \
+    --environment "${environment}" \
+    --secret psu_api_key psu_private_key psu_kid \
+    --region eu-west-2 \
+    --cluster artilleryio-cluster \
+    --security-group-ids "${security_group}" \
+    --subnet-ids "${vpc_subnets}" \
+    --task-role-name "${artillery_worker_role_name}" \
+    --env-file runtimeenv.env \
+    --output notify_load_test.json \
+    artillery/notify_load_test.yml
+
+npx artillery report notify_load_test.json 

scripts/run_psu_load_test.sh

@@ -59,7 +59,7 @@ npx artillery run-fargate \
     --security-group-ids "${security_group}" \
     --subnet-ids "${vpc_subnets}" \
     --task-role-name "${artillery_worker_role_name}" \
-    --dotenv runtimeenv.env \
+    --env-file runtimeenv.env \
     --output psu_load_test.json \
     artillery/psu_load_test.yml
 

scripts/test_cpsu_load_test.sh

@@ -54,9 +54,9 @@ set -e
 # Run the Artillery test locally
 npx artillery run \
     -e "${environment}" \
-    --dotenv /workspaces/eps-load-test/runtimeenv.env \
+    --env-file /workspaces/eps-load-test/runtimeenv.env \
     --output /workspaces/eps-load-test/cpsu_load_test.json \
     /workspaces/eps-load-test/artillery/cpsu_load_test.yml
 
 # Generate a report from the test results
-npx /workspaces/eps-load-test/artillery report cpsu_load_test.json
+npx artillery report cpsu_load_test.json