New: [AEA-5411] - Create load test for PSU Notify

.devcontainer/Dockerfile

@@ -9,7 +9,7 @@ RUN apt-get update \
     jq apt-transport-https ca-certificates gnupg-agent \
     software-properties-common bash-completion python3-pip make libbz2-dev \
     libreadline-dev libsqlite3-dev wget llvm libncurses5-dev libncursesw5-dev \
-    xz-utils tk-dev liblzma-dev netcat libyaml-dev
+    xz-utils tk-dev liblzma-dev netcat-traditional libyaml-dev
 
 # install aws stuff
 RUN wget -O /tmp/awscliv2.zip "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" && \

.github/workflows/run_notify_load.yml

@@ -8,21 +8,25 @@ on:
         required: true
         default: 'ref'
       arrivalRate:
-        description: 'The number of new users to add every second'
+        description: 'How many requests to send per second during the main test'
         required: true
-        default: '275'
+        default: '100'
       duration:
         description: 'The duration of the main test'
         required: true
         default: '900'
       rampUpDuration:
-        description: 'The duration to ramp up to the arrival rate'
+        description: 'The duration of ramp-up phase of the test'
         required: true
         default: '900'
       maxVusers:
-        description: 'Maximum number of vusers to create'
+        description: 'Maximum number of virtual users to create'
         required: true
-        default: '10000'
+        default: '100'
+      artillery_key:
+        description: 'Your Artillery cloud API key (optional – omit to run without recording)'
+        required: false
+        default: ''
 
 jobs:
   run_artillery:
@@ -40,6 +44,11 @@ jobs:
           echo "## duration : ${{ github.event.inputs.duration }}" >> "$GITHUB_STEP_SUMMARY"
           echo "## rampUpDuration : ${{ github.event.inputs.rampUpDuration }}" >> "$GITHUB_STEP_SUMMARY"
           echo "## maxVusers : ${{ github.event.inputs.maxVusers }}" >> "$GITHUB_STEP_SUMMARY"
+          if [ -n "${{ github.event.inputs.artillery_key }}" ]; then
+            echo "## artillery_key        : (provided)" >> "$GITHUB_STEP_SUMMARY"
+          else
+            echo "## artillery_key        : (not provided)" >> "$GITHUB_STEP_SUMMARY"
+          fi
 
       - name: Checkout repo
         uses: actions/checkout@v4
@@ -65,7 +74,7 @@ jobs:
         with:
           asdf_branch: v0.14.0
         env:
-            PYTHON_CONFIGURE_OPTS: --enable-shared
+          PYTHON_CONFIGURE_OPTS: --enable-shared
 
       - name: Install Dependencies
         run: make install
@@ -94,6 +103,7 @@ jobs:
           duration: ${{ github.event.inputs.duration }}
           rampUpDuration: ${{ github.event.inputs.rampUpDuration }}
           maxVusers: ${{ github.event.inputs.maxVusers }}
+          artillery_key: ${{ github.event.inputs.artillery_key }}
         run: |
           ./scripts/run_notify_load_test.sh 
 

Makefile

@@ -10,6 +10,9 @@ compile-node:
 
 compile: compile-node
 
+aws-configure:
+	aws configure sso --region eu-west-2
+
 aws-login:
 	aws sso login --sso-session sso-session
 
@@ -36,4 +39,7 @@ local-cpsu:
 	./scripts/test_cpsu_load_test.sh
 
 local-psu:
-	./scripts/test_psu_load_test.sh
+	./scripts/test_psu_load_test.sh
+
+local-notify:
+	./scripts/test_notify_load_test.sh

artillery/helper/odscodes.mjs

@@ -0,0 +1,8 @@
+// These ODS codes should match the values in AWS parameter store. They're assumed to be enabled in the whitelist
+export const allowedOdsCodes = [
+    "FA565"
+]
+
+export const blockedOdsCodes = [
+    "B3J1Z"
+]

artillery/helper/psu.mjs

@@ -6,17 +6,19 @@ const logger = pino()
 let oauthToken
 let tokenExpiryTime
 
-export function getBody(isValid = true) {
+export function getBody(
+  isValid = true, 
+  status = "in-progress",
+  odsCode = "C9Z10", 
+  nhsNumber = "9449304130",
+  businessStatus = "With Pharmacy",
+) {
   // If this is intended to be a failed request, mangle the prescription ID.
   const prescriptionID = isValid ? shortPrescId() : invalidShortPrescId();
 
   const task_identifier = uuidv4()
   const prescriptionOrderItemNumber = uuidv4()
-  const nhsNumber = "9449304130"
   const currentTimestamp = new Date().toISOString()
-  const odsCode = "C9Z1O"
-  const status = "in-progress"
-  const businessStatus = "With Pharmacy"
   const body = {
     resourceType: "Bundle",
     type: "transaction",
@@ -112,6 +114,8 @@ export async function getSharedAuthToken(vuContext) {
     const api_key = process.env.psu_api_key
     const kid = process.env.psu_kid
 
+    logger.info("Secrets:", {privateKey, api_key, kid})
+
     // And use them to fetch the access token
     const response = await getAccessToken(logger, vuContext.vars.target, privateKey, api_key, kid)
 
@@ -130,7 +134,6 @@ export async function getPSUParams(requestParams, vuContext) {
   const isValid = vuContext.scenario.tags.isValid
   const body = getBody(isValid)
 
-  requestParams.json = body
   // This sets the body of the request and some variables so headers are unique
   requestParams.json = body
   vuContext.vars.x_request_id = uuidv4()

artillery/notify_entrypoint.mjs

@@ -0,0 +1,106 @@
+import {v4 as uuidv4}   from "uuid"
+import pino from "pino"
+import {getSharedAuthToken, getBody} from "./helper/psu.mjs"
+import {allowedOdsCodes, blockedOdsCodes} from "./helper/odscodes.mjs"
+
+export { getSharedAuthToken }
+
+const logger = pino()
+
+function computeCheckDigit(nhsNumber) {
+    const factors = [10,9,8,7,6,5,4,3,2]
+    let total = 0
+
+    for (let i = 0; i < 9; i++) {
+        total += parseInt(nhsNumber.charAt(i),10) * factors[i]
+    }
+
+    const rem = total % 11
+    let d = 11 - rem
+    if (d === 11) d = 0
+
+    return d
+}
+
+function generateValidNhsNumber() {
+  while (true) {
+    const partial = Array.from({length:9},() => Math.floor(Math.random()*10)).join("")
+    const cd = computeCheckDigit(partial)
+    if (cd < 10) return partial + cd
+  }
+}
+
+// Apparently Math.sampleNormal isn't a function? Do a quick Box-Muller transform instead
+function sampleNormal(mean = 0, sd = 1) {
+  let u = 0, v = 0;
+  // avoid zeros because of log(0)
+  while (u === 0) u = Math.random();
+  while (v === 0) v = Math.random();
+  const z = Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
+  return z * sd + mean;
+}
+
+function initUserContextVars(context) {
+  context.vars.nhsNumber = generateValidNhsNumber()
+
+  let prescriptionCount = Math.round(sampleNormal(3,1))
+  if (prescriptionCount < 1) prescriptionCount = 1 // just truncate at 1.
+  context.vars.prescriptionCount  = prescriptionCount
+  context.vars.loopcount = 0
+}
+
+export function initUserAllowed(context, events, done) {
+  logger.info("Initializing user context variables for allowed ODS codes")
+  initUserContextVars(context)
+
+  // Generate data for a patient with an allowed ODS code
+  context.vars.odsCode = allowedOdsCodes[Math.floor(Math.random() * allowedOdsCodes.length)]
+
+  logger.info(`[ALLOWED] Patient ${context.vars.nhsNumber}, ODS ${context.vars.odsCode} has ${context.vars.prescriptionCount} prescriptions`)
+  done()
+}
+
+export function initUserBlocked(context, events, done) {
+  logger.info("Initializing user context variables for allowed ODS codes")
+  initUserContextVars(context)
+
+  // Generate data for a patient with a blocked ODS code
+  context.vars.odsCode = blockedOdsCodes[Math.floor(Math.random() * blockedOdsCodes.length)]
+
+  logger.info(`[BLOCKED] Patient ${context.vars.nhsNumber}, ODS ${context.vars.odsCode} has ${context.vars.prescriptionCount} prescriptions`)
+  done()
+}
+
+export function generatePrescData(requestParams, context, ee, next) {
+  const isAllowed = allowedOdsCodes.includes(context.vars.odsCode);
+  const logPrefix = isAllowed ? "[ALLOWED]" : "[BLOCKED]";
+
+  logger.debug(`${logPrefix} Generating a prescription for patient ${context.vars.nhsNumber}`)
+  const body = getBody(
+    true,                   /* isValid */   
+    "completed",            /* status */    
+    context.vars.odsCode,   /* odsCode */   
+    context.vars.nhsNumber, /* nhsNumber */ 
+    "ready to collect"      /* Item status */
+  )
+  // The body is fine - it works when I put it in postman
+
+  requestParams.json = body
+  context.vars.x_request_id = uuidv4()
+  context.vars.x_correlation_id = uuidv4()
+
+  context.vars.loopcount += 1
+
+  // Wait this long between requests
+  let meanDelay = 10 // seconds
+  let stdDevDelay = 10 // seconds
+  let delay = 0
+  if (context.vars.loopcount < context.vars.prescriptionCount) {
+    delay = sampleNormal(meanDelay, stdDevDelay)
+    while (delay < 0) delay = sampleNormal(meanDelay, stdDevDelay)
+  }
+
+  context.vars.nextDelay = delay
+  logger.debug(`${logPrefix} Patient ${context.vars.nhsNumber} (on prescription update ${context.vars.loopcount}/${context.vars.prescriptionCount}) will think for ${context.vars.nextDelay} seconds`)
+  next()
+}

artillery/notify_load_test.yml

@@ -0,0 +1,65 @@
+config:
+  processor: "./notify_entrypoint.mjs"
+  plugins:
+    expect: {}
+    apdex: {}
+  phases:
+    - name: ramp up phase
+      duration: "{{ $env.rampUpDuration }}"
+      arrivalRate: 1
+      rampTo: "{{ $env.arrivalRate }}"
+      maxVusers: "{{ $env.maxVusers }}"
+    - name: run phase
+      duration: "{{ $env.duration }}"
+      arrivalRate: "{{ $env.arrivalRate }}"
+      maxVusers: "{{ $env.maxVusers }}"
+  environments:
+    dev:
+      target: https://internal-dev.api.service.nhs.uk/
+    ref:
+      target: https://ref.api.service.nhs.uk/
+    int:
+      target: https://int.api.service.nhs.uk/
+    pr:
+      target: https://psu-pr-{{ prNumber }}.dev.eps.national.nhs.uk/
+
+before:
+  flow:
+    - function: getSharedAuthToken
+
+scenarios:
+  - name: Allowed ODS code scenario
+    weight: 1
+    flow:
+      - function: initUserAllowed
+      - loop:
+          - function: getSharedAuthToken
+          - post:
+              url: "/prescription-status-update/"
+              beforeRequest: "generatePrescData"
+              headers:
+                Authorization: "Bearer {{ authToken }}"
+                x-request-id:     "{{ x_request_id }}"
+                x-correlation-id: "{{ x_correlation_id }}"
+              expect:
+                - statusCode: 201
+          - think: "{{ nextDelay }}seconds"
+        count: "{{ prescriptionCount }}"
+
+  - name: Blocked ODS code scenario
+    weight: 0
+    flow:
+      - function: initUserBlocked
+      - loop:
+          - function: getSharedAuthToken
+          - post:
+              url: "/prescription-status-update/"
+              beforeRequest: "generatePrescData"
+              headers:
+                Authorization: "Bearer {{ authToken }}"
+                x-request-id:     "{{ x_request_id }}"
+                x-correlation-id: "{{ x_correlation_id }}"
+              expect:
+                - statusCode: 201
+          - think: "{{ nextDelay }}seconds"
+        count: "{{ prescriptionCount }}"

scripts/run_cpsu_load_test.sh

@@ -57,7 +57,7 @@ npx artillery run-fargate \
     --security-group-ids "${security_group}" \
     --subnet-ids "${vpc_subnets}" \
     --task-role-name "${artillery_worker_role_name}" \
-    --dotenv runtimeenv.env \
+    --env-file runtimeenv.env \
     --output cpsu_load_test.json \
     artillery/cpsu_load_test.yml
 

scripts/run_notify_load_test.sh

@@ -0,0 +1,72 @@
+#!/usr/bin/env bash
+
+
+if [ -z "${environment}" ]; then
+    echo "environment is unset or set to the empty string"
+    exit 1
+fi
+
+if [ -z "${maxVusers}" ]; then
+    echo "maxVusers is unset or set to the empty string"
+    exit 1
+fi
+
+if [ -z "${duration}" ]; then
+    echo "duration is unset or set to the empty string"
+    exit 1
+fi
+
+if [ -z "${rampUpDuration}" ]; then
+    echo "rampUpDuration is unset or set to the empty string"
+    exit 1
+fi
+
+if [ -z "${arrivalRate}" ]; then
+    echo "arrivalRate is unset or set to the empty string"
+    exit 1
+fi
+
+if ! [[ "${environment}" =~ ^(dev|ref)$ ]]; then 
+    echo "environment must be dev or ref"
+    exit 1
+fi
+
+security_group=$(aws cloudformation list-exports --output json | jq -r '.Exports[] | select(.Name == "artillery-resources:ArtillerySecurityGroupId") | .Value' | grep -o '[^:]*$')
+export security_group
+vpc_subnets=$(aws cloudformation list-exports --output json | jq -r '.Exports[] | select(.Name == "vpc-resources:PrivateSubnets") | .Value' | grep -o '[^:]*$')
+export vpc_subnets
+
+artillery_worker_role_name=$(aws cloudformation list-exports --output json | jq -r '.Exports[] | select(.Name == "artillery-resources:ArtilleryWorkerRoleName") | .Value' | grep -o '[^:]*$')
+export artillery_worker_role_name
+
+if [ -z "${artillery_key}" ]; then
+    echo "artillery_key is unset. Running without --record to Artillery Cloud."
+    RECORD_ARGS=""
+else
+    RECORD_ARGS="--record --key ${artillery_key}"
+fi
+
+cat <<EOF > runtimeenv.env
+maxVusers=${maxVusers}
+duration=${duration}
+arrivalRate=${arrivalRate}
+rampUpDuration=${rampUpDuration}
+EOF
+
+echo ${launch_config}
+
+# shellcheck disable=SC2090,SC2086
+npx artillery run-fargate \
+    --environment "${environment}" \
+    --secret psu_api_key psu_private_key psu_kid \
+    --region eu-west-2 \
+    --cluster artilleryio-cluster \
+    --security-group-ids "${security_group}" \
+    --subnet-ids "${vpc_subnets}" \
+    --task-role-name "${artillery_worker_role_name}" \
+    --env-file runtimeenv.env \
+    --output notify_load_test.json \
+    $RECORD_ARGS \
+    artillery/notify_load_test.yml
+
+npx artillery report notify_load_test.json