VED-812: Add e2e test pipeline

.github/workflows/deploy-backend.yml

@@ -42,62 +42,54 @@ on:
         type: string
         description: Set the sub environment name e.g. pr-xxx, or green/blue in higher environments
 
+env: # Sonarcloud - do not allow direct usage of untrusted data
+  APIGEE_ENVIRONMENT: ${{ inputs.apigee_environment }}
+  ENVIRONMENT: ${{ inputs.environment }}
+  SUB_ENVIRONMENT: ${{ inputs.sub_environment }}
+
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   terraform-plan:
     runs-on: ubuntu-latest
     environment:
       name: ${{ inputs.environment }}
-    env: # Sonarcloud - do not allow direct usage of untrusted data
-      APIGEE_ENVIRONMENT: ${{ inputs.apigee_environment }}
-      BACKEND_ENVIRONMENT: ${{ inputs.environment }}
-      BACKEND_SUB_ENVIRONMENT: ${{ inputs.sub_environment }}
-    permissions:
-      id-token: write
-      contents: read
     steps:
+      - name: Checkout
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+
       - name: Connect to AWS
         uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838
         with:
           aws-region: eu-west-2
           role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
           role-session-name: github-actions
 
-      - name: Whoami
-        run: aws sts get-caller-identity
-
-      - name: Checkout
-        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
-
       - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
         with:
           terraform_version: "1.12.2"
 
       - name: Terraform Init
-        working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
-        run: make init apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
+        working-directory: terraform
+        run: make init
 
       - name: Terraform Plan
-        working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
-        run: make plan-ci apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
+        working-directory: terraform
+        run: make plan-ci
 
       - name: Save Terraform Plan
         uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
           name: tfplan
-          path: ${{ vars.TERRAFORM_DIR_PATH }}/tfplan
+          path: terraform/tfplan
 
   terraform-apply:
     needs: terraform-plan
     runs-on: ubuntu-latest
     environment:
       name: ${{ inputs.environment }}
-    env: # Sonarcloud - do not allow direct usage of untrusted data
-      APIGEE_ENVIRONMENT: ${{ inputs.apigee_environment }}
-      BACKEND_ENVIRONMENT: ${{ inputs.environment }}
-      BACKEND_SUB_ENVIRONMENT: ${{ inputs.sub_environment }}
-    permissions:
-      id-token: write
-      contents: read
     steps:
       - name: Checkout
         uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
@@ -116,16 +108,16 @@ jobs:
         uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0
         with:
           name: tfplan
-          path: ${{ vars.TERRAFORM_DIR_PATH }}
+          path: terraform
 
       - name: Terraform Init
-        working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
-        run: make init apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
+        working-directory: terraform
+        run: make init
 
       - name: Terraform Apply
-        working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
+        working-directory: terraform
         run: |
-          make apply-ci apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
+          make apply-ci
           echo "ID_SYNC_QUEUE_ARN=$(make -s output name=id_sync_queue_arn)" >> $GITHUB_ENV
 
       - name: Install poetry
@@ -137,12 +129,14 @@ jobs:
         with:
           python-version: 3.11
           cache: "poetry"
+          cache-dependency-path: |
+            lambdas/mns_subscription/poetry.lock
+            lambdas/shared/poetry.lock
 
       - name: Create MNS Subscription
         if: ${{ inputs.environment == 'dev' && inputs.create_mns_subscription }}
         working-directory: "./lambdas/mns_subscription"
         env:
-          APIGEE_ENVIRONMENT: ${{ inputs.apigee_environment }}
           SQS_ARN: ${{ env.ID_SYNC_QUEUE_ARN }}
         run: |
           poetry install --no-root

.github/workflows/pr-teardown.yml

@@ -43,7 +43,7 @@ jobs:
           terraform_version: "1.12.2"
 
       - name: Terraform Init and extract MNS SQS QUEUE ARN
-        working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
+        working-directory: terraform
         run: |
           make init apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
           make workspace apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT
@@ -56,6 +56,9 @@ jobs:
         with:
           python-version: 3.11
           cache: "poetry"
+          cache-dependency-path: |
+            lambdas/mns_subscription/poetry.lock
+            lambdas/shared/poetry.lock
 
       - name: Unsubscribe MNS
         working-directory: "./lambdas/mns_subscription"
@@ -68,6 +71,6 @@ jobs:
           make unsubscribe
 
       - name: Terraform Destroy
-        working-directory: ${{ vars.TERRAFORM_DIR_PATH }}
+        working-directory: terraform
         run: |
           make destroy apigee_environment=$APIGEE_ENVIRONMENT environment=$BACKEND_ENVIRONMENT sub_environment=$BACKEND_SUB_ENVIRONMENT

.github/workflows/run-e2e-tests.yml

@@ -0,0 +1,228 @@
+name: Run e2e Tests
+
+on:
+  workflow_call:
+    inputs:
+      apigee_environment:
+        required: true
+        type: string
+      environment:
+        required: true
+        type: string
+      sub_environment:
+        required: true
+        type: string
+  workflow_dispatch:
+    inputs:
+      apigee_environment:
+        type: choice
+        description: Select the Apigee proxy environment
+        options:
+          - internal-dev
+          - int
+          - ref
+          - prod
+      environment:
+        type: string
+        description: Select the backend environment
+        options:
+          - dev
+          - preprod
+          - prod
+      sub_environment:
+        type: string
+        description: Set the sub environment name e.g. pr-xxx, or green/blue in higher environments
+
+env:
+  APIGEE_ENVIRONMENT: ${{ inputs.apigee_environment }}
+  ENVIRONMENT: ${{ inputs.environment }}
+  SUB_ENVIRONMENT: ${{ inputs.sub_environment }}
+  SERVICE_BASE_PATH: ${{ startsWith(inputs.sub_environment, 'pr-') && format('immunisation-fhir-api/FHIR/R4-{0}', inputs.sub_environment) || 'immunisation-fhir-api/FHIR/R4' }}
+  PROXY_NAME: ${{ startsWith(inputs.sub_environment, 'pr-') && format('immunisation-fhir-api-{0}', inputs.sub_environment) || format('immunisation-fhir-api-{0}', inputs.apigee_environment) }}
+  STATUS_API_KEY: ${{ secrets.STATUS_API_KEY }}
+  SOURCE_COMMIT_ID: ${{ github.sha }}
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  wait-for-deployment:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.apigee_environment }}
+    outputs:
+      # Workaround for environment-level variables being unavailable in `jobs.<job-id>.if`.
+      RUN_BATCH_E2E_TESTS: ${{ vars.RUN_BATCH_E2E_TESTS }}
+    steps:
+      - name: Wait for API to be available
+        run: |
+          endpoint=""
+          if [[ ${APIGEE_ENVIRONMENT} =~ "prod" ]]; then
+            endpoint="https://api.service.nhs.uk/${SERVICE_BASE_PATH}/_status"
+          else
+            endpoint="https://${APIGEE_ENVIRONMENT}.api.service.nhs.uk/${SERVICE_BASE_PATH}/_status"
+          fi
+
+          counter=0
+          while [[ ${counter} -lt 31 ]]; do
+            response=$(curl -H "apikey: ${STATUS_API_KEY}" -s "${endpoint}")
+
+            response_code=$(jq -r '.checks.healthcheck.responseCode' <<< "${response}")
+            response_body=$(jq -r '.checks.healthcheck.outcome' <<< "${response}")
+            status=$(jq -r '.status' <<< "${response}")
+            commitId=$(jq -r '.commitId' <<< "${response}")
+
+            if [[ "${response_code}" -eq 200 ]] && [[ "${response_body}" == "OK" ]] && [[ "${status}" == "pass" ]]; then
+              echo "Status test successful"
+              if [[ "${commitId}" == "${SOURCE_COMMIT_ID}" ]]; then
+                echo "Commit hash test successful"
+                break
+              else
+                echo "Waiting for ${endpoint} to return the correct commit hash..."
+              fi
+            else
+              echo "Waiting for ${endpoint} to return a 200 response with 'OK' body..."
+            fi
+
+            ((counter=counter+1)) # Increment counter by 1
+            echo "Attempt ${counter}"
+            sleep 30
+          done
+
+          if [[ ${counter} -eq 31 ]]; then
+              echo "Status test failed: Maximum number of attempts reached"
+              echo "Last response received:"
+              echo "${response}"
+              exit 1
+          fi
+
+  e2e-tests:
+    runs-on: ubuntu-latest
+    needs: [wait-for-deployment]
+    environment: ${{ inputs.apigee_environment }}
+    env:
+      APIGEE_USERNAME: ${{ vars.APIGEE_USERNAME }}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+
+      - name: Connect to AWS
+        if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
+        uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838
+        with:
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
+          role-session-name: github-actions
+
+      - uses: hashicorp/setup-terraform@b9cd54a3c349d3f38e8881555d616ced269862dd
+        if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
+        with:
+          terraform_version: "1.12.2"
+
+      - name: Terraform Init
+        if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
+        working-directory: terraform
+        run: make init
+
+      - name: Set Terraform workspace
+        if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
+        working-directory: terraform
+        run: make workspace
+
+      - name: Read Terraform outputs
+        if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
+        working-directory: terraform
+        run: |
+          echo "IMMS_DELTA_TABLE_NAME=$(make -s output name=imms_delta_table_name)" >> $GITHUB_ENV
+          echo "AWS_DOMAIN_NAME=$(make -s output name=service_domain_name)" >> $GITHUB_ENV
+          echo "DYNAMODB_TABLE_NAME=$(make -s output name=dynamodb_table_name)" >> $GITHUB_ENV
+          echo "AWS_SQS_QUEUE_NAME=$(make -s output name=aws_sqs_queue_name)" >> $GITHUB_ENV
+          echo "AWS_SNS_TOPIC_NAME=$(make -s output name=aws_sns_topic_name)" >> $GITHUB_ENV
+
+      - name: Install poetry
+        run: pip install poetry==2.1.4
+
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c
+        with:
+          python-version: 3.11
+          cache: "poetry"
+          cache-dependency-path: "e2e/poetry.lock"
+
+      - name: Install e2e test dependencies
+        working-directory: e2e
+        run: poetry install --no-root
+
+      - name: Get Apigee access token
+        if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
+        working-directory: e2e
+        env:
+          APIGEE_PASSWORD: ${{ secrets.APIGEE_PASSWORD }}
+          APIGEE_BASIC_AUTH_TOKEN: ${{ secrets.APIGEE_BASIC_AUTH_TOKEN }}
+          APIGEE_OTP_KEY: ${{ secrets.APIGEE_OTP_KEY }}
+        run: |
+          CODE=$(poetry run python utils/compute_totp_code.py "$APIGEE_OTP_KEY")
+          echo "::add-mask::$CODE"
+
+          echo "Requesting access token from Apigee..."
+          response=$(curl -s -X POST "https://login.apigee.com/oauth/token" \
+            -H "Content-Type: application/x-www-form-urlencoded" \
+            -H "Accept: application/json;charset=utf-8" \
+            -H "Authorization: Basic $APIGEE_BASIC_AUTH_TOKEN" \
+            -d "username=$APIGEE_USERNAME&password=$APIGEE_PASSWORD&mfa_token=$CODE&grant_type=password")
+
+          token=$(jq -e -r '.access_token' <<< "$response")
+          echo "::add-mask::$token"
+          echo "APIGEE_ACCESS_TOKEN=$token" >> $GITHUB_ENV
+
+      - name: Run proxy deployment e2e test suite
+        working-directory: e2e
+        run: poetry run python -m unittest test_deployment
+
+      - name: Run proxy e2e test suite
+        if: ${{ vars.RUN_PROXY_E2E_TESTS == 'true' }}
+        working-directory: e2e
+        run: poetry run python -m unittest test_proxy
+
+      - name: Run sandbox e2e test suite
+        if: ${{ vars.RUN_SANDBOX_E2E_TESTS == 'true' }}
+        working-directory: e2e
+        run: poetry run python -m unittest test_proxy.TestProxyHealthcheck
+
+      - name: Run full e2e test suite
+        if: ${{ vars.RUN_FULL_E2E_TESTS == 'true' }}
+        working-directory: e2e
+        run: poetry run python -m unittest
+
+  batch-e2e-tests:
+    needs: [wait-for-deployment, e2e-tests]
+    # Only actually depend on wait-for-deployment, but run after e2e-tests
+    if: ${{ !cancelled() && needs.wait-for-deployment.result == 'success' && needs.wait-for-deployment.outputs.RUN_BATCH_E2E_TESTS == 'true' }}
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.apigee_environment }}
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8
+
+      - name: Connect to AWS
+        uses: aws-actions/configure-aws-credentials@a03048d87541d1d9fcf2ecf528a4a65ba9bd7838
+        with:
+          aws-region: eu-west-2
+          role-to-assume: arn:aws:iam::${{ vars.AWS_ACCOUNT_ID }}:role/auto-ops
+          role-session-name: github-actions
+
+      - name: Install poetry
+        run: pip install poetry==2.1.4
+
+      - uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c
+        with:
+          python-version: 3.11
+          cache: "poetry"
+          cache-dependency-path: "e2e_batch/poetry.lock"
+
+      - name: Install e2e test dependencies
+        working-directory: e2e_batch
+        run: poetry install --no-root
+
+      - name: Run batch e2e test suite
+        working-directory: e2e_batch
+        env:
+          ENVIRONMENT: ${{ inputs.sub_environment }}
+        run: poetry run python -m unittest -c -v