feat: [DTOSS-11798] testing environment poc

.github/workflows/cicd-1-pull-request-devtest.yaml

@@ -1,20 +1,162 @@
-name: CI/CD pull request - devtest
+name: 'CI/CD pull request - devtest'
 
 on:
   push:
     tags:
           - 'devtest'
 
+permissions:
+  contents: read
+  id-token: write
+  pull-requests: write
+
 jobs:
-  print-debug-info:
+  metadata:
+    name: "Set CI/CD metadata"
     runs-on: ubuntu-latest
+    timeout-minutes: 1
+    permissions:
+      pull-requests: read
+    outputs:
+      build_datetime_london: ${{ steps.variables.outputs.build_datetime_london }}
+      build_datetime: ${{ steps.variables.outputs.build_datetime }}
+      build_timestamp: ${{ steps.variables.outputs.build_timestamp }}
+      build_epoch: ${{ steps.variables.outputs.build_epoch }}
+      nodejs_version: ${{ steps.variables.outputs.nodejs_version }}
+      python_version: ${{ steps.variables.outputs.python_version }}
+      terraform_version: ${{ steps.variables.outputs.terraform_version }}
+      environment_tag: ${{ steps.variables.outputs.environment_tag }}
+      version: ${{ steps.variables.outputs.version }}
+      does_pull_request_exist: ${{ steps.pr_exists.outputs.does_pull_request_exist }}
     steps:
-      - name: Display Information
+      - name: "Checkout code"
+        uses: actions/checkout@v4
+        with:
+          submodules: 'true'
+      - name: "Set CI/CD variables"
+        id: variables
+        run: |
+          datetime=$(date -u +'%Y-%m-%dT%H:%M:%S%z')
+          BUILD_DATETIME=$datetime make version-create-effective-file
+          echo "build_datetime_london=$(TZ=Europe/London date --date=$datetime +'%Y-%m-%dT%H:%M:%S%z')" >> $GITHUB_OUTPUT
+          echo "build_datetime=$datetime" >> $GITHUB_OUTPUT
+          echo "build_timestamp=$(date --date=$datetime -u +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT
+          echo "build_epoch=$(date --date=$datetime -u +'%s')" >> $GITHUB_OUTPUT
+          echo "nodejs_version=$(grep "^nodejs" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
+          echo "python_version=$(grep "^nodejs" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
+          echo "terraform_version=$(grep "^terraform" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT
+          echo "version=$(head -n 1 .version 2> /dev/null || echo unknown)" >> $GITHUB_OUTPUT
+          echo "environment_tag=development" >> $GITHUB_OUTPUT
+      - name: "Check if pull request exists for this branch"
+        id: pr_exists
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          branch_name=${GITHUB_HEAD_REF:-$(echo $GITHUB_REF | sed 's#refs/heads/##')}
+          echo "Current branch is '$branch_name'"
+          if gh pr list --head $branch_name | grep -q .; then
+            echo "Pull request exists"
+            echo "does_pull_request_exist=true" >> $GITHUB_OUTPUT
+          else
+            echo "Pull request doesn't exist"
+            echo "does_pull_request_exist=false" >> $GITHUB_OUTPUT
+          fi
+      - name: "List variables"
         run: |
-          echo "Workflow triggered by tag!"
-          echo "--------------------------------------"
-          echo "Tag/Ref Name : $GITHUB_REF"
-          echo "Commit Hash  : $GITHUB_SHA"
-          echo "Triggered By : $GITHUB_ACTOR"
-          echo "Event Name   : $GITHUB_EVENT_NAME"
-          echo "--------------------------------------"
+          export BUILD_DATETIME_LONDON="${{ steps.variables.outputs.build_datetime_london }}"
+          export BUILD_DATETIME="${{ steps.variables.outputs.build_datetime }}"
+          export BUILD_TIMESTAMP="${{ steps.variables.outputs.build_timestamp }}"
+          export BUILD_EPOCH="${{ steps.variables.outputs.build_epoch }}"
+          export NODEJS_VERSION="${{ steps.variables.outputs.nodejs_version }}"
+          export PYTHON_VERSION="${{ steps.variables.outputs.python_version }}"
+          export TERRAFORM_VERSION="${{ steps.variables.outputs.terraform_version }}"
+          export ENVIRONMENT_TAG="${{ steps.variables.outputs.environment_tag }}"
+          export VERSION="${{ steps.variables.outputs.version }}"
+          export DOES_PULL_REQUEST_EXIST="${{ steps.pr_exists.outputs.does_pull_request_exist }}"
+          make list-variables
+  commit-stage: # Recommended maximum execution time is 2 minutes
+    name: "Commit stage"
+    needs: [metadata]
+    uses: ./.github/workflows/stage-1-commit.yaml
+    with:
+      build_datetime: "${{ needs.metadata.outputs.build_datetime }}"
+      build_timestamp: "${{ needs.metadata.outputs.build_timestamp }}"
+      build_epoch: "${{ needs.metadata.outputs.build_epoch }}"
+      nodejs_version: "${{ needs.metadata.outputs.nodejs_version }}"
+      python_version: "${{ needs.metadata.outputs.python_version }}"
+      terraform_version: "${{ needs.metadata.outputs.terraform_version }}"
+      version: "${{ needs.metadata.outputs.version }}"
+  # test-stage: # Recommended maximum execution time is 5 minutes
+  #   name: 'Test stage'
+  #   needs: [metadata]
+  #   uses: ./.github/workflows/stage-2-test.yaml
+  #   with:
+  #     unit_test_dir: tests/UnitTests
+  #     app_dir: application/CohortManager
+  #     build_datetime: '${{ needs.metadata.outputs.build_datetime }}'
+  #     build_timestamp: '${{ needs.metadata.outputs.build_timestamp }}'
+  #     build_epoch: '${{ needs.metadata.outputs.build_epoch }}'
+  #     nodejs_version: '${{ needs.metadata.outputs.nodejs_version }}'
+  #     python_version: '${{ needs.metadata.outputs.python_version }}'
+  #     terraform_version: '${{ needs.metadata.outputs.terraform_version }}'
+  #     version: '${{ needs.metadata.outputs.version }}'
+  #   secrets: inherit
+  # analysis-stage: # Recommended maximum execution time is 5 minutes
+  #   name: "Analysis stage"
+  #   needs: [metadata, commit-stage, test-stage]
+  #   uses: ./.github/workflows/stage-2-analyse.yaml
+  #   secrets:
+  #     sonar_token: ${{ secrets.SONAR_TOKEN }}
+  #   with:
+  #     unit_test_dir: tests/UnitTests
+  #     build_datetime: "${{ needs.metadata.outputs.build_datetime }}"
+  #     build_timestamp: "${{ needs.metadata.outputs.build_timestamp }}"
+  #     build_epoch: "${{ needs.metadata.outputs.build_epoch }}"
+  #     nodejs_version: "${{ needs.metadata.outputs.nodejs_version }}"
+  #     python_version: "${{ needs.metadata.outputs.python_version }}"
+  #     terraform_version: "${{ needs.metadata.outputs.terraform_version }}"
+  #     version: "${{ needs.metadata.outputs.version }}"
+  build-image-stage: # Recommended maximum execution time is 3 minutes
+    name: "Image build stage"
+    needs: [metadata, commit-stage] #[metadata, commit-stage, test-stage, analysis-stage]
+    uses: ./.github/workflows/stage-3-build-images-devtest.yaml
+    secrets:
+      client_id: ${{ secrets.AZURE_CLIENT_ID }}
+      tenant_id: ${{ secrets.AZURE_TENANT_ID }}
+      subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }}
+      acr_devtest_name: ${{ secrets.ACR_DEVTEST_NAME }}
+    with:
+      docker_compose_file: application/CohortManager/compose.yaml
+      excluded_containers_csv_list: azurite,azurite-setup,sql-server
+      environment_tag: ${{ needs.metadata.outputs.environment_tag }}
+      function_app_source_code_path: application/CohortManager/src
+      project_name: cohort-manager
+      build_all_images: true
+  # deploy-stage:
+  #   if: contains(github.event.pull_request.labels.*.name, 'deploy')
+  #   name: Deploy review app pr-${{ github.event.pull_request.number }}
+  #   needs: [build-image-stage]
+  #   permissions:
+  #     id-token: write
+  #     contents: read
+  #   uses: ./.github/workflows/stage-4-deploy.yaml
+  #   with:
+  #     environments: '["review"]'
+  #     commit_sha: ${{ github.event.pull_request.head.sha }}
+  #     pr_number: ${{ github.event.pull_request.number }}
+  #   secrets: inherit
+  # post-url:
+  #   if: contains(github.event.pull_request.labels.*.name, 'deploy')
+  #   name: Post URL pr-${{ github.event.pull_request.number }} to PR comments
+  #   runs-on: ubuntu-latest
+  #   needs: [deploy-stage]
+  #   permissions:
+  #     pull-requests: write
+  #   steps:
+  #     - name: Post URL to PR comments
+  #       uses: marocchino/sticky-pull-request-comment@5060d4700a91de252c87eeddd2da026382d9298a
+  #       with:
+  #         message: |
+  #           The review app is available at this URL:
+  #           https://pr-${{ github.event.pull_request.number }}.manage-breast-screening.non-live.screening.nhs.uk
+  #           You must authenticate with HTTP basic authentication. Ask the team for credentials.

.github/workflows/stage-3-build-images-devtest.yaml

@@ -0,0 +1,201 @@
+name: 'Docker Image CI - devtest'
+
+on:
+  workflow_call:
+    inputs:
+      environment_tag:
+        description: Environment of the deployment
+        required: true
+        type: string
+        default: development
+      docker_compose_file:
+        description: The path of the compose.yaml file needed to build docker images
+        required: true
+        type: string
+      function_app_source_code_path:
+        description: The source path of the function app source code for the docker builds
+        required: true
+        type: string
+      project_name:
+        description: The name of the project
+        required: true
+        type: string
+      excluded_containers_csv_list:
+        description: Excluded containers in a comma separated list
+        required: true
+        type: string
+      build_all_images:
+        description: Build all images (true) or only changed ones (false)
+        required: false
+        type: boolean
+        default: false
+
+    secrets:
+      client_id:
+        description: 'The Azure Client ID.'
+        required: true
+      tenant_id:
+        description: 'The Azure Tenant ID.'
+        required: true
+      subscription_id:
+        description: 'The Azure Subscription ID.'
+        required: true
+      acr_devtest_name:
+        description: 'The name of the Azure Container Registry.'
+        required: true
+
+jobs:
+  get-functions:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+      id-token: write
+    outputs:
+      FUNC_NAMES: ${{ steps.get-function-names.outputs.FUNC_NAMES }}
+      DOCKER_COMPOSE_DIR: ${{ steps.get-function-names.outputs.DOCKER_COMPOSE_DIR }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Checkout dtos-devops-templates repository
+        uses: actions/checkout@v4
+        with:
+          repository: NHSDigital/dtos-devops-templates
+          path: templates
+          ref: main
+
+      - name: Determine which Docker container(s) to build
+        id: get-function-names
+        env:
+          COMPOSE_FILES_CSV: ${{ inputs.docker_compose_file }}
+          EXCLUDED_CONTAINERS_CSV: ${{ inputs.excluded_containers_csv_list }}
+          SOURCE_CODE_PATH: ${{ inputs.function_app_source_code_path }}
+          MANUAL_BUILD_ALL: ${{ inputs.build_all_images || false }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: bash scripts/deployment/get-docker-names.sh
+
+  build-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+      pull-requests: read
+    needs: get-functions
+    strategy:
+      matrix:
+        function: ${{ fromJSON(needs.get-functions.outputs.FUNC_NAMES) }}
+    if: needs.get-functions.outputs.FUNC_NAMES != '[]'
+    outputs:
+      pr_num_tag: ${{ env.PR_NUM_TAG }}
+      short_commit_hash: ${{ env.COMMIT_HASH_TAG }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          fetch-depth: 1
+          submodules: 'true'
+
+      - name: Checkout dtos-devops-templates repository
+        uses: actions/checkout@v4
+        with:
+          repository: NHSDigital/dtos-devops-templates
+          path: templates
+          ref: main
+
+      - name: Az CLI login
+        uses: azure/login@v2
+        with:
+          client-id: ${{ secrets.client_id }}
+          tenant-id: ${{ secrets.tenant_id }}
+          subscription-id: ${{ secrets.subscription_id }}
+
+      - name: Azure Container Registry login
+        env:
+          ACR_DEVTEST_NAME: ${{ secrets.acr_devtest_name }}
+        run: az acr login --name ${ACR_DEVTEST_NAME}
+
+      - name: Create Tags
+        env:
+          GH_TOKEN: ${{ github.token }}
+          ENVIRONMENT_TAG: ${{ inputs.environment_tag }}
+        continue-on-error: false
+        run: |
+          echo "The branch is: ${GITHUB_REF}"
+
+          if [[ "${GITHUB_REF}" == refs/pull/*/merge ]]; then
+            PR_NUM_TAG=$(echo "${GITHUB_REF}" | sed 's/refs\/pull\/\([0-9]*\)\/merge/\1/')
+          else
+            PULLS_JSON=$(gh api /repos/{owner}/{repo}/commits/${GITHUB_SHA}/pulls)
+            ORIGINATING_BRANCH=$(echo ${PULLS_JSON} | jq -r '.[].head.ref' | python3 -c "import sys, urllib.parse; print(urllib.parse.quote_plus(sys.stdin.read().strip()))")
+            echo "ORIGINATING_BRANCH: ${ORIGINATING_BRANCH}"
+            PR_NUM_TAG=$(echo ${PULLS_JSON} | jq -r '.[].number')
+          fi
+
+          echo "PR_NUM_TAG: pr${PR_NUM_TAG}"
+          echo "PR_NUM_TAG=pr${PR_NUM_TAG}" >> ${GITHUB_ENV}
+
+          SHORT_COMMIT_HASH=$(git rev-parse --short ${GITHUB_SHA})
+          echo "Commit hash tag: ${SHORT_COMMIT_HASH}"
+          echo "COMMIT_HASH_TAG=${SHORT_COMMIT_HASH}" >> ${GITHUB_ENV}
+
+          echo "ENVIRONMENT_TAG=${ENVIRONMENT_TAG}" >> ${GITHUB_ENV}
+
+      - name: Build and Push Image
+        working-directory: ${{ steps.get-function-names.outputs.DOCKER_COMPOSE_DIR }}
+        continue-on-error: false
+        env:
+          COMPOSE_FILE: ${{ inputs.docker_compose_file }}
+          PROJECT_NAME: ${{ inputs.project_name }}
+          ACR_DEVTEST_NAME: ${{ secrets.acr_devtest_name }}
+        run: |
+          function=${{ matrix.function }}
+
+          echo PROJECT_NAME: ${PROJECT_NAME}
+
+          if [ -z "${function}" ]; then
+            echo "Function variable is empty. Skipping Docker build."
+            exit 0
+          fi
+
+          # Build the image
+          docker compose -f ${COMPOSE_FILE//,/ -f } -p ${PROJECT_NAME} --profile "*" build --no-cache --pull ${function}
+
+          repo_name="${ACR_DEVTEST_NAME}.azurecr.io/${PROJECT_NAME}-${function}"
+          echo $(repo_name)
+
+          # Tag the image
+          echo "Tag the image:"
+          docker tag ${PROJECT_NAME}-$ {function}:latest "$repo_name:${COMMIT_HASH_TAG}"
+          docker tag ${PROJECT_NAME}-${function}:latest "$repo_name:${PR_NUM_TAG}"
+          docker tag ${PROJECT_NAME}-${function}:latest "$repo_name:${ENVIRONMENT_TAG}"
+
+          # If this variable is set, the create-sbom-report.sh script will scan this docker image instead.
+          export CHECK_DOCKER_IMAGE=${PROJECT_NAME}-${function}:latest
+          export FORCE_USE_DOCKER=true
+
+          export PR_NUM_TAG=${PR_NUM_TAG}
+          echo "PR_NUM_TAG=${PR_NUM_TAG}" >> ${GITHUB_ENV}
+
+          # Push the image to the repository
+          docker push "${repo_name}:${COMMIT_HASH_TAG}"
+          if [ "${PR_NUM_TAG}" != 'pr' ]; then
+            docker push "${repo_name}:${PR_NUM_TAG}"
+          fi
+          docker push "${repo_name}:${ENVIRONMENT_TAG}"
+
+      - name: Cleanup the docker images
+        env:
+          PROJECT_NAME: ${{ inputs.project_name }}
+          ACR_DEVTEST_NAME: ${{ secrets.acr_devtest_name }}
+        run: |
+          function=${{ matrix.function }}
+          repo_name="${ACR_DEVTEST_NAME}.azurecr.io/${PROJECT_NAME}-${function}"
+
+          # Remove the images
+          docker rmi "${repo_name}:${COMMIT_HASH_TAG}"
+          docker rmi "${repo_name}:${PR_NUM_TAG}"
+          docker rmi "${repo_name}:${ENVIRONMENT_TAG}"
+          docker rmi ${PROJECT_NAME}-${function}:latest