diff --git a/.github/workflows/cicd-1-pull-request-devtest.yaml b/.github/workflows/cicd-1-pull-request-devtest.yaml index c9b312ff7..45523797e 100644 --- a/.github/workflows/cicd-1-pull-request-devtest.yaml +++ b/.github/workflows/cicd-1-pull-request-devtest.yaml @@ -1,20 +1,163 @@ -name: CI/CD pull request - devtest +name: 'CI/CD pull request - devtest' on: push: tags: - 'devtest' +permissions: + contents: read + id-token: write + pull-requests: write + jobs: - print-debug-info: + metadata: + name: "Set CI/CD metadata" runs-on: ubuntu-latest + timeout-minutes: 1 + permissions: + pull-requests: read + outputs: + build_datetime_london: ${{ steps.variables.outputs.build_datetime_london }} + build_datetime: ${{ steps.variables.outputs.build_datetime }} + build_timestamp: ${{ steps.variables.outputs.build_timestamp }} + build_epoch: ${{ steps.variables.outputs.build_epoch }} + nodejs_version: ${{ steps.variables.outputs.nodejs_version }} + python_version: ${{ steps.variables.outputs.python_version }} + terraform_version: ${{ steps.variables.outputs.terraform_version }} + environment_tag: ${{ steps.variables.outputs.environment_tag }} + version: ${{ steps.variables.outputs.version }} + does_pull_request_exist: ${{ steps.pr_exists.outputs.does_pull_request_exist }} steps: - - name: Display Information + - name: "Checkout code" + uses: actions/checkout@v4 + with: + submodules: 'true' + - name: "Set CI/CD variables" + id: variables + run: | + datetime=$(date -u +'%Y-%m-%dT%H:%M:%S%z') + BUILD_DATETIME=$datetime make version-create-effective-file + echo "build_datetime_london=$(TZ=Europe/London date --date=$datetime +'%Y-%m-%dT%H:%M:%S%z')" >> $GITHUB_OUTPUT + echo "build_datetime=$datetime" >> $GITHUB_OUTPUT + echo "build_timestamp=$(date --date=$datetime -u +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT + echo "build_epoch=$(date --date=$datetime -u +'%s')" >> $GITHUB_OUTPUT + echo "nodejs_version=$(grep "^nodejs" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT + echo "python_version=$(grep "^nodejs" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT + echo "terraform_version=$(grep "^terraform" .tool-versions | cut -f2 -d' ')" >> $GITHUB_OUTPUT + echo "version=$(head -n 1 .version 2> /dev/null || echo unknown)" >> $GITHUB_OUTPUT + echo "environment_tag=development" >> $GITHUB_OUTPUT + - name: "Check if pull request exists for this branch" + id: pr_exists + env: + GH_TOKEN: ${{ github.token }} + run: | + branch_name=${GITHUB_HEAD_REF:-$(echo $GITHUB_REF | sed 's#refs/heads/##')} + echo "Current branch is '$branch_name'" + if gh pr list --head $branch_name | grep -q .; then + echo "Pull request exists" + echo "does_pull_request_exist=true" >> $GITHUB_OUTPUT + else + echo "Pull request doesn't exist" + echo "does_pull_request_exist=false" >> $GITHUB_OUTPUT + fi + - name: "List variables" run: | - echo "Workflow triggered by tag!" - echo "--------------------------------------" - echo "Tag/Ref Name : $GITHUB_REF" - echo "Commit Hash : $GITHUB_SHA" - echo "Triggered By : $GITHUB_ACTOR" - echo "Event Name : $GITHUB_EVENT_NAME" - echo "--------------------------------------" + export BUILD_DATETIME_LONDON="${{ steps.variables.outputs.build_datetime_london }}" + export BUILD_DATETIME="${{ steps.variables.outputs.build_datetime }}" + export BUILD_TIMESTAMP="${{ steps.variables.outputs.build_timestamp }}" + export BUILD_EPOCH="${{ steps.variables.outputs.build_epoch }}" + export NODEJS_VERSION="${{ steps.variables.outputs.nodejs_version }}" + export PYTHON_VERSION="${{ steps.variables.outputs.python_version }}" + export TERRAFORM_VERSION="${{ steps.variables.outputs.terraform_version }}" + export ENVIRONMENT_TAG="${{ steps.variables.outputs.environment_tag }}" + export VERSION="${{ steps.variables.outputs.version }}" + export DOES_PULL_REQUEST_EXIST="${{ steps.pr_exists.outputs.does_pull_request_exist }}" + make list-variables + commit-stage: # Recommended maximum execution time is 2 minutes + name: "Commit stage" + needs: [metadata] + uses: ./.github/workflows/stage-1-commit.yaml + with: + build_datetime: "${{ needs.metadata.outputs.build_datetime }}" + build_timestamp: "${{ needs.metadata.outputs.build_timestamp }}" + build_epoch: "${{ needs.metadata.outputs.build_epoch }}" + nodejs_version: "${{ needs.metadata.outputs.nodejs_version }}" + python_version: "${{ needs.metadata.outputs.python_version }}" + terraform_version: "${{ needs.metadata.outputs.terraform_version }}" + version: "${{ needs.metadata.outputs.version }}" + # test-stage: # Recommended maximum execution time is 5 minutes + # name: 'Test stage' + # needs: [metadata] + # uses: ./.github/workflows/stage-2-test.yaml + # with: + # unit_test_dir: tests/UnitTests + # app_dir: application/CohortManager + # build_datetime: '${{ needs.metadata.outputs.build_datetime }}' + # build_timestamp: '${{ needs.metadata.outputs.build_timestamp }}' + # build_epoch: '${{ needs.metadata.outputs.build_epoch }}' + # nodejs_version: '${{ needs.metadata.outputs.nodejs_version }}' + # python_version: '${{ needs.metadata.outputs.python_version }}' + # terraform_version: '${{ needs.metadata.outputs.terraform_version }}' + # version: '${{ needs.metadata.outputs.version }}' + # secrets: inherit + # analysis-stage: # Recommended maximum execution time is 5 minutes + # name: "Analysis stage" + # needs: [metadata, commit-stage, test-stage] + # uses: ./.github/workflows/stage-2-analyse.yaml + # secrets: + # sonar_token: ${{ secrets.SONAR_TOKEN }} + # with: + # unit_test_dir: tests/UnitTests + # build_datetime: "${{ needs.metadata.outputs.build_datetime }}" + # build_timestamp: "${{ needs.metadata.outputs.build_timestamp }}" + # build_epoch: "${{ needs.metadata.outputs.build_epoch }}" + # nodejs_version: "${{ needs.metadata.outputs.nodejs_version }}" + # python_version: "${{ needs.metadata.outputs.python_version }}" + # terraform_version: "${{ needs.metadata.outputs.terraform_version }}" + # version: "${{ needs.metadata.outputs.version }}" + build-image-stage: # Recommended maximum execution time is 3 minutes + name: "Image build stage" + needs: [metadata, commit-stage] #[metadata, commit-stage, test-stage, analysis-stage] + uses: ./.github/workflows/stage-3-build-images-devtest.yaml + secrets: + client_id: ${{ secrets.AZURE_CLIENT_ID }} + tenant_id: ${{ secrets.AZURE_TENANT_ID }} + #subscription_id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + subscription_id_dev: ${{ secrets.AZURE_SUBSCRIPTION_ID_DEV }} + acr_devtest_name: ${{ secrets.ACR_DEVTEST_NAME }} + with: + docker_compose_file: application/CohortManager/compose.yaml + excluded_containers_csv_list: azurite,azurite-setup,sql-server + environment_tag: ${{ needs.metadata.outputs.environment_tag }} + function_app_source_code_path: application/CohortManager/src + project_name: cohort-manager + build_all_images: true + # deploy-stage: + # if: contains(github.event.pull_request.labels.*.name, 'deploy') + # name: Deploy review app pr-${{ github.event.pull_request.number }} + # needs: [build-image-stage] + # permissions: + # id-token: write + # contents: read + # uses: ./.github/workflows/stage-4-deploy.yaml + # with: + # environments: '["review"]' + # commit_sha: ${{ github.event.pull_request.head.sha }} + # pr_number: ${{ github.event.pull_request.number }} + # secrets: inherit + # post-url: + # if: contains(github.event.pull_request.labels.*.name, 'deploy') + # name: Post URL pr-${{ github.event.pull_request.number }} to PR comments + # runs-on: ubuntu-latest + # needs: [deploy-stage] + # permissions: + # pull-requests: write + # steps: + # - name: Post URL to PR comments + # uses: marocchino/sticky-pull-request-comment@5060d4700a91de252c87eeddd2da026382d9298a + # with: + # message: | + # The review app is available at this URL: + # https://pr-${{ github.event.pull_request.number }}.manage-breast-screening.non-live.screening.nhs.uk + # You must authenticate with HTTP basic authentication. Ask the team for credentials. diff --git a/.github/workflows/stage-3-build-images-devtest.yaml b/.github/workflows/stage-3-build-images-devtest.yaml new file mode 100644 index 000000000..ec3021186 --- /dev/null +++ b/.github/workflows/stage-3-build-images-devtest.yaml @@ -0,0 +1,204 @@ +name: 'Docker Image CI - devtest' + +on: + workflow_call: + inputs: + environment_tag: + description: Environment of the deployment + required: true + type: string + default: development + docker_compose_file: + description: The path of the compose.yaml file needed to build docker images + required: true + type: string + function_app_source_code_path: + description: The source path of the function app source code for the docker builds + required: true + type: string + project_name: + description: The name of the project + required: true + type: string + excluded_containers_csv_list: + description: Excluded containers in a comma separated list + required: true + type: string + build_all_images: + description: Build all images (true) or only changed ones (false) + required: false + type: boolean + default: false + + secrets: + client_id: + description: 'The Azure Client ID.' + required: true + tenant_id: + description: 'The Azure Tenant ID.' + required: true + # subscription_id: + # description: 'The Azure Subscription ID.' + # required: true + subscription_id_dev: + description: 'The Azure Development Subscription ID.' + required: true + acr_devtest_name: + description: 'The name of the Azure Container Registry.' + required: true + +jobs: + get-functions: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + id-token: write + outputs: + FUNC_NAMES: ${{ steps.get-function-names.outputs.FUNC_NAMES }} + DOCKER_COMPOSE_DIR: ${{ steps.get-function-names.outputs.DOCKER_COMPOSE_DIR }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 2 + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Checkout dtos-devops-templates repository + uses: actions/checkout@v4 + with: + repository: NHSDigital/dtos-devops-templates + path: templates + ref: main + + - name: Determine which Docker container(s) to build + id: get-function-names + env: + COMPOSE_FILES_CSV: ${{ inputs.docker_compose_file }} + EXCLUDED_CONTAINERS_CSV: ${{ inputs.excluded_containers_csv_list }} + SOURCE_CODE_PATH: ${{ inputs.function_app_source_code_path }} + MANUAL_BUILD_ALL: ${{ inputs.build_all_images || false }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: bash scripts/deployment/get-docker-names.sh + + build-and-push: + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + pull-requests: read + needs: get-functions + strategy: + matrix: + function: ${{ fromJSON(needs.get-functions.outputs.FUNC_NAMES) }} + if: needs.get-functions.outputs.FUNC_NAMES != '[]' + outputs: + pr_num_tag: ${{ env.PR_NUM_TAG }} + short_commit_hash: ${{ env.COMMIT_HASH_TAG }} + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + fetch-depth: 1 + submodules: 'true' + + - name: Checkout dtos-devops-templates repository + uses: actions/checkout@v4 + with: + repository: NHSDigital/dtos-devops-templates + path: templates + ref: main + + - name: Az CLI login + uses: azure/login@v2 + with: + client-id: ${{ secrets.client_id }} + tenant-id: ${{ secrets.tenant_id }} + subscription-id: ${{ secrets.subscription_id_dev }} + + - name: Azure Container Registry login + env: + ACR_DEVTEST_NAME: ${{ secrets.acr_devtest_name }} + run: az acr login --name ${ACR_DEVTEST_NAME} + + - name: Create Tags + env: + GH_TOKEN: ${{ github.token }} + ENVIRONMENT_TAG: ${{ inputs.environment_tag }} + continue-on-error: false + run: | + echo "The branch is: ${GITHUB_REF}" + + if [[ "${GITHUB_REF}" == refs/pull/*/merge ]]; then + PR_NUM_TAG=$(echo "${GITHUB_REF}" | sed 's/refs\/pull\/\([0-9]*\)\/merge/\1/') + else + PULLS_JSON=$(gh api /repos/{owner}/{repo}/commits/${GITHUB_SHA}/pulls) + ORIGINATING_BRANCH=$(echo ${PULLS_JSON} | jq -r '.[].head.ref' | python3 -c "import sys, urllib.parse; print(urllib.parse.quote_plus(sys.stdin.read().strip()))") + echo "ORIGINATING_BRANCH: ${ORIGINATING_BRANCH}" + PR_NUM_TAG=$(echo ${PULLS_JSON} | jq -r '.[].number') + fi + + echo "PR_NUM_TAG: pr${PR_NUM_TAG}" + echo "PR_NUM_TAG=pr${PR_NUM_TAG}" >> ${GITHUB_ENV} + + SHORT_COMMIT_HASH=$(git rev-parse --short ${GITHUB_SHA}) + echo "Commit hash tag: ${SHORT_COMMIT_HASH}" + echo "COMMIT_HASH_TAG=${SHORT_COMMIT_HASH}" >> ${GITHUB_ENV} + + echo "ENVIRONMENT_TAG=${ENVIRONMENT_TAG}" >> ${GITHUB_ENV} + + - name: Build and Push Image + working-directory: ${{ steps.get-function-names.outputs.DOCKER_COMPOSE_DIR }} + continue-on-error: false + env: + COMPOSE_FILE: ${{ inputs.docker_compose_file }} + PROJECT_NAME: ${{ inputs.project_name }} + ACR_DEVTEST_NAME: ${{ secrets.acr_devtest_name }} + run: | + function=${{ matrix.function }} + + echo PROJECT_NAME: ${PROJECT_NAME} + + if [ -z "${function}" ]; then + echo "Function variable is empty. Skipping Docker build." + exit 0 + fi + + # Build the image + docker compose -f ${COMPOSE_FILE//,/ -f } -p ${PROJECT_NAME} --profile "*" build --no-cache --pull ${function} + + repo_name="${ACR_DEVTEST_NAME}.azurecr.io/${PROJECT_NAME}-${function}" + echo $(repo_name) + + # Tag the image + echo "Tag the image:" + docker tag ${PROJECT_NAME}-${function}:latest "$repo_name:${COMMIT_HASH_TAG}" + docker tag ${PROJECT_NAME}-${function}:latest "$repo_name:${PR_NUM_TAG}" + docker tag ${PROJECT_NAME}-${function}:latest "$repo_name:${ENVIRONMENT_TAG}" + + # If this variable is set, the create-sbom-report.sh script will scan this docker image instead. + export CHECK_DOCKER_IMAGE=${PROJECT_NAME}-${function}:latest + export FORCE_USE_DOCKER=true + + export PR_NUM_TAG=${PR_NUM_TAG} + echo "PR_NUM_TAG=${PR_NUM_TAG}" >> ${GITHUB_ENV} + + # Push the image to the repository + docker push "${repo_name}:${COMMIT_HASH_TAG}" + if [ "${PR_NUM_TAG}" != 'pr' ]; then + docker push "${repo_name}:${PR_NUM_TAG}" + fi + docker push "${repo_name}:${ENVIRONMENT_TAG}" + + - name: Cleanup the docker images + env: + PROJECT_NAME: ${{ inputs.project_name }} + ACR_DEVTEST_NAME: ${{ secrets.acr_devtest_name }} + run: | + function=${{ matrix.function }} + repo_name="${ACR_DEVTEST_NAME}.azurecr.io/${PROJECT_NAME}-${function}" + + # Remove the images + docker rmi "${repo_name}:${COMMIT_HASH_TAG}" + docker rmi "${repo_name}:${PR_NUM_TAG}" + docker rmi "${repo_name}:${ENVIRONMENT_TAG}" + docker rmi ${PROJECT_NAME}-${function}:latest diff --git a/infrastructure/tf-core/container_registry.tf b/infrastructure/tf-core/container_registry.tf new file mode 100644 index 000000000..1105ef008 --- /dev/null +++ b/infrastructure/tf-core/container_registry.tf @@ -0,0 +1,30 @@ +module "acr" { + for_each = var.features.acr_enabled ? var.regions : {} + + source = "../../../dtos-devops-templates/infrastructure/modules/container-registry" + + name = module.regions_config[each.key].names.azure-container-registry #-${lower(each.key.name_suffix)}" + resource_group_name = azurerm_resource_group.core[each.key].name + location = each.key + + admin_enabled = var.container_registry.admin_enabled + + log_analytics_workspace_id = data.terraform_remote_state.audit.outputs.log_analytics_workspace_id[local.primary_region] + monitor_diagnostic_setting_acr_enabled_logs = local.monitor_diagnostic_setting_acr_enabled_logs + monitor_diagnostic_setting_acr_metrics = local.monitor_diagnostic_setting_acr_metrics + + uai_name = var.container_registry.uai_name + sku = var.container_registry.sku + public_network_access_enabled = var.features.public_network_access_enabled + + # Private Endpoint Configuration if enabled + private_endpoint_properties = var.features.private_endpoints_enabled ? { + private_dns_zone_ids = [data.terraform_remote_state.hub.outputs.private_dns_zones["${each.key}-container_registry"].id] + private_endpoint_enabled = var.features.private_endpoints_enabled + private_endpoint_subnet_id = module.subnets["${module.regions_config[each.key].names.subnet}-pep"].id + private_endpoint_resource_group_name = azurerm_resource_group.rg_private_endpoints[each.key].name + private_service_connection_is_manual = var.features.private_service_connection_is_manual + } : null + + tags = var.tags +} diff --git a/infrastructure/tf-core/diagnostic_settings.tf b/infrastructure/tf-core/diagnostic_settings.tf index 8309155ba..ff59480f8 100644 --- a/infrastructure/tf-core/diagnostic_settings.tf +++ b/infrastructure/tf-core/diagnostic_settings.tf @@ -1,4 +1,8 @@ locals { + # ACR + monitor_diagnostic_setting_acr_enabled_logs = ["ContainerRegistryRepositoryEvents", "ContainerRegistryLoginEvents"] + monitor_diagnostic_setting_acr_metrics = ["AllMetrics"] + # APPSERVICEPLAN monitor_diagnostic_setting_appserviceplan_metrics = ["AllMetrics"] diff --git a/infrastructure/tf-core/environments/development.tfvars b/infrastructure/tf-core/environments/development.tfvars index e45814c91..55b1623a8 100644 --- a/infrastructure/tf-core/environments/development.tfvars +++ b/infrastructure/tf-core/environments/development.tfvars @@ -4,7 +4,7 @@ environment = "DEV" environment_hub = "dev" features = { - acr_enabled = false + acr_enabled = true api_management_enabled = false event_grid_enabled = false private_endpoints_enabled = true @@ -272,6 +272,13 @@ container_apps = { } } +container_registry = { + name_suffix = "devtest" + admin_enabled = false + uai_name = "dtos-cohort-manager-acr-push" + sku = "Premium" +} + diagnostic_settings = { metric_enabled = true } @@ -867,7 +874,7 @@ function_apps = { ] env_vars_static = { AcceptableLatencyThresholdMs = "500" - MaxRetryCount=3 + MaxRetryCount = 3 } } diff --git a/infrastructure/tf-core/variables.tf b/infrastructure/tf-core/variables.tf index ba256cb77..f39b61f6d 100644 --- a/infrastructure/tf-core/variables.tf +++ b/infrastructure/tf-core/variables.tf @@ -615,3 +615,13 @@ variable "function_app_slots" { function_app_slot_enabled = optional(bool, false) })) } + +variable "container_registry" { + description = "Configuration of the Azure Container Registry used for feature testing" + type = object({ + name_suffix = optional(string, "") + admin_enabled = optional(bool, false) + uai_name = optional(string, "dtos-cohort-manager-acr-push") + sku = optional(string, "Premium") + }) +}