diff --git a/.github/actions/scan-secrets/action.yaml b/.github/actions/scan-secrets/action.yaml new file mode 100644 index 000000000..d555ef177 --- /dev/null +++ b/.github/actions/scan-secrets/action.yaml @@ -0,0 +1,10 @@ +name: "Scan secrets" +description: "Scan secrets" +runs: + using: "composite" + steps: + - name: "Scan secrets" + shell: bash + run: | + # Please do not change this `check=whole-history` setting, as new patterns may be added or history may be rewritten. + check=whole-history ./scripts/githooks/scan-secrets.sh \ No newline at end of file diff --git a/.gitleaksignore b/.gitleaksignore new file mode 100644 index 000000000..141f0ed5a --- /dev/null +++ b/.gitleaksignore @@ -0,0 +1,3 @@ +# SEE: https://github.com/gitleaks/gitleaks/blob/master/README.md#gitleaksignore + +cd9c0efec38c5d63053dd865e5d4e207c0760d91:docs/guides/Perform_static_analysis.md:generic-api-key:37 \ No newline at end of file diff --git a/README.md b/README.md index 0524ad641..531289844 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,6 @@ https://nhsd-confluence.digital.nhs.uk/spaces/APM/pages/1226682275/Pipeline+Quer Note: Projects running Python version 3.13 or later do not need any pipeline modifications. - ## Scripts * `template.py` - cli for basic jinja templating * `test_pull_request_deployments.py` - cli for testing utils against other repositories diff --git a/ansible/Makefile b/ansible/Makefile index b422c7f6e..01f3e7bea 100644 --- a/ansible/Makefile +++ b/ansible/Makefile @@ -43,7 +43,12 @@ create-build-env-vars: guard-build_label guard-out_dir @poetry run ansible-playbook -i local create-build-env-vars.yml deploy-ecs-proxies: guard-account guard-build_label guard-service_id guard-APIGEE_ENVIRONMENT guard-PROXY_VARS_FILE - @poetry run ansible-playbook -i local deploy-ecs-proxies.yml + @echo "MAKE DEBUG: use_ecs_tag=${use_ecs_tag}" + @poetry run ansible-playbook -i local deploy-ecs-proxies.yml \ + -e "use_ecs_tag=${use_ecs_tag}" + +deploy-ecs-proxies-retag: guard-build_label guard-service_id guard-PROXY_VARS_FILE + @poetry run ansible-playbook -i local deploy-ecs-proxies-retag.yml deploy-apigee-proxy: guard-FULLY_QUALIFIED_SERVICE_NAME guard-SERVICE_BASE_PATH guard-APIGEE_ENVIRONMENT guard-APIGEE_ORGANIZATION guard-APIGEE_ACCESS_TOKEN guard-PROXY_DIR guard-PING @poetry run ansible-playbook -i local deploy-apigee-proxy.yml diff --git a/ansible/deploy-ecs-proxies-retag.yml b/ansible/deploy-ecs-proxies-retag.yml new file mode 100644 index 000000000..7718d0c4e --- /dev/null +++ b/ansible/deploy-ecs-proxies-retag.yml @@ -0,0 +1,30 @@ +- name: deploy ecs proxies retag + hosts: 127.0.0.1 + connection: local + gather_facts: no + + vars: + service_id: "{{ lookup('env','service_id') }}" + APIGEE_ENVIRONMENT: "{{ lookup('env','APIGEE_ENVIRONMENT') }}" + account: "{{ lookup('env','account') }}" + + pre_tasks: + - name: Show CONTAINER_VARS_FILE from environment + debug: + msg: "CONTAINER_VARS_FILE={{ lookup('env','CONTAINER_VARS_FILE') }}" + + - name: include container vars + include_vars: + file: "{{ lookup('env', 'CONTAINER_VARS_FILE') | expandvars | expanduser | realpath }}" + + - name: Debug docker_containers + debug: + var: docker_containers + + - name: Debug containers + debug: + var: containers + + roles: + - setup-facts + - deploy-ecs-proxies-retag diff --git a/ansible/deploy-ecs-proxies.yml b/ansible/deploy-ecs-proxies.yml index 43a54df26..35c488b79 100644 --- a/ansible/deploy-ecs-proxies.yml +++ b/ansible/deploy-ecs-proxies.yml @@ -43,6 +43,19 @@ include_vars: file: "{{ lookup('env', 'PROXY_VARS_FILE') | expandvars | expanduser | realpath }}" + - name: load use_ecs_tag from environment + set_fact: + use_ecs_tag: "{{ lookup('env','use_ecs_tag') | default('false') }}" + + - name: normalise use_ecs_tag to boolean + set_fact: + use_ecs_tag: "{{ use_ecs_tag | lower == 'true' }}" + + - name: debug use_ecs_tag type + debug: + msg: "VALUEDEPLOYYAML={{ use_ecs_tag }} TYPE={{ use_ecs_tag | type_debug }}" + + roles: - setup-facts - deploy-ecs-proxies \ No newline at end of file diff --git a/ansible/ecr-lifecycle/ecr_lifecycle.json b/ansible/ecr-lifecycle/ecr_lifecycle.json new file mode 100644 index 000000000..ee111e937 --- /dev/null +++ b/ansible/ecr-lifecycle/ecr_lifecycle.json @@ -0,0 +1,36 @@ +{ + "rules": [ + { + "rulePriority": 1, + "description": "Keep the 6 most recent ECS deployment images tagged ecs- (release images)", + "selection": { + "tagStatus": "tagged", + "tagPrefixList": ["ecs-"], + "countType": "imageCountMoreThan", + "countNumber": 6 + }, + "action": { "type": "expire" } + }, + { + "rulePriority": 2, + "description": "Never expire the 'latest' tag", + "selection": { + "tagStatus": "tagged", + "tagPrefixList": ["latest"], + "countType": "imageCountMoreThan", + "countNumber": 9999 + }, + "action": { "type": "expire" } + }, + { + "rulePriority": 3, + "description": "Keep the 6 most recent build images (all tags)", + "selection": { + "tagStatus": "any", + "countType": "imageCountMoreThan", + "countNumber": 6 + }, + "action": { "type": "expire" } + } + ] +} diff --git a/ansible/roles/build-ecs-proxies/tasks/main.yml b/ansible/roles/build-ecs-proxies/tasks/main.yml index 284b881d3..e6863769d 100644 --- a/ansible/roles/build-ecs-proxies/tasks/main.yml +++ b/ansible/roles/build-ecs-proxies/tasks/main.yml @@ -30,6 +30,29 @@ with_items: "{{ new_repos }}" when: new_repos +# TO DO- Add back in once confirmed lifecycle policy to be applied to all new repos. + +# - name: Read lifecycle policy file +# ansible.builtin.slurp: +# src: "{{ playbook_dir }}/ecr-lifecycle/ecr_lifecycle.json" +# register: desired_policy_raw +# when: new_repos | length > 0 + +# - name: Decode lifecycle policy JSON +# set_fact: +# desired_policy_json: "{{ desired_policy_raw.content | b64decode | from_json }}" +# when: new_repos | length > 0 + +# - name: Apply lifecycle policy to each new repo +# ansible.builtin.command: > +# {{ aws_cmd }} ecr put-lifecycle-policy +# --repository-name {{ item }} +# --lifecycle-policy-text '{{ desired_policy_json | to_json }}' +# with_items: "{{ new_repos }}" +# register: lifecycle_update +# ignore_errors: yes +# when: new_repos | length > 0 + - name: ecr login shell: "eval $({{ aws_cmd }} ecr get-login --no-include-email)" changed_when: no diff --git a/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf b/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf index 29eb55a3e..775b01a11 100644 --- a/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf +++ b/ansible/roles/create-api-deployment-pre-reqs/templates/terraform/iam.tf @@ -69,6 +69,8 @@ data "aws_iam_policy_document" "ecs-execution-role" { "ecr:DescribeRepositories", "ecr:ListImages", "ecr:DescribeImages", + "ecr:GetLifecyclePolicy", + "ecr:PutLifecyclePolicy", "s3:GetObject" ] @@ -173,6 +175,18 @@ data "aws_iam_policy_document" "deploy-user" { } + statement { + actions = [ + "ecr:GetLifecyclePolicy", + "ecr:PutLifecyclePolicy" + ] + + resources = [ + "arn:aws:ecr:${local.region}:${local.account_id}:repository/${var.service_id}", + "arn:aws:ecr:${local.region}:${local.account_id}:repository/${var.service_id}_*" + ] + } + statement { actions = [ "s3:ListBucket", diff --git a/ansible/roles/create-ecr-build-role/vars/main.yml b/ansible/roles/create-ecr-build-role/vars/main.yml index c40db5b1a..817fd7bb0 100644 --- a/ansible/roles/create-ecr-build-role/vars/main.yml +++ b/ansible/roles/create-ecr-build-role/vars/main.yml @@ -44,6 +44,7 @@ aws_ecs_policy: - "ecr:StartImageScan" - "ecr:StartLifecyclePolicyPreview" - "ecr:UploadLayerPart" + - "ecr:PutLifecyclePolicy" Resource: [ "arn:aws:ecr:{{ aws_region }}:{{ aws_account_id }}:repository/{{ service_id }}_*" ] diff --git a/ansible/roles/deploy-ecs-proxies-retag/tasks/main.yml b/ansible/roles/deploy-ecs-proxies-retag/tasks/main.yml new file mode 100644 index 000000000..145e10c95 --- /dev/null +++ b/ansible/roles/deploy-ecs-proxies-retag/tasks/main.yml @@ -0,0 +1,95 @@ +- name: Ensure docker_containers is loaded + include_vars: + file: "{{ lookup('env', 'CONTAINER_VARS_FILE') }}" + when: docker_containers is not defined + +- name: Debug docker_containers (retag role) + debug: + var: docker_containers + +- name: Debug containers (retag role) + debug: + var: containers + +- name: Debug repo_names (retag role) + debug: + var: repo_names + +- name: Login to ECR + shell: > + {{ aws_cmd }} ecr get-login-password --region {{ aws_region }} + | docker login --username AWS --password-stdin {{ ecr_registry }} + +- name: Debug pulling image + debug: + msg: "Pulling {{ ecr_registry }}/{{ item }}:{{ build_label }}" + loop: "{{ repo_names }}" + loop_control: + label: "{{ item }}" + +- name: Pull existing image + ansible.builtin.command: + cmd: > + docker pull {{ ecr_registry }}/{{ item }}:{{ build_label }} + loop: "{{ repo_names }}" + loop_control: + label: "{{ item }}" + register: pull_results + +- name: Debug retagging image + debug: + msg: "Retagging {{ item.item }}:{{ build_label }} → ecs-{{ build_label }}" + loop: "{{ pull_results.results }}" + loop_control: + label: "{{ item.item }}" + when: + - item.rc == 0 + - item.item == "canary_canary-api" + +- name: Retag image + ansible.builtin.command: + cmd: > + docker tag + {{ ecr_registry }}/{{ item.item }}:{{ build_label }} + {{ ecr_registry }}/{{ item.item }}:ecs-{{ build_label }} + loop: "{{ pull_results.results }}" + loop_control: + label: "{{ item.item }}" + when: + - item.rc == 0 + - item.item == "canary_canary-api" + +- name: Debug pushing image + debug: + msg: "Pushing ecs-{{ build_label }} for {{ item.item }}" + loop: "{{ pull_results.results }}" + loop_control: + label: "{{ item.item }}" + when: + - item.rc == 0 + - item.item == "canary_canary-api" + +- name: Push new tag + ansible.builtin.command: + cmd: > + docker push {{ ecr_registry }}/{{ item.item }}:ecs-{{ build_label }} + loop: "{{ pull_results.results }}" + loop_control: + label: "{{ item.item }}" + when: + - item.rc == 0 + - item.item == "canary_canary-api" + +# - name: Delete old tag from ECR +# ansible.builtin.command: +# cmd: > +# aws ecr batch-delete-image +# --repository-name {{ item.item }} +# --image-ids imageTag={{ build_label }} +# --region {{ aws_region }} +# loop: "{{ pull_results.results }}" +# loop_control: +# label: "{{ item.item }}" +# when: +# - item.rc == 0 +# - item.item == "canary_canary-api" diff --git a/ansible/roles/deploy-ecs-proxies-retag/vars/main.yml b/ansible/roles/deploy-ecs-proxies-retag/vars/main.yml new file mode 100644 index 000000000..da9d210b6 --- /dev/null +++ b/ansible/roles/deploy-ecs-proxies-retag/vars/main.yml @@ -0,0 +1,6 @@ +--- +build_label: "{{ lookup('env', 'build_label') }}" +containers: "{{ docker_containers | json_query('[].name') | unique | sort }}" +repo_names: "{{ containers | map('regex_replace', '^(.*)$', service_id + '_\\1') | list }}" +base_dir: "{{ playbook_dir }}/../.." + diff --git a/ansible/roles/deploy-ecs-proxies/tasks/main.yml b/ansible/roles/deploy-ecs-proxies/tasks/main.yml index 668c8cb0e..0796bf5b7 100644 --- a/ansible/roles/deploy-ecs-proxies/tasks/main.yml +++ b/ansible/roles/deploy-ecs-proxies/tasks/main.yml @@ -71,14 +71,23 @@ with_filetree: "{{ '../templates' }}" when: item.state == 'file' + - name: debug use_ecs_tag before terraform + debug: + msg: "ANSIBLE: use_ecs_tag={{ use_ecs_tag }} TF_VAR_use_ecs_tag={{ use_ecs_tag | ternary('true','false') }}" + - name: terraform plan - shell: "make -C {{ out_dir }}/terraform clean plan args='-no-color -lock-timeout=30m -out tfplan.out'" # noqa 305 + shell: "TF_VAR_use_ecs_tag={{ use_ecs_tag | ternary('true','false') }} make -C {{ out_dir }}/terraform clean plan args='-no-color -lock-timeout=30m -out tfplan.out'" # noqa 305 register: tfplan failed_when: tfplan.rc not in (0, 2) when: not do_not_terraform + - name: print full terraform plan output (lines) + debug: + var: tfplan.stdout_lines + when: tfplan is defined + - name: terraform apply - shell: "make -C {{ out_dir }}/terraform apply-plan args='-no-color -lock-timeout=30m --auto-approve tfplan.out'" # noqa 305 + shell: "TF_VAR_use_ecs_tag={{ use_ecs_tag | ternary('true','false') }} make -C {{ out_dir }}/terraform apply-plan args='-no-color -lock-timeout=30m --auto-approve tfplan.out'" # noqa 305 register: tfapply when: not do_not_terraform diff --git a/ansible/roles/deploy-ecs-proxies/templates/terraform/locals.tf b/ansible/roles/deploy-ecs-proxies/templates/terraform/locals.tf index c01c869d5..d44cfceab 100644 --- a/ansible/roles/deploy-ecs-proxies/templates/terraform/locals.tf +++ b/ansible/roles/deploy-ecs-proxies/templates/terraform/locals.tf @@ -43,18 +43,41 @@ locals { } - ecs_service = [ - {% for container in ecs_service %} - {{ - ( - container - | combine( - {'image': '${local.account_id}.dkr.ecr.eu-west-2.amazonaws.com/' + service_id + '_' + container.name + ':' + build_label } +ecs_service = [ +{% for container in ecs_service %} + + # DEBUG: print the boolean value Jinja sees + {% if use_ecs_tag %} + {% set _ = print("DEBUG: use_ecs_tag = TRUE") %} + {% else %} + {% set _ = print("DEBUG: use_ecs_tag = FALSE") %} + {% endif %} + + {% set image_tag = ( + '${local.account_id}.dkr.ecr.eu-west-2.amazonaws.com/' + + service_id + '_' + container.name + + ( + ":ecs-" + build_label + if use_ecs_tag and container.name == "canary-api" + else ":" + build_label ) - ) | to_json - }}, - {% endfor %} - ] + ) + %} + + + {{ + ( + container + | combine( + { + 'image': image_tag + } + ) + ) | to_json + }}, + +{% endfor %} +] exposed_service = element(matchkeys(local.ecs_service, local.ecs_service.*.expose, list(true)), 0) diff --git a/ansible/roles/deploy-ecs-proxies/templates/terraform/variables.tf b/ansible/roles/deploy-ecs-proxies/templates/terraform/variables.tf index 3b88aad59..361c5df98 100644 --- a/ansible/roles/deploy-ecs-proxies/templates/terraform/variables.tf +++ b/ansible/roles/deploy-ecs-proxies/templates/terraform/variables.tf @@ -71,4 +71,9 @@ variable "autoscaling_scale_out_cooldown" { variable "deregistration_delay" { type = number +} + +variable "use_ecs_tag" { + type = bool + description = "Whether to use ecs- prefixed tag for canary-api" } \ No newline at end of file diff --git a/azure/build-prereqs.yml b/azure/build-prereqs.yml index 2f3944a3a..11374528f 100644 --- a/azure/build-prereqs.yml +++ b/azure/build-prereqs.yml @@ -4,14 +4,13 @@ parameters: default: 'utils' steps: - # - bash: | - # echo "Setting python tool cache path" - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]:/agent/_work/_tool/Python/3.13.7/x64/lib/" - # displayName: 'Set python tool cache path' - bash: | - PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \ - | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \ - | sort -V | tail -n 1 | cut -d- -f1) + pyversion="3.13" + PATCH=$(curl -fsSL https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json \ + | jq -r --arg pyversion "$pyversion" '[ .[] | select(.stable == true) | .version + | select(test("^" + ($pyversion|gsub("\\.";"\\.")) + "\\.\\d+$")) | split(".") | map(tonumber)] | max | join(".")') + + echo "##vso[task.setvariable variable=PY_VER]$PATCH" echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" displayName: 'Query and set python tool cache path' @@ -21,13 +20,13 @@ steps: name: UsePy displayName: 'Use Python 3.13' inputs: - versionSpec: '3.13' + versionSpec: '$(PY_VER)' - - bash: | - echo "Checking the python version in use to set LD_LIBRARY_PATH" - echo "Python location: $(UsePy.pythonLocation)" - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - displayName: 'Set LD_LIBRARY_PATH' + # - bash: | + # echo "Checking the python version in use to set LD_LIBRARY_PATH" + # echo "Python location: $(UsePy.pythonLocation)" + # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" + # displayName: 'Set LD_LIBRARY_PATH' - bash: | diff --git a/azure/cleanup-ecs-pr-proxies.yml b/azure/cleanup-ecs-pr-proxies.yml index 070823ceb..373ed4a68 100644 --- a/azure/cleanup-ecs-pr-proxies.yml +++ b/azure/cleanup-ecs-pr-proxies.yml @@ -61,24 +61,26 @@ jobs: displayName: cache utils pre-requisites - bash: | - PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \ - | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \ - | sort -V | tail -n 1 | cut -d- -f1) + pyversion="3.13" + PATCH=$(curl -fsSL https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json \ + | jq -r --arg pyversion "$pyversion" '[ .[] | select(.stable == true) | .version + | select(test("^" + ($pyversion|gsub("\\.";"\\.")) + "\\.\\d+$")) | split(".") | map(tonumber)] | max | join(".")') echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" - displayName: 'Query and set python tool cache path' + displayName: 'Query and set python tool cache path' + - task: UsePythonVersion@0 name: UsePy - displayName: "Use Python 3.13" + displayName: 'Use Python 3.13' inputs: - versionSpec: 3.13 + versionSpec: '$(PY_VER)' - - bash: | - echo "Checking the python version in use to set LD_LIBRARY_PATH" - echo "Python location: $(UsePy.pythonLocation)" - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - displayName: 'Set LD_LIBRARY_PATH' + # - bash: | + # echo "Checking the python version in use to set LD_LIBRARY_PATH" + # echo "Python location: $(UsePy.pythonLocation)" + # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" + # displayName: 'Set LD_LIBRARY_PATH' - bash: | make install diff --git a/azure/common/apigee-build.yml b/azure/common/apigee-build.yml index 0c28d76e7..873fe838a 100644 --- a/azure/common/apigee-build.yml +++ b/azure/common/apigee-build.yml @@ -143,32 +143,29 @@ jobs: parameters: service_name: "${{ parameters.service_name }}" - # - bash: | - # echo "Setting python tool cache path" - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]:/agent/_work/_tool/Python/${{ parameters.python_version }}/x64/lib/" - # displayName: 'Set python tool cache path' - - bash: | - PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \ - | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \ - | sort -V | tail -n 1 | cut -d- -f1) + pyversion=${{ parameters.python_version }} + PATCH=$(curl -fsSL https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json \ + | jq -r --arg pyversion "$pyversion" '[ .[] | select(.stable == true) | .version + | select(test("^" + ($pyversion|gsub("\\.";"\\.")) + "\\.\\d+$")) | split(".") | map(tonumber)] | max | join(".")') + + echo "##vso[task.setvariable variable=PY_VER]$PATCH" echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" - displayName: 'Query and set python tool cache path' + displayName: 'Query and set python tool cache path' + - task: UsePythonVersion@0 name: UsePy - displayName: "Use Python ${{ parameters.python_version }}" + displayName: 'Use Python ${{ parameters.python_version }}' inputs: - versionSpec: ${{ parameters.python_version }} - + versionSpec: '$(PY_VER)' - - bash: | - echo "Checking the python version in use to set LD_LIBRARY_PATH" - echo "Python location: $(UsePy.pythonLocation)" - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - displayName: 'Set LD_LIBRARY_PATH' - + # - bash: | + # echo "Checking the python version in use to set LD_LIBRARY_PATH" + # echo "Python location: $(UsePy.pythonLocation)" + # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" + # displayName: 'Set LD_LIBRARY_PATH' - ${{ each cache_step in parameters.cache_steps }}: - ${{ cache_step }} diff --git a/azure/common/deploy-stage.yml b/azure/common/deploy-stage.yml index 1c3746a50..4951b60c3 100644 --- a/azure/common/deploy-stage.yml +++ b/azure/common/deploy-stage.yml @@ -125,24 +125,28 @@ stages: aws_account: "${{ parameters.aws_account }}" - bash: | - PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \ - | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \ - | sort -V | tail -n 1 | cut -d- -f1) + pyversion="3.13" + PATCH=$(curl -fsSL https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json \ + | jq -r --arg pyversion "$pyversion" '[ .[] | select(.stable == true) | .version + | select(test("^" + ($pyversion|gsub("\\.";"\\.")) + "\\.\\d+$")) | split(".") | map(tonumber)] | max | join(".")') + + echo "##vso[task.setvariable variable=PY_VER]$PATCH" echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" - displayName: 'Query and set python tool cache path' + displayName: 'Query and set python tool cache path' + - task: UsePythonVersion@0 name: UsePy - displayName: "Use Python 3.13" + displayName: 'Use Python 3.13' inputs: - versionSpec: 3.13 + versionSpec: '$(PY_VER)' - - bash: | - echo "Checking the python version in use to set LD_LIBRARY_PATH" - echo "Python location: $(UsePy.pythonLocation)" - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - displayName: 'Set LD_LIBRARY_PATH' + # - bash: | + # echo "Checking the python version in use to set LD_LIBRARY_PATH" + # echo "Python location: $(UsePy.pythonLocation)" + # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" + # displayName: 'Set LD_LIBRARY_PATH' - template: "../components/set-facts.yml" parameters: diff --git a/azure/deploy-ecs-proxies-retag.yml b/azure/deploy-ecs-proxies-retag.yml new file mode 100644 index 000000000..02679c720 --- /dev/null +++ b/azure/deploy-ecs-proxies-retag.yml @@ -0,0 +1,22 @@ +parameters: + - name: 'container_vars' + type: string + default: 'ecs-proxies-containers.yml' + + - name: 'env_vars_dir' + type: string + default: './' + + - name: 'utils_dir' + type: string + default: 'utils' + +steps: + - bash: | + set -e + source "${{ parameters.env_vars_dir }}/.build_env_vars" + + export CONTAINER_VARS_FILE="$SERVICE_DIR/ecs-proxies-containers.yml + + make --no-print-directory -C ${{ parameters.utils_dir }}/ansible deploy-ecs-proxies-retag + displayName: "Retag ECS proxies" diff --git a/azure/templates/deploy-service.yml b/azure/templates/deploy-service.yml index a67b52457..3d4474374 100644 --- a/azure/templates/deploy-service.yml +++ b/azure/templates/deploy-service.yml @@ -191,12 +191,43 @@ steps: displayName: Create ECS Prerequisites condition: and(succeeded(), eq(variables['deploy_containers'], 'true')) + - bash: | + echo "##vso[task.setvariable variable=SERVICE_BASE_PATH_FOR_CONDITION]${{ parameters.service_base_path }}" + displayName: Set SERVICE_BASE_PATH_FOR_CONDITION + + + - bash: | + set -e + + export CONTAINER_VARS_FILE="$SERVICE_DIR/ecs-proxies-containers.yml" + + echo "DEBUG: CONTAINER_VARS_FILE='${CONTAINER_VARS_FILE}'" + echo "DEBUG (evaluated old path): $BUILD_SOURCESDIRECTORY/${{ parameters.service_name }}/${{ parameters.service_name }}-$(BRANCH_NAME)+$(Build.BuildID)/ecs-proxies-containers.yml" + + source $(SERVICE_DIR)/.build_env_vars + export SERVICE_BASE_PATH="${{ parameters.service_base_path }}" + echo "DEBUG: SERVICE_BASE_PATH='${SERVICE_BASE_PATH}'" + + export ASSUMED_VERSION=$(echo $SERVICE_ARTIFACT_NAME | grep -E -o "v[0-9]+\.[0-9]+\.[0-9]+-?[a-z]*" || true | tail -1) + export DEPLOYED_VERSION=${ASSUMED_VERSION:-${{ parameters.fully_qualified_service_name }}} + + account=${{ parameters.aws_account }} \ + CONTAINER_VARS_FILE="${CONTAINER_VARS_FILE}" \ + SOURCE_COMMIT_ID="$(Build.SourceVersion)" \ + RELEASE_RELEASEID="$(Build.BuildId)" \ + make --no-print-directory -C $(UTILS_DIR)/ansible deploy-ecs-proxies-retag + + displayName: Retag ECS image + condition: and(succeeded(), ne(variables['DEPLOY_ROLE'], ''), not(contains(variables['SERVICE_BASE_PATH_FOR_CONDITION'], '-pr-'))) + + + - template: ../components/aws-assume-role.yml parameters: role: "$(DEPLOY_ROLE)" profile: "$(DEPLOY_ROLE)" aws_account: "${{ parameters.aws_account }}" - + - bash: | set -e proxy_vars_file="$(PROXY_VARS_FILE)" @@ -209,6 +240,18 @@ steps: else export DEPLOYED_VERSION="${{ parameters.fully_qualified_service_name }}" fi + + + if [[ "$SERVICE_BASE_PATH_FOR_CONDITION" == *"-pr-"* ]]; then + export TF_VAR_use_ecs_tag=false + else + export TF_VAR_use_ecs_tag=true + fi + + echo "DEBUG: TF_VAR_use_ecs_tag='${TF_VAR_use_ecs_tag}'" + echo "DEBUG: SERVICE_BASE_PATH='${SERVICE_BASE_PATH}'" + + export use_ecs_tag=$TF_VAR_use_ecs_tag account=${{ parameters.aws_account }} \ PROXY_VARS_FILE="${proxy_vars_file}" \ @@ -216,6 +259,7 @@ steps: RELEASE_RELEASEID="$(Build.BuildId)" \ SERVICE_BASE_PATH=${{ parameters.service_base_path }} \ APIGEE_ENVIRONMENT=${{ parameters.apigee_environment }} \ + use_ecs_tag="${use_ecs_tag}" \ make --no-print-directory -C $(UTILS_DIR)/ansible deploy-ecs-proxies displayName: Deploy ECS proxies @@ -348,6 +392,7 @@ steps: displayName: Disable _status monitoring condition: and(succeeded(), eq(variables['check_and_enable_status'], 'false'), eq(variables['is_pull_request'], 'false')) + - bash: | set -euo pipefail diff --git a/azure/utils-pr-pipeline.yml b/azure/utils-pr-pipeline.yml index 5996b767a..7fff0c5f5 100644 --- a/azure/utils-pr-pipeline.yml +++ b/azure/utils-pr-pipeline.yml @@ -29,31 +29,29 @@ jobs: workspace: clean: all steps: - # - bash: | - # echo "Setting python tool cache path" - # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]:/agent/_work/_tool/Python/3.13.7/x64/lib/" - # displayName: 'Set python tool cache path' - - bash: | - PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \ - | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \ - | sort -V | tail -n 1 | cut -d- -f1) + pyversion="3.13" + PATCH=$(curl -fsSL https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json \ + | jq -r --arg pyversion "$pyversion" '[ .[] | select(.stable == true) | .version + | select(test("^" + ($pyversion|gsub("\\.";"\\.")) + "\\.\\d+$")) | split(".") | map(tonumber)] | max | join(".")') + + echo "##vso[task.setvariable variable=PY_VER]$PATCH" echo "Resolved latest python version: $PATCH" echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/" - displayName: 'Query and set python tool cache path' + displayName: 'Query and set python tool cache path' - task: UsePythonVersion@0 name: UsePy displayName: 'Use Python 3.13' inputs: - versionSpec: 3.13 + versionSpec: '$(PY_VER)' - - bash: | - echo "Checking the python version in use to set LD_LIBRARY_PATH" - echo "Python location: $(UsePy.pythonLocation)" - echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" - displayName: 'Set LD_LIBRARY_PATH' + # - bash: | + # echo "Checking the python version in use to set LD_LIBRARY_PATH" + # echo "Python location: $(UsePy.pythonLocation)" + # echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib" + # displayName: 'Set LD_LIBRARY_PATH' - bash: | instance_id="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)" diff --git a/poetry.lock b/poetry.lock index 1097d45e6..9c8b6532b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1009,14 +1009,14 @@ testing = ["coverage", "pytest", "pytest-benchmark"] [[package]] name = "pyasn1" -version = "0.6.1" +version = "0.6.2" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "pyasn1-0.6.1-py3-none-any.whl", hash = "sha256:0d632f46f2ba09143da3a8afe9e33fb6f92fa2320ab7e886e2d0f7672af84629"}, - {file = "pyasn1-0.6.1.tar.gz", hash = "sha256:6f580d2bdd84365380830acf45550f2511469f673cb4a5ae3857a3170128b034"}, + {file = "pyasn1-0.6.2-py3-none-any.whl", hash = "sha256:1eb26d860996a18e9b6ed05e7aae0e9fc21619fcee6af91cca9bad4fbea224bf"}, + {file = "pyasn1-0.6.2.tar.gz", hash = "sha256:9b59a2b25ba7e4f8197db7686c09fb33e658b98339fadb826e9512629017833b"}, ] [[package]] @@ -1421,14 +1421,14 @@ files = [ [[package]] name = "urllib3" -version = "2.6.0" +version = "2.6.3" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" groups = ["main"] files = [ - {file = "urllib3-2.6.0-py3-none-any.whl", hash = "sha256:c90f7a39f716c572c4e3e58509581ebd83f9b59cced005b7db7ad2d22b0db99f"}, - {file = "urllib3-2.6.0.tar.gz", hash = "sha256:cb9bcef5a4b345d5da5d145dc3e30834f58e8018828cbc724d30b4cb7d4d49f1"}, + {file = "urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4"}, + {file = "urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed"}, ] [package.extras] @@ -1440,4 +1440,4 @@ zstd = ["backports-zstd (>=1.0.0) ; python_version < \"3.14\""] [metadata] lock-version = "2.1" python-versions = "^3.13" -content-hash = "3c42aeaa2140b74054ef175e76ffdd49ae48d3d0e72d4651ad1961b58df72ad4" +content-hash = "8790b640d2ff2c972d93c012fd1c84621ef938fca3730d310246a3b5a282e6bf" diff --git a/pyproject.toml b/pyproject.toml index f9f8cbe4b..891782983 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -27,7 +27,7 @@ pytest-xdist = "^3.0" lxml = "^5.0" boto3 = "^1.34.0" ansible-core = "^2.19.4" -urllib3 = "^2.6.0" +urllib3 = "^2.6.3" cryptography = "44.0.1" click="^8.1.2" diff --git a/scripts/config/gitleaks.toml b/scripts/config/gitleaks.toml new file mode 100644 index 000000000..7388a6b6f --- /dev/null +++ b/scripts/config/gitleaks.toml @@ -0,0 +1,19 @@ +# SEE: https://github.com/gitleaks/gitleaks/#configuration + +[extend] +useDefault = true # SEE: https://github.com/gitleaks/gitleaks/blob/master/config/gitleaks.toml + +[[rules]] +description = "IPv4" +id = "ipv4" +regex = '''[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}''' + +[rules.allowlist] +regexTarget = "match" +regexes = [ + # Exclude the private network IPv4 addresses as well as the DNS servers for Google and OpenDNS + '''(127\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|10\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}|172\.(1[6-9]|2[0-9]|3[0-1])\.[0-9]{1,3}\.[0-9]{1,3}|192\.168\.[0-9]{1,3}\.[0-9]{1,3}|0\.0\.0\.0|255\.255\.255\.255|8\.8\.8\.8|8\.8\.4\.4|208\.67\.222\.222|208\.67\.220\.220)''', +] + +[allowlist] +paths = ['''.terraform.lock.hcl''', '''poetry.lock''', '''yarn.lock'''] \ No newline at end of file diff --git a/scripts/config/pre-commit.yaml b/scripts/config/pre-commit.yaml new file mode 100644 index 000000000..de8831c79 --- /dev/null +++ b/scripts/config/pre-commit.yaml @@ -0,0 +1,40 @@ +repos: +- repo: local + hooks: + - id: scan-secrets + name: Scan secrets + entry: ./scripts/githooks/scan-secrets.sh + args: ["check=staged-changes"] + language: script + pass_filenames: false +- repo: local + hooks: + - id: check-file-format + name: Check file format + entry: ./scripts/githooks/check-file-format.sh + args: ["check=staged-changes"] + language: script + pass_filenames: false +- repo: local + hooks: + - id: check-markdown-format + name: Check Markdown format + entry: ./scripts/githooks/check-markdown-format.sh + args: ["check=staged-changes"] + language: script + pass_filenames: false +- repo: local + hooks: + - id: check-english-usage + name: Check English usage + entry: ./scripts/githooks/check-english-usage.sh + args: ["check=staged-changes"] + language: script + pass_filenames: false +- repo: local + hooks: + - id: lint-terraform + name: Lint Terraform + entry: ./scripts/githooks/check-terraform-format.sh + language: script + pass_filenames: false \ No newline at end of file diff --git a/scripts/githooks/scan-secrets.sh b/scripts/githooks/scan-secrets.sh new file mode 100644 index 000000000..be4b7914b --- /dev/null +++ b/scripts/githooks/scan-secrets.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# WARNING: Please, DO NOT edit this file! It is maintained in the Repository Template (https://github.com/nhs-england-tools/repository-template). Raise a PR instead. + +set -euo pipefail + +# Pre-commit git hook to scan for secrets hard-coded in the codebase. This is a +# gitleaks command wrapper. It will run gitleaks natively if it is installed, +# otherwise it will run it in a Docker container. +# +# Usage: +# $ [options] ./scan-secrets.sh +# +# Options: +# check={whole-history,last-commit,staged-changes} # Type of the check to run, default is 'staged-changes' +# FORCE_USE_DOCKER=true # If set to true the command is run in a Docker container, default is 'false' +# VERBOSE=true # Show all the executed commands, default is 'false' +# +# Exit codes: +# 0 - No leaks present +# 1 - Leaks or error encountered +# 126 - Unknown flag + +# ============================================================================== + +function main() { + + cd "$(git rev-parse --show-toplevel)" + + if command -v gitleaks > /dev/null 2>&1 && ! is-arg-true "${FORCE_USE_DOCKER:-false}"; then + dir="$PWD" + cmd="$(get-cmd-to-run)" run-gitleaks-natively + else + dir="/workdir" + cmd="$(get-cmd-to-run)" run-gitleaks-in-docker + fi +} + +# Get Gitleaks command to execute and configuration. +# Arguments (provided as environment variables): +# dir=[project's top-level directory] +function get-cmd-to-run() { + + check=${check:-staged-changes} + case $check in + "whole-history") + cmd="detect --source $dir --verbose --redact" + ;; + "last-commit") + cmd="detect --source $dir --verbose --redact --log-opts -1" + ;; + "staged-changes") + cmd="protect --source $dir --verbose --staged" + ;; + esac + # Include base line file if it exists + if [ -f "$dir/scripts/config/.gitleaks-baseline.json" ]; then + cmd="$cmd --baseline-path $dir/scripts/config/.gitleaks-baseline.json" + fi + # Include the config file + cmd="$cmd --config $dir/scripts/config/gitleaks.toml" + + echo "$cmd" +} + +# Run Gitleaks natively. +# Arguments (provided as environment variables): +# cmd=[command to run] +function run-gitleaks-natively() { + + # shellcheck disable=SC2086 + gitleaks $cmd +} + +# Run Gitleaks in a Docker container. +# Arguments (provided as environment variables): +# cmd=[command to run] +# dir=[directory to mount as a volume] +function run-gitleaks-in-docker() { + + # shellcheck disable=SC1091 + source ./scripts/docker/docker.lib.sh + + # shellcheck disable=SC2155 + local image=$(name=ghcr.io/gitleaks/gitleaks docker-get-image-version-and-pull) + # shellcheck disable=SC2086 + docker run --rm --platform linux/amd64 \ + --volume "$PWD:$dir" \ + --workdir $dir \ + "$image" \ + $cmd +} + +# ============================================================================== + +function is-arg-true() { + + if [[ "$1" =~ ^(true|yes|y|on|1|TRUE|YES|Y|ON)$ ]]; then + return 0 + else + return 1 + fi +} + +# ============================================================================== + +is-arg-true "${VERBOSE:-false}" && set -x + +main "$@" + +exit 0 \ No newline at end of file