Merge pull request #531 from NHSDigital/APIM-6679-Python-Update

Updated the python and dependencies to latest stable versions to support python 3.13

Author: sathiya-nhs

.github/workflows/publish.yml

@@ -12,10 +12,10 @@ jobs:
               with:
                   fetch-depth: 0  # This causes all history to be fetched, which is required for calculate-version to function
 
-            - name: Install Python 3.8
+            - name: Install Python 3.13
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: 3.13
 
             - name: Install poetry
               run: pip install poetry

README.md

@@ -1,6 +1,25 @@
 # api-management-utils
 Scripts and utilities used across API managment platform and services 
 
+
+################################################################################
+###########               Python upgrade to 3.13                     ###########
+###########       Utils Repo has been updated to python 3.13         ###########
+################################################################################
+
+########### We are continuing to support python 3.8/9(which are currently out of support) until January 26th 2026 ###########
+###########          After the deadline your pipelines will fail if you are using python version 3.8/9            ###########
+
+Python upgrade related changes
+###############################
+Projects using Python versions older than 3.13 and extending their pipeline with the utils repository must update their pipelines to ensure compatibility with the latest changes.
+For detailed guidance, please refer to the APIM FAQ page:
+https://nhsd-confluence.digital.nhs.uk/spaces/APM/pages/1226682275/Pipeline+Queries
+
+Note: Projects running Python version 3.13 or later do not need any pipeline modifications.
+
+
+
 ## Scripts
 * `template.py` - cli for basic jinja templating
 * `test_pull_request_deployments.py` - cli for testing utils against other repositories

ansible/collections/ansible_collections/nhsd/apigee/Makefile

@@ -4,10 +4,10 @@ update-schema:
 	@PYTHONPATH=../../../ poetry run python scripts/update_schema.py
 
 unit-test:
-	@poetry run ansible-test units --python=3.8
+	@poetry run ansible-test units --python=3.13
 
 integration-test:
-	@poetry run ansible-test integration --python=3.8
+	@poetry run ansible-test integration --python=3.13
 
 test: unit-test integration-test
 

ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/utils.py

@@ -5,6 +5,7 @@
 import typing
 
 from ansible_collections.nhsd.apigee.plugins.module_utils import constants
+from ansible.utils.unsafe_proxy import AnsibleUnsafeText
 
 
 def exclude_keys(dict_, keys_to_ignore):
@@ -19,7 +20,7 @@ def delta(before, after, keys_to_ignore=None):
             before,
             after,
             ignore_order=True,
-            ignore_type_in_groups=[(ansible.utils.unsafe_proxy.AnsibleUnsafeText,str)],
+            ignore_type_in_groups=[(AnsibleUnsafeText,str)],
             exclude_paths=[f"root['{key}']" for key in keys_to_ignore],
         ).to_json()
     )

ansible/collections/ansible_collections/nhsd/apigee/roles/deploy_manifest/tasks/main.yml

@@ -54,7 +54,7 @@
   # since products can only be created if they reference proxies
   # that exist.  This safety check can be removed when the
   # manifest manages proxies too.
-  when: item.name | regex_search('^' + SERVICE_NAME + '-' + PULL_REQUEST | default(APIGEE_ENVIRONMENT))
+  when: (item.name is match('^' ~ SERVICE_NAME ~ '-' ~ (PULL_REQUEST | default(APIGEE_ENVIRONMENT))))
 
 # - name: deploy apigee specs
 #   nhsd.apigee.deploy_spec:

ansible/collections/ansible_collections/nhsd/apigee/tests/integration/targets/test_validate_manifest/tasks/main.yml

@@ -17,7 +17,6 @@
 - name: validate test manifest files
   include_role:
     name: validate_manifest
-    vars_from: "{{ DIST_DIR }}/vars.yml"
   loop: "{{ DIST_DIRS }}"
   loop_control:
     loop_var: DIST_DIR

ansible/deploy-apigee-proxy.yml

@@ -10,7 +10,7 @@
     APIGEE_ORGANIZATION: "{{ lookup('env', 'APIGEE_ORGANIZATION') }}"
     APIGEE_ACCESS_TOKEN: "{{ lookup('env', 'APIGEE_ACCESS_TOKEN') }}"
     PROXY_DIR: "{{ lookup('env', 'PROXY_DIR') }}"
-    PING: "{{ (lookup('env', 'PING') | length > 0) | ternary(lookup('env', 'PING'), 'true') }}"
+    PING: "{{ (lookup('env', 'PING') | default('true')) | bool }}"
 
   pre_tasks:
     - name: check FULLY_QUALIFIED_SERVICE_NAME

ansible/roles/build-ecs-proxies/tasks/build-container.yml

@@ -11,7 +11,7 @@
 
 - name: pull latest if exists - to ensure cache  {{ item }}
   ansible.builtin.command:
-    cmd: "docker pull {{ ecr_registry }}/{{ item }}:{{ describe_images.stdout }}"
+    cmd: "docker pull {{ ecr_registry }}/{{ service_id }}_{{ item }}:{{ describe_images.stdout }}"
   when: describe_images.stdout != 'None'
   ignore_errors: true
   no_log: true

ansible/roles/build-ecs-proxies/tasks/main.yml

@@ -35,5 +35,5 @@
   changed_when: no
 
 - name: build and push ecr images
-  include: build-container.yml
+  include_tasks: build-container.yml
   with_items: "{{ containers }}"

azure/build-prereqs.yml

@@ -4,9 +4,31 @@ parameters:
     default: 'utils'
 
 steps:
+  # - bash: |
+  #     echo "Setting python tool cache path"
+  #     echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]:/agent/_work/_tool/Python/3.13.7/x64/lib/"
+  #   displayName: 'Set python tool cache path'
+  - bash: |
+      PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \
+        | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \
+        | sort -V | tail -n 1 | cut -d- -f1)
+      echo "Resolved latest python version: $PATCH"
+      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/"
+    displayName: 'Query and set python tool cache path'  
+
+
   - task: UsePythonVersion@0
+    name: UsePy
+    displayName: 'Use Python 3.13'
     inputs:
-      versionSpec: '3.8'
+      versionSpec: '3.13'
+
+  - bash: |
+      echo "Checking the python version in use to set LD_LIBRARY_PATH"
+      echo "Python location: $(UsePy.pythonLocation)"
+      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib"
+    displayName: 'Set LD_LIBRARY_PATH'
+  
 
   - bash: |
       tfenv use 0.14.6