Updated the python and dependencies to latest stable versions to supp…

.github/workflows/publish.yml

@@ -12,10 +12,10 @@ jobs:
               with:
                   fetch-depth: 0  # This causes all history to be fetched, which is required for calculate-version to function
 
-            - name: Install Python 3.8
+            - name: Install Python 3.13
               uses: actions/setup-python@v5
               with:
-                  python-version: 3.8
+                  python-version: 3.13
 
             - name: Install poetry
               run: pip install poetry

README.md

@@ -1,6 +1,25 @@
 # api-management-utils
 Scripts and utilities used across API managment platform and services 
 
+
+################################################################################
+###########               Python upgrade to 3.13                     ###########
+###########       Utils Repo has been updated to python 3.13         ###########
+################################################################################
+
+########### We are continuing to support python 3.8/9(which are currently out of support) until January 26th 2026 ###########
+###########          After the deadline your pipelines will fail if you are using python version 3.8/9            ###########
+
+Python upgrade related changes
+###############################
+Projects using Python versions older than 3.13 and extending their pipeline with the utils repository must update their pipelines to ensure compatibility with the latest changes.
+For detailed guidance, please refer to the APIM FAQ page:
+https://nhsd-confluence.digital.nhs.uk/spaces/APM/pages/1226682275/Pipeline+Queries
+
+Note: Projects running Python version 3.13 or later do not need any pipeline modifications.
+
+
+
 ## Scripts
 * `template.py` - cli for basic jinja templating
 * `test_pull_request_deployments.py` - cli for testing utils against other repositories

ansible/collections/ansible_collections/nhsd/apigee/Makefile

@@ -4,10 +4,10 @@ update-schema:
 	@PYTHONPATH=../../../ poetry run python scripts/update_schema.py
 
 unit-test:
-	@poetry run ansible-test units --python=3.8
+	@poetry run ansible-test units --python=3.13
 
 integration-test:
-	@poetry run ansible-test integration --python=3.8
+	@poetry run ansible-test integration --python=3.13
 
 test: unit-test integration-test
 

ansible/collections/ansible_collections/nhsd/apigee/plugins/module_utils/utils.py

@@ -5,6 +5,7 @@
 import typing
 
 from ansible_collections.nhsd.apigee.plugins.module_utils import constants
+from ansible.utils.unsafe_proxy import AnsibleUnsafeText
 
 
 def exclude_keys(dict_, keys_to_ignore):
@@ -19,7 +20,7 @@ def delta(before, after, keys_to_ignore=None):
             before,
             after,
             ignore_order=True,
-            ignore_type_in_groups=[(ansible.utils.unsafe_proxy.AnsibleUnsafeText,str)],
+            ignore_type_in_groups=[(AnsibleUnsafeText,str)],
             exclude_paths=[f"root['{key}']" for key in keys_to_ignore],
         ).to_json()
     )

ansible/collections/ansible_collections/nhsd/apigee/roles/deploy_manifest/tasks/main.yml

@@ -54,7 +54,7 @@
   # since products can only be created if they reference proxies
   # that exist.  This safety check can be removed when the
   # manifest manages proxies too.
-  when: item.name | regex_search('^' + SERVICE_NAME + '-' + PULL_REQUEST | default(APIGEE_ENVIRONMENT))
+  when: (item.name is match('^' ~ SERVICE_NAME ~ '-' ~ (PULL_REQUEST | default(APIGEE_ENVIRONMENT))))
 
 # - name: deploy apigee specs
 #   nhsd.apigee.deploy_spec:

ansible/collections/ansible_collections/nhsd/apigee/tests/integration/targets/test_validate_manifest/tasks/main.yml

@@ -17,7 +17,6 @@
 - name: validate test manifest files
   include_role:
     name: validate_manifest
-    vars_from: "{{ DIST_DIR }}/vars.yml"
   loop: "{{ DIST_DIRS }}"
   loop_control:
     loop_var: DIST_DIR

ansible/deploy-apigee-proxy.yml

@@ -10,7 +10,7 @@
     APIGEE_ORGANIZATION: "{{ lookup('env', 'APIGEE_ORGANIZATION') }}"
     APIGEE_ACCESS_TOKEN: "{{ lookup('env', 'APIGEE_ACCESS_TOKEN') }}"
     PROXY_DIR: "{{ lookup('env', 'PROXY_DIR') }}"
-    PING: "{{ (lookup('env', 'PING') | length > 0) | ternary(lookup('env', 'PING'), 'true') }}"
+    PING: "{{ (lookup('env', 'PING') | default('true')) | bool }}"
 
   pre_tasks:
     - name: check FULLY_QUALIFIED_SERVICE_NAME

ansible/roles/build-ecs-proxies/tasks/build-container.yml

@@ -11,7 +11,7 @@
 
 - name: pull latest if exists - to ensure cache  {{ item }}
   ansible.builtin.command:
-    cmd: "docker pull {{ ecr_registry }}/{{ item }}:{{ describe_images.stdout }}"
+    cmd: "docker pull {{ ecr_registry }}/{{ service_id }}_{{ item }}:{{ describe_images.stdout }}"
   when: describe_images.stdout != 'None'
   ignore_errors: true
   no_log: true

ansible/roles/build-ecs-proxies/tasks/main.yml

@@ -35,5 +35,5 @@
   changed_when: no
 
 - name: build and push ecr images
-  include: build-container.yml
+  include_tasks: build-container.yml
   with_items: "{{ containers }}"

azure/build-prereqs.yml

@@ -4,9 +4,31 @@ parameters:
     default: 'utils'
 
 steps:
+  # - bash: |
+  #     echo "Setting python tool cache path"
+  #     echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]:/agent/_work/_tool/Python/3.13.7/x64/lib/"
+  #   displayName: 'Set python tool cache path'
+  - bash: |
+      PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \
+        | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \
+        | sort -V | tail -n 1 | cut -d- -f1)
+      echo "Resolved latest python version: $PATCH"
+      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/"
+    displayName: 'Query and set python tool cache path'  
+
+
   - task: UsePythonVersion@0
+    name: UsePy
+    displayName: 'Use Python 3.13'
     inputs:
-      versionSpec: '3.8'
+      versionSpec: '3.13'
+
+  - bash: |
+      echo "Checking the python version in use to set LD_LIBRARY_PATH"
+      echo "Python location: $(UsePy.pythonLocation)"
+      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib"
+    displayName: 'Set LD_LIBRARY_PATH'
+
 
   - bash: |
       tfenv use 0.14.6

azure/common/apigee-build.yml

@@ -29,7 +29,7 @@ parameters:
     default: []
   - name: python_version
     type: string
-    default: "3.8"
+    default: "3.13"
   - name: agent_pool
     type: string
     default: "AWS-ECS"
@@ -143,11 +143,33 @@ jobs:
         parameters:
           service_name: "${{ parameters.service_name }}"
 
+      # - bash: |
+      #     echo "Setting python tool cache path"
+      #     echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]:/agent/_work/_tool/Python/${{ parameters.python_version }}/x64/lib/"
+      #   displayName: 'Set python tool cache path'
+
+      - bash: |
+          PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \
+              | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \
+              | sort -V | tail -n 1 | cut -d- -f1)
+          echo "Resolved latest python version: $PATCH"
+          echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/"
+        displayName: 'Query and set python tool cache path'      
+
       - task: UsePythonVersion@0
+        name: UsePy
         displayName: "Use Python ${{ parameters.python_version }}"
         inputs:
           versionSpec: ${{ parameters.python_version }}
 
+
+      - bash: |
+          echo "Checking the python version in use to set LD_LIBRARY_PATH"
+          echo "Python location: $(UsePy.pythonLocation)"
+          echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib"
+        displayName: 'Set LD_LIBRARY_PATH'
+
+
       - ${{ each cache_step in parameters.cache_steps }}:
           - ${{ cache_step }}
 

azure/common/apigee-deployment.yml

@@ -150,7 +150,7 @@ parameters:
     default: 'DEPRECATED'
   - name: python_version
     type: string
-    default: "3.8"
+    default: "3.13"
   - name: agent_pool
     type: string
     default: "AWS-ECS"

azure/common/deploy-stage.yml

@@ -123,11 +123,26 @@ stages:
                     role: "auto-ops"
                     profile: "apm_${{ parameters.aws_account }}"
                     aws_account: "${{ parameters.aws_account }}"
+
+                - bash: |
+                    PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \
+                      | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \
+                      | sort -V | tail -n 1 | cut -d- -f1)
+                    echo "Resolved latest python version: $PATCH"
+                    echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/"
+                  displayName: 'Query and set python tool cache path' 
 
                 - task: UsePythonVersion@0
-                  displayName: "Use Python ${{ parameters.python_version }}"
+                  name: UsePy
+                  displayName: "Use Python 3.13"
                   inputs:
-                    versionSpec: ${{ parameters.python_version }}
+                    versionSpec: 3.13
+
+                - bash: |
+                    echo "Checking the python version in use to set LD_LIBRARY_PATH"
+                    echo "Python location: $(UsePy.pythonLocation)"
+                    echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib"
+                  displayName: 'Set LD_LIBRARY_PATH'
 
                 - template: "../components/set-facts.yml"
                   parameters:

azure/utils-pr-pipeline.yml

@@ -29,10 +29,31 @@ jobs:
     workspace:
       clean: all
     steps:
+      # - bash: |
+      #     echo "Setting python tool cache path"
+      #     echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]:/agent/_work/_tool/Python/3.13.7/x64/lib/"
+      #   displayName: 'Set python tool cache path'
+
+      - bash: |
+          PATCH=$(curl -s https://api.github.com/repos/actions/python-versions/releases \
+              | jq -r '[.[] | .tag_name | select(startswith("3.13"))] | .[]' \
+              | sort -V | tail -n 1 | cut -d- -f1)
+          echo "Resolved latest python version: $PATCH"
+          echo "##vso[task.setvariable variable=LD_LIBRARY_PATH;]/agent/_work/_tool/Python/${PATCH}/x64/lib/"
+        displayName: 'Query and set python tool cache path'
+
+
       - task: UsePythonVersion@0
-        displayName: 'Use Python 3.8'
+        name: UsePy
+        displayName: 'Use Python 3.13'
         inputs:
-          versionSpec: 3.8
+          versionSpec: 3.13
+
+      - bash: |
+          echo "Checking the python version in use to set LD_LIBRARY_PATH"
+          echo "Python location: $(UsePy.pythonLocation)"
+          echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(UsePy.pythonLocation)/lib"
+        displayName: 'Set LD_LIBRARY_PATH'
 
       - bash: |
           instance_id="$(curl -s http://169.254.169.254/latest/meta-data/instance-id)"
@@ -103,12 +124,12 @@ jobs:
         displayName: Install utils pre-requisites
 
       - bash: |
-          ANSIBLE_FORCE_COLOR=yes poetry run ansible-test units --python=3.8
+          ANSIBLE_FORCE_COLOR=yes poetry run ansible-test units --python=3.13
         workingDirectory: ansible/collections/ansible_collections/nhsd/apigee
         displayName: Unit test ansible
 
       - bash: |
-          ANSIBLE_FORCE_COLOR=yes poetry run ansible-test integration --python=3.8
+          ANSIBLE_FORCE_COLOR=yes poetry run ansible-test integration --python=3.13
         workingDirectory: ansible/collections/ansible_collections/nhsd/apigee
         displayName: Integration test ansible