Add the steps to reboot the computes after update.

roles/update/README.md

@@ -16,5 +16,7 @@ Role to run update
 * `cifmw_update_ctl_plane_max_fail`: (Integer) For continuous control plane testing, maximum number of failures allowed. Default to 3.
 * `cifmw_update_ctl_plane_max_tries`: (Integer) For continuous control plane testing, number of retries allowed to stop and destroy the last vm created. Each retry is 5 seconds apart. Default to 84, so 7 minutes.
 * `cifmw_update_openstackclient_pod_timeout`: (Integer) Maximum number of seconds to wait for the openstackclient Pod to be available during control plane testing, as it is being restarted during update.  Default to `10` seconds.
-
+* `cifmw_update_reboot_test`: (Boolean) Activate the reboot test after update. Default to `False`.
+* `cifmw_update_ansible_ssh_private_key_file`: (String) Define the path to the private key file used for the compute nodes.
+* `cifmw_update_wait_retries_reboot`: (Integer) Number of retries to wait for a compute node reboot. One retry is done every five seconds. Default to 60, so five minutes.
 ## Examples

roles/update/defaults/main.yml

@@ -37,7 +37,15 @@ cifmw_update_timestamper_cmd: >-
 cifmw_update_ping_start_script: "{{ cifmw_update_artifacts_basedir }}/l3_agent_start_ping.sh"
 cifmw_update_ping_stop_script: "{{ cifmw_update_artifacts_basedir }}/l3_agent_stop_ping.sh"
 
+# Operation in the openstack namespace
+cifmw_update_openstack_cmd: >-
+  oc rsh -n {{ cifmw_update_namespace }} openstackclient openstack
+
 ## User facing
+cifmw_update_reboot_test: false
+cifmw_update_ansible_ssh_private_key_file: >-
+  "{{ ansible_ssh_private_key_file | default(ansible_user_dir ~ '/.ssh/id_cifw') }}"
+cifmw_update_wait_retries_reboot: 60
 
 cifmw_update_ping_test: false
 cifmw_update_create_volume: false

roles/update/molecule/default/prepare.yml

@@ -28,7 +28,8 @@
     - role: ci_setup
     - role: install_yamls
   tasks:
-    - name: Set custom cifmw PATH reusable fact
+    - name: Set custom some reusable facts
       ansible.builtin.set_fact:
         cifmw_path: "{{ ansible_user_dir }}/.crc/bin:{{ ansible_user_dir }}/.crc/bin/oc:{{ ansible_user_dir }}/bin:{{ ansible_env.PATH }}"
+        cifmw_update_reboot_test: false
         cacheable: true

roles/update/tasks/main.yml

@@ -74,3 +74,8 @@
     - not cifmw_update_run_dryrun | bool
   ansible.builtin.command: |
     {{ cifmw_update_artifacts_basedir }}/control_plane_test_stop.sh
+
+- name: Reboot the compute nodes
+  ansible.builtin.include_tasks: reboot_computes.yml
+  when:
+    - cifmw_update_reboot_test | bool

roles/update/tasks/reboot_computes.yml

@@ -0,0 +1,71 @@
+- name: Register storage backend type
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_openstack_cmd }} volume service list -f json |
+      jq -r -c '.[] | select(.Binary | contains("cinder-volume")) | .Host'
+  register: storage_backend
+  changed_when: false
+
+- name: Get the list of OpenStack hypervisors
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: |
+    {{ cifmw_update_openstack_cmd }} hypervisor list -f json
+  register: hypervisor_list
+  changed_when: false
+
+- name: Parse the hypervisor list to extract hostnames
+  ansible.builtin.set_fact:
+    hypervisor_hostnames: "{{ hypervisor_list.stdout | from_json | map(attribute='Hypervisor Hostname') | list }}"
+
+- name: Create a reboot monitor servers script
+  ansible.builtin.template:
+    src: "monitor_servers.sh.j2"
+    dest: "{{ cifmw_update_artifacts_basedir }}/monitor_servers.sh"
+    mode: "0775"
+
+- name: Start the monitor servers script
+  ansible.builtin.shell: |
+    nohup {{ cifmw_update_artifacts_basedir }}/monitor_servers.sh &> /dev/null &
+    echo $!
+  register: monitor_servers_job
+
+- name: Create a monitor placement monitor script
+  ansible.builtin.template:
+    src: "monitor_vm_placement.sh.j2"
+    dest: "{{ cifmw_update_artifacts_basedir }}/monitor_vm_placement.sh"
+    mode: "0775"
+
+- name: Start the monitor placement script
+  ansible.builtin.shell: |
+    nohup {{ cifmw_update_artifacts_basedir }}/monitor_vm_placement.sh &> /dev/null &
+    echo $!
+  register: monitor_placement_job
+
+- name: Iterate over each hypervisor for the reboot sequence
+  ansible.builtin.include_tasks: reboot_hypervisor.yml
+  loop: "{{ hypervisor_hostnames }}"
+  loop_control:
+    loop_var: hypervisor
+
+- name: Stop the monitor servers script if running
+  ansible.builtin.shell: |
+    if kill -0 {{ monitor_servers_job.stdout }} &>/dev/null; then
+      kill {{ monitor_servers_job.stdout }}
+    fi
+  register: kill_result
+  failed_when: kill_result.rc not in [0, 1] # We can still have a race
+                                            # between kill -0 and
+                                            # kill, even if unlikely.
+
+- name: Stop the monitor placement script if running
+  ansible.builtin.shell: |
+    if kill -0 {{ monitor_placement_job.stdout }} &>/dev/null; then
+      kill {{ monitor_placement_job.stdout }}
+    fi
+  register: kill_result
+  failed_when: kill_result.rc not in [0, 1]

roles/update/tasks/reboot_hypervisor.yml

@@ -0,0 +1,113 @@
+---
+- name: Extract short hostname from FQDN
+  ansible.builtin.set_fact:
+    cifmw_update_hypervisor_short_name: "{{ hypervisor.split('.')[0] }}"
+
+- name: Display current stage
+  ansible.builtin.debug:
+    msg: "Rebooting {{ cifmw_update_hypervisor_short_name }}"
+
+- name: Define command for nova interaction
+  ansible.builtin.set_fact:
+    cifmw_update_bash_cmd: >-
+      oc rsh -n {{ cifmw_update_namespace }} openstackclient bash -c
+
+- name: Check active VMs on hypervisor
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_openstack_cmd }} server list --all --host {{ hypervisor }} -f json
+    | jq -r -c '.[] | select(.Status | contains("ACTIVE") or contains("PAUSED")) | .ID'
+  register: active_vms
+  changed_when: false
+
+- name: Evacuate VMs if they are running
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    {{ cifmw_update_bash_cmd }} ". cloudrc &&
+    nova host-evacuate-live
+    {% if 'ceph' not in storage_backend.stdout %}
+    --block-migrate
+    {% endif %}
+    {{ hypervisor }}"
+  when:
+    - active_vms.stdout != ''
+    - hypervisor_hostnames | length > 1
+  changed_when: true
+
+- name: Wait for compute node to get quiesced
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_openstack_cmd }} server list --all --host {{ hypervisor }} -f json
+    | jq -r -c '[.[] | select(.Status |
+    contains("ACTIVE") or contains("PAUSED") or contains("MIGRATING"))]
+    | length'
+  register: compute_node_instances
+  until: compute_node_instances.stdout.find("0") > -1
+  retries: 30
+  delay: 5
+  when:
+    - active_vms.stdout != ''
+    - hypervisor_hostnames | length > 1
+
+- name: Reboot the hypervisors using CR
+  ansible.builtin.include_tasks: reboot_hypervisor_using_cr.yml
+
+- name: Perform sanity checks post-reboot
+  ansible.builtin.include_tasks: reboot_hypervisor_sanity_checks.yml
+  vars:
+    current_hypervisor: "{{ hypervisor }}"
+
+- name: Display current stage
+  ansible.builtin.debug:
+    msg: "Migrate back {{ item }} to {{ cifmw_update_hypervisor_short_name }}."
+  with_items: "{{ active_vms.stdout_lines }}"
+
+- name: Migrate back VMs post-reboot
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_bash_cmd }} ". cloudrc &&
+    nova live-migration
+    {% if 'ceph' not in storage_backend.stdout %}
+    --block-migrate
+    {% endif %}
+    {{ item }} {{ hypervisor }}";
+    {{ cifmw_update_openstack_cmd }} server show {{ item }} -f json |
+    jq -r -c '. | .["OS-EXT-SRV-ATTR:host"]'
+  register: instance_migration_result
+  until: instance_migration_result.stdout.find(hypervisor) > -1
+  retries: 30
+  delay: 5
+  with_items: "{{ active_vms.stdout_lines }}"
+  when:
+    - active_vms.stdout != ''
+    - hypervisor_hostnames | length > 1
+
+- name: Start VMs post-reboot when only one hypervisor is present
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_openstack_cmd }} server start {{ item }};
+    sleep 5;
+    {{ cifmw_update_openstack_cmd }} server show {{ item }} -f json
+    | jq -r .status
+  register: instance_migration_result
+  until: instance_migration_result.stdout == "ACTIVE"
+  retries: 30
+  delay: 1                      # We already wait 5s in the shell.
+  with_items: "{{ active_vms.stdout_lines }}"
+  when:
+    - active_vms.stdout != ''
+    - hypervisor_hostnames | length == 1

roles/update/tasks/reboot_hypervisor_sanity_checks.yml

@@ -0,0 +1,50 @@
+---
+- name: Display current stage
+  ansible.builtin.debug:
+    msg: |
+      Testing the status of the services for {{ current_hypervisor }} after reboot.
+
+- name: Verify nova-compute services
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_openstack_cmd }} compute service list
+    --host {{ current_hypervisor }} -f json
+    | jq -r -c '.[]
+    | select(.Binary | contains("nova-compute")) | .State'
+  register: nova_compute_status
+  until: nova_compute_status.stdout == 'up'
+  retries: 30
+  delay: 5
+
+- name: Verify ovn-controller services
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_openstack_cmd }} network agent list
+    --host {{ current_hypervisor }} -f json
+    | jq -r -c '.[]
+    | select(.Binary | contains("ovn-controller")) | .Alive'
+  register: ovn_controller_status
+  until: ovn_controller_status.stdout == 'true'
+  retries: 30
+  delay: 5
+
+- name: Verify networking-ovn-metadata-agent
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    {{ cifmw_update_openstack_cmd }} network agent list
+    --host {{ current_hypervisor }} -f json
+    | jq -r -c '.[]
+    | select(.Binary | contains("neutron-ovn-metadata-agent")) | .Alive'
+  register: networking_ovn_metadata_status
+  until: networking_ovn_metadata_status.stdout == 'true'
+  retries: 30
+  delay: 5

roles/update/tasks/reboot_hypervisor_using_cr.yml

@@ -0,0 +1,62 @@
+---
+- name: Fetch NodeSets for the Reboot OpenStackDataPlaneDeployment
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.shell: >-
+    set -o pipefail;
+    oc -n {{ cifmw_update_namespace }}
+    get openstackdataplanenodeset -o name
+    | awk -F'/' '{print $2}'
+  register: cifmw_update_node_sets
+  changed_when: false
+
+- name: Construct the Reboot CR name
+  ansible.builtin.set_fact:
+    cifmw_reboot_dep_name: >-
+      {{
+      'reboot-' ~ cifmw_update_hypervisor_short_name ~ '-' ~
+      lookup('pipe', 'date +%Y%m%d%H%S')
+      }}
+
+- name: Create the OpenStackDataPlaneDeployment CR used for reboot
+  ansible.builtin.copy:
+    dest: "{{ cifmw_update_artifacts_basedir }}/{{ cifmw_reboot_dep_name }}.yaml"
+    content: "{{ _content | to_nice_yaml }}"
+  vars:
+    _content:
+      apiVersion: dataplane.openstack.org/v1beta1
+      kind: OpenStackDataPlaneDeployment
+      metadata:
+        name: "{{ cifmw_reboot_dep_name }}"
+        namespace: "{{ cifmw_update_namespace }}"
+      spec:
+        nodeSets: "{{ cifmw_update_node_sets.stdout
+        | split('\n')
+        | map('trim')
+        | reject('equalto', '')
+        | list
+        }}"
+        servicesOverride:
+          - reboot-os
+        ansibleExtraVars:
+          edpm_reboot_strategy: force
+        ansibleLimit: "{{ cifmw_update_hypervisor_short_name }}"
+
+- name: Create the OpenStackDataPlaneDeployment CR to trigger a reboot
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.command: >-
+    oc -n {{ cifmw_update_namespace }}
+    create -f {{ cifmw_update_artifacts_basedir }}/{{ cifmw_reboot_dep_name }}.yaml
+
+- name: Wait for the reboot to finish
+  environment:
+    KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
+    PATH: "{{ cifmw_path | default(ansible_env.PATH) }}"
+  ansible.builtin.command: >-
+    oc -n {{ cifmw_update_namespace }}
+    wait --for=condition=Ready
+    openstackdataplanedeployment/{{ cifmw_reboot_dep_name }}
+    --timeout={{ (cifmw_update_wait_retries_reboot | int * 5) }}s