feat(telco-kpi): add lock-free job preemption based on OCP version priority (#72894)

Problem:
Multiple Telco KPI Prow jobs compete for the same baremetal host. Lower OCP
version jobs can block higher version jobs for extended periods, delaying
critical testing for newer releases.

Solution:
Implement a lock-free preemption mechanism where higher OCP version jobs can
signal lower version jobs to quit, freeing the baremetal host sooner.

How it works:
1. WAITING PHASE: Each job creates a unique waiting file on the bastion BEFORE
   attempting to acquire the lock: <lock>.waiting.<nanosecond_timestamp>.<ocp_version>
   Example: spoke-baremetal-50-7c-6f-5c-47-8c.lock.waiting.1766568440841947242.4.22
   This ensures the job's presence is visible even if it immediately gets the lock.

2. LOCK ACQUISITION: When a job acquires the lock, it checks for higher priority
   waiters BEFORE removing its own waiting file (deferred deletion). If higher
   priority found, it releases the lock and keeps its waiting file for retry.
   Only when no higher priority is found does it remove its waiting file.

3. PERIODIC CHECKS: While holding the lock, the job periodically checks for
   higher priority waiters at key points:
   - cluster-install: every QUIT_CHECK_INTERVAL iterations (default: 3)
   - oslat test: before running tests
   - cpu-util test: before running tests

4. QUIT MODES:
   - 'graceful' (exit 0): Used by test steps. Allows remaining steps like
     PTP reporting to complete. Job exits cleanly.
   - 'force' (exit 1): Used by cluster-install. If installation is interrupted,
     remaining steps are meaningless. Job aborts immediately.

5. CLEANUP: Each job always removes its own waiting file during cleanup,
   regardless of whether it acquired the lock. This prevents orphaned files.

Priority logic:
- ONLY the OCP version determines priority (e.g., 4.22 > 4.20)
- The nanosecond timestamp is NOT used for priority decisions
- Same-version jobs (e.g., two 4.22 jobs) compete equally for the lock
- Timestamp is used solely for: (1) unique filenames, (2) self-cleanup

Key benefits:
- Lock-free: No shared mutable state, each job manages its own file
- Race-safe: Nanosecond timestamps ensure unique filenames
- Deferred deletion: Waiting file persists until validation passes
- Self-cleaning: Jobs clean up only their own files
- Configurable: QUIT_CHECK_INTERVAL controls check frequency

New shared functions:
- extract_ocp_version: Gets version from JOB_NAME
- create_waiting_request_file: Creates unique waiting file
- remove_own_waiting_file: Removes job's waiting file
- check_for_higher_priority_waiter: Scans waiting files for higher version
- should_quit: Determines if quit is needed
- check_for_quit: Main entry point (supports graceful/force modes)

Signed-off-by: Carlos Cardenosa <ccardeno@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>

Author: ccardenosa

ci-operator/step-registry/telcov10n/metal-single-node-spoke-kpis/hacks/clean-up/telcov10n-metal-single-node-spoke-kpis-hacks-clean-up-commands.sh

@@ -15,16 +15,11 @@ function release_locked_host {
   network_spoke_mac_address=$(cat $SHARED_DIR/hosts.yaml|grep 'mac:'|awk -F'mac:' '{print $2}'|tr -d '[:blank:]')
   local spoke_lock_filename="/var/run/lock/ztp-baremetal-pool/spoke-baremetal-${network_spoke_mac_address//:/-}.lock"
 
-  echo "************ telcov10n Releasing Lock for the host used by this Spoke cluster deployemnt ************"
+  echo "************ telcov10n Releasing Lock for the host used by this Spoke cluster deployment ************"
 
   set -x
-  timeout -s 9 10m ssh "${SSHOPTS[@]}" "root@${AUX_HOST}" bash -s --  \
-    "${spoke_lock_filename}" << 'EOF'
-set -o nounset
-set -o errexit
-set -o pipefail
-sudo rm -fv ${1}
-EOF
+  timeout -s 9 10m ssh "${SSHOPTS[@]}" "root@${AUX_HOST}" \
+    "sudo rm -fv ${spoke_lock_filename} && echo 'Lock released successfully.'"
   set +x
 }
 
@@ -48,17 +43,44 @@ function server_poweroff {
 
 }
 
+function cleanup_waiting_file {
+  # Always clean up our waiting file, regardless of lock status
+
+  if [ ! -f "${SHARED_DIR}/own_waiting_file.txt" ]; then
+    echo "[INFO] No waiting file to clean up."
+    return 0
+  fi
+
+  local own_waiting_file
+  own_waiting_file=$(cat "${SHARED_DIR}/own_waiting_file.txt")
+
+  if [ -z "${own_waiting_file}" ]; then
+    echo "[INFO] Waiting file path is empty, nothing to clean up."
+    return 0
+  fi
+
+  echo "************ telcov10n Cleaning up waiting file ************"
+  echo "[INFO] Removing waiting file: ${own_waiting_file}"
+
+  timeout -s 9 2m ssh "${SSHOPTS[@]}" "root@${AUX_HOST}" \
+    "rm -fv ${own_waiting_file} 2>/dev/null || true"
+}
+
 function main {
 
+  # Setup SSH access once for all operations
+  setup_aux_host_ssh_access
+
+  # Always clean up our waiting file first (handles both timeout and normal cases)
+  cleanup_waiting_file
+
+  # Only do lock-related cleanup if we hold the lock
   local does_the_current_job_hold_a_lock_to_use_a_baremetal_server
   does_the_current_job_hold_a_lock_to_use_a_baremetal_server=$( \
     cat ${SHARED_DIR}/do_you_hold_the_lock_for_the_sno_spoke_cluster_server.txt || echo "no")
 
   if [ "${does_the_current_job_hold_a_lock_to_use_a_baremetal_server}" == "yes" ]; then
-
-    setup_aux_host_ssh_access
     server_poweroff
-
     # This must be run the latest one since it releases its server lock
     release_locked_host
   fi

ci-operator/step-registry/telcov10n/metal-single-node-spoke-kpis/hacks/deploy/telcov10n-metal-single-node-spoke-kpis-hacks-deploy-commands.sh

@@ -9,6 +9,27 @@ echo "************ telcov10n Fix user IDs in a container ************"
 
 source ${SHARED_DIR}/common-telcov10n-bash-functions.sh
 
+function extract_and_set_ocp_version {
+
+  echo "************ telcov10n Extracting OCP version from JOB_NAME ************"
+
+  echo "[INFO] JOB_NAME: ${JOB_NAME:-not set}"
+
+  OCP_VERSION=$(extract_ocp_version)
+
+  if [ -z "${OCP_VERSION}" ]; then
+    echo "[ERROR] Could not extract OCP version from JOB_NAME"
+    exit 1
+  fi
+
+  echo "[INFO] OCP Version: ${OCP_VERSION}"
+
+  # Store OCP version for other steps
+  echo -n "${OCP_VERSION}" >| ${SHARED_DIR}/ocp_version.txt
+
+  export OCP_VERSION
+}
+
 function define_spoke_cluster_name {
 
   #### Spoke cluster
@@ -36,6 +57,83 @@ function set_spoke_cluster_kubeconfig {
   export KUBECONFIG="${SHARED_DIR}/spoke-${secret_kubeconfig}.yaml"
 }
 
+# Track if we've already created the waiting request file (stored path for cleanup)
+WAITING_FILE_PATH=""
+
+function create_waiting_request_on_bastion {
+
+  local spoke_lock_filename="${1}"
+
+  # Only create the waiting file once per session
+  # Each job gets a unique file (with timestamp) that only it will delete
+  if [ -n "${WAITING_FILE_PATH}" ]; then
+    return 0
+  fi
+
+  echo
+  echo "************ telcov10n Registering wait request before lock attempt ************"
+  echo
+
+  local waiting_file
+  waiting_file=$(create_waiting_request_file "${AUX_HOST}" "${spoke_lock_filename}" "${OCP_VERSION}")
+
+  if [ -n "${waiting_file}" ]; then
+    echo "[INFO] Created waiting request file: ${waiting_file}"
+    echo "       This signals that a job with OCP version ${OCP_VERSION} is waiting."
+    WAITING_FILE_PATH="${waiting_file}"
+    # Store the path in SHARED_DIR for cleanup step
+    echo -n "${waiting_file}" >| ${SHARED_DIR}/own_waiting_file.txt
+  else
+    echo "[WARNING] Failed to create waiting request file."
+  fi
+
+  echo
+}
+
+function validate_lock_for_higher_priority {
+
+  local spoke_lock_filename="${1}"
+
+  echo
+  echo "************ telcov10n Validating lock acquisition for priority ************"
+  echo
+
+  # Check if there's a higher priority job waiting BEFORE removing our waiting file
+  # This way, if we need to release the lock, our waiting file stays intact
+  local check_result
+  check_result=$(check_for_higher_priority_waiter "${AUX_HOST}" "${spoke_lock_filename}" "${OCP_VERSION}")
+
+  if [[ "${check_result}" == quit:* ]]; then
+    local higher_version=${check_result#quit:}
+    echo
+    echo "[WARNING] Lock acquired but a higher priority job is waiting!"
+    echo "          Current job version: ${OCP_VERSION}"
+    echo "          Higher version waiting: ${higher_version}"
+    echo "          Releasing lock to allow higher priority job to proceed..."
+    echo "          (Keeping own waiting file for next attempt)"
+    echo
+    # Release the lock to let the higher priority job acquire it
+    # Keep our waiting file - we're still waiting!
+    timeout -s 9 10m ssh "${SSHOPTS[@]}" "root@${AUX_HOST}" "rm -fv ${spoke_lock_filename}"
+    return 1
+  fi
+
+  echo "[INFO] No higher priority jobs waiting. Proceeding with lock."
+
+  # NOW remove our waiting file since we're proceeding
+  if [ -n "${WAITING_FILE_PATH}" ]; then
+    echo "[INFO] Removing own waiting file: ${WAITING_FILE_PATH}"
+    remove_own_waiting_file "${AUX_HOST}" "${WAITING_FILE_PATH}"
+    WAITING_FILE_PATH=""
+    rm -f ${SHARED_DIR}/own_waiting_file.txt 2>/dev/null || true
+  fi
+
+  # Store lock filename for later use by other steps
+  echo -n "${spoke_lock_filename}" >| ${SHARED_DIR}/spoke_lock_filename.txt
+
+  return 0
+}
+
 function select_baremetal_host_from_pool {
 
   echo "************ telcov10n select a baremetal host from the pool ************"
@@ -60,13 +158,24 @@ function select_baremetal_host_from_pool {
       local network_spoke_mac_address
       network_spoke_mac_address="$(cat ${baremetal_host_path}/network_spoke_mac_address)"
       local spoke_lock_filename="/var/run/lock/ztp-baremetal-pool/spoke-baremetal-${network_spoke_mac_address//:/-}.lock"
+
+      # Create waiting request file BEFORE trying to acquire lock (only once)
+      # This ensures our presence is visible even if we immediately get the lock
+      create_waiting_request_on_bastion "${spoke_lock_filename}"
+
       try_to_lock_host "${AUX_HOST}" "${spoke_lock_filename}" "${host_lock_timestamp}" "${LOCK_TIMEOUT}"
-      [[ "$(check_the_host_was_locked "${AUX_HOST}" "${spoke_lock_filename}" "${host_lock_timestamp}")" == "locked" ]] &&
-      {
-        update_host_and_master_yaml_files "$(dirname ${host})" ;
-        echo -n "yes" >| ${SHARED_DIR}/do_you_hold_the_lock_for_the_sno_spoke_cluster_server.txt
-        return 0 ;
-      }
+      if [[ "$(check_the_host_was_locked "${AUX_HOST}" "${spoke_lock_filename}" "${host_lock_timestamp}")" == "locked" ]]; then
+        # Validate that no higher priority job is waiting
+        if validate_lock_for_higher_priority "${spoke_lock_filename}"; then
+          update_host_and_master_yaml_files "$(dirname ${host})"
+          echo -n "yes" >| ${SHARED_DIR}/do_you_hold_the_lock_for_the_sno_spoke_cluster_server.txt
+          return 0
+        else
+          # Higher priority job is waiting, lock was released
+          # Our waiting file is still intact (not removed until validation passes)
+          echo "[INFO] Will retry acquiring lock..."
+        fi
+      fi
     fi
   done
 
@@ -218,6 +327,7 @@ function hack_spoke_deployment {
 function main {
 
   setup_aux_host_ssh_access
+  extract_and_set_ocp_version
   define_spoke_cluster_name
   set_spoke_cluster_kubeconfig
   hack_spoke_deployment

ci-operator/step-registry/telcov10n/metal-single-node-spoke-kpis/hacks/deploy/telcov10n-metal-single-node-spoke-kpis-hacks-deploy-ref.yaml

@@ -46,4 +46,6 @@ ref:
       If cluster endpoints are reachables through a socks5 proxy
   documentation: |-
     This step allows to adapt the SNO Spoke cluster deployment for
-    the new baremetal server pool in the new lab location
+    the new baremetal server pool in the new lab location.
+    OCP version is automatically extracted from RELEASE_IMAGE_LATEST for
+    graceful quit priority when multiple jobs compete for the same baremetal host.

ci-operator/step-registry/telcov10n/metal-single-node-spoke-kpis/tests/cpu-util/telcov10n-metal-single-node-spoke-kpis-tests-cpu-util-commands.sh

@@ -247,14 +247,21 @@ function test_kpis {
 
   echo "************ telcov10n Run CPU Utilization Telco KPIs test ************"
 
+  # Check for graceful quit request before starting this test
+  check_for_quit "cpu_utils_test" "graceful"
+
   make_up_inventory
   make_up_remote_test_command
   make_up_ansible_playbook
   run_ansible_playbook
   setup_test_result_for_component_readiness
+
+  # Mark successful completion
+  echo -n "completed" >| ${SHARED_DIR}/cpu_util_test_status.txt
 }
 
 function main {
+  setup_ssh_and_lock_info
   set_spoke_cluster_kubeconfig
   test_kpis
 }

ci-operator/step-registry/telcov10n/metal-single-node-spoke-kpis/tests/cpu-util/telcov10n-metal-single-node-spoke-kpis-tests-cpu-util-ref.yaml

@@ -53,3 +53,7 @@ ref:
       [see: https://github.com/neisw/ci-test-mapping/blob/main/pkg/components/telcoperformance/component.go]
   documentation: |-
     This step allows to verify the SNO Spoke cluster deployed through its kubeconfig.
+    OCP version is loaded from SHARED_DIR/ocp_version.txt (set by deploy step).
+    If a graceful quit is requested by a higher version job, this test will be skipped
+    to release the baremetal host lock faster. The oslat test will have already completed
+    by this point, so PTP reporting can still collect those results.

ci-operator/step-registry/telcov10n/metal-single-node-spoke-kpis/tests/oslat/telcov10n-metal-single-node-spoke-kpis-tests-oslat-commands.sh

@@ -239,14 +239,21 @@ function test_kpis {
 
   echo "************ telcov10n Run oslat Telco KPIs test ************"
 
+  # Check for graceful quit request before starting this test
+  check_for_quit "oslat_test" "graceful"
+
   make_up_inventory
   make_up_remote_test_command
   make_up_ansible_playbook
   run_ansible_playbook
   setup_test_result_for_component_readiness
+
+  # Mark successful completion
+  echo -n "completed" >| ${SHARED_DIR}/oslat_test_status.txt
 }
 
 function main {
+  setup_ssh_and_lock_info
   set_spoke_cluster_kubeconfig
   test_kpis
 }

ci-operator/step-registry/telcov10n/metal-single-node-spoke-kpis/tests/oslat/telcov10n-metal-single-node-spoke-kpis-tests-oslat-ref.yaml

@@ -46,3 +46,6 @@ ref:
       [see: https://github.com/neisw/ci-test-mapping/blob/main/pkg/components/telcoperformance/component.go]
   documentation: |-
     This step allows to verify the SNO Spoke cluster deployed through its kubeconfig.
+    OCP version is loaded from SHARED_DIR/ocp_version.txt (set by deploy step).
+    If a graceful quit is requested by a higher version job, this test will be skipped
+    and the job will exit gracefully to release the baremetal host lock.

ci-operator/step-registry/telcov10n/metal-single-node-spoke/cluster/install/telcov10n-metal-single-node-spoke-cluster-install-commands.sh

@@ -160,6 +160,10 @@ function checking_installation_progress {
   timeout=$(date -d "${ABORT_INSTALLATION_TIMEOUT}" +%s)
   abort_installation=/tmp/abort.installation
 
+  # Counter for quit check - only check every QUIT_CHECK_INTERVAL iterations
+  local quit_check_counter=0
+  local quit_check_interval="${QUIT_CHECK_INTERVAL:-3}"
+
   while true; do
 
     test -f ${abort_installation} && {
@@ -203,6 +207,14 @@ function checking_installation_progress {
         echo "$ touch ${abort_installation}"
       fi
 
+      # Check for quit request every N iterations (QUIT_CHECK_INTERVAL)
+      # Use "force" mode since if interrupted, the rest of the steps are meaningless (cluster not ready)
+      ((quit_check_counter++))
+      if [ "${quit_check_counter}" -ge "${quit_check_interval}" ]; then
+        check_for_quit "cluster_installation_progress" "force"
+        quit_check_counter=0
+      fi
+
       sleep ${refresh_timing:="10m"} ;
     } || echo
   done
@@ -238,6 +250,10 @@ function get_and_save_kubeconfig_and_creds {
 }
 
 function main {
+
+  # Setup SSH and load lock info for quit checks
+  setup_ssh_and_lock_info
+
   set_hub_cluster_kubeconfig
   generate_cluster_image_set
   create_spoke_namespace

ci-operator/step-registry/telcov10n/metal-single-node-spoke/cluster/install/telcov10n-metal-single-node-spoke-cluster-install-ref.yaml

@@ -39,6 +39,14 @@ ref:
       Set the amount of time, the step must wait at most, before unconditionally forcing the workflow to continue
       with the next steps which will clean up and free all resources used in the deployment of this cluster
       (MIN: "REFRESH_TIME value", default: "1 hours + 45 min", MAX: "2 hours - REFRESH_TIME value").
+  - name: QUIT_CHECK_INTERVAL
+    default: "3"
+    documentation: |-
+      Number of REFRESH_TIME iterations between each quit condition check. A higher priority job (newer OCP version)
+      waiting for the baremetal host lock can request the current job to quit. This setting controls how often
+      the installation loop checks for such quit requests. Uses "force" mode (exit 1) since an interrupted
+      installation leaves the cluster unusable. Default is 3 (check every 3 iterations, i.e., every 9 min
+      with default REFRESH_TIME of 3m).
   - name: BIOS_SETTINGS
     default: "{}"
     documentation: |-