Merge pull request #5990 from ggiguash/retry-observability-image-pull

USHIFT-6431: Retry Loki and Prometheus image pull

Author: openshift-merge-bot[bot]

scripts/pull_retry.sh

@@ -1,15 +1,18 @@
 #!/bin/bash
-set -xuo pipefail
+set -euo pipefail
+set -x
 #
 # Use this script to avoid image pull getting stuck once MicroShift tries to
 # do so in runtime.
 #
+PULL_CMD=${PULL_CMD:-"sudo crictl pull"}
 
 function pull_image() {
     local -r image=$1
     local rc=0
     for _ in $(seq 3); do
-        timeout 5m sudo crictl pull "${image}" && return
+        # shellcheck disable=SC2086
+        timeout 5m ${PULL_CMD} "${image}" && return
         rc=$?
         sleep 1
     done

test/bin/manage_loki.sh

@@ -9,6 +9,7 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 # shellcheck source=test/bin/common.sh
 source "${SCRIPTDIR}/common.sh"
 
+LOKI_IMAGE="docker.io/grafana/loki"
 DEFAULT_HOST_PORT="3100"
 
 usage() {
@@ -38,14 +39,17 @@ action_start() {
     local host_port="${1:-${DEFAULT_HOST_PORT}}"
     local container_name="loki"
 
+    # Prefetch the image with retries
+    PULL_CMD="podman pull" "${SCRIPTDIR}/../../scripts/pull_retry.sh" "${LOKI_IMAGE}"
+
     echo "Stopping previous instance of Loki container ${container_name} (if any)"
     action_stop "${host_port}"
 
     echo "Starting Loki container ${container_name} on host port ${host_port}"
     podman run -d --rm --name "${container_name}" \
         -p "${host_port}:3100" \
         --user root \
-        docker.io/grafana/loki > /dev/null
+        "${LOKI_IMAGE}" > /dev/null
 }
 
 if [ $# -eq 0 ]; then

test/bin/manage_prometheus.sh

@@ -10,6 +10,7 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 source "${SCRIPTDIR}/common.sh"
 
 PROMETHEUS_DIR="${IMAGEDIR}/prometheus"
+PROMETHEUS_IMAGE="quay.io/prometheus/prometheus"
 DEFAULT_HOST_PORT="9091"
 
 usage() {
@@ -40,6 +41,9 @@ action_start() {
     local host_port="${1:-${DEFAULT_HOST_PORT}}"
     local container_name="prometheus"
 
+    # Prefetch the image with retries
+    PULL_CMD="podman pull" "${SCRIPTDIR}/../../scripts/pull_retry.sh" "${PROMETHEUS_IMAGE}"
+
     mkdir -p "${PROMETHEUS_DIR}"
     PROM_CONFIG="${PROMETHEUS_DIR}/prometheus.yml"
     # Empty configuration file will take all defaults.
@@ -54,7 +58,7 @@ action_start() {
     podman run -d --rm --name "${container_name}" \
         -p "${host_port}:9090" \
         -v "${PROMETHEUS_DIR}:/etc/prometheus:Z" \
-        quay.io/prometheus/prometheus \
+        "${PROMETHEUS_IMAGE}" \
         --config.file=/etc/prometheus/prometheus.yml \
         --web.enable-remote-write-receiver > /dev/null
 }