Install Cluster Observability Operator in zuul jobs

Watcher relies on MetricStorage provided by telemetry which uses
OpenShift Cluster Observability Operator. This patch is installing the
operator before deploying the watcher operator so that we have it
running before doing kuttl preparation steps.

Author: amoralej

ci/playbooks/create-coo-subscription.yaml

@@ -0,0 +1,33 @@
+---
+- name: Create the COO subscription
+  ansible.builtin.shell:
+    cmd: |
+      oc create -f - <<EOF
+      apiVersion: operators.coreos.com/v1alpha1
+      kind: Subscription
+      metadata:
+        name: cluster-observability-operator
+        namespace: openshift-operators
+      spec:
+        channel: development
+        installPlanApproval: Automatic
+        name: cluster-observability-operator
+        source: redhat-operators
+        sourceNamespace: openshift-marketplace
+      EOF
+  register: output
+
+  # need to have a wait here, since the csv is not created immediately. There is a slight delay, during which time, the oc wait command would fail, since there's no resource to watch
+- name: Wait for the required resource to be created
+  ansible.builtin.command:
+    cmd:
+      oc get csv --namespace=openshift-operators -l operators.coreos.com/cluster-observability-operator.openshift-operators
+  delay: 10
+  retries: 20
+  register: output
+  until: output.stdout_lines | length != 0
+
+- name: Wait for the resources to be available
+  ansible.builtin.command:
+    cmd: |
+      oc wait --timeout=300s --for jsonpath="{.status.phase}"=Succeeded csv --namespace=openshift-operators -l operators.coreos.com/cluster-observability-operator.openshift-operators

ci/playbooks/deploy_watcher_service.yaml

@@ -8,6 +8,9 @@
     KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}"
     PATH: "{{ cifmw_path }}"
   tasks:
+    - name: Import tasks for creating COO subscription
+      ansible.builtin.include_tasks: "create-coo-subscription.yaml"
+
     - name: Install Watcher Operator
       cifmw.general.ci_script:
         output_dir: "{{ cifmw_basedir }}/artifacts"