Quarantine flaky delete test (CrunchyData#3290)

* Quarantine flaky delete test

The `delete` test that looked at event timestamps to make sure the
replica deleted before the primary occasionally flaked out.
This PR removes that timestamp checking, quarantining that version
of the test for future debugging; and changes the in-use test to
simply verify that a cluster with replica deletes.
This PR also fixes an error in the delete tests where the -delete.yaml
was incorrectly set up.

Issue [sc-15009]

Author: benjaminjb

testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/10--cluster.yaml

@@ -0,0 +1,29 @@
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: delete-switchover-with-timestamp
+spec:
+  postgresVersion: ${KUTTL_PG_VERSION}
+  patroni:
+    switchover:
+      enabled: true
+  instances:
+    - name: instance1
+      replicas: 2
+      dataVolumeClaimSpec:
+        accessModes:
+        - "ReadWriteOnce"
+        resources:
+          requests:
+            storage: 1Gi
+  backups:
+    pgbackrest:
+      repos:
+      - name: repo1
+        volume:
+          volumeClaimSpec:
+            accessModes:
+            - "ReadWriteOnce"
+            resources:
+              requests:
+                storage: 1Gi

testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/10-assert.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: postgres-operator.crunchydata.com/v1beta1
+kind: PostgresCluster
+metadata:
+  name: delete-switchover-with-timestamp
+status:
+  instances:
+    - name: instance1
+      readyReplicas: 2
+      replicas: 2
+      updatedReplicas: 2
+---
+# Patroni labels and readiness happen separately.
+# The next step expects to find pods by their role label; wait for them here.
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
+    postgres-operator.crunchydata.com/role: master
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
+    postgres-operator.crunchydata.com/role: replica
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
+    postgres-operator.crunchydata.com/pgbackrest-backup: replica-create
+status:
+  succeeded: 1

testing/kuttl/e2e/delete/11-annotate.yaml …ca-and-check-timestamps/11-annotate.yamltesting/kuttl/e2e/delete/11-annotate.yaml renamed to testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/11-annotate.yaml testing/kuttl/e2e/delete/11-annotate.yaml renamed to testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/11-annotate.yaml

@@ -15,5 +15,5 @@ commands:
 
   # Annotate the cluster to trigger a switchover.
   - script: |
-      kubectl annotate --namespace="${NAMESPACE}" postgrescluster/delete-switchover \
+      kubectl annotate --namespace="${NAMESPACE}" postgrescluster/delete-switchover-with-timestamp \
         "postgres-operator.crunchydata.com/trigger-switchover=$(date)"

testing/kuttl/e2e/delete/12-assert.yaml …lica-and-check-timestamps/12-assert.yamltesting/kuttl/e2e/delete/12-assert.yaml renamed to testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/12-assert.yaml testing/kuttl/e2e/delete/12-assert.yaml renamed to testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/12-assert.yaml

@@ -4,7 +4,7 @@ apiVersion: v1
 kind: Pod
 metadata:
   labels:
-    postgres-operator.crunchydata.com/cluster: delete-switchover
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
     postgres-operator.crunchydata.com/data: postgres
     postgres-operator.crunchydata.com/role: master
     testing/role-before: replica
@@ -14,7 +14,7 @@ apiVersion: v1
 kind: Pod
 metadata:
   labels:
-    postgres-operator.crunchydata.com/cluster: delete-switchover
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
     postgres-operator.crunchydata.com/data: postgres
     postgres-operator.crunchydata.com/role: replica
     testing/role-before: master
@@ -23,7 +23,7 @@ metadata:
 apiVersion: postgres-operator.crunchydata.com/v1beta1
 kind: PostgresCluster
 metadata:
-  name: delete-switchover
+  name: delete-switchover-with-timestamp
 status:
   instances:
     - name: instance1

…/delete/13-delete-cluster-and-check.yaml …estamps/13-delete-cluster-and-check.yamltesting/kuttl/e2e/delete/13-delete-cluster-and-check.yaml renamed to testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/13-delete-cluster-and-check.yaml testing/kuttl/e2e/delete/13-delete-cluster-and-check.yaml renamed to testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/13-delete-cluster-and-check.yaml

@@ -24,7 +24,7 @@ commands:
       if [ -z "$PRIMARY" ]; then exit 1; fi      
       if [ -z "$REPLICA" ]; then exit 1; fi
 
-      kubectl delete postgrescluster -n "${NAMESPACE}" delete-switchover
+      kubectl delete postgrescluster -n "${NAMESPACE}" delete-switchover-with-timestamp
       
       kubectl wait "pod/${REPLICA}" --namespace "${NAMESPACE}" --for=delete --timeout=180s
 

testing/kuttl/e2e/delete/14-errors.yaml …lica-and-check-timestamps/14-errors.yamltesting/kuttl/e2e/delete/14-errors.yaml renamed to testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/14-errors.yaml testing/kuttl/e2e/delete/14-errors.yaml renamed to testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/14-errors.yaml

@@ -1,36 +1,42 @@
 ---
 apiVersion: postgres-operator.crunchydata.com/v1beta1
 kind: PostgresCluster
-labels:
-    postgres-operator.crunchydata.com/cluster: delete-switchover
+metadata:
+  name: delete-switchover-with-timestamp
 ---
 apiVersion: apps/v1
 kind: StatefulSet
-labels:
-    postgres-operator.crunchydata.com/cluster: delete-switchover
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
 ---
 # Patroni DCS objects are not owned by the PostgresCluster.
 apiVersion: v1
 kind: Endpoints
-labels:
-    postgres-operator.crunchydata.com/cluster: delete-switchover
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
 ---
 apiVersion: v1
 kind: Pod
-labels:
-    postgres-operator.crunchydata.com/cluster: delete-switchover
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
 ---
 apiVersion: v1
 kind: Service
-labels:
-    postgres-operator.crunchydata.com/cluster: delete-switchover
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
 ---
 apiVersion: v1
 kind: Secret
-labels:
-    postgres-operator.crunchydata.com/cluster: delete-switchover
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp
 ---
 apiVersion: v1
 kind: ConfigMap
-labels:
-    postgres-operator.crunchydata.com/cluster: delete-switchover
+metadata:
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-switchover-with-timestamp

testing/kuttl/e2e-other/delete-with-replica-and-check-timestamps/README.md

@@ -0,0 +1,7 @@
+This test originally existed as the second test-case in the `delete` KUTTL test.
+The test as written was prone to occasional flakes, sometimes due to missing events
+(which were being used to check the timestamp of the container delete event).
+
+After discussion, we decided that this behavior (replica deleting before the primary)
+was no longer required in v5, and the decision was made to sequester this test-case for
+further testing and refinement.

testing/kuttl/e2e/delete-namespace/02-errors.yaml

@@ -1,43 +1,49 @@
 ---
 apiVersion: postgres-operator.crunchydata.com/v1beta1
 kind: PostgresCluster
-namespace: kuttl-test-delete-namespace
-labels:
-    postgres-operator.crunchydata.com/cluster: delete
+metadata:
+  namespace: kuttl-test-delete-namespace
+  name: delete-namespace
 ---
 apiVersion: apps/v1
 kind: StatefulSet
-namespace: kuttl-test-delete-namespace
-labels:
-    postgres-operator.crunchydata.com/cluster: delete
+metadata:
+  namespace: kuttl-test-delete-namespace
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-namespace
 ---
 # Patroni DCS objects are not owned by the PostgresCluster.
 apiVersion: v1
 kind: Endpoints
-namespace: kuttl-test-delete-namespace
-labels:
-    postgres-operator.crunchydata.com/cluster: delete
+metadata:
+  namespace: kuttl-test-delete-namespace
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-namespace
 ---
 apiVersion: v1
 kind: Pod
-namespace: kuttl-test-delete-namespace
-labels:
-    postgres-operator.crunchydata.com/cluster: delete
+metadata:
+  namespace: kuttl-test-delete-namespace
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-namespace
 ---
 apiVersion: v1
 kind: Service
-namespace: kuttl-test-delete-namespace
-labels:
-    postgres-operator.crunchydata.com/cluster: delete
+metadata:
+  namespace: kuttl-test-delete-namespace
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-namespace
 ---
 apiVersion: v1
 kind: Secret
-namespace: kuttl-test-delete-namespace
-labels:
-    postgres-operator.crunchydata.com/cluster: delete
+metadata:
+  namespace: kuttl-test-delete-namespace
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-namespace
 ---
 apiVersion: v1
 kind: ConfigMap
-namespace: kuttl-test-delete-namespace
-labels:
-    postgres-operator.crunchydata.com/cluster: delete
+metadata:
+  namespace: kuttl-test-delete-namespace
+  labels:
+    postgres-operator.crunchydata.com/cluster: delete-namespace

testing/kuttl/e2e/delete/02-errors.yaml

@@ -1,36 +1,42 @@
 ---
 apiVersion: postgres-operator.crunchydata.com/v1beta1
 kind: PostgresCluster
-labels:
-    postgres-operator.crunchydata.com/cluster: delete
+metadata:
+  name: delete
 ---
 apiVersion: apps/v1
 kind: StatefulSet
-labels:
+metadata:
+  labels:
     postgres-operator.crunchydata.com/cluster: delete
 ---
 # Patroni DCS objects are not owned by the PostgresCluster.
 apiVersion: v1
 kind: Endpoints
-labels:
+metadata:
+  labels:
     postgres-operator.crunchydata.com/cluster: delete
 ---
 apiVersion: v1
 kind: Pod
-labels:
+metadata:
+  labels:
     postgres-operator.crunchydata.com/cluster: delete
 ---
 apiVersion: v1
 kind: Service
-labels:
+metadata:
+  labels:
     postgres-operator.crunchydata.com/cluster: delete
 ---
 apiVersion: v1
 kind: Secret
-labels:
+metadata:
+  labels:
     postgres-operator.crunchydata.com/cluster: delete
 ---
 apiVersion: v1
 kind: ConfigMap
-labels:
+metadata:
+  labels:
     postgres-operator.crunchydata.com/cluster: delete

testing/kuttl/e2e/delete/10--cluster.yaml

@@ -1,7 +1,7 @@
 apiVersion: postgres-operator.crunchydata.com/v1beta1
 kind: PostgresCluster
 metadata:
-  name: delete-switchover
+  name: delete-with-replica
 spec:
   postgresVersion: ${KUTTL_PG_VERSION}
   patroni: