feat(pgbouncer): trigger reconnection on PostgreSQL primary failover

build/crd/crunchy/generated/postgres-operator.crunchydata.com_postgresclusters.yaml

@@ -22972,6 +22972,12 @@ spec:
                 properties:
                   pgBouncer:
                     properties:
+                      lastFailoverPrimaryUID:
+                        description: |-
+                          Identifies the primary pod UID when failover signal (SIGTERM) was last triggered.
+                          Used to detect failovers and trigger PgBouncer container restart for fresh
+                          connection pool and DNS lookup to the correct primary.
+                        type: string
                       postgresRevision:
                         description: |-
                           Identifies the revision of PgBouncer assets that have been installed into

config/crd/bases/postgres-operator.crunchydata.com_postgresclusters.yaml

@@ -22870,6 +22870,12 @@ spec:
                 properties:
                   pgBouncer:
                     properties:
+                      lastFailoverPrimaryUID:
+                        description: |-
+                          Identifies the primary pod UID when failover signal (SIGTERM) was last triggered.
+                          Used to detect failovers and trigger PgBouncer container restart for fresh
+                          connection pool and DNS lookup to the correct primary.
+                        type: string
                       postgresRevision:
                         description: |-
                           Identifies the revision of PgBouncer assets that have been installed into

deploy/crd.yaml

@@ -51472,6 +51472,12 @@ spec:
                 properties:
                   pgBouncer:
                     properties:
+                      lastFailoverPrimaryUID:
+                        description: |-
+                          Identifies the primary pod UID when failover signal (SIGTERM) was last triggered.
+                          Used to detect failovers and trigger PgBouncer container restart for fresh
+                          connection pool and DNS lookup to the correct primary.
+                        type: string
                       postgresRevision:
                         description: |-
                           Identifies the revision of PgBouncer assets that have been installed into

e2e-tests/functions

@@ -54,6 +54,7 @@ deploy_operator() {
 		disable_telemetry=false
 	fi
 	yq eval '.spec.template.spec.containers[0].image = "'${IMAGE}'"' "${DEPLOY_DIR}/${cw_prefix}operator.yaml" \
+    	| yq eval '.spec.template.spec.containers[0].imagePullPolicy = "IfNotPresent"' - \
 		| yq eval '(.spec.template.spec.containers[] | select(.name=="operator") | .env[] | select(.name=="DISABLE_TELEMETRY") | .value) = "'${disable_telemetry}'"' - \
 		| kubectl -n "${OPERATOR_NS:-$NAMESPACE}" apply -f -
 }

e2e-tests/tests/pgbouncer-failover/00-assert.yaml

@@ -0,0 +1,12 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: percona-postgresql-operator
+status:
+  readyReplicas: 1
+  replicas: 1
+  updatedReplicas: 1

e2e-tests/tests/pgbouncer-failover/00-deploy-operator.yaml

@@ -0,0 +1,14 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+      init_temp_dir # do this only in the first TestStep
+
+      deploy_operator
+      deploy_client
+      deploy_s3_secrets

e2e-tests/tests/pgbouncer-failover/01-assert.yaml

@@ -0,0 +1,16 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: pgbouncer-failover
+status:
+  postgres:
+    ready: 2
+    size: 2
+  pgbouncer:
+    ready: 2
+    size: 2
+  state: ready

e2e-tests/tests/pgbouncer-failover/01-create-cluster.yaml

@@ -0,0 +1,15 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 10
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      get_cr \
+        | yq eval '
+            .spec.proxy.pgBouncer.replicas=2 |
+            .spec.instances[].replicas=2' - \
+        | kubectl -n "${NAMESPACE}" apply -f -

e2e-tests/tests/pgbouncer-failover/02-assert.yaml

@@ -0,0 +1,17 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+timeout: 120
+---
+apiVersion: pgv2.percona.com/v2
+kind: PerconaPGCluster
+metadata:
+  name: pgbouncer-failover
+status:
+  postgres:
+    ready: 2
+    size: 2
+  pgbouncer:
+    # PgBouncer should be back to ready state with 2 replicas
+    ready: 2
+    size: 2
+  state: ready

e2e-tests/tests/pgbouncer-failover/02-trigger-switchover.yaml

@@ -0,0 +1,55 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+timeout: 60
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      LEADER=$(kubectl get pods -n "${NAMESPACE}" \
+        -l postgres-operator.crunchydata.com/role=primary \
+        -o jsonpath='{.items[0].metadata.name}')
+
+      echo "Current leader: $LEADER"
+
+      REPLICA=$(kubectl get pods -n "${NAMESPACE}" \
+        -l postgres-operator.crunchydata.com/role=replica \
+        -o jsonpath='{.items[0].metadata.name}')
+
+      echo "Replica to promote: $REPLICA"
+
+      kubectl get pods -n "${NAMESPACE}" \
+        -l postgres-operator.crunchydata.com/role=pgbouncer \
+        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \
+        > /tmp/pgbouncer-pods-before.txt
+
+      echo "PgBouncer pods before switchover:"
+      cat /tmp/pgbouncer-pods-before.txt
+
+      kubectl exec "$LEADER" -c database -n "${NAMESPACE}" -- \
+        patronictl switchover \
+        --leader "$LEADER" \
+        --candidate "$REPLICA" \
+        --force
+
+      echo "Switchover triggered successfully"
+
+      # Wait for PgBouncer pods to be recreated (they should be deleted and recreated)
+      sleep 15
+
+      kubectl get pods -n "${NAMESPACE}" \
+        -l postgres-operator.crunchydata.com/role=pgbouncer \
+        -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' \
+        > /tmp/pgbouncer-pods-after.txt
+
+      echo "PgBouncer pods after switchover:"
+      cat /tmp/pgbouncer-pods-after.txt
+
+      if diff /tmp/pgbouncer-pods-before.txt /tmp/pgbouncer-pods-after.txt > /dev/null; then
+        echo "ERROR: PgBouncer pods were NOT recreated after failover!"
+        exit 1
+      else
+        echo "SUCCESS: PgBouncer pods were recreated after failover"
+      fi

e2e-tests/tests/pgbouncer-failover/03-cleanup.yaml

@@ -0,0 +1,21 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+  - apiVersion: pgv2.percona.com/v2
+    kind: PerconaPGCluster
+    metadata:
+      name: pgbouncer-failover
+  - apiVersion: postgres-operator.crunchydata.com/v1beta1
+    kind: PostgresCluster
+    metadata:
+      name: pgbouncer-failover
+commands:
+  - script: |-
+      set -o errexit
+      set -o xtrace
+
+      source ../../functions
+
+      remove_all_finalizers
+      destroy_operator
+    timeout: 60

internal/controller/postgrescluster/instance.go

@@ -262,11 +262,11 @@ func newObservedInstances(
 	return &observed
 }
 
-// writablePod looks at observedInstances and finds an instance that matches
+// WritablePod looks at observedInstances and finds an instance that matches
 // a few conditions. The instance should be non-terminating, running, and
 // writable i.e. the instance with the primary. If such an instance exists, it
 // is returned along with the instance pod.
-func (observed *observedInstances) writablePod(container string) (*corev1.Pod, *Instance) {
+func (observed *observedInstances) WritablePod(container string) (*corev1.Pod, *Instance) {
 	if observed == nil {
 		return nil, nil
 	}

internal/controller/postgrescluster/instance_test.go

@@ -381,7 +381,7 @@ func TestWritablePod(t *testing.T) {
 	t.Run("empty observed", func(t *testing.T) {
 		observed := &observedInstances{}
 
-		pod, instance := observed.writablePod("container")
+		pod, instance := observed.WritablePod("container")
 		assert.Assert(t, pod == nil)
 		assert.Assert(t, instance == nil)
 	})
@@ -415,7 +415,7 @@ func TestWritablePod(t *testing.T) {
 		terminating, known := observed.forCluster[0].IsTerminating()
 		assert.Assert(t, terminating && known)
 
-		pod, instance := observed.writablePod("container")
+		pod, instance := observed.WritablePod("container")
 		assert.Assert(t, pod == nil)
 		assert.Assert(t, instance == nil)
 	})
@@ -447,7 +447,7 @@ func TestWritablePod(t *testing.T) {
 		running, known := observed.forCluster[0].IsRunning(container)
 		assert.Check(t, !running && known)
 
-		pod, instance := observed.writablePod("container")
+		pod, instance := observed.WritablePod("container")
 		assert.Assert(t, pod == nil)
 		assert.Assert(t, instance == nil)
 	})
@@ -480,7 +480,7 @@ func TestWritablePod(t *testing.T) {
 		writable, known := observed.forCluster[0].IsWritable()
 		assert.Check(t, !writable && known)
 
-		pod, instance := observed.writablePod("container")
+		pod, instance := observed.WritablePod("container")
 		assert.Assert(t, pod == nil)
 		assert.Assert(t, instance == nil)
 	})
@@ -517,7 +517,7 @@ func TestWritablePod(t *testing.T) {
 		running, known := observed.forCluster[0].IsRunning(container)
 		assert.Check(t, running && known)
 
-		pod, instance := observed.writablePod("container")
+		pod, instance := observed.WritablePod("container")
 		assert.Assert(t, pod != nil)
 		assert.Assert(t, instance != nil)
 	})