Skip to content

Commit e35722b

Browse files
cbandycbandy
committed
Add an end-to-end test for in-place PITR
Issue: [sc-13322]
1 parent ad0a2a8 commit e35722b

File tree

11 files changed

+248
-16
lines changed

11 files changed

+248
-16
lines changed

‎testing/kuttl/e2e/pgbackrest-restore/01--create-cluster.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ spec:
1515
max_connections: 200
1616
instances:
1717
- dataVolumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } }
18+
replicas: 2
1819
backups:
1920
pgbackrest:
2021
manual:

‎testing/kuttl/e2e/pgbackrest-restore/07--annotate.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,14 @@ apiVersion: kuttl.dev/v1beta1
44
kind: TestStep
55
commands:
66
- script: |
7-
INSTANCE=$(
8-
kubectl get statefulset.apps --namespace "${NAMESPACE}" \
7+
PRIMARY=$(
8+
kubectl get pod --namespace "${NAMESPACE}" \
99
--output name --selector '
1010
postgres-operator.crunchydata.com/cluster=original,
11-
postgres-operator.crunchydata.com/instance'
11+
postgres-operator.crunchydata.com/role=master'
1212
)
1313
START=$(
14-
kubectl exec --namespace "${NAMESPACE}" "${INSTANCE}" \
14+
kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
1515
-- psql -qAt --command 'SELECT pg_postmaster_start_time()'
1616
)
1717
kubectl annotate --namespace "${NAMESPACE}" postgrescluster/original \

‎testing/kuttl/e2e/pgbackrest-restore/07--update-cluster.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ spec:
1414
max_connections: 1000
1515
instances:
1616
- dataVolumeClaimSpec: { accessModes: [ReadWriteOnce], resources: { requests: { storage: 1Gi } } }
17+
replicas: 2
1718
backups:
1819
pgbackrest:
1920
manual:

‎testing/kuttl/e2e/pgbackrest-restore/08--wait-restart.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,23 +7,23 @@ commands:
77
kubectl get --namespace "${NAMESPACE}" postgrescluster/original \
88
--output 'go-template={{ index .metadata.annotations "testing/start-before" }}'
99
)
10-
INSTANCE=$(
11-
kubectl get statefulset.apps --namespace "${NAMESPACE}" --output name \
12-
--selector '
10+
PRIMARY=$(
11+
kubectl get pod --namespace "${NAMESPACE}" \
12+
--output name --selector '
1313
postgres-operator.crunchydata.com/cluster=original,
14-
postgres-operator.crunchydata.com/instance'
14+
postgres-operator.crunchydata.com/role=master'
1515
)
1616
1717
# Wait for PostgreSQL to restart.
1818
while true; do
1919
START=$(
20-
kubectl exec --namespace "${NAMESPACE}" "${INSTANCE}" \
20+
kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
2121
-- psql -qAt --command 'SELECT pg_postmaster_start_time()'
2222
)
2323
if [ "${START}" ] && [ "${START}" != "${BEFORE}" ]; then break; else sleep 1; fi
2424
done
2525
echo "${START} != ${BEFORE}"
2626
2727
# Reset counters in the "pg_stat_archiver" view.
28-
kubectl exec --namespace "${NAMESPACE}" "${INSTANCE}" \
28+
kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
2929
-- psql -qb --command "SELECT pg_stat_reset_shared('archiver')"

‎testing/kuttl/e2e/pgbackrest-restore/10--wait-archived.yaml‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,17 @@ apiVersion: kuttl.dev/v1beta1
33
kind: TestStep
44
commands:
55
- script: |
6-
INSTANCE=$(
7-
kubectl get statefulset.apps --namespace "${NAMESPACE}" --output name \
8-
--selector '
6+
PRIMARY=$(
7+
kubectl get pod --namespace "${NAMESPACE}" \
8+
--output name --selector '
99
postgres-operator.crunchydata.com/cluster=original,
10-
postgres-operator.crunchydata.com/instance'
10+
postgres-operator.crunchydata.com/role=master'
1111
)
1212
1313
# Wait for the data to be sent to the WAL archive. A prior step reset the
1414
# "pg_stat_archiver" counters, so anything more than zero should suffice.
15-
kubectl exec --namespace "${NAMESPACE}" "${INSTANCE}" \
16-
-- psql -qb --command '
15+
kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
16+
-- psql -qb --set ON_ERROR_STOP=1 --command '
1717
DO $$
1818
BEGIN
1919
PERFORM pg_switch_wal(); -- at least once

‎testing/kuttl/e2e/pgbackrest-restore/14--lose-data.yaml‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestStep
4+
commands:
5+
- script: |
6+
PRIMARY=$(
7+
kubectl get pod --namespace "${NAMESPACE}" \
8+
--output name --selector '
9+
postgres-operator.crunchydata.com/cluster=original,
10+
postgres-operator.crunchydata.com/role=master'
11+
)
12+
OBJECTIVE=$(
13+
kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
14+
-- psql -qAt --command 'SELECT clock_timestamp()'
15+
)
16+
17+
# Store the recovery objective for later steps.
18+
kubectl annotate --namespace "${NAMESPACE}" postgrescluster/original \
19+
"testing/objective=${OBJECTIVE}"
20+
21+
# A reason to restore. Wait for the change to be sent to the WAL archive.
22+
kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" \
23+
-- psql -qb original --set ON_ERROR_STOP=1 \
24+
--command 'DROP TABLE important' \
25+
--command "SELECT pg_stat_reset_shared('archiver')" \
26+
--command '
27+
DO $$
28+
BEGIN
29+
LOOP
30+
PERFORM pg_switch_wal() FROM pg_stat_archiver WHERE archived_count = 0;
31+
EXIT WHEN NOT FOUND;
32+
PERFORM pg_sleep(1); -- seconds
33+
ROLLBACK; -- fresh stats
34+
END LOOP;
35+
END $$'
36+
37+
# The replica should also need to be restored.
38+
- script: |
39+
REPLICA=$(
40+
kubectl get pod --namespace "${NAMESPACE}" \
41+
--output name --selector '
42+
postgres-operator.crunchydata.com/cluster=original,
43+
postgres-operator.crunchydata.com/role=replica'
44+
)
45+
46+
kubectl exec --stdin --namespace "${NAMESPACE}" "${REPLICA}" \
47+
-- psql -qb original --set ON_ERROR_STOP=1 \
48+
--file=- <<'SQL'
49+
DO $$
50+
BEGIN
51+
ASSERT to_regclass('important') IS NULL, 'expected no table';
52+
PERFORM * FROM information_schema.tables WHERE table_name = 'important';
53+
ASSERT NOT FOUND, 'expected no table';
54+
END $$
55+
SQL

‎testing/kuttl/e2e/pgbackrest-restore/15--in-place-pitr.yaml‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
---
2+
apiVersion: kuttl.dev/v1beta1
3+
kind: TestStep
4+
commands:
5+
- script: |
6+
TARGET_JSON=$(
7+
kubectl get --namespace "${NAMESPACE}" postgrescluster/original \
8+
--output 'go-template={{ index .metadata.annotations "testing/objective" | printf "--target=%q" | printf "%q" }}'
9+
)
10+
11+
# Configure the cluster for an in-place point-in-time restore (PITR).
12+
kubectl patch --namespace "${NAMESPACE}" postgrescluster/original \
13+
--type 'merge' --patch '
14+
{"spec":{"backups":{"pgbackrest":{"restore":{
15+
"enabled": true,
16+
"repoName": "repo1",
17+
"options": ["--type=time", '"${TARGET_JSON}"']
18+
}}}}}'
19+
20+
# Annotate the cluster to trigger the restore.
21+
kubectl annotate --namespace="${NAMESPACE}" postgrescluster/original \
22+
'postgres-operator.crunchydata.com/pgbackrest-restore=one'

‎testing/kuttl/e2e/pgbackrest-restore/15-assert.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
# Wait for the restore to complete and the cluster to come online.
3+
apiVersion: postgres-operator.crunchydata.com/v1beta1
4+
kind: PostgresCluster
5+
metadata:
6+
name: original
7+
status:
8+
instances:
9+
- name: '00'
10+
replicas: 2
11+
readyReplicas: 2
12+
updatedReplicas: 2
13+
pgbackrest:
14+
restore:
15+
id: one
16+
finished: true

‎testing/kuttl/e2e/pgbackrest-restore/16--check-data.yaml‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
---
2+
# Confirm that data was restored to the point-in-time.
3+
apiVersion: batch/v1
4+
kind: Job
5+
metadata:
6+
name: original-pitr-primary-data
7+
labels: { postgres-operator-test: kuttl }
8+
spec:
9+
backoffLimit: 3
10+
template:
11+
metadata:
12+
labels: { postgres-operator-test: kuttl }
13+
spec:
14+
restartPolicy: Never
15+
containers:
16+
- name: psql
17+
image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:centos8-14.1-0
18+
env:
19+
- name: PGURI
20+
valueFrom: { secretKeyRef: { name: original-pguser-original, key: uri } }
21+
22+
# Do not wait indefinitely.
23+
- { name: PGCONNECT_TIMEOUT, value: '5' }
24+
25+
# Note: the `$$$$` is reduced to `$$` by Kubernetes.
26+
# - https://kubernetes.io/docs/tasks/inject-data-application/
27+
command:
28+
- psql
29+
- $(PGURI)
30+
- -qa
31+
- --set=ON_ERROR_STOP=1
32+
- --command
33+
- |
34+
DO $$$$
35+
DECLARE
36+
restored jsonb;
37+
BEGIN
38+
SELECT jsonb_agg(important) INTO restored FROM important;
39+
ASSERT restored = '[
40+
{"data":"treasure"}, {"data":"water"}, {"data":"socks"}
41+
]', format('got %L', restored);
42+
END $$$$;
43+
44+
---
45+
# Confirm that replicas are also restored and streaming from the primary.
46+
apiVersion: batch/v1
47+
kind: Job
48+
metadata:
49+
name: original-pitr-replica-data
50+
labels: { postgres-operator-test: kuttl }
51+
spec:
52+
backoffLimit: 3
53+
template:
54+
metadata:
55+
labels: { postgres-operator-test: kuttl }
56+
spec:
57+
restartPolicy: Never
58+
containers:
59+
- name: psql
60+
image: registry.developers.crunchydata.com/crunchydata/crunchy-postgres:centos8-14.1-0
61+
env:
62+
- name: PGPORT
63+
valueFrom: { secretKeyRef: { name: original-pguser-original, key: port } }
64+
- name: PGDATABASE
65+
valueFrom: { secretKeyRef: { name: original-pguser-original, key: dbname } }
66+
- name: PGUSER
67+
valueFrom: { secretKeyRef: { name: original-pguser-original, key: user } }
68+
- name: PGPASSWORD
69+
valueFrom: { secretKeyRef: { name: original-pguser-original, key: password } }
70+
71+
# The user secret does not contain the replica service.
72+
- name: NAMESPACE
73+
valueFrom: { fieldRef: { fieldPath: metadata.namespace } }
74+
- name: PGHOST
75+
value: "original-replicas.$(NAMESPACE).svc"
76+
77+
# Do not wait indefinitely.
78+
- { name: PGCONNECT_TIMEOUT, value: '5' }
79+
80+
# Note: the `$$$$` is reduced to `$$` by Kubernetes.
81+
# - https://kubernetes.io/docs/tasks/inject-data-application/
82+
command:
83+
- psql
84+
- -qa
85+
- --set=ON_ERROR_STOP=1
86+
- --command
87+
- |
88+
DO $$$$
89+
DECLARE
90+
restored jsonb;
91+
BEGIN
92+
ASSERT pg_is_in_recovery(), 'expected replica';
93+
-- only users with "pg_read_all_settings" role may examine "primary_conninfo"
94+
-- ASSERT current_setting('primary_conninfo') <> '', 'expected streaming';
95+
96+
SELECT jsonb_agg(important) INTO restored FROM important;
97+
ASSERT restored = '[
98+
{"data":"treasure"}, {"data":"water"}, {"data":"socks"}
99+
]', format('got %L', restored);
100+
END $$$$;

‎testing/kuttl/e2e/pgbackrest-restore/16-assert.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
apiVersion: batch/v1
3+
kind: Job
4+
metadata:
5+
name: original-pitr-primary-data
6+
status:
7+
succeeded: 1
8+
9+
---
10+
apiVersion: batch/v1
11+
kind: Job
12+
metadata:
13+
name: original-pitr-replica-data
14+
status:
15+
succeeded: 1

0 commit comments

Comments
 (0)