diff --git a/testing/chainsaw/README b/testing/chainsaw/README new file mode 100644 index 0000000..f5843a2 --- /dev/null +++ b/testing/chainsaw/README @@ -0,0 +1,22 @@ + +# README + +## Install PGO + +```shell +kubectl apply -k kustomize/install/namespace +kubectl apply --server-side -k kustomize/install/default +``` + +## Configure Image Pull Secrets + +If your images are not publicly accessible, you will need to configure ImagePullSecrets. +Kyverno Policies can be used to inject them into the tests for you. +See [README](./policies/README) for the details. + + +## Run Chainsaw + +```shell +chainsaw test --config ./e2e/config.yaml --values ./e2e/values.yaml --test-dir e2e +``` diff --git a/testing/chainsaw/e2e/backup/chainsaw-test.yaml b/testing/chainsaw/e2e/backup/chainsaw-test.yaml new file mode 100644 index 0000000..23719ef --- /dev/null +++ b/testing/chainsaw/e2e/backup/chainsaw-test.yaml @@ -0,0 +1,658 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-replica-create-backup +spec: + bindings: + - name: cluster + value: check-replica-create-backup + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=check-replica-create-backup', + 'postgres-operator.crunchydata.com/pgbackrest-backup=replica-create', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: Verify COMMAND_OPTS matches "--stanza=db --repo=1" + try: + - sleep: + duration: 10s + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + - name: "SELECTOR" + value: ($backupSelector) + entrypoint: kubectl + args: + - get + - pod + - -n + - $NAMESPACE + - -l + - $SELECTOR + - -o + - jsonpath="{.items[*].spec.containers[*].env[?(@.name=='COMMAND_OPTS')].value}" + check: + (contains($stdout, '--stanza=db --repo=1')): true +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-backup-command +spec: + bindings: + - name: cluster + value: check-backup-command + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=check-backup-command', + 'postgres-operator.crunchydata.com/pgbackrest-backup=manual', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: "Sleep 10s" + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Trigger a CLI backup with --options="--type=full" + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: kubectl-pgo + args: + - -n + - $NAMESPACE + - backup + - $CLUSTER + - --repoName + - repo1 + - --options + - --type=full + - sleep: + duration: 10s + + - name: "Sleep 10s" + try: + - sleep: + duration: 10s + + - name: Verify COMMAND_OPTS matches "--stanza=db --repo=1 --type=full" + try: + - sleep: + duration: 10s + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + - name: "SELECTOR" + value: ($backupSelector) + entrypoint: kubectl + args: + - get + - pod + - -n + - $NAMESPACE + - -l + - $SELECTOR + - -o + - jsonpath="{.items[*].spec.containers[*].env[?(@.name=='COMMAND_OPTS')].value}" + check: + ($stdout): '"--stanza=db --repo=1 --type=full"' + + - name: Wait for CLI backup to succeed" + try: + - wait: + apiVersion: v1 + kind: Pod + selector: ($backupSelector) + timeout: 300s + for: + jsonPath: + path: '{.status.phase}' + value: 'Succeeded' +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-backup-command-longer-options +spec: + bindings: + - name: cluster + value: check-backup-command-longer-options + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=check-backup-command-longer-options', + 'postgres-operator.crunchydata.com/pgbackrest-backup=manual', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Trigger a backup through CLI with longer options + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: kubectl-pgo + args: + - -n + - $NAMESPACE + - backup + - $CLUSTER + - --repoName + - repo1 + - --options + - --type=full --start-fast=n + - sleep: + duration: 10s + + - name: Verify COMMAND_OPTS matches "--stanza=db --repo=1 --type=full --start-fast=n" + try: + - sleep: + duration: 10s + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + - name: "SELECTOR" + value: ($backupSelector) + entrypoint: kubectl + args: + - get + - pod + - -n + - $NAMESPACE + - -l + - $SELECTOR + - -o + - jsonpath="{.items[*].spec.containers[*].env[?(@.name=='COMMAND_OPTS')].value}" + check: + ($stdout): '"--stanza=db --repo=1 --type=full --start-fast=n"' + + - name: Wait for CLI backup to succeed" + try: + - wait: + apiVersion: v1 + kind: Pod + selector: ($backupSelector) + timeout: 300s + for: + jsonPath: + path: '{.status.phase}' + value: 'Succeeded' +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-backup-command-multiple-flags +spec: + bindings: + - name: cluster + value: check-backup-command-multiple-flags + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=check-backup-command-multiple-flags', + 'postgres-operator.crunchydata.com/pgbackrest-backup=manual', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: "Sleep 10s" + try: + - sleep: + duration: 10s + + - name: Trigger a backup through CLI with multiple flags + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: kubectl-pgo + args: + - -n + - $NAMESPACE + - backup + - $CLUSTER + - --repoName + - repo1 + - --options + - --type=full + - --options + - --start-fast=n + - sleep: + duration: 10s + + - name: Verify COMMAND_OPTS matches "--stanza=db --repo=1 --type=full --start-fast=n" + try: + - sleep: + duration: 10s + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + - name: "SELECTOR" + value: ($backupSelector) + entrypoint: kubectl + args: + - get + - pod + - -n + - $NAMESPACE + - -l + - $SELECTOR + - -o + - jsonpath="{.items[*].spec.containers[*].env[?(@.name=='COMMAND_OPTS')].value}" + check: + ($stdout): '"--stanza=db --repo=1 --type=full --start-fast=n"' + + - name: Wait for CLI backup to succeed" + try: + - wait: + apiVersion: v1 + kind: Pod + selector: ($backupSelector) + timeout: 300s + for: + jsonPath: + path: '{.status.phase}' + value: 'Succeeded' +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-backup-command-no-flags +spec: + bindings: + - name: cluster + value: check-backup-command-no-flags + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=check-backup-command-no-flags', + 'postgres-operator.crunchydata.com/pgbackrest-backup=manual', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: Trigger a backup through CLI with no flags + try: + - description: Get prior annotation + command: + outputs: + - name: prior_annotation + value: ($stdout) + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + - name: "GOTEMPLATE" + value: go-template={{ if .metadata.annotations }}{{ index .metadata.annotations "postgres-operator.crunchydata.com/pgbackrest-backup" }}{{ else }}not-found{{ end }} + entrypoint: kubectl + args: + - get + - postgrescluster + - $CLUSTER + - -n + - $NAMESPACE + - --output + - '$GOTEMPLATE' + + + - description: Print Prior Annotation + script: + env: + - name: INPUT + value: ($prior_annotation) + content: | + echo $(date) + echo $INPUT + + - description: Start backup with PGO CLI + command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: kubectl-pgo + args: + - -n + - $NAMESPACE + - backup + - $CLUSTER + - --repoName + - repo1 + - sleep: + duration: 10s + + - description: Get current annotation + command: + outputs: + - name: current_annotation + value: ($stdout) + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + - name: "GOTEMPLATE" + value: go-template={{ if .metadata.annotations }}{{ index .metadata.annotations "postgres-operator.crunchydata.com/pgbackrest-backup" }}{{ else }}not-found{{ end }} + entrypoint: kubectl + args: + - get + - postgrescluster + - $CLUSTER + - -n + - $NAMESPACE + - --output + - '$GOTEMPLATE' + + - description: Create ConfigMap to compare annotations + apply: + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + name: compare-annotations + data: + key1: ($prior_annotation) + key2: ($current_annotation) + + - description: Compare annotations + assert: + timeout: 30s + resource: + apiVersion: v1 + kind: ConfigMap + metadata: + name: compare-annotations + data: + (key1 != key2): true + (key2 != 'not-found'): true + + - name: Wait for CLI backup to succeed" + try: + - wait: + apiVersion: v1 + kind: Pod + selector: ($backupSelector) + timeout: 300s + for: + jsonPath: + path: '{.status.phase}' + value: 'Succeeded' +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-backup-command-force-conflicts +spec: + bindings: + - name: cluster + value: check-backup-command-force-conflicts + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: Change ownership of spec.backups.pgbackrest.manual.repoName + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + envsubst < change-ownership.yaml | kubectl apply --server-side -n ${NAMESPACE} --field-manager=test-manager -f - + + - assert: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + spec: + backups: + pgbackrest: + manual: + repoName: repo3 + + + - name: Attempt a backup on Repo1 without force conflicts and get an error + try: + - description: Start backup with PGO CLI + command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: kubectl-pgo + args: + - -n + - $NAMESPACE + - backup + - $CLUSTER + - --repoName + - repo1 + - --options + - --type=full + check: + (contains($stderr, 'conflict with "test-manager"')): true + - sleep: + duration: 10s + + - name: Attempt a backup on Repo1 with force conflicts and succeed + try: + - description: Start backup with PGO CLI + command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: kubectl-pgo + args: + - -n + - $NAMESPACE + - backup + - $CLUSTER + - --repoName + - repo1 + - --options + - --type=full + - --force-conflicts + check: + (contains($stdout, 'backup initiated')): true + - sleep: + duration: 10s \ No newline at end of file diff --git a/testing/chainsaw/e2e/backup/change-ownership.yaml b/testing/chainsaw/e2e/backup/change-ownership.yaml new file mode 100644 index 0000000..b126e6a --- /dev/null +++ b/testing/chainsaw/e2e/backup/change-ownership.yaml @@ -0,0 +1,9 @@ +apiVersion: postgres-operator.crunchydata.com/v1beta1 +kind: PostgresCluster +metadata: + name: ${CLUSTER} +spec: + backups: + pgbackrest: + manual: + repoName: repo3 \ No newline at end of file diff --git a/testing/chainsaw/e2e/config.yaml b/testing/chainsaw/e2e/config.yaml new file mode 100644 index 0000000..caa43a9 --- /dev/null +++ b/testing/chainsaw/e2e/config.yaml @@ -0,0 +1,12 @@ +apiVersion: chainsaw.kyverno.io/v1alpha2 +kind: Configuration +metadata: + name: end-to-end +spec: + namespace: + template: + metadata: + labels: { postgres-operator-test: chainsaw } + timeouts: + assert: 3m + cleanup: 3m diff --git a/testing/chainsaw/e2e/create/chainsaw-test.yaml b/testing/chainsaw/e2e/create/chainsaw-test.yaml new file mode 100644 index 0000000..710ab0c --- /dev/null +++ b/testing/chainsaw/e2e/create/chainsaw-test.yaml @@ -0,0 +1,146 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: create-cluster +spec: + bindings: + - name: cluster + value: create-cluster + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: psql + value: + image: ($values.images.psql) + connect: { name: PGCONNECT_TIMEOUT, value: '5' } + + - name: confirmQuery + value: | + DO $script$ + DECLARE + pg_is_in_recovery boolean; + pgVer TEXT; + BEGIN + SELECT pg_is_in_recovery() INTO pg_is_in_recovery; + SELECT postgresVersion into pgVer; + RAISE NOTICE 'pgVer is %', pgVer; + RAISE NOTICE 'version() is %', version(); + ASSERT pg_is_in_recovery = FALSE AND + position('PostgreSQL ' || pgVer in version()) > 0; + END $script$; + + steps: + + - name: 'Create Cluster with PGO CLI' + use: + template: '../templates/create-cluster.yaml' + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: 'Verify Postgres is running correct version' + description: > + Confirm that Postgres is running and using the expected version + use: + template: '../templates/psql-data.yaml' + with: + bindings: + - name: target + value: ($cluster) + - name: job + value: (join('-', [$cluster, 'verify-version'])) + - name: command + value: (replace_all($confirmQuery, 'postgresVersion', $postgresVersion)) +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: create-cluster-without-backups +spec: + bindings: + - name: cluster + value: create-cluster-without-backups + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: psql + value: + image: ($values.images.psql) + connect: { name: PGCONNECT_TIMEOUT, value: '5' } + + - name: confirmQuery + value: | + DO $script$ + DECLARE + pg_is_in_recovery boolean; + pgVer TEXT; + BEGIN + SELECT pg_is_in_recovery() INTO pg_is_in_recovery; + SELECT postgresVersion into pgVer; + RAISE NOTICE 'pgVer is %', pgVer; + RAISE NOTICE 'version() is %', version(); + ASSERT pg_is_in_recovery = FALSE AND + position('PostgreSQL ' || pgVer in version()) > 0; + END $script$; + + steps: + + - name: 'Create Cluster with PGO CLI' + use: + template: '../templates/create-cluster-without-backups.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Verify Postgres is running correct version' + description: > + Confirm that Postgres is running and using the expected version + use: + template: '../templates/psql-data.yaml' + with: + bindings: + - name: target + value: ($cluster) + - name: job + value: (join('-', [$cluster, 'verify-version'])) + - name: command + value: (replace_all($confirmQuery, 'postgresVersion', $postgresVersion)) + + - name: Confirm spec.backups is not in the manifest + try: + - error: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + spec: + backups: {} \ No newline at end of file diff --git a/testing/chainsaw/e2e/delete/chainsaw-test.yaml b/testing/chainsaw/e2e/delete/chainsaw-test.yaml new file mode 100644 index 0000000..731afd1 --- /dev/null +++ b/testing/chainsaw/e2e/delete/chainsaw-test.yaml @@ -0,0 +1,88 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: chainsaw-delete-cluster-no +spec: + bindings: + - name: cluster + value: chainsaw-delete-cluster-no + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: run 'delete cluster' with confirm 'n' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo 'n' | kubectl pgo delete postgrescluster $CLUSTER --namespace=$NAMESPACE" + timeout: 10s + + - name: confirm cluster did not delete + try: + - assert: + timeout: 30s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + namespace: ($namespace) +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: chainsaw-delete-cluster-yes +spec: + bindings: + - name: cluster + value: chainsaw-delete-cluster-yes + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: run 'delete cluster' with confirm 'y' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo 'y' | kubectl pgo delete postgrescluster $CLUSTER --namespace=$NAMESPACE" + timeout: 10s + + - name: "Sleep 60s" + try: + - sleep: + duration: 60s + + - name: confirm cluster deleted + try: + - error: + timeout: 500s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + namespace: ($namespace) diff --git a/testing/chainsaw/e2e/restore/chainsaw-test.yaml b/testing/chainsaw/e2e/restore/chainsaw-test.yaml new file mode 100644 index 0000000..e55022a --- /dev/null +++ b/testing/chainsaw/e2e/restore/chainsaw-test.yaml @@ -0,0 +1,713 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-restore-no-repo-name +spec: + bindings: + - name: cluster + value: check-restore-no-repo-name + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=CLUSTER', + 'postgres-operator.crunchydata.com/pgbackrest-backup=BACKUP_TYPE', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: run 'restore' with no options + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "kubectl-pgo restore $CLUSTER -n $NAMESPACE" + timeout: 10s + check: + (contains($stderr, 'Required value')): true + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: Confirm spec.backups.pgbackrest.restore is not in the manifest + try: + - error: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + spec: + backups: + pgbackrest: + restore: {} +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-restore-confirm-no +spec: + bindings: + - name: cluster + value: check-restore-confirm-no + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=CLUSTER', + 'postgres-operator.crunchydata.com/pgbackrest-backup=BACKUP_TYPE', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: run 'restore' with confirm 'no' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo no | kubectl-pgo restore $CLUSTER -n $NAMESPACE --repoName repo1" + timeout: 10s + check: + (contains($stderr, '')): true + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: Confirm spec.backups.pgbackrest.restore is not in the manifest + try: + - error: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + spec: + backups: + pgbackrest: + restore: {} +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-restore-confirm-yes +spec: + bindings: + - name: cluster + value: check-restore-confirm-yes + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=CLUSTER', + 'postgres-operator.crunchydata.com/pgbackrest-backup=BACKUP_TYPE', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: run 'restore' with confirm 'yes' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo yes | kubectl-pgo restore $CLUSTER -n $NAMESPACE --repoName repo1" + timeout: 10s + check: + (contains($stdout, 'patched')): true + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: Confirm restore annotation is present + try: + - assert: + timeout: 60s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + annotations: + postgres-operator.crunchydata.com/pgbackrest-restore: {} + + - name: Confirm manifest is correct after the restore + try: + - assert: + timeout: 60s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + spec: + backups: + pgbackrest: + restore: + enabled: true + repoName: repo1 + status: + pgbackrest: + restore: + succeeded: 1 + instances: + - replicas: 2 +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-restore-confirm-yes-with-options +spec: + bindings: + - name: cluster + value: check-restore-confirm-yes-with-options + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=CLUSTER', + 'postgres-operator.crunchydata.com/pgbackrest-backup=BACKUP_TYPE', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: run 'restore' with confirm 'yes' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo yes | kubectl-pgo restore $CLUSTER -n $NAMESPACE --repoName repo1 --options=--buffer-size=8MiB --options=--io-timeout=120 --options=--process-max=2" + timeout: 10s + check: + (contains($stdout, 'patched')): true + (contains($stdout, 'options:[--buffer-size=8MiB --io-timeout=120 --process-max=2] repoName:repo1')): true + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: Confirm restore annotation is present + try: + - assert: + timeout: 60s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + annotations: + postgres-operator.crunchydata.com/pgbackrest-restore: {} + + - name: Confirm manifest is correct after the restore + try: + - assert: + timeout: 60s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + spec: + backups: + pgbackrest: + restore: + enabled: true + repoName: repo1 + options: + - --buffer-size=8MiB + - --io-timeout=120 + - --process-max=2 + status: + pgbackrest: + restore: + succeeded: 1 + instances: + - replicas: 2 +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-restore-disable-restores +spec: + bindings: + - name: cluster + value: check-restore-disable-restores + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=CLUSTER', + 'postgres-operator.crunchydata.com/pgbackrest-backup=BACKUP_TYPE', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: run 'restore disable' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "kubectl-pgo restore disable $CLUSTER -n $NAMESPACE" + timeout: 10s + check: + (contains($stdout, 'patched')): true + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: Confirm spec.backups.pgbackrest.restore is not in the manifest + try: + - error: + timeout: 60s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + spec: + backups: + pgbackrest: + restore: {} +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-restore-predefined-settings +spec: + bindings: + - name: cluster + value: check-restore-predefined-settings + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=CLUSTER', + 'postgres-operator.crunchydata.com/pgbackrest-backup=BACKUP_TYPE', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: Change ownership of spec.backups.pgbackrest.restore to kubectl-patch + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + kubectl patch postgrescluster $CLUSTER -n $NAMESPACE --type=merge -p '{ + "spec": { + "backups": { + "pgbackrest": { + "restore": { "enabled": true, "repoName": "repo1" } + } + } + } + }' + check: + (contains($stdout, 'patched')): true + + - name: run 'restore' with confirm 'yes' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo yes | kubectl-pgo restore $CLUSTER -n $NAMESPACE --repoName repo1 --options=--buffer-size=8MiB" + timeout: 10s + check: + (contains($stdout, 'patched')): true + (contains($stdout, 'options:[--buffer-size=8MiB] repoName:repo1')): true + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: Confirm restore annotation is present + try: + - assert: + timeout: 60s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + annotations: + postgres-operator.crunchydata.com/pgbackrest-restore: {} + + - name: Confirm manifest is correct after the restore + try: + - assert: + timeout: 60s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + spec: + backups: + pgbackrest: + restore: + enabled: true + repoName: repo1 + options: + - --buffer-size=8MiB + status: + pgbackrest: + restore: + succeeded: 1 + instances: + - replicas: 2 +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: check-restore-force-conflicts +spec: + bindings: + - name: cluster + value: check-restore-force-conflicts + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + - name: backupSelector + value: (join(',', [ + 'postgres-operator.crunchydata.com/cluster=CLUSTER', + 'postgres-operator.crunchydata.com/pgbackrest-backup=BACKUP_TYPE', + 'postgres-operator.crunchydata.com/pgbackrest-repo=repo1' + ])) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 10s + try: + - sleep: + duration: 10s + + - name: Change ownership of spec.backups.pgbackrest.restore to kubectl-patch + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + kubectl patch postgrescluster $CLUSTER -n $NAMESPACE --type=merge -p '{ + "spec": { + "backups": { + "pgbackrest": { + "restore": { "enabled": false, "repoName": "repo1" } + } + } + } + }' + check: + (contains($stdout, 'patched')): true + + - name: Change ownership of annotation to kubectl-annotate + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + kubectl --namespace "${NAMESPACE}" annotate postgrescluster/$CLUSTER \ + --overwrite 'postgres-operator.crunchydata.com/pgbackrest-restore=anything' + check: + (contains($stdout, 'annotated')): true + + - name: run 'restore' with confirm 'yes' and expect error + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo yes | kubectl-pgo restore $CLUSTER -n $NAMESPACE --repoName repo1" + timeout: 10s + check: + (contains($stderr, 'Apply failed')): true + (contains($stderr, '2 conflicts')): true + + - name: run 'restore' with confirm 'yes' and --force-conflicts + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + pgbackrest_restore_annotation() { + kubectl --namespace "${NAMESPACE}" get postgrescluster/$CLUSTER \ + --output "jsonpath-as-json={.metadata.annotations['postgres-operator\.crunchydata\.com/pgbackrest-restore']}" + } + kubectl --namespace "${NAMESPACE}" annotate postgrescluster/$CLUSTER \ + postgres-operator.crunchydata.com/pgbackrest-restore="$(date)" --overwrite || exit + PRIOR=$(pgbackrest_restore_annotation) + # Running restore will update the annotation. + echo yes | kubectl-pgo --namespace="${NAMESPACE}" restore $CLUSTER --repoName="repo1" --force-conflicts + CURRENT=$(pgbackrest_restore_annotation) + if [ "${CURRENT}" != "${PRIOR}" ]; then + echo "restore was successful" + exit 0 + fi + printf 'Expected annotation to change, got PRIOR %s CURRENT %s' "${PRIOR}" "${CURRENT}" + echo "RESULT from taking restore: ${RESULT}" + exit 1 + check: + (contains($stdout, 'restore was successful')): true diff --git a/testing/chainsaw/e2e/show/chainsaw-test.yaml b/testing/chainsaw/e2e/show/chainsaw-test.yaml new file mode 100644 index 0000000..2645f31 --- /dev/null +++ b/testing/chainsaw/e2e/show/chainsaw-test.yaml @@ -0,0 +1,402 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: chainsaw-show +spec: + bindings: + - name: cluster + value: chainsaw-show + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + + - name: Show pgbackrest info + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + + SELECTOR=postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master + PRIMARY=$(kubectl get pod --namespace "${NAMESPACE}" --output name --selector ${SELECTOR}) + EXEC_INFO=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- \ + pgbackrest info + ) + CLI_INFO=$( + kubectl-pgo --namespace "${NAMESPACE}" show backup $CLUSTER + ) + status=$? + if [ "$status" -ne 0 ]; then + echo "pgo command unsuccessful" + exit 1 + fi + + # check command output is not empty and equals 'exec' output + if [ -n "$CLI_INFO" ] && [ "$EXEC_INFO" = "$CLI_INFO" ]; then + exit 0 + fi + + exit 1 + + - name: Show pgbackrest info repo1 + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + + SELECTOR=postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master + PRIMARY=$(kubectl get pod --namespace "${NAMESPACE}" --output name --selector ${SELECTOR}) + EXEC_INFO=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- \ + pgbackrest info --repo=1 + ) + CLI_INFO=$( + kubectl-pgo --namespace "${NAMESPACE}" show backup $CLUSTER --repoName=repo1 + ) + status=$? + if [ "$status" -ne 0 ]; then + echo "pgo command unsuccessful" + exit 1 + fi + + # check command output is not empty and equals 'exec' output + if [ -n "$CLI_INFO" ] && [ "$EXEC_INFO" = "$CLI_INFO" ]; then + exit 0 + fi + + exit 1 + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: Show pgbackrest info output json + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + + SELECTOR=postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master + PRIMARY=$(kubectl get pod --namespace "${NAMESPACE}" --output name --selector ${SELECTOR}) + EXEC_INFO=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- \ + pgbackrest info --output=json + ) + CLI_INFO=$( + kubectl-pgo --namespace "${NAMESPACE}" show backup $CLUSTER --output=json + ) + status=$? + if [ "$status" -ne 0 ]; then + echo "pgo command unsuccessful" + exit 1 + fi + + # check command output is not empty and equals 'exec' output + if [ -n "$CLI_INFO" ] && [ "$EXEC_INFO" = "$CLI_INFO" ]; then + exit 0 + fi + + exit 1 + + - name: Show pgbackrest info output bad format + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + + SELECTOR=postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master + PRIMARY=$(kubectl get pod --namespace "${NAMESPACE}" --output name --selector ${SELECTOR}) + CLI_INFO=$( + kubectl-pgo --namespace "${NAMESPACE}" show backup $CLUSTER -o bad 2>&1 + ) + status=$? + if [ "$status" -ne 1 ]; then + echo "expected bad format to fail" + exit 1 + fi + + # check command output is not empty and contains the expected error + # Note: case is used as it allows for the use of a wildcard (*) + # and is POSIX compliant + case "$CLI_INFO" in + "") + exit 1 + ;; + *"must be one of \"text\", \"json\""*) + exit 0 + ;; + esac + + exit 1 + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + + - name: Show patronictl list + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + + SELECTOR=postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master + PRIMARY=$(kubectl get pod --namespace "${NAMESPACE}" --output name --selector ${SELECTOR}) + EXEC_INFO=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- \ + patronictl list + ) + CLI_HA=$( + kubectl-pgo --namespace "${NAMESPACE}" show ha $CLUSTER + ) + status=$? + if [ "$status" -ne 0 ]; then + echo "pgo command unsuccessful" + exit 1 + fi + + # check command output is not empty and equals 'exec' output + if [ -n "$CLI_HA" ] && [ "$EXEC_INFO" = "$CLI_HA" ]; then + exit 0 + fi + + exit 1 + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + + - name: Show patronictl list output json + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + + SELECTOR=postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master + PRIMARY=$(kubectl get pod --namespace "${NAMESPACE}" --output name --selector ${SELECTOR}) + EXEC_INFO=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- \ + patronictl list -f json + ) + CLI_HA=$( + kubectl-pgo --namespace "${NAMESPACE}" show ha $CLUSTER -o json + ) + status=$? + if [ "$status" -ne 0 ]; then + echo "pgo command unsuccessful" + exit 1 + fi + + # check command output is not empty and equals 'exec' output + if [ -n "$CLI_HA" ] && [ "$EXEC_INFO" = "$CLI_HA" ]; then + exit 0 + fi + + exit 1 + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: Show patronictl list output bad + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + + SELECTOR=postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master + PRIMARY=$(kubectl get pod --namespace "${NAMESPACE}" --output name --selector ${SELECTOR}) + + CLI_HA=$( + kubectl-pgo --namespace "${NAMESPACE}" show ha $CLUSTER -o bad 2>&1 + ) + status=$? + if [ "$status" -ne 1 ]; then + echo "expected bad format to fail" + exit 1 + fi + + # check command output contains the expected error + # Note: case is used as it allows for the use of a wildcard (*) + # and is POSIX compliant + case "$CLI_HA" in + "") + exit 1 + ;; + *"must be one of \"pretty\", \"tsv\", \"json\", \"yaml\""*) + exit 0 + ;; + esac + + exit 1 + + + - name: Show user + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + SELECTOR=postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master + PRIMARY=$(kubectl get pod --namespace "${NAMESPACE}" --output name --selector ${SELECTOR}) + + CLI_USER=$( + kubectl-pgo --namespace "${NAMESPACE}" show user --cluster $CLUSTER + ) + + status=$? + if [ "$status" -ne 0 ]; then + echo "pgo command unsuccessful" + exit 1 + fi + + # expected output + SHOW_USER_OUTPUT=" + CLUSTER USERNAME + chainsaw-show chainsaw-show" + + + # check command output is not empty and equals the expected output + if [ -z ${CLI_USER} ] || [ "${CLI_USER}" != "${SHOW_USER_OUTPUT}" ]; then + echo "pgo command output unexpected: expected ${SHOW_USER_OUTPUT} got ${CLI_USER}" + exit 1 + fi + + CLI_USER_SENSITIVE=$( + echo yes | kubectl-pgo --namespace "${NAMESPACE}" show user --cluster ${CLUSTER} --show-connection-info + ) + + SECRET_DATA=$(kubectl get secret -n "${NAMESPACE}" ${CLUSTER}-pguser-${CLUSTER} -o jsonpath={.data}) + + PASSWORD=$(echo "${SECRET_DATA}" | jq -r .password | base64 -d) + USER=$(echo "${SECRET_DATA}" | jq -r .user | base64 -d) + HOST=$(echo "${SECRET_DATA}" | jq -r .host | base64 -d) + PORT=$(echo "${SECRET_DATA}" | jq -r .port | base64 -d) + + # check command output is not empty and contains the connection URL field + # Note: case is used as it allows for the use of a wildcard (*) + # and is POSIX compliant + case "$CLI_USER_SENSITIVE" in + "") + exit 1 + ;; + *"postgres://${USER}:${PASSWORD}@${HOST}:${PORT}/${CLUSTER}"*) + exit 0 + ;; + esac + + echo "pgo command output for connection info unexpected: got ${CLI_USER_SENSITIVE}" + exit 1 + + + - name: Show entire command + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + content: | + SELECTOR=postgres-operator.crunchydata.com/cluster=$CLUSTER,postgres-operator.crunchydata.com/role=master + PRIMARY=$(kubectl get pod --namespace "${NAMESPACE}" --output name --selector ${SELECTOR}) + + PGBACKREST_INFO_EXEC=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- \ + pgbackrest info + ) + + PATRONI_LIST_EXEC=$( + kubectl exec --namespace "${NAMESPACE}" "${PRIMARY}" -- \ + patronictl list + ) + + EXPECTED_SHOW_COMMAND="BACKUP + + $PGBACKREST_INFO_EXEC + + HA + + $PATRONI_LIST_EXEC" + + echo $EXPECTED_SHOW_COMMAND + + CLI_SHOW=$( + kubectl-pgo --namespace "${NAMESPACE}" show $CLUSTER + ) + + status=$? + if [ "$status" -ne 0 ]; then + echo "pgo command unsuccessful" + exit 1 + fi + + # check command output is not empty and equals expected output + if [ -n "$CLI_SHOW" ] && [ "$EXPECTED_SHOW_COMMAND" = "$CLI_SHOW" ]; then + exit 0 + fi + + exit 1 + + diff --git a/testing/chainsaw/e2e/start-stop/chainsaw-test.yaml b/testing/chainsaw/e2e/start-stop/chainsaw-test.yaml new file mode 100644 index 0000000..2017f6d --- /dev/null +++ b/testing/chainsaw/e2e/start-stop/chainsaw-test.yaml @@ -0,0 +1,255 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: stop-cluster-no-with-force-conflicts +spec: + + bindings: + - name: cluster + value: stop-cluster-no-with-force-conflicts + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: run 'stop cluster' with confirm 'n' and force-conflicts + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo 'n' | kubectl-pgo stop $CLUSTER --force-conflicts --namespace=$NAMESPACE" + timeout: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: stop-cluster-yes-with-force-conflicts +spec: + + bindings: + - name: cluster + value: stop-cluster-yes-with-force-conflicts + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: run 'stop cluster' with confirm 'y' and force-conflicts + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo 'y' | kubectl-pgo stop $CLUSTER --force-conflicts --namespace=$NAMESPACE" + timeout: 10s + - sleep: + duration: 10s + + - name: confirm cluster did stop + try: + - assert: + timeout: 30s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + namespace: ($namespace) + spec: + shutdown: true + - sleep: + duration: 10s +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: start-cluster +spec: + + bindings: + - name: cluster + value: start-cluster + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: "Sleep 30s" + try: + - sleep: + duration: 30s + + - name: run 'stop cluster' with confirm 'y' and force-conflicts + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "echo 'y' | kubectl-pgo stop $CLUSTER --force-conflicts --namespace=$NAMESPACE" + timeout: 10s + - sleep: + duration: 10s + + - name: confirm cluster did stop + try: + - assert: + timeout: 30s + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + namespace: ($namespace) + spec: + shutdown: true + - sleep: + duration: 10s + + - name: run 'start cluster' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "kubectl-pgo start $CLUSTER --namespace=$NAMESPACE" + timeout: 10s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' +--- +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: start-cluster-already-running +spec: + + bindings: + - name: cluster + value: start-cluster-already-running + + - name: postgresVersion + value: (to_string($values.versions.postgres)) + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: run 'start cluster' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "sleep 5 && kubectl-pgo start $CLUSTER -n $NAMESPACE" + timeout: 10s + check: + (contains($stdout, 'start initiated')): true + + - name: run 'start cluster' + try: + - command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + entrypoint: "sh" + args: + - "-c" + - "sleep 5 && kubectl-pgo start $CLUSTER -n $NAMESPACE" + timeout: 10s + check: + (contains($stdout, 'Cluster already Started')): true + diff --git a/testing/chainsaw/e2e/support/chainsaw-test.yaml b/testing/chainsaw/e2e/support/chainsaw-test.yaml new file mode 100644 index 0000000..f85a522 --- /dev/null +++ b/testing/chainsaw/e2e/support/chainsaw-test.yaml @@ -0,0 +1,932 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: kuttl-support-cluster +spec: + bindings: + - name: cluster + value: kuttl-support-cluster + + - name: postgresVersion + value: 16 + + - name: psql + value: + image: ($values.images.psql) + connect: { name: PGCONNECT_TIMEOUT, value: '5' } + + - name: confirmQuery + value: | + DO $script$ + DECLARE + pg_is_in_recovery boolean; + pgVer TEXT; + BEGIN + SELECT pg_is_in_recovery() INTO pg_is_in_recovery; + SELECT postgresVersion into pgVer; + RAISE NOTICE 'pgVer is %', pgVer; + RAISE NOTICE 'version() is %', version(); + ASSERT pg_is_in_recovery = FALSE AND + position('PostgreSQL ' || pgVer in version()) > 0; + END $script$; + + steps: + + - name: 'Create Cluster from manifest' + use: + template: '../templates/create-cluster-from-manifest.yaml' + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: 'Confirm cluster is created' + use: + template: '../templates/confirm-created.yaml' + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: 'Confirm Replica backup completed' + use: + template: '../templates/replica-backup-complete.yaml' + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: Run support export + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: kuttl-support-cluster + content: | + kubectl-pgo --namespace $NAMESPACE --operator-namespace postgres-operator support export $CLUSTER -o . + check: + (contains($stdout, 'PGO CLI Support Export Tool')): true + + - name: Expand support export + try: + - script: + content: | + tar -xzf ./crunchy_k8s_support_export_*.tar.gz + + + - name: Check support export + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: kuttl-support-cluster + content: | + CLEANUP="rm -r ./kuttl-support-cluster ./operator ./crunchy_k8s_support_export_*.tar.gz" + check_file() { + if [ ! -s ./"${1}" ] + then + echo "Expected ${1} file to not be empty" + eval "$CLEANUP" + exit 1 + else + echo "Found ${1}" + fi + } + check_exists() { + if [ -f ./"${1}" ] + then + echo "Expected ${1} file to exist" + eval "$CLEANUP" + exit 1 + else + echo "Found ${1}" + fi + } + + # check that the PGO CLI version is recorded + VER=$(cat ./kuttl-support-cluster/pgo-cli-version) + echo "$VER" | grep -E "v[0-9]+\.[0-9]+\.[0-9]+$" + STATUS=$? + [ "$STATUS" = 0 ] || { + echo "Expected PGO CLI version, got:" + echo "${VER}" + eval "$CLEANUP" + exit 1 + } + + # check that the cluster-names file exists and is not empty + check_file "kuttl-support-cluster/cluster-names" + + # check that the system-time file exists and is not empty + check_file "kuttl-support-cluster/system-time" + + # check that the context file exists and is not empty + check_file "kuttl-support-cluster/current-context" + + # check that the patroni info file exists and is not empty + check_file "kuttl-support-cluster/patroni-info" + + # check that the pgbackrest info file exists and is not empty + check_file "kuttl-support-cluster/pgbackrest-info" + + # check that the plugin list file exists and is not empty + # the file will at least include kubectl-pgo + check_file "kuttl-support-cluster/plugin-list" + + # check that the operator file exists and is not empty + # the list file will not be empty for the requested Kubernetes types + check_file "operator/deployments/list" + check_file "operator/replicasets/list" + check_file "operator/pods/list" + + # check for expected gzip compression level + FILE_INFO=$(file ./crunchy_k8s_support_export_*.tar.gz) + case "${FILE_INFO}" in + *'gzip compressed data, max compression'*) + ;; + *) + echo "Expected gzip max compression message, got:" + echo "${FILE_INFO}" + eval "$CLEANUP" + exit 1 + ;; + esac + + # Node directory and list file path + DIR="./kuttl-support-cluster/nodes/" + LIST="${DIR}list" + + # check for expected table header in the list file + KV=$(awk 'NR==1 {print $9}' $LIST) + [ "${KV}" = '|KERNEL-VERSION' ] || { + echo "Expected KERNEL-VERSION header, got:" + echo "${KV}" + eval "$CLEANUP" + exit 1 + } + + # check for a .yaml file with the name of the first Node in the list file + NODE="$(awk 'NR==2 {print $1}' $LIST).yaml" + + if [ ! -f "${DIR}${NODE}" ] + then + echo "Expected directory with file ${NODE}, got:" + ls ${DIR} + eval "$CLEANUP" + exit 1 + fi + + # check that the events file exist and is not empty + check_file "kuttl-support-cluster/events" + + # check that logs exist for the PG + # use `check_exists` so we can use a wildcard + check_exists "kuttl-support-cluster/pods/kuttl-support-cluster-00-*-0/pgdata/pg16/log/postgresql-*.log" + + EVENTS="./kuttl-support-cluster/events" + # check that the events file contains the expected string + if ! grep -Fq "Started container postgres-startup" $EVENTS + then + echo "Events file does not contain expected string" + eval "$CLEANUP" + exit 1 + fi + + PROCESSES_DIR="./kuttl-support-cluster/processes/" + + # Check for the files that contain an expected pgBackRest server process. + # Expected to be found in the Postgres instance Pod's 'database', + # 'replication-cert-copy', 'pgbackrest', and 'pgbackrest-config' containers + # and the pgBackRest repo Pod's 'pgbackrest' and 'pgbackrest-config' + # containers, i.e. 6 files total, but test will pass if at least one is found. + found=$(grep -lR "pgbackrest server" ${PROCESSES_DIR} | wc -l) + if [ "${found}" -lt 1 ]; then + echo "Expected to find pgBackRest process, got ${found}" + eval "$CLEANUP" + exit 1 + fi + + # Check for the files that contain an expected Postgres process. Expected + # to be found in the Postgres instance Pod's 'database', 'replication-cert-copy', + # 'pgbackrest', and 'pgbackrest-config' containers, i.e. 4 files total, but + # test will pass if at least one is found. + found=$(grep -lR "postgres -D /pgdata/pg" ${PROCESSES_DIR} | wc -l) + if [ "${found}" -lt 1 ]; then + echo "Expected to find Postgres process, got ${found}" + eval "$CLEANUP" + exit 1 + fi + + # check that the PGO CLI log file contains expected messages + CLI_LOG="./kuttl-support-cluster/cli.log" + + # info output includes expected heading + if ! grep -Fq -- "- INFO - | PGO CLI Support Export Tool" $CLI_LOG + then + echo "PGO CLI log does not contain expected info message" + eval "$CLEANUP" + exit 1 + fi + + # debug output includes cluster name argument + if ! grep -Fq -- "- DEBUG - Arg - PostgresCluster Name: kuttl-support-cluster" $CLI_LOG + then + echo "PGO CLI log does not contain cluster name debug message" + eval "$CLEANUP" + exit 1 + fi + + rm -r ./kuttl-support-cluster ./operator ./crunchy_k8s_support_export_*.tar.gz + + + - name: Run support export for invalid cluster + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: kuttl-support-cluster + content: | + kubectl-pgo --namespace $NAMESPACE --operator-namespace postgres-operator support export invalid -o . + check: + (contains($stderr, '"invalid" not found')): true + + + + - name: 'Create Limit Range' + try: + - apply: + resource: + apiVersion: v1 + kind: LimitRange + metadata: + name: kuttl-test-limitrange + spec: + limits: + - type: PersistentVolumeClaim + max: + storage: 2Gi + min: + storage: 500Mi + + - name: 'Create Ingress' + try: + - apply: + resource: + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: kuttl-test-ingress + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + spec: + ingressClassName: simple-example + rules: + - http: + paths: + - path: /testpath + pathType: Prefix + backend: + service: + name: test + port: + number: 80 + + + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: Run support export + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: kuttl-support-cluster + content: | + kubectl-pgo --namespace $NAMESPACE --operator-namespace postgres-operator support export $CLUSTER -o . + check: + (contains($stdout, 'PGO CLI Support Export Tool')): true + + - name: Expand support export + try: + - script: + content: | + tar -xzf ./crunchy_k8s_support_export_*.tar.gz + + + - name: Check support export + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: kuttl-support-cluster + content: | + CLEANUP="rm -r ./kuttl-support-cluster ./crunchy_k8s_support_export_*.tar.gz" + + # LimitRange directory and list file path + LR_DIR="./kuttl-support-cluster/limitranges/" + LR_LIST="${LR_DIR}list" + + # check for limitrange object name + LRO=$(awk 'NR==2 {print $1}' "${LR_LIST}") + [ "${LRO}" = 'kuttl-test-limitrange' ] || { + echo "Expected 'kuttl-test-limitrange' limitrange, got:" + echo "${LRO}" + eval "$CLEANUP" + exit 1 + } + + # check for a .yaml file for the limitrange object + LR_FILE="${LR_DIR}kuttl-test-limitrange.yaml" + if [ ! -f ${LR_FILE} ] + then + echo "Expected directory with file, ${LR_FILE}, got:" + ls ${LR_DIR} + eval "$CLEANUP" + exit 1 + fi + + # Ingress directory and list file path + I_DIR="./kuttl-support-cluster/ingresses/" + I_LIST="${I_DIR}list" + + # check for ingress object name + IO=$(awk 'NR==2 {print $1}' ${I_LIST}) + [ "${IO}" = 'kuttl-test-ingress' ] || { + echo "Expected 'kuttl-test-ingress' ingress, got:" + echo "${IO}" + eval "$CLEANUP" + exit 1 + } + + # check for a .yaml file for the ingress object + I_FILE="${I_DIR}kuttl-test-ingress.yaml" + if [ ! -f ${I_FILE} ] + then + echo "Expected directory with file, ${I_FILE}, got:" + ls ${I_DIR} + eval "$CLEANUP" + exit 1 + fi + + rm -r ./kuttl-support-cluster ./operator ./crunchy_k8s_support_export_*.tar.gz + + + + - name: Delete resources + try: + - delete: + ref: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: kuttl-support-cluster + - delete: + ref: + apiVersion: v1 + kind: LimitRange + name: kuttl-test-limitrange + - delete: + ref: + apiVersion: networking.k8s.io/v1 + kind: Ingress + name: kuttl-test-ingress + + + - name: 'Create Monitoring Cluster' + try: + - apply: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: kuttl-support-monitoring-cluster + spec: + postgresVersion: 16 + instances: + - dataVolumeClaimSpec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 1Gi } } + backups: + pgbackrest: + repos: + - name: repo1 + volume: + volumeClaimSpec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 1Gi } } + monitoring: + pgmonitor: + exporter: {} + - apply: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: crunchy-prometheus + spec: + selector: + matchLabels: + app.kubernetes.io/name: postgres-operator-monitoring + name: crunchy-prometheus + template: + metadata: + labels: + app.kubernetes.io/name: postgres-operator-monitoring + name: crunchy-prometheus + spec: + containers: + - image: prom/prometheus:v2.33.5 + name: prometheus + # Override default command to avoid 'permission denied' error in some + # environments (e.g. Openshift). Echo 'hello' so the Pod log is not empty. + command: ["sh", "-c", "while true; do echo hello; sleep 10;done"] + - apply: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: crunchy-grafana + spec: + selector: + matchLabels: + app.kubernetes.io/name: postgres-operator-monitoring + name: crunchy-grafana + template: + metadata: + labels: + app.kubernetes.io/name: postgres-operator-monitoring + name: crunchy-grafana + spec: + containers: + - image: grafana/grafana:8.5.10 + name: grafana + - apply: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: crunchy-alertmanager + spec: + selector: + matchLabels: + app.kubernetes.io/name: postgres-operator-monitoring + name: crunchy-alertmanager + template: + metadata: + labels: + app.kubernetes.io/name: postgres-operator-monitoring + name: crunchy-alertmanager + spec: + containers: + - image: prom/alertmanager:v0.22.2 + name: alertmanager + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: Assert resources exist + try: + - assert: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: kuttl-support-monitoring-cluster + spec: + backups: + pgbackrest: + repos: + - name: repo1 + volume: + volumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + instances: + - dataVolumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + replicas: 1 + postgresVersion: 16 + status: + instances: + - name: "00" + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + monitoring: {} + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: crunchy-prometheus + status: + availableReplicas: 1 + observedGeneration: 1 + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: crunchy-grafana + status: + availableReplicas: 1 + observedGeneration: 1 + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + - assert: + resource: + apiVersion: apps/v1 + kind: Deployment + metadata: + name: crunchy-alertmanager + status: + availableReplicas: 1 + observedGeneration: 1 + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + + + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: Run support export + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: kuttl-support-monitoring-cluster + content: | + kubectl-pgo --namespace $NAMESPACE support export kuttl-support-monitoring-cluster -o . + check: + (contains($stdout, 'PGO CLI Support Export Tool')): true + + - name: Expand support export + try: + - script: + content: | + tar -xzf ./crunchy_k8s_support_export_*.tar.gz + + + - name: Check support export + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: kuttl-support-monitoring-cluster + content: | + CLEANUP="rm -r ./kuttl-support-monitoring-cluster ./monitoring ./crunchy_k8s_support_export_*.tar.gz" + CLUSTER_DIR="./kuttl-support-monitoring-cluster/pods/" + MONITORING_DIR="./monitoring/pods/" + + # check for exporter, prometheus, grafana and alertmanager logs + found=$(find ${CLUSTER_DIR} -name "*exporter.log" | wc -l) + if [ "${found}" -eq 0 ]; then + echo "exporter not found" + eval "$CLEANUP" + exit 1 + fi + + found=$(find ${MONITORING_DIR} -name "*prometheus.log" | wc -l) + if [ "${found}" -eq 0 ]; then + echo "prometheus not found" + eval "$CLEANUP" + exit 1 + fi + + found=$(find ${MONITORING_DIR} -name "*grafana.log" | wc -l) + if [ "${found}" -eq 0 ]; then + echo "grafana not found" + eval "$CLEANUP" + exit 1 + fi + + found=$(find ${MONITORING_DIR} -name "*alertmanager.log" | wc -l) + if [ "${found}" -eq 0 ]; then + echo "alertmanager not found" + eval "$CLEANUP" + exit 1 + fi + + rm -r ./kuttl-support-monitoring-cluster ./monitoring ./operator ./crunchy_k8s_support_export_*.tar.gz + + + - name: Delete resources + try: + - delete: + ref: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + name: kuttl-support-monitoring-cluster + - delete: + ref: + apiVersion: apps/v1 + kind: Deployment + name: crunchy-prometheus + - delete: + ref: + apiVersion: apps/v1 + kind: Deployment + name: crunchy-alertmanager + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + + - name: 'Create Instrumentation Cluster' + try: + - apply: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: kuttl-support-instrumentation + spec: + postgresVersion: 16 + instances: + - dataVolumeClaimSpec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 1Gi } } + backups: + pgbackrest: + repos: + - name: repo1 + volume: + volumeClaimSpec: + accessModes: [ReadWriteOnce] + resources: { requests: { storage: 1Gi } } + instrumentation: {} + + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + - name: Assert resources exist + try: + - assert: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: kuttl-support-instrumentation + spec: + backups: + pgbackrest: + repos: + - name: repo1 + volume: + volumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + instances: + - dataVolumeClaimSpec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: 1Gi + replicas: 1 + postgresVersion: 16 + status: + instances: + - name: "00" + readyReplicas: 1 + replicas: 1 + updatedReplicas: 1 + monitoring: {} + + + - name: Sleep for 30s + try: + - sleep: + duration: 30s + + + - name: Run support export + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: kuttl-support-monitoring-cluster + content: | + kubectl-pgo --namespace $NAMESPACE --operator-namespace postgres-operator support export kuttl-support-instrumentation -o . + check: + (contains($stdout, 'PGO CLI Support Export Tool')): true + + - name: Expand support export + try: + - script: + content: | + tar -xzf ./crunchy_k8s_support_export_*.tar.gz + + + - name: Check support export + try: + - script: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: kuttl-support-monitoring-cluster + content: | + CLEANUP="rm -r ./kuttl-support-instrumentation ./operator ./crunchy_k8s_support_export_*.tar.gz" + check_file() { + if [ ! -s ./"${1}" ] + then + echo "Expected ${1} file to not be empty" + eval "$CLEANUP" + exit 1 + else + echo "Found ${1}" + fi + } + check_exists() { + if [ -f ./"${1}" ] + then + echo "Expected ${1} file to exist" + eval "$CLEANUP" + exit 1 + else + echo "Found ${1}" + fi + } + + # check that the PGO CLI version is recorded + VER=$(cat ./kuttl-support-instrumentation/pgo-cli-version) + echo "$VER" | grep -E "v[0-9]+\.[0-9]+\.[0-9]+$" + STATUS=$? + [ "$STATUS" = 0 ] || { + echo "Expected PGO CLI version, got:" + echo "${VER}" + eval "$CLEANUP" + exit 1 + } + + # check that the cluster-names file exists and is not empty + check_file "kuttl-support-instrumentation/cluster-names" + + # check that the system-time file exists and is not empty + check_file "kuttl-support-instrumentation/system-time" + + # check that the context file exists and is not empty + check_file "kuttl-support-instrumentation/current-context" + + # check that the patroni info file exists and is not empty + check_file "kuttl-support-instrumentation/patroni-info" + + # check that the pgbackrest info file exists and is not empty + check_file "kuttl-support-instrumentation/pgbackrest-info" + + # check that the plugin list file exists and is not empty + # the file will at least include kubectl-pgo + check_file "kuttl-support-instrumentation/plugin-list" + + # check that the operator file exists and is not empty + # the list file will not be empty for the requested Kubernetes types + check_file "operator/deployments/list" + check_file "operator/replicasets/list" + check_file "operator/pods/list" + + # check for expected gzip compression level + FILE_INFO=$(file ./crunchy_k8s_support_export_*.tar.gz) + case "${FILE_INFO}" in + *'gzip compressed data, max compression'*) + ;; + *) + echo "Expected gzip max compression message, got:" + echo "${FILE_INFO}" + eval "$CLEANUP" + exit 1 + ;; + esac + + # Node directory and list file path + DIR="./kuttl-support-instrumentation/nodes/" + LIST="${DIR}list" + + # check for expected table header in the list file + KV=$(awk 'NR==1 {print $9}' $LIST) + [ "${KV}" = '|KERNEL-VERSION' ] || { + echo "Expected KERNEL-VERSION header, got:" + echo "${KV}" + eval "$CLEANUP" + exit 1 + } + + # check for a .yaml file with the name of the first Node in the list file + NODE="$(awk 'NR==2 {print $1}' $LIST).yaml" + + if [ ! -f "${DIR}${NODE}" ] + then + echo "Expected directory with file ${NODE}, got:" + ls ${DIR} + eval "$CLEANUP" + exit 1 + fi + + # check that the events file exist and is not empty + check_file "kuttl-support-instrumentation/events" + + # check that logs exist for the PG + # use `check_exists` so we can use a wildcard + check_exists "kuttl-support-instrumentation/pods/kuttl-support-cluster-00-*-0/pgdata/logs/postgresql-*.json" + + EVENTS="./kuttl-support-instrumentation/events" + # check that the events file contains the expected string + if ! grep -Fq "Started container postgres-startup" $EVENTS + then + echo "Events file does not contain expected string" + eval "$CLEANUP" + exit 1 + fi + + PROCESSES_DIR="./kuttl-support-instrumentation/processes/" + + # Check for the files that contain an expected pgBackRest server process. + # Expected to be found in the Postgres instance Pod's 'collector', 'database', + # 'replication-cert-copy', 'pgbackrest', and 'pgbackrest-config' containers + # and the pgBackRest repo Pod's 'pgbackrest' and 'pgbackrest-config' + # containers, i.e. 6 files total, but test will pass if at least one is found. + found=$(grep -lR "pgbackrest server" ${PROCESSES_DIR} | wc -l) + if [ "${found}" -lt 1 ]; then + echo "Expected to find pgBackRest process, got ${found}" + eval "$CLEANUP" + exit 1 + fi + + # Check for the files that contain an expected Postgres process. Expected + # to be found in the Postgres instance Pod's 'collector', 'database', 'replication-cert-copy', + # 'pgbackrest', and 'pgbackrest-config' containers, i.e. 5 files total, but + # test will pass if at least one is found. + found=$(grep -lR "postgres -D /pgdata/pg" ${PROCESSES_DIR} | wc -l) + if [ "${found}" -lt 1 ]; then + echo "Expected to find Postgres process, got ${found}" + eval "$CLEANUP" + exit 1 + fi + + # check that the PGO CLI log file contains expected messages + CLI_LOG="./kuttl-support-instrumentation/cli.log" + + # info output includes expected heading + if ! grep -Fq -- "- INFO - | PGO CLI Support Export Tool" $CLI_LOG + then + echo "PGO CLI log does not contain expected info message" + eval "$CLEANUP" + exit 1 + fi + + # debug output includes cluster name argument + if ! grep -Fq -- "- DEBUG - Arg - PostgresCluster Name: kuttl-support-instrumentation" $CLI_LOG + then + echo "PGO CLI log does not contain cluster name debug message" + eval "$CLEANUP" + exit 1 + fi + + rm -r ./kuttl-support-instrumentation ./monitoring ./operator ./crunchy_k8s_support_export_*.tar.gz + diff --git a/testing/chainsaw/e2e/templates/confirm-created.yaml b/testing/chainsaw/e2e/templates/confirm-created.yaml new file mode 100644 index 0000000..7a2cafe --- /dev/null +++ b/testing/chainsaw/e2e/templates/confirm-created.yaml @@ -0,0 +1,24 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: confirm-created +spec: + try: + - description: > + Confirm the cluster is created + assert: + resource: + apiVersion: v1 + kind: Pod + metadata: + labels: + postgres-operator.crunchydata.com/cluster: ($cluster) + status: + (containerStatuses[?name == 'database']): + - name: database + ready: true + catch: + - describe: + apiVersion: v1 + kind: Pod + selector: (concat('postgres-operator.crunchydata.com/cluster=', $cluster)) diff --git a/testing/chainsaw/e2e/templates/create-cluster-from-manifest.yaml b/testing/chainsaw/e2e/templates/create-cluster-from-manifest.yaml new file mode 100644 index 0000000..c1c773f --- /dev/null +++ b/testing/chainsaw/e2e/templates/create-cluster-from-manifest.yaml @@ -0,0 +1,36 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: create-cluster-from-manifest +spec: + try: + - + description: > + Create a standard cluster with two replicas and a single pgBackRest repository + apply: + resource: + apiVersion: postgres-operator.crunchydata.com/v1beta1 + kind: PostgresCluster + metadata: + name: ($cluster) + spec: + postgresVersion: (to_number($postgresVersion)) + instances: + - replicas: 2 + dataVolumeClaimSpec: { accessModes: [ "ReadWriteOnce" ], resources: { requests: { storage: 1Gi } } } + backups: + pgbackrest: + global: + log-level-console: detail + log-level-file: detail + start-fast: 'y' + repos: + - name: repo1 + volume: + volumeClaimSpec: { accessModes: ["ReadWriteOnce"], resources: { requests: { storage: 1Gi } } } + + catch: + - describe: + apiVersion: v1 + kind: Pod + selector: (concat('postgres-operator.crunchydata.com/cluster=', $cluster)) diff --git a/testing/chainsaw/e2e/templates/create-cluster-without-backups.yaml b/testing/chainsaw/e2e/templates/create-cluster-without-backups.yaml new file mode 100644 index 0000000..23e6695 --- /dev/null +++ b/testing/chainsaw/e2e/templates/create-cluster-without-backups.yaml @@ -0,0 +1,27 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: create-cluster-without-backups +spec: + try: + - + description: > + Create a PostgresCluster with the PGO CLI without backups + command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + - name: "PGVERSION" + value: ($postgresVersion) + entrypoint: "sh" + args: + - "-c" + - "echo yes | kubectl-pgo --namespace $NAMESPACE create postgrescluster --pg-major-version $PGVERSION --disable-backups $CLUSTER" + timeout: 10s + catch: + - describe: + apiVersion: v1 + kind: Pod + selector: (concat('postgres-operator.crunchydata.com/cluster=', $cluster)) diff --git a/testing/chainsaw/e2e/templates/create-cluster.yaml b/testing/chainsaw/e2e/templates/create-cluster.yaml new file mode 100644 index 0000000..c3d4faf --- /dev/null +++ b/testing/chainsaw/e2e/templates/create-cluster.yaml @@ -0,0 +1,31 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: create-cluster +spec: + try: + - + description: > + Create a PostgresCluster with the PGO CLI + command: + env: + - name: "NAMESPACE" + value: ($namespace) + - name: "CLUSTER" + value: ($cluster) + - name: "PGVERSION" + value: ($postgresVersion) + entrypoint: "kubectl" + args: + - "pgo" + - "create" + - "postgrescluster" + - "$CLUSTER" + - "--pg-major-version=$PGVERSION" + - "--namespace=$NAMESPACE" + timeout: 10s + catch: + - describe: + apiVersion: v1 + kind: Pod + selector: (concat('postgres-operator.crunchydata.com/cluster=', $cluster)) diff --git a/testing/chainsaw/e2e/templates/psql-data.yaml b/testing/chainsaw/e2e/templates/psql-data.yaml new file mode 100644 index 0000000..d950f82 --- /dev/null +++ b/testing/chainsaw/e2e/templates/psql-data.yaml @@ -0,0 +1,74 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: psql-data +spec: + bindings: + - name: target + value: 'The name of the PostgresCluster' + - name: job + value: 'The name of the job' + - name: command + value: 'The command to run on the psql pod' + + try: + + - command: + entrypoint: 'true' + skipCommandOutput: true + outputs: + - name: secret + value: (join('-', [$target, 'pguser', $target])) + + - + description: > + Run a command through psql + apply: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: ($job) + spec: + backoffLimit: 3 + template: + spec: + restartPolicy: Never + containers: + - name: psql + image: ($psql.image) + env: + - ($psql.connect) + - name: PGHOST + valueFrom: { secretKeyRef: { name: ($secret), key: host } } + - name: PGPORT + valueFrom: { secretKeyRef: { name: ($secret), key: port } } + - name: PGDATABASE + valueFrom: { secretKeyRef: { name: ($secret), key: dbname } } + - name: PGUSER + valueFrom: { secretKeyRef: { name: ($secret), key: user } } + - name: PGPASSWORD + valueFrom: { secretKeyRef: { name: ($secret), key: password } } + command: + - psql + - -qa + - --set=ON_ERROR_STOP=1 + - --command + - ($command) + + - assert: + resource: + apiVersion: batch/v1 + kind: Job + metadata: + name: ($job) + status: + succeeded: 1 + + catch: + - + description: > + Read all log lines from the job pods + podLogs: + selector: (join('', ['batch.kubernetes.io/job-name in (', $job, ')'])) + tail: -1 diff --git a/testing/chainsaw/e2e/templates/replica-backup-complete.yaml b/testing/chainsaw/e2e/templates/replica-backup-complete.yaml new file mode 100644 index 0000000..520d435 --- /dev/null +++ b/testing/chainsaw/e2e/templates/replica-backup-complete.yaml @@ -0,0 +1,25 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: StepTemplate +metadata: + name: replica-backup-complete +spec: + try: + - sleep: + duration: 30s + + - description: Wait for initial backup to complete' + wait: + for: + jsonPath: + path: '{.status.succeeded}' + value: "1" + timeout: 5m + apiVersion: batch/v1 + kind: Job + namespace: ($namespace) + selector: postgres-operator.crunchydata.com/pgbackrest-backup=replica-create + catch: + - describe: + apiVersion: v1 + kind: Pod + selector: (concat('postgres-operator.crunchydata.com/cluster=', $cluster)) diff --git a/testing/chainsaw/e2e/values.yaml b/testing/chainsaw/e2e/values.yaml new file mode 100644 index 0000000..0c8a3ce --- /dev/null +++ b/testing/chainsaw/e2e/values.yaml @@ -0,0 +1,5 @@ +versions: + postgres: '17' + +images: + psql: 'registry.developers.crunchydata.com/crunchydata/crunchy-postgres:ubi9-17.5-2520' diff --git a/testing/chainsaw/e2e/version/chainsaw-test.yaml b/testing/chainsaw/e2e/version/chainsaw-test.yaml new file mode 100644 index 0000000..1fc5fa9 --- /dev/null +++ b/testing/chainsaw/e2e/version/chainsaw-test.yaml @@ -0,0 +1,50 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + creationTimestamp: null + name: version +spec: + steps: + - name: Run kubectl pgo version + try: + - script: + content: | + contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; } + + VERSION_OUTPUT=$(kubectl pgo version) + { contains "${VERSION_OUTPUT}" "Client Version:"; } || { + echo "Expected: Client Version:*" + echo "Actual: ${VERSION_OUTPUT}" + exit 1 + } + - script: + content: | + contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; } + + OPERATOR_VERSION=$( + kubectl get crd postgresclusters.postgres-operator.crunchydata.com \ + -o go-template='{{ index .metadata.labels "app.kubernetes.io/version" }}' + ) + + VERSION_OUTPUT=$(kubectl pgo version) + + { contains "${VERSION_OUTPUT}" "Operator Version: v${OPERATOR_VERSION}"; } || { + echo "Expected: ${OPERATOR_VERSION}" + echo "Actual: ${VERSION_OUTPUT}" + exit 1 + } + - name: Run kubectl pgo version --client + try: + - script: + content: | + contains() { bash -ceu '[[ "$1" == *"$2"* ]]' - "$@"; } + + VERSION_OUTPUT=$(KUBECONFIG=blah kubectl pgo version --client) + + { contains "${VERSION_OUTPUT}" "Client Version:"; } || { + echo "Expected: Client Version:*" + echo "Actual: ${VERSION_OUTPUT}" + exit 1 + } + diff --git a/testing/chainsaw/policies/README b/testing/chainsaw/policies/README new file mode 100644 index 0000000..7e67161 --- /dev/null +++ b/testing/chainsaw/policies/README @@ -0,0 +1,49 @@ + +# Kyverno Policies + +## Install Kyverno + + +Each 1.x version of Kyverno supports a range of Kubernetes version. Consult the +chart for appropriate versions for your test. + +The latest is v1.15 and is not included in the website documentation. + + +| Kyverno Version | Kubernetes Min | Kubernetes Max | +|-----------------|----------------|----------------| +| 1.12.x | 1.26 | 1.29 | +| 1.13.x | 1.28 | 1.31 | +| 1.14.x | 1.29 | 1.32 | + +- https://kyverno.io/docs/installation/#compatibility-matrix + + +```shell +kubectl create -f https://github.com/kyverno/kyverno/releases/download/v1.15.1/install.yaml +``` + +## Image Pull Secret Policies + +Execute these commands after Kyverno successfully deploys. + +```shell +kubectl apply -f v1.15.1/00-rbac.yaml +kubectl apply -f v1.15.1/01-inject-imagepullsecret-to-namespace.yaml +kubectl apply -f v1.15.1/02-add-imagepullsecrets.yaml +``` + +## Create Registry Secrets + +```shell +kubectl create secret docker-registry crunchy-access-portal -n default \ + --docker-server=registry.crunchydata.com \ + --docker-username=$CRUNCHY_AP_USERNAME \ + --docker-email=$CRUNCHY_AP_USERNAME \ + --docker-password=$CRUNCHY_AP_PWD + +kubectl create secret docker-registry crunchy-harbor -n default \ + --docker-server registry.internal.crunchydata.com \ + --docker-username=$HARBOR_USER \ + --docker-password=$HARBOR_PWD +``` diff --git a/testing/chainsaw/policies/v1.15.1/00-rbac.yaml b/testing/chainsaw/policies/v1.15.1/00-rbac.yaml new file mode 100644 index 0000000..03b4311 --- /dev/null +++ b/testing/chainsaw/policies/v1.15.1/00-rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kyverno:list-get-secrets + labels: + rbac.kyverno.io/aggregate-to-admission-controller: "true" +rules: +- apiGroups: + - '' + resources: + - secrets + verbs: + - list + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: kyverno:create-update-secrets + labels: + rbac.kyverno.io/aggregate-to-background-controller: "true" +rules: +- apiGroups: + - '' + resources: + - secrets + verbs: + - create + - update + - delete + - get + - list \ No newline at end of file diff --git a/testing/chainsaw/policies/v1.15.1/01-inject-imagepullsecret-to-namespace.yaml b/testing/chainsaw/policies/v1.15.1/01-inject-imagepullsecret-to-namespace.yaml new file mode 100644 index 0000000..90e55fe --- /dev/null +++ b/testing/chainsaw/policies/v1.15.1/01-inject-imagepullsecret-to-namespace.yaml @@ -0,0 +1,58 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: inject-imagepullsecret-to-namespace + annotations: + policies.kyverno.io/title: Clone ImagePullSecret to New Namespaces + policies.kyverno.io/subject: Namespace + policies.kyverno.io/description: >- + Clones imagePullSecrets to new namespaces (except system namespaces). +spec: + generateExisting: true + rules: + - name: clone-crunchy-access-portal-secret + match: + any: + - resources: + kinds: + - Namespace + exclude: + any: + - resources: + namespaces: + - kube-system + - kube-node-lease + - kube-public + - kyverno + generate: + apiVersion: v1 + kind: Secret + name: crunchy-access-portal + namespace: "{{ request.object.metadata.name }}" + synchronize: true + clone: + namespace: default + name: crunchy-access-portal + - name: clone-crunchy-harbor-secret + match: + any: + - resources: + kinds: + - Namespace + exclude: + any: + - resources: + namespaces: + - kube-system + - kube-node-lease + - kube-public + - kyverno + generate: + apiVersion: v1 + kind: Secret + name: crunchy-harbor + namespace: "{{ request.object.metadata.name }}" + synchronize: true + clone: + namespace: default + name: crunchy-harbor \ No newline at end of file diff --git a/testing/chainsaw/policies/v1.15.1/02-add-imagepullsecrets.yaml b/testing/chainsaw/policies/v1.15.1/02-add-imagepullsecrets.yaml new file mode 100644 index 0000000..e82e787 --- /dev/null +++ b/testing/chainsaw/policies/v1.15.1/02-add-imagepullsecrets.yaml @@ -0,0 +1,53 @@ +apiVersion: kyverno.io/v1 +kind: ClusterPolicy +metadata: + name: add-imagepullsecrets + annotations: + policies.kyverno.io/title: Add ImagePullSecrets for Multiple Registries + policies.kyverno.io/subject: Pod + policies.kyverno.io/description: >- + Conditionally adds imagePullSecrets to pods based on the registry of their container images. +spec: + rules: + - name: add-crunchy-access-portal-imagepullsecret + match: + any: + - resources: + kinds: + - Pod + context: + - name: images_in_crunchy_access_portal + variable: + jmesPath: "[request.object.spec.containers[*].image.contains(@, 'registry.crunchydata.com')][]" + default: [] + preconditions: + all: + - key: true + operator: In + value: "{{ images_in_crunchy_access_portal }}" + mutate: + patchStrategicMerge: + spec: + imagePullSecrets: + - name: crunchy-access-portal + - name: add-crunchy-harbor-imagepullsecret + match: + any: + - resources: + kinds: + - Pod + context: + - name: images_in_crunchy_harbor + variable: + jmesPath: "[request.object.spec.containers[*].image.contains(@, 'registry.internal.crunchydata.com')][]" + default: [] + preconditions: + all: + - key: true + operator: In + value: "{{ images_in_crunchy_harbor }}" + mutate: + patchStrategicMerge: + spec: + imagePullSecrets: + - name: crunchy-harbor \ No newline at end of file