diff --git a/percona/controller/pgcluster/backup.go b/percona/controller/pgcluster/backup.go
index 44639f1b1f..890485579c 100644
--- a/percona/controller/pgcluster/backup.go
+++ b/percona/controller/pgcluster/backup.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/pkg/errors"
 	batchv1 "k8s.io/api/batch/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/util/retry"
@@ -67,7 +68,10 @@ func (r *PGClusterReconciler) cleanupOutdatedBackups(ctx context.Context, cr *v2
 		}
 
 		for _, pgBackup := range pbList {
-			if pgBackup.Status.State != v2.BackupSucceeded || pgBackup.CompareVersion("2.4.0") < 0 {
+			if (pgBackup.Status.State != v2.BackupSucceeded &&
+				pgBackup.Status.State != v2.BackupFailed &&
+				pgBackup.Status.State != v2.BackupStarting &&
+				pgBackup.Status.State != v2.BackupRunning) || pgBackup.CompareVersion("2.4.0") < 0 {
 				continue
 			}
 
@@ -87,14 +91,19 @@ func (r *PGClusterReconciler) cleanupOutdatedBackups(ctx context.Context, cr *v2
 			// After the pg-backup is deleted, the job is not deleted immediately.
 			// We need to set the DeletionTimestamp for a job so that `reconcileBackupJob` doesn't create a new pg-backup before the job deletion.
 			job := new(batchv1.Job)
-			if err := r.Client.Get(ctx, types.NamespacedName{Name: pgBackup.Status.JobName, Namespace: pgBackup.Namespace}, job); err != nil {
+			err := r.Client.Get(ctx, types.NamespacedName{Name: pgBackup.Status.JobName, Namespace: pgBackup.Namespace}, job)
+			if err != nil && !k8serrors.IsNotFound(err) {
 				return errors.Wrap(err, "get backup job")
 			}
-			prop := metav1.DeletePropagationForeground
-			if err := r.Client.Delete(ctx, job, &client.DeleteOptions{
-				PropagationPolicy: &prop,
-			}); err != nil {
-				return errors.Wrapf(err, "delete job %s/%s", job.Name, job.Namespace)
+
+			// Only try to delete the job if it exists
+			if err == nil {
+				prop := metav1.DeletePropagationForeground
+				if err := r.Client.Delete(ctx, job, &client.DeleteOptions{
+					PropagationPolicy: &prop,
+				}); err != nil {
+					return errors.Wrapf(err, "delete job %s/%s", job.Name, job.Namespace)
+				}
 			}
 			if err := r.Client.Delete(ctx, &pgBackup); err != nil {
 				return errors.Wrapf(err, "delete backup %s/%s", pgBackup.Name, pgBackup.Namespace)