Execute Kubernetes tests in parallel

Issue: [sc-12208]

Author: cbandy

.golangci.yaml

@@ -64,6 +64,12 @@ issues:
       path: _test[.]go$
       text: \`(net/http/httptest|[^`]*testing[^`]*)`
 
+    # Allow setupTestEnv and teardownTestEnv to remain unused for a while.
+    # TODO(cbandy): Remove these functions after setupKubernetes has landed.
+    - linters: [deadcode, unused]
+      path: internal/controller/postgrescluster/helpers_test.go
+      text: \`(setupTestEnv|teardownTestEnv)`
+
 run:
   build-tags:
     - envtest

internal/controller/postgrescluster/apply_test.go

@@ -37,26 +37,18 @@ import (
 	"k8s.io/client-go/discovery"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-	"sigs.k8s.io/controller-runtime/pkg/envtest"
+
+	"github.com/crunchydata/postgres-operator/internal/testing/require"
 )
 
 func TestServerSideApply(t *testing.T) {
-	// TODO: Update tests that include envtest package to better handle
-	// running in parallel
-	// t.Parallel()
-
 	ctx := context.Background()
-	env := &envtest.Environment{}
-	config, err := env.Start()
-	assert.NilError(t, err)
-	t.Cleanup(func() { assert.Check(t, env.Stop()) })
-
-	cc, err := client.New(config, client.Options{})
-	assert.NilError(t, err)
+	env, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 0)
 
 	ns := setupNamespace(t, cc)
 
-	dc, err := discovery.NewDiscoveryClientForConfig(config)
+	dc, err := discovery.NewDiscoveryClientForConfig(env.Config)
 	assert.NilError(t, err)
 
 	server, err := dc.ServerVersion()

internal/controller/postgrescluster/cluster_test.go

@@ -43,6 +43,7 @@ import (
 
 	"github.com/crunchydata/postgres-operator/internal/initialize"
 	"github.com/crunchydata/postgres-operator/internal/naming"
+	"github.com/crunchydata/postgres-operator/internal/testing/require"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
@@ -93,13 +94,11 @@ var gvks = []schema.GroupVersionKind{{
 }}
 
 func TestCustomLabels(t *testing.T) {
-	t.Parallel()
-
-	env, cc, config := setupTestEnv(t, ControllerName)
-	t.Cleanup(func() { teardownTestEnv(t, env) })
+	env, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 2)
 
 	reconciler := &Reconciler{}
-	ctx, cancel := setupManager(t, config, func(mgr manager.Manager) {
+	ctx, cancel := setupManager(t, env.Config, func(mgr manager.Manager) {
 		reconciler = &Reconciler{
 			Client:   cc,
 			Owner:    client.FieldOwner(t.Name()),
@@ -351,13 +350,11 @@ func TestCustomLabels(t *testing.T) {
 }
 
 func TestCustomAnnotations(t *testing.T) {
-	t.Parallel()
-
-	env, cc, config := setupTestEnv(t, ControllerName)
-	t.Cleanup(func() { teardownTestEnv(t, env) })
+	env, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 2)
 
 	reconciler := &Reconciler{}
-	ctx, cancel := setupManager(t, config, func(mgr manager.Manager) {
+	ctx, cancel := setupManager(t, env.Config, func(mgr manager.Manager) {
 		reconciler = &Reconciler{
 			Client:   cc,
 			Owner:    client.FieldOwner(t.Name()),
@@ -614,20 +611,18 @@ func TestContainerSecurityContext(t *testing.T) {
 		t.Skip("Test requires pods to be created")
 	}
 
-	t.Parallel()
-
-	env, cc, config := setupTestEnv(t, ControllerName)
-	t.Cleanup(func() { teardownTestEnv(t, env) })
+	env, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 1)
 
 	reconciler := &Reconciler{}
-	ctx, cancel := setupManager(t, config, func(mgr manager.Manager) {
+	ctx, cancel := setupManager(t, env.Config, func(mgr manager.Manager) {
 		reconciler = &Reconciler{
 			Client:   cc,
 			Owner:    client.FieldOwner(t.Name()),
 			Recorder: mgr.GetEventRecorderFor(ControllerName),
 			Tracer:   otel.Tracer(t.Name()),
 		}
-		podExec, err := newPodExecutor(config)
+		podExec, err := newPodExecutor(env.Config)
 		assert.NilError(t, err)
 		reconciler.PodExec = podExec
 	})
@@ -691,8 +686,8 @@ func TestContainerSecurityContext(t *testing.T) {
 }
 
 func TestGenerateClusterPrimaryService(t *testing.T) {
-	env, cc, _ := setupTestEnv(t, ControllerName)
-	t.Cleanup(func() { teardownTestEnv(t, env) })
+	_, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 0)
 
 	reconciler := &Reconciler{Client: cc}
 
@@ -792,8 +787,8 @@ subsets:
 
 func TestReconcileClusterPrimaryService(t *testing.T) {
 	ctx := context.Background()
-	env, cc, _ := setupTestEnv(t, ControllerName)
-	t.Cleanup(func() { teardownTestEnv(t, env) })
+	_, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 1)
 
 	reconciler := &Reconciler{Client: cc, Owner: client.FieldOwner(t.Name())}
 
@@ -813,8 +808,8 @@ func TestReconcileClusterPrimaryService(t *testing.T) {
 }
 
 func TestGenerateClusterReplicaServiceIntent(t *testing.T) {
-	env, cc, _ := setupTestEnv(t, ControllerName)
-	t.Cleanup(func() { teardownTestEnv(t, env) })
+	_, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 0)
 
 	reconciler := &Reconciler{Client: cc}
 

internal/controller/postgrescluster/controller_ref_manager_test.go

@@ -29,25 +29,23 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	"github.com/crunchydata/postgres-operator/internal/naming"
+	"github.com/crunchydata/postgres-operator/internal/testing/require"
 )
 
 func TestManageControllerRefs(t *testing.T) {
+	tEnv, tClient := setupKubernetes(t)
+	require.ParallelCapacity(t, 1)
 
-	// setup the test environment and ensure a clean teardown
-	tEnv, tClient, cfg := setupTestEnv(t, ControllerName)
 	r := &Reconciler{}
-	ctx, cancel := setupManager(t, cfg, func(mgr manager.Manager) {
+	ctx, cancel := setupManager(t, tEnv.Config, func(mgr manager.Manager) {
 		r = &Reconciler{
 			Client:   mgr.GetClient(),
 			Recorder: mgr.GetEventRecorderFor(ControllerName),
 			Tracer:   otel.Tracer(ControllerName),
 			Owner:    ControllerName,
 		}
 	})
-	t.Cleanup(func() {
-		teardownManager(cancel, t)
-		teardownTestEnv(t, tEnv)
-	})
+	t.Cleanup(func() { teardownManager(cancel, t) })
 
 	clusterName := "hippo"
 

internal/controller/postgrescluster/controller_test.go

@@ -42,13 +42,14 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"github.com/crunchydata/postgres-operator/internal/naming"
+	"github.com/crunchydata/postgres-operator/internal/testing/require"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
 func TestDeleteControlled(t *testing.T) {
 	ctx := context.Background()
-	tEnv, cc, _ := setupTestEnv(t, t.Name())
-	t.Cleanup(func() { teardownTestEnv(t, tEnv) })
+	_, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 1)
 
 	ns := setupNamespace(t, cc)
 	reconciler := Reconciler{Client: cc}

internal/controller/postgrescluster/delete_test.go

@@ -22,7 +22,6 @@ import (
 	"context"
 	"io"
 	"os"
-	"path/filepath"
 	"strings"
 	"testing"
 	"time"
@@ -34,47 +33,27 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/tools/record"
 	"sigs.k8s.io/controller-runtime/pkg/client"
-	"sigs.k8s.io/controller-runtime/pkg/envtest"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/yaml"
 
 	"github.com/crunchydata/postgres-operator/internal/patroni"
+	"github.com/crunchydata/postgres-operator/internal/testing/require"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
 
 func TestReconcilerHandleDelete(t *testing.T) {
 	if !strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") {
 		t.Skip("requires a running garbage collection controller")
 	}
-	// TODO: Update tests that include envtest package to better handle
-	// running in parallel
-	// t.Parallel()
 
 	ctx := context.Background()
-	env := &envtest.Environment{
-		CRDDirectoryPaths: []string{
-			filepath.Join("..", "..", "..", "config", "crd", "bases"),
-		},
-	}
-
-	options := client.Options{}
-	options.Scheme = runtime.NewScheme()
-	assert.NilError(t, scheme.AddToScheme(options.Scheme))
-	assert.NilError(t, v1beta1.AddToScheme(options.Scheme))
-
-	config, err := env.Start()
-	assert.NilError(t, err)
-	t.Cleanup(func() { assert.Check(t, env.Stop()) })
-
-	cc, err := client.New(config, options)
-	assert.NilError(t, err)
+	env, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 2)
 
 	ns := setupNamespace(t, cc)
 	reconciler := Reconciler{
@@ -84,7 +63,8 @@ func TestReconcilerHandleDelete(t *testing.T) {
 		Tracer:   otel.Tracer(t.Name()),
 	}
 
-	reconciler.PodExec, err = newPodExecutor(config)
+	var err error
+	reconciler.PodExec, err = newPodExecutor(env.Config)
 	assert.NilError(t, err)
 
 	mustReconcile := func(t *testing.T, cluster *v1beta1.PostgresCluster) reconcile.Result {
@@ -353,28 +333,9 @@ func TestReconcilerHandleDeleteNamespace(t *testing.T) {
 		t.Skip("requires a running garbage collection controller")
 	}
 
-	// TODO: Update tests that include envtest package to better handle
-	// running in parallel
-	// t.Parallel()
-
 	ctx := context.Background()
-	env := &envtest.Environment{
-		CRDDirectoryPaths: []string{
-			filepath.Join("..", "..", "..", "config", "crd", "bases"),
-		},
-	}
-
-	options := client.Options{}
-	options.Scheme = runtime.NewScheme()
-	assert.NilError(t, scheme.AddToScheme(options.Scheme))
-	assert.NilError(t, v1beta1.AddToScheme(options.Scheme))
-
-	config, err := env.Start()
-	assert.NilError(t, err)
-	t.Cleanup(func() { assert.Check(t, env.Stop()) })
-
-	cc, err := client.New(config, options)
-	assert.NilError(t, err)
+	env, cc := setupKubernetes(t)
+	require.ParallelCapacity(t, 2)
 
 	ns := setupNamespace(t, cc)
 
@@ -385,11 +346,12 @@ func TestReconcilerHandleDeleteNamespace(t *testing.T) {
 		Stop    context.CancelFunc
 	}
 
+	var err error
 	mm.Context, mm.Stop = context.WithCancel(context.Background())
 	mm.Error = make(chan error, 1)
-	mm.Manager, err = manager.New(config, manager.Options{
+	mm.Manager, err = manager.New(env.Config, manager.Options{
 		Namespace: ns.Name,
-		Scheme:    options.Scheme,
+		Scheme:    cc.Scheme(),
 
 		HealthProbeBindAddress: "0", // disable
 		MetricsBindAddress:     "0", // disable

internal/controller/postgrescluster/helpers_test.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 	"strconv"
+	"sync"
 	"testing"
 	"time"
 
@@ -69,6 +70,60 @@ func marshalMatches(actual interface{}, expected string) cmp.Comparison {
 	return cmp.MarshalMatches(actual, expected)
 }
 
+var kubernetes struct {
+	sync.Mutex
+
+	env   *envtest.Environment
+	count int
+}
+
+// setupKubernetes starts or connects to a Kubernetes API and returns a client
+// that uses it. When starting a local API, the client is a member of the
+// "system:masters" group. It also creates any CRDs present in the
+// "/config/crd/bases" directory. When any of these fail, it calls t.Fatal.
+// It deletes CRDs and stops the local API using t.Cleanup.
+func setupKubernetes(t testing.TB) (*envtest.Environment, client.Client) {
+	t.Helper()
+
+	kubernetes.Lock()
+	defer kubernetes.Unlock()
+
+	if kubernetes.env == nil {
+		env := &envtest.Environment{
+			CRDDirectoryPaths: []string{
+				filepath.Join("..", "..", "..", "config", "crd", "bases"),
+			},
+		}
+
+		_, err := env.Start()
+		assert.NilError(t, err)
+
+		kubernetes.env = env
+	}
+
+	kubernetes.count++
+
+	t.Cleanup(func() {
+		kubernetes.Lock()
+		defer kubernetes.Unlock()
+
+		kubernetes.count--
+
+		if kubernetes.count == 0 {
+			assert.Check(t, kubernetes.env.Stop())
+			kubernetes.env = nil
+		}
+	})
+
+	scheme, err := runtime.CreatePostgresOperatorScheme()
+	assert.NilError(t, err)
+
+	client, err := client.New(kubernetes.env.Config, client.Options{Scheme: scheme})
+	assert.NilError(t, err)
+
+	return kubernetes.env, client
+}
+
 // setupNamespace creates a random namespace that will be deleted by t.Cleanup.
 // When creation fails, it calls t.Fatal. The caller may delete the namespace
 // at any time.
@@ -137,6 +192,8 @@ func testCluster() *v1beta1.PostgresCluster {
 
 // setupTestEnv configures and starts an EnvTest instance of etcd and the Kubernetes API server
 // for test usage, as well as creates a new client instance.
+//
+// Deprecated: use setupKubernetes instead.
 func setupTestEnv(t *testing.T,
 	_ string) (*envtest.Environment, client.Client, *rest.Config) {
 
@@ -147,6 +204,7 @@ func setupTestEnv(t *testing.T,
 	if err != nil {
 		t.Fatal(err)
 	}
+	t.Log("DEPRECATED: Use setupKubernetes instead of setupTestEnv.")
 	t.Log("Test environment started")
 
 	pgoScheme, err := runtime.CreatePostgresOperatorScheme()