WIP: Added a functional test

Author: auniyal61

test/functional/nova_multicell_test.go

@@ -952,3 +952,198 @@ var _ = Describe("Nova multi cell", func() {
 		})
 	})
 })
+
+var _ = Describe("Nova multi cell deletion", func() {
+	BeforeEach(func() {
+		apiMariaDBAccount, apiMariaDBSecret := mariadb.CreateMariaDBAccountAndSecret(
+			novaNames.APIMariaDBDatabaseAccount, mariadbv1.MariaDBAccountSpec{})
+		DeferCleanup(k8sClient.Delete, ctx, apiMariaDBAccount)
+		DeferCleanup(k8sClient.Delete, ctx, apiMariaDBSecret)
+
+		cell0Account, cell0Secret := mariadb.CreateMariaDBAccountAndSecret(
+			cell0.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{})
+		DeferCleanup(k8sClient.Delete, ctx, cell0Account)
+		DeferCleanup(k8sClient.Delete, ctx, cell0Secret)
+
+		// cell1Account, cell1Secret := mariadb.CreateMariaDBAccountAndSecret(
+		// 	cell1.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{})
+		// DeferCleanup(k8sClient.Delete, ctx, cell1Account)
+		// DeferCleanup(k8sClient.Delete, ctx, cell1Secret)
+
+		mariadb.CreateMariaDBAccountAndSecret(
+			cell1.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{})
+		// DeferCleanup(k8sClient.Delete, ctx, cell1Account)
+		// DeferCleanup(k8sClient.Delete, ctx, cell1Secret)
+
+	})
+
+	When("Nova CR instance is created with 3 cells", func() {
+		BeforeEach(func() {
+
+			mariadb.CreateMariaDBAccountAndSecret(
+				cell2.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{})
+
+			// mariadb.CreateMariaDBAccountAndSecret(
+			// 	cell3.MariaDBAccountName, mariadbv1.MariaDBAccountSpec{})
+
+			// TODO(bogdando): deduplicate this into CreateNovaWith3CellsAndEnsureReady()
+			DeferCleanup(k8sClient.Delete, ctx, CreateNovaSecretFor3Cells(novaNames.NovaName.Namespace, SecretName))
+			DeferCleanup(k8sClient.Delete, ctx, CreateNovaMessageBusSecret(cell0))
+			DeferCleanup(k8sClient.Delete, ctx, CreateNovaMessageBusSecret(cell1))
+			DeferCleanup(k8sClient.Delete, ctx, CreateNovaMessageBusSecret(cell2))
+			DeferCleanup(k8sClient.Delete, ctx, CreateNovaMessageBusSecret(cell3))
+
+			serviceSpec := corev1.ServiceSpec{Ports: []corev1.ServicePort{{Port: 3306}}}
+			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(novaNames.APIMariaDBDatabaseName.Namespace, novaNames.APIMariaDBDatabaseName.Name, serviceSpec))
+			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(cell0.MariaDBDatabaseName.Namespace, cell0.MariaDBDatabaseName.Name, serviceSpec))
+			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(cell1.MariaDBDatabaseName.Namespace, cell1.MariaDBDatabaseName.Name, serviceSpec))
+			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(cell2.MariaDBDatabaseName.Namespace, cell2.MariaDBDatabaseName.Name, serviceSpec))
+			DeferCleanup(mariadb.DeleteDBService, mariadb.CreateDBService(cell3.MariaDBDatabaseName.Namespace, cell3.MariaDBDatabaseName.Name, serviceSpec))
+
+			spec := GetDefaultNovaSpec()
+			cell0Template := GetDefaultNovaCellTemplate()
+			cell0Template["cellDatabaseInstance"] = cell0.MariaDBDatabaseName.Name
+			cell0Template["cellDatabaseAccount"] = cell0.MariaDBAccountName.Name
+
+			cell1Template := GetDefaultNovaCellTemplate()
+			cell1Template["cellDatabaseInstance"] = cell1.MariaDBDatabaseName.Name
+			cell1Template["cellDatabaseAccount"] = cell1.MariaDBAccountName.Name
+			cell1Template["cellMessageBusInstance"] = cell1.TransportURLName.Name
+			cell1Template["passwordSelectors"] = map[string]interface{}{
+				"database": "NovaCell1DatabasePassword",
+			}
+			cell1Template["novaComputeTemplates"] = map[string]interface{}{
+				ironicComputeName: GetDefaultNovaComputeTemplate(),
+			}
+			cell1Memcached := "memcached1"
+			cell1Template["memcachedInstance"] = cell1Memcached
+
+			cell2Template := GetDefaultNovaCellTemplate()
+			cell2Template["cellDatabaseInstance"] = cell2.MariaDBDatabaseName.Name
+			cell2Template["cellDatabaseAccount"] = cell2.MariaDBAccountName.Name
+			cell2Template["cellMessageBusInstance"] = cell2.TransportURLName.Name
+			cell2Template["hasAPIAccess"] = false
+			cell2Template["passwordSelectors"] = map[string]interface{}{
+				"database": "NovaCell2DatabasePassword",
+			}
+
+			cell3Template := GetDefaultNovaCellTemplate()
+			cell3Template["cellDatabaseInstance"] = cell3.MariaDBDatabaseName.Name
+			cell3Template["cellDatabaseAccount"] = cell3.MariaDBAccountName.Name
+			cell3Template["cellMessageBusInstance"] = cell3.TransportURLName.Name
+			cell3Template["hasAPIAccess"] = false
+			cell3Template["passwordSelectors"] = map[string]interface{}{
+				"database": "NovaCell3DatabasePassword",
+			}
+
+			spec["cellTemplates"] = map[string]interface{}{
+				"cell0": cell0Template,
+				"cell1": cell1Template,
+				"cell2": cell2Template,
+				"cell3": cell3Template,
+			}
+
+			spec["apiDatabaseInstance"] = novaNames.APIMariaDBDatabaseName.Name
+			spec["apiMessageBusInstance"] = cell0.TransportURLName.Name
+
+			DeferCleanup(th.DeleteInstance, CreateNova(novaNames.NovaName, spec))
+			DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(novaNames.NovaName.Namespace))
+			memcachedSpecCell1 := memcachedv1.MemcachedSpec{
+				MemcachedSpecCore: memcachedv1.MemcachedSpecCore{
+					Replicas: ptr.To(int32(3)),
+				},
+			}
+			memcachedNamespace := types.NamespacedName{
+				Name:      cell1Memcached,
+				Namespace: novaNames.NovaName.Namespace,
+			}
+			DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(novaNames.NovaName.Namespace, cell1Memcached, memcachedSpecCell1))
+			infra.SimulateMemcachedReady(memcachedNamespace)
+
+			memcachedSpec := memcachedv1.MemcachedSpec{
+				MemcachedSpecCore: memcachedv1.MemcachedSpecCore{
+					Replicas: ptr.To(int32(3)),
+				},
+			}
+
+			DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(novaNames.NovaName.Namespace, MemcachedInstance, memcachedSpec))
+			infra.SimulateMemcachedReady(novaNames.MemcachedNamespace)
+			keystone.SimulateKeystoneServiceReady(novaNames.KeystoneServiceName)
+
+			// Pre-requisite simulation
+			mariadb.SimulateMariaDBDatabaseCompleted(novaNames.APIMariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(novaNames.APIMariaDBDatabaseAccount)
+			mariadb.SimulateMariaDBDatabaseCompleted(cell0.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(cell0.MariaDBAccountName)
+			infra.SimulateTransportURLReady(cell0.TransportURLName)
+			th.SimulateJobSuccess(cell0.DBSyncJobName)
+			th.SimulateStatefulSetReplicaReady(cell0.ConductorStatefulSetName)
+			th.SimulateJobSuccess(cell0.CellMappingJobName)
+
+			th.SimulateStatefulSetReplicaReady(novaNames.APIStatefulSetName)
+			keystone.SimulateKeystoneEndpointReady(novaNames.APIKeystoneEndpointName)
+			th.SimulateStatefulSetReplicaReady(novaNames.SchedulerStatefulSetName)
+			th.SimulateStatefulSetReplicaReady(novaNames.MetadataStatefulSetName)
+
+			// cell1
+			mariadb.SimulateMariaDBDatabaseCompleted(cell1.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(cell1.MariaDBAccountName)
+			infra.SimulateTransportURLReady(cell1.TransportURLName)
+			th.SimulateStatefulSetReplicaReady(cell1.NovaComputeStatefulSetName)
+			th.SimulateStatefulSetReplicaReady(cell1.NoVNCProxyStatefulSetName)
+			th.SimulateJobSuccess(cell1.DBSyncJobName)
+			th.SimulateStatefulSetReplicaReady(cell1.ConductorStatefulSetName)
+			th.SimulateJobSuccess(cell1.CellMappingJobName)
+			th.SimulateJobSuccess(cell1.HostDiscoveryJobName)
+
+			// Create/Simulate DB for cell2 and cell3
+			mariadb.SimulateMariaDBDatabaseCompleted(cell2.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(cell2.MariaDBAccountName)
+			infra.SimulateTransportURLReady(cell2.TransportURLName)
+			th.SimulateStatefulSetReplicaReady(cell2.NoVNCProxyStatefulSetName)
+			th.SimulateJobSuccess(cell2.DBSyncJobName)
+			th.SimulateStatefulSetReplicaReady(cell2.ConductorStatefulSetName)
+			th.SimulateJobSuccess(cell2.CellMappingJobName)
+
+			mariadb.SimulateMariaDBDatabaseCompleted(cell3.MariaDBDatabaseName)
+			mariadb.SimulateMariaDBAccountCompleted(cell3.MariaDBAccountName)
+			infra.SimulateTransportURLReady(cell3.TransportURLName)
+			th.SimulateStatefulSetReplicaReady(cell3.NoVNCProxyStatefulSetName)
+			th.SimulateJobSuccess(cell3.DBSyncJobName)
+			th.SimulateStatefulSetReplicaReady(cell3.ConductorStatefulSetName)
+			th.SimulateJobSuccess(cell3.CellMappingJobName)
+
+		})
+
+		It("delete cell1 and cell2, verify for cell1", func() {
+
+			nova := GetNova(novaNames.NovaName)
+			Expect(nova.Status.RegisteredCells).To(HaveKey(cell0.CellCRName.Name), "cell0 is not in the RegisteredCells", nova.Status.RegisteredCells)
+			Expect(nova.Status.RegisteredCells).To(HaveKey(cell1.CellCRName.Name), "cell1 is not in the RegisteredCells", nova.Status.RegisteredCells)
+			Expect(nova.Status.RegisteredCells).To(HaveKey(cell2.CellCRName.Name), "cell2 is not in the RegisteredCells", nova.Status.RegisteredCells)
+
+			// manually delete DB for cell2, to reproduce the error in cell2 deletion
+			Expect(k8sClient.Delete(ctx, mariadb.GetMariaDBDatabase(cell1.MariaDBDatabaseName))).To(Succeed())
+
+			// delete cell1 and then cell2,
+			// the overall call should succeed
+			Eventually(func(g Gomega) {
+				nova := GetNova(novaNames.NovaName)
+				delete(nova.Spec.CellTemplates, "cell1")
+				delete(nova.Spec.CellTemplates, "cell2")
+				g.Expect(k8sClient.Update(ctx, nova)).To(Succeed())
+			}, timeout, interval).Should(Succeed())
+
+			Eventually(func(g Gomega) {
+				// test cell2 is not present in CR
+				nova := GetNova(novaNames.NovaName)
+				g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell1.CellCRName.Name))
+				g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell2.CellCRName.Name))
+			}, timeout, interval).Should(Succeed())
+
+			NovaCellNotExists(cell1.CellCRName)
+			NovaCellNotExists(cell2.CellCRName)
+
+		})
+	})
+})

test/functional/suite_test.go

@@ -83,6 +83,7 @@ var (
 	cell0     CellNames
 	cell1     CellNames
 	cell2     CellNames
+	cell3     CellNames
 )
 
 func TestAPIs(t *testing.T) {
@@ -280,8 +281,9 @@ var _ = BeforeEach(func() {
 		Name:      uuid.New().String()[:25],
 	}
 
-	novaNames = GetNovaNames(novaName, []string{"cell0", "cell1", "cell2"})
+	novaNames = GetNovaNames(novaName, []string{"cell0", "cell1", "cell2", "cell3"})
 	cell0 = novaNames.Cells["cell0"]
 	cell1 = novaNames.Cells["cell1"]
 	cell2 = novaNames.Cells["cell2"]
+	cell3 = novaNames.Cells["cell3"]
 })