From 5be74fd6cbda058a4670f683aa1626946a88c121 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Wed, 22 Jan 2025 10:05:40 +0100 Subject: [PATCH 1/5] Add TLS options to SPEC In order to validate internal connections with TLS we need to add the standard TLS options to the CRs. Note that, it will be copied from the top level one to all subCRs so it only will be exposed in main Watcher CRD top level. --- .../watcher.openstack.org_watcherapis.yaml | 30 +++++++++++++++++++ api/bases/watcher.openstack.org_watchers.yaml | 30 +++++++++++++++++++ api/v1beta1/common_types.go | 7 +++++ api/v1beta1/zz_generated.deepcopy.go | 1 + .../watcher.openstack.org_watcherapis.yaml | 30 +++++++++++++++++++ .../bases/watcher.openstack.org_watchers.yaml | 30 +++++++++++++++++++ ...atcher-operator.clusterserviceversion.yaml | 8 +++++ controllers/watcher_controller.go | 3 ++ pkg/watcher/dbsync.go | 6 ++++ pkg/watcherapi/deployment.go | 6 ++++ tests/functional/base_test.go | 3 ++ tests/functional/watcher_controller_test.go | 5 +++- 12 files changed, 158 insertions(+), 1 deletion(-) diff --git a/api/bases/watcher.openstack.org_watcherapis.yaml b/api/bases/watcher.openstack.org_watcherapis.yaml index 7c40340f..01bc2c5d 100644 --- a/api/bases/watcher.openstack.org_watcherapis.yaml +++ b/api/bases/watcher.openstack.org_watcherapis.yaml @@ -194,6 +194,36 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - secret - serviceAccount diff --git a/api/bases/watcher.openstack.org_watchers.yaml b/api/bases/watcher.openstack.org_watchers.yaml index 949f895a..1e4be5ad 100644 --- a/api/bases/watcher.openstack.org_watchers.yaml +++ b/api/bases/watcher.openstack.org_watchers.yaml @@ -224,6 +224,36 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - apiContainerImageURL - apiServiceTemplate diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index 963acc6b..0ae4bf57 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -17,7 +17,9 @@ limitations under the License. package v1beta1 import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + corev1 "k8s.io/api/core/v1" ) @@ -55,6 +57,11 @@ type WatcherCommon struct { // NodeSelector to target subset of worker nodes running this component. Setting here overrides // any global NodeSelector settings within the Watcher CR. NodeSelector *map[string]string `json:"nodeSelector,omitempty"` + + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // TLS - Parameters related to the TLS + TLS tls.API `json:"tls,omitempty"` } // WatcherTemplate defines the fields used in the top level CR diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 911e8320..f231193c 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -338,6 +338,7 @@ func (in *WatcherCommon) DeepCopyInto(out *WatcherCommon) { } } } + in.TLS.DeepCopyInto(&out.TLS) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherCommon. diff --git a/config/crd/bases/watcher.openstack.org_watcherapis.yaml b/config/crd/bases/watcher.openstack.org_watcherapis.yaml index 7c40340f..01bc2c5d 100644 --- a/config/crd/bases/watcher.openstack.org_watcherapis.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherapis.yaml @@ -194,6 +194,36 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - secret - serviceAccount diff --git a/config/crd/bases/watcher.openstack.org_watchers.yaml b/config/crd/bases/watcher.openstack.org_watchers.yaml index 949f895a..1e4be5ad 100644 --- a/config/crd/bases/watcher.openstack.org_watchers.yaml +++ b/config/crd/bases/watcher.openstack.org_watchers.yaml @@ -224,6 +224,36 @@ spec: description: ServiceUser - optional username used for this service to register in keystone type: string + tls: + description: TLS - Parameters related to the TLS + properties: + api: + description: API tls type which encapsulates for API services + properties: + internal: + description: Internal GenericService - holds the secret for + the internal endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + public: + description: Public GenericService - holds the secret for + the public endpoint + properties: + secretName: + description: SecretName - holding the cert, key for the + service + type: string + type: object + type: object + caBundleSecretName: + description: CaBundleSecretName - holding the CA certs in a pre-created + bundle file + type: string + type: object required: - apiContainerImageURL - apiServiceTemplate diff --git a/config/manifests/bases/watcher-operator.clusterserviceversion.yaml b/config/manifests/bases/watcher-operator.clusterserviceversion.yaml index e42dd9f3..2c30c2ea 100644 --- a/config/manifests/bases/watcher-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/watcher-operator.clusterserviceversion.yaml @@ -23,6 +23,10 @@ spec: displayName: Watcher API kind: WatcherAPI name: watcherapis.watcher.openstack.org + specDescriptors: + - description: TLS - Parameters related to the TLS + displayName: TLS + path: tls version: v1beta1 - description: WatcherApplier is the Schema for the watcherappliers API displayName: Watcher Applier @@ -39,6 +43,10 @@ spec: displayName: Watcher kind: Watcher name: watchers.watcher.openstack.org + specDescriptors: + - description: TLS - Parameters related to the TLS + displayName: TLS + path: tls version: v1beta1 description: The Watcher Operator project displayName: Watcher Operator diff --git a/controllers/watcher_controller.go b/controllers/watcher_controller.go index 634960ae..534c2bfc 100644 --- a/controllers/watcher_controller.go +++ b/controllers/watcher_controller.go @@ -774,6 +774,9 @@ func (r *WatcherReconciler) ensureAPI( watcherAPISpec.NodeSelector = instance.Spec.NodeSelector } + // We need to have TLS defined in SubCRs to have some values available + watcherAPISpec.TLS = instance.Spec.TLS + apiDeployment := &watcherv1beta1.WatcherAPI{ ObjectMeta: metav1.ObjectMeta{ Name: fmt.Sprintf("%s-api", instance.Name), diff --git a/pkg/watcher/dbsync.go b/pkg/watcher/dbsync.go index c5dbe7d9..00ebd436 100644 --- a/pkg/watcher/dbsync.go +++ b/pkg/watcher/dbsync.go @@ -56,6 +56,12 @@ func DbSyncJob(instance *watcherv1beta1.Watcher, labels map[string]string, annot }, } + // Create mount for bundle CA if defined in TLS.CaBundleSecretName + if instance.Spec.TLS.CaBundleSecretName != "" { + dbSyncVolume = append(dbSyncVolume, instance.Spec.TLS.CreateVolume()) + dbSyncMounts = append(dbSyncMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + args := []string{"-c", DBSyncCommand} runAsUser := int64(0) diff --git a/pkg/watcherapi/deployment.go b/pkg/watcherapi/deployment.go index e2c5a679..7581573b 100644 --- a/pkg/watcherapi/deployment.go +++ b/pkg/watcherapi/deployment.go @@ -65,6 +65,12 @@ func Deployment( } apiVolumeMounts = append(apiVolumeMounts, watcher.GetLogVolumeMount()...) + // Create mount for bundle CA if defined in TLS.CaBundleSecretName + if instance.Spec.TLS.CaBundleSecretName != "" { + apiVolumes = append(apiVolumes, instance.Spec.TLS.CreateVolume()) + apiVolumeMounts = append(apiVolumeMounts, instance.Spec.TLS.CreateVolumeMounts(nil)...) + } + deployment := &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: instance.Name, diff --git a/tests/functional/base_test.go b/tests/functional/base_test.go index 459199f0..5be15f53 100644 --- a/tests/functional/base_test.go +++ b/tests/functional/base_test.go @@ -48,6 +48,9 @@ func GetNonDefaultWatcherSpec() map[string]interface{} { "replicas": 2, "nodeSelector": map[string]string{"foo": "bar"}, }, + "tls": map[string]interface{}{ + "caBundleSecretName": "combined-ca-bundle", + }, } } diff --git a/tests/functional/watcher_controller_test.go b/tests/functional/watcher_controller_test.go index ffcf7dba..81a75453 100644 --- a/tests/functional/watcher_controller_test.go +++ b/tests/functional/watcher_controller_test.go @@ -51,6 +51,7 @@ var _ = Describe("Watcher controller with minimal spec values", func() { Expect(*(Watcher.Spec.RabbitMqClusterName)).Should(Equal("rabbitmq")) Expect(Watcher.Spec.ServiceUser).Should(Equal("watcher")) Expect(Watcher.Spec.PreserveJobs).Should(BeFalse()) + Expect(Watcher.Spec.TLS.CaBundleSecretName).Should(Equal("")) }) It("should have the Status fields initialized", func() { @@ -623,6 +624,7 @@ var _ = Describe("Watcher controller", func() { Expect(Watcher.Spec.Secret).Should(Equal("test-osp-secret")) Expect(Watcher.Spec.PreserveJobs).Should(BeTrue()) Expect(*(Watcher.Spec.RabbitMqClusterName)).Should(Equal("rabbitmq")) + Expect(Watcher.Spec.TLS.CaBundleSecretName).Should(Equal("combined-ca-bundle")) }) It("Should create watcher service with custom values", func() { @@ -751,12 +753,13 @@ var _ = Describe("Watcher controller", func() { Expect(WatcherAPI.Spec.ServiceAccount).To(Equal("watcher-watcher")) Expect(int(*WatcherAPI.Spec.Replicas)).To(Equal(2)) Expect(*WatcherAPI.Spec.NodeSelector).To(Equal(map[string]string{"foo": "bar"})) + Expect(WatcherAPI.Spec.TLS.CaBundleSecretName).Should(Equal("combined-ca-bundle")) // Assert that the watcher deployment is created deployment := th.GetDeployment(watcherTest.WatcherAPIDeployment) Expect(deployment.Spec.Template.Spec.ServiceAccountName).To(Equal("watcher-watcher")) Expect(int(*deployment.Spec.Replicas)).To(Equal(2)) - Expect(deployment.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(deployment.Spec.Template.Spec.Volumes).To(HaveLen(4)) Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(2)) Expect(deployment.Spec.Selector.MatchLabels).To(Equal(map[string]string{"service": "watcher-api"})) From 58d5132377867ea5c1bef229af531b1764def11e Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Fri, 17 Jan 2025 10:50:14 +0100 Subject: [PATCH 2/5] Support SSL in access to mariadb Database This patch adds support for SSL to the Database depending on the configuration of TLS in watcher and watcherapi spec. --- controllers/watcher_common.go | 2 + controllers/watcher_controller.go | 13 +++- controllers/watcherapi_controller.go | 21 +++++- tests/functional/watcher_controller_test.go | 16 +++++ .../functional/watcherapi_controller_test.go | 67 +++++++++++++++++-- 5 files changed, 111 insertions(+), 8 deletions(-) diff --git a/controllers/watcher_common.go b/controllers/watcher_common.go index 6e55b2ce..d3dcec45 100644 --- a/controllers/watcher_common.go +++ b/controllers/watcher_common.go @@ -36,6 +36,8 @@ var ( const ( // TransportURLSelector is the name of key in the secret created by TransportURL TransportURLSelector = "transport_url" + // DatabaseAccount is the name of key in the secret for the name of the Database Acount object + DatabaseAccount = "database_account" // DatabaseUsername is the name of key in the secret for the user name used to login to the database DatabaseUsername = "database_username" // DatabaseUsername is the name of key in the secret for the password used to login to the database diff --git a/controllers/watcher_controller.go b/controllers/watcher_controller.go index 534c2bfc..4b3aaf10 100644 --- a/controllers/watcher_controller.go +++ b/controllers/watcher_controller.go @@ -41,6 +41,7 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/labels" common_rbac "github.com/openstack-k8s-operators/lib-common/modules/common/rbac" "github.com/openstack-k8s-operators/lib-common/modules/common/secret" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -634,13 +635,20 @@ func (r *WatcherReconciler) generateServiceConfigDBSync( Log := r.GetLogger(ctx) Log.Info("generateServiceConfigs - reconciling config for Watcher CR") - customData := map[string]string{} + var tlsCfg *tls.Service + if instance.Spec.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + // customData hold any customization for the service. + customData := map[string]string{ + "my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } labels := labels.GetLabels(instance, labels.GetGroupLabel(watcher.ServiceName), map[string]string{}) databaseAccount := db.GetAccount() databaseSecret := db.GetSecret() templateParameters := map[string]interface{}{ - "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?charset=utf8", + "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", databaseAccount.Spec.UserName, string(databaseSecret.Data[mariadbv1.DatabasePasswordSelector]), db.GetDatabaseHostname(), @@ -721,6 +729,7 @@ func (r *WatcherReconciler) createSubLevelSecret( data := map[string]string{ instance.Spec.PasswordSelectors.Service: string(inputSecret.Data[instance.Spec.PasswordSelectors.Service]), TransportURLSelector: string(transportURLSecret.Data[TransportURLSelector]), + DatabaseAccount: databaseAccount.Name, DatabaseUsername: databaseAccount.Spec.UserName, DatabasePassword: string(databaseSecret.Data[mariadbv1.DatabasePasswordSelector]), DatabaseHostname: db.GetDatabaseHostname(), diff --git a/controllers/watcherapi_controller.go b/controllers/watcherapi_controller.go index ee080201..aa0865c1 100644 --- a/controllers/watcherapi_controller.go +++ b/controllers/watcherapi_controller.go @@ -43,9 +43,12 @@ import ( "github.com/openstack-k8s-operators/lib-common/modules/common/helper" "github.com/openstack-k8s-operators/lib-common/modules/common/labels" "github.com/openstack-k8s-operators/lib-common/modules/common/service" + "github.com/openstack-k8s-operators/lib-common/modules/common/tls" "github.com/openstack-k8s-operators/lib-common/modules/common/util" + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" + "github.com/openstack-k8s-operators/watcher-operator/pkg/watcher" "github.com/openstack-k8s-operators/watcher-operator/pkg/watcherapi" @@ -160,6 +163,7 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) []string{ instance.Spec.PasswordSelectors.Service, TransportURLSelector, + DatabaseAccount, }, helper.GetClient(), &instance.Status.Conditions, @@ -280,16 +284,29 @@ func (r *WatcherAPIReconciler) generateServiceConfigs( if err != nil { return err } + + databaseAccount := string(secret.Data[DatabaseAccount]) + db, err := mariadbv1.GetDatabaseByNameAndAccount(ctx, helper, watcher.DatabaseCRName, databaseAccount, instance.Namespace) + if err != nil { + return err + } // customData hold any customization for the service. // NOTE jgilaber making an empty map for now, we'll probably want to // implement CustomServiceConfig later - customData := map[string]string{} + var tlsCfg *tls.Service + if instance.Spec.TLS.Ca.CaBundleSecretName != "" { + tlsCfg = &tls.Service{} + } + // customData hold any customization for the service. + customData := map[string]string{ + "my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf + } databaseUsername := string(secret.Data[DatabaseUsername]) databaseHostname := string(secret.Data[DatabaseHostname]) databasePassword := string(secret.Data[DatabasePassword]) templateParameters := map[string]interface{}{ - "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?charset=utf8", + "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", databaseUsername, databasePassword, databaseHostname, diff --git a/tests/functional/watcher_controller_test.go b/tests/functional/watcher_controller_test.go index 81a75453..236bae3d 100644 --- a/tests/functional/watcher_controller_test.go +++ b/tests/functional/watcher_controller_test.go @@ -345,6 +345,7 @@ var _ = Describe("Watcher controller", func() { Expect(createdSecret).ShouldNot(BeNil()) Expect(createdSecret.Data["WatcherPassword"]).To(Equal([]byte("password"))) Expect(createdSecret.Data["transport_url"]).To(Equal([]byte("rabbit://rabbitmq-secret/fake"))) + Expect(createdSecret.Data["database_account"]).To(Equal([]byte("watcher"))) // Check WatcherAPI is created WatcherAPI := GetWatcherAPI(watcherTest.WatcherAPI) @@ -746,6 +747,21 @@ var _ = Describe("Watcher controller", func() { Expect(Watcher.Status.Hash[watcherv1beta1.DbSyncHash]).ShouldNot(BeNil()) // Check WatcherAPI is created with non-default values + watcherAPI := &watcherv1beta1.WatcherAPI{} + Expect(k8sClient.Get(ctx, + types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: watcherTest.Instance.Name + "-api"}, + watcherAPI)).Should(Succeed()) + + // Check the config-data volume of watcherapi has expected info + apiConfigSecret := th.GetSecret( + types.NamespacedName{ + Name: watcherTest.Instance.Name + "-api-config-data", + Namespace: watcherTest.Instance.Namespace, + }, + ) + Expect(apiConfigSecret).ShouldNot(BeNil()) + Expect(apiConfigSecret.Data["my.cnf"]).To(Equal([]byte("[client]\nssl=0"))) + WatcherAPI := GetWatcherAPI(watcherTest.WatcherAPI) //Expect(WatcherAPI.Spec.Replicas).To(Equal(int(1))) Expect(WatcherAPI.Spec.ContainerImage).To(Equal("fake-API-Container-URL")) diff --git a/tests/functional/watcherapi_controller_test.go b/tests/functional/watcherapi_controller_test.go index 6e4244ce..cb6cebeb 100644 --- a/tests/functional/watcherapi_controller_test.go +++ b/tests/functional/watcherapi_controller_test.go @@ -10,6 +10,7 @@ import ( memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" + "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -108,11 +109,40 @@ var _ = Describe("WatcherAPI controller", func() { secret := th.CreateSecret( watcherTest.InternalTopLevelSecretName, map[string][]byte{ - "WatcherPassword": []byte("service-password"), - "transport_url": []byte("url"), + "WatcherPassword": []byte("service-password"), + "transport_url": []byte("url"), + "database_username": []byte("username"), + "database_password": []byte("password"), + "database_hostname": []byte("hostname"), + "database_account": []byte("watcher"), }, ) DeferCleanup(k8sClient.Delete, ctx, secret) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + watcherTest.WatcherAPI.Namespace, + "openstack", + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + mariadb.CreateMariaDBAccountAndSecret( + watcherTest.WatcherDatabaseAccount, + v1beta1.MariaDBAccountSpec{ + UserName: "watcher", + }, + ) + mariadb.CreateMariaDBDatabase( + watcherTest.WatcherAPI.Namespace, + "watcher", + v1beta1.MariaDBDatabaseSpec{ + Name: "watcher", + }, + ) + mariadb.SimulateMariaDBAccountCompleted(watcherTest.WatcherDatabaseAccount) + mariadb.SimulateMariaDBDatabaseCompleted(watcherTest.WatcherDatabaseName) DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace)) memcachedSpec := memcachedv1.MemcachedSpec{ @@ -122,6 +152,7 @@ var _ = Describe("WatcherAPI controller", func() { } DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.WatcherAPI.Namespace, MemcachedInstance, memcachedSpec)) infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace) + }) It("should have input ready", func() { th.ExpectCondition( @@ -264,6 +295,7 @@ var _ = Describe("WatcherAPI controller", func() { "database_username": []byte("username"), "database_password": []byte("password"), "database_hostname": []byte("hostname"), + "database_account": []byte("watcher"), }, ) DeferCleanup(k8sClient.Delete, ctx, secret) @@ -299,6 +331,7 @@ var _ = Describe("WatcherAPI controller", func() { "database_username": []byte("username"), "database_password": []byte("password"), "database_hostname": []byte("hostname"), + "database_account": []byte("watcher"), }, ) DeferCleanup(k8sClient.Delete, ctx, secret) @@ -348,8 +381,9 @@ var _ = Describe("WatcherAPI controller", func() { secret := th.CreateSecret( watcherTest.InternalTopLevelSecretName, map[string][]byte{ - "WatcherPassword": []byte("service-password"), - "transport_url": []byte("url"), + "WatcherPassword": []byte("service-password"), + "transport_url": []byte("url"), + "database_account": []byte("watcher"), }, ) DeferCleanup(k8sClient.Delete, ctx, secret) @@ -371,6 +405,31 @@ var _ = Describe("WatcherAPI controller", func() { } DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.WatcherAPI.Namespace, MemcachedInstance, memcachedSpec)) infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace) + DeferCleanup( + mariadb.DeleteDBService, + mariadb.CreateDBService( + watcherTest.WatcherAPI.Namespace, + "openstack", + corev1.ServiceSpec{ + Ports: []corev1.ServicePort{{Port: 3306}}, + }, + ), + ) + mariadb.CreateMariaDBAccountAndSecret( + watcherTest.WatcherDatabaseAccount, + v1beta1.MariaDBAccountSpec{ + UserName: "watcher", + }, + ) + mariadb.CreateMariaDBDatabase( + watcherTest.WatcherAPI.Namespace, + "watcher", + v1beta1.MariaDBDatabaseSpec{ + Name: "watcher", + }, + ) + mariadb.SimulateMariaDBAccountCompleted(watcherTest.WatcherDatabaseAccount) + mariadb.SimulateMariaDBDatabaseCompleted(watcherTest.WatcherDatabaseName) }) It("creates MetalLB service", func() { From a7a07ce43c4044179ee563cbfbb97affa29d58e1 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Fri, 17 Jan 2025 12:00:35 +0100 Subject: [PATCH 3/5] Add TLS support to memcached connection By default memcached is created with TLS support by openstack operators. This patch adds support to access memcached instance with TLS enabled. --- controllers/watcherapi_controller.go | 16 +++++++++------- templates/watcher/config/00-default.conf | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/controllers/watcherapi_controller.go b/controllers/watcherapi_controller.go index aa0865c1..34624093 100644 --- a/controllers/watcherapi_controller.go +++ b/controllers/watcherapi_controller.go @@ -312,13 +312,15 @@ func (r *WatcherAPIReconciler) generateServiceConfigs( databaseHostname, watcher.DatabaseName, ), - "KeystoneAuthURL": keystoneInternalURL, - "ServicePassword": string(secret.Data[instance.Spec.PasswordSelectors.Service]), - "ServiceUser": instance.Spec.ServiceUser, - "TransportURL": string(secret.Data[TransportURLSelector]), - "MemcachedServers": memcachedInstance.GetMemcachedServerListString(), - "LogFile": fmt.Sprintf("%s%s.log", watcher.WatcherLogPath, instance.Name), - "APIPublicPort": fmt.Sprintf("%d", watcher.WatcherPublicPort), + "KeystoneAuthURL": keystoneInternalURL, + "ServicePassword": string(secret.Data[instance.Spec.PasswordSelectors.Service]), + "ServiceUser": instance.Spec.ServiceUser, + "TransportURL": string(secret.Data[TransportURLSelector]), + "MemcachedServers": memcachedInstance.GetMemcachedServerListString(), + "MemcachedServersWithInet": memcachedInstance.GetMemcachedServerListWithInetString(), + "MemcachedTLS": memcachedInstance.GetMemcachedTLSSupport(), + "LogFile": fmt.Sprintf("%s%s.log", watcher.WatcherLogPath, instance.Name), + "APIPublicPort": fmt.Sprintf("%d", watcher.WatcherPublicPort), } // create httpd vhost template parameters diff --git a/templates/watcher/config/00-default.conf b/templates/watcher/config/00-default.conf index 7ffeccac..13514e7e 100644 --- a/templates/watcher/config/00-default.conf +++ b/templates/watcher/config/00-default.conf @@ -18,9 +18,10 @@ driver = messagingv2 {{ if (index . "KeystoneAuthURL") }} [keystone_authtoken] -{{ if (index . "MemcachedServers") }} -memcached_servers = {{ .MemcachedServers }} -{{ end }} +{{if (index . "MemcachedServersWithInet")}} +memcached_servers={{ .MemcachedServersWithInet }} +{{end}} + # TODO jgilaber implement handling this option when we add tls support # cafile = /var/lib/ca-bundle.pem project_domain_name = Default @@ -56,5 +57,13 @@ datasources = ceilometer {{ if (index . "MemcachedServers") }} [cache] -memcached_servers = {{ .MemcachedServers }} -{{ end }} +{{if .MemcachedTLS}} +backend = dogpile.cache.pymemcache +memcache_servers={{ .MemcachedServers }} +{{else}} +backend = dogpile.cache.memcached +memcache_servers={{ .MemcachedServersWithInet }} +{{end}} +enabled=true +tls_enabled={{ .MemcachedTLS }} +{{end}} From 57fcd161160b12d38f1e274d375608aefc9342c2 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Fri, 17 Jan 2025 12:24:28 +0100 Subject: [PATCH 4/5] Configure CA Cert file for keystone and watcher clients When connecting to internal services, watcher may find they have TLS enabled. This patch is adding the required parameter `cafile` based on the presence of TLS.CaBundleSecretName parameter which defines de Secret of the CA bundle containing the cert used for internal communications. --- controllers/watcherapi_controller.go | 6 ++++++ templates/watcher/config/00-default.conf | 11 ++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/controllers/watcherapi_controller.go b/controllers/watcherapi_controller.go index 34624093..fc148325 100644 --- a/controllers/watcherapi_controller.go +++ b/controllers/watcherapi_controller.go @@ -305,6 +305,11 @@ func (r *WatcherAPIReconciler) generateServiceConfigs( databaseUsername := string(secret.Data[DatabaseUsername]) databaseHostname := string(secret.Data[DatabaseHostname]) databasePassword := string(secret.Data[DatabasePassword]) + + var CaFilePath string + if instance.Spec.TLS.CaBundleSecretName != "" { + CaFilePath = tls.DownstreamTLSCABundlePath + } templateParameters := map[string]interface{}{ "DatabaseConnection": fmt.Sprintf("mysql+pymysql://%s:%s@%s/%s?read_default_file=/etc/my.cnf", databaseUsername, @@ -321,6 +326,7 @@ func (r *WatcherAPIReconciler) generateServiceConfigs( "MemcachedTLS": memcachedInstance.GetMemcachedTLSSupport(), "LogFile": fmt.Sprintf("%s%s.log", watcher.WatcherLogPath, instance.Name), "APIPublicPort": fmt.Sprintf("%d", watcher.WatcherPublicPort), + "CaFilePath": CaFilePath, } // create httpd vhost template parameters diff --git a/templates/watcher/config/00-default.conf b/templates/watcher/config/00-default.conf index 13514e7e..be77d48c 100644 --- a/templates/watcher/config/00-default.conf +++ b/templates/watcher/config/00-default.conf @@ -21,9 +21,6 @@ driver = messagingv2 {{if (index . "MemcachedServersWithInet")}} memcached_servers={{ .MemcachedServersWithInet }} {{end}} - -# TODO jgilaber implement handling this option when we add tls support -# cafile = /var/lib/ca-bundle.pem project_domain_name = Default project_name = service user_domain_name = Default @@ -32,12 +29,13 @@ username = {{ .ServiceUser }} auth_url = {{ .KeystoneAuthURL }} interface = internal auth_type = password +{{if .CaFilePath}} +cafile = {{ .CaFilePath }} +{{ end }} {{ end }} {{ if (index . "KeystoneAuthURL") }} [watcher_clients_auth] -# TODO jgilaber implement handling this option when we add tls support -# cafile = /var/lib/ca-bundle.pem project_domain_name = Default project_name = service user_domain_name = Default @@ -46,6 +44,9 @@ username = {{ .ServiceUser }} auth_url = {{ .KeystoneAuthURL }} interface = internal auth_type = password +{{if .CaFilePath}} +cafile = {{ .CaFilePath }} +{{ end }} {{ end }} From 7ca018b0388787c10ffcfcb99127fc85fa8a45e2 Mon Sep 17 00:00:00 2001 From: Alfredo Moralejo Date: Tue, 21 Jan 2025 13:29:26 +0100 Subject: [PATCH 5/5] Add tests to kuttl scenarios to test TLS backends config This patch checks proper values of config files for TLS enablement. --- config/samples/watcher_v1beta1_watcher.yaml | 2 ++ .../kuttl/test-suites/default/common/deploy-with-defaults.yaml | 2 ++ tests/kuttl/test-suites/default/watcher/01-assert.yaml | 2 ++ .../default/watcher/04-deploy-with-precreated-account.yaml | 2 ++ 4 files changed, 8 insertions(+) diff --git a/config/samples/watcher_v1beta1_watcher.yaml b/config/samples/watcher_v1beta1_watcher.yaml index 441eaf8b..3205c24f 100644 --- a/config/samples/watcher_v1beta1_watcher.yaml +++ b/config/samples/watcher_v1beta1_watcher.yaml @@ -10,3 +10,5 @@ metadata: name: watcher spec: databaseInstance: "openstack" + tls: + caBundleSecretName: "combined-ca-bundle" diff --git a/tests/kuttl/test-suites/default/common/deploy-with-defaults.yaml b/tests/kuttl/test-suites/default/common/deploy-with-defaults.yaml index 3b969bae..01251717 100644 --- a/tests/kuttl/test-suites/default/common/deploy-with-defaults.yaml +++ b/tests/kuttl/test-suites/default/common/deploy-with-defaults.yaml @@ -5,3 +5,5 @@ metadata: namespace: watcher-kuttl-default spec: databaseInstance: "openstack" + tls: + caBundleSecretName: "combined-ca-bundle" diff --git a/tests/kuttl/test-suites/default/watcher/01-assert.yaml b/tests/kuttl/test-suites/default/watcher/01-assert.yaml index 617bf4b6..98e0a9e1 100644 --- a/tests/kuttl/test-suites/default/watcher/01-assert.yaml +++ b/tests/kuttl/test-suites/default/watcher/01-assert.yaml @@ -324,6 +324,8 @@ commands: SERVICEID=$(oc exec -n watcher-kuttl-default openstackclient -- openstack service list -f value -c Name -c Type -c ID | grep watcher| awk '{print $1}') [ $(oc get -n watcher-kuttl-default keystoneservice watcher -o jsonpath={.status.serviceID}) == ${SERVICEID} ] [ -n "$(oc get -n watcher-kuttl-default watcher watcher-kuttl -o jsonpath={.status.hash.dbsync})" ] + [ "$(oc get -n watcher-kuttl-default secret watcher-kuttl-api-config-data -o jsonpath='{.data.my\.cnf}'|base64 -d|grep -c 'ssl=1')" == 1 ] + [ "$(oc get -n watcher-kuttl-default secret watcher-kuttl-api-config-data -o jsonpath='{.data.00-default\.conf}'|base64 -d|grep -c 'cafile = /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem')" == 2 ] # If we are running the container locally, skip following test if [ "$(oc get pods -n openstack-operators -o name -l openstack.org/operator-name=watcher)" == "" ]; then exit 0 diff --git a/tests/kuttl/test-suites/default/watcher/04-deploy-with-precreated-account.yaml b/tests/kuttl/test-suites/default/watcher/04-deploy-with-precreated-account.yaml index 952168d4..93f5ac55 100644 --- a/tests/kuttl/test-suites/default/watcher/04-deploy-with-precreated-account.yaml +++ b/tests/kuttl/test-suites/default/watcher/04-deploy-with-precreated-account.yaml @@ -8,3 +8,5 @@ spec: databaseAccount: watcher-precreated apiServiceTemplate: replicas: 2 + tls: + caBundleSecretName: "combined-ca-bundle"