Add CustomServiceConfig

api/bases/watcher.openstack.org_watcherapis.yaml

@@ -43,6 +43,12 @@ spec:
                 description: The service specific Container Image URL (will be set
                   to environmental default if empty)
                 type: string
+              customServiceConfig:
+                description: |-
+                  CustomServiceConfig - customize the service config using this parameter to change service defaults,
+                  or overwrite rendered information using raw OpenStack config format. The content gets added to
+                  to /etc/<service>/<service>.conf.d directory as a custom config file.
+                type: string
               memcachedInstance:
                 default: memcached
                 description: MemcachedInstance is the name of the Memcached CR that

api/bases/watcher.openstack.org_watchers.yaml

@@ -47,6 +47,12 @@ spec:
                   replicas: 1
                 description: APIServiceTemplate - define the watcher-api service
                 properties:
+                  customServiceConfig:
+                    description: |-
+                      CustomServiceConfig - customize the service config using this parameter to change service defaults,
+                      or overwrite rendered information using raw OpenStack config format. The content gets added to
+                      to /etc/<service>/<service>.conf.d directory as a custom config file.
+                    type: string
                   nodeSelector:
                     additionalProperties:
                       type: string
@@ -167,6 +173,12 @@ spec:
               applierContainerImageURL:
                 description: ApplierContainerImageURL
                 type: string
+              customServiceConfig:
+                description: |-
+                  CustomServiceConfig - customize the service config using this parameter to change service defaults,
+                  or overwrite rendered information using raw OpenStack config format. The content gets added to
+                  to /etc/<service>/<service>.conf.d directory as a custom config file.
+                type: string
               databaseAccount:
                 default: watcher
                 description: DatabaseAccount - MariaDBAccount CR name used for watcher

api/v1beta1/common_types.go

@@ -58,6 +58,12 @@ type WatcherCommon struct {
 	// any global NodeSelector settings within the Watcher CR.
 	NodeSelector *map[string]string `json:"nodeSelector,omitempty"`
 
+	// +kubebuilder:validation:Optional
+	// CustomServiceConfig - customize the service config using this parameter to change service defaults,
+	// or overwrite rendered information using raw OpenStack config format. The content gets added to
+	// to /etc/<service>/<service>.conf.d directory as a custom config file.
+	CustomServiceConfig string `json:"customServiceConfig,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// TLS - Parameters related to the TLS
@@ -148,6 +154,12 @@ type WatcherSubCrsTemplate struct {
 	// NodeSelector to target subset of worker nodes running this component. Setting here overrides
 	// any global NodeSelector settings within the Watcher CR.
 	NodeSelector *map[string]string `json:"nodeSelector,omitempty"`
+
+	// +kubebuilder:validation:Optional
+	// CustomServiceConfig - customize the service config using this parameter to change service defaults,
+	// or overwrite rendered information using raw OpenStack config format. The content gets added to
+	// to /etc/<service>/<service>.conf.d directory as a custom config file.
+	CustomServiceConfig string `json:"customServiceConfig,omitempty"`
 }
 
 // MetalLBConfig to configure the MetalLB loadbalancer service

config/crd/bases/watcher.openstack.org_watcherapis.yaml

@@ -43,6 +43,12 @@ spec:
                 description: The service specific Container Image URL (will be set
                   to environmental default if empty)
                 type: string
+              customServiceConfig:
+                description: |-
+                  CustomServiceConfig - customize the service config using this parameter to change service defaults,
+                  or overwrite rendered information using raw OpenStack config format. The content gets added to
+                  to /etc/<service>/<service>.conf.d directory as a custom config file.
+                type: string
               memcachedInstance:
                 default: memcached
                 description: MemcachedInstance is the name of the Memcached CR that

config/crd/bases/watcher.openstack.org_watchers.yaml

@@ -47,6 +47,12 @@ spec:
                   replicas: 1
                 description: APIServiceTemplate - define the watcher-api service
                 properties:
+                  customServiceConfig:
+                    description: |-
+                      CustomServiceConfig - customize the service config using this parameter to change service defaults,
+                      or overwrite rendered information using raw OpenStack config format. The content gets added to
+                      to /etc/<service>/<service>.conf.d directory as a custom config file.
+                    type: string
                   nodeSelector:
                     additionalProperties:
                       type: string
@@ -167,6 +173,12 @@ spec:
               applierContainerImageURL:
                 description: ApplierContainerImageURL
                 type: string
+              customServiceConfig:
+                description: |-
+                  CustomServiceConfig - customize the service config using this parameter to change service defaults,
+                  or overwrite rendered information using raw OpenStack config format. The content gets added to
+                  to /etc/<service>/<service>.conf.d directory as a custom config file.
+                type: string
               databaseAccount:
                 default: watcher
                 description: DatabaseAccount - MariaDBAccount CR name used for watcher

controllers/watcher_controller.go

@@ -641,7 +641,8 @@ func (r *WatcherReconciler) generateServiceConfigDBSync(
 	}
 	// customData hold any customization for the service.
 	customData := map[string]string{
-		"my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
+		watcher.GlobalCustomConfigFileName: instance.Spec.CustomServiceConfig,
+		"my.cnf":                           db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
 	}
 
 	labels := labels.GetLabels(instance, labels.GetGroupLabel(watcher.ServiceName), map[string]string{})
@@ -733,6 +734,7 @@ func (r *WatcherReconciler) createSubLevelSecret(
 		DatabaseUsername:                        databaseAccount.Spec.UserName,
 		DatabasePassword:                        string(databaseSecret.Data[mariadbv1.DatabasePasswordSelector]),
 		DatabaseHostname:                        db.GetDatabaseHostname(),
+		watcher.GlobalCustomConfigFileName:      instance.Spec.CustomServiceConfig,
 	}
 	secretName := instance.Name
 
@@ -762,11 +764,12 @@ func (r *WatcherReconciler) ensureAPI(
 	watcherAPISpec := watcherv1beta1.WatcherAPISpec{
 		Secret: instance.Name,
 		WatcherCommon: watcherv1beta1.WatcherCommon{
-			ServiceUser:       instance.Spec.ServiceUser,
-			PasswordSelectors: instance.Spec.PasswordSelectors,
-			MemcachedInstance: instance.Spec.MemcachedInstance,
-			NodeSelector:      instance.Spec.APIServiceTemplate.NodeSelector,
-			PreserveJobs:      instance.Spec.PreserveJobs,
+			ServiceUser:         instance.Spec.ServiceUser,
+			PasswordSelectors:   instance.Spec.PasswordSelectors,
+			MemcachedInstance:   instance.Spec.MemcachedInstance,
+			NodeSelector:        instance.Spec.APIServiceTemplate.NodeSelector,
+			PreserveJobs:        instance.Spec.PreserveJobs,
+			CustomServiceConfig: instance.Spec.APIServiceTemplate.CustomServiceConfig,
 		},
 		WatcherSubCrsCommon: watcherv1beta1.WatcherSubCrsCommon{
 			ContainerImage: instance.Spec.APIContainerImageURL,

controllers/watcherapi_controller.go

@@ -164,6 +164,7 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 			instance.Spec.PasswordSelectors.Service,
 			TransportURLSelector,
 			DatabaseAccount,
+			watcher.GlobalCustomConfigFileName,
 		},
 		helper.GetClient(),
 		&instance.Status.Conditions,
@@ -299,7 +300,9 @@ func (r *WatcherAPIReconciler) generateServiceConfigs(
 	}
 	// customData hold any customization for the service.
 	customData := map[string]string{
-		"my.cnf": db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
+		watcher.GlobalCustomConfigFileName:  string(secret.Data[watcher.GlobalCustomConfigFileName]),
+		watcher.ServiceCustomConfigFileName: instance.Spec.CustomServiceConfig,
+		"my.cnf":                            db.GetDatabaseClientConfig(tlsCfg), //(mschuppert) for now just get the default my.cnf
 	}
 
 	databaseUsername := string(secret.Data[DatabaseUsername])

pkg/watcher/constants.go

@@ -20,8 +20,11 @@ const (
 	// DefaultsConfigFileName - File name with default configuration
 	DefaultsConfigFileName = "00-default.conf"
 
-	// CustomConfigFileName - File name with custom configuration
-	CustomConfigFileName = "01-custom.conf"
+	// GlobalCustomConfigFileName - File name with custom configuration define in Watcher
+	GlobalCustomConfigFileName = "01-global-custom.conf"
+
+	// ServiceCustomConfigFileName - File name with custom configuration define in SubCRs
+	ServiceCustomConfigFileName = "02-service-custom.conf"
 
 	// LogVolume is the default logVolume name used to mount logs
 	LogVolume = "logs"

templates/watcher/config/watcher-api-config.json

@@ -7,6 +7,20 @@
       "owner": "watcher",
       "perm": "0600"
     },
+    {
+      "source": "/var/lib/config-data/default/01-global-custom.conf",
+      "dest": "/etc/watcher/watcher.conf.d/01-global-custom.conf",
+      "owner": "watcher",
+      "perm": "0600",
+      "optional": true
+    },
+    {
+      "source": "/var/lib/config-data/default/02-service-custom.conf",
+      "dest": "/etc/watcher/watcher.conf.d/02-service-custom.conf",
+      "owner": "watcher",
+      "perm": "0600",
+      "optional": true
+    },
     {
       "source": "/var/lib/config-data/default/10-watcher-wsgi-main.conf",
       "dest": "/etc/httpd/conf.d/10-watcher-wsgi-main.conf",

tests/functional/base_test.go

@@ -44,9 +44,11 @@ func GetNonDefaultWatcherSpec() map[string]interface{} {
 		"preserveJobs":         true,
 		"databaseInstance":     "fakeopenstack",
 		"serviceUser":          "fakeuser",
+		"customServiceConfig":  "# Global config",
 		"apiServiceTemplate": map[string]interface{}{
-			"replicas":     2,
-			"nodeSelector": map[string]string{"foo": "bar"},
+			"replicas":            2,
+			"nodeSelector":        map[string]string{"foo": "bar"},
+			"customServiceConfig": "# Service config",
 		},
 		"tls": map[string]interface{}{
 			"caBundleSecretName": "combined-ca-bundle",

tests/functional/watcher_controller_test.go

@@ -52,6 +52,8 @@ var _ = Describe("Watcher controller with minimal spec values", func() {
 			Expect(Watcher.Spec.ServiceUser).Should(Equal("watcher"))
 			Expect(Watcher.Spec.PreserveJobs).Should(BeFalse())
 			Expect(Watcher.Spec.TLS.CaBundleSecretName).Should(Equal(""))
+			Expect(Watcher.Spec.CustomServiceConfig).Should(Equal(""))
+			Expect(Watcher.Spec.APIServiceTemplate.CustomServiceConfig).Should(Equal(""))
 		})
 
 		It("should have the Status fields initialized", func() {
@@ -93,6 +95,7 @@ var _ = Describe("Watcher controller", func() {
 			Expect(Watcher.Spec.Secret).Should(Equal("test-osp-secret"))
 			Expect(*(Watcher.Spec.RabbitMqClusterName)).Should(Equal("rabbitmq"))
 			Expect(Watcher.Spec.PreserveJobs).Should(BeFalse())
+
 		})
 
 		It("should have the Status fields initialized", func() {
@@ -346,6 +349,7 @@ var _ = Describe("Watcher controller", func() {
 			Expect(createdSecret.Data["WatcherPassword"]).To(Equal([]byte("password")))
 			Expect(createdSecret.Data["transport_url"]).To(Equal([]byte("rabbit://rabbitmq-secret/fake")))
 			Expect(createdSecret.Data["database_account"]).To(Equal([]byte("watcher")))
+			Expect(createdSecret.Data["01-global-custom.conf"]).To(Equal([]byte("")))
 
 			// Check WatcherAPI is created
 			WatcherAPI := GetWatcherAPI(watcherTest.WatcherAPI)
@@ -355,6 +359,7 @@ var _ = Describe("Watcher controller", func() {
 			Expect(WatcherAPI.Spec.ServiceAccount).To(Equal("watcher-watcher"))
 			Expect(int(*WatcherAPI.Spec.Replicas)).To(Equal(1))
 			Expect(WatcherAPI.Spec.NodeSelector).To(BeNil())
+			Expect(WatcherAPI.Spec.CustomServiceConfig).To(Equal(""))
 
 			// Assert that the watcher deployment is created
 			deployment := th.GetDeployment(watcherTest.WatcherAPIDeployment)
@@ -626,6 +631,8 @@ var _ = Describe("Watcher controller", func() {
 			Expect(Watcher.Spec.PreserveJobs).Should(BeTrue())
 			Expect(*(Watcher.Spec.RabbitMqClusterName)).Should(Equal("rabbitmq"))
 			Expect(Watcher.Spec.TLS.CaBundleSecretName).Should(Equal("combined-ca-bundle"))
+			Expect(Watcher.Spec.CustomServiceConfig).Should(Equal("# Global config"))
+			Expect(Watcher.Spec.APIServiceTemplate.CustomServiceConfig).Should(Equal("# Service config"))
 		})
 
 		It("Should create watcher service with custom values", func() {
@@ -746,6 +753,13 @@ var _ = Describe("Watcher controller", func() {
 			Watcher := GetWatcher(watcherTest.Instance)
 			Expect(Watcher.Status.Hash[watcherv1beta1.DbSyncHash]).ShouldNot(BeNil())
 
+			// assert that the top level secret is created with proper content
+			createdSecret := th.GetSecret(watcherTest.Watcher)
+			Expect(createdSecret).ShouldNot(BeNil())
+			Expect(createdSecret.Data["WatcherPassword"]).To(Equal([]byte("password")))
+			Expect(createdSecret.Data["transport_url"]).To(Equal([]byte("rabbit://rabbitmq-secret/fake")))
+			Expect(createdSecret.Data["01-global-custom.conf"]).To(Equal([]byte("# Global config")))
+
 			// Check WatcherAPI is created with non-default values
 			watcherAPI := &watcherv1beta1.WatcherAPI{}
 			Expect(k8sClient.Get(ctx,
@@ -770,6 +784,7 @@ var _ = Describe("Watcher controller", func() {
 			Expect(int(*WatcherAPI.Spec.Replicas)).To(Equal(2))
 			Expect(*WatcherAPI.Spec.NodeSelector).To(Equal(map[string]string{"foo": "bar"}))
 			Expect(WatcherAPI.Spec.TLS.CaBundleSecretName).Should(Equal("combined-ca-bundle"))
+			Expect(WatcherAPI.Spec.CustomServiceConfig).Should(Equal("# Service config"))
 
 			// Assert that the watcher deployment is created
 			deployment := th.GetDeployment(watcherTest.WatcherAPIDeployment)
@@ -779,7 +794,14 @@ var _ = Describe("Watcher controller", func() {
 			Expect(deployment.Spec.Template.Spec.Containers).To(HaveLen(2))
 			Expect(deployment.Spec.Selector.MatchLabels).To(Equal(map[string]string{"service": "watcher-api"}))
 
+			// Assert that the required custom configuration is applied in the config secret
+			// assert that the top level secret is created with proper content
+			createdConfigSecret := th.GetSecret(types.NamespacedName{Namespace: watcherTest.Instance.Namespace, Name: watcherTest.Instance.Name + "-api-config-data"})
+			Expect(createdConfigSecret).ShouldNot(BeNil())
+			Expect(createdConfigSecret.Data["01-global-custom.conf"]).Should(Equal([]byte("# Global config")))
+			Expect(createdConfigSecret.Data["02-service-custom.conf"]).Should(Equal([]byte("# Service config")))
 		})
+
 	})
 
 })

tests/functional/watcherapi_controller_test.go

@@ -109,12 +109,13 @@ var _ = Describe("WatcherAPI controller", func() {
 			secret := th.CreateSecret(
 				watcherTest.InternalTopLevelSecretName,
 				map[string][]byte{
-					"WatcherPassword":   []byte("service-password"),
-					"transport_url":     []byte("url"),
-					"database_username": []byte("username"),
-					"database_password": []byte("password"),
-					"database_hostname": []byte("hostname"),
-					"database_account":  []byte("watcher"),
+					"WatcherPassword":       []byte("service-password"),
+					"transport_url":         []byte("url"),
+					"database_username":     []byte("username"),
+					"database_password":     []byte("password"),
+					"database_hostname":     []byte("hostname"),
+					"database_account":      []byte("watcher"),
+					"01-global-custom.conf": []byte(""),
 				},
 			)
 			DeferCleanup(k8sClient.Delete, ctx, secret)
@@ -290,12 +291,13 @@ var _ = Describe("WatcherAPI controller", func() {
 			secret := th.CreateSecret(
 				watcherTest.InternalTopLevelSecretName,
 				map[string][]byte{
-					"WatcherPassword":   []byte("service-password"),
-					"transport_url":     []byte("url"),
-					"database_username": []byte("username"),
-					"database_password": []byte("password"),
-					"database_hostname": []byte("hostname"),
-					"database_account":  []byte("watcher"),
+					"WatcherPassword":       []byte("service-password"),
+					"transport_url":         []byte("url"),
+					"database_username":     []byte("username"),
+					"database_password":     []byte("password"),
+					"database_hostname":     []byte("hostname"),
+					"database_account":      []byte("watcher"),
+					"01-global-custom.conf": []byte(""),
 				},
 			)
 			DeferCleanup(k8sClient.Delete, ctx, secret)
@@ -326,12 +328,13 @@ var _ = Describe("WatcherAPI controller", func() {
 			secret := th.CreateSecret(
 				watcherTest.InternalTopLevelSecretName,
 				map[string][]byte{
-					"WatcherPassword":   []byte("service-password"),
-					"transport_url":     []byte("url"),
-					"database_username": []byte("username"),
-					"database_password": []byte("password"),
-					"database_hostname": []byte("hostname"),
-					"database_account":  []byte("watcher"),
+					"WatcherPassword":       []byte("service-password"),
+					"transport_url":         []byte("url"),
+					"database_username":     []byte("username"),
+					"database_password":     []byte("password"),
+					"database_hostname":     []byte("hostname"),
+					"database_account":      []byte("watcher"),
+					"01-global-custom.conf": []byte(""),
 				},
 			)
 			DeferCleanup(k8sClient.Delete, ctx, secret)
@@ -381,9 +384,10 @@ var _ = Describe("WatcherAPI controller", func() {
 			secret := th.CreateSecret(
 				watcherTest.InternalTopLevelSecretName,
 				map[string][]byte{
-					"WatcherPassword":  []byte("service-password"),
-					"transport_url":    []byte("url"),
-					"database_account": []byte("watcher"),
+					"WatcherPassword":       []byte("service-password"),
+					"transport_url":         []byte("url"),
+					"database_account":      []byte("watcher"),
+					"01-global-custom.conf": []byte(""),
 				},
 			)
 			DeferCleanup(k8sClient.Delete, ctx, secret)

tests/kuttl/test-suites/default/watcher/04-assert.yaml

@@ -10,10 +10,14 @@ spec:
   databaseInstance: openstack
   passwordSelectors:
     service: WatcherPassword
+  customServiceConfig: |
+    # Global config
   secret: osp-secret
   apiServiceTemplate:
     replicas: 2
     resources: {}
+    customServiceConfig: |
+      # Service config
 status:
   apiServiceReadyCount: 2
   conditions:
@@ -227,3 +231,18 @@ spec:
 status:
   readyReplicas: 2
   replicas: 2
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+namespaced: true
+commands:
+  - script: |
+      set -euxo pipefail
+      oc project watcher-kuttl-default
+      APIPOD=$(oc get pods -n watcher-kuttl-default -l "service=watcher-api" -ocustom-columns=:metadata.name|grep -v ^$|head -1)
+      if [ -n "${APIPOD}" ]; then
+        [ $(echo $(oc rsh -c watcher-api ${APIPOD} cat /etc/watcher/watcher.conf.d/01-global-custom.conf) |grep -c "^# Global config") == 1 ]
+        [ $(echo $(oc rsh -c watcher-api ${APIPOD} cat /etc/watcher/watcher.conf.d/02-service-custom.conf) |grep -c "^# Service config") == 1 ]
+      else
+        exit 1
+      fi