From dc2ce6a7876067f59b8c228ecf8dbbfc3e747417 Mon Sep 17 00:00:00 2001 From: jgilaber Date: Thu, 9 Jan 2025 13:07:42 +0100 Subject: [PATCH 1/6] Create WatcherAPI service Create the WatcherAPI service. Adds new fields to the spec related to the service. --- .../watcher.openstack.org_watcherapis.yaml | 159 ++++++++++++++++++ api/v1beta1/watcherapi_types.go | 12 ++ api/v1beta1/zz_generated.deepcopy.go | 24 +++ .../watcher.openstack.org_watcherapis.yaml | 159 ++++++++++++++++++ controllers/watcherapi_controller.go | 122 +++++++++++++- pkg/watcher/constants.go | 3 + 6 files changed, 477 insertions(+), 2 deletions(-) diff --git a/api/bases/watcher.openstack.org_watcherapis.yaml b/api/bases/watcher.openstack.org_watcherapis.yaml index 4e57de24..59b16225 100644 --- a/api/bases/watcher.openstack.org_watcherapis.yaml +++ b/api/bases/watcher.openstack.org_watcherapis.yaml @@ -55,6 +55,165 @@ spec: NodeSelector to target subset of worker nodes running this component. Setting here overrides any global NodeSelector settings within the Watcher CR. type: object + override: + description: |- + Override, provides the ability to override the generated manifest of + several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object passwordSelectors: default: service: WatcherPassword diff --git a/api/v1beta1/watcherapi_types.go b/api/v1beta1/watcherapi_types.go index c7e08173..370dfa67 100644 --- a/api/v1beta1/watcherapi_types.go +++ b/api/v1beta1/watcherapi_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -33,6 +34,17 @@ type WatcherAPISpec struct { Secret string `json:"secret"` WatcherSubCrsCommon `json:",inline"` + + // Override, provides the ability to override the generated manifest of + // several child resources. + Override APIOverrideSpec `json:"override,omitempty"` +} + +// APIOverrideSpec to override the generated manifest of several child resources. +type APIOverrideSpec struct { + // Override configuration for the Service created to serve traffic to the cluster. + // The key must be the endpoint type (public, internal) + Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` } // WatcherAPIStatus defines the observed state of WatcherAPI diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index e67c3232..9b9ceb29 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -22,9 +22,32 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIOverrideSpec) DeepCopyInto(out *APIOverrideSpec) { + *out = *in + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = make(map[service.Endpoint]service.RoutedOverrideSpec, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIOverrideSpec. +func (in *APIOverrideSpec) DeepCopy() *APIOverrideSpec { + if in == nil { + return nil + } + out := new(APIOverrideSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PasswordSelector) DeepCopyInto(out *PasswordSelector) { *out = *in @@ -131,6 +154,7 @@ func (in *WatcherAPISpec) DeepCopyInto(out *WatcherAPISpec) { *out = *in in.WatcherCommon.DeepCopyInto(&out.WatcherCommon) in.WatcherSubCrsCommon.DeepCopyInto(&out.WatcherSubCrsCommon) + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherAPISpec. diff --git a/config/crd/bases/watcher.openstack.org_watcherapis.yaml b/config/crd/bases/watcher.openstack.org_watcherapis.yaml index 4e57de24..59b16225 100644 --- a/config/crd/bases/watcher.openstack.org_watcherapis.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherapis.yaml @@ -55,6 +55,165 @@ spec: NodeSelector to target subset of worker nodes running this component. Setting here overrides any global NodeSelector settings within the Watcher CR. type: object + override: + description: |- + Override, provides the ability to override the generated manifest of + several child resources. + properties: + service: + additionalProperties: + description: |- + RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic + to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. + properties: + endpointURL: + type: string + metadata: + description: |- + EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. + Only labels and annotations are included. + properties: + annotations: + additionalProperties: + type: string + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: http://kubernetes.io/docs/user-guide/annotations + type: object + labels: + additionalProperties: + type: string + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: http://kubernetes.io/docs/user-guide/labels + type: object + type: object + spec: + description: |- + OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec + Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, + IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy + properties: + externalName: + description: |- + externalName is the external reference that discovery mechanisms will + return as an alias for this service (e.g. a DNS CNAME record). No + proxying will be involved. Must be a lowercase RFC-1123 hostname + (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". + type: string + externalTrafficPolicy: + description: |- + externalTrafficPolicy describes how nodes distribute service traffic they + receive on one of the Service's "externally-facing" addresses (NodePorts, + ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure + the service in a way that assumes that external load balancers will take care + of balancing the service traffic between nodes, and so each node will deliver + traffic only to the node-local endpoints of the service, without masquerading + the client source IP. (Traffic mistakenly sent to a node with no endpoints will + be dropped.) The default value, "Cluster", uses the standard behavior of + routing to all endpoints evenly (possibly modified by topology and other + features). Note that traffic sent to an External IP or LoadBalancer IP from + within the cluster will always get "Cluster" semantics, but clients sending to + a NodePort from within the cluster may need to take traffic policy into account + when picking a node. + type: string + internalTrafficPolicy: + description: |- + InternalTrafficPolicy describes how nodes distribute service traffic they + receive on the ClusterIP. If set to "Local", the proxy will assume that pods + only want to talk to endpoints of the service on the same node as the pod, + dropping the traffic if there are no local endpoints. The default value, + "Cluster", uses the standard behavior of routing to all endpoints evenly + (possibly modified by topology and other features). + type: string + ipFamilyPolicy: + description: |- + IPFamilyPolicy represents the dual-stack-ness requested or required by + this Service. If there is no value provided, then this field will be set + to SingleStack. Services can be "SingleStack" (a single IP family), + "PreferDualStack" (two IP families on dual-stack configured clusters or + a single IP family on single-stack clusters), or "RequireDualStack" + (two IP families on dual-stack configured clusters, otherwise fail). The + ipFamilies and clusterIPs fields depend on the value of this field. This + field will be wiped when updating a service to type ExternalName. + type: string + loadBalancerClass: + description: |- + loadBalancerClass is the class of the load balancer implementation this Service belongs to. + If specified, the value of this field must be a label-style identifier, with an optional prefix, + e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. + This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load + balancer implementation is used, today this is typically done through the cloud provider integration, + but should apply for any default implementation. If set, it is assumed that a load balancer + implementation is watching for Services with a matching class. Any default load balancer + implementation (e.g. cloud providers) should ignore Services that set this field. + This field can only be set when creating or updating a Service to type 'LoadBalancer'. + Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. + type: string + loadBalancerSourceRanges: + description: |- + If specified and supported by the platform, this will restrict traffic through the cloud-provider + load-balancer will be restricted to the specified client IPs. This field will be ignored if the + cloud-provider does not support the feature." + More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ + items: + type: string + type: array + sessionAffinity: + description: |- + Supports "ClientIP" and "None". Used to maintain session affinity. + Enable client IP based session affinity. + Must be ClientIP or None. + Defaults to None. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies + type: string + sessionAffinityConfig: + description: sessionAffinityConfig contains the configurations + of session affinity. + properties: + clientIP: + description: clientIP contains the configurations + of Client IP based session affinity. + properties: + timeoutSeconds: + description: |- + timeoutSeconds specifies the seconds of ClientIP type session sticky time. + The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". + Default value is 10800(for 3 hours). + format: int32 + type: integer + type: object + type: object + type: + description: |- + type determines how the Service is exposed. Defaults to ClusterIP. Valid + options are ExternalName, ClusterIP, NodePort, and LoadBalancer. + "ClusterIP" allocates a cluster-internal IP address for load-balancing + to endpoints. Endpoints are determined by the selector or if that is not + specified, by manual construction of an Endpoints object or + EndpointSlice objects. If clusterIP is "None", no virtual IP is + allocated and the endpoints are published as a set of endpoints rather + than a virtual IP. + "NodePort" builds on ClusterIP and allocates a port on every node which + routes to the same endpoints as the clusterIP. + "LoadBalancer" builds on NodePort and creates an external load-balancer + (if supported in the current cloud) which routes to the same endpoints + as the clusterIP. + "ExternalName" aliases this service to the specified externalName. + Several other fields do not apply to ExternalName services. + More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types + type: string + type: object + type: object + description: |- + Override configuration for the Service created to serve traffic to the cluster. + The key must be the endpoint type (public, internal) + type: object + type: object passwordSelectors: default: service: WatcherPassword diff --git a/controllers/watcherapi_controller.go b/controllers/watcherapi_controller.go index aa5c471d..d61b1e0e 100644 --- a/controllers/watcherapi_controller.go +++ b/controllers/watcherapi_controller.go @@ -69,6 +69,7 @@ func (r *WatcherAPIReconciler) GetLogger(ctx context.Context) logr.Logger { //+kubebuilder:rbac:groups=watcher.openstack.org,resources=watcherapis/status,verbs=get;update;patch //+kubebuilder:rbac:groups=watcher.openstack.org,resources=watcherapis/finalizers,verbs=update //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete; +//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete; //+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneapis,verbs=get;list;watch; //+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneservices,verbs=get;list;watch;create;update;patch;delete; //+kubebuilder:rbac:groups=keystone.openstack.org,resources=keystoneendpoints,verbs=get;list;watch;create;update;patch;delete; @@ -221,6 +222,14 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) return result, err } + apiEndpoints, result, err := r.ensureServiceExposed(ctx, helper, instance) + if (err != nil || result != ctrl.Result{}) { + // We can ignore RequeueAfter as we are watching the Service resource + // but we have to return while waiting for the service to be exposed + return ctrl.Result{}, err + } + _ = apiEndpoints // we'll use the apiEndpoints later + // We reached the end of the Reconcile, update the Ready condition based on // the sub conditions if instance.Status.Conditions.AllSubConditionIsTrue() { @@ -228,6 +237,7 @@ func (r *WatcherAPIReconciler) Reconcile(ctx context.Context, req ctrl.Request) condition.ReadyCondition, condition.ReadyMessage) } + Log.Info(fmt.Sprintf("Successfully reconciled WatcherAPI instance '%s'", instance.Name)) return ctrl.Result{}, nil } @@ -355,6 +365,112 @@ func (r *WatcherAPIReconciler) createDeployment( return ctrl.Result{}, nil } +func (r *WatcherAPIReconciler) ensureServiceExposed( + ctx context.Context, + helper *helper.Helper, + instance *watcherv1beta1.WatcherAPI, +) (map[string]string, ctrl.Result, error) { + Log := r.GetLogger(ctx) + Log.Info(fmt.Sprintf("Defining WatcherAPI services '%s'", instance.Name)) + + ports := map[service.Endpoint]endpoint.Data{ + service.EndpointPublic: { + Port: watcher.WatcherPublicPort, + }, + service.EndpointInternal: { + Port: watcher.WatcherPublicPort, + }, + } + + apiEndpoints := make(map[string]string) + + for endpointType, data := range ports { + endpointTypeStr := string(endpointType) + endpointName := watcher.ServiceName + "-" + endpointTypeStr + exportLabels := util.MergeStringMaps( + getAPIServiceLabels(), + map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }, + ) + svcOverride := instance.Spec.Override.Service[endpointType] + + svc, err := service.NewService( + service.GenericService(&service.GenericServiceDetails{ + Name: endpointName, + Namespace: instance.Namespace, + Labels: exportLabels, + Selector: getAPIServiceLabels(), + Port: service.GenericServicePort{ + Name: endpointName, + Port: data.Port, + Protocol: corev1.ProtocolTCP, + }, + }), + 5, + &svcOverride.OverrideSpec, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error(), + )) + return nil, ctrl.Result{}, err + } + + svc.AddAnnotation(map[string]string{ + service.AnnotationEndpointKey: endpointTypeStr, + }) + + // add Annotation to whether creating an ingress is required or not + if endpointType == service.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "true", + }) + } else { + svc.AddAnnotation(map[string]string{ + service.AnnotationIngressCreateKey: "false", + }) + if svc.GetServiceType() == corev1.ServiceTypeLoadBalancer { + svc.AddAnnotation(map[string]string{ + service.AnnotationHostnameKey: svc.GetServiceHostname(), // add annotation to register service name in dnsmasq + }) + } + } + + ctrlResult, err := svc.CreateOrPatch(ctx, helper) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.CreateServiceReadyErrorMessage, + err.Error(), + )) + return nil, ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.CreateServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.CreateServiceReadyRunningMessage)) + return nil, ctrlResult, nil + } + + apiEndpoints[endpointTypeStr], err = svc.GetAPIEndpoint(svcOverride.EndpointURL, data.Protocol, data.Path) + if err != nil { + return nil, ctrl.Result{}, err + } + } + + instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) + + return apiEndpoints, ctrl.Result{}, nil +} + func (r *WatcherAPIReconciler) reconcileDelete(ctx context.Context, instance *watcherv1beta1.WatcherAPI, helper *helper.Helper) (ctrl.Result, error) { Log := r.GetLogger(ctx) Log.Info(fmt.Sprintf("Reconcile Service '%s' delete started", instance.Name)) @@ -387,9 +503,10 @@ func (r *WatcherAPIReconciler) initStatus(instance *watcherv1beta1.WatcherAPI) e // failure/in-progress operation condition.UnknownCondition(condition.ReadyCondition, condition.InitReason, condition.ReadyInitMessage), condition.UnknownCondition(condition.InputReadyCondition, condition.InitReason, condition.InputReadyInitMessage), - condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyMessage), + condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), - condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyMessage), + condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + condition.UnknownCondition(condition.CreateServiceReadyCondition, condition.InitReason, condition.CreateServiceReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -422,6 +539,7 @@ func (r *WatcherAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { For(&watcherv1beta1.WatcherAPI{}). Owns(&corev1.Secret{}). Owns(&appsv1.Deployment{}). + Owns(&corev1.Service{}). Watches( &corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), diff --git a/pkg/watcher/constants.go b/pkg/watcher/constants.go index fe9ff464..387a03d7 100644 --- a/pkg/watcher/constants.go +++ b/pkg/watcher/constants.go @@ -29,6 +29,9 @@ const ( // WatcherPublicPort - public port of watcher containers WatcherPublicPort int32 = 9322 + // WatcherPublicPort - internal port of watcher containers + WatcherInternalPort int32 = 9322 + // WatcherLogPath is the path used by WatcherAPI to stream/store its logs WatcherLogPath = "/var/log/watcher/" From ff9e35a495212a82bf7825c7fbff0b76466b5891 Mon Sep 17 00:00:00 2001 From: jgilaber Date: Wed, 15 Jan 2025 16:32:12 +0100 Subject: [PATCH 2/6] Expose the services using lib-common functions Temporary solution to expose the service while the operator is installed standalone. After we integrate watcher-operator with the openstack-operator, we'll rely on it to expose the services like the other do, and this code should be removed. --- .../watcher.openstack.org_watcherapis.yaml | 200 ++++-------------- api/v1beta1/common_types.go | 30 +++ api/v1beta1/watcherapi_types.go | 14 +- api/v1beta1/zz_generated.deepcopy.go | 27 +-- .../watcher.openstack.org_watcherapis.yaml | 200 ++++-------------- config/rbac/role.yaml | 12 ++ controllers/watcherapi_controller.go | 131 +++++------- go.mod | 2 +- main.go | 2 + 9 files changed, 197 insertions(+), 421 deletions(-) diff --git a/api/bases/watcher.openstack.org_watcherapis.yaml b/api/bases/watcher.openstack.org_watcherapis.yaml index 59b16225..40c62ca9 100644 --- a/api/bases/watcher.openstack.org_watcherapis.yaml +++ b/api/bases/watcher.openstack.org_watcherapis.yaml @@ -43,6 +43,47 @@ spec: description: The service specific Container Image URL (will be set to environmental default if empty) type: string + externalEndpoints: + description: ExternalEndpoints, expose a VIP via MetalLB on the pre-created + address pool + items: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + endpoint: + description: Endpoint, OpenStack endpoint this service maps + to + enum: + - internal + - public + type: string + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from the pool + if available. Using a list to allow dual stack (IPv4/IPv6) + support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with multiple + services + type: boolean + sharedIPKey: + default: "" + description: |- + SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + SharedIP is true, but no SharedIPKey specified. + type: string + required: + - ipAddressPool + type: object + type: array memcachedInstance: default: memcached description: MemcachedInstance is the name of the Memcached CR that @@ -55,165 +96,6 @@ spec: NodeSelector to target subset of worker nodes running this component. Setting here overrides any global NodeSelector settings within the Watcher CR. type: object - override: - description: |- - Override, provides the ability to override the generated manifest of - several child resources. - properties: - service: - additionalProperties: - description: |- - RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic - to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. - properties: - endpointURL: - type: string - metadata: - description: |- - EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. - Only labels and annotations are included. - properties: - annotations: - additionalProperties: - type: string - description: |- - Annotations is an unstructured key value map stored with a resource that may be - set by external tools to store and retrieve arbitrary metadata. They are not - queryable and should be preserved when modifying objects. - More info: http://kubernetes.io/docs/user-guide/annotations - type: object - labels: - additionalProperties: - type: string - description: |- - Map of string keys and values that can be used to organize and categorize - (scope and select) objects. May match selectors of replication controllers - and services. - More info: http://kubernetes.io/docs/user-guide/labels - type: object - type: object - spec: - description: |- - OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec - Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, - IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy - properties: - externalName: - description: |- - externalName is the external reference that discovery mechanisms will - return as an alias for this service (e.g. a DNS CNAME record). No - proxying will be involved. Must be a lowercase RFC-1123 hostname - (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: |- - externalTrafficPolicy describes how nodes distribute service traffic they - receive on one of the Service's "externally-facing" addresses (NodePorts, - ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure - the service in a way that assumes that external load balancers will take care - of balancing the service traffic between nodes, and so each node will deliver - traffic only to the node-local endpoints of the service, without masquerading - the client source IP. (Traffic mistakenly sent to a node with no endpoints will - be dropped.) The default value, "Cluster", uses the standard behavior of - routing to all endpoints evenly (possibly modified by topology and other - features). Note that traffic sent to an External IP or LoadBalancer IP from - within the cluster will always get "Cluster" semantics, but clients sending to - a NodePort from within the cluster may need to take traffic policy into account - when picking a node. - type: string - internalTrafficPolicy: - description: |- - InternalTrafficPolicy describes how nodes distribute service traffic they - receive on the ClusterIP. If set to "Local", the proxy will assume that pods - only want to talk to endpoints of the service on the same node as the pod, - dropping the traffic if there are no local endpoints. The default value, - "Cluster", uses the standard behavior of routing to all endpoints evenly - (possibly modified by topology and other features). - type: string - ipFamilyPolicy: - description: |- - IPFamilyPolicy represents the dual-stack-ness requested or required by - this Service. If there is no value provided, then this field will be set - to SingleStack. Services can be "SingleStack" (a single IP family), - "PreferDualStack" (two IP families on dual-stack configured clusters or - a single IP family on single-stack clusters), or "RequireDualStack" - (two IP families on dual-stack configured clusters, otherwise fail). The - ipFamilies and clusterIPs fields depend on the value of this field. This - field will be wiped when updating a service to type ExternalName. - type: string - loadBalancerClass: - description: |- - loadBalancerClass is the class of the load balancer implementation this Service belongs to. - If specified, the value of this field must be a label-style identifier, with an optional prefix, - e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. - This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load - balancer implementation is used, today this is typically done through the cloud provider integration, - but should apply for any default implementation. If set, it is assumed that a load balancer - implementation is watching for Services with a matching class. Any default load balancer - implementation (e.g. cloud providers) should ignore Services that set this field. - This field can only be set when creating or updating a Service to type 'LoadBalancer'. - Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerSourceRanges: - description: |- - If specified and supported by the platform, this will restrict traffic through the cloud-provider - load-balancer will be restricted to the specified client IPs. This field will be ignored if the - cloud-provider does not support the feature." - More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ - items: - type: string - type: array - sessionAffinity: - description: |- - Supports "ClientIP" and "None". Used to maintain session affinity. - Enable client IP based session affinity. - Must be ClientIP or None. - Defaults to None. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations - of session affinity. - properties: - clientIP: - description: clientIP contains the configurations - of Client IP based session affinity. - properties: - timeoutSeconds: - description: |- - timeoutSeconds specifies the seconds of ClientIP type session sticky time. - The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". - Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - type: - description: |- - type determines how the Service is exposed. Defaults to ClusterIP. Valid - options are ExternalName, ClusterIP, NodePort, and LoadBalancer. - "ClusterIP" allocates a cluster-internal IP address for load-balancing - to endpoints. Endpoints are determined by the selector or if that is not - specified, by manual construction of an Endpoints object or - EndpointSlice objects. If clusterIP is "None", no virtual IP is - allocated and the endpoints are published as a set of endpoints rather - than a virtual IP. - "NodePort" builds on ClusterIP and allocates a port on every node which - routes to the same endpoints as the clusterIP. - "LoadBalancer" builds on NodePort and creates an external load-balancer - (if supported in the current cloud) which routes to the same endpoints - as the clusterIP. - "ExternalName" aliases this service to the specified externalName. - Several other fields do not apply to ExternalName services. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types - type: string - type: object - type: object - description: |- - Override configuration for the Service created to serve traffic to the cluster. - The key must be the endpoint type (public, internal) - type: object - type: object passwordSelectors: default: service: WatcherPassword diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index f648a812..bb0a40fe 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta1 import ( + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" ) @@ -143,6 +144,35 @@ type WatcherSubCrsTemplate struct { NodeSelector *map[string]string `json:"nodeSelector,omitempty"` } +// MetalLBConfig to configure the MetalLB loadbalancer service +type MetalLBConfig struct { + // +kubebuilder:validation:Optional + // +kubebuilder:validation:Enum=internal;public + // Endpoint, OpenStack endpoint this service maps to + Endpoint service.Endpoint `json:"endpoint"` + + // +kubebuilder:validation:Required + // +kubebuilder:validation:MinLength=1 + // IPAddressPool expose VIP via MetalLB on the IPAddressPool + IPAddressPool string `json:"ipAddressPool"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default=true + // SharedIP if true, VIP/VIPs get shared with multiple services + SharedIP bool `json:"sharedIP"` + + // +kubebuilder:validation:Optional + // +kubebuilder:default="" + // SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + // Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + // SharedIP is true, but no SharedIPKey specified. + SharedIPKey string `json:"sharedIPKey"` + + // +kubebuilder:validation:Optional + // LoadBalancerIPs, request given IPs from the pool if available. Using a list to allow dual stack (IPv4/IPv6) support + LoadBalancerIPs []string `json:"loadBalancerIPs"` +} + type WatcherImages struct { // +kubebuilder:validation:Required // APIContainerImageURL diff --git a/api/v1beta1/watcherapi_types.go b/api/v1beta1/watcherapi_types.go index 370dfa67..b390a624 100644 --- a/api/v1beta1/watcherapi_types.go +++ b/api/v1beta1/watcherapi_types.go @@ -18,7 +18,6 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" - "github.com/openstack-k8s-operators/lib-common/modules/common/service" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -35,16 +34,9 @@ type WatcherAPISpec struct { WatcherSubCrsCommon `json:",inline"` - // Override, provides the ability to override the generated manifest of - // several child resources. - Override APIOverrideSpec `json:"override,omitempty"` -} - -// APIOverrideSpec to override the generated manifest of several child resources. -type APIOverrideSpec struct { - // Override configuration for the Service created to serve traffic to the cluster. - // The key must be the endpoint type (public, internal) - Service map[service.Endpoint]service.RoutedOverrideSpec `json:"service,omitempty"` + // +kubebuilder:validation:Optional + // ExternalEndpoints, expose a VIP via MetalLB on the pre-created address pool + ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` } // WatcherAPIStatus defines the observed state of WatcherAPI diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 9b9ceb29..86b7e597 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -22,28 +22,25 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" - "github.com/openstack-k8s-operators/lib-common/modules/common/service" "k8s.io/apimachinery/pkg/runtime" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *APIOverrideSpec) DeepCopyInto(out *APIOverrideSpec) { +func (in *MetalLBConfig) DeepCopyInto(out *MetalLBConfig) { *out = *in - if in.Service != nil { - in, out := &in.Service, &out.Service - *out = make(map[service.Endpoint]service.RoutedOverrideSpec, len(*in)) - for key, val := range *in { - (*out)[key] = *val.DeepCopy() - } + if in.LoadBalancerIPs != nil { + in, out := &in.LoadBalancerIPs, &out.LoadBalancerIPs + *out = make([]string, len(*in)) + copy(*out, *in) } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIOverrideSpec. -func (in *APIOverrideSpec) DeepCopy() *APIOverrideSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetalLBConfig. +func (in *MetalLBConfig) DeepCopy() *MetalLBConfig { if in == nil { return nil } - out := new(APIOverrideSpec) + out := new(MetalLBConfig) in.DeepCopyInto(out) return out } @@ -154,7 +151,13 @@ func (in *WatcherAPISpec) DeepCopyInto(out *WatcherAPISpec) { *out = *in in.WatcherCommon.DeepCopyInto(&out.WatcherCommon) in.WatcherSubCrsCommon.DeepCopyInto(&out.WatcherSubCrsCommon) - in.Override.DeepCopyInto(&out.Override) + if in.ExternalEndpoints != nil { + in, out := &in.ExternalEndpoints, &out.ExternalEndpoints + *out = make([]MetalLBConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherAPISpec. diff --git a/config/crd/bases/watcher.openstack.org_watcherapis.yaml b/config/crd/bases/watcher.openstack.org_watcherapis.yaml index 59b16225..40c62ca9 100644 --- a/config/crd/bases/watcher.openstack.org_watcherapis.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherapis.yaml @@ -43,6 +43,47 @@ spec: description: The service specific Container Image URL (will be set to environmental default if empty) type: string + externalEndpoints: + description: ExternalEndpoints, expose a VIP via MetalLB on the pre-created + address pool + items: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + endpoint: + description: Endpoint, OpenStack endpoint this service maps + to + enum: + - internal + - public + type: string + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on the IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from the pool + if available. Using a list to allow dual stack (IPv4/IPv6) + support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with multiple + services + type: boolean + sharedIPKey: + default: "" + description: |- + SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + SharedIP is true, but no SharedIPKey specified. + type: string + required: + - ipAddressPool + type: object + type: array memcachedInstance: default: memcached description: MemcachedInstance is the name of the Memcached CR that @@ -55,165 +96,6 @@ spec: NodeSelector to target subset of worker nodes running this component. Setting here overrides any global NodeSelector settings within the Watcher CR. type: object - override: - description: |- - Override, provides the ability to override the generated manifest of - several child resources. - properties: - service: - additionalProperties: - description: |- - RoutedOverrideSpec - a routed service override configuration for the Service created to serve traffic - to the cluster. Allows for the manifest of the created Service to be overwritten with custom configuration. - properties: - endpointURL: - type: string - metadata: - description: |- - EmbeddedLabelsAnnotations is an embedded subset of the fields included in k8s.io/apimachinery/pkg/apis/meta/v1.ObjectMeta. - Only labels and annotations are included. - properties: - annotations: - additionalProperties: - type: string - description: |- - Annotations is an unstructured key value map stored with a resource that may be - set by external tools to store and retrieve arbitrary metadata. They are not - queryable and should be preserved when modifying objects. - More info: http://kubernetes.io/docs/user-guide/annotations - type: object - labels: - additionalProperties: - type: string - description: |- - Map of string keys and values that can be used to organize and categorize - (scope and select) objects. May match selectors of replication controllers - and services. - More info: http://kubernetes.io/docs/user-guide/labels - type: object - type: object - spec: - description: |- - OverrideServiceSpec is a subset of the fields included in https://pkg.go.dev/k8s.io/api@v0.26.6/core/v1#ServiceSpec - Limited to Type, SessionAffinity, LoadBalancerSourceRanges, ExternalName, ExternalTrafficPolicy, SessionAffinityConfig, - IPFamilyPolicy, LoadBalancerClass and InternalTrafficPolicy - properties: - externalName: - description: |- - externalName is the external reference that discovery mechanisms will - return as an alias for this service (e.g. a DNS CNAME record). No - proxying will be involved. Must be a lowercase RFC-1123 hostname - (https://tools.ietf.org/html/rfc1123) and requires `type` to be "ExternalName". - type: string - externalTrafficPolicy: - description: |- - externalTrafficPolicy describes how nodes distribute service traffic they - receive on one of the Service's "externally-facing" addresses (NodePorts, - ExternalIPs, and LoadBalancer IPs). If set to "Local", the proxy will configure - the service in a way that assumes that external load balancers will take care - of balancing the service traffic between nodes, and so each node will deliver - traffic only to the node-local endpoints of the service, without masquerading - the client source IP. (Traffic mistakenly sent to a node with no endpoints will - be dropped.) The default value, "Cluster", uses the standard behavior of - routing to all endpoints evenly (possibly modified by topology and other - features). Note that traffic sent to an External IP or LoadBalancer IP from - within the cluster will always get "Cluster" semantics, but clients sending to - a NodePort from within the cluster may need to take traffic policy into account - when picking a node. - type: string - internalTrafficPolicy: - description: |- - InternalTrafficPolicy describes how nodes distribute service traffic they - receive on the ClusterIP. If set to "Local", the proxy will assume that pods - only want to talk to endpoints of the service on the same node as the pod, - dropping the traffic if there are no local endpoints. The default value, - "Cluster", uses the standard behavior of routing to all endpoints evenly - (possibly modified by topology and other features). - type: string - ipFamilyPolicy: - description: |- - IPFamilyPolicy represents the dual-stack-ness requested or required by - this Service. If there is no value provided, then this field will be set - to SingleStack. Services can be "SingleStack" (a single IP family), - "PreferDualStack" (two IP families on dual-stack configured clusters or - a single IP family on single-stack clusters), or "RequireDualStack" - (two IP families on dual-stack configured clusters, otherwise fail). The - ipFamilies and clusterIPs fields depend on the value of this field. This - field will be wiped when updating a service to type ExternalName. - type: string - loadBalancerClass: - description: |- - loadBalancerClass is the class of the load balancer implementation this Service belongs to. - If specified, the value of this field must be a label-style identifier, with an optional prefix, - e.g. "internal-vip" or "example.com/internal-vip". Unprefixed names are reserved for end-users. - This field can only be set when the Service type is 'LoadBalancer'. If not set, the default load - balancer implementation is used, today this is typically done through the cloud provider integration, - but should apply for any default implementation. If set, it is assumed that a load balancer - implementation is watching for Services with a matching class. Any default load balancer - implementation (e.g. cloud providers) should ignore Services that set this field. - This field can only be set when creating or updating a Service to type 'LoadBalancer'. - Once set, it can not be changed. This field will be wiped when a service is updated to a non 'LoadBalancer' type. - type: string - loadBalancerSourceRanges: - description: |- - If specified and supported by the platform, this will restrict traffic through the cloud-provider - load-balancer will be restricted to the specified client IPs. This field will be ignored if the - cloud-provider does not support the feature." - More info: https://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/ - items: - type: string - type: array - sessionAffinity: - description: |- - Supports "ClientIP" and "None". Used to maintain session affinity. - Enable client IP based session affinity. - Must be ClientIP or None. - Defaults to None. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#virtual-ips-and-service-proxies - type: string - sessionAffinityConfig: - description: sessionAffinityConfig contains the configurations - of session affinity. - properties: - clientIP: - description: clientIP contains the configurations - of Client IP based session affinity. - properties: - timeoutSeconds: - description: |- - timeoutSeconds specifies the seconds of ClientIP type session sticky time. - The value must be >0 && <=86400(for 1 day) if ServiceAffinity == "ClientIP". - Default value is 10800(for 3 hours). - format: int32 - type: integer - type: object - type: object - type: - description: |- - type determines how the Service is exposed. Defaults to ClusterIP. Valid - options are ExternalName, ClusterIP, NodePort, and LoadBalancer. - "ClusterIP" allocates a cluster-internal IP address for load-balancing - to endpoints. Endpoints are determined by the selector or if that is not - specified, by manual construction of an Endpoints object or - EndpointSlice objects. If clusterIP is "None", no virtual IP is - allocated and the endpoints are published as a set of endpoints rather - than a virtual IP. - "NodePort" builds on ClusterIP and allocates a port on every node which - routes to the same endpoints as the clusterIP. - "LoadBalancer" builds on NodePort and creates an external load-balancer - (if supported in the current cloud) which routes to the same endpoints - as the clusterIP. - "ExternalName" aliases this service to the specified externalName. - Several other fields do not apply to ExternalName services. - More info: https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types - type: string - type: object - type: object - description: |- - Override configuration for the Service created to serve traffic to the cluster. - The key must be the endpoint type (public, internal) - type: object - type: object passwordSelectors: default: service: WatcherPassword diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index f054a1f4..c1752ce0 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -188,6 +188,18 @@ rules: - patch - update - watch +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - security.openshift.io resourceNames: diff --git a/controllers/watcherapi_controller.go b/controllers/watcherapi_controller.go index d61b1e0e..f6695ae4 100644 --- a/controllers/watcherapi_controller.go +++ b/controllers/watcherapi_controller.go @@ -19,6 +19,8 @@ package controllers import ( "context" "fmt" + "net/url" + "strings" "time" ctrl "sigs.k8s.io/controller-runtime" @@ -47,6 +49,7 @@ import ( "github.com/openstack-k8s-operators/watcher-operator/pkg/watcher" "github.com/openstack-k8s-operators/watcher-operator/pkg/watcherapi" + routev1 "github.com/openshift/api/route/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" k8s_errors "k8s.io/apimachinery/pkg/api/errors" @@ -76,6 +79,7 @@ func (r *WatcherAPIReconciler) GetLogger(ctx context.Context) logr.Logger { //+kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds,verbs=get;list;watch;update;patch //+kubebuilder:rbac:groups=memcached.openstack.org,resources=memcacheds/finalizers,verbs=update;patch //+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete; +//+kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete; // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. @@ -382,91 +386,59 @@ func (r *WatcherAPIReconciler) ensureServiceExposed( }, } - apiEndpoints := make(map[string]string) - - for endpointType, data := range ports { - endpointTypeStr := string(endpointType) - endpointName := watcher.ServiceName + "-" + endpointTypeStr - exportLabels := util.MergeStringMaps( - getAPIServiceLabels(), - map[string]string{ - service.AnnotationEndpointKey: endpointTypeStr, - }, - ) - svcOverride := instance.Spec.Override.Service[endpointType] - - svc, err := service.NewService( - service.GenericService(&service.GenericServiceDetails{ - Name: endpointName, - Namespace: instance.Namespace, - Labels: exportLabels, - Selector: getAPIServiceLabels(), - Port: service.GenericServicePort{ - Name: endpointName, - Port: data.Port, - Protocol: corev1.ProtocolTCP, - }, - }), - 5, - &svcOverride.OverrideSpec, - ) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.CreateServiceReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.CreateServiceReadyErrorMessage, - err.Error(), - )) - return nil, ctrl.Result{}, err + for _, metallbcfg := range instance.Spec.ExternalEndpoints { + portCfg := ports[metallbcfg.Endpoint] + portCfg.MetalLB = &endpoint.MetalLBData{ + IPAddressPool: metallbcfg.IPAddressPool, + SharedIP: metallbcfg.SharedIP, + SharedIPKey: metallbcfg.SharedIPKey, + LoadBalancerIPs: metallbcfg.LoadBalancerIPs, } - svc.AddAnnotation(map[string]string{ - service.AnnotationEndpointKey: endpointTypeStr, - }) - - // add Annotation to whether creating an ingress is required or not - if endpointType == service.EndpointPublic && svc.GetServiceType() == corev1.ServiceTypeClusterIP { - svc.AddAnnotation(map[string]string{ - service.AnnotationIngressCreateKey: "true", - }) - } else { - svc.AddAnnotation(map[string]string{ - service.AnnotationIngressCreateKey: "false", - }) - if svc.GetServiceType() == corev1.ServiceTypeLoadBalancer { - svc.AddAnnotation(map[string]string{ - service.AnnotationHostnameKey: svc.GetServiceHostname(), // add annotation to register service name in dnsmasq - }) - } - } + ports[metallbcfg.Endpoint] = portCfg + } - ctrlResult, err := svc.CreateOrPatch(ctx, helper) - if err != nil { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.CreateServiceReadyCondition, - condition.ErrorReason, - condition.SeverityWarning, - condition.CreateServiceReadyErrorMessage, - err.Error(), - )) - return nil, ctrlResult, err - } else if (ctrlResult != ctrl.Result{}) { - instance.Status.Conditions.Set(condition.FalseCondition( - condition.CreateServiceReadyCondition, - condition.RequestedReason, - condition.SeverityInfo, - condition.CreateServiceReadyRunningMessage)) - return nil, ctrlResult, nil - } + apiEndpoints, ctrlResult, err := endpoint.ExposeEndpoints( + ctx, + helper, + watcher.ServiceName, + getAPIServiceLabels(), + ports, + r.RequeueTimeout, + ) + if err != nil { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.ErrorReason, + condition.SeverityWarning, + condition.ExposeServiceReadyErrorMessage, + err.Error(), + )) + return nil, ctrlResult, err + } else if (ctrlResult != ctrl.Result{}) { + instance.Status.Conditions.Set(condition.FalseCondition( + condition.ExposeServiceReadyCondition, + condition.RequestedReason, + condition.SeverityInfo, + condition.ExposeServiceReadyRunningMessage, + )) + return nil, ctrlResult, nil + } - apiEndpoints[endpointTypeStr], err = svc.GetAPIEndpoint(svcOverride.EndpointURL, data.Protocol, data.Path) - if err != nil { - return nil, ctrl.Result{}, err + // fix wrongly formatted endpoint url gotten from the lib-common + // ExposeEndpoints function + for endpointType, endpointURL := range apiEndpoints { + // fix repeated '://' in endpoint url + endpointURL = strings.Replace(endpointURL, "://://", "://", 1) + if endpointType == string(service.EndpointPublic) { + // remove trailing port number + url, _ := url.Parse(endpointURL) + endpointURL = fmt.Sprintf("%s://%s", url.Scheme, url.Hostname()) } + apiEndpoints[endpointType] = endpointURL } - instance.Status.Conditions.MarkTrue(condition.CreateServiceReadyCondition, condition.CreateServiceReadyMessage) + instance.Status.Conditions.MarkTrue(condition.ExposeServiceReadyCondition, condition.ExposeServiceReadyMessage) return apiEndpoints, ctrl.Result{}, nil } @@ -506,7 +478,7 @@ func (r *WatcherAPIReconciler) initStatus(instance *watcherv1beta1.WatcherAPI) e condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), condition.UnknownCondition(condition.MemcachedReadyCondition, condition.InitReason, condition.MemcachedReadyInitMessage), condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), - condition.UnknownCondition(condition.CreateServiceReadyCondition, condition.InitReason, condition.CreateServiceReadyInitMessage), + condition.UnknownCondition(condition.ExposeServiceReadyCondition, condition.InitReason, condition.ExposeServiceReadyInitMessage), ) instance.Status.Conditions.Init(&cl) @@ -540,6 +512,7 @@ func (r *WatcherAPIReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&corev1.Secret{}). Owns(&appsv1.Deployment{}). Owns(&corev1.Service{}). + Owns(&routev1.Route{}). Watches( &corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.findObjectsForSrc), diff --git a/go.mod b/go.mod index a350da60..e9625565 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/google/uuid v1.6.0 github.com/onsi/ginkgo/v2 v2.20.1 github.com/onsi/gomega v1.34.1 + github.com/openshift/api v3.9.0+incompatible github.com/openstack-k8s-operators/infra-operator/apis v0.5.0 github.com/openstack-k8s-operators/keystone-operator/api v0.5.1-0.20241023160107-bd8e671350e1 github.com/openstack-k8s-operators/lib-common/modules/common v0.5.1-0.20241029151503-4878b3fa3333 @@ -49,7 +50,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/openshift/api v3.9.0+incompatible // indirect github.com/openstack-k8s-operators/lib-common/modules/openstack v0.5.1-0.20241029151503-4878b3fa3333 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_golang v1.18.0 // indirect diff --git a/main.go b/main.go index f8c4afaf..82adbc52 100644 --- a/main.go +++ b/main.go @@ -27,6 +27,7 @@ import ( "k8s.io/client-go/kubernetes" _ "k8s.io/client-go/plugin/pkg/client/auth" + routev1 "github.com/openshift/api/route/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" @@ -60,6 +61,7 @@ func init() { utilruntime.Must(rabbitmqv1.AddToScheme(scheme)) utilruntime.Must(keystonev1.AddToScheme(scheme)) utilruntime.Must(memcachedv1.AddToScheme(scheme)) + utilruntime.Must(routev1.AddToScheme(scheme)) //+kubebuilder:scaffold:scheme } From b8fe02acf0b25b2b2cd2b11bf9d9b95d41e87b46 Mon Sep 17 00:00:00 2001 From: jgilaber Date: Thu, 9 Jan 2025 15:49:13 +0100 Subject: [PATCH 3/6] Add tests for WatcherAPI service addition --- tests/functional/watcher_test_data.go | 10 +++++++++ .../functional/watcherapi_controller_test.go | 13 ++++++++++++ .../default/watcher/01-assert.yaml | 21 +++++++++++++++++++ .../default/watcher/04-assert.yaml | 4 ++++ .../default/watcher/05-errors.yaml | 16 ++++++++++++++ 5 files changed, 64 insertions(+) diff --git a/tests/functional/watcher_test_data.go b/tests/functional/watcher_test_data.go index c4d06dd7..2d7a5c9d 100644 --- a/tests/functional/watcher_test_data.go +++ b/tests/functional/watcher_test_data.go @@ -47,6 +47,8 @@ type WatcherTestData struct { RoleBindingName types.NamespacedName WatcherDBSync types.NamespacedName WatcherAPIDeployment types.NamespacedName + WatcherPublicServiceName types.NamespacedName + WatcherInternalServiceName types.NamespacedName } // GetWatcherTestData is a function that initialize the WatcherTestData @@ -115,5 +117,13 @@ func GetWatcherTestData(watcherName types.NamespacedName) WatcherTestData { Namespace: watcherName.Namespace, Name: "watcher-api", }, + WatcherPublicServiceName: types.NamespacedName{ + Namespace: watcherName.Namespace, + Name: "watcher-public", + }, + WatcherInternalServiceName: types.NamespacedName{ + Namespace: watcherName.Namespace, + Name: "watcher-internal", + }, } } diff --git a/tests/functional/watcherapi_controller_test.go b/tests/functional/watcherapi_controller_test.go index 2e0ae8ce..b367bb48 100644 --- a/tests/functional/watcherapi_controller_test.go +++ b/tests/functional/watcherapi_controller_test.go @@ -183,6 +183,19 @@ var _ = Describe("WatcherAPI controller", func() { Expect(container.LivenessProbe.HTTPGet.Port.IntVal).To(Equal(int32(9322))) Expect(container.ReadinessProbe.HTTPGet.Port.IntVal).To(Equal(int32(9322))) }) + It("creates the public and internal services for the watcher-api service", func() { + th.ExpectCondition( + watcherTest.WatcherAPI, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.CreateServiceReadyCondition, + corev1.ConditionTrue, + ) + public := th.GetService(watcherTest.WatcherPublicServiceName) + Expect(public.Labels["service"]).To(Equal("watcher-api")) + internal := th.GetService(watcherTest.WatcherInternalServiceName) + Expect(internal.Labels["service"]).To(Equal("watcher-api")) + + }) }) When("the secret is created but missing fields", func() { BeforeEach(func() { diff --git a/tests/kuttl/test-suites/default/watcher/01-assert.yaml b/tests/kuttl/test-suites/default/watcher/01-assert.yaml index 7e6d2ea1..40517afd 100644 --- a/tests/kuttl/test-suites/default/watcher/01-assert.yaml +++ b/tests/kuttl/test-suites/default/watcher/01-assert.yaml @@ -204,6 +204,10 @@ status: reason: Ready status: "True" type: Ready + - message: Create service completed + reason: Ready + status: "True" + type: CreateServiceReady - message: Deployment completed reason: Ready status: "True" @@ -250,6 +254,23 @@ spec: status: phase: Running --- +apiVersion: v1 +kind: Service +metadata: + labels: + endpoint: internal + service: watcher-api + name: watcher-internal +spec: + ports: + - name: watcher-internal + port: 9322 + protocol: TCP + targetPort: 9322 + selector: + service: watcher-api + type: ClusterIP +--- apiVersion: kuttl.dev/v1beta1 kind: TestAssert namespaced: true diff --git a/tests/kuttl/test-suites/default/watcher/04-assert.yaml b/tests/kuttl/test-suites/default/watcher/04-assert.yaml index 0f45843c..d9e128b7 100644 --- a/tests/kuttl/test-suites/default/watcher/04-assert.yaml +++ b/tests/kuttl/test-suites/default/watcher/04-assert.yaml @@ -185,6 +185,10 @@ status: reason: Ready status: "True" type: Ready + - message: Create service completed + reason: Ready + status: "True" + type: CreateServiceReady - message: Deployment completed reason: Ready status: "True" diff --git a/tests/kuttl/test-suites/default/watcher/05-errors.yaml b/tests/kuttl/test-suites/default/watcher/05-errors.yaml index 26c20cfd..317c7362 100644 --- a/tests/kuttl/test-suites/default/watcher/05-errors.yaml +++ b/tests/kuttl/test-suites/default/watcher/05-errors.yaml @@ -67,3 +67,19 @@ kind: Pod metadata: labels: service: watcher-api +--- +apiVersion: v1 +kind: Service +metadata: + labels: + endpoint: internal + service: watcher-api + name: watcher-internal +--- +apiVersion: v1 +kind: Service +metadata: + labels: + endpoint: public + service: watcher-api + name: watcher-public From c95e461d7873e4813ad369c7ad12966fdbe0aa7b Mon Sep 17 00:00:00 2001 From: jgilaber Date: Fri, 17 Jan 2025 12:47:49 +0100 Subject: [PATCH 4/6] Expose service override fields to top-level CR --- api/bases/watcher.openstack.org_watchers.yaml | 42 +++++++++++++++++++ api/v1beta1/watcherapi_types.go | 4 ++ api/v1beta1/zz_generated.deepcopy.go | 7 ++++ .../bases/watcher.openstack.org_watchers.yaml | 42 +++++++++++++++++++ controllers/watcher_controller.go | 1 + 5 files changed, 96 insertions(+) diff --git a/api/bases/watcher.openstack.org_watchers.yaml b/api/bases/watcher.openstack.org_watchers.yaml index d22ad36a..f2144259 100644 --- a/api/bases/watcher.openstack.org_watchers.yaml +++ b/api/bases/watcher.openstack.org_watchers.yaml @@ -47,6 +47,48 @@ spec: replicas: 1 description: APIServiceTemplate - define the watcher-api service properties: + externalEndpoints: + description: ExternalEndpoints, expose a VIP via MetalLB on the + pre-created address pool + items: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + endpoint: + description: Endpoint, OpenStack endpoint this service maps + to + enum: + - internal + - public + type: string + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on the + IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from the + pool if available. Using a list to allow dual stack (IPv4/IPv6) + support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with + multiple services + type: boolean + sharedIPKey: + default: "" + description: |- + SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + SharedIP is true, but no SharedIPKey specified. + type: string + required: + - ipAddressPool + type: object + type: array nodeSelector: additionalProperties: type: string diff --git a/api/v1beta1/watcherapi_types.go b/api/v1beta1/watcherapi_types.go index b390a624..f32a8851 100644 --- a/api/v1beta1/watcherapi_types.go +++ b/api/v1beta1/watcherapi_types.go @@ -61,6 +61,10 @@ type WatcherAPIStatus struct { // create a WatcherAPI via higher level CRDs. type WatcherAPITemplate struct { WatcherSubCrsTemplate `json:",inline"` + + // +kubebuilder:validation:Optional + // ExternalEndpoints, expose a VIP via MetalLB on the pre-created address pool + ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 86b7e597..9ca28528 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -203,6 +203,13 @@ func (in *WatcherAPIStatus) DeepCopy() *WatcherAPIStatus { func (in *WatcherAPITemplate) DeepCopyInto(out *WatcherAPITemplate) { *out = *in in.WatcherSubCrsTemplate.DeepCopyInto(&out.WatcherSubCrsTemplate) + if in.ExternalEndpoints != nil { + in, out := &in.ExternalEndpoints, &out.ExternalEndpoints + *out = make([]MetalLBConfig, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherAPITemplate. diff --git a/config/crd/bases/watcher.openstack.org_watchers.yaml b/config/crd/bases/watcher.openstack.org_watchers.yaml index d22ad36a..f2144259 100644 --- a/config/crd/bases/watcher.openstack.org_watchers.yaml +++ b/config/crd/bases/watcher.openstack.org_watchers.yaml @@ -47,6 +47,48 @@ spec: replicas: 1 description: APIServiceTemplate - define the watcher-api service properties: + externalEndpoints: + description: ExternalEndpoints, expose a VIP via MetalLB on the + pre-created address pool + items: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + endpoint: + description: Endpoint, OpenStack endpoint this service maps + to + enum: + - internal + - public + type: string + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on the + IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from the + pool if available. Using a list to allow dual stack (IPv4/IPv6) + support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with + multiple services + type: boolean + sharedIPKey: + default: "" + description: |- + SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + SharedIP is true, but no SharedIPKey specified. + type: string + required: + - ipAddressPool + type: object + type: array nodeSelector: additionalProperties: type: string diff --git a/controllers/watcher_controller.go b/controllers/watcher_controller.go index 79331a57..7abd4c80 100644 --- a/controllers/watcher_controller.go +++ b/controllers/watcher_controller.go @@ -765,6 +765,7 @@ func (r *WatcherReconciler) ensureAPI( Resources: instance.Spec.APIServiceTemplate.Resources, ServiceAccount: "watcher-" + instance.Name, }, + ExternalEndpoints: instance.Spec.APIServiceTemplate.ExternalEndpoints, } // If NodeSelector is not specified in Watcher APIServiceTemplate, the current From cebf13526c0b7aa29636552d2219b7fbbe8c8b0a Mon Sep 17 00:00:00 2001 From: jgilaber Date: Fri, 17 Jan 2025 10:43:44 +0100 Subject: [PATCH 5/6] Add tests for exposing WatcherAPI service --- tests/functional/suite_test.go | 6 ++ tests/functional/watcher_test_data.go | 10 ++ .../functional/watcherapi_controller_test.go | 101 ++++++++++++------ .../default/watcher/01-assert.yaml | 41 ++++++- .../default/watcher/04-assert.yaml | 8 +- .../default/watcher/05-errors.yaml | 12 ++- 6 files changed, 132 insertions(+), 46 deletions(-) diff --git a/tests/functional/suite_test.go b/tests/functional/suite_test.go index 63d0e883..d17c5795 100644 --- a/tests/functional/suite_test.go +++ b/tests/functional/suite_test.go @@ -28,6 +28,7 @@ import ( metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" + routev1 "github.com/openshift/api/route/v1" memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" rabbitmqv1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1" keystonev1 "github.com/openstack-k8s-operators/keystone-operator/api/v1beta1" @@ -97,6 +98,8 @@ var _ = BeforeSuite(func() { keystoneCRDs, err := test.GetCRDDirFromModule( "github.com/openstack-k8s-operators/keystone-operator/api", "../../go.mod", "bases") Expect(err).ShouldNot(HaveOccurred()) + routev1CRDs, err := test.GetOpenShiftCRDDir("route/v1", "../../go.mod") + Expect(err).ShouldNot(HaveOccurred()) By("bootstrapping test environment") testEnv = &envtest.Environment{ @@ -105,6 +108,7 @@ var _ = BeforeSuite(func() { mariaDBCRDs, rabbitmqCRDs, keystoneCRDs, + routev1CRDs, }, ErrorIfCRDPathMissing: true, @@ -136,6 +140,8 @@ var _ = BeforeSuite(func() { Expect(err).NotTo(HaveOccurred()) err = memcachedv1.AddToScheme(scheme.Scheme) Expect(err).NotTo(HaveOccurred()) + err = routev1.AddToScheme(scheme.Scheme) + Expect(err).NotTo(HaveOccurred()) logger = ctrl.Log.WithName("---Test---") //+kubebuilder:scaffold:scheme diff --git a/tests/functional/watcher_test_data.go b/tests/functional/watcher_test_data.go index 2d7a5c9d..b201e5c7 100644 --- a/tests/functional/watcher_test_data.go +++ b/tests/functional/watcher_test_data.go @@ -49,6 +49,8 @@ type WatcherTestData struct { WatcherAPIDeployment types.NamespacedName WatcherPublicServiceName types.NamespacedName WatcherInternalServiceName types.NamespacedName + WatcherRouteName types.NamespacedName + WatcherInternalRouteName types.NamespacedName } // GetWatcherTestData is a function that initialize the WatcherTestData @@ -125,5 +127,13 @@ func GetWatcherTestData(watcherName types.NamespacedName) WatcherTestData { Namespace: watcherName.Namespace, Name: "watcher-internal", }, + WatcherRouteName: types.NamespacedName{ + Namespace: watcherName.Namespace, + Name: "watcher-public", + }, + WatcherInternalRouteName: types.NamespacedName{ + Namespace: watcherName.Namespace, + Name: "watcher-internal", + }, } } diff --git a/tests/functional/watcherapi_controller_test.go b/tests/functional/watcherapi_controller_test.go index b367bb48..693d0f2f 100644 --- a/tests/functional/watcherapi_controller_test.go +++ b/tests/functional/watcherapi_controller_test.go @@ -10,7 +10,6 @@ import ( memcachedv1 "github.com/openstack-k8s-operators/infra-operator/apis/memcached/v1beta1" condition "github.com/openstack-k8s-operators/lib-common/modules/common/condition" . "github.com/openstack-k8s-operators/lib-common/modules/common/test/helpers" - mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" watcherv1beta1 "github.com/openstack-k8s-operators/watcher-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/utils/ptr" @@ -114,14 +113,6 @@ var _ = Describe("WatcherAPI controller", func() { }, ) DeferCleanup(k8sClient.Delete, ctx, secret) - mariadb.CreateMariaDBDatabase(watcherTest.WatcherDatabaseName.Namespace, watcherTest.WatcherDatabaseName.Name, mariadbv1.MariaDBDatabaseSpec{}) - DeferCleanup(k8sClient.Delete, ctx, mariadb.GetMariaDBDatabase(watcherTest.WatcherDatabaseName)) - - mariadb.SimulateMariaDBTLSDatabaseCompleted(watcherTest.WatcherDatabaseName) - apiMariaDBAccount, apiMariaDBSecret := mariadb.CreateMariaDBAccountAndSecret( - watcherTest.WatcherDatabaseAccount, mariadbv1.MariaDBAccountSpec{}) - DeferCleanup(k8sClient.Delete, ctx, apiMariaDBAccount) - DeferCleanup(k8sClient.Delete, ctx, apiMariaDBSecret) DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace)) memcachedSpec := memcachedv1.MemcachedSpec{ @@ -183,18 +174,18 @@ var _ = Describe("WatcherAPI controller", func() { Expect(container.LivenessProbe.HTTPGet.Port.IntVal).To(Equal(int32(9322))) Expect(container.ReadinessProbe.HTTPGet.Port.IntVal).To(Equal(int32(9322))) }) - It("creates the public and internal services for the watcher-api service", func() { + It("exposes the watcher-api service", func() { th.ExpectCondition( watcherTest.WatcherAPI, ConditionGetterFunc(WatcherAPIConditionGetter), - condition.CreateServiceReadyCondition, + condition.ExposeServiceReadyCondition, corev1.ConditionTrue, ) public := th.GetService(watcherTest.WatcherPublicServiceName) Expect(public.Labels["service"]).To(Equal("watcher-api")) internal := th.GetService(watcherTest.WatcherInternalServiceName) Expect(internal.Labels["service"]).To(Equal("watcher-api")) - + th.AssertRouteExists(watcherTest.WatcherRouteName) }) }) When("the secret is created but missing fields", func() { @@ -204,14 +195,6 @@ var _ = Describe("WatcherAPI controller", func() { map[string][]byte{}, ) DeferCleanup(k8sClient.Delete, ctx, secret) - mariadb.CreateMariaDBDatabase(watcherTest.WatcherDatabaseName.Namespace, watcherTest.WatcherDatabaseName.Name, mariadbv1.MariaDBDatabaseSpec{}) - DeferCleanup(k8sClient.Delete, ctx, mariadb.GetMariaDBDatabase(watcherTest.WatcherDatabaseName)) - - mariadb.SimulateMariaDBTLSDatabaseCompleted(watcherTest.WatcherDatabaseName) - apiMariaDBAccount, apiMariaDBSecret := mariadb.CreateMariaDBAccountAndSecret( - watcherTest.WatcherDatabaseAccount, mariadbv1.MariaDBAccountSpec{}) - DeferCleanup(k8sClient.Delete, ctx, apiMariaDBAccount) - DeferCleanup(k8sClient.Delete, ctx, apiMariaDBSecret) DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) }) It("should have input false", func() { @@ -265,14 +248,7 @@ var _ = Describe("WatcherAPI controller", func() { }, ) DeferCleanup(k8sClient.Delete, ctx, secret) - mariadb.CreateMariaDBDatabase(watcherTest.WatcherDatabaseName.Namespace, watcherTest.WatcherDatabaseName.Name, mariadbv1.MariaDBDatabaseSpec{}) - DeferCleanup(k8sClient.Delete, ctx, mariadb.GetMariaDBDatabase(watcherTest.WatcherDatabaseName)) - mariadb.SimulateMariaDBTLSDatabaseCompleted(watcherTest.WatcherDatabaseName) - apiMariaDBAccount, apiMariaDBSecret := mariadb.CreateMariaDBAccountAndSecret( - watcherTest.WatcherDatabaseAccount, mariadbv1.MariaDBAccountSpec{}) - DeferCleanup(k8sClient.Delete, ctx, apiMariaDBAccount) - DeferCleanup(k8sClient.Delete, ctx, apiMariaDBSecret) DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, GetDefaultWatcherAPISpec())) }) It("should have input ready true", func() { @@ -307,14 +283,6 @@ var _ = Describe("WatcherAPI controller", func() { }, ) DeferCleanup(k8sClient.Delete, ctx, secret) - mariadb.CreateMariaDBDatabase(watcherTest.WatcherDatabaseName.Namespace, watcherTest.WatcherDatabaseName.Name, mariadbv1.MariaDBDatabaseSpec{}) - DeferCleanup(k8sClient.Delete, ctx, mariadb.GetMariaDBDatabase(watcherTest.WatcherDatabaseName)) - - mariadb.SimulateMariaDBTLSDatabaseCompleted(watcherTest.WatcherDatabaseName) - apiMariaDBAccount, apiMariaDBSecret := mariadb.CreateMariaDBAccountAndSecret( - watcherTest.WatcherDatabaseAccount, mariadbv1.MariaDBAccountSpec{}) - DeferCleanup(k8sClient.Delete, ctx, apiMariaDBAccount) - DeferCleanup(k8sClient.Delete, ctx, apiMariaDBSecret) memcachedSpec := memcachedv1.MemcachedSpec{ MemcachedSpecCore: memcachedv1.MemcachedSpecCore{ Replicas: ptr.To(int32(1)), @@ -356,4 +324,67 @@ var _ = Describe("WatcherAPI controller", func() { ) }) }) + When("WatcherAPI is created with extrenalEndpoints", func() { + BeforeEach(func() { + secret := th.CreateSecret( + watcherTest.InternalTopLevelSecretName, + map[string][]byte{ + "WatcherPassword": []byte("service-password"), + "transport_url": []byte("url"), + }, + ) + DeferCleanup(k8sClient.Delete, ctx, secret) + spec := GetDefaultWatcherAPISpec() + var externalEndpoints []interface{} + externalEndpoints = append( + externalEndpoints, map[string]interface{}{ + "endpoint": "internal", + "ipAddressPool": "osp-internalapi", + "loadBalancerIPs": []string{"internal-lb-ip-1", "internal-lb-ip-2"}, + }, + ) + spec["externalEndpoints"] = externalEndpoints + DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, spec)) + DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace)) + memcachedSpec := memcachedv1.MemcachedSpec{ + MemcachedSpecCore: memcachedv1.MemcachedSpecCore{ + Replicas: ptr.To(int32(1)), + }, + } + DeferCleanup(infra.DeleteMemcached, infra.CreateMemcached(watcherTest.WatcherAPI.Namespace, MemcachedInstance, memcachedSpec)) + infra.SimulateMemcachedReady(watcherTest.MemcachedNamespace) + + }) + It("creates MetalLB service", func() { + th.SimulateDeploymentReplicaReady(watcherTest.WatcherAPIDeployment) + + // As the public endpoint is not mentioned in the ExternalEndpoints + // a generic Service and a Route is created + public := th.GetService(watcherTest.WatcherPublicServiceName) + Expect(public.Annotations).NotTo(HaveKey("metallb.universe.tf/address-pool")) + Expect(public.Annotations).NotTo(HaveKey("metallb.universe.tf/allow-shared-ip")) + Expect(public.Annotations).NotTo(HaveKey("metallb.universe.tf/loadBalancerIPs")) + th.AssertRouteExists(watcherTest.WatcherRouteName) + + // As the internal endpoint is configure in ExternalEndpoints it + // does not get a Route but a Service with MetalLB annotations + // instead + internal := th.GetService(watcherTest.WatcherInternalServiceName) + Expect(internal.Annotations).To(HaveKeyWithValue("metallb.universe.tf/address-pool", "osp-internalapi")) + Expect(internal.Annotations).To(HaveKeyWithValue("metallb.universe.tf/allow-shared-ip", "osp-internalapi")) + Expect(internal.Annotations).To(HaveKeyWithValue("metallb.universe.tf/loadBalancerIPs", "internal-lb-ip-1,internal-lb-ip-2")) + th.AssertRouteNotExists(watcherTest.WatcherInternalRouteName) + + // simulate that the internal service got a LoadBalancerIP + // assigned + th.SimulateLoadBalancerServiceIP(watcherTest.WatcherInternalServiceName) + + th.ExpectCondition( + watcherTest.WatcherAPI, + ConditionGetterFunc(WatcherAPIConditionGetter), + condition.ReadyCondition, + corev1.ConditionTrue, + ) + }) + }) }) diff --git a/tests/kuttl/test-suites/default/watcher/01-assert.yaml b/tests/kuttl/test-suites/default/watcher/01-assert.yaml index 40517afd..9ae80f76 100644 --- a/tests/kuttl/test-suites/default/watcher/01-assert.yaml +++ b/tests/kuttl/test-suites/default/watcher/01-assert.yaml @@ -204,14 +204,14 @@ status: reason: Ready status: "True" type: Ready - - message: Create service completed - reason: Ready - status: "True" - type: CreateServiceReady - message: Deployment completed reason: Ready status: "True" type: DeploymentReady + - message: Exposing service completed + reason: Ready + status: "True" + type: ExposeServiceReady - message: Input data complete reason: Ready status: "True" @@ -258,7 +258,24 @@ apiVersion: v1 kind: Service metadata: labels: - endpoint: internal + public: "true" + service: watcher-api + name: watcher-public +spec: + ports: + - name: watcher-public + port: 9322 + protocol: TCP + targetPort: 9322 + selector: + service: watcher-api + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + internal: "true" service: watcher-api name: watcher-internal spec: @@ -271,6 +288,20 @@ spec: service: watcher-api type: ClusterIP --- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + public: "true" + service: watcher-api + name: watcher-public +spec: + port: + targetPort: watcher-public + to: + kind: Service + name: watcher-public +--- apiVersion: kuttl.dev/v1beta1 kind: TestAssert namespaced: true diff --git a/tests/kuttl/test-suites/default/watcher/04-assert.yaml b/tests/kuttl/test-suites/default/watcher/04-assert.yaml index d9e128b7..6bd0fd51 100644 --- a/tests/kuttl/test-suites/default/watcher/04-assert.yaml +++ b/tests/kuttl/test-suites/default/watcher/04-assert.yaml @@ -185,14 +185,14 @@ status: reason: Ready status: "True" type: Ready - - message: Create service completed - reason: Ready - status: "True" - type: CreateServiceReady - message: Deployment completed reason: Ready status: "True" type: DeploymentReady + - message: Exposing service completed + reason: Ready + status: "True" + type: ExposeServiceReady - message: Input data complete reason: Ready status: "True" diff --git a/tests/kuttl/test-suites/default/watcher/05-errors.yaml b/tests/kuttl/test-suites/default/watcher/05-errors.yaml index 317c7362..633d9dd9 100644 --- a/tests/kuttl/test-suites/default/watcher/05-errors.yaml +++ b/tests/kuttl/test-suites/default/watcher/05-errors.yaml @@ -72,7 +72,7 @@ apiVersion: v1 kind: Service metadata: labels: - endpoint: internal + internal: "true" service: watcher-api name: watcher-internal --- @@ -80,6 +80,14 @@ apiVersion: v1 kind: Service metadata: labels: - endpoint: public + public: "true" + service: watcher-api + name: watcher-public +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + public: "true" service: watcher-api name: watcher-public From 8d7d0ed6f7abfd6e537b37c33526b04528f2d186 Mon Sep 17 00:00:00 2001 From: jgilaber Date: Mon, 20 Jan 2025 19:36:23 +0100 Subject: [PATCH 6/6] Use Override spec field instead of ExternalEndpoints Use the Override spec field that is used in the other operators, so once we change the code to expose the services to use the same method used by the openstack operator, the spec will go through less changes. --- .../watcher.openstack.org_watcherapis.yaml | 86 ++--- api/bases/watcher.openstack.org_watchers.yaml | 87 ++--- api/v1beta1/common_types.go | 6 - api/v1beta1/watcherapi_types.go | 22 +- api/v1beta1/zz_generated.deepcopy.go | 39 +- .../watcher.openstack.org_watcherapis.yaml | 86 ++--- .../bases/watcher.openstack.org_watchers.yaml | 87 ++--- controllers/watcher_controller.go | 2 +- controllers/watcherapi_controller.go | 15 +- .../functional/watcherapi_controller_test.go | 23 +- .../00-cleanup-watcher.yaml | 1 + .../01-assert.yaml | 337 ++++++++++++++++++ .../01-deploy-with-defaults.yaml | 14 + .../05-assert.yaml | 7 + .../05-cleanup-watcher.yaml | 1 + .../05-errors.yaml | 93 +++++ 16 files changed, 696 insertions(+), 210 deletions(-) create mode 120000 tests/kuttl/test-suites/default/watcher-api-service-override/00-cleanup-watcher.yaml create mode 100644 tests/kuttl/test-suites/default/watcher-api-service-override/01-assert.yaml create mode 100644 tests/kuttl/test-suites/default/watcher-api-service-override/01-deploy-with-defaults.yaml create mode 100644 tests/kuttl/test-suites/default/watcher-api-service-override/05-assert.yaml create mode 120000 tests/kuttl/test-suites/default/watcher-api-service-override/05-cleanup-watcher.yaml create mode 100644 tests/kuttl/test-suites/default/watcher-api-service-override/05-errors.yaml diff --git a/api/bases/watcher.openstack.org_watcherapis.yaml b/api/bases/watcher.openstack.org_watcherapis.yaml index 40c62ca9..7c40340f 100644 --- a/api/bases/watcher.openstack.org_watcherapis.yaml +++ b/api/bases/watcher.openstack.org_watcherapis.yaml @@ -43,47 +43,6 @@ spec: description: The service specific Container Image URL (will be set to environmental default if empty) type: string - externalEndpoints: - description: ExternalEndpoints, expose a VIP via MetalLB on the pre-created - address pool - items: - description: MetalLBConfig to configure the MetalLB loadbalancer - service - properties: - endpoint: - description: Endpoint, OpenStack endpoint this service maps - to - enum: - - internal - - public - type: string - ipAddressPool: - description: IPAddressPool expose VIP via MetalLB on the IPAddressPool - minLength: 1 - type: string - loadBalancerIPs: - description: LoadBalancerIPs, request given IPs from the pool - if available. Using a list to allow dual stack (IPv4/IPv6) - support - items: - type: string - type: array - sharedIP: - default: true - description: SharedIP if true, VIP/VIPs get shared with multiple - services - type: boolean - sharedIPKey: - default: "" - description: |- - SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. - Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if - SharedIP is true, but no SharedIPKey specified. - type: string - required: - - ipAddressPool - type: object - type: array memcachedInstance: default: memcached description: MemcachedInstance is the name of the Memcached CR that @@ -96,6 +55,51 @@ spec: NodeSelector to target subset of worker nodes running this component. Setting here overrides any global NodeSelector settings within the Watcher CR. type: object + override: + description: |- + Override, provides the ability to override the generated manifest of + several child resources. + properties: + service: + additionalProperties: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on the + IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from the + pool if available. Using a list to allow dual stack (IPv4/IPv6) + support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with + multiple services + type: boolean + sharedIPKey: + default: "" + description: |- + SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + SharedIP is true, but no SharedIPKey specified. + type: string + required: + - ipAddressPool + type: object + description: |- + Override configuration for the Service created to serve traffic to + the cluster. + The key must be the endpoint type (public, internal) + temporarily use MetalLBConfig struct, later we'll switch to + service.RoutedOverrideSpec + type: object + type: object passwordSelectors: default: service: WatcherPassword diff --git a/api/bases/watcher.openstack.org_watchers.yaml b/api/bases/watcher.openstack.org_watchers.yaml index f2144259..949f895a 100644 --- a/api/bases/watcher.openstack.org_watchers.yaml +++ b/api/bases/watcher.openstack.org_watchers.yaml @@ -47,48 +47,6 @@ spec: replicas: 1 description: APIServiceTemplate - define the watcher-api service properties: - externalEndpoints: - description: ExternalEndpoints, expose a VIP via MetalLB on the - pre-created address pool - items: - description: MetalLBConfig to configure the MetalLB loadbalancer - service - properties: - endpoint: - description: Endpoint, OpenStack endpoint this service maps - to - enum: - - internal - - public - type: string - ipAddressPool: - description: IPAddressPool expose VIP via MetalLB on the - IPAddressPool - minLength: 1 - type: string - loadBalancerIPs: - description: LoadBalancerIPs, request given IPs from the - pool if available. Using a list to allow dual stack (IPv4/IPv6) - support - items: - type: string - type: array - sharedIP: - default: true - description: SharedIP if true, VIP/VIPs get shared with - multiple services - type: boolean - sharedIPKey: - default: "" - description: |- - SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. - Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if - SharedIP is true, but no SharedIPKey specified. - type: string - required: - - ipAddressPool - type: object - type: array nodeSelector: additionalProperties: type: string @@ -96,6 +54,51 @@ spec: NodeSelector to target subset of worker nodes running this component. Setting here overrides any global NodeSelector settings within the Watcher CR. type: object + override: + description: |- + Override, provides the ability to override the generated manifest of + several child resources. + properties: + service: + additionalProperties: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on + the IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from + the pool if available. Using a list to allow dual + stack (IPv4/IPv6) support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with + multiple services + type: boolean + sharedIPKey: + default: "" + description: |- + SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + SharedIP is true, but no SharedIPKey specified. + type: string + required: + - ipAddressPool + type: object + description: |- + Override configuration for the Service created to serve traffic to + the cluster. + The key must be the endpoint type (public, internal) + temporarily use MetalLBConfig struct, later we'll switch to + service.RoutedOverrideSpec + type: object + type: object replicas: default: 1 description: Replicas of Watcher service to run diff --git a/api/v1beta1/common_types.go b/api/v1beta1/common_types.go index bb0a40fe..963acc6b 100644 --- a/api/v1beta1/common_types.go +++ b/api/v1beta1/common_types.go @@ -17,7 +17,6 @@ limitations under the License. package v1beta1 import ( - "github.com/openstack-k8s-operators/lib-common/modules/common/service" "github.com/openstack-k8s-operators/lib-common/modules/common/util" corev1 "k8s.io/api/core/v1" ) @@ -146,11 +145,6 @@ type WatcherSubCrsTemplate struct { // MetalLBConfig to configure the MetalLB loadbalancer service type MetalLBConfig struct { - // +kubebuilder:validation:Optional - // +kubebuilder:validation:Enum=internal;public - // Endpoint, OpenStack endpoint this service maps to - Endpoint service.Endpoint `json:"endpoint"` - // +kubebuilder:validation:Required // +kubebuilder:validation:MinLength=1 // IPAddressPool expose VIP via MetalLB on the IPAddressPool diff --git a/api/v1beta1/watcherapi_types.go b/api/v1beta1/watcherapi_types.go index f32a8851..7f9b8e72 100644 --- a/api/v1beta1/watcherapi_types.go +++ b/api/v1beta1/watcherapi_types.go @@ -18,6 +18,7 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -35,8 +36,9 @@ type WatcherAPISpec struct { WatcherSubCrsCommon `json:",inline"` // +kubebuilder:validation:Optional - // ExternalEndpoints, expose a VIP via MetalLB on the pre-created address pool - ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` + // Override, provides the ability to override the generated manifest of + // several child resources. + Override APIOverrideSpec `json:"override,omitempty"` } // WatcherAPIStatus defines the observed state of WatcherAPI @@ -57,14 +59,26 @@ type WatcherAPIStatus struct { Hash map[string]string `json:"hash,omitempty"` } +// APIOverrideSpec to override the generated manifest of several child +// resources. +type APIOverrideSpec struct { + // Override configuration for the Service created to serve traffic to + // the cluster. + // The key must be the endpoint type (public, internal) + // temporarily use MetalLBConfig struct, later we'll switch to + // service.RoutedOverrideSpec + Service map[service.Endpoint]MetalLBConfig `json:"service,omitempty"` +} + // WatcherAPITemplate defines the input parameters specified by the user to // create a WatcherAPI via higher level CRDs. type WatcherAPITemplate struct { WatcherSubCrsTemplate `json:",inline"` // +kubebuilder:validation:Optional - // ExternalEndpoints, expose a VIP via MetalLB on the pre-created address pool - ExternalEndpoints []MetalLBConfig `json:"externalEndpoints,omitempty"` + // Override, provides the ability to override the generated manifest of + // several child resources. + Override APIOverrideSpec `json:"override,omitempty"` } //+kubebuilder:object:root=true diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 9ca28528..911e8320 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -22,9 +22,32 @@ package v1beta1 import ( "github.com/openstack-k8s-operators/lib-common/modules/common/condition" + "github.com/openstack-k8s-operators/lib-common/modules/common/service" "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *APIOverrideSpec) DeepCopyInto(out *APIOverrideSpec) { + *out = *in + if in.Service != nil { + in, out := &in.Service, &out.Service + *out = make(map[service.Endpoint]MetalLBConfig, len(*in)) + for key, val := range *in { + (*out)[key] = *val.DeepCopy() + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new APIOverrideSpec. +func (in *APIOverrideSpec) DeepCopy() *APIOverrideSpec { + if in == nil { + return nil + } + out := new(APIOverrideSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetalLBConfig) DeepCopyInto(out *MetalLBConfig) { *out = *in @@ -151,13 +174,7 @@ func (in *WatcherAPISpec) DeepCopyInto(out *WatcherAPISpec) { *out = *in in.WatcherCommon.DeepCopyInto(&out.WatcherCommon) in.WatcherSubCrsCommon.DeepCopyInto(&out.WatcherSubCrsCommon) - if in.ExternalEndpoints != nil { - in, out := &in.ExternalEndpoints, &out.ExternalEndpoints - *out = make([]MetalLBConfig, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherAPISpec. @@ -203,13 +220,7 @@ func (in *WatcherAPIStatus) DeepCopy() *WatcherAPIStatus { func (in *WatcherAPITemplate) DeepCopyInto(out *WatcherAPITemplate) { *out = *in in.WatcherSubCrsTemplate.DeepCopyInto(&out.WatcherSubCrsTemplate) - if in.ExternalEndpoints != nil { - in, out := &in.ExternalEndpoints, &out.ExternalEndpoints - *out = make([]MetalLBConfig, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } + in.Override.DeepCopyInto(&out.Override) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WatcherAPITemplate. diff --git a/config/crd/bases/watcher.openstack.org_watcherapis.yaml b/config/crd/bases/watcher.openstack.org_watcherapis.yaml index 40c62ca9..7c40340f 100644 --- a/config/crd/bases/watcher.openstack.org_watcherapis.yaml +++ b/config/crd/bases/watcher.openstack.org_watcherapis.yaml @@ -43,47 +43,6 @@ spec: description: The service specific Container Image URL (will be set to environmental default if empty) type: string - externalEndpoints: - description: ExternalEndpoints, expose a VIP via MetalLB on the pre-created - address pool - items: - description: MetalLBConfig to configure the MetalLB loadbalancer - service - properties: - endpoint: - description: Endpoint, OpenStack endpoint this service maps - to - enum: - - internal - - public - type: string - ipAddressPool: - description: IPAddressPool expose VIP via MetalLB on the IPAddressPool - minLength: 1 - type: string - loadBalancerIPs: - description: LoadBalancerIPs, request given IPs from the pool - if available. Using a list to allow dual stack (IPv4/IPv6) - support - items: - type: string - type: array - sharedIP: - default: true - description: SharedIP if true, VIP/VIPs get shared with multiple - services - type: boolean - sharedIPKey: - default: "" - description: |- - SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. - Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if - SharedIP is true, but no SharedIPKey specified. - type: string - required: - - ipAddressPool - type: object - type: array memcachedInstance: default: memcached description: MemcachedInstance is the name of the Memcached CR that @@ -96,6 +55,51 @@ spec: NodeSelector to target subset of worker nodes running this component. Setting here overrides any global NodeSelector settings within the Watcher CR. type: object + override: + description: |- + Override, provides the ability to override the generated manifest of + several child resources. + properties: + service: + additionalProperties: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on the + IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from the + pool if available. Using a list to allow dual stack (IPv4/IPv6) + support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with + multiple services + type: boolean + sharedIPKey: + default: "" + description: |- + SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + SharedIP is true, but no SharedIPKey specified. + type: string + required: + - ipAddressPool + type: object + description: |- + Override configuration for the Service created to serve traffic to + the cluster. + The key must be the endpoint type (public, internal) + temporarily use MetalLBConfig struct, later we'll switch to + service.RoutedOverrideSpec + type: object + type: object passwordSelectors: default: service: WatcherPassword diff --git a/config/crd/bases/watcher.openstack.org_watchers.yaml b/config/crd/bases/watcher.openstack.org_watchers.yaml index f2144259..949f895a 100644 --- a/config/crd/bases/watcher.openstack.org_watchers.yaml +++ b/config/crd/bases/watcher.openstack.org_watchers.yaml @@ -47,48 +47,6 @@ spec: replicas: 1 description: APIServiceTemplate - define the watcher-api service properties: - externalEndpoints: - description: ExternalEndpoints, expose a VIP via MetalLB on the - pre-created address pool - items: - description: MetalLBConfig to configure the MetalLB loadbalancer - service - properties: - endpoint: - description: Endpoint, OpenStack endpoint this service maps - to - enum: - - internal - - public - type: string - ipAddressPool: - description: IPAddressPool expose VIP via MetalLB on the - IPAddressPool - minLength: 1 - type: string - loadBalancerIPs: - description: LoadBalancerIPs, request given IPs from the - pool if available. Using a list to allow dual stack (IPv4/IPv6) - support - items: - type: string - type: array - sharedIP: - default: true - description: SharedIP if true, VIP/VIPs get shared with - multiple services - type: boolean - sharedIPKey: - default: "" - description: |- - SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. - Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if - SharedIP is true, but no SharedIPKey specified. - type: string - required: - - ipAddressPool - type: object - type: array nodeSelector: additionalProperties: type: string @@ -96,6 +54,51 @@ spec: NodeSelector to target subset of worker nodes running this component. Setting here overrides any global NodeSelector settings within the Watcher CR. type: object + override: + description: |- + Override, provides the ability to override the generated manifest of + several child resources. + properties: + service: + additionalProperties: + description: MetalLBConfig to configure the MetalLB loadbalancer + service + properties: + ipAddressPool: + description: IPAddressPool expose VIP via MetalLB on + the IPAddressPool + minLength: 1 + type: string + loadBalancerIPs: + description: LoadBalancerIPs, request given IPs from + the pool if available. Using a list to allow dual + stack (IPv4/IPv6) support + items: + type: string + type: array + sharedIP: + default: true + description: SharedIP if true, VIP/VIPs get shared with + multiple services + type: boolean + sharedIPKey: + default: "" + description: |- + SharedIPKey specifies the sharing key which gets set as the annotation on the LoadBalancer service. + Services which share the same VIP must have the same SharedIPKey. Defaults to the IPAddressPool if + SharedIP is true, but no SharedIPKey specified. + type: string + required: + - ipAddressPool + type: object + description: |- + Override configuration for the Service created to serve traffic to + the cluster. + The key must be the endpoint type (public, internal) + temporarily use MetalLBConfig struct, later we'll switch to + service.RoutedOverrideSpec + type: object + type: object replicas: default: 1 description: Replicas of Watcher service to run diff --git a/controllers/watcher_controller.go b/controllers/watcher_controller.go index 7abd4c80..634960ae 100644 --- a/controllers/watcher_controller.go +++ b/controllers/watcher_controller.go @@ -765,7 +765,7 @@ func (r *WatcherReconciler) ensureAPI( Resources: instance.Spec.APIServiceTemplate.Resources, ServiceAccount: "watcher-" + instance.Name, }, - ExternalEndpoints: instance.Spec.APIServiceTemplate.ExternalEndpoints, + Override: instance.Spec.APIServiceTemplate.Override, } // If NodeSelector is not specified in Watcher APIServiceTemplate, the current diff --git a/controllers/watcherapi_controller.go b/controllers/watcherapi_controller.go index f6695ae4..0bac4e6d 100644 --- a/controllers/watcherapi_controller.go +++ b/controllers/watcherapi_controller.go @@ -386,16 +386,17 @@ func (r *WatcherAPIReconciler) ensureServiceExposed( }, } - for _, metallbcfg := range instance.Spec.ExternalEndpoints { - portCfg := ports[metallbcfg.Endpoint] + for endpointType := range instance.Spec.Override.Service { + svcOverride := instance.Spec.Override.Service[endpointType] + portCfg := ports[endpointType] portCfg.MetalLB = &endpoint.MetalLBData{ - IPAddressPool: metallbcfg.IPAddressPool, - SharedIP: metallbcfg.SharedIP, - SharedIPKey: metallbcfg.SharedIPKey, - LoadBalancerIPs: metallbcfg.LoadBalancerIPs, + IPAddressPool: svcOverride.IPAddressPool, + SharedIP: svcOverride.SharedIP, + SharedIPKey: svcOverride.SharedIPKey, + LoadBalancerIPs: svcOverride.LoadBalancerIPs, } - ports[metallbcfg.Endpoint] = portCfg + ports[endpointType] = portCfg } apiEndpoints, ctrlResult, err := endpoint.ExposeEndpoints( diff --git a/tests/functional/watcherapi_controller_test.go b/tests/functional/watcherapi_controller_test.go index 693d0f2f..52dd7fe6 100644 --- a/tests/functional/watcherapi_controller_test.go +++ b/tests/functional/watcherapi_controller_test.go @@ -324,7 +324,7 @@ var _ = Describe("WatcherAPI controller", func() { ) }) }) - When("WatcherAPI is created with extrenalEndpoints", func() { + When("WatcherAPI is created with service overrides", func() { BeforeEach(func() { secret := th.CreateSecret( watcherTest.InternalTopLevelSecretName, @@ -335,15 +335,14 @@ var _ = Describe("WatcherAPI controller", func() { ) DeferCleanup(k8sClient.Delete, ctx, secret) spec := GetDefaultWatcherAPISpec() - var externalEndpoints []interface{} - externalEndpoints = append( - externalEndpoints, map[string]interface{}{ - "endpoint": "internal", - "ipAddressPool": "osp-internalapi", - "loadBalancerIPs": []string{"internal-lb-ip-1", "internal-lb-ip-2"}, - }, - ) - spec["externalEndpoints"] = externalEndpoints + apiOverrideSpec := map[string]interface{}{} + endpoint := map[string]interface{}{} + internalEndpoint := map[string]interface{}{} + endpoint["ipAddressPool"] = "osp-internalapi" + endpoint["loadBalancerIPs"] = []string{"internal-lb-ip-1", "internal-lb-ip-2"} + internalEndpoint["internal"] = endpoint + apiOverrideSpec["service"] = internalEndpoint + spec["override"] = apiOverrideSpec DeferCleanup(th.DeleteInstance, CreateWatcherAPI(watcherTest.WatcherAPI, spec)) DeferCleanup(keystone.DeleteKeystoneAPI, keystone.CreateKeystoneAPI(watcherTest.WatcherAPI.Namespace)) memcachedSpec := memcachedv1.MemcachedSpec{ @@ -358,7 +357,7 @@ var _ = Describe("WatcherAPI controller", func() { It("creates MetalLB service", func() { th.SimulateDeploymentReplicaReady(watcherTest.WatcherAPIDeployment) - // As the public endpoint is not mentioned in the ExternalEndpoints + // As the public endpoint is not mentioned in the service override // a generic Service and a Route is created public := th.GetService(watcherTest.WatcherPublicServiceName) Expect(public.Annotations).NotTo(HaveKey("metallb.universe.tf/address-pool")) @@ -366,7 +365,7 @@ var _ = Describe("WatcherAPI controller", func() { Expect(public.Annotations).NotTo(HaveKey("metallb.universe.tf/loadBalancerIPs")) th.AssertRouteExists(watcherTest.WatcherRouteName) - // As the internal endpoint is configure in ExternalEndpoints it + // As the internal endpoint is configure in the service override it // does not get a Route but a Service with MetalLB annotations // instead internal := th.GetService(watcherTest.WatcherInternalServiceName) diff --git a/tests/kuttl/test-suites/default/watcher-api-service-override/00-cleanup-watcher.yaml b/tests/kuttl/test-suites/default/watcher-api-service-override/00-cleanup-watcher.yaml new file mode 120000 index 00000000..92ed6e0b --- /dev/null +++ b/tests/kuttl/test-suites/default/watcher-api-service-override/00-cleanup-watcher.yaml @@ -0,0 +1 @@ +../common/cleanup-watcher.yaml \ No newline at end of file diff --git a/tests/kuttl/test-suites/default/watcher-api-service-override/01-assert.yaml b/tests/kuttl/test-suites/default/watcher-api-service-override/01-assert.yaml new file mode 100644 index 00000000..f3e5b95b --- /dev/null +++ b/tests/kuttl/test-suites/default/watcher-api-service-override/01-assert.yaml @@ -0,0 +1,337 @@ +apiVersion: watcher.openstack.org/v1beta1 +kind: Watcher +metadata: + finalizers: + - openstack.org/watcher + name: watcher-kuttl + namespace: watcher-kuttl-default +spec: + apiContainerImageURL: "quay.io/podified-master-centos9/openstack-watcher-api:current-podified" + decisionengineContainerImageURL: "quay.io/podified-master-centos9/openstack-watcher-decision-engine:current-podified" + applierContainerImageURL: "quay.io/podified-master-centos9/openstack-watcher-applier:current-podified" + databaseAccount: watcher + databaseInstance: openstack + passwordSelectors: + service: WatcherPassword + preserveJobs: false + rabbitMqClusterName: rabbitmq + secret: osp-secret + serviceUser: watcher + apiServiceTemplate: + replicas: 1 + resources: {} +status: + apiServiceReadyCount: 1 + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: DB create completed + reason: Ready + status: "True" + type: DBReady + - message: DBsync completed + reason: Ready + status: "True" + type: DBSyncReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: Setup complete + reason: Ready + status: "True" + type: KeystoneServiceReady + - message: MariaDBAccount creation complete + reason: Ready + status: "True" + type: MariaDBAccountReady + - message: RoleBinding created + reason: Ready + status: "True" + type: RoleBindingReady + - message: Role created + reason: Ready + status: "True" + type: RoleReady + - message: ServiceAccount created + reason: Ready + status: "True" + type: ServiceAccountReady + - message: Service config create completed + reason: Ready + status: "True" + type: ServiceConfigReady + - message: Setup complete + reason: Ready + status: "True" + type: WatcherAPIReady + - message: WatcherRabbitMQTransportURL successfully created + reason: Ready + status: "True" + type: WatcherRabbitMQTransportURLReady +--- +apiVersion: v1 +kind: Secret +metadata: + name: watcher-db-secret + namespace: watcher-kuttl-default + finalizers: + - openstack.org/watcher +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBAccount +metadata: + name: watcher + namespace: watcher-kuttl-default + finalizers: + - openstack.org/watcher + - openstack.org/mariadbaccount + labels: + mariaDBDatabaseName: watcher +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + name: watcher + namespace: watcher-kuttl-default + finalizers: + - openstack.org/watcher + - openstack.org/mariadbdatabase + - openstack.org/mariadbaccount-watcher +--- +apiVersion: rabbitmq.openstack.org/v1beta1 +kind: TransportURL +metadata: + generation: 1 + labels: + service: watcher + name: watcher-kuttl-watcher-transport + namespace: watcher-kuttl-default +spec: + rabbitmqClusterName: rabbitmq +status: + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: TransportURL completed + reason: Ready + status: "True" + type: TransportURLReady + secretName: rabbitmq-transport-url-watcher-kuttl-watcher-transport +--- +apiVersion: v1 +kind: Secret +metadata: + name: rabbitmq-transport-url-watcher-kuttl-watcher-transport + namespace: watcher-kuttl-default +--- +apiVersion: v1 +kind: Secret +metadata: + name: watcher-kuttl +--- +apiVersion: keystone.openstack.org/v1beta1 +kind: KeystoneService +metadata: + name: watcher + finalizers: + - openstack.org/watcher + - openstack.org/keystoneservice +spec: + enabled: true + passwordSelector: WatcherPassword + secret: osp-secret + serviceDescription: Watcher Service + serviceName: watcher + serviceType: infra-optim + serviceUser: watcher +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: watcher-watcher-kuttl +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: watcher-watcher-kuttl-rolebinding + namespace: watcher-kuttl-default +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: watcher-watcher-kuttl-role +subjects: +- kind: ServiceAccount + name: watcher-watcher-kuttl + namespace: watcher-kuttl-default +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + service: watcher + name: watcher-kuttl-db-sync +--- +apiVersion: v1 +kind: Secret +metadata: + name: watcher-kuttl-config-data +--- +apiVersion: watcher.openstack.org/v1beta1 +kind: WatcherAPI +metadata: + finalizers: + - openstack.org/watcherapi + name: watcher-kuttl-api +spec: + containerImage: quay.io/podified-master-centos9/openstack-watcher-api:current-podified + memcachedInstance: memcached + passwordSelectors: + service: WatcherPassword + preserveJobs: false + replicas: 1 + resources: {} + secret: watcher-kuttl + serviceAccount: watcher-watcher-kuttl + serviceUser: watcher +status: + conditions: + - message: Setup complete + reason: Ready + status: "True" + type: Ready + - message: Deployment completed + reason: Ready + status: "True" + type: DeploymentReady + - message: Exposing service completed + reason: Ready + status: "True" + type: ExposeServiceReady + - message: Input data complete + reason: Ready + status: "True" + type: InputReady + - message: " Memcached instance has been provisioned" + reason: Ready + status: "True" + type: MemcachedReady + - message: Service config create completed + reason: Ready + status: "True" + type: ServiceConfigReady +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: watcher-kuttl-api + labels: + service: watcher-api +spec: + replicas: 1 + template: + spec: + containers: + - name: watcher-kuttl-api-log + - name: watcher-api +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + service: watcher-api +spec: + containers: + - name: watcher-kuttl-api-log + - name: watcher-api +status: + phase: Running +--- +apiVersion: v1 +kind: Service +metadata: + labels: + public: "true" + service: watcher-api + name: watcher-public +spec: + ports: + - name: watcher-public + port: 9322 + protocol: TCP + targetPort: 9322 + selector: + service: watcher-api + type: ClusterIP +--- +apiVersion: v1 +kind: Service +metadata: + labels: + internal: "true" + service: watcher-api + name: watcher-internal +spec: + allocateLoadBalancerNodePorts: true + ports: + - name: watcher-internal + port: 9322 + protocol: TCP + targetPort: 9322 + selector: + service: watcher-api + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 172.17.0.82 +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + public: "true" + service: watcher-api + name: watcher-public +spec: + port: + targetPort: watcher-public + to: + kind: Service + name: watcher-public +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +namespaced: true +commands: + - script: | + set -euxo pipefail + oc exec -n watcher-kuttl-default openstackclient -- openstack service list -f value -c Name -c Type |[ $(grep -c ^watcher) == 1 ] + SERVICEID=$(oc exec -n watcher-kuttl-default openstackclient -- openstack service list -f value -c Name -c Type -c ID | grep watcher| awk '{print $1}') + [ $(oc get -n watcher-kuttl-default keystoneservice watcher -o jsonpath={.status.serviceID}) == ${SERVICEID} ] + [ -n "$(oc get -n watcher-kuttl-default watcher watcher-kuttl -o jsonpath={.status.hash.dbsync})" ] + # If we are running the container locally, skip following test + if [ "$(oc get pods -n openstack-operators -o name -l openstack.org/operator-name=watcher)" == "" ]; then + exit 0 + fi + env_variables=$(oc set env $(oc get pods -n openstack-operators -o name -l openstack.org/operator-name=watcher) -n openstack-operators --list) + counter=0 + for i in ${env_variables}; do + if echo ${i} | grep '_URL_DEFAULT' &> /dev/null; then + echo ${i} + counter=$((counter + 1)) + fi + done + if [ ${counter} -lt 3 ]; then + echo "Error: Less than 3 _URL_DEFAULT variables found." + exit 1 + else + echo "Success: ${counter} _URL_DEFAULT variables found." + fi diff --git a/tests/kuttl/test-suites/default/watcher-api-service-override/01-deploy-with-defaults.yaml b/tests/kuttl/test-suites/default/watcher-api-service-override/01-deploy-with-defaults.yaml new file mode 100644 index 00000000..bcd84992 --- /dev/null +++ b/tests/kuttl/test-suites/default/watcher-api-service-override/01-deploy-with-defaults.yaml @@ -0,0 +1,14 @@ +apiVersion: watcher.openstack.org/v1beta1 +kind: Watcher +metadata: + name: watcher-kuttl + namespace: watcher-kuttl-default +spec: + databaseInstance: "openstack" + apiServiceTemplate: + override: + service: + internal: + ipAddressPool: "internalapi" + loadBalancerIPs: + - 172.17.0.82 diff --git a/tests/kuttl/test-suites/default/watcher-api-service-override/05-assert.yaml b/tests/kuttl/test-suites/default/watcher-api-service-override/05-assert.yaml new file mode 100644 index 00000000..89371130 --- /dev/null +++ b/tests/kuttl/test-suites/default/watcher-api-service-override/05-assert.yaml @@ -0,0 +1,7 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +namespaced: true +commands: + - script: | + set -ex + oc exec -n watcher-kuttl-default openstackclient -- openstack service list -f value -c Name -c Type | [ $(grep -c ^watcher) == 0 ] diff --git a/tests/kuttl/test-suites/default/watcher-api-service-override/05-cleanup-watcher.yaml b/tests/kuttl/test-suites/default/watcher-api-service-override/05-cleanup-watcher.yaml new file mode 120000 index 00000000..92ed6e0b --- /dev/null +++ b/tests/kuttl/test-suites/default/watcher-api-service-override/05-cleanup-watcher.yaml @@ -0,0 +1 @@ +../common/cleanup-watcher.yaml \ No newline at end of file diff --git a/tests/kuttl/test-suites/default/watcher-api-service-override/05-errors.yaml b/tests/kuttl/test-suites/default/watcher-api-service-override/05-errors.yaml new file mode 100644 index 00000000..633d9dd9 --- /dev/null +++ b/tests/kuttl/test-suites/default/watcher-api-service-override/05-errors.yaml @@ -0,0 +1,93 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: watcher-watcher-kuttl +--- +apiVersion: keystone.openstack.org/v1beta1 +kind: KeystoneService +metadata: + name: watcher +--- +apiVersion: v1 +kind: Secret +metadata: + name: rabbitmq-transport-url-watcher-kuttl-watcher-transport +--- +apiVersion: v1 +kind: Secret +metadata: + name: watcher-kuttl +--- +apiVersion: rabbitmq.openstack.org/v1beta1 +kind: TransportURL +metadata: + name: watcher-kuttl-watcher-transport +--- +apiVersion: mariadb.openstack.org/v1beta1 +kind: MariaDBDatabase +metadata: + name: watcher +--- +apiVersion: v1 +kind: Secret +metadata: + name: watcher-db-secret +--- +apiVersion: batch/v1 +kind: Job +metadata: + labels: + service: watcher + name: watcher-kuttl-db-sync +--- +apiVersion: v1 +kind: Secret +metadata: + name: watcher-kuttl-config-data +--- +apiVersion: watcher.openstack.org/v1beta1 +kind: Watcher +metadata: + name: watcher-kuttl +--- +apiVersion: watcher.openstack.org/v1beta1 +kind: WatcherAPI +metadata: + finalizers: + - openstack.org/watcherapi + name: watcher-kuttl-api +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: watcher-kuttl-api +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + service: watcher-api +--- +apiVersion: v1 +kind: Service +metadata: + labels: + internal: "true" + service: watcher-api + name: watcher-internal +--- +apiVersion: v1 +kind: Service +metadata: + labels: + public: "true" + service: watcher-api + name: watcher-public +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + public: "true" + service: watcher-api + name: watcher-public